diff --git a/packages/claude-agents/agents.md b/packages/claude-agents/agents.md index c7202e2e..8ff60679 100644 --- a/packages/claude-agents/agents.md +++ b/packages/claude-agents/agents.md @@ -31,6 +31,7 @@ That means the LaunchDarkly flag variation must have `provider.name == "Anthropi | `src/launchdarkly_ai_claude_agents/native_graph.py` | `to_claude_agents()` native graph adapter | | `src/launchdarkly_ai_claude_agents/builtins.py` | Pre-constructed `NativeTool` sentinels for Claude built-in tools | | `src/launchdarkly_ai_claude_agents/__init__.py` | Package exports | +| `src/launchdarkly_ai_claude_agents/_version.py` | `PACKAGE_NAME` and `__version__`, sent with `$ld:ai:sdk:info` and `$ld:ai:sdk:usage` | --- diff --git a/packages/claude-agents/src/launchdarkly_ai_claude_agents/__init__.py b/packages/claude-agents/src/launchdarkly_ai_claude_agents/__init__.py index 8b2dcab6..d74b486c 100644 --- a/packages/claude-agents/src/launchdarkly_ai_claude_agents/__init__.py +++ b/packages/claude-agents/src/launchdarkly_ai_claude_agents/__init__.py @@ -3,11 +3,10 @@ See https://launchdarkly.com/docs for usage. """ -__version__ = "0.2.4" # x-release-please-version - from launchdarkly_ai_server import register_ai_sdk_package from . import native_graph # noqa: F401 +from ._version import PACKAGE_NAME, __version__ from .builtins import ( ClaudeBash, ClaudeEdit, @@ -50,4 +49,4 @@ "to_claude_agents", ] -register_ai_sdk_package("launchdarkly-ai-claude-agents", __version__) +register_ai_sdk_package(PACKAGE_NAME, __version__) diff --git a/packages/claude-agents/src/launchdarkly_ai_claude_agents/_version.py b/packages/claude-agents/src/launchdarkly_ai_claude_agents/_version.py new file mode 100644 index 00000000..b6dada91 --- /dev/null +++ b/packages/claude-agents/src/launchdarkly_ai_claude_agents/_version.py @@ -0,0 +1,4 @@ +"""This package's identity, as sent with ``$ld:ai:sdk:info`` and ``$ld:ai:sdk:usage``.""" + +PACKAGE_NAME = "launchdarkly-ai-claude-agents" +__version__ = "0.2.4" # x-release-please-version diff --git a/packages/claude-agents/src/launchdarkly_ai_claude_agents/graph.py b/packages/claude-agents/src/launchdarkly_ai_claude_agents/graph.py index 83d6e311..1311a6c2 100644 --- a/packages/claude-agents/src/launchdarkly_ai_claude_agents/graph.py +++ b/packages/claude-agents/src/launchdarkly_ai_claude_agents/graph.py @@ -4,8 +4,10 @@ from typing import Any -from launchdarkly_ai_claude_agents.handler import create_claude_agents_handler -from launchdarkly_ai_server import graph +from launchdarkly_ai_claude_agents.handler import _create_claude_agents_handler +from launchdarkly_ai_server import _graph, report_usage + +from ._version import PACKAGE_NAME, __version__ def claude_graph(key: str, **options: Any) -> Any: @@ -15,4 +17,5 @@ def claude_graph(key: str, **options: Any) -> Any: Equivalent to ``graph(key, handlers=[create_claude_agents_handler()], **options)``. Use the base ``graph()`` directly for multi-provider graphs. """ - return graph(key, handlers=[create_claude_agents_handler()], **options) + report_usage("claude-agents.claudeGraph", PACKAGE_NAME, __version__) + return _graph(key, handlers=[_create_claude_agents_handler()], **options) diff --git a/packages/claude-agents/src/launchdarkly_ai_claude_agents/handler.py b/packages/claude-agents/src/launchdarkly_ai_claude_agents/handler.py index 32ee9fd6..eb7665e1 100644 --- a/packages/claude-agents/src/launchdarkly_ai_claude_agents/handler.py +++ b/packages/claude-agents/src/launchdarkly_ai_claude_agents/handler.py @@ -35,19 +35,21 @@ ProviderHandler, SpanMessage, SpanMessagePart, + _config, + _create_handler, compose_history, - config, content_to_text, - create_handler, end_span_once, end_unfinished_spans, parse_template, + report_usage, set_conversation_id_if_absent, set_input_content_attributes, set_output_content_attributes, set_tool_call_content_attributes, ) +from ._version import PACKAGE_NAME, __version__ from .spans import ( MCP_TOOL_PREFIX, TOOL_MCP_NAME, @@ -508,6 +510,12 @@ def create_claude_agents_handler(*, capture_content: bool = False) -> ProviderHa Set *capture_content* to put prompts, model output, tool arguments and tool results on the emitted spans. It defaults to off. See TELEMETRY-CONTRACT.md section 7. """ + report_usage("claude-agents.createClaudeAgentsHandler", PACKAGE_NAME, __version__) + return _create_claude_agents_handler(capture_content=capture_content) + + +def _create_claude_agents_handler(*, capture_content: bool = False) -> ProviderHandler: + """Non-reporting :func:`create_claude_agents_handler`, used by this package's wrappers.""" async def _call_impl( config: AiConfigRep, @@ -694,7 +702,7 @@ def _stream_impl( capture_content=capture_content, ) - return create_handler( + return _create_handler( ("Anthropic", "agent"), _call_impl, # type: ignore[arg-type] _stream_impl, # type: ignore[arg-type] @@ -886,10 +894,11 @@ def claude_agents( **kwargs: Any, ) -> Any: """Convenience wrapper: creates a handler and calls config(...).invoke().""" + report_usage("claude-agents.claudeAgents", PACKAGE_NAME, __version__) variables = kwargs.pop("variables", None) capture_content = kwargs.pop("capture_content", False) - return config( + return _config( key=config_key, - handler=create_claude_agents_handler(capture_content=capture_content), + handler=_create_claude_agents_handler(capture_content=capture_content), **kwargs, ).invoke(user_input, context, variables=variables) diff --git a/packages/claude-agents/src/launchdarkly_ai_claude_agents/native_graph.py b/packages/claude-agents/src/launchdarkly_ai_claude_agents/native_graph.py index b0ffadb9..9f92ab0b 100644 --- a/packages/claude-agents/src/launchdarkly_ai_claude_agents/native_graph.py +++ b/packages/claude-agents/src/launchdarkly_ai_claude_agents/native_graph.py @@ -18,9 +18,12 @@ NativeTool, get_client, make_track_data, + report_usage, to_ld_context, ) +from ._version import PACKAGE_NAME, __version__ + try: from opentelemetry import trace from opentelemetry.trace import StatusCode as SpanStatusCode @@ -207,6 +210,7 @@ def to_claude_agents( Returns an object with a ``.invoke(input, variables)`` coroutine. """ + report_usage("claude-agents.toClaudeAgents", PACKAGE_NAME, __version__) _opts = opts or {} async def invoke( diff --git a/packages/claude-agents/tests/test_graph.py b/packages/claude-agents/tests/test_graph.py index be7ff327..0e6f05d1 100644 --- a/packages/claude-agents/tests/test_graph.py +++ b/packages/claude-agents/tests/test_graph.py @@ -10,14 +10,14 @@ class TestClaudeGraph: def test_graph_called_with_correct_key(self) -> None: - with patch("launchdarkly_ai_claude_agents.graph.graph") as mock_graph: + with patch("launchdarkly_ai_claude_agents.graph._graph") as mock_graph: mock_graph.return_value = MagicMock() claude_graph("my-flag-key") mock_graph.assert_called_once() assert mock_graph.call_args[0][0] == "my-flag-key" def test_handlers_pre_populated(self) -> None: - with patch("launchdarkly_ai_claude_agents.graph.graph") as mock_graph: + with patch("launchdarkly_ai_claude_agents.graph._graph") as mock_graph: mock_graph.return_value = MagicMock() claude_graph("key") kw = mock_graph.call_args[1] @@ -26,14 +26,14 @@ def test_handlers_pre_populated(self) -> None: def test_user_supplied_options_forwarded(self) -> None: ctx = {"kind": "user", "key": "u1"} - with patch("launchdarkly_ai_claude_agents.graph.graph") as mock_graph: + with patch("launchdarkly_ai_claude_agents.graph._graph") as mock_graph: mock_graph.return_value = MagicMock() claude_graph("key", context=ctx) kw = mock_graph.call_args[1] assert kw.get("context") == ctx def test_user_cannot_override_handlers(self) -> None: - with patch("launchdarkly_ai_claude_agents.graph.graph") as mock_graph: + with patch("launchdarkly_ai_claude_agents.graph._graph") as mock_graph: mock_graph.return_value = MagicMock() claude_graph("key", extra_kwarg=42) kw = mock_graph.call_args[1] diff --git a/packages/claude-agents/tests/test_handler.py b/packages/claude-agents/tests/test_handler.py index 5651391e..83aef4de 100644 --- a/packages/claude-agents/tests/test_handler.py +++ b/packages/claude-agents/tests/test_handler.py @@ -1604,7 +1604,7 @@ def _fake_config(**_kwargs: Any) -> Any: return instance monkeypatch.setattr( - "launchdarkly_ai_claude_agents.handler.config", _fake_config + "launchdarkly_ai_claude_agents.handler._config", _fake_config ) result = await claude_agents("cfg-key", "hi", {"key": "u1"}) assert result == {"output": "ok"} @@ -1633,7 +1633,7 @@ def test_config_key_forwarded_as_key(self) -> None: mock_config_fn = MagicMock(return_value=mock_config_instance) mock_config_instance.invoke = MagicMock(return_value="result") - with patch.object(_handler_mod, "config", mock_config_fn): + with patch.object(_handler_mod, "_config", mock_config_fn): from launchdarkly_ai_claude_agents.handler import claude_agents ctx = {"kind": "user", "key": "u1"} @@ -1656,7 +1656,7 @@ def test_callable_without_extra_kwargs(self) -> None: mock_config_fn = MagicMock(return_value=mock_config_instance) mock_config_instance.invoke = MagicMock(return_value="result") - with patch.object(_handler_mod, "config", mock_config_fn): + with patch.object(_handler_mod, "_config", mock_config_fn): from launchdarkly_ai_claude_agents.handler import claude_agents ctx = {"kind": "user", "key": "u1"} diff --git a/packages/claude-agents/tests/test_sdk_usage.py b/packages/claude-agents/tests/test_sdk_usage.py new file mode 100644 index 00000000..fd576f0f --- /dev/null +++ b/packages/claude-agents/tests/test_sdk_usage.py @@ -0,0 +1,149 @@ +"""Tests for TESTING.md §3.27 — claude-agents.""" + +from types import SimpleNamespace +from typing import Any +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +import launchdarkly_ai_server.lifecycle as lifecycle_module +from launchdarkly_ai_claude_agents import ( + claude_agents, + claude_graph, + create_claude_agents_handler, + to_claude_agents, +) +from launchdarkly_ai_claude_agents._version import PACKAGE_NAME +from launchdarkly_ai_claude_agents._version import __version__ as PACKAGE_VERSION +from launchdarkly_ai_server import SDK_INFO_CONTEXT, __version__, init_client, shutdown + +CONTEXT = {"kind": "user", "key": "user-1"} + + +def _fake() -> MagicMock: + client = MagicMock() + client.track = MagicMock() + client.variation = AsyncMock(return_value=None) + client.flush = AsyncMock() + client.close = AsyncMock() + return client + + +async def _settle(value: Any) -> None: + if hasattr(value, "__await__"): + try: + await value + except Exception: + return + + +@pytest.fixture +async def client() -> Any: + await shutdown() + installed = _fake() + with patch.object(lifecycle_module, "_setup_telemetry", return_value=None): + await init_client(client=installed) + installed.track.reset_mock() + yield installed + await shutdown() + + +@pytest.mark.parametrize( + ("helper", "call"), + [ + ("claude-agents.claudeAgents", lambda: claude_agents("k", "q", CONTEXT)), + ("claude-agents.createClaudeAgentsHandler", create_claude_agents_handler), + ("claude-agents.claudeGraph", lambda: claude_graph("k")), + ("claude-agents.toClaudeAgents", lambda: to_claude_agents(None)), + ], +) +async def test_helper_sends_one_usage_event( + client: MagicMock, helper: str, call: Any +) -> None: + try: + await _settle(call()) + except Exception: + pass + calls = [ + entry + for entry in client.track.call_args_list + if entry.args[0] == "$ld:ai:sdk:usage" + ] + assert [entry.args[2].get("helper") for entry in calls] == [helper] + assert calls[0].args[1] == SDK_INFO_CONTEXT + assert calls[0].args[2] == { + "aiSdkName": "launchdarkly-ai-server", + "aiSdkVersion": __version__, + "aiSdkLanguage": "python", + "helper": helper, + "helperPackageName": PACKAGE_NAME, + "helperPackageVersion": PACKAGE_VERSION, + } + assert calls[0].args[3] == 1 + + +def _helpers(client: MagicMock) -> list[str]: + return [ + entry.args[2].get("helper") + for entry in client.track.call_args_list + if entry.args[0] == "$ld:ai:sdk:usage" + ] + + +async def _drain(stream: Any) -> None: + try: + async for _ in stream: + pass + except Exception: + return + + +async def test_graph_wrapper_invoke_and_stream_report_only_the_wrapper( + client: MagicMock, +) -> None: + """The returned graph's methods are not separate ``client.graph.*`` calls.""" + instance = claude_graph("k") + await _settle(instance.invoke("q", CONTEXT)) + await _drain(instance.stream("q", CONTEXT)) + assert _helpers(client) == ["claude-agents.claudeGraph"] + + +async def test_native_adapter_invoke_reports_only_the_adapter( + client: MagicMock, +) -> None: + async def _definition() -> Any: + return SimpleNamespace(enabled=False, key="k") + + runner = to_claude_agents(_definition(), {"context": CONTEXT}) + with pytest.raises(ValueError, match="disabled"): + await runner.invoke("q") + assert _helpers(client) == ["claude-agents.toClaudeAgents"] + + +async def test_held_helper_flushes_with_its_package() -> None: + """A helper called before init keeps its package fields until the flush.""" + await shutdown() + lifecycle_module._reset_for_testing() + try: + create_claude_agents_handler() + except Exception: + pass # The factory may need credentials; it reports first. + installed = _fake() + with patch.object(lifecycle_module, "_setup_telemetry", return_value=None): + await init_client(client=installed) + calls = [ + entry.args[2] + for entry in installed.track.call_args_list + if entry.args[0] == "$ld:ai:sdk:usage" + ] + assert calls == [ + { + "aiSdkName": "launchdarkly-ai-server", + "aiSdkVersion": __version__, + "aiSdkLanguage": "python", + "helper": "claude-agents.createClaudeAgentsHandler", + "helperPackageName": PACKAGE_NAME, + "helperPackageVersion": PACKAGE_VERSION, + } + ] + await shutdown() diff --git a/packages/claude-messages/agents.md b/packages/claude-messages/agents.md index 908d8f26..95705470 100644 --- a/packages/claude-messages/agents.md +++ b/packages/claude-messages/agents.md @@ -28,6 +28,7 @@ That means the LaunchDarkly flag variation must have `provider.name == "Anthropi |---|---| | `src/launchdarkly_ai_claude_messages/handler.py` | All implementation — message building, tool schema conversion, tool-use loop, telemetry | | `src/launchdarkly_ai_claude_messages/__init__.py` | Package exports | +| `src/launchdarkly_ai_claude_messages/_version.py` | `PACKAGE_NAME` and `__version__`, sent with `$ld:ai:sdk:info` and `$ld:ai:sdk:usage` | --- diff --git a/packages/claude-messages/src/launchdarkly_ai_claude_messages/__init__.py b/packages/claude-messages/src/launchdarkly_ai_claude_messages/__init__.py index 86b4b319..768d0930 100644 --- a/packages/claude-messages/src/launchdarkly_ai_claude_messages/__init__.py +++ b/packages/claude-messages/src/launchdarkly_ai_claude_messages/__init__.py @@ -3,12 +3,11 @@ See https://launchdarkly.com/docs for usage. """ -__version__ = "0.2.3" # x-release-please-version - from launchdarkly_ai_server import register_ai_sdk_package +from ._version import PACKAGE_NAME, __version__ from .handler import claude_messages, create_claude_messages_handler __all__ = ["claude_messages", "create_claude_messages_handler"] -register_ai_sdk_package("launchdarkly-ai-claude-messages", __version__) +register_ai_sdk_package(PACKAGE_NAME, __version__) diff --git a/packages/claude-messages/src/launchdarkly_ai_claude_messages/_version.py b/packages/claude-messages/src/launchdarkly_ai_claude_messages/_version.py new file mode 100644 index 00000000..7c067e2c --- /dev/null +++ b/packages/claude-messages/src/launchdarkly_ai_claude_messages/_version.py @@ -0,0 +1,4 @@ +"""This package's identity, as sent with ``$ld:ai:sdk:info`` and ``$ld:ai:sdk:usage``.""" + +PACKAGE_NAME = "launchdarkly-ai-claude-messages" +__version__ = "0.2.3" # x-release-please-version diff --git a/packages/claude-messages/src/launchdarkly_ai_claude_messages/handler.py b/packages/claude-messages/src/launchdarkly_ai_claude_messages/handler.py index 0de014a3..9d01c14f 100644 --- a/packages/claude-messages/src/launchdarkly_ai_claude_messages/handler.py +++ b/packages/claude-messages/src/launchdarkly_ai_claude_messages/handler.py @@ -11,20 +11,22 @@ ProviderHandler, SpanMessage, SpanMessagePart, + _config, + _create_handler, compose_history, - config, content_to_text, - create_handler, end_span_once, end_unfinished_spans, is_content_blocks, parse_template, + report_usage, set_input_content_attributes, set_output_content_attributes, set_tool_call_content_attributes, to_semconv_finish_reason, ) +from ._version import PACKAGE_NAME, __version__ from .spans import ( RawRunUsage, fail_span, @@ -359,6 +361,16 @@ def create_claude_messages_handler(*, capture_content: bool = False) -> Provider meaning models, token counts, timings and tool names, until a caller asks for more. Turning this on sends the text of every request and response to whatever collector the SDK is pointed at. """ + report_usage( + "claude-messages.createClaudeMessagesHandler", PACKAGE_NAME, __version__ + ) + return _create_claude_messages_handler(capture_content=capture_content) + + +def _create_claude_messages_handler( + *, capture_content: bool = False +) -> ProviderHandler: + """Non-reporting :func:`create_claude_messages_handler`, used by this package's wrappers.""" import importlib anthropic_mod = importlib.import_module("anthropic") @@ -457,7 +469,7 @@ def _stream_impl( capture_content=capture_content, ) - return create_handler( + return _create_handler( ("Anthropic", "messages"), _call_impl, # type: ignore[arg-type] _stream_impl, # type: ignore[arg-type] @@ -708,10 +720,11 @@ def claude_messages( # Both are lifted out of kwargs: capture_content configures the handler, variables belong to # the invocation. Leaving either in would pass it to config(), which takes neither, so a caller # asking for content on spans got a TypeError instead of content. + report_usage("claude-messages.claudeMessages", PACKAGE_NAME, __version__) variables = kwargs.pop("variables", None) capture_content = kwargs.pop("capture_content", False) - return config( + return _config( key=config_key, - handler=create_claude_messages_handler(capture_content=capture_content), + handler=_create_claude_messages_handler(capture_content=capture_content), **kwargs, ).invoke(user_input, context, variables=variables) diff --git a/packages/claude-messages/tests/test_handler.py b/packages/claude-messages/tests/test_handler.py index f07336fe..0a0353d9 100644 --- a/packages/claude-messages/tests/test_handler.py +++ b/packages/claude-messages/tests/test_handler.py @@ -1191,7 +1191,7 @@ def test_calls_through_to_model_call(self, mock_anthropic: MagicMock) -> None: mock_config_fn = MagicMock(return_value=mock_config_instance) mock_config_instance.invoke = MagicMock(return_value="result") - with patch.object(handler_mod, "config", mock_config_fn): + with patch.object(handler_mod, "_config", mock_config_fn): from launchdarkly_ai_claude_messages.handler import claude_messages ctx = {"kind": "user", "key": "u1"} @@ -1214,7 +1214,7 @@ def test_callable_without_extra_kwargs(self) -> None: mock_config_fn = MagicMock(return_value=mock_config_instance) mock_config_instance.invoke = MagicMock(return_value="result") - with patch.object(handler_mod, "config", mock_config_fn): + with patch.object(handler_mod, "_config", mock_config_fn): from launchdarkly_ai_claude_messages.handler import claude_messages ctx = {"kind": "user", "key": "u1"} @@ -1811,8 +1811,8 @@ def _factory(*args: Any, capture_content: bool = False, **kw: Any) -> Any: fake_config = MagicMock() fake_config.return_value.invoke = MagicMock(return_value="ok") with ( - patch.object(handler_mod, "create_claude_messages_handler", _factory), - patch.object(handler_mod, "config", fake_config), + patch.object(handler_mod, "_create_claude_messages_handler", _factory), + patch.object(handler_mod, "_config", fake_config), ): handler_mod.claude_messages("k", "q", {}, **kwargs) seen["config_kwargs"] = fake_config.call_args.kwargs diff --git a/packages/claude-messages/tests/test_sdk_usage.py b/packages/claude-messages/tests/test_sdk_usage.py new file mode 100644 index 00000000..7ecc294b --- /dev/null +++ b/packages/claude-messages/tests/test_sdk_usage.py @@ -0,0 +1,106 @@ +"""Tests for TESTING.md §3.27 — claude-messages.""" + +from typing import Any +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +import launchdarkly_ai_server.lifecycle as lifecycle_module +from launchdarkly_ai_claude_messages import ( + claude_messages, + create_claude_messages_handler, +) +from launchdarkly_ai_claude_messages._version import PACKAGE_NAME +from launchdarkly_ai_claude_messages._version import __version__ as PACKAGE_VERSION +from launchdarkly_ai_server import SDK_INFO_CONTEXT, __version__, init_client, shutdown + +CONTEXT = {"kind": "user", "key": "user-1"} + + +def _fake() -> MagicMock: + client = MagicMock() + client.track = MagicMock() + client.variation = AsyncMock(return_value=None) + client.flush = AsyncMock() + client.close = AsyncMock() + return client + + +async def _settle(value: Any) -> None: + if hasattr(value, "__await__"): + try: + await value + except Exception: + return + + +@pytest.fixture +async def client() -> Any: + await shutdown() + installed = _fake() + with patch.object(lifecycle_module, "_setup_telemetry", return_value=None): + await init_client(client=installed) + installed.track.reset_mock() + yield installed + await shutdown() + + +@pytest.mark.parametrize( + ("helper", "call"), + [ + ("claude-messages.claudeMessages", lambda: claude_messages("k", "q", CONTEXT)), + ("claude-messages.createClaudeMessagesHandler", create_claude_messages_handler), + ], +) +async def test_helper_sends_one_usage_event( + client: MagicMock, helper: str, call: Any +) -> None: + try: + await _settle(call()) + except Exception: + pass + calls = [ + entry + for entry in client.track.call_args_list + if entry.args[0] == "$ld:ai:sdk:usage" + ] + assert [entry.args[2].get("helper") for entry in calls] == [helper] + assert calls[0].args[1] == SDK_INFO_CONTEXT + assert calls[0].args[2] == { + "aiSdkName": "launchdarkly-ai-server", + "aiSdkVersion": __version__, + "aiSdkLanguage": "python", + "helper": helper, + "helperPackageName": PACKAGE_NAME, + "helperPackageVersion": PACKAGE_VERSION, + } + assert calls[0].args[3] == 1 + + +async def test_held_helper_flushes_with_its_package() -> None: + """A helper called before init keeps its package fields until the flush.""" + await shutdown() + lifecycle_module._reset_for_testing() + try: + create_claude_messages_handler() + except Exception: + pass # The factory may need credentials; it reports first. + installed = _fake() + with patch.object(lifecycle_module, "_setup_telemetry", return_value=None): + await init_client(client=installed) + calls = [ + entry.args[2] + for entry in installed.track.call_args_list + if entry.args[0] == "$ld:ai:sdk:usage" + ] + assert calls == [ + { + "aiSdkName": "launchdarkly-ai-server", + "aiSdkVersion": __version__, + "aiSdkLanguage": "python", + "helper": "claude-messages.createClaudeMessagesHandler", + "helperPackageName": PACKAGE_NAME, + "helperPackageVersion": PACKAGE_VERSION, + } + ] + await shutdown() diff --git a/packages/client/src/launchdarkly_ai_server/__init__.py b/packages/client/src/launchdarkly_ai_server/__init__.py index 6e77b0bf..dbac999d 100644 --- a/packages/client/src/launchdarkly_ai_server/__init__.py +++ b/packages/client/src/launchdarkly_ai_server/__init__.py @@ -6,6 +6,7 @@ __version__ = "0.2.4" # x-release-please-version from .client import ConfigInstance, config +from .client import _config as _config from .content import ( SpanMessage, SpanMessagePart, @@ -40,6 +41,7 @@ init_evaluations, ) from .graph import GraphInstance, graph, resolve_graph +from .graph import _graph as _graph from .history import ( any_multimodal, compose_history, @@ -64,6 +66,7 @@ resolve_tools, ) from .sdk_info import SDK_INFO_CONTEXT, SDK_INFO_EVENT, register_ai_sdk_package +from .sdk_usage import report_usage from .tracking import execute_and_stream, execute_and_track, wrap_tool_handlers from .types import ( NATIVE_TOOL_KEY, @@ -127,6 +130,7 @@ to_ld_context, to_usage_dict, ) +from .utils import _create_handler as _create_handler __all__ = [ # noqa: RUF022 # types @@ -241,6 +245,7 @@ "SDK_INFO_CONTEXT", "SDK_INFO_EVENT", "register_ai_sdk_package", + "report_usage", # judges "build_judge_tasks", "run_judge", diff --git a/packages/client/src/launchdarkly_ai_server/client.py b/packages/client/src/launchdarkly_ai_server/client.py index f9f598f2..0f77383f 100644 --- a/packages/client/src/launchdarkly_ai_server/client.py +++ b/packages/client/src/launchdarkly_ai_server/client.py @@ -5,9 +5,10 @@ from typing import Any from .conversation import bind_conversation_id -from .judges import build_judge_tasks, run_judges +from .judges import _build_judge_tasks, run_judges from .lifecycle import extract_variation from .registry import resolve_handlers, resolve_tools +from .sdk_usage import report_usage from .tracking import execute_and_stream, execute_and_track from .types import ( AiConfigRep, @@ -72,6 +73,16 @@ async def invoke( context: LDContext, variables: dict[str, Any] | None = None, history: list[dict[str, Any]] | None = None, + ) -> ProviderResponse[Any]: + report_usage("client.config.invoke") + return await self._invoke(user_input, context, variables, history) + + async def _invoke( + self, + user_input: str | None, + context: LDContext, + variables: dict[str, Any] | None = None, + history: list[dict[str, Any]] | None = None, ) -> ProviderResponse[Any]: resolved_handler_list = resolve_handlers( self._registry, self._normalize_handlers() @@ -116,7 +127,7 @@ async def invoke( usage_obj = to_usage_dict(usage) if self._skip_judges: - judge_tasks = await build_judge_tasks( + judge_tasks = await _build_judge_tasks( config=config, user_context=context, handler=handler, @@ -163,8 +174,20 @@ def stream( Deliberately not an ``async def`` with ``yield``: a generator body does not run until the first ``__anext__``, by which point a :func:`conversation_id` block wrapped around this call has already exited. Binding here — at call time — is what lets a caller hand the - generator off and iterate it later. + generator off and iterate it later. The usage report runs here for the same reason: it + fires on the call, before any iteration. """ + report_usage("client.config.stream") + return self._stream(user_input, context, variables, history) + + def _stream( + self, + user_input: str | None, + context: LDContext, + variables: dict[str, Any] | None = None, + history: list[dict[str, Any]] | None = None, + ) -> AsyncGenerator[StreamEvent, None]: + """Non-reporting :meth:`stream`: binds the conversation id at call time.""" return bind_conversation_id( self._stream_events(user_input, context, variables, history) ) @@ -232,6 +255,32 @@ async def _stream_events( } +class _InternalConfigInstance(ConfigInstance): + """A ``ConfigInstance`` whose ``invoke`` and ``stream`` do not report usage. + + Returned by :func:`_config` for SDK wrappers: the wrapper reports itself, and the + config call it makes is not a call from the application. + """ + + async def invoke( + self, + user_input: str | None, + context: LDContext, + variables: dict[str, Any] | None = None, + history: list[dict[str, Any]] | None = None, + ) -> ProviderResponse[Any]: + return await self._invoke(user_input, context, variables, history) + + def stream( + self, + user_input: str | None, + context: LDContext, + variables: dict[str, Any] | None = None, + history: list[dict[str, Any]] | None = None, + ) -> AsyncGenerator[StreamEvent, None]: + return self._stream(user_input, context, variables, history) + + def config( *, key: str, @@ -259,3 +308,21 @@ def config( registry=registry, skip_judges=skip_judges, ) + + +def _config( + *, + key: str, + handler: ProviderHandler | list[ProviderHandler] | None = None, + tool_handlers: dict[str, Callable[..., Any] | NativeTool] | None = None, + registry: Any = None, + skip_judges: bool = False, +) -> ConfigInstance: + """Non-reporting :func:`config` for SDK wrappers.""" + return _InternalConfigInstance( + key=key, + handler=handler, + tool_handlers=tool_handlers, + registry=registry, + skip_judges=skip_judges, + ) diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/module.py b/packages/client/src/launchdarkly_ai_server/evaluations/module.py index 873b5ea1..2348690f 100644 --- a/packages/client/src/launchdarkly_ai_server/evaluations/module.py +++ b/packages/client/src/launchdarkly_ai_server/evaluations/module.py @@ -711,6 +711,9 @@ def init_evaluations( ``project_key`` names the project that holds the evaluations, the tools, and the datasets this module uses. """ + from ..sdk_usage import report_usage + + report_usage("client.init_evaluations") resolved_project_key = (project_key or "").strip() or _env("LD_PROJECT_KEY") if not resolved_project_key: raise EvaluationsError( diff --git a/packages/client/src/launchdarkly_ai_server/graph.py b/packages/client/src/launchdarkly_ai_server/graph.py index 02e81f55..beb39d97 100644 --- a/packages/client/src/launchdarkly_ai_server/graph.py +++ b/packages/client/src/launchdarkly_ai_server/graph.py @@ -837,6 +837,9 @@ async def resolve_graph( ``context`` is a keyword-only argument, mirroring the TypeScript ``resolveGraph(key, { context, handlers, toolHandlers, registry })`` shape. """ + from .sdk_usage import report_usage + + report_usage("client.resolveGraph") resolved_handlers = resolve_handlers(registry, handlers) resolved_tools = resolve_tools(registry, tool_handlers) options = { @@ -869,6 +872,19 @@ async def invoke( variables: dict[str, Any] | None = None, history: list[dict[str, Any]] | None = None, ) -> ProviderGraphResponse: + from .sdk_usage import report_usage + + report_usage("client.graph.invoke") + return await self._invoke(user_input, context, variables, history) + + async def _invoke( + self, + user_input: str | None, + context: LDContext, + variables: dict[str, Any] | None = None, + history: list[dict[str, Any]] | None = None, + ) -> ProviderGraphResponse: + """Non-reporting :meth:`invoke`.""" from opentelemetry import trace from .judges import run_judges @@ -1066,8 +1082,22 @@ def stream( Deliberately not an ``async def`` with ``yield``: a generator body does not run until the first ``__anext__``, by which point a ``conversation_id`` / caller span scope wrapped around this call may have already exited. Binding the conversation id and capturing the OTel parent - here — at call time — matches ``config().stream()`` and the TypeScript graph stream. + here — at call time — matches ``config().stream()`` and the TypeScript graph stream. The + usage report also runs here, on the call, before any iteration. """ + from .sdk_usage import report_usage + + report_usage("client.graph.stream") + return self._stream(user_input, context, variables, history) + + def _stream( + self, + user_input: str | None, + context: LDContext, + variables: dict[str, Any] | None = None, + history: list[dict[str, Any]] | None = None, + ) -> AsyncGenerator[GraphStreamEvent, None]: + """Non-reporting :meth:`stream`: binds the conversation id and OTel parent at call time.""" from opentelemetry import context as otel_context caller_context = otel_context.get_current() @@ -1289,6 +1319,32 @@ async def _stream_events( end_span_once(span, ended, abandoned=True, cancelled=cancelled) +class _InternalGraphInstance(GraphInstance): + """A ``GraphInstance`` whose ``invoke`` and ``stream`` do not report usage. + + Returned by :func:`_graph` for SDK graph wrappers: the wrapper reports itself, and the + graph calls made through the object it returns are not separate helper calls. + """ + + async def invoke( + self, + user_input: str | None, + context: LDContext, + variables: dict[str, Any] | None = None, + history: list[dict[str, Any]] | None = None, + ) -> ProviderGraphResponse: + return await self._invoke(user_input, context, variables, history) + + def stream( + self, + user_input: str | None, + context: LDContext, + variables: dict[str, Any] | None = None, + history: list[dict[str, Any]] | None = None, + ) -> AsyncGenerator[GraphStreamEvent, None]: + return self._stream(user_input, context, variables, history) + + def graph( key: str, *, @@ -1309,3 +1365,21 @@ def graph( "graph_judge": graph_judge, } return GraphInstance(key=key, options=options) + + +def _graph( + key: str, + *, + handlers: list[ProviderHandler] | None = None, + tool_handlers: dict[str, Callable[..., Any] | NativeTool] | None = None, + registry: Any = None, + graph_judge: str | None = None, +) -> GraphInstance: + """Non-reporting :func:`graph` for SDK graph wrappers.""" + options = { + "handlers": handlers, + "tool_handlers": tool_handlers, + "registry": registry, + "graph_judge": graph_judge, + } + return _InternalGraphInstance(key=key, options=options) diff --git a/packages/client/src/launchdarkly_ai_server/judges.py b/packages/client/src/launchdarkly_ai_server/judges.py index 56d7e032..d3e8377b 100644 --- a/packages/client/src/launchdarkly_ai_server/judges.py +++ b/packages/client/src/launchdarkly_ai_server/judges.py @@ -11,6 +11,7 @@ numeric_score, parse_judge_response, ) +from .sdk_usage import report_usage from .types import ( AiConfigRep, JudgeResult, @@ -232,6 +233,31 @@ async def build_judge_tasks( ``samplingRate`` causes them to be skipped are excluded from the list. Returns an empty list when no active judges are configured. """ + report_usage("client.buildJudgeTasks") + return await _build_judge_tasks( + config=config, + user_context=user_context, + handler=handler, + handlers=handlers, + llm_response=llm_response, + base_track_data=base_track_data, + user_input=user_input, + trajectory=trajectory, + ) + + +async def _build_judge_tasks( + *, + config: AiConfigRep, + user_context: LDContext, + handler: ProviderHandler, + handlers: list[ProviderHandler] | None = None, + llm_response: str, + base_track_data: TrackData, + user_input: str | None = None, + trajectory: str = "", +) -> list[JudgeTask]: + """Non-reporting :func:`build_judge_tasks`, used by ``config().invoke``.""" from .lifecycle import extract_variation judge_config_block = ( @@ -351,6 +377,7 @@ async def run_judge( Returns ``None`` when no compatible handler is found or the response cannot be parsed as ``{"score": ..., "reasoning": ...}``. """ + report_usage("client.runJudge") from .tracking import execute_and_track def _matches(h: ProviderHandler) -> bool: diff --git a/packages/client/src/launchdarkly_ai_server/lifecycle.py b/packages/client/src/launchdarkly_ai_server/lifecycle.py index 18e311c6..f9bc8497 100644 --- a/packages/client/src/launchdarkly_ai_server/lifecycle.py +++ b/packages/client/src/launchdarkly_ai_server/lifecycle.py @@ -7,6 +7,7 @@ from typing import Any from .sdk_info import flush_ai_sdk_info, reset_ai_sdk_info +from .sdk_usage import flush_sdk_usage, report_usage, reset_sdk_usage from .types import InitClientOptions from .utils import model_stamps_from_meta @@ -142,6 +143,7 @@ async def init_client( # Idempotent — if already initialized, return the existing client if _client is not None: flush_ai_sdk_info(_client) + flush_sdk_usage(_client) return _client # BYOC path — pre-initialized client @@ -149,6 +151,7 @@ async def init_client( _client = client _setup_telemetry(opts.get("sdkKey", "byoc"), opts) flush_ai_sdk_info(_client) + flush_sdk_usage(_client) return _client # Resolve SDK key @@ -196,6 +199,7 @@ async def init_client( _client = ld_client _setup_telemetry(sdk_key, opts) flush_ai_sdk_info(_client) + flush_sdk_usage(_client) return _client @@ -213,6 +217,7 @@ async def shutdown() -> None: _client = None _tracer_provider = None reset_ai_sdk_info() + reset_sdk_usage() if local_provider is not None: try: @@ -270,6 +275,7 @@ async def inspect_config( - ``config`` (dict | None): the parsed AI config, or None when disabled/invalid - ``meta`` (dict | None): the variation metadata, or None when unreachable """ + report_usage("client.inspectConfig") from .types_validation import parse_ai_config from .utils import to_ld_context # late import avoids circular dependency diff --git a/packages/client/src/launchdarkly_ai_server/sdk_usage.py b/packages/client/src/launchdarkly_ai_server/sdk_usage.py new file mode 100644 index 00000000..5152709b --- /dev/null +++ b/packages/client/src/launchdarkly_ai_server/sdk_usage.py @@ -0,0 +1,129 @@ +"""Records which public AI SDK helpers an application calls. + +One ``$ld:ai:sdk:usage`` event per helper per client. A call made before a +client exists is held and sent on the next init, on the same path as sdk-info. + +Each event names the core client (``aiSdkName`` / ``aiSdkVersion``) and the +package the helper belongs to (``helperPackageName`` / ``helperPackageVersion``). +Handler packages are versioned separately from the core, and sdk-info sends +every package on one shared context, so only this event ties a helper call to +the handler release that ran it. + +Only a call from outside the SDK reports. Each reporting helper has a +non-reporting internal version, and SDK code calls that version instead, so +this module never needs to know who the caller is. +""" + +from __future__ import annotations + +import threading +from typing import Any + +from .sdk_info import SDK_INFO_CONTEXT + +SDK_USAGE_EVENT = "$ld:ai:sdk:usage" +_SDK_USAGE_LANGUAGE = "python" +_SDK_NAME = "launchdarkly-ai-server" + +# A helper's package: (name, version), or None for the core client. +_Package = tuple[str, str] | None + +# Held helpers keep their package so the flushed event is complete. +_pending: dict[str, _Package] = {} +_reported: set[str] = set() +# Guards _pending and _reported. A helper is marked reported under the lock +# before it is delivered, so concurrent first calls send it once. The client is +# read under the lock too: init sets the client and then flushes, and the flush +# waits here, so a helper held just as init runs is still sent by that flush. +# Delivery runs outside the lock so a slow track() does not block other callers. +_lock = threading.Lock() + + +def _peek_client() -> Any: + from . import lifecycle + + return lifecycle._client + + +def _version() -> str: + from . import __version__ + + return __version__ + + +def _deliver(client: Any, helper: str, package: _Package) -> None: + from .utils import to_ld_context + + try: + core_version = _version() + package_name, package_version = package or (_SDK_NAME, core_version) + client.track( + SDK_USAGE_EVENT, + to_ld_context(client, SDK_INFO_CONTEXT), + { + "aiSdkName": _SDK_NAME, + "aiSdkVersion": core_version, + "aiSdkLanguage": _SDK_USAGE_LANGUAGE, + "helper": helper, + "helperPackageName": package_name, + "helperPackageVersion": package_version, + }, + 1, + ) + except Exception: + # A throw from track, or from building the payload, must not fail the helper. + return + + +def report_usage( + helper: str, + package_name: str | None = None, + package_version: str | None = None, +) -> None: + """Record a public helper. + + A handler package passes the same *package_name* and *package_version* it + gives :func:`register_ai_sdk_package`. A core client helper passes neither, + and the event names the core client. Pass both or neither; passing only one + reports the core client. + + Sends immediately when a client exists, otherwise holds the helper until + :func:`flush_sdk_usage`. No-ops when already reported or held; the package + is not part of that key. Never raises. + """ + try: + package: _Package = None + if package_name is not None and package_version is not None: + package = (package_name, package_version) + with _lock: + client = _peek_client() + if helper in _reported or helper in _pending: + return + if client is None: + _pending[helper] = package + return + _reported.add(helper) + _deliver(client, helper, package) + except Exception: + with _lock: + _pending.pop(helper, None) + _reported.add(helper) + + +def flush_sdk_usage(client: Any) -> None: + """Send every helper that was recorded before a client existed.""" + with _lock: + if not _pending: + return + held = list(_pending.items()) + _pending.clear() + _reported.update(helper for helper, _ in held) + for helper, package in held: + _deliver(client, helper, package) + + +def reset_sdk_usage() -> None: + """Drop the reported set so the next client hears each helper again.""" + with _lock: + _reported.clear() + _pending.clear() diff --git a/packages/client/src/launchdarkly_ai_server/utils.py b/packages/client/src/launchdarkly_ai_server/utils.py index 2958de6e..7c656abc 100644 --- a/packages/client/src/launchdarkly_ai_server/utils.py +++ b/packages/client/src/launchdarkly_ai_server/utils.py @@ -6,6 +6,7 @@ from dataclasses import dataclass, field from typing import Any, Literal +from .sdk_usage import report_usage from .types import ( AiConfigRep, GraphNode, @@ -28,6 +29,17 @@ def create_handler( Wraps a plain async callable in a :class:`ProviderHandler` with the given ``provides_for`` metadata and optional streaming implementation. """ + report_usage("client.createHandler") + return _create_handler(provides_for, fn, stream_fn, capture_content) + + +def _create_handler( + provides_for: tuple[str, Literal["agent", "messages"]], + fn: _HandlerFn, + stream_fn: _StreamFn | None = None, + capture_content: bool = False, +) -> ProviderHandler: + """Non-reporting :func:`create_handler`, used by the package factories.""" return ProviderHandler( fn=fn, provides_for=provides_for, diff --git a/packages/client/tests/test_client.py b/packages/client/tests/test_client.py index d7357308..1ea9f66a 100644 --- a/packages/client/tests/test_client.py +++ b/packages/client/tests/test_client.py @@ -22,6 +22,15 @@ # --------------------------------------------------------------------------- CONTEXT = {"kind": "user", "key": "u1"} +_SDK_BOOKKEEPING = {"$ld:ai:sdk:usage", "$ld:ai:sdk:info"} + + +def _generation_calls(client: MagicMock) -> list[Any]: + return [ + call + for call in client.track.call_args_list + if call[0][0] not in _SDK_BOOKKEEPING + ] def _make_client(response: Any = "Hello", usage: dict | None = None) -> MagicMock: @@ -127,8 +136,8 @@ async def test_track_data_copies_model_key_and_version_from_ld_meta( assert result.track_data["modelKey"] == "my-model" assert result.track_data["modelVersion"] == 3 assert isinstance(result.track_data["modelVersion"], int) - assert mock_ld_client.track.call_args_list - for call in mock_ld_client.track.call_args_list: + assert _generation_calls(mock_ld_client) + for call in _generation_calls(mock_ld_client): payload = call[0][2] assert payload["modelKey"] == "my-model" assert payload["modelVersion"] == 3 @@ -141,7 +150,7 @@ async def test_track_data_omits_model_key_and_version_when_absent( assert result.track_data is not None assert "modelKey" not in result.track_data assert "modelVersion" not in result.track_data - for call in mock_ld_client.track.call_args_list: + for call in _generation_calls(mock_ld_client): payload = call[0][2] assert "modelKey" not in payload assert "modelVersion" not in payload @@ -174,8 +183,8 @@ async def test_stream_track_data_copies_model_key_and_version( m = config(key="flag", handler=_make_handler(stream_chunks=["a", "b"])) async for _ in m.stream("q", CONTEXT): pass - assert mock_ld_client.track.call_args_list - for call in mock_ld_client.track.call_args_list: + assert _generation_calls(mock_ld_client) + for call in _generation_calls(mock_ld_client): payload = call[0][2] assert payload["modelKey"] == "my-model" assert payload["modelVersion"] == 3 diff --git a/packages/client/tests/test_sdk_usage.py b/packages/client/tests/test_sdk_usage.py new file mode 100644 index 00000000..0bf87082 --- /dev/null +++ b/packages/client/tests/test_sdk_usage.py @@ -0,0 +1,500 @@ +"""Tests for TESTING.md §3.27 helper usage events.""" + +from typing import Any +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +import launchdarkly_ai_server.lifecycle as lifecycle_module +import launchdarkly_ai_server.sdk_usage as sdk_usage_module +from launchdarkly_ai_server import ( + SDK_INFO_CONTEXT, + ProviderHandler, + __version__, + build_judge_tasks, + config, + create_handler, + graph, + init_client, + init_evaluations, + inspect_config, + resolve_graph, + run_judge, + shutdown, +) + +CONTEXT = {"kind": "user", "key": "user-1"} +HELPER = "client.buildJudgeTasks" + + +def _client(variation: Any = None) -> MagicMock: + client = MagicMock() + client.track = MagicMock() + client.variation = AsyncMock(return_value=variation) + client.flush = AsyncMock() + client.close = AsyncMock() + return client + + +async def _install(variation: Any = None) -> MagicMock: + await shutdown() + client = _client(variation) + with patch.object(lifecycle_module, "_setup_telemetry", return_value=None): + await init_client(client=client) + client.track.reset_mock() + return client + + +def _usage(client: MagicMock, helper: str | None = None) -> list[Any]: + return [ + call + for call in client.track.call_args_list + if call.args[0] == "$ld:ai:sdk:usage" + and (helper is None or call.args[2].get("helper") == helper) + ] + + +def _helpers(client: MagicMock) -> list[str]: + return [call.args[2].get("helper") for call in _usage(client)] + + +_USAGE_FIELDS = ( + "helper", + "aiSdkName", + "aiSdkVersion", + "aiSdkLanguage", + "helperPackageName", + "helperPackageVersion", +) + +_ENABLED_CONFIG: dict[str, Any] = { + "model": {"name": "gpt-4o"}, + "provider": {"name": "OpenAI"}, + "instructions": "Be helpful.", + "_ldMeta": { + "enabled": True, + "variationKey": "v1", + "version": 1, + "mode": "messages", + }, +} + + +def _expect_usage(client: MagicMock, helper: str) -> None: + calls = _usage(client, helper) + assert len(calls) == 1 + assert calls[0].args[1] == SDK_INFO_CONTEXT + assert calls[0].args[2] == { + "aiSdkName": "launchdarkly-ai-server", + "aiSdkVersion": __version__, + "aiSdkLanguage": "python", + "helper": helper, + "helperPackageName": "launchdarkly-ai-server", + "helperPackageVersion": __version__, + } + # A client.* helper belongs to the core package. + payload = calls[0].args[2] + assert payload["helperPackageName"] == payload["aiSdkName"] + assert payload["helperPackageVersion"] == payload["aiSdkVersion"] + assert calls[0].args[3] == 1 + + +async def _settle(value: Any) -> None: + if hasattr(value, "__await__"): + try: + await value + except Exception: + return + + +def _judge_kwargs() -> dict[str, Any]: + return { + "config": { + "model": {"name": "m"}, + "provider": {"name": "OpenAI"}, + "instructions": "i", + }, + "user_context": CONTEXT, + "handler": MagicMock(), + "llm_response": "", + "base_track_data": { + "runId": "r", + "configKey": "k", + "variationKey": "v", + "version": 1, + "modelName": "m", + "providerName": "OpenAI", + }, + } + + +def _call(helper: str) -> Any: + if helper == "client.config.invoke": + return config(key="k").invoke("q", CONTEXT) + if helper == "client.config.stream": + return config(key="k").stream("q", CONTEXT) + if helper == "client.graph.invoke": + return graph("k").invoke("q", CONTEXT) + if helper == "client.graph.stream": + return graph("k").stream("q", CONTEXT) + if helper == "client.resolveGraph": + return resolve_graph("k", context=CONTEXT) + if helper == "client.inspectConfig": + return inspect_config("k", CONTEXT) + if helper == "client.runJudge": + return run_judge(MagicMock(), []) + if helper == "client.buildJudgeTasks": + return build_judge_tasks(**_judge_kwargs()) + if helper == "client.createHandler": + + async def _fn(*_args: Any, **_kwargs: Any) -> dict[str, Any]: + return {"output": ""} + + create_handler(("OpenAI", "messages"), _fn) + return None + try: + init_evaluations() + except Exception: + return None + return None + + +@pytest.fixture +async def client() -> Any: + installed = await _install() + yield installed + await shutdown() + + +@pytest.mark.parametrize( + "helper", + [ + "client.config.invoke", + "client.config.stream", + "client.graph.invoke", + "client.graph.stream", + "client.resolveGraph", + "client.inspectConfig", + "client.runJudge", + "client.buildJudgeTasks", + "client.createHandler", + "client.init_evaluations", + ], +) +async def test_helper_sends_one_usage_event(client: MagicMock, helper: str) -> None: + await _settle(_call(helper)) + _expect_usage(client, helper) + # One call emits only its own helper: config.invoke emits no client.createHandler, + # client.buildJudgeTasks, and so on. + assert _helpers(client) == [helper] + if helper == "client.graph.invoke": + assert _usage(client, "client.graph.stream") == [] + if helper == "client.graph.stream": + assert _usage(client, "client.graph.invoke") == [] + + +async def test_repeat_call_sends_nothing_until_shutdown(client: MagicMock) -> None: + await build_judge_tasks(**_judge_kwargs()) + _expect_usage(client, HELPER) + await build_judge_tasks(**_judge_kwargs()) + assert len(_usage(client, HELPER)) == 1 + + await shutdown() + fresh = await _install() + await build_judge_tasks(**_judge_kwargs()) + _expect_usage(fresh, HELPER) + + +async def test_helper_before_client_is_sent_on_init() -> None: + await shutdown() + lifecycle_module._reset_for_testing() + await build_judge_tasks(**_judge_kwargs()) + client = _client() + assert _usage(client, HELPER) == [] + with patch.object(lifecycle_module, "_setup_telemetry", return_value=None): + await init_client(client=client) + _expect_usage(client, HELPER) + await shutdown() + + +async def test_held_helper_is_sent_on_sdk_key_init() -> None: + await shutdown() + lifecycle_module._reset_for_testing() + await build_judge_tasks(**_judge_kwargs()) + stub = _client() + ld_module = MagicMock() + ld_module.Config = MagicMock(return_value=MagicMock()) + ld_module.LDClient = MagicMock(return_value=stub) + with ( + patch.object(lifecycle_module, "_setup_telemetry", return_value=None), + patch("importlib.import_module", return_value=ld_module), + ): + await init_client({"sdkKey": "sdk-key"}) + ld_module.LDClient.assert_called_once() + _expect_usage(stub, HELPER) + await shutdown() + + +async def test_test_setter_does_not_flush_but_already_initialized_init_does() -> None: + await shutdown() + lifecycle_module._reset_for_testing() + await build_judge_tasks(**_judge_kwargs()) + client = _client() + lifecycle_module._set_client_for_testing(client) + assert _usage(client) == [] + + # init_client returns the existing client on this path and flushes what was held. + assert await init_client() is client + _expect_usage(client, HELPER) + await shutdown() + + +async def test_track_failure_does_not_fail_the_helper_or_retry( + client: MagicMock, +) -> None: + client.track.side_effect = RuntimeError("client closed") + result = await build_judge_tasks(**_judge_kwargs()) + assert result == [] + assert len(_usage(client, HELPER)) == 1 + + client.track.reset_mock(side_effect=True) + await build_judge_tasks(**_judge_kwargs()) + assert _usage(client, HELPER) == [] + + +async def test_payload_failure_does_not_fail_the_helper_or_retry( + client: MagicMock, +) -> None: + with patch.object( + sdk_usage_module, "_version", side_effect=RuntimeError("no version") + ) as version: + assert await build_judge_tasks(**_judge_kwargs()) == [] + assert await build_judge_tasks(**_judge_kwargs()) == [] + # Building the payload threw before track, once. The helper counts as reported. + assert version.call_count == 1 + assert _usage(client) == [] + await build_judge_tasks(**_judge_kwargs()) + assert _usage(client) == [] + + +async def test_held_payload_failure_on_flush_is_swallowed() -> None: + await shutdown() + lifecycle_module._reset_for_testing() + await build_judge_tasks(**_judge_kwargs()) + client = _client() + with ( + patch.object(lifecycle_module, "_setup_telemetry", return_value=None), + patch.object( + sdk_usage_module, "_version", side_effect=RuntimeError("no version") + ), + ): + assert await init_client(client=client) is client + assert _usage(client) == [] + await build_judge_tasks(**_judge_kwargs()) + assert _usage(client) == [] + await shutdown() + + +async def test_config_invoke_does_not_report_internal_judge_tasks() -> None: + """``skip_judges`` builds judge tasks through the internal path.""" + judged = { + **_ENABLED_CONFIG, + "judgeConfiguration": {"judges": [{"key": "judge", "samplingRate": 1}]}, + } + installed = await _install(judged) + try: + + async def _handler(*_args: Any, **_kwargs: Any) -> dict[str, Any]: + return {"output": "ok"} + + result = await config( + key="flag", + handler=ProviderHandler(fn=_handler, provides_for=("OpenAI", "messages")), + skip_judges=True, + ).invoke("q", CONTEXT) + assert result.judge_tasks + assert _helpers(installed) == ["client.config.invoke"] + finally: + await shutdown() + + +@pytest.mark.parametrize("method", ["invoke", "stream"]) +async def test_helpers_called_from_user_handler_and_tool_still_report( + method: str, +) -> None: + """Code the application passes in is outside the SDK, under invoke and stream alike.""" + installed = await _install(_ENABLED_CONFIG) + try: + + async def _lookup() -> str: + await build_judge_tasks(**_judge_kwargs()) + return "found" + + async def _handler( + _config: Any, + _input: Any, + tool_handlers: dict[str, Any], + *_a: Any, + **_k: Any, + ) -> dict[str, Any]: + create_handler(("OpenAI", "messages"), _handler) + return {"output": await tool_handlers["lookup"]()} + + instance = config( + key="flag", + handler=ProviderHandler(fn=_handler, provides_for=("OpenAI", "messages")), + tool_handlers={"lookup": _lookup}, + ) + if method == "invoke": + await instance.invoke("q", CONTEXT) + else: + async for _ in instance.stream("q", CONTEXT): + pass + + assert _helpers(installed) == [ + f"client.config.{method}", + "client.createHandler", + "client.buildJudgeTasks", + ] + tool_calls = [ + call + for call in installed.track.call_args_list + if call.args[0] == "$ld:ai:tool_call" + ] + assert len(tool_calls) == 1 + assert tool_calls[0].args[2]["toolKey"] == "lookup" + for field in _USAGE_FIELDS: + assert field not in tool_calls[0].args[2] + finally: + await shutdown() + + +def _graph_variation(key: str, _ctx: Any, _default: Any) -> Any: + if key == "graph-key": + return {"root": "root-node", "edges": {"root-node": [{"key": "leaf-node"}]}} + return {**_ENABLED_CONFIG, "provider": {"name": "TestProvider"}} + + +@pytest.mark.parametrize("method", ["invoke", "stream"]) +async def test_graph_events_gain_no_usage_fields(method: str) -> None: + installed = await _install() + installed.variation = AsyncMock(side_effect=_graph_variation) + try: + + async def _handler(*_args: Any, **_kwargs: Any) -> dict[str, Any]: + return {"output": "ok", "usage": {"input_tokens": 1, "output_tokens": 1}} + + instance = graph( + "graph-key", + handlers=[ + ProviderHandler(fn=_handler, provides_for=("TestProvider", "messages")) + ], + ) + if method == "invoke": + await instance.invoke("q", CONTEXT) + else: + async for _ in instance.stream("q", CONTEXT): + pass + + assert _helpers(installed) == [f"client.graph.{method}"] + graph_calls = [ + call + for call in installed.track.call_args_list + if str(call.args[0]).startswith("$ld:ai:graph:") + ] + assert "$ld:ai:graph:invocation_success" in [c.args[0] for c in graph_calls] + for call in graph_calls: + for field in _USAGE_FIELDS: + assert field not in call.args[2] + finally: + await shutdown() + + +async def test_generation_events_gain_no_usage_fields() -> None: + await shutdown() + try: + installed = await _install( + { + "model": {"name": "gpt-4o"}, + "provider": {"name": "OpenAI"}, + "instructions": "Be helpful.", + "_ldMeta": { + "enabled": True, + "variationKey": "v1", + "version": 1, + "mode": "messages", + }, + } + ) + + async def _handler(*_args: Any, **_kwargs: Any) -> dict[str, Any]: + return {"output": "ok", "usage": {"input_tokens": 2, "output_tokens": 3}} + + await config( + key="flag", + handler=ProviderHandler(fn=_handler, provides_for=("OpenAI", "messages")), + ).invoke("q", CONTEXT) + + _expect_usage(installed, "client.config.invoke") + others = [ + call + for call in installed.track.call_args_list + if call.args[0] not in {"$ld:ai:sdk:usage", "$ld:ai:sdk:info"} + ] + names = [call.args[0] for call in others] + assert "$ld:ai:duration:total" in names + assert "$ld:ai:generation:success" in names + assert "$ld:ai:tokens:total" in names + for call in others: + for field in _USAGE_FIELDS: + assert field not in call.args[2] + finally: + await shutdown() + + +def _package_payload(helper: str) -> dict[str, Any]: + return { + "aiSdkName": "launchdarkly-ai-server", + "aiSdkVersion": __version__, + "aiSdkLanguage": "python", + "helper": helper, + "helperPackageName": "launchdarkly-ai-example", + "helperPackageVersion": "9.8.7", + } + + +async def test_package_helper_names_its_package_and_the_core( + client: MagicMock, +) -> None: + sdk_usage_module.report_usage("example.helper", "launchdarkly-ai-example", "9.8.7") + calls = _usage(client, "example.helper") + assert [call.args[2] for call in calls] == [_package_payload("example.helper")] + + # The package is not part of the dedupe key. + sdk_usage_module.report_usage("example.helper", "launchdarkly-ai-other", "1.0.0") + assert len(_usage(client, "example.helper")) == 1 + + +async def test_held_package_helper_flushes_with_its_package() -> None: + await shutdown() + lifecycle_module._reset_for_testing() + sdk_usage_module.report_usage("example.helper", "launchdarkly-ai-example", "9.8.7") + await build_judge_tasks(**_judge_kwargs()) + client = _client() + with patch.object(lifecycle_module, "_setup_telemetry", return_value=None): + await init_client(client=client) + assert [call.args[2] for call in _usage(client, "example.helper")] == [ + _package_payload("example.helper") + ] + _expect_usage(client, HELPER) + await shutdown() + + +async def test_package_name_without_version_reports_the_core( + client: MagicMock, +) -> None: + sdk_usage_module.report_usage("example.helper", "launchdarkly-ai-example") + calls = _usage(client, "example.helper") + assert len(calls) == 1 + assert calls[0].args[2]["helperPackageName"] == "launchdarkly-ai-server" + assert calls[0].args[2]["helperPackageVersion"] == __version__ diff --git a/packages/langchain-agents/agents.md b/packages/langchain-agents/agents.md index 37a5bdbf..e771dc13 100644 --- a/packages/langchain-agents/agents.md +++ b/packages/langchain-agents/agents.md @@ -34,6 +34,7 @@ The handler is model-agnostic — it accepts any `BaseChatModel`. The default is | `src/launchdarkly_ai_langchain_agents/graph.py` | `langchain_graph()` convenience wrapper around `graph()` | | `src/launchdarkly_ai_langchain_agents/native_graph.py` | `to_lang_graph()` native graph adapter | | `src/launchdarkly_ai_langchain_agents/__init__.py` | Package exports | +| `src/launchdarkly_ai_langchain_agents/_version.py` | `PACKAGE_NAME` and `__version__`, sent with `$ld:ai:sdk:info` and `$ld:ai:sdk:usage` | --- diff --git a/packages/langchain-agents/src/launchdarkly_ai_langchain_agents/__init__.py b/packages/langchain-agents/src/launchdarkly_ai_langchain_agents/__init__.py index bef1b8b6..a38273bc 100644 --- a/packages/langchain-agents/src/launchdarkly_ai_langchain_agents/__init__.py +++ b/packages/langchain-agents/src/launchdarkly_ai_langchain_agents/__init__.py @@ -3,10 +3,9 @@ See https://launchdarkly.com/docs for usage. """ -__version__ = "0.2.4" # x-release-please-version - from launchdarkly_ai_server import register_ai_sdk_package +from ._version import PACKAGE_NAME, __version__ from .graph import langchain_graph from .handler import create_langchain_agents_handler, langchain_agents from .native_graph import to_lang_graph @@ -18,4 +17,4 @@ "to_lang_graph", ] -register_ai_sdk_package("launchdarkly-ai-langchain-agents", __version__) +register_ai_sdk_package(PACKAGE_NAME, __version__) diff --git a/packages/langchain-agents/src/launchdarkly_ai_langchain_agents/_version.py b/packages/langchain-agents/src/launchdarkly_ai_langchain_agents/_version.py new file mode 100644 index 00000000..b492a4b0 --- /dev/null +++ b/packages/langchain-agents/src/launchdarkly_ai_langchain_agents/_version.py @@ -0,0 +1,4 @@ +"""This package's identity, as sent with ``$ld:ai:sdk:info`` and ``$ld:ai:sdk:usage``.""" + +PACKAGE_NAME = "launchdarkly-ai-langchain-agents" +__version__ = "0.2.4" # x-release-please-version diff --git a/packages/langchain-agents/src/launchdarkly_ai_langchain_agents/graph.py b/packages/langchain-agents/src/launchdarkly_ai_langchain_agents/graph.py index 95dc0c11..4c83f00d 100644 --- a/packages/langchain-agents/src/launchdarkly_ai_langchain_agents/graph.py +++ b/packages/langchain-agents/src/launchdarkly_ai_langchain_agents/graph.py @@ -4,8 +4,10 @@ from typing import Any -from launchdarkly_ai_langchain_agents.handler import create_langchain_agents_handler -from launchdarkly_ai_server import graph +from launchdarkly_ai_langchain_agents.handler import _create_langchain_agents_handler +from launchdarkly_ai_server import _graph, report_usage + +from ._version import PACKAGE_NAME, __version__ def langchain_graph(key: str, llm: Any = None, **options: Any) -> Any: @@ -15,4 +17,5 @@ def langchain_graph(key: str, llm: Any = None, **options: Any) -> Any: Equivalent to ``graph(key, handlers=[create_langchain_agents_handler(llm)], **options)``. Use the base ``graph()`` directly for multi-provider graphs. """ - return graph(key, handlers=[create_langchain_agents_handler(llm)], **options) + report_usage("langchain-agents.langchainGraph", PACKAGE_NAME, __version__) + return _graph(key, handlers=[_create_langchain_agents_handler(llm)], **options) diff --git a/packages/langchain-agents/src/launchdarkly_ai_langchain_agents/handler.py b/packages/langchain-agents/src/launchdarkly_ai_langchain_agents/handler.py index 41c69ed6..ef600b48 100644 --- a/packages/langchain-agents/src/launchdarkly_ai_langchain_agents/handler.py +++ b/packages/langchain-agents/src/launchdarkly_ai_langchain_agents/handler.py @@ -17,9 +17,9 @@ ProviderHandler, SpanMessage, SpanMessagePart, + _config, + _create_handler, compose_history, - config, - create_handler, create_run_usage, end_span_once, end_unfinished_spans, @@ -27,10 +27,12 @@ lang_chain_span_messages, lang_chain_span_usage, parse_template, + report_usage, set_input_content_attributes, set_output_content_attributes, ) +from ._version import PACKAGE_NAME, __version__ from .messages import to_lang_chain_messages from .spans import ( build_span_callbacks, @@ -265,6 +267,16 @@ def create_langchain_agents_handler( emitted spans. It defaults to off. Conversation content is PII, so a run emits only metadata, meaning models, token counts, timings and tool names, until a caller asks for more. """ + report_usage( + "langchain-agents.createLangChainAgentsHandler", PACKAGE_NAME, __version__ + ) + return _create_langchain_agents_handler(llm, capture_content=capture_content) + + +def _create_langchain_agents_handler( + llm: Any = None, *, capture_content: bool = False +) -> ProviderHandler: + """Non-reporting :func:`create_langchain_agents_handler`, used by this package's wrappers.""" async def _call_impl( config: AiConfigRep, @@ -412,7 +424,7 @@ def _stream_impl( capture_content=capture_content, ) - return create_handler( + return _create_handler( ("*", "agent"), _call_impl, # type: ignore[arg-type] _stream_impl, # type: ignore[arg-type] @@ -570,10 +582,11 @@ def langchain_agents( # Both are lifted out of kwargs: capture_content configures the handler, variables belong to # the invocation. Leaving either in would pass it to config(), which takes neither, so a caller # asking for content on spans got a TypeError instead of content. + report_usage("langchain-agents.langchainAgents", PACKAGE_NAME, __version__) variables = kwargs.pop("variables", None) capture_content = kwargs.pop("capture_content", False) - return config( + return _config( key=config_key, - handler=create_langchain_agents_handler(capture_content=capture_content), + handler=_create_langchain_agents_handler(capture_content=capture_content), **kwargs, ).invoke(user_input, context, variables=variables) diff --git a/packages/langchain-agents/src/launchdarkly_ai_langchain_agents/native_graph.py b/packages/langchain-agents/src/launchdarkly_ai_langchain_agents/native_graph.py index 190e5f87..039fe0fc 100644 --- a/packages/langchain-agents/src/launchdarkly_ai_langchain_agents/native_graph.py +++ b/packages/langchain-agents/src/launchdarkly_ai_langchain_agents/native_graph.py @@ -20,9 +20,11 @@ get_client, make_track_data, parse_template, + report_usage, to_ld_context, ) +from ._version import PACKAGE_NAME, __version__ from .messages import to_lang_chain_messages try: @@ -132,6 +134,7 @@ def to_lang_graph( {"context": ctx}, ).invoke("I was double charged") """ + report_usage("langchain-agents.toLangGraph", PACKAGE_NAME, __version__) _opts = opts or {} async def invoke( diff --git a/packages/langchain-agents/tests/test_graph.py b/packages/langchain-agents/tests/test_graph.py index ce5e455a..e6ea186f 100644 --- a/packages/langchain-agents/tests/test_graph.py +++ b/packages/langchain-agents/tests/test_graph.py @@ -10,14 +10,14 @@ class TestLangChainGraph: def test_graph_called_with_correct_key(self) -> None: - with patch("launchdarkly_ai_langchain_agents.graph.graph") as mock_graph: + with patch("launchdarkly_ai_langchain_agents.graph._graph") as mock_graph: mock_graph.return_value = MagicMock() langchain_graph("my-flag-key") mock_graph.assert_called_once() assert mock_graph.call_args[0][0] == "my-flag-key" def test_handlers_pre_populated_with_langchain_agents_handler(self) -> None: - with patch("launchdarkly_ai_langchain_agents.graph.graph") as mock_graph: + with patch("launchdarkly_ai_langchain_agents.graph._graph") as mock_graph: mock_graph.return_value = MagicMock() langchain_graph("key") kw = mock_graph.call_args[1] @@ -26,14 +26,14 @@ def test_handlers_pre_populated_with_langchain_agents_handler(self) -> None: def test_user_supplied_options_forwarded(self) -> None: ctx = {"kind": "user", "key": "u1"} - with patch("launchdarkly_ai_langchain_agents.graph.graph") as mock_graph: + with patch("launchdarkly_ai_langchain_agents.graph._graph") as mock_graph: mock_graph.return_value = MagicMock() langchain_graph("key", context=ctx) kw = mock_graph.call_args[1] assert kw.get("context") == ctx def test_user_cannot_override_handlers(self) -> None: - with patch("launchdarkly_ai_langchain_agents.graph.graph") as mock_graph: + with patch("launchdarkly_ai_langchain_agents.graph._graph") as mock_graph: mock_graph.return_value = MagicMock() langchain_graph("key", extra=1) kw = mock_graph.call_args[1] @@ -42,7 +42,7 @@ def test_user_cannot_override_handlers(self) -> None: def test_llm_option_forwarded(self) -> None: mock_llm = MagicMock() - with patch("launchdarkly_ai_langchain_agents.graph.graph") as mock_graph: + with patch("launchdarkly_ai_langchain_agents.graph._graph") as mock_graph: mock_graph.return_value = MagicMock() langchain_graph("key", llm=mock_llm) # llm is consumed by create_langchain_agents_handler(llm), not forwarded to graph diff --git a/packages/langchain-agents/tests/test_handler.py b/packages/langchain-agents/tests/test_handler.py index e43cb3b0..c6d3b2fa 100644 --- a/packages/langchain-agents/tests/test_handler.py +++ b/packages/langchain-agents/tests/test_handler.py @@ -1326,7 +1326,7 @@ def test_config_key_forwarded_as_key(self) -> None: mock_config_fn = MagicMock(return_value=mock_config_instance) mock_config_instance.invoke = MagicMock(return_value="result") - with patch.object(handler_mod, "config", mock_config_fn): + with patch.object(handler_mod, "_config", mock_config_fn): from launchdarkly_ai_langchain_agents.handler import langchain_agents ctx = {"kind": "user", "key": "u1"} @@ -1348,7 +1348,7 @@ def test_callable_without_extra_kwargs(self) -> None: mock_config_fn = MagicMock(return_value=mock_config_instance) mock_config_instance.invoke = MagicMock(return_value="result") - with patch.object(handler_mod, "config", mock_config_fn): + with patch.object(handler_mod, "_config", mock_config_fn): from launchdarkly_ai_langchain_agents.handler import langchain_agents ctx = {"kind": "user", "key": "u1"} @@ -1836,8 +1836,8 @@ def _factory(*args: Any, capture_content: bool = False, **kw: Any) -> Any: fake_config = MagicMock() fake_config.return_value.invoke = MagicMock(return_value="ok") with ( - patch.object(handler_mod, "create_langchain_agents_handler", _factory), - patch.object(handler_mod, "config", fake_config), + patch.object(handler_mod, "_create_langchain_agents_handler", _factory), + patch.object(handler_mod, "_config", fake_config), ): handler_mod.langchain_agents("k", "q", {}, **kwargs) seen["config_kwargs"] = fake_config.call_args.kwargs diff --git a/packages/langchain-agents/tests/test_sdk_usage.py b/packages/langchain-agents/tests/test_sdk_usage.py new file mode 100644 index 00000000..27a0ced8 --- /dev/null +++ b/packages/langchain-agents/tests/test_sdk_usage.py @@ -0,0 +1,155 @@ +"""Tests for TESTING.md §3.27 — langchain-agents.""" + +from types import SimpleNamespace +from typing import Any +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +import launchdarkly_ai_server.lifecycle as lifecycle_module +from launchdarkly_ai_langchain_agents import ( + create_langchain_agents_handler, + langchain_agents, + langchain_graph, + to_lang_graph, +) +from launchdarkly_ai_langchain_agents._version import PACKAGE_NAME +from launchdarkly_ai_langchain_agents._version import __version__ as PACKAGE_VERSION +from launchdarkly_ai_server import SDK_INFO_CONTEXT, __version__, init_client, shutdown + +CONTEXT = {"kind": "user", "key": "user-1"} + + +def _fake() -> MagicMock: + client = MagicMock() + client.track = MagicMock() + client.variation = AsyncMock(return_value=None) + client.flush = AsyncMock() + client.close = AsyncMock() + return client + + +async def _settle(value: Any) -> None: + if hasattr(value, "__await__"): + try: + await value + except Exception: + return + + +@pytest.fixture +async def client() -> Any: + await shutdown() + installed = _fake() + with patch.object(lifecycle_module, "_setup_telemetry", return_value=None): + await init_client(client=installed) + installed.track.reset_mock() + yield installed + await shutdown() + + +@pytest.mark.parametrize( + ("helper", "call"), + [ + ( + "langchain-agents.langchainAgents", + lambda: langchain_agents("k", "q", CONTEXT), + ), + ( + "langchain-agents.createLangChainAgentsHandler", + create_langchain_agents_handler, + ), + ("langchain-agents.langchainGraph", lambda: langchain_graph("k")), + ("langchain-agents.toLangGraph", lambda: to_lang_graph(None)), + ], +) +async def test_helper_sends_one_usage_event( + client: MagicMock, helper: str, call: Any +) -> None: + try: + await _settle(call()) + except Exception: + pass + calls = [ + entry + for entry in client.track.call_args_list + if entry.args[0] == "$ld:ai:sdk:usage" + ] + assert [entry.args[2].get("helper") for entry in calls] == [helper] + assert calls[0].args[1] == SDK_INFO_CONTEXT + assert calls[0].args[2] == { + "aiSdkName": "launchdarkly-ai-server", + "aiSdkVersion": __version__, + "aiSdkLanguage": "python", + "helper": helper, + "helperPackageName": PACKAGE_NAME, + "helperPackageVersion": PACKAGE_VERSION, + } + assert calls[0].args[3] == 1 + + +def _helpers(client: MagicMock) -> list[str]: + return [ + entry.args[2].get("helper") + for entry in client.track.call_args_list + if entry.args[0] == "$ld:ai:sdk:usage" + ] + + +async def _drain(stream: Any) -> None: + try: + async for _ in stream: + pass + except Exception: + return + + +async def test_graph_wrapper_invoke_and_stream_report_only_the_wrapper( + client: MagicMock, +) -> None: + """The returned graph's methods are not separate ``client.graph.*`` calls.""" + instance = langchain_graph("k") + await _settle(instance.invoke("q", CONTEXT)) + await _drain(instance.stream("q", CONTEXT)) + assert _helpers(client) == ["langchain-agents.langchainGraph"] + + +async def test_native_adapter_invoke_reports_only_the_adapter( + client: MagicMock, +) -> None: + async def _definition() -> Any: + return SimpleNamespace(enabled=False, key="k") + + runner = to_lang_graph(_definition(), {"context": CONTEXT}) + with pytest.raises(ValueError, match="disabled"): + await runner.invoke("q") + assert _helpers(client) == ["langchain-agents.toLangGraph"] + + +async def test_held_helper_flushes_with_its_package() -> None: + """A helper called before init keeps its package fields until the flush.""" + await shutdown() + lifecycle_module._reset_for_testing() + try: + create_langchain_agents_handler() + except Exception: + pass # The factory may need credentials; it reports first. + installed = _fake() + with patch.object(lifecycle_module, "_setup_telemetry", return_value=None): + await init_client(client=installed) + calls = [ + entry.args[2] + for entry in installed.track.call_args_list + if entry.args[0] == "$ld:ai:sdk:usage" + ] + assert calls == [ + { + "aiSdkName": "launchdarkly-ai-server", + "aiSdkVersion": __version__, + "aiSdkLanguage": "python", + "helper": "langchain-agents.createLangChainAgentsHandler", + "helperPackageName": PACKAGE_NAME, + "helperPackageVersion": PACKAGE_VERSION, + } + ] + await shutdown() diff --git a/packages/langchain-messages/agents.md b/packages/langchain-messages/agents.md index 78314502..6361a8d3 100644 --- a/packages/langchain-messages/agents.md +++ b/packages/langchain-messages/agents.md @@ -32,6 +32,7 @@ The handler is model-agnostic — it accepts any `BaseChatModel`. The default is |---|---| | `src/launchdarkly_ai_langchain_messages/handler.py` | All implementation — message building, tool binding, invoke loop, telemetry | | `src/launchdarkly_ai_langchain_messages/__init__.py` | Package exports | +| `src/launchdarkly_ai_langchain_messages/_version.py` | `PACKAGE_NAME` and `__version__`, sent with `$ld:ai:sdk:info` and `$ld:ai:sdk:usage` | --- diff --git a/packages/langchain-messages/src/launchdarkly_ai_langchain_messages/__init__.py b/packages/langchain-messages/src/launchdarkly_ai_langchain_messages/__init__.py index f6bf8df9..103ca420 100644 --- a/packages/langchain-messages/src/launchdarkly_ai_langchain_messages/__init__.py +++ b/packages/langchain-messages/src/launchdarkly_ai_langchain_messages/__init__.py @@ -3,12 +3,11 @@ See https://launchdarkly.com/docs for usage. """ -__version__ = "0.2.3" # x-release-please-version - from launchdarkly_ai_server import register_ai_sdk_package +from ._version import PACKAGE_NAME, __version__ from .handler import create_langchain_messages_handler, langchain_messages __all__ = ["create_langchain_messages_handler", "langchain_messages"] -register_ai_sdk_package("launchdarkly-ai-langchain-messages", __version__) +register_ai_sdk_package(PACKAGE_NAME, __version__) diff --git a/packages/langchain-messages/src/launchdarkly_ai_langchain_messages/_version.py b/packages/langchain-messages/src/launchdarkly_ai_langchain_messages/_version.py new file mode 100644 index 00000000..bc2964f0 --- /dev/null +++ b/packages/langchain-messages/src/launchdarkly_ai_langchain_messages/_version.py @@ -0,0 +1,4 @@ +"""This package's identity, as sent with ``$ld:ai:sdk:info`` and ``$ld:ai:sdk:usage``.""" + +PACKAGE_NAME = "launchdarkly-ai-langchain-messages" +__version__ = "0.2.3" # x-release-please-version diff --git a/packages/langchain-messages/src/launchdarkly_ai_langchain_messages/handler.py b/packages/langchain-messages/src/launchdarkly_ai_langchain_messages/handler.py index b30e8b80..e9089166 100644 --- a/packages/langchain-messages/src/launchdarkly_ai_langchain_messages/handler.py +++ b/packages/langchain-messages/src/launchdarkly_ai_langchain_messages/handler.py @@ -13,9 +13,9 @@ SpanMessage, SpanMessagePart, SpanUsage, + _config, + _create_handler, compose_history, - config, - create_handler, create_run_usage, end_span_once, end_unfinished_spans, @@ -27,11 +27,13 @@ lang_chain_span_usage, number_or_zero, parse_template, + report_usage, set_input_content_attributes, set_output_content_attributes, set_tool_call_content_attributes, ) +from ._version import PACKAGE_NAME, __version__ from .spans import ( fail_span, finish_model_span, @@ -384,6 +386,14 @@ def create_langchain_messages_handler( emitted spans. It defaults to off. Conversation content is PII, so a run emits only metadata, meaning models, token counts, timings and tool names, until a caller asks for more. """ + report_usage("langchain-messages.createLangChainHandler", PACKAGE_NAME, __version__) + return _create_langchain_messages_handler(llm, capture_content=capture_content) + + +def _create_langchain_messages_handler( + llm: Any = None, *, capture_content: bool = False +) -> ProviderHandler: + """Non-reporting :func:`create_langchain_messages_handler`, used by this package's wrappers.""" async def _call_impl( config: AiConfigRep, @@ -694,7 +704,7 @@ def _stream_impl( capture_content=capture_content, ) - return create_handler( + return _create_handler( ("*", "messages"), _call_impl, # type: ignore[arg-type] _stream_impl, # type: ignore[arg-type] @@ -1013,12 +1023,13 @@ def langchain_messages( # Both are lifted out of kwargs: capture_content configures the handler, variables belong to # the invocation. Leaving either in would pass it to config(), which takes neither, so a caller # asking for content on spans got a TypeError instead of content. + report_usage("langchain-messages.langchainMessages", PACKAGE_NAME, __version__) variables = kwargs.pop("variables", None) capture_content = kwargs.pop("capture_content", False) - return config( + return _config( key=config_key, - handler=create_langchain_messages_handler( - llm=llm, capture_content=capture_content + handler=_create_langchain_messages_handler( + llm, capture_content=capture_content ), **kwargs, ).invoke(user_input, context, variables=variables) diff --git a/packages/langchain-messages/tests/test_handler.py b/packages/langchain-messages/tests/test_handler.py index d00e43fe..c305be6d 100644 --- a/packages/langchain-messages/tests/test_handler.py +++ b/packages/langchain-messages/tests/test_handler.py @@ -1109,7 +1109,7 @@ def test_calls_through_to_model_call(self) -> None: mock_config_fn = MagicMock(return_value=mock_config_instance) mock_config_instance.invoke = MagicMock(return_value="result") - with patch.object(handler_mod, "config", mock_config_fn): + with patch.object(handler_mod, "_config", mock_config_fn): from launchdarkly_ai_langchain_messages.handler import langchain_messages ctx = {"kind": "user", "key": "u1"} @@ -1132,7 +1132,7 @@ def test_callable_without_extra_kwargs(self) -> None: mock_config_fn = MagicMock(return_value=mock_config_instance) mock_config_instance.invoke = MagicMock(return_value="result") - with patch.object(handler_mod, "config", mock_config_fn): + with patch.object(handler_mod, "_config", mock_config_fn): from launchdarkly_ai_langchain_messages.handler import langchain_messages ctx = {"kind": "user", "key": "u1"} @@ -1642,8 +1642,8 @@ def _factory(*args: Any, capture_content: bool = False, **kw: Any) -> Any: fake_config = MagicMock() fake_config.return_value.invoke = MagicMock(return_value="ok") with ( - patch.object(handler_mod, "create_langchain_messages_handler", _factory), - patch.object(handler_mod, "config", fake_config), + patch.object(handler_mod, "_create_langchain_messages_handler", _factory), + patch.object(handler_mod, "_config", fake_config), ): handler_mod.langchain_messages("k", "q", {}, **kwargs) seen["config_kwargs"] = fake_config.call_args.kwargs diff --git a/packages/langchain-messages/tests/test_sdk_usage.py b/packages/langchain-messages/tests/test_sdk_usage.py new file mode 100644 index 00000000..e297e4c6 --- /dev/null +++ b/packages/langchain-messages/tests/test_sdk_usage.py @@ -0,0 +1,112 @@ +"""Tests for TESTING.md §3.27 — langchain-messages.""" + +from typing import Any +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +import launchdarkly_ai_server.lifecycle as lifecycle_module +from launchdarkly_ai_langchain_messages import ( + create_langchain_messages_handler, + langchain_messages, +) +from launchdarkly_ai_langchain_messages._version import PACKAGE_NAME +from launchdarkly_ai_langchain_messages._version import __version__ as PACKAGE_VERSION +from launchdarkly_ai_server import SDK_INFO_CONTEXT, __version__, init_client, shutdown + +CONTEXT = {"kind": "user", "key": "user-1"} + + +def _fake() -> MagicMock: + client = MagicMock() + client.track = MagicMock() + client.variation = AsyncMock(return_value=None) + client.flush = AsyncMock() + client.close = AsyncMock() + return client + + +async def _settle(value: Any) -> None: + if hasattr(value, "__await__"): + try: + await value + except Exception: + return + + +@pytest.fixture +async def client() -> Any: + await shutdown() + installed = _fake() + with patch.object(lifecycle_module, "_setup_telemetry", return_value=None): + await init_client(client=installed) + installed.track.reset_mock() + yield installed + await shutdown() + + +@pytest.mark.parametrize( + ("helper", "call"), + [ + ( + "langchain-messages.langchainMessages", + lambda: langchain_messages("k", "q", CONTEXT), + ), + ( + "langchain-messages.createLangChainHandler", + create_langchain_messages_handler, + ), + ], +) +async def test_helper_sends_one_usage_event( + client: MagicMock, helper: str, call: Any +) -> None: + try: + await _settle(call()) + except Exception: + pass + calls = [ + entry + for entry in client.track.call_args_list + if entry.args[0] == "$ld:ai:sdk:usage" + ] + assert [entry.args[2].get("helper") for entry in calls] == [helper] + assert calls[0].args[1] == SDK_INFO_CONTEXT + assert calls[0].args[2] == { + "aiSdkName": "launchdarkly-ai-server", + "aiSdkVersion": __version__, + "aiSdkLanguage": "python", + "helper": helper, + "helperPackageName": PACKAGE_NAME, + "helperPackageVersion": PACKAGE_VERSION, + } + assert calls[0].args[3] == 1 + + +async def test_held_helper_flushes_with_its_package() -> None: + """A helper called before init keeps its package fields until the flush.""" + await shutdown() + lifecycle_module._reset_for_testing() + try: + create_langchain_messages_handler() + except Exception: + pass # The factory may need credentials; it reports first. + installed = _fake() + with patch.object(lifecycle_module, "_setup_telemetry", return_value=None): + await init_client(client=installed) + calls = [ + entry.args[2] + for entry in installed.track.call_args_list + if entry.args[0] == "$ld:ai:sdk:usage" + ] + assert calls == [ + { + "aiSdkName": "launchdarkly-ai-server", + "aiSdkVersion": __version__, + "aiSdkLanguage": "python", + "helper": "langchain-messages.createLangChainHandler", + "helperPackageName": PACKAGE_NAME, + "helperPackageVersion": PACKAGE_VERSION, + } + ] + await shutdown() diff --git a/packages/openai-agents/agents.md b/packages/openai-agents/agents.md index f86aee58..414dcc9f 100644 --- a/packages/openai-agents/agents.md +++ b/packages/openai-agents/agents.md @@ -31,6 +31,7 @@ That means the LaunchDarkly flag variation must have `provider.name == "OpenAI"` | `src/launchdarkly_ai_openai_agents/native_graph.py` | `to_openai_agents()` native graph adapter | | `src/launchdarkly_ai_openai_agents/utils.py` | Shared utility helpers (e.g. `await_coroutine_or_run`) | | `src/launchdarkly_ai_openai_agents/__init__.py` | Package exports | +| `src/launchdarkly_ai_openai_agents/_version.py` | `PACKAGE_NAME` and `__version__`, sent with `$ld:ai:sdk:info` and `$ld:ai:sdk:usage` | --- diff --git a/packages/openai-agents/src/launchdarkly_ai_openai_agents/__init__.py b/packages/openai-agents/src/launchdarkly_ai_openai_agents/__init__.py index 04e788e3..0475189c 100644 --- a/packages/openai-agents/src/launchdarkly_ai_openai_agents/__init__.py +++ b/packages/openai-agents/src/launchdarkly_ai_openai_agents/__init__.py @@ -3,11 +3,10 @@ See https://launchdarkly.com/docs for usage. """ -__version__ = "0.2.4" # x-release-please-version - from launchdarkly_ai_server import register_ai_sdk_package from . import native_graph # noqa: F401 +from ._version import PACKAGE_NAME, __version__ from .graph import openai_graph from .handler import create_openai_agent_handler, openai_agents from .native_graph import to_openai_agents @@ -21,4 +20,4 @@ "to_openai_agents", ] -register_ai_sdk_package("launchdarkly-ai-openai-agents", __version__) +register_ai_sdk_package(PACKAGE_NAME, __version__) diff --git a/packages/openai-agents/src/launchdarkly_ai_openai_agents/_version.py b/packages/openai-agents/src/launchdarkly_ai_openai_agents/_version.py new file mode 100644 index 00000000..ccb6ea43 --- /dev/null +++ b/packages/openai-agents/src/launchdarkly_ai_openai_agents/_version.py @@ -0,0 +1,4 @@ +"""This package's identity, as sent with ``$ld:ai:sdk:info`` and ``$ld:ai:sdk:usage``.""" + +PACKAGE_NAME = "launchdarkly-ai-openai-agents" +__version__ = "0.2.4" # x-release-please-version diff --git a/packages/openai-agents/src/launchdarkly_ai_openai_agents/graph.py b/packages/openai-agents/src/launchdarkly_ai_openai_agents/graph.py index bac10979..b509f688 100644 --- a/packages/openai-agents/src/launchdarkly_ai_openai_agents/graph.py +++ b/packages/openai-agents/src/launchdarkly_ai_openai_agents/graph.py @@ -4,8 +4,10 @@ from typing import Any -from launchdarkly_ai_openai_agents.handler import create_openai_agent_handler -from launchdarkly_ai_server import graph +from launchdarkly_ai_openai_agents.handler import _create_openai_agent_handler +from launchdarkly_ai_server import _graph, report_usage + +from ._version import PACKAGE_NAME, __version__ def openai_graph(key: str, **options: Any) -> Any: @@ -15,4 +17,5 @@ def openai_graph(key: str, **options: Any) -> Any: Equivalent to ``graph(key, handlers=[create_openai_agent_handler()], **options)``. Use the base ``graph()`` directly for multi-provider graphs. """ - return graph(key, handlers=[create_openai_agent_handler()], **options) + report_usage("openai-agents.openaiGraph", PACKAGE_NAME, __version__) + return _graph(key, handlers=[_create_openai_agent_handler()], **options) diff --git a/packages/openai-agents/src/launchdarkly_ai_openai_agents/handler.py b/packages/openai-agents/src/launchdarkly_ai_openai_agents/handler.py index ac97a9aa..355aaab9 100644 --- a/packages/openai-agents/src/launchdarkly_ai_openai_agents/handler.py +++ b/packages/openai-agents/src/launchdarkly_ai_openai_agents/handler.py @@ -30,21 +30,23 @@ ProviderHandler, RunUsage, SpanUsage, + _config, + _create_handler, compose_history, - config, content_to_text, - create_handler, create_run_usage, end_span_once, end_unfinished_spans, image_block_to_url, parse_template, + report_usage, set_input_content_attributes, set_output_content_attributes, set_tool_call_content_attributes, text_message, ) +from ._version import PACKAGE_NAME, __version__ from .spans import ( derive_finish_reason, fail_span, @@ -464,6 +466,12 @@ def create_openai_agent_handler(*, capture_content: bool = False) -> ProviderHan emitted spans. It defaults to off. Conversation content is PII, so a run emits only metadata, meaning models, token counts, timings and tool names, until a caller asks for more. """ + report_usage("openai-agents.createOpenAIAgentHandler", PACKAGE_NAME, __version__) + return _create_openai_agent_handler(capture_content=capture_content) + + +def _create_openai_agent_handler(*, capture_content: bool = False) -> ProviderHandler: + """Non-reporting :func:`create_openai_agent_handler`, used by this package's wrappers.""" async def _call_impl( config: AiConfigRep, @@ -560,7 +568,7 @@ def _stream_impl( capture_content=capture_content, ) - return create_handler( + return _create_handler( ("OpenAI", "agent"), _call_impl, # type: ignore[arg-type] _stream_impl, # type: ignore[arg-type] @@ -712,10 +720,11 @@ def openai_agents( # Both are lifted out of kwargs: capture_content configures the handler, variables belong to # the invocation. Leaving either in would pass it to config(), which takes neither, so a caller # asking for content on spans got a TypeError instead of content. + report_usage("openai-agents.openaiAgents", PACKAGE_NAME, __version__) variables = kwargs.pop("variables", None) capture_content = kwargs.pop("capture_content", False) - return config( + return _config( key=config_key, - handler=create_openai_agent_handler(capture_content=capture_content), + handler=_create_openai_agent_handler(capture_content=capture_content), **kwargs, ).invoke(user_input, context, variables=variables) diff --git a/packages/openai-agents/src/launchdarkly_ai_openai_agents/native_graph.py b/packages/openai-agents/src/launchdarkly_ai_openai_agents/native_graph.py index ac8b0a90..eb771ab8 100644 --- a/packages/openai-agents/src/launchdarkly_ai_openai_agents/native_graph.py +++ b/packages/openai-agents/src/launchdarkly_ai_openai_agents/native_graph.py @@ -20,9 +20,11 @@ get_client, make_track_data, parse_template, + report_usage, to_ld_context, ) +from ._version import PACKAGE_NAME, __version__ from .handler import _parse_message_content, _to_openai_agent_items try: @@ -102,6 +104,7 @@ def to_openai_agents( {"context": ctx}, ).invoke("I was double charged") """ + report_usage("openai-agents.toOpenAIAgents", PACKAGE_NAME, __version__) _opts = opts or {} async def invoke( diff --git a/packages/openai-agents/tests/test_graph.py b/packages/openai-agents/tests/test_graph.py index 29c6af40..b7eeab0f 100644 --- a/packages/openai-agents/tests/test_graph.py +++ b/packages/openai-agents/tests/test_graph.py @@ -10,14 +10,14 @@ class TestOpenAIGraph: def test_graph_called_with_correct_key(self) -> None: - with patch("launchdarkly_ai_openai_agents.graph.graph") as mock_graph: + with patch("launchdarkly_ai_openai_agents.graph._graph") as mock_graph: mock_graph.return_value = MagicMock() openai_graph("my-flag-key") mock_graph.assert_called_once() assert mock_graph.call_args[0][0] == "my-flag-key" def test_handlers_pre_populated_with_openai_agent_handler(self) -> None: - with patch("launchdarkly_ai_openai_agents.graph.graph") as mock_graph: + with patch("launchdarkly_ai_openai_agents.graph._graph") as mock_graph: mock_graph.return_value = MagicMock() openai_graph("key") kw = mock_graph.call_args[1] @@ -26,14 +26,14 @@ def test_handlers_pre_populated_with_openai_agent_handler(self) -> None: def test_user_supplied_options_forwarded(self) -> None: ctx = {"kind": "user", "key": "u1"} - with patch("launchdarkly_ai_openai_agents.graph.graph") as mock_graph: + with patch("launchdarkly_ai_openai_agents.graph._graph") as mock_graph: mock_graph.return_value = MagicMock() openai_graph("key", context=ctx) kw = mock_graph.call_args[1] assert kw.get("context") == ctx def test_user_cannot_override_handlers(self) -> None: - with patch("launchdarkly_ai_openai_agents.graph.graph") as mock_graph: + with patch("launchdarkly_ai_openai_agents.graph._graph") as mock_graph: mock_graph.return_value = MagicMock() openai_graph("key", extra=1) kw = mock_graph.call_args[1] diff --git a/packages/openai-agents/tests/test_handler.py b/packages/openai-agents/tests/test_handler.py index 75e20003..fe0d4af2 100644 --- a/packages/openai-agents/tests/test_handler.py +++ b/packages/openai-agents/tests/test_handler.py @@ -1225,7 +1225,7 @@ def test_config_key_forwarded_as_key(self) -> None: mock_config_fn = MagicMock(return_value=mock_config_instance) mock_config_instance.invoke = MagicMock(return_value="result") - with patch.object(handler_mod, "config", mock_config_fn): + with patch.object(handler_mod, "_config", mock_config_fn): ctx = {"kind": "user", "key": "u1"} openai_agents("my-flag", "hello", ctx) @@ -1242,7 +1242,7 @@ def test_callable_without_extra_kwargs(self) -> None: mock_config_fn = MagicMock(return_value=mock_config_instance) mock_config_instance.invoke = MagicMock(return_value="result") - with patch.object(handler_mod, "config", mock_config_fn): + with patch.object(handler_mod, "_config", mock_config_fn): ctx = {"kind": "user", "key": "u1"} openai_agents("my-flag", "hello", ctx) @@ -1691,8 +1691,8 @@ def _factory(*, capture_content: bool = False) -> Any: fake_config = MagicMock() fake_config.return_value.invoke = MagicMock(return_value="ok") with ( - patch.object(handler_mod, "create_openai_agent_handler", _factory), - patch.object(handler_mod, "config", fake_config), + patch.object(handler_mod, "_create_openai_agent_handler", _factory), + patch.object(handler_mod, "_config", fake_config), ): handler_mod.openai_agents("k", "q", {}, capture_content=True) @@ -1712,8 +1712,8 @@ def _factory(*, capture_content: bool = False) -> Any: fake_config = MagicMock() fake_config.return_value.invoke = MagicMock(return_value="ok") with ( - patch.object(handler_mod, "create_openai_agent_handler", _factory), - patch.object(handler_mod, "config", fake_config), + patch.object(handler_mod, "_create_openai_agent_handler", _factory), + patch.object(handler_mod, "_config", fake_config), ): handler_mod.openai_agents("k", "q", {}) diff --git a/packages/openai-agents/tests/test_sdk_usage.py b/packages/openai-agents/tests/test_sdk_usage.py new file mode 100644 index 00000000..73621084 --- /dev/null +++ b/packages/openai-agents/tests/test_sdk_usage.py @@ -0,0 +1,149 @@ +"""Tests for TESTING.md §3.27 — openai-agents.""" + +from types import SimpleNamespace +from typing import Any +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +import launchdarkly_ai_server.lifecycle as lifecycle_module +from launchdarkly_ai_openai_agents import ( + create_openai_agent_handler, + openai_agents, + openai_graph, + to_openai_agents, +) +from launchdarkly_ai_openai_agents._version import PACKAGE_NAME +from launchdarkly_ai_openai_agents._version import __version__ as PACKAGE_VERSION +from launchdarkly_ai_server import SDK_INFO_CONTEXT, __version__, init_client, shutdown + +CONTEXT = {"kind": "user", "key": "user-1"} + + +def _fake() -> MagicMock: + client = MagicMock() + client.track = MagicMock() + client.variation = AsyncMock(return_value=None) + client.flush = AsyncMock() + client.close = AsyncMock() + return client + + +async def _settle(value: Any) -> None: + if hasattr(value, "__await__"): + try: + await value + except Exception: + return + + +@pytest.fixture +async def client() -> Any: + await shutdown() + installed = _fake() + with patch.object(lifecycle_module, "_setup_telemetry", return_value=None): + await init_client(client=installed) + installed.track.reset_mock() + yield installed + await shutdown() + + +@pytest.mark.parametrize( + ("helper", "call"), + [ + ("openai-agents.openaiAgents", lambda: openai_agents("k", "q", CONTEXT)), + ("openai-agents.createOpenAIAgentHandler", create_openai_agent_handler), + ("openai-agents.openaiGraph", lambda: openai_graph("k")), + ("openai-agents.toOpenAIAgents", lambda: to_openai_agents(None)), + ], +) +async def test_helper_sends_one_usage_event( + client: MagicMock, helper: str, call: Any +) -> None: + try: + await _settle(call()) + except Exception: + pass + calls = [ + entry + for entry in client.track.call_args_list + if entry.args[0] == "$ld:ai:sdk:usage" + ] + assert [entry.args[2].get("helper") for entry in calls] == [helper] + assert calls[0].args[1] == SDK_INFO_CONTEXT + assert calls[0].args[2] == { + "aiSdkName": "launchdarkly-ai-server", + "aiSdkVersion": __version__, + "aiSdkLanguage": "python", + "helper": helper, + "helperPackageName": PACKAGE_NAME, + "helperPackageVersion": PACKAGE_VERSION, + } + assert calls[0].args[3] == 1 + + +def _helpers(client: MagicMock) -> list[str]: + return [ + entry.args[2].get("helper") + for entry in client.track.call_args_list + if entry.args[0] == "$ld:ai:sdk:usage" + ] + + +async def _drain(stream: Any) -> None: + try: + async for _ in stream: + pass + except Exception: + return + + +async def test_graph_wrapper_invoke_and_stream_report_only_the_wrapper( + client: MagicMock, +) -> None: + """The returned graph's methods are not separate ``client.graph.*`` calls.""" + instance = openai_graph("k") + await _settle(instance.invoke("q", CONTEXT)) + await _drain(instance.stream("q", CONTEXT)) + assert _helpers(client) == ["openai-agents.openaiGraph"] + + +async def test_native_adapter_invoke_reports_only_the_adapter( + client: MagicMock, +) -> None: + async def _definition() -> Any: + return SimpleNamespace(enabled=False, key="k") + + runner = to_openai_agents(_definition(), {"context": CONTEXT}) + with pytest.raises(ValueError, match="disabled"): + await runner.invoke("q") + assert _helpers(client) == ["openai-agents.toOpenAIAgents"] + + +async def test_held_helper_flushes_with_its_package() -> None: + """A helper called before init keeps its package fields until the flush.""" + await shutdown() + lifecycle_module._reset_for_testing() + try: + create_openai_agent_handler() + except Exception: + pass # The factory may need credentials; it reports first. + installed = _fake() + with patch.object(lifecycle_module, "_setup_telemetry", return_value=None): + await init_client(client=installed) + calls = [ + entry.args[2] + for entry in installed.track.call_args_list + if entry.args[0] == "$ld:ai:sdk:usage" + ] + assert calls == [ + { + "aiSdkName": "launchdarkly-ai-server", + "aiSdkVersion": __version__, + "aiSdkLanguage": "python", + "helper": "openai-agents.createOpenAIAgentHandler", + "helperPackageName": PACKAGE_NAME, + "helperPackageVersion": PACKAGE_VERSION, + } + ] + await shutdown() diff --git a/packages/openai-messages/agents.md b/packages/openai-messages/agents.md index 15a858c1..5f29cae4 100644 --- a/packages/openai-messages/agents.md +++ b/packages/openai-messages/agents.md @@ -28,6 +28,7 @@ That means the LaunchDarkly flag variation must have `provider.name == "OpenAI"` |---|---| | `src/launchdarkly_ai_openai_messages/handler.py` | All implementation — message building, tool schema conversion, function-call loop, telemetry | | `src/launchdarkly_ai_openai_messages/__init__.py` | Package exports | +| `src/launchdarkly_ai_openai_messages/_version.py` | `PACKAGE_NAME` and `__version__`, sent with `$ld:ai:sdk:info` and `$ld:ai:sdk:usage` | --- diff --git a/packages/openai-messages/src/launchdarkly_ai_openai_messages/__init__.py b/packages/openai-messages/src/launchdarkly_ai_openai_messages/__init__.py index e9242442..f556a943 100644 --- a/packages/openai-messages/src/launchdarkly_ai_openai_messages/__init__.py +++ b/packages/openai-messages/src/launchdarkly_ai_openai_messages/__init__.py @@ -3,12 +3,11 @@ See https://launchdarkly.com/docs for usage. """ -__version__ = "0.2.3" # x-release-please-version - from launchdarkly_ai_server import register_ai_sdk_package +from ._version import PACKAGE_NAME, __version__ from .handler import create_openai_messages_handler, openai_messages __all__ = ["create_openai_messages_handler", "openai_messages"] -register_ai_sdk_package("launchdarkly-ai-openai-messages", __version__) +register_ai_sdk_package(PACKAGE_NAME, __version__) diff --git a/packages/openai-messages/src/launchdarkly_ai_openai_messages/_version.py b/packages/openai-messages/src/launchdarkly_ai_openai_messages/_version.py new file mode 100644 index 00000000..8559e759 --- /dev/null +++ b/packages/openai-messages/src/launchdarkly_ai_openai_messages/_version.py @@ -0,0 +1,4 @@ +"""This package's identity, as sent with ``$ld:ai:sdk:info`` and ``$ld:ai:sdk:usage``.""" + +PACKAGE_NAME = "launchdarkly-ai-openai-messages" +__version__ = "0.2.3" # x-release-please-version diff --git a/packages/openai-messages/src/launchdarkly_ai_openai_messages/handler.py b/packages/openai-messages/src/launchdarkly_ai_openai_messages/handler.py index a404fdcb..6afc2b49 100644 --- a/packages/openai-messages/src/launchdarkly_ai_openai_messages/handler.py +++ b/packages/openai-messages/src/launchdarkly_ai_openai_messages/handler.py @@ -12,21 +12,23 @@ RunUsage, SpanMessage, SpanMessagePart, + _config, + _create_handler, compose_history, - config, content_to_text, - create_handler, create_run_usage, end_span_once, end_unfinished_spans, image_block_to_url, is_content_blocks, parse_template, + report_usage, set_input_content_attributes, set_output_content_attributes, set_tool_call_content_attributes, ) +from ._version import PACKAGE_NAME, __version__ from .spans import ( fail_span, finish_model_span, @@ -219,6 +221,14 @@ def create_openai_messages_handler(*, capture_content: bool = False) -> Provider emitted spans. It defaults to off. Conversation content is PII, so a run emits only metadata, meaning models, token counts, timings and tool names, until a caller asks for more. """ + report_usage("openai-messages.createOpenAIHandler", PACKAGE_NAME, __version__) + return _create_openai_messages_handler(capture_content=capture_content) + + +def _create_openai_messages_handler( + *, capture_content: bool = False +) -> ProviderHandler: + """Non-reporting :func:`create_openai_messages_handler`, used by this package's wrappers.""" import importlib openai_mod = importlib.import_module("openai") @@ -407,7 +417,7 @@ def _stream_impl( capture_content=capture_content, ) - return create_handler( + return _create_handler( ("OpenAI", "messages"), _call_impl, # type: ignore[arg-type] _stream_impl, # type: ignore[arg-type] @@ -641,10 +651,11 @@ def openai_messages( # Both are lifted out of kwargs: capture_content configures the handler, variables belong to # the invocation. Leaving either in would pass it to config(), which takes neither, so a caller # asking for content on spans got a TypeError instead of content. + report_usage("openai-messages.openaiMessages", PACKAGE_NAME, __version__) variables = kwargs.pop("variables", None) capture_content = kwargs.pop("capture_content", False) - return config( + return _config( key=config_key, - handler=create_openai_messages_handler(capture_content=capture_content), + handler=_create_openai_messages_handler(capture_content=capture_content), **kwargs, ).invoke(user_input, context, variables=variables) diff --git a/packages/openai-messages/tests/test_handler.py b/packages/openai-messages/tests/test_handler.py index 4668976f..35da2f9d 100644 --- a/packages/openai-messages/tests/test_handler.py +++ b/packages/openai-messages/tests/test_handler.py @@ -1167,7 +1167,7 @@ def test_calls_through_to_model_call(self, mock_openai: MagicMock) -> None: mock_config_fn = MagicMock(return_value=mock_config_instance) mock_config_instance.invoke = MagicMock(return_value="result") - with patch.object(handler_mod, "config", mock_config_fn): + with patch.object(handler_mod, "_config", mock_config_fn): from launchdarkly_ai_openai_messages.handler import openai_messages ctx = {"kind": "user", "key": "u1"} @@ -1190,7 +1190,7 @@ def test_callable_without_extra_kwargs(self, mock_openai: MagicMock) -> None: mock_config_fn = MagicMock(return_value=mock_config_instance) mock_config_instance.invoke = MagicMock(return_value="result") - with patch.object(handler_mod, "config", mock_config_fn): + with patch.object(handler_mod, "_config", mock_config_fn): from launchdarkly_ai_openai_messages.handler import openai_messages ctx = {"kind": "user", "key": "u1"} @@ -1866,8 +1866,8 @@ def _factory(*args: Any, capture_content: bool = False, **kw: Any) -> Any: fake_config = MagicMock() fake_config.return_value.invoke = MagicMock(return_value="ok") with ( - patch.object(handler_mod, "create_openai_messages_handler", _factory), - patch.object(handler_mod, "config", fake_config), + patch.object(handler_mod, "_create_openai_messages_handler", _factory), + patch.object(handler_mod, "_config", fake_config), ): handler_mod.openai_messages("k", "q", {}, **kwargs) seen["config_kwargs"] = fake_config.call_args.kwargs diff --git a/packages/openai-messages/tests/test_sdk_usage.py b/packages/openai-messages/tests/test_sdk_usage.py new file mode 100644 index 00000000..ec3ab5fc --- /dev/null +++ b/packages/openai-messages/tests/test_sdk_usage.py @@ -0,0 +1,107 @@ +"""Tests for TESTING.md §3.27 — openai-messages.""" + +from typing import Any +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +import launchdarkly_ai_server.lifecycle as lifecycle_module +from launchdarkly_ai_openai_messages import ( + create_openai_messages_handler, + openai_messages, +) +from launchdarkly_ai_openai_messages._version import PACKAGE_NAME +from launchdarkly_ai_openai_messages._version import __version__ as PACKAGE_VERSION +from launchdarkly_ai_server import SDK_INFO_CONTEXT, __version__, init_client, shutdown + +CONTEXT = {"kind": "user", "key": "user-1"} +ANONYMOUS = SDK_INFO_CONTEXT + + +def _fake() -> MagicMock: + client = MagicMock() + client.track = MagicMock() + client.variation = AsyncMock(return_value=None) + client.flush = AsyncMock() + client.close = AsyncMock() + return client + + +async def _settle(value: Any) -> None: + if hasattr(value, "__await__"): + try: + await value + except Exception: + return + + +@pytest.fixture +async def client() -> Any: + await shutdown() + installed = _fake() + with patch.object(lifecycle_module, "_setup_telemetry", return_value=None): + await init_client(client=installed) + installed.track.reset_mock() + yield installed + await shutdown() + + +@pytest.mark.parametrize( + ("helper", "call"), + [ + ("openai-messages.openaiMessages", lambda: openai_messages("k", "q", CONTEXT)), + ("openai-messages.createOpenAIHandler", create_openai_messages_handler), + ], +) +async def test_helper_sends_one_usage_event( + client: MagicMock, helper: str, call: Any +) -> None: + try: + await _settle(call()) + except Exception: + pass + calls = [ + entry + for entry in client.track.call_args_list + if entry.args[0] == "$ld:ai:sdk:usage" + ] + assert [entry.args[2].get("helper") for entry in calls] == [helper] + assert calls[0].args[1] == ANONYMOUS + assert calls[0].args[2] == { + "aiSdkName": "launchdarkly-ai-server", + "aiSdkVersion": __version__, + "aiSdkLanguage": "python", + "helper": helper, + "helperPackageName": PACKAGE_NAME, + "helperPackageVersion": PACKAGE_VERSION, + } + assert calls[0].args[3] == 1 + + +async def test_held_helper_flushes_with_its_package() -> None: + """A helper called before init keeps its package fields until the flush.""" + await shutdown() + lifecycle_module._reset_for_testing() + try: + create_openai_messages_handler() + except Exception: + pass # The factory may need credentials; it reports first. + installed = _fake() + with patch.object(lifecycle_module, "_setup_telemetry", return_value=None): + await init_client(client=installed) + calls = [ + entry.args[2] + for entry in installed.track.call_args_list + if entry.args[0] == "$ld:ai:sdk:usage" + ] + assert calls == [ + { + "aiSdkName": "launchdarkly-ai-server", + "aiSdkVersion": __version__, + "aiSdkLanguage": "python", + "helper": "openai-messages.createOpenAIHandler", + "helperPackageName": PACKAGE_NAME, + "helperPackageVersion": PACKAGE_VERSION, + } + ] + await shutdown() diff --git a/release-please-config.json b/release-please-config.json index 9d2a04e3..d68c9835 100644 --- a/release-please-config.json +++ b/release-please-config.json @@ -31,7 +31,7 @@ "bump-patch-for-minor-pre-major": true, "include-v-in-tag": false, "extra-files": [ - "src/launchdarkly_ai_claude_agents/__init__.py" + "src/launchdarkly_ai_claude_agents/_version.py" ], "component": "launchdarkly-ai-claude-agents" }, @@ -42,7 +42,7 @@ "bump-patch-for-minor-pre-major": true, "include-v-in-tag": false, "extra-files": [ - "src/launchdarkly_ai_claude_messages/__init__.py" + "src/launchdarkly_ai_claude_messages/_version.py" ], "component": "launchdarkly-ai-claude-messages" }, @@ -53,7 +53,7 @@ "bump-patch-for-minor-pre-major": true, "include-v-in-tag": false, "extra-files": [ - "src/launchdarkly_ai_openai_agents/__init__.py" + "src/launchdarkly_ai_openai_agents/_version.py" ], "component": "launchdarkly-ai-openai-agents" }, @@ -64,7 +64,7 @@ "bump-patch-for-minor-pre-major": true, "include-v-in-tag": false, "extra-files": [ - "src/launchdarkly_ai_openai_messages/__init__.py" + "src/launchdarkly_ai_openai_messages/_version.py" ], "component": "launchdarkly-ai-openai-messages" }, @@ -75,7 +75,7 @@ "bump-patch-for-minor-pre-major": true, "include-v-in-tag": false, "extra-files": [ - "src/launchdarkly_ai_langchain_agents/__init__.py" + "src/launchdarkly_ai_langchain_agents/_version.py" ], "component": "launchdarkly-ai-langchain-agents" }, @@ -86,7 +86,7 @@ "bump-patch-for-minor-pre-major": true, "include-v-in-tag": false, "extra-files": [ - "src/launchdarkly_ai_langchain_messages/__init__.py" + "src/launchdarkly_ai_langchain_messages/_version.py" ], "component": "launchdarkly-ai-langchain-messages" }