From a5d6045bc3f71e54322af76cf6eef423d8f1c9af Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Thu, 5 Nov 2026 08:49:14 +0100 Subject: [PATCH 1/7] feat(anthropic): Record reasoning parts in output --- sentry_sdk/integrations/anthropic.py | 24 +- .../integrations/anthropic/test_anthropic.py | 1702 ++++++++--------- 2 files changed, 824 insertions(+), 902 deletions(-) diff --git a/sentry_sdk/integrations/anthropic.py b/sentry_sdk/integrations/anthropic.py index 868b94e792..811208122d 100644 --- a/sentry_sdk/integrations/anthropic.py +++ b/sentry_sdk/integrations/anthropic.py @@ -85,7 +85,7 @@ ToolUnionParam, ) - from sentry_sdk._types import TextPart + from sentry_sdk._types import ReasoningPart, TextPart class _PatchedRawMessageStream(Stream[Any]): _span: Union[Span, StreamedSpan] @@ -731,30 +731,30 @@ def _set_output_data( record_outputs = True if record_outputs: - output_messages: "dict[str, list[Any]]" = { - "response": [], - "tool": [], - } + parts: "list[Union[TextPart, ReasoningPart]]" = [] + tool_calls = [] for output in content_blocks: if output["type"] == "text": - output_messages["response"].append(output["text"]) + parts.append({"type": "text", "content": output["text"]}) + elif output["type"] == "thinking": + parts.append({"type": "reasoning", "content": output.get("thinking")}) elif output["type"] == "tool_use": - output_messages["tool"].append(output) + tool_calls.append(output) - if len(output_messages["tool"]) > 0: + if len(tool_calls) > 0: set_data_normalized( span, SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS, - output_messages["tool"], + tool_calls, unpack=False, ) - if len(output_messages["response"]) > 0: + if len(parts) > 0: set_data_normalized( span, - SPANDATA.GEN_AI_RESPONSE_TEXT, - output_messages["response"], + SPANDATA.GEN_AI_OUTPUT_MESSAGES, + [{"role": "assistant", "parts": parts}], unpack=False, ) diff --git a/tests/integrations/anthropic/test_anthropic.py b/tests/integrations/anthropic/test_anthropic.py index 19a9f846ad..abb3746b11 100644 --- a/tests/integrations/anthropic/test_anthropic.py +++ b/tests/integrations/anthropic/test_anthropic.py @@ -23,6 +23,12 @@ async def __call__(self, *args, **kwargs): from anthropic.types.message_delta_event import MessageDeltaEvent from anthropic.types.message_start_event import MessageStartEvent +try: + from anthropic.types import ThinkingBlock +except ImportError: + ThinkingBlock = None + + try: from anthropic import APIStatusError from anthropic.types import ErrorResponse, OverloadedError @@ -78,7 +84,18 @@ async def __call__(self, *args, **kwargs): id="msg_01XFDUDYJgAACzvnptvVoYEL", model="model", role="assistant", - content=[TextBlock(type="text", text="Hi, I'm Claude.")], + content=( + [ + ThinkingBlock( + type="thinking", + thinking="Let me reason step by step...", + signature="sig123", + ), + TextBlock(type="text", text="Hi, I'm Claude."), + ] + if ThinkingBlock is not None + else [TextBlock(type="text", text="Hi, I'm Claude.")] + ), type="message", stop_reason="end_turn", usage=Usage(input_tokens=10, output_tokens=20), @@ -138,6 +155,83 @@ def data_collection_tool_use_message(): ) +@pytest.fixture +def streaming_anthropic_response(): + def inner(text_deltas, stop_reason): + return [ + MessageStartEvent( + message=Message( + id="msg_01XFDUDYJgAACzvnptvVoYEL", + model="model", + role="assistant", + content=[TextBlock(type="text", text="")], + type="message", + stop_reason="end_turn", + usage=Usage(input_tokens=10, output_tokens=20), + ), + type="message_start", + ), + ContentBlockStartEvent( + type="content_block_start", + index=0, + content_block=TextBlock(type="text", text=""), + ), + *[ + ContentBlockDeltaEvent( + delta=TextDelta(text=text, type="text_delta"), + index=0, + type="content_block_delta", + ) + for text in text_deltas + ], + ContentBlockStopEvent(type="content_block_stop", index=0), + MessageDeltaEvent( + delta=Delta(stop_reason=stop_reason), + usage=MessageDeltaUsage(output_tokens=10), + type="message_delta", + ), + ] + + return inner + + +@pytest.fixture +def streaming_anthropic_error_response(): + return [ + MessageStartEvent( + message=Message( + id="msg_01XFDUDYJgAACzvnptvVoYEL", + model="model", + role="assistant", + content=[TextBlock(type="text", text="")], + type="message", + stop_reason="end_turn", + usage=Usage(input_tokens=10, output_tokens=20), + ), + type="message_start", + ), + ContentBlockStartEvent( + type="content_block_start", + index=0, + content_block=TextBlock(type="text", text=""), + ), + ContentBlockDeltaEvent( + delta=TextDelta(text="Hi", type="text_delta"), + index=0, + type="content_block_delta", + ), + ContentBlockDeltaEvent( + delta=TextDelta(text="!", type="text_delta"), + index=0, + type="content_block_delta", + ), + ErrorResponse( + type="error", + error=OverloadedError(message="Overloaded", type="overloaded_error"), + ), + ] + + @pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.parametrize("stream_gen_ai_spans", [True, False]) @pytest.mark.parametrize( @@ -219,12 +313,25 @@ def test_nonstreaming_create_message( "content": "Hello, Claude", }, ] - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ["Hi, I'm Claude."] - ) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": ( + [ + { + "type": "reasoning", + "content": "Let me reason step by step...", + }, + {"type": "text", "content": "Hi, I'm Claude."}, + ] + if ThinkingBlock is not None + else [{"type": "text", "content": "Hi, I'm Claude."}] + ), + } + ] else: assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["attributes"] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 @@ -276,12 +383,25 @@ def test_nonstreaming_create_message( "content": "Hello, Claude", }, ] - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ["Hi, I'm Claude."] - ) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": ( + [ + { + "type": "reasoning", + "content": "Let me reason step by step...", + }, + {"type": "text", "content": "Hi, I'm Claude."}, + ] + if ThinkingBlock is not None + else [{"type": "text", "content": "Hi, I'm Claude."}] + ), + } + ] else: assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["attributes"] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 @@ -328,12 +448,25 @@ def test_nonstreaming_create_message( span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "Hello, Claude"}]' ) - assert span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ["Hi, I'm Claude."] - ) + assert json.loads(span["data"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": ( + [ + { + "type": "reasoning", + "content": "Let me reason step by step...", + }, + {"type": "text", "content": "Hi, I'm Claude."}, + ] + if ThinkingBlock is not None + else [{"type": "text", "content": "Hi, I'm Claude."}] + ), + } + ] else: assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["data"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["data"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["data"] assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 @@ -748,15 +881,23 @@ def test_nonstreaming_create_message_data_collection_outputs( assert span_data[SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 if outputs_collected: - assert span_data[SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - [DATA_COLLECTION_EXPECTED_RESPONSE_TEXT] - ) + assert json.loads(span_data[SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": [ + { + "type": "text", + "content": DATA_COLLECTION_EXPECTED_RESPONSE_TEXT, + } + ], + } + ] assert ( json.loads(span_data[SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS]) == DATA_COLLECTION_EXPECTED_TOOL_CALLS ) else: - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span_data + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span_data assert SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS not in span_data @@ -875,15 +1016,23 @@ async def test_nonstreaming_create_message_data_collection_outputs_async( assert span_data[SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 if outputs_collected: - assert span_data[SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - [DATA_COLLECTION_EXPECTED_RESPONSE_TEXT] - ) + assert json.loads(span_data[SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": [ + { + "type": "text", + "content": DATA_COLLECTION_EXPECTED_RESPONSE_TEXT, + } + ], + } + ] assert ( json.loads(span_data[SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS]) == DATA_COLLECTION_EXPECTED_TOOL_CALLS ) else: - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span_data + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span_data assert SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS not in span_data @@ -969,12 +1118,25 @@ async def test_nonstreaming_create_message_async( "content": "Hello, Claude", }, ] - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ["Hi, I'm Claude."] - ) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": ( + [ + { + "type": "reasoning", + "content": "Let me reason step by step...", + }, + {"type": "text", "content": "Hi, I'm Claude."}, + ] + if ThinkingBlock is not None + else [{"type": "text", "content": "Hi, I'm Claude."}] + ), + } + ] else: assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["attributes"] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 @@ -1024,12 +1186,25 @@ async def test_nonstreaming_create_message_async( "content": "Hello, Claude", }, ] - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ["Hi, I'm Claude."] - ) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": ( + [ + { + "type": "reasoning", + "content": "Let me reason step by step...", + }, + {"type": "text", "content": "Hi, I'm Claude."}, + ] + if ThinkingBlock is not None + else [{"type": "text", "content": "Hi, I'm Claude."}] + ), + } + ] else: assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["attributes"] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 @@ -1073,12 +1248,25 @@ async def test_nonstreaming_create_message_async( span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "Hello, Claude"}]' ) - assert span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ["Hi, I'm Claude."] - ) + assert json.loads(span["data"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": ( + [ + { + "type": "reasoning", + "content": "Let me reason step by step...", + }, + {"type": "text", "content": "Hi, I'm Claude."}, + ] + if ThinkingBlock is not None + else [{"type": "text", "content": "Hi, I'm Claude."}] + ), + } + ] else: assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["data"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["data"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["data"] assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 @@ -1110,43 +1298,16 @@ def test_streaming_create_message( server_side_event_chunks, stream_gen_ai_spans, span_streaming, + streaming_anthropic_response, ): client = Anthropic(api_key="z") response = get_model_response( server_side_event_chunks( - [ - MessageStartEvent( - message=EXAMPLE_MESSAGE, - type="message_start", - ), - ContentBlockStartEvent( - type="content_block_start", - index=0, - content_block=TextBlock(type="text", text=""), - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="Hi", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="!", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockDeltaEvent( - delta=TextDelta(text=" I'm Claude!", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockStopEvent(type="content_block_stop", index=0), - MessageDeltaEvent( - delta=Delta(stop_reason="max_tokens"), - usage=MessageDeltaUsage(output_tokens=10), - type="message_delta", - ), - ] + streaming_anthropic_response( + text_deltas=("Hi", "!", " I'm Claude!"), + stop_reason="max_tokens", + ) ) ) @@ -1209,13 +1370,16 @@ def test_streaming_create_message( "content": "Hello, Claude", }, ] - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ["Hi! I'm Claude!"] - ) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + } + ] else: assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["attributes"] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 @@ -1269,13 +1433,16 @@ def test_streaming_create_message( "content": "Hello, Claude", }, ] - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ["Hi! I'm Claude!"] - ) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + } + ] else: assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["attributes"] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 @@ -1322,13 +1489,16 @@ def test_streaming_create_message( span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "Hello, Claude"}]' ) - assert span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ["Hi! I'm Claude!"] - ) + assert json.loads(span["data"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + } + ] else: assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["data"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["data"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["data"] assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 @@ -1384,6 +1554,7 @@ def test_streaming_create_message_data_collection( server_side_event_chunks, stream_gen_ai_spans, span_streaming, + streaming_anthropic_response, ): sentry_init_kwargs = dict( integrations=[AnthropicIntegration(include_prompts=include_prompts)], @@ -1401,28 +1572,10 @@ def test_streaming_create_message_data_collection( response = get_model_response( server_side_event_chunks( - [ - MessageStartEvent( - message=EXAMPLE_MESSAGE, - type="message_start", - ), - ContentBlockStartEvent( - type="content_block_start", - index=0, - content_block=TextBlock(type="text", text=""), - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="Hi", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockStopEvent(type="content_block_stop", index=0), - MessageDeltaEvent( - delta=Delta(stop_reason="max_tokens"), - usage=MessageDeltaUsage(output_tokens=10), - type="message_delta", - ), - ] + streaming_anthropic_response( + text_deltas=("Hi",), + stop_reason="max_tokens", + ) ) ) @@ -1533,6 +1686,7 @@ def test_streaming_create_message_data_collection_outputs( server_side_event_chunks, stream_gen_ai_spans, span_streaming, + streaming_anthropic_response, ): sentry_init_kwargs = dict( integrations=[AnthropicIntegration(include_prompts=include_prompts)], @@ -1550,33 +1704,10 @@ def test_streaming_create_message_data_collection_outputs( response = get_model_response( server_side_event_chunks( - [ - MessageStartEvent( - message=EXAMPLE_MESSAGE, - type="message_start", - ), - ContentBlockStartEvent( - type="content_block_start", - index=0, - content_block=TextBlock(type="text", text=""), - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="Hi", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="! I'm Claude!", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockStopEvent(type="content_block_stop", index=0), - MessageDeltaEvent( - delta=Delta(stop_reason="max_tokens"), - usage=MessageDeltaUsage(output_tokens=10), - type="message_delta", - ), - ] + streaming_anthropic_response( + text_deltas=("Hi", "! I'm Claude!"), + stop_reason="max_tokens", + ) ) ) @@ -1627,11 +1758,14 @@ def test_streaming_create_message_data_collection_outputs( assert span_data[SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 if outputs_collected: - assert span_data[SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ["Hi! I'm Claude!"] - ) + assert json.loads(span_data[SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + } + ] else: - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span_data + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span_data @pytest.mark.parametrize("span_streaming", [True, False]) @@ -1644,43 +1778,16 @@ def test_streaming_create_message_close( server_side_event_chunks, stream_gen_ai_spans, span_streaming, + streaming_anthropic_response, ): client = Anthropic(api_key="z") response = get_model_response( server_side_event_chunks( - [ - MessageStartEvent( - message=EXAMPLE_MESSAGE, - type="message_start", - ), - ContentBlockStartEvent( - type="content_block_start", - index=0, - content_block=TextBlock(type="text", text=""), - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="Hi", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="!", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockDeltaEvent( - delta=TextDelta(text=" I'm Claude!", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockStopEvent(type="content_block_stop", index=0), - MessageDeltaEvent( - delta=Delta(stop_reason="max_tokens"), - usage=MessageDeltaUsage(output_tokens=10), - type="message_delta", - ), - ] + streaming_anthropic_response( + text_deltas=("Hi", "!", " I'm Claude!"), + stop_reason="max_tokens", + ) ) ) @@ -1734,7 +1841,9 @@ def test_streaming_create_message_close( span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "Hello, Claude"}]' ) - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps(["Hi!"]) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + {"role": "assistant", "parts": [{"type": "text", "content": "Hi!"}]} + ] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 @@ -1780,7 +1889,9 @@ def test_streaming_create_message_close( span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "Hello, Claude"}]' ) - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps(["Hi!"]) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + {"role": "assistant", "parts": [{"type": "text", "content": "Hi!"}]} + ] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 @@ -1825,7 +1936,9 @@ def test_streaming_create_message_close( span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "Hello, Claude"}]' ) - assert span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps(["Hi!"]) + assert json.loads(span["data"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + {"role": "assistant", "parts": [{"type": "text", "content": "Hi!"}]} + ] assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 @@ -1850,39 +1963,12 @@ def test_streaming_create_message_api_error( server_side_event_chunks, stream_gen_ai_spans, span_streaming, + streaming_anthropic_error_response, ): client = Anthropic(api_key="z") response = get_model_response( - server_side_event_chunks( - [ - MessageStartEvent( - message=EXAMPLE_MESSAGE, - type="message_start", - ), - ContentBlockStartEvent( - type="content_block_start", - index=0, - content_block=TextBlock(type="text", text=""), - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="Hi", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="!", type="text_delta"), - index=0, - type="content_block_delta", - ), - ErrorResponse( - type="error", - error=OverloadedError( - message="Overloaded", type="overloaded_error" - ), - ), - ] - ) + server_side_event_chunks(streaming_anthropic_error_response) ) sentry_init( @@ -1934,7 +2020,9 @@ def test_streaming_create_message_api_error( span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "Hello, Claude"}]' ) - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps(["Hi!"]) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + {"role": "assistant", "parts": [{"type": "text", "content": "Hi!"}]} + ] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 @@ -1979,7 +2067,9 @@ def test_streaming_create_message_api_error( span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "Hello, Claude"}]' ) - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps(["Hi!"]) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + {"role": "assistant", "parts": [{"type": "text", "content": "Hi!"}]} + ] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 @@ -2025,7 +2115,9 @@ def test_streaming_create_message_api_error( span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "Hello, Claude"}]' ) - assert span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps(["Hi!"]) + assert json.loads(span["data"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + {"role": "assistant", "parts": [{"type": "text", "content": "Hi!"}]} + ] assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 @@ -2061,43 +2153,16 @@ def test_stream_messages( server_side_event_chunks, stream_gen_ai_spans, span_streaming, + streaming_anthropic_response, ): client = Anthropic(api_key="z") response = get_model_response( server_side_event_chunks( - [ - MessageStartEvent( - message=EXAMPLE_MESSAGE, - type="message_start", - ), - ContentBlockStartEvent( - type="content_block_start", - index=0, - content_block=TextBlock(type="text", text=""), - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="Hi", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="!", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockDeltaEvent( - delta=TextDelta(text=" I'm Claude!", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockStopEvent(type="content_block_stop", index=0), - MessageDeltaEvent( - delta=Delta(stop_reason="max_tokens"), - usage=MessageDeltaUsage(output_tokens=10), - type="message_delta", - ), - ] + streaming_anthropic_response( + text_deltas=("Hi", "!", " I'm Claude!"), + stop_reason="max_tokens", + ) ) ) @@ -2160,13 +2225,16 @@ def test_stream_messages( "content": "Hello, Claude", }, ] - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ["Hi! I'm Claude!"] - ) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + } + ] else: assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["attributes"] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 @@ -2220,13 +2288,16 @@ def test_stream_messages( "content": "Hello, Claude", }, ] - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ["Hi! I'm Claude!"] - ) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + } + ] else: assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["attributes"] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 @@ -2273,12 +2344,15 @@ def test_stream_messages( span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "Hello, Claude"}]' ) - assert span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ["Hi! I'm Claude!"] - ) + assert json.loads(span["data"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + } + ] else: assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["data"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["data"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["data"] assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 @@ -2344,6 +2418,7 @@ def test_stream_messages_data_collection_outputs( server_side_event_chunks, stream_gen_ai_spans, span_streaming, + streaming_anthropic_response, ): sentry_init_kwargs = dict( integrations=[AnthropicIntegration(include_prompts=include_prompts)], @@ -2361,33 +2436,10 @@ def test_stream_messages_data_collection_outputs( response = get_model_response( server_side_event_chunks( - [ - MessageStartEvent( - message=EXAMPLE_MESSAGE, - type="message_start", - ), - ContentBlockStartEvent( - type="content_block_start", - index=0, - content_block=TextBlock(type="text", text=""), - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="Hi", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="! I'm Claude!", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockStopEvent(type="content_block_stop", index=0), - MessageDeltaEvent( - delta=Delta(stop_reason="max_tokens"), - usage=MessageDeltaUsage(output_tokens=10), - type="message_delta", - ), - ] + streaming_anthropic_response( + text_deltas=("Hi", "! I'm Claude!"), + stop_reason="max_tokens", + ) ) ) @@ -2439,11 +2491,14 @@ def test_stream_messages_data_collection_outputs( assert span_data[SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 if outputs_collected: - assert span_data[SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ["Hi! I'm Claude!"] - ) + assert json.loads(span_data[SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + } + ] else: - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span_data + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span_data @pytest.mark.parametrize("span_streaming", [True, False]) @@ -2456,43 +2511,16 @@ def test_stream_messages_close( server_side_event_chunks, stream_gen_ai_spans, span_streaming, + streaming_anthropic_response, ): client = Anthropic(api_key="z") response = get_model_response( server_side_event_chunks( - [ - MessageStartEvent( - message=EXAMPLE_MESSAGE, - type="message_start", - ), - ContentBlockStartEvent( - type="content_block_start", - index=0, - content_block=TextBlock(type="text", text=""), - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="Hi", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="!", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockDeltaEvent( - delta=TextDelta(text=" I'm Claude!", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockStopEvent(type="content_block_stop", index=0), - MessageDeltaEvent( - delta=Delta(stop_reason="max_tokens"), - usage=MessageDeltaUsage(output_tokens=10), - type="message_delta", - ), - ] + streaming_anthropic_response( + text_deltas=("Hi", "!", " I'm Claude!"), + stop_reason="max_tokens", + ) ) ) @@ -2550,7 +2578,9 @@ def test_stream_messages_close( span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "Hello, Claude"}]' ) - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps(["Hi!"]) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + {"role": "assistant", "parts": [{"type": "text", "content": "Hi!"}]} + ] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 @@ -2600,7 +2630,9 @@ def test_stream_messages_close( span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "Hello, Claude"}]' ) - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps(["Hi!"]) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + {"role": "assistant", "parts": [{"type": "text", "content": "Hi!"}]} + ] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 @@ -2649,7 +2681,9 @@ def test_stream_messages_close( span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "Hello, Claude"}]' ) - assert span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps(["Hi!"]) + assert json.loads(span["data"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + {"role": "assistant", "parts": [{"type": "text", "content": "Hi!"}]} + ] assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 @@ -2674,39 +2708,12 @@ def test_stream_messages_api_error( server_side_event_chunks, stream_gen_ai_spans, span_streaming, + streaming_anthropic_error_response, ): client = Anthropic(api_key="z") response = get_model_response( - server_side_event_chunks( - [ - MessageStartEvent( - message=EXAMPLE_MESSAGE, - type="message_start", - ), - ContentBlockStartEvent( - type="content_block_start", - index=0, - content_block=TextBlock(type="text", text=""), - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="Hi", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="!", type="text_delta"), - index=0, - type="content_block_delta", - ), - ErrorResponse( - type="error", - error=OverloadedError( - message="Overloaded", type="overloaded_error" - ), - ), - ] - ) + server_side_event_chunks(streaming_anthropic_error_response) ) sentry_init( @@ -2758,7 +2765,9 @@ def test_stream_messages_api_error( span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "Hello, Claude"}]' ) - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps(["Hi!"]) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + {"role": "assistant", "parts": [{"type": "text", "content": "Hi!"}]} + ] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 @@ -2804,7 +2813,9 @@ def test_stream_messages_api_error( span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "Hello, Claude"}]' ) - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps(["Hi!"]) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + {"role": "assistant", "parts": [{"type": "text", "content": "Hi!"}]} + ] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 @@ -2850,7 +2861,9 @@ def test_stream_messages_api_error( span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "Hello, Claude"}]' ) - assert span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps(["Hi!"]) + assert json.loads(span["data"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + {"role": "assistant", "parts": [{"type": "text", "content": "Hi!"}]} + ] assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 @@ -2888,46 +2901,19 @@ async def test_streaming_create_message_async( server_side_event_chunks, stream_gen_ai_spans, span_streaming, + streaming_anthropic_response, ): client = AsyncAnthropic(api_key="z") response = get_model_response( async_iterator( server_side_event_chunks( - [ - MessageStartEvent( - message=EXAMPLE_MESSAGE, - type="message_start", - ), - ContentBlockStartEvent( - type="content_block_start", - index=0, - content_block=TextBlock(type="text", text=""), - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="Hi", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="!", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockDeltaEvent( - delta=TextDelta(text=" I'm Claude!", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockStopEvent(type="content_block_stop", index=0), - MessageDeltaEvent( - delta=Delta(stop_reason="max_tokens"), - usage=MessageDeltaUsage(output_tokens=10), - type="message_delta", - ), - ] + streaming_anthropic_response( + text_deltas=("Hi", "!", " I'm Claude!"), + stop_reason="max_tokens", + ) ) - ), + ) ) sentry_init( @@ -2990,13 +2976,16 @@ async def test_streaming_create_message_async( "content": "Hello, Claude", }, ] - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ["Hi! I'm Claude!"] - ) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + } + ] else: assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["attributes"] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 @@ -3049,13 +3038,16 @@ async def test_streaming_create_message_async( "content": "Hello, Claude", }, ] - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ["Hi! I'm Claude!"] - ) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + } + ] else: assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["attributes"] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 @@ -3103,13 +3095,16 @@ async def test_streaming_create_message_async( span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "Hello, Claude"}]' ) - assert span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ["Hi! I'm Claude!"] - ) + assert json.loads(span["data"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + } + ] else: assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["data"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["data"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["data"] assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 @@ -3177,6 +3172,7 @@ async def test_streaming_create_message_data_collection_outputs_async( server_side_event_chunks, stream_gen_ai_spans, span_streaming, + streaming_anthropic_response, ): sentry_init_kwargs = dict( integrations=[AnthropicIntegration(include_prompts=include_prompts)], @@ -3195,35 +3191,12 @@ async def test_streaming_create_message_data_collection_outputs_async( response = get_model_response( async_iterator( server_side_event_chunks( - [ - MessageStartEvent( - message=EXAMPLE_MESSAGE, - type="message_start", - ), - ContentBlockStartEvent( - type="content_block_start", - index=0, - content_block=TextBlock(type="text", text=""), - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="Hi", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="! I'm Claude!", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockStopEvent(type="content_block_stop", index=0), - MessageDeltaEvent( - delta=Delta(stop_reason="max_tokens"), - usage=MessageDeltaUsage(output_tokens=10), - type="message_delta", - ), - ] + streaming_anthropic_response( + text_deltas=("Hi", "! I'm Claude!"), + stop_reason="max_tokens", + ) ) - ), + ) ) create_kwargs = dict( @@ -3273,11 +3246,14 @@ async def test_streaming_create_message_data_collection_outputs_async( assert span_data[SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 if outputs_collected: - assert span_data[SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ["Hi! I'm Claude!"] - ) + assert json.loads(span_data[SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + } + ] else: - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span_data + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span_data @pytest.mark.parametrize("span_streaming", [True, False]) @@ -3292,44 +3268,17 @@ async def test_streaming_create_message_async_close( server_side_event_chunks, stream_gen_ai_spans, span_streaming, + streaming_anthropic_response, ): client = AsyncAnthropic(api_key="z") response = get_model_response( async_iterator( server_side_event_chunks( - [ - MessageStartEvent( - message=EXAMPLE_MESSAGE, - type="message_start", - ), - ContentBlockStartEvent( - type="content_block_start", - index=0, - content_block=TextBlock(type="text", text=""), - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="Hi", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="!", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockDeltaEvent( - delta=TextDelta(text=" I'm Claude!", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockStopEvent(type="content_block_stop", index=0), - MessageDeltaEvent( - delta=Delta(stop_reason="max_tokens"), - usage=MessageDeltaUsage(output_tokens=10), - type="message_delta", - ), - ] + streaming_anthropic_response( + text_deltas=("Hi", "!", " I'm Claude!"), + stop_reason="max_tokens", + ) ) ) ) @@ -3383,7 +3332,9 @@ async def test_streaming_create_message_async_close( span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "Hello, Claude"}]' ) - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps(["Hi!"]) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + {"role": "assistant", "parts": [{"type": "text", "content": "Hi!"}]} + ] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 @@ -3428,7 +3379,9 @@ async def test_streaming_create_message_async_close( span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "Hello, Claude"}]' ) - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps(["Hi!"]) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + {"role": "assistant", "parts": [{"type": "text", "content": "Hi!"}]} + ] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 @@ -3472,7 +3425,9 @@ async def test_streaming_create_message_async_close( span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "Hello, Claude"}]' ) - assert span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps(["Hi!"]) + assert json.loads(span["data"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + {"role": "assistant", "parts": [{"type": "text", "content": "Hi!"}]} + ] assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 @@ -3499,41 +3454,12 @@ async def test_streaming_create_message_async_api_error( server_side_event_chunks, stream_gen_ai_spans, span_streaming, + streaming_anthropic_error_response, ): client = AsyncAnthropic(api_key="z") response = get_model_response( - async_iterator( - server_side_event_chunks( - [ - MessageStartEvent( - message=EXAMPLE_MESSAGE, - type="message_start", - ), - ContentBlockStartEvent( - type="content_block_start", - index=0, - content_block=TextBlock(type="text", text=""), - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="Hi", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="!", type="text_delta"), - index=0, - type="content_block_delta", - ), - ErrorResponse( - type="error", - error=OverloadedError( - message="Overloaded", type="overloaded_error" - ), - ), - ] - ) - ) + async_iterator(server_side_event_chunks(streaming_anthropic_error_response)) ) sentry_init( @@ -3585,7 +3511,9 @@ async def test_streaming_create_message_async_api_error( span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "Hello, Claude"}]' ) - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps(["Hi!"]) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + {"role": "assistant", "parts": [{"type": "text", "content": "Hi!"}]} + ] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 @@ -3631,7 +3559,9 @@ async def test_streaming_create_message_async_api_error( span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "Hello, Claude"}]' ) - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps(["Hi!"]) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + {"role": "assistant", "parts": [{"type": "text", "content": "Hi!"}]} + ] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 @@ -3677,7 +3607,9 @@ async def test_streaming_create_message_async_api_error( span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "Hello, Claude"}]' ) - assert span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps(["Hi!"]) + assert json.loads(span["data"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + {"role": "assistant", "parts": [{"type": "text", "content": "Hi!"}]} + ] assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 @@ -3715,46 +3647,19 @@ async def test_stream_message_async( server_side_event_chunks, stream_gen_ai_spans, span_streaming, + streaming_anthropic_response, ): client = AsyncAnthropic(api_key="z") response = get_model_response( async_iterator( server_side_event_chunks( - [ - MessageStartEvent( - message=EXAMPLE_MESSAGE, - type="message_start", - ), - ContentBlockStartEvent( - type="content_block_start", - index=0, - content_block=TextBlock(type="text", text=""), - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="Hi", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="!", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockDeltaEvent( - delta=TextDelta(text=" I'm Claude!", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockStopEvent(type="content_block_stop", index=0), - MessageDeltaEvent( - delta=Delta(), - usage=MessageDeltaUsage(output_tokens=10), - type="message_delta", - ), - ] + streaming_anthropic_response( + text_deltas=("Hi", "!", " I'm Claude!"), + stop_reason=None, + ) ) - ), + ) ) sentry_init( @@ -3817,13 +3722,16 @@ async def test_stream_message_async( "content": "Hello, Claude", }, ] - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ["Hi! I'm Claude!"] - ) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + } + ] else: assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["attributes"] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 @@ -3874,13 +3782,16 @@ async def test_stream_message_async( "content": "Hello, Claude", }, ] - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ["Hi! I'm Claude!"] - ) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + } + ] else: assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["attributes"] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 @@ -3925,13 +3836,16 @@ async def test_stream_message_async( span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "Hello, Claude"}]' ) - assert span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ["Hi! I'm Claude!"] - ) + assert json.loads(span["data"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + } + ] else: assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["data"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["data"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["data"] assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 @@ -3998,6 +3912,7 @@ async def test_stream_messages_data_collection_outputs_async( server_side_event_chunks, stream_gen_ai_spans, span_streaming, + streaming_anthropic_response, ): sentry_init_kwargs = dict( integrations=[AnthropicIntegration(include_prompts=include_prompts)], @@ -4016,35 +3931,12 @@ async def test_stream_messages_data_collection_outputs_async( response = get_model_response( async_iterator( server_side_event_chunks( - [ - MessageStartEvent( - message=EXAMPLE_MESSAGE, - type="message_start", - ), - ContentBlockStartEvent( - type="content_block_start", - index=0, - content_block=TextBlock(type="text", text=""), - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="Hi", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="! I'm Claude!", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockStopEvent(type="content_block_stop", index=0), - MessageDeltaEvent( - delta=Delta(stop_reason="max_tokens"), - usage=MessageDeltaUsage(output_tokens=10), - type="message_delta", - ), - ] + streaming_anthropic_response( + text_deltas=("Hi", "! I'm Claude!"), + stop_reason="max_tokens", + ) ) - ), + ) ) stream_kwargs = dict( @@ -4093,11 +3985,14 @@ async def test_stream_messages_data_collection_outputs_async( assert span_data[SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 if outputs_collected: - assert span_data[SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ["Hi! I'm Claude!"] - ) + assert json.loads(span_data[SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + } + ] else: - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span_data + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span_data @pytest.mark.parametrize("span_streaming", [True, False]) @@ -4112,45 +4007,16 @@ async def test_stream_messages_async_api_error( capture_events, capture_items, get_model_response, - async_iterator, - server_side_event_chunks, - stream_gen_ai_spans, - span_streaming, -): - client = AsyncAnthropic(api_key="z") - - response = get_model_response( - async_iterator( - server_side_event_chunks( - [ - MessageStartEvent( - message=EXAMPLE_MESSAGE, - type="message_start", - ), - ContentBlockStartEvent( - type="content_block_start", - index=0, - content_block=TextBlock(type="text", text=""), - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="Hi", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="!", type="text_delta"), - index=0, - type="content_block_delta", - ), - ErrorResponse( - type="error", - error=OverloadedError( - message="Overloaded", type="overloaded_error" - ), - ), - ] - ) - ) + async_iterator, + server_side_event_chunks, + stream_gen_ai_spans, + span_streaming, + streaming_anthropic_error_response, +): + client = AsyncAnthropic(api_key="z") + + response = get_model_response( + async_iterator(server_side_event_chunks(streaming_anthropic_error_response)) ) sentry_init( @@ -4202,7 +4068,9 @@ async def test_stream_messages_async_api_error( span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "Hello, Claude"}]' ) - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps(["Hi!"]) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + {"role": "assistant", "parts": [{"type": "text", "content": "Hi!"}]} + ] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 @@ -4246,7 +4114,9 @@ async def test_stream_messages_async_api_error( span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "Hello, Claude"}]' ) - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps(["Hi!"]) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + {"role": "assistant", "parts": [{"type": "text", "content": "Hi!"}]} + ] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 @@ -4292,7 +4162,9 @@ async def test_stream_messages_async_api_error( span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "Hello, Claude"}]' ) - assert span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps(["Hi!"]) + assert json.loads(span["data"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + {"role": "assistant", "parts": [{"type": "text", "content": "Hi!"}]} + ] assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 @@ -4319,44 +4191,17 @@ async def test_stream_messages_async_close( server_side_event_chunks, stream_gen_ai_spans, span_streaming, + streaming_anthropic_response, ): client = AsyncAnthropic(api_key="z") response = get_model_response( async_iterator( server_side_event_chunks( - [ - MessageStartEvent( - message=EXAMPLE_MESSAGE, - type="message_start", - ), - ContentBlockStartEvent( - type="content_block_start", - index=0, - content_block=TextBlock(type="text", text=""), - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="Hi", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="!", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockDeltaEvent( - delta=TextDelta(text=" I'm Claude!", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockStopEvent(type="content_block_stop", index=0), - MessageDeltaEvent( - delta=Delta(stop_reason="max_tokens"), - usage=MessageDeltaUsage(output_tokens=10), - type="message_delta", - ), - ] + streaming_anthropic_response( + text_deltas=("Hi", "!", " I'm Claude!"), + stop_reason="max_tokens", + ) ) ) ) @@ -4418,7 +4263,9 @@ async def test_stream_messages_async_close( span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "Hello, Claude"}]' ) - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps(["Hi!"]) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + {"role": "assistant", "parts": [{"type": "text", "content": "Hi!"}]} + ] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 @@ -4468,7 +4315,9 @@ async def test_stream_messages_async_close( span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "Hello, Claude"}]' ) - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps(["Hi!"]) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + {"role": "assistant", "parts": [{"type": "text", "content": "Hi!"}]} + ] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 @@ -4520,7 +4369,9 @@ async def test_stream_messages_async_close( span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "Hello, Claude"}]' ) - assert span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps(["Hi!"]) + assert json.loads(span["data"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + {"role": "assistant", "parts": [{"type": "text", "content": "Hi!"}]} + ] assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 @@ -4675,12 +4526,17 @@ def test_streaming_create_message_with_input_json_delta( span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "What is the weather like in San Francisco?"}]' ) - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ['{"location": "San Francisco, CA"}'] - ) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": [ + {"type": "text", "content": '{"location": "San Francisco, CA"}'} + ], + } + ] else: assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["attributes"] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 366 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 41 @@ -4721,12 +4577,17 @@ def test_streaming_create_message_with_input_json_delta( span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "What is the weather like in San Francisco?"}]' ) - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ['{"location": "San Francisco, CA"}'] - ) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": [ + {"type": "text", "content": '{"location": "San Francisco, CA"}'} + ], + } + ] else: assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["attributes"] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 366 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 41 @@ -4768,12 +4629,17 @@ def test_streaming_create_message_with_input_json_delta( span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "What is the weather like in San Francisco?"}]' ) - assert span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ['{"location": "San Francisco, CA"}'] - ) + assert json.loads(span["data"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": [ + {"type": "text", "content": '{"location": "San Francisco, CA"}'} + ], + } + ] else: assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["data"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["data"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["data"] assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 366 assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 41 @@ -4925,12 +4791,17 @@ def test_stream_messages_with_input_json_delta( span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "What is the weather like in San Francisco?"}]' ) - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ['{"location": "San Francisco, CA"}'] - ) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": [ + {"type": "text", "content": '{"location": "San Francisco, CA"}'} + ], + } + ] else: assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["attributes"] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 366 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 41 @@ -4971,12 +4842,17 @@ def test_stream_messages_with_input_json_delta( span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "What is the weather like in San Francisco?"}]' ) - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ['{"location": "San Francisco, CA"}'] - ) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": [ + {"type": "text", "content": '{"location": "San Francisco, CA"}'} + ], + } + ] else: assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["attributes"] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 366 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 41 @@ -5017,12 +4893,17 @@ def test_stream_messages_with_input_json_delta( span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "What is the weather like in San Francisco?"}]' ) - assert span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ['{"location": "San Francisco, CA"}'] - ) + assert json.loads(span["data"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": [ + {"type": "text", "content": '{"location": "San Francisco, CA"}'} + ], + } + ] else: assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["data"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["data"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["data"] assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 366 assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 41 @@ -5181,13 +5062,18 @@ async def test_streaming_create_message_with_input_json_delta_async( span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "What is the weather like in San Francisco?"}]' ) - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ['{"location": "San Francisco, CA"}'] - ) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": [ + {"type": "text", "content": '{"location": "San Francisco, CA"}'} + ], + } + ] else: assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["attributes"] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 366 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 41 @@ -5228,13 +5114,18 @@ async def test_streaming_create_message_with_input_json_delta_async( span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "What is the weather like in San Francisco?"}]' ) - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ['{"location": "San Francisco, CA"}'] - ) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": [ + {"type": "text", "content": '{"location": "San Francisco, CA"}'} + ], + } + ] else: assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["attributes"] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 366 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 41 @@ -5275,13 +5166,18 @@ async def test_streaming_create_message_with_input_json_delta_async( span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "What is the weather like in San Francisco?"}]' ) - assert span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ['{"location": "San Francisco, CA"}'] - ) + assert json.loads(span["data"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": [ + {"type": "text", "content": '{"location": "San Francisco, CA"}'} + ], + } + ] else: assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["data"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["data"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["data"] assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 366 assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 41 @@ -5441,12 +5337,17 @@ async def test_stream_message_with_input_json_delta_async( span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "What is the weather like in San Francisco?"}]' ) - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ['{"location": "San Francisco, CA"}'] - ) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": [ + {"type": "text", "content": '{"location": "San Francisco, CA"}'} + ], + } + ] else: assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["attributes"] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 366 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 41 @@ -5488,12 +5389,17 @@ async def test_stream_message_with_input_json_delta_async( span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "What is the weather like in San Francisco?"}]' ) - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ['{"location": "San Francisco, CA"}'] - ) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": [ + {"type": "text", "content": '{"location": "San Francisco, CA"}'} + ], + } + ] else: assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["attributes"] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 366 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 41 @@ -5535,13 +5441,18 @@ async def test_stream_message_with_input_json_delta_async( span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "What is the weather like in San Francisco?"}]' ) - assert span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ['{"location": "San Francisco, CA"}'] - ) + assert json.loads(span["data"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": [ + {"type": "text", "content": '{"location": "San Francisco, CA"}'} + ], + } + ] else: assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["data"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["data"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["data"] assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 366 assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 41 @@ -6061,9 +5972,14 @@ def test_set_output_data_with_input_json_delta(sentry_init): content_blocks=[{"text": "".join(json_deltas), "type": "text"}], ) - assert span._data.get(SPANDATA.GEN_AI_RESPONSE_TEXT) == json.dumps( - ["{'test': 'data','more': 'json'}"] - ) + assert json.loads(span._data.get(SPANDATA.GEN_AI_OUTPUT_MESSAGES)) == [ + { + "role": "assistant", + "parts": [ + {"type": "text", "content": "{'test': 'data','more': 'json'}"} + ], + } + ] assert span._data.get(SPANDATA.GEN_AI_USAGE_INPUT_TOKENS) == 10 assert span._data.get(SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS) == 20 assert span._data.get(SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS) == 30 @@ -6361,13 +6277,26 @@ def test_nonstreaming_create_message_with_system_prompt( assert len(stored_messages) == 1 assert stored_messages[0]["role"] == "user" assert stored_messages[0]["content"] == "Hello, Claude" - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ["Hi, I'm Claude."] - ) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": ( + [ + { + "type": "reasoning", + "content": "Let me reason step by step...", + }, + {"type": "text", "content": "Hi, I'm Claude."}, + ] + if ThinkingBlock is not None + else [{"type": "text", "content": "Hi, I'm Claude."}] + ), + } + ] else: assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["attributes"] assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["attributes"] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 @@ -6423,13 +6352,26 @@ def test_nonstreaming_create_message_with_system_prompt( assert len(stored_messages) == 1 assert stored_messages[0]["role"] == "user" assert stored_messages[0]["content"] == "Hello, Claude" - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ["Hi, I'm Claude."] - ) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": ( + [ + { + "type": "reasoning", + "content": "Let me reason step by step...", + }, + {"type": "text", "content": "Hi, I'm Claude."}, + ] + if ThinkingBlock is not None + else [{"type": "text", "content": "Hi, I'm Claude."}] + ), + } + ] else: assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["attributes"] assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["attributes"] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 @@ -6484,13 +6426,26 @@ def test_nonstreaming_create_message_with_system_prompt( assert len(stored_messages) == 1 assert stored_messages[0]["role"] == "user" assert stored_messages[0]["content"] == "Hello, Claude" - assert span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ["Hi, I'm Claude."] - ) + assert json.loads(span["data"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": ( + [ + { + "type": "reasoning", + "content": "Let me reason step by step...", + }, + {"type": "text", "content": "Hi, I'm Claude."}, + ] + if ThinkingBlock is not None + else [{"type": "text", "content": "Hi, I'm Claude."}] + ), + } + ] else: assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["data"] assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["data"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["data"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["data"] assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 @@ -6586,13 +6541,26 @@ async def test_nonstreaming_create_message_with_system_prompt_async( assert len(stored_messages) == 1 assert stored_messages[0]["role"] == "user" assert stored_messages[0]["content"] == "Hello, Claude" - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ["Hi, I'm Claude."] - ) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": ( + [ + { + "type": "reasoning", + "content": "Let me reason step by step...", + }, + {"type": "text", "content": "Hi, I'm Claude."}, + ] + if ThinkingBlock is not None + else [{"type": "text", "content": "Hi, I'm Claude."}] + ), + } + ] else: assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["attributes"] assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["attributes"] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 @@ -6648,13 +6616,26 @@ async def test_nonstreaming_create_message_with_system_prompt_async( assert len(stored_messages) == 1 assert stored_messages[0]["role"] == "user" assert stored_messages[0]["content"] == "Hello, Claude" - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ["Hi, I'm Claude."] - ) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": ( + [ + { + "type": "reasoning", + "content": "Let me reason step by step...", + }, + {"type": "text", "content": "Hi, I'm Claude."}, + ] + if ThinkingBlock is not None + else [{"type": "text", "content": "Hi, I'm Claude."}] + ), + } + ] else: assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["attributes"] assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["attributes"] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 @@ -6709,13 +6690,26 @@ async def test_nonstreaming_create_message_with_system_prompt_async( assert len(stored_messages) == 1 assert stored_messages[0]["role"] == "user" assert stored_messages[0]["content"] == "Hello, Claude" - assert span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ["Hi, I'm Claude."] - ) + assert json.loads(span["data"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": ( + [ + { + "type": "reasoning", + "content": "Let me reason step by step...", + }, + {"type": "text", "content": "Hi, I'm Claude."}, + ] + if ThinkingBlock is not None + else [{"type": "text", "content": "Hi, I'm Claude."}] + ), + } + ] else: assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["data"] assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["data"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["data"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["data"] assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 @@ -6745,44 +6739,17 @@ def test_streaming_create_message_with_system_prompt( server_side_event_chunks, stream_gen_ai_spans, span_streaming, + streaming_anthropic_response, ): """Test that system prompts are properly captured in streaming mode.""" client = Anthropic(api_key="z") response = get_model_response( server_side_event_chunks( - [ - MessageStartEvent( - message=EXAMPLE_MESSAGE, - type="message_start", - ), - ContentBlockStartEvent( - type="content_block_start", - index=0, - content_block=TextBlock(type="text", text=""), - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="Hi", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="!", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockDeltaEvent( - delta=TextDelta(text=" I'm Claude!", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockStopEvent(type="content_block_stop", index=0), - MessageDeltaEvent( - delta=Delta(), - usage=MessageDeltaUsage(output_tokens=10), - type="message_delta", - ), - ] + streaming_anthropic_response( + text_deltas=("Hi", "!", " I'm Claude!"), + stop_reason=None, + ) ) ) @@ -6850,14 +6817,17 @@ def test_streaming_create_message_with_system_prompt( assert len(stored_messages) == 1 assert stored_messages[0]["role"] == "user" assert stored_messages[0]["content"] == "Hello, Claude" - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ["Hi! I'm Claude!"] - ) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + } + ] else: assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["attributes"] assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["attributes"] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 @@ -6913,14 +6883,17 @@ def test_streaming_create_message_with_system_prompt( assert len(stored_messages) == 1 assert stored_messages[0]["role"] == "user" assert stored_messages[0]["content"] == "Hello, Claude" - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ["Hi! I'm Claude!"] - ) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + } + ] else: assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["attributes"] assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["attributes"] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 @@ -6975,14 +6948,17 @@ def test_streaming_create_message_with_system_prompt( assert len(stored_messages) == 1 assert stored_messages[0]["role"] == "user" assert stored_messages[0]["content"] == "Hello, Claude" - assert span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ["Hi! I'm Claude!"] - ) + assert json.loads(span["data"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + } + ] else: assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["data"] assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["data"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["data"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["data"] assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 @@ -7011,44 +6987,17 @@ def test_stream_messages_with_system_prompt( server_side_event_chunks, stream_gen_ai_spans, span_streaming, + streaming_anthropic_response, ): """Test that system prompts are properly captured in streaming mode.""" client = Anthropic(api_key="z") response = get_model_response( server_side_event_chunks( - [ - MessageStartEvent( - message=EXAMPLE_MESSAGE, - type="message_start", - ), - ContentBlockStartEvent( - type="content_block_start", - index=0, - content_block=TextBlock(type="text", text=""), - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="Hi", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="!", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockDeltaEvent( - delta=TextDelta(text=" I'm Claude!", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockStopEvent(type="content_block_stop", index=0), - MessageDeltaEvent( - delta=Delta(), - usage=MessageDeltaUsage(output_tokens=10), - type="message_delta", - ), - ] + streaming_anthropic_response( + text_deltas=("Hi", "!", " I'm Claude!"), + stop_reason=None, + ) ) ) @@ -7112,13 +7061,16 @@ def test_stream_messages_with_system_prompt( assert len(stored_messages) == 1 assert stored_messages[0]["role"] == "user" assert stored_messages[0]["content"] == "Hello, Claude" - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ["Hi! I'm Claude!"] - ) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + } + ] else: assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["attributes"] assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["attributes"] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 @@ -7170,13 +7122,16 @@ def test_stream_messages_with_system_prompt( assert len(stored_messages) == 1 assert stored_messages[0]["role"] == "user" assert stored_messages[0]["content"] == "Hello, Claude" - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ["Hi! I'm Claude!"] - ) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + } + ] else: assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["attributes"] assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["attributes"] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 @@ -7226,13 +7181,16 @@ def test_stream_messages_with_system_prompt( assert len(stored_messages) == 1 assert stored_messages[0]["role"] == "user" assert stored_messages[0]["content"] == "Hello, Claude" - assert span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ["Hi! I'm Claude!"] - ) + assert json.loads(span["data"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + } + ] else: assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["data"] assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["data"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["data"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["data"] assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 @@ -7263,6 +7221,7 @@ async def test_stream_message_with_system_prompt_async( server_side_event_chunks, stream_gen_ai_spans, span_streaming, + streaming_anthropic_response, ): """Test that system prompts are properly captured in streaming mode (async).""" client = AsyncAnthropic(api_key="z") @@ -7270,38 +7229,10 @@ async def test_stream_message_with_system_prompt_async( response = get_model_response( async_iterator( server_side_event_chunks( - [ - MessageStartEvent( - message=EXAMPLE_MESSAGE, - type="message_start", - ), - ContentBlockStartEvent( - type="content_block_start", - index=0, - content_block=TextBlock(type="text", text=""), - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="Hi", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="!", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockDeltaEvent( - delta=TextDelta(text=" I'm Claude!", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockStopEvent(type="content_block_stop", index=0), - MessageDeltaEvent( - delta=Delta(), - usage=MessageDeltaUsage(output_tokens=10), - type="message_delta", - ), - ] + streaming_anthropic_response( + text_deltas=("Hi", "!", " I'm Claude!"), + stop_reason=None, + ) ) ) ) @@ -7368,13 +7299,16 @@ async def test_stream_message_with_system_prompt_async( assert len(stored_messages) == 1 assert stored_messages[0]["role"] == "user" assert stored_messages[0]["content"] == "Hello, Claude" - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ["Hi! I'm Claude!"] - ) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + } + ] else: assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["attributes"] assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["attributes"] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 @@ -7428,13 +7362,16 @@ async def test_stream_message_with_system_prompt_async( assert len(stored_messages) == 1 assert stored_messages[0]["role"] == "user" assert stored_messages[0]["content"] == "Hello, Claude" - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ["Hi! I'm Claude!"] - ) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + } + ] else: assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["attributes"] assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["attributes"] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 @@ -7486,14 +7423,17 @@ async def test_stream_message_with_system_prompt_async( assert len(stored_messages) == 1 assert stored_messages[0]["role"] == "user" assert stored_messages[0]["content"] == "Hello, Claude" - assert span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ["Hi! I'm Claude!"] - ) + assert json.loads(span["data"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + } + ] else: assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["data"] assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["data"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["data"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["data"] assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 @@ -7524,6 +7464,7 @@ async def test_streaming_create_message_with_system_prompt_async( server_side_event_chunks, stream_gen_ai_spans, span_streaming, + streaming_anthropic_response, ): """Test that system prompts are properly captured in streaming mode (async).""" client = AsyncAnthropic(api_key="z") @@ -7531,38 +7472,10 @@ async def test_streaming_create_message_with_system_prompt_async( response = get_model_response( async_iterator( server_side_event_chunks( - [ - MessageStartEvent( - message=EXAMPLE_MESSAGE, - type="message_start", - ), - ContentBlockStartEvent( - type="content_block_start", - index=0, - content_block=TextBlock(type="text", text=""), - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="Hi", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="!", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockDeltaEvent( - delta=TextDelta(text=" I'm Claude!", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockStopEvent(type="content_block_stop", index=0), - MessageDeltaEvent( - delta=Delta(), - usage=MessageDeltaUsage(output_tokens=10), - type="message_delta", - ), - ] + streaming_anthropic_response( + text_deltas=("Hi", "!", " I'm Claude!"), + stop_reason=None, + ) ) ) ) @@ -7632,14 +7545,17 @@ async def test_streaming_create_message_with_system_prompt_async( assert len(stored_messages) == 1 assert stored_messages[0]["role"] == "user" assert stored_messages[0]["content"] == "Hello, Claude" - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ["Hi! I'm Claude!"] - ) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + } + ] else: assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["attributes"] assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["attributes"] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 @@ -7696,14 +7612,17 @@ async def test_streaming_create_message_with_system_prompt_async( assert len(stored_messages) == 1 assert stored_messages[0]["role"] == "user" assert stored_messages[0]["content"] == "Hello, Claude" - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ["Hi! I'm Claude!"] - ) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + } + ] else: assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["attributes"] assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["attributes"] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 @@ -7756,14 +7675,17 @@ async def test_streaming_create_message_with_system_prompt_async( assert len(stored_messages) == 1 assert stored_messages[0]["role"] == "user" assert stored_messages[0]["content"] == "Hello, Claude" - assert span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ["Hi! I'm Claude!"] - ) + assert json.loads(span["data"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + } + ] else: assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["data"] assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["data"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["data"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["data"] assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 From 510b381a731aeb86bb96397e26c775257a8849ea Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Thu, 5 Nov 2026 09:33:12 +0100 Subject: [PATCH 2/7] feat(anthropic): Record reasoning parts in streaming output --- sentry_sdk/integrations/anthropic.py | 43 +- .../integrations/anthropic/test_anthropic.py | 887 +++++++++++++----- 2 files changed, 696 insertions(+), 234 deletions(-) diff --git a/sentry_sdk/integrations/anthropic.py b/sentry_sdk/integrations/anthropic.py index 811208122d..79cb9a2dba 100644 --- a/sentry_sdk/integrations/anthropic.py +++ b/sentry_sdk/integrations/anthropic.py @@ -64,6 +64,11 @@ except ImportError: raise DidNotEnable("Anthropic not installed") +try: + from anthropic.types import ThinkingDelta +except ImportError: + ThinkingDelta = None + if TYPE_CHECKING: from typing import ( Any, @@ -365,22 +370,33 @@ def _collect_ai_data( event: "MessageStreamEvent", model: "str | None", usage: "_RecordedUsage", - content_blocks: "list[str]", + content_blocks: "list[Any]", response_id: "str | None" = None, finish_reason: "str | None" = None, -) -> "tuple[str | None, _RecordedUsage, list[str], str | None, str | None]": +) -> "tuple[str | None, _RecordedUsage, list[Any], str | None, str | None]": """ Collect model information, token usage, and collect content blocks from the AI streaming response. """ with capture_internal_exceptions(): if hasattr(event, "type"): if event.type == "content_block_start": - pass + block_type = getattr(event.content_block, "type", "text") + content_blocks.append({"type": block_type, "text": ""}) elif event.type == "content_block_delta": - if hasattr(event.delta, "text"): - content_blocks.append(event.delta.text) - elif hasattr(event.delta, "partial_json"): - content_blocks.append(event.delta.partial_json) + delta = event.delta + if ThinkingDelta is not None and isinstance(delta, ThinkingDelta): + chunk = delta.thinking + elif hasattr(delta, "text"): + chunk = delta.text + elif hasattr(delta, "partial_json"): + chunk = delta.partial_json + else: + chunk = None + if chunk is not None: + if content_blocks: + content_blocks[-1]["text"] += chunk + else: + content_blocks.append({"type": "text", "text": chunk}) elif event.type == "content_block_stop": pass @@ -1042,7 +1058,7 @@ def _set_streaming_output_data( integration: "AnthropicIntegration", model: "Optional[str]", usage: "_RecordedUsage", - content_blocks: "list[str]", + content_blocks: "list[Any]", response_id: "Optional[str]", finish_reason: "Optional[str]", ) -> None: @@ -1057,6 +1073,15 @@ def _set_streaming_output_data( + (usage.cache_write_input_tokens or 0) ) + merged_blocks = [] + for block in content_blocks: + block_type = block.get("type", "text") + text = block.get("text", "") + if block_type == "thinking": + merged_blocks.append({"type": "thinking", "thinking": text}) + else: + merged_blocks.append({"type": "text", "text": text}) + _set_output_data( span=span, integration=integration, @@ -1065,7 +1090,7 @@ def _set_streaming_output_data( output_tokens=usage.output_tokens, cache_read_input_tokens=usage.cache_read_input_tokens, cache_write_input_tokens=usage.cache_write_input_tokens, - content_blocks=[{"text": "".join(content_blocks), "type": "text"}], + content_blocks=merged_blocks, response_id=response_id, finish_reason=finish_reason, ) diff --git a/tests/integrations/anthropic/test_anthropic.py b/tests/integrations/anthropic/test_anthropic.py index abb3746b11..59781e29b6 100644 --- a/tests/integrations/anthropic/test_anthropic.py +++ b/tests/integrations/anthropic/test_anthropic.py @@ -24,9 +24,10 @@ async def __call__(self, *args, **kwargs): from anthropic.types.message_start_event import MessageStartEvent try: - from anthropic.types import ThinkingBlock + from anthropic.types import ThinkingBlock, ThinkingDelta except ImportError: ThinkingBlock = None + ThinkingDelta = None try: @@ -158,6 +159,29 @@ def data_collection_tool_use_message(): @pytest.fixture def streaming_anthropic_response(): def inner(text_deltas, stop_reason): + thinking_events = [] + if ThinkingBlock is not None: + thinking_events = [ + ContentBlockStartEvent( + type="content_block_start", + index=0, + content_block=ThinkingBlock( + type="thinking", + thinking="", + signature="sig123", + ), + ), + ContentBlockDeltaEvent( + delta=ThinkingDelta( + thinking="Let me reason step by step...", + type="thinking_delta", + ), + index=0, + type="content_block_delta", + ), + ContentBlockStopEvent(type="content_block_stop", index=0), + ] + text_index = 1 if ThinkingBlock is not None else 0 return [ MessageStartEvent( message=Message( @@ -171,20 +195,21 @@ def inner(text_deltas, stop_reason): ), type="message_start", ), + *thinking_events, ContentBlockStartEvent( type="content_block_start", - index=0, + index=text_index, content_block=TextBlock(type="text", text=""), ), *[ ContentBlockDeltaEvent( delta=TextDelta(text=text, type="text_delta"), - index=0, + index=text_index, type="content_block_delta", ) for text in text_deltas ], - ContentBlockStopEvent(type="content_block_stop", index=0), + ContentBlockStopEvent(type="content_block_stop", index=text_index), MessageDeltaEvent( delta=Delta(stop_reason=stop_reason), usage=MessageDeltaUsage(output_tokens=10), @@ -313,21 +338,19 @@ def test_nonstreaming_create_message( "content": "Hello, Claude", }, ] + expected_parts = ( + [ + { + "type": "reasoning", + "content": "Let me reason step by step...", + }, + {"type": "text", "content": "Hi, I'm Claude."}, + ] + if ThinkingBlock is not None + else [{"type": "text", "content": "Hi, I'm Claude."}] + ) assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ - { - "role": "assistant", - "parts": ( - [ - { - "type": "reasoning", - "content": "Let me reason step by step...", - }, - {"type": "text", "content": "Hi, I'm Claude."}, - ] - if ThinkingBlock is not None - else [{"type": "text", "content": "Hi, I'm Claude."}] - ), - } + {"role": "assistant", "parts": expected_parts} ] else: assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] @@ -383,21 +406,19 @@ def test_nonstreaming_create_message( "content": "Hello, Claude", }, ] + expected_parts = ( + [ + { + "type": "reasoning", + "content": "Let me reason step by step...", + }, + {"type": "text", "content": "Hi, I'm Claude."}, + ] + if ThinkingBlock is not None + else [{"type": "text", "content": "Hi, I'm Claude."}] + ) assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ - { - "role": "assistant", - "parts": ( - [ - { - "type": "reasoning", - "content": "Let me reason step by step...", - }, - {"type": "text", "content": "Hi, I'm Claude."}, - ] - if ThinkingBlock is not None - else [{"type": "text", "content": "Hi, I'm Claude."}] - ), - } + {"role": "assistant", "parts": expected_parts} ] else: assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] @@ -448,21 +469,19 @@ def test_nonstreaming_create_message( span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "Hello, Claude"}]' ) + expected_parts = ( + [ + { + "type": "reasoning", + "content": "Let me reason step by step...", + }, + {"type": "text", "content": "Hi, I'm Claude."}, + ] + if ThinkingBlock is not None + else [{"type": "text", "content": "Hi, I'm Claude."}] + ) assert json.loads(span["data"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ - { - "role": "assistant", - "parts": ( - [ - { - "type": "reasoning", - "content": "Let me reason step by step...", - }, - {"type": "text", "content": "Hi, I'm Claude."}, - ] - if ThinkingBlock is not None - else [{"type": "text", "content": "Hi, I'm Claude."}] - ), - } + {"role": "assistant", "parts": expected_parts} ] else: assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["data"] @@ -1118,21 +1137,19 @@ async def test_nonstreaming_create_message_async( "content": "Hello, Claude", }, ] + expected_parts = ( + [ + { + "type": "reasoning", + "content": "Let me reason step by step...", + }, + {"type": "text", "content": "Hi, I'm Claude."}, + ] + if ThinkingBlock is not None + else [{"type": "text", "content": "Hi, I'm Claude."}] + ) assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ - { - "role": "assistant", - "parts": ( - [ - { - "type": "reasoning", - "content": "Let me reason step by step...", - }, - {"type": "text", "content": "Hi, I'm Claude."}, - ] - if ThinkingBlock is not None - else [{"type": "text", "content": "Hi, I'm Claude."}] - ), - } + {"role": "assistant", "parts": expected_parts} ] else: assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] @@ -1186,21 +1203,19 @@ async def test_nonstreaming_create_message_async( "content": "Hello, Claude", }, ] + expected_parts = ( + [ + { + "type": "reasoning", + "content": "Let me reason step by step...", + }, + {"type": "text", "content": "Hi, I'm Claude."}, + ] + if ThinkingBlock is not None + else [{"type": "text", "content": "Hi, I'm Claude."}] + ) assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ - { - "role": "assistant", - "parts": ( - [ - { - "type": "reasoning", - "content": "Let me reason step by step...", - }, - {"type": "text", "content": "Hi, I'm Claude."}, - ] - if ThinkingBlock is not None - else [{"type": "text", "content": "Hi, I'm Claude."}] - ), - } + {"role": "assistant", "parts": expected_parts} ] else: assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] @@ -1248,21 +1263,19 @@ async def test_nonstreaming_create_message_async( span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "Hello, Claude"}]' ) + expected_parts = ( + [ + { + "type": "reasoning", + "content": "Let me reason step by step...", + }, + {"type": "text", "content": "Hi, I'm Claude."}, + ] + if ThinkingBlock is not None + else [{"type": "text", "content": "Hi, I'm Claude."}] + ) assert json.loads(span["data"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ - { - "role": "assistant", - "parts": ( - [ - { - "type": "reasoning", - "content": "Let me reason step by step...", - }, - {"type": "text", "content": "Hi, I'm Claude."}, - ] - if ThinkingBlock is not None - else [{"type": "text", "content": "Hi, I'm Claude."}] - ), - } + {"role": "assistant", "parts": expected_parts} ] else: assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["data"] @@ -1373,7 +1386,17 @@ def test_streaming_create_message( assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ { "role": "assistant", - "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + "parts": ( + [ + { + "type": "reasoning", + "content": "Let me reason step by step...", + }, + {"type": "text", "content": "Hi! I'm Claude!"}, + ] + if ThinkingBlock is not None + else [{"type": "text", "content": "Hi! I'm Claude!"}] + ), } ] @@ -1436,7 +1459,17 @@ def test_streaming_create_message( assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ { "role": "assistant", - "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + "parts": ( + [ + { + "type": "reasoning", + "content": "Let me reason step by step...", + }, + {"type": "text", "content": "Hi! I'm Claude!"}, + ] + if ThinkingBlock is not None + else [{"type": "text", "content": "Hi! I'm Claude!"}] + ), } ] @@ -1492,7 +1525,17 @@ def test_streaming_create_message( assert json.loads(span["data"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ { "role": "assistant", - "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + "parts": ( + [ + { + "type": "reasoning", + "content": "Let me reason step by step...", + }, + {"type": "text", "content": "Hi! I'm Claude!"}, + ] + if ThinkingBlock is not None + else [{"type": "text", "content": "Hi! I'm Claude!"}] + ), } ] @@ -1761,7 +1804,17 @@ def test_streaming_create_message_data_collection_outputs( assert json.loads(span_data[SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ { "role": "assistant", - "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + "parts": ( + [ + { + "type": "reasoning", + "content": "Let me reason step by step...", + }, + {"type": "text", "content": "Hi! I'm Claude!"}, + ] + if ThinkingBlock is not None + else [{"type": "text", "content": "Hi! I'm Claude!"}] + ), } ] else: @@ -1819,7 +1872,7 @@ def test_streaming_create_message_close( max_tokens=1024, messages=messages, model="model", stream=True ) - for _ in range(4): + for _ in range(7 if ThinkingBlock is not None else 4): next(messages) messages.close() @@ -1842,7 +1895,20 @@ def test_streaming_create_message_close( == '[{"role": "user", "content": "Hello, Claude"}]' ) assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ - {"role": "assistant", "parts": [{"type": "text", "content": "Hi!"}]} + { + "role": "assistant", + "parts": ( + [ + { + "type": "reasoning", + "content": "Let me reason step by step...", + }, + {"type": "text", "content": "Hi!"}, + ] + if ThinkingBlock is not None + else [{"type": "text", "content": "Hi!"}] + ), + } ] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 @@ -1865,7 +1931,7 @@ def test_streaming_create_message_close( max_tokens=1024, messages=messages, model="model", stream=True ) - for _ in range(4): + for _ in range(7 if ThinkingBlock is not None else 4): next(messages) messages.close() @@ -1890,7 +1956,20 @@ def test_streaming_create_message_close( == '[{"role": "user", "content": "Hello, Claude"}]' ) assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ - {"role": "assistant", "parts": [{"type": "text", "content": "Hi!"}]} + { + "role": "assistant", + "parts": ( + [ + { + "type": "reasoning", + "content": "Let me reason step by step...", + }, + {"type": "text", "content": "Hi!"}, + ] + if ThinkingBlock is not None + else [{"type": "text", "content": "Hi!"}] + ), + } ] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 @@ -1913,7 +1992,7 @@ def test_streaming_create_message_close( max_tokens=1024, messages=messages, model="model", stream=True ) - for _ in range(4): + for _ in range(7 if ThinkingBlock is not None else 4): next(messages) messages.close() @@ -1937,7 +2016,20 @@ def test_streaming_create_message_close( == '[{"role": "user", "content": "Hello, Claude"}]' ) assert json.loads(span["data"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ - {"role": "assistant", "parts": [{"type": "text", "content": "Hi!"}]} + { + "role": "assistant", + "parts": ( + [ + { + "type": "reasoning", + "content": "Let me reason step by step...", + }, + {"type": "text", "content": "Hi!"}, + ] + if ThinkingBlock is not None + else [{"type": "text", "content": "Hi!"}] + ), + } ] assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 @@ -2228,7 +2320,17 @@ def test_stream_messages( assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ { "role": "assistant", - "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + "parts": ( + [ + { + "type": "reasoning", + "content": "Let me reason step by step...", + }, + {"type": "text", "content": "Hi! I'm Claude!"}, + ] + if ThinkingBlock is not None + else [{"type": "text", "content": "Hi! I'm Claude!"}] + ), } ] @@ -2291,7 +2393,17 @@ def test_stream_messages( assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ { "role": "assistant", - "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + "parts": ( + [ + { + "type": "reasoning", + "content": "Let me reason step by step...", + }, + {"type": "text", "content": "Hi! I'm Claude!"}, + ] + if ThinkingBlock is not None + else [{"type": "text", "content": "Hi! I'm Claude!"}] + ), } ] @@ -2347,7 +2459,17 @@ def test_stream_messages( assert json.loads(span["data"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ { "role": "assistant", - "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + "parts": ( + [ + { + "type": "reasoning", + "content": "Let me reason step by step...", + }, + {"type": "text", "content": "Hi! I'm Claude!"}, + ] + if ThinkingBlock is not None + else [{"type": "text", "content": "Hi! I'm Claude!"}] + ), } ] else: @@ -2494,7 +2616,17 @@ def test_stream_messages_data_collection_outputs( assert json.loads(span_data[SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ { "role": "assistant", - "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + "parts": ( + [ + { + "type": "reasoning", + "content": "Let me reason step by step...", + }, + {"type": "text", "content": "Hi! I'm Claude!"}, + ] + if ThinkingBlock is not None + else [{"type": "text", "content": "Hi! I'm Claude!"}] + ), } ] else: @@ -2552,7 +2684,7 @@ def test_stream_messages_close( messages=messages, model="model", ) as stream: - for _ in range(4): + for _ in range(7 if ThinkingBlock is not None else 4): next(stream) # New versions add TextEvent, so consume one more event. @@ -2579,7 +2711,20 @@ def test_stream_messages_close( == '[{"role": "user", "content": "Hello, Claude"}]' ) assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ - {"role": "assistant", "parts": [{"type": "text", "content": "Hi!"}]} + { + "role": "assistant", + "parts": ( + [ + { + "type": "reasoning", + "content": "Let me reason step by step...", + }, + {"type": "text", "content": "Hi! I'm Claude!"}, + ] + if ThinkingBlock is not None + else [{"type": "text", "content": "Hi!"}] + ), + } ] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 @@ -2602,7 +2747,7 @@ def test_stream_messages_close( messages=messages, model="model", ) as stream: - for _ in range(4): + for _ in range(7 if ThinkingBlock is not None else 4): next(stream) # New versions add TextEvent, so consume one more event. @@ -2631,7 +2776,20 @@ def test_stream_messages_close( == '[{"role": "user", "content": "Hello, Claude"}]' ) assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ - {"role": "assistant", "parts": [{"type": "text", "content": "Hi!"}]} + { + "role": "assistant", + "parts": ( + [ + { + "type": "reasoning", + "content": "Let me reason step by step...", + }, + {"type": "text", "content": "Hi! I'm Claude!"}, + ] + if ThinkingBlock is not None + else [{"type": "text", "content": "Hi!"}] + ), + } ] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 @@ -2654,7 +2812,7 @@ def test_stream_messages_close( messages=messages, model="model", ) as stream: - for _ in range(4): + for _ in range(7 if ThinkingBlock is not None else 4): next(stream) # New versions add TextEvent, so consume one more event. @@ -2682,7 +2840,20 @@ def test_stream_messages_close( == '[{"role": "user", "content": "Hello, Claude"}]' ) assert json.loads(span["data"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ - {"role": "assistant", "parts": [{"type": "text", "content": "Hi!"}]} + { + "role": "assistant", + "parts": ( + [ + { + "type": "reasoning", + "content": "Let me reason step by step...", + }, + {"type": "text", "content": "Hi! I'm Claude!"}, + ] + if ThinkingBlock is not None + else [{"type": "text", "content": "Hi!"}] + ), + } ] assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 @@ -2979,7 +3150,17 @@ async def test_streaming_create_message_async( assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ { "role": "assistant", - "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + "parts": ( + [ + { + "type": "reasoning", + "content": "Let me reason step by step...", + }, + {"type": "text", "content": "Hi! I'm Claude!"}, + ] + if ThinkingBlock is not None + else [{"type": "text", "content": "Hi! I'm Claude!"}] + ), } ] @@ -3041,7 +3222,17 @@ async def test_streaming_create_message_async( assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ { "role": "assistant", - "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + "parts": ( + [ + { + "type": "reasoning", + "content": "Let me reason step by step...", + }, + {"type": "text", "content": "Hi! I'm Claude!"}, + ] + if ThinkingBlock is not None + else [{"type": "text", "content": "Hi! I'm Claude!"}] + ), } ] @@ -3098,7 +3289,17 @@ async def test_streaming_create_message_async( assert json.loads(span["data"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ { "role": "assistant", - "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + "parts": ( + [ + { + "type": "reasoning", + "content": "Let me reason step by step...", + }, + {"type": "text", "content": "Hi! I'm Claude!"}, + ] + if ThinkingBlock is not None + else [{"type": "text", "content": "Hi! I'm Claude!"}] + ), } ] @@ -3249,7 +3450,17 @@ async def test_streaming_create_message_data_collection_outputs_async( assert json.loads(span_data[SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ { "role": "assistant", - "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + "parts": ( + [ + { + "type": "reasoning", + "content": "Let me reason step by step...", + }, + {"type": "text", "content": "Hi! I'm Claude!"}, + ] + if ThinkingBlock is not None + else [{"type": "text", "content": "Hi! I'm Claude!"}] + ), } ] else: @@ -3311,7 +3522,7 @@ async def test_streaming_create_message_async_close( max_tokens=1024, messages=messages, model="model", stream=True ) - for _ in range(4): + for _ in range(7 if ThinkingBlock is not None else 4): await messages.__anext__() await messages.close() @@ -3333,7 +3544,20 @@ async def test_streaming_create_message_async_close( == '[{"role": "user", "content": "Hello, Claude"}]' ) assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ - {"role": "assistant", "parts": [{"type": "text", "content": "Hi!"}]} + { + "role": "assistant", + "parts": ( + [ + { + "type": "reasoning", + "content": "Let me reason step by step...", + }, + {"type": "text", "content": "Hi!"}, + ] + if ThinkingBlock is not None + else [{"type": "text", "content": "Hi!"}] + ), + } ] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 @@ -3356,7 +3580,7 @@ async def test_streaming_create_message_async_close( max_tokens=1024, messages=messages, model="model", stream=True ) - for _ in range(4): + for _ in range(7 if ThinkingBlock is not None else 4): await messages.__anext__() await messages.close() @@ -3380,7 +3604,20 @@ async def test_streaming_create_message_async_close( == '[{"role": "user", "content": "Hello, Claude"}]' ) assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ - {"role": "assistant", "parts": [{"type": "text", "content": "Hi!"}]} + { + "role": "assistant", + "parts": ( + [ + { + "type": "reasoning", + "content": "Let me reason step by step...", + }, + {"type": "text", "content": "Hi!"}, + ] + if ThinkingBlock is not None + else [{"type": "text", "content": "Hi!"}] + ), + } ] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 @@ -3403,7 +3640,7 @@ async def test_streaming_create_message_async_close( max_tokens=1024, messages=messages, model="model", stream=True ) - for _ in range(4): + for _ in range(7 if ThinkingBlock is not None else 4): await messages.__anext__() await messages.close() @@ -3426,7 +3663,20 @@ async def test_streaming_create_message_async_close( == '[{"role": "user", "content": "Hello, Claude"}]' ) assert json.loads(span["data"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ - {"role": "assistant", "parts": [{"type": "text", "content": "Hi!"}]} + { + "role": "assistant", + "parts": ( + [ + { + "type": "reasoning", + "content": "Let me reason step by step...", + }, + {"type": "text", "content": "Hi!"}, + ] + if ThinkingBlock is not None + else [{"type": "text", "content": "Hi!"}] + ), + } ] assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 @@ -3725,7 +3975,17 @@ async def test_stream_message_async( assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ { "role": "assistant", - "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + "parts": ( + [ + { + "type": "reasoning", + "content": "Let me reason step by step...", + }, + {"type": "text", "content": "Hi! I'm Claude!"}, + ] + if ThinkingBlock is not None + else [{"type": "text", "content": "Hi! I'm Claude!"}] + ), } ] @@ -3785,7 +4045,17 @@ async def test_stream_message_async( assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ { "role": "assistant", - "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + "parts": ( + [ + { + "type": "reasoning", + "content": "Let me reason step by step...", + }, + {"type": "text", "content": "Hi! I'm Claude!"}, + ] + if ThinkingBlock is not None + else [{"type": "text", "content": "Hi! I'm Claude!"}] + ), } ] @@ -3839,7 +4109,17 @@ async def test_stream_message_async( assert json.loads(span["data"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ { "role": "assistant", - "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + "parts": ( + [ + { + "type": "reasoning", + "content": "Let me reason step by step...", + }, + {"type": "text", "content": "Hi! I'm Claude!"}, + ] + if ThinkingBlock is not None + else [{"type": "text", "content": "Hi! I'm Claude!"}] + ), } ] @@ -3988,7 +4268,17 @@ async def test_stream_messages_data_collection_outputs_async( assert json.loads(span_data[SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ { "role": "assistant", - "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + "parts": ( + [ + { + "type": "reasoning", + "content": "Let me reason step by step...", + }, + {"type": "text", "content": "Hi! I'm Claude!"}, + ] + if ThinkingBlock is not None + else [{"type": "text", "content": "Hi! I'm Claude!"}] + ), } ] else: @@ -4235,7 +4525,7 @@ async def test_stream_messages_async_close( messages=messages, model="model", ) as stream: - for _ in range(4): + for _ in range(7 if ThinkingBlock is not None else 4): await stream.__anext__() # New versions add TextEvent, so consume one more event. @@ -4264,7 +4554,20 @@ async def test_stream_messages_async_close( == '[{"role": "user", "content": "Hello, Claude"}]' ) assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ - {"role": "assistant", "parts": [{"type": "text", "content": "Hi!"}]} + { + "role": "assistant", + "parts": ( + [ + { + "type": "reasoning", + "content": "Let me reason step by step...", + }, + {"type": "text", "content": "Hi! I'm Claude!"}, + ] + if ThinkingBlock is not None + else [{"type": "text", "content": "Hi!"}] + ), + } ] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 @@ -4288,7 +4591,7 @@ async def test_stream_messages_async_close( messages=messages, model="model", ) as stream: - for _ in range(4): + for _ in range(7 if ThinkingBlock is not None else 4): await stream.__anext__() # New versions add TextEvent, so consume one more event. @@ -4316,7 +4619,20 @@ async def test_stream_messages_async_close( == '[{"role": "user", "content": "Hello, Claude"}]' ) assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ - {"role": "assistant", "parts": [{"type": "text", "content": "Hi!"}]} + { + "role": "assistant", + "parts": ( + [ + { + "type": "reasoning", + "content": "Let me reason step by step...", + }, + {"type": "text", "content": "Hi! I'm Claude!"}, + ] + if ThinkingBlock is not None + else [{"type": "text", "content": "Hi!"}] + ), + } ] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 @@ -4340,7 +4656,7 @@ async def test_stream_messages_async_close( messages=messages, model="model", ) as stream: - for _ in range(4): + for _ in range(7 if ThinkingBlock is not None else 4): await stream.__anext__() # New versions add TextEvent, so consume one more event. @@ -4370,7 +4686,20 @@ async def test_stream_messages_async_close( == '[{"role": "user", "content": "Hello, Claude"}]' ) assert json.loads(span["data"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ - {"role": "assistant", "parts": [{"type": "text", "content": "Hi!"}]} + { + "role": "assistant", + "parts": ( + [ + { + "type": "reasoning", + "content": "Let me reason step by step...", + }, + {"type": "text", "content": "Hi! I'm Claude!"}, + ] + if ThinkingBlock is not None + else [{"type": "text", "content": "Hi!"}] + ), + } ] assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 @@ -5940,7 +6269,7 @@ def test_collect_ai_data_with_input_json_delta(span_streaming): assert model is None assert new_usage.input_tokens == usage.input_tokens assert new_usage.output_tokens == usage.output_tokens - assert new_content_blocks == ["test"] + assert new_content_blocks == [{"type": "text", "text": "test"}] assert response_id is None assert finish_reason is None @@ -6277,21 +6606,19 @@ def test_nonstreaming_create_message_with_system_prompt( assert len(stored_messages) == 1 assert stored_messages[0]["role"] == "user" assert stored_messages[0]["content"] == "Hello, Claude" + expected_parts = ( + [ + { + "type": "reasoning", + "content": "Let me reason step by step...", + }, + {"type": "text", "content": "Hi, I'm Claude."}, + ] + if ThinkingBlock is not None + else [{"type": "text", "content": "Hi, I'm Claude."}] + ) assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ - { - "role": "assistant", - "parts": ( - [ - { - "type": "reasoning", - "content": "Let me reason step by step...", - }, - {"type": "text", "content": "Hi, I'm Claude."}, - ] - if ThinkingBlock is not None - else [{"type": "text", "content": "Hi, I'm Claude."}] - ), - } + {"role": "assistant", "parts": expected_parts} ] else: assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["attributes"] @@ -6352,21 +6679,19 @@ def test_nonstreaming_create_message_with_system_prompt( assert len(stored_messages) == 1 assert stored_messages[0]["role"] == "user" assert stored_messages[0]["content"] == "Hello, Claude" + expected_parts = ( + [ + { + "type": "reasoning", + "content": "Let me reason step by step...", + }, + {"type": "text", "content": "Hi, I'm Claude."}, + ] + if ThinkingBlock is not None + else [{"type": "text", "content": "Hi, I'm Claude."}] + ) assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ - { - "role": "assistant", - "parts": ( - [ - { - "type": "reasoning", - "content": "Let me reason step by step...", - }, - {"type": "text", "content": "Hi, I'm Claude."}, - ] - if ThinkingBlock is not None - else [{"type": "text", "content": "Hi, I'm Claude."}] - ), - } + {"role": "assistant", "parts": expected_parts} ] else: assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["attributes"] @@ -6426,21 +6751,19 @@ def test_nonstreaming_create_message_with_system_prompt( assert len(stored_messages) == 1 assert stored_messages[0]["role"] == "user" assert stored_messages[0]["content"] == "Hello, Claude" + expected_parts = ( + [ + { + "type": "reasoning", + "content": "Let me reason step by step...", + }, + {"type": "text", "content": "Hi, I'm Claude."}, + ] + if ThinkingBlock is not None + else [{"type": "text", "content": "Hi, I'm Claude."}] + ) assert json.loads(span["data"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ - { - "role": "assistant", - "parts": ( - [ - { - "type": "reasoning", - "content": "Let me reason step by step...", - }, - {"type": "text", "content": "Hi, I'm Claude."}, - ] - if ThinkingBlock is not None - else [{"type": "text", "content": "Hi, I'm Claude."}] - ), - } + {"role": "assistant", "parts": expected_parts} ] else: assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["data"] @@ -6541,21 +6864,19 @@ async def test_nonstreaming_create_message_with_system_prompt_async( assert len(stored_messages) == 1 assert stored_messages[0]["role"] == "user" assert stored_messages[0]["content"] == "Hello, Claude" + expected_parts = ( + [ + { + "type": "reasoning", + "content": "Let me reason step by step...", + }, + {"type": "text", "content": "Hi, I'm Claude."}, + ] + if ThinkingBlock is not None + else [{"type": "text", "content": "Hi, I'm Claude."}] + ) assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ - { - "role": "assistant", - "parts": ( - [ - { - "type": "reasoning", - "content": "Let me reason step by step...", - }, - {"type": "text", "content": "Hi, I'm Claude."}, - ] - if ThinkingBlock is not None - else [{"type": "text", "content": "Hi, I'm Claude."}] - ), - } + {"role": "assistant", "parts": expected_parts} ] else: assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["attributes"] @@ -6616,21 +6937,19 @@ async def test_nonstreaming_create_message_with_system_prompt_async( assert len(stored_messages) == 1 assert stored_messages[0]["role"] == "user" assert stored_messages[0]["content"] == "Hello, Claude" + expected_parts = ( + [ + { + "type": "reasoning", + "content": "Let me reason step by step...", + }, + {"type": "text", "content": "Hi, I'm Claude."}, + ] + if ThinkingBlock is not None + else [{"type": "text", "content": "Hi, I'm Claude."}] + ) assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ - { - "role": "assistant", - "parts": ( - [ - { - "type": "reasoning", - "content": "Let me reason step by step...", - }, - {"type": "text", "content": "Hi, I'm Claude."}, - ] - if ThinkingBlock is not None - else [{"type": "text", "content": "Hi, I'm Claude."}] - ), - } + {"role": "assistant", "parts": expected_parts} ] else: assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["attributes"] @@ -6690,21 +7009,19 @@ async def test_nonstreaming_create_message_with_system_prompt_async( assert len(stored_messages) == 1 assert stored_messages[0]["role"] == "user" assert stored_messages[0]["content"] == "Hello, Claude" + expected_parts = ( + [ + { + "type": "reasoning", + "content": "Let me reason step by step...", + }, + {"type": "text", "content": "Hi, I'm Claude."}, + ] + if ThinkingBlock is not None + else [{"type": "text", "content": "Hi, I'm Claude."}] + ) assert json.loads(span["data"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ - { - "role": "assistant", - "parts": ( - [ - { - "type": "reasoning", - "content": "Let me reason step by step...", - }, - {"type": "text", "content": "Hi, I'm Claude."}, - ] - if ThinkingBlock is not None - else [{"type": "text", "content": "Hi, I'm Claude."}] - ), - } + {"role": "assistant", "parts": expected_parts} ] else: assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["data"] @@ -6820,7 +7137,17 @@ def test_streaming_create_message_with_system_prompt( assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ { "role": "assistant", - "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + "parts": ( + [ + { + "type": "reasoning", + "content": "Let me reason step by step...", + }, + {"type": "text", "content": "Hi! I'm Claude!"}, + ] + if ThinkingBlock is not None + else [{"type": "text", "content": "Hi! I'm Claude!"}] + ), } ] @@ -6886,7 +7213,17 @@ def test_streaming_create_message_with_system_prompt( assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ { "role": "assistant", - "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + "parts": ( + [ + { + "type": "reasoning", + "content": "Let me reason step by step...", + }, + {"type": "text", "content": "Hi! I'm Claude!"}, + ] + if ThinkingBlock is not None + else [{"type": "text", "content": "Hi! I'm Claude!"}] + ), } ] @@ -6951,7 +7288,17 @@ def test_streaming_create_message_with_system_prompt( assert json.loads(span["data"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ { "role": "assistant", - "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + "parts": ( + [ + { + "type": "reasoning", + "content": "Let me reason step by step...", + }, + {"type": "text", "content": "Hi! I'm Claude!"}, + ] + if ThinkingBlock is not None + else [{"type": "text", "content": "Hi! I'm Claude!"}] + ), } ] @@ -7064,7 +7411,17 @@ def test_stream_messages_with_system_prompt( assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ { "role": "assistant", - "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + "parts": ( + [ + { + "type": "reasoning", + "content": "Let me reason step by step...", + }, + {"type": "text", "content": "Hi! I'm Claude!"}, + ] + if ThinkingBlock is not None + else [{"type": "text", "content": "Hi! I'm Claude!"}] + ), } ] else: @@ -7125,7 +7482,17 @@ def test_stream_messages_with_system_prompt( assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ { "role": "assistant", - "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + "parts": ( + [ + { + "type": "reasoning", + "content": "Let me reason step by step...", + }, + {"type": "text", "content": "Hi! I'm Claude!"}, + ] + if ThinkingBlock is not None + else [{"type": "text", "content": "Hi! I'm Claude!"}] + ), } ] else: @@ -7184,7 +7551,17 @@ def test_stream_messages_with_system_prompt( assert json.loads(span["data"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ { "role": "assistant", - "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + "parts": ( + [ + { + "type": "reasoning", + "content": "Let me reason step by step...", + }, + {"type": "text", "content": "Hi! I'm Claude!"}, + ] + if ThinkingBlock is not None + else [{"type": "text", "content": "Hi! I'm Claude!"}] + ), } ] else: @@ -7302,7 +7679,17 @@ async def test_stream_message_with_system_prompt_async( assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ { "role": "assistant", - "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + "parts": ( + [ + { + "type": "reasoning", + "content": "Let me reason step by step...", + }, + {"type": "text", "content": "Hi! I'm Claude!"}, + ] + if ThinkingBlock is not None + else [{"type": "text", "content": "Hi! I'm Claude!"}] + ), } ] else: @@ -7365,7 +7752,17 @@ async def test_stream_message_with_system_prompt_async( assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ { "role": "assistant", - "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + "parts": ( + [ + { + "type": "reasoning", + "content": "Let me reason step by step...", + }, + {"type": "text", "content": "Hi! I'm Claude!"}, + ] + if ThinkingBlock is not None + else [{"type": "text", "content": "Hi! I'm Claude!"}] + ), } ] else: @@ -7426,7 +7823,17 @@ async def test_stream_message_with_system_prompt_async( assert json.loads(span["data"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ { "role": "assistant", - "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + "parts": ( + [ + { + "type": "reasoning", + "content": "Let me reason step by step...", + }, + {"type": "text", "content": "Hi! I'm Claude!"}, + ] + if ThinkingBlock is not None + else [{"type": "text", "content": "Hi! I'm Claude!"}] + ), } ] @@ -7548,7 +7955,17 @@ async def test_streaming_create_message_with_system_prompt_async( assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ { "role": "assistant", - "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + "parts": ( + [ + { + "type": "reasoning", + "content": "Let me reason step by step...", + }, + {"type": "text", "content": "Hi! I'm Claude!"}, + ] + if ThinkingBlock is not None + else [{"type": "text", "content": "Hi! I'm Claude!"}] + ), } ] @@ -7615,7 +8032,17 @@ async def test_streaming_create_message_with_system_prompt_async( assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ { "role": "assistant", - "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + "parts": ( + [ + { + "type": "reasoning", + "content": "Let me reason step by step...", + }, + {"type": "text", "content": "Hi! I'm Claude!"}, + ] + if ThinkingBlock is not None + else [{"type": "text", "content": "Hi! I'm Claude!"}] + ), } ] @@ -7678,7 +8105,17 @@ async def test_streaming_create_message_with_system_prompt_async( assert json.loads(span["data"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ { "role": "assistant", - "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + "parts": ( + [ + { + "type": "reasoning", + "content": "Let me reason step by step...", + }, + {"type": "text", "content": "Hi! I'm Claude!"}, + ] + if ThinkingBlock is not None + else [{"type": "text", "content": "Hi! I'm Claude!"}] + ), } ] From 224885db3e2f0e9799c6e71b81fcb1993a96f703 Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Thu, 5 Nov 2026 08:49:14 +0100 Subject: [PATCH 3/7] feat(anthropic): Record reasoning parts in output --- sentry_sdk/integrations/anthropic.py | 24 +- .../integrations/anthropic/test_anthropic.py | 1654 ++++++++--------- 2 files changed, 788 insertions(+), 890 deletions(-) diff --git a/sentry_sdk/integrations/anthropic.py b/sentry_sdk/integrations/anthropic.py index 868b94e792..811208122d 100644 --- a/sentry_sdk/integrations/anthropic.py +++ b/sentry_sdk/integrations/anthropic.py @@ -85,7 +85,7 @@ ToolUnionParam, ) - from sentry_sdk._types import TextPart + from sentry_sdk._types import ReasoningPart, TextPart class _PatchedRawMessageStream(Stream[Any]): _span: Union[Span, StreamedSpan] @@ -731,30 +731,30 @@ def _set_output_data( record_outputs = True if record_outputs: - output_messages: "dict[str, list[Any]]" = { - "response": [], - "tool": [], - } + parts: "list[Union[TextPart, ReasoningPart]]" = [] + tool_calls = [] for output in content_blocks: if output["type"] == "text": - output_messages["response"].append(output["text"]) + parts.append({"type": "text", "content": output["text"]}) + elif output["type"] == "thinking": + parts.append({"type": "reasoning", "content": output.get("thinking")}) elif output["type"] == "tool_use": - output_messages["tool"].append(output) + tool_calls.append(output) - if len(output_messages["tool"]) > 0: + if len(tool_calls) > 0: set_data_normalized( span, SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS, - output_messages["tool"], + tool_calls, unpack=False, ) - if len(output_messages["response"]) > 0: + if len(parts) > 0: set_data_normalized( span, - SPANDATA.GEN_AI_RESPONSE_TEXT, - output_messages["response"], + SPANDATA.GEN_AI_OUTPUT_MESSAGES, + [{"role": "assistant", "parts": parts}], unpack=False, ) diff --git a/tests/integrations/anthropic/test_anthropic.py b/tests/integrations/anthropic/test_anthropic.py index 19a9f846ad..81062e90d1 100644 --- a/tests/integrations/anthropic/test_anthropic.py +++ b/tests/integrations/anthropic/test_anthropic.py @@ -23,6 +23,12 @@ async def __call__(self, *args, **kwargs): from anthropic.types.message_delta_event import MessageDeltaEvent from anthropic.types.message_start_event import MessageStartEvent +try: + from anthropic.types import ThinkingBlock +except ImportError: + ThinkingBlock = None + + try: from anthropic import APIStatusError from anthropic.types import ErrorResponse, OverloadedError @@ -78,7 +84,18 @@ async def __call__(self, *args, **kwargs): id="msg_01XFDUDYJgAACzvnptvVoYEL", model="model", role="assistant", - content=[TextBlock(type="text", text="Hi, I'm Claude.")], + content=( + [ + ThinkingBlock( + type="thinking", + thinking="Let me reason step by step...", + signature="sig123", + ), + TextBlock(type="text", text="Hi, I'm Claude."), + ] + if ThinkingBlock is not None + else [TextBlock(type="text", text="Hi, I'm Claude.")] + ), type="message", stop_reason="end_turn", usage=Usage(input_tokens=10, output_tokens=20), @@ -138,6 +155,83 @@ def data_collection_tool_use_message(): ) +@pytest.fixture +def streaming_anthropic_response(): + def inner(text_deltas, stop_reason): + return [ + MessageStartEvent( + message=Message( + id="msg_01XFDUDYJgAACzvnptvVoYEL", + model="model", + role="assistant", + content=[TextBlock(type="text", text="")], + type="message", + stop_reason="end_turn", + usage=Usage(input_tokens=10, output_tokens=20), + ), + type="message_start", + ), + ContentBlockStartEvent( + type="content_block_start", + index=0, + content_block=TextBlock(type="text", text=""), + ), + *[ + ContentBlockDeltaEvent( + delta=TextDelta(text=text, type="text_delta"), + index=0, + type="content_block_delta", + ) + for text in text_deltas + ], + ContentBlockStopEvent(type="content_block_stop", index=0), + MessageDeltaEvent( + delta=Delta(stop_reason=stop_reason), + usage=MessageDeltaUsage(output_tokens=10), + type="message_delta", + ), + ] + + return inner + + +@pytest.fixture +def streaming_anthropic_error_response(): + return [ + MessageStartEvent( + message=Message( + id="msg_01XFDUDYJgAACzvnptvVoYEL", + model="model", + role="assistant", + content=[TextBlock(type="text", text="")], + type="message", + stop_reason="end_turn", + usage=Usage(input_tokens=10, output_tokens=20), + ), + type="message_start", + ), + ContentBlockStartEvent( + type="content_block_start", + index=0, + content_block=TextBlock(type="text", text=""), + ), + ContentBlockDeltaEvent( + delta=TextDelta(text="Hi", type="text_delta"), + index=0, + type="content_block_delta", + ), + ContentBlockDeltaEvent( + delta=TextDelta(text="!", type="text_delta"), + index=0, + type="content_block_delta", + ), + ErrorResponse( + type="error", + error=OverloadedError(message="Overloaded", type="overloaded_error"), + ), + ] + + @pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.parametrize("stream_gen_ai_spans", [True, False]) @pytest.mark.parametrize( @@ -219,12 +313,23 @@ def test_nonstreaming_create_message( "content": "Hello, Claude", }, ] - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ["Hi, I'm Claude."] + expected_parts = ( + [ + { + "type": "reasoning", + "content": "Let me reason step by step...", + }, + {"type": "text", "content": "Hi, I'm Claude."}, + ] + if ThinkingBlock is not None + else [{"type": "text", "content": "Hi, I'm Claude."}] ) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + {"role": "assistant", "parts": expected_parts} + ] else: assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["attributes"] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 @@ -276,12 +381,23 @@ def test_nonstreaming_create_message( "content": "Hello, Claude", }, ] - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ["Hi, I'm Claude."] + expected_parts = ( + [ + { + "type": "reasoning", + "content": "Let me reason step by step...", + }, + {"type": "text", "content": "Hi, I'm Claude."}, + ] + if ThinkingBlock is not None + else [{"type": "text", "content": "Hi, I'm Claude."}] ) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + {"role": "assistant", "parts": expected_parts} + ] else: assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["attributes"] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 @@ -328,12 +444,23 @@ def test_nonstreaming_create_message( span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "Hello, Claude"}]' ) - assert span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ["Hi, I'm Claude."] + expected_parts = ( + [ + { + "type": "reasoning", + "content": "Let me reason step by step...", + }, + {"type": "text", "content": "Hi, I'm Claude."}, + ] + if ThinkingBlock is not None + else [{"type": "text", "content": "Hi, I'm Claude."}] ) + assert json.loads(span["data"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + {"role": "assistant", "parts": expected_parts} + ] else: assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["data"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["data"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["data"] assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 @@ -748,15 +875,23 @@ def test_nonstreaming_create_message_data_collection_outputs( assert span_data[SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 if outputs_collected: - assert span_data[SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - [DATA_COLLECTION_EXPECTED_RESPONSE_TEXT] - ) + assert json.loads(span_data[SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": [ + { + "type": "text", + "content": DATA_COLLECTION_EXPECTED_RESPONSE_TEXT, + } + ], + } + ] assert ( json.loads(span_data[SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS]) == DATA_COLLECTION_EXPECTED_TOOL_CALLS ) else: - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span_data + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span_data assert SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS not in span_data @@ -875,15 +1010,23 @@ async def test_nonstreaming_create_message_data_collection_outputs_async( assert span_data[SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 if outputs_collected: - assert span_data[SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - [DATA_COLLECTION_EXPECTED_RESPONSE_TEXT] - ) + assert json.loads(span_data[SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": [ + { + "type": "text", + "content": DATA_COLLECTION_EXPECTED_RESPONSE_TEXT, + } + ], + } + ] assert ( json.loads(span_data[SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS]) == DATA_COLLECTION_EXPECTED_TOOL_CALLS ) else: - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span_data + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span_data assert SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS not in span_data @@ -969,12 +1112,23 @@ async def test_nonstreaming_create_message_async( "content": "Hello, Claude", }, ] - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ["Hi, I'm Claude."] + expected_parts = ( + [ + { + "type": "reasoning", + "content": "Let me reason step by step...", + }, + {"type": "text", "content": "Hi, I'm Claude."}, + ] + if ThinkingBlock is not None + else [{"type": "text", "content": "Hi, I'm Claude."}] ) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + {"role": "assistant", "parts": expected_parts} + ] else: assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["attributes"] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 @@ -1024,12 +1178,23 @@ async def test_nonstreaming_create_message_async( "content": "Hello, Claude", }, ] - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ["Hi, I'm Claude."] + expected_parts = ( + [ + { + "type": "reasoning", + "content": "Let me reason step by step...", + }, + {"type": "text", "content": "Hi, I'm Claude."}, + ] + if ThinkingBlock is not None + else [{"type": "text", "content": "Hi, I'm Claude."}] ) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + {"role": "assistant", "parts": expected_parts} + ] else: assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["attributes"] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 @@ -1073,12 +1238,23 @@ async def test_nonstreaming_create_message_async( span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "Hello, Claude"}]' ) - assert span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ["Hi, I'm Claude."] + expected_parts = ( + [ + { + "type": "reasoning", + "content": "Let me reason step by step...", + }, + {"type": "text", "content": "Hi, I'm Claude."}, + ] + if ThinkingBlock is not None + else [{"type": "text", "content": "Hi, I'm Claude."}] ) + assert json.loads(span["data"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + {"role": "assistant", "parts": expected_parts} + ] else: assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["data"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["data"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["data"] assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 @@ -1110,43 +1286,16 @@ def test_streaming_create_message( server_side_event_chunks, stream_gen_ai_spans, span_streaming, + streaming_anthropic_response, ): client = Anthropic(api_key="z") response = get_model_response( server_side_event_chunks( - [ - MessageStartEvent( - message=EXAMPLE_MESSAGE, - type="message_start", - ), - ContentBlockStartEvent( - type="content_block_start", - index=0, - content_block=TextBlock(type="text", text=""), - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="Hi", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="!", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockDeltaEvent( - delta=TextDelta(text=" I'm Claude!", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockStopEvent(type="content_block_stop", index=0), - MessageDeltaEvent( - delta=Delta(stop_reason="max_tokens"), - usage=MessageDeltaUsage(output_tokens=10), - type="message_delta", - ), - ] + streaming_anthropic_response( + text_deltas=("Hi", "!", " I'm Claude!"), + stop_reason="max_tokens", + ) ) ) @@ -1209,13 +1358,16 @@ def test_streaming_create_message( "content": "Hello, Claude", }, ] - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ["Hi! I'm Claude!"] - ) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + } + ] else: assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["attributes"] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 @@ -1269,13 +1421,16 @@ def test_streaming_create_message( "content": "Hello, Claude", }, ] - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ["Hi! I'm Claude!"] - ) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + } + ] else: assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["attributes"] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 @@ -1322,13 +1477,16 @@ def test_streaming_create_message( span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "Hello, Claude"}]' ) - assert span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ["Hi! I'm Claude!"] - ) + assert json.loads(span["data"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + } + ] else: assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["data"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["data"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["data"] assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 @@ -1384,6 +1542,7 @@ def test_streaming_create_message_data_collection( server_side_event_chunks, stream_gen_ai_spans, span_streaming, + streaming_anthropic_response, ): sentry_init_kwargs = dict( integrations=[AnthropicIntegration(include_prompts=include_prompts)], @@ -1401,28 +1560,10 @@ def test_streaming_create_message_data_collection( response = get_model_response( server_side_event_chunks( - [ - MessageStartEvent( - message=EXAMPLE_MESSAGE, - type="message_start", - ), - ContentBlockStartEvent( - type="content_block_start", - index=0, - content_block=TextBlock(type="text", text=""), - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="Hi", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockStopEvent(type="content_block_stop", index=0), - MessageDeltaEvent( - delta=Delta(stop_reason="max_tokens"), - usage=MessageDeltaUsage(output_tokens=10), - type="message_delta", - ), - ] + streaming_anthropic_response( + text_deltas=("Hi",), + stop_reason="max_tokens", + ) ) ) @@ -1533,6 +1674,7 @@ def test_streaming_create_message_data_collection_outputs( server_side_event_chunks, stream_gen_ai_spans, span_streaming, + streaming_anthropic_response, ): sentry_init_kwargs = dict( integrations=[AnthropicIntegration(include_prompts=include_prompts)], @@ -1550,33 +1692,10 @@ def test_streaming_create_message_data_collection_outputs( response = get_model_response( server_side_event_chunks( - [ - MessageStartEvent( - message=EXAMPLE_MESSAGE, - type="message_start", - ), - ContentBlockStartEvent( - type="content_block_start", - index=0, - content_block=TextBlock(type="text", text=""), - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="Hi", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="! I'm Claude!", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockStopEvent(type="content_block_stop", index=0), - MessageDeltaEvent( - delta=Delta(stop_reason="max_tokens"), - usage=MessageDeltaUsage(output_tokens=10), - type="message_delta", - ), - ] + streaming_anthropic_response( + text_deltas=("Hi", "! I'm Claude!"), + stop_reason="max_tokens", + ) ) ) @@ -1627,11 +1746,14 @@ def test_streaming_create_message_data_collection_outputs( assert span_data[SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 if outputs_collected: - assert span_data[SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ["Hi! I'm Claude!"] - ) + assert json.loads(span_data[SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + } + ] else: - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span_data + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span_data @pytest.mark.parametrize("span_streaming", [True, False]) @@ -1644,43 +1766,16 @@ def test_streaming_create_message_close( server_side_event_chunks, stream_gen_ai_spans, span_streaming, + streaming_anthropic_response, ): client = Anthropic(api_key="z") response = get_model_response( server_side_event_chunks( - [ - MessageStartEvent( - message=EXAMPLE_MESSAGE, - type="message_start", - ), - ContentBlockStartEvent( - type="content_block_start", - index=0, - content_block=TextBlock(type="text", text=""), - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="Hi", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="!", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockDeltaEvent( - delta=TextDelta(text=" I'm Claude!", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockStopEvent(type="content_block_stop", index=0), - MessageDeltaEvent( - delta=Delta(stop_reason="max_tokens"), - usage=MessageDeltaUsage(output_tokens=10), - type="message_delta", - ), - ] + streaming_anthropic_response( + text_deltas=("Hi", "!", " I'm Claude!"), + stop_reason="max_tokens", + ) ) ) @@ -1734,7 +1829,9 @@ def test_streaming_create_message_close( span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "Hello, Claude"}]' ) - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps(["Hi!"]) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + {"role": "assistant", "parts": [{"type": "text", "content": "Hi!"}]} + ] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 @@ -1780,7 +1877,9 @@ def test_streaming_create_message_close( span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "Hello, Claude"}]' ) - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps(["Hi!"]) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + {"role": "assistant", "parts": [{"type": "text", "content": "Hi!"}]} + ] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 @@ -1825,7 +1924,9 @@ def test_streaming_create_message_close( span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "Hello, Claude"}]' ) - assert span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps(["Hi!"]) + assert json.loads(span["data"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + {"role": "assistant", "parts": [{"type": "text", "content": "Hi!"}]} + ] assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 @@ -1850,39 +1951,12 @@ def test_streaming_create_message_api_error( server_side_event_chunks, stream_gen_ai_spans, span_streaming, + streaming_anthropic_error_response, ): client = Anthropic(api_key="z") response = get_model_response( - server_side_event_chunks( - [ - MessageStartEvent( - message=EXAMPLE_MESSAGE, - type="message_start", - ), - ContentBlockStartEvent( - type="content_block_start", - index=0, - content_block=TextBlock(type="text", text=""), - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="Hi", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="!", type="text_delta"), - index=0, - type="content_block_delta", - ), - ErrorResponse( - type="error", - error=OverloadedError( - message="Overloaded", type="overloaded_error" - ), - ), - ] - ) + server_side_event_chunks(streaming_anthropic_error_response) ) sentry_init( @@ -1934,7 +2008,9 @@ def test_streaming_create_message_api_error( span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "Hello, Claude"}]' ) - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps(["Hi!"]) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + {"role": "assistant", "parts": [{"type": "text", "content": "Hi!"}]} + ] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 @@ -1979,7 +2055,9 @@ def test_streaming_create_message_api_error( span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "Hello, Claude"}]' ) - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps(["Hi!"]) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + {"role": "assistant", "parts": [{"type": "text", "content": "Hi!"}]} + ] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 @@ -2025,7 +2103,9 @@ def test_streaming_create_message_api_error( span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "Hello, Claude"}]' ) - assert span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps(["Hi!"]) + assert json.loads(span["data"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + {"role": "assistant", "parts": [{"type": "text", "content": "Hi!"}]} + ] assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 @@ -2061,43 +2141,16 @@ def test_stream_messages( server_side_event_chunks, stream_gen_ai_spans, span_streaming, + streaming_anthropic_response, ): client = Anthropic(api_key="z") response = get_model_response( server_side_event_chunks( - [ - MessageStartEvent( - message=EXAMPLE_MESSAGE, - type="message_start", - ), - ContentBlockStartEvent( - type="content_block_start", - index=0, - content_block=TextBlock(type="text", text=""), - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="Hi", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="!", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockDeltaEvent( - delta=TextDelta(text=" I'm Claude!", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockStopEvent(type="content_block_stop", index=0), - MessageDeltaEvent( - delta=Delta(stop_reason="max_tokens"), - usage=MessageDeltaUsage(output_tokens=10), - type="message_delta", - ), - ] + streaming_anthropic_response( + text_deltas=("Hi", "!", " I'm Claude!"), + stop_reason="max_tokens", + ) ) ) @@ -2160,13 +2213,16 @@ def test_stream_messages( "content": "Hello, Claude", }, ] - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ["Hi! I'm Claude!"] - ) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + } + ] else: assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["attributes"] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 @@ -2220,13 +2276,16 @@ def test_stream_messages( "content": "Hello, Claude", }, ] - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ["Hi! I'm Claude!"] - ) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + } + ] else: assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["attributes"] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 @@ -2273,12 +2332,15 @@ def test_stream_messages( span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "Hello, Claude"}]' ) - assert span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ["Hi! I'm Claude!"] - ) + assert json.loads(span["data"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + } + ] else: assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["data"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["data"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["data"] assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 @@ -2344,6 +2406,7 @@ def test_stream_messages_data_collection_outputs( server_side_event_chunks, stream_gen_ai_spans, span_streaming, + streaming_anthropic_response, ): sentry_init_kwargs = dict( integrations=[AnthropicIntegration(include_prompts=include_prompts)], @@ -2361,33 +2424,10 @@ def test_stream_messages_data_collection_outputs( response = get_model_response( server_side_event_chunks( - [ - MessageStartEvent( - message=EXAMPLE_MESSAGE, - type="message_start", - ), - ContentBlockStartEvent( - type="content_block_start", - index=0, - content_block=TextBlock(type="text", text=""), - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="Hi", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="! I'm Claude!", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockStopEvent(type="content_block_stop", index=0), - MessageDeltaEvent( - delta=Delta(stop_reason="max_tokens"), - usage=MessageDeltaUsage(output_tokens=10), - type="message_delta", - ), - ] + streaming_anthropic_response( + text_deltas=("Hi", "! I'm Claude!"), + stop_reason="max_tokens", + ) ) ) @@ -2439,11 +2479,14 @@ def test_stream_messages_data_collection_outputs( assert span_data[SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 if outputs_collected: - assert span_data[SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ["Hi! I'm Claude!"] - ) + assert json.loads(span_data[SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + } + ] else: - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span_data + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span_data @pytest.mark.parametrize("span_streaming", [True, False]) @@ -2456,43 +2499,16 @@ def test_stream_messages_close( server_side_event_chunks, stream_gen_ai_spans, span_streaming, + streaming_anthropic_response, ): client = Anthropic(api_key="z") response = get_model_response( server_side_event_chunks( - [ - MessageStartEvent( - message=EXAMPLE_MESSAGE, - type="message_start", - ), - ContentBlockStartEvent( - type="content_block_start", - index=0, - content_block=TextBlock(type="text", text=""), - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="Hi", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="!", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockDeltaEvent( - delta=TextDelta(text=" I'm Claude!", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockStopEvent(type="content_block_stop", index=0), - MessageDeltaEvent( - delta=Delta(stop_reason="max_tokens"), - usage=MessageDeltaUsage(output_tokens=10), - type="message_delta", - ), - ] + streaming_anthropic_response( + text_deltas=("Hi", "!", " I'm Claude!"), + stop_reason="max_tokens", + ) ) ) @@ -2550,7 +2566,9 @@ def test_stream_messages_close( span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "Hello, Claude"}]' ) - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps(["Hi!"]) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + {"role": "assistant", "parts": [{"type": "text", "content": "Hi!"}]} + ] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 @@ -2600,7 +2618,9 @@ def test_stream_messages_close( span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "Hello, Claude"}]' ) - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps(["Hi!"]) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + {"role": "assistant", "parts": [{"type": "text", "content": "Hi!"}]} + ] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 @@ -2649,7 +2669,9 @@ def test_stream_messages_close( span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "Hello, Claude"}]' ) - assert span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps(["Hi!"]) + assert json.loads(span["data"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + {"role": "assistant", "parts": [{"type": "text", "content": "Hi!"}]} + ] assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 @@ -2674,39 +2696,12 @@ def test_stream_messages_api_error( server_side_event_chunks, stream_gen_ai_spans, span_streaming, + streaming_anthropic_error_response, ): client = Anthropic(api_key="z") response = get_model_response( - server_side_event_chunks( - [ - MessageStartEvent( - message=EXAMPLE_MESSAGE, - type="message_start", - ), - ContentBlockStartEvent( - type="content_block_start", - index=0, - content_block=TextBlock(type="text", text=""), - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="Hi", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="!", type="text_delta"), - index=0, - type="content_block_delta", - ), - ErrorResponse( - type="error", - error=OverloadedError( - message="Overloaded", type="overloaded_error" - ), - ), - ] - ) + server_side_event_chunks(streaming_anthropic_error_response) ) sentry_init( @@ -2758,7 +2753,9 @@ def test_stream_messages_api_error( span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "Hello, Claude"}]' ) - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps(["Hi!"]) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + {"role": "assistant", "parts": [{"type": "text", "content": "Hi!"}]} + ] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 @@ -2804,7 +2801,9 @@ def test_stream_messages_api_error( span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "Hello, Claude"}]' ) - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps(["Hi!"]) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + {"role": "assistant", "parts": [{"type": "text", "content": "Hi!"}]} + ] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 @@ -2850,7 +2849,9 @@ def test_stream_messages_api_error( span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "Hello, Claude"}]' ) - assert span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps(["Hi!"]) + assert json.loads(span["data"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + {"role": "assistant", "parts": [{"type": "text", "content": "Hi!"}]} + ] assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 @@ -2888,46 +2889,19 @@ async def test_streaming_create_message_async( server_side_event_chunks, stream_gen_ai_spans, span_streaming, + streaming_anthropic_response, ): client = AsyncAnthropic(api_key="z") response = get_model_response( async_iterator( server_side_event_chunks( - [ - MessageStartEvent( - message=EXAMPLE_MESSAGE, - type="message_start", - ), - ContentBlockStartEvent( - type="content_block_start", - index=0, - content_block=TextBlock(type="text", text=""), - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="Hi", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="!", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockDeltaEvent( - delta=TextDelta(text=" I'm Claude!", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockStopEvent(type="content_block_stop", index=0), - MessageDeltaEvent( - delta=Delta(stop_reason="max_tokens"), - usage=MessageDeltaUsage(output_tokens=10), - type="message_delta", - ), - ] + streaming_anthropic_response( + text_deltas=("Hi", "!", " I'm Claude!"), + stop_reason="max_tokens", + ) ) - ), + ) ) sentry_init( @@ -2990,13 +2964,16 @@ async def test_streaming_create_message_async( "content": "Hello, Claude", }, ] - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ["Hi! I'm Claude!"] - ) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + } + ] else: assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["attributes"] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 @@ -3049,13 +3026,16 @@ async def test_streaming_create_message_async( "content": "Hello, Claude", }, ] - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ["Hi! I'm Claude!"] - ) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + } + ] else: assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["attributes"] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 @@ -3103,13 +3083,16 @@ async def test_streaming_create_message_async( span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "Hello, Claude"}]' ) - assert span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ["Hi! I'm Claude!"] - ) + assert json.loads(span["data"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + } + ] else: assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["data"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["data"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["data"] assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 @@ -3177,6 +3160,7 @@ async def test_streaming_create_message_data_collection_outputs_async( server_side_event_chunks, stream_gen_ai_spans, span_streaming, + streaming_anthropic_response, ): sentry_init_kwargs = dict( integrations=[AnthropicIntegration(include_prompts=include_prompts)], @@ -3195,35 +3179,12 @@ async def test_streaming_create_message_data_collection_outputs_async( response = get_model_response( async_iterator( server_side_event_chunks( - [ - MessageStartEvent( - message=EXAMPLE_MESSAGE, - type="message_start", - ), - ContentBlockStartEvent( - type="content_block_start", - index=0, - content_block=TextBlock(type="text", text=""), - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="Hi", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="! I'm Claude!", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockStopEvent(type="content_block_stop", index=0), - MessageDeltaEvent( - delta=Delta(stop_reason="max_tokens"), - usage=MessageDeltaUsage(output_tokens=10), - type="message_delta", - ), - ] + streaming_anthropic_response( + text_deltas=("Hi", "! I'm Claude!"), + stop_reason="max_tokens", + ) ) - ), + ) ) create_kwargs = dict( @@ -3273,11 +3234,14 @@ async def test_streaming_create_message_data_collection_outputs_async( assert span_data[SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 if outputs_collected: - assert span_data[SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ["Hi! I'm Claude!"] - ) + assert json.loads(span_data[SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + } + ] else: - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span_data + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span_data @pytest.mark.parametrize("span_streaming", [True, False]) @@ -3292,44 +3256,17 @@ async def test_streaming_create_message_async_close( server_side_event_chunks, stream_gen_ai_spans, span_streaming, + streaming_anthropic_response, ): client = AsyncAnthropic(api_key="z") response = get_model_response( async_iterator( server_side_event_chunks( - [ - MessageStartEvent( - message=EXAMPLE_MESSAGE, - type="message_start", - ), - ContentBlockStartEvent( - type="content_block_start", - index=0, - content_block=TextBlock(type="text", text=""), - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="Hi", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="!", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockDeltaEvent( - delta=TextDelta(text=" I'm Claude!", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockStopEvent(type="content_block_stop", index=0), - MessageDeltaEvent( - delta=Delta(stop_reason="max_tokens"), - usage=MessageDeltaUsage(output_tokens=10), - type="message_delta", - ), - ] + streaming_anthropic_response( + text_deltas=("Hi", "!", " I'm Claude!"), + stop_reason="max_tokens", + ) ) ) ) @@ -3383,7 +3320,9 @@ async def test_streaming_create_message_async_close( span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "Hello, Claude"}]' ) - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps(["Hi!"]) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + {"role": "assistant", "parts": [{"type": "text", "content": "Hi!"}]} + ] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 @@ -3428,7 +3367,9 @@ async def test_streaming_create_message_async_close( span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "Hello, Claude"}]' ) - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps(["Hi!"]) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + {"role": "assistant", "parts": [{"type": "text", "content": "Hi!"}]} + ] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 @@ -3472,7 +3413,9 @@ async def test_streaming_create_message_async_close( span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "Hello, Claude"}]' ) - assert span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps(["Hi!"]) + assert json.loads(span["data"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + {"role": "assistant", "parts": [{"type": "text", "content": "Hi!"}]} + ] assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 @@ -3499,41 +3442,12 @@ async def test_streaming_create_message_async_api_error( server_side_event_chunks, stream_gen_ai_spans, span_streaming, + streaming_anthropic_error_response, ): client = AsyncAnthropic(api_key="z") response = get_model_response( - async_iterator( - server_side_event_chunks( - [ - MessageStartEvent( - message=EXAMPLE_MESSAGE, - type="message_start", - ), - ContentBlockStartEvent( - type="content_block_start", - index=0, - content_block=TextBlock(type="text", text=""), - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="Hi", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="!", type="text_delta"), - index=0, - type="content_block_delta", - ), - ErrorResponse( - type="error", - error=OverloadedError( - message="Overloaded", type="overloaded_error" - ), - ), - ] - ) - ) + async_iterator(server_side_event_chunks(streaming_anthropic_error_response)) ) sentry_init( @@ -3585,7 +3499,9 @@ async def test_streaming_create_message_async_api_error( span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "Hello, Claude"}]' ) - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps(["Hi!"]) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + {"role": "assistant", "parts": [{"type": "text", "content": "Hi!"}]} + ] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 @@ -3631,7 +3547,9 @@ async def test_streaming_create_message_async_api_error( span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "Hello, Claude"}]' ) - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps(["Hi!"]) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + {"role": "assistant", "parts": [{"type": "text", "content": "Hi!"}]} + ] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 @@ -3677,7 +3595,9 @@ async def test_streaming_create_message_async_api_error( span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "Hello, Claude"}]' ) - assert span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps(["Hi!"]) + assert json.loads(span["data"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + {"role": "assistant", "parts": [{"type": "text", "content": "Hi!"}]} + ] assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 @@ -3715,46 +3635,19 @@ async def test_stream_message_async( server_side_event_chunks, stream_gen_ai_spans, span_streaming, + streaming_anthropic_response, ): client = AsyncAnthropic(api_key="z") response = get_model_response( async_iterator( server_side_event_chunks( - [ - MessageStartEvent( - message=EXAMPLE_MESSAGE, - type="message_start", - ), - ContentBlockStartEvent( - type="content_block_start", - index=0, - content_block=TextBlock(type="text", text=""), - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="Hi", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="!", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockDeltaEvent( - delta=TextDelta(text=" I'm Claude!", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockStopEvent(type="content_block_stop", index=0), - MessageDeltaEvent( - delta=Delta(), - usage=MessageDeltaUsage(output_tokens=10), - type="message_delta", - ), - ] + streaming_anthropic_response( + text_deltas=("Hi", "!", " I'm Claude!"), + stop_reason=None, + ) ) - ), + ) ) sentry_init( @@ -3817,13 +3710,16 @@ async def test_stream_message_async( "content": "Hello, Claude", }, ] - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ["Hi! I'm Claude!"] - ) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + } + ] else: assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["attributes"] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 @@ -3874,13 +3770,16 @@ async def test_stream_message_async( "content": "Hello, Claude", }, ] - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ["Hi! I'm Claude!"] - ) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + } + ] else: assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["attributes"] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 @@ -3925,13 +3824,16 @@ async def test_stream_message_async( span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "Hello, Claude"}]' ) - assert span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ["Hi! I'm Claude!"] - ) + assert json.loads(span["data"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + } + ] else: assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["data"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["data"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["data"] assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 @@ -3998,6 +3900,7 @@ async def test_stream_messages_data_collection_outputs_async( server_side_event_chunks, stream_gen_ai_spans, span_streaming, + streaming_anthropic_response, ): sentry_init_kwargs = dict( integrations=[AnthropicIntegration(include_prompts=include_prompts)], @@ -4016,35 +3919,12 @@ async def test_stream_messages_data_collection_outputs_async( response = get_model_response( async_iterator( server_side_event_chunks( - [ - MessageStartEvent( - message=EXAMPLE_MESSAGE, - type="message_start", - ), - ContentBlockStartEvent( - type="content_block_start", - index=0, - content_block=TextBlock(type="text", text=""), - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="Hi", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="! I'm Claude!", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockStopEvent(type="content_block_stop", index=0), - MessageDeltaEvent( - delta=Delta(stop_reason="max_tokens"), - usage=MessageDeltaUsage(output_tokens=10), - type="message_delta", - ), - ] + streaming_anthropic_response( + text_deltas=("Hi", "! I'm Claude!"), + stop_reason="max_tokens", + ) ) - ), + ) ) stream_kwargs = dict( @@ -4093,11 +3973,14 @@ async def test_stream_messages_data_collection_outputs_async( assert span_data[SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 if outputs_collected: - assert span_data[SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ["Hi! I'm Claude!"] - ) + assert json.loads(span_data[SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + } + ] else: - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span_data + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span_data @pytest.mark.parametrize("span_streaming", [True, False]) @@ -4112,45 +3995,16 @@ async def test_stream_messages_async_api_error( capture_events, capture_items, get_model_response, - async_iterator, - server_side_event_chunks, - stream_gen_ai_spans, - span_streaming, -): - client = AsyncAnthropic(api_key="z") - - response = get_model_response( - async_iterator( - server_side_event_chunks( - [ - MessageStartEvent( - message=EXAMPLE_MESSAGE, - type="message_start", - ), - ContentBlockStartEvent( - type="content_block_start", - index=0, - content_block=TextBlock(type="text", text=""), - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="Hi", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="!", type="text_delta"), - index=0, - type="content_block_delta", - ), - ErrorResponse( - type="error", - error=OverloadedError( - message="Overloaded", type="overloaded_error" - ), - ), - ] - ) - ) + async_iterator, + server_side_event_chunks, + stream_gen_ai_spans, + span_streaming, + streaming_anthropic_error_response, +): + client = AsyncAnthropic(api_key="z") + + response = get_model_response( + async_iterator(server_side_event_chunks(streaming_anthropic_error_response)) ) sentry_init( @@ -4202,7 +4056,9 @@ async def test_stream_messages_async_api_error( span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "Hello, Claude"}]' ) - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps(["Hi!"]) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + {"role": "assistant", "parts": [{"type": "text", "content": "Hi!"}]} + ] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 @@ -4246,7 +4102,9 @@ async def test_stream_messages_async_api_error( span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "Hello, Claude"}]' ) - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps(["Hi!"]) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + {"role": "assistant", "parts": [{"type": "text", "content": "Hi!"}]} + ] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 @@ -4292,7 +4150,9 @@ async def test_stream_messages_async_api_error( span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "Hello, Claude"}]' ) - assert span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps(["Hi!"]) + assert json.loads(span["data"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + {"role": "assistant", "parts": [{"type": "text", "content": "Hi!"}]} + ] assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 @@ -4319,44 +4179,17 @@ async def test_stream_messages_async_close( server_side_event_chunks, stream_gen_ai_spans, span_streaming, + streaming_anthropic_response, ): client = AsyncAnthropic(api_key="z") response = get_model_response( async_iterator( server_side_event_chunks( - [ - MessageStartEvent( - message=EXAMPLE_MESSAGE, - type="message_start", - ), - ContentBlockStartEvent( - type="content_block_start", - index=0, - content_block=TextBlock(type="text", text=""), - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="Hi", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="!", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockDeltaEvent( - delta=TextDelta(text=" I'm Claude!", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockStopEvent(type="content_block_stop", index=0), - MessageDeltaEvent( - delta=Delta(stop_reason="max_tokens"), - usage=MessageDeltaUsage(output_tokens=10), - type="message_delta", - ), - ] + streaming_anthropic_response( + text_deltas=("Hi", "!", " I'm Claude!"), + stop_reason="max_tokens", + ) ) ) ) @@ -4418,7 +4251,9 @@ async def test_stream_messages_async_close( span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "Hello, Claude"}]' ) - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps(["Hi!"]) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + {"role": "assistant", "parts": [{"type": "text", "content": "Hi!"}]} + ] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 @@ -4468,7 +4303,9 @@ async def test_stream_messages_async_close( span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "Hello, Claude"}]' ) - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps(["Hi!"]) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + {"role": "assistant", "parts": [{"type": "text", "content": "Hi!"}]} + ] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 @@ -4520,7 +4357,9 @@ async def test_stream_messages_async_close( span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "Hello, Claude"}]' ) - assert span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps(["Hi!"]) + assert json.loads(span["data"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + {"role": "assistant", "parts": [{"type": "text", "content": "Hi!"}]} + ] assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 @@ -4675,12 +4514,17 @@ def test_streaming_create_message_with_input_json_delta( span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "What is the weather like in San Francisco?"}]' ) - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ['{"location": "San Francisco, CA"}'] - ) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": [ + {"type": "text", "content": '{"location": "San Francisco, CA"}'} + ], + } + ] else: assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["attributes"] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 366 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 41 @@ -4721,12 +4565,17 @@ def test_streaming_create_message_with_input_json_delta( span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "What is the weather like in San Francisco?"}]' ) - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ['{"location": "San Francisco, CA"}'] - ) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": [ + {"type": "text", "content": '{"location": "San Francisco, CA"}'} + ], + } + ] else: assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["attributes"] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 366 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 41 @@ -4768,12 +4617,17 @@ def test_streaming_create_message_with_input_json_delta( span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "What is the weather like in San Francisco?"}]' ) - assert span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ['{"location": "San Francisco, CA"}'] - ) + assert json.loads(span["data"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": [ + {"type": "text", "content": '{"location": "San Francisco, CA"}'} + ], + } + ] else: assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["data"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["data"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["data"] assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 366 assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 41 @@ -4925,12 +4779,17 @@ def test_stream_messages_with_input_json_delta( span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "What is the weather like in San Francisco?"}]' ) - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ['{"location": "San Francisco, CA"}'] - ) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": [ + {"type": "text", "content": '{"location": "San Francisco, CA"}'} + ], + } + ] else: assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["attributes"] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 366 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 41 @@ -4971,12 +4830,17 @@ def test_stream_messages_with_input_json_delta( span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "What is the weather like in San Francisco?"}]' ) - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ['{"location": "San Francisco, CA"}'] - ) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": [ + {"type": "text", "content": '{"location": "San Francisco, CA"}'} + ], + } + ] else: assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["attributes"] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 366 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 41 @@ -5017,12 +4881,17 @@ def test_stream_messages_with_input_json_delta( span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "What is the weather like in San Francisco?"}]' ) - assert span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ['{"location": "San Francisco, CA"}'] - ) + assert json.loads(span["data"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": [ + {"type": "text", "content": '{"location": "San Francisco, CA"}'} + ], + } + ] else: assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["data"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["data"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["data"] assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 366 assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 41 @@ -5181,13 +5050,18 @@ async def test_streaming_create_message_with_input_json_delta_async( span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "What is the weather like in San Francisco?"}]' ) - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ['{"location": "San Francisco, CA"}'] - ) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": [ + {"type": "text", "content": '{"location": "San Francisco, CA"}'} + ], + } + ] else: assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["attributes"] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 366 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 41 @@ -5228,13 +5102,18 @@ async def test_streaming_create_message_with_input_json_delta_async( span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "What is the weather like in San Francisco?"}]' ) - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ['{"location": "San Francisco, CA"}'] - ) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": [ + {"type": "text", "content": '{"location": "San Francisco, CA"}'} + ], + } + ] else: assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["attributes"] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 366 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 41 @@ -5275,13 +5154,18 @@ async def test_streaming_create_message_with_input_json_delta_async( span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "What is the weather like in San Francisco?"}]' ) - assert span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ['{"location": "San Francisco, CA"}'] - ) + assert json.loads(span["data"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": [ + {"type": "text", "content": '{"location": "San Francisco, CA"}'} + ], + } + ] else: assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["data"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["data"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["data"] assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 366 assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 41 @@ -5441,12 +5325,17 @@ async def test_stream_message_with_input_json_delta_async( span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "What is the weather like in San Francisco?"}]' ) - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ['{"location": "San Francisco, CA"}'] - ) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": [ + {"type": "text", "content": '{"location": "San Francisco, CA"}'} + ], + } + ] else: assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["attributes"] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 366 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 41 @@ -5488,12 +5377,17 @@ async def test_stream_message_with_input_json_delta_async( span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "What is the weather like in San Francisco?"}]' ) - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ['{"location": "San Francisco, CA"}'] - ) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": [ + {"type": "text", "content": '{"location": "San Francisco, CA"}'} + ], + } + ] else: assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["attributes"] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 366 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 41 @@ -5535,13 +5429,18 @@ async def test_stream_message_with_input_json_delta_async( span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "What is the weather like in San Francisco?"}]' ) - assert span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ['{"location": "San Francisco, CA"}'] - ) + assert json.loads(span["data"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": [ + {"type": "text", "content": '{"location": "San Francisco, CA"}'} + ], + } + ] else: assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["data"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["data"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["data"] assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 366 assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 41 @@ -6061,9 +5960,14 @@ def test_set_output_data_with_input_json_delta(sentry_init): content_blocks=[{"text": "".join(json_deltas), "type": "text"}], ) - assert span._data.get(SPANDATA.GEN_AI_RESPONSE_TEXT) == json.dumps( - ["{'test': 'data','more': 'json'}"] - ) + assert json.loads(span._data.get(SPANDATA.GEN_AI_OUTPUT_MESSAGES)) == [ + { + "role": "assistant", + "parts": [ + {"type": "text", "content": "{'test': 'data','more': 'json'}"} + ], + } + ] assert span._data.get(SPANDATA.GEN_AI_USAGE_INPUT_TOKENS) == 10 assert span._data.get(SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS) == 20 assert span._data.get(SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS) == 30 @@ -6361,13 +6265,24 @@ def test_nonstreaming_create_message_with_system_prompt( assert len(stored_messages) == 1 assert stored_messages[0]["role"] == "user" assert stored_messages[0]["content"] == "Hello, Claude" - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ["Hi, I'm Claude."] + expected_parts = ( + [ + { + "type": "reasoning", + "content": "Let me reason step by step...", + }, + {"type": "text", "content": "Hi, I'm Claude."}, + ] + if ThinkingBlock is not None + else [{"type": "text", "content": "Hi, I'm Claude."}] ) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + {"role": "assistant", "parts": expected_parts} + ] else: assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["attributes"] assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["attributes"] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 @@ -6423,13 +6338,24 @@ def test_nonstreaming_create_message_with_system_prompt( assert len(stored_messages) == 1 assert stored_messages[0]["role"] == "user" assert stored_messages[0]["content"] == "Hello, Claude" - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ["Hi, I'm Claude."] + expected_parts = ( + [ + { + "type": "reasoning", + "content": "Let me reason step by step...", + }, + {"type": "text", "content": "Hi, I'm Claude."}, + ] + if ThinkingBlock is not None + else [{"type": "text", "content": "Hi, I'm Claude."}] ) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + {"role": "assistant", "parts": expected_parts} + ] else: assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["attributes"] assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["attributes"] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 @@ -6484,13 +6410,24 @@ def test_nonstreaming_create_message_with_system_prompt( assert len(stored_messages) == 1 assert stored_messages[0]["role"] == "user" assert stored_messages[0]["content"] == "Hello, Claude" - assert span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ["Hi, I'm Claude."] + expected_parts = ( + [ + { + "type": "reasoning", + "content": "Let me reason step by step...", + }, + {"type": "text", "content": "Hi, I'm Claude."}, + ] + if ThinkingBlock is not None + else [{"type": "text", "content": "Hi, I'm Claude."}] ) + assert json.loads(span["data"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + {"role": "assistant", "parts": expected_parts} + ] else: assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["data"] assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["data"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["data"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["data"] assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 @@ -6586,13 +6523,24 @@ async def test_nonstreaming_create_message_with_system_prompt_async( assert len(stored_messages) == 1 assert stored_messages[0]["role"] == "user" assert stored_messages[0]["content"] == "Hello, Claude" - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ["Hi, I'm Claude."] + expected_parts = ( + [ + { + "type": "reasoning", + "content": "Let me reason step by step...", + }, + {"type": "text", "content": "Hi, I'm Claude."}, + ] + if ThinkingBlock is not None + else [{"type": "text", "content": "Hi, I'm Claude."}] ) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + {"role": "assistant", "parts": expected_parts} + ] else: assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["attributes"] assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["attributes"] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 @@ -6648,13 +6596,24 @@ async def test_nonstreaming_create_message_with_system_prompt_async( assert len(stored_messages) == 1 assert stored_messages[0]["role"] == "user" assert stored_messages[0]["content"] == "Hello, Claude" - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ["Hi, I'm Claude."] + expected_parts = ( + [ + { + "type": "reasoning", + "content": "Let me reason step by step...", + }, + {"type": "text", "content": "Hi, I'm Claude."}, + ] + if ThinkingBlock is not None + else [{"type": "text", "content": "Hi, I'm Claude."}] ) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + {"role": "assistant", "parts": expected_parts} + ] else: assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["attributes"] assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["attributes"] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 @@ -6709,13 +6668,24 @@ async def test_nonstreaming_create_message_with_system_prompt_async( assert len(stored_messages) == 1 assert stored_messages[0]["role"] == "user" assert stored_messages[0]["content"] == "Hello, Claude" - assert span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ["Hi, I'm Claude."] + expected_parts = ( + [ + { + "type": "reasoning", + "content": "Let me reason step by step...", + }, + {"type": "text", "content": "Hi, I'm Claude."}, + ] + if ThinkingBlock is not None + else [{"type": "text", "content": "Hi, I'm Claude."}] ) + assert json.loads(span["data"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + {"role": "assistant", "parts": expected_parts} + ] else: assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["data"] assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["data"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["data"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["data"] assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 @@ -6745,44 +6715,17 @@ def test_streaming_create_message_with_system_prompt( server_side_event_chunks, stream_gen_ai_spans, span_streaming, + streaming_anthropic_response, ): """Test that system prompts are properly captured in streaming mode.""" client = Anthropic(api_key="z") response = get_model_response( server_side_event_chunks( - [ - MessageStartEvent( - message=EXAMPLE_MESSAGE, - type="message_start", - ), - ContentBlockStartEvent( - type="content_block_start", - index=0, - content_block=TextBlock(type="text", text=""), - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="Hi", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="!", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockDeltaEvent( - delta=TextDelta(text=" I'm Claude!", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockStopEvent(type="content_block_stop", index=0), - MessageDeltaEvent( - delta=Delta(), - usage=MessageDeltaUsage(output_tokens=10), - type="message_delta", - ), - ] + streaming_anthropic_response( + text_deltas=("Hi", "!", " I'm Claude!"), + stop_reason=None, + ) ) ) @@ -6850,14 +6793,17 @@ def test_streaming_create_message_with_system_prompt( assert len(stored_messages) == 1 assert stored_messages[0]["role"] == "user" assert stored_messages[0]["content"] == "Hello, Claude" - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ["Hi! I'm Claude!"] - ) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + } + ] else: assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["attributes"] assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["attributes"] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 @@ -6913,14 +6859,17 @@ def test_streaming_create_message_with_system_prompt( assert len(stored_messages) == 1 assert stored_messages[0]["role"] == "user" assert stored_messages[0]["content"] == "Hello, Claude" - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ["Hi! I'm Claude!"] - ) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + } + ] else: assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["attributes"] assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["attributes"] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 @@ -6975,14 +6924,17 @@ def test_streaming_create_message_with_system_prompt( assert len(stored_messages) == 1 assert stored_messages[0]["role"] == "user" assert stored_messages[0]["content"] == "Hello, Claude" - assert span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ["Hi! I'm Claude!"] - ) + assert json.loads(span["data"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + } + ] else: assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["data"] assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["data"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["data"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["data"] assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 @@ -7011,44 +6963,17 @@ def test_stream_messages_with_system_prompt( server_side_event_chunks, stream_gen_ai_spans, span_streaming, + streaming_anthropic_response, ): """Test that system prompts are properly captured in streaming mode.""" client = Anthropic(api_key="z") response = get_model_response( server_side_event_chunks( - [ - MessageStartEvent( - message=EXAMPLE_MESSAGE, - type="message_start", - ), - ContentBlockStartEvent( - type="content_block_start", - index=0, - content_block=TextBlock(type="text", text=""), - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="Hi", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="!", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockDeltaEvent( - delta=TextDelta(text=" I'm Claude!", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockStopEvent(type="content_block_stop", index=0), - MessageDeltaEvent( - delta=Delta(), - usage=MessageDeltaUsage(output_tokens=10), - type="message_delta", - ), - ] + streaming_anthropic_response( + text_deltas=("Hi", "!", " I'm Claude!"), + stop_reason=None, + ) ) ) @@ -7112,13 +7037,16 @@ def test_stream_messages_with_system_prompt( assert len(stored_messages) == 1 assert stored_messages[0]["role"] == "user" assert stored_messages[0]["content"] == "Hello, Claude" - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ["Hi! I'm Claude!"] - ) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + } + ] else: assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["attributes"] assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["attributes"] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 @@ -7170,13 +7098,16 @@ def test_stream_messages_with_system_prompt( assert len(stored_messages) == 1 assert stored_messages[0]["role"] == "user" assert stored_messages[0]["content"] == "Hello, Claude" - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ["Hi! I'm Claude!"] - ) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + } + ] else: assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["attributes"] assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["attributes"] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 @@ -7226,13 +7157,16 @@ def test_stream_messages_with_system_prompt( assert len(stored_messages) == 1 assert stored_messages[0]["role"] == "user" assert stored_messages[0]["content"] == "Hello, Claude" - assert span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ["Hi! I'm Claude!"] - ) + assert json.loads(span["data"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + } + ] else: assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["data"] assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["data"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["data"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["data"] assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 @@ -7263,6 +7197,7 @@ async def test_stream_message_with_system_prompt_async( server_side_event_chunks, stream_gen_ai_spans, span_streaming, + streaming_anthropic_response, ): """Test that system prompts are properly captured in streaming mode (async).""" client = AsyncAnthropic(api_key="z") @@ -7270,38 +7205,10 @@ async def test_stream_message_with_system_prompt_async( response = get_model_response( async_iterator( server_side_event_chunks( - [ - MessageStartEvent( - message=EXAMPLE_MESSAGE, - type="message_start", - ), - ContentBlockStartEvent( - type="content_block_start", - index=0, - content_block=TextBlock(type="text", text=""), - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="Hi", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="!", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockDeltaEvent( - delta=TextDelta(text=" I'm Claude!", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockStopEvent(type="content_block_stop", index=0), - MessageDeltaEvent( - delta=Delta(), - usage=MessageDeltaUsage(output_tokens=10), - type="message_delta", - ), - ] + streaming_anthropic_response( + text_deltas=("Hi", "!", " I'm Claude!"), + stop_reason=None, + ) ) ) ) @@ -7368,13 +7275,16 @@ async def test_stream_message_with_system_prompt_async( assert len(stored_messages) == 1 assert stored_messages[0]["role"] == "user" assert stored_messages[0]["content"] == "Hello, Claude" - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ["Hi! I'm Claude!"] - ) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + } + ] else: assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["attributes"] assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["attributes"] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 @@ -7428,13 +7338,16 @@ async def test_stream_message_with_system_prompt_async( assert len(stored_messages) == 1 assert stored_messages[0]["role"] == "user" assert stored_messages[0]["content"] == "Hello, Claude" - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ["Hi! I'm Claude!"] - ) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + } + ] else: assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["attributes"] assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["attributes"] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 @@ -7486,14 +7399,17 @@ async def test_stream_message_with_system_prompt_async( assert len(stored_messages) == 1 assert stored_messages[0]["role"] == "user" assert stored_messages[0]["content"] == "Hello, Claude" - assert span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ["Hi! I'm Claude!"] - ) + assert json.loads(span["data"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + } + ] else: assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["data"] assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["data"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["data"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["data"] assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 @@ -7524,6 +7440,7 @@ async def test_streaming_create_message_with_system_prompt_async( server_side_event_chunks, stream_gen_ai_spans, span_streaming, + streaming_anthropic_response, ): """Test that system prompts are properly captured in streaming mode (async).""" client = AsyncAnthropic(api_key="z") @@ -7531,38 +7448,10 @@ async def test_streaming_create_message_with_system_prompt_async( response = get_model_response( async_iterator( server_side_event_chunks( - [ - MessageStartEvent( - message=EXAMPLE_MESSAGE, - type="message_start", - ), - ContentBlockStartEvent( - type="content_block_start", - index=0, - content_block=TextBlock(type="text", text=""), - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="Hi", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="!", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockDeltaEvent( - delta=TextDelta(text=" I'm Claude!", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockStopEvent(type="content_block_stop", index=0), - MessageDeltaEvent( - delta=Delta(), - usage=MessageDeltaUsage(output_tokens=10), - type="message_delta", - ), - ] + streaming_anthropic_response( + text_deltas=("Hi", "!", " I'm Claude!"), + stop_reason=None, + ) ) ) ) @@ -7632,14 +7521,17 @@ async def test_streaming_create_message_with_system_prompt_async( assert len(stored_messages) == 1 assert stored_messages[0]["role"] == "user" assert stored_messages[0]["content"] == "Hello, Claude" - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ["Hi! I'm Claude!"] - ) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + } + ] else: assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["attributes"] assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["attributes"] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 @@ -7696,14 +7588,17 @@ async def test_streaming_create_message_with_system_prompt_async( assert len(stored_messages) == 1 assert stored_messages[0]["role"] == "user" assert stored_messages[0]["content"] == "Hello, Claude" - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ["Hi! I'm Claude!"] - ) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + } + ] else: assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["attributes"] assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["attributes"] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 @@ -7756,14 +7651,17 @@ async def test_streaming_create_message_with_system_prompt_async( assert len(stored_messages) == 1 assert stored_messages[0]["role"] == "user" assert stored_messages[0]["content"] == "Hello, Claude" - assert span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ["Hi! I'm Claude!"] - ) + assert json.loads(span["data"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + } + ] else: assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["data"] assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["data"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["data"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["data"] assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 From 07587e7c85eff362f11cae200f4916ff82fb3d61 Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Thu, 5 Nov 2026 09:33:12 +0100 Subject: [PATCH 4/7] feat(anthropic): Record reasoning parts in streaming output --- sentry_sdk/integrations/anthropic.py | 43 +- .../integrations/anthropic/test_anthropic.py | 575 ++++++++++++++++-- 2 files changed, 552 insertions(+), 66 deletions(-) diff --git a/sentry_sdk/integrations/anthropic.py b/sentry_sdk/integrations/anthropic.py index 811208122d..79cb9a2dba 100644 --- a/sentry_sdk/integrations/anthropic.py +++ b/sentry_sdk/integrations/anthropic.py @@ -64,6 +64,11 @@ except ImportError: raise DidNotEnable("Anthropic not installed") +try: + from anthropic.types import ThinkingDelta +except ImportError: + ThinkingDelta = None + if TYPE_CHECKING: from typing import ( Any, @@ -365,22 +370,33 @@ def _collect_ai_data( event: "MessageStreamEvent", model: "str | None", usage: "_RecordedUsage", - content_blocks: "list[str]", + content_blocks: "list[Any]", response_id: "str | None" = None, finish_reason: "str | None" = None, -) -> "tuple[str | None, _RecordedUsage, list[str], str | None, str | None]": +) -> "tuple[str | None, _RecordedUsage, list[Any], str | None, str | None]": """ Collect model information, token usage, and collect content blocks from the AI streaming response. """ with capture_internal_exceptions(): if hasattr(event, "type"): if event.type == "content_block_start": - pass + block_type = getattr(event.content_block, "type", "text") + content_blocks.append({"type": block_type, "text": ""}) elif event.type == "content_block_delta": - if hasattr(event.delta, "text"): - content_blocks.append(event.delta.text) - elif hasattr(event.delta, "partial_json"): - content_blocks.append(event.delta.partial_json) + delta = event.delta + if ThinkingDelta is not None and isinstance(delta, ThinkingDelta): + chunk = delta.thinking + elif hasattr(delta, "text"): + chunk = delta.text + elif hasattr(delta, "partial_json"): + chunk = delta.partial_json + else: + chunk = None + if chunk is not None: + if content_blocks: + content_blocks[-1]["text"] += chunk + else: + content_blocks.append({"type": "text", "text": chunk}) elif event.type == "content_block_stop": pass @@ -1042,7 +1058,7 @@ def _set_streaming_output_data( integration: "AnthropicIntegration", model: "Optional[str]", usage: "_RecordedUsage", - content_blocks: "list[str]", + content_blocks: "list[Any]", response_id: "Optional[str]", finish_reason: "Optional[str]", ) -> None: @@ -1057,6 +1073,15 @@ def _set_streaming_output_data( + (usage.cache_write_input_tokens or 0) ) + merged_blocks = [] + for block in content_blocks: + block_type = block.get("type", "text") + text = block.get("text", "") + if block_type == "thinking": + merged_blocks.append({"type": "thinking", "thinking": text}) + else: + merged_blocks.append({"type": "text", "text": text}) + _set_output_data( span=span, integration=integration, @@ -1065,7 +1090,7 @@ def _set_streaming_output_data( output_tokens=usage.output_tokens, cache_read_input_tokens=usage.cache_read_input_tokens, cache_write_input_tokens=usage.cache_write_input_tokens, - content_blocks=[{"text": "".join(content_blocks), "type": "text"}], + content_blocks=merged_blocks, response_id=response_id, finish_reason=finish_reason, ) diff --git a/tests/integrations/anthropic/test_anthropic.py b/tests/integrations/anthropic/test_anthropic.py index 81062e90d1..59781e29b6 100644 --- a/tests/integrations/anthropic/test_anthropic.py +++ b/tests/integrations/anthropic/test_anthropic.py @@ -24,9 +24,10 @@ async def __call__(self, *args, **kwargs): from anthropic.types.message_start_event import MessageStartEvent try: - from anthropic.types import ThinkingBlock + from anthropic.types import ThinkingBlock, ThinkingDelta except ImportError: ThinkingBlock = None + ThinkingDelta = None try: @@ -158,6 +159,29 @@ def data_collection_tool_use_message(): @pytest.fixture def streaming_anthropic_response(): def inner(text_deltas, stop_reason): + thinking_events = [] + if ThinkingBlock is not None: + thinking_events = [ + ContentBlockStartEvent( + type="content_block_start", + index=0, + content_block=ThinkingBlock( + type="thinking", + thinking="", + signature="sig123", + ), + ), + ContentBlockDeltaEvent( + delta=ThinkingDelta( + thinking="Let me reason step by step...", + type="thinking_delta", + ), + index=0, + type="content_block_delta", + ), + ContentBlockStopEvent(type="content_block_stop", index=0), + ] + text_index = 1 if ThinkingBlock is not None else 0 return [ MessageStartEvent( message=Message( @@ -171,20 +195,21 @@ def inner(text_deltas, stop_reason): ), type="message_start", ), + *thinking_events, ContentBlockStartEvent( type="content_block_start", - index=0, + index=text_index, content_block=TextBlock(type="text", text=""), ), *[ ContentBlockDeltaEvent( delta=TextDelta(text=text, type="text_delta"), - index=0, + index=text_index, type="content_block_delta", ) for text in text_deltas ], - ContentBlockStopEvent(type="content_block_stop", index=0), + ContentBlockStopEvent(type="content_block_stop", index=text_index), MessageDeltaEvent( delta=Delta(stop_reason=stop_reason), usage=MessageDeltaUsage(output_tokens=10), @@ -1361,7 +1386,17 @@ def test_streaming_create_message( assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ { "role": "assistant", - "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + "parts": ( + [ + { + "type": "reasoning", + "content": "Let me reason step by step...", + }, + {"type": "text", "content": "Hi! I'm Claude!"}, + ] + if ThinkingBlock is not None + else [{"type": "text", "content": "Hi! I'm Claude!"}] + ), } ] @@ -1424,7 +1459,17 @@ def test_streaming_create_message( assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ { "role": "assistant", - "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + "parts": ( + [ + { + "type": "reasoning", + "content": "Let me reason step by step...", + }, + {"type": "text", "content": "Hi! I'm Claude!"}, + ] + if ThinkingBlock is not None + else [{"type": "text", "content": "Hi! I'm Claude!"}] + ), } ] @@ -1480,7 +1525,17 @@ def test_streaming_create_message( assert json.loads(span["data"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ { "role": "assistant", - "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + "parts": ( + [ + { + "type": "reasoning", + "content": "Let me reason step by step...", + }, + {"type": "text", "content": "Hi! I'm Claude!"}, + ] + if ThinkingBlock is not None + else [{"type": "text", "content": "Hi! I'm Claude!"}] + ), } ] @@ -1749,7 +1804,17 @@ def test_streaming_create_message_data_collection_outputs( assert json.loads(span_data[SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ { "role": "assistant", - "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + "parts": ( + [ + { + "type": "reasoning", + "content": "Let me reason step by step...", + }, + {"type": "text", "content": "Hi! I'm Claude!"}, + ] + if ThinkingBlock is not None + else [{"type": "text", "content": "Hi! I'm Claude!"}] + ), } ] else: @@ -1807,7 +1872,7 @@ def test_streaming_create_message_close( max_tokens=1024, messages=messages, model="model", stream=True ) - for _ in range(4): + for _ in range(7 if ThinkingBlock is not None else 4): next(messages) messages.close() @@ -1830,7 +1895,20 @@ def test_streaming_create_message_close( == '[{"role": "user", "content": "Hello, Claude"}]' ) assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ - {"role": "assistant", "parts": [{"type": "text", "content": "Hi!"}]} + { + "role": "assistant", + "parts": ( + [ + { + "type": "reasoning", + "content": "Let me reason step by step...", + }, + {"type": "text", "content": "Hi!"}, + ] + if ThinkingBlock is not None + else [{"type": "text", "content": "Hi!"}] + ), + } ] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 @@ -1853,7 +1931,7 @@ def test_streaming_create_message_close( max_tokens=1024, messages=messages, model="model", stream=True ) - for _ in range(4): + for _ in range(7 if ThinkingBlock is not None else 4): next(messages) messages.close() @@ -1878,7 +1956,20 @@ def test_streaming_create_message_close( == '[{"role": "user", "content": "Hello, Claude"}]' ) assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ - {"role": "assistant", "parts": [{"type": "text", "content": "Hi!"}]} + { + "role": "assistant", + "parts": ( + [ + { + "type": "reasoning", + "content": "Let me reason step by step...", + }, + {"type": "text", "content": "Hi!"}, + ] + if ThinkingBlock is not None + else [{"type": "text", "content": "Hi!"}] + ), + } ] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 @@ -1901,7 +1992,7 @@ def test_streaming_create_message_close( max_tokens=1024, messages=messages, model="model", stream=True ) - for _ in range(4): + for _ in range(7 if ThinkingBlock is not None else 4): next(messages) messages.close() @@ -1925,7 +2016,20 @@ def test_streaming_create_message_close( == '[{"role": "user", "content": "Hello, Claude"}]' ) assert json.loads(span["data"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ - {"role": "assistant", "parts": [{"type": "text", "content": "Hi!"}]} + { + "role": "assistant", + "parts": ( + [ + { + "type": "reasoning", + "content": "Let me reason step by step...", + }, + {"type": "text", "content": "Hi!"}, + ] + if ThinkingBlock is not None + else [{"type": "text", "content": "Hi!"}] + ), + } ] assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 @@ -2216,7 +2320,17 @@ def test_stream_messages( assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ { "role": "assistant", - "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + "parts": ( + [ + { + "type": "reasoning", + "content": "Let me reason step by step...", + }, + {"type": "text", "content": "Hi! I'm Claude!"}, + ] + if ThinkingBlock is not None + else [{"type": "text", "content": "Hi! I'm Claude!"}] + ), } ] @@ -2279,7 +2393,17 @@ def test_stream_messages( assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ { "role": "assistant", - "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + "parts": ( + [ + { + "type": "reasoning", + "content": "Let me reason step by step...", + }, + {"type": "text", "content": "Hi! I'm Claude!"}, + ] + if ThinkingBlock is not None + else [{"type": "text", "content": "Hi! I'm Claude!"}] + ), } ] @@ -2335,7 +2459,17 @@ def test_stream_messages( assert json.loads(span["data"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ { "role": "assistant", - "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + "parts": ( + [ + { + "type": "reasoning", + "content": "Let me reason step by step...", + }, + {"type": "text", "content": "Hi! I'm Claude!"}, + ] + if ThinkingBlock is not None + else [{"type": "text", "content": "Hi! I'm Claude!"}] + ), } ] else: @@ -2482,7 +2616,17 @@ def test_stream_messages_data_collection_outputs( assert json.loads(span_data[SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ { "role": "assistant", - "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + "parts": ( + [ + { + "type": "reasoning", + "content": "Let me reason step by step...", + }, + {"type": "text", "content": "Hi! I'm Claude!"}, + ] + if ThinkingBlock is not None + else [{"type": "text", "content": "Hi! I'm Claude!"}] + ), } ] else: @@ -2540,7 +2684,7 @@ def test_stream_messages_close( messages=messages, model="model", ) as stream: - for _ in range(4): + for _ in range(7 if ThinkingBlock is not None else 4): next(stream) # New versions add TextEvent, so consume one more event. @@ -2567,7 +2711,20 @@ def test_stream_messages_close( == '[{"role": "user", "content": "Hello, Claude"}]' ) assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ - {"role": "assistant", "parts": [{"type": "text", "content": "Hi!"}]} + { + "role": "assistant", + "parts": ( + [ + { + "type": "reasoning", + "content": "Let me reason step by step...", + }, + {"type": "text", "content": "Hi! I'm Claude!"}, + ] + if ThinkingBlock is not None + else [{"type": "text", "content": "Hi!"}] + ), + } ] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 @@ -2590,7 +2747,7 @@ def test_stream_messages_close( messages=messages, model="model", ) as stream: - for _ in range(4): + for _ in range(7 if ThinkingBlock is not None else 4): next(stream) # New versions add TextEvent, so consume one more event. @@ -2619,7 +2776,20 @@ def test_stream_messages_close( == '[{"role": "user", "content": "Hello, Claude"}]' ) assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ - {"role": "assistant", "parts": [{"type": "text", "content": "Hi!"}]} + { + "role": "assistant", + "parts": ( + [ + { + "type": "reasoning", + "content": "Let me reason step by step...", + }, + {"type": "text", "content": "Hi! I'm Claude!"}, + ] + if ThinkingBlock is not None + else [{"type": "text", "content": "Hi!"}] + ), + } ] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 @@ -2642,7 +2812,7 @@ def test_stream_messages_close( messages=messages, model="model", ) as stream: - for _ in range(4): + for _ in range(7 if ThinkingBlock is not None else 4): next(stream) # New versions add TextEvent, so consume one more event. @@ -2670,7 +2840,20 @@ def test_stream_messages_close( == '[{"role": "user", "content": "Hello, Claude"}]' ) assert json.loads(span["data"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ - {"role": "assistant", "parts": [{"type": "text", "content": "Hi!"}]} + { + "role": "assistant", + "parts": ( + [ + { + "type": "reasoning", + "content": "Let me reason step by step...", + }, + {"type": "text", "content": "Hi! I'm Claude!"}, + ] + if ThinkingBlock is not None + else [{"type": "text", "content": "Hi!"}] + ), + } ] assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 @@ -2967,7 +3150,17 @@ async def test_streaming_create_message_async( assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ { "role": "assistant", - "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + "parts": ( + [ + { + "type": "reasoning", + "content": "Let me reason step by step...", + }, + {"type": "text", "content": "Hi! I'm Claude!"}, + ] + if ThinkingBlock is not None + else [{"type": "text", "content": "Hi! I'm Claude!"}] + ), } ] @@ -3029,7 +3222,17 @@ async def test_streaming_create_message_async( assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ { "role": "assistant", - "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + "parts": ( + [ + { + "type": "reasoning", + "content": "Let me reason step by step...", + }, + {"type": "text", "content": "Hi! I'm Claude!"}, + ] + if ThinkingBlock is not None + else [{"type": "text", "content": "Hi! I'm Claude!"}] + ), } ] @@ -3086,7 +3289,17 @@ async def test_streaming_create_message_async( assert json.loads(span["data"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ { "role": "assistant", - "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + "parts": ( + [ + { + "type": "reasoning", + "content": "Let me reason step by step...", + }, + {"type": "text", "content": "Hi! I'm Claude!"}, + ] + if ThinkingBlock is not None + else [{"type": "text", "content": "Hi! I'm Claude!"}] + ), } ] @@ -3237,7 +3450,17 @@ async def test_streaming_create_message_data_collection_outputs_async( assert json.loads(span_data[SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ { "role": "assistant", - "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + "parts": ( + [ + { + "type": "reasoning", + "content": "Let me reason step by step...", + }, + {"type": "text", "content": "Hi! I'm Claude!"}, + ] + if ThinkingBlock is not None + else [{"type": "text", "content": "Hi! I'm Claude!"}] + ), } ] else: @@ -3299,7 +3522,7 @@ async def test_streaming_create_message_async_close( max_tokens=1024, messages=messages, model="model", stream=True ) - for _ in range(4): + for _ in range(7 if ThinkingBlock is not None else 4): await messages.__anext__() await messages.close() @@ -3321,7 +3544,20 @@ async def test_streaming_create_message_async_close( == '[{"role": "user", "content": "Hello, Claude"}]' ) assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ - {"role": "assistant", "parts": [{"type": "text", "content": "Hi!"}]} + { + "role": "assistant", + "parts": ( + [ + { + "type": "reasoning", + "content": "Let me reason step by step...", + }, + {"type": "text", "content": "Hi!"}, + ] + if ThinkingBlock is not None + else [{"type": "text", "content": "Hi!"}] + ), + } ] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 @@ -3344,7 +3580,7 @@ async def test_streaming_create_message_async_close( max_tokens=1024, messages=messages, model="model", stream=True ) - for _ in range(4): + for _ in range(7 if ThinkingBlock is not None else 4): await messages.__anext__() await messages.close() @@ -3368,7 +3604,20 @@ async def test_streaming_create_message_async_close( == '[{"role": "user", "content": "Hello, Claude"}]' ) assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ - {"role": "assistant", "parts": [{"type": "text", "content": "Hi!"}]} + { + "role": "assistant", + "parts": ( + [ + { + "type": "reasoning", + "content": "Let me reason step by step...", + }, + {"type": "text", "content": "Hi!"}, + ] + if ThinkingBlock is not None + else [{"type": "text", "content": "Hi!"}] + ), + } ] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 @@ -3391,7 +3640,7 @@ async def test_streaming_create_message_async_close( max_tokens=1024, messages=messages, model="model", stream=True ) - for _ in range(4): + for _ in range(7 if ThinkingBlock is not None else 4): await messages.__anext__() await messages.close() @@ -3414,7 +3663,20 @@ async def test_streaming_create_message_async_close( == '[{"role": "user", "content": "Hello, Claude"}]' ) assert json.loads(span["data"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ - {"role": "assistant", "parts": [{"type": "text", "content": "Hi!"}]} + { + "role": "assistant", + "parts": ( + [ + { + "type": "reasoning", + "content": "Let me reason step by step...", + }, + {"type": "text", "content": "Hi!"}, + ] + if ThinkingBlock is not None + else [{"type": "text", "content": "Hi!"}] + ), + } ] assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 @@ -3713,7 +3975,17 @@ async def test_stream_message_async( assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ { "role": "assistant", - "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + "parts": ( + [ + { + "type": "reasoning", + "content": "Let me reason step by step...", + }, + {"type": "text", "content": "Hi! I'm Claude!"}, + ] + if ThinkingBlock is not None + else [{"type": "text", "content": "Hi! I'm Claude!"}] + ), } ] @@ -3773,7 +4045,17 @@ async def test_stream_message_async( assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ { "role": "assistant", - "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + "parts": ( + [ + { + "type": "reasoning", + "content": "Let me reason step by step...", + }, + {"type": "text", "content": "Hi! I'm Claude!"}, + ] + if ThinkingBlock is not None + else [{"type": "text", "content": "Hi! I'm Claude!"}] + ), } ] @@ -3827,7 +4109,17 @@ async def test_stream_message_async( assert json.loads(span["data"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ { "role": "assistant", - "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + "parts": ( + [ + { + "type": "reasoning", + "content": "Let me reason step by step...", + }, + {"type": "text", "content": "Hi! I'm Claude!"}, + ] + if ThinkingBlock is not None + else [{"type": "text", "content": "Hi! I'm Claude!"}] + ), } ] @@ -3976,7 +4268,17 @@ async def test_stream_messages_data_collection_outputs_async( assert json.loads(span_data[SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ { "role": "assistant", - "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + "parts": ( + [ + { + "type": "reasoning", + "content": "Let me reason step by step...", + }, + {"type": "text", "content": "Hi! I'm Claude!"}, + ] + if ThinkingBlock is not None + else [{"type": "text", "content": "Hi! I'm Claude!"}] + ), } ] else: @@ -4223,7 +4525,7 @@ async def test_stream_messages_async_close( messages=messages, model="model", ) as stream: - for _ in range(4): + for _ in range(7 if ThinkingBlock is not None else 4): await stream.__anext__() # New versions add TextEvent, so consume one more event. @@ -4252,7 +4554,20 @@ async def test_stream_messages_async_close( == '[{"role": "user", "content": "Hello, Claude"}]' ) assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ - {"role": "assistant", "parts": [{"type": "text", "content": "Hi!"}]} + { + "role": "assistant", + "parts": ( + [ + { + "type": "reasoning", + "content": "Let me reason step by step...", + }, + {"type": "text", "content": "Hi! I'm Claude!"}, + ] + if ThinkingBlock is not None + else [{"type": "text", "content": "Hi!"}] + ), + } ] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 @@ -4276,7 +4591,7 @@ async def test_stream_messages_async_close( messages=messages, model="model", ) as stream: - for _ in range(4): + for _ in range(7 if ThinkingBlock is not None else 4): await stream.__anext__() # New versions add TextEvent, so consume one more event. @@ -4304,7 +4619,20 @@ async def test_stream_messages_async_close( == '[{"role": "user", "content": "Hello, Claude"}]' ) assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ - {"role": "assistant", "parts": [{"type": "text", "content": "Hi!"}]} + { + "role": "assistant", + "parts": ( + [ + { + "type": "reasoning", + "content": "Let me reason step by step...", + }, + {"type": "text", "content": "Hi! I'm Claude!"}, + ] + if ThinkingBlock is not None + else [{"type": "text", "content": "Hi!"}] + ), + } ] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 @@ -4328,7 +4656,7 @@ async def test_stream_messages_async_close( messages=messages, model="model", ) as stream: - for _ in range(4): + for _ in range(7 if ThinkingBlock is not None else 4): await stream.__anext__() # New versions add TextEvent, so consume one more event. @@ -4358,7 +4686,20 @@ async def test_stream_messages_async_close( == '[{"role": "user", "content": "Hello, Claude"}]' ) assert json.loads(span["data"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ - {"role": "assistant", "parts": [{"type": "text", "content": "Hi!"}]} + { + "role": "assistant", + "parts": ( + [ + { + "type": "reasoning", + "content": "Let me reason step by step...", + }, + {"type": "text", "content": "Hi! I'm Claude!"}, + ] + if ThinkingBlock is not None + else [{"type": "text", "content": "Hi!"}] + ), + } ] assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 @@ -5928,7 +6269,7 @@ def test_collect_ai_data_with_input_json_delta(span_streaming): assert model is None assert new_usage.input_tokens == usage.input_tokens assert new_usage.output_tokens == usage.output_tokens - assert new_content_blocks == ["test"] + assert new_content_blocks == [{"type": "text", "text": "test"}] assert response_id is None assert finish_reason is None @@ -6796,7 +7137,17 @@ def test_streaming_create_message_with_system_prompt( assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ { "role": "assistant", - "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + "parts": ( + [ + { + "type": "reasoning", + "content": "Let me reason step by step...", + }, + {"type": "text", "content": "Hi! I'm Claude!"}, + ] + if ThinkingBlock is not None + else [{"type": "text", "content": "Hi! I'm Claude!"}] + ), } ] @@ -6862,7 +7213,17 @@ def test_streaming_create_message_with_system_prompt( assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ { "role": "assistant", - "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + "parts": ( + [ + { + "type": "reasoning", + "content": "Let me reason step by step...", + }, + {"type": "text", "content": "Hi! I'm Claude!"}, + ] + if ThinkingBlock is not None + else [{"type": "text", "content": "Hi! I'm Claude!"}] + ), } ] @@ -6927,7 +7288,17 @@ def test_streaming_create_message_with_system_prompt( assert json.loads(span["data"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ { "role": "assistant", - "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + "parts": ( + [ + { + "type": "reasoning", + "content": "Let me reason step by step...", + }, + {"type": "text", "content": "Hi! I'm Claude!"}, + ] + if ThinkingBlock is not None + else [{"type": "text", "content": "Hi! I'm Claude!"}] + ), } ] @@ -7040,7 +7411,17 @@ def test_stream_messages_with_system_prompt( assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ { "role": "assistant", - "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + "parts": ( + [ + { + "type": "reasoning", + "content": "Let me reason step by step...", + }, + {"type": "text", "content": "Hi! I'm Claude!"}, + ] + if ThinkingBlock is not None + else [{"type": "text", "content": "Hi! I'm Claude!"}] + ), } ] else: @@ -7101,7 +7482,17 @@ def test_stream_messages_with_system_prompt( assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ { "role": "assistant", - "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + "parts": ( + [ + { + "type": "reasoning", + "content": "Let me reason step by step...", + }, + {"type": "text", "content": "Hi! I'm Claude!"}, + ] + if ThinkingBlock is not None + else [{"type": "text", "content": "Hi! I'm Claude!"}] + ), } ] else: @@ -7160,7 +7551,17 @@ def test_stream_messages_with_system_prompt( assert json.loads(span["data"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ { "role": "assistant", - "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + "parts": ( + [ + { + "type": "reasoning", + "content": "Let me reason step by step...", + }, + {"type": "text", "content": "Hi! I'm Claude!"}, + ] + if ThinkingBlock is not None + else [{"type": "text", "content": "Hi! I'm Claude!"}] + ), } ] else: @@ -7278,7 +7679,17 @@ async def test_stream_message_with_system_prompt_async( assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ { "role": "assistant", - "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + "parts": ( + [ + { + "type": "reasoning", + "content": "Let me reason step by step...", + }, + {"type": "text", "content": "Hi! I'm Claude!"}, + ] + if ThinkingBlock is not None + else [{"type": "text", "content": "Hi! I'm Claude!"}] + ), } ] else: @@ -7341,7 +7752,17 @@ async def test_stream_message_with_system_prompt_async( assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ { "role": "assistant", - "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + "parts": ( + [ + { + "type": "reasoning", + "content": "Let me reason step by step...", + }, + {"type": "text", "content": "Hi! I'm Claude!"}, + ] + if ThinkingBlock is not None + else [{"type": "text", "content": "Hi! I'm Claude!"}] + ), } ] else: @@ -7402,7 +7823,17 @@ async def test_stream_message_with_system_prompt_async( assert json.loads(span["data"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ { "role": "assistant", - "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + "parts": ( + [ + { + "type": "reasoning", + "content": "Let me reason step by step...", + }, + {"type": "text", "content": "Hi! I'm Claude!"}, + ] + if ThinkingBlock is not None + else [{"type": "text", "content": "Hi! I'm Claude!"}] + ), } ] @@ -7524,7 +7955,17 @@ async def test_streaming_create_message_with_system_prompt_async( assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ { "role": "assistant", - "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + "parts": ( + [ + { + "type": "reasoning", + "content": "Let me reason step by step...", + }, + {"type": "text", "content": "Hi! I'm Claude!"}, + ] + if ThinkingBlock is not None + else [{"type": "text", "content": "Hi! I'm Claude!"}] + ), } ] @@ -7591,7 +8032,17 @@ async def test_streaming_create_message_with_system_prompt_async( assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ { "role": "assistant", - "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + "parts": ( + [ + { + "type": "reasoning", + "content": "Let me reason step by step...", + }, + {"type": "text", "content": "Hi! I'm Claude!"}, + ] + if ThinkingBlock is not None + else [{"type": "text", "content": "Hi! I'm Claude!"}] + ), } ] @@ -7654,7 +8105,17 @@ async def test_streaming_create_message_with_system_prompt_async( assert json.loads(span["data"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ { "role": "assistant", - "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + "parts": ( + [ + { + "type": "reasoning", + "content": "Let me reason step by step...", + }, + {"type": "text", "content": "Hi! I'm Claude!"}, + ] + if ThinkingBlock is not None + else [{"type": "text", "content": "Hi! I'm Claude!"}] + ), } ] From 08af830d998407ad1a83fd5bf5fcb0b6c9f9e36d Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Thu, 5 Nov 2026 10:34:57 +0100 Subject: [PATCH 5/7] merge --- sentry_sdk/integrations/anthropic.py | 51 ++++++++----------- .../integrations/anthropic/test_anthropic.py | 14 +++-- 2 files changed, 33 insertions(+), 32 deletions(-) diff --git a/sentry_sdk/integrations/anthropic.py b/sentry_sdk/integrations/anthropic.py index 79cb9a2dba..9052108b86 100644 --- a/sentry_sdk/integrations/anthropic.py +++ b/sentry_sdk/integrations/anthropic.py @@ -65,9 +65,10 @@ raise DidNotEnable("Anthropic not installed") try: - from anthropic.types import ThinkingDelta + from anthropic.types import ThinkingBlock except ImportError: - ThinkingDelta = None + ThinkingBlock = None # type: ignore[assignment,misc] + if TYPE_CHECKING: from typing import ( @@ -380,23 +381,22 @@ def _collect_ai_data( with capture_internal_exceptions(): if hasattr(event, "type"): if event.type == "content_block_start": - block_type = getattr(event.content_block, "type", "text") - content_blocks.append({"type": block_type, "text": ""}) - elif event.type == "content_block_delta": - delta = event.delta - if ThinkingDelta is not None and isinstance(delta, ThinkingDelta): - chunk = delta.thinking - elif hasattr(delta, "text"): - chunk = delta.text - elif hasattr(delta, "partial_json"): - chunk = delta.partial_json + if ThinkingBlock is not None and isinstance( + event.content_block, ThinkingBlock + ): + content_blocks.append({"type": "thinking", "thinking": ""}) + else: + content_blocks.append({"type": "text", "text": ""}) + elif event.type == "content_block_delta" and content_blocks: + block = content_blocks[-1] + if block["type"] == "thinking": + block["thinking"] += event.delta.thinking else: - chunk = None - if chunk is not None: - if content_blocks: - content_blocks[-1]["text"] += chunk - else: - content_blocks.append({"type": "text", "text": chunk}) + chunk = getattr(event.delta, "text", None) or getattr( + event.delta, "partial_json", None + ) + if chunk: + block["text"] += chunk elif event.type == "content_block_stop": pass @@ -754,7 +754,9 @@ def _set_output_data( if output["type"] == "text": parts.append({"type": "text", "content": output["text"]}) elif output["type"] == "thinking": - parts.append({"type": "reasoning", "content": output.get("thinking")}) + parts.append( + {"type": "reasoning", "content": output.get("thinking", "")} + ) elif output["type"] == "tool_use": tool_calls.append(output) @@ -1073,15 +1075,6 @@ def _set_streaming_output_data( + (usage.cache_write_input_tokens or 0) ) - merged_blocks = [] - for block in content_blocks: - block_type = block.get("type", "text") - text = block.get("text", "") - if block_type == "thinking": - merged_blocks.append({"type": "thinking", "thinking": text}) - else: - merged_blocks.append({"type": "text", "text": text}) - _set_output_data( span=span, integration=integration, @@ -1090,7 +1083,7 @@ def _set_streaming_output_data( output_tokens=usage.output_tokens, cache_read_input_tokens=usage.cache_read_input_tokens, cache_write_input_tokens=usage.cache_write_input_tokens, - content_blocks=merged_blocks, + content_blocks=content_blocks, response_id=response_id, finish_reason=finish_reason, ) diff --git a/tests/integrations/anthropic/test_anthropic.py b/tests/integrations/anthropic/test_anthropic.py index 59781e29b6..f175cb379d 100644 --- a/tests/integrations/anthropic/test_anthropic.py +++ b/tests/integrations/anthropic/test_anthropic.py @@ -6250,7 +6250,14 @@ async def test_span_origin_async( ) @pytest.mark.parametrize("span_streaming", [True, False]) def test_collect_ai_data_with_input_json_delta(span_streaming): - event = ContentBlockDeltaEvent( + start_event = ContentBlockStartEvent( + type="content_block_start", + index=0, + content_block=ToolUseBlock( + type="tool_use", id="toolu_123", name="get_weather", input={} + ), + ) + delta_event = ContentBlockDeltaEvent( delta=InputJSONDelta(partial_json="test", type="input_json_delta"), index=0, type="content_block_delta", @@ -6261,10 +6268,11 @@ def test_collect_ai_data_with_input_json_delta(span_streaming): usage.output_tokens = 20 usage.input_tokens = 10 - content_blocks = [] + content_blocks: list = [] + _collect_ai_data(start_event, model, usage, content_blocks) model, new_usage, new_content_blocks, response_id, finish_reason = _collect_ai_data( - event, model, usage, content_blocks + delta_event, model, usage, content_blocks ) assert model is None assert new_usage.input_tokens == usage.input_tokens From d6f725bc9e3b064d9451088d288bd10b9dd6fa6c Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Thu, 5 Nov 2026 10:54:30 +0100 Subject: [PATCH 6/7] use square bracket access since it's output --- sentry_sdk/integrations/anthropic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sentry_sdk/integrations/anthropic.py b/sentry_sdk/integrations/anthropic.py index 811208122d..33ebbbdc5e 100644 --- a/sentry_sdk/integrations/anthropic.py +++ b/sentry_sdk/integrations/anthropic.py @@ -738,7 +738,7 @@ def _set_output_data( if output["type"] == "text": parts.append({"type": "text", "content": output["text"]}) elif output["type"] == "thinking": - parts.append({"type": "reasoning", "content": output.get("thinking")}) + parts.append({"type": "reasoning", "content": output["thinking"]}) elif output["type"] == "tool_use": tool_calls.append(output) From 13567049fdd6abc1c366baf876dcac8e603b3153 Mon Sep 17 00:00:00 2001 From: Alexander Alderman Webb Date: Thu, 5 Nov 2026 11:33:17 +0100 Subject: [PATCH 7/7] remove some local vars in tests --- sentry_sdk/integrations/anthropic.py | 18 ++- .../integrations/anthropic/test_anthropic.py | 142 +++++++----------- 2 files changed, 65 insertions(+), 95 deletions(-) diff --git a/sentry_sdk/integrations/anthropic.py b/sentry_sdk/integrations/anthropic.py index 8d178cd1ce..6af004f209 100644 --- a/sentry_sdk/integrations/anthropic.py +++ b/sentry_sdk/integrations/anthropic.py @@ -387,14 +387,16 @@ def _collect_ai_data( content_blocks.append({"type": "thinking", "thinking": ""}) elif event.content_block.type in ("text", "tool_use"): content_blocks.append({"type": "text", "text": ""}) - elif event.type == "content_block_delta" and content_blocks: - delta = event.delta - if ThinkingDelta is not None and isinstance(delta, ThinkingDelta): - content_blocks[-1]["thinking"] += delta.thinking - elif hasattr(delta, "text"): - content_blocks[-1]["text"] += delta.text - elif hasattr(delta, "partial_json"): - content_blocks[-1]["text"] += delta.partial_json + else: + content_blocks.append({"type": event.content_block.type}) + elif event.type == "content_block_delta": + block = content_blocks[event.index] + if ThinkingDelta is not None and isinstance(event.delta, ThinkingDelta): + block["thinking"] += event.delta.thinking + elif hasattr(event.delta, "text"): + block["text"] += event.delta.text + elif hasattr(event.delta, "partial_json"): + block["text"] += event.delta.partial_json elif event.type == "content_block_stop": pass diff --git a/tests/integrations/anthropic/test_anthropic.py b/tests/integrations/anthropic/test_anthropic.py index 0020a43fc3..2e6fe7c0dc 100644 --- a/tests/integrations/anthropic/test_anthropic.py +++ b/tests/integrations/anthropic/test_anthropic.py @@ -222,42 +222,46 @@ def inner(text_deltas, stop_reason): @pytest.fixture def streaming_anthropic_text_response(): - def inner(text_deltas, stop_reason): - return [ - MessageStartEvent( - message=Message( - id="msg_01XFDUDYJgAACzvnptvVoYEL", - model="model", - role="assistant", - content=[TextBlock(type="text", text="")], - type="message", - stop_reason="end_turn", - usage=Usage(input_tokens=10, output_tokens=20), - ), - type="message_start", - ), - ContentBlockStartEvent( - type="content_block_start", - index=0, - content_block=TextBlock(type="text", text=""), - ), - *[ - ContentBlockDeltaEvent( - delta=TextDelta(text=text, type="text_delta"), - index=0, - type="content_block_delta", - ) - for text in text_deltas - ], - ContentBlockStopEvent(type="content_block_stop", index=0), - MessageDeltaEvent( - delta=Delta(stop_reason=stop_reason), - usage=MessageDeltaUsage(output_tokens=10), - type="message_delta", + return [ + MessageStartEvent( + message=Message( + id="msg_01XFDUDYJgAACzvnptvVoYEL", + model="model", + role="assistant", + content=[TextBlock(type="text", text="")], + type="message", + stop_reason="end_turn", + usage=Usage(input_tokens=10, output_tokens=20), ), - ] - - return inner + type="message_start", + ), + ContentBlockStartEvent( + type="content_block_start", + index=0, + content_block=TextBlock(type="text", text=""), + ), + ContentBlockDeltaEvent( + delta=TextDelta(text="Hi", type="text_delta"), + index=0, + type="content_block_delta", + ), + ContentBlockDeltaEvent( + delta=TextDelta(text="!", type="text_delta"), + index=0, + type="content_block_delta", + ), + ContentBlockDeltaEvent( + delta=TextDelta(text=" I'm Claude!", type="text_delta"), + index=0, + type="content_block_delta", + ), + ContentBlockStopEvent(type="content_block_stop", index=0), + MessageDeltaEvent( + delta=Delta(stop_reason="max_tokens"), + usage=MessageDeltaUsage(output_tokens=10), + type="message_delta", + ), + ] @pytest.fixture @@ -1868,12 +1872,7 @@ def test_streaming_create_message_close( client = Anthropic(api_key="z") response = get_model_response( - server_side_event_chunks( - streaming_anthropic_text_response( - text_deltas=("Hi", "!", " I'm Claude!"), - stop_reason="max_tokens", - ) - ) + server_side_event_chunks(streaming_anthropic_text_response) ) sentry_init( @@ -1926,9 +1925,8 @@ def test_streaming_create_message_close( span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "Hello, Claude"}]' ) - expected_parts = [{"type": "text", "content": "Hi!"}] assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ - {"role": "assistant", "parts": expected_parts} + {"role": "assistant", "parts": [{"type": "text", "content": "Hi!"}]} ] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 @@ -1975,9 +1973,8 @@ def test_streaming_create_message_close( span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "Hello, Claude"}]' ) - expected_parts = [{"type": "text", "content": "Hi!"}] assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ - {"role": "assistant", "parts": expected_parts} + {"role": "assistant", "parts": [{"type": "text", "content": "Hi!"}]} ] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 @@ -2023,9 +2020,8 @@ def test_streaming_create_message_close( span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "Hello, Claude"}]' ) - expected_parts = [{"type": "text", "content": "Hi!"}] assert json.loads(span["data"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ - {"role": "assistant", "parts": expected_parts} + {"role": "assistant", "parts": [{"type": "text", "content": "Hi!"}]} ] assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 @@ -2636,12 +2632,7 @@ def test_stream_messages_close( client = Anthropic(api_key="z") response = get_model_response( - server_side_event_chunks( - streaming_anthropic_text_response( - text_deltas=("Hi", "!", " I'm Claude!"), - stop_reason="max_tokens", - ) - ) + server_side_event_chunks(streaming_anthropic_text_response) ) sentry_init( @@ -2698,9 +2689,8 @@ def test_stream_messages_close( span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "Hello, Claude"}]' ) - expected_parts = [{"type": "text", "content": "Hi!"}] assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ - {"role": "assistant", "parts": expected_parts} + {"role": "assistant", "parts": [{"type": "text", "content": "Hi!"}]} ] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 @@ -2751,9 +2741,8 @@ def test_stream_messages_close( span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "Hello, Claude"}]' ) - expected_parts = [{"type": "text", "content": "Hi!"}] assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ - {"role": "assistant", "parts": expected_parts} + {"role": "assistant", "parts": [{"type": "text", "content": "Hi!"}]} ] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 @@ -2803,9 +2792,8 @@ def test_stream_messages_close( span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "Hello, Claude"}]' ) - expected_parts = [{"type": "text", "content": "Hi!"}] assert json.loads(span["data"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ - {"role": "assistant", "parts": expected_parts} + {"role": "assistant", "parts": [{"type": "text", "content": "Hi!"}]} ] assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 @@ -3428,14 +3416,7 @@ async def test_streaming_create_message_async_close( client = AsyncAnthropic(api_key="z") response = get_model_response( - async_iterator( - server_side_event_chunks( - streaming_anthropic_text_response( - text_deltas=("Hi", "!", " I'm Claude!"), - stop_reason="max_tokens", - ) - ) - ) + async_iterator(server_side_event_chunks(streaming_anthropic_text_response)) ) sentry_init( @@ -3487,9 +3468,8 @@ async def test_streaming_create_message_async_close( span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "Hello, Claude"}]' ) - expected_parts = [{"type": "text", "content": "Hi!"}] assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ - {"role": "assistant", "parts": expected_parts} + {"role": "assistant", "parts": [{"type": "text", "content": "Hi!"}]} ] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 @@ -3535,9 +3515,8 @@ async def test_streaming_create_message_async_close( span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "Hello, Claude"}]' ) - expected_parts = [{"type": "text", "content": "Hi!"}] assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ - {"role": "assistant", "parts": expected_parts} + {"role": "assistant", "parts": [{"type": "text", "content": "Hi!"}]} ] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 @@ -3582,9 +3561,8 @@ async def test_streaming_create_message_async_close( span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "Hello, Claude"}]' ) - expected_parts = [{"type": "text", "content": "Hi!"}] assert json.loads(span["data"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ - {"role": "assistant", "parts": expected_parts} + {"role": "assistant", "parts": [{"type": "text", "content": "Hi!"}]} ] assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 @@ -4386,14 +4364,7 @@ async def test_stream_messages_async_close( client = AsyncAnthropic(api_key="z") response = get_model_response( - async_iterator( - server_side_event_chunks( - streaming_anthropic_text_response( - text_deltas=("Hi", "!", " I'm Claude!"), - stop_reason="max_tokens", - ) - ) - ) + async_iterator(server_side_event_chunks(streaming_anthropic_text_response)) ) sentry_init( @@ -4453,9 +4424,8 @@ async def test_stream_messages_async_close( span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "Hello, Claude"}]' ) - expected_parts = [{"type": "text", "content": "Hi!"}] assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ - {"role": "assistant", "parts": expected_parts} + {"role": "assistant", "parts": [{"type": "text", "content": "Hi!"}]} ] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 @@ -4506,9 +4476,8 @@ async def test_stream_messages_async_close( span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "Hello, Claude"}]' ) - expected_parts = [{"type": "text", "content": "Hi!"}] assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ - {"role": "assistant", "parts": expected_parts} + {"role": "assistant", "parts": [{"type": "text", "content": "Hi!"}]} ] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 @@ -4561,9 +4530,8 @@ async def test_stream_messages_async_close( span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "Hello, Claude"}]' ) - expected_parts = [{"type": "text", "content": "Hi!"}] assert json.loads(span["data"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ - {"role": "assistant", "parts": expected_parts} + {"role": "assistant", "parts": [{"type": "text", "content": "Hi!"}]} ] assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10