diff --git a/sentry_sdk/integrations/anthropic.py b/sentry_sdk/integrations/anthropic.py index 868b94e792..33ebbbdc5e 100644 --- a/sentry_sdk/integrations/anthropic.py +++ b/sentry_sdk/integrations/anthropic.py @@ -85,7 +85,7 @@ ToolUnionParam, ) - from sentry_sdk._types import TextPart + from sentry_sdk._types import ReasoningPart, TextPart class _PatchedRawMessageStream(Stream[Any]): _span: Union[Span, StreamedSpan] @@ -731,30 +731,30 @@ def _set_output_data( record_outputs = True if record_outputs: - output_messages: "dict[str, list[Any]]" = { - "response": [], - "tool": [], - } + parts: "list[Union[TextPart, ReasoningPart]]" = [] + tool_calls = [] for output in content_blocks: if output["type"] == "text": - output_messages["response"].append(output["text"]) + parts.append({"type": "text", "content": output["text"]}) + elif output["type"] == "thinking": + parts.append({"type": "reasoning", "content": output["thinking"]}) elif output["type"] == "tool_use": - output_messages["tool"].append(output) + tool_calls.append(output) - if len(output_messages["tool"]) > 0: + if len(tool_calls) > 0: set_data_normalized( span, SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS, - output_messages["tool"], + tool_calls, unpack=False, ) - if len(output_messages["response"]) > 0: + if len(parts) > 0: set_data_normalized( span, - SPANDATA.GEN_AI_RESPONSE_TEXT, - output_messages["response"], + SPANDATA.GEN_AI_OUTPUT_MESSAGES, + [{"role": "assistant", "parts": parts}], unpack=False, ) diff --git a/tests/integrations/anthropic/test_anthropic.py b/tests/integrations/anthropic/test_anthropic.py index 19a9f846ad..81062e90d1 100644 --- a/tests/integrations/anthropic/test_anthropic.py +++ b/tests/integrations/anthropic/test_anthropic.py @@ -23,6 +23,12 @@ async def __call__(self, *args, **kwargs): from anthropic.types.message_delta_event import MessageDeltaEvent from anthropic.types.message_start_event import MessageStartEvent +try: + from anthropic.types import ThinkingBlock +except ImportError: + ThinkingBlock = None + + try: from anthropic import APIStatusError from anthropic.types import ErrorResponse, OverloadedError @@ -78,7 +84,18 @@ async def __call__(self, *args, **kwargs): id="msg_01XFDUDYJgAACzvnptvVoYEL", model="model", role="assistant", - content=[TextBlock(type="text", text="Hi, I'm Claude.")], + content=( + [ + ThinkingBlock( + type="thinking", + thinking="Let me reason step by step...", + signature="sig123", + ), + TextBlock(type="text", text="Hi, I'm Claude."), + ] + if ThinkingBlock is not None + else [TextBlock(type="text", text="Hi, I'm Claude.")] + ), type="message", stop_reason="end_turn", usage=Usage(input_tokens=10, output_tokens=20), @@ -138,6 +155,83 @@ def data_collection_tool_use_message(): ) +@pytest.fixture +def streaming_anthropic_response(): + def inner(text_deltas, stop_reason): + return [ + MessageStartEvent( + message=Message( + id="msg_01XFDUDYJgAACzvnptvVoYEL", + model="model", + role="assistant", + content=[TextBlock(type="text", text="")], + type="message", + stop_reason="end_turn", + usage=Usage(input_tokens=10, output_tokens=20), + ), + type="message_start", + ), + ContentBlockStartEvent( + type="content_block_start", + index=0, + content_block=TextBlock(type="text", text=""), + ), + *[ + ContentBlockDeltaEvent( + delta=TextDelta(text=text, type="text_delta"), + index=0, + type="content_block_delta", + ) + for text in text_deltas + ], + ContentBlockStopEvent(type="content_block_stop", index=0), + MessageDeltaEvent( + delta=Delta(stop_reason=stop_reason), + usage=MessageDeltaUsage(output_tokens=10), + type="message_delta", + ), + ] + + return inner + + +@pytest.fixture +def streaming_anthropic_error_response(): + return [ + MessageStartEvent( + message=Message( + id="msg_01XFDUDYJgAACzvnptvVoYEL", + model="model", + role="assistant", + content=[TextBlock(type="text", text="")], + type="message", + stop_reason="end_turn", + usage=Usage(input_tokens=10, output_tokens=20), + ), + type="message_start", + ), + ContentBlockStartEvent( + type="content_block_start", + index=0, + content_block=TextBlock(type="text", text=""), + ), + ContentBlockDeltaEvent( + delta=TextDelta(text="Hi", type="text_delta"), + index=0, + type="content_block_delta", + ), + ContentBlockDeltaEvent( + delta=TextDelta(text="!", type="text_delta"), + index=0, + type="content_block_delta", + ), + ErrorResponse( + type="error", + error=OverloadedError(message="Overloaded", type="overloaded_error"), + ), + ] + + @pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.parametrize("stream_gen_ai_spans", [True, False]) @pytest.mark.parametrize( @@ -219,12 +313,23 @@ def test_nonstreaming_create_message( "content": "Hello, Claude", }, ] - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ["Hi, I'm Claude."] + expected_parts = ( + [ + { + "type": "reasoning", + "content": "Let me reason step by step...", + }, + {"type": "text", "content": "Hi, I'm Claude."}, + ] + if ThinkingBlock is not None + else [{"type": "text", "content": "Hi, I'm Claude."}] ) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + {"role": "assistant", "parts": expected_parts} + ] else: assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["attributes"] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 @@ -276,12 +381,23 @@ def test_nonstreaming_create_message( "content": "Hello, Claude", }, ] - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ["Hi, I'm Claude."] + expected_parts = ( + [ + { + "type": "reasoning", + "content": "Let me reason step by step...", + }, + {"type": "text", "content": "Hi, I'm Claude."}, + ] + if ThinkingBlock is not None + else [{"type": "text", "content": "Hi, I'm Claude."}] ) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + {"role": "assistant", "parts": expected_parts} + ] else: assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["attributes"] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 @@ -328,12 +444,23 @@ def test_nonstreaming_create_message( span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "Hello, Claude"}]' ) - assert span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ["Hi, I'm Claude."] + expected_parts = ( + [ + { + "type": "reasoning", + "content": "Let me reason step by step...", + }, + {"type": "text", "content": "Hi, I'm Claude."}, + ] + if ThinkingBlock is not None + else [{"type": "text", "content": "Hi, I'm Claude."}] ) + assert json.loads(span["data"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + {"role": "assistant", "parts": expected_parts} + ] else: assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["data"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["data"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["data"] assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 @@ -748,15 +875,23 @@ def test_nonstreaming_create_message_data_collection_outputs( assert span_data[SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 if outputs_collected: - assert span_data[SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - [DATA_COLLECTION_EXPECTED_RESPONSE_TEXT] - ) + assert json.loads(span_data[SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": [ + { + "type": "text", + "content": DATA_COLLECTION_EXPECTED_RESPONSE_TEXT, + } + ], + } + ] assert ( json.loads(span_data[SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS]) == DATA_COLLECTION_EXPECTED_TOOL_CALLS ) else: - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span_data + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span_data assert SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS not in span_data @@ -875,15 +1010,23 @@ async def test_nonstreaming_create_message_data_collection_outputs_async( assert span_data[SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 if outputs_collected: - assert span_data[SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - [DATA_COLLECTION_EXPECTED_RESPONSE_TEXT] - ) + assert json.loads(span_data[SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": [ + { + "type": "text", + "content": DATA_COLLECTION_EXPECTED_RESPONSE_TEXT, + } + ], + } + ] assert ( json.loads(span_data[SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS]) == DATA_COLLECTION_EXPECTED_TOOL_CALLS ) else: - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span_data + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span_data assert SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS not in span_data @@ -969,12 +1112,23 @@ async def test_nonstreaming_create_message_async( "content": "Hello, Claude", }, ] - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ["Hi, I'm Claude."] + expected_parts = ( + [ + { + "type": "reasoning", + "content": "Let me reason step by step...", + }, + {"type": "text", "content": "Hi, I'm Claude."}, + ] + if ThinkingBlock is not None + else [{"type": "text", "content": "Hi, I'm Claude."}] ) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + {"role": "assistant", "parts": expected_parts} + ] else: assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["attributes"] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 @@ -1024,12 +1178,23 @@ async def test_nonstreaming_create_message_async( "content": "Hello, Claude", }, ] - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ["Hi, I'm Claude."] + expected_parts = ( + [ + { + "type": "reasoning", + "content": "Let me reason step by step...", + }, + {"type": "text", "content": "Hi, I'm Claude."}, + ] + if ThinkingBlock is not None + else [{"type": "text", "content": "Hi, I'm Claude."}] ) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + {"role": "assistant", "parts": expected_parts} + ] else: assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["attributes"] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 @@ -1073,12 +1238,23 @@ async def test_nonstreaming_create_message_async( span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "Hello, Claude"}]' ) - assert span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ["Hi, I'm Claude."] + expected_parts = ( + [ + { + "type": "reasoning", + "content": "Let me reason step by step...", + }, + {"type": "text", "content": "Hi, I'm Claude."}, + ] + if ThinkingBlock is not None + else [{"type": "text", "content": "Hi, I'm Claude."}] ) + assert json.loads(span["data"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + {"role": "assistant", "parts": expected_parts} + ] else: assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["data"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["data"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["data"] assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 @@ -1110,43 +1286,16 @@ def test_streaming_create_message( server_side_event_chunks, stream_gen_ai_spans, span_streaming, + streaming_anthropic_response, ): client = Anthropic(api_key="z") response = get_model_response( server_side_event_chunks( - [ - MessageStartEvent( - message=EXAMPLE_MESSAGE, - type="message_start", - ), - ContentBlockStartEvent( - type="content_block_start", - index=0, - content_block=TextBlock(type="text", text=""), - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="Hi", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="!", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockDeltaEvent( - delta=TextDelta(text=" I'm Claude!", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockStopEvent(type="content_block_stop", index=0), - MessageDeltaEvent( - delta=Delta(stop_reason="max_tokens"), - usage=MessageDeltaUsage(output_tokens=10), - type="message_delta", - ), - ] + streaming_anthropic_response( + text_deltas=("Hi", "!", " I'm Claude!"), + stop_reason="max_tokens", + ) ) ) @@ -1209,13 +1358,16 @@ def test_streaming_create_message( "content": "Hello, Claude", }, ] - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ["Hi! I'm Claude!"] - ) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + } + ] else: assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["attributes"] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 @@ -1269,13 +1421,16 @@ def test_streaming_create_message( "content": "Hello, Claude", }, ] - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ["Hi! I'm Claude!"] - ) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + } + ] else: assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["attributes"] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 @@ -1322,13 +1477,16 @@ def test_streaming_create_message( span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "Hello, Claude"}]' ) - assert span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ["Hi! I'm Claude!"] - ) + assert json.loads(span["data"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + } + ] else: assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["data"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["data"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["data"] assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 @@ -1384,6 +1542,7 @@ def test_streaming_create_message_data_collection( server_side_event_chunks, stream_gen_ai_spans, span_streaming, + streaming_anthropic_response, ): sentry_init_kwargs = dict( integrations=[AnthropicIntegration(include_prompts=include_prompts)], @@ -1401,28 +1560,10 @@ def test_streaming_create_message_data_collection( response = get_model_response( server_side_event_chunks( - [ - MessageStartEvent( - message=EXAMPLE_MESSAGE, - type="message_start", - ), - ContentBlockStartEvent( - type="content_block_start", - index=0, - content_block=TextBlock(type="text", text=""), - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="Hi", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockStopEvent(type="content_block_stop", index=0), - MessageDeltaEvent( - delta=Delta(stop_reason="max_tokens"), - usage=MessageDeltaUsage(output_tokens=10), - type="message_delta", - ), - ] + streaming_anthropic_response( + text_deltas=("Hi",), + stop_reason="max_tokens", + ) ) ) @@ -1533,6 +1674,7 @@ def test_streaming_create_message_data_collection_outputs( server_side_event_chunks, stream_gen_ai_spans, span_streaming, + streaming_anthropic_response, ): sentry_init_kwargs = dict( integrations=[AnthropicIntegration(include_prompts=include_prompts)], @@ -1550,33 +1692,10 @@ def test_streaming_create_message_data_collection_outputs( response = get_model_response( server_side_event_chunks( - [ - MessageStartEvent( - message=EXAMPLE_MESSAGE, - type="message_start", - ), - ContentBlockStartEvent( - type="content_block_start", - index=0, - content_block=TextBlock(type="text", text=""), - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="Hi", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="! I'm Claude!", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockStopEvent(type="content_block_stop", index=0), - MessageDeltaEvent( - delta=Delta(stop_reason="max_tokens"), - usage=MessageDeltaUsage(output_tokens=10), - type="message_delta", - ), - ] + streaming_anthropic_response( + text_deltas=("Hi", "! I'm Claude!"), + stop_reason="max_tokens", + ) ) ) @@ -1627,11 +1746,14 @@ def test_streaming_create_message_data_collection_outputs( assert span_data[SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 if outputs_collected: - assert span_data[SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ["Hi! I'm Claude!"] - ) + assert json.loads(span_data[SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + } + ] else: - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span_data + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span_data @pytest.mark.parametrize("span_streaming", [True, False]) @@ -1644,43 +1766,16 @@ def test_streaming_create_message_close( server_side_event_chunks, stream_gen_ai_spans, span_streaming, + streaming_anthropic_response, ): client = Anthropic(api_key="z") response = get_model_response( server_side_event_chunks( - [ - MessageStartEvent( - message=EXAMPLE_MESSAGE, - type="message_start", - ), - ContentBlockStartEvent( - type="content_block_start", - index=0, - content_block=TextBlock(type="text", text=""), - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="Hi", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="!", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockDeltaEvent( - delta=TextDelta(text=" I'm Claude!", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockStopEvent(type="content_block_stop", index=0), - MessageDeltaEvent( - delta=Delta(stop_reason="max_tokens"), - usage=MessageDeltaUsage(output_tokens=10), - type="message_delta", - ), - ] + streaming_anthropic_response( + text_deltas=("Hi", "!", " I'm Claude!"), + stop_reason="max_tokens", + ) ) ) @@ -1734,7 +1829,9 @@ def test_streaming_create_message_close( span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "Hello, Claude"}]' ) - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps(["Hi!"]) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + {"role": "assistant", "parts": [{"type": "text", "content": "Hi!"}]} + ] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 @@ -1780,7 +1877,9 @@ def test_streaming_create_message_close( span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "Hello, Claude"}]' ) - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps(["Hi!"]) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + {"role": "assistant", "parts": [{"type": "text", "content": "Hi!"}]} + ] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 @@ -1825,7 +1924,9 @@ def test_streaming_create_message_close( span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "Hello, Claude"}]' ) - assert span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps(["Hi!"]) + assert json.loads(span["data"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + {"role": "assistant", "parts": [{"type": "text", "content": "Hi!"}]} + ] assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 @@ -1850,39 +1951,12 @@ def test_streaming_create_message_api_error( server_side_event_chunks, stream_gen_ai_spans, span_streaming, + streaming_anthropic_error_response, ): client = Anthropic(api_key="z") response = get_model_response( - server_side_event_chunks( - [ - MessageStartEvent( - message=EXAMPLE_MESSAGE, - type="message_start", - ), - ContentBlockStartEvent( - type="content_block_start", - index=0, - content_block=TextBlock(type="text", text=""), - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="Hi", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="!", type="text_delta"), - index=0, - type="content_block_delta", - ), - ErrorResponse( - type="error", - error=OverloadedError( - message="Overloaded", type="overloaded_error" - ), - ), - ] - ) + server_side_event_chunks(streaming_anthropic_error_response) ) sentry_init( @@ -1934,7 +2008,9 @@ def test_streaming_create_message_api_error( span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "Hello, Claude"}]' ) - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps(["Hi!"]) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + {"role": "assistant", "parts": [{"type": "text", "content": "Hi!"}]} + ] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 @@ -1979,7 +2055,9 @@ def test_streaming_create_message_api_error( span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "Hello, Claude"}]' ) - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps(["Hi!"]) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + {"role": "assistant", "parts": [{"type": "text", "content": "Hi!"}]} + ] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 @@ -2025,7 +2103,9 @@ def test_streaming_create_message_api_error( span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "Hello, Claude"}]' ) - assert span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps(["Hi!"]) + assert json.loads(span["data"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + {"role": "assistant", "parts": [{"type": "text", "content": "Hi!"}]} + ] assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 @@ -2061,43 +2141,16 @@ def test_stream_messages( server_side_event_chunks, stream_gen_ai_spans, span_streaming, + streaming_anthropic_response, ): client = Anthropic(api_key="z") response = get_model_response( server_side_event_chunks( - [ - MessageStartEvent( - message=EXAMPLE_MESSAGE, - type="message_start", - ), - ContentBlockStartEvent( - type="content_block_start", - index=0, - content_block=TextBlock(type="text", text=""), - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="Hi", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="!", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockDeltaEvent( - delta=TextDelta(text=" I'm Claude!", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockStopEvent(type="content_block_stop", index=0), - MessageDeltaEvent( - delta=Delta(stop_reason="max_tokens"), - usage=MessageDeltaUsage(output_tokens=10), - type="message_delta", - ), - ] + streaming_anthropic_response( + text_deltas=("Hi", "!", " I'm Claude!"), + stop_reason="max_tokens", + ) ) ) @@ -2160,13 +2213,16 @@ def test_stream_messages( "content": "Hello, Claude", }, ] - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ["Hi! I'm Claude!"] - ) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + } + ] else: assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["attributes"] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 @@ -2220,13 +2276,16 @@ def test_stream_messages( "content": "Hello, Claude", }, ] - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ["Hi! I'm Claude!"] - ) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + } + ] else: assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["attributes"] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 @@ -2273,12 +2332,15 @@ def test_stream_messages( span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "Hello, Claude"}]' ) - assert span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ["Hi! I'm Claude!"] - ) + assert json.loads(span["data"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + } + ] else: assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["data"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["data"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["data"] assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 @@ -2344,6 +2406,7 @@ def test_stream_messages_data_collection_outputs( server_side_event_chunks, stream_gen_ai_spans, span_streaming, + streaming_anthropic_response, ): sentry_init_kwargs = dict( integrations=[AnthropicIntegration(include_prompts=include_prompts)], @@ -2361,33 +2424,10 @@ def test_stream_messages_data_collection_outputs( response = get_model_response( server_side_event_chunks( - [ - MessageStartEvent( - message=EXAMPLE_MESSAGE, - type="message_start", - ), - ContentBlockStartEvent( - type="content_block_start", - index=0, - content_block=TextBlock(type="text", text=""), - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="Hi", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="! I'm Claude!", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockStopEvent(type="content_block_stop", index=0), - MessageDeltaEvent( - delta=Delta(stop_reason="max_tokens"), - usage=MessageDeltaUsage(output_tokens=10), - type="message_delta", - ), - ] + streaming_anthropic_response( + text_deltas=("Hi", "! I'm Claude!"), + stop_reason="max_tokens", + ) ) ) @@ -2439,11 +2479,14 @@ def test_stream_messages_data_collection_outputs( assert span_data[SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 if outputs_collected: - assert span_data[SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ["Hi! I'm Claude!"] - ) + assert json.loads(span_data[SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + } + ] else: - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span_data + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span_data @pytest.mark.parametrize("span_streaming", [True, False]) @@ -2456,43 +2499,16 @@ def test_stream_messages_close( server_side_event_chunks, stream_gen_ai_spans, span_streaming, + streaming_anthropic_response, ): client = Anthropic(api_key="z") response = get_model_response( server_side_event_chunks( - [ - MessageStartEvent( - message=EXAMPLE_MESSAGE, - type="message_start", - ), - ContentBlockStartEvent( - type="content_block_start", - index=0, - content_block=TextBlock(type="text", text=""), - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="Hi", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="!", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockDeltaEvent( - delta=TextDelta(text=" I'm Claude!", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockStopEvent(type="content_block_stop", index=0), - MessageDeltaEvent( - delta=Delta(stop_reason="max_tokens"), - usage=MessageDeltaUsage(output_tokens=10), - type="message_delta", - ), - ] + streaming_anthropic_response( + text_deltas=("Hi", "!", " I'm Claude!"), + stop_reason="max_tokens", + ) ) ) @@ -2550,7 +2566,9 @@ def test_stream_messages_close( span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "Hello, Claude"}]' ) - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps(["Hi!"]) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + {"role": "assistant", "parts": [{"type": "text", "content": "Hi!"}]} + ] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 @@ -2600,7 +2618,9 @@ def test_stream_messages_close( span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "Hello, Claude"}]' ) - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps(["Hi!"]) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + {"role": "assistant", "parts": [{"type": "text", "content": "Hi!"}]} + ] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 @@ -2649,7 +2669,9 @@ def test_stream_messages_close( span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "Hello, Claude"}]' ) - assert span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps(["Hi!"]) + assert json.loads(span["data"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + {"role": "assistant", "parts": [{"type": "text", "content": "Hi!"}]} + ] assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 @@ -2674,39 +2696,12 @@ def test_stream_messages_api_error( server_side_event_chunks, stream_gen_ai_spans, span_streaming, + streaming_anthropic_error_response, ): client = Anthropic(api_key="z") response = get_model_response( - server_side_event_chunks( - [ - MessageStartEvent( - message=EXAMPLE_MESSAGE, - type="message_start", - ), - ContentBlockStartEvent( - type="content_block_start", - index=0, - content_block=TextBlock(type="text", text=""), - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="Hi", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="!", type="text_delta"), - index=0, - type="content_block_delta", - ), - ErrorResponse( - type="error", - error=OverloadedError( - message="Overloaded", type="overloaded_error" - ), - ), - ] - ) + server_side_event_chunks(streaming_anthropic_error_response) ) sentry_init( @@ -2758,7 +2753,9 @@ def test_stream_messages_api_error( span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "Hello, Claude"}]' ) - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps(["Hi!"]) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + {"role": "assistant", "parts": [{"type": "text", "content": "Hi!"}]} + ] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 @@ -2804,7 +2801,9 @@ def test_stream_messages_api_error( span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "Hello, Claude"}]' ) - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps(["Hi!"]) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + {"role": "assistant", "parts": [{"type": "text", "content": "Hi!"}]} + ] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 @@ -2850,7 +2849,9 @@ def test_stream_messages_api_error( span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "Hello, Claude"}]' ) - assert span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps(["Hi!"]) + assert json.loads(span["data"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + {"role": "assistant", "parts": [{"type": "text", "content": "Hi!"}]} + ] assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 @@ -2888,46 +2889,19 @@ async def test_streaming_create_message_async( server_side_event_chunks, stream_gen_ai_spans, span_streaming, + streaming_anthropic_response, ): client = AsyncAnthropic(api_key="z") response = get_model_response( async_iterator( server_side_event_chunks( - [ - MessageStartEvent( - message=EXAMPLE_MESSAGE, - type="message_start", - ), - ContentBlockStartEvent( - type="content_block_start", - index=0, - content_block=TextBlock(type="text", text=""), - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="Hi", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="!", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockDeltaEvent( - delta=TextDelta(text=" I'm Claude!", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockStopEvent(type="content_block_stop", index=0), - MessageDeltaEvent( - delta=Delta(stop_reason="max_tokens"), - usage=MessageDeltaUsage(output_tokens=10), - type="message_delta", - ), - ] + streaming_anthropic_response( + text_deltas=("Hi", "!", " I'm Claude!"), + stop_reason="max_tokens", + ) ) - ), + ) ) sentry_init( @@ -2990,13 +2964,16 @@ async def test_streaming_create_message_async( "content": "Hello, Claude", }, ] - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ["Hi! I'm Claude!"] - ) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + } + ] else: assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["attributes"] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 @@ -3049,13 +3026,16 @@ async def test_streaming_create_message_async( "content": "Hello, Claude", }, ] - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ["Hi! I'm Claude!"] - ) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + } + ] else: assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["attributes"] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 @@ -3103,13 +3083,16 @@ async def test_streaming_create_message_async( span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "Hello, Claude"}]' ) - assert span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ["Hi! I'm Claude!"] - ) + assert json.loads(span["data"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + } + ] else: assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["data"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["data"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["data"] assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 @@ -3177,6 +3160,7 @@ async def test_streaming_create_message_data_collection_outputs_async( server_side_event_chunks, stream_gen_ai_spans, span_streaming, + streaming_anthropic_response, ): sentry_init_kwargs = dict( integrations=[AnthropicIntegration(include_prompts=include_prompts)], @@ -3195,35 +3179,12 @@ async def test_streaming_create_message_data_collection_outputs_async( response = get_model_response( async_iterator( server_side_event_chunks( - [ - MessageStartEvent( - message=EXAMPLE_MESSAGE, - type="message_start", - ), - ContentBlockStartEvent( - type="content_block_start", - index=0, - content_block=TextBlock(type="text", text=""), - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="Hi", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="! I'm Claude!", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockStopEvent(type="content_block_stop", index=0), - MessageDeltaEvent( - delta=Delta(stop_reason="max_tokens"), - usage=MessageDeltaUsage(output_tokens=10), - type="message_delta", - ), - ] + streaming_anthropic_response( + text_deltas=("Hi", "! I'm Claude!"), + stop_reason="max_tokens", + ) ) - ), + ) ) create_kwargs = dict( @@ -3273,11 +3234,14 @@ async def test_streaming_create_message_data_collection_outputs_async( assert span_data[SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 if outputs_collected: - assert span_data[SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ["Hi! I'm Claude!"] - ) + assert json.loads(span_data[SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + } + ] else: - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span_data + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span_data @pytest.mark.parametrize("span_streaming", [True, False]) @@ -3292,44 +3256,17 @@ async def test_streaming_create_message_async_close( server_side_event_chunks, stream_gen_ai_spans, span_streaming, + streaming_anthropic_response, ): client = AsyncAnthropic(api_key="z") response = get_model_response( async_iterator( server_side_event_chunks( - [ - MessageStartEvent( - message=EXAMPLE_MESSAGE, - type="message_start", - ), - ContentBlockStartEvent( - type="content_block_start", - index=0, - content_block=TextBlock(type="text", text=""), - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="Hi", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="!", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockDeltaEvent( - delta=TextDelta(text=" I'm Claude!", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockStopEvent(type="content_block_stop", index=0), - MessageDeltaEvent( - delta=Delta(stop_reason="max_tokens"), - usage=MessageDeltaUsage(output_tokens=10), - type="message_delta", - ), - ] + streaming_anthropic_response( + text_deltas=("Hi", "!", " I'm Claude!"), + stop_reason="max_tokens", + ) ) ) ) @@ -3383,7 +3320,9 @@ async def test_streaming_create_message_async_close( span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "Hello, Claude"}]' ) - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps(["Hi!"]) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + {"role": "assistant", "parts": [{"type": "text", "content": "Hi!"}]} + ] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 @@ -3428,7 +3367,9 @@ async def test_streaming_create_message_async_close( span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "Hello, Claude"}]' ) - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps(["Hi!"]) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + {"role": "assistant", "parts": [{"type": "text", "content": "Hi!"}]} + ] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 @@ -3472,7 +3413,9 @@ async def test_streaming_create_message_async_close( span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "Hello, Claude"}]' ) - assert span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps(["Hi!"]) + assert json.loads(span["data"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + {"role": "assistant", "parts": [{"type": "text", "content": "Hi!"}]} + ] assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 @@ -3499,41 +3442,12 @@ async def test_streaming_create_message_async_api_error( server_side_event_chunks, stream_gen_ai_spans, span_streaming, + streaming_anthropic_error_response, ): client = AsyncAnthropic(api_key="z") response = get_model_response( - async_iterator( - server_side_event_chunks( - [ - MessageStartEvent( - message=EXAMPLE_MESSAGE, - type="message_start", - ), - ContentBlockStartEvent( - type="content_block_start", - index=0, - content_block=TextBlock(type="text", text=""), - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="Hi", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="!", type="text_delta"), - index=0, - type="content_block_delta", - ), - ErrorResponse( - type="error", - error=OverloadedError( - message="Overloaded", type="overloaded_error" - ), - ), - ] - ) - ) + async_iterator(server_side_event_chunks(streaming_anthropic_error_response)) ) sentry_init( @@ -3585,7 +3499,9 @@ async def test_streaming_create_message_async_api_error( span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "Hello, Claude"}]' ) - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps(["Hi!"]) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + {"role": "assistant", "parts": [{"type": "text", "content": "Hi!"}]} + ] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 @@ -3631,7 +3547,9 @@ async def test_streaming_create_message_async_api_error( span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "Hello, Claude"}]' ) - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps(["Hi!"]) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + {"role": "assistant", "parts": [{"type": "text", "content": "Hi!"}]} + ] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 @@ -3677,7 +3595,9 @@ async def test_streaming_create_message_async_api_error( span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "Hello, Claude"}]' ) - assert span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps(["Hi!"]) + assert json.loads(span["data"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + {"role": "assistant", "parts": [{"type": "text", "content": "Hi!"}]} + ] assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 @@ -3715,46 +3635,19 @@ async def test_stream_message_async( server_side_event_chunks, stream_gen_ai_spans, span_streaming, + streaming_anthropic_response, ): client = AsyncAnthropic(api_key="z") response = get_model_response( async_iterator( server_side_event_chunks( - [ - MessageStartEvent( - message=EXAMPLE_MESSAGE, - type="message_start", - ), - ContentBlockStartEvent( - type="content_block_start", - index=0, - content_block=TextBlock(type="text", text=""), - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="Hi", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="!", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockDeltaEvent( - delta=TextDelta(text=" I'm Claude!", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockStopEvent(type="content_block_stop", index=0), - MessageDeltaEvent( - delta=Delta(), - usage=MessageDeltaUsage(output_tokens=10), - type="message_delta", - ), - ] + streaming_anthropic_response( + text_deltas=("Hi", "!", " I'm Claude!"), + stop_reason=None, + ) ) - ), + ) ) sentry_init( @@ -3817,13 +3710,16 @@ async def test_stream_message_async( "content": "Hello, Claude", }, ] - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ["Hi! I'm Claude!"] - ) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + } + ] else: assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["attributes"] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 @@ -3874,13 +3770,16 @@ async def test_stream_message_async( "content": "Hello, Claude", }, ] - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ["Hi! I'm Claude!"] - ) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + } + ] else: assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["attributes"] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 @@ -3925,13 +3824,16 @@ async def test_stream_message_async( span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "Hello, Claude"}]' ) - assert span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ["Hi! I'm Claude!"] - ) + assert json.loads(span["data"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + } + ] else: assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["data"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["data"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["data"] assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 @@ -3998,6 +3900,7 @@ async def test_stream_messages_data_collection_outputs_async( server_side_event_chunks, stream_gen_ai_spans, span_streaming, + streaming_anthropic_response, ): sentry_init_kwargs = dict( integrations=[AnthropicIntegration(include_prompts=include_prompts)], @@ -4016,35 +3919,12 @@ async def test_stream_messages_data_collection_outputs_async( response = get_model_response( async_iterator( server_side_event_chunks( - [ - MessageStartEvent( - message=EXAMPLE_MESSAGE, - type="message_start", - ), - ContentBlockStartEvent( - type="content_block_start", - index=0, - content_block=TextBlock(type="text", text=""), - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="Hi", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="! I'm Claude!", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockStopEvent(type="content_block_stop", index=0), - MessageDeltaEvent( - delta=Delta(stop_reason="max_tokens"), - usage=MessageDeltaUsage(output_tokens=10), - type="message_delta", - ), - ] + streaming_anthropic_response( + text_deltas=("Hi", "! I'm Claude!"), + stop_reason="max_tokens", + ) ) - ), + ) ) stream_kwargs = dict( @@ -4093,11 +3973,14 @@ async def test_stream_messages_data_collection_outputs_async( assert span_data[SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 if outputs_collected: - assert span_data[SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ["Hi! I'm Claude!"] - ) + assert json.loads(span_data[SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + } + ] else: - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span_data + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span_data @pytest.mark.parametrize("span_streaming", [True, False]) @@ -4112,45 +3995,16 @@ async def test_stream_messages_async_api_error( capture_events, capture_items, get_model_response, - async_iterator, - server_side_event_chunks, - stream_gen_ai_spans, - span_streaming, -): - client = AsyncAnthropic(api_key="z") - - response = get_model_response( - async_iterator( - server_side_event_chunks( - [ - MessageStartEvent( - message=EXAMPLE_MESSAGE, - type="message_start", - ), - ContentBlockStartEvent( - type="content_block_start", - index=0, - content_block=TextBlock(type="text", text=""), - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="Hi", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="!", type="text_delta"), - index=0, - type="content_block_delta", - ), - ErrorResponse( - type="error", - error=OverloadedError( - message="Overloaded", type="overloaded_error" - ), - ), - ] - ) - ) + async_iterator, + server_side_event_chunks, + stream_gen_ai_spans, + span_streaming, + streaming_anthropic_error_response, +): + client = AsyncAnthropic(api_key="z") + + response = get_model_response( + async_iterator(server_side_event_chunks(streaming_anthropic_error_response)) ) sentry_init( @@ -4202,7 +4056,9 @@ async def test_stream_messages_async_api_error( span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "Hello, Claude"}]' ) - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps(["Hi!"]) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + {"role": "assistant", "parts": [{"type": "text", "content": "Hi!"}]} + ] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 @@ -4246,7 +4102,9 @@ async def test_stream_messages_async_api_error( span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "Hello, Claude"}]' ) - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps(["Hi!"]) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + {"role": "assistant", "parts": [{"type": "text", "content": "Hi!"}]} + ] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 @@ -4292,7 +4150,9 @@ async def test_stream_messages_async_api_error( span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "Hello, Claude"}]' ) - assert span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps(["Hi!"]) + assert json.loads(span["data"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + {"role": "assistant", "parts": [{"type": "text", "content": "Hi!"}]} + ] assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 @@ -4319,44 +4179,17 @@ async def test_stream_messages_async_close( server_side_event_chunks, stream_gen_ai_spans, span_streaming, + streaming_anthropic_response, ): client = AsyncAnthropic(api_key="z") response = get_model_response( async_iterator( server_side_event_chunks( - [ - MessageStartEvent( - message=EXAMPLE_MESSAGE, - type="message_start", - ), - ContentBlockStartEvent( - type="content_block_start", - index=0, - content_block=TextBlock(type="text", text=""), - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="Hi", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="!", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockDeltaEvent( - delta=TextDelta(text=" I'm Claude!", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockStopEvent(type="content_block_stop", index=0), - MessageDeltaEvent( - delta=Delta(stop_reason="max_tokens"), - usage=MessageDeltaUsage(output_tokens=10), - type="message_delta", - ), - ] + streaming_anthropic_response( + text_deltas=("Hi", "!", " I'm Claude!"), + stop_reason="max_tokens", + ) ) ) ) @@ -4418,7 +4251,9 @@ async def test_stream_messages_async_close( span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "Hello, Claude"}]' ) - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps(["Hi!"]) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + {"role": "assistant", "parts": [{"type": "text", "content": "Hi!"}]} + ] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 @@ -4468,7 +4303,9 @@ async def test_stream_messages_async_close( span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "Hello, Claude"}]' ) - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps(["Hi!"]) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + {"role": "assistant", "parts": [{"type": "text", "content": "Hi!"}]} + ] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 @@ -4520,7 +4357,9 @@ async def test_stream_messages_async_close( span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "Hello, Claude"}]' ) - assert span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps(["Hi!"]) + assert json.loads(span["data"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + {"role": "assistant", "parts": [{"type": "text", "content": "Hi!"}]} + ] assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 @@ -4675,12 +4514,17 @@ def test_streaming_create_message_with_input_json_delta( span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "What is the weather like in San Francisco?"}]' ) - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ['{"location": "San Francisco, CA"}'] - ) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": [ + {"type": "text", "content": '{"location": "San Francisco, CA"}'} + ], + } + ] else: assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["attributes"] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 366 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 41 @@ -4721,12 +4565,17 @@ def test_streaming_create_message_with_input_json_delta( span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "What is the weather like in San Francisco?"}]' ) - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ['{"location": "San Francisco, CA"}'] - ) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": [ + {"type": "text", "content": '{"location": "San Francisco, CA"}'} + ], + } + ] else: assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["attributes"] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 366 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 41 @@ -4768,12 +4617,17 @@ def test_streaming_create_message_with_input_json_delta( span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "What is the weather like in San Francisco?"}]' ) - assert span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ['{"location": "San Francisco, CA"}'] - ) + assert json.loads(span["data"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": [ + {"type": "text", "content": '{"location": "San Francisco, CA"}'} + ], + } + ] else: assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["data"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["data"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["data"] assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 366 assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 41 @@ -4925,12 +4779,17 @@ def test_stream_messages_with_input_json_delta( span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "What is the weather like in San Francisco?"}]' ) - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ['{"location": "San Francisco, CA"}'] - ) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": [ + {"type": "text", "content": '{"location": "San Francisco, CA"}'} + ], + } + ] else: assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["attributes"] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 366 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 41 @@ -4971,12 +4830,17 @@ def test_stream_messages_with_input_json_delta( span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "What is the weather like in San Francisco?"}]' ) - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ['{"location": "San Francisco, CA"}'] - ) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": [ + {"type": "text", "content": '{"location": "San Francisco, CA"}'} + ], + } + ] else: assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["attributes"] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 366 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 41 @@ -5017,12 +4881,17 @@ def test_stream_messages_with_input_json_delta( span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "What is the weather like in San Francisco?"}]' ) - assert span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ['{"location": "San Francisco, CA"}'] - ) + assert json.loads(span["data"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": [ + {"type": "text", "content": '{"location": "San Francisco, CA"}'} + ], + } + ] else: assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["data"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["data"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["data"] assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 366 assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 41 @@ -5181,13 +5050,18 @@ async def test_streaming_create_message_with_input_json_delta_async( span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "What is the weather like in San Francisco?"}]' ) - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ['{"location": "San Francisco, CA"}'] - ) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": [ + {"type": "text", "content": '{"location": "San Francisco, CA"}'} + ], + } + ] else: assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["attributes"] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 366 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 41 @@ -5228,13 +5102,18 @@ async def test_streaming_create_message_with_input_json_delta_async( span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "What is the weather like in San Francisco?"}]' ) - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ['{"location": "San Francisco, CA"}'] - ) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": [ + {"type": "text", "content": '{"location": "San Francisco, CA"}'} + ], + } + ] else: assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["attributes"] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 366 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 41 @@ -5275,13 +5154,18 @@ async def test_streaming_create_message_with_input_json_delta_async( span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "What is the weather like in San Francisco?"}]' ) - assert span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ['{"location": "San Francisco, CA"}'] - ) + assert json.loads(span["data"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": [ + {"type": "text", "content": '{"location": "San Francisco, CA"}'} + ], + } + ] else: assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["data"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["data"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["data"] assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 366 assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 41 @@ -5441,12 +5325,17 @@ async def test_stream_message_with_input_json_delta_async( span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "What is the weather like in San Francisco?"}]' ) - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ['{"location": "San Francisco, CA"}'] - ) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": [ + {"type": "text", "content": '{"location": "San Francisco, CA"}'} + ], + } + ] else: assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["attributes"] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 366 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 41 @@ -5488,12 +5377,17 @@ async def test_stream_message_with_input_json_delta_async( span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "What is the weather like in San Francisco?"}]' ) - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ['{"location": "San Francisco, CA"}'] - ) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": [ + {"type": "text", "content": '{"location": "San Francisco, CA"}'} + ], + } + ] else: assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["attributes"] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 366 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 41 @@ -5535,13 +5429,18 @@ async def test_stream_message_with_input_json_delta_async( span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "What is the weather like in San Francisco?"}]' ) - assert span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ['{"location": "San Francisco, CA"}'] - ) + assert json.loads(span["data"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": [ + {"type": "text", "content": '{"location": "San Francisco, CA"}'} + ], + } + ] else: assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["data"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["data"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["data"] assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 366 assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 41 @@ -6061,9 +5960,14 @@ def test_set_output_data_with_input_json_delta(sentry_init): content_blocks=[{"text": "".join(json_deltas), "type": "text"}], ) - assert span._data.get(SPANDATA.GEN_AI_RESPONSE_TEXT) == json.dumps( - ["{'test': 'data','more': 'json'}"] - ) + assert json.loads(span._data.get(SPANDATA.GEN_AI_OUTPUT_MESSAGES)) == [ + { + "role": "assistant", + "parts": [ + {"type": "text", "content": "{'test': 'data','more': 'json'}"} + ], + } + ] assert span._data.get(SPANDATA.GEN_AI_USAGE_INPUT_TOKENS) == 10 assert span._data.get(SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS) == 20 assert span._data.get(SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS) == 30 @@ -6361,13 +6265,24 @@ def test_nonstreaming_create_message_with_system_prompt( assert len(stored_messages) == 1 assert stored_messages[0]["role"] == "user" assert stored_messages[0]["content"] == "Hello, Claude" - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ["Hi, I'm Claude."] + expected_parts = ( + [ + { + "type": "reasoning", + "content": "Let me reason step by step...", + }, + {"type": "text", "content": "Hi, I'm Claude."}, + ] + if ThinkingBlock is not None + else [{"type": "text", "content": "Hi, I'm Claude."}] ) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + {"role": "assistant", "parts": expected_parts} + ] else: assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["attributes"] assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["attributes"] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 @@ -6423,13 +6338,24 @@ def test_nonstreaming_create_message_with_system_prompt( assert len(stored_messages) == 1 assert stored_messages[0]["role"] == "user" assert stored_messages[0]["content"] == "Hello, Claude" - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ["Hi, I'm Claude."] + expected_parts = ( + [ + { + "type": "reasoning", + "content": "Let me reason step by step...", + }, + {"type": "text", "content": "Hi, I'm Claude."}, + ] + if ThinkingBlock is not None + else [{"type": "text", "content": "Hi, I'm Claude."}] ) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + {"role": "assistant", "parts": expected_parts} + ] else: assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["attributes"] assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["attributes"] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 @@ -6484,13 +6410,24 @@ def test_nonstreaming_create_message_with_system_prompt( assert len(stored_messages) == 1 assert stored_messages[0]["role"] == "user" assert stored_messages[0]["content"] == "Hello, Claude" - assert span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ["Hi, I'm Claude."] + expected_parts = ( + [ + { + "type": "reasoning", + "content": "Let me reason step by step...", + }, + {"type": "text", "content": "Hi, I'm Claude."}, + ] + if ThinkingBlock is not None + else [{"type": "text", "content": "Hi, I'm Claude."}] ) + assert json.loads(span["data"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + {"role": "assistant", "parts": expected_parts} + ] else: assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["data"] assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["data"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["data"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["data"] assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 @@ -6586,13 +6523,24 @@ async def test_nonstreaming_create_message_with_system_prompt_async( assert len(stored_messages) == 1 assert stored_messages[0]["role"] == "user" assert stored_messages[0]["content"] == "Hello, Claude" - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ["Hi, I'm Claude."] + expected_parts = ( + [ + { + "type": "reasoning", + "content": "Let me reason step by step...", + }, + {"type": "text", "content": "Hi, I'm Claude."}, + ] + if ThinkingBlock is not None + else [{"type": "text", "content": "Hi, I'm Claude."}] ) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + {"role": "assistant", "parts": expected_parts} + ] else: assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["attributes"] assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["attributes"] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 @@ -6648,13 +6596,24 @@ async def test_nonstreaming_create_message_with_system_prompt_async( assert len(stored_messages) == 1 assert stored_messages[0]["role"] == "user" assert stored_messages[0]["content"] == "Hello, Claude" - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ["Hi, I'm Claude."] + expected_parts = ( + [ + { + "type": "reasoning", + "content": "Let me reason step by step...", + }, + {"type": "text", "content": "Hi, I'm Claude."}, + ] + if ThinkingBlock is not None + else [{"type": "text", "content": "Hi, I'm Claude."}] ) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + {"role": "assistant", "parts": expected_parts} + ] else: assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["attributes"] assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["attributes"] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 @@ -6709,13 +6668,24 @@ async def test_nonstreaming_create_message_with_system_prompt_async( assert len(stored_messages) == 1 assert stored_messages[0]["role"] == "user" assert stored_messages[0]["content"] == "Hello, Claude" - assert span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ["Hi, I'm Claude."] + expected_parts = ( + [ + { + "type": "reasoning", + "content": "Let me reason step by step...", + }, + {"type": "text", "content": "Hi, I'm Claude."}, + ] + if ThinkingBlock is not None + else [{"type": "text", "content": "Hi, I'm Claude."}] ) + assert json.loads(span["data"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + {"role": "assistant", "parts": expected_parts} + ] else: assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["data"] assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["data"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["data"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["data"] assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 20 @@ -6745,44 +6715,17 @@ def test_streaming_create_message_with_system_prompt( server_side_event_chunks, stream_gen_ai_spans, span_streaming, + streaming_anthropic_response, ): """Test that system prompts are properly captured in streaming mode.""" client = Anthropic(api_key="z") response = get_model_response( server_side_event_chunks( - [ - MessageStartEvent( - message=EXAMPLE_MESSAGE, - type="message_start", - ), - ContentBlockStartEvent( - type="content_block_start", - index=0, - content_block=TextBlock(type="text", text=""), - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="Hi", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="!", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockDeltaEvent( - delta=TextDelta(text=" I'm Claude!", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockStopEvent(type="content_block_stop", index=0), - MessageDeltaEvent( - delta=Delta(), - usage=MessageDeltaUsage(output_tokens=10), - type="message_delta", - ), - ] + streaming_anthropic_response( + text_deltas=("Hi", "!", " I'm Claude!"), + stop_reason=None, + ) ) ) @@ -6850,14 +6793,17 @@ def test_streaming_create_message_with_system_prompt( assert len(stored_messages) == 1 assert stored_messages[0]["role"] == "user" assert stored_messages[0]["content"] == "Hello, Claude" - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ["Hi! I'm Claude!"] - ) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + } + ] else: assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["attributes"] assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["attributes"] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 @@ -6913,14 +6859,17 @@ def test_streaming_create_message_with_system_prompt( assert len(stored_messages) == 1 assert stored_messages[0]["role"] == "user" assert stored_messages[0]["content"] == "Hello, Claude" - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ["Hi! I'm Claude!"] - ) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + } + ] else: assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["attributes"] assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["attributes"] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 @@ -6975,14 +6924,17 @@ def test_streaming_create_message_with_system_prompt( assert len(stored_messages) == 1 assert stored_messages[0]["role"] == "user" assert stored_messages[0]["content"] == "Hello, Claude" - assert span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ["Hi! I'm Claude!"] - ) + assert json.loads(span["data"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + } + ] else: assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["data"] assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["data"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["data"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["data"] assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 @@ -7011,44 +6963,17 @@ def test_stream_messages_with_system_prompt( server_side_event_chunks, stream_gen_ai_spans, span_streaming, + streaming_anthropic_response, ): """Test that system prompts are properly captured in streaming mode.""" client = Anthropic(api_key="z") response = get_model_response( server_side_event_chunks( - [ - MessageStartEvent( - message=EXAMPLE_MESSAGE, - type="message_start", - ), - ContentBlockStartEvent( - type="content_block_start", - index=0, - content_block=TextBlock(type="text", text=""), - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="Hi", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="!", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockDeltaEvent( - delta=TextDelta(text=" I'm Claude!", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockStopEvent(type="content_block_stop", index=0), - MessageDeltaEvent( - delta=Delta(), - usage=MessageDeltaUsage(output_tokens=10), - type="message_delta", - ), - ] + streaming_anthropic_response( + text_deltas=("Hi", "!", " I'm Claude!"), + stop_reason=None, + ) ) ) @@ -7112,13 +7037,16 @@ def test_stream_messages_with_system_prompt( assert len(stored_messages) == 1 assert stored_messages[0]["role"] == "user" assert stored_messages[0]["content"] == "Hello, Claude" - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ["Hi! I'm Claude!"] - ) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + } + ] else: assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["attributes"] assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["attributes"] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 @@ -7170,13 +7098,16 @@ def test_stream_messages_with_system_prompt( assert len(stored_messages) == 1 assert stored_messages[0]["role"] == "user" assert stored_messages[0]["content"] == "Hello, Claude" - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ["Hi! I'm Claude!"] - ) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + } + ] else: assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["attributes"] assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["attributes"] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 @@ -7226,13 +7157,16 @@ def test_stream_messages_with_system_prompt( assert len(stored_messages) == 1 assert stored_messages[0]["role"] == "user" assert stored_messages[0]["content"] == "Hello, Claude" - assert span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ["Hi! I'm Claude!"] - ) + assert json.loads(span["data"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + } + ] else: assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["data"] assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["data"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["data"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["data"] assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 @@ -7263,6 +7197,7 @@ async def test_stream_message_with_system_prompt_async( server_side_event_chunks, stream_gen_ai_spans, span_streaming, + streaming_anthropic_response, ): """Test that system prompts are properly captured in streaming mode (async).""" client = AsyncAnthropic(api_key="z") @@ -7270,38 +7205,10 @@ async def test_stream_message_with_system_prompt_async( response = get_model_response( async_iterator( server_side_event_chunks( - [ - MessageStartEvent( - message=EXAMPLE_MESSAGE, - type="message_start", - ), - ContentBlockStartEvent( - type="content_block_start", - index=0, - content_block=TextBlock(type="text", text=""), - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="Hi", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="!", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockDeltaEvent( - delta=TextDelta(text=" I'm Claude!", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockStopEvent(type="content_block_stop", index=0), - MessageDeltaEvent( - delta=Delta(), - usage=MessageDeltaUsage(output_tokens=10), - type="message_delta", - ), - ] + streaming_anthropic_response( + text_deltas=("Hi", "!", " I'm Claude!"), + stop_reason=None, + ) ) ) ) @@ -7368,13 +7275,16 @@ async def test_stream_message_with_system_prompt_async( assert len(stored_messages) == 1 assert stored_messages[0]["role"] == "user" assert stored_messages[0]["content"] == "Hello, Claude" - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ["Hi! I'm Claude!"] - ) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + } + ] else: assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["attributes"] assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["attributes"] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 @@ -7428,13 +7338,16 @@ async def test_stream_message_with_system_prompt_async( assert len(stored_messages) == 1 assert stored_messages[0]["role"] == "user" assert stored_messages[0]["content"] == "Hello, Claude" - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ["Hi! I'm Claude!"] - ) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + } + ] else: assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["attributes"] assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["attributes"] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 @@ -7486,14 +7399,17 @@ async def test_stream_message_with_system_prompt_async( assert len(stored_messages) == 1 assert stored_messages[0]["role"] == "user" assert stored_messages[0]["content"] == "Hello, Claude" - assert span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ["Hi! I'm Claude!"] - ) + assert json.loads(span["data"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + } + ] else: assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["data"] assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["data"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["data"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["data"] assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 @@ -7524,6 +7440,7 @@ async def test_streaming_create_message_with_system_prompt_async( server_side_event_chunks, stream_gen_ai_spans, span_streaming, + streaming_anthropic_response, ): """Test that system prompts are properly captured in streaming mode (async).""" client = AsyncAnthropic(api_key="z") @@ -7531,38 +7448,10 @@ async def test_streaming_create_message_with_system_prompt_async( response = get_model_response( async_iterator( server_side_event_chunks( - [ - MessageStartEvent( - message=EXAMPLE_MESSAGE, - type="message_start", - ), - ContentBlockStartEvent( - type="content_block_start", - index=0, - content_block=TextBlock(type="text", text=""), - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="Hi", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockDeltaEvent( - delta=TextDelta(text="!", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockDeltaEvent( - delta=TextDelta(text=" I'm Claude!", type="text_delta"), - index=0, - type="content_block_delta", - ), - ContentBlockStopEvent(type="content_block_stop", index=0), - MessageDeltaEvent( - delta=Delta(), - usage=MessageDeltaUsage(output_tokens=10), - type="message_delta", - ), - ] + streaming_anthropic_response( + text_deltas=("Hi", "!", " I'm Claude!"), + stop_reason=None, + ) ) ) ) @@ -7632,14 +7521,17 @@ async def test_streaming_create_message_with_system_prompt_async( assert len(stored_messages) == 1 assert stored_messages[0]["role"] == "user" assert stored_messages[0]["content"] == "Hello, Claude" - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ["Hi! I'm Claude!"] - ) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + } + ] else: assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["attributes"] assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["attributes"] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 @@ -7696,14 +7588,17 @@ async def test_streaming_create_message_with_system_prompt_async( assert len(stored_messages) == 1 assert stored_messages[0]["role"] == "user" assert stored_messages[0]["content"] == "Hello, Claude" - assert span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ["Hi! I'm Claude!"] - ) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + } + ] else: assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["attributes"] assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["attributes"] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10 @@ -7756,14 +7651,17 @@ async def test_streaming_create_message_with_system_prompt_async( assert len(stored_messages) == 1 assert stored_messages[0]["role"] == "user" assert stored_messages[0]["content"] == "Hello, Claude" - assert span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] == json.dumps( - ["Hi! I'm Claude!"] - ) + assert json.loads(span["data"][SPANDATA.GEN_AI_OUTPUT_MESSAGES]) == [ + { + "role": "assistant", + "parts": [{"type": "text", "content": "Hi! I'm Claude!"}], + } + ] else: assert SPANDATA.GEN_AI_SYSTEM_INSTRUCTIONS not in span["data"] assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["data"] - assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["data"] + assert SPANDATA.GEN_AI_OUTPUT_MESSAGES not in span["data"] assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 10 assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 10