1010from collections .abc import Awaitable , Callable , Mapping , Sequence
1111from dataclasses import dataclass
1212from datetime import UTC , datetime
13+ from math import isfinite
1314from typing import Any , Literal , cast
1415
15- from ..content import tool_result_string
16+ from ..canonical_json import canonical_json
1617from ..judge_scoring import (
1718 FORMATTING_INSTRUCTIONS ,
1819 build_message_history ,
20+ judge_text ,
1921 numeric_score ,
2022 parse_judge_response ,
2123 typesafe_judge_entries ,
@@ -95,6 +97,17 @@ def _provides_for(
9597 return handler_key (handler )
9698
9799
100+ def _describe_value (value : Any ) -> str :
101+ """A value in an error message, as JSON rather than a Python repr, so it reads the same in every SDK.
102+
103+ Spelled as ``JSON.stringify`` spells it, except that a non-finite number is
104+ ``NaN`` / ``Infinity``, and a value JSON cannot hold is its ``str()``.
105+ """
106+ if isinstance (value , float ) and not isfinite (value ):
107+ return json .dumps (value )
108+ return canonical_json (value , sort_keys = False )
109+
110+
98111def describe_handlers (handlers : Sequence [EvalHandler ]) -> str :
99112 """Return the ``provides_for`` of each handler, for an error message."""
100113 return ", " .join (repr (_provides_for (handler )) for handler in handlers )
@@ -733,9 +746,7 @@ def _emit_generation_events(
733746 }
734747 if dataset .id is None :
735748 del identity ["datasetId" ]
736- event_id = hashlib .sha256 (
737- json .dumps (identity , sort_keys = True , separators = ("," , ":" )).encode ()
738- ).hexdigest ()
749+ event_id = hashlib .sha256 (canonical_json (identity ).encode ()).hexdigest ()
739750 error = result .get ("error" )
740751 generated = {
741752 "status" : result ["status" ],
@@ -758,9 +769,7 @@ def _emit_generation_events(
758769 "outputTokens" : normalized_usage ["output" ],
759770 }
760771 content_hash = hashlib .sha256 (
761- json .dumps (
762- generated , sort_keys = True , separators = ("," , ":" ), default = str
763- ).encode ()
772+ canonical_json (generated ).encode ()
764773 ).hexdigest ()
765774 emitted_at = datetime .now (UTC ).isoformat ().replace ("+00:00" , "Z" )
766775 payload : dict [str , Any ] = {
@@ -824,7 +833,7 @@ def _judge_variables(
824833 variables .update (
825834 {
826835 "input" : row_result .get ("input" ) or "" ,
827- "response_to_evaluate" : tool_result_string (output ),
836+ "response_to_evaluate" : judge_text (output ),
828837 "message_history" : build_message_history (
829838 user_input = row_result .get ("input" ),
830839 trajectory = trajectory ,
@@ -901,7 +910,7 @@ async def _run_scorer_for_result(
901910 if score is None or score < 0 or score > 1 :
902911 message = (
903912 "scorer fn must return a finite number between 0 and 1, "
904- f"got { score_value !r } "
913+ f"got { _describe_value ( score_value ) } "
905914 )
906915 if scorer .name not in warned_scorers :
907916 warned_scorers .add (scorer .name )
@@ -992,7 +1001,7 @@ async def _run_ld_judge_for_result(
9921001 base ,
9931002 started_clock ,
9941003 "invalid_score" ,
995- f"judge score must be a number between 0 and 1, got { raw_score !r } " ,
1004+ f"judge score must be a number between 0 and 1, got { _describe_value ( raw_score ) } " ,
9961005 )
9971006 completed = datetime .now (UTC )
9981007 # No verdict: the SDK reports the score and LaunchDarkly rules on it. The
@@ -1085,9 +1094,7 @@ def _emit_evaluation_events(
10851094 }
10861095 if dataset .id is None :
10871096 del identity ["datasetId" ]
1088- event_id = hashlib .sha256 (
1089- json .dumps (identity , sort_keys = True , separators = ("," , ":" )).encode ()
1090- ).hexdigest ()
1097+ event_id = hashlib .sha256 (canonical_json (identity ).encode ()).hexdigest ()
10911098 emitted_at = datetime .now (UTC ).isoformat ().replace ("+00:00" , "Z" )
10921099 usage : TokenUsage | None = None
10931100 if result ["kind" ] == "judge" and isinstance (
0 commit comments