Skip to content

Commit 52f69be

Browse files
jeffdupontclaude
andcommitted
Merge origin/main into fix/scorer-rejects-boolean-scores
Keep this branch's rule that a bool is an invalid score, against main's coercion. The invalid_score message and warning now name the value with _describe_value from #209 (true, null, "high", NaN), and the test asserts that rendering. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2 parents bc596c0 + 7cee8c8 commit 52f69be

9 files changed

Lines changed: 417 additions & 43 deletions

File tree

Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
"""RFC 8785 JSON Canonicalization Scheme (JCS), for hashes every SDK must agree on.
2+
3+
JCS spells JSON the way ECMAScript's ``JSON.stringify`` does: no whitespace,
4+
object keys sorted by UTF-16 code unit, non-ASCII written as is, and numbers in
5+
ECMAScript's shortest round-trip form. That is the native form in JavaScript
6+
and close to it in Go, so the only language that needs this module is Python,
7+
whose ``json.dumps`` writes ``1.0`` where JCS writes ``1``.
8+
"""
9+
10+
from __future__ import annotations
11+
12+
import json
13+
import re
14+
from decimal import Decimal
15+
from math import isfinite
16+
from typing import Any
17+
18+
# Above this an int has no exact IEEE double, so it is written as the double
19+
# nearest to it, as a JavaScript JSON.parse reads it.
20+
_MAX_SAFE_INTEGER = 2**53 - 1
21+
_SURROGATE = re.compile("[\ud800-\udfff]")
22+
23+
24+
def canonical_json(value: Any, *, sort_keys: bool = True) -> str:
25+
"""The JCS text of a JSON-like value.
26+
27+
With ``sort_keys=False`` object keys keep their insertion order, which is
28+
``JSON.stringify``'s output exactly: the form judge text uses, so a judge
29+
reads the same text in every SDK. It is not JCS, so never hash it.
30+
31+
The value is first normalized through ``json.dumps(..., default=str)``, so
32+
a tuple is a list, a non-string key is the string ``json.dumps`` writes for
33+
it, and a value JSON cannot hold is its ``str()``. A non-finite number is
34+
written ``null``, as ``JSON.stringify`` writes it.
35+
"""
36+
return _encode(json.loads(json.dumps(value, default=str)), sort_keys)
37+
38+
39+
def _encode(value: Any, sort_keys: bool = True) -> str:
40+
if value is None or value is True or value is False:
41+
return json.dumps(value)
42+
if isinstance(value, int):
43+
if abs(value) <= _MAX_SAFE_INTEGER:
44+
return str(value)
45+
try:
46+
return _number(float(value))
47+
except OverflowError:
48+
return "null"
49+
if isinstance(value, float):
50+
return _number(value)
51+
if isinstance(value, str):
52+
return _string(value)
53+
if isinstance(value, list):
54+
return "[" + ",".join(_encode(item, sort_keys) for item in value) + "]"
55+
keys = sorted(value, key=_utf16) if sort_keys else list(value)
56+
return (
57+
"{"
58+
+ ",".join(f"{_string(key)}:{_encode(value[key], sort_keys)}" for key in keys)
59+
+ "}"
60+
)
61+
62+
63+
def _number(value: float) -> str:
64+
"""ECMAScript's Number::toString, from the shortest round-trip digits ``repr`` gives."""
65+
if not isfinite(value):
66+
return "null"
67+
if value == 0:
68+
return "0"
69+
_, digit_tuple, exponent = Decimal(repr(abs(value))).as_tuple()
70+
# The value is 0.<digits> x 10^n, the form Number::toString starts from.
71+
n = len(digit_tuple) + int(exponent)
72+
digits = "".join(map(str, digit_tuple)).rstrip("0")
73+
k = len(digits)
74+
if k <= n <= 21:
75+
text = digits + "0" * (n - k)
76+
elif 0 < n <= 21:
77+
text = f"{digits[:n]}.{digits[n:]}"
78+
elif -6 < n <= 0:
79+
text = f"0.{'0' * -n}{digits}"
80+
else:
81+
fraction = f".{digits[1:]}" if k > 1 else ""
82+
text = f"{digits[0]}{fraction}e{'+' if n - 1 >= 0 else '-'}{abs(n - 1)}"
83+
return ("-" if value < 0 else "") + text
84+
85+
86+
def _string(text: str) -> str:
87+
# A surrogate pair held as two code points is joined into one character, as
88+
# it is in a JavaScript string; a lone surrogate is escaped, as
89+
# JSON.stringify escapes it, so the text always has a UTF-8 encoding.
90+
joined = text.encode("utf-16", "surrogatepass").decode("utf-16", "surrogatepass")
91+
return _SURROGATE.sub(
92+
lambda match: f"\\u{ord(match.group()):04x}",
93+
json.dumps(joined, ensure_ascii=False),
94+
)
95+
96+
97+
def _utf16(key: str) -> bytes:
98+
return key.encode("utf-16-be", "surrogatepass")

‎packages/client/src/launchdarkly_ai_server/client.py‎

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,12 @@
11
from __future__ import annotations
22

3-
import json
43
from collections.abc import AsyncGenerator, Callable
54
from contextlib import aclosing
65
from typing import Any
76

87
from .agui import StableProjector, judge_completed_stream
98
from .conversation import bind_conversation_id
9+
from .judge_scoring import judge_text
1010
from .judges import _build_judge_tasks, run_judges
1111
from .lifecycle import extract_variation
1212
from .registry import resolve_handlers, resolve_tools
@@ -120,11 +120,8 @@ async def _invoke(
120120
config.get("outputFormat") if isinstance(config, dict) else None,
121121
)
122122

123-
llm_str = (
124-
parsed_response
125-
if isinstance(parsed_response, str)
126-
else json.dumps(parsed_response)
127-
)
123+
# The same text an offline judge reads, so a rubric sees one spelling.
124+
llm_str = judge_text(parsed_response)
128125

129126
usage_obj = to_usage_dict(usage)
130127

‎packages/client/src/launchdarkly_ai_server/evaluations/runner.py‎

Lines changed: 20 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -10,12 +10,14 @@
1010
from collections.abc import Awaitable, Callable, Mapping, Sequence
1111
from dataclasses import dataclass
1212
from datetime import UTC, datetime
13+
from math import isfinite
1314
from typing import Any, Literal, cast
1415

15-
from ..content import tool_result_string
16+
from ..canonical_json import canonical_json
1617
from ..judge_scoring import (
1718
FORMATTING_INSTRUCTIONS,
1819
build_message_history,
20+
judge_text,
1921
numeric_score,
2022
parse_judge_response,
2123
typesafe_judge_entries,
@@ -95,6 +97,17 @@ def _provides_for(
9597
return handler_key(handler)
9698

9799

100+
def _describe_value(value: Any) -> str:
101+
"""A value in an error message, as JSON rather than a Python repr, so it reads the same in every SDK.
102+
103+
Spelled as ``JSON.stringify`` spells it, except that a non-finite number is
104+
``NaN`` / ``Infinity``, and a value JSON cannot hold is its ``str()``.
105+
"""
106+
if isinstance(value, float) and not isfinite(value):
107+
return json.dumps(value)
108+
return canonical_json(value, sort_keys=False)
109+
110+
98111
def describe_handlers(handlers: Sequence[EvalHandler]) -> str:
99112
"""Return the ``provides_for`` of each handler, for an error message."""
100113
return ", ".join(repr(_provides_for(handler)) for handler in handlers)
@@ -733,9 +746,7 @@ def _emit_generation_events(
733746
}
734747
if dataset.id is None:
735748
del identity["datasetId"]
736-
event_id = hashlib.sha256(
737-
json.dumps(identity, sort_keys=True, separators=(",", ":")).encode()
738-
).hexdigest()
749+
event_id = hashlib.sha256(canonical_json(identity).encode()).hexdigest()
739750
error = result.get("error")
740751
generated = {
741752
"status": result["status"],
@@ -758,9 +769,7 @@ def _emit_generation_events(
758769
"outputTokens": normalized_usage["output"],
759770
}
760771
content_hash = hashlib.sha256(
761-
json.dumps(
762-
generated, sort_keys=True, separators=(",", ":"), default=str
763-
).encode()
772+
canonical_json(generated).encode()
764773
).hexdigest()
765774
emitted_at = datetime.now(UTC).isoformat().replace("+00:00", "Z")
766775
payload: dict[str, Any] = {
@@ -824,7 +833,7 @@ def _judge_variables(
824833
variables.update(
825834
{
826835
"input": row_result.get("input") or "",
827-
"response_to_evaluate": tool_result_string(output),
836+
"response_to_evaluate": judge_text(output),
828837
"message_history": build_message_history(
829838
user_input=row_result.get("input"),
830839
trajectory=trajectory,
@@ -901,7 +910,7 @@ async def _run_scorer_for_result(
901910
if score is None or score < 0 or score > 1:
902911
message = (
903912
"scorer fn must return a finite number between 0 and 1, "
904-
f"got {score_value!r}"
913+
f"got {_describe_value(score_value)}"
905914
)
906915
if scorer.name not in warned_scorers:
907916
warned_scorers.add(scorer.name)
@@ -992,7 +1001,7 @@ async def _run_ld_judge_for_result(
9921001
base,
9931002
started_clock,
9941003
"invalid_score",
995-
f"judge score must be a number between 0 and 1, got {raw_score!r}",
1004+
f"judge score must be a number between 0 and 1, got {_describe_value(raw_score)}",
9961005
)
9971006
completed = datetime.now(UTC)
9981007
# No verdict: the SDK reports the score and LaunchDarkly rules on it. The
@@ -1085,9 +1094,7 @@ def _emit_evaluation_events(
10851094
}
10861095
if dataset.id is None:
10871096
del identity["datasetId"]
1088-
event_id = hashlib.sha256(
1089-
json.dumps(identity, sort_keys=True, separators=(",", ":")).encode()
1090-
).hexdigest()
1097+
event_id = hashlib.sha256(canonical_json(identity).encode()).hexdigest()
10911098
emitted_at = datetime.now(UTC).isoformat().replace("+00:00", "Z")
10921099
usage: TokenUsage | None = None
10931100
if result["kind"] == "judge" and isinstance(

‎packages/client/src/launchdarkly_ai_server/judge_scoring.py‎

Lines changed: 49 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@
1616
from math import isfinite
1717
from typing import Any
1818

19-
from .content import tool_result_string
19+
from .canonical_json import canonical_json
2020
from .utils import parse_json_with_possible_fences
2121

2222
logger = logging.getLogger(__name__)
@@ -42,25 +42,52 @@ def build_message_history(
4242
"""The conversation a judge is shown, as its ``message_history`` variable.
4343
4444
Ordered as it happened: what was asked, what the agent did, what it
45-
answered, then how to format the verdict. Empty parts are skipped, so a
46-
run with no tools yields the history it did before trajectories existed.
45+
answered, then how to format the verdict. An empty part (see
46+
:func:`_is_empty_part`) is skipped, so a run with no tools yields the
47+
history it did before trajectories existed.
4748
4849
The formatting block is appended here, not by callers: judges built from
4950
the AI Library's default templates read the JSON shape from
5051
``{{message_history}}``, and one that stopped being told it would return
5152
prose and fail every result as invalid output.
5253
53-
A non-string output is shown as JSON, by the rule tool results follow, so a
54-
rubric reads the same text in every SDK.
54+
Every other part is rendered by the rule tool results follow, so a
55+
non-string output is shown as JSON and a rubric reads the same text in
56+
every SDK. An output of ``0`` or ``False`` is an answer, and is shown.
5557
"""
56-
output_text = tool_result_string(output) if output else None
5758
return "\n\n".join(
58-
str(part)
59-
for part in (user_input, trajectory, output_text, FORMATTING_INSTRUCTIONS)
60-
if part
59+
judge_text(part)
60+
for part in (user_input, trajectory, output, FORMATTING_INSTRUCTIONS)
61+
if not _is_empty_part(part)
6162
)
6263

6364

65+
def judge_text(value: Any) -> str:
66+
"""A value as a judge is shown it: a string as it is, ``None`` as ``""``, anything else as JSON.
67+
68+
The JSON is ``JSON.stringify``'s spelling (no whitespace, keys in insertion
69+
order, ``1`` for ``1.0``), not ``json.dumps``'s, so a rubric reads the same
70+
text in every SDK.
71+
"""
72+
if isinstance(value, str):
73+
return value
74+
if value is None:
75+
return ""
76+
return canonical_json(value, sort_keys=False)
77+
78+
79+
def _is_empty_part(value: Any) -> bool:
80+
"""``None``, ``""``, an empty list or an empty mapping, tested before rendering.
81+
82+
An empty container is skipped, as it was before structured outputs were
83+
rendered as JSON. Python truthiness is not the test: it would also drop
84+
``0`` and ``False``, which are answers.
85+
"""
86+
if value is None or value == "":
87+
return True
88+
return isinstance(value, (list, tuple, Mapping)) and not value
89+
90+
6491
def numeric_score(score: Any) -> float | None:
6592
"""Return ``score`` as a float only when it already is a finite number.
6693
@@ -129,6 +156,10 @@ def parse_judge_response(raw: Any) -> tuple[Any, str]:
129156
already-decoded mapping. The score is returned untouched — callers apply
130157
their own policy to non-numeric values via :func:`numeric_score`.
131158
159+
The reasoning is the first of ``reasoning`` and ``reason`` that is not an
160+
empty part (:func:`_is_empty_part`), rendered by the tool-result rule: a
161+
string as it is, anything else as JSON, never as a Python repr.
162+
132163
Raises ``ValueError`` when the response is not a non-empty JSON object.
133164
"""
134165
parsed: Any
@@ -140,8 +171,15 @@ def parse_judge_response(raw: Any) -> tuple[Any, str]:
140171
parsed = None
141172
if not isinstance(parsed, Mapping) or not parsed:
142173
raise ValueError("Invalid JSON from judge")
143-
reasoning = parsed.get("reasoning") or parsed.get("reason") or ""
144-
return parsed.get("score"), str(reasoning)
174+
reasoning = next(
175+
(
176+
judge_text(value)
177+
for value in (parsed.get("reasoning"), parsed.get("reason"))
178+
if not _is_empty_part(value)
179+
),
180+
"",
181+
)
182+
return parsed.get("score"), reasoning
145183

146184

147185
def without_output_format(config: Any, judge_key: str) -> Any:

0 commit comments

Comments
 (0)