Skip to content

Commit ae45422

Browse files
fix(evaluation): record NOT_EVALUATED instead of dropping invocations with zero auto-rater samples
LlmAsJudge.evaluate_invocations skipped straight to `continue` whenever an invocation's auto-rater call produced zero samples (e.g. the judge model's stream ended without emitting a response). The invocation was silently omitted from per_invocation_results entirely, shrinking the denominator downstream with no trace that anything went wrong. Same defect family as #6682 (NOT_EVALUATED metrics masked by a passing one) but one level up: here an invocation vanishes before it ever gets an eval_status. Append a PerInvocationResult defaulting to NOT_EVALUATED instead, consistent with how a genuinely-graded but missing metric is already represented elsewhere in this module.
1 parent 2e878ed commit ae45422

2 files changed

Lines changed: 71 additions & 0 deletions

File tree

src/google/adk/evaluation/llm_as_judge.py

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -191,6 +191,16 @@ async def evaluate_invocations(
191191
)
192192
)
193193
if not invocation_result_samples:
194+
# The auto-rater produced no samples for this invocation (e.g. the
195+
# judge model's stream ended without emitting a response). Record it
196+
# as not evaluated instead of silently dropping it from the results,
197+
# which would shrink the denominator downstream with no trace.
198+
per_invocation_results.append(
199+
PerInvocationResult(
200+
actual_invocation=actual,
201+
expected_invocation=expected,
202+
)
203+
)
194204
continue
195205
per_invocation_results.append(
196206
self.aggregate_per_invocation_samples(invocation_result_samples)

tests/unittests/evaluation/test_llm_as_judge.py

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -289,3 +289,64 @@ async def test_evaluate_invocations_grades_criterion_only_metric(
289289
assert [r.eval_status for r in result.per_invocation_results] == [
290290
EvalStatus.PASSED
291291
]
292+
293+
294+
@pytest.mark.asyncio
295+
async def test_evaluate_invocations_records_not_evaluated_when_no_samples_produced(
296+
mocker,
297+
):
298+
"""An invocation whose auto-rater call produces zero samples (e.g. the judge
299+
300+
model's stream ends without emitting a response) must still show up in
301+
per_invocation_results as NOT_EVALUATED, not vanish from the results
302+
entirely. Silently dropping it shrinks the denominator downstream with no
303+
trace that anything went wrong.
304+
"""
305+
judge = PerInvocationReportingLlmAsJudge(
306+
eval_metric=EvalMetric(
307+
metric_name="test_metric",
308+
criterion=LlmAsAJudgeCriterion(
309+
threshold=0.5,
310+
judge_model_options=JudgeModelOptions(
311+
judge_model="gemini-2.5-flash",
312+
judge_model_config=genai_types.GenerateContentConfig(),
313+
num_samples=1,
314+
),
315+
),
316+
),
317+
criterion_type=LlmAsAJudgeCriterion,
318+
)
319+
320+
empty_judge_model = mocker.MagicMock()
321+
322+
async def mock_generate_content_async_no_response(llm_request):
323+
del llm_request
324+
return
325+
yield # pragma: no cover -- makes this an async generator.
326+
327+
empty_judge_model.generate_content_async = (
328+
mock_generate_content_async_no_response
329+
)
330+
judge._judge_model = empty_judge_model
331+
332+
actual_invocations = [
333+
Invocation(
334+
invocation_id="id1",
335+
user_content=genai_types.Content(
336+
parts=[genai_types.Part(text="user content 1")],
337+
role="user",
338+
),
339+
final_response=genai_types.Content(
340+
parts=[genai_types.Part(text="final response 1")],
341+
role="model",
342+
),
343+
)
344+
]
345+
346+
result = await judge.evaluate_invocations(actual_invocations)
347+
348+
assert len(result.per_invocation_results) == 1
349+
assert (
350+
result.per_invocation_results[0].eval_status == EvalStatus.NOT_EVALUATED
351+
)
352+
assert result.per_invocation_results[0].score is None

0 commit comments

Comments
 (0)