test: cover LLMJudge.score()

tests/test_judge.py

@@ -0,0 +1,127 @@
+"""Tests for LLMJudge.score().
+
+The judge's scoring logic was previously untested (only the Pydantic types in
+test_scoring.py were covered). These tests exercise score() fully offline by
+patching the OpenAI client, so no API key or network call is needed. They pin
+the two things score() is responsible for: (1) forwarding the request with the
+configured model/schema and a formatted prompt, and (2) combining the LLM's
+structured output with the input's index/configuration into a ScoringOutput.
+"""
+
+from unittest.mock import MagicMock, patch
+
+from finretrieval.scoring.judge import LLMJudge, LLMJudgeOutput
+from finretrieval.scoring.types import ScoringInput, ScoringOutput
+
+
+def _make_input(**overrides) -> ScoringInput:
+    base = {
+        "index": 7,
+        "question": "What was Apple's revenue in Q1 2024?",
+        "answer": "$1.5 billion",
+        "value": "1.5",
+        "unit": "billion",
+        "period": "Q1 2024",
+        "ticker": "AAPL",
+        "company": "Apple Inc.",
+        "configuration": "opus4.5",
+        "response": "Apple reported $1.5 billion in revenue for Q1 2024.",
+    }
+    base.update(overrides)
+    return ScoringInput(**base)
+
+
+def _completion_returning(llm_output: LLMJudgeOutput) -> MagicMock:
+    # Mirror the shape score() reads: completion.choices[0].message.parsed
+    completion = MagicMock()
+    choice = MagicMock()
+    choice.message.parsed = llm_output
+    completion.choices = [choice]
+    return completion
+
+
+@patch("finretrieval.scoring.judge.OpenAI")
+def test_score_combines_llm_output_with_input_fields(_mock_openai):
+    judge = LLMJudge(model="test-model")
+    llm_output = LLMJudgeOutput(
+        is_correct=True,
+        expected_value="1.5",
+        expected_unit="Billion",
+        expected_currency="USD",
+        extracted_value="1.5",
+        extracted_unit="Billion",
+        extracted_currency="USD",
+        could_extract=True,
+        error_reason=None,
+    )
+    judge.client.beta.chat.completions.parse = MagicMock(
+        return_value=_completion_returning(llm_output)
+    )
+
+    result = judge.score(_make_input(index=7, configuration="opus4.5"))
+
+    assert isinstance(result, ScoringOutput)
+    # index and configuration must come from the INPUT, not the LLM output.
+    assert result.index == 7
+    assert result.configuration == "opus4.5"
+    # LLM-provided fields are mapped through unchanged.
+    assert result.is_correct is True
+    assert result.expected_value == "1.5"
+    assert result.expected_unit == "Billion"
+    assert result.expected_currency == "USD"
+    assert result.extracted_value == "1.5"
+    assert result.could_extract is True
+    assert result.error_reason is None
+
+
+@patch("finretrieval.scoring.judge.OpenAI")
+def test_score_calls_client_with_model_schema_and_temperature(_mock_openai):
+    judge = LLMJudge(model="test-model")
+    parse = MagicMock(
+        return_value=_completion_returning(
+            LLMJudgeOutput(
+                is_correct=False,
+                expected_value="150",
+                expected_unit="",
+                could_extract=False,
+                error_reason="no numeric value found",
+            )
+        )
+    )
+    judge.client.beta.chat.completions.parse = parse
+
+    judge.score(_make_input())
+
+    _, kwargs = parse.call_args
+    assert kwargs["model"] == "test-model"
+    assert kwargs["temperature"] == 0
+    assert kwargs["response_format"] is LLMJudgeOutput
+
+
+@patch("finretrieval.scoring.judge.OpenAI")
+def test_score_prompt_contains_input_and_uses_na_when_unit_missing(_mock_openai):
+    judge = LLMJudge()
+    parse = MagicMock(
+        return_value=_completion_returning(
+            LLMJudgeOutput(
+                is_correct=False,
+                expected_value="150",
+                expected_unit="",
+                could_extract=False,
+                error_reason="no numeric value found",
+            )
+        )
+    )
+    judge.client.beta.chat.completions.parse = parse
+
+    result = judge.score(
+        _make_input(unit=None, company="Tesla", ticker="TSLA", response="n/a")
+    )
+
+    _, kwargs = parse.call_args
+    user_prompt = kwargs["messages"][1]["content"]
+    assert "Tesla (TSLA)" in user_prompt
+    # A missing unit renders as the "N/A" fallback, not a literal "None".
+    assert "GROUND TRUTH UNIT: N/A" in user_prompt
+    assert result.could_extract is False
+    assert result.error_reason == "no numeric value found"