diff --git a/tests/test_judge.py b/tests/test_judge.py new file mode 100644 index 0000000..cae1658 --- /dev/null +++ b/tests/test_judge.py @@ -0,0 +1,127 @@ +"""Tests for LLMJudge.score(). + +The judge's scoring logic was previously untested (only the Pydantic types in +test_scoring.py were covered). These tests exercise score() fully offline by +patching the OpenAI client, so no API key or network call is needed. They pin +the two things score() is responsible for: (1) forwarding the request with the +configured model/schema and a formatted prompt, and (2) combining the LLM's +structured output with the input's index/configuration into a ScoringOutput. +""" + +from unittest.mock import MagicMock, patch + +from finretrieval.scoring.judge import LLMJudge, LLMJudgeOutput +from finretrieval.scoring.types import ScoringInput, ScoringOutput + + +def _make_input(**overrides) -> ScoringInput: + base = { + "index": 7, + "question": "What was Apple's revenue in Q1 2024?", + "answer": "$1.5 billion", + "value": "1.5", + "unit": "billion", + "period": "Q1 2024", + "ticker": "AAPL", + "company": "Apple Inc.", + "configuration": "opus4.5", + "response": "Apple reported $1.5 billion in revenue for Q1 2024.", + } + base.update(overrides) + return ScoringInput(**base) + + +def _completion_returning(llm_output: LLMJudgeOutput) -> MagicMock: + # Mirror the shape score() reads: completion.choices[0].message.parsed + completion = MagicMock() + choice = MagicMock() + choice.message.parsed = llm_output + completion.choices = [choice] + return completion + + +@patch("finretrieval.scoring.judge.OpenAI") +def test_score_combines_llm_output_with_input_fields(_mock_openai): + judge = LLMJudge(model="test-model") + llm_output = LLMJudgeOutput( + is_correct=True, + expected_value="1.5", + expected_unit="Billion", + expected_currency="USD", + extracted_value="1.5", + extracted_unit="Billion", + extracted_currency="USD", + could_extract=True, + error_reason=None, + ) + judge.client.beta.chat.completions.parse = MagicMock( + return_value=_completion_returning(llm_output) + ) + + result = judge.score(_make_input(index=7, configuration="opus4.5")) + + assert isinstance(result, ScoringOutput) + # index and configuration must come from the INPUT, not the LLM output. + assert result.index == 7 + assert result.configuration == "opus4.5" + # LLM-provided fields are mapped through unchanged. + assert result.is_correct is True + assert result.expected_value == "1.5" + assert result.expected_unit == "Billion" + assert result.expected_currency == "USD" + assert result.extracted_value == "1.5" + assert result.could_extract is True + assert result.error_reason is None + + +@patch("finretrieval.scoring.judge.OpenAI") +def test_score_calls_client_with_model_schema_and_temperature(_mock_openai): + judge = LLMJudge(model="test-model") + parse = MagicMock( + return_value=_completion_returning( + LLMJudgeOutput( + is_correct=False, + expected_value="150", + expected_unit="", + could_extract=False, + error_reason="no numeric value found", + ) + ) + ) + judge.client.beta.chat.completions.parse = parse + + judge.score(_make_input()) + + _, kwargs = parse.call_args + assert kwargs["model"] == "test-model" + assert kwargs["temperature"] == 0 + assert kwargs["response_format"] is LLMJudgeOutput + + +@patch("finretrieval.scoring.judge.OpenAI") +def test_score_prompt_contains_input_and_uses_na_when_unit_missing(_mock_openai): + judge = LLMJudge() + parse = MagicMock( + return_value=_completion_returning( + LLMJudgeOutput( + is_correct=False, + expected_value="150", + expected_unit="", + could_extract=False, + error_reason="no numeric value found", + ) + ) + ) + judge.client.beta.chat.completions.parse = parse + + result = judge.score( + _make_input(unit=None, company="Tesla", ticker="TSLA", response="n/a") + ) + + _, kwargs = parse.call_args + user_prompt = kwargs["messages"][1]["content"] + assert "Tesla (TSLA)" in user_prompt + # A missing unit renders as the "N/A" fallback, not a literal "None". + assert "GROUND TRUTH UNIT: N/A" in user_prompt + assert result.could_extract is False + assert result.error_reason == "no numeric value found"