diff --git a/CHANGELOG.md b/CHANGELOG.md index e0edecc..9a4c3fa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,15 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [Unreleased] +## [0.2.0] - 2026-07-11 + +### Changed + +- Enforce schema-backed judge responses through RubyLLM structured output when supported +- Raise `RubricLLM::JudgeError` for empty, malformed, missing-score, non-numeric, or out-of-range judge responses instead of returning `nil` or clamping invalid scores +- Require `ruby_llm ~> 1.13` for named schema payload support in structured output +- Record judge failures per metric in `RubricLLM.evaluate` and `RubricLLM.evaluate_batch` as a `nil` score with the error message in details and continue the run, while non-judge errors propagate +- Remove dead score clamping and nil-response guards from LLM metrics now that the judge validates the response contract ## [0.1.2] - 2026-04-30 diff --git a/lib/rubric_llm/evaluator.rb b/lib/rubric_llm/evaluator.rb index 83be19a..0a3747d 100644 --- a/lib/rubric_llm/evaluator.rb +++ b/lib/rubric_llm/evaluator.rb @@ -30,7 +30,7 @@ def call(question:, answer:, context: [], ground_truth: nil) result = metric.call(question:, answer:, context:, ground_truth:) scores[name] = result[:score] details[name] = result[:details] - rescue StandardError => e + rescue JudgeError => e scores[name] = nil details[name] = { error: e.message } end diff --git a/lib/rubric_llm/judge.rb b/lib/rubric_llm/judge.rb index 183d69b..74442bb 100644 --- a/lib/rubric_llm/judge.rb +++ b/lib/rubric_llm/judge.rb @@ -4,6 +4,24 @@ module RubricLLM class Judge + METRIC_RESPONSE_SCHEMA = { + name: "rubric_llm_metric_response", + strict: false, + schema: { + type: "object", + properties: { + score: { type: "number", minimum: 0.0, maximum: 1.0 }, + reasoning: { type: "string" }, + claims: { type: "array", items: { type: "object" } }, + context_scores: { type: "array", items: { type: "object" } }, + covered_facts: { type: "array", items: { type: "object" } }, + discrepancies: { type: "array", items: { type: "object" } } + }, + required: ["score"], + additionalProperties: true + } + }.freeze + attr_reader :config def initialize(config:) @@ -20,13 +38,19 @@ def call(system_prompt:, user_prompt:) chat = RubyLLM.chat(model: config.judge_model, provider: config.judge_provider) chat.with_temperature(config.temperature) chat.with_params(max_tokens: config.max_tokens) + apply_response_schema(chat) full_system_prompt = build_system_prompt(system_prompt) chat.with_instructions(full_system_prompt) response = chat.ask(user_prompt) - parse_json(response.content) + content = response.content + validate_response!(content.is_a?(Hash) ? content : parse_json(content)) rescue StandardError => e - raise JudgeError, "Judge call failed: #{e.message}" if attempts > config.max_retries + if attempts > config.max_retries + raise e if e.is_a?(JudgeError) + + raise JudgeError, "Judge call failed: #{e.message}" + end sleep(config.retry_base_delay * (2**(attempts - 1))) retry @@ -36,25 +60,55 @@ def call(system_prompt:, user_prompt:) # Parse JSON from LLM output with multiple strategies: # 1. Direct JSON.parse # 2. Extract from markdown code fence - # 3. Return nil (never raises) + # 3. Raise JudgeError for malformed output def parse_json(text) - return nil if text.nil? || text.strip.empty? + raise JudgeError, "Judge response was empty" if text.nil? || text.strip.empty? - # Try direct parse JSON.parse(text) - rescue JSON::ParserError - # Try extracting from code fence + rescue JSON::ParserError => e if (match = text.match(/```(?:json)?\s*\n?(.*?)\n?\s*```/m)) begin - JSON.parse(match[1]) - rescue JSON::ParserError - nil + return JSON.parse(match[1]) + rescue JSON::ParserError => e + raise JudgeError, "Judge response code fence was not valid JSON: #{e.message}" end end + + raise JudgeError, "Judge response was not valid JSON: #{e.message}" end private + def apply_response_schema(chat) + return chat unless chat.respond_to?(:with_schema) + return chat unless structured_output_supported?(chat) + + chat.with_schema(METRIC_RESPONSE_SCHEMA) + end + + def structured_output_supported?(chat) + return true unless chat.respond_to?(:model) + return true unless chat.model.respond_to?(:structured_output?) + + chat.model.structured_output? + end + + def validate_response!(response) + raise JudgeError, "Judge response must be a JSON object" unless response.is_a?(Hash) + raise JudgeError, "Judge response missing required score" unless response.key?("score") + + score = parse_score(response["score"]) + raise JudgeError, "Judge response score must be between 0.0 and 1.0" unless score.finite? && score.between?(0.0, 1.0) + + response + end + + def parse_score(score) + Float(score) + rescue ArgumentError, TypeError + raise JudgeError, "Judge response score must be numeric" + end + def build_system_prompt(base_prompt) return base_prompt unless config.custom_prompt diff --git a/lib/rubric_llm/metrics/base.rb b/lib/rubric_llm/metrics/base.rb index fd810c3..ff54033 100644 --- a/lib/rubric_llm/metrics/base.rb +++ b/lib/rubric_llm/metrics/base.rb @@ -19,10 +19,7 @@ def call(question:, answer:, context: [], ground_truth: nil, **) private def judge_eval(system_prompt:, user_prompt:) - result = judge.call(system_prompt:, user_prompt:) - return { score: nil, details: { error: "No response from judge" } } if result.nil? - - result + judge.call(system_prompt:, user_prompt:) end end end diff --git a/lib/rubric_llm/metrics/context_precision.rb b/lib/rubric_llm/metrics/context_precision.rb index 7681f37..5fd85fb 100644 --- a/lib/rubric_llm/metrics/context_precision.rb +++ b/lib/rubric_llm/metrics/context_precision.rb @@ -34,10 +34,8 @@ def call(question:, context: [], **) private def normalize(result) - return { score: nil, details: result } unless result.is_a?(Hash) && result["score"] - { - score: Float(result["score"]).clamp(0.0, 1.0), + score: Float(result["score"]), details: { context_scores: result["context_scores"], reasoning: result["reasoning"] diff --git a/lib/rubric_llm/metrics/context_recall.rb b/lib/rubric_llm/metrics/context_recall.rb index 5198403..73e1eb4 100644 --- a/lib/rubric_llm/metrics/context_recall.rb +++ b/lib/rubric_llm/metrics/context_recall.rb @@ -35,10 +35,8 @@ def call(context: [], ground_truth: nil, **) private def normalize(result) - return { score: nil, details: result } unless result.is_a?(Hash) && result["score"] - { - score: Float(result["score"]).clamp(0.0, 1.0), + score: Float(result["score"]), details: { covered_facts: result["covered_facts"], reasoning: result["reasoning"] diff --git a/lib/rubric_llm/metrics/correctness.rb b/lib/rubric_llm/metrics/correctness.rb index ae72789..3a7a8e4 100644 --- a/lib/rubric_llm/metrics/correctness.rb +++ b/lib/rubric_llm/metrics/correctness.rb @@ -34,10 +34,8 @@ def call(question:, answer:, ground_truth: nil, **) private def normalize(result) - return { score: nil, details: result } unless result.is_a?(Hash) && result["score"] - { - score: Float(result["score"]).clamp(0.0, 1.0), + score: Float(result["score"]), details: { reasoning: result["reasoning"] } } end diff --git a/lib/rubric_llm/metrics/factual_accuracy.rb b/lib/rubric_llm/metrics/factual_accuracy.rb index 832d806..eaf14ac 100644 --- a/lib/rubric_llm/metrics/factual_accuracy.rb +++ b/lib/rubric_llm/metrics/factual_accuracy.rb @@ -33,10 +33,8 @@ def call(answer:, ground_truth: nil, **) private def normalize(result) - return { score: nil, details: result } unless result.is_a?(Hash) && result["score"] - { - score: Float(result["score"]).clamp(0.0, 1.0), + score: Float(result["score"]), details: { discrepancies: result["discrepancies"], reasoning: result["reasoning"] diff --git a/lib/rubric_llm/metrics/faithfulness.rb b/lib/rubric_llm/metrics/faithfulness.rb index 8a047a2..af206a7 100644 --- a/lib/rubric_llm/metrics/faithfulness.rb +++ b/lib/rubric_llm/metrics/faithfulness.rb @@ -36,10 +36,8 @@ def call(question:, answer:, context: [], **) private def normalize(result) - return { score: nil, details: result } unless result.is_a?(Hash) && result["score"] - { - score: Float(result["score"]).clamp(0.0, 1.0), + score: Float(result["score"]), details: { claims: result["claims"], reasoning: result["reasoning"] diff --git a/lib/rubric_llm/metrics/relevance.rb b/lib/rubric_llm/metrics/relevance.rb index 98a6e9a..c539d99 100644 --- a/lib/rubric_llm/metrics/relevance.rb +++ b/lib/rubric_llm/metrics/relevance.rb @@ -30,10 +30,8 @@ def call(question:, answer:, **) private def normalize(result) - return { score: nil, details: result } unless result.is_a?(Hash) && result["score"] - { - score: Float(result["score"]).clamp(0.0, 1.0), + score: Float(result["score"]), details: { reasoning: result["reasoning"] } } end diff --git a/lib/rubric_llm/retrieval_result.rb b/lib/rubric_llm/retrieval_result.rb index 908ea89..749733c 100644 --- a/lib/rubric_llm/retrieval_result.rb +++ b/lib/rubric_llm/retrieval_result.rb @@ -51,7 +51,7 @@ def ndcg(k: retrieved.size) end def hit_rate - retrieved.any? { |doc| relevant.include?(doc) } ? 1.0 : 0.0 + relevant.intersect?(retrieved) ? 1.0 : 0.0 end def to_h diff --git a/lib/rubric_llm/version.rb b/lib/rubric_llm/version.rb index f8ffbb9..e7f7818 100644 --- a/lib/rubric_llm/version.rb +++ b/lib/rubric_llm/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module RubricLLM - VERSION = "0.1.2" + VERSION = "0.2.0" end diff --git a/rubric_llm.gemspec b/rubric_llm.gemspec index d38591b..b4d4b5c 100644 --- a/rubric_llm.gemspec +++ b/rubric_llm.gemspec @@ -37,5 +37,5 @@ Gem::Specification.new do |spec| spec.require_paths = ["lib"] spec.extra_rdoc_files = Dir["README.md", "CHANGELOG.md", "LICENSE.txt"] - spec.add_dependency "ruby_llm", "~> 1.0" + spec.add_dependency "ruby_llm", "~> 1.13" end diff --git a/test/metrics/test_factual_accuracy.rb b/test/metrics/test_factual_accuracy.rb index 4fa247c..fd5073a 100644 --- a/test/metrics/test_factual_accuracy.rb +++ b/test/metrics/test_factual_accuracy.rb @@ -35,11 +35,17 @@ def test_nil_without_ground_truth assert_equal "No ground truth provided", result[:details][:error] end - def test_handles_nil_judge_response + def test_raises_for_empty_judge_response stub_judge_response("") - metric = RubricLLM::Metrics::FactualAccuracy.new(judge: RubricLLM::Judge.new(config: RubricLLM.config)) - result = metric.call(answer: "a", ground_truth: "b") + metric = RubricLLM::Metrics::FactualAccuracy.new(judge: RubricLLM::Judge.new(config: no_retry_config)) - assert_nil result[:score] + error = assert_raises(RubricLLM::JudgeError) { metric.call(answer: "a", ground_truth: "b") } + assert_includes error.message, "empty" + end + + private + + def no_retry_config + RubricLLM::Config.new(max_retries: 0, retry_base_delay: 0.0) end end diff --git a/test/metrics/test_faithfulness.rb b/test/metrics/test_faithfulness.rb index 41d730a..bf97ac1 100644 --- a/test/metrics/test_faithfulness.rb +++ b/test/metrics/test_faithfulness.rb @@ -19,12 +19,12 @@ def test_returns_score_and_details assert result[:details][:reasoning] end - def test_handles_nil_judge_response + def test_raises_for_empty_judge_response stub_judge_response("") - metric = RubricLLM::Metrics::Faithfulness.new(judge: RubricLLM::Judge.new(config: RubricLLM.config)) - result = metric.call(question: "q", answer: "a", context: ["c"]) + metric = RubricLLM::Metrics::Faithfulness.new(judge: RubricLLM::Judge.new(config: no_retry_config)) - assert_nil result[:score] + error = assert_raises(RubricLLM::JudgeError) { metric.call(question: "q", answer: "a", context: ["c"]) } + assert_includes error.message, "empty" end def test_nil_without_context @@ -38,4 +38,10 @@ def test_nil_without_context assert_equal "No context provided", result[:details][:error] assert_nil chat.last_user_prompt end + + private + + def no_retry_config + RubricLLM::Config.new(max_retries: 0, retry_base_delay: 0.0) + end end diff --git a/test/metrics/test_relevance.rb b/test/metrics/test_relevance.rb index eba2192..c3f0782 100644 --- a/test/metrics/test_relevance.rb +++ b/test/metrics/test_relevance.rb @@ -13,19 +13,25 @@ def test_returns_score assert_in_delta 0.85, result[:score] end - def test_clamps_score_above_one + def test_raises_for_score_above_one stub_judge_response('{"score": 1.5, "reasoning": "over"}') - metric = RubricLLM::Metrics::Relevance.new(judge: RubricLLM::Judge.new(config: RubricLLM.config)) - result = metric.call(question: "q", answer: "a") + metric = RubricLLM::Metrics::Relevance.new(judge: RubricLLM::Judge.new(config: no_retry_config)) - assert_in_delta 1.0, result[:score] + error = assert_raises(RubricLLM::JudgeError) { metric.call(question: "q", answer: "a") } + assert_includes error.message, "between 0.0 and 1.0" end - def test_clamps_score_below_zero + def test_raises_for_score_below_zero stub_judge_response('{"score": -0.3, "reasoning": "under"}') - metric = RubricLLM::Metrics::Relevance.new(judge: RubricLLM::Judge.new(config: RubricLLM.config)) - result = metric.call(question: "q", answer: "a") + metric = RubricLLM::Metrics::Relevance.new(judge: RubricLLM::Judge.new(config: no_retry_config)) + + error = assert_raises(RubricLLM::JudgeError) { metric.call(question: "q", answer: "a") } + assert_includes error.message, "between 0.0 and 1.0" + end + + private - assert_in_delta 0.0, result[:score] + def no_retry_config + RubricLLM::Config.new(max_retries: 0, retry_base_delay: 0.0) end end diff --git a/test/test_evaluator.rb b/test/test_evaluator.rb index 9cb78e7..f747a0d 100644 --- a/test/test_evaluator.rb +++ b/test/test_evaluator.rb @@ -5,6 +5,14 @@ class TestEvaluator < Minitest::Test include TestSetup + class BrokenMetric + def initialize(judge:); end + + def call(**) + raise "programming error" + end + end + def test_evaluate_returns_result stub_judge_response('{"score": 0.9, "reasoning": "good"}') result = RubricLLM.evaluate( @@ -45,14 +53,24 @@ def test_evaluate_default_metrics_include_factual_accuracy end def test_evaluate_handles_judge_failure - stub_judge_response("completely broken response") + stub_judge_response("") result = RubricLLM.evaluate( question: "test", answer: "test", - metrics: [RubricLLM::Metrics::Relevance] + metrics: [RubricLLM::Metrics::Relevance], + config: RubricLLM::Config.new(max_retries: 0, retry_base_delay: 0.0) ) assert_nil result.scores[:relevance] + assert_match(/empty/i, result.details[:relevance][:error]) + end + + def test_evaluate_propagates_non_judge_errors + error = assert_raises(RuntimeError) do + RubricLLM::Evaluator.new(config: RubricLLM::Config.new, metrics: [BrokenMetric]).call(question: "test", answer: "test") + end + + assert_equal "programming error", error.message end def test_evaluate_with_custom_prompt diff --git a/test/test_helper.rb b/test/test_helper.rb index 219cc34..aaafd72 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -1,6 +1,7 @@ # frozen_string_literal: true $LOAD_PATH.unshift File.expand_path("../lib", __dir__) +require "json" require "rubric_llm" require "minitest/autorun" @@ -16,7 +17,7 @@ def initialize(content) class FakeChat attr_accessor :response_content - attr_reader :last_system_prompt, :last_user_prompt, :last_params, :last_attachments, :call_count + attr_reader :last_system_prompt, :last_user_prompt, :last_params, :last_attachments, :last_schema, :call_count def initialize(response_content: '{"score": 0.9, "reasoning": "test"}', fail_times: 0, error_class: RuntimeError) @response_content = response_content @@ -53,13 +54,27 @@ def ask(prompt, with: nil, **) raise @error_class, "transient failure" if @call_count <= @fail_times - FakeResponse.new(response_content) + content = @last_schema ? normalize_schema_response(response_content) : response_content + FakeResponse.new(content) end def with_params(**params) @last_params = params self end + + def with_schema(schema) + @last_schema = schema + self + end + + private + + def normalize_schema_response(content) + JSON.parse(content) + rescue JSON::ParserError + content + end end def self.chat(**) diff --git a/test/test_judge.rb b/test/test_judge.rb index 949944b..05e7028 100644 --- a/test/test_judge.rb +++ b/test/test_judge.rb @@ -31,19 +31,22 @@ def test_parse_json_code_fence_no_lang def test_parse_json_nil_input judge = RubricLLM::Judge.new(config: RubricLLM.config) - assert_nil judge.parse_json(nil) + error = assert_raises(RubricLLM::JudgeError) { judge.parse_json(nil) } + assert_includes error.message, "empty" end def test_parse_json_empty_input judge = RubricLLM::Judge.new(config: RubricLLM.config) - assert_nil judge.parse_json("") + error = assert_raises(RubricLLM::JudgeError) { judge.parse_json("") } + assert_includes error.message, "empty" end def test_parse_json_unparseable judge = RubricLLM::Judge.new(config: RubricLLM.config) - assert_nil judge.parse_json("This is not JSON at all") + error = assert_raises(RubricLLM::JudgeError) { judge.parse_json("This is not JSON at all") } + assert_includes error.message, "not valid JSON" end def test_call_returns_parsed_json diff --git a/test/test_judge_contract.rb b/test/test_judge_contract.rb new file mode 100644 index 0000000..587e513 --- /dev/null +++ b/test/test_judge_contract.rb @@ -0,0 +1,101 @@ +# frozen_string_literal: true + +require "test_helper" + +class TestJudgeContract < Minitest::Test + include TestSetup + + def test_call_applies_metric_response_schema + chat = RubyLLMStub::FakeChat.new(response_content: '{"score": 0.95, "reasoning": "excellent"}') + RubyLLMStub.fake_chat = chat + + result = judge.call(system_prompt: "test", user_prompt: "test") + + assert_in_delta(0.95, result["score"]) + assert_equal RubricLLM::Judge::METRIC_RESPONSE_SCHEMA, chat.last_schema + end + + def test_call_accepts_schema_parsed_hash_content + chat = RubyLLMStub::FakeChat.new(response_content: '{"score": 0.95, "reasoning": "excellent"}') + RubyLLMStub.fake_chat = chat + + result = judge.call(system_prompt: "test", user_prompt: "test") + + assert_equal({ "score" => 0.95, "reasoning" => "excellent" }, result) + end + + def test_call_raises_for_schema_parsed_hash_missing_score + chat = RubyLLMStub::FakeChat.new(response_content: '{"reasoning": "missing"}') + RubyLLMStub.fake_chat = chat + + error = assert_raises(RubricLLM::JudgeError) do + judge.call(system_prompt: "test", user_prompt: "test") + end + + assert_includes error.message, "missing required score" + end + + def test_call_raises_for_malformed_json + chat = RubyLLMStub::FakeChat.new(response_content: "This is not JSON") + RubyLLMStub.fake_chat = chat + + error = assert_raises(RubricLLM::JudgeError) do + judge.call(system_prompt: "test", user_prompt: "test") + end + + assert_includes error.message, "not valid JSON" + assert_equal 1, chat.call_count + end + + def test_call_retries_judge_contract_failures + chat = RubyLLMStub::FakeChat.new(response_content: "This is not JSON") + RubyLLMStub.fake_chat = chat + + retrying_judge = RubricLLM::Judge.new(config: RubricLLM::Config.new(max_retries: 1, retry_base_delay: 0.0)) + + assert_raises(RubricLLM::JudgeError) do + retrying_judge.call(system_prompt: "test", user_prompt: "test") + end + + assert_equal 2, chat.call_count + end + + def test_call_raises_for_missing_score + chat = RubyLLMStub::FakeChat.new(response_content: '{"reasoning": "missing"}') + RubyLLMStub.fake_chat = chat + + error = assert_raises(RubricLLM::JudgeError) do + judge.call(system_prompt: "test", user_prompt: "test") + end + + assert_includes error.message, "missing required score" + end + + def test_call_raises_for_non_numeric_score + chat = RubyLLMStub::FakeChat.new(response_content: '{"score": "excellent", "reasoning": "bad score"}') + RubyLLMStub.fake_chat = chat + + error = assert_raises(RubricLLM::JudgeError) do + judge.call(system_prompt: "test", user_prompt: "test") + end + + assert_includes error.message, "must be numeric" + end + + def test_call_raises_for_out_of_range_score + chat = RubyLLMStub::FakeChat.new(response_content: '{"score": 1.1, "reasoning": "too high"}') + RubyLLMStub.fake_chat = chat + + error = assert_raises(RubricLLM::JudgeError) do + judge.call(system_prompt: "test", user_prompt: "test") + end + + assert_includes error.message, "between 0.0 and 1.0" + end + + private + + def judge + RubricLLM::Judge.new(config: RubricLLM::Config.new(max_retries: 0, retry_base_delay: 0.0)) + end +end diff --git a/test/test_rspec_matchers.rb b/test/test_rspec_matchers.rb index e57793e..1b774e3 100644 --- a/test/test_rspec_matchers.rb +++ b/test/test_rspec_matchers.rb @@ -34,12 +34,12 @@ def test_faithfulness_matcher_custom_threshold assert_includes matcher.failure_message, "0.9" end - def test_faithfulness_matcher_nil_score + def test_faithfulness_matcher_raises_for_empty_judge_response stub_judge_response("") - matcher = RubricLLM::RSpecMatchers::FaithfulnessMatcher.new(["context"]) + matcher = RubricLLM::RSpecMatchers::FaithfulnessMatcher.new(["context"]).with_config(no_retry_config) - refute matcher.matches?("answer") - assert_includes matcher.failure_message, "nil" + error = assert_raises(RubricLLM::JudgeError) { matcher.matches?("answer") } + assert_includes error.message, "empty" end def test_faithfulness_negated_message @@ -117,11 +117,12 @@ def test_hallucination_matcher_no_hallucination assert_includes matcher.failure_message, "hallucination" end - def test_hallucination_matcher_nil_score_counts_as_hallucination + def test_hallucination_matcher_raises_for_empty_judge_response stub_judge_response("") - matcher = RubricLLM::RSpecMatchers::HallucinationMatcher.new(["context"]) + matcher = RubricLLM::RSpecMatchers::HallucinationMatcher.new(["context"]).with_config(no_retry_config) - assert matcher.matches?("answer") + error = assert_raises(RubricLLM::JudgeError) { matcher.matches?("answer") } + assert_includes error.message, "empty" end def test_hallucination_negated_message @@ -154,4 +155,10 @@ def test_dsl_helpers_exist assert_respond_to obj, :be_relevant_to assert_respond_to obj, :hallucinate_from end + + private + + def no_retry_config + RubricLLM::Config.new(max_retries: 0, retry_base_delay: 0.0) + end end