Release 0.2.0 with schema-enforced judge responses

CHANGELOG.md

@@ -5,7 +5,15 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
-## [Unreleased]
+## [0.2.0] - 2026-07-11
+
+### Changed
+
+- Enforce schema-backed judge responses through RubyLLM structured output when supported
+- Raise `RubricLLM::JudgeError` for empty, malformed, missing-score, non-numeric, or out-of-range judge responses instead of returning `nil` or clamping invalid scores
+- Require `ruby_llm ~> 1.13` for named schema payload support in structured output
+- Record judge failures per metric in `RubricLLM.evaluate` and `RubricLLM.evaluate_batch` as a `nil` score with the error message in details and continue the run, while non-judge errors propagate
+- Remove dead score clamping and nil-response guards from LLM metrics now that the judge validates the response contract
 
 ## [0.1.2] - 2026-04-30
 

lib/rubric_llm/evaluator.rb

@@ -30,7 +30,7 @@ def call(question:, answer:, context: [], ground_truth: nil)
         result = metric.call(question:, answer:, context:, ground_truth:)
         scores[name] = result[:score]
         details[name] = result[:details]
-      rescue StandardError => e
+      rescue JudgeError => e
         scores[name] = nil
         details[name] = { error: e.message }
       end

lib/rubric_llm/judge.rb

@@ -4,6 +4,24 @@
 
 module RubricLLM
   class Judge
+    METRIC_RESPONSE_SCHEMA = {
+      name: "rubric_llm_metric_response",
+      strict: false,
+      schema: {
+        type: "object",
+        properties: {
+          score: { type: "number", minimum: 0.0, maximum: 1.0 },
+          reasoning: { type: "string" },
+          claims: { type: "array", items: { type: "object" } },
+          context_scores: { type: "array", items: { type: "object" } },
+          covered_facts: { type: "array", items: { type: "object" } },
+          discrepancies: { type: "array", items: { type: "object" } }
+        },
+        required: ["score"],
+        additionalProperties: true
+      }
+    }.freeze
+
     attr_reader :config
 
     def initialize(config:)
@@ -20,13 +38,19 @@ def call(system_prompt:, user_prompt:)
         chat = RubyLLM.chat(model: config.judge_model, provider: config.judge_provider)
         chat.with_temperature(config.temperature)
         chat.with_params(max_tokens: config.max_tokens)
+        apply_response_schema(chat)
 
         full_system_prompt = build_system_prompt(system_prompt)
         chat.with_instructions(full_system_prompt)
         response = chat.ask(user_prompt)
-        parse_json(response.content)
+        content = response.content
+        validate_response!(content.is_a?(Hash) ? content : parse_json(content))
       rescue StandardError => e
-        raise JudgeError, "Judge call failed: #{e.message}" if attempts > config.max_retries
+        if attempts > config.max_retries
+          raise e if e.is_a?(JudgeError)
+
+          raise JudgeError, "Judge call failed: #{e.message}"
+        end
 
         sleep(config.retry_base_delay * (2**(attempts - 1)))
         retry
@@ -36,25 +60,55 @@ def call(system_prompt:, user_prompt:)
     # Parse JSON from LLM output with multiple strategies:
     # 1. Direct JSON.parse
     # 2. Extract from markdown code fence
-    # 3. Return nil (never raises)
+    # 3. Raise JudgeError for malformed output
     def parse_json(text)
-      return nil if text.nil? || text.strip.empty?
+      raise JudgeError, "Judge response was empty" if text.nil? || text.strip.empty?
 
-      # Try direct parse
       JSON.parse(text)
-    rescue JSON::ParserError
-      # Try extracting from code fence
+    rescue JSON::ParserError => e
       if (match = text.match(/```(?:json)?\s*\n?(.*?)\n?\s*```/m))
         begin
-          JSON.parse(match[1])
-        rescue JSON::ParserError
-          nil
+          return JSON.parse(match[1])
+        rescue JSON::ParserError => e
+          raise JudgeError, "Judge response code fence was not valid JSON: #{e.message}"
         end
       end
+
+      raise JudgeError, "Judge response was not valid JSON: #{e.message}"
     end
 
     private
 
+    def apply_response_schema(chat)
+      return chat unless chat.respond_to?(:with_schema)
+      return chat unless structured_output_supported?(chat)
+
+      chat.with_schema(METRIC_RESPONSE_SCHEMA)
+    end
+
+    def structured_output_supported?(chat)
+      return true unless chat.respond_to?(:model)
+      return true unless chat.model.respond_to?(:structured_output?)
+
+      chat.model.structured_output?
+    end
+
+    def validate_response!(response)
+      raise JudgeError, "Judge response must be a JSON object" unless response.is_a?(Hash)
+      raise JudgeError, "Judge response missing required score" unless response.key?("score")
+
+      score = parse_score(response["score"])
+      raise JudgeError, "Judge response score must be between 0.0 and 1.0" unless score.finite? && score.between?(0.0, 1.0)
+
+      response
+    end
+
+    def parse_score(score)
+      Float(score)
+    rescue ArgumentError, TypeError
+      raise JudgeError, "Judge response score must be numeric"
+    end
+
     def build_system_prompt(base_prompt)
       return base_prompt unless config.custom_prompt
 

lib/rubric_llm/metrics/base.rb

@@ -19,10 +19,7 @@ def call(question:, answer:, context: [], ground_truth: nil, **)
       private
 
       def judge_eval(system_prompt:, user_prompt:)
-        result = judge.call(system_prompt:, user_prompt:)
-        return { score: nil, details: { error: "No response from judge" } } if result.nil?
-
-        result
+        judge.call(system_prompt:, user_prompt:)
       end
     end
   end

lib/rubric_llm/metrics/context_precision.rb

@@ -34,10 +34,8 @@ def call(question:, context: [], **)
       private
 
       def normalize(result)
-        return { score: nil, details: result } unless result.is_a?(Hash) && result["score"]
-
         {
-          score: Float(result["score"]).clamp(0.0, 1.0),
+          score: Float(result["score"]),
           details: {
             context_scores: result["context_scores"],
             reasoning: result["reasoning"]

lib/rubric_llm/metrics/context_recall.rb

@@ -35,10 +35,8 @@ def call(context: [], ground_truth: nil, **)
       private
 
       def normalize(result)
-        return { score: nil, details: result } unless result.is_a?(Hash) && result["score"]
-
         {
-          score: Float(result["score"]).clamp(0.0, 1.0),
+          score: Float(result["score"]),
           details: {
             covered_facts: result["covered_facts"],
             reasoning: result["reasoning"]

lib/rubric_llm/metrics/correctness.rb

@@ -34,10 +34,8 @@ def call(question:, answer:, ground_truth: nil, **)
       private
 
       def normalize(result)
-        return { score: nil, details: result } unless result.is_a?(Hash) && result["score"]
-
         {
-          score: Float(result["score"]).clamp(0.0, 1.0),
+          score: Float(result["score"]),
           details: { reasoning: result["reasoning"] }
         }
       end

lib/rubric_llm/metrics/factual_accuracy.rb

@@ -33,10 +33,8 @@ def call(answer:, ground_truth: nil, **)
       private
 
       def normalize(result)
-        return { score: nil, details: result } unless result.is_a?(Hash) && result["score"]
-
         {
-          score: Float(result["score"]).clamp(0.0, 1.0),
+          score: Float(result["score"]),
           details: {
             discrepancies: result["discrepancies"],
             reasoning: result["reasoning"]

lib/rubric_llm/metrics/faithfulness.rb

@@ -36,10 +36,8 @@ def call(question:, answer:, context: [], **)
       private
 
       def normalize(result)
-        return { score: nil, details: result } unless result.is_a?(Hash) && result["score"]
-
         {
-          score: Float(result["score"]).clamp(0.0, 1.0),
+          score: Float(result["score"]),
           details: {
             claims: result["claims"],
             reasoning: result["reasoning"]

lib/rubric_llm/metrics/relevance.rb

@@ -30,10 +30,8 @@ def call(question:, answer:, **)
       private
 
       def normalize(result)
-        return { score: nil, details: result } unless result.is_a?(Hash) && result["score"]
-
         {
-          score: Float(result["score"]).clamp(0.0, 1.0),
+          score: Float(result["score"]),
           details: { reasoning: result["reasoning"] }
         }
       end

lib/rubric_llm/retrieval_result.rb

@@ -51,7 +51,7 @@ def ndcg(k: retrieved.size)
     end
 
     def hit_rate
-      retrieved.any? { |doc| relevant.include?(doc) } ? 1.0 : 0.0
+      relevant.intersect?(retrieved) ? 1.0 : 0.0
     end
 
     def to_h

lib/rubric_llm/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module RubricLLM
-  VERSION = "0.1.2"
+  VERSION = "0.2.0"
 end

rubric_llm.gemspec

@@ -37,5 +37,5 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
   spec.extra_rdoc_files = Dir["README.md", "CHANGELOG.md", "LICENSE.txt"]
 
-  spec.add_dependency "ruby_llm", "~> 1.0"
+  spec.add_dependency "ruby_llm", "~> 1.13"
 end

test/metrics/test_factual_accuracy.rb

@@ -35,11 +35,17 @@ def test_nil_without_ground_truth
     assert_equal "No ground truth provided", result[:details][:error]
   end
 
-  def test_handles_nil_judge_response
+  def test_raises_for_empty_judge_response
     stub_judge_response("")
-    metric = RubricLLM::Metrics::FactualAccuracy.new(judge: RubricLLM::Judge.new(config: RubricLLM.config))
-    result = metric.call(answer: "a", ground_truth: "b")
+    metric = RubricLLM::Metrics::FactualAccuracy.new(judge: RubricLLM::Judge.new(config: no_retry_config))
 
-    assert_nil result[:score]
+    error = assert_raises(RubricLLM::JudgeError) { metric.call(answer: "a", ground_truth: "b") }
+    assert_includes error.message, "empty"
+  end
+
+  private
+
+  def no_retry_config
+    RubricLLM::Config.new(max_retries: 0, retry_base_delay: 0.0)
   end
 end

test/metrics/test_faithfulness.rb

@@ -19,12 +19,12 @@ def test_returns_score_and_details
     assert result[:details][:reasoning]
   end
 
-  def test_handles_nil_judge_response
+  def test_raises_for_empty_judge_response
     stub_judge_response("")
-    metric = RubricLLM::Metrics::Faithfulness.new(judge: RubricLLM::Judge.new(config: RubricLLM.config))
-    result = metric.call(question: "q", answer: "a", context: ["c"])
+    metric = RubricLLM::Metrics::Faithfulness.new(judge: RubricLLM::Judge.new(config: no_retry_config))
 
-    assert_nil result[:score]
+    error = assert_raises(RubricLLM::JudgeError) { metric.call(question: "q", answer: "a", context: ["c"]) }
+    assert_includes error.message, "empty"
   end
 
   def test_nil_without_context
@@ -38,4 +38,10 @@ def test_nil_without_context
     assert_equal "No context provided", result[:details][:error]
     assert_nil chat.last_user_prompt
   end
+
+  private
+
+  def no_retry_config
+    RubricLLM::Config.new(max_retries: 0, retry_base_delay: 0.0)
+  end
 end

test/metrics/test_relevance.rb

@@ -13,19 +13,25 @@ def test_returns_score
     assert_in_delta 0.85, result[:score]
   end
 
-  def test_clamps_score_above_one
+  def test_raises_for_score_above_one
     stub_judge_response('{"score": 1.5, "reasoning": "over"}')
-    metric = RubricLLM::Metrics::Relevance.new(judge: RubricLLM::Judge.new(config: RubricLLM.config))
-    result = metric.call(question: "q", answer: "a")
+    metric = RubricLLM::Metrics::Relevance.new(judge: RubricLLM::Judge.new(config: no_retry_config))
 
-    assert_in_delta 1.0, result[:score]
+    error = assert_raises(RubricLLM::JudgeError) { metric.call(question: "q", answer: "a") }
+    assert_includes error.message, "between 0.0 and 1.0"
   end
 
-  def test_clamps_score_below_zero
+  def test_raises_for_score_below_zero
     stub_judge_response('{"score": -0.3, "reasoning": "under"}')
-    metric = RubricLLM::Metrics::Relevance.new(judge: RubricLLM::Judge.new(config: RubricLLM.config))
-    result = metric.call(question: "q", answer: "a")
+    metric = RubricLLM::Metrics::Relevance.new(judge: RubricLLM::Judge.new(config: no_retry_config))
+
+    error = assert_raises(RubricLLM::JudgeError) { metric.call(question: "q", answer: "a") }
+    assert_includes error.message, "between 0.0 and 1.0"
+  end
+
+  private
 
-    assert_in_delta 0.0, result[:score]
+  def no_retry_config
+    RubricLLM::Config.new(max_retries: 0, retry_base_delay: 0.0)
   end
 end

test/test_evaluator.rb

@@ -5,6 +5,14 @@
 class TestEvaluator < Minitest::Test
   include TestSetup
 
+  class BrokenMetric
+    def initialize(judge:); end
+
+    def call(**)
+      raise "programming error"
+    end
+  end
+
   def test_evaluate_returns_result
     stub_judge_response('{"score": 0.9, "reasoning": "good"}')
     result = RubricLLM.evaluate(
@@ -45,14 +53,24 @@ def test_evaluate_default_metrics_include_factual_accuracy
   end
 
   def test_evaluate_handles_judge_failure
-    stub_judge_response("completely broken response")
+    stub_judge_response("")
     result = RubricLLM.evaluate(
       question: "test",
       answer: "test",
-      metrics: [RubricLLM::Metrics::Relevance]
+      metrics: [RubricLLM::Metrics::Relevance],
+      config: RubricLLM::Config.new(max_retries: 0, retry_base_delay: 0.0)
     )
 
     assert_nil result.scores[:relevance]
+    assert_match(/empty/i, result.details[:relevance][:error])
+  end
+
+  def test_evaluate_propagates_non_judge_errors
+    error = assert_raises(RuntimeError) do
+      RubricLLM::Evaluator.new(config: RubricLLM::Config.new, metrics: [BrokenMetric]).call(question: "test", answer: "test")
+    end
+
+    assert_equal "programming error", error.message
   end
 
   def test_evaluate_with_custom_prompt