From e404b72a97708a9b0e17811e70c88871b971c514 Mon Sep 17 00:00:00 2001
From: David Paluy <dpaluy@users.noreply.github.com>
Date: Thu, 30 Apr 2026 00:56:39 -0400
Subject: [PATCH 1/6] release: surface judge contract failures in 0.2.0

Judge outputs now use RubyLLM structured response schemas when available, with local validation rejecting malformed payloads and invalid score contracts. The version and changelog move this into 0.2.0 because callers can now receive JudgeError where bad model output previously became nil or was clamped.

Constraint: GitHub issue #3 requires schema-backed judge responses and explicit failures for malformed, missing-score, and out-of-range responses
Rejected: Keep clamping out-of-range scores | hides judge contract violations that should fail loudly
Confidence: high
Scope-risk: moderate
Directive: Do not reintroduce nil-on-malformed judge output without updating the issue #3 contract tests
Tested: bundle exec rake; gem build rubric_llm.gemspec; git diff --check
Not-tested: Live provider structured-output API call
Related: GH-3
---
 CHANGELOG.md                          |  7 +++
 lib/rubric_llm/judge.rb               | 73 ++++++++++++++++++++----
 lib/rubric_llm/version.rb             |  2 +-
 test/metrics/test_factual_accuracy.rb | 14 +++--
 test/metrics/test_faithfulness.rb     | 14 +++--
 test/metrics/test_relevance.rb        | 22 +++++---
 test/test_evaluator.rb                |  3 +-
 test/test_helper.rb                   |  7 ++-
 test/test_judge.rb                    |  9 ++-
 test/test_judge_contract.rb           | 81 +++++++++++++++++++++++++++
 test/test_rspec_matchers.rb           | 21 ++++---
 11 files changed, 214 insertions(+), 39 deletions(-)
 create mode 100644 test/test_judge_contract.rb

diff --git a/CHANGELOG.md b/CHANGELOG.md
index e0edecc..3d31199 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -7,6 +7,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [0.2.0] - 2026-04-30
+
+### Changed
+
+- Enforce schema-backed judge responses through RubyLLM structured output when supported
+- Raise `RubricLLM::JudgeError` for empty, malformed, missing-score, non-numeric, or out-of-range judge responses instead of returning `nil` or clamping invalid scores
+
 ## [0.1.2] - 2026-04-30
 
 ### Added
diff --git a/lib/rubric_llm/judge.rb b/lib/rubric_llm/judge.rb
index 183d69b..9444ab4 100644
--- a/lib/rubric_llm/judge.rb
+++ b/lib/rubric_llm/judge.rb
@@ -4,6 +4,24 @@
 
 module RubricLLM
   class Judge
+    METRIC_RESPONSE_SCHEMA = {
+      name: "rubric_llm_metric_response",
+      strict: false,
+      schema: {
+        type: "object",
+        properties: {
+          score: { type: "number", minimum: 0.0, maximum: 1.0 },
+          reasoning: { type: "string" },
+          claims: { type: "array", items: { type: "object" } },
+          context_scores: { type: "array", items: { type: "object" } },
+          covered_facts: { type: "array", items: { type: "object" } },
+          discrepancies: { type: "array", items: { type: "object" } }
+        },
+        required: ["score"],
+        additionalProperties: true
+      }
+    }.freeze
+
     attr_reader :config
 
     def initialize(config:)
@@ -20,13 +38,18 @@ def call(system_prompt:, user_prompt:)
         chat = RubyLLM.chat(model: config.judge_model, provider: config.judge_provider)
         chat.with_temperature(config.temperature)
         chat.with_params(max_tokens: config.max_tokens)
+        apply_response_schema(chat)
 
         full_system_prompt = build_system_prompt(system_prompt)
         chat.with_instructions(full_system_prompt)
         response = chat.ask(user_prompt)
-        parse_json(response.content)
+        validate_response!(parse_json(response.content))
       rescue StandardError => e
-        raise JudgeError, "Judge call failed: #{e.message}" if attempts > config.max_retries
+        if attempts > config.max_retries
+          raise e if e.is_a?(JudgeError)
+
+          raise JudgeError, "Judge call failed: #{e.message}"
+        end
 
         sleep(config.retry_base_delay * (2**(attempts - 1)))
         retry
@@ -36,25 +59,55 @@ def call(system_prompt:, user_prompt:)
     # Parse JSON from LLM output with multiple strategies:
     # 1. Direct JSON.parse
     # 2. Extract from markdown code fence
-    # 3. Return nil (never raises)
+    # 3. Raise JudgeError for malformed output
     def parse_json(text)
-      return nil if text.nil? || text.strip.empty?
+      raise JudgeError, "Judge response was empty" if text.nil? || text.strip.empty?
 
-      # Try direct parse
       JSON.parse(text)
-    rescue JSON::ParserError
-      # Try extracting from code fence
+    rescue JSON::ParserError => e
       if (match = text.match(/```(?:json)?\s*\n?(.*?)\n?\s*```/m))
         begin
-          JSON.parse(match[1])
-        rescue JSON::ParserError
-          nil
+          return JSON.parse(match[1])
+        rescue JSON::ParserError => e
+          raise JudgeError, "Judge response code fence was not valid JSON: #{e.message}"
         end
       end
+
+      raise JudgeError, "Judge response was not valid JSON: #{e.message}"
     end
 
     private
 
+    def apply_response_schema(chat)
+      return chat unless chat.respond_to?(:with_schema)
+      return chat unless structured_output_supported?(chat)
+
+      chat.with_schema(METRIC_RESPONSE_SCHEMA)
+    end
+
+    def structured_output_supported?(chat)
+      return true unless chat.respond_to?(:model)
+      return true unless chat.model.respond_to?(:structured_output?)
+
+      chat.model.structured_output?
+    end
+
+    def validate_response!(response)
+      raise JudgeError, "Judge response must be a JSON object" unless response.is_a?(Hash)
+      raise JudgeError, "Judge response missing required score" unless response.key?("score")
+
+      score = parse_score(response["score"])
+      raise JudgeError, "Judge response score must be between 0.0 and 1.0" unless score.finite? && score.between?(0.0, 1.0)
+
+      response
+    end
+
+    def parse_score(score)
+      Float(score)
+    rescue ArgumentError, TypeError
+      raise JudgeError, "Judge response score must be numeric"
+    end
+
     def build_system_prompt(base_prompt)
       return base_prompt unless config.custom_prompt
 
diff --git a/lib/rubric_llm/version.rb b/lib/rubric_llm/version.rb
index f8ffbb9..e7f7818 100644
--- a/lib/rubric_llm/version.rb
+++ b/lib/rubric_llm/version.rb
@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module RubricLLM
-  VERSION = "0.1.2"
+  VERSION = "0.2.0"
 end
diff --git a/test/metrics/test_factual_accuracy.rb b/test/metrics/test_factual_accuracy.rb
index 4fa247c..fd5073a 100644
--- a/test/metrics/test_factual_accuracy.rb
+++ b/test/metrics/test_factual_accuracy.rb
@@ -35,11 +35,17 @@ def test_nil_without_ground_truth
     assert_equal "No ground truth provided", result[:details][:error]
   end
 
-  def test_handles_nil_judge_response
+  def test_raises_for_empty_judge_response
     stub_judge_response("")
-    metric = RubricLLM::Metrics::FactualAccuracy.new(judge: RubricLLM::Judge.new(config: RubricLLM.config))
-    result = metric.call(answer: "a", ground_truth: "b")
+    metric = RubricLLM::Metrics::FactualAccuracy.new(judge: RubricLLM::Judge.new(config: no_retry_config))
 
-    assert_nil result[:score]
+    error = assert_raises(RubricLLM::JudgeError) { metric.call(answer: "a", ground_truth: "b") }
+    assert_includes error.message, "empty"
+  end
+
+  private
+
+  def no_retry_config
+    RubricLLM::Config.new(max_retries: 0, retry_base_delay: 0.0)
   end
 end
diff --git a/test/metrics/test_faithfulness.rb b/test/metrics/test_faithfulness.rb
index 41d730a..bf97ac1 100644
--- a/test/metrics/test_faithfulness.rb
+++ b/test/metrics/test_faithfulness.rb
@@ -19,12 +19,12 @@ def test_returns_score_and_details
     assert result[:details][:reasoning]
   end
 
-  def test_handles_nil_judge_response
+  def test_raises_for_empty_judge_response
     stub_judge_response("")
-    metric = RubricLLM::Metrics::Faithfulness.new(judge: RubricLLM::Judge.new(config: RubricLLM.config))
-    result = metric.call(question: "q", answer: "a", context: ["c"])
+    metric = RubricLLM::Metrics::Faithfulness.new(judge: RubricLLM::Judge.new(config: no_retry_config))
 
-    assert_nil result[:score]
+    error = assert_raises(RubricLLM::JudgeError) { metric.call(question: "q", answer: "a", context: ["c"]) }
+    assert_includes error.message, "empty"
   end
 
   def test_nil_without_context
@@ -38,4 +38,10 @@ def test_nil_without_context
     assert_equal "No context provided", result[:details][:error]
     assert_nil chat.last_user_prompt
   end
+
+  private
+
+  def no_retry_config
+    RubricLLM::Config.new(max_retries: 0, retry_base_delay: 0.0)
+  end
 end
diff --git a/test/metrics/test_relevance.rb b/test/metrics/test_relevance.rb
index eba2192..c3f0782 100644
--- a/test/metrics/test_relevance.rb
+++ b/test/metrics/test_relevance.rb
@@ -13,19 +13,25 @@ def test_returns_score
     assert_in_delta 0.85, result[:score]
   end
 
-  def test_clamps_score_above_one
+  def test_raises_for_score_above_one
     stub_judge_response('{"score": 1.5, "reasoning": "over"}')
-    metric = RubricLLM::Metrics::Relevance.new(judge: RubricLLM::Judge.new(config: RubricLLM.config))
-    result = metric.call(question: "q", answer: "a")
+    metric = RubricLLM::Metrics::Relevance.new(judge: RubricLLM::Judge.new(config: no_retry_config))
 
-    assert_in_delta 1.0, result[:score]
+    error = assert_raises(RubricLLM::JudgeError) { metric.call(question: "q", answer: "a") }
+    assert_includes error.message, "between 0.0 and 1.0"
   end
 
-  def test_clamps_score_below_zero
+  def test_raises_for_score_below_zero
     stub_judge_response('{"score": -0.3, "reasoning": "under"}')
-    metric = RubricLLM::Metrics::Relevance.new(judge: RubricLLM::Judge.new(config: RubricLLM.config))
-    result = metric.call(question: "q", answer: "a")
+    metric = RubricLLM::Metrics::Relevance.new(judge: RubricLLM::Judge.new(config: no_retry_config))
+
+    error = assert_raises(RubricLLM::JudgeError) { metric.call(question: "q", answer: "a") }
+    assert_includes error.message, "between 0.0 and 1.0"
+  end
+
+  private
 
-    assert_in_delta 0.0, result[:score]
+  def no_retry_config
+    RubricLLM::Config.new(max_retries: 0, retry_base_delay: 0.0)
   end
 end
diff --git a/test/test_evaluator.rb b/test/test_evaluator.rb
index 9cb78e7..65d09e2 100644
--- a/test/test_evaluator.rb
+++ b/test/test_evaluator.rb
@@ -49,7 +49,8 @@ def test_evaluate_handles_judge_failure
     result = RubricLLM.evaluate(
       question: "test",
       answer: "test",
-      metrics: [RubricLLM::Metrics::Relevance]
+      metrics: [RubricLLM::Metrics::Relevance],
+      config: RubricLLM::Config.new(max_retries: 0, retry_base_delay: 0.0)
     )
 
     assert_nil result.scores[:relevance]
diff --git a/test/test_helper.rb b/test/test_helper.rb
index 219cc34..bd41c75 100644
--- a/test/test_helper.rb
+++ b/test/test_helper.rb
@@ -16,7 +16,7 @@ def initialize(content)
 
   class FakeChat
     attr_accessor :response_content
-    attr_reader :last_system_prompt, :last_user_prompt, :last_params, :last_attachments, :call_count
+    attr_reader :last_system_prompt, :last_user_prompt, :last_params, :last_attachments, :last_schema, :call_count
 
     def initialize(response_content: '{"score": 0.9, "reasoning": "test"}', fail_times: 0, error_class: RuntimeError)
       @response_content = response_content
@@ -60,6 +60,11 @@ def with_params(**params)
       @last_params = params
       self
     end
+
+    def with_schema(schema)
+      @last_schema = schema
+      self
+    end
   end
 
   def self.chat(**)
diff --git a/test/test_judge.rb b/test/test_judge.rb
index 949944b..05e7028 100644
--- a/test/test_judge.rb
+++ b/test/test_judge.rb
@@ -31,19 +31,22 @@ def test_parse_json_code_fence_no_lang
   def test_parse_json_nil_input
     judge = RubricLLM::Judge.new(config: RubricLLM.config)
 
-    assert_nil judge.parse_json(nil)
+    error = assert_raises(RubricLLM::JudgeError) { judge.parse_json(nil) }
+    assert_includes error.message, "empty"
   end
 
   def test_parse_json_empty_input
     judge = RubricLLM::Judge.new(config: RubricLLM.config)
 
-    assert_nil judge.parse_json("")
+    error = assert_raises(RubricLLM::JudgeError) { judge.parse_json("") }
+    assert_includes error.message, "empty"
   end
 
   def test_parse_json_unparseable
     judge = RubricLLM::Judge.new(config: RubricLLM.config)
 
-    assert_nil judge.parse_json("This is not JSON at all")
+    error = assert_raises(RubricLLM::JudgeError) { judge.parse_json("This is not JSON at all") }
+    assert_includes error.message, "not valid JSON"
   end
 
   def test_call_returns_parsed_json
diff --git a/test/test_judge_contract.rb b/test/test_judge_contract.rb
new file mode 100644
index 0000000..8550873
--- /dev/null
+++ b/test/test_judge_contract.rb
@@ -0,0 +1,81 @@
+# frozen_string_literal: true
+
+require "test_helper"
+
+class TestJudgeContract < Minitest::Test
+  include TestSetup
+
+  def test_call_applies_metric_response_schema
+    chat = RubyLLMStub::FakeChat.new(response_content: '{"score": 0.95, "reasoning": "excellent"}')
+    RubyLLMStub.fake_chat = chat
+
+    result = judge.call(system_prompt: "test", user_prompt: "test")
+
+    assert_in_delta(0.95, result["score"])
+    assert_equal RubricLLM::Judge::METRIC_RESPONSE_SCHEMA, chat.last_schema
+  end
+
+  def test_call_raises_for_malformed_json
+    chat = RubyLLMStub::FakeChat.new(response_content: "This is not JSON")
+    RubyLLMStub.fake_chat = chat
+
+    error = assert_raises(RubricLLM::JudgeError) do
+      judge.call(system_prompt: "test", user_prompt: "test")
+    end
+
+    assert_includes error.message, "not valid JSON"
+    assert_equal 1, chat.call_count
+  end
+
+  def test_call_retries_judge_contract_failures
+    chat = RubyLLMStub::FakeChat.new(response_content: "This is not JSON")
+    RubyLLMStub.fake_chat = chat
+
+    retrying_judge = RubricLLM::Judge.new(config: RubricLLM::Config.new(max_retries: 1, retry_base_delay: 0.0))
+
+    assert_raises(RubricLLM::JudgeError) do
+      retrying_judge.call(system_prompt: "test", user_prompt: "test")
+    end
+
+    assert_equal 2, chat.call_count
+  end
+
+  def test_call_raises_for_missing_score
+    chat = RubyLLMStub::FakeChat.new(response_content: '{"reasoning": "missing"}')
+    RubyLLMStub.fake_chat = chat
+
+    error = assert_raises(RubricLLM::JudgeError) do
+      judge.call(system_prompt: "test", user_prompt: "test")
+    end
+
+    assert_includes error.message, "missing required score"
+  end
+
+  def test_call_raises_for_non_numeric_score
+    chat = RubyLLMStub::FakeChat.new(response_content: '{"score": "excellent", "reasoning": "bad score"}')
+    RubyLLMStub.fake_chat = chat
+
+    error = assert_raises(RubricLLM::JudgeError) do
+      judge.call(system_prompt: "test", user_prompt: "test")
+    end
+
+    assert_includes error.message, "must be numeric"
+  end
+
+  def test_call_raises_for_out_of_range_score
+    chat = RubyLLMStub::FakeChat.new(response_content: '{"score": 1.1, "reasoning": "too high"}')
+    RubyLLMStub.fake_chat = chat
+
+    error = assert_raises(RubricLLM::JudgeError) do
+      judge.call(system_prompt: "test", user_prompt: "test")
+    end
+
+    assert_includes error.message, "between 0.0 and 1.0"
+  end
+
+  private
+
+  def judge
+    RubricLLM::Judge.new(config: RubricLLM::Config.new(max_retries: 0, retry_base_delay: 0.0))
+  end
+end
diff --git a/test/test_rspec_matchers.rb b/test/test_rspec_matchers.rb
index e57793e..1b774e3 100644
--- a/test/test_rspec_matchers.rb
+++ b/test/test_rspec_matchers.rb
@@ -34,12 +34,12 @@ def test_faithfulness_matcher_custom_threshold
     assert_includes matcher.failure_message, "0.9"
   end
 
-  def test_faithfulness_matcher_nil_score
+  def test_faithfulness_matcher_raises_for_empty_judge_response
     stub_judge_response("")
-    matcher = RubricLLM::RSpecMatchers::FaithfulnessMatcher.new(["context"])
+    matcher = RubricLLM::RSpecMatchers::FaithfulnessMatcher.new(["context"]).with_config(no_retry_config)
 
-    refute matcher.matches?("answer")
-    assert_includes matcher.failure_message, "nil"
+    error = assert_raises(RubricLLM::JudgeError) { matcher.matches?("answer") }
+    assert_includes error.message, "empty"
   end
 
   def test_faithfulness_negated_message
@@ -117,11 +117,12 @@ def test_hallucination_matcher_no_hallucination
     assert_includes matcher.failure_message, "hallucination"
   end
 
-  def test_hallucination_matcher_nil_score_counts_as_hallucination
+  def test_hallucination_matcher_raises_for_empty_judge_response
     stub_judge_response("")
-    matcher = RubricLLM::RSpecMatchers::HallucinationMatcher.new(["context"])
+    matcher = RubricLLM::RSpecMatchers::HallucinationMatcher.new(["context"]).with_config(no_retry_config)
 
-    assert matcher.matches?("answer")
+    error = assert_raises(RubricLLM::JudgeError) { matcher.matches?("answer") }
+    assert_includes error.message, "empty"
   end
 
   def test_hallucination_negated_message
@@ -154,4 +155,10 @@ def test_dsl_helpers_exist
     assert_respond_to obj, :be_relevant_to
     assert_respond_to obj, :hallucinate_from
   end
+
+  private
+
+  def no_retry_config
+    RubricLLM::Config.new(max_retries: 0, retry_base_delay: 0.0)
+  end
 end

From 5e52728f71c47e50646635727b38b781711a239f Mon Sep 17 00:00:00 2001
From: David Paluy <dpaluy@users.noreply.github.com>
Date: Sat, 11 Jul 2026 13:12:19 -0500
Subject: [PATCH 2/6] fix: require ruby_llm ~> 1.13 for schema payload support

The named schema wrapper passed to chat.with_schema is only
normalized correctly by ruby_llm >= 1.13; older 1.x versions embed
the wrapper verbatim as the schema definition. Also corrects the
0.2.0 changelog date.

CodeFactory builder
---
 CHANGELOG.md       | 3 ++-
 rubric_llm.gemspec | 2 +-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 3d31199..a105375 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -7,12 +7,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
-## [0.2.0] - 2026-04-30
+## [0.2.0] - 2026-07-11
 
 ### Changed
 
 - Enforce schema-backed judge responses through RubyLLM structured output when supported
 - Raise `RubricLLM::JudgeError` for empty, malformed, missing-score, non-numeric, or out-of-range judge responses instead of returning `nil` or clamping invalid scores
+- Require `ruby_llm ~> 1.13` for named schema payload support in structured output
 
 ## [0.1.2] - 2026-04-30
 
diff --git a/rubric_llm.gemspec b/rubric_llm.gemspec
index d38591b..b4d4b5c 100644
--- a/rubric_llm.gemspec
+++ b/rubric_llm.gemspec
@@ -37,5 +37,5 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
   spec.extra_rdoc_files = Dir["README.md", "CHANGELOG.md", "LICENSE.txt"]
 
-  spec.add_dependency "ruby_llm", "~> 1.0"
+  spec.add_dependency "ruby_llm", "~> 1.13"
 end

From bef82a04661fa2393c443c5881899fe607692ccc Mon Sep 17 00:00:00 2001
From: David Paluy <dpaluy@users.noreply.github.com>
Date: Sat, 11 Jul 2026 13:16:26 -0500
Subject: [PATCH 3/6] fix: handle schema-parsed Hash content in Judge#call

ruby_llm >= 1.13 auto-parses response content into a Hash when a
schema is set, so parse_json crashed on every valid schema-enforced
response. Pass Hash content directly to validation and make the
test stub emulate ruby_llm's schema response parsing.

CodeFactory builder
---
 lib/rubric_llm/judge.rb     |  3 ++-
 test/test_helper.rb         | 12 +++++++++++-
 test/test_judge_contract.rb | 20 ++++++++++++++++++++
 3 files changed, 33 insertions(+), 2 deletions(-)

diff --git a/lib/rubric_llm/judge.rb b/lib/rubric_llm/judge.rb
index 9444ab4..74442bb 100644
--- a/lib/rubric_llm/judge.rb
+++ b/lib/rubric_llm/judge.rb
@@ -43,7 +43,8 @@ def call(system_prompt:, user_prompt:)
         full_system_prompt = build_system_prompt(system_prompt)
         chat.with_instructions(full_system_prompt)
         response = chat.ask(user_prompt)
-        validate_response!(parse_json(response.content))
+        content = response.content
+        validate_response!(content.is_a?(Hash) ? content : parse_json(content))
       rescue StandardError => e
         if attempts > config.max_retries
           raise e if e.is_a?(JudgeError)
diff --git a/test/test_helper.rb b/test/test_helper.rb
index bd41c75..aaafd72 100644
--- a/test/test_helper.rb
+++ b/test/test_helper.rb
@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 $LOAD_PATH.unshift File.expand_path("../lib", __dir__)
+require "json"
 require "rubric_llm"
 require "minitest/autorun"
 
@@ -53,7 +54,8 @@ def ask(prompt, with: nil, **)
 
       raise @error_class, "transient failure" if @call_count <= @fail_times
 
-      FakeResponse.new(response_content)
+      content = @last_schema ? normalize_schema_response(response_content) : response_content
+      FakeResponse.new(content)
     end
 
     def with_params(**params)
@@ -65,6 +67,14 @@ def with_schema(schema)
       @last_schema = schema
       self
     end
+
+    private
+
+    def normalize_schema_response(content)
+      JSON.parse(content)
+    rescue JSON::ParserError
+      content
+    end
   end
 
   def self.chat(**)
diff --git a/test/test_judge_contract.rb b/test/test_judge_contract.rb
index 8550873..587e513 100644
--- a/test/test_judge_contract.rb
+++ b/test/test_judge_contract.rb
@@ -15,6 +15,26 @@ def test_call_applies_metric_response_schema
     assert_equal RubricLLM::Judge::METRIC_RESPONSE_SCHEMA, chat.last_schema
   end
 
+  def test_call_accepts_schema_parsed_hash_content
+    chat = RubyLLMStub::FakeChat.new(response_content: '{"score": 0.95, "reasoning": "excellent"}')
+    RubyLLMStub.fake_chat = chat
+
+    result = judge.call(system_prompt: "test", user_prompt: "test")
+
+    assert_equal({ "score" => 0.95, "reasoning" => "excellent" }, result)
+  end
+
+  def test_call_raises_for_schema_parsed_hash_missing_score
+    chat = RubyLLMStub::FakeChat.new(response_content: '{"reasoning": "missing"}')
+    RubyLLMStub.fake_chat = chat
+
+    error = assert_raises(RubricLLM::JudgeError) do
+      judge.call(system_prompt: "test", user_prompt: "test")
+    end
+
+    assert_includes error.message, "missing required score"
+  end
+
   def test_call_raises_for_malformed_json
     chat = RubyLLMStub::FakeChat.new(response_content: "This is not JSON")
     RubyLLMStub.fake_chat = chat

From e2ca9c5b6131224a6e59df39d6604c8d868ff408 Mon Sep 17 00:00:00 2001
From: David Paluy <dpaluy@users.noreply.github.com>
Date: Sat, 11 Jul 2026 13:16:26 -0500
Subject: [PATCH 4/6] chore: pin development ruby to 4.0.5

CodeFactory builder
---
 .ruby-version | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 .ruby-version

diff --git a/.ruby-version b/.ruby-version
new file mode 100644
index 0000000..7636e75
--- /dev/null
+++ b/.ruby-version
@@ -0,0 +1 @@
+4.0.5

From 5ea672cdcb0872f3aaf8d79910e278ad8e1c83bd Mon Sep 17 00:00:00 2001
From: David Paluy <dpaluy@users.noreply.github.com>
Date: Sat, 11 Jul 2026 14:58:06 -0500
Subject: [PATCH 5/6] refactor: align evaluator and metrics with judge response
 contract

Judge#call now guarantees a validated Hash or raises JudgeError, so
the metric-level nil guards and score clamping were unreachable.
Narrow the evaluator rescue to JudgeError: judge failures still
record nil scores with error details so batch runs continue, but
programming errors now propagate instead of hiding as nil scores.

CodeFactory builder
---
 CHANGELOG.md                                |  2 ++
 lib/rubric_llm/evaluator.rb                 |  2 +-
 lib/rubric_llm/metrics/base.rb              |  5 +----
 lib/rubric_llm/metrics/context_precision.rb |  4 +---
 lib/rubric_llm/metrics/context_recall.rb    |  4 +---
 lib/rubric_llm/metrics/correctness.rb       |  4 +---
 lib/rubric_llm/metrics/factual_accuracy.rb  |  4 +---
 lib/rubric_llm/metrics/faithfulness.rb      |  4 +---
 lib/rubric_llm/metrics/relevance.rb         |  4 +---
 test/test_evaluator.rb                      | 19 ++++++++++++++++++-
 10 files changed, 28 insertions(+), 24 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index a105375..0bf6b9f 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -14,6 +14,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Enforce schema-backed judge responses through RubyLLM structured output when supported
 - Raise `RubricLLM::JudgeError` for empty, malformed, missing-score, non-numeric, or out-of-range judge responses instead of returning `nil` or clamping invalid scores
 - Require `ruby_llm ~> 1.13` for named schema payload support in structured output
+- Record judge failures per metric in `RubricLLM.evaluate` and `RubricLLM.evaluate_batch` as a `nil` score with the error message in details and continue the run, while non-judge errors propagate
+- Remove dead score clamping and nil-response guards from LLM metrics now that the judge validates the response contract
 
 ## [0.1.2] - 2026-04-30
 
diff --git a/lib/rubric_llm/evaluator.rb b/lib/rubric_llm/evaluator.rb
index 83be19a..0a3747d 100644
--- a/lib/rubric_llm/evaluator.rb
+++ b/lib/rubric_llm/evaluator.rb
@@ -30,7 +30,7 @@ def call(question:, answer:, context: [], ground_truth: nil)
         result = metric.call(question:, answer:, context:, ground_truth:)
         scores[name] = result[:score]
         details[name] = result[:details]
-      rescue StandardError => e
+      rescue JudgeError => e
         scores[name] = nil
         details[name] = { error: e.message }
       end
diff --git a/lib/rubric_llm/metrics/base.rb b/lib/rubric_llm/metrics/base.rb
index fd810c3..ff54033 100644
--- a/lib/rubric_llm/metrics/base.rb
+++ b/lib/rubric_llm/metrics/base.rb
@@ -19,10 +19,7 @@ def call(question:, answer:, context: [], ground_truth: nil, **)
       private
 
       def judge_eval(system_prompt:, user_prompt:)
-        result = judge.call(system_prompt:, user_prompt:)
-        return { score: nil, details: { error: "No response from judge" } } if result.nil?
-
-        result
+        judge.call(system_prompt:, user_prompt:)
       end
     end
   end
diff --git a/lib/rubric_llm/metrics/context_precision.rb b/lib/rubric_llm/metrics/context_precision.rb
index 7681f37..5fd85fb 100644
--- a/lib/rubric_llm/metrics/context_precision.rb
+++ b/lib/rubric_llm/metrics/context_precision.rb
@@ -34,10 +34,8 @@ def call(question:, context: [], **)
       private
 
       def normalize(result)
-        return { score: nil, details: result } unless result.is_a?(Hash) && result["score"]
-
         {
-          score: Float(result["score"]).clamp(0.0, 1.0),
+          score: Float(result["score"]),
           details: {
             context_scores: result["context_scores"],
             reasoning: result["reasoning"]
diff --git a/lib/rubric_llm/metrics/context_recall.rb b/lib/rubric_llm/metrics/context_recall.rb
index 5198403..73e1eb4 100644
--- a/lib/rubric_llm/metrics/context_recall.rb
+++ b/lib/rubric_llm/metrics/context_recall.rb
@@ -35,10 +35,8 @@ def call(context: [], ground_truth: nil, **)
       private
 
       def normalize(result)
-        return { score: nil, details: result } unless result.is_a?(Hash) && result["score"]
-
         {
-          score: Float(result["score"]).clamp(0.0, 1.0),
+          score: Float(result["score"]),
           details: {
             covered_facts: result["covered_facts"],
             reasoning: result["reasoning"]
diff --git a/lib/rubric_llm/metrics/correctness.rb b/lib/rubric_llm/metrics/correctness.rb
index ae72789..3a7a8e4 100644
--- a/lib/rubric_llm/metrics/correctness.rb
+++ b/lib/rubric_llm/metrics/correctness.rb
@@ -34,10 +34,8 @@ def call(question:, answer:, ground_truth: nil, **)
       private
 
       def normalize(result)
-        return { score: nil, details: result } unless result.is_a?(Hash) && result["score"]
-
         {
-          score: Float(result["score"]).clamp(0.0, 1.0),
+          score: Float(result["score"]),
           details: { reasoning: result["reasoning"] }
         }
       end
diff --git a/lib/rubric_llm/metrics/factual_accuracy.rb b/lib/rubric_llm/metrics/factual_accuracy.rb
index 832d806..eaf14ac 100644
--- a/lib/rubric_llm/metrics/factual_accuracy.rb
+++ b/lib/rubric_llm/metrics/factual_accuracy.rb
@@ -33,10 +33,8 @@ def call(answer:, ground_truth: nil, **)
       private
 
       def normalize(result)
-        return { score: nil, details: result } unless result.is_a?(Hash) && result["score"]
-
         {
-          score: Float(result["score"]).clamp(0.0, 1.0),
+          score: Float(result["score"]),
           details: {
             discrepancies: result["discrepancies"],
             reasoning: result["reasoning"]
diff --git a/lib/rubric_llm/metrics/faithfulness.rb b/lib/rubric_llm/metrics/faithfulness.rb
index 8a047a2..af206a7 100644
--- a/lib/rubric_llm/metrics/faithfulness.rb
+++ b/lib/rubric_llm/metrics/faithfulness.rb
@@ -36,10 +36,8 @@ def call(question:, answer:, context: [], **)
       private
 
       def normalize(result)
-        return { score: nil, details: result } unless result.is_a?(Hash) && result["score"]
-
         {
-          score: Float(result["score"]).clamp(0.0, 1.0),
+          score: Float(result["score"]),
           details: {
             claims: result["claims"],
             reasoning: result["reasoning"]
diff --git a/lib/rubric_llm/metrics/relevance.rb b/lib/rubric_llm/metrics/relevance.rb
index 98a6e9a..c539d99 100644
--- a/lib/rubric_llm/metrics/relevance.rb
+++ b/lib/rubric_llm/metrics/relevance.rb
@@ -30,10 +30,8 @@ def call(question:, answer:, **)
       private
 
       def normalize(result)
-        return { score: nil, details: result } unless result.is_a?(Hash) && result["score"]
-
         {
-          score: Float(result["score"]).clamp(0.0, 1.0),
+          score: Float(result["score"]),
           details: { reasoning: result["reasoning"] }
         }
       end
diff --git a/test/test_evaluator.rb b/test/test_evaluator.rb
index 65d09e2..f747a0d 100644
--- a/test/test_evaluator.rb
+++ b/test/test_evaluator.rb
@@ -5,6 +5,14 @@
 class TestEvaluator < Minitest::Test
   include TestSetup
 
+  class BrokenMetric
+    def initialize(judge:); end
+
+    def call(**)
+      raise "programming error"
+    end
+  end
+
   def test_evaluate_returns_result
     stub_judge_response('{"score": 0.9, "reasoning": "good"}')
     result = RubricLLM.evaluate(
@@ -45,7 +53,7 @@ def test_evaluate_default_metrics_include_factual_accuracy
   end
 
   def test_evaluate_handles_judge_failure
-    stub_judge_response("completely broken response")
+    stub_judge_response("")
     result = RubricLLM.evaluate(
       question: "test",
       answer: "test",
@@ -54,6 +62,15 @@ def test_evaluate_handles_judge_failure
     )
 
     assert_nil result.scores[:relevance]
+    assert_match(/empty/i, result.details[:relevance][:error])
+  end
+
+  def test_evaluate_propagates_non_judge_errors
+    error = assert_raises(RuntimeError) do
+      RubricLLM::Evaluator.new(config: RubricLLM::Config.new, metrics: [BrokenMetric]).call(question: "test", answer: "test")
+    end
+
+    assert_equal "programming error", error.message
   end
 
   def test_evaluate_with_custom_prompt

From 355c713fdfff9c7846e6da2bd0bd517c6c4013c3 Mon Sep 17 00:00:00 2001
From: David Paluy <dpaluy@users.noreply.github.com>
Date: Sat, 11 Jul 2026 15:01:03 -0500
Subject: [PATCH 6/6] style: satisfy Style/ArrayIntersect in
 RetrievalResult#hit_rate

Newer rubocop on CI flags the any?/include? pattern. Use
Set#intersect? with relevant as receiver since Array#intersect?
rejects the Set argument.

CodeFactory builder
---
 .ruby-version                      | 1 -
 CHANGELOG.md                       | 2 --
 lib/rubric_llm/retrieval_result.rb | 2 +-
 3 files changed, 1 insertion(+), 4 deletions(-)
 delete mode 100644 .ruby-version

diff --git a/.ruby-version b/.ruby-version
deleted file mode 100644
index 7636e75..0000000
--- a/.ruby-version
+++ /dev/null
@@ -1 +0,0 @@
-4.0.5
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 0bf6b9f..9a4c3fa 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,8 +5,6 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
-## [Unreleased]
-
 ## [0.2.0] - 2026-07-11
 
 ### Changed
diff --git a/lib/rubric_llm/retrieval_result.rb b/lib/rubric_llm/retrieval_result.rb
index 908ea89..749733c 100644
--- a/lib/rubric_llm/retrieval_result.rb
+++ b/lib/rubric_llm/retrieval_result.rb
@@ -51,7 +51,7 @@ def ndcg(k: retrieved.size)
     end
 
     def hit_rate
-      retrieved.any? { |doc| relevant.include?(doc) } ? 1.0 : 0.0
+      relevant.intersect?(retrieved) ? 1.0 : 0.0
     end
 
     def to_h