From 2c25c3bc9c99879f2b38d8d030d49ef11a217325 Mon Sep 17 00:00:00 2001 From: David Paluy Date: Sat, 11 Jul 2026 17:16:08 -0500 Subject: [PATCH 1/3] chore: bump ruby_llm to ~> 1.16 and add csv runtime dependency csv moved from a default gem to a bundled gem in Ruby 3.4, so Bundler-managed consumers hit a LoadError on Report#export_csv without an explicit dependency. CodeFactory builder --- rubric_llm.gemspec | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/rubric_llm.gemspec b/rubric_llm.gemspec index b4d4b5c..0e9603c 100644 --- a/rubric_llm.gemspec +++ b/rubric_llm.gemspec @@ -37,5 +37,6 @@ Gem::Specification.new do |spec| spec.require_paths = ["lib"] spec.extra_rdoc_files = Dir["README.md", "CHANGELOG.md", "LICENSE.txt"] - spec.add_dependency "ruby_llm", "~> 1.13" + spec.add_dependency "csv" + spec.add_dependency "ruby_llm", "~> 1.16" end From bb69e2beadd8177cf0b115b50c4ab5cae2f896e7 Mon Sep 17 00:00:00 2001 From: David Paluy Date: Sat, 11 Jul 2026 17:16:14 -0500 Subject: [PATCH 2/3] fix(core): validate batch dataset upfront before any LLM call Add validate_dataset! to evaluate_batch so malformed samples (non-hash, missing or nil :question/:answer, string or symbol keys) raise ArgumentError with the offending index before any API spend, making sequential and concurrent modes fail identically. Remove the redundant csv Gemfile entry now that it is a gemspec runtime dependency. Closes #5 CodeFactory builder --- Gemfile | 1 - lib/rubric_llm.rb | 17 ++++++++ lib/rubric_llm/report.rb | 2 +- test/test_batch_validation.rb | 75 +++++++++++++++++++++++++++++++++++ 4 files changed, 93 insertions(+), 2 deletions(-) create mode 100644 test/test_batch_validation.rb diff --git a/Gemfile b/Gemfile index 7b45873..98f5a3f 100644 --- a/Gemfile +++ b/Gemfile @@ -4,7 +4,6 @@ source "https://rubygems.org" gemspec -gem "csv" gem "minitest" gem "rake" gem "rdoc" diff --git a/lib/rubric_llm.rb b/lib/rubric_llm.rb index 17af220..446e8de 100644 --- a/lib/rubric_llm.rb +++ b/lib/rubric_llm.rb @@ -56,6 +56,7 @@ def evaluate(question:, answer:, context: [], ground_truth: nil, metrics: nil, # report = RubricLLM.evaluate_batch(dataset) # report = RubricLLM.evaluate_batch(dataset, concurrency: 4) def evaluate_batch(dataset, metrics: nil, config: self.config, custom_prompt: nil, concurrency: nil) + validate_dataset!(dataset) config = apply_custom_prompt(config, custom_prompt) pool_size = concurrency || config.concurrency start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC) @@ -87,6 +88,22 @@ def evaluate_retrieval(retrieved:, relevant:) private + def validate_dataset!(dataset) + dataset.each_with_index do |sample, index| + raise ArgumentError, "sample at index #{index} is not a Hash" unless sample.is_a?(Hash) + + question_present = sample.key?(:question) || sample.key?("question") + answer_present = sample.key?(:answer) || sample.key?("answer") + raise ArgumentError, "sample at index #{index} is missing :question" unless question_present + raise ArgumentError, "sample at index #{index} is missing :answer" unless answer_present + + question_provided = !sample[:question].nil? || !sample["question"].nil? + answer_provided = !sample[:answer].nil? || !sample["answer"].nil? + raise ArgumentError, "sample at index #{index} has nil :question" unless question_provided + raise ArgumentError, "sample at index #{index} has nil :answer" unless answer_provided + end + end + def evaluate_sample(evaluator, sample) sample = normalize_sample(sample) evaluator.call( diff --git a/lib/rubric_llm/report.rb b/lib/rubric_llm/report.rb index 214752e..544d11d 100644 --- a/lib/rubric_llm/report.rb +++ b/lib/rubric_llm/report.rb @@ -42,7 +42,7 @@ def summary end def export_csv(path) - require "csv" # optional dependency — add `gem "csv"` to your Gemfile if missing + require "csv" metrics = all_metric_names CSV.open(path, "w") do |csv| csv << ["question", "answer", "overall", *metrics] diff --git a/test/test_batch_validation.rb b/test/test_batch_validation.rb new file mode 100644 index 0000000..03199ba --- /dev/null +++ b/test/test_batch_validation.rb @@ -0,0 +1,75 @@ +# frozen_string_literal: true + +require "test_helper" + +class TestBatchValidation < Minitest::Test + include TestSetup + + def test_evaluate_batch_rejects_non_hash_samples_before_judge_calls + chat = RubyLLMStub::FakeChat.new + RubyLLMStub.fake_chat = chat + + error = assert_raises(ArgumentError) do + RubricLLM.evaluate_batch( + [{ question: "q1", answer: "a1" }, "not a sample"], + metrics: [RubricLLM::Metrics::Relevance] + ) + end + + assert_match "sample at index 1 is not a Hash", error.message + assert_equal 0, chat.call_count + end + + def test_evaluate_batch_rejects_samples_missing_required_keys + [{ question: "q1" }, { "answer" => "a1" }].each_with_index do |sample, index| + error = assert_raises(ArgumentError) do + RubricLLM.evaluate_batch([sample], metrics: [RubricLLM::Metrics::Relevance]) + end + + assert_match "sample at index 0", error.message + assert_match(index.zero? ? ":answer" : ":question", error.message) + end + end + + def test_evaluate_batch_rejects_samples_with_nil_required_values_before_judge_calls + [{ question: nil, answer: "a1" }, { "question" => "q2", "answer" => nil }].each do |sample| + chat = RubyLLMStub::FakeChat.new + RubyLLMStub.fake_chat = chat + + error = assert_raises(ArgumentError) do + RubricLLM.evaluate_batch([sample], metrics: [RubricLLM::Metrics::Relevance]) + end + + assert_match "sample at index 0 has nil", error.message + assert_equal 0, chat.call_count + end + end + + def test_evaluate_batch_accepts_complete_symbol_and_string_keyed_samples + stub_judge_response('{"score": 0.9, "reasoning": "ok"}') + + report = RubricLLM.evaluate_batch( + [{ question: "symbol question", answer: "symbol answer" }, + { "question" => "string question", "answer" => "string answer" }], + metrics: [RubricLLM::Metrics::Relevance] + ) + + assert_equal 2, report.results.size + end + + def test_evaluate_batch_validates_before_starting_concurrent_work + chat = RubyLLMStub::FakeChat.new + RubyLLMStub.fake_chat = chat + + error = assert_raises(ArgumentError) do + RubricLLM.evaluate_batch( + [{ question: "q1", answer: "a1" }, { question: "q2" }], + metrics: [RubricLLM::Metrics::Relevance], + concurrency: 2 + ) + end + + assert_match "sample at index 1 is missing :answer", error.message + assert_equal 0, chat.call_count + end +end From 4e6570366a8aa9eaeb2f6790a9d1625a89ec3bb4 Mon Sep 17 00:00:00 2001 From: David Paluy Date: Sat, 11 Jul 2026 17:37:27 -0500 Subject: [PATCH 3/3] chore: bump version to 0.4.0 CodeFactory builder --- CHANGELOG.md | 11 +++++++++++ lib/rubric_llm/version.rb | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e12f2ef..92d5d69 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,17 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.4.0] - 2026-07-11 + +### Added + +- `RubricLLM.evaluate_batch` validates every sample upfront (must be a Hash with non-nil `:question` and `:answer`, string or symbol keys) and raises `ArgumentError` with the offending index before any LLM call, so sequential and concurrent modes fail identically and without API spend + +### Changed + +- Add `csv` as a runtime dependency; `csv` moved from a default gem to a bundled gem in Ruby 3.4, so consumers previously hit a `LoadError` on `Report#export_csv` +- Require `ruby_llm ~> 1.16` + ## [0.3.0] - 2026-07-11 ### Changed diff --git a/lib/rubric_llm/version.rb b/lib/rubric_llm/version.rb index 5e614f9..7f3eede 100644 --- a/lib/rubric_llm/version.rb +++ b/lib/rubric_llm/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module RubricLLM - VERSION = "0.3.0" + VERSION = "0.4.0" end