diff --git a/CHANGELOG.md b/CHANGELOG.md index e12f2ef..92d5d69 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,17 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.4.0] - 2026-07-11 + +### Added + +- `RubricLLM.evaluate_batch` validates every sample upfront (must be a Hash with non-nil `:question` and `:answer`, string or symbol keys) and raises `ArgumentError` with the offending index before any LLM call, so sequential and concurrent modes fail identically and without API spend + +### Changed + +- Add `csv` as a runtime dependency; `csv` moved from a default gem to a bundled gem in Ruby 3.4, so consumers previously hit a `LoadError` on `Report#export_csv` +- Require `ruby_llm ~> 1.16` + ## [0.3.0] - 2026-07-11 ### Changed diff --git a/Gemfile b/Gemfile index 7b45873..98f5a3f 100644 --- a/Gemfile +++ b/Gemfile @@ -4,7 +4,6 @@ source "https://rubygems.org" gemspec -gem "csv" gem "minitest" gem "rake" gem "rdoc" diff --git a/lib/rubric_llm.rb b/lib/rubric_llm.rb index 17af220..446e8de 100644 --- a/lib/rubric_llm.rb +++ b/lib/rubric_llm.rb @@ -56,6 +56,7 @@ def evaluate(question:, answer:, context: [], ground_truth: nil, metrics: nil, # report = RubricLLM.evaluate_batch(dataset) # report = RubricLLM.evaluate_batch(dataset, concurrency: 4) def evaluate_batch(dataset, metrics: nil, config: self.config, custom_prompt: nil, concurrency: nil) + validate_dataset!(dataset) config = apply_custom_prompt(config, custom_prompt) pool_size = concurrency || config.concurrency start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC) @@ -87,6 +88,22 @@ def evaluate_retrieval(retrieved:, relevant:) private + def validate_dataset!(dataset) + dataset.each_with_index do |sample, index| + raise ArgumentError, "sample at index #{index} is not a Hash" unless sample.is_a?(Hash) + + question_present = sample.key?(:question) || sample.key?("question") + answer_present = sample.key?(:answer) || sample.key?("answer") + raise ArgumentError, "sample at index #{index} is missing :question" unless question_present + raise ArgumentError, "sample at index #{index} is missing :answer" unless answer_present + + question_provided = !sample[:question].nil? || !sample["question"].nil? + answer_provided = !sample[:answer].nil? || !sample["answer"].nil? + raise ArgumentError, "sample at index #{index} has nil :question" unless question_provided + raise ArgumentError, "sample at index #{index} has nil :answer" unless answer_provided + end + end + def evaluate_sample(evaluator, sample) sample = normalize_sample(sample) evaluator.call( diff --git a/lib/rubric_llm/report.rb b/lib/rubric_llm/report.rb index 214752e..544d11d 100644 --- a/lib/rubric_llm/report.rb +++ b/lib/rubric_llm/report.rb @@ -42,7 +42,7 @@ def summary end def export_csv(path) - require "csv" # optional dependency — add `gem "csv"` to your Gemfile if missing + require "csv" metrics = all_metric_names CSV.open(path, "w") do |csv| csv << ["question", "answer", "overall", *metrics] diff --git a/lib/rubric_llm/version.rb b/lib/rubric_llm/version.rb index 5e614f9..7f3eede 100644 --- a/lib/rubric_llm/version.rb +++ b/lib/rubric_llm/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module RubricLLM - VERSION = "0.3.0" + VERSION = "0.4.0" end diff --git a/rubric_llm.gemspec b/rubric_llm.gemspec index b4d4b5c..0e9603c 100644 --- a/rubric_llm.gemspec +++ b/rubric_llm.gemspec @@ -37,5 +37,6 @@ Gem::Specification.new do |spec| spec.require_paths = ["lib"] spec.extra_rdoc_files = Dir["README.md", "CHANGELOG.md", "LICENSE.txt"] - spec.add_dependency "ruby_llm", "~> 1.13" + spec.add_dependency "csv" + spec.add_dependency "ruby_llm", "~> 1.16" end diff --git a/test/test_batch_validation.rb b/test/test_batch_validation.rb new file mode 100644 index 0000000..03199ba --- /dev/null +++ b/test/test_batch_validation.rb @@ -0,0 +1,75 @@ +# frozen_string_literal: true + +require "test_helper" + +class TestBatchValidation < Minitest::Test + include TestSetup + + def test_evaluate_batch_rejects_non_hash_samples_before_judge_calls + chat = RubyLLMStub::FakeChat.new + RubyLLMStub.fake_chat = chat + + error = assert_raises(ArgumentError) do + RubricLLM.evaluate_batch( + [{ question: "q1", answer: "a1" }, "not a sample"], + metrics: [RubricLLM::Metrics::Relevance] + ) + end + + assert_match "sample at index 1 is not a Hash", error.message + assert_equal 0, chat.call_count + end + + def test_evaluate_batch_rejects_samples_missing_required_keys + [{ question: "q1" }, { "answer" => "a1" }].each_with_index do |sample, index| + error = assert_raises(ArgumentError) do + RubricLLM.evaluate_batch([sample], metrics: [RubricLLM::Metrics::Relevance]) + end + + assert_match "sample at index 0", error.message + assert_match(index.zero? ? ":answer" : ":question", error.message) + end + end + + def test_evaluate_batch_rejects_samples_with_nil_required_values_before_judge_calls + [{ question: nil, answer: "a1" }, { "question" => "q2", "answer" => nil }].each do |sample| + chat = RubyLLMStub::FakeChat.new + RubyLLMStub.fake_chat = chat + + error = assert_raises(ArgumentError) do + RubricLLM.evaluate_batch([sample], metrics: [RubricLLM::Metrics::Relevance]) + end + + assert_match "sample at index 0 has nil", error.message + assert_equal 0, chat.call_count + end + end + + def test_evaluate_batch_accepts_complete_symbol_and_string_keyed_samples + stub_judge_response('{"score": 0.9, "reasoning": "ok"}') + + report = RubricLLM.evaluate_batch( + [{ question: "symbol question", answer: "symbol answer" }, + { "question" => "string question", "answer" => "string answer" }], + metrics: [RubricLLM::Metrics::Relevance] + ) + + assert_equal 2, report.results.size + end + + def test_evaluate_batch_validates_before_starting_concurrent_work + chat = RubyLLMStub::FakeChat.new + RubyLLMStub.fake_chat = chat + + error = assert_raises(ArgumentError) do + RubricLLM.evaluate_batch( + [{ question: "q1", answer: "a1" }, { question: "q2" }], + metrics: [RubricLLM::Metrics::Relevance], + concurrency: 2 + ) + end + + assert_match "sample at index 1 is missing :answer", error.message + assert_equal 0, chat.call_count + end +end