diff --git a/configs/eval-dataset-deepeval.yml b/configs/eval-dataset-deepeval.yml new file mode 100644 index 0000000..8854290 --- /dev/null +++ b/configs/eval-dataset-deepeval.yml @@ -0,0 +1,14 @@ +dataset_generation: + provider: deepeval + options: + model: gpt-4o-mini + max_goldens_per_context: 1 + limit_chunks: 50 + min_context_chars: 120 + include_expected_output: true + max_concurrent: 10 + +llm: + provider: openai + model: gpt-4o-mini + temperature: 0.3 diff --git a/docs/deepeval.md b/docs/deepeval.md new file mode 100644 index 0000000..cacfe31 --- /dev/null +++ b/docs/deepeval.md @@ -0,0 +1,91 @@ +# DeepEval evaluation and synthesis + +DeepEval is available as an optional dev evaluation provider alongside the +existing RAGAS provider. Use it when you want one framework for both RAG +component scoring and synthetic golden generation from prepared xrag Chunks. + +## Why DeepEval for xrag + +The DeepEval RAG guide recommends evaluating retrieval and generation together: +retrieval quality is measured with contextual precision, recall, and relevancy; +generation quality is measured with faithfulness and answer relevancy. Its +Synthesizer can generate goldens from prepared contexts, which maps cleanly to +xrag's Chunk artifacts because xrag already controls parsing, chunk metadata, +and Retrieval Text. + +## Installation + +DeepEval is an optional tool dependency. Install it in your active dev +environment before selecting the provider: + +```bash +uv pip install deepeval +``` + +## Full-eval provider + +Set `evaluation.provider: deepeval` in any RAG config and keep `--full-eval` +on the eval command: + +```yaml +evaluation: + provider: deepeval + options: + judge_model: gpt-4o-mini + metrics: + - faithfulness + - answer_relevancy + - context_precision + - context_recall + - context_relevancy + threshold: 0.7 + include_reason: true +``` + +Supported metric names: + +- `faithfulness` +- `answer_relevancy` +- `contextual_precision` or `context_precision` +- `contextual_recall` or `context_recall` +- `contextual_relevancy` or `context_relevancy` + +The eval runner still writes the same artifacts as RAGAS runs: +`eval_results.json`, `per_question.jsonl`, `summary.txt`, and the append-only +`experiments/eval/experiments.jsonl`. Deterministic retrieval metrics remain the +benchmark gate; DeepEval metrics are added only for `--full-eval` runs. + +## Synthetic data provider + +Use `configs/eval-dataset-deepeval.yml` to generate DeepEval goldens from xrag +Chunk contexts: + +```bash +uv run python -m xrag.cli eval create-dataset \ + --input data/dev/ir_document/latest/chunks.json \ + --config configs/eval-dataset-deepeval.yml \ + --output-dir data/artifacts/ir_document/eval-deepeval +``` + +The provider calls `Synthesizer.generate_goldens_from_contexts()` rather than +`generate_goldens_from_docs()` because xrag already parsed documents into +canonical Chunks with `doc_id`, `chunk_id`, page, and source-element metadata. +Each generated golden is converted back into the existing `testset.jsonl` schema +with the source chunk pinned as `evidence_chunk_ids`. + +Important options: + +- `model`: DeepEval synthesizer model. +- `limit_chunks`: maximum number of chunks to submit. +- `min_context_chars`: skips tiny chunks that do not provide enough signal. +- `max_goldens_per_context`: generated questions per chunk context. +- `include_expected_output`: keep enabled for RAG scoring. +- `max_concurrent`: reduce if the model provider rate-limits. + +## Benchmark discipline + +DeepEval synthetic rows are LLM-generated, but they are not a replacement for the +canonical curated/LLM xrag benchmark unless explicitly promoted. After any eval +run, keep following the repository benchmark rules: compile the benchmark, +append a new `docs/BENCHMARK.md` row, and update key findings and phase gate +status when there is new evidence. diff --git a/scripts/eval/README.md b/scripts/eval/README.md index 6818507..009b119 100644 --- a/scripts/eval/README.md +++ b/scripts/eval/README.md @@ -78,5 +78,5 @@ the per-metric tolerance bands exits non-zero. ```bash scripts/eval/verify_chunk_schema.sh # deterministic-only (~12 min) -scripts/eval/verify_chunk_schema.sh --full-eval # include RAGAS judge (~50 min) +scripts/eval/verify_chunk_schema.sh --full-eval # include configured judge metrics (~50 min) ``` diff --git a/tests/unit/test_deepeval_eval.py b/tests/unit/test_deepeval_eval.py new file mode 100644 index 0000000..cc40576 --- /dev/null +++ b/tests/unit/test_deepeval_eval.py @@ -0,0 +1,102 @@ +from __future__ import annotations + +import sys +import types +from dataclasses import dataclass +from typing import Any + +import pytest + +from xrag.config.models import ProviderConfig + + +class _FakeMetric: + def __init__(self, **kwargs: Any) -> None: + self.kwargs = kwargs + self.score: float | None = None + self.reason = "grounded" + + def measure(self, test_case: Any) -> None: + self.score = 1.0 if test_case.actual_output else 0.0 + + +def _install_fake_deepeval(monkeypatch: pytest.MonkeyPatch) -> None: + deepeval = types.ModuleType("deepeval") + metrics = types.ModuleType("deepeval.metrics") + test_case_mod = types.ModuleType("deepeval.test_case") + + @dataclass + class LLMTestCase: + input: str + actual_output: str + expected_output: str = "" + retrieval_context: list[str] | None = None + + for name in ( + "AnswerRelevancyMetric", + "ContextualPrecisionMetric", + "ContextualRecallMetric", + "ContextualRelevancyMetric", + "FaithfulnessMetric", + ): + setattr(metrics, name, _FakeMetric) + test_case_mod.LLMTestCase = LLMTestCase + + monkeypatch.setitem(sys.modules, "deepeval", deepeval) + monkeypatch.setitem(sys.modules, "deepeval.metrics", metrics) + monkeypatch.setitem(sys.modules, "deepeval.test_case", test_case_mod) + sys.modules.pop("tools.evaluation.deepeval_eval", None) + + +def test_build_evaluator_returns_deepeval_provider(monkeypatch: pytest.MonkeyPatch) -> None: + """Factory lazily builds the DeepEval provider.""" + _install_fake_deepeval(monkeypatch) + + from tools.evaluation.deepeval_eval import DeepEvalEvaluator + from tools.evaluation.factory import build_evaluator + + evaluator = build_evaluator(ProviderConfig(provider="deepeval", options={})) + + assert isinstance(evaluator, DeepEvalEvaluator) + + +def test_deepeval_evaluator_scores_and_aliases_metrics( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """DeepEval scores use xrag's existing aggregate/per-question contract.""" + _install_fake_deepeval(monkeypatch) + + from tools.evaluation.deepeval_eval import DeepEvalEvaluator + + evaluator = DeepEvalEvaluator() + output = evaluator.evaluate( + [ + { + "query": "What is revenue?", + "answer": "$10M", + "contexts": ["Revenue was $10M."], + "ground_truth": "$10M", + } + ], + { + "metrics": ["faithfulness", "contextual_precision"], + "judge_model": "gpt-4o-mini", + "include_reason": True, + }, + ) + + assert output["scores"] == {"faithfulness": 1.0, "context_precision": 1.0} + assert output["per_question"][0]["query"] == "What is revenue?" + assert output["per_question"][0]["context_precision_reason"] == "grounded" + + +def test_deepeval_evaluator_rejects_unknown_metric( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Unknown metric names fail fast before any judge calls are made.""" + _install_fake_deepeval(monkeypatch) + + from tools.evaluation.deepeval_eval import DeepEvalEvaluator + + with pytest.raises(ValueError, match="unknown deepeval metrics"): + DeepEvalEvaluator().evaluate([], {"metrics": ["bogus"]}) diff --git a/tests/unit/test_deepeval_synthesizer.py b/tests/unit/test_deepeval_synthesizer.py new file mode 100644 index 0000000..8fc0b8e --- /dev/null +++ b/tests/unit/test_deepeval_synthesizer.py @@ -0,0 +1,114 @@ +from __future__ import annotations + +import sys +import types +from dataclasses import dataclass +from typing import Any + +import pytest + +from xrag.config.models import ProviderConfig + + +@dataclass +class _Golden: + input: str + expected_output: str + source_file: str + + +class _FakeSynthesizer: + last_kwargs: dict[str, Any] = {} + last_contexts: list[list[str]] = [] + + def __init__(self, **kwargs: Any) -> None: + type(self).last_kwargs = kwargs + + def generate_goldens_from_contexts(self, **kwargs: Any) -> list[_Golden]: + type(self).last_contexts = kwargs["contexts"] + return [ + _Golden( + input="What was reported?", + expected_output="Revenue was $10M.", + source_file="doc:chunk:1", + ) + ] + + +def _install_fake_deepeval_synthesizer(monkeypatch: pytest.MonkeyPatch) -> None: + deepeval = types.ModuleType("deepeval") + synthesizer_mod = types.ModuleType("deepeval.synthesizer") + synthesizer_mod.Synthesizer = _FakeSynthesizer + monkeypatch.setitem(sys.modules, "deepeval", deepeval) + monkeypatch.setitem(sys.modules, "deepeval.synthesizer", synthesizer_mod) + sys.modules.pop("tools.dataset_generation.deepeval_synthesizer", None) + + +def test_build_dataset_generator_returns_deepeval_provider( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Dataset generator factory exposes DeepEval synthesis.""" + _install_fake_deepeval_synthesizer(monkeypatch) + + from tools.dataset_generation.deepeval_synthesizer import ( + DeepEvalSynthesizerDatasetGenerator, + ) + from tools.dataset_generation.factory import build_dataset_generator + + generator = build_dataset_generator(ProviderConfig(provider="deepeval", options={})) + + assert isinstance(generator, DeepEvalSynthesizerDatasetGenerator) + + +def test_deepeval_synthesizer_maps_goldens_to_testset_schema( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """DeepEval goldens are converted into xrag testset rows with pinned evidence.""" + _install_fake_deepeval_synthesizer(monkeypatch) + + from tools.dataset_generation.deepeval_synthesizer import ( + DeepEvalSynthesizerDatasetGenerator, + ) + + chunks = [ + { + "chunk_id": "doc:chunk:1", + "doc_id": "doc", + "chunk_strategy": "section_table", + "content_with_weight": "Revenue was $10M. " * 10, + "source_element_ids": ["el1"], + "page_number": 3, + } + ] + rows = DeepEvalSynthesizerDatasetGenerator().run( + chunks, + { + "model": "gpt-4o-mini", + "limit_chunks": 1, + "min_context_chars": 10, + "max_goldens_per_context": 1, + }, + ) + + assert _FakeSynthesizer.last_kwargs["model"] == "gpt-4o-mini" + assert _FakeSynthesizer.last_contexts == [[chunks[0]["content_with_weight"].strip()]] + assert rows == [ + { + "example_id": "doc:qa:0001", + "question": "What was reported?", + "answer": "Revenue was $10M.", + "answer_type": "string", + "reasoning_type": "deepeval_synthetic", + "difficulty": "medium", + "doc_id": "doc", + "chunk_strategy": "section_table", + "evidence_chunk_ids": ["doc:chunk:1"], + "evidence_element_ids": ["el1"], + "page_numbers": [3], + "metadata": { + "generator_provider": "deepeval", + "category": "deepeval_synthetic", + "validation_status": "valid", + }, + } + ] diff --git a/tools/dataset_generation/deepeval_synthesizer.py b/tools/dataset_generation/deepeval_synthesizer.py new file mode 100644 index 0000000..e3d77ef --- /dev/null +++ b/tools/dataset_generation/deepeval_synthesizer.py @@ -0,0 +1,133 @@ +from __future__ import annotations + +from typing import Any + +from deepeval.synthesizer import Synthesizer + +from tools.dataset_generation.base import BaseDatasetGenerator +from tools.dataset_generation.template import _validate_examples + + +def _chunk_text(chunk: dict[str, Any]) -> str: + """Return the best available Retrieval Text for synthetic generation.""" + for key in ("content_with_weight", "content", "text", "body"): + value = chunk.get(key) + if isinstance(value, str) and value.strip(): + return value.strip() + return "" + + +def _chunk_metadata(chunk: dict[str, Any]) -> tuple[str, str, list[str], list[int]]: + """Extract common xrag chunk metadata for the testset row schema.""" + page = chunk.get("page_number") + return ( + chunk.get("doc_id", "document"), + chunk.get("chunk_strategy", "section_table"), + list(chunk.get("source_element_ids") or []), + [page] if page else [], + ) + + +def _golden_value(golden: Any, name: str, default: Any = None) -> Any: + """Read a DeepEval Golden field from objects or test doubles.""" + if hasattr(golden, name): + return getattr(golden, name) + if isinstance(golden, dict): + return golden.get(name, default) + return default + + +class DeepEvalSynthesizerDatasetGenerator(BaseDatasetGenerator): + """Generate xrag testset rows from Chunk contexts with DeepEval Synthesizer.""" + + def run(self, chunks: list[dict[str, Any]], options: dict[str, Any]) -> list[dict[str, Any]]: + """Generate synthetic QA items from prepared xrag chunks. + + Args: + chunks: Flattened list of chunk dicts from chunks.json. + options: Provider options. Supports ``model``, ``max_concurrent``, + ``async_mode``, ``include_expected_output``, + ``max_goldens_per_context``, ``limit_chunks``, and + ``min_context_chars``. + + Returns: + Validated QA example dicts matching the testset JSONL schema. + """ + min_context_chars = int(options.get("min_context_chars", 120)) + limit_chunks = int(options.get("limit_chunks", 0)) + selected_chunks = [ + chunk for chunk in chunks if len(_chunk_text(chunk)) >= min_context_chars + ] + if limit_chunks > 0: + selected_chunks = selected_chunks[:limit_chunks] + + if not selected_chunks: + return [] + + contexts = [[_chunk_text(chunk)] for chunk in selected_chunks] + source_files = [ + str(chunk.get("chunk_id", f"chunk-{idx}")) + for idx, chunk in enumerate(selected_chunks) + ] + + synthesizer_kwargs: dict[str, Any] = {} + for key in ("model", "max_concurrent", "async_mode", "cost_tracking"): + if key in options: + synthesizer_kwargs[key] = options[key] + synthesizer = Synthesizer(**synthesizer_kwargs) + goldens = synthesizer.generate_goldens_from_contexts( + contexts=contexts, + include_expected_output=bool(options.get("include_expected_output", True)), + max_goldens_per_context=int(options.get("max_goldens_per_context", 1)), + source_files=source_files, + ) + + examples: list[dict[str, Any]] = [] + for idx, golden in enumerate(goldens, start=1): + source_file = _golden_value(golden, "source_file") or _golden_value( + golden, "source_files", [None] + ) + if isinstance(source_file, list): + source_file = source_file[0] if source_file else None + chunk = _chunk_by_id(selected_chunks, str(source_file)) or selected_chunks[ + min(idx - 1, len(selected_chunks) - 1) + ] + doc_id, chunk_strategy, element_ids, pages = _chunk_metadata(chunk) + chunk_id = str(chunk.get("chunk_id", source_file or "")) + question = str(_golden_value(golden, "input", "")).strip() + answer = str(_golden_value(golden, "expected_output", "")).strip() + if not question or not answer: + continue + examples.append( + { + "example_id": f"{doc_id}:qa:{idx:04d}", + "question": question, + "answer": answer, + "answer_type": "string", + "reasoning_type": "deepeval_synthetic", + "difficulty": "medium", + "doc_id": doc_id, + "chunk_strategy": chunk_strategy, + "evidence_chunk_ids": [chunk_id] if chunk_id else [], + "evidence_element_ids": element_ids, + "page_numbers": pages, + "metadata": { + "generator_provider": "deepeval", + "category": "deepeval_synthetic", + "validation_status": "valid", + }, + } + ) + + valid_chunk_ids = {str(chunk.get("chunk_id")) for chunk in selected_chunks} + return _validate_examples(examples, valid_chunk_ids) + + +def _chunk_by_id(chunks: list[dict[str, Any]], chunk_id: str | None) -> dict[str, Any] | None: + """Find a chunk by ``chunk_id``.""" + if not chunk_id: + return None + for chunk in chunks: + if str(chunk.get("chunk_id")) == chunk_id: + return chunk + return None diff --git a/tools/dataset_generation/factory.py b/tools/dataset_generation/factory.py index ef0100e..7c22c7f 100644 --- a/tools/dataset_generation/factory.py +++ b/tools/dataset_generation/factory.py @@ -25,4 +25,10 @@ def build_dataset_generator( if llm_config is None: raise ValueError("llm dataset generator requires an llm config section") return LlmDatasetGenerator(llm_config=llm_config) + if config.provider == "deepeval": + from tools.dataset_generation.deepeval_synthesizer import ( + DeepEvalSynthesizerDatasetGenerator, + ) + + return DeepEvalSynthesizerDatasetGenerator() raise ValueError(f"unsupported dataset_generation provider: {config.provider}") diff --git a/tools/evaluation/deepeval_eval.py b/tools/evaluation/deepeval_eval.py new file mode 100644 index 0000000..b78859a --- /dev/null +++ b/tools/evaluation/deepeval_eval.py @@ -0,0 +1,123 @@ +from __future__ import annotations + +import statistics +from collections.abc import Callable +from typing import Any + +from deepeval.metrics import ( + AnswerRelevancyMetric, + ContextualPrecisionMetric, + ContextualRecallMetric, + ContextualRelevancyMetric, + FaithfulnessMetric, +) +from deepeval.test_case import LLMTestCase + +from tools.evaluation.base import BaseEvaluator + +MetricFactory = Callable[[dict[str, Any]], Any] + + +_DEFAULT_METRICS = ["faithfulness", "answer_relevancy", "contextual_precision"] + + +def _metric_options(options: dict[str, Any]) -> dict[str, Any]: + """Extract constructor options shared by DeepEval metric classes.""" + metric_options: dict[str, Any] = {} + if "threshold" in options: + metric_options["threshold"] = float(options["threshold"]) + if "judge_model" in options: + metric_options["model"] = options["judge_model"] + elif "model" in options: + metric_options["model"] = options["model"] + if "include_reason" in options: + metric_options["include_reason"] = bool(options["include_reason"]) + if "async_mode" in options: + metric_options["async_mode"] = bool(options["async_mode"]) + return metric_options + + +def _build_metric(cls: type, options: dict[str, Any]) -> Any: + """Instantiate a DeepEval metric using the common option subset.""" + return cls(**_metric_options(options)) + + +_METRIC_REGISTRY: dict[str, MetricFactory] = { + "faithfulness": lambda options: _build_metric(FaithfulnessMetric, options), + "answer_relevancy": lambda options: _build_metric(AnswerRelevancyMetric, options), + "contextual_precision": lambda options: _build_metric(ContextualPrecisionMetric, options), + "context_precision": lambda options: _build_metric(ContextualPrecisionMetric, options), + "contextual_recall": lambda options: _build_metric(ContextualRecallMetric, options), + "context_recall": lambda options: _build_metric(ContextualRecallMetric, options), + "contextual_relevancy": lambda options: _build_metric(ContextualRelevancyMetric, options), + "context_relevancy": lambda options: _build_metric(ContextualRelevancyMetric, options), +} + +_OUTPUT_ALIASES = { + "contextual_precision": "context_precision", + "contextual_recall": "context_recall", + "contextual_relevancy": "context_relevancy", +} + + +class DeepEvalEvaluator(BaseEvaluator): + """DeepEval-based RAG evaluation provider.""" + + def evaluate(self, results: list[dict[str, Any]], options: dict[str, Any]) -> dict[str, Any]: + """Score RAG results with DeepEval single-turn RAG metrics. + + Args: + results: List of dicts with ``query``, ``answer``, ``contexts``, + and ``ground_truth`` keys. + options: Optional knobs: + - ``metrics``: list of metric names. Supported: + ``faithfulness``, ``answer_relevancy``, + ``contextual_precision`` / ``context_precision``, + ``contextual_recall`` / ``context_recall``, and + ``contextual_relevancy`` / ``context_relevancy``. + - ``judge_model`` or ``model``: DeepEval judge model. + - ``threshold``: minimum passing threshold per metric. + - ``include_reason``: include judge reasons in per-question + output when DeepEval returns them. + - ``async_mode``: passed through to metric constructors. + + Returns: + Dict with ``scores`` (aggregate) and ``per_question`` (per-item). + """ + metric_names = options.get("metrics") or _DEFAULT_METRICS + unknown = [m for m in metric_names if m not in _METRIC_REGISTRY] + if unknown: + raise ValueError( + f"unknown deepeval metrics: {unknown}. Supported: {sorted(_METRIC_REGISTRY)}" + ) + + per_question: list[dict[str, Any]] = [] + raw_scores: dict[str, list[float]] = {} + + for result in results: + test_case = LLMTestCase( + input=result["query"], + actual_output=result["answer"], + expected_output=result.get("ground_truth", ""), + retrieval_context=result.get("contexts", []), + ) + entry: dict[str, Any] = {"query": result["query"]} + for metric_name in metric_names: + metric = _METRIC_REGISTRY[metric_name](options) + metric.measure(test_case) + output_name = _OUTPUT_ALIASES.get(metric_name, metric_name) + score = getattr(metric, "score", None) + entry[output_name] = float(score) if score is not None else None + reason = getattr(metric, "reason", None) + if reason: + entry[f"{output_name}_reason"] = reason + if entry[output_name] is not None: + raw_scores.setdefault(output_name, []).append(entry[output_name]) + per_question.append(entry) + + aggregate = { + metric_name: float(statistics.mean(values)) + for metric_name, values in raw_scores.items() + if values + } + return {"scores": aggregate, "per_question": per_question} diff --git a/tools/evaluation/factory.py b/tools/evaluation/factory.py index a8378bd..368577a 100644 --- a/tools/evaluation/factory.py +++ b/tools/evaluation/factory.py @@ -8,7 +8,7 @@ def build_evaluator(config: ProviderConfig) -> BaseEvaluator: """Build the configured evaluator implementation. Provider modules are imported lazily so that core retrieve/RAG paths - don't require the `[eval]` extra (RAGAS + judge-LLM stack). + don't require optional judge stacks (RAGAS, DeepEval, etc.). Args: config: Evaluation provider config. @@ -17,12 +17,11 @@ def build_evaluator(config: ProviderConfig) -> BaseEvaluator: Concrete evaluator implementation. """ if config.provider == "ragas": - try: - from tools.evaluation.ragas_eval import RagasEvaluator - except ImportError as e: # pragma: no cover — extras-gating - raise ImportError( - "RAGAS evaluation support requires the [eval] extra. " - "Install with `pip install 'xrag[eval]'`." - ) from e + from tools.evaluation.ragas_eval import RagasEvaluator + return RagasEvaluator() + if config.provider == "deepeval": + from tools.evaluation.deepeval_eval import DeepEvalEvaluator + + return DeepEvalEvaluator() raise ValueError(f"unsupported evaluation provider: {config.provider}") diff --git a/tools/pipelines/eval_run.py b/tools/pipelines/eval_run.py index 3e9cef8..d08b3ff 100644 --- a/tools/pipelines/eval_run.py +++ b/tools/pipelines/eval_run.py @@ -52,9 +52,9 @@ def run_eval_pipeline( max_workers: Concurrency for retrieve+generate phase. Each question is independent (pure I/O) so threads scale well up to the provider's rate limit. Default 1 (sequential). - full_eval: When True, run RAGAS LLM-judge metrics - (``faithfulness``, ``answer_relevancy``, ``context_precision``) - on top of the deterministic retrieval metrics. Off by default + full_eval: When True, run configured LLM-judge metrics + (for example ``faithfulness``, ``answer_relevancy``, + ``context_precision``) on top of deterministic retrieval metrics. Off by default so the inner-loop run is fast and cheap; flip on for promotion-grade rows in BENCHMARK.md. @@ -237,43 +237,48 @@ def _process(idx: int, q: EvalQuestion) -> tuple[int, dict[str, Any]]: evaluation_options = evaluation_config.options eval_output: dict[str, Any] = {"scores": {}, "per_question": []} - ragas_elapsed = 0.0 + judge_elapsed = 0.0 if full_eval: - _log("=== RAGAS Scoring ===") - # RAGAS judges expect answer/contexts/ground_truth fields. Error + evaluator_name = evaluation_config.provider + _log(f"=== {evaluator_name} Scoring ===") + # Judge providers expect answer/contexts/ground_truth fields. Error # records have none, so they're filtered out of judge inputs (the # row remains in per_question.jsonl with its error block). - ragas_records = [r for r in eval_records if "error" not in r] + judge_records = [r for r in eval_records if "error" not in r] metric_names = evaluation_options.get("metrics") or ["faithfulness", "answer_relevancy"] # Expected LLM-call count so progress can be reasoned about. # Each metric typically issues ~1 judge call per sample; `answer_relevancy` # asks the LLM for N (default 3) generations per sample. `context_precision` # scales with retrieved contexts (top_k per sample). - expected_calls = len(ragas_records) * len(metric_names) + expected_calls = len(judge_records) * len(metric_names) _log( f"Evaluator: {evaluation_config.provider} " f"(metrics={metric_names}, " - f"samples={len(ragas_records)}, " + f"samples={len(judge_records)}, " f"≈{expected_calls} judge jobs, " f"workers={evaluation_options.get('max_workers', 32)}, " f"judge={evaluation_options.get('judge_model', 'gpt-4o-mini')})" ) - _log(" (RAGAS tqdm bar follows — one bar per metric)") - t_ragas = time.time() + if evaluation_config.provider == "ragas": + _log(" (RAGAS tqdm bar follows — one bar per metric)") + t_judge = time.time() evaluator = build_evaluator(evaluation_config) - eval_output = evaluator.evaluate(ragas_records, evaluation_options) - ragas_elapsed = time.time() - t_ragas - if ragas_records: + eval_output = evaluator.evaluate(judge_records, evaluation_options) + judge_elapsed = time.time() - t_judge + if judge_records: _log( - f"RAGAS completed in {ragas_elapsed:.1f}s " - f"({ragas_elapsed / len(ragas_records):.2f}s/sample)" + f"{evaluation_config.provider} completed in {judge_elapsed:.1f}s " + f"({judge_elapsed / len(judge_records):.2f}s/sample)" ) else: - _log(f"RAGAS completed in {ragas_elapsed:.1f}s (no successful records to score)") + _log( + f"{evaluation_config.provider} completed in " + f"{judge_elapsed:.1f}s (no successful records to score)" + ) else: _log( - "=== RAGAS Scoring SKIPPED (deterministic-only; rerun with --full-eval to include) ===" + "=== Judge Scoring SKIPPED (deterministic-only; rerun with --full-eval to include) ===" ) scores: dict[str, float] = dict(eval_output.get("scores", {})) @@ -281,7 +286,8 @@ def _process(idx: int, q: EvalQuestion) -> tuple[int, dict[str, Any]]: scores.update(_aggregate_latency_metrics(eval_records)) scores.update(_aggregate_context_metrics(eval_records)) if full_eval: - scores["latency.ragas_s"] = round(ragas_elapsed, 1) + scores["latency.ragas_s"] = round(judge_elapsed, 1) + scores["latency.judge_s"] = round(judge_elapsed, 1) eval_output["scores"] = scores scores_by_category = _aggregate_by_category(eval_records) @@ -336,7 +342,7 @@ def _process(idx: int, q: EvalQuestion) -> tuple[int, dict[str, Any]]: _log("") _log("Latency") _log("-" * 60) - _log(_format_latency_block(scores, pipeline_elapsed, ragas_elapsed)) + _log(_format_latency_block(scores, pipeline_elapsed, judge_elapsed)) _log("") _log("Context") _log("-" * 60) @@ -344,7 +350,7 @@ def _process(idx: int, q: EvalQuestion) -> tuple[int, dict[str, Any]]: _log("") _log(f"Experiment: {resolved_dir}") _log(" config.yml — full config snapshot") - _log(" eval_results.json — aggregate scores + per-question RAGAS") + _log(" eval_results.json — aggregate scores + per-question judge metrics") _log(" per_question.jsonl — per-question details with latency") _log(" summary.txt — human-readable summary") @@ -457,12 +463,12 @@ def _build_summary_text( def _format_latency_block( scores: dict[str, float], pipeline_elapsed: float, - ragas_elapsed: float, + judge_elapsed: float, ) -> str: - """Render the latency breakdown: retrieval / generation / e2e / ragas / total. + """Render the latency breakdown: retrieval / generation / e2e / judge / total. Per-question metrics are aggregated as mean / p50 / p95. - Phase totals (ragas, pipeline) are wall-clock seconds. + Phase totals (judge, pipeline) are wall-clock seconds. """ rows = [ f" {'Phase':<12s} {'Mean':>9s} {'P50':>9s} {'P95':>9s} (per-question)", @@ -480,12 +486,12 @@ def _format_latency_block( continue rows.append(f" {label:<12s} {mean:>7.0f}ms {p50:>7.0f}ms {p95:>7.0f}ms") rows.append("") - if ragas_elapsed > 0: - rows.append(f" {'RAGAS total':<12s} {ragas_elapsed:>7.1f}s (whole eval-score phase)") + if judge_elapsed > 0: + rows.append(f" {'Judge total':<12s} {judge_elapsed:>7.1f}s (whole eval-score phase)") else: - rows.append(f" {'RAGAS total':<12s} skipped (rerun with --full-eval to include)") + rows.append(f" {'Judge total':<12s} skipped (rerun with --full-eval to include)") rows.append( - f" {'Pipeline':<12s} {pipeline_elapsed:>7.1f}s (end-to-end: retrieve+generate+ragas+I/O)" + f" {'Pipeline':<12s} {pipeline_elapsed:>7.1f}s (end-to-end: retrieve+generate+judge+I/O)" ) return "\n".join(rows) diff --git a/xrag/cli.py b/xrag/cli.py index 8de4363..1d2d017 100644 --- a/xrag/cli.py +++ b/xrag/cli.py @@ -1042,8 +1042,8 @@ def eval_run_command( bool, typer.Option( "--full-eval", - help="Run RAGAS LLM-judge metrics (faithfulness, answer_relevancy, " - "context_precision) on top of the deterministic retrieval " + help="Run configured LLM-judge metrics (for example faithfulness, " + "answer_relevancy, context_precision) on top of deterministic retrieval " "metrics. Off by default for the fast inner loop; flip on for " "promotion-grade rows in BENCHMARK.md.", ), @@ -1062,7 +1062,7 @@ def eval_run_command( workers: Concurrency for the retrieve+generate phase. tenant_id: Optional override for pipeline.tenant_id. doc_id: Optional repeatable doc_id filter. - full_eval: Include RAGAS LLM-judge metrics in the run. + full_eval: Include configured LLM-judge metrics in the run. pretty: Whether to pretty-print the JSON result. """ parsed_config = load_rag_config(config)