From e4a3e0cc3b5f007061537be8df7a96e530b8613d Mon Sep 17 00:00:00 2001 From: Markus Klein Date: Thu, 27 Aug 2026 17:56:48 +0200 Subject: [PATCH 1/6] test: Composed benchmarks tested Dataset to Prompt --- src/eval_framework/benchmarks/arc_de.py | 33 ++-- .../benchmarks/csqa_ellamind.py | 48 +++-- .../benchmarks/piqa_ellamind.py | 53 +++-- .../benchmarks/siqa_ellamind.py | 48 +++-- .../benchmarks/test_csqa_ellamind.py | 186 +++++++++++++----- .../benchmarks/test_piqa_ellamind.py | 171 +++++++++++----- .../benchmarks/test_siqa_ellamind.py | 176 +++++++++++------ .../tests_eval_framework/benchmarks/utils.py | 45 +++++ 8 files changed, 557 insertions(+), 203 deletions(-) create mode 100644 tests/tests_eval_framework/benchmarks/utils.py diff --git a/src/eval_framework/benchmarks/arc_de.py b/src/eval_framework/benchmarks/arc_de.py index 75d364ec4..49513f2ac 100644 --- a/src/eval_framework/benchmarks/arc_de.py +++ b/src/eval_framework/benchmarks/arc_de.py @@ -7,6 +7,7 @@ from eval_framework.contract import Benchmark from eval_framework.subjects import NoSubject from eval_framework.tasks.base import Language +from eval_framework.tasks.dataset_loading import DatasetPolicy from eval_framework.tasks.dataset_revisions import pinned_by_framework from eval_framework.tasks.task_style import ClozeStyle, answer_key_to_index @@ -28,16 +29,22 @@ def read(self, item: dict[str, Any]) -> ChoiceFields: ) -# ARC-DE as cloze/ranked classification. -# https://huggingface.co/datasets/LeoLM/ArcChallenge_de -ARC_DE_BENCHMARK: Benchmark = ComposedBenchmark.compose( - id="ARC_DE", - display_name="ARC German", - styler=ClozeStyle(question_prefix="Frage: ", cue_text="Antwort:"), - reader=ArcDeReader(), - sample_split="test", - fewshot_split="validation", - subjects=NoSubject(), - dataset_policy=pinned_by_framework("LeoLM/ArcChallenge_de"), - language=Language.DEU, -) +def arc_de(dataset: DatasetPolicy | None = None) -> Benchmark: + """ARC-DE as cloze/ranked classification. + + https://huggingface.co/datasets/LeoLM/ArcChallenge_de + """ + return ComposedBenchmark.compose( + id="ARC_DE", + display_name="ARC German", + styler=ClozeStyle(question_prefix="Frage: ", cue_text="Antwort:"), + reader=ArcDeReader(), + sample_split="test", + fewshot_split="validation", + subjects=NoSubject(), + dataset_policy=dataset if dataset is not None else pinned_by_framework("LeoLM/ArcChallenge_de"), + language=Language.DEU, + ) + + +ARC_DE_BENCHMARK: Benchmark = arc_de() diff --git a/src/eval_framework/benchmarks/csqa_ellamind.py b/src/eval_framework/benchmarks/csqa_ellamind.py index 1beee7835..8ebbc5b6d 100644 --- a/src/eval_framework/benchmarks/csqa_ellamind.py +++ b/src/eval_framework/benchmarks/csqa_ellamind.py @@ -12,6 +12,7 @@ from eval_framework.contract import Benchmark from eval_framework.subjects import ListOfSubjects from eval_framework.tasks.base import Language +from eval_framework.tasks.dataset_loading import DatasetPolicy from eval_framework.tasks.dataset_revisions import pinned_by_framework from eval_framework.tasks.task_style import BPBStyle, ClozeStyle, MCStyle, TaskStyler, shuffle_correct_with_distractors @@ -34,14 +35,9 @@ def read(self, item: dict[str, Any]) -> ChoiceFields: return ChoiceFields(raw_question=item["question"], choices=choices, correct_index=correct_index) -# One styler per format, all sharing the German prefix/cue. The easy/hard distractor axis belongs to -# the reader, so it is orthogonal to the styler choice. -CSQA_ELLAMIND_CLOZE_STYLER = ClozeStyle.for_language(Language.DEU) -CSQA_ELLAMIND_MC_STYLER = MCStyle.for_language(Language.DEU) -CSQA_ELLAMIND_BPB_STYLER = BPBStyle.for_language(Language.DEU) - - -def _csqa_ellamind_benchmark(id: str, styler: TaskStyler, distractor_level: Literal["easy", "hard"]) -> Benchmark: +def _csqa_ellamind_benchmark( + id: str, styler: TaskStyler, distractor_level: Literal["easy", "hard"], dataset: DatasetPolicy | None +) -> Benchmark: return ComposedBenchmark.compose( id=id, styler=styler, @@ -49,15 +45,39 @@ def _csqa_ellamind_benchmark(id: str, styler: TaskStyler, distractor_level: Lite sample_split="validation", fewshot_split="validation", subjects=ListOfSubjects(["deu"]), - dataset_policy=pinned_by_framework("ellamind/csqa-multilingual"), + dataset_policy=dataset if dataset is not None else pinned_by_framework("ellamind/csqa-multilingual"), language=Language.DEU, ) +def csqa_ellamind_mc_easy_de(dataset: DatasetPolicy | None = None) -> Benchmark: + return _csqa_ellamind_benchmark("CSQA_ELLAMIND_MC_EASY_DE", MCStyle.for_language(Language.DEU), "easy", dataset) + + +def csqa_ellamind_mc_hard_de(dataset: DatasetPolicy | None = None) -> Benchmark: + return _csqa_ellamind_benchmark("CSQA_ELLAMIND_MC_HARD_DE", MCStyle.for_language(Language.DEU), "hard", dataset) + + +def csqa_ellamind_cloze_easy_de(dataset: DatasetPolicy | None = None) -> Benchmark: + return _csqa_ellamind_benchmark( + "CSQA_ELLAMIND_CLOZE_EASY_DE", ClozeStyle.for_language(Language.DEU), "easy", dataset + ) + + +def csqa_ellamind_cloze_hard_de(dataset: DatasetPolicy | None = None) -> Benchmark: + return _csqa_ellamind_benchmark( + "CSQA_ELLAMIND_CLOZE_HARD_DE", ClozeStyle.for_language(Language.DEU), "hard", dataset + ) + + +def csqa_ellamind_bpb_de(dataset: DatasetPolicy | None = None) -> Benchmark: + return _csqa_ellamind_benchmark("CSQA_ELLAMIND_BPB_DE", BPBStyle.for_language(Language.DEU), "easy", dataset) + + CSQA_ELLAMIND_BENCHMARKS: list[Benchmark] = [ - _csqa_ellamind_benchmark("CSQA_ELLAMIND_MC_EASY_DE", CSQA_ELLAMIND_MC_STYLER, "easy"), - _csqa_ellamind_benchmark("CSQA_ELLAMIND_MC_HARD_DE", CSQA_ELLAMIND_MC_STYLER, "hard"), - _csqa_ellamind_benchmark("CSQA_ELLAMIND_CLOZE_EASY_DE", CSQA_ELLAMIND_CLOZE_STYLER, "easy"), - _csqa_ellamind_benchmark("CSQA_ELLAMIND_CLOZE_HARD_DE", CSQA_ELLAMIND_CLOZE_STYLER, "hard"), - _csqa_ellamind_benchmark("CSQA_ELLAMIND_BPB_DE", CSQA_ELLAMIND_BPB_STYLER, "easy"), + csqa_ellamind_mc_easy_de(), + csqa_ellamind_mc_hard_de(), + csqa_ellamind_cloze_easy_de(), + csqa_ellamind_cloze_hard_de(), + csqa_ellamind_bpb_de(), ] diff --git a/src/eval_framework/benchmarks/piqa_ellamind.py b/src/eval_framework/benchmarks/piqa_ellamind.py index 8556ab71b..76ea4ecf2 100644 --- a/src/eval_framework/benchmarks/piqa_ellamind.py +++ b/src/eval_framework/benchmarks/piqa_ellamind.py @@ -12,6 +12,7 @@ from eval_framework.contract import Benchmark from eval_framework.subjects import ListOfSubjects from eval_framework.tasks.base import Language +from eval_framework.tasks.dataset_loading import DatasetPolicy from eval_framework.tasks.dataset_revisions import pinned_by_framework from eval_framework.tasks.task_style import BPBStyle, ClozeStyle, MCStyle, TaskStyler, shuffle_correct_with_distractors @@ -37,14 +38,10 @@ def read(self, item: dict[str, Any]) -> ChoiceFields: _QUESTION_PREFIX = "Ziel: " _CUE_TEXT = "Antwort:" -# One styler per format (all sharing the German prefix/cue). The easy/hard distractor axis belongs to -# the reader, so it is orthogonal to the styler choice. -PIQA_ELLAMIND_CLOZE_STYLER = ClozeStyle(question_prefix=_QUESTION_PREFIX, cue_text=_CUE_TEXT) -PIQA_ELLAMIND_MC_STYLER = MCStyle(question_prefix=_QUESTION_PREFIX, cue_text=_CUE_TEXT) -PIQA_ELLAMIND_BPB_STYLER = BPBStyle(question_prefix=_QUESTION_PREFIX, cue_text=_CUE_TEXT) - -def _piqa_ellamind_benchmark(id: str, styler: TaskStyler, distractor_level: Literal["easy", "hard"]) -> Benchmark: +def _piqa_ellamind_benchmark( + id: str, styler: TaskStyler, distractor_level: Literal["easy", "hard"], dataset: DatasetPolicy | None +) -> Benchmark: return ComposedBenchmark.compose( id=id, styler=styler, @@ -52,15 +49,45 @@ def _piqa_ellamind_benchmark(id: str, styler: TaskStyler, distractor_level: Lite sample_split="validation", fewshot_split="validation", subjects=ListOfSubjects(["deu"]), - dataset_policy=pinned_by_framework("ellamind/piqa-multilingual"), + dataset_policy=dataset if dataset is not None else pinned_by_framework("ellamind/piqa-multilingual"), language=Language.DEU, ) +def piqa_ellamind_cloze_easy_de(dataset: DatasetPolicy | None = None) -> Benchmark: + return _piqa_ellamind_benchmark( + "PIQA_ELLAMIND_CLOZE_EASY_DE", ClozeStyle(question_prefix=_QUESTION_PREFIX, cue_text=_CUE_TEXT), "easy", dataset + ) + + +def piqa_ellamind_cloze_hard_de(dataset: DatasetPolicy | None = None) -> Benchmark: + return _piqa_ellamind_benchmark( + "PIQA_ELLAMIND_CLOZE_HARD_DE", ClozeStyle(question_prefix=_QUESTION_PREFIX, cue_text=_CUE_TEXT), "hard", dataset + ) + + +def piqa_ellamind_mc_easy_de(dataset: DatasetPolicy | None = None) -> Benchmark: + return _piqa_ellamind_benchmark( + "PIQA_ELLAMIND_MC_EASY_DE", MCStyle(question_prefix=_QUESTION_PREFIX, cue_text=_CUE_TEXT), "easy", dataset + ) + + +def piqa_ellamind_mc_hard_de(dataset: DatasetPolicy | None = None) -> Benchmark: + return _piqa_ellamind_benchmark( + "PIQA_ELLAMIND_MC_HARD_DE", MCStyle(question_prefix=_QUESTION_PREFIX, cue_text=_CUE_TEXT), "hard", dataset + ) + + +def piqa_ellamind_bpb_de(dataset: DatasetPolicy | None = None) -> Benchmark: + return _piqa_ellamind_benchmark( + "PIQA_ELLAMIND_BPB_DE", BPBStyle(question_prefix=_QUESTION_PREFIX, cue_text=_CUE_TEXT), "easy", dataset + ) + + PIQA_ELLAMIND_BENCHMARKS: list[Benchmark] = [ - _piqa_ellamind_benchmark("PIQA_ELLAMIND_CLOZE_EASY_DE", PIQA_ELLAMIND_CLOZE_STYLER, "easy"), - _piqa_ellamind_benchmark("PIQA_ELLAMIND_CLOZE_HARD_DE", PIQA_ELLAMIND_CLOZE_STYLER, "hard"), - _piqa_ellamind_benchmark("PIQA_ELLAMIND_MC_EASY_DE", PIQA_ELLAMIND_MC_STYLER, "easy"), - _piqa_ellamind_benchmark("PIQA_ELLAMIND_MC_HARD_DE", PIQA_ELLAMIND_MC_STYLER, "hard"), - _piqa_ellamind_benchmark("PIQA_ELLAMIND_BPB_DE", PIQA_ELLAMIND_BPB_STYLER, "easy"), + piqa_ellamind_cloze_easy_de(), + piqa_ellamind_cloze_hard_de(), + piqa_ellamind_mc_easy_de(), + piqa_ellamind_mc_hard_de(), + piqa_ellamind_bpb_de(), ] diff --git a/src/eval_framework/benchmarks/siqa_ellamind.py b/src/eval_framework/benchmarks/siqa_ellamind.py index 4f5ab6c0a..2c43d070b 100644 --- a/src/eval_framework/benchmarks/siqa_ellamind.py +++ b/src/eval_framework/benchmarks/siqa_ellamind.py @@ -12,6 +12,7 @@ from eval_framework.contract import Benchmark from eval_framework.subjects import ListOfSubjects from eval_framework.tasks.base import Language +from eval_framework.tasks.dataset_loading import DatasetPolicy from eval_framework.tasks.dataset_revisions import pinned_by_framework from eval_framework.tasks.task_style import BPBStyle, ClozeStyle, MCStyle, TaskStyler, shuffle_correct_with_distractors @@ -37,14 +38,9 @@ def read(self, item: dict[str, Any]) -> ChoiceFields: ) -# One styler per format, all sharing the German prefix/cue. The easy/hard distractor axis belongs to -# the reader, so it is orthogonal to the styler choice. -SIQA_ELLAMIND_CLOZE_STYLER = ClozeStyle.for_language(Language.DEU) -SIQA_ELLAMIND_MC_STYLER = MCStyle.for_language(Language.DEU) -SIQA_ELLAMIND_BPB_STYLER = BPBStyle.for_language(Language.DEU) - - -def _siqa_ellamind_benchmark(id: str, styler: TaskStyler, distractor_level: Literal["easy", "hard"]) -> Benchmark: +def _siqa_ellamind_benchmark( + id: str, styler: TaskStyler, distractor_level: Literal["easy", "hard"], dataset: DatasetPolicy | None +) -> Benchmark: return ComposedBenchmark.compose( id=id, styler=styler, @@ -52,15 +48,39 @@ def _siqa_ellamind_benchmark(id: str, styler: TaskStyler, distractor_level: Lite sample_split="validation", fewshot_split="validation", subjects=ListOfSubjects(["deu"]), - dataset_policy=pinned_by_framework("ellamind/siqa-multilingual"), + dataset_policy=dataset if dataset is not None else pinned_by_framework("ellamind/siqa-multilingual"), language=Language.DEU, ) +def siqa_ellamind_mc_easy_de(dataset: DatasetPolicy | None = None) -> Benchmark: + return _siqa_ellamind_benchmark("SIQA_ELLAMIND_MC_EASY_DE", MCStyle.for_language(Language.DEU), "easy", dataset) + + +def siqa_ellamind_mc_hard_de(dataset: DatasetPolicy | None = None) -> Benchmark: + return _siqa_ellamind_benchmark("SIQA_ELLAMIND_MC_HARD_DE", MCStyle.for_language(Language.DEU), "hard", dataset) + + +def siqa_ellamind_cloze_easy_de(dataset: DatasetPolicy | None = None) -> Benchmark: + return _siqa_ellamind_benchmark( + "SIQA_ELLAMIND_CLOZE_EASY_DE", ClozeStyle.for_language(Language.DEU), "easy", dataset + ) + + +def siqa_ellamind_cloze_hard_de(dataset: DatasetPolicy | None = None) -> Benchmark: + return _siqa_ellamind_benchmark( + "SIQA_ELLAMIND_CLOZE_HARD_DE", ClozeStyle.for_language(Language.DEU), "hard", dataset + ) + + +def siqa_ellamind_bpb_de(dataset: DatasetPolicy | None = None) -> Benchmark: + return _siqa_ellamind_benchmark("SIQA_ELLAMIND_BPB_DE", BPBStyle.for_language(Language.DEU), "easy", dataset) + + SIQA_ELLAMIND_BENCHMARKS: list[Benchmark] = [ - _siqa_ellamind_benchmark("SIQA_ELLAMIND_MC_EASY_DE", SIQA_ELLAMIND_MC_STYLER, "easy"), - _siqa_ellamind_benchmark("SIQA_ELLAMIND_MC_HARD_DE", SIQA_ELLAMIND_MC_STYLER, "hard"), - _siqa_ellamind_benchmark("SIQA_ELLAMIND_CLOZE_EASY_DE", SIQA_ELLAMIND_CLOZE_STYLER, "easy"), - _siqa_ellamind_benchmark("SIQA_ELLAMIND_CLOZE_HARD_DE", SIQA_ELLAMIND_CLOZE_STYLER, "hard"), - _siqa_ellamind_benchmark("SIQA_ELLAMIND_BPB_DE", SIQA_ELLAMIND_BPB_STYLER, "easy"), + siqa_ellamind_mc_easy_de(), + siqa_ellamind_mc_hard_de(), + siqa_ellamind_cloze_easy_de(), + siqa_ellamind_cloze_hard_de(), + siqa_ellamind_bpb_de(), ] diff --git a/tests/tests_eval_framework/benchmarks/test_csqa_ellamind.py b/tests/tests_eval_framework/benchmarks/test_csqa_ellamind.py index 7a5b04d8a..8d116dfcd 100644 --- a/tests/tests_eval_framework/benchmarks/test_csqa_ellamind.py +++ b/tests/tests_eval_framework/benchmarks/test_csqa_ellamind.py @@ -1,26 +1,30 @@ -"""Tests for the German CSQA (EllaMind) tasks. +"""Specification of the German CSQA (EllaMind) tasks. -- formatter hash test for every CSQA variant -- offline test that the reader (item -> ChoiceFields) and the chosen styler produce the expected - prompt content. Message assembly (roles / fewshot / cue placement) is generic and covered in - ``test_composed_benchmark``; the reader logic itself is covered in ``test_choices``. +Each spec test builds the real benchmark (via its ``csqa_ellamind_*_de`` constructor) over a fictional +dataset and asserts the assembled messages, ground truth, and scored completions — so this file reads +as CSQA's prompt spec, with ``composed.py`` an implementation detail. ``test_formatter_hash`` separately +pins the real benchmarks against the real HuggingFace data. """ +from collections.abc import Callable from dataclasses import dataclass from typing import Any import pytest from eval_framework.benchmarks.csqa_ellamind import ( - CSQA_ELLAMIND_BPB_STYLER, - CSQA_ELLAMIND_CLOZE_STYLER, - CSQA_ELLAMIND_MC_STYLER, CsqaReader, + csqa_ellamind_bpb_de, + csqa_ellamind_cloze_easy_de, + csqa_ellamind_cloze_hard_de, + csqa_ellamind_mc_easy_de, + csqa_ellamind_mc_hard_de, ) +from eval_framework.contract import Benchmark from eval_framework.tasks.registry import Registry from eval_framework.tasks.task_names import register_csqa_ellamind_tasks -from eval_framework.tasks.task_style import TaskStyler -from template_formatting.formatter import BaseFormatter, ConcatFormatter, Llama3Formatter +from template_formatting.formatter import BaseFormatter, ConcatFormatter, Llama3Formatter, Message, Role +from tests.tests_eval_framework.benchmarks.utils import DatasetStub, first_sample from tests.tests_eval_framework.tasks.benchmarks.utils import run_formatter_hash_test # Registry for this test suite only holding csqa_ellamind tasks @@ -36,75 +40,165 @@ def test_formatter_hash(task_name: str, formatter_cls: type[BaseFormatter]) -> N # --------------------------------------------------------------------------- -# Offline test: reader + chosen styler produce the expected prompt content (no Eval, no dataset) +# Prompt spec: build the real benchmark over fictional rows, assert the assembled messages # --------------------------------------------------------------------------- -# A fictional row in the CSQA format (NOT a real dataset example). Choices are shuffled -# deterministically (seed: question + correct_answer), which places the correct answer at index 0. +# Fictional rows in the CSQA format (NOT real dataset examples). Choices are shuffled deterministically +# (seed: question + correct_answer). _EVAL_ROW: dict[str, Any] = { "question": "Wo bewahrt man frische Milch am besten auf?", "correct_answer": "Im Kühlschrank", "easy_distractors": ["Auf dem Mond", "In einem Schuh", "Im Vulkan"], "hard_distractors": ["In der Speisekammer", "Auf der Fensterbank", "Im Keller"], } +_FEWSHOT_ROW: dict[str, Any] = { + "question": "Womit schreibt man normalerweise auf Papier?", + "correct_answer": "Mit einem Stift", + "easy_distractors": ["Mit einer Banane", "Mit einer Wolke", "Mit einem Stein"], + "hard_distractors": ["Mit einem Pinsel", "Mit Kreide", "Mit einer Tastatur"], +} + +_EVAL_Q = "Frage: Wo bewahrt man frische Milch am besten auf?" @dataclass(frozen=True) class _ExpectedPrompt: - instruction: str - cue: str + messages: list[Message] ground_truth: str - completions: list[str] + possible_completions: list[str] +# --- Zero-shot --- _MC_EASY = _ExpectedPrompt( - instruction="Frage: Wo bewahrt man frische Milch am besten auf?\n" - "A. Im Kühlschrank\nB. Auf dem Mond\nC. In einem Schuh\nD. Im Vulkan\n", - cue="Antwort:", + messages=[ + Message( + role=Role.USER, content=f"{_EVAL_Q}\nA. Im Kühlschrank\nB. Auf dem Mond\nC. In einem Schuh\nD. Im Vulkan\n" + ), + Message(role=Role.ASSISTANT, content="Antwort:"), + ], ground_truth=" A", - completions=[" A", " B", " C", " D"], + possible_completions=[" A", " B", " C", " D"], ) _MC_HARD = _ExpectedPrompt( - instruction="Frage: Wo bewahrt man frische Milch am besten auf?\n" - "A. Im Kühlschrank\nB. In der Speisekammer\nC. Auf der Fensterbank\nD. Im Keller\n", - cue="Antwort:", + messages=[ + Message( + role=Role.USER, + content=f"{_EVAL_Q}\nA. Im Kühlschrank\nB. In der Speisekammer\nC. Auf der Fensterbank\nD. Im Keller\n", + ), + Message(role=Role.ASSISTANT, content="Antwort:"), + ], ground_truth=" A", - completions=[" A", " B", " C", " D"], + possible_completions=[" A", " B", " C", " D"], ) -# Cloze/BPB show no options, so the prompt text is identical; only the scored completions differ. +# Cloze/BPB show no options, so the assembled messages are identical; only the scored completions differ. +_CLOZE_MESSAGES = [ + Message(role=Role.USER, content=f"{_EVAL_Q}\n"), + Message(role=Role.ASSISTANT, content="Antwort:"), +] _CLOZE_EASY = _ExpectedPrompt( - instruction="Frage: Wo bewahrt man frische Milch am besten auf?\n", - cue="Antwort:", + messages=_CLOZE_MESSAGES, ground_truth=" Im Kühlschrank", - completions=[" Im Kühlschrank", " Auf dem Mond", " In einem Schuh", " Im Vulkan"], + possible_completions=[" Im Kühlschrank", " Auf dem Mond", " In einem Schuh", " Im Vulkan"], ) _CLOZE_HARD = _ExpectedPrompt( - instruction="Frage: Wo bewahrt man frische Milch am besten auf?\n", - cue="Antwort:", + messages=_CLOZE_MESSAGES, ground_truth=" Im Kühlschrank", - completions=[" Im Kühlschrank", " In der Speisekammer", " Auf der Fensterbank", " Im Keller"], + possible_completions=[" Im Kühlschrank", " In der Speisekammer", " Auf der Fensterbank", " Im Keller"], ) _BPB = _ExpectedPrompt( - instruction="Frage: Wo bewahrt man frische Milch am besten auf?\n", - cue="Antwort:", + messages=_CLOZE_MESSAGES, ground_truth=" Im Kühlschrank", - completions=[" Im Kühlschrank"], # BPB scores only the gold continuation + possible_completions=[" Im Kühlschrank"], # BPB scores only the gold continuation +) + + +@pytest.mark.parametrize( + "make_benchmark, expected", + [ + pytest.param(csqa_ellamind_mc_easy_de, _MC_EASY, id="mc_easy"), + pytest.param(csqa_ellamind_mc_hard_de, _MC_HARD, id="mc_hard"), + pytest.param(csqa_ellamind_cloze_easy_de, _CLOZE_EASY, id="cloze_easy"), + pytest.param(csqa_ellamind_cloze_hard_de, _CLOZE_HARD, id="cloze_hard"), + pytest.param(csqa_ellamind_bpb_de, _BPB, id="bpb"), + ], ) +def test_csqa_zeroshot_prompt(make_benchmark: Callable[..., Benchmark], expected: _ExpectedPrompt) -> None: + benchmark = make_benchmark(dataset=DatasetStub({"validation": [_EVAL_ROW]})) + sample = first_sample(benchmark, num_fewshot=0) + assert sample.messages == expected.messages + assert sample.ground_truth == expected.ground_truth + assert sample.possible_completions == expected.possible_completions + + +# --- One-shot: fewshot row rendered with its answer, then the eval row's own zero-shot prompt --- +_MC_EASY_FEWSHOT_MESSAGES = [ + Message( + role=Role.USER, + content="Frage: Womit schreibt man normalerweise auf Papier?\n" + "A. Mit einer Wolke\nB. Mit einem Stift\nC. Mit einem Stein\nD. Mit einer Banane\n", + ), + Message(role=Role.ASSISTANT, content="Antwort: B"), +] +_MC_HARD_FEWSHOT_MESSAGES = [ + Message( + role=Role.USER, + content="Frage: Womit schreibt man normalerweise auf Papier?\n" + "A. Mit Kreide\nB. Mit einem Stift\nC. Mit einer Tastatur\nD. Mit einem Pinsel\n", + ), + Message(role=Role.ASSISTANT, content="Antwort: B"), +] +_CLOZE_FEWSHOT_MESSAGES = [ + Message(role=Role.USER, content="Frage: Womit schreibt man normalerweise auf Papier?\n"), + Message(role=Role.ASSISTANT, content="Antwort: Mit einem Stift"), +] + + +def _oneshot(fewshot_messages: list[Message], eval_expected: _ExpectedPrompt) -> _ExpectedPrompt: + return _ExpectedPrompt( + messages=[*fewshot_messages, *eval_expected.messages], + ground_truth=eval_expected.ground_truth, + possible_completions=eval_expected.possible_completions, + ) @pytest.mark.parametrize( - "reader, styler, expected", + "make_benchmark, expected", [ - pytest.param(CsqaReader("easy"), CSQA_ELLAMIND_MC_STYLER, _MC_EASY, id="mc_easy"), - pytest.param(CsqaReader("hard"), CSQA_ELLAMIND_MC_STYLER, _MC_HARD, id="mc_hard"), - pytest.param(CsqaReader("easy"), CSQA_ELLAMIND_CLOZE_STYLER, _CLOZE_EASY, id="cloze_easy"), - pytest.param(CsqaReader("hard"), CSQA_ELLAMIND_CLOZE_STYLER, _CLOZE_HARD, id="cloze_hard"), - pytest.param(CsqaReader("easy"), CSQA_ELLAMIND_BPB_STYLER, _BPB, id="bpb"), + pytest.param(csqa_ellamind_mc_easy_de, _oneshot(_MC_EASY_FEWSHOT_MESSAGES, _MC_EASY), id="mc_easy"), + pytest.param(csqa_ellamind_mc_hard_de, _oneshot(_MC_HARD_FEWSHOT_MESSAGES, _MC_HARD), id="mc_hard"), + pytest.param(csqa_ellamind_cloze_easy_de, _oneshot(_CLOZE_FEWSHOT_MESSAGES, _CLOZE_EASY), id="cloze_easy"), + pytest.param(csqa_ellamind_cloze_hard_de, _oneshot(_CLOZE_FEWSHOT_MESSAGES, _CLOZE_HARD), id="cloze_hard"), + pytest.param(csqa_ellamind_bpb_de, _oneshot(_CLOZE_FEWSHOT_MESSAGES, _BPB), id="bpb"), ], ) -def test_csqa_prompt_content(reader: CsqaReader, styler: TaskStyler, expected: _ExpectedPrompt) -> None: - fields = reader.read(_EVAL_ROW) - assert styler.get_instruction_text(fields.raw_question, fields.choices) == expected.instruction - assert styler.get_cue_text() == expected.cue - assert styler.get_ground_truth(fields.choices, fields.correct_index) == expected.ground_truth - assert styler.get_possible_completions(fields.choices, fields.correct_index) == expected.completions +def test_csqa_oneshot_prompt(make_benchmark: Callable[..., Benchmark], expected: _ExpectedPrompt) -> None: + benchmark = make_benchmark(dataset=DatasetStub({"validation": [_FEWSHOT_ROW, _EVAL_ROW]})) + sample = first_sample(benchmark, num_fewshot=1) + assert sample.messages == expected.messages + assert sample.ground_truth == expected.ground_truth + assert sample.possible_completions == expected.possible_completions + + +# --------------------------------------------------------------------------- +# Distractor shuffling: deterministic, correct answer preserved, easy/hard draw from different pools +# --------------------------------------------------------------------------- +def test_csqa_shuffling_is_deterministic_and_uses_expected_distractor_set() -> None: + item: dict[str, Any] = { + "question": "Was essen Pandas am liebsten?", + "correct_answer": "Bambus", + "easy_distractors": ["Pizza", "Eis", "Schokolade"], + "hard_distractors": ["Blätter", "Gräser", "Kräuter"], + } + + easy_1 = CsqaReader("easy").read(item) + easy_2 = CsqaReader("easy").read(item) + hard = CsqaReader("hard").read(item) + + # Deterministic for identical input. + assert (easy_1.choices, easy_1.correct_index) == (easy_2.choices, easy_2.correct_index) + # Correct index points to the correct answer. + assert easy_1.choices[easy_1.correct_index] == "Bambus" + assert hard.choices[hard.correct_index] == "Bambus" + # Easy and hard draw from different distractor pools. + assert set(easy_1.choices) == {"Bambus", "Pizza", "Eis", "Schokolade"} + assert set(hard.choices) == {"Bambus", "Blätter", "Gräser", "Kräuter"} diff --git a/tests/tests_eval_framework/benchmarks/test_piqa_ellamind.py b/tests/tests_eval_framework/benchmarks/test_piqa_ellamind.py index 2190b2756..36115da1e 100644 --- a/tests/tests_eval_framework/benchmarks/test_piqa_ellamind.py +++ b/tests/tests_eval_framework/benchmarks/test_piqa_ellamind.py @@ -1,26 +1,36 @@ -"""Tests for the German PIQA (EllaMind) tasks. +"""Specification of the German PIQA (EllaMind) tasks. -- formatter hash test for every PIQA variant -- offline test that the reader (item -> ChoiceFields) and the chosen styler produce the expected - prompt content. Message assembly (roles / fewshot / cue placement) is generic and covered in - ``test_composed_benchmark``. +Each spec test builds the real benchmark (via its ``piqa_ellamind_*_de`` constructor) over a fictional +dataset and asserts the assembled messages, ground truth, and scored completions — so this file reads +as PIQA's prompt spec, with ``composed.py`` an implementation detail. ``test_formatter_hash`` separately +pins the real benchmarks against the real HuggingFace data. """ +from collections.abc import Callable from dataclasses import dataclass from typing import Any import pytest from eval_framework.benchmarks.piqa_ellamind import ( - PIQA_ELLAMIND_BPB_STYLER, - PIQA_ELLAMIND_CLOZE_STYLER, - PIQA_ELLAMIND_MC_STYLER, - PiqaReader, + piqa_ellamind_bpb_de, + piqa_ellamind_cloze_easy_de, + piqa_ellamind_cloze_hard_de, + piqa_ellamind_mc_easy_de, + piqa_ellamind_mc_hard_de, ) +from eval_framework.contract import Benchmark from eval_framework.tasks.registry import Registry from eval_framework.tasks.task_names import register_piqa_ellamind_tasks -from eval_framework.tasks.task_style import TaskStyler -from template_formatting.formatter import BaseFormatter, ConcatFormatter, Llama3Formatter, NoStripConcatFormatter +from template_formatting.formatter import ( + BaseFormatter, + ConcatFormatter, + Llama3Formatter, + Message, + NoStripConcatFormatter, + Role, +) +from tests.tests_eval_framework.benchmarks.utils import DatasetStub, first_sample from tests.tests_eval_framework.tasks.benchmarks.utils import run_formatter_hash_test # Registry for this test suite only holding piqa_ellamind tasks @@ -36,75 +46,142 @@ def test_formatter_hash(task_name: str, formatter_cls: type[BaseFormatter]) -> N # --------------------------------------------------------------------------- -# Offline test: reader + chosen styler produce the expected prompt content (no Eval, no dataset) +# Prompt spec: build the real benchmark over fictional rows, assert the assembled messages # --------------------------------------------------------------------------- -# A fictional row in the PIQA format (NOT a real dataset example). Choices are shuffled -# deterministically (seed: goal + correct_solution), which places the correct answer at index 0. +# Fictional rows in the PIQA format (NOT real dataset examples). Choices are shuffled deterministically +# (seed: goal + correct_solution), which places the correct answer at index 0. _EVAL_ROW: dict[str, Any] = { "goal": "Martin möchte einen Nagel in die Wand schlagen.", "correct_solution": "Er verwendet einen Hammer.", "easy_distractor": "Er verwendet eine Schere.", "hard_distractor": "Er verwendet eine Zange.", } +_FEWSHOT_ROW: dict[str, Any] = { + "goal": "Prabhu will Wasser aufkochen.", + "correct_solution": "Er stellt den Topf auf den Herd.", + "easy_distractor": "Er schreit den Topf an, bis er warm wird.", + "hard_distractor": "Er stellt den Topf über Nacht in den Kühlschrank.", +} + +_QUESTION = "Ziel: Martin möchte einen Nagel in die Wand schlagen." @dataclass(frozen=True) class _ExpectedPrompt: - instruction: str - cue: str + messages: list[Message] ground_truth: str - completions: list[str] + possible_completions: list[str] +# --- Zero-shot: the eval row on its own --- _MC_EASY = _ExpectedPrompt( - instruction="Ziel: Martin möchte einen Nagel in die Wand schlagen.\n" - "A. Er verwendet einen Hammer.\nB. Er verwendet eine Schere.\n", - cue="Antwort:", + messages=[ + Message(role=Role.USER, content=f"{_QUESTION}\nA. Er verwendet einen Hammer.\nB. Er verwendet eine Schere.\n"), + Message(role=Role.ASSISTANT, content="Antwort:"), + ], ground_truth=" A", - completions=[" A", " B"], + possible_completions=[" A", " B"], ) _MC_HARD = _ExpectedPrompt( - instruction="Ziel: Martin möchte einen Nagel in die Wand schlagen.\n" - "A. Er verwendet einen Hammer.\nB. Er verwendet eine Zange.\n", - cue="Antwort:", + messages=[ + Message(role=Role.USER, content=f"{_QUESTION}\nA. Er verwendet einen Hammer.\nB. Er verwendet eine Zange.\n"), + Message(role=Role.ASSISTANT, content="Antwort:"), + ], ground_truth=" A", - completions=[" A", " B"], + possible_completions=[" A", " B"], ) -# Cloze/BPB show no options, so the prompt text is identical; only the scored completions differ. +# Cloze/BPB show no options, so the assembled messages are identical; only the scored completions differ. +_CLOZE_MESSAGES = [ + Message(role=Role.USER, content=f"{_QUESTION}\n"), + Message(role=Role.ASSISTANT, content="Antwort:"), +] _CLOZE_EASY = _ExpectedPrompt( - instruction="Ziel: Martin möchte einen Nagel in die Wand schlagen.\n", - cue="Antwort:", + messages=_CLOZE_MESSAGES, ground_truth=" Er verwendet einen Hammer.", - completions=[" Er verwendet einen Hammer.", " Er verwendet eine Schere."], + possible_completions=[" Er verwendet einen Hammer.", " Er verwendet eine Schere."], ) _CLOZE_HARD = _ExpectedPrompt( - instruction="Ziel: Martin möchte einen Nagel in die Wand schlagen.\n", - cue="Antwort:", + messages=_CLOZE_MESSAGES, ground_truth=" Er verwendet einen Hammer.", - completions=[" Er verwendet einen Hammer.", " Er verwendet eine Zange."], + possible_completions=[" Er verwendet einen Hammer.", " Er verwendet eine Zange."], ) _BPB = _ExpectedPrompt( - instruction="Ziel: Martin möchte einen Nagel in die Wand schlagen.\n", - cue="Antwort:", + messages=_CLOZE_MESSAGES, ground_truth=" Er verwendet einen Hammer.", - completions=[" Er verwendet einen Hammer."], # BPB scores only the gold continuation + possible_completions=[" Er verwendet einen Hammer."], # BPB scores only the gold continuation +) + + +@pytest.mark.parametrize( + "make_benchmark, expected", + [ + pytest.param(piqa_ellamind_mc_easy_de, _MC_EASY, id="mc_easy"), + pytest.param(piqa_ellamind_mc_hard_de, _MC_HARD, id="mc_hard"), + pytest.param(piqa_ellamind_cloze_easy_de, _CLOZE_EASY, id="cloze_easy"), + pytest.param(piqa_ellamind_cloze_hard_de, _CLOZE_HARD, id="cloze_hard"), + pytest.param(piqa_ellamind_bpb_de, _BPB, id="bpb"), + ], ) +def test_piqa_zeroshot_prompt(make_benchmark: Callable[..., Benchmark], expected: _ExpectedPrompt) -> None: + # Given the real PIQA benchmark over a single fictional row + benchmark = make_benchmark(dataset=DatasetStub({"validation": [_EVAL_ROW]})) + # When we assemble its first sample (zero-shot) + sample = first_sample(benchmark, num_fewshot=0) + # Then the messages, ground truth, and scored completions are exactly: + assert sample.messages == expected.messages + assert sample.ground_truth == expected.ground_truth + assert sample.possible_completions == expected.possible_completions + + +# --- One-shot: the fewshot row is rendered with its answer, then the eval row (its own zero-shot prompt) --- +_MC_EASY_FEWSHOT_MESSAGES = [ + Message( + role=Role.USER, + content="Ziel: Prabhu will Wasser aufkochen.\n" + "A. Er stellt den Topf auf den Herd.\nB. Er schreit den Topf an, bis er warm wird.\n", + ), + Message(role=Role.ASSISTANT, content="Antwort: A"), +] +_MC_HARD_FEWSHOT_MESSAGES = [ + Message( + role=Role.USER, + content="Ziel: Prabhu will Wasser aufkochen.\n" + "A. Er stellt den Topf auf den Herd.\nB. Er stellt den Topf über Nacht in den Kühlschrank.\n", + ), + Message(role=Role.ASSISTANT, content="Antwort: A"), +] +_CLOZE_FEWSHOT_MESSAGES = [ + Message(role=Role.USER, content="Ziel: Prabhu will Wasser aufkochen.\n"), + Message(role=Role.ASSISTANT, content="Antwort: Er stellt den Topf auf den Herd."), +] + + +def _oneshot(fewshot_messages: list[Message], eval_expected: _ExpectedPrompt) -> _ExpectedPrompt: + return _ExpectedPrompt( + messages=[*fewshot_messages, *eval_expected.messages], + ground_truth=eval_expected.ground_truth, + possible_completions=eval_expected.possible_completions, + ) @pytest.mark.parametrize( - "reader, styler, expected", + "make_benchmark, expected", [ - pytest.param(PiqaReader("easy"), PIQA_ELLAMIND_MC_STYLER, _MC_EASY, id="mc_easy"), - pytest.param(PiqaReader("hard"), PIQA_ELLAMIND_MC_STYLER, _MC_HARD, id="mc_hard"), - pytest.param(PiqaReader("easy"), PIQA_ELLAMIND_CLOZE_STYLER, _CLOZE_EASY, id="cloze_easy"), - pytest.param(PiqaReader("hard"), PIQA_ELLAMIND_CLOZE_STYLER, _CLOZE_HARD, id="cloze_hard"), - pytest.param(PiqaReader("easy"), PIQA_ELLAMIND_BPB_STYLER, _BPB, id="bpb"), + pytest.param(piqa_ellamind_mc_easy_de, _oneshot(_MC_EASY_FEWSHOT_MESSAGES, _MC_EASY), id="mc_easy"), + pytest.param(piqa_ellamind_mc_hard_de, _oneshot(_MC_HARD_FEWSHOT_MESSAGES, _MC_HARD), id="mc_hard"), + pytest.param(piqa_ellamind_cloze_easy_de, _oneshot(_CLOZE_FEWSHOT_MESSAGES, _CLOZE_EASY), id="cloze_easy"), + pytest.param(piqa_ellamind_cloze_hard_de, _oneshot(_CLOZE_FEWSHOT_MESSAGES, _CLOZE_HARD), id="cloze_hard"), + pytest.param(piqa_ellamind_bpb_de, _oneshot(_CLOZE_FEWSHOT_MESSAGES, _BPB), id="bpb"), ], ) -def test_piqa_prompt_content(reader: PiqaReader, styler: TaskStyler, expected: _ExpectedPrompt) -> None: - fields = reader.read(_EVAL_ROW) - assert styler.get_instruction_text(fields.raw_question, fields.choices) == expected.instruction - assert styler.get_cue_text() == expected.cue - assert styler.get_ground_truth(fields.choices, fields.correct_index) == expected.ground_truth - assert styler.get_possible_completions(fields.choices, fields.correct_index) == expected.completions +def test_piqa_oneshot_prompt(make_benchmark: Callable[..., Benchmark], expected: _ExpectedPrompt) -> None: + # Given the real PIQA benchmark over a fewshot row followed by the eval row (fewshot row first so the + # seed-42 shuffle puts the eval row first, making it the sample and the other the fewshot example) + benchmark = make_benchmark(dataset=DatasetStub({"validation": [_FEWSHOT_ROW, _EVAL_ROW]})) + # When we assemble its first sample (one-shot) + sample = first_sample(benchmark, num_fewshot=1) + # Then the fewshot example precedes the eval prompt, and ground truth / completions are the eval row's: + assert sample.messages == expected.messages + assert sample.ground_truth == expected.ground_truth + assert sample.possible_completions == expected.possible_completions diff --git a/tests/tests_eval_framework/benchmarks/test_siqa_ellamind.py b/tests/tests_eval_framework/benchmarks/test_siqa_ellamind.py index 5f9a9c4da..62072481e 100644 --- a/tests/tests_eval_framework/benchmarks/test_siqa_ellamind.py +++ b/tests/tests_eval_framework/benchmarks/test_siqa_ellamind.py @@ -1,26 +1,30 @@ -"""Tests for the German Social IQa (EllaMind) tasks. +"""Specification of the German Social IQa (EllaMind) tasks. -- formatter hash test for every SIQA variant -- offline test that the reader (item -> ChoiceFields) and the chosen styler produce the expected - prompt content. Message assembly (roles / fewshot / cue placement) is generic and covered in - ``test_composed_benchmark``. +Each spec test builds the real benchmark (via its ``siqa_ellamind_*_de`` constructor) over a fictional +dataset and asserts the assembled messages, ground truth, and scored completions — so this file reads +as SIQA's prompt spec, with ``composed.py`` an implementation detail. ``test_formatter_hash`` separately +pins the real benchmarks against the real HuggingFace data. The shown question is the context followed +by the question. """ +from collections.abc import Callable from dataclasses import dataclass from typing import Any import pytest from eval_framework.benchmarks.siqa_ellamind import ( - SIQA_ELLAMIND_BPB_STYLER, - SIQA_ELLAMIND_CLOZE_STYLER, - SIQA_ELLAMIND_MC_STYLER, - SiqaReader, + siqa_ellamind_bpb_de, + siqa_ellamind_cloze_easy_de, + siqa_ellamind_cloze_hard_de, + siqa_ellamind_mc_easy_de, + siqa_ellamind_mc_hard_de, ) +from eval_framework.contract import Benchmark from eval_framework.tasks.registry import Registry from eval_framework.tasks.task_names import register_siqa_ellamind_tasks -from eval_framework.tasks.task_style import TaskStyler -from template_formatting.formatter import BaseFormatter, ConcatFormatter, Llama3Formatter +from template_formatting.formatter import BaseFormatter, ConcatFormatter, Llama3Formatter, Message, Role +from tests.tests_eval_framework.benchmarks.utils import DatasetStub, first_sample from tests.tests_eval_framework.tasks.benchmarks.utils import run_formatter_hash_test # Registry for this test suite only holding siqa_ellamind tasks @@ -36,76 +40,136 @@ def test_formatter_hash(task_name: str, formatter_cls: type[BaseFormatter]) -> N # --------------------------------------------------------------------------- -# Offline test: reader + chosen styler produce the expected prompt content (no Eval, no dataset) +# Prompt spec: build the real benchmark over fictional rows, assert the assembled messages # --------------------------------------------------------------------------- -# A fictional row in the SIQA format (NOT a real dataset example). The shown question is the context -# followed by the question; choices are shuffled deterministically (seed: question + correct_answer). +# Fictional rows in the SIQA format (NOT real dataset examples). Choices are shuffled deterministically +# (seed: question + correct_answer). _EVAL_ROW: dict[str, Any] = { - "context": "Alex hat den ganzen Tag im Garten gearbeitet.", - "question": "Wie fühlt sich Alex danach?", - "correct_answer": "Erschöpft", - "easy_distractors": ["Gelangweilt", "Neugierig", "Hungrig"], - "hard_distractors": ["Zufrieden", "Entspannt", "Stolz"], + "context": "Max kommt nach Hause und findet sein Zimmer aufgeräumt.", + "question": "Was hat Max als nächstes getan?", + "correct_answer": "Er hat sich bedankt.", + "easy_distractors": ["Er ist wütend geworden.", "Er ist gegangen."], + "hard_distractors": ["Er hat gegessen.", "Er hat geschlafen."], +} +_FEWSHOT_ROW: dict[str, Any] = { + "context": "Lisa hat ihr Buch vergessen.", + "question": "Wie hat Lisa sich gefühlt?", + "correct_answer": "Ärgerlich.", + "easy_distractors": ["Glücklich.", "Müde."], + "hard_distractors": ["Neugierig.", "Aufgeregt."], } -_QUESTION = "Frage: Alex hat den ganzen Tag im Garten gearbeitet. Wie fühlt sich Alex danach?" +# The shown question is "Frage: " + context + " " + question. +_EVAL_Q = "Frage: Max kommt nach Hause und findet sein Zimmer aufgeräumt. Was hat Max als nächstes getan?" +_FEWSHOT_Q = "Frage: Lisa hat ihr Buch vergessen. Wie hat Lisa sich gefühlt?" @dataclass(frozen=True) class _ExpectedPrompt: - instruction: str - cue: str + messages: list[Message] ground_truth: str - completions: list[str] + possible_completions: list[str] +# --- Zero-shot --- _MC_EASY = _ExpectedPrompt( - instruction=f"{_QUESTION}\nA. Hungrig\nB. Neugierig\nC. Gelangweilt\nD. Erschöpft\n", - cue="Antwort:", - ground_truth=" D", - completions=[" A", " B", " C", " D"], + messages=[ + Message( + role=Role.USER, + content=f"{_EVAL_Q}\nA. Er ist wütend geworden.\nB. Er hat sich bedankt.\nC. Er ist gegangen.\n", + ), + Message(role=Role.ASSISTANT, content="Antwort:"), + ], + ground_truth=" B", + possible_completions=[" A", " B", " C"], ) _MC_HARD = _ExpectedPrompt( - instruction=f"{_QUESTION}\nA. Stolz\nB. Entspannt\nC. Zufrieden\nD. Erschöpft\n", - cue="Antwort:", - ground_truth=" D", - completions=[" A", " B", " C", " D"], + messages=[ + Message( + role=Role.USER, content=f"{_EVAL_Q}\nA. Er hat gegessen.\nB. Er hat sich bedankt.\nC. Er hat geschlafen.\n" + ), + Message(role=Role.ASSISTANT, content="Antwort:"), + ], + ground_truth=" B", + possible_completions=[" A", " B", " C"], ) -# Cloze/BPB show no options, so the prompt text is identical; only the scored completions differ. +# Cloze/BPB show no options, so the assembled messages are identical; only the scored completions differ. +_CLOZE_MESSAGES = [ + Message(role=Role.USER, content=f"{_EVAL_Q}\n"), + Message(role=Role.ASSISTANT, content="Antwort:"), +] _CLOZE_EASY = _ExpectedPrompt( - instruction=f"{_QUESTION}\n", - cue="Antwort:", - ground_truth=" Erschöpft", - completions=[" Hungrig", " Neugierig", " Gelangweilt", " Erschöpft"], + messages=_CLOZE_MESSAGES, + ground_truth=" Er hat sich bedankt.", + possible_completions=[" Er ist wütend geworden.", " Er hat sich bedankt.", " Er ist gegangen."], ) _CLOZE_HARD = _ExpectedPrompt( - instruction=f"{_QUESTION}\n", - cue="Antwort:", - ground_truth=" Erschöpft", - completions=[" Stolz", " Entspannt", " Zufrieden", " Erschöpft"], + messages=_CLOZE_MESSAGES, + ground_truth=" Er hat sich bedankt.", + possible_completions=[" Er hat gegessen.", " Er hat sich bedankt.", " Er hat geschlafen."], ) _BPB = _ExpectedPrompt( - instruction=f"{_QUESTION}\n", - cue="Antwort:", - ground_truth=" Erschöpft", - completions=[" Erschöpft"], # BPB scores only the gold continuation + messages=_CLOZE_MESSAGES, + ground_truth=" Er hat sich bedankt.", + possible_completions=[" Er hat sich bedankt."], # BPB scores only the gold continuation +) + + +@pytest.mark.parametrize( + "make_benchmark, expected", + [ + pytest.param(siqa_ellamind_mc_easy_de, _MC_EASY, id="mc_easy"), + pytest.param(siqa_ellamind_mc_hard_de, _MC_HARD, id="mc_hard"), + pytest.param(siqa_ellamind_cloze_easy_de, _CLOZE_EASY, id="cloze_easy"), + pytest.param(siqa_ellamind_cloze_hard_de, _CLOZE_HARD, id="cloze_hard"), + pytest.param(siqa_ellamind_bpb_de, _BPB, id="bpb"), + ], ) +def test_siqa_zeroshot_prompt(make_benchmark: Callable[..., Benchmark], expected: _ExpectedPrompt) -> None: + benchmark = make_benchmark(dataset=DatasetStub({"validation": [_EVAL_ROW]})) + sample = first_sample(benchmark, num_fewshot=0) + assert sample.messages == expected.messages + assert sample.ground_truth == expected.ground_truth + assert sample.possible_completions == expected.possible_completions + + +# --- One-shot: fewshot row rendered with its answer, then the eval row's own zero-shot prompt --- +_MC_EASY_FEWSHOT_MESSAGES = [ + Message(role=Role.USER, content=f"{_FEWSHOT_Q}\nA. Ärgerlich.\nB. Müde.\nC. Glücklich.\n"), + Message(role=Role.ASSISTANT, content="Antwort: A"), +] +_MC_HARD_FEWSHOT_MESSAGES = [ + Message(role=Role.USER, content=f"{_FEWSHOT_Q}\nA. Ärgerlich.\nB. Aufgeregt.\nC. Neugierig.\n"), + Message(role=Role.ASSISTANT, content="Antwort: A"), +] +_CLOZE_FEWSHOT_MESSAGES = [ + Message(role=Role.USER, content=f"{_FEWSHOT_Q}\n"), + Message(role=Role.ASSISTANT, content="Antwort: Ärgerlich."), +] + + +def _oneshot(fewshot_messages: list[Message], eval_expected: _ExpectedPrompt) -> _ExpectedPrompt: + return _ExpectedPrompt( + messages=[*fewshot_messages, *eval_expected.messages], + ground_truth=eval_expected.ground_truth, + possible_completions=eval_expected.possible_completions, + ) @pytest.mark.parametrize( - "reader, styler, expected", + "make_benchmark, expected", [ - pytest.param(SiqaReader("easy"), SIQA_ELLAMIND_MC_STYLER, _MC_EASY, id="mc_easy"), - pytest.param(SiqaReader("hard"), SIQA_ELLAMIND_MC_STYLER, _MC_HARD, id="mc_hard"), - pytest.param(SiqaReader("easy"), SIQA_ELLAMIND_CLOZE_STYLER, _CLOZE_EASY, id="cloze_easy"), - pytest.param(SiqaReader("hard"), SIQA_ELLAMIND_CLOZE_STYLER, _CLOZE_HARD, id="cloze_hard"), - pytest.param(SiqaReader("easy"), SIQA_ELLAMIND_BPB_STYLER, _BPB, id="bpb"), + pytest.param(siqa_ellamind_mc_easy_de, _oneshot(_MC_EASY_FEWSHOT_MESSAGES, _MC_EASY), id="mc_easy"), + pytest.param(siqa_ellamind_mc_hard_de, _oneshot(_MC_HARD_FEWSHOT_MESSAGES, _MC_HARD), id="mc_hard"), + pytest.param(siqa_ellamind_cloze_easy_de, _oneshot(_CLOZE_FEWSHOT_MESSAGES, _CLOZE_EASY), id="cloze_easy"), + pytest.param(siqa_ellamind_cloze_hard_de, _oneshot(_CLOZE_FEWSHOT_MESSAGES, _CLOZE_HARD), id="cloze_hard"), + pytest.param(siqa_ellamind_bpb_de, _oneshot(_CLOZE_FEWSHOT_MESSAGES, _BPB), id="bpb"), ], ) -def test_siqa_prompt_content(reader: SiqaReader, styler: TaskStyler, expected: _ExpectedPrompt) -> None: - fields = reader.read(_EVAL_ROW) - assert styler.get_instruction_text(fields.raw_question, fields.choices) == expected.instruction - assert styler.get_cue_text() == expected.cue - assert styler.get_ground_truth(fields.choices, fields.correct_index) == expected.ground_truth - assert styler.get_possible_completions(fields.choices, fields.correct_index) == expected.completions +def test_siqa_oneshot_prompt(make_benchmark: Callable[..., Benchmark], expected: _ExpectedPrompt) -> None: + benchmark = make_benchmark(dataset=DatasetStub({"validation": [_FEWSHOT_ROW, _EVAL_ROW]})) + sample = first_sample(benchmark, num_fewshot=1) + assert sample.messages == expected.messages + assert sample.ground_truth == expected.ground_truth + assert sample.possible_completions == expected.possible_completions diff --git a/tests/tests_eval_framework/benchmarks/utils.py b/tests/tests_eval_framework/benchmarks/utils.py new file mode 100644 index 000000000..3249cab95 --- /dev/null +++ b/tests/tests_eval_framework/benchmarks/utils.py @@ -0,0 +1,45 @@ +"""Test helpers for specifying composed benchmarks offline. + +A benchmark's dataset is an injected policy, so a test can build the real benchmark over a fictional +in-memory dataset and assert its assembled messages — no download, and ``composed.py`` stays an +implementation detail. +""" + +from typing import Any, final, override + +from datasets import Dataset, DatasetDict + +from eval_framework.contract import Benchmark, Sample +from eval_framework.tasks.dataset_loading import DatasetLoader, DatasetPolicy + + +@final +class DatasetStub(DatasetPolicy, DatasetLoader): + """A fictional in-memory dataset, injected in place of a benchmark's pinned Hugging Face policy.""" + + def __init__(self, splits: dict[str, list[dict[str, Any]]]) -> None: + self._splits = splits + + @override + def loader(self, custom_hf_revision: str | None) -> DatasetLoader: + return self + + @override + def documentation(self) -> str: + return "fictional in-memory dataset" + + @override + def load(self, name: str | None) -> DatasetDict: + return DatasetDict({split: Dataset.from_list(rows) for split, rows in self._splits.items()}) + + @override + def metadata(self) -> dict[str, str]: + return {"dataset_path": "stub"} + + +def first_sample(benchmark: Benchmark, *, num_fewshot: int, custom_subjects: list[str] | None = None) -> Sample: + """Build the benchmark's eval (seed 42) and return its first assembled sample.""" + evaluation = benchmark.create( + num_fewshot=num_fewshot, custom_subjects=custom_subjects, custom_hf_revision=None, seed=42 + ) + return next(iter(evaluation.iterate_samples(1))) From a81a8a4ee16bb2c3a0ec8396ddf557c84d95dcb5 Mon Sep 17 00:00:00 2001 From: Markus Klein Date: Thu, 27 Aug 2026 23:06:44 +0200 Subject: [PATCH 2/6] refactor: gpqa_ellamind converted to composed implementation --- .../benchmarks/gpqa_ellamind.py | 123 ++++++++ .../tasks/benchmarks/gpqa_ellamind.py | 114 ------- src/eval_framework/tasks/task_names.py | 11 +- .../benchmarks/test_gpqa_ellamind.py | 176 +++++++++++ .../tasks/benchmarks/task-prompts-hashes.json | 15 +- .../tasks/benchmarks/test_gpqa_ellamind.py | 291 ------------------ 6 files changed, 308 insertions(+), 422 deletions(-) create mode 100644 src/eval_framework/benchmarks/gpqa_ellamind.py delete mode 100644 src/eval_framework/tasks/benchmarks/gpqa_ellamind.py create mode 100644 tests/tests_eval_framework/benchmarks/test_gpqa_ellamind.py delete mode 100644 tests/tests_eval_framework/tasks/benchmarks/test_gpqa_ellamind.py diff --git a/src/eval_framework/benchmarks/gpqa_ellamind.py b/src/eval_framework/benchmarks/gpqa_ellamind.py new file mode 100644 index 000000000..1f1633d54 --- /dev/null +++ b/src/eval_framework/benchmarks/gpqa_ellamind.py @@ -0,0 +1,123 @@ +"""German GPQA (Graduate-level Professional QA, EllaMind) tasks. + +https://huggingface.co/datasets/ellamind/gpqa-multilingual + +GPQA uses a single distractor set (``incorrect_answers``). The diamond variants restrict evaluation to +the diamond subset — the 198 hardest questions (``is_diamond``) from the original GPQA-Diamond benchmark. +""" + +from typing import Any, final, override + +from datasets import DatasetDict + +from eval_framework.choices import ChoiceFields, ChoiceReader +from eval_framework.composed import ComposedBenchmark +from eval_framework.contract import Benchmark +from eval_framework.subjects import ListOfSubjects +from eval_framework.tasks.base import Language +from eval_framework.tasks.dataset_loading import DatasetLoader, DatasetPolicy +from eval_framework.tasks.dataset_revisions import pinned_by_framework +from eval_framework.tasks.task_style import BPBStyle, ClozeStyle, MCStyle, TaskStyler, shuffle_correct_with_distractors + + +@final +class GpqaReader(ChoiceReader): + """Reads a GPQA item: a single ``incorrect_answers`` distractor set, shuffled in with the correct answer.""" + + @override + def read(self, item: dict[str, Any]) -> ChoiceFields: + choices, correct_index = shuffle_correct_with_distractors( + correct=item["correct_answer"], + distractors=item["incorrect_answers"], + seed_text=item["question"] + item["correct_answer"], + ) + return ChoiceFields(raw_question=item["question"], choices=choices, correct_index=correct_index) + + +@final +class _DiamondFilteredLoader(DatasetLoader): + """Restricts a loader's every split to the diamond subset (``is_diamond``).""" + + def __init__(self, inner: DatasetLoader) -> None: + self._inner = inner + + @override + def load(self, name: str | None) -> DatasetDict: + loaded = self._inner.load(name) + return DatasetDict({split: data.filter(lambda row: row["is_diamond"]) for split, data in loaded.items()}) + + @override + def metadata(self) -> dict[str, str]: + return self._inner.metadata() + + +@final +class _DiamondOnly(DatasetPolicy): + """Wraps a dataset policy to serve only the diamond subset — the 198 hardest GPQA questions.""" + + def __init__(self, inner: DatasetPolicy) -> None: + self._inner = inner + + @override + def loader(self, custom_hf_revision: str | None) -> DatasetLoader: + return _DiamondFilteredLoader(self._inner.loader(custom_hf_revision)) + + @override + def documentation(self) -> str: + return self._inner.documentation() + + +def _gpqa_ellamind_benchmark(id: str, styler: TaskStyler, dataset: DatasetPolicy | None) -> Benchmark: + return ComposedBenchmark.compose( + id=id, + styler=styler, + reader=GpqaReader(), + sample_split="train", + fewshot_split="train", + subjects=ListOfSubjects(["deu"]), + dataset_policy=dataset if dataset is not None else pinned_by_framework("ellamind/gpqa-multilingual"), + language=Language.DEU, + ) + + +def _gpqa_ellamind_diamond_benchmark(id: str, styler: TaskStyler, dataset: DatasetPolicy | None) -> Benchmark: + source = dataset if dataset is not None else pinned_by_framework("ellamind/gpqa-multilingual") + return _gpqa_ellamind_benchmark(id, styler, _DiamondOnly(source)) + + +def gpqa_ellamind_mc_de(dataset: DatasetPolicy | None = None) -> Benchmark: + return _gpqa_ellamind_benchmark("GPQA_ELLAMIND_MC_DE", MCStyle.for_language(Language.DEU), dataset) + + +def gpqa_ellamind_cloze_de(dataset: DatasetPolicy | None = None) -> Benchmark: + return _gpqa_ellamind_benchmark("GPQA_ELLAMIND_CLOZE_DE", ClozeStyle.for_language(Language.DEU), dataset) + + +def gpqa_ellamind_bpb_de(dataset: DatasetPolicy | None = None) -> Benchmark: + return _gpqa_ellamind_benchmark("GPQA_ELLAMIND_BPB_DE", BPBStyle.for_language(Language.DEU), dataset) + + +def gpqa_ellamind_diamond_mc_de(dataset: DatasetPolicy | None = None) -> Benchmark: + return _gpqa_ellamind_diamond_benchmark("GPQA_ELLAMIND_DIAMOND_MC_DE", MCStyle.for_language(Language.DEU), dataset) + + +def gpqa_ellamind_diamond_cloze_de(dataset: DatasetPolicy | None = None) -> Benchmark: + return _gpqa_ellamind_diamond_benchmark( + "GPQA_ELLAMIND_DIAMOND_CLOZE_DE", ClozeStyle.for_language(Language.DEU), dataset + ) + + +def gpqa_ellamind_diamond_bpb_de(dataset: DatasetPolicy | None = None) -> Benchmark: + return _gpqa_ellamind_diamond_benchmark( + "GPQA_ELLAMIND_DIAMOND_BPB_DE", BPBStyle.for_language(Language.DEU), dataset + ) + + +GPQA_ELLAMIND_BENCHMARKS: list[Benchmark] = [ + gpqa_ellamind_mc_de(), + gpqa_ellamind_cloze_de(), + gpqa_ellamind_diamond_mc_de(), + gpqa_ellamind_diamond_cloze_de(), + gpqa_ellamind_bpb_de(), + gpqa_ellamind_diamond_bpb_de(), +] diff --git a/src/eval_framework/tasks/benchmarks/gpqa_ellamind.py b/src/eval_framework/tasks/benchmarks/gpqa_ellamind.py deleted file mode 100644 index f1b3309fb..000000000 --- a/src/eval_framework/tasks/benchmarks/gpqa_ellamind.py +++ /dev/null @@ -1,114 +0,0 @@ -"""German GPQA (Graduate-level Professional QA, EllaMind) tasks. - -https://huggingface.co/datasets/ellamind/gpqa-multilingual - -GPQA uses a single distractor set (``incorrect_answers``). Its diamond subset (the 198 -hardest questions) is exposed via ``_DIAMOND_ONLY = True`` on the subclass. -""" - -from typing import Any - -from eval_framework.tasks.base import BaseTask, Language -from eval_framework.tasks.dataset_revisions import HF_REVISIONS_LOCKFILE -from eval_framework.tasks.task_style import BPBStyle, ClozeStyle, MCStyle, shuffle_correct_with_distractors - - -class _GPQA_ELLAMIND_DE_Base(BaseTask[str]): - """Non-registered base for German GPQA (EllaMind) variants. - - Dataset: https://huggingface.co/datasets/ellamind/gpqa-multilingual - - The diamond subset (``is_diamond=True``, 198 of 448 items) is the hardest - questions from the original GPQA-Diamond benchmark. Set ``_DIAMOND_ONLY = - True`` on a subclass to restrict evaluation to that subset. - - Note, we don't use `domain` or `subdomain` fields at the moment (they are not translated). - Also, we don't use an initial system prompt, different to the base GPQA task. - """ - - DATASET_PATH = "ellamind/gpqa-multilingual" - SAMPLE_SPLIT = "train" - FEWSHOT_SPLIT = "train" - SUBJECTS = ["deu"] - LANGUAGE = Language.DEU - _DIAMOND_ONLY: bool = False - - def _load_dataset(self, subject: str) -> None: - super()._load_dataset(subject) - if self._DIAMOND_ONLY: - self.dataset = { - split: [item for item in items if item["is_diamond"]] for split, items in self.dataset.items() - } - - def _shuffled(self, item: dict[str, Any]) -> tuple[list[str], int]: - return shuffle_correct_with_distractors( - correct=item["correct_answer"], - distractors=item["incorrect_answers"], - seed_text=item["question"] + item["correct_answer"], - ) - - def _get_raw_question(self, item: dict[str, Any]) -> str: - return item["question"] - - def _get_choices(self, item: dict[str, Any]) -> list[str]: - return self._shuffled(item)[0] - - def _get_correct_index(self, item: dict[str, Any]) -> int: - return self._shuffled(item)[1] - - -class GPQA_ELLAMIND_MC_DE(_GPQA_ELLAMIND_DE_Base): - """German GPQA - MC format (all 448 items).""" - - REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE - - NAME = "GPQA_ELLAMIND_MC_DE" - TASK_STYLER = MCStyle().for_language(Language.DEU) - - -class GPQA_ELLAMIND_CLOZE_DE(_GPQA_ELLAMIND_DE_Base): - """German GPQA - Cloze format (all 448 items).""" - - REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE - - NAME = "GPQA_ELLAMIND_CLOZE_DE" - TASK_STYLER = ClozeStyle().for_language(Language.DEU) - - -class GPQA_ELLAMIND_DIAMOND_MC_DE(_GPQA_ELLAMIND_DE_Base): - """German GPQA - MC format, diamond subset (198 hardest items).""" - - REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE - - NAME = "GPQA_ELLAMIND_DIAMOND_MC_DE" - _DIAMOND_ONLY = True - TASK_STYLER = MCStyle().for_language(Language.DEU) - - -class GPQA_ELLAMIND_DIAMOND_CLOZE_DE(_GPQA_ELLAMIND_DE_Base): - """German GPQA - Cloze format, diamond subset (198 hardest items).""" - - REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE - - NAME = "GPQA_ELLAMIND_DIAMOND_CLOZE_DE" - _DIAMOND_ONLY = True - TASK_STYLER = ClozeStyle().for_language(Language.DEU) - - -class GPQA_ELLAMIND_BPB_DE(_GPQA_ELLAMIND_DE_Base): - """German GPQA - BPB format (all 448 items).""" - - REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE - - NAME = "GPQA_ELLAMIND_BPB_DE" - TASK_STYLER = BPBStyle().for_language(Language.DEU) - - -class GPQA_ELLAMIND_DIAMOND_BPB_DE(_GPQA_ELLAMIND_DE_Base): - """German GPQA - BPB format, diamond subset (198 hardest items).""" - - REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE - - NAME = "GPQA_ELLAMIND_DIAMOND_BPB_DE" - _DIAMOND_ONLY = True - TASK_STYLER = BPBStyle().for_language(Language.DEU) diff --git a/src/eval_framework/tasks/task_names.py b/src/eval_framework/tasks/task_names.py index 40dc5fdf1..482cdc974 100644 --- a/src/eval_framework/tasks/task_names.py +++ b/src/eval_framework/tasks/task_names.py @@ -2,6 +2,7 @@ from eval_framework.benchmarks.arc_de import ARC_DE_BENCHMARK from eval_framework.benchmarks.csqa_ellamind import CSQA_ELLAMIND_BENCHMARKS +from eval_framework.benchmarks.gpqa_ellamind import GPQA_ELLAMIND_BENCHMARKS from eval_framework.benchmarks.piqa_ellamind import PIQA_ELLAMIND_BENCHMARKS from eval_framework.benchmarks.siqa_ellamind import SIQA_ELLAMIND_BENCHMARKS from eval_framework.tasks.base import BaseTask @@ -247,14 +248,8 @@ def register_csqa_ellamind_tasks(registry: Registry) -> None: def register_gpqa_ellamind_tasks(registry: Registry) -> None: """Register gpqa_ellamind benchmark tasks.""" - register_lazy_task("eval_framework.tasks.benchmarks.gpqa_ellamind.GPQA_ELLAMIND_MC_DE", registry=registry) - register_lazy_task("eval_framework.tasks.benchmarks.gpqa_ellamind.GPQA_ELLAMIND_CLOZE_DE", registry=registry) - register_lazy_task("eval_framework.tasks.benchmarks.gpqa_ellamind.GPQA_ELLAMIND_DIAMOND_MC_DE", registry=registry) - register_lazy_task( - "eval_framework.tasks.benchmarks.gpqa_ellamind.GPQA_ELLAMIND_DIAMOND_CLOZE_DE", registry=registry - ) - register_lazy_task("eval_framework.tasks.benchmarks.gpqa_ellamind.GPQA_ELLAMIND_BPB_DE", registry=registry) - register_lazy_task("eval_framework.tasks.benchmarks.gpqa_ellamind.GPQA_ELLAMIND_DIAMOND_BPB_DE", registry=registry) + for benchmark in GPQA_ELLAMIND_BENCHMARKS: + registry.add(benchmark) def register_gsm8k_ellamind_tasks(registry: Registry) -> None: diff --git a/tests/tests_eval_framework/benchmarks/test_gpqa_ellamind.py b/tests/tests_eval_framework/benchmarks/test_gpqa_ellamind.py new file mode 100644 index 000000000..590b8b9ef --- /dev/null +++ b/tests/tests_eval_framework/benchmarks/test_gpqa_ellamind.py @@ -0,0 +1,176 @@ +"""Specification of the German GPQA (EllaMind) tasks. + +Each spec test builds the real benchmark (via its ``gpqa_ellamind_*_de`` constructor) over a fictional +dataset and asserts the assembled messages, ground truth, and scored completions. This is an Open +Source codebase, and the actual dataset should not leak in order to prevent it to become part of the +training data itself. Therfore it is important for the data to be fictional. +""" + +from collections.abc import Callable +from dataclasses import dataclass +from typing import Any + +import pytest + +from eval_framework.benchmarks.gpqa_ellamind import ( + gpqa_ellamind_bpb_de, + gpqa_ellamind_cloze_de, + gpqa_ellamind_diamond_bpb_de, + gpqa_ellamind_diamond_cloze_de, + gpqa_ellamind_diamond_mc_de, + gpqa_ellamind_mc_de, +) +from eval_framework.contract import Benchmark +from eval_framework.tasks.registry import Registry +from eval_framework.tasks.task_names import register_gpqa_ellamind_tasks +from template_formatting.formatter import BaseFormatter, ConcatFormatter, Llama3Formatter, Message, Role +from tests.tests_eval_framework.benchmarks.utils import DatasetStub, first_sample +from tests.tests_eval_framework.tasks.benchmarks.utils import run_formatter_hash_test + +# Registry for this test suite only holding gpqa_ellamind tasks +_gpqa_ellamind_registry = Registry() +register_gpqa_ellamind_tasks(registry=_gpqa_ellamind_registry) + + +@pytest.mark.formatter_hash +@pytest.mark.parametrize("formatter_cls", [Llama3Formatter, ConcatFormatter]) +@pytest.mark.parametrize("task_name", _gpqa_ellamind_registry.task_names()) +def test_formatter_hash(task_name: str, formatter_cls: type[BaseFormatter]) -> None: + run_formatter_hash_test(task_name, formatter_cls, registry=_gpqa_ellamind_registry) + + +# --------------------------------------------------------------------------- +# Prompt spec: build the real benchmark over fictional rows, assert the assembled messages +# --------------------------------------------------------------------------- + +# Fictional rows in the GPQA format (NOT real dataset examples). Choices are shuffled deterministically +# (seed: question + correct_answer). Both are diamond rows, so they survive the diamond variants' filter +# and render identically to the full variants. +_EVAL_ROW: dict[str, Any] = { + "question": "Was ist die SI-Einheit des elektrischen Widerstands?", + "correct_answer": "Ohm", + "incorrect_answers": ["Volt", "Ampere", "Watt"], + "is_diamond": True, +} +_FEWSHOT_ROW: dict[str, Any] = { + "question": "Was ist die SI-Einheit der Temperatur?", + "correct_answer": "Kelvin", + "incorrect_answers": ["Celsius", "Fahrenheit", "Joule"], + "is_diamond": True, +} + +_EVAL_Q = "Frage: Was ist die SI-Einheit des elektrischen Widerstands?" +_FEWSHOT_Q = "Frage: Was ist die SI-Einheit der Temperatur?" + + +@dataclass(frozen=True) +class _ExpectedPrompt: + messages: list[Message] + ground_truth: str + possible_completions: list[str] + + +# --- Zero-shot --- +_MC = _ExpectedPrompt( + messages=[ + Message(role=Role.USER, content=f"{_EVAL_Q}\nA. Ohm\nB. Volt\nC. Watt\nD. Ampere\n"), + Message(role=Role.ASSISTANT, content="Antwort:"), + ], + ground_truth=" A", + possible_completions=[" A", " B", " C", " D"], +) +# Cloze/BPB show no options, so the assembled messages are identical; only the scored completions differ. +_CLOZE_MESSAGES = [ + Message(role=Role.USER, content=f"{_EVAL_Q}\n"), + Message(role=Role.ASSISTANT, content="Antwort:"), +] +_CLOZE = _ExpectedPrompt( + messages=_CLOZE_MESSAGES, + ground_truth=" Ohm", + possible_completions=[" Ohm", " Volt", " Watt", " Ampere"], +) +_BPB = _ExpectedPrompt( + messages=_CLOZE_MESSAGES, + ground_truth=" Ohm", + possible_completions=[" Ohm"], # BPB scores only the gold continuation +) + + +@pytest.mark.parametrize( + "make_benchmark, expected", + [ + pytest.param(gpqa_ellamind_mc_de, _MC, id="mc"), + pytest.param(gpqa_ellamind_diamond_mc_de, _MC, id="diamond_mc"), + pytest.param(gpqa_ellamind_cloze_de, _CLOZE, id="cloze"), + pytest.param(gpqa_ellamind_diamond_cloze_de, _CLOZE, id="diamond_cloze"), + pytest.param(gpqa_ellamind_bpb_de, _BPB, id="bpb"), + pytest.param(gpqa_ellamind_diamond_bpb_de, _BPB, id="diamond_bpb"), + ], +) +def test_gpqa_zeroshot_prompt(make_benchmark: Callable[..., Benchmark], expected: _ExpectedPrompt) -> None: + benchmark = make_benchmark(dataset=DatasetStub({"train": [_EVAL_ROW]})) + sample = first_sample(benchmark, num_fewshot=0) + assert sample.messages == expected.messages + assert sample.ground_truth == expected.ground_truth + assert sample.possible_completions == expected.possible_completions + + +# --- One-shot: fewshot row rendered with its answer, then the eval row's own zero-shot prompt --- +_MC_FEWSHOT_MESSAGES = [ + Message(role=Role.USER, content=f"{_FEWSHOT_Q}\nA. Celsius\nB. Fahrenheit\nC. Joule\nD. Kelvin\n"), + Message(role=Role.ASSISTANT, content="Antwort: D"), +] +_CLOZE_FEWSHOT_MESSAGES = [ + Message(role=Role.USER, content=f"{_FEWSHOT_Q}\n"), + Message(role=Role.ASSISTANT, content="Antwort: Kelvin"), +] + + +def _oneshot(fewshot_messages: list[Message], eval_expected: _ExpectedPrompt) -> _ExpectedPrompt: + return _ExpectedPrompt( + messages=[*fewshot_messages, *eval_expected.messages], + ground_truth=eval_expected.ground_truth, + possible_completions=eval_expected.possible_completions, + ) + + +@pytest.mark.parametrize( + "make_benchmark, expected", + [ + pytest.param(gpqa_ellamind_mc_de, _oneshot(_MC_FEWSHOT_MESSAGES, _MC), id="mc"), + pytest.param(gpqa_ellamind_diamond_mc_de, _oneshot(_MC_FEWSHOT_MESSAGES, _MC), id="diamond_mc"), + pytest.param(gpqa_ellamind_cloze_de, _oneshot(_CLOZE_FEWSHOT_MESSAGES, _CLOZE), id="cloze"), + pytest.param(gpqa_ellamind_diamond_cloze_de, _oneshot(_CLOZE_FEWSHOT_MESSAGES, _CLOZE), id="diamond_cloze"), + pytest.param(gpqa_ellamind_bpb_de, _oneshot(_CLOZE_FEWSHOT_MESSAGES, _BPB), id="bpb"), + pytest.param(gpqa_ellamind_diamond_bpb_de, _oneshot(_CLOZE_FEWSHOT_MESSAGES, _BPB), id="diamond_bpb"), + ], +) +def test_gpqa_oneshot_prompt(make_benchmark: Callable[..., Benchmark], expected: _ExpectedPrompt) -> None: + benchmark = make_benchmark(dataset=DatasetStub({"train": [_FEWSHOT_ROW, _EVAL_ROW]})) + sample = first_sample(benchmark, num_fewshot=1) + assert sample.messages == expected.messages + assert sample.ground_truth == expected.ground_truth + assert sample.possible_completions == expected.possible_completions + + +# --------------------------------------------------------------------------- +# Diamond subset: the diamond variant keeps only is_diamond rows; the full variant keeps them all +# --------------------------------------------------------------------------- +def test_gpqa_diamond_variant_keeps_only_diamond_rows() -> None: + # Given a dataset mixing diamond and non-diamond rows + rows: list[dict[str, Any]] = [ + {"question": "Q1", "correct_answer": "A", "incorrect_answers": ["x", "y", "z"], "is_diamond": True}, + {"question": "Q2", "correct_answer": "A", "incorrect_answers": ["x", "y", "z"], "is_diamond": False}, + {"question": "Q3", "correct_answer": "A", "incorrect_answers": ["x", "y", "z"], "is_diamond": True}, + ] + diamond = gpqa_ellamind_diamond_mc_de(dataset=DatasetStub({"train": rows})) + full = gpqa_ellamind_mc_de(dataset=DatasetStub({"train": rows})) + + # When we assemble all samples for each + diamond_samples = list(diamond.create(0, None, None, seed=42).iterate_samples()) + full_samples = list(full.create(0, None, None, seed=42).iterate_samples()) + + # Then the diamond variant drops the non-diamond row (Q2); the full variant keeps all three + assert len(diamond_samples) == 2 + assert all("Q2" not in sample.messages[0].content for sample in diamond_samples) + assert len(full_samples) == 3 diff --git a/tests/tests_eval_framework/tasks/benchmarks/task-prompts-hashes.json b/tests/tests_eval_framework/tasks/benchmarks/task-prompts-hashes.json index 9fb7a8705..efe8af8f8 100644 --- a/tests/tests_eval_framework/tasks/benchmarks/task-prompts-hashes.json +++ b/tests/tests_eval_framework/tasks/benchmarks/task-prompts-hashes.json @@ -77,15 +77,12 @@ "GPQA_ELLAMIND_CLOZE_DE.ConcatFormatter": "17012ddfedd5d172777d14829cbc59d9", "GPQA_ELLAMIND_CLOZE_DE.Llama3Formatter": "7564a2287e4f05c07258ecb8c7296430", "GPQA_ELLAMIND_CLOZE_DE.NoStripConcatFormatter": "17012ddfedd5d172777d14829cbc59d9", - "GPQA_ELLAMIND_DIAMOND_BPB_DE.ConcatFormatter": "512aba3ac575edf2e436f3b56c39c02f", - "GPQA_ELLAMIND_DIAMOND_BPB_DE.Llama3Formatter": "2e8d48dc1e72ecc36f7089a212344ce8", - "GPQA_ELLAMIND_DIAMOND_BPB_DE.NoStripConcatFormatter": "512aba3ac575edf2e436f3b56c39c02f", - "GPQA_ELLAMIND_DIAMOND_CLOZE_DE.ConcatFormatter": "4a98fcb0a244406408b51998cdf774c1", - "GPQA_ELLAMIND_DIAMOND_CLOZE_DE.Llama3Formatter": "83a2c5f2e1d730d3597c460e26da0d7b", - "GPQA_ELLAMIND_DIAMOND_CLOZE_DE.NoStripConcatFormatter": "4a98fcb0a244406408b51998cdf774c1", - "GPQA_ELLAMIND_DIAMOND_MC_DE.ConcatFormatter": "fbc610535e08cf35d419128e87a9e81f", - "GPQA_ELLAMIND_DIAMOND_MC_DE.Llama3Formatter": "facfff72edfc0cea3586151cf68c3b5a", - "GPQA_ELLAMIND_DIAMOND_MC_DE.NoStripConcatFormatter": "fbc610535e08cf35d419128e87a9e81f", + "GPQA_ELLAMIND_DIAMOND_BPB_DE.ConcatFormatter": "d9d6884002b3444a3e86cf381549fb7a", + "GPQA_ELLAMIND_DIAMOND_BPB_DE.Llama3Formatter": "c0c41d144555d68643d6aa859313225c", + "GPQA_ELLAMIND_DIAMOND_CLOZE_DE.ConcatFormatter": "187ce4452e77c41a476e985293692506", + "GPQA_ELLAMIND_DIAMOND_CLOZE_DE.Llama3Formatter": "7c19fd18ab53dd2b841d5a3ebdff52ab", + "GPQA_ELLAMIND_DIAMOND_MC_DE.ConcatFormatter": "e621619202beca3dd218f299dd8b3952", + "GPQA_ELLAMIND_DIAMOND_MC_DE.Llama3Formatter": "7061ee6ebeeb37d940bb0a79ff53b237", "GPQA_ELLAMIND_MC_DE.ConcatFormatter": "d75bea2373f0e995a2d8fed079611e97", "GPQA_ELLAMIND_MC_DE.Llama3Formatter": "3fb1cc28b6f4a78111489dc14b0f07cf", "GPQA_ELLAMIND_MC_DE.NoStripConcatFormatter": "d75bea2373f0e995a2d8fed079611e97", diff --git a/tests/tests_eval_framework/tasks/benchmarks/test_gpqa_ellamind.py b/tests/tests_eval_framework/tasks/benchmarks/test_gpqa_ellamind.py deleted file mode 100644 index 84e51b6b3..000000000 --- a/tests/tests_eval_framework/tasks/benchmarks/test_gpqa_ellamind.py +++ /dev/null @@ -1,291 +0,0 @@ -"""Tests for the German GPQA (EllaMind) tasks. - -Tests: -- formatter hash test for every GPQA variant -- offline prompt assembly tests -- diamond-subset filtering test (offline) -""" - -from typing import Any - -import pytest - -import eval_framework.tasks.benchmarks.gpqa_ellamind as gpqa_ellamind -from eval_framework.tasks.base import BaseTask -from eval_framework.tasks.registry import Registry -from eval_framework.tasks.task_names import register_gpqa_ellamind_tasks -from template_formatting.formatter import ( - BaseFormatter, - ConcatFormatter, - Llama3Formatter, - Message, - Role, -) -from tests.tests_eval_framework.tasks.benchmarks.utils import ( - ExpectedPrompt, - assert_offline_oneshot_prompt, - assert_offline_zeroshot_prompt, - run_formatter_hash_test, -) - -# Registry for this test suite only holding gpqa_ellamind tasks -_gpqa_ellamind_registry = Registry() -register_gpqa_ellamind_tasks(registry=_gpqa_ellamind_registry) - -# --------------------------------------------------------------------------- -# Formatter hash tests (Hugging Face) -# --------------------------------------------------------------------------- - - -@pytest.mark.formatter_hash -@pytest.mark.parametrize("formatter_cls", [Llama3Formatter, ConcatFormatter]) -@pytest.mark.parametrize("task_name", _gpqa_ellamind_registry.task_names()) -def test_formatter_hash(task_name: str, formatter_cls: type[BaseFormatter]) -> None: - run_formatter_hash_test(task_name, formatter_cls, registry=_gpqa_ellamind_registry) - - -# --------------------------------------------------------------------------- -# Offline prompt assembly tests (use fictional dataset) -# --------------------------------------------------------------------------- - -_SUBJECT = "deu" - -# Fictional rows following the GPQA format. NOT real examples from the GPQA dataset. -# Option order and correct letters are shuffled deterministically (seed: question+answer). -_EVAL_ROW: dict[str, Any] = { - "question": "Was ist die SI-Einheit des elektrischen Widerstands?", - "correct_answer": "Ohm", - "incorrect_answers": ["Volt", "Ampere", "Watt"], - "is_diamond": True, -} - -_FEWSHOT_ROW: dict[str, Any] = { - "question": "Was ist die SI-Einheit der Temperatur?", - "correct_answer": "Kelvin", - "incorrect_answers": ["Celsius", "Fahrenheit", "Joule"], - "is_diamond": True, -} - -# Expected prompts (messages, flat concat, ground truth, completions). -# --- GPQA_ELLAMIND_MC_DE --- -_MC_ZEROSHOT = ExpectedPrompt( - messages=[ - Message( - role=Role.USER, - content="Frage: Was ist die SI-Einheit des elektrischen Widerstands?\nA. Ohm\nB. Volt\nC. Watt\nD. Ampere\n", - ), - Message(role=Role.ASSISTANT, content="Antwort:"), - ], - concat="""\ -Frage: Was ist die SI-Einheit des elektrischen Widerstands? -A. Ohm -B. Volt -C. Watt -D. Ampere -Antwort:""", - ground_truth=" A", - completions=[" A", " B", " C", " D"], -) - -_MC_FEWSHOT = ExpectedPrompt( - messages=[ - Message( - role=Role.USER, - content="Frage: Was ist die SI-Einheit der Temperatur?\nA. Celsius\nB. Fahrenheit\nC. Joule\nD. Kelvin\n", - ), - Message(role=Role.ASSISTANT, content="Antwort: D"), - Message( - role=Role.USER, - content="Frage: Was ist die SI-Einheit des elektrischen Widerstands?\nA. Ohm\nB. Volt\nC. Watt\nD. Ampere\n", - ), - Message(role=Role.ASSISTANT, content="Antwort:"), - ], - concat="""\ -Frage: Was ist die SI-Einheit der Temperatur? -A. Celsius -B. Fahrenheit -C. Joule -D. Kelvin -Antwort: D - -Frage: Was ist die SI-Einheit des elektrischen Widerstands? -A. Ohm -B. Volt -C. Watt -D. Ampere -Antwort:""", - ground_truth=_MC_ZEROSHOT.ground_truth, - completions=_MC_ZEROSHOT.completions, -) - -# --- GPQA_ELLAMIND_CLOZE_DE --- -_CLOZE_ZEROSHOT = ExpectedPrompt( - messages=[ - Message(role=Role.USER, content="Frage: Was ist die SI-Einheit des elektrischen Widerstands?\n"), - Message(role=Role.ASSISTANT, content="Antwort:"), - ], - concat="""\ -Frage: Was ist die SI-Einheit des elektrischen Widerstands? -Antwort:""", - ground_truth=" Ohm", - completions=[" Ohm", " Volt", " Watt", " Ampere"], -) - -_CLOZE_FEWSHOT = ExpectedPrompt( - messages=[ - Message(role=Role.USER, content="Frage: Was ist die SI-Einheit der Temperatur?\n"), - Message(role=Role.ASSISTANT, content="Antwort: Kelvin"), - Message(role=Role.USER, content="Frage: Was ist die SI-Einheit des elektrischen Widerstands?\n"), - Message(role=Role.ASSISTANT, content="Antwort:"), - ], - concat="""\ -Frage: Was ist die SI-Einheit der Temperatur? -Antwort: Kelvin - -Frage: Was ist die SI-Einheit des elektrischen Widerstands? -Antwort:""", - ground_truth=_CLOZE_ZEROSHOT.ground_truth, - completions=_CLOZE_ZEROSHOT.completions, -) - -# --- GPQA_ELLAMIND_BPB_DE --- -# Same prompt as cloze; BPB scores only the gold continuation. -_cloze_ground_truth = _CLOZE_ZEROSHOT.ground_truth -assert isinstance(_cloze_ground_truth, str) # narrow the type: cloze ground_truth is always a str - -_BPB_ZEROSHOT = ExpectedPrompt( - messages=_CLOZE_ZEROSHOT.messages, - concat=_CLOZE_ZEROSHOT.concat, - ground_truth=_cloze_ground_truth, - completions=[_cloze_ground_truth], -) - -_BPB_FEWSHOT = ExpectedPrompt( - messages=_CLOZE_FEWSHOT.messages, - concat=_CLOZE_FEWSHOT.concat, - ground_truth=_cloze_ground_truth, - completions=[_cloze_ground_truth], -) - - -# --- TESTS --- -def test_gpqa_ellamind_mc_de_offline_prompt_formatting() -> None: - assert_offline_zeroshot_prompt( - gpqa_ellamind.GPQA_ELLAMIND_MC_DE, - eval_row=_EVAL_ROW, - subjects=[_SUBJECT], - expected=_MC_ZEROSHOT, - ) - assert_offline_oneshot_prompt( - gpqa_ellamind.GPQA_ELLAMIND_MC_DE, - eval_row=_EVAL_ROW, - fewshot_row=_FEWSHOT_ROW, - subjects=[_SUBJECT], - expected=_MC_FEWSHOT, - ) - - -def test_gpqa_ellamind_diamond_mc_de_offline_prompt_formatting() -> None: - assert_offline_zeroshot_prompt( - gpqa_ellamind.GPQA_ELLAMIND_DIAMOND_MC_DE, - eval_row=_EVAL_ROW, - subjects=[_SUBJECT], - expected=_MC_ZEROSHOT, - ) - assert_offline_oneshot_prompt( - gpqa_ellamind.GPQA_ELLAMIND_DIAMOND_MC_DE, - eval_row=_EVAL_ROW, - fewshot_row=_FEWSHOT_ROW, - subjects=[_SUBJECT], - expected=_MC_FEWSHOT, - ) - - -def test_gpqa_ellamind_cloze_de_offline_prompt_formatting() -> None: - assert_offline_zeroshot_prompt( - gpqa_ellamind.GPQA_ELLAMIND_CLOZE_DE, - eval_row=_EVAL_ROW, - subjects=[_SUBJECT], - expected=_CLOZE_ZEROSHOT, - ) - assert_offline_oneshot_prompt( - gpqa_ellamind.GPQA_ELLAMIND_CLOZE_DE, - eval_row=_EVAL_ROW, - fewshot_row=_FEWSHOT_ROW, - subjects=[_SUBJECT], - expected=_CLOZE_FEWSHOT, - ) - - -def test_gpqa_ellamind_diamond_cloze_de_offline_prompt_formatting() -> None: - assert_offline_zeroshot_prompt( - gpqa_ellamind.GPQA_ELLAMIND_DIAMOND_CLOZE_DE, - eval_row=_EVAL_ROW, - subjects=[_SUBJECT], - expected=_CLOZE_ZEROSHOT, - ) - assert_offline_oneshot_prompt( - gpqa_ellamind.GPQA_ELLAMIND_DIAMOND_CLOZE_DE, - eval_row=_EVAL_ROW, - fewshot_row=_FEWSHOT_ROW, - subjects=[_SUBJECT], - expected=_CLOZE_FEWSHOT, - ) - - -def test_gpqa_ellamind_bpb_de_offline_prompt_formatting() -> None: - assert_offline_zeroshot_prompt( - gpqa_ellamind.GPQA_ELLAMIND_BPB_DE, - eval_row=_EVAL_ROW, - subjects=[_SUBJECT], - expected=_BPB_ZEROSHOT, - ) - assert_offline_oneshot_prompt( - gpqa_ellamind.GPQA_ELLAMIND_BPB_DE, - eval_row=_EVAL_ROW, - fewshot_row=_FEWSHOT_ROW, - subjects=[_SUBJECT], - expected=_BPB_FEWSHOT, - ) - - -def test_gpqa_ellamind_diamond_bpb_de_offline_prompt_formatting() -> None: - assert_offline_zeroshot_prompt( - gpqa_ellamind.GPQA_ELLAMIND_DIAMOND_BPB_DE, - eval_row=_EVAL_ROW, - subjects=[_SUBJECT], - expected=_BPB_ZEROSHOT, - ) - assert_offline_oneshot_prompt( - gpqa_ellamind.GPQA_ELLAMIND_DIAMOND_BPB_DE, - eval_row=_EVAL_ROW, - fewshot_row=_FEWSHOT_ROW, - subjects=[_SUBJECT], - expected=_BPB_FEWSHOT, - ) - - -# --------------------------------------------------------------------------- -# Diamond-subset filtering test (offline) -# --------------------------------------------------------------------------- - - -def test_gpqa_diamond_variant_filters_to_diamond_rows(monkeypatch: pytest.MonkeyPatch) -> None: - """Diamond variant should keep only rows with `is_diamond=True`.""" - - def fake_base_load_dataset(self: BaseTask, subject: str) -> None: - _ = subject - self.dataset = { - self.SAMPLE_SPLIT: [ - {"is_diamond": True, "question": "Q1"}, - {"is_diamond": False, "question": "Q2"}, - {"is_diamond": True, "question": "Q3"}, - ] - } - - monkeypatch.setattr(BaseTask, "_load_dataset", fake_base_load_dataset) - task = gpqa_ellamind.GPQA_ELLAMIND_DIAMOND_MC_DE(num_fewshot=0) - task._load_dataset("deu") - - assert len(task.dataset[task.SAMPLE_SPLIT]) == 2 - assert all(item["is_diamond"] for item in task.dataset[task.SAMPLE_SPLIT]) From 5804336d9c0b9916420a3ab2d90a87935871aaf0 Mon Sep 17 00:00:00 2001 From: Markus Klein Date: Fri, 28 Aug 2026 09:57:09 +0200 Subject: [PATCH 3/6] refactor: simpleqa_ellamind migrated to composed implementation --- .../benchmarks/simpleqa_ellamind.py | 91 +++++ .../tasks/benchmarks/simpleqa_ellamind.py | 94 ------ src/eval_framework/tasks/task_names.py | 16 +- .../benchmarks/test_simpleqa_ellamind.py | 172 ++++++++++ .../benchmarks/test_simpleqa_ellamind.py | 316 ------------------ 5 files changed, 266 insertions(+), 423 deletions(-) create mode 100644 src/eval_framework/benchmarks/simpleqa_ellamind.py delete mode 100644 src/eval_framework/tasks/benchmarks/simpleqa_ellamind.py create mode 100644 tests/tests_eval_framework/benchmarks/test_simpleqa_ellamind.py delete mode 100644 tests/tests_eval_framework/tasks/benchmarks/test_simpleqa_ellamind.py diff --git a/src/eval_framework/benchmarks/simpleqa_ellamind.py b/src/eval_framework/benchmarks/simpleqa_ellamind.py new file mode 100644 index 000000000..138508385 --- /dev/null +++ b/src/eval_framework/benchmarks/simpleqa_ellamind.py @@ -0,0 +1,91 @@ +"""German SimpleQA (verified, EllaMind) tasks. + +https://huggingface.co/datasets/ellamind/simpleqa-verified-multilingual + +SimpleQA supplies separate easy and hard distractors. The ``answer_aliases`` field is unused. +""" + +from typing import Any, Literal, final, override + +from eval_framework.choices import ChoiceFields, ChoiceReader +from eval_framework.composed import ComposedBenchmark +from eval_framework.contract import Benchmark +from eval_framework.subjects import ListOfSubjects +from eval_framework.tasks.base import Language +from eval_framework.tasks.dataset_loading import DatasetPolicy +from eval_framework.tasks.dataset_revisions import pinned_by_framework +from eval_framework.tasks.task_style import BPBStyle, ClozeStyle, MCStyle, TaskStyler, shuffle_correct_with_distractors + + +@final +class SimpleqaReader(ChoiceReader): + """Reads a SimpleQA item: easy/hard distractor lists per level, shuffled in with the correct answer.""" + + def __init__(self, distractor_level: Literal["easy", "hard"]) -> None: + self._distractor_level = distractor_level + + @override + def read(self, item: dict[str, Any]) -> ChoiceFields: + distractors = item["easy_distractors"] if self._distractor_level == "easy" else item["hard_distractors"] + choices, correct_index = shuffle_correct_with_distractors( + correct=item["answer"], + distractors=distractors, + seed_text=item["question"] + item["answer"], + ) + return ChoiceFields(raw_question=item["question"], choices=choices, correct_index=correct_index) + + +def _simpleqa_ellamind_benchmark( + id: str, styler: TaskStyler, distractor_level: Literal["easy", "hard"], dataset: DatasetPolicy | None +) -> Benchmark: + return ComposedBenchmark.compose( + id=id, + styler=styler, + reader=SimpleqaReader(distractor_level), + sample_split="eval", + fewshot_split="eval", + subjects=ListOfSubjects(["deu"]), + dataset_policy=dataset + if dataset is not None + else pinned_by_framework("ellamind/simpleqa-verified-multilingual"), + language=Language.DEU, + ) + + +def simpleqa_ellamind_mc_easy_de(dataset: DatasetPolicy | None = None) -> Benchmark: + return _simpleqa_ellamind_benchmark( + "SIMPLEQA_ELLAMIND_MC_EASY_DE", MCStyle.for_language(Language.DEU), "easy", dataset + ) + + +def simpleqa_ellamind_mc_hard_de(dataset: DatasetPolicy | None = None) -> Benchmark: + return _simpleqa_ellamind_benchmark( + "SIMPLEQA_ELLAMIND_MC_HARD_DE", MCStyle.for_language(Language.DEU), "hard", dataset + ) + + +def simpleqa_ellamind_cloze_easy_de(dataset: DatasetPolicy | None = None) -> Benchmark: + return _simpleqa_ellamind_benchmark( + "SIMPLEQA_ELLAMIND_CLOZE_EASY_DE", ClozeStyle.for_language(Language.DEU), "easy", dataset + ) + + +def simpleqa_ellamind_cloze_hard_de(dataset: DatasetPolicy | None = None) -> Benchmark: + return _simpleqa_ellamind_benchmark( + "SIMPLEQA_ELLAMIND_CLOZE_HARD_DE", ClozeStyle.for_language(Language.DEU), "hard", dataset + ) + + +def simpleqa_ellamind_bpb_de(dataset: DatasetPolicy | None = None) -> Benchmark: + return _simpleqa_ellamind_benchmark( + "SIMPLEQA_ELLAMIND_BPB_DE", BPBStyle.for_language(Language.DEU), "easy", dataset + ) + + +SIMPLEQA_ELLAMIND_BENCHMARKS: list[Benchmark] = [ + simpleqa_ellamind_mc_easy_de(), + simpleqa_ellamind_mc_hard_de(), + simpleqa_ellamind_cloze_easy_de(), + simpleqa_ellamind_cloze_hard_de(), + simpleqa_ellamind_bpb_de(), +] diff --git a/src/eval_framework/tasks/benchmarks/simpleqa_ellamind.py b/src/eval_framework/tasks/benchmarks/simpleqa_ellamind.py deleted file mode 100644 index e20f504b4..000000000 --- a/src/eval_framework/tasks/benchmarks/simpleqa_ellamind.py +++ /dev/null @@ -1,94 +0,0 @@ -"""German SimpleQA (verified, EllaMind) tasks. - -https://huggingface.co/datasets/ellamind/simpleqa-verified-multilingual - -SimpleQA supplies separate easy and hard distractors. Each base class uses a -``_DISTRACTOR_LEVEL`` class attribute (``"easy"`` or ``"hard"``) that the registered -subclass overrides. -""" - -from typing import Any, Literal - -from eval_framework.tasks.base import BaseTask, Language -from eval_framework.tasks.dataset_revisions import HF_REVISIONS_LOCKFILE -from eval_framework.tasks.task_style import BPBStyle, ClozeStyle, MCStyle, shuffle_correct_with_distractors - - -class _SIMPLEQA_ELLAMIND_DE_Base(BaseTask[str]): - """Non-registered base for German SimpleQA (EllaMind) variants. - - Dataset: https://huggingface.co/datasets/ellamind/simpleqa-verified-multilingual - - We don't use their `answer_aliases` field for Cloze and MC variants. - """ - - DATASET_PATH = "ellamind/simpleqa-verified-multilingual" - SAMPLE_SPLIT = "eval" - FEWSHOT_SPLIT = "eval" - SUBJECTS = ["deu"] - LANGUAGE = Language.DEU - _DISTRACTOR_LEVEL: Literal["easy", "hard"] = "easy" - - def _shuffled(self, item: dict[str, Any]) -> tuple[list[str], int]: - distractors = item["easy_distractors"] if self._DISTRACTOR_LEVEL == "easy" else item["hard_distractors"] - return shuffle_correct_with_distractors( - correct=item["answer"], - distractors=distractors, - seed_text=item["question"] + item["answer"], - ) - - def _get_raw_question(self, item: dict[str, Any]) -> str: - return item["question"] - - def _get_choices(self, item: dict[str, Any]) -> list[str]: - return self._shuffled(item)[0] - - def _get_correct_index(self, item: dict[str, Any]) -> int: - return self._shuffled(item)[1] - - -class SIMPLEQA_ELLAMIND_MC_EASY_DE(_SIMPLEQA_ELLAMIND_DE_Base): - """German SimpleQA - MC format with easy distractors.""" - - REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE - - NAME = "SIMPLEQA_ELLAMIND_MC_EASY_DE" - TASK_STYLER = MCStyle().for_language(Language.DEU) - - -class SIMPLEQA_ELLAMIND_MC_HARD_DE(_SIMPLEQA_ELLAMIND_DE_Base): - """German SimpleQA - MC format with hard distractors.""" - - REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE - - NAME = "SIMPLEQA_ELLAMIND_MC_HARD_DE" - _DISTRACTOR_LEVEL = "hard" - TASK_STYLER = MCStyle().for_language(Language.DEU) - - -class SIMPLEQA_ELLAMIND_CLOZE_EASY_DE(_SIMPLEQA_ELLAMIND_DE_Base): - """German SimpleQA - Cloze format with easy distractors.""" - - REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE - - NAME = "SIMPLEQA_ELLAMIND_CLOZE_EASY_DE" - TASK_STYLER = ClozeStyle().for_language(Language.DEU) - - -class SIMPLEQA_ELLAMIND_CLOZE_HARD_DE(_SIMPLEQA_ELLAMIND_DE_Base): - """German SimpleQA - Cloze format with hard distractors.""" - - REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE - - NAME = "SIMPLEQA_ELLAMIND_CLOZE_HARD_DE" - _DISTRACTOR_LEVEL = "hard" - TASK_STYLER = ClozeStyle().for_language(Language.DEU) - - -class SIMPLEQA_ELLAMIND_BPB_DE(SIMPLEQA_ELLAMIND_CLOZE_EASY_DE): - """German SimpleQA - BPB format (distractor set is irrelevant for BPB).""" - - REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE - - NAME = "SIMPLEQA_ELLAMIND_BPB_DE" - TASK_STYLER = BPBStyle().for_language(Language.DEU) diff --git a/src/eval_framework/tasks/task_names.py b/src/eval_framework/tasks/task_names.py index 482cdc974..8bc66b26f 100644 --- a/src/eval_framework/tasks/task_names.py +++ b/src/eval_framework/tasks/task_names.py @@ -4,6 +4,7 @@ from eval_framework.benchmarks.csqa_ellamind import CSQA_ELLAMIND_BENCHMARKS from eval_framework.benchmarks.gpqa_ellamind import GPQA_ELLAMIND_BENCHMARKS from eval_framework.benchmarks.piqa_ellamind import PIQA_ELLAMIND_BENCHMARKS +from eval_framework.benchmarks.simpleqa_ellamind import SIMPLEQA_ELLAMIND_BENCHMARKS from eval_framework.benchmarks.siqa_ellamind import SIQA_ELLAMIND_BENCHMARKS from eval_framework.tasks.base import BaseTask from eval_framework.tasks.registry import Registry, register_lazy_task @@ -317,19 +318,8 @@ def register_piqa_ellamind_tasks(registry: Registry) -> None: def register_simpleqa_ellamind_tasks(registry: Registry) -> None: """Register simpleqa_ellamind benchmark tasks.""" - register_lazy_task( - "eval_framework.tasks.benchmarks.simpleqa_ellamind.SIMPLEQA_ELLAMIND_MC_EASY_DE", registry=registry - ) - register_lazy_task( - "eval_framework.tasks.benchmarks.simpleqa_ellamind.SIMPLEQA_ELLAMIND_MC_HARD_DE", registry=registry - ) - register_lazy_task( - "eval_framework.tasks.benchmarks.simpleqa_ellamind.SIMPLEQA_ELLAMIND_CLOZE_EASY_DE", registry=registry - ) - register_lazy_task( - "eval_framework.tasks.benchmarks.simpleqa_ellamind.SIMPLEQA_ELLAMIND_CLOZE_HARD_DE", registry=registry - ) - register_lazy_task("eval_framework.tasks.benchmarks.simpleqa_ellamind.SIMPLEQA_ELLAMIND_BPB_DE", registry=registry) + for benchmark in SIMPLEQA_ELLAMIND_BENCHMARKS: + registry.add(benchmark) def register_siqa_ellamind_tasks(registry: Registry) -> None: diff --git a/tests/tests_eval_framework/benchmarks/test_simpleqa_ellamind.py b/tests/tests_eval_framework/benchmarks/test_simpleqa_ellamind.py new file mode 100644 index 000000000..0b89db231 --- /dev/null +++ b/tests/tests_eval_framework/benchmarks/test_simpleqa_ellamind.py @@ -0,0 +1,172 @@ +"""Specification of the German SimpleQA (verified, EllaMind) tasks. + +Each spec test builds the real benchmark (via its ``simpleqa_ellamind_*_de`` constructor) over a fictional +dataset and asserts the assembled messages, ground truth, and scored completions — so this file reads as +SimpleQA's prompt spec, with ``composed.py`` an implementation detail. The rows are fictional so this open +source codebase does not leak the real dataset into training data. ``test_formatter_hash`` separately pins +the real benchmarks against the real HuggingFace data. +""" + +from collections.abc import Callable +from dataclasses import dataclass +from typing import Any + +import pytest + +from eval_framework.benchmarks.simpleqa_ellamind import ( + simpleqa_ellamind_bpb_de, + simpleqa_ellamind_cloze_easy_de, + simpleqa_ellamind_cloze_hard_de, + simpleqa_ellamind_mc_easy_de, + simpleqa_ellamind_mc_hard_de, +) +from eval_framework.contract import Benchmark +from eval_framework.tasks.registry import Registry +from eval_framework.tasks.task_names import register_simpleqa_ellamind_tasks +from template_formatting.formatter import BaseFormatter, ConcatFormatter, Llama3Formatter, Message, Role +from tests.tests_eval_framework.benchmarks.utils import DatasetStub, first_sample +from tests.tests_eval_framework.tasks.benchmarks.utils import run_formatter_hash_test + +# Registry for this test suite only holding simpleqa_ellamind tasks +_simpleqa_ellamind_registry = Registry() +register_simpleqa_ellamind_tasks(registry=_simpleqa_ellamind_registry) + + +@pytest.mark.formatter_hash +@pytest.mark.parametrize("formatter_cls", [Llama3Formatter, ConcatFormatter]) +@pytest.mark.parametrize("task_name", _simpleqa_ellamind_registry.task_names()) +def test_formatter_hash(task_name: str, formatter_cls: type[BaseFormatter]) -> None: + run_formatter_hash_test(task_name, formatter_cls, registry=_simpleqa_ellamind_registry) + + +# --------------------------------------------------------------------------- +# Prompt spec: build the real benchmark over fictional rows, assert the assembled messages +# --------------------------------------------------------------------------- + +# Fictional rows in the SimpleQA format (NOT real dataset examples). Choices are shuffled deterministically +# (seed: question + answer). +_EVAL_ROW: dict[str, Any] = { + "question": "Welches Jahr haben?", + "answer": "2026", + "easy_distractors": ["1954", "1974", "1990"], + "hard_distractors": ["2024", "2025", "2027"], +} +_FEWSHOT_ROW: dict[str, Any] = { + "question": "Was ist die Hauptstadt von Frankreich?", + "answer": "Paris", + "easy_distractors": ["London", "Berlin", "Madrid"], + "hard_distractors": ["Lyon", "Bordeaux", "Marseille"], +} + +_EVAL_Q = "Frage: Welches Jahr haben?" + + +@dataclass(frozen=True) +class _ExpectedPrompt: + messages: list[Message] + ground_truth: str + possible_completions: list[str] + + +# --- Zero-shot --- +_MC_EASY = _ExpectedPrompt( + messages=[ + Message(role=Role.USER, content=f"{_EVAL_Q}\nA. 1990\nB. 1954\nC. 1974\nD. 2026\n"), + Message(role=Role.ASSISTANT, content="Antwort:"), + ], + ground_truth=" D", + possible_completions=[" A", " B", " C", " D"], +) +_MC_HARD = _ExpectedPrompt( + messages=[ + Message(role=Role.USER, content=f"{_EVAL_Q}\nA. 2027\nB. 2024\nC. 2025\nD. 2026\n"), + Message(role=Role.ASSISTANT, content="Antwort:"), + ], + ground_truth=" D", + possible_completions=[" A", " B", " C", " D"], +) +# Cloze/BPB show no options, so the assembled messages are identical; only the scored completions differ. +_CLOZE_MESSAGES = [ + Message(role=Role.USER, content=f"{_EVAL_Q}\n"), + Message(role=Role.ASSISTANT, content="Antwort:"), +] +_CLOZE_EASY = _ExpectedPrompt( + messages=_CLOZE_MESSAGES, + ground_truth=" 2026", + possible_completions=[" 1990", " 1954", " 1974", " 2026"], +) +_CLOZE_HARD = _ExpectedPrompt( + messages=_CLOZE_MESSAGES, + ground_truth=" 2026", + possible_completions=[" 2027", " 2024", " 2025", " 2026"], +) +_BPB = _ExpectedPrompt( + messages=_CLOZE_MESSAGES, + ground_truth=" 2026", + possible_completions=[" 2026"], # BPB scores only the gold continuation +) + + +@pytest.mark.parametrize( + "make_benchmark, expected", + [ + pytest.param(simpleqa_ellamind_mc_easy_de, _MC_EASY, id="mc_easy"), + pytest.param(simpleqa_ellamind_mc_hard_de, _MC_HARD, id="mc_hard"), + pytest.param(simpleqa_ellamind_cloze_easy_de, _CLOZE_EASY, id="cloze_easy"), + pytest.param(simpleqa_ellamind_cloze_hard_de, _CLOZE_HARD, id="cloze_hard"), + pytest.param(simpleqa_ellamind_bpb_de, _BPB, id="bpb"), + ], +) +def test_simpleqa_zeroshot_prompt(make_benchmark: Callable[..., Benchmark], expected: _ExpectedPrompt) -> None: + benchmark = make_benchmark(dataset=DatasetStub({"eval": [_EVAL_ROW]})) + sample = first_sample(benchmark, num_fewshot=0) + assert sample.messages == expected.messages + assert sample.ground_truth == expected.ground_truth + assert sample.possible_completions == expected.possible_completions + + +# --- One-shot: fewshot row rendered with its answer, then the eval row's own zero-shot prompt --- +_MC_EASY_FEWSHOT_MESSAGES = [ + Message( + role=Role.USER, + content="Frage: Was ist die Hauptstadt von Frankreich?\nA. Paris\nB. Berlin\nC. London\nD. Madrid\n", + ), + Message(role=Role.ASSISTANT, content="Antwort: A"), +] +_MC_HARD_FEWSHOT_MESSAGES = [ + Message( + role=Role.USER, + content="Frage: Was ist die Hauptstadt von Frankreich?\nA. Paris\nB. Bordeaux\nC. Lyon\nD. Marseille\n", + ), + Message(role=Role.ASSISTANT, content="Antwort: A"), +] +_CLOZE_FEWSHOT_MESSAGES = [ + Message(role=Role.USER, content="Frage: Was ist die Hauptstadt von Frankreich?\n"), + Message(role=Role.ASSISTANT, content="Antwort: Paris"), +] + + +def _oneshot(fewshot_messages: list[Message], eval_expected: _ExpectedPrompt) -> _ExpectedPrompt: + return _ExpectedPrompt( + messages=[*fewshot_messages, *eval_expected.messages], + ground_truth=eval_expected.ground_truth, + possible_completions=eval_expected.possible_completions, + ) + + +@pytest.mark.parametrize( + "make_benchmark, expected", + [ + pytest.param(simpleqa_ellamind_mc_easy_de, _oneshot(_MC_EASY_FEWSHOT_MESSAGES, _MC_EASY), id="mc_easy"), + pytest.param(simpleqa_ellamind_mc_hard_de, _oneshot(_MC_HARD_FEWSHOT_MESSAGES, _MC_HARD), id="mc_hard"), + pytest.param(simpleqa_ellamind_cloze_easy_de, _oneshot(_CLOZE_FEWSHOT_MESSAGES, _CLOZE_EASY), id="cloze_easy"), + pytest.param(simpleqa_ellamind_cloze_hard_de, _oneshot(_CLOZE_FEWSHOT_MESSAGES, _CLOZE_HARD), id="cloze_hard"), + pytest.param(simpleqa_ellamind_bpb_de, _oneshot(_CLOZE_FEWSHOT_MESSAGES, _BPB), id="bpb"), + ], +) +def test_simpleqa_oneshot_prompt(make_benchmark: Callable[..., Benchmark], expected: _ExpectedPrompt) -> None: + benchmark = make_benchmark(dataset=DatasetStub({"eval": [_FEWSHOT_ROW, _EVAL_ROW]})) + sample = first_sample(benchmark, num_fewshot=1) + assert sample.messages == expected.messages + assert sample.ground_truth == expected.ground_truth + assert sample.possible_completions == expected.possible_completions diff --git a/tests/tests_eval_framework/tasks/benchmarks/test_simpleqa_ellamind.py b/tests/tests_eval_framework/tasks/benchmarks/test_simpleqa_ellamind.py deleted file mode 100644 index f8a72b111..000000000 --- a/tests/tests_eval_framework/tasks/benchmarks/test_simpleqa_ellamind.py +++ /dev/null @@ -1,316 +0,0 @@ -"""Tests for the German SimpleQA (EllaMind) tasks. - -Tests: -- formatter hash test for every SimpleQA variant -- offline prompt assembly tests -""" - -from typing import Any - -import pytest - -import eval_framework.tasks.benchmarks.simpleqa_ellamind as simpleqa_ellamind -from eval_framework.tasks.registry import Registry -from eval_framework.tasks.task_names import register_simpleqa_ellamind_tasks -from template_formatting.formatter import ( - BaseFormatter, - ConcatFormatter, - Llama3Formatter, - Message, - Role, -) -from tests.tests_eval_framework.tasks.benchmarks.utils import ( - ExpectedPrompt, - assert_offline_oneshot_prompt, - assert_offline_zeroshot_prompt, - run_formatter_hash_test, -) - -# Registry for this test suite only holding simpleqa_ellamind tasks -_simpleqa_ellamind_registry = Registry() -register_simpleqa_ellamind_tasks(registry=_simpleqa_ellamind_registry) - -# --------------------------------------------------------------------------- -# Formatter hash tests (Hugging Face) -# --------------------------------------------------------------------------- - - -@pytest.mark.formatter_hash -@pytest.mark.parametrize("formatter_cls", [Llama3Formatter, ConcatFormatter]) -@pytest.mark.parametrize("task_name", _simpleqa_ellamind_registry.task_names()) -def test_formatter_hash(task_name: str, formatter_cls: type[BaseFormatter]) -> None: - run_formatter_hash_test(task_name, formatter_cls, registry=_simpleqa_ellamind_registry) - - -# --------------------------------------------------------------------------- -# Offline prompt assembly tests (use fictional dataset) -# --------------------------------------------------------------------------- - -_SUBJECT = "deu" - -# Fictional rows following the SimpleQA format. NOT real examples from the SimpleQA dataset. -# Option order and correct letters are shuffled deterministically (seed: question+answer). -_EVAL_ROW: dict[str, Any] = { - "question": "Welches Jahr haben?", - "answer": "2026", - "easy_distractors": ["1954", "1974", "1990"], - "hard_distractors": ["2024", "2025", "2027"], -} - -_FEWSHOT_ROW: dict[str, Any] = { - "question": "Was ist die Hauptstadt von Frankreich?", - "answer": "Paris", - "easy_distractors": ["London", "Berlin", "Madrid"], - "hard_distractors": ["Lyon", "Bordeaux", "Marseille"], -} - -# Expected prompts (messages, flat concat, ground truth, completions). -# --- SIMPLEQA_ELLAMIND_MC_EASY_DE --- -_MC_EASY_ZEROSHOT = ExpectedPrompt( - messages=[ - Message( - role=Role.USER, - content="Frage: Welches Jahr haben?\nA. 1990\nB. 1954\nC. 1974\nD. 2026\n", - ), - Message(role=Role.ASSISTANT, content="Antwort:"), - ], - concat="""\ -Frage: Welches Jahr haben? -A. 1990 -B. 1954 -C. 1974 -D. 2026 -Antwort:""", - ground_truth=" D", - completions=[" A", " B", " C", " D"], -) - -_MC_EASY_FEWSHOT = ExpectedPrompt( - messages=[ - Message( - role=Role.USER, - content="Frage: Was ist die Hauptstadt von Frankreich?\nA. Paris\nB. Berlin\nC. London\nD. Madrid\n", - ), - Message(role=Role.ASSISTANT, content="Antwort: A"), - Message( - role=Role.USER, - content="Frage: Welches Jahr haben?\nA. 1990\nB. 1954\nC. 1974\nD. 2026\n", - ), - Message(role=Role.ASSISTANT, content="Antwort:"), - ], - concat="""\ -Frage: Was ist die Hauptstadt von Frankreich? -A. Paris -B. Berlin -C. London -D. Madrid -Antwort: A - -Frage: Welches Jahr haben? -A. 1990 -B. 1954 -C. 1974 -D. 2026 -Antwort:""", - ground_truth=_MC_EASY_ZEROSHOT.ground_truth, - completions=_MC_EASY_ZEROSHOT.completions, -) - -# --- SIMPLEQA_ELLAMIND_MC_HARD_DE --- -_MC_HARD_ZEROSHOT = ExpectedPrompt( - messages=[ - Message( - role=Role.USER, - content="Frage: Welches Jahr haben?\nA. 2027\nB. 2024\nC. 2025\nD. 2026\n", - ), - Message(role=Role.ASSISTANT, content="Antwort:"), - ], - concat="""\ -Frage: Welches Jahr haben? -A. 2027 -B. 2024 -C. 2025 -D. 2026 -Antwort:""", - ground_truth=_MC_EASY_ZEROSHOT.ground_truth, - completions=_MC_EASY_ZEROSHOT.completions, -) - -_MC_HARD_FEWSHOT = ExpectedPrompt( - messages=[ - Message( - role=Role.USER, - content="Frage: Was ist die Hauptstadt von Frankreich?\nA. Paris\nB. Bordeaux\nC. Lyon\nD. Marseille\n", - ), - Message(role=Role.ASSISTANT, content="Antwort: A"), - Message( - role=Role.USER, - content="Frage: Welches Jahr haben?\nA. 2027\nB. 2024\nC. 2025\nD. 2026\n", - ), - Message(role=Role.ASSISTANT, content="Antwort:"), - ], - concat="""\ -Frage: Was ist die Hauptstadt von Frankreich? -A. Paris -B. Bordeaux -C. Lyon -D. Marseille -Antwort: A - -Frage: Welches Jahr haben? -A. 2027 -B. 2024 -C. 2025 -D. 2026 -Antwort:""", - ground_truth=_MC_HARD_ZEROSHOT.ground_truth, - completions=_MC_HARD_ZEROSHOT.completions, -) - -# --- SIMPLEQA_ELLAMIND_CLOZE_EASY_DE --- -# Cloze prompts show no options, so the easy/hard variants share the same prompt; only the -# scored completions differ. -_CLOZE_EASY_ZEROSHOT = ExpectedPrompt( - messages=[ - Message(role=Role.USER, content="Frage: Welches Jahr haben?\n"), - Message(role=Role.ASSISTANT, content="Antwort:"), - ], - concat="""\ -Frage: Welches Jahr haben? -Antwort:""", - ground_truth=" 2026", - completions=[" 1990", " 1954", " 1974", " 2026"], -) - -_CLOZE_EASY_FEWSHOT = ExpectedPrompt( - messages=[ - Message(role=Role.USER, content="Frage: Was ist die Hauptstadt von Frankreich?\n"), - Message(role=Role.ASSISTANT, content="Antwort: Paris"), - Message(role=Role.USER, content="Frage: Welches Jahr haben?\n"), - Message(role=Role.ASSISTANT, content="Antwort:"), - ], - concat="""\ -Frage: Was ist die Hauptstadt von Frankreich? -Antwort: Paris - -Frage: Welches Jahr haben? -Antwort:""", - ground_truth=_CLOZE_EASY_ZEROSHOT.ground_truth, - completions=_CLOZE_EASY_ZEROSHOT.completions, -) - -# --- SIMPLEQA_ELLAMIND_CLOZE_HARD_DE --- -# Same prompt as cloze-easy; only the (hard) distractor completions differ. -_CLOZE_HARD_ZEROSHOT = ExpectedPrompt( - messages=_CLOZE_EASY_ZEROSHOT.messages, - concat=_CLOZE_EASY_ZEROSHOT.concat, - ground_truth=_CLOZE_EASY_ZEROSHOT.ground_truth, - completions=[" 2027", " 2024", " 2025", " 2026"], -) - -_CLOZE_HARD_FEWSHOT = ExpectedPrompt( - messages=_CLOZE_EASY_FEWSHOT.messages, - concat=_CLOZE_EASY_FEWSHOT.concat, - ground_truth=_CLOZE_HARD_ZEROSHOT.ground_truth, - completions=_CLOZE_HARD_ZEROSHOT.completions, -) - -# --- SIMPLEQA_ELLAMIND_BPB_DE --- -# Same prompt as cloze-easy; BPB scores only the gold continuation. -_cloze_ground_truth = _CLOZE_EASY_ZEROSHOT.ground_truth -assert isinstance(_cloze_ground_truth, str) # narrow the type: cloze ground_truth is always a str - -_BPB_ZEROSHOT = ExpectedPrompt( - messages=_CLOZE_EASY_ZEROSHOT.messages, - concat=_CLOZE_EASY_ZEROSHOT.concat, - ground_truth=_cloze_ground_truth, - completions=[_cloze_ground_truth], -) - -_BPB_FEWSHOT = ExpectedPrompt( - messages=_CLOZE_EASY_FEWSHOT.messages, - concat=_CLOZE_EASY_FEWSHOT.concat, - ground_truth=_cloze_ground_truth, - completions=[_cloze_ground_truth], -) - - -# --- TESTS --- -def test_simpleqa_ellamind_mc_easy_de_offline_prompt_formatting() -> None: - assert_offline_zeroshot_prompt( - simpleqa_ellamind.SIMPLEQA_ELLAMIND_MC_EASY_DE, - eval_row=_EVAL_ROW, - subjects=[_SUBJECT], - expected=_MC_EASY_ZEROSHOT, - ) - assert_offline_oneshot_prompt( - simpleqa_ellamind.SIMPLEQA_ELLAMIND_MC_EASY_DE, - eval_row=_EVAL_ROW, - fewshot_row=_FEWSHOT_ROW, - subjects=[_SUBJECT], - expected=_MC_EASY_FEWSHOT, - ) - - -def test_simpleqa_ellamind_mc_hard_de_offline_prompt_formatting() -> None: - assert_offline_zeroshot_prompt( - simpleqa_ellamind.SIMPLEQA_ELLAMIND_MC_HARD_DE, - eval_row=_EVAL_ROW, - subjects=[_SUBJECT], - expected=_MC_HARD_ZEROSHOT, - ) - assert_offline_oneshot_prompt( - simpleqa_ellamind.SIMPLEQA_ELLAMIND_MC_HARD_DE, - eval_row=_EVAL_ROW, - fewshot_row=_FEWSHOT_ROW, - subjects=[_SUBJECT], - expected=_MC_HARD_FEWSHOT, - ) - - -def test_simpleqa_ellamind_cloze_easy_de_offline_prompt_formatting() -> None: - assert_offline_zeroshot_prompt( - simpleqa_ellamind.SIMPLEQA_ELLAMIND_CLOZE_EASY_DE, - eval_row=_EVAL_ROW, - subjects=[_SUBJECT], - expected=_CLOZE_EASY_ZEROSHOT, - ) - assert_offline_oneshot_prompt( - simpleqa_ellamind.SIMPLEQA_ELLAMIND_CLOZE_EASY_DE, - eval_row=_EVAL_ROW, - fewshot_row=_FEWSHOT_ROW, - subjects=[_SUBJECT], - expected=_CLOZE_EASY_FEWSHOT, - ) - - -def test_simpleqa_ellamind_cloze_hard_de_offline_prompt_formatting() -> None: - assert_offline_zeroshot_prompt( - simpleqa_ellamind.SIMPLEQA_ELLAMIND_CLOZE_HARD_DE, - eval_row=_EVAL_ROW, - subjects=[_SUBJECT], - expected=_CLOZE_HARD_ZEROSHOT, - ) - assert_offline_oneshot_prompt( - simpleqa_ellamind.SIMPLEQA_ELLAMIND_CLOZE_HARD_DE, - eval_row=_EVAL_ROW, - fewshot_row=_FEWSHOT_ROW, - subjects=[_SUBJECT], - expected=_CLOZE_HARD_FEWSHOT, - ) - - -def test_simpleqa_ellamind_bpb_de_offline_prompt_formatting() -> None: - assert_offline_zeroshot_prompt( - simpleqa_ellamind.SIMPLEQA_ELLAMIND_BPB_DE, - eval_row=_EVAL_ROW, - subjects=[_SUBJECT], - expected=_BPB_ZEROSHOT, - ) - assert_offline_oneshot_prompt( - simpleqa_ellamind.SIMPLEQA_ELLAMIND_BPB_DE, - eval_row=_EVAL_ROW, - fewshot_row=_FEWSHOT_ROW, - subjects=[_SUBJECT], - expected=_BPB_FEWSHOT, - ) From 3cd415926ed3ab33163d68e6c2d54db36d2988fc Mon Sep 17 00:00:00 2001 From: Markus Klein Date: Fri, 28 Aug 2026 10:09:06 +0200 Subject: [PATCH 4/6] refactor: hellaswag_ellamind migrated to composed benchmark --- .../benchmarks/hellaswag_ellamind.py | 82 ++++++++ .../tasks/benchmarks/hellaswag_ellamind.py | 73 ------- src/eval_framework/tasks/task_names.py | 12 +- .../benchmarks/test_hellaswag_ellamind.py | 152 ++++++++++++++ .../benchmarks/test_hellaswag_ellamind.py | 192 ------------------ 5 files changed, 237 insertions(+), 274 deletions(-) create mode 100644 src/eval_framework/benchmarks/hellaswag_ellamind.py delete mode 100644 src/eval_framework/tasks/benchmarks/hellaswag_ellamind.py create mode 100644 tests/tests_eval_framework/benchmarks/test_hellaswag_ellamind.py delete mode 100644 tests/tests_eval_framework/tasks/benchmarks/test_hellaswag_ellamind.py diff --git a/src/eval_framework/benchmarks/hellaswag_ellamind.py b/src/eval_framework/benchmarks/hellaswag_ellamind.py new file mode 100644 index 000000000..34aaa6080 --- /dev/null +++ b/src/eval_framework/benchmarks/hellaswag_ellamind.py @@ -0,0 +1,82 @@ +"""German HellaSwag (EllaMind) tasks. + +https://huggingface.co/datasets/ellamind/hellaswag-multilingual + +HellaSwag is a sentence-completion task: the prompt is a partial sentence (``"{activity}: {context}"``) +and the model scores full-sentence endings. There is no natural MC variant. HellaSwag supplies separate +easy and hard distractors. +""" + +from typing import Any, Literal, final, override + +from eval_framework.choices import ChoiceFields, ChoiceReader +from eval_framework.composed import ComposedBenchmark +from eval_framework.contract import Benchmark +from eval_framework.subjects import ListOfSubjects +from eval_framework.tasks.base import Language +from eval_framework.tasks.dataset_loading import DatasetPolicy +from eval_framework.tasks.dataset_revisions import pinned_by_framework +from eval_framework.tasks.task_style import BPBStyle, ClozeStyle, TaskStyler, shuffle_correct_with_distractors + + +@final +class HellaswagReader(ChoiceReader): + """Reads a HellaSwag item: the partial sentence ``"{activity}: {context}"``, with the easy/hard + full-sentence endings for the level shuffled in with the correct ending.""" + + def __init__(self, distractor_level: Literal["easy", "hard"]) -> None: + self._distractor_level = distractor_level + + @override + def read(self, item: dict[str, Any]) -> ChoiceFields: + distractors = item["easy_distractors"] if self._distractor_level == "easy" else item["hard_distractors"] + choices, correct_index = shuffle_correct_with_distractors( + correct=item["correct_ending"], + distractors=distractors, + seed_text=item["context"] + item["correct_ending"], + ) + return ChoiceFields( + raw_question=f"{item['activity'].strip()}: {item['context'].strip()}", + choices=choices, + correct_index=correct_index, + ) + + +def _hellaswag_ellamind_benchmark( + id: str, styler: TaskStyler, distractor_level: Literal["easy", "hard"], dataset: DatasetPolicy | None +) -> Benchmark: + return ComposedBenchmark.compose( + id=id, + styler=styler, + reader=HellaswagReader(distractor_level), + sample_split="validation", + fewshot_split="validation", + subjects=ListOfSubjects(["deu"]), + dataset_policy=dataset if dataset is not None else pinned_by_framework("ellamind/hellaswag-multilingual"), + language=Language.DEU, + ) + + +# Sentence-completion: no question prefix, no cue, the continuation follows the context directly. +def _cloze_completion_style() -> ClozeStyle: + return ClozeStyle(question_prefix="", trailing_newline=False, cue_text="") + + +def hellaswag_ellamind_easy_de(dataset: DatasetPolicy | None = None) -> Benchmark: + return _hellaswag_ellamind_benchmark("HELLASWAG_ELLAMIND_EASY_DE", _cloze_completion_style(), "easy", dataset) + + +def hellaswag_ellamind_hard_de(dataset: DatasetPolicy | None = None) -> Benchmark: + return _hellaswag_ellamind_benchmark("HELLASWAG_ELLAMIND_HARD_DE", _cloze_completion_style(), "hard", dataset) + + +def hellaswag_ellamind_bpb_de(dataset: DatasetPolicy | None = None) -> Benchmark: + styler = BPBStyle(question_prefix="", trailing_newline=False, cue_text="") + return _hellaswag_ellamind_benchmark("HELLASWAG_ELLAMIND_BPB_DE", styler, "easy", dataset) + + +HELLASWAG_ELLAMIND_BENCHMARKS: list[Benchmark] = [ + hellaswag_ellamind_easy_de(), + hellaswag_ellamind_hard_de(), + hellaswag_ellamind_bpb_de(), +] diff --git a/src/eval_framework/tasks/benchmarks/hellaswag_ellamind.py b/src/eval_framework/tasks/benchmarks/hellaswag_ellamind.py deleted file mode 100644 index 9f0ac3d8a..000000000 --- a/src/eval_framework/tasks/benchmarks/hellaswag_ellamind.py +++ /dev/null @@ -1,73 +0,0 @@ -"""German HellaSwag (EllaMind) tasks. - -https://huggingface.co/datasets/ellamind/hellaswag-multilingual - -HellaSwag supplies separate easy and hard distractors. Each task class uses a -``_DISTRACTOR_LEVEL`` class attribute (``"easy"`` or ``"hard"``). -""" - -from typing import Any, Literal - -from eval_framework.tasks.base import BaseTask, Language -from eval_framework.tasks.dataset_revisions import HF_REVISIONS_LOCKFILE -from eval_framework.tasks.task_style import BPBStyle, ClozeStyle, shuffle_correct_with_distractors - - -class HELLASWAG_ELLAMIND_EASY_DE(BaseTask[str]): - """German HellaSwag - Cloze (sentence-completion) format with easy distractors. - - Dataset: https://huggingface.co/datasets/ellamind/hellaswag-multilingual - - HellaSwag is a *sentence-completion* task: the prompt is a partial sentence - (``"{activity}: {context}"``) and the model scores full sentence endings. - There is no natural MC variant for this task (would be possible, but not natural). - - Set ``_DISTRACTOR_LEVEL = "easy"`` or ``"hard"`` on the task class. - """ - - REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE - - NAME = "HELLASWAG_ELLAMIND_EASY_DE" - DATASET_PATH = "ellamind/hellaswag-multilingual" - SAMPLE_SPLIT = "validation" - FEWSHOT_SPLIT = "validation" - SUBJECTS = ["deu"] - LANGUAGE = Language.DEU - _DISTRACTOR_LEVEL: Literal["easy", "hard"] = "easy" - # Sentence-completion: no question prefix, no cue, continuation follows directly - TASK_STYLER = ClozeStyle(question_prefix="", trailing_newline=False, cue_text="") - - def _shuffled(self, item: dict[str, Any]) -> tuple[list[str], int]: - distractors = item["easy_distractors"] if self._DISTRACTOR_LEVEL == "easy" else item["hard_distractors"] - return shuffle_correct_with_distractors( - correct=item["correct_ending"], - distractors=distractors, - seed_text=item["context"] + item["correct_ending"], - ) - - def _get_raw_question(self, item: dict[str, Any]) -> str: - return f"{item['activity'].strip()}: {item['context'].strip()}" - - def _get_choices(self, item: dict[str, Any]) -> list[str]: - return self._shuffled(item)[0] - - def _get_correct_index(self, item: dict[str, Any]) -> int: - return self._shuffled(item)[1] - - -class HELLASWAG_ELLAMIND_HARD_DE(HELLASWAG_ELLAMIND_EASY_DE): - """German HellaSwag - Cloze (sentence-completion) format with hard distractors.""" - - REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE - - NAME = "HELLASWAG_ELLAMIND_HARD_DE" - _DISTRACTOR_LEVEL = "hard" - - -class HELLASWAG_ELLAMIND_BPB_DE(HELLASWAG_ELLAMIND_EASY_DE): - """German HellaSwag - BPB format.""" - - REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE - - NAME = "HELLASWAG_ELLAMIND_BPB_DE" - TASK_STYLER = BPBStyle(question_prefix="", trailing_newline=False, cue_text="") diff --git a/src/eval_framework/tasks/task_names.py b/src/eval_framework/tasks/task_names.py index 8bc66b26f..710927e79 100644 --- a/src/eval_framework/tasks/task_names.py +++ b/src/eval_framework/tasks/task_names.py @@ -3,6 +3,7 @@ from eval_framework.benchmarks.arc_de import ARC_DE_BENCHMARK from eval_framework.benchmarks.csqa_ellamind import CSQA_ELLAMIND_BENCHMARKS from eval_framework.benchmarks.gpqa_ellamind import GPQA_ELLAMIND_BENCHMARKS +from eval_framework.benchmarks.hellaswag_ellamind import HELLASWAG_ELLAMIND_BENCHMARKS from eval_framework.benchmarks.piqa_ellamind import PIQA_ELLAMIND_BENCHMARKS from eval_framework.benchmarks.simpleqa_ellamind import SIMPLEQA_ELLAMIND_BENCHMARKS from eval_framework.benchmarks.siqa_ellamind import SIQA_ELLAMIND_BENCHMARKS @@ -263,15 +264,8 @@ def register_gsm8k_ellamind_tasks(registry: Registry) -> None: def register_hellaswag_ellamind_tasks(registry: Registry) -> None: """Register hellaswag_ellamind benchmark tasks.""" - register_lazy_task( - "eval_framework.tasks.benchmarks.hellaswag_ellamind.HELLASWAG_ELLAMIND_EASY_DE", registry=registry - ) - register_lazy_task( - "eval_framework.tasks.benchmarks.hellaswag_ellamind.HELLASWAG_ELLAMIND_HARD_DE", registry=registry - ) - register_lazy_task( - "eval_framework.tasks.benchmarks.hellaswag_ellamind.HELLASWAG_ELLAMIND_BPB_DE", registry=registry - ) + for benchmark in HELLASWAG_ELLAMIND_BENCHMARKS: + registry.add(benchmark) def register_hendrycks_math_ellamind_tasks(registry: Registry) -> None: diff --git a/tests/tests_eval_framework/benchmarks/test_hellaswag_ellamind.py b/tests/tests_eval_framework/benchmarks/test_hellaswag_ellamind.py new file mode 100644 index 000000000..83647b356 --- /dev/null +++ b/tests/tests_eval_framework/benchmarks/test_hellaswag_ellamind.py @@ -0,0 +1,152 @@ +"""Specification of the German HellaSwag (EllaMind) tasks. + +Each spec test builds the real benchmark (via its ``hellaswag_ellamind_*_de`` constructor) over a fictional +dataset and asserts the assembled messages, ground truth, and scored completions — so this file reads as +HellaSwag's prompt spec, with ``composed.py`` an implementation detail. The rows are fictional so this open +source codebase does not leak the real dataset into training data. HellaSwag is sentence-completion: no +question prefix and no assistant cue, so the prompt is just the partial sentence and the model scores full +endings. ``test_formatter_hash`` separately pins the real benchmarks against the real HuggingFace data. +""" + +from collections.abc import Callable +from dataclasses import dataclass +from typing import Any + +import pytest + +from eval_framework.benchmarks.hellaswag_ellamind import ( + hellaswag_ellamind_bpb_de, + hellaswag_ellamind_easy_de, + hellaswag_ellamind_hard_de, +) +from eval_framework.contract import Benchmark +from eval_framework.tasks.registry import Registry +from eval_framework.tasks.task_names import register_hellaswag_ellamind_tasks +from template_formatting.formatter import ( + BaseFormatter, + ConcatFormatter, + Llama3Formatter, + Message, + NoStripConcatFormatter, + Role, +) +from tests.tests_eval_framework.benchmarks.utils import DatasetStub, first_sample +from tests.tests_eval_framework.tasks.benchmarks.utils import run_formatter_hash_test + +# Registry for this test suite only holding hellaswag_ellamind tasks +_hellaswag_ellamind_registry = Registry() +register_hellaswag_ellamind_tasks(registry=_hellaswag_ellamind_registry) + + +@pytest.mark.formatter_hash +@pytest.mark.parametrize("formatter_cls", [Llama3Formatter, ConcatFormatter, NoStripConcatFormatter]) +@pytest.mark.parametrize("task_name", _hellaswag_ellamind_registry.task_names()) +def test_formatter_hash(task_name: str, formatter_cls: type[BaseFormatter]) -> None: + run_formatter_hash_test(task_name, formatter_cls, registry=_hellaswag_ellamind_registry) + + +# --------------------------------------------------------------------------- +# Prompt spec: build the real benchmark over fictional rows, assert the assembled messages +# --------------------------------------------------------------------------- + +# Fictional rows in the HellaSwag format (NOT real dataset examples). Endings are shuffled deterministically +# (seed: context + correct_ending). +_EVAL_ROW: dict[str, Any] = { + "activity": "Kochen", + "context": "Die Zwiebeln werden in der Pfanne angebraten", + "correct_ending": "bis sie goldbraun sind.", + "easy_distractors": ["mit einem Schraubenzieher.", "auf dem Dach.", "im Schwimmbad."], + "hard_distractors": ["bis sie gefroren sind.", "bis sie roh sind.", "bis sie trocken sind."], +} +_FEWSHOT_ROW: dict[str, Any] = { + "activity": "Sport", + "context": "Der Spieler rennt über das Feld", + "correct_ending": "und macht ein Tor.", + "easy_distractors": ["und liest ein Buch.", "und kocht Suppe.", "und schläft ein."], + "hard_distractors": ["und springt ins Wasser.", "und setzt sich hin.", "und kommt nicht an."], +} + + +@dataclass(frozen=True) +class _ExpectedPrompt: + messages: list[Message] + ground_truth: str + possible_completions: list[str] + + +# Sentence-completion: the prompt is just the partial sentence, with no assistant cue after it. +_ZEROSHOT_MESSAGES = [Message(role=Role.USER, content="Kochen: Die Zwiebeln werden in der Pfanne angebraten")] + +# Easy/hard/BPB share the same prompt and ground truth; only the scored completions differ. +_EASY = _ExpectedPrompt( + messages=_ZEROSHOT_MESSAGES, + ground_truth=" bis sie goldbraun sind.", + possible_completions=[ + " auf dem Dach.", + " bis sie goldbraun sind.", + " mit einem Schraubenzieher.", + " im Schwimmbad.", + ], +) +_HARD = _ExpectedPrompt( + messages=_ZEROSHOT_MESSAGES, + ground_truth=" bis sie goldbraun sind.", + possible_completions=[ + " bis sie roh sind.", + " bis sie goldbraun sind.", + " bis sie gefroren sind.", + " bis sie trocken sind.", + ], +) +_BPB = _ExpectedPrompt( + messages=_ZEROSHOT_MESSAGES, + ground_truth=" bis sie goldbraun sind.", + possible_completions=[" bis sie goldbraun sind."], # BPB scores only the gold continuation +) + + +@pytest.mark.parametrize( + "make_benchmark, expected", + [ + pytest.param(hellaswag_ellamind_easy_de, _EASY, id="easy"), + pytest.param(hellaswag_ellamind_hard_de, _HARD, id="hard"), + pytest.param(hellaswag_ellamind_bpb_de, _BPB, id="bpb"), + ], +) +def test_hellaswag_zeroshot_prompt(make_benchmark: Callable[..., Benchmark], expected: _ExpectedPrompt) -> None: + benchmark = make_benchmark(dataset=DatasetStub({"validation": [_EVAL_ROW]})) + sample = first_sample(benchmark, num_fewshot=0) + assert sample.messages == expected.messages + assert sample.ground_truth == expected.ground_truth + assert sample.possible_completions == expected.possible_completions + + +# One-shot: the fewshot partial sentence, its full ending as the assistant turn, then the eval prompt. +_FEWSHOT_MESSAGES = [ + Message(role=Role.USER, content="Sport: Der Spieler rennt über das Feld"), + Message(role=Role.ASSISTANT, content=" und macht ein Tor."), +] + + +def _oneshot(eval_expected: _ExpectedPrompt) -> _ExpectedPrompt: + return _ExpectedPrompt( + messages=[*_FEWSHOT_MESSAGES, *eval_expected.messages], + ground_truth=eval_expected.ground_truth, + possible_completions=eval_expected.possible_completions, + ) + + +@pytest.mark.parametrize( + "make_benchmark, expected", + [ + pytest.param(hellaswag_ellamind_easy_de, _oneshot(_EASY), id="easy"), + pytest.param(hellaswag_ellamind_hard_de, _oneshot(_HARD), id="hard"), + pytest.param(hellaswag_ellamind_bpb_de, _oneshot(_BPB), id="bpb"), + ], +) +def test_hellaswag_oneshot_prompt(make_benchmark: Callable[..., Benchmark], expected: _ExpectedPrompt) -> None: + benchmark = make_benchmark(dataset=DatasetStub({"validation": [_FEWSHOT_ROW, _EVAL_ROW]})) + sample = first_sample(benchmark, num_fewshot=1) + assert sample.messages == expected.messages + assert sample.ground_truth == expected.ground_truth + assert sample.possible_completions == expected.possible_completions diff --git a/tests/tests_eval_framework/tasks/benchmarks/test_hellaswag_ellamind.py b/tests/tests_eval_framework/tasks/benchmarks/test_hellaswag_ellamind.py deleted file mode 100644 index 12ad6dfcb..000000000 --- a/tests/tests_eval_framework/tasks/benchmarks/test_hellaswag_ellamind.py +++ /dev/null @@ -1,192 +0,0 @@ -"""Tests for the German HellaSwag (EllaMind) tasks. - -Tests: -- formatter hash test for every HellaSwag variant -- offline prompt assembly tests - -HellaSwag is a sentence-completion task: the prompt is the partial sentence -``"{activity}: {context}"`` with no question/answer cue, and the model scores full sentence -endings. The easy/hard/BPB variants therefore share the same prompt and differ only in the -scored completions. -""" - -from typing import Any - -import pytest - -import eval_framework.tasks.benchmarks.hellaswag_ellamind as hellaswag_ellamind -from eval_framework.tasks.registry import Registry -from eval_framework.tasks.task_names import register_hellaswag_ellamind_tasks -from template_formatting.formatter import ( - BaseFormatter, - ConcatFormatter, - Llama3Formatter, - Message, - NoStripConcatFormatter, - Role, -) -from tests.tests_eval_framework.tasks.benchmarks.utils import ( - ExpectedPrompt, - assert_offline_oneshot_prompt, - assert_offline_zeroshot_prompt, - run_formatter_hash_test, -) - -# Registry for this test suite only holding hellaswag_ellamind tasks -_hellaswag_ellamind_registry = Registry() -register_hellaswag_ellamind_tasks(registry=_hellaswag_ellamind_registry) - -# --------------------------------------------------------------------------- -# Formatter hash tests (Hugging Face) -# --------------------------------------------------------------------------- - - -@pytest.mark.formatter_hash -@pytest.mark.parametrize("formatter_cls", [Llama3Formatter, ConcatFormatter, NoStripConcatFormatter]) -@pytest.mark.parametrize("task_name", _hellaswag_ellamind_registry.task_names()) -def test_formatter_hash(task_name: str, formatter_cls: type[BaseFormatter]) -> None: - run_formatter_hash_test(task_name, formatter_cls, registry=_hellaswag_ellamind_registry) - - -# --------------------------------------------------------------------------- -# Offline prompt assembly tests (use fictional dataset) -# --------------------------------------------------------------------------- - -_SUBJECT = "deu" - -# Fictional rows following the HellaSwag format. NOT real examples from the HellaSwag dataset. -# Option order and correct letters are shuffled deterministically (seed: context+correct_ending). -_EVAL_ROW: dict[str, Any] = { - "activity": "Kochen", - "context": "Die Zwiebeln werden in der Pfanne angebraten", - "correct_ending": "bis sie goldbraun sind.", - "easy_distractors": ["mit einem Schraubenzieher.", "auf dem Dach.", "im Schwimmbad."], - "hard_distractors": ["bis sie gefroren sind.", "bis sie roh sind.", "bis sie trocken sind."], -} - -_FEWSHOT_ROW: dict[str, Any] = { - "activity": "Sport", - "context": "Der Spieler rennt über das Feld", - "correct_ending": "und macht ein Tor.", - "easy_distractors": ["und liest ein Buch.", "und kocht Suppe.", "und schläft ein."], - "hard_distractors": ["und springt ins Wasser.", "und setzt sich hin.", "und kommt nicht an."], -} - -# Expected prompts (messages, flat concat, ground truth, completions). -# --- HELLASWAG_ELLAMIND_EASY_DE --- -_EASY_ZEROSHOT = ExpectedPrompt( - messages=[ - Message(role=Role.USER, content="Kochen: Die Zwiebeln werden in der Pfanne angebraten"), - ], - concat="""\ -Kochen: Die Zwiebeln werden in der Pfanne angebraten""", - ground_truth=" bis sie goldbraun sind.", - completions=[ - " auf dem Dach.", - " bis sie goldbraun sind.", - " mit einem Schraubenzieher.", - " im Schwimmbad.", - ], -) - -_EASY_FEWSHOT = ExpectedPrompt( - messages=[ - Message(role=Role.USER, content="Sport: Der Spieler rennt über das Feld"), - Message(role=Role.ASSISTANT, content=" und macht ein Tor."), - Message(role=Role.USER, content="Kochen: Die Zwiebeln werden in der Pfanne angebraten"), - ], - concat="""\ -Sport: Der Spieler rennt über das Feld und macht ein Tor. - -Kochen: Die Zwiebeln werden in der Pfanne angebraten""", - ground_truth=_EASY_ZEROSHOT.ground_truth, - completions=_EASY_ZEROSHOT.completions, -) - -# --- HELLASWAG_ELLAMIND_HARD_DE --- -_HARD_ZEROSHOT = ExpectedPrompt( - messages=_EASY_ZEROSHOT.messages, - concat=_EASY_ZEROSHOT.concat, - ground_truth=_EASY_ZEROSHOT.ground_truth, - completions=[ - " bis sie roh sind.", - " bis sie goldbraun sind.", - " bis sie gefroren sind.", - " bis sie trocken sind.", - ], -) - -_HARD_FEWSHOT = ExpectedPrompt( - messages=_EASY_FEWSHOT.messages, - concat=_EASY_FEWSHOT.concat, - ground_truth=_EASY_ZEROSHOT.ground_truth, - completions=_HARD_ZEROSHOT.completions, -) - -# --- HELLASWAG_ELLAMIND_BPB_DE --- -# Same prompt as easy/hard; BPB scores only the gold continuation. -_easy_ground_truth = _EASY_ZEROSHOT.ground_truth -assert isinstance(_easy_ground_truth, str) # narrow the type: easy ground_truth is always a str - -_BPB_ZEROSHOT = ExpectedPrompt( - messages=_EASY_ZEROSHOT.messages, - concat=_EASY_ZEROSHOT.concat, - ground_truth=_easy_ground_truth, - completions=[_easy_ground_truth], -) - -_BPB_FEWSHOT = ExpectedPrompt( - messages=_EASY_FEWSHOT.messages, - concat=_EASY_FEWSHOT.concat, - ground_truth=_easy_ground_truth, - completions=[_easy_ground_truth], -) - - -# --- TESTS --- -def test_hellaswag_ellamind_easy_de_offline_prompt_formatting() -> None: - assert_offline_zeroshot_prompt( - hellaswag_ellamind.HELLASWAG_ELLAMIND_EASY_DE, - eval_row=_EVAL_ROW, - subjects=[_SUBJECT], - expected=_EASY_ZEROSHOT, - ) - assert_offline_oneshot_prompt( - hellaswag_ellamind.HELLASWAG_ELLAMIND_EASY_DE, - eval_row=_EVAL_ROW, - fewshot_row=_FEWSHOT_ROW, - subjects=[_SUBJECT], - expected=_EASY_FEWSHOT, - ) - - -def test_hellaswag_ellamind_hard_de_offline_prompt_formatting() -> None: - assert_offline_zeroshot_prompt( - hellaswag_ellamind.HELLASWAG_ELLAMIND_HARD_DE, - eval_row=_EVAL_ROW, - subjects=[_SUBJECT], - expected=_HARD_ZEROSHOT, - ) - assert_offline_oneshot_prompt( - hellaswag_ellamind.HELLASWAG_ELLAMIND_HARD_DE, - eval_row=_EVAL_ROW, - fewshot_row=_FEWSHOT_ROW, - subjects=[_SUBJECT], - expected=_HARD_FEWSHOT, - ) - - -def test_hellaswag_ellamind_bpb_de_offline_prompt_formatting() -> None: - assert_offline_zeroshot_prompt( - hellaswag_ellamind.HELLASWAG_ELLAMIND_BPB_DE, - eval_row=_EVAL_ROW, - subjects=[_SUBJECT], - expected=_BPB_ZEROSHOT, - ) - assert_offline_oneshot_prompt( - hellaswag_ellamind.HELLASWAG_ELLAMIND_BPB_DE, - eval_row=_EVAL_ROW, - fewshot_row=_FEWSHOT_ROW, - subjects=[_SUBJECT], - expected=_BPB_FEWSHOT, - ) From 7f0fd9d8d8915d85e9b302e152073e821b1e5b77 Mon Sep 17 00:00:00 2001 From: Markus Klein Date: Fri, 28 Aug 2026 10:21:05 +0200 Subject: [PATCH 5/6] refactor: NoSubject is default for composed benchmarks --- src/eval_framework/benchmarks/arc_de.py | 2 -- src/eval_framework/composed.py | 9 +++++---- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/src/eval_framework/benchmarks/arc_de.py b/src/eval_framework/benchmarks/arc_de.py index 49513f2ac..c3b36deae 100644 --- a/src/eval_framework/benchmarks/arc_de.py +++ b/src/eval_framework/benchmarks/arc_de.py @@ -5,7 +5,6 @@ from eval_framework.choices import ChoiceFields, ChoiceReader from eval_framework.composed import ComposedBenchmark from eval_framework.contract import Benchmark -from eval_framework.subjects import NoSubject from eval_framework.tasks.base import Language from eval_framework.tasks.dataset_loading import DatasetPolicy from eval_framework.tasks.dataset_revisions import pinned_by_framework @@ -41,7 +40,6 @@ def arc_de(dataset: DatasetPolicy | None = None) -> Benchmark: reader=ArcDeReader(), sample_split="test", fewshot_split="validation", - subjects=NoSubject(), dataset_policy=dataset if dataset is not None else pinned_by_framework("LeoLM/ArcChallenge_de"), language=Language.DEU, ) diff --git a/src/eval_framework/composed.py b/src/eval_framework/composed.py index 68f734a15..eb21e11a6 100644 --- a/src/eval_framework/composed.py +++ b/src/eval_framework/composed.py @@ -18,7 +18,7 @@ from eval_framework.metrics.efficiency.token_counters import TokenCounts from eval_framework.shared.errors import raise_errors from eval_framework.shared.types import Completion, Error, RawCompletion -from eval_framework.subjects import Subjects, SubjectsSelector +from eval_framework.subjects import NoSubject, Subjects, SubjectsSelector from eval_framework.tasks.base import RANDOM_SEED, Language from eval_framework.tasks.dataset_loading import DatasetLoader, DatasetPolicy from eval_framework.tasks.markdown_doc import markdown_doc as render_markdown_doc @@ -311,16 +311,17 @@ def compose( reader: ChoiceReader, sample_split: str, fewshot_split: str, - subjects: SubjectsSelector, + subjects: SubjectsSelector | None = None, dataset_policy: DatasetPolicy, language: LanguageSpec, display_name: str | None = None, ) -> Self: - """Build a ``ComposedBenchmark`` from its inputs; ``display_name`` defaults to ``id``.""" + """Build a ``ComposedBenchmark`` from its inputs; ``subjects`` defaults to ``NoSubject`` + (a single unnamed slice) and ``display_name`` to ``id``.""" return cls( id=id, display_name=display_name if display_name is not None else id, - subjects=subjects, + subjects=subjects if subjects is not None else NoSubject(), styler=styler, reader=reader, sample_split=sample_split, From af500ca56d31ac049147b77521f0ff81e65921cc Mon Sep 17 00:00:00 2001 From: Markus Klein Date: Fri, 28 Aug 2026 11:05:33 +0200 Subject: [PATCH 6/6] refactor: hle_ellamind migrated to composed implementation --- .../benchmarks/gpqa_ellamind.py | 39 +-- src/eval_framework/benchmarks/hle_ellamind.py | 81 +++++ .../tasks/benchmarks/hle_ellamind.py | 101 ------ src/eval_framework/tasks/dataset_loading.py | 42 ++- src/eval_framework/tasks/task_names.py | 8 +- .../benchmarks/test_hle_ellamind.py | 186 +++++++++++ .../tasks/benchmarks/task-prompts-hashes.json | 10 +- .../tasks/benchmarks/test_hle_ellamind.py | 313 ------------------ 8 files changed, 317 insertions(+), 463 deletions(-) create mode 100644 src/eval_framework/benchmarks/hle_ellamind.py delete mode 100644 src/eval_framework/tasks/benchmarks/hle_ellamind.py create mode 100644 tests/tests_eval_framework/benchmarks/test_hle_ellamind.py delete mode 100644 tests/tests_eval_framework/tasks/benchmarks/test_hle_ellamind.py diff --git a/src/eval_framework/benchmarks/gpqa_ellamind.py b/src/eval_framework/benchmarks/gpqa_ellamind.py index 1f1633d54..f96c88675 100644 --- a/src/eval_framework/benchmarks/gpqa_ellamind.py +++ b/src/eval_framework/benchmarks/gpqa_ellamind.py @@ -8,14 +8,12 @@ from typing import Any, final, override -from datasets import DatasetDict - from eval_framework.choices import ChoiceFields, ChoiceReader from eval_framework.composed import ComposedBenchmark from eval_framework.contract import Benchmark from eval_framework.subjects import ListOfSubjects from eval_framework.tasks.base import Language -from eval_framework.tasks.dataset_loading import DatasetLoader, DatasetPolicy +from eval_framework.tasks.dataset_loading import DatasetPolicy, Subset from eval_framework.tasks.dataset_revisions import pinned_by_framework from eval_framework.tasks.task_style import BPBStyle, ClozeStyle, MCStyle, TaskStyler, shuffle_correct_with_distractors @@ -34,39 +32,6 @@ def read(self, item: dict[str, Any]) -> ChoiceFields: return ChoiceFields(raw_question=item["question"], choices=choices, correct_index=correct_index) -@final -class _DiamondFilteredLoader(DatasetLoader): - """Restricts a loader's every split to the diamond subset (``is_diamond``).""" - - def __init__(self, inner: DatasetLoader) -> None: - self._inner = inner - - @override - def load(self, name: str | None) -> DatasetDict: - loaded = self._inner.load(name) - return DatasetDict({split: data.filter(lambda row: row["is_diamond"]) for split, data in loaded.items()}) - - @override - def metadata(self) -> dict[str, str]: - return self._inner.metadata() - - -@final -class _DiamondOnly(DatasetPolicy): - """Wraps a dataset policy to serve only the diamond subset — the 198 hardest GPQA questions.""" - - def __init__(self, inner: DatasetPolicy) -> None: - self._inner = inner - - @override - def loader(self, custom_hf_revision: str | None) -> DatasetLoader: - return _DiamondFilteredLoader(self._inner.loader(custom_hf_revision)) - - @override - def documentation(self) -> str: - return self._inner.documentation() - - def _gpqa_ellamind_benchmark(id: str, styler: TaskStyler, dataset: DatasetPolicy | None) -> Benchmark: return ComposedBenchmark.compose( id=id, @@ -82,7 +47,7 @@ def _gpqa_ellamind_benchmark(id: str, styler: TaskStyler, dataset: DatasetPolicy def _gpqa_ellamind_diamond_benchmark(id: str, styler: TaskStyler, dataset: DatasetPolicy | None) -> Benchmark: source = dataset if dataset is not None else pinned_by_framework("ellamind/gpqa-multilingual") - return _gpqa_ellamind_benchmark(id, styler, _DiamondOnly(source)) + return _gpqa_ellamind_benchmark(id, styler, Subset(source, keep=lambda row: row["is_diamond"])) def gpqa_ellamind_mc_de(dataset: DatasetPolicy | None = None) -> Benchmark: diff --git a/src/eval_framework/benchmarks/hle_ellamind.py b/src/eval_framework/benchmarks/hle_ellamind.py new file mode 100644 index 000000000..48720a932 --- /dev/null +++ b/src/eval_framework/benchmarks/hle_ellamind.py @@ -0,0 +1,81 @@ +"""German HLE (Humanity's Last Exam, EllaMind) tasks. + +https://huggingface.co/datasets/ellamind/hle-multilingual + +HLE uses a single distractor set (``incorrect_answers``). The NATIVE variants restrict evaluation to the +items that are natively multiple-choice in the original benchmark (``answer_type == "multipleChoice"``). +""" + +from typing import Any, final, override + +from eval_framework.choices import ChoiceFields, ChoiceReader +from eval_framework.composed import ComposedBenchmark +from eval_framework.contract import Benchmark +from eval_framework.subjects import ListOfSubjects +from eval_framework.tasks.base import Language +from eval_framework.tasks.dataset_loading import DatasetPolicy, Subset +from eval_framework.tasks.dataset_revisions import pinned_by_framework +from eval_framework.tasks.task_style import BPBStyle, ClozeStyle, MCStyle, TaskStyler, shuffle_correct_with_distractors + + +@final +class HleReader(ChoiceReader): + """Reads an HLE item: a single ``incorrect_answers`` distractor set, shuffled in with the correct answer.""" + + @override + def read(self, item: dict[str, Any]) -> ChoiceFields: + choices, correct_index = shuffle_correct_with_distractors( + correct=item["correct_answer"], + distractors=item["incorrect_answers"], + seed_text=item["question"] + item["correct_answer"], + ) + return ChoiceFields(raw_question=item["question"], choices=choices, correct_index=correct_index) + + +def _hle_ellamind_benchmark(id: str, styler: TaskStyler, dataset: DatasetPolicy | None) -> Benchmark: + return ComposedBenchmark.compose( + id=id, + styler=styler, + reader=HleReader(), + sample_split="test", + fewshot_split="test", + subjects=ListOfSubjects(["deu"]), + dataset_policy=dataset if dataset is not None else pinned_by_framework("ellamind/hle-multilingual"), + language=Language.DEU, + ) + + +def _hle_ellamind_native_benchmark(id: str, styler: TaskStyler, dataset: DatasetPolicy | None) -> Benchmark: + source = dataset if dataset is not None else pinned_by_framework("ellamind/hle-multilingual") + return _hle_ellamind_benchmark(id, styler, Subset(source, keep=lambda row: row["answer_type"] == "multipleChoice")) + + +def hle_ellamind_mc_de(dataset: DatasetPolicy | None = None) -> Benchmark: + return _hle_ellamind_benchmark("HLE_ELLAMIND_MC_DE", MCStyle.for_language(Language.DEU), dataset) + + +def hle_ellamind_cloze_de(dataset: DatasetPolicy | None = None) -> Benchmark: + return _hle_ellamind_benchmark("HLE_ELLAMIND_CLOZE_DE", ClozeStyle.for_language(Language.DEU), dataset) + + +def hle_ellamind_mc_native_de(dataset: DatasetPolicy | None = None) -> Benchmark: + return _hle_ellamind_native_benchmark("HLE_ELLAMIND_MC_NATIVE_DE", MCStyle.for_language(Language.DEU), dataset) + + +def hle_ellamind_cloze_native_de(dataset: DatasetPolicy | None = None) -> Benchmark: + return _hle_ellamind_native_benchmark( + "HLE_ELLAMIND_CLOZE_NATIVE_DE", ClozeStyle.for_language(Language.DEU), dataset + ) + + +def hle_ellamind_bpb_de(dataset: DatasetPolicy | None = None) -> Benchmark: + return _hle_ellamind_benchmark("HLE_ELLAMIND_BPB_DE", BPBStyle.for_language(Language.DEU), dataset) + + +HLE_ELLAMIND_BENCHMARKS: list[Benchmark] = [ + hle_ellamind_mc_de(), + hle_ellamind_cloze_de(), + hle_ellamind_mc_native_de(), + hle_ellamind_cloze_native_de(), + hle_ellamind_bpb_de(), +] diff --git a/src/eval_framework/tasks/benchmarks/hle_ellamind.py b/src/eval_framework/tasks/benchmarks/hle_ellamind.py deleted file mode 100644 index 1f3f567f5..000000000 --- a/src/eval_framework/tasks/benchmarks/hle_ellamind.py +++ /dev/null @@ -1,101 +0,0 @@ -"""German HLE (Humanity's Last Exam, EllaMind) tasks. - -https://huggingface.co/datasets/ellamind/hle-multilingual - -HLE uses a single distractor set (``incorrect_answers``). The natively -multiple-choice subset is exposed via ``_NATIVE_MC_ONLY = True`` on the subclass. -""" - -from typing import Any - -from eval_framework.tasks.base import BaseTask, Language -from eval_framework.tasks.dataset_revisions import HF_REVISIONS_LOCKFILE -from eval_framework.tasks.task_style import BPBStyle, ClozeStyle, MCStyle, shuffle_correct_with_distractors - - -class _HLE_ELLAMIND_DE_Base(BaseTask[str]): - """Non-registered base for German HLE (EllaMind) variants. - - Dataset: https://huggingface.co/datasets/ellamind/hle-multilingual - - Set ``_NATIVE_MC_ONLY = True`` to restrict to the ~half of items that are - natively multiple-choice in the original benchmark. - """ - - DATASET_PATH = "ellamind/hle-multilingual" - SAMPLE_SPLIT = "test" - FEWSHOT_SPLIT = "test" - SUBJECTS = ["deu"] - LANGUAGE = Language.DEU - _NATIVE_MC_ONLY: bool = False - - def _load_dataset(self, subject: str) -> None: - super()._load_dataset(subject) - if self._NATIVE_MC_ONLY: - self.dataset = { - split: [item for item in items if item["answer_type"] == "multipleChoice"] - for split, items in self.dataset.items() - } - - def _shuffled(self, item: dict[str, Any]) -> tuple[list[str], int]: - return shuffle_correct_with_distractors( - correct=item["correct_answer"], - distractors=item["incorrect_answers"], - seed_text=item["question"] + item["correct_answer"], - ) - - def _get_raw_question(self, item: dict[str, Any]) -> str: - return item["question"] - - def _get_choices(self, item: dict[str, Any]) -> list[str]: - return self._shuffled(item)[0] - - def _get_correct_index(self, item: dict[str, Any]) -> int: - return self._shuffled(item)[1] - - -class HLE_ELLAMIND_MC_DE(_HLE_ELLAMIND_DE_Base): - """German HLE - MC format (all 800 items).""" - - REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE - - NAME = "HLE_ELLAMIND_MC_DE" - TASK_STYLER = MCStyle().for_language(Language.DEU) - - -class HLE_ELLAMIND_CLOZE_DE(_HLE_ELLAMIND_DE_Base): - """German HLE - Cloze format (all 800 items).""" - - REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE - - NAME = "HLE_ELLAMIND_CLOZE_DE" - TASK_STYLER = ClozeStyle().for_language(Language.DEU) - - -class HLE_ELLAMIND_MC_NATIVE_DE(_HLE_ELLAMIND_DE_Base): - """German HLE - MC format, native multiple-choice items only.""" - - REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE - - NAME = "HLE_ELLAMIND_MC_NATIVE_DE" - _NATIVE_MC_ONLY = True - TASK_STYLER = MCStyle().for_language(Language.DEU) - - -class HLE_ELLAMIND_CLOZE_NATIVE_DE(_HLE_ELLAMIND_DE_Base): - """German HLE - Cloze format, native multiple-choice items only.""" - - REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE - - NAME = "HLE_ELLAMIND_CLOZE_NATIVE_DE" - _NATIVE_MC_ONLY = True - TASK_STYLER = ClozeStyle().for_language(Language.DEU) - - -class HLE_ELLAMIND_BPB_DE(_HLE_ELLAMIND_DE_Base): - """German HLE - BPB format (all 800 items).""" - - REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE - - NAME = "HLE_ELLAMIND_BPB_DE" - TASK_STYLER = BPBStyle().for_language(Language.DEU) diff --git a/src/eval_framework/tasks/dataset_loading.py b/src/eval_framework/tasks/dataset_loading.py index d9c280ded..a16fb792b 100644 --- a/src/eval_framework/tasks/dataset_loading.py +++ b/src/eval_framework/tasks/dataset_loading.py @@ -3,8 +3,9 @@ import os from abc import ABC, abstractmethod +from collections.abc import Callable from pathlib import Path -from typing import cast, final, override +from typing import Any, cast, final, override from datasets import DatasetDict, DownloadConfig, load_dataset @@ -57,3 +58,42 @@ def loader(self, custom_hf_revision: str | None) -> DatasetLoader: ... def documentation(self) -> str: """Markdown for the task's ``## Dataset`` doc section, describing where the dataset comes from.""" ... + + +@final +class _SubsetLoader(DatasetLoader): + """Loads another loader's splits, keeping only the rows for which ``keep`` returns true.""" + + def __init__(self, inner: DatasetLoader, keep: Callable[[dict[str, Any]], bool]) -> None: + self._inner = inner + self._keep = keep + + @override + def load(self, name: str | None) -> DatasetDict: + loaded = self._inner.load(name) + return DatasetDict({split: data.filter(self._keep) for split, data in loaded.items()}) + + @override + def metadata(self) -> dict[str, str]: + return self._inner.metadata() + + +@final +class Subset(DatasetPolicy): + """Restricts another policy's dataset to the rows for which ``keep`` returns true, in every split. + + A benchmark whose items are a row-filtered subset of a larger dataset (e.g. GPQA's diamond subset, + HLE's natively-multiple-choice subset) wraps the base policy in a ``Subset``. + """ + + def __init__(self, inner: DatasetPolicy, keep: Callable[[dict[str, Any]], bool]) -> None: + self._inner = inner + self._keep = keep + + @override + def loader(self, custom_hf_revision: str | None) -> DatasetLoader: + return _SubsetLoader(self._inner.loader(custom_hf_revision), self._keep) + + @override + def documentation(self) -> str: + return self._inner.documentation() diff --git a/src/eval_framework/tasks/task_names.py b/src/eval_framework/tasks/task_names.py index 710927e79..9797fd9a6 100644 --- a/src/eval_framework/tasks/task_names.py +++ b/src/eval_framework/tasks/task_names.py @@ -4,6 +4,7 @@ from eval_framework.benchmarks.csqa_ellamind import CSQA_ELLAMIND_BENCHMARKS from eval_framework.benchmarks.gpqa_ellamind import GPQA_ELLAMIND_BENCHMARKS from eval_framework.benchmarks.hellaswag_ellamind import HELLASWAG_ELLAMIND_BENCHMARKS +from eval_framework.benchmarks.hle_ellamind import HLE_ELLAMIND_BENCHMARKS from eval_framework.benchmarks.piqa_ellamind import PIQA_ELLAMIND_BENCHMARKS from eval_framework.benchmarks.simpleqa_ellamind import SIMPLEQA_ELLAMIND_BENCHMARKS from eval_framework.benchmarks.siqa_ellamind import SIQA_ELLAMIND_BENCHMARKS @@ -281,11 +282,8 @@ def register_hendrycks_math_ellamind_tasks(registry: Registry) -> None: def register_hle_ellamind_tasks(registry: Registry) -> None: """Register hle_ellamind benchmark tasks.""" - register_lazy_task("eval_framework.tasks.benchmarks.hle_ellamind.HLE_ELLAMIND_MC_DE", registry=registry) - register_lazy_task("eval_framework.tasks.benchmarks.hle_ellamind.HLE_ELLAMIND_CLOZE_DE", registry=registry) - register_lazy_task("eval_framework.tasks.benchmarks.hle_ellamind.HLE_ELLAMIND_MC_NATIVE_DE", registry=registry) - register_lazy_task("eval_framework.tasks.benchmarks.hle_ellamind.HLE_ELLAMIND_CLOZE_NATIVE_DE", registry=registry) - register_lazy_task("eval_framework.tasks.benchmarks.hle_ellamind.HLE_ELLAMIND_BPB_DE", registry=registry) + for benchmark in HLE_ELLAMIND_BENCHMARKS: + registry.add(benchmark) def register_humaneval_ellamind_tasks(registry: Registry) -> None: diff --git a/tests/tests_eval_framework/benchmarks/test_hle_ellamind.py b/tests/tests_eval_framework/benchmarks/test_hle_ellamind.py new file mode 100644 index 000000000..f09a5ddec --- /dev/null +++ b/tests/tests_eval_framework/benchmarks/test_hle_ellamind.py @@ -0,0 +1,186 @@ +"""Specification of the German HLE (Humanity's Last Exam, EllaMind) tasks. + +Each spec test builds the real benchmark (via its ``hle_ellamind_*_de`` constructor) over a fictional +dataset and asserts the assembled messages, ground truth, and scored completions — so this file reads as +HLE's prompt spec, with ``composed.py`` an implementation detail. The rows are fictional so this open +source codebase does not leak the real dataset into training data. NATIVE variants render identically to +their full counterparts; they differ only by restricting the dataset to natively-multiple-choice items +(``answer_type == "multipleChoice"``). ``test_formatter_hash`` separately pins the real benchmarks against +the real HuggingFace data. +""" + +from collections.abc import Callable +from dataclasses import dataclass +from typing import Any + +import pytest + +from eval_framework.benchmarks.hle_ellamind import ( + hle_ellamind_bpb_de, + hle_ellamind_cloze_de, + hle_ellamind_cloze_native_de, + hle_ellamind_mc_de, + hle_ellamind_mc_native_de, +) +from eval_framework.contract import Benchmark +from eval_framework.tasks.registry import Registry +from eval_framework.tasks.task_names import register_hle_ellamind_tasks +from template_formatting.formatter import BaseFormatter, ConcatFormatter, Llama3Formatter, Message, Role +from tests.tests_eval_framework.benchmarks.utils import DatasetStub, first_sample +from tests.tests_eval_framework.tasks.benchmarks.utils import run_formatter_hash_test + +# Registry for this test suite only holding hle_ellamind tasks +_hle_ellamind_registry = Registry() +register_hle_ellamind_tasks(registry=_hle_ellamind_registry) + + +@pytest.mark.formatter_hash +@pytest.mark.parametrize("formatter_cls", [Llama3Formatter, ConcatFormatter]) +@pytest.mark.parametrize("task_name", _hle_ellamind_registry.task_names()) +def test_formatter_hash(task_name: str, formatter_cls: type[BaseFormatter]) -> None: + run_formatter_hash_test(task_name, formatter_cls, registry=_hle_ellamind_registry) + + +# --------------------------------------------------------------------------- +# Prompt spec: build the real benchmark over fictional rows, assert the assembled messages +# --------------------------------------------------------------------------- + +# Fictional rows in the HLE format (NOT real dataset examples). Choices are shuffled deterministically +# (seed: question + correct_answer). Both are multipleChoice rows, so they survive the NATIVE filter and +# render identically to the full variants. +_EVAL_ROW: dict[str, Any] = { + "question": "Was ist die Antwort auf die Frage nach dem Leben, dem Universum und allem?", + "correct_answer": "42", + "incorrect_answers": ["41", "43", "44"], + "answer_type": "multipleChoice", +} +_FEWSHOT_ROW: dict[str, Any] = { + "question": "Was ist die Summe der Innenwinkel eines Dreiecks?", + "correct_answer": "180", + "incorrect_answers": ["90", "270", "360"], + "answer_type": "multipleChoice", +} + +_EVAL_Q = "Frage: Was ist die Antwort auf die Frage nach dem Leben, dem Universum und allem?" +_FEWSHOT_Q = "Frage: Was ist die Summe der Innenwinkel eines Dreiecks?" + + +@dataclass(frozen=True) +class _ExpectedPrompt: + messages: list[Message] + ground_truth: str + possible_completions: list[str] + + +# --- Zero-shot --- +_MC = _ExpectedPrompt( + messages=[ + Message(role=Role.USER, content=f"{_EVAL_Q}\nA. 44\nB. 43\nC. 41\nD. 42\n"), + Message(role=Role.ASSISTANT, content="Antwort:"), + ], + ground_truth=" D", + possible_completions=[" A", " B", " C", " D"], +) +# Cloze/BPB show no options, so the assembled messages are identical; only the scored completions differ. +_CLOZE_MESSAGES = [ + Message(role=Role.USER, content=f"{_EVAL_Q}\n"), + Message(role=Role.ASSISTANT, content="Antwort:"), +] +_CLOZE = _ExpectedPrompt( + messages=_CLOZE_MESSAGES, + ground_truth=" 42", + possible_completions=[" 44", " 43", " 41", " 42"], +) +_BPB = _ExpectedPrompt( + messages=_CLOZE_MESSAGES, + ground_truth=" 42", + possible_completions=[" 42"], # BPB scores only the gold continuation +) + + +@pytest.mark.parametrize( + "make_benchmark, expected", + [ + pytest.param(hle_ellamind_mc_de, _MC, id="mc"), + pytest.param(hle_ellamind_mc_native_de, _MC, id="mc_native"), + pytest.param(hle_ellamind_cloze_de, _CLOZE, id="cloze"), + pytest.param(hle_ellamind_cloze_native_de, _CLOZE, id="cloze_native"), + pytest.param(hle_ellamind_bpb_de, _BPB, id="bpb"), + ], +) +def test_hle_zeroshot_prompt(make_benchmark: Callable[..., Benchmark], expected: _ExpectedPrompt) -> None: + benchmark = make_benchmark(dataset=DatasetStub({"test": [_EVAL_ROW]})) + sample = first_sample(benchmark, num_fewshot=0) + assert sample.messages == expected.messages + assert sample.ground_truth == expected.ground_truth + assert sample.possible_completions == expected.possible_completions + + +# --- One-shot: fewshot row rendered with its answer, then the eval row's own zero-shot prompt --- +_MC_FEWSHOT_MESSAGES = [ + Message(role=Role.USER, content=f"{_FEWSHOT_Q}\nA. 270\nB. 180\nC. 360\nD. 90\n"), + Message(role=Role.ASSISTANT, content="Antwort: B"), +] +_CLOZE_FEWSHOT_MESSAGES = [ + Message(role=Role.USER, content=f"{_FEWSHOT_Q}\n"), + Message(role=Role.ASSISTANT, content="Antwort: 180"), +] + + +def _oneshot(fewshot_messages: list[Message], eval_expected: _ExpectedPrompt) -> _ExpectedPrompt: + return _ExpectedPrompt( + messages=[*fewshot_messages, *eval_expected.messages], + ground_truth=eval_expected.ground_truth, + possible_completions=eval_expected.possible_completions, + ) + + +@pytest.mark.parametrize( + "make_benchmark, expected", + [ + pytest.param(hle_ellamind_mc_de, _oneshot(_MC_FEWSHOT_MESSAGES, _MC), id="mc"), + pytest.param(hle_ellamind_mc_native_de, _oneshot(_MC_FEWSHOT_MESSAGES, _MC), id="mc_native"), + pytest.param(hle_ellamind_cloze_de, _oneshot(_CLOZE_FEWSHOT_MESSAGES, _CLOZE), id="cloze"), + pytest.param(hle_ellamind_cloze_native_de, _oneshot(_CLOZE_FEWSHOT_MESSAGES, _CLOZE), id="cloze_native"), + pytest.param(hle_ellamind_bpb_de, _oneshot(_CLOZE_FEWSHOT_MESSAGES, _BPB), id="bpb"), + ], +) +def test_hle_oneshot_prompt(make_benchmark: Callable[..., Benchmark], expected: _ExpectedPrompt) -> None: + benchmark = make_benchmark(dataset=DatasetStub({"test": [_FEWSHOT_ROW, _EVAL_ROW]})) + sample = first_sample(benchmark, num_fewshot=1) + assert sample.messages == expected.messages + assert sample.ground_truth == expected.ground_truth + assert sample.possible_completions == expected.possible_completions + + +# --------------------------------------------------------------------------- +# NATIVE subset: the native variant keeps only answer_type == "multipleChoice" rows; the full variant keeps all +# --------------------------------------------------------------------------- +def test_hle_native_variant_keeps_only_multiple_choice_rows() -> None: + # Given a dataset mixing multipleChoice and other answer types + rows: list[dict[str, Any]] = [ + { + "question": "Q1", + "correct_answer": "a", + "incorrect_answers": ["x", "y", "z"], + "answer_type": "multipleChoice", + }, + {"question": "Q2", "correct_answer": "a", "incorrect_answers": ["x", "y", "z"], "answer_type": "exactMatch"}, + { + "question": "Q3", + "correct_answer": "a", + "incorrect_answers": ["x", "y", "z"], + "answer_type": "multipleChoice", + }, + ] + native = hle_ellamind_mc_native_de(dataset=DatasetStub({"test": rows})) + full = hle_ellamind_mc_de(dataset=DatasetStub({"test": rows})) + + # When we assemble all samples for each + native_samples = list(native.create(0, None, None, seed=42).iterate_samples()) + full_samples = list(full.create(0, None, None, seed=42).iterate_samples()) + + # Then the native variant drops the non-multipleChoice row (Q2); the full variant keeps all three + assert len(native_samples) == 2 + assert all("Q2" not in sample.messages[0].content for sample in native_samples) + assert len(full_samples) == 3 diff --git a/tests/tests_eval_framework/tasks/benchmarks/task-prompts-hashes.json b/tests/tests_eval_framework/tasks/benchmarks/task-prompts-hashes.json index efe8af8f8..aa582c677 100644 --- a/tests/tests_eval_framework/tasks/benchmarks/task-prompts-hashes.json +++ b/tests/tests_eval_framework/tasks/benchmarks/task-prompts-hashes.json @@ -129,15 +129,13 @@ "HLE_ELLAMIND_CLOZE_DE.ConcatFormatter": "e2cba5768045decaee0008f5da6c150f", "HLE_ELLAMIND_CLOZE_DE.Llama3Formatter": "0ccacce6a6b2a7635b468d323044c01a", "HLE_ELLAMIND_CLOZE_DE.NoStripConcatFormatter": "e2cba5768045decaee0008f5da6c150f", - "HLE_ELLAMIND_CLOZE_NATIVE_DE.ConcatFormatter": "cf536ff11003860b3613fde181521542", - "HLE_ELLAMIND_CLOZE_NATIVE_DE.Llama3Formatter": "b315840e131b2ef5f4441d1634b0ab3c", - "HLE_ELLAMIND_CLOZE_NATIVE_DE.NoStripConcatFormatter": "cf536ff11003860b3613fde181521542", + "HLE_ELLAMIND_CLOZE_NATIVE_DE.ConcatFormatter": "1d4ebe9df4492ba52d2b59d72d3e5166", + "HLE_ELLAMIND_CLOZE_NATIVE_DE.Llama3Formatter": "ebea106202f324ea36202b5f6efffe01", "HLE_ELLAMIND_MC_DE.ConcatFormatter": "5341d9a509b1df21253533edebd4f541", "HLE_ELLAMIND_MC_DE.Llama3Formatter": "bf4b67cfc8603c59b9348c8c8ef8ff2c", "HLE_ELLAMIND_MC_DE.NoStripConcatFormatter": "5341d9a509b1df21253533edebd4f541", - "HLE_ELLAMIND_MC_NATIVE_DE.ConcatFormatter": "492963639f9b08ae42c4fd1656d08be8", - "HLE_ELLAMIND_MC_NATIVE_DE.Llama3Formatter": "0f0f58b6f1e3052d35f54bf75e743756", - "HLE_ELLAMIND_MC_NATIVE_DE.NoStripConcatFormatter": "492963639f9b08ae42c4fd1656d08be8", + "HLE_ELLAMIND_MC_NATIVE_DE.ConcatFormatter": "3af4f151a636ff3af02cb009d91c1746", + "HLE_ELLAMIND_MC_NATIVE_DE.Llama3Formatter": "cac03440f07b73e9cd9e47b19f807312", "HumanEvalBPB.ConcatFormatter": "a87291b6ceeb714b3a6f36ff1e572944", "HumanEvalBPB.Llama3Formatter": "a8356d0d5ffd85c3265a6c16aa603823", "HumanEvalBPB.NoStripConcatFormatter": "092d620f0926dce92d6111e3a51d7051", diff --git a/tests/tests_eval_framework/tasks/benchmarks/test_hle_ellamind.py b/tests/tests_eval_framework/tasks/benchmarks/test_hle_ellamind.py deleted file mode 100644 index 88bde5920..000000000 --- a/tests/tests_eval_framework/tasks/benchmarks/test_hle_ellamind.py +++ /dev/null @@ -1,313 +0,0 @@ -"""Tests for the German HLE (EllaMind) tasks. - -Tests: -- formatter hash test for every HLE variant -- offline prompt assembly tests -- native multiple-choice filtering test (offline) -""" - -from typing import Any - -import pytest - -import eval_framework.tasks.benchmarks.hle_ellamind as hle_ellamind -from eval_framework.tasks.base import BaseTask -from eval_framework.tasks.registry import Registry -from eval_framework.tasks.task_names import register_hle_ellamind_tasks -from template_formatting.formatter import ( - BaseFormatter, - ConcatFormatter, - Llama3Formatter, - Message, - Role, -) -from tests.tests_eval_framework.tasks.benchmarks.utils import ( - ExpectedPrompt, - assert_offline_oneshot_prompt, - assert_offline_zeroshot_prompt, - run_formatter_hash_test, -) - -# Registry for this test suite only holding hle_ellamind tasks -_hle_ellamind_registry = Registry() -register_hle_ellamind_tasks(registry=_hle_ellamind_registry) - -# --------------------------------------------------------------------------- -# Formatter hash tests (Hugging Face) -# --------------------------------------------------------------------------- - - -@pytest.mark.formatter_hash -@pytest.mark.parametrize("formatter_cls", [Llama3Formatter, ConcatFormatter]) -@pytest.mark.parametrize("task_name", _hle_ellamind_registry.task_names()) -def test_formatter_hash(task_name: str, formatter_cls: type[BaseFormatter]) -> None: - run_formatter_hash_test(task_name, formatter_cls, registry=_hle_ellamind_registry) - - -# --------------------------------------------------------------------------- -# Offline prompt assembly tests (use fictional dataset) -# --------------------------------------------------------------------------- - -_SUBJECT = "deu" - -# Fictional rows following the HLE format. NOT real examples from the HLE dataset. -# Option order and correct letters are shuffled deterministically (seed: question+answer). -_EVAL_ROW: dict[str, Any] = { - "question": "Was ist die Antwort auf die Frage nach dem Leben, dem Universum und allem?", - "correct_answer": "42", - "incorrect_answers": ["41", "43", "44"], - "answer_type": "multipleChoice", -} - -_FEWSHOT_ROW: dict[str, Any] = { - "question": "Was ist die Summe der Innenwinkel eines Dreiecks?", - "correct_answer": "180", - "incorrect_answers": ["90", "270", "360"], - "answer_type": "multipleChoice", -} - -# Expected prompts (messages, flat concat, ground truth, completions). -# --- HLE_ELLAMIND_MC_DE --- -_MC_ZEROSHOT = ExpectedPrompt( - messages=[ - Message( - role=Role.USER, - content="Frage: Was ist die Antwort auf die Frage nach dem Leben, dem Universum und allem?\nA. 44\nB. 43\nC. 41\nD. 42\n", - ), - Message(role=Role.ASSISTANT, content="Antwort:"), - ], - concat="""\ -Frage: Was ist die Antwort auf die Frage nach dem Leben, dem Universum und allem? -A. 44 -B. 43 -C. 41 -D. 42 -Antwort:""", - ground_truth=" D", - completions=[" A", " B", " C", " D"], -) - -_MC_FEWSHOT = ExpectedPrompt( - messages=[ - Message( - role=Role.USER, - content="Frage: Was ist die Summe der Innenwinkel eines Dreiecks?\nA. 270\nB. 180\nC. 360\nD. 90\n", - ), - Message(role=Role.ASSISTANT, content="Antwort: B"), - Message( - role=Role.USER, - content="Frage: Was ist die Antwort auf die Frage nach dem Leben, dem Universum und allem?\nA. 44\nB. 43\nC. 41\nD. 42\n", - ), - Message(role=Role.ASSISTANT, content="Antwort:"), - ], - concat="""\ -Frage: Was ist die Summe der Innenwinkel eines Dreiecks? -A. 270 -B. 180 -C. 360 -D. 90 -Antwort: B - -Frage: Was ist die Antwort auf die Frage nach dem Leben, dem Universum und allem? -A. 44 -B. 43 -C. 41 -D. 42 -Antwort:""", - ground_truth=_MC_ZEROSHOT.ground_truth, - completions=_MC_ZEROSHOT.completions, -) - -# --- HLE_ELLAMIND_MC_NATIVE_DE --- -# Same prompt render as MC. -_MC_NATIVE_ZEROSHOT = ExpectedPrompt( - messages=_MC_ZEROSHOT.messages, - concat=_MC_ZEROSHOT.concat, - ground_truth=_MC_ZEROSHOT.ground_truth, - completions=_MC_ZEROSHOT.completions, -) - -_MC_NATIVE_FEWSHOT = ExpectedPrompt( - messages=_MC_FEWSHOT.messages, - concat=_MC_FEWSHOT.concat, - ground_truth=_MC_ZEROSHOT.ground_truth, - completions=_MC_ZEROSHOT.completions, -) - -# --- HLE_ELLAMIND_CLOZE_DE --- -_CLOZE_ZEROSHOT = ExpectedPrompt( - messages=[ - Message( - role=Role.USER, - content="Frage: Was ist die Antwort auf die Frage nach dem Leben, dem Universum und allem?\n", - ), - Message(role=Role.ASSISTANT, content="Antwort:"), - ], - concat="""\ -Frage: Was ist die Antwort auf die Frage nach dem Leben, dem Universum und allem? -Antwort:""", - ground_truth=" 42", - completions=[" 44", " 43", " 41", " 42"], -) - -_CLOZE_FEWSHOT = ExpectedPrompt( - messages=[ - Message(role=Role.USER, content="Frage: Was ist die Summe der Innenwinkel eines Dreiecks?\n"), - Message(role=Role.ASSISTANT, content="Antwort: 180"), - Message( - role=Role.USER, - content="Frage: Was ist die Antwort auf die Frage nach dem Leben, dem Universum und allem?\n", - ), - Message(role=Role.ASSISTANT, content="Antwort:"), - ], - concat="""\ -Frage: Was ist die Summe der Innenwinkel eines Dreiecks? -Antwort: 180 - -Frage: Was ist die Antwort auf die Frage nach dem Leben, dem Universum und allem? -Antwort:""", - ground_truth=_CLOZE_ZEROSHOT.ground_truth, - completions=_CLOZE_ZEROSHOT.completions, -) - -# --- HLE_ELLAMIND_CLOZE_NATIVE_DE --- -# Same prompt render as Cloze. -_CLOZE_NATIVE_ZEROSHOT = ExpectedPrompt( - messages=_CLOZE_ZEROSHOT.messages, - concat=_CLOZE_ZEROSHOT.concat, - ground_truth=_CLOZE_ZEROSHOT.ground_truth, - completions=_CLOZE_ZEROSHOT.completions, -) - -_CLOZE_NATIVE_FEWSHOT = ExpectedPrompt( - messages=_CLOZE_FEWSHOT.messages, - concat=_CLOZE_FEWSHOT.concat, - ground_truth=_CLOZE_ZEROSHOT.ground_truth, - completions=_CLOZE_ZEROSHOT.completions, -) - -# --- HLE_ELLAMIND_BPB_DE --- -# Same prompt as cloze; BPB scores only the gold continuation. -_cloze_ground_truth = _CLOZE_ZEROSHOT.ground_truth -assert isinstance(_cloze_ground_truth, str) # narrow the type: cloze ground_truth is always a str - -_BPB_ZEROSHOT = ExpectedPrompt( - messages=_CLOZE_ZEROSHOT.messages, - concat=_CLOZE_ZEROSHOT.concat, - ground_truth=_cloze_ground_truth, - completions=[_cloze_ground_truth], -) - -_BPB_FEWSHOT = ExpectedPrompt( - messages=_CLOZE_FEWSHOT.messages, - concat=_CLOZE_FEWSHOT.concat, - ground_truth=_cloze_ground_truth, - completions=[_cloze_ground_truth], -) - - -# --- TESTS --- -def test_hle_ellamind_mc_de_offline_prompt_formatting() -> None: - assert_offline_zeroshot_prompt( - hle_ellamind.HLE_ELLAMIND_MC_DE, - eval_row=_EVAL_ROW, - subjects=[_SUBJECT], - expected=_MC_ZEROSHOT, - ) - assert_offline_oneshot_prompt( - hle_ellamind.HLE_ELLAMIND_MC_DE, - eval_row=_EVAL_ROW, - fewshot_row=_FEWSHOT_ROW, - subjects=[_SUBJECT], - expected=_MC_FEWSHOT, - ) - - -def test_hle_ellamind_mc_native_de_offline_prompt_formatting() -> None: - assert_offline_zeroshot_prompt( - hle_ellamind.HLE_ELLAMIND_MC_NATIVE_DE, - eval_row=_EVAL_ROW, - subjects=[_SUBJECT], - expected=_MC_NATIVE_ZEROSHOT, - ) - assert_offline_oneshot_prompt( - hle_ellamind.HLE_ELLAMIND_MC_NATIVE_DE, - eval_row=_EVAL_ROW, - fewshot_row=_FEWSHOT_ROW, - subjects=[_SUBJECT], - expected=_MC_NATIVE_FEWSHOT, - ) - - -def test_hle_ellamind_cloze_de_offline_prompt_formatting() -> None: - assert_offline_zeroshot_prompt( - hle_ellamind.HLE_ELLAMIND_CLOZE_DE, - eval_row=_EVAL_ROW, - subjects=[_SUBJECT], - expected=_CLOZE_ZEROSHOT, - ) - assert_offline_oneshot_prompt( - hle_ellamind.HLE_ELLAMIND_CLOZE_DE, - eval_row=_EVAL_ROW, - fewshot_row=_FEWSHOT_ROW, - subjects=[_SUBJECT], - expected=_CLOZE_FEWSHOT, - ) - - -def test_hle_ellamind_cloze_native_de_offline_prompt_formatting() -> None: - assert_offline_zeroshot_prompt( - hle_ellamind.HLE_ELLAMIND_CLOZE_NATIVE_DE, - eval_row=_EVAL_ROW, - subjects=[_SUBJECT], - expected=_CLOZE_NATIVE_ZEROSHOT, - ) - assert_offline_oneshot_prompt( - hle_ellamind.HLE_ELLAMIND_CLOZE_NATIVE_DE, - eval_row=_EVAL_ROW, - fewshot_row=_FEWSHOT_ROW, - subjects=[_SUBJECT], - expected=_CLOZE_NATIVE_FEWSHOT, - ) - - -def test_hle_ellamind_bpb_de_offline_prompt_formatting() -> None: - assert_offline_zeroshot_prompt( - hle_ellamind.HLE_ELLAMIND_BPB_DE, - eval_row=_EVAL_ROW, - subjects=[_SUBJECT], - expected=_BPB_ZEROSHOT, - ) - assert_offline_oneshot_prompt( - hle_ellamind.HLE_ELLAMIND_BPB_DE, - eval_row=_EVAL_ROW, - fewshot_row=_FEWSHOT_ROW, - subjects=[_SUBJECT], - expected=_BPB_FEWSHOT, - ) - - -# --------------------------------------------------------------------------- -# Native multiple-choice filtering test (offline) -# --------------------------------------------------------------------------- - - -def test_hle_native_variant_filters_to_multiple_choice_rows(monkeypatch: pytest.MonkeyPatch) -> None: - """Native HLE variants should keep only `answer_type == "multipleChoice"`.""" - - def fake_base_load_dataset(self: BaseTask, subject: str) -> None: - _ = subject - self.dataset = { - self.SAMPLE_SPLIT: [ - {"answer_type": "multipleChoice", "question": "Q1"}, - {"answer_type": "shortAnswer", "question": "Q2"}, - {"answer_type": "multipleChoice", "question": "Q3"}, - ] - } - - monkeypatch.setattr(BaseTask, "_load_dataset", fake_base_load_dataset) - task = hle_ellamind.HLE_ELLAMIND_MC_NATIVE_DE(num_fewshot=0) - task._load_dataset("deu") - - assert len(task.dataset[task.SAMPLE_SPLIT]) == 2 - assert all(item["answer_type"] == "multipleChoice" for item in task.dataset[task.SAMPLE_SPLIT])