composed benchmarks

src/eval_framework/benchmarks/arc_de.py

@@ -5,8 +5,8 @@
 from eval_framework.choices import ChoiceFields, ChoiceReader
 from eval_framework.composed import ComposedBenchmark
 from eval_framework.contract import Benchmark
-from eval_framework.subjects import NoSubject
 from eval_framework.tasks.base import Language
+from eval_framework.tasks.dataset_loading import DatasetPolicy
 from eval_framework.tasks.dataset_revisions import pinned_by_framework
 from eval_framework.tasks.task_style import ClozeStyle, answer_key_to_index
 
@@ -28,16 +28,21 @@ def read(self, item: dict[str, Any]) -> ChoiceFields:
         )
 
 
-# ARC-DE as cloze/ranked classification.
-# https://huggingface.co/datasets/LeoLM/ArcChallenge_de
-ARC_DE_BENCHMARK: Benchmark = ComposedBenchmark.compose(
-    id="ARC_DE",
-    display_name="ARC German",
-    styler=ClozeStyle(question_prefix="Frage: ", cue_text="Antwort:"),
-    reader=ArcDeReader(),
-    sample_split="test",
-    fewshot_split="validation",
-    subjects=NoSubject(),
-    dataset_policy=pinned_by_framework("LeoLM/ArcChallenge_de"),
-    language=Language.DEU,
-)
+def arc_de(dataset: DatasetPolicy | None = None) -> Benchmark:
+    """ARC-DE as cloze/ranked classification.
+
+    https://huggingface.co/datasets/LeoLM/ArcChallenge_de
+    """
+    return ComposedBenchmark.compose(
+        id="ARC_DE",
+        display_name="ARC German",
+        styler=ClozeStyle(question_prefix="Frage: ", cue_text="Antwort:"),
+        reader=ArcDeReader(),
+        sample_split="test",
+        fewshot_split="validation",
+        dataset_policy=dataset if dataset is not None else pinned_by_framework("LeoLM/ArcChallenge_de"),
+        language=Language.DEU,
+    )
+
+
+ARC_DE_BENCHMARK: Benchmark = arc_de()

src/eval_framework/benchmarks/csqa_ellamind.py

@@ -12,6 +12,7 @@
 from eval_framework.contract import Benchmark
 from eval_framework.subjects import ListOfSubjects
 from eval_framework.tasks.base import Language
+from eval_framework.tasks.dataset_loading import DatasetPolicy
 from eval_framework.tasks.dataset_revisions import pinned_by_framework
 from eval_framework.tasks.task_style import BPBStyle, ClozeStyle, MCStyle, TaskStyler, shuffle_correct_with_distractors
 
@@ -34,30 +35,49 @@ def read(self, item: dict[str, Any]) -> ChoiceFields:
         return ChoiceFields(raw_question=item["question"], choices=choices, correct_index=correct_index)
 
 
-# One styler per format, all sharing the German prefix/cue. The easy/hard distractor axis belongs to
-# the reader, so it is orthogonal to the styler choice.
-CSQA_ELLAMIND_CLOZE_STYLER = ClozeStyle.for_language(Language.DEU)
-CSQA_ELLAMIND_MC_STYLER = MCStyle.for_language(Language.DEU)
-CSQA_ELLAMIND_BPB_STYLER = BPBStyle.for_language(Language.DEU)
-
-
-def _csqa_ellamind_benchmark(id: str, styler: TaskStyler, distractor_level: Literal["easy", "hard"]) -> Benchmark:
+def _csqa_ellamind_benchmark(
+    id: str, styler: TaskStyler, distractor_level: Literal["easy", "hard"], dataset: DatasetPolicy | None
+) -> Benchmark:
     return ComposedBenchmark.compose(
         id=id,
         styler=styler,
         reader=CsqaReader(distractor_level),
         sample_split="validation",
         fewshot_split="validation",
         subjects=ListOfSubjects(["deu"]),
-        dataset_policy=pinned_by_framework("ellamind/csqa-multilingual"),
+        dataset_policy=dataset if dataset is not None else pinned_by_framework("ellamind/csqa-multilingual"),
         language=Language.DEU,
     )
 
 
+def csqa_ellamind_mc_easy_de(dataset: DatasetPolicy | None = None) -> Benchmark:
+    return _csqa_ellamind_benchmark("CSQA_ELLAMIND_MC_EASY_DE", MCStyle.for_language(Language.DEU), "easy", dataset)
+
+
+def csqa_ellamind_mc_hard_de(dataset: DatasetPolicy | None = None) -> Benchmark:
+    return _csqa_ellamind_benchmark("CSQA_ELLAMIND_MC_HARD_DE", MCStyle.for_language(Language.DEU), "hard", dataset)
+
+
+def csqa_ellamind_cloze_easy_de(dataset: DatasetPolicy | None = None) -> Benchmark:
+    return _csqa_ellamind_benchmark(
+        "CSQA_ELLAMIND_CLOZE_EASY_DE", ClozeStyle.for_language(Language.DEU), "easy", dataset
+    )
+
+
+def csqa_ellamind_cloze_hard_de(dataset: DatasetPolicy | None = None) -> Benchmark:
+    return _csqa_ellamind_benchmark(
+        "CSQA_ELLAMIND_CLOZE_HARD_DE", ClozeStyle.for_language(Language.DEU), "hard", dataset
+    )
+
+
+def csqa_ellamind_bpb_de(dataset: DatasetPolicy | None = None) -> Benchmark:
+    return _csqa_ellamind_benchmark("CSQA_ELLAMIND_BPB_DE", BPBStyle.for_language(Language.DEU), "easy", dataset)
+
+
 CSQA_ELLAMIND_BENCHMARKS: list[Benchmark] = [
-    _csqa_ellamind_benchmark("CSQA_ELLAMIND_MC_EASY_DE", CSQA_ELLAMIND_MC_STYLER, "easy"),
-    _csqa_ellamind_benchmark("CSQA_ELLAMIND_MC_HARD_DE", CSQA_ELLAMIND_MC_STYLER, "hard"),
-    _csqa_ellamind_benchmark("CSQA_ELLAMIND_CLOZE_EASY_DE", CSQA_ELLAMIND_CLOZE_STYLER, "easy"),
-    _csqa_ellamind_benchmark("CSQA_ELLAMIND_CLOZE_HARD_DE", CSQA_ELLAMIND_CLOZE_STYLER, "hard"),
-    _csqa_ellamind_benchmark("CSQA_ELLAMIND_BPB_DE", CSQA_ELLAMIND_BPB_STYLER, "easy"),
+    csqa_ellamind_mc_easy_de(),
+    csqa_ellamind_mc_hard_de(),
+    csqa_ellamind_cloze_easy_de(),
+    csqa_ellamind_cloze_hard_de(),
+    csqa_ellamind_bpb_de(),
 ]

src/eval_framework/benchmarks/gpqa_ellamind.py

@@ -0,0 +1,88 @@
+"""German GPQA (Graduate-level Professional QA, EllaMind) tasks.
+
+https://huggingface.co/datasets/ellamind/gpqa-multilingual
+
+GPQA uses a single distractor set (``incorrect_answers``). The diamond variants restrict evaluation to
+the diamond subset — the 198 hardest questions (``is_diamond``) from the original GPQA-Diamond benchmark.
+"""
+
+from typing import Any, final, override
+
+from eval_framework.choices import ChoiceFields, ChoiceReader
+from eval_framework.composed import ComposedBenchmark
+from eval_framework.contract import Benchmark
+from eval_framework.subjects import ListOfSubjects
+from eval_framework.tasks.base import Language
+from eval_framework.tasks.dataset_loading import DatasetPolicy, Subset
+from eval_framework.tasks.dataset_revisions import pinned_by_framework
+from eval_framework.tasks.task_style import BPBStyle, ClozeStyle, MCStyle, TaskStyler, shuffle_correct_with_distractors
+
+
+@final
+class GpqaReader(ChoiceReader):
+    """Reads a GPQA item: a single ``incorrect_answers`` distractor set, shuffled in with the correct answer."""
+
+    @override
+    def read(self, item: dict[str, Any]) -> ChoiceFields:
+        choices, correct_index = shuffle_correct_with_distractors(
+            correct=item["correct_answer"],
+            distractors=item["incorrect_answers"],
+            seed_text=item["question"] + item["correct_answer"],
+        )
+        return ChoiceFields(raw_question=item["question"], choices=choices, correct_index=correct_index)
+
+
+def _gpqa_ellamind_benchmark(id: str, styler: TaskStyler, dataset: DatasetPolicy | None) -> Benchmark:
+    return ComposedBenchmark.compose(
+        id=id,
+        styler=styler,
+        reader=GpqaReader(),
+        sample_split="train",
+        fewshot_split="train",
+        subjects=ListOfSubjects(["deu"]),
+        dataset_policy=dataset if dataset is not None else pinned_by_framework("ellamind/gpqa-multilingual"),
+        language=Language.DEU,
+    )
+
+
+def _gpqa_ellamind_diamond_benchmark(id: str, styler: TaskStyler, dataset: DatasetPolicy | None) -> Benchmark:
+    source = dataset if dataset is not None else pinned_by_framework("ellamind/gpqa-multilingual")
+    return _gpqa_ellamind_benchmark(id, styler, Subset(source, keep=lambda row: row["is_diamond"]))
+
+
+def gpqa_ellamind_mc_de(dataset: DatasetPolicy | None = None) -> Benchmark:
+    return _gpqa_ellamind_benchmark("GPQA_ELLAMIND_MC_DE", MCStyle.for_language(Language.DEU), dataset)
+
+
+def gpqa_ellamind_cloze_de(dataset: DatasetPolicy | None = None) -> Benchmark:
+    return _gpqa_ellamind_benchmark("GPQA_ELLAMIND_CLOZE_DE", ClozeStyle.for_language(Language.DEU), dataset)
+
+
+def gpqa_ellamind_bpb_de(dataset: DatasetPolicy | None = None) -> Benchmark:
+    return _gpqa_ellamind_benchmark("GPQA_ELLAMIND_BPB_DE", BPBStyle.for_language(Language.DEU), dataset)
+
+
+def gpqa_ellamind_diamond_mc_de(dataset: DatasetPolicy | None = None) -> Benchmark:
+    return _gpqa_ellamind_diamond_benchmark("GPQA_ELLAMIND_DIAMOND_MC_DE", MCStyle.for_language(Language.DEU), dataset)
+
+
+def gpqa_ellamind_diamond_cloze_de(dataset: DatasetPolicy | None = None) -> Benchmark:
+    return _gpqa_ellamind_diamond_benchmark(
+        "GPQA_ELLAMIND_DIAMOND_CLOZE_DE", ClozeStyle.for_language(Language.DEU), dataset
+    )
+
+
+def gpqa_ellamind_diamond_bpb_de(dataset: DatasetPolicy | None = None) -> Benchmark:
+    return _gpqa_ellamind_diamond_benchmark(
+        "GPQA_ELLAMIND_DIAMOND_BPB_DE", BPBStyle.for_language(Language.DEU), dataset
+    )
+
+
+GPQA_ELLAMIND_BENCHMARKS: list[Benchmark] = [
+    gpqa_ellamind_mc_de(),
+    gpqa_ellamind_cloze_de(),
+    gpqa_ellamind_diamond_mc_de(),
+    gpqa_ellamind_diamond_cloze_de(),
+    gpqa_ellamind_bpb_de(),
+    gpqa_ellamind_diamond_bpb_de(),
+]

src/eval_framework/benchmarks/hellaswag_ellamind.py

@@ -0,0 +1,82 @@
+"""German HellaSwag (EllaMind) tasks.
+
+https://huggingface.co/datasets/ellamind/hellaswag-multilingual
+
+HellaSwag is a sentence-completion task: the prompt is a partial sentence (``"{activity}: {context}"``)
+and the model scores full-sentence endings. There is no natural MC variant. HellaSwag supplies separate
+easy and hard distractors.
+"""
+
+from typing import Any, Literal, final, override
+
+from eval_framework.choices import ChoiceFields, ChoiceReader
+from eval_framework.composed import ComposedBenchmark
+from eval_framework.contract import Benchmark
+from eval_framework.subjects import ListOfSubjects
+from eval_framework.tasks.base import Language
+from eval_framework.tasks.dataset_loading import DatasetPolicy
+from eval_framework.tasks.dataset_revisions import pinned_by_framework
+from eval_framework.tasks.task_style import BPBStyle, ClozeStyle, TaskStyler, shuffle_correct_with_distractors
+
+
+@final
+class HellaswagReader(ChoiceReader):
+    """Reads a HellaSwag item: the partial sentence ``"{activity}: {context}"``, with the easy/hard
+    full-sentence endings for the level shuffled in with the correct ending."""
+
+    def __init__(self, distractor_level: Literal["easy", "hard"]) -> None:
+        self._distractor_level = distractor_level
+
+    @override
+    def read(self, item: dict[str, Any]) -> ChoiceFields:
+        distractors = item["easy_distractors"] if self._distractor_level == "easy" else item["hard_distractors"]
+        choices, correct_index = shuffle_correct_with_distractors(
+            correct=item["correct_ending"],
+            distractors=distractors,
+            seed_text=item["context"] + item["correct_ending"],
+        )
+        return ChoiceFields(
+            raw_question=f"{item['activity'].strip()}: {item['context'].strip()}",
+            choices=choices,
+            correct_index=correct_index,
+        )
+
+
+def _hellaswag_ellamind_benchmark(
+    id: str, styler: TaskStyler, distractor_level: Literal["easy", "hard"], dataset: DatasetPolicy | None
+) -> Benchmark:
+    return ComposedBenchmark.compose(
+        id=id,
+        styler=styler,
+        reader=HellaswagReader(distractor_level),
+        sample_split="validation",
+        fewshot_split="validation",
+        subjects=ListOfSubjects(["deu"]),
+        dataset_policy=dataset if dataset is not None else pinned_by_framework("ellamind/hellaswag-multilingual"),
+        language=Language.DEU,
+    )
+
+
+# Sentence-completion: no question prefix, no cue, the continuation follows the context directly.
+def _cloze_completion_style() -> ClozeStyle:
+    return ClozeStyle(question_prefix="", trailing_newline=False, cue_text="")
+
+
+def hellaswag_ellamind_easy_de(dataset: DatasetPolicy | None = None) -> Benchmark:
+    return _hellaswag_ellamind_benchmark("HELLASWAG_ELLAMIND_EASY_DE", _cloze_completion_style(), "easy", dataset)
+
+
+def hellaswag_ellamind_hard_de(dataset: DatasetPolicy | None = None) -> Benchmark:
+    return _hellaswag_ellamind_benchmark("HELLASWAG_ELLAMIND_HARD_DE", _cloze_completion_style(), "hard", dataset)
+
+
+def hellaswag_ellamind_bpb_de(dataset: DatasetPolicy | None = None) -> Benchmark:
+    styler = BPBStyle(question_prefix="", trailing_newline=False, cue_text="")
+    return _hellaswag_ellamind_benchmark("HELLASWAG_ELLAMIND_BPB_DE", styler, "easy", dataset)
+
+
+HELLASWAG_ELLAMIND_BENCHMARKS: list[Benchmark] = [
+    hellaswag_ellamind_easy_de(),
+    hellaswag_ellamind_hard_de(),
+    hellaswag_ellamind_bpb_de(),
+]

src/eval_framework/benchmarks/hle_ellamind.py

@@ -0,0 +1,81 @@
+"""German HLE (Humanity's Last Exam, EllaMind) tasks.
+
+https://huggingface.co/datasets/ellamind/hle-multilingual
+
+HLE uses a single distractor set (``incorrect_answers``). The NATIVE variants restrict evaluation to the
+items that are natively multiple-choice in the original benchmark (``answer_type == "multipleChoice"``).
+"""
+
+from typing import Any, final, override
+
+from eval_framework.choices import ChoiceFields, ChoiceReader
+from eval_framework.composed import ComposedBenchmark
+from eval_framework.contract import Benchmark
+from eval_framework.subjects import ListOfSubjects
+from eval_framework.tasks.base import Language
+from eval_framework.tasks.dataset_loading import DatasetPolicy, Subset
+from eval_framework.tasks.dataset_revisions import pinned_by_framework
+from eval_framework.tasks.task_style import BPBStyle, ClozeStyle, MCStyle, TaskStyler, shuffle_correct_with_distractors
+
+
+@final
+class HleReader(ChoiceReader):
+    """Reads an HLE item: a single ``incorrect_answers`` distractor set, shuffled in with the correct answer."""
+
+    @override
+    def read(self, item: dict[str, Any]) -> ChoiceFields:
+        choices, correct_index = shuffle_correct_with_distractors(
+            correct=item["correct_answer"],
+            distractors=item["incorrect_answers"],
+            seed_text=item["question"] + item["correct_answer"],
+        )
+        return ChoiceFields(raw_question=item["question"], choices=choices, correct_index=correct_index)
+
+
+def _hle_ellamind_benchmark(id: str, styler: TaskStyler, dataset: DatasetPolicy | None) -> Benchmark:
+    return ComposedBenchmark.compose(
+        id=id,
+        styler=styler,
+        reader=HleReader(),
+        sample_split="test",
+        fewshot_split="test",
+        subjects=ListOfSubjects(["deu"]),
+        dataset_policy=dataset if dataset is not None else pinned_by_framework("ellamind/hle-multilingual"),
+        language=Language.DEU,
+    )
+
+
+def _hle_ellamind_native_benchmark(id: str, styler: TaskStyler, dataset: DatasetPolicy | None) -> Benchmark:
+    source = dataset if dataset is not None else pinned_by_framework("ellamind/hle-multilingual")
+    return _hle_ellamind_benchmark(id, styler, Subset(source, keep=lambda row: row["answer_type"] == "multipleChoice"))
+
+
+def hle_ellamind_mc_de(dataset: DatasetPolicy | None = None) -> Benchmark:
+    return _hle_ellamind_benchmark("HLE_ELLAMIND_MC_DE", MCStyle.for_language(Language.DEU), dataset)
+
+
+def hle_ellamind_cloze_de(dataset: DatasetPolicy | None = None) -> Benchmark:
+    return _hle_ellamind_benchmark("HLE_ELLAMIND_CLOZE_DE", ClozeStyle.for_language(Language.DEU), dataset)
+
+
+def hle_ellamind_mc_native_de(dataset: DatasetPolicy | None = None) -> Benchmark:
+    return _hle_ellamind_native_benchmark("HLE_ELLAMIND_MC_NATIVE_DE", MCStyle.for_language(Language.DEU), dataset)
+
+
+def hle_ellamind_cloze_native_de(dataset: DatasetPolicy | None = None) -> Benchmark:
+    return _hle_ellamind_native_benchmark(
+        "HLE_ELLAMIND_CLOZE_NATIVE_DE", ClozeStyle.for_language(Language.DEU), dataset
+    )
+
+
+def hle_ellamind_bpb_de(dataset: DatasetPolicy | None = None) -> Benchmark:
+    return _hle_ellamind_benchmark("HLE_ELLAMIND_BPB_DE", BPBStyle.for_language(Language.DEU), dataset)
+
+
+HLE_ELLAMIND_BENCHMARKS: list[Benchmark] = [
+    hle_ellamind_mc_de(),
+    hle_ellamind_cloze_de(),
+    hle_ellamind_mc_native_de(),
+    hle_ellamind_cloze_native_de(),
+    hle_ellamind_bpb_de(),
+]