diff --git a/docs/benchmarks_and_metrics.md b/docs/benchmarks_and_metrics.md index 1c76429cd..d81db67c0 100644 --- a/docs/benchmarks_and_metrics.md +++ b/docs/benchmarks_and_metrics.md @@ -7,7 +7,7 @@ Currently, the framework covers a wide range of pre-training and post-training b | **Task** | **Capability** | **Benchmarks** | **Long Context** | | ------------------- | -------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------- | | Logical Reasoning | Math | `AIME2024`, `AIME2025`, `AIME2026`, `GSM8K`, `GSM8K_EU20_DE`, `GSM8K_EU20_FR`, `GSM8KEvalHarness`, `GSM8KReasoning`, `MATH`, `MATH500`, `MATH500Minerva`, `MATHLvl5`, `MATHMinerva`, `MATHMinervaBPB`, `MATHMinervaEvalHarness`, `TableBench` | `InfiniteBench_MathFind` | -| Logical Reasoning | Programming | `BigCodeBench`, `BigCodeBenchHard`, `BigCodeBenchInstruct`, `BigCodeBenchHardInstruct`, `HumanEval`, `HumanEvalInstruct`, `MBPP`, `MBPP_PROMPT_WITHOUT_TESTS`, `MBPP_SANITIZED`, `MBPP_PROMPT_WITHOUT_TESTS_SANITIZED` | `InfiniteBench_CodeRun` | +| Logical Reasoning | Programming | `BigCodeBench`, `BigCodeBenchHard`, `BigCodeBenchInstruct`, `BigCodeBenchHardInstruct`, `HumanEval`, `HumanEvalInstruct`, `MBPP`, `MBPP_PROMPT_WITHOUT_TESTS` | `InfiniteBench_CodeRun` | | Logical Reasoning | Puzzle | `SPHYR` | | | Output Control | Structure | `IFEval`, `IFEvalDe`, `IFEvalFiSv`, `RenderableStructEval`, `StructEval` | | | Text Distillation | Aggregation | | `ZERO_SCROLLS_SPACE_DIGEST` | @@ -22,7 +22,7 @@ Currently, the framework covers a wide range of pre-training and post-training b | **Task** | **Capability** | **Benchmarks** | **Long Context** | | ----------------- | -------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------- | | Output Control | Bias | `WINOGENDER` | | -| Text Distillation | Classification | `ARC`, `ARC_DE`, `ARC_EU20_DE`, `ARC_EU20_FR`, `ARC_FI`, `ARC_IDK`, `BELEBELE`, `ChemBench`, `CommonsenseQACloze`, `CommonsenseQAMC`, `CommonsenseQAFullTextCloze`, `FullTextMMLU`, `GlobalMMLU`, `GlobalMMLU_German`, `GPQA`, `GPQA_IDK`, `INCLUDE`, `MMLU`, `MMLU_DE`, `MMLU_EU20_DE`, `MMLU_EU20_FR`, `MMMLU`, `MMMLU_German`, `MMLU_PRO`, `MMLU_PRO_IDK`, `OPENBOOKQA`, `OPENBOOKQA_IDK`, `PIQA`, `PIQA_IDK`, `SCIQ`, `SCIQEvalHarness`, `SocialIQACloze`, `SocialIQAMC`, `SocialIQAMC_OLMES`, `TRUTHFULQA`, `TRUTHFULQA_EU20_DE`, `TRUTHFULQA_EU20_FR`, `TRUTHFULQA_IDK` | | +| Text Distillation | Classification | `ARC`, `ARC_DE`, `ARC_EU20_DE`, `ARC_EU20_FR`, `ARC_FI`, `ARC_IDK`, `BELEBELE`, `ChemBench`, `CommonsenseQACloze`, `CommonsenseQAMC`, `CommonsenseQAFullTextCloze`, `FullTextMMLU`, `GlobalMMLU`, `GPQA`, `GPQA_IDK`, `INCLUDE`, `MMLU`, `MMLU_DE`, `MMLU_EU20_DE`, `MMLU_EU20_FR`, `MMMLU`, `MMMLU_German`, `MMLU_PRO`, `MMLU_PRO_IDK`, `OPENBOOKQA`, `OPENBOOKQA_IDK`, `PIQA`, `PIQA_IDK`, `SCIQ`, `SCIQEvalHarness`, `SocialIQACloze`, `SocialIQAMC`, `SocialIQAMC_OLMES`, `TRUTHFULQA`, `TRUTHFULQA_EU20_DE`, `TRUTHFULQA_EU20_FR`, `TRUTHFULQA_IDK` | | | Text Distillation | QA | `DropCloze`, `DropMC`, `LabBenchCloze`, `LabBenchMC`, `MedQACloze`, `MedQAMC`, `NaturalQsOpen`, `NaturalQsOpenCloze`, `NaturalQsOpenMC` | `QUALITY`, `ZERO_SCROLLS_QUALITY` | | Text Generation | Open QA | `CASEHOLD` | | | Logical Reasoning | Closed QA | | `InfiniteBench_EnMC` | diff --git a/src/eval_framework/tasks/benchmarks/global_mmlu.py b/src/eval_framework/tasks/benchmarks/global_mmlu.py index da3985039..ee12447f6 100644 --- a/src/eval_framework/tasks/benchmarks/global_mmlu.py +++ b/src/eval_framework/tasks/benchmarks/global_mmlu.py @@ -533,10 +533,3 @@ def _get_ground_truth(self, item: dict[str, Any]) -> str | None: def _get_possible_completions(self, item: dict[str, Any]) -> list[str] | None: return [f" {key}" for key in self.keys] - - -class GlobalMMLU_German(GlobalMMLU): - REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE - NAME = "GlobalMMLU_German" - SUBJECTS = [("de", subject) for subject in MMLU_SUBJECTS] - LANGUAGE = Language.DEU diff --git a/src/eval_framework/tasks/benchmarks/gpqa.py b/src/eval_framework/tasks/benchmarks/gpqa.py index e63f91c13..5488abde6 100644 --- a/src/eval_framework/tasks/benchmarks/gpqa.py +++ b/src/eval_framework/tasks/benchmarks/gpqa.py @@ -177,6 +177,8 @@ class GPQA_COT(GPQA): NAME = "GPQA_COT" RESPONSE_TYPE = ResponseType.COMPLETION METRICS = [AccuracyCompletion] + # "gpqa_diamond" can be selected via task_subjects=["gpqa_diamond"] + SUBJECTS = ["gpqa_extended", "gpqa_diamond"] ANS_RE = re.compile(r"Therefore, the answer is \(([ABCDEFGHIJ])\)") def __init__(self, num_fewshot: int = 0) -> None: @@ -228,8 +230,3 @@ def _get_ground_truth(self, item: dict[str, Any]) -> str | None: # index 1 selects the letter answer_key = choices[correct_answer_position][1] return answer_key - - -class GPQA_DIAMOND_COT(GPQA_COT): - NAME = "GPQA_DIAMOND_COT" - SUBJECTS = ["gpqa_diamond"] diff --git a/src/eval_framework/tasks/benchmarks/mbpp.py b/src/eval_framework/tasks/benchmarks/mbpp.py index 052ecdd81..39e10f0f9 100644 --- a/src/eval_framework/tasks/benchmarks/mbpp.py +++ b/src/eval_framework/tasks/benchmarks/mbpp.py @@ -145,11 +145,6 @@ def _get_possible_completions(self, item: dict[str, Any]) -> list[str] | None: return [gt] if gt else None -class MBPP_SANITIZED(MBPP): - NAME = "MBPP_SANITZED" - SUBJECTS = ["sanitized"] - - class MBPP_PROMPT_WITHOUT_TESTS(MBPP): """ MBPP provides both the problem statement and the test cases upfront. It says, "Here's the problem and here are the @@ -211,11 +206,6 @@ def post_process_generated_completion(self, completion_text: str, sample: Sample return function_header + code -class MBPP_PROMPT_WITHOUT_TESTS_SANITIZED(MBPP_PROMPT_WITHOUT_TESTS): - NAME = "MBPP_PROMPT_WITHOUT_TESTS_SANITIZED" - SUBJECTS = ["sanitized"] - - _OLMES_FEWSHOT_EXAMPLES: list[dict[str, Any]] = [ { "text": "Write a function to find the similar elements from the given two tuple lists.", diff --git a/src/eval_framework/tasks/task_names.py b/src/eval_framework/tasks/task_names.py index 401b076e5..8a5fd62b2 100644 --- a/src/eval_framework/tasks/task_names.py +++ b/src/eval_framework/tasks/task_names.py @@ -81,7 +81,7 @@ def register_piqa_tasks(registry: Registry) -> None: def register_gpqa_tasks(registry: Registry) -> None: """Register gpqa benchmark tasks.""" register_lazy_task("eval_framework.tasks.benchmarks.gpqa.GPQA_OLMES", registry=registry) - register_lazy_task("eval_framework.tasks.benchmarks.gpqa.GPQA_DIAMOND_COT", registry=registry) + register_lazy_task("eval_framework.tasks.benchmarks.gpqa.GPQA_COT", registry=registry) def register_gsm8k_tasks(registry: Registry) -> None: @@ -181,7 +181,6 @@ def register_mmlu_pro_tasks(registry: Registry) -> None: def register_global_mmlu_tasks(registry: Registry) -> None: """Register global_mmlu benchmark tasks.""" register_lazy_task("eval_framework.tasks.benchmarks.global_mmlu.GlobalMMLU", registry=registry) - register_lazy_task("eval_framework.tasks.benchmarks.global_mmlu.GlobalMMLU_German", registry=registry) def register_sciq_tasks(registry: Registry) -> None: diff --git a/tests/tests_eval_framework/tasks/benchmarks/task-prompts-hashes.json b/tests/tests_eval_framework/tasks/benchmarks/task-prompts-hashes.json index 9fb7a8705..21470f9e1 100644 --- a/tests/tests_eval_framework/tasks/benchmarks/task-prompts-hashes.json +++ b/tests/tests_eval_framework/tasks/benchmarks/task-prompts-hashes.json @@ -68,9 +68,8 @@ "GOLDENSWAG_IDK.ConcatFormatter": "59ce1a466cbe4e53055e3185d658e4d7", "GOLDENSWAG_IDK.Llama3Formatter": "2f0bf16b82db9a645d9a1c9fb710c0d1", "GOLDENSWAG_IDK.NoStripConcatFormatter": "59ce1a466cbe4e53055e3185d658e4d7", - "GPQA_DIAMOND_COT.ConcatFormatter": "02dab00b8b83f736a40318e1bd1dfe07", - "GPQA_DIAMOND_COT.Llama3Formatter": "528f1dd854a642a0f7ca5ecd3e14c330", - "GPQA_DIAMOND_COT.NoStripConcatFormatter": "02dab00b8b83f736a40318e1bd1dfe07", + "GPQA_COT.ConcatFormatter": "18aa0d783c0047205791e9d61b0deeef", + "GPQA_COT.Llama3Formatter": "b821534cc47354f27e5c0457a96135bd", "GPQA_ELLAMIND_BPB_DE.ConcatFormatter": "0508ba54e6c240045bbe5fd9d67b574d", "GPQA_ELLAMIND_BPB_DE.Llama3Formatter": "038fba029ca2811659fd2a0140e097b9", "GPQA_ELLAMIND_BPB_DE.NoStripConcatFormatter": "0508ba54e6c240045bbe5fd9d67b574d", @@ -107,9 +106,6 @@ "GlobalMMLU.ConcatFormatter": "85ea2761878e7bf61a2d9c0ed44a460e", "GlobalMMLU.Llama3Formatter": "bf8fff177651c02c54ae5bde8336c9ce", "GlobalMMLU.NoStripConcatFormatter": "85ea2761878e7bf61a2d9c0ed44a460e", - "GlobalMMLU_German.ConcatFormatter": "f6c08eeec8ba39b04937bba8717b14a3", - "GlobalMMLU_German.Llama3Formatter": "73f90c386bf7fb3d35435f227b3dbda8", - "GlobalMMLU_German.NoStripConcatFormatter": "f6c08eeec8ba39b04937bba8717b14a3", "GridDifference.EXTRACTED_GRID": "c3463fc837dbf31c6d20e4c6a135f14f", "HELLASWAG.ConcatFormatter": "fb0db820785df792f9bb2c948d7fb4d6", "HELLASWAG.Llama3Formatter": "9b88a5a4b97c3287ef9750918a7d38e0",