From 9a6138decdb6a57dea167a8808428f56b38b2547 Mon Sep 17 00:00:00 2001 From: koen Date: Fri, 21 Aug 2026 15:26:38 +0000 Subject: [PATCH 1/2] Implement Bayesian accuracy --- src/eval_framework/evaluation_generator.py | 9 ++- src/eval_framework/metrics/base.py | 4 ++ .../loglikelihood/accuracy_loglikelihood.py | 62 +++++++++++++++++++ src/eval_framework/tasks/benchmarks/arc.py | 9 ++- src/eval_framework/tasks/benchmarks/arc_de.py | 8 ++- src/eval_framework/tasks/benchmarks/copa.py | 4 +- src/eval_framework/tasks/benchmarks/csqa.py | 15 ++++- src/eval_framework/tasks/benchmarks/drop.py | 3 + .../tasks/benchmarks/global_mmlu.py | 8 ++- .../tasks/benchmarks/goldenswag.py | 2 + src/eval_framework/tasks/benchmarks/gpqa.py | 4 +- .../tasks/benchmarks/hellaswag.py | 9 ++- src/eval_framework/tasks/benchmarks/medqa.py | 8 ++- src/eval_framework/tasks/benchmarks/mmlu.py | 16 ++++- .../tasks/benchmarks/mmlu_pro.py | 4 +- src/eval_framework/tasks/benchmarks/piqa.py | 9 ++- src/eval_framework/tasks/benchmarks/sciq.py | 12 +++- .../tasks/benchmarks/social_iqa.py | 2 + .../tasks/benchmarks/winogrande.py | 9 ++- src/eval_framework/tasks/task_style.py | 3 + .../metrics/test_accuracy_loglikelihood.py | 62 ++++++++++++++++++- .../tasks/test_task_style.py | 7 +++ tests/tests_eval_framework/test_end_to_end.py | 6 +- 23 files changed, 254 insertions(+), 21 deletions(-) diff --git a/src/eval_framework/evaluation_generator.py b/src/eval_framework/evaluation_generator.py index 00a7e5f5f..fe1963b84 100644 --- a/src/eval_framework/evaluation_generator.py +++ b/src/eval_framework/evaluation_generator.py @@ -1,5 +1,6 @@ import logging import math +from typing import Any, cast import numpy as np import pandas as pd @@ -49,18 +50,20 @@ def _run_metric_calculators(self, responses: list[Completion | Loglikelihood]) - """ llm_judge = None for metric_class in self.metrics: - metric: BaseMetric + raw_metric: BaseMetric[Any] if issubclass(metric_class, BaseLLMJudgeMetric): if llm_judge is None: llm_judge = self.config.llm_judge() - metric = metric_class( + raw_metric = metric_class( llm_judge=llm_judge, randomize_order=self.config.randomize_judge_order, ) else: - metric = metric_class() + raw_metric = metric_class() + metric = cast(BaseMetric[Completion | Loglikelihood], raw_metric) metric.fail_on_error = self.config.fail_on_error + metric.prepare(responses) logger.info(f"Starting calculation of {metric.NAME}") for response in tqdm(responses, desc=f"Calculating {metric.NAME}", disable=get_disable_bar_flag()): if f"{response.subject}_{response.id}_{metric.__class__.__name__}" in subject_result_id_existing: diff --git a/src/eval_framework/metrics/base.py b/src/eval_framework/metrics/base.py index b5afa076f..bb33e0e54 100644 --- a/src/eval_framework/metrics/base.py +++ b/src/eval_framework/metrics/base.py @@ -48,6 +48,10 @@ def NAMES(cls) -> list[str]: def calculate(self, response: Response) -> list[MetricResult]: raise NotImplementedError + def prepare(self, responses: list[Response]) -> None: + """Prepare metric before calculating per-response results. + This is needed for metrics that depend on variables derived from all of the responses.""" + def _record_or_raise(self, exc: Exception) -> list[MetricResult]: """Infra failure (e.g. a Docker image-pull rate limit): abort when fail_on_error is set, otherwise record a per-sample error so the run continues.""" diff --git a/src/eval_framework/metrics/loglikelihood/accuracy_loglikelihood.py b/src/eval_framework/metrics/loglikelihood/accuracy_loglikelihood.py index 69ba7d3ce..c9a3850e2 100644 --- a/src/eval_framework/metrics/loglikelihood/accuracy_loglikelihood.py +++ b/src/eval_framework/metrics/loglikelihood/accuracy_loglikelihood.py @@ -1,5 +1,8 @@ +import numpy as np + from eval_framework.metrics.base import BaseMetric, MetricResult from eval_framework.shared.types import Loglikelihood +from eval_framework.utils.helpers import count_bytes class AccuracyLoglikelihood(BaseMetric[Loglikelihood]): @@ -51,6 +54,65 @@ def calculate(self, response: Loglikelihood) -> list[MetricResult]: ] +class AccuracyBayesianLoglikelihood(BaseMetric[Loglikelihood]): + """Accuracy after adjusting the loglikelihoods for the byte-length bias of the completion. + See https://arxiv.org/html/2607.12767v1 for more details. + """ + + NAME = "Accuracy Bayesian Loglikelihood" + + def __init__(self) -> None: + self.length_decay = 0.0 + + def prepare(self, responses: list[Loglikelihood]) -> None: + """Estimating the length decay factor. + See Equation (24) in https://arxiv.org/html/2607.12767v1 + """ + numerator = 0.0 + denominator = 0.0 + + for response in responses: + if response.error is not None: + continue + + num_candidates = len(response.loglikelihoods) + if num_candidates <= 1: + continue + + lengths = np.array([count_bytes(completion) for completion in response.loglikelihoods], dtype=float) + loglikelihoods = np.array(list(response.loglikelihoods.values()), dtype=float) + length_differences = lengths - np.mean(lengths) + loglikelihood_differences = loglikelihoods - np.mean(loglikelihoods) + + local_denominator = num_candidates * np.sum(length_differences**2) + if local_denominator == 0: + continue + + numerator += float(num_candidates * np.sum(length_differences * loglikelihood_differences)) + denominator += float(local_denominator) + + self.length_decay = 0.0 if denominator == 0 else numerator / denominator + + def calculate(self, response: Loglikelihood) -> list[MetricResult]: + if response.error is not None: + return [MetricResult(metric_name=self.NAME, value=None, higher_is_better=True, error=response.error)] + + corrected_loglikelihoods = { + completion: loglikelihood - self.length_decay * count_bytes(completion) + for completion, loglikelihood in response.loglikelihoods.items() + } + completion_text = max(corrected_loglikelihoods, key=corrected_loglikelihoods.get) # type: ignore[arg-type] + + return [ + MetricResult( + metric_name=self.NAME, + value=float(completion_text in response.ground_truth_list), + higher_is_better=True, + error=response.error, + ) + ] + + class PartialEvalAccuracy(BaseMetric[Loglikelihood]): """An accuracy metric for partial evaluation tasks, e.g. WinograndeCloze. diff --git a/src/eval_framework/tasks/benchmarks/arc.py b/src/eval_framework/tasks/benchmarks/arc.py index bfe1cf066..325ff23f7 100644 --- a/src/eval_framework/tasks/benchmarks/arc.py +++ b/src/eval_framework/tasks/benchmarks/arc.py @@ -1,6 +1,7 @@ from typing import Any from eval_framework.metrics.loglikelihood.accuracy_loglikelihood import ( + AccuracyBayesianLoglikelihood, AccuracyLoglikelihood, AccuracyNormLoglikelihood, ) @@ -23,7 +24,12 @@ class ARC(BaseTask[str]): SAMPLE_SPLIT = "test" FEWSHOT_SPLIT = "train" RESPONSE_TYPE = ResponseType.LOGLIKELIHOODS - METRICS = [AccuracyLoglikelihood, AccuracyNormLoglikelihood, BitsPerByteLoglikelihood] + METRICS = [ + AccuracyLoglikelihood, + AccuracyNormLoglikelihood, + AccuracyBayesianLoglikelihood, + BitsPerByteLoglikelihood, + ] SUBJECTS = ["ARC-Easy", "ARC-Challenge"] LANGUAGE = Language.ENG @@ -85,6 +91,7 @@ class ARC_IDK(ARC): METRICS = [ AccuracyLoglikelihood, AccuracyNormLoglikelihood, + AccuracyBayesianLoglikelihood, ConfidenceWeightedAccuracy, DistributionalCorrectnessScore, TernaryScore, diff --git a/src/eval_framework/tasks/benchmarks/arc_de.py b/src/eval_framework/tasks/benchmarks/arc_de.py index fcf4b00a7..9ce194a76 100644 --- a/src/eval_framework/tasks/benchmarks/arc_de.py +++ b/src/eval_framework/tasks/benchmarks/arc_de.py @@ -1,6 +1,7 @@ from typing import Any from eval_framework.metrics.loglikelihood.accuracy_loglikelihood import ( + AccuracyBayesianLoglikelihood, AccuracyLoglikelihood, AccuracyNormLoglikelihood, ) @@ -20,7 +21,12 @@ class ARC_DE(BaseTask[str]): SAMPLE_SPLIT = "test" FEWSHOT_SPLIT = "validation" RESPONSE_TYPE = ResponseType.LOGLIKELIHOODS - METRICS = [AccuracyLoglikelihood, AccuracyNormLoglikelihood, BitsPerByteLoglikelihood] + METRICS = [ + AccuracyLoglikelihood, + AccuracyNormLoglikelihood, + AccuracyBayesianLoglikelihood, + BitsPerByteLoglikelihood, + ] SUBJECTS = [NO_SUBJECT] LANGUAGE = Language.DEU diff --git a/src/eval_framework/tasks/benchmarks/copa.py b/src/eval_framework/tasks/benchmarks/copa.py index d8ecf21a0..ed4951a18 100644 --- a/src/eval_framework/tasks/benchmarks/copa.py +++ b/src/eval_framework/tasks/benchmarks/copa.py @@ -1,6 +1,7 @@ from typing import Any from eval_framework.metrics.loglikelihood.accuracy_loglikelihood import ( + AccuracyBayesianLoglikelihood, AccuracyLoglikelihood, AccuracyNormLoglikelihood, ) @@ -24,7 +25,7 @@ class COPAEvalHarness(BaseTask[str]): SAMPLE_SPLIT = "validation" # 100 examples (same split as lm-eval) FEWSHOT_SPLIT = "test" # 500 examples RESPONSE_TYPE = ResponseType.LOGLIKELIHOODS - METRICS = [AccuracyLoglikelihood, AccuracyNormLoglikelihood] + METRICS = [AccuracyLoglikelihood, AccuracyNormLoglikelihood, AccuracyBayesianLoglikelihood] SUBJECTS = ["copa"] LANGUAGE = Language.ENG @@ -94,6 +95,7 @@ class COPA_IDKEvalHarness(COPAEvalHarness): METRICS = [ AccuracyLoglikelihood, AccuracyNormLoglikelihood, + AccuracyBayesianLoglikelihood, ConfidenceWeightedAccuracy, DistributionalCorrectnessScore, TernaryScore, diff --git a/src/eval_framework/tasks/benchmarks/csqa.py b/src/eval_framework/tasks/benchmarks/csqa.py index f718a44ab..ea3ea2882 100644 --- a/src/eval_framework/tasks/benchmarks/csqa.py +++ b/src/eval_framework/tasks/benchmarks/csqa.py @@ -1,6 +1,7 @@ from typing import Any from eval_framework.metrics.loglikelihood.accuracy_loglikelihood import ( + AccuracyBayesianLoglikelihood, AccuracyLoglikelihood, AccuracyNormLoglikelihood, ) @@ -20,7 +21,12 @@ class CommonsenseQACloze(BaseTask[str]): SAMPLE_SPLIT = "validation" FEWSHOT_SPLIT = "validation" RESPONSE_TYPE = ResponseType.LOGLIKELIHOODS - METRICS = [AccuracyLoglikelihood, AccuracyNormLoglikelihood, BitsPerByteLoglikelihood] + METRICS = [ + AccuracyLoglikelihood, + AccuracyNormLoglikelihood, + AccuracyBayesianLoglikelihood, + BitsPerByteLoglikelihood, + ] SUBJECTS = [NO_SUBJECT] LANGUAGE = Language.ENG @@ -57,7 +63,12 @@ class CommonsenseQAFullTextCloze(CommonsenseQACloze): REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE NAME = "CommonsenseQAFullTextCloze" - METRICS = [AccuracyLoglikelihood, AccuracyNormLoglikelihood, BitsPerByteLoglikelihood] + METRICS = [ + AccuracyLoglikelihood, + AccuracyNormLoglikelihood, + AccuracyBayesianLoglikelihood, + BitsPerByteLoglikelihood, + ] def _get_ground_truth(self, item: dict[str, Any]) -> str | None: correct_label = item["answerKey"] diff --git a/src/eval_framework/tasks/benchmarks/drop.py b/src/eval_framework/tasks/benchmarks/drop.py index 5bc6a25b5..55bb40b95 100644 --- a/src/eval_framework/tasks/benchmarks/drop.py +++ b/src/eval_framework/tasks/benchmarks/drop.py @@ -5,6 +5,7 @@ DropMetricContext, ) from eval_framework.metrics.loglikelihood.accuracy_loglikelihood import ( + AccuracyBayesianLoglikelihood, AccuracyLoglikelihood, AccuracyNormLoglikelihood, ) @@ -174,6 +175,7 @@ class DropMC(BaseTask[str]): METRICS = [ AccuracyLoglikelihood, AccuracyNormLoglikelihood, + AccuracyBayesianLoglikelihood, BitsPerByteLoglikelihood, ] SUBJECTS = [NO_SUBJECT] @@ -245,6 +247,7 @@ class DropCloze(BaseTask[str]): METRICS = [ AccuracyLoglikelihood, AccuracyNormLoglikelihood, + AccuracyBayesianLoglikelihood, BitsPerByteLoglikelihood, ] SUBJECTS = [NO_SUBJECT] diff --git a/src/eval_framework/tasks/benchmarks/global_mmlu.py b/src/eval_framework/tasks/benchmarks/global_mmlu.py index da3985039..3da5b390d 100644 --- a/src/eval_framework/tasks/benchmarks/global_mmlu.py +++ b/src/eval_framework/tasks/benchmarks/global_mmlu.py @@ -3,6 +3,7 @@ from typing import Any from eval_framework.metrics.loglikelihood.accuracy_loglikelihood import ( + AccuracyBayesianLoglikelihood, AccuracyLoglikelihood, AccuracyNormLoglikelihood, ) @@ -477,7 +478,12 @@ class GlobalMMLU(BaseTask[tuple[str, str]]): SAMPLE_SPLIT = "test" FEWSHOT_SPLIT = "dev" RESPONSE_TYPE = ResponseType.LOGLIKELIHOODS - METRICS = [AccuracyLoglikelihood, AccuracyNormLoglikelihood, BitsPerByteLoglikelihood] + METRICS = [ + AccuracyLoglikelihood, + AccuracyNormLoglikelihood, + AccuracyBayesianLoglikelihood, + BitsPerByteLoglikelihood, + ] SUBJECTS = list(product(GLOBAL_MMLU_LANGUAGES, MMLU_SUBJECTS)) LANGUAGE: Language | dict[str, Language] | None = { str((lang_code.split("_")[0], subject)): LANGUAGE_NAME_MAP[lang_code] diff --git a/src/eval_framework/tasks/benchmarks/goldenswag.py b/src/eval_framework/tasks/benchmarks/goldenswag.py index aae6b5209..28077231f 100644 --- a/src/eval_framework/tasks/benchmarks/goldenswag.py +++ b/src/eval_framework/tasks/benchmarks/goldenswag.py @@ -1,6 +1,7 @@ from typing import Any from eval_framework.metrics.loglikelihood.accuracy_loglikelihood import ( + AccuracyBayesianLoglikelihood, AccuracyLoglikelihood, AccuracyNormLoglikelihood, ) @@ -29,6 +30,7 @@ class GOLDENSWAG_IDK(GOLDENSWAG): METRICS = [ AccuracyLoglikelihood, AccuracyNormLoglikelihood, + AccuracyBayesianLoglikelihood, ConfidenceWeightedAccuracy, DistributionalCorrectnessScore, TernaryScore, diff --git a/src/eval_framework/tasks/benchmarks/gpqa.py b/src/eval_framework/tasks/benchmarks/gpqa.py index e63f91c13..33324f0c8 100644 --- a/src/eval_framework/tasks/benchmarks/gpqa.py +++ b/src/eval_framework/tasks/benchmarks/gpqa.py @@ -6,6 +6,7 @@ from eval_framework.metrics.completion.accuracy_completion import AccuracyCompletion from eval_framework.metrics.loglikelihood.accuracy_loglikelihood import ( + AccuracyBayesianLoglikelihood, AccuracyLoglikelihood, AccuracyNormLoglikelihood, ) @@ -29,7 +30,7 @@ class GPQA(BaseTask[str]): SAMPLE_SPLIT = "train" FEWSHOT_SPLIT = "train" RESPONSE_TYPE = ResponseType.LOGLIKELIHOODS - METRICS = [AccuracyLoglikelihood, AccuracyNormLoglikelihood] + METRICS = [AccuracyLoglikelihood, AccuracyNormLoglikelihood, AccuracyBayesianLoglikelihood] SUBJECTS = ["gpqa_extended"] # ["gpqa_diamond", "gpqa_extended", "gpqa_main", "gpqa_experts"] LANGUAGE = Language.ENG @@ -156,6 +157,7 @@ class GPQA_IDK(GPQA): METRICS = [ AccuracyLoglikelihood, AccuracyNormLoglikelihood, + AccuracyBayesianLoglikelihood, ConfidenceWeightedAccuracy, DistributionalCorrectnessScore, TernaryScore, diff --git a/src/eval_framework/tasks/benchmarks/hellaswag.py b/src/eval_framework/tasks/benchmarks/hellaswag.py index 962ceba76..af3010252 100644 --- a/src/eval_framework/tasks/benchmarks/hellaswag.py +++ b/src/eval_framework/tasks/benchmarks/hellaswag.py @@ -2,6 +2,7 @@ from typing import Any from eval_framework.metrics.loglikelihood.accuracy_loglikelihood import ( + AccuracyBayesianLoglikelihood, AccuracyLoglikelihood, AccuracyNormLoglikelihood, ) @@ -24,7 +25,12 @@ class HELLASWAG(BaseTask[str]): SAMPLE_SPLIT = "validation" FEWSHOT_SPLIT = "train" RESPONSE_TYPE = ResponseType.LOGLIKELIHOODS - METRICS = [AccuracyLoglikelihood, AccuracyNormLoglikelihood, BitsPerByteLoglikelihood] + METRICS = [ + AccuracyLoglikelihood, + AccuracyNormLoglikelihood, + AccuracyBayesianLoglikelihood, + BitsPerByteLoglikelihood, + ] SUBJECTS = [NO_SUBJECT] LANGUAGE = Language.ENG @@ -62,6 +68,7 @@ class HELLASWAG_IDK(HELLASWAG): METRICS = [ AccuracyLoglikelihood, AccuracyNormLoglikelihood, + AccuracyBayesianLoglikelihood, ConfidenceWeightedAccuracy, DistributionalCorrectnessScore, TernaryScore, diff --git a/src/eval_framework/tasks/benchmarks/medqa.py b/src/eval_framework/tasks/benchmarks/medqa.py index 9354f701e..4e7987553 100644 --- a/src/eval_framework/tasks/benchmarks/medqa.py +++ b/src/eval_framework/tasks/benchmarks/medqa.py @@ -5,6 +5,7 @@ from typing import Any from eval_framework.metrics.loglikelihood.accuracy_loglikelihood import ( + AccuracyBayesianLoglikelihood, AccuracyLoglikelihood, AccuracyNormLoglikelihood, ) @@ -24,7 +25,12 @@ class MedQACloze(BaseTask[str]): SAMPLE_SPLIT = "test" FEWSHOT_SPLIT = "dev" RESPONSE_TYPE = ResponseType.LOGLIKELIHOODS - METRICS = [AccuracyLoglikelihood, AccuracyNormLoglikelihood, BitsPerByteLoglikelihood] + METRICS = [ + AccuracyLoglikelihood, + AccuracyNormLoglikelihood, + AccuracyBayesianLoglikelihood, + BitsPerByteLoglikelihood, + ] SUBJECTS = [NO_SUBJECT] LANGUAGE = Language.ENG diff --git a/src/eval_framework/tasks/benchmarks/mmlu.py b/src/eval_framework/tasks/benchmarks/mmlu.py index 92726fdb4..66e2749f2 100644 --- a/src/eval_framework/tasks/benchmarks/mmlu.py +++ b/src/eval_framework/tasks/benchmarks/mmlu.py @@ -3,6 +3,7 @@ from eval_framework.metrics.completion.accuracy_completion import AccuracyCompletion from eval_framework.metrics.loglikelihood.accuracy_loglikelihood import ( + AccuracyBayesianLoglikelihood, AccuracyLoglikelihood, AccuracyNormLoglikelihood, ) @@ -85,7 +86,12 @@ class MMLU(BaseTask[str]): SAMPLE_SPLIT = "test" FEWSHOT_SPLIT = "dev" RESPONSE_TYPE = ResponseType.LOGLIKELIHOODS - METRICS = [AccuracyLoglikelihood, AccuracyNormLoglikelihood, BitsPerByteLoglikelihood] + METRICS = [ + AccuracyLoglikelihood, + AccuracyNormLoglikelihood, + AccuracyBayesianLoglikelihood, + BitsPerByteLoglikelihood, + ] SUBJECTS = MMLU_SUBJECTS LANGUAGE = Language.ENG @@ -139,7 +145,12 @@ class FullTextMMLU(MMLU): """MMLU dataset but where the model is expected to replicate choice text, rather than just the key.""" NAME = "Full Text MMLU" - METRICS = [AccuracyLoglikelihood, AccuracyNormLoglikelihood, BitsPerByteLoglikelihood] + METRICS = [ + AccuracyLoglikelihood, + AccuracyNormLoglikelihood, + AccuracyBayesianLoglikelihood, + BitsPerByteLoglikelihood, + ] def _get_initial_prompt_text(self, item: dict[str, Any]) -> str: subject_name = self._get_subject_name(item) @@ -164,6 +175,7 @@ class MMLU_IDK(MMLU): METRICS = [ AccuracyLoglikelihood, AccuracyNormLoglikelihood, + AccuracyBayesianLoglikelihood, ConfidenceWeightedAccuracy, DistributionalCorrectnessScore, TernaryScore, diff --git a/src/eval_framework/tasks/benchmarks/mmlu_pro.py b/src/eval_framework/tasks/benchmarks/mmlu_pro.py index 261bde0ac..79bdd5640 100644 --- a/src/eval_framework/tasks/benchmarks/mmlu_pro.py +++ b/src/eval_framework/tasks/benchmarks/mmlu_pro.py @@ -4,6 +4,7 @@ from eval_framework.metrics.completion.accuracy_completion import AccuracyCompletion from eval_framework.metrics.loglikelihood.accuracy_loglikelihood import ( + AccuracyBayesianLoglikelihood, AccuracyLoglikelihood, AccuracyNormLoglikelihood, ) @@ -42,7 +43,7 @@ class MMLU_PRO(BaseTask[str]): SAMPLE_SPLIT = "test" FEWSHOT_SPLIT = "test" RESPONSE_TYPE = ResponseType.LOGLIKELIHOODS - METRICS = [AccuracyLoglikelihood, AccuracyNormLoglikelihood] + METRICS = [AccuracyLoglikelihood, AccuracyNormLoglikelihood, AccuracyBayesianLoglikelihood] SUBJECTS = MMLU_PRO_SUBJECTS LANGUAGE = Language.ENG @@ -114,6 +115,7 @@ class MMLU_PRO_IDK(MMLU_PRO): METRICS = [ AccuracyLoglikelihood, AccuracyNormLoglikelihood, + AccuracyBayesianLoglikelihood, ConfidenceWeightedAccuracy, DistributionalCorrectnessScore, TernaryScore, diff --git a/src/eval_framework/tasks/benchmarks/piqa.py b/src/eval_framework/tasks/benchmarks/piqa.py index c65766a8e..113607840 100644 --- a/src/eval_framework/tasks/benchmarks/piqa.py +++ b/src/eval_framework/tasks/benchmarks/piqa.py @@ -1,6 +1,7 @@ from typing import Any from eval_framework.metrics.loglikelihood.accuracy_loglikelihood import ( + AccuracyBayesianLoglikelihood, AccuracyLoglikelihood, AccuracyNormLoglikelihood, ) @@ -23,7 +24,12 @@ class PIQA(BaseTask[str]): SAMPLE_SPLIT = "validation" # 1838 examples (same split as lm-eval) FEWSHOT_SPLIT = "test" # 3084 examples RESPONSE_TYPE = ResponseType.LOGLIKELIHOODS - METRICS = [AccuracyLoglikelihood, AccuracyNormLoglikelihood, BitsPerByteLoglikelihood] + METRICS = [ + AccuracyLoglikelihood, + AccuracyNormLoglikelihood, + AccuracyBayesianLoglikelihood, + BitsPerByteLoglikelihood, + ] SUBJECTS = [NO_SUBJECT] LANGUAGE = Language.ENG @@ -82,6 +88,7 @@ class PIQA_IDK(PIQA): METRICS = [ AccuracyLoglikelihood, AccuracyNormLoglikelihood, + AccuracyBayesianLoglikelihood, ConfidenceWeightedAccuracy, DistributionalCorrectnessScore, TernaryScore, diff --git a/src/eval_framework/tasks/benchmarks/sciq.py b/src/eval_framework/tasks/benchmarks/sciq.py index 9cb056de8..25a7adfec 100644 --- a/src/eval_framework/tasks/benchmarks/sciq.py +++ b/src/eval_framework/tasks/benchmarks/sciq.py @@ -3,6 +3,7 @@ from typing import Any from eval_framework.metrics.loglikelihood.accuracy_loglikelihood import ( + AccuracyBayesianLoglikelihood, AccuracyLoglikelihood, AccuracyNormLoglikelihood, ) @@ -40,7 +41,12 @@ class SCIQ(BaseTask[str]): SAMPLE_SPLIT = "validation" # 1000 examples (same split as lm-eval) FEWSHOT_SPLIT = "test" # 1000 examples RESPONSE_TYPE = ResponseType.LOGLIKELIHOODS - METRICS = [AccuracyLoglikelihood, AccuracyNormLoglikelihood, BitsPerByteLoglikelihood] + METRICS = [ + AccuracyLoglikelihood, + AccuracyNormLoglikelihood, + AccuracyBayesianLoglikelihood, + BitsPerByteLoglikelihood, + ] SUBJECTS = [NO_SUBJECT] LANGUAGE = Language.ENG @@ -99,6 +105,7 @@ class SCIQ_IDK(SCIQ): METRICS = [ AccuracyLoglikelihood, AccuracyNormLoglikelihood, + AccuracyBayesianLoglikelihood, ConfidenceWeightedAccuracy, DistributionalCorrectnessScore, TernaryScore, @@ -127,7 +134,7 @@ class SCIQEvalHarness(SCIQ): SAMPLE_SPLIT = "validation" # 1000 examples (same split as lm-eval) FEWSHOT_SPLIT = "test" # 1000 examples RESPONSE_TYPE = ResponseType.LOGLIKELIHOODS - METRICS = [AccuracyLoglikelihood, AccuracyNormLoglikelihood] + METRICS = [AccuracyLoglikelihood, AccuracyNormLoglikelihood, AccuracyBayesianLoglikelihood] SUBJECTS = [NO_SUBJECT] LANGUAGE = Language.ENG @@ -141,6 +148,7 @@ class SCIQEvalHarness_IDK(SCIQEvalHarness): METRICS = [ AccuracyLoglikelihood, AccuracyNormLoglikelihood, + AccuracyBayesianLoglikelihood, ConfidenceWeightedAccuracy, DistributionalCorrectnessScore, TernaryScore, diff --git a/src/eval_framework/tasks/benchmarks/social_iqa.py b/src/eval_framework/tasks/benchmarks/social_iqa.py index e53d12f6c..86d80fe25 100644 --- a/src/eval_framework/tasks/benchmarks/social_iqa.py +++ b/src/eval_framework/tasks/benchmarks/social_iqa.py @@ -14,6 +14,7 @@ from datasets import Dataset, DatasetDict, DownloadConfig, load_dataset from eval_framework.metrics.loglikelihood.accuracy_loglikelihood import ( + AccuracyBayesianLoglikelihood, AccuracyLoglikelihood, AccuracyNormLoglikelihood, ) @@ -161,6 +162,7 @@ class SocialIQACloze(BaseTask[str]): METRICS = [ AccuracyLoglikelihood, AccuracyNormLoglikelihood, + AccuracyBayesianLoglikelihood, BitsPerByteLoglikelihood, ] SUBJECTS = [NO_SUBJECT] diff --git a/src/eval_framework/tasks/benchmarks/winogrande.py b/src/eval_framework/tasks/benchmarks/winogrande.py index 992a03fbf..c3849e4bc 100644 --- a/src/eval_framework/tasks/benchmarks/winogrande.py +++ b/src/eval_framework/tasks/benchmarks/winogrande.py @@ -1,6 +1,7 @@ from typing import Any from eval_framework.metrics.loglikelihood.accuracy_loglikelihood import ( + AccuracyBayesianLoglikelihood, AccuracyLoglikelihood, AccuracyNormLoglikelihood, PartialEvalAccuracy, @@ -25,7 +26,12 @@ class WINOGRANDE(BaseTask[str]): SAMPLE_SPLIT = "validation" FEWSHOT_SPLIT = "train" RESPONSE_TYPE = ResponseType.LOGLIKELIHOODS - METRICS = [AccuracyLoglikelihood, AccuracyNormLoglikelihood, BitsPerByteLoglikelihood] + METRICS = [ + AccuracyLoglikelihood, + AccuracyNormLoglikelihood, + AccuracyBayesianLoglikelihood, + BitsPerByteLoglikelihood, + ] SUBJECTS = ["winogrande_xl"] LANGUAGE = Language.ENG @@ -84,6 +90,7 @@ class WINOGRANDE_IDK(WINOGRANDE): METRICS = [ AccuracyLoglikelihood, AccuracyNormLoglikelihood, + AccuracyBayesianLoglikelihood, ConfidenceWeightedAccuracy, DistributionalCorrectnessScore, TernaryScore, diff --git a/src/eval_framework/tasks/task_style.py b/src/eval_framework/tasks/task_style.py index 3bee81936..44102bdd5 100644 --- a/src/eval_framework/tasks/task_style.py +++ b/src/eval_framework/tasks/task_style.py @@ -64,6 +64,7 @@ class ARC_BPB(_ARC_Base): from eval_framework.metrics.completion.accuracy_completion import AccuracyCompletion from eval_framework.metrics.loglikelihood.accuracy_loglikelihood import ( + AccuracyBayesianLoglikelihood, AccuracyLoglikelihood, AccuracyNormLoglikelihood, ) @@ -177,6 +178,7 @@ class MCStyle(TaskStyler): metrics: list[type["BaseMetric"]] = [ AccuracyLoglikelihood, AccuracyNormLoglikelihood, + AccuracyBayesianLoglikelihood, BitsPerByteLoglikelihood, ] task_style = TaskStyle.MULTIPLE_CHOICE @@ -303,6 +305,7 @@ class ClozeStyle(TaskStyler): metrics: list[type["BaseMetric"]] = [ AccuracyLoglikelihood, AccuracyNormLoglikelihood, + AccuracyBayesianLoglikelihood, BitsPerByteLoglikelihood, ] task_style = TaskStyle.CLOZE diff --git a/tests/tests_eval_framework/metrics/test_accuracy_loglikelihood.py b/tests/tests_eval_framework/metrics/test_accuracy_loglikelihood.py index ac2b445c3..1e4744b8a 100644 --- a/tests/tests_eval_framework/metrics/test_accuracy_loglikelihood.py +++ b/tests/tests_eval_framework/metrics/test_accuracy_loglikelihood.py @@ -1,10 +1,29 @@ import pytest from eval_framework.metrics.loglikelihood.accuracy_loglikelihood import ( + AccuracyBayesianLoglikelihood, AccuracyLoglikelihood, AccuracyNormLoglikelihood, ) -from eval_framework.shared.types import Loglikelihood +from eval_framework.shared.types import Error, Loglikelihood + + +def create_loglikelihood( + loglikelihoods: dict[str, float], + ground_truth: str | list[str], + *, + error: Error | None = None, +) -> Loglikelihood: + return Loglikelihood( + id=1, + subject="test", + ground_truth=ground_truth, + prompt="test", + prompt_num_tokens=None, + loglikelihoods=loglikelihoods, + loglikelihoods_num_tokens={}, + error=error, + ) @pytest.mark.parametrize( @@ -59,3 +78,44 @@ def test_accuracy_norm_loglikelihood(response: Loglikelihood, expected_value: fl assert results[0].value == pytest.approx(expected_value) assert results[0].metric_name == "Accuracy Normalized Loglikelihood" assert results[0].higher_is_better is True + + +def test_accuracy_bayesian_estimates_centered_slope_and_corrects_scores() -> None: + steep_response = create_loglikelihood({"a": -1.0, "aaa": -5.0}, "a") + flat_response = create_loglikelihood({"a": -1.0, "aaa": -1.0}, ["aaa", "other"]) + metric = AccuracyBayesianLoglikelihood() + + metric.prepare([steep_response, flat_response]) + results = metric.calculate(flat_response) + + assert metric.length_decay == pytest.approx(-1.0) + assert results[0].value == 1.0 + assert results[0].metric_name == "Accuracy Bayesian Loglikelihood" + assert results[0].higher_is_better is True + + +def test_accuracy_bayesian_uses_utf8_byte_lengths() -> None: + metric = AccuracyBayesianLoglikelihood() + + metric.prepare([create_loglikelihood({"é": -2.0, "aaa": -4.0}, "é")]) + + assert metric.length_decay == pytest.approx(-2.0) + + +def test_accuracy_bayesian_ignores_errored_responses_when_estimating_slope() -> None: + error = Error(error_class="RuntimeError", message="failed", traceback="trace") + valid_response = create_loglikelihood({"a": -1.0, "aaa": -3.0}, "a") + errored_response = create_loglikelihood({"a": -1.0, "aaa": -101.0}, "a", error=error) + metric = AccuracyBayesianLoglikelihood() + + metric.prepare([valid_response, errored_response]) + + assert metric.length_decay == pytest.approx(-1.0) + + +def test_accuracy_bayesian_falls_back_to_zero_without_length_variation() -> None: + metric = AccuracyBayesianLoglikelihood() + + metric.prepare([create_loglikelihood({"a": -1.0, "b": -2.0}, "a")]) + + assert metric.length_decay == 0.0 diff --git a/tests/tests_eval_framework/tasks/test_task_style.py b/tests/tests_eval_framework/tasks/test_task_style.py index 985185a4a..e2f6a43eb 100644 --- a/tests/tests_eval_framework/tasks/test_task_style.py +++ b/tests/tests_eval_framework/tasks/test_task_style.py @@ -366,12 +366,14 @@ def test_response_type_from_styler(self) -> None: def test_metrics_from_styler(self) -> None: from eval_framework.metrics.loglikelihood.accuracy_loglikelihood import ( + AccuracyBayesianLoglikelihood, AccuracyLoglikelihood, AccuracyNormLoglikelihood, ) assert AccuracyLoglikelihood in self.task.TASK_STYLER.metrics assert AccuracyNormLoglikelihood in self.task.TASK_STYLER.metrics + assert AccuracyBayesianLoglikelihood in self.task.TASK_STYLER.metrics assert BitsPerByteLoglikelihood in self.task.TASK_STYLER.metrics @@ -404,6 +406,11 @@ def test_metadata_includes_task_style(self) -> None: meta = self.task.get_metadata() assert meta["task_style"] == TaskStyle.CLOZE.value + def test_metrics_include_bayesian_accuracy(self) -> None: + from eval_framework.metrics.loglikelihood.accuracy_loglikelihood import AccuracyBayesianLoglikelihood + + assert AccuracyBayesianLoglikelihood in self.task.TASK_STYLER.metrics + class TestBaseTaskMCCompletionStyle: def setup_method(self) -> None: diff --git a/tests/tests_eval_framework/test_end_to_end.py b/tests/tests_eval_framework/test_end_to_end.py index 7c400ef9f..a051165fc 100644 --- a/tests/tests_eval_framework/test_end_to_end.py +++ b/tests/tests_eval_framework/test_end_to_end.py @@ -40,7 +40,11 @@ def test_automatic_tasks(tmp_path: Path, test_llms: BaseLLM) -> None: output_dir = tmp_path / "eval" task_name = HELLASWAG.NAME - expected_results = {"Accuracy Loglikelihood": 0.4, "Accuracy Normalized Loglikelihood": 0.6} + expected_results = { + "Accuracy Loglikelihood": 0.4, + "Accuracy Normalized Loglikelihood": 0.6, + "Accuracy Bayesian Loglikelihood": 0.5, + } num_fewshot = 2 num_samples = 10 From 8d4d4b630ab21ddeb5d39d2a80e2ae7cace0679c Mon Sep 17 00:00:00 2001 From: koen Date: Fri, 21 Aug 2026 15:42:05 +0000 Subject: [PATCH 2/2] feat(metrics): add Bayesian accuracy --- .../metrics/loglikelihood/accuracy_loglikelihood.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/eval_framework/metrics/loglikelihood/accuracy_loglikelihood.py b/src/eval_framework/metrics/loglikelihood/accuracy_loglikelihood.py index c9a3850e2..8bf19c70e 100644 --- a/src/eval_framework/metrics/loglikelihood/accuracy_loglikelihood.py +++ b/src/eval_framework/metrics/loglikelihood/accuracy_loglikelihood.py @@ -65,7 +65,8 @@ def __init__(self) -> None: self.length_decay = 0.0 def prepare(self, responses: list[Loglikelihood]) -> None: - """Estimating the length decay factor. + """ + Estimating the length decay factor. See Equation (24) in https://arxiv.org/html/2607.12767v1 """ numerator = 0.0