Feature: Bayesian accuracy

src/eval_framework/evaluation_generator.py

@@ -1,5 +1,6 @@
 import logging
 import math
+from typing import Any, cast
 
 import numpy as np
 import pandas as pd
@@ -49,18 +50,20 @@ def _run_metric_calculators(self, responses: list[Completion | Loglikelihood]) -
         """
         llm_judge = None
         for metric_class in self.metrics:
-            metric: BaseMetric
+            raw_metric: BaseMetric[Any]
             if issubclass(metric_class, BaseLLMJudgeMetric):
                 if llm_judge is None:
                     llm_judge = self.config.llm_judge()
-                metric = metric_class(
+                raw_metric = metric_class(
                     llm_judge=llm_judge,
                     randomize_order=self.config.randomize_judge_order,
                 )
             else:
-                metric = metric_class()
+                raw_metric = metric_class()
+            metric = cast(BaseMetric[Completion | Loglikelihood], raw_metric)
             metric.fail_on_error = self.config.fail_on_error
 
+            metric.prepare(responses)
             logger.info(f"Starting calculation of {metric.NAME}")
             for response in tqdm(responses, desc=f"Calculating {metric.NAME}", disable=get_disable_bar_flag()):
                 if f"{response.subject}_{response.id}_{metric.__class__.__name__}" in subject_result_id_existing:

src/eval_framework/metrics/base.py

@@ -48,6 +48,10 @@ def NAMES(cls) -> list[str]:
     def calculate(self, response: Response) -> list[MetricResult]:
         raise NotImplementedError
 
+    def prepare(self, responses: list[Response]) -> None:
+        """Prepare metric before calculating per-response results.
+        This is needed for metrics that depend on variables derived from all of the responses."""
+
     def _record_or_raise(self, exc: Exception) -> list[MetricResult]:
         """Infra failure (e.g. a Docker image-pull rate limit): abort when fail_on_error is set,
         otherwise record a per-sample error so the run continues."""

src/eval_framework/metrics/loglikelihood/accuracy_loglikelihood.py

@@ -1,5 +1,8 @@
+import numpy as np
+
 from eval_framework.metrics.base import BaseMetric, MetricResult
 from eval_framework.shared.types import Loglikelihood
+from eval_framework.utils.helpers import count_bytes
 
 
 class AccuracyLoglikelihood(BaseMetric[Loglikelihood]):
@@ -51,6 +54,66 @@ def calculate(self, response: Loglikelihood) -> list[MetricResult]:
         ]
 
 
+class AccuracyBayesianLoglikelihood(BaseMetric[Loglikelihood]):
+    """Accuracy after adjusting the loglikelihoods for the byte-length bias of the completion.
+    See https://arxiv.org/html/2607.12767v1 for more details.
+    """
+
+    NAME = "Accuracy Bayesian Loglikelihood"
+
+    def __init__(self) -> None:
+        self.length_decay = 0.0
+
+    def prepare(self, responses: list[Loglikelihood]) -> None:
+        """
+        Estimating the length decay factor.
+        See Equation (24) in https://arxiv.org/html/2607.12767v1
+        """
+        numerator = 0.0
+        denominator = 0.0
+
+        for response in responses:
+            if response.error is not None:
+                continue
+
+            num_candidates = len(response.loglikelihoods)
+            if num_candidates <= 1:
+                continue
+
+            lengths = np.array([count_bytes(completion) for completion in response.loglikelihoods], dtype=float)
+            loglikelihoods = np.array(list(response.loglikelihoods.values()), dtype=float)
+            length_differences = lengths - np.mean(lengths)
+            loglikelihood_differences = loglikelihoods - np.mean(loglikelihoods)
+
+            local_denominator = num_candidates * np.sum(length_differences**2)
+            if local_denominator == 0:
+                continue
+
+            numerator += float(num_candidates * np.sum(length_differences * loglikelihood_differences))
+            denominator += float(local_denominator)
+
+        self.length_decay = 0.0 if denominator == 0 else numerator / denominator
+
+    def calculate(self, response: Loglikelihood) -> list[MetricResult]:
+        if response.error is not None:
+            return [MetricResult(metric_name=self.NAME, value=None, higher_is_better=True, error=response.error)]
+
+        corrected_loglikelihoods = {
+            completion: loglikelihood - self.length_decay * count_bytes(completion)
+            for completion, loglikelihood in response.loglikelihoods.items()
+        }
+        completion_text = max(corrected_loglikelihoods, key=corrected_loglikelihoods.get)  # type: ignore[arg-type]
+
+        return [
+            MetricResult(
+                metric_name=self.NAME,
+                value=float(completion_text in response.ground_truth_list),
+                higher_is_better=True,
+                error=response.error,
+            )
+        ]
+
+
 class PartialEvalAccuracy(BaseMetric[Loglikelihood]):
     """An accuracy metric for partial evaluation tasks, e.g. WinograndeCloze.
 

src/eval_framework/tasks/benchmarks/arc.py

@@ -1,6 +1,7 @@
 from typing import Any
 
 from eval_framework.metrics.loglikelihood.accuracy_loglikelihood import (
+    AccuracyBayesianLoglikelihood,
     AccuracyLoglikelihood,
     AccuracyNormLoglikelihood,
 )
@@ -23,7 +24,12 @@ class ARC(BaseTask[str]):
     SAMPLE_SPLIT = "test"
     FEWSHOT_SPLIT = "train"
     RESPONSE_TYPE = ResponseType.LOGLIKELIHOODS
-    METRICS = [AccuracyLoglikelihood, AccuracyNormLoglikelihood, BitsPerByteLoglikelihood]
+    METRICS = [
+        AccuracyLoglikelihood,
+        AccuracyNormLoglikelihood,
+        AccuracyBayesianLoglikelihood,
+        BitsPerByteLoglikelihood,
+    ]
     SUBJECTS = ["ARC-Easy", "ARC-Challenge"]
     LANGUAGE = Language.ENG
 
@@ -85,6 +91,7 @@ class ARC_IDK(ARC):
     METRICS = [
         AccuracyLoglikelihood,
         AccuracyNormLoglikelihood,
+        AccuracyBayesianLoglikelihood,
         ConfidenceWeightedAccuracy,
         DistributionalCorrectnessScore,
         TernaryScore,

src/eval_framework/tasks/benchmarks/arc_de.py

@@ -1,6 +1,7 @@
 from typing import Any
 
 from eval_framework.metrics.loglikelihood.accuracy_loglikelihood import (
+    AccuracyBayesianLoglikelihood,
     AccuracyLoglikelihood,
     AccuracyNormLoglikelihood,
 )
@@ -20,7 +21,12 @@ class ARC_DE(BaseTask[str]):
     SAMPLE_SPLIT = "test"
     FEWSHOT_SPLIT = "validation"
     RESPONSE_TYPE = ResponseType.LOGLIKELIHOODS
-    METRICS = [AccuracyLoglikelihood, AccuracyNormLoglikelihood, BitsPerByteLoglikelihood]
+    METRICS = [
+        AccuracyLoglikelihood,
+        AccuracyNormLoglikelihood,
+        AccuracyBayesianLoglikelihood,
+        BitsPerByteLoglikelihood,
+    ]
     SUBJECTS = [NO_SUBJECT]
     LANGUAGE = Language.DEU
 

src/eval_framework/tasks/benchmarks/copa.py

@@ -1,6 +1,7 @@
 from typing import Any
 
 from eval_framework.metrics.loglikelihood.accuracy_loglikelihood import (
+    AccuracyBayesianLoglikelihood,
     AccuracyLoglikelihood,
     AccuracyNormLoglikelihood,
 )
@@ -24,7 +25,7 @@ class COPAEvalHarness(BaseTask[str]):
     SAMPLE_SPLIT = "validation"  # 100 examples (same split as lm-eval)
     FEWSHOT_SPLIT = "test"  # 500 examples
     RESPONSE_TYPE = ResponseType.LOGLIKELIHOODS
-    METRICS = [AccuracyLoglikelihood, AccuracyNormLoglikelihood]
+    METRICS = [AccuracyLoglikelihood, AccuracyNormLoglikelihood, AccuracyBayesianLoglikelihood]
     SUBJECTS = ["copa"]
     LANGUAGE = Language.ENG
 
@@ -94,6 +95,7 @@ class COPA_IDKEvalHarness(COPAEvalHarness):
     METRICS = [
         AccuracyLoglikelihood,
         AccuracyNormLoglikelihood,
+        AccuracyBayesianLoglikelihood,
         ConfidenceWeightedAccuracy,
         DistributionalCorrectnessScore,
         TernaryScore,

src/eval_framework/tasks/benchmarks/csqa.py

@@ -1,6 +1,7 @@
 from typing import Any
 
 from eval_framework.metrics.loglikelihood.accuracy_loglikelihood import (
+    AccuracyBayesianLoglikelihood,
     AccuracyLoglikelihood,
     AccuracyNormLoglikelihood,
 )
@@ -20,7 +21,12 @@ class CommonsenseQACloze(BaseTask[str]):
     SAMPLE_SPLIT = "validation"
     FEWSHOT_SPLIT = "validation"
     RESPONSE_TYPE = ResponseType.LOGLIKELIHOODS
-    METRICS = [AccuracyLoglikelihood, AccuracyNormLoglikelihood, BitsPerByteLoglikelihood]
+    METRICS = [
+        AccuracyLoglikelihood,
+        AccuracyNormLoglikelihood,
+        AccuracyBayesianLoglikelihood,
+        BitsPerByteLoglikelihood,
+    ]
     SUBJECTS = [NO_SUBJECT]
     LANGUAGE = Language.ENG
 
@@ -57,7 +63,12 @@ class CommonsenseQAFullTextCloze(CommonsenseQACloze):
     REVISION_LOCKFILE = HF_REVISIONS_LOCKFILE
 
     NAME = "CommonsenseQAFullTextCloze"
-    METRICS = [AccuracyLoglikelihood, AccuracyNormLoglikelihood, BitsPerByteLoglikelihood]
+    METRICS = [
+        AccuracyLoglikelihood,
+        AccuracyNormLoglikelihood,
+        AccuracyBayesianLoglikelihood,
+        BitsPerByteLoglikelihood,
+    ]
 
     def _get_ground_truth(self, item: dict[str, Any]) -> str | None:
         correct_label = item["answerKey"]

src/eval_framework/tasks/benchmarks/drop.py

@@ -5,6 +5,7 @@
     DropMetricContext,
 )
 from eval_framework.metrics.loglikelihood.accuracy_loglikelihood import (
+    AccuracyBayesianLoglikelihood,
     AccuracyLoglikelihood,
     AccuracyNormLoglikelihood,
 )
@@ -174,6 +175,7 @@ class DropMC(BaseTask[str]):
     METRICS = [
         AccuracyLoglikelihood,
         AccuracyNormLoglikelihood,
+        AccuracyBayesianLoglikelihood,
         BitsPerByteLoglikelihood,
     ]
     SUBJECTS = [NO_SUBJECT]
@@ -245,6 +247,7 @@ class DropCloze(BaseTask[str]):
     METRICS = [
         AccuracyLoglikelihood,
         AccuracyNormLoglikelihood,
+        AccuracyBayesianLoglikelihood,
         BitsPerByteLoglikelihood,
     ]
     SUBJECTS = [NO_SUBJECT]

src/eval_framework/tasks/benchmarks/global_mmlu.py

@@ -3,6 +3,7 @@
 from typing import Any
 
 from eval_framework.metrics.loglikelihood.accuracy_loglikelihood import (
+    AccuracyBayesianLoglikelihood,
     AccuracyLoglikelihood,
     AccuracyNormLoglikelihood,
 )
@@ -477,7 +478,12 @@ class GlobalMMLU(BaseTask[tuple[str, str]]):
     SAMPLE_SPLIT = "test"
     FEWSHOT_SPLIT = "dev"
     RESPONSE_TYPE = ResponseType.LOGLIKELIHOODS
-    METRICS = [AccuracyLoglikelihood, AccuracyNormLoglikelihood, BitsPerByteLoglikelihood]
+    METRICS = [
+        AccuracyLoglikelihood,
+        AccuracyNormLoglikelihood,
+        AccuracyBayesianLoglikelihood,
+        BitsPerByteLoglikelihood,
+    ]
     SUBJECTS = list(product(GLOBAL_MMLU_LANGUAGES, MMLU_SUBJECTS))
     LANGUAGE: Language | dict[str, Language] | None = {
         str((lang_code.split("_")[0], subject)): LANGUAGE_NAME_MAP[lang_code]

src/eval_framework/tasks/benchmarks/goldenswag.py

@@ -1,6 +1,7 @@
 from typing import Any
 
 from eval_framework.metrics.loglikelihood.accuracy_loglikelihood import (
+    AccuracyBayesianLoglikelihood,
     AccuracyLoglikelihood,
     AccuracyNormLoglikelihood,
 )
@@ -29,6 +30,7 @@ class GOLDENSWAG_IDK(GOLDENSWAG):
     METRICS = [
         AccuracyLoglikelihood,
         AccuracyNormLoglikelihood,
+        AccuracyBayesianLoglikelihood,
         ConfidenceWeightedAccuracy,
         DistributionalCorrectnessScore,
         TernaryScore,

src/eval_framework/tasks/benchmarks/gpqa.py

@@ -6,6 +6,7 @@
 
 from eval_framework.metrics.completion.accuracy_completion import AccuracyCompletion
 from eval_framework.metrics.loglikelihood.accuracy_loglikelihood import (
+    AccuracyBayesianLoglikelihood,
     AccuracyLoglikelihood,
     AccuracyNormLoglikelihood,
 )
@@ -29,7 +30,7 @@ class GPQA(BaseTask[str]):
     SAMPLE_SPLIT = "train"
     FEWSHOT_SPLIT = "train"
     RESPONSE_TYPE = ResponseType.LOGLIKELIHOODS
-    METRICS = [AccuracyLoglikelihood, AccuracyNormLoglikelihood]
+    METRICS = [AccuracyLoglikelihood, AccuracyNormLoglikelihood, AccuracyBayesianLoglikelihood]
     SUBJECTS = ["gpqa_extended"]  # ["gpqa_diamond", "gpqa_extended", "gpqa_main", "gpqa_experts"]
     LANGUAGE = Language.ENG
 
@@ -156,6 +157,7 @@ class GPQA_IDK(GPQA):
     METRICS = [
         AccuracyLoglikelihood,
         AccuracyNormLoglikelihood,
+        AccuracyBayesianLoglikelihood,
         ConfidenceWeightedAccuracy,
         DistributionalCorrectnessScore,
         TernaryScore,

src/eval_framework/tasks/benchmarks/hellaswag.py

@@ -2,6 +2,7 @@
 from typing import Any
 
 from eval_framework.metrics.loglikelihood.accuracy_loglikelihood import (
+    AccuracyBayesianLoglikelihood,
     AccuracyLoglikelihood,
     AccuracyNormLoglikelihood,
 )
@@ -24,7 +25,12 @@ class HELLASWAG(BaseTask[str]):
     SAMPLE_SPLIT = "validation"
     FEWSHOT_SPLIT = "train"
     RESPONSE_TYPE = ResponseType.LOGLIKELIHOODS
-    METRICS = [AccuracyLoglikelihood, AccuracyNormLoglikelihood, BitsPerByteLoglikelihood]
+    METRICS = [
+        AccuracyLoglikelihood,
+        AccuracyNormLoglikelihood,
+        AccuracyBayesianLoglikelihood,
+        BitsPerByteLoglikelihood,
+    ]
     SUBJECTS = [NO_SUBJECT]
     LANGUAGE = Language.ENG
 
@@ -62,6 +68,7 @@ class HELLASWAG_IDK(HELLASWAG):
     METRICS = [
         AccuracyLoglikelihood,
         AccuracyNormLoglikelihood,
+        AccuracyBayesianLoglikelihood,
         ConfidenceWeightedAccuracy,
         DistributionalCorrectnessScore,
         TernaryScore,

src/eval_framework/tasks/benchmarks/medqa.py

@@ -5,6 +5,7 @@
 from typing import Any
 
 from eval_framework.metrics.loglikelihood.accuracy_loglikelihood import (
+    AccuracyBayesianLoglikelihood,
     AccuracyLoglikelihood,
     AccuracyNormLoglikelihood,
 )
@@ -24,7 +25,12 @@ class MedQACloze(BaseTask[str]):
     SAMPLE_SPLIT = "test"
     FEWSHOT_SPLIT = "dev"
     RESPONSE_TYPE = ResponseType.LOGLIKELIHOODS
-    METRICS = [AccuracyLoglikelihood, AccuracyNormLoglikelihood, BitsPerByteLoglikelihood]
+    METRICS = [
+        AccuracyLoglikelihood,
+        AccuracyNormLoglikelihood,
+        AccuracyBayesianLoglikelihood,
+        BitsPerByteLoglikelihood,
+    ]
     SUBJECTS = [NO_SUBJECT]
     LANGUAGE = Language.ENG