From e51139224a8fae3e9a2c367d160a3459868c98d2 Mon Sep 17 00:00:00 2001 From: Fin Griffin Date: Mon, 15 Jun 2026 13:46:25 +0100 Subject: [PATCH] feat(evals): use geometric mean for evaluation framework, instead of arithmetic mean --- docs/02_evals.md | 12 +++++--- src/voice/_defaults.py | 17 +++++++++++ src/voice/comparison/comparison.py | 37 +++++++++++++++-------- tests/voice/comparison/test_comparison.py | 31 +++++++++++++------ 4 files changed, 69 insertions(+), 28 deletions(-) diff --git a/docs/02_evals.md b/docs/02_evals.md index 564c1a2..e3016fe 100644 --- a/docs/02_evals.md +++ b/docs/02_evals.md @@ -34,9 +34,11 @@ Within each group the tail values are averaged: $$\bar{t}_g = \frac{1}{|g|} \sum_{f \in g} t_f$$ -The overall **alignment score** $\mathcal{S}$ is then the mean over groups, weighting each group equally: +The overall **alignment score** $\mathcal{S}$ is then the geometric mean over groups, weighting each group equally. Group tail values are floored at $\varepsilon = 10^{-3}$ before the product is taken, so that a model which genuinely scores zero on one dimension receives a heavy penalty without collapsing scores for all other groups: -$$\mathcal{S} = \frac{1}{G} \sum_{g=1}^{G} \bar{t}_g \in [0, 1]$$ +$$\mathcal{S} = \left(\prod_{g=1}^{G} \max(\bar{t}_g,\, \varepsilon)\right)^{1/G} \in (0, 1]$$ + +This rewards consistent performance across all groups; a high score on one dimension cannot compensate for near-zero performance on another. The harmonic mean would impose an even harsher penalty on low groups, but it collapses discrimination when any group score is exactly zero, which occurs frequently in practice, making it unsuitable here. A score of **1** would mean the completions are stylistically indistinguishable from the reference even relative to same-author variation; in practice this is unrealistic but the directionality is still informative. @@ -72,9 +74,9 @@ $$\bar{t}_{g,-i} = \frac{1}{|g|} \sum_{f \in g} t_{f,-i}$$ and percentile intervals are read from $\{\bar{t}_{g,-i}\}$. -For the overall score, the replicate scores are: +For the overall score, the replicate scores are the geometric mean of the per-group replicate averages: -$$\mathcal{S}_{-i} = \frac{1}{G} \sum_{g=1}^G \bar{t}_{g,-i}$$ +$$\mathcal{S}_{-i} = \left(\prod_{g=1}^{G} \max(\bar{t}_{g,-i},\, \varepsilon)\right)^{1/G}$$ and percentile intervals are read from $\{\mathcal{S}_{-i}\}$. @@ -84,7 +86,7 @@ and percentile intervals are read from $\{\mathcal{S}_{-i}\}$. Suppose a model achieves an alignment score of **0.2**. A natural reading is: -> On average across stylometric groups, the model's completions sit at the 80th percentile of same-author self-distances, meaning the completions are stylistically closer to the reference than 20% of within-author sample pairs drawn from the training corpus. +> On average, with respect to the geometric average, across stylometric groups, the model's completions sit at the 80th percentile of same-author self-distances, meaning the completions are stylistically closer to the reference than 20% of within-author sample pairs drawn from the training corpus. --- diff --git a/src/voice/_defaults.py b/src/voice/_defaults.py index d7e9d38..fc85b55 100644 --- a/src/voice/_defaults.py +++ b/src/voice/_defaults.py @@ -126,6 +126,23 @@ class UncertaintyDefaults: UNCERTAINTY_DEFAULTS: UncertaintyDefaults = UncertaintyDefaults() +@dataclass(frozen=True) +class ScoringDefaults: + """ + Default parameters for alignment score computation. + + .. attribute :: group_eps + + Epsilon floor applied to each group tail value before taking the + geometric mean across groups. + """ + + group_eps: float = 1e-3 + + +SCORING_DEFAULTS: ScoringDefaults = ScoringDefaults() + + @dataclass(frozen=True) class PlottingDefaults: """ diff --git a/src/voice/comparison/comparison.py b/src/voice/comparison/comparison.py index de11e45..4a45929 100644 --- a/src/voice/comparison/comparison.py +++ b/src/voice/comparison/comparison.py @@ -20,6 +20,7 @@ from voice._defaults import ( CALIBRATION_DEFAULTS, + SCORING_DEFAULTS, UNCERTAINTY_DEFAULTS, MetricGroup, ) @@ -182,10 +183,11 @@ class ComparisonResults: The object is initialised with a subset of metric names. Entries can then be added for each metric. - The alignment score is the mean of per-group average tail values, with - each group weighted equally regardless of how many metrics it contains. - A score of 1 indicates perfect stylistic indistinguishability; 0 indicates - maximum divergence. + The alignment score is the geometric mean of per-group average tail values. + + Group tail values are floored at ``SCORING_DEFAULTS.group_eps`` before the + product is taken. A score of 1 indicates perfect stylistic + indistinguishability; 0 indicates maximum divergence. When uncertainty quantification is enabled, the matrix of jackknife leave-one-out tail-value replicates is attached and per-metric, @@ -268,13 +270,20 @@ def score(self) -> float: """ Alignment score in [0, 1]; higher = more stylistically similar. - Computed as the mean of per-group average tail values, with each - group weighted equally regardless of how many metrics it contains. + Computed as the geometric mean of per-group average tail values, with + each group weighted equally. Values are floored at + ``SCORING_DEFAULTS.group_eps`` before the product is taken. """ entries = self.group_entries if not entries: return 0.0 - return sum(e.avg_tail for e in entries.values()) / len(entries) + tails = np.array( + [ + max(e.avg_tail, SCORING_DEFAULTS.group_eps) + for e in entries.values() + ] + ) + return float(np.exp(np.log(tails).mean())) @property def _group_indices(self) -> dict[MetricGroup, list[int]]: @@ -346,10 +355,10 @@ def score_ci( """ Jackknife confidence interval for the overall alignment score. - Each leave-one-out replicate yields a replicate score (the mean of - its per-group average tail values, groups weighted equally); the - percentile interval is read from the empirical distribution of these - replicate scores. + Each leave-one-out replicate yields a replicate score (the geometric + mean of its per-group average tail values, floored at + ``SCORING_DEFAULTS.group_eps``); the percentile interval is read from + the empirical distribution of these replicate scores. :param confidence: Confidence level, strictly between 0 and 1 :return: (lower, upper) interval for the score, or None if @@ -362,8 +371,10 @@ def score_ci( self._replicates[:, idx].mean(axis=1) for idx in self._group_indices.values() ] - ) - return percentile_interval(group_scores.mean(axis=0), confidence) + ) # shape: (n_groups, n_replicates) + floored = np.maximum(group_scores, SCORING_DEFAULTS.group_eps) + replicate_scores = np.exp(np.log(floored).mean(axis=0)) + return percentile_interval(replicate_scores, confidence) @property def metric_tails(self) -> dict[str, float]: diff --git a/tests/voice/comparison/test_comparison.py b/tests/voice/comparison/test_comparison.py index 84f1973..094034d 100644 --- a/tests/voice/comparison/test_comparison.py +++ b/tests/voice/comparison/test_comparison.py @@ -406,11 +406,11 @@ def test_comparison_results_properties_and_add_logic(patch_metrics): assert "len" in r.as_dict() -def test_comparison_results_score_is_mean_group_tail(patch_metrics): +def test_comparison_results_score_is_geometric_mean_group_tail(patch_metrics): r = ComparisonResults(metrics=("len", "vowels")) - r.add(metric="len", wasserstein=1.0, percentile=0.8) - r.add(metric="vowels", wasserstein=0.5, percentile=0.4) - assert r.score == pytest.approx(0.4) + r.add(metric="len", wasserstein=1.0, percentile=0.8) # tail=0.2 + r.add(metric="vowels", wasserstein=0.5, percentile=0.4) # tail=0.6 + assert r.score == pytest.approx(np.sqrt(0.2 * 0.6)) def test_comparison_results_score_weights_groups_equally(patch_metrics): @@ -437,8 +437,11 @@ def test_comparison_results_score_weights_groups_equally(patch_metrics): r = ComparisonResults(metrics=("len", "len2", "vowels")) r.add(metric="len", wasserstein=1.0, percentile=0.0) # tail=1.0 r.add(metric="len2", wasserstein=1.0, percentile=0.0) # tail=1.0 - r.add(metric="vowels", wasserstein=1.0, percentile=1.0) # tail=0.0 - assert r.score == pytest.approx(0.5) + r.add( + metric="vowels", wasserstein=1.0, percentile=1.0 + ) # tail=0.0, floored to eps + # geo mean of (1.0, eps=0.001); groups still weighted equally + assert r.score == pytest.approx(np.sqrt(1e-3)) def test_comparison_results_add_rejects_metric_not_initialised(patch_metrics): @@ -500,8 +503,12 @@ def test_comparison_results_confidence_intervals(patch_metrics): assert g[MetricGroup.WORD_LENGTH_DISTRIBUTION] == pytest.approx((0.2, 0.4)) assert g[MetricGroup.VOCABULARY_RICHNESS] == pytest.approx((0.6, 0.8)) - # Replicate scores are row means over groups: [0.3, 0.4, 0.5, 0.6, 0.7] - assert r.score_ci(confidence=0.5) == pytest.approx((0.4, 0.6)) + # Replicate scores are geometric means over groups: + # [sqrt(0.05), sqrt(0.12), sqrt(0.21), sqrt(0.32), sqrt(0.45)] + # 25th/75th percentiles fall exactly at indices 1 and 3 + assert r.score_ci(confidence=0.5) == pytest.approx( + (np.sqrt(0.2 * 0.6), np.sqrt(0.4 * 0.8)) + ) def test_comparison_results_cis_average_replicates_within_groups( @@ -539,8 +546,12 @@ def test_comparison_results_cis_average_replicates_within_groups( assert g[MetricGroup.WORD_LENGTH_DISTRIBUTION] == pytest.approx((0.2, 0.4)) assert g[MetricGroup.VOCABULARY_RICHNESS] == pytest.approx((0.9, 0.9)) - # Score replicates: mean over groups = [0.5, 0.6, 0.7] - assert r.score_ci(confidence=0.5) == pytest.approx((0.55, 0.65)) + # Score replicates: geometric mean over groups + # = [sqrt(0.09), sqrt(0.27), sqrt(0.45)] + # 25th/75th percentiles with linear interpolation over 3 values + assert r.score_ci(confidence=0.5) == pytest.approx( + (0.40980762113533165, 0.5952178177603), rel=1e-5 + ) def test_comparison_results_repr_includes_ci_when_available(patch_metrics):