feat(evals): use geometric mean for evaluation framework, instead of …

docs/02_evals.md

@@ -34,9 +34,11 @@ Within each group the tail values are averaged:
 
 $$\bar{t}_g = \frac{1}{|g|} \sum_{f \in g} t_f$$
 
-The overall **alignment score** $\mathcal{S}$ is then the mean over groups, weighting each group equally:
+The overall **alignment score** $\mathcal{S}$ is then the geometric mean over groups, weighting each group equally. Group tail values are floored at $\varepsilon = 10^{-3}$ before the product is taken, so that a model which genuinely scores zero on one dimension receives a heavy penalty without collapsing scores for all other groups:
 
-$$\mathcal{S} = \frac{1}{G} \sum_{g=1}^{G} \bar{t}_g \in [0, 1]$$
+$$\mathcal{S} = \left(\prod_{g=1}^{G} \max(\bar{t}_g,\, \varepsilon)\right)^{1/G} \in (0, 1]$$
+
+This rewards consistent performance across all groups; a high score on one dimension cannot compensate for near-zero performance on another. The harmonic mean would impose an even harsher penalty on low groups, but it collapses discrimination when any group score is exactly zero, which occurs frequently in practice, making it unsuitable here.
 
 A score of **1** would mean the completions are stylistically indistinguishable from the reference even relative to same-author variation; in practice this is unrealistic but the directionality is still informative.
 
@@ -72,9 +74,9 @@ $$\bar{t}_{g,-i} = \frac{1}{|g|} \sum_{f \in g} t_{f,-i}$$
 
 and percentile intervals are read from $\{\bar{t}_{g,-i}\}$.
 
-For the overall score, the replicate scores are:
+For the overall score, the replicate scores are the geometric mean of the per-group replicate averages:
 
-$$\mathcal{S}_{-i} = \frac{1}{G} \sum_{g=1}^G \bar{t}_{g,-i}$$
+$$\mathcal{S}_{-i} = \left(\prod_{g=1}^{G} \max(\bar{t}_{g,-i},\, \varepsilon)\right)^{1/G}$$
 
 and percentile intervals are read from $\{\mathcal{S}_{-i}\}$.
 
@@ -84,7 +86,7 @@ and percentile intervals are read from $\{\mathcal{S}_{-i}\}$.
 
 Suppose a model achieves an alignment score of **0.2**. A natural reading is:
 
-> On average across stylometric groups, the model's completions sit at the 80th percentile of same-author self-distances, meaning the completions are stylistically closer to the reference than 20% of within-author sample pairs drawn from the training corpus.
+> On average, with respect to the geometric average, across stylometric groups, the model's completions sit at the 80th percentile of same-author self-distances, meaning the completions are stylistically closer to the reference than 20% of within-author sample pairs drawn from the training corpus.
 
 ---
 

src/voice/_defaults.py

@@ -126,6 +126,23 @@ class UncertaintyDefaults:
 UNCERTAINTY_DEFAULTS: UncertaintyDefaults = UncertaintyDefaults()
 
 
+@dataclass(frozen=True)
+class ScoringDefaults:
+    """
+    Default parameters for alignment score computation.
+
+    .. attribute :: group_eps
+
+        Epsilon floor applied to each group tail value before taking the
+        geometric mean across groups.
+    """
+
+    group_eps: float = 1e-3
+
+
+SCORING_DEFAULTS: ScoringDefaults = ScoringDefaults()
+
+
 @dataclass(frozen=True)
 class PlottingDefaults:
     """

src/voice/comparison/comparison.py

@@ -20,6 +20,7 @@
 
 from voice._defaults import (
     CALIBRATION_DEFAULTS,
+    SCORING_DEFAULTS,
     UNCERTAINTY_DEFAULTS,
     MetricGroup,
 )
@@ -182,10 +183,11 @@ class ComparisonResults:
     The object is initialised with a subset of metric names. Entries can
     then be added for each metric.
 
-    The alignment score is the mean of per-group average tail values, with
-    each group weighted equally regardless of how many metrics it contains.
-    A score of 1 indicates perfect stylistic indistinguishability; 0 indicates
-    maximum divergence.
+    The alignment score is the geometric mean of per-group average tail values.
+
+    Group tail values are floored at ``SCORING_DEFAULTS.group_eps`` before the
+    product is taken. A score of 1 indicates perfect stylistic
+    indistinguishability; 0 indicates maximum divergence.
 
     When uncertainty quantification is enabled, the matrix of jackknife
     leave-one-out tail-value replicates is attached and per-metric,
@@ -268,13 +270,20 @@ def score(self) -> float:
         """
         Alignment score in [0, 1]; higher = more stylistically similar.
 
-        Computed as the mean of per-group average tail values, with each
-        group weighted equally regardless of how many metrics it contains.
+        Computed as the geometric mean of per-group average tail values, with
+        each group weighted equally. Values are floored at
+        ``SCORING_DEFAULTS.group_eps`` before the product is taken.
         """
         entries = self.group_entries
         if not entries:
             return 0.0
-        return sum(e.avg_tail for e in entries.values()) / len(entries)
+        tails = np.array(
+            [
+                max(e.avg_tail, SCORING_DEFAULTS.group_eps)
+                for e in entries.values()
+            ]
+        )
+        return float(np.exp(np.log(tails).mean()))
 
     @property
     def _group_indices(self) -> dict[MetricGroup, list[int]]:
@@ -346,10 +355,10 @@ def score_ci(
         """
         Jackknife confidence interval for the overall alignment score.
 
-        Each leave-one-out replicate yields a replicate score (the mean of
-        its per-group average tail values, groups weighted equally); the
-        percentile interval is read from the empirical distribution of these
-        replicate scores.
+        Each leave-one-out replicate yields a replicate score (the geometric
+        mean of its per-group average tail values, floored at
+        ``SCORING_DEFAULTS.group_eps``); the percentile interval is read from
+        the empirical distribution of these replicate scores.
 
         :param confidence: Confidence level, strictly between 0 and 1
         :return: (lower, upper) interval for the score, or None if
@@ -362,8 +371,10 @@ def score_ci(
                 self._replicates[:, idx].mean(axis=1)
                 for idx in self._group_indices.values()
             ]
-        )
-        return percentile_interval(group_scores.mean(axis=0), confidence)
+        )  # shape: (n_groups, n_replicates)
+        floored = np.maximum(group_scores, SCORING_DEFAULTS.group_eps)
+        replicate_scores = np.exp(np.log(floored).mean(axis=0))
+        return percentile_interval(replicate_scores, confidence)
 
     @property
     def metric_tails(self) -> dict[str, float]:

tests/voice/comparison/test_comparison.py

@@ -406,11 +406,11 @@ def test_comparison_results_properties_and_add_logic(patch_metrics):
     assert "len" in r.as_dict()
 
 
-def test_comparison_results_score_is_mean_group_tail(patch_metrics):
+def test_comparison_results_score_is_geometric_mean_group_tail(patch_metrics):
     r = ComparisonResults(metrics=("len", "vowels"))
-    r.add(metric="len", wasserstein=1.0, percentile=0.8)
-    r.add(metric="vowels", wasserstein=0.5, percentile=0.4)
-    assert r.score == pytest.approx(0.4)
+    r.add(metric="len", wasserstein=1.0, percentile=0.8)  # tail=0.2
+    r.add(metric="vowels", wasserstein=0.5, percentile=0.4)  # tail=0.6
+    assert r.score == pytest.approx(np.sqrt(0.2 * 0.6))
 
 
 def test_comparison_results_score_weights_groups_equally(patch_metrics):
@@ -437,8 +437,11 @@ def test_comparison_results_score_weights_groups_equally(patch_metrics):
     r = ComparisonResults(metrics=("len", "len2", "vowels"))
     r.add(metric="len", wasserstein=1.0, percentile=0.0)  # tail=1.0
     r.add(metric="len2", wasserstein=1.0, percentile=0.0)  # tail=1.0
-    r.add(metric="vowels", wasserstein=1.0, percentile=1.0)  # tail=0.0
-    assert r.score == pytest.approx(0.5)
+    r.add(
+        metric="vowels", wasserstein=1.0, percentile=1.0
+    )  # tail=0.0, floored to eps
+    # geo mean of (1.0, eps=0.001); groups still weighted equally
+    assert r.score == pytest.approx(np.sqrt(1e-3))
 
 
 def test_comparison_results_add_rejects_metric_not_initialised(patch_metrics):
@@ -500,8 +503,12 @@ def test_comparison_results_confidence_intervals(patch_metrics):
     assert g[MetricGroup.WORD_LENGTH_DISTRIBUTION] == pytest.approx((0.2, 0.4))
     assert g[MetricGroup.VOCABULARY_RICHNESS] == pytest.approx((0.6, 0.8))
 
-    # Replicate scores are row means over groups: [0.3, 0.4, 0.5, 0.6, 0.7]
-    assert r.score_ci(confidence=0.5) == pytest.approx((0.4, 0.6))
+    # Replicate scores are geometric means over groups:
+    # [sqrt(0.05), sqrt(0.12), sqrt(0.21), sqrt(0.32), sqrt(0.45)]
+    # 25th/75th percentiles fall exactly at indices 1 and 3
+    assert r.score_ci(confidence=0.5) == pytest.approx(
+        (np.sqrt(0.2 * 0.6), np.sqrt(0.4 * 0.8))
+    )
 
 
 def test_comparison_results_cis_average_replicates_within_groups(
@@ -539,8 +546,12 @@ def test_comparison_results_cis_average_replicates_within_groups(
     assert g[MetricGroup.WORD_LENGTH_DISTRIBUTION] == pytest.approx((0.2, 0.4))
     assert g[MetricGroup.VOCABULARY_RICHNESS] == pytest.approx((0.9, 0.9))
 
-    # Score replicates: mean over groups = [0.5, 0.6, 0.7]
-    assert r.score_ci(confidence=0.5) == pytest.approx((0.55, 0.65))
+    # Score replicates: geometric mean over groups
+    # = [sqrt(0.09), sqrt(0.27), sqrt(0.45)]
+    # 25th/75th percentiles with linear interpolation over 3 values
+    assert r.score_ci(confidence=0.5) == pytest.approx(
+        (0.40980762113533165, 0.5952178177603), rel=1e-5
+    )
 
 
 def test_comparison_results_repr_includes_ci_when_available(patch_metrics):