From 20401130f8c9ecfbc41a882f69176a6cf82d5d5c Mon Sep 17 00:00:00 2001
From: Fin Griffin <fingriffin2023@outlook.com>
Date: Thu, 21 May 2026 16:09:09 +0100
Subject: [PATCH 1/5] feat(docs): add docs for data, evals and stylometry

---
 README.md                | 13 ++++++-
 cspell/library-words.txt |  2 +
 cspell/project-words.txt |  1 +
 docs/00_data.md          | 22 +++++++++++
 docs/01_stylometry.md    | 52 ++++++++++++++++++++++++++
 docs/02_evals.md         | 80 ++++++++++++++++++++++++++++++++++++++++
 6 files changed, 169 insertions(+), 1 deletion(-)
 create mode 100644 docs/00_data.md
 create mode 100644 docs/01_stylometry.md
 create mode 100644 docs/02_evals.md

diff --git a/README.md b/README.md
index 9c1ae2b..f6a441d 100644
--- a/README.md
+++ b/README.md
@@ -1,3 +1,14 @@
-# VOICE - Fine-tuning for Stylistic Fidelity
+# 🗣️ VOICE
+
+...
 
 ---
+
+## Documentation
+
+```
+docs/
+├── 00_data.md         — Included datasets
+├── 01_evals.md        — Evaluation suite: scoring and interpretation
+└── 02_stylometry.md   — Stylometric metrics: definition and catalogue
+```
diff --git a/cspell/library-words.txt b/cspell/library-words.txt
index dc8d8fa..3e2f1e1 100644
--- a/cspell/library-words.txt
+++ b/cspell/library-words.txt
@@ -9,3 +9,5 @@ ylabel
 frameon
 allclose
 linalg
+mathcal
+mathbb
diff --git a/cspell/project-words.txt b/cspell/project-words.txt
index 43b5325..68a0486 100644
--- a/cspell/project-words.txt
+++ b/cspell/project-words.txt
@@ -12,3 +12,4 @@ bonf
 bonferroni
 aeiou
 stylometrically
+evals
diff --git a/docs/00_data.md b/docs/00_data.md
new file mode 100644
index 0000000..e504061
--- /dev/null
+++ b/docs/00_data.md
@@ -0,0 +1,22 @@
+# Datasets
+
+---
+
+Two example datasets are included, each containing press conference Q&A transcripts split into train, validation and test sets:
+
+| President | HuggingFace |
+|---|---|
+| Barack Obama | [`AccelerateScience/bo-press-conference-qa`](https://huggingface.co/datasets/AccelerateScience/bo-press-conference-qa) |
+| George W. Bush | [`AccelerateScience/gwb-press-conference-qa`](https://huggingface.co/datasets/AccelerateScience/gwb-press-conference-qa) |
+
+Each example is a single JSONL record in chat format:
+
+```json
+{
+  "messages": [
+    {"role": "system",    "content": "You are president Barack Obama ..."},
+    {"role": "user",      "content": "<press conference question>"},
+    {"role": "assistant", "content": "<president's answer>"}
+  ]
+}
+```
diff --git a/docs/01_stylometry.md b/docs/01_stylometry.md
new file mode 100644
index 0000000..ee053e4
--- /dev/null
+++ b/docs/01_stylometry.md
@@ -0,0 +1,52 @@
+# Stylometric Metrics
+
+---
+
+A **stylometric metric** is any function
+
+$$f : \mathcal{T} \rightarrow \mathbb{R}$$
+
+that maps a text string to a real scalar capturing some surface property of writing style. VOICE treats each metric as a distribution over a corpus: given a set of texts, $f$ is applied to each one to produce a sample from the author's stylometric distribution for that feature.
+
+## Implemented Metrics
+
+### Word Length Distribution
+Moments of the per-word character length distribution.
+
+| Metric | Description |
+|---|---|
+| `avg_word_length` | Mean word length |
+| `std_word_length` | Standard deviation of word length |
+| `skew_word_length` | Skewness of word length |
+| `kurtosis_word_length` | Kurtosis of word length |
+
+### Vocabulary Richness
+Type–token and word statistics measuring lexical diversity.
+
+| Metric | Description |
+|---|---|
+| `type_token_ratio` | Unique tokens / total tokens |
+| `moving_avg_type_token_ratio` | TTR averaged over a sliding window (MATTR) |
+| `hapax_legomena_ratio` | Fraction of words appearing exactly once |
+| `dis_legomena_ratio` | Fraction of words appearing exactly twice |
+| `tri_legomena_ratio` | Fraction of words appearing exactly three times |
+
+### Function Words
+| Metric | Description |
+|---|---|
+| `function_word_ratio` | Proportion of tokens drawn from a closed function-word list |
+
+### Character N-gram Diversity
+Type–token ratio and MATTR computed over character $n$-grams for $n \in \{3, 4, 5\}$.
+
+| Metric | Description |
+|---|---|
+| `char_{n}gram_type_token_ratio` | Character $n$-gram TTR |
+| `char_{n}gram_moving_avg_type_token_ratio` | Character $n$-gram MATTR |
+
+### Text Length
+| Metric | Description |
+|---|---|
+| `num_words` | Total word count |
+
+> **Note:** `num_words` may be deprecated in a future release. It does not tend to be used as a signature for authorship attribution in the broader stylometry literature.
diff --git a/docs/02_evals.md b/docs/02_evals.md
new file mode 100644
index 0000000..4b55fce
--- /dev/null
+++ b/docs/02_evals.md
@@ -0,0 +1,80 @@
+# Evaluation Suite
+
+---
+
+## How Scoring Works
+
+For a set of model completions $\mathcal{C} = \{c_1, \dots, c_n\}$ and a set of reference answers $\mathcal{D} = \{d_1, \dots, d_n\}$ drawn from the same underlying question corpus, VOICE computes an **alignment score** in $[0, 1]$ measuring how stylometrically similar the completions are to the reference author. Both $\mathcal{C}$ and $\mathcal{D}$ are responses to the same set of prompts, so any observed distributional difference reflects style rather than content.
+
+### Step 1: Wasserstein Distance
+
+For each metric $f$, apply it to every text to get empirical distributions:
+
+$$X = \{f(c_i)\}_{i=1}^n \quad \text{and} \quad Y = \{f(d_i)\}_{i=1}^n$$
+
+and compute their Wasserstein-1 distance $W(X, Y)$. A smaller distance means the two distributions are closer in shape.
+
+### Step 2: Calibration
+
+To interpret $W(X, Y)$ we need to know what distance is *expected even between two samples of the same size from the same author*. VOICE estimates this from the **training split** of $\mathcal{D}$ by bootstrap resampling: repeatedly drawing two non-overlapping subsets of size $n$ and computing their mutual Wasserstein distance. This yields a reference distribution $\mathcal{W}_0$ of self-distances that characterises natural within-author variation under the same sample size as the completions being evaluated.
+
+### Step 3: Tail Value
+
+The observed distance is located within $\mathcal{W}_0$ via its empirical CDF:
+
+$$p = \hat{F}_{\mathcal{W}_0}(W(X, Y)), \qquad \tau = 1 - p$$
+
+$\tau$ is the **tail value**: the fraction of self-distances that *exceed* the observed distance. A high $\tau$ means the model's completions are no further from the reference than typical same-author samples, i.e. they are stylistically consistent with the reference author.
+
+### Step 4: Group and Overall Score
+
+Metrics are organised into groups (see [Stylometric Metrics](01_stylometry.md)). Each group captures a distinct linguistic object — word length, vocabulary richness, and so on — and the metrics within it represent different ways of characterising that same object. Because they study the same underlying property, within-group metrics are highly correlated; averaging within groups before aggregating across them prevents any single linguistic dimension from dominating the score simply by having more metrics defined for it.
+
+Within each group the tail values are averaged:
+
+$$\bar{\tau}_g = \frac{1}{|g|} \sum_{f \in g} \tau_f$$
+
+The overall **alignment score** is then the mean over groups, weighting each group equally:
+
+$$\text{score} = \frac{1}{G} \sum_{g=1}^{G} \bar{\tau}_g \in [0, 1]$$
+
+A score of **1** would mean the completions are stylistically indistinguishable from the reference even relative to same-author variation; in practice this is unrealistic but the directionality is still informative.
+
+---
+
+## This Is Not a Hypothesis Test
+
+The tail value $\tau$ resembles a p-value but is not one and should not be interpreted as such.
+
+A classical hypothesis test asks whether there is sufficient evidence to reject a null hypothesis. That framing entails a binary decision and is sensitive to sample size: with enough data, even negligible stylistic differences will cross any fixed significance threshold.
+
+VOICE is not concerned with whether two distributions are statistically indistinguishable. It asks how similar they are, on a continuous scale, relative to the natural within-author variation present in the training corpus. The calibration distribution $\mathcal{W}_0$ serves as a normalisation baseline, not a null to be rejected. There is no significance threshold and no p-value, only a score.
+
+---
+
+## Example
+
+Suppose a model achieves an alignment score of **0.2**. A natural reading is:
+
+> On average across stylometric groups, the model's completions sit at the 80th percentile of same-author self-distances, meaning the completions are stylistically closer to the reference than 20% of within-author sample pairs drawn from the training corpus.
+
+---
+
+## Diagnostics
+
+The API exposes intermediate results for deeper inspection:
+
+```python
+results = make_comparison(completions, true_ds)
+
+# Per-metric tail values
+results.metric_tails   # dict[str, float]
+
+# Per-group average tail values
+results.group_tails    # dict[MetricGroup, float]
+
+# Overall score
+results.score          # float
+```
+
+Per-metric tails are useful for identifying which stylometric dimensions are misaligned; per-group tails aggregate correlated metrics and correspond directly to the terms summed in the overall score.

From 99cf6c0aa4ca845f632e24fa81413f7f4dcf3e95 Mon Sep 17 00:00:00 2001
From: Fin Griffin <fingriffin2023@outlook.com>
Date: Tue, 9 Jun 2026 16:18:22 +0100
Subject: [PATCH 2/5] feat(stylometry): remove number of tokens/words as a
 stylometric metric

---
 src/voice/_defaults.py                    |  5 -----
 src/voice/stylometry/__init__.py          |  2 --
 src/voice/stylometry/metrics.py           | 18 ----------------
 tests/voice/comparison/test_comparison.py | 26 ++++++++++++++---------
 tests/voice/stylometry/test_metrics.py    | 20 ++++++-----------
 5 files changed, 23 insertions(+), 48 deletions(-)

diff --git a/src/voice/_defaults.py b/src/voice/_defaults.py
index 7118cc0..da6b078 100644
--- a/src/voice/_defaults.py
+++ b/src/voice/_defaults.py
@@ -31,17 +31,12 @@ class MetricGroup(str, Enum):
     .. attribute :: CHAR_NGRAM_DIVERSITY
 
         Character n-gram TTR and MATTR across n = 3, 4, 5.
-
-    .. attribute :: TEXT_LENGTH
-
-        Raw token count.
     """
 
     WORD_LENGTH_DISTRIBUTION = "word_length_distribution"
     VOCABULARY_RICHNESS = "vocabulary_richness"
     FUNCTION_WORDS = "function_words"
     CHAR_NGRAM_DIVERSITY = "char_ngram_diversity"
-    TEXT_LENGTH = "text_length"
 
 
 @dataclass(frozen=True)
diff --git a/src/voice/stylometry/__init__.py b/src/voice/stylometry/__init__.py
index e4da059..1e52bc6 100644
--- a/src/voice/stylometry/__init__.py
+++ b/src/voice/stylometry/__init__.py
@@ -19,7 +19,6 @@
     calculate_hapax_legomena_ratio,
     calculate_kurtosis_word_length,
     calculate_moving_avg_type_token_ratio,
-    calculate_num_words,
     calculate_skew_word_length,
     calculate_std_word_length,
     calculate_tri_legomena_ratio,
@@ -38,7 +37,6 @@
     "calculate_std_word_length",
     "calculate_skew_word_length",
     "calculate_kurtosis_word_length",
-    "calculate_num_words",
     "calculate_hapax_legomena_ratio",
     "calculate_dis_legomena_ratio",
     "calculate_tri_legomena_ratio",
diff --git a/src/voice/stylometry/metrics.py b/src/voice/stylometry/metrics.py
index 3f4666c..634c963 100644
--- a/src/voice/stylometry/metrics.py
+++ b/src/voice/stylometry/metrics.py
@@ -9,7 +9,6 @@
     - Standard deviation of word length (`std_word_length`)
     - Skewness of word length (`skew_word_length`)
     - Standard kurtosis of word length (`kurtosis_word_length`)
-    - Total number of words (`num_words`)
     - Hapax legomena ratio (`hapax_legomena_ratio`)
     - Dis legomena ratio (`dis_legomena_ratio`)
     - Tri legomena ratio (`tri_legomena_ratio`)
@@ -235,23 +234,6 @@ def calculate_kurtosis_word_length(text: str) -> float:
 # -----------------------------------------------------------------------------
 
 
-@metric(
-    "num_words",
-    group=MetricGroup.TEXT_LENGTH,
-    description="Number of words in text",
-)
-def calculate_num_words(text: str) -> float:
-    """
-    Calculate the number of words in a string.
-
-    Number of words in an empty string is 0.0.
-
-    :param text: Provided text
-    :return: Number of words as a float
-    """
-    return float(len(word_tokenize(text)))
-
-
 @metric(
     "hapax_legomena_ratio",
     group=MetricGroup.VOCABULARY_RICHNESS,
diff --git a/tests/voice/comparison/test_comparison.py b/tests/voice/comparison/test_comparison.py
index 82666d3..b328b72 100644
--- a/tests/voice/comparison/test_comparison.py
+++ b/tests/voice/comparison/test_comparison.py
@@ -85,7 +85,7 @@ def metric_registry():
     return {
         "len": _Metric(
             fn=lambda s: float(len(s)),
-            group=MetricGroup.TEXT_LENGTH,
+            group=MetricGroup.WORD_LENGTH_DISTRIBUTION,
         ),
         "vowels": _Metric(
             fn=lambda s: float(sum(c in "aeiou" for c in s.lower())),
@@ -253,7 +253,9 @@ def raising_metric(s: str) -> float:
         comparison,
         "get_metrics",
         lambda: {
-            "len": _Metric(fn=raising_metric, group=MetricGroup.TEXT_LENGTH)
+            "len": _Metric(
+                fn=raising_metric, group=MetricGroup.WORD_LENGTH_DISTRIBUTION
+            )
         },
     )
 
@@ -281,7 +283,9 @@ def test_self_wasserstein_distribution_is_zero_for_constant_metric(
         comparison,
         "get_metrics",
         lambda: {
-            "const": _Metric(fn=lambda _s: 1.0, group=MetricGroup.TEXT_LENGTH)
+            "const": _Metric(
+                fn=lambda _s: 1.0, group=MetricGroup.WORD_LENGTH_DISTRIBUTION
+            )
         },
     )
 
@@ -305,7 +309,7 @@ def test_self_wasserstein_distribution_is_zero_for_constant_metric(
 def test_comparison_entry_is_frozen_and_slots():
     e = ComparisonEntry(
         metric="m",
-        group=MetricGroup.TEXT_LENGTH,
+        group=MetricGroup.WORD_LENGTH_DISTRIBUTION,
         wasserstein=1.0,
         percentile=0.9,
         tail=0.1,
@@ -317,7 +321,7 @@ def test_comparison_entry_is_frozen_and_slots():
 
 def test_comparison_group_entry_is_frozen_and_slots():
     e = GroupEntry(
-        group=MetricGroup.TEXT_LENGTH,
+        group=MetricGroup.WORD_LENGTH_DISTRIBUTION,
         avg_percentile=0.9,
         avg_tail=0.1,
     )
@@ -341,7 +345,7 @@ def test_comparison_results_properties_and_add_logic(patch_metrics):
     r.add(metric="len", wasserstein=np.float64(1.25), percentile=0.9)
     e = r.get("len")
     assert e.metric == "len"
-    assert e.group == MetricGroup.TEXT_LENGTH
+    assert e.group == MetricGroup.WORD_LENGTH_DISTRIBUTION
     assert isinstance(e.wasserstein, float)
     assert isinstance(e.percentile, float)
     assert e.tail == pytest.approx(0.1)
@@ -351,7 +355,7 @@ def test_comparison_results_properties_and_add_logic(patch_metrics):
     assert e2.group == MetricGroup.VOCABULARY_RICHNESS
 
     # Group-level checks
-    ge_len = r.get_group(MetricGroup.TEXT_LENGTH)
+    ge_len = r.get_group(MetricGroup.WORD_LENGTH_DISTRIBUTION)
     assert ge_len.avg_percentile == pytest.approx(0.9)
     assert ge_len.avg_tail == pytest.approx(0.1)
 
@@ -375,10 +379,12 @@ def test_comparison_results_score_weights_groups_equally(patch_metrics):
     # Add a second TEXT_LENGTH metric to verify equal group weighting.
     registry = {
         "len": _Metric(
-            fn=lambda s: float(len(s)), group=MetricGroup.TEXT_LENGTH
+            fn=lambda s: float(len(s)),
+            group=MetricGroup.WORD_LENGTH_DISTRIBUTION,
         ),
         "len2": _Metric(
-            fn=lambda s: float(len(s)), group=MetricGroup.TEXT_LENGTH
+            fn=lambda s: float(len(s)),
+            group=MetricGroup.WORD_LENGTH_DISTRIBUTION,
         ),
         "vowels": _Metric(
             fn=lambda s: float(sum(c in "aeiou" for c in s.lower())),
@@ -418,7 +424,7 @@ def test_comparison_results_group_tails(patch_metrics):
     r.add(metric="vowels", wasserstein=0.5, percentile=0.3)
 
     tails = r.group_tails
-    assert tails[MetricGroup.TEXT_LENGTH] == pytest.approx(0.2)
+    assert tails[MetricGroup.WORD_LENGTH_DISTRIBUTION] == pytest.approx(0.2)
     assert tails[MetricGroup.VOCABULARY_RICHNESS] == pytest.approx(0.7)
 
 
diff --git a/tests/voice/stylometry/test_metrics.py b/tests/voice/stylometry/test_metrics.py
index 69fc947..8adc336 100644
--- a/tests/voice/stylometry/test_metrics.py
+++ b/tests/voice/stylometry/test_metrics.py
@@ -80,7 +80,6 @@ def test_get_metrics_contains_expected_metric_names():
         "std_word_length",
         "skew_word_length",
         "kurtosis_word_length",
-        "num_words",
         "hapax_legomena_ratio",
         "dis_legomena_ratio",
         "tri_legomena_ratio",
@@ -100,15 +99,19 @@ def test_get_metrics_contains_expected_metric_names():
 def test_metric_decorator_rejects_duplicate_names(monkeypatch):
     # Isolate decorator behaviour by swapping the module registry.
     monkeypatch.setattr(metrics, "_REGISTRY", {})
-    metrics.metric("x", group=MetricGroup.TEXT_LENGTH)(lambda t: 0.0)
+    metrics.metric("x", group=MetricGroup.WORD_LENGTH_DISTRIBUTION)(
+        lambda t: 0.0
+    )
     with pytest.raises(ValueError, match=r"Duplicate metric name 'x'"):
-        metrics.metric("x", group=MetricGroup.TEXT_LENGTH)(lambda t: 1.0)
+        metrics.metric("x", group=MetricGroup.WORD_LENGTH_DISTRIBUTION)(
+            lambda t: 1.0
+        )
 
 
 def test_metric_spec_is_frozen():
     spec = metrics.MetricSpec(
         fn=lambda t: 0.0,
-        group=MetricGroup.TEXT_LENGTH,
+        group=MetricGroup.WORD_LENGTH_DISTRIBUTION,
         description="x",
     )
     assert type(spec).__dataclass_params__.frozen is True
@@ -152,7 +155,6 @@ def test_metric_groups_cover_all_enum_values():
     "std_word_length": MetricGroup.WORD_LENGTH_DISTRIBUTION,
     "skew_word_length": MetricGroup.WORD_LENGTH_DISTRIBUTION,
     "kurtosis_word_length": MetricGroup.WORD_LENGTH_DISTRIBUTION,
-    "num_words": MetricGroup.TEXT_LENGTH,
     "hapax_legomena_ratio": MetricGroup.VOCABULARY_RICHNESS,
     "dis_legomena_ratio": MetricGroup.VOCABULARY_RICHNESS,
     "tri_legomena_ratio": MetricGroup.VOCABULARY_RICHNESS,
@@ -256,13 +258,6 @@ def fake_moment(units, order: int) -> float:
 # -----------------------------------------------------------------------------
 
 
-def test_num_words_counts_tokens(monkeypatch):
-    monkeypatch.setattr(
-        metrics, "word_tokenize", lambda t: ["a", "b", "c", "d"]
-    )
-    assert metrics.calculate_num_words("ignored") == 4.0
-
-
 def test_hapax_dis_tri_legomena_ratios_exact_on_handmade_tokens(monkeypatch):
     tokens = ["a", "b", "b", "c", "c", "c"]
     monkeypatch.setattr(metrics, "word_tokenize", lambda t: tokens)
@@ -457,6 +452,5 @@ def test_all_metrics_run_on_simple_sentence_smoke():
 
 def test_selected_metrics_on_tiny_text_regression():
     text = "a bb ccc"
-    assert metrics.calculate_num_words(text) == 3.0
     assert metrics.calculate_avg_word_length(text) == pytest.approx(2.0)
     assert metrics.calculate_hapax_legomena_ratio(text) == pytest.approx(1.0)

From c44664ae5dda52d553b4e7bbcf2bd0d2e449d471 Mon Sep 17 00:00:00 2001
From: Fin Griffin <fingriffin2023@outlook.com>
Date: Wed, 10 Jun 2026 10:32:28 +0100
Subject: [PATCH 3/5] feat(docs): add UQ to eval suite

---
 docs/02_evals.md | 48 ++++++++++++++++++++++++++++++++++++++++--------
 1 file changed, 40 insertions(+), 8 deletions(-)

diff --git a/docs/02_evals.md b/docs/02_evals.md
index 4b55fce..871d3ca 100644
--- a/docs/02_evals.md
+++ b/docs/02_evals.md
@@ -22,21 +22,21 @@ To interpret $W(X, Y)$ we need to know what distance is *expected even between t
 
 The observed distance is located within $\mathcal{W}_0$ via its empirical CDF:
 
-$$p = \hat{F}_{\mathcal{W}_0}(W(X, Y)), \qquad \tau = 1 - p$$
+$$p = \hat{F}_{\mathcal{W}_0}(W(X, Y)), \qquad t = 1 - p$$
 
-$\tau$ is the **tail value**: the fraction of self-distances that *exceed* the observed distance. A high $\tau$ means the model's completions are no further from the reference than typical same-author samples, i.e. they are stylistically consistent with the reference author.
+$t$ is the **tail value**: the fraction of self-distances that *exceed* the observed distance. A high $t$ means the model's completions are no further from the reference than typical same-author samples, i.e. they are stylistically consistent with the reference author.
 
 ### Step 4: Group and Overall Score
 
-Metrics are organised into groups (see [Stylometric Metrics](01_stylometry.md)). Each group captures a distinct linguistic object — word length, vocabulary richness, and so on — and the metrics within it represent different ways of characterising that same object. Because they study the same underlying property, within-group metrics are highly correlated; averaging within groups before aggregating across them prevents any single linguistic dimension from dominating the score simply by having more metrics defined for it.
+Metrics are organised into groups (see [Stylometric Metrics](01_stylometry.md)). Each group captures a distinct linguistic object (word length, vocabulary richness, .etc) and the metrics within it represent different ways of characterising that same object. Because they study the same underlying property, within-group metrics are highly correlated; averaging within groups before aggregating across them prevents any single linguistic dimension from dominating the score simply by having more metrics defined for it.
 
 Within each group the tail values are averaged:
 
-$$\bar{\tau}_g = \frac{1}{|g|} \sum_{f \in g} \tau_f$$
+$$\bar{t}_g = \frac{1}{|g|} \sum_{f \in g} t_f$$
 
-The overall **alignment score** is then the mean over groups, weighting each group equally:
+The overall **alignment score** $\mathcal{S}$ is then the mean over groups, weighting each group equally:
 
-$$\text{score} = \frac{1}{G} \sum_{g=1}^{G} \bar{\tau}_g \in [0, 1]$$
+$$\mathcal{S} = \frac{1}{G} \sum_{g=1}^{G} \bar{t}_g \in [0, 1]$$
 
 A score of **1** would mean the completions are stylistically indistinguishable from the reference even relative to same-author variation; in practice this is unrealistic but the directionality is still informative.
 
@@ -44,7 +44,7 @@ A score of **1** would mean the completions are stylistically indistinguishable
 
 ## This Is Not a Hypothesis Test
 
-The tail value $\tau$ resembles a p-value but is not one and should not be interpreted as such.
+The tail value $t$ resembles a p-value but is not one and should not be interpreted as such.
 
 A classical hypothesis test asks whether there is sufficient evidence to reject a null hypothesis. That framing entails a binary decision and is sensitive to sample size: with enough data, even negligible stylistic differences will cross any fixed significance threshold.
 
@@ -52,6 +52,38 @@ VOICE is not concerned with whether two distributions are statistically indistin
 
 ---
 
+## Uncertainty Quantification
+
+The uncertainty in $\mathcal{S}$ arises from the finite number of completions $n$. It is estimated using jackknife resampling over the completion set, operated directly at the level of tail values so that no analytical propagation through the calibration CDF is required.
+
+### Jackknife Replicates
+
+For each $i = 1, \dots, n$, remove $c_i$ and its paired reference $d_i$ and run the full pipeline (recomputing the Wasserstein distance and locating it within $\mathcal{W}_0$) for every metric $f$. This yields a matrix of leave-one-out tail values $\{t_{f,-i}\}$, or equivalently $n$ replicate vectors $\mathbf{t}_{-i} \in \mathbb{R}^M$ where $M$ is the total number of metrics across all groups.
+
+### Covariance Matrix
+
+The jackknife covariance matrix $\hat{\Sigma} \in \mathbb{R}^{M \times M}$ is estimated from these replicates:
+
+$$\hat{\Sigma}_{f,f'} = \frac{n-1}{n} \sum_{i=1}^n \left(t_{f,-i} - \bar{t}_{f,-\cdot}\right)\left(t_{f',-i} - \bar{t}_{f',-\cdot}\right)$$
+
+where $\bar{t}_{f,-\cdot} = \frac{1}{n}\sum_i t_{f,-i}$. This matrix captures the full correlation structure across metrics, both within and between groups, without requiring any independence assumptions.
+
+### Propagation to Per-Metric, Group and Overall Uncertainty
+
+The variance of an individual tail value is the corresponding diagonal entry:
+
+$$\hat{\sigma}_{t_f}^2 = \hat{\Sigma}_{ff}$$
+
+For a group score $\bar{t}_g$, the variance is obtained from the submatrix $\hat{\Sigma}_g$ of $\hat{\Sigma}$ corresponding to metrics within group $g$, with uniform weights $\mathbf{w}_g = \frac{1}{|g|}\mathbf{1}$:
+
+$$\hat{\sigma}_{\bar{t}_g}^2 = \mathbf{w}_g^\top \hat{\Sigma}_g \mathbf{w}_g$$
+
+The overall score $\mathcal{S} = \mathbf{w}^\top \mathbf{t}$ is a linear combination of all per-metric tail values, where the weight for metric $f$ in group $g$ is $w_f = \frac{1}{G \cdot |g|}$. Its variance is:
+
+$$\hat{\sigma}_{\mathcal{S}}^2 = \mathbf{w}^\top \hat{\Sigma} \mathbf{w}$$
+
+---
+
 ## Example
 
 Suppose a model achieves an alignment score of **0.2**. A natural reading is:
@@ -77,4 +109,4 @@ results.group_tails    # dict[MetricGroup, float]
 results.score          # float
 ```
 
-Per-metric tails are useful for identifying which stylometric dimensions are misaligned; per-group tails aggregate correlated metrics and correspond directly to the terms summed in the overall score.
+Per-metric tails are useful for identifying which stylometric dimensions are misaligned; per-group tails aggregate correlated metrics and correspond directly to the terms summed in the overall score.
\ No newline at end of file

From 4be56f4553c7b5309bb8f6aa64e2a0f4a72a4cee Mon Sep 17 00:00:00 2001
From: Fin Griffin <fingriffin2023@outlook.com>
Date: Wed, 10 Jun 2026 12:03:27 +0100
Subject: [PATCH 4/5] feat(evals): add assymetric confidence intervals to tail
 scores

---
 .pre-commit-config.yaml                   |  13 -
 cspell/project-words.txt                  |   1 +
 docs/02_evals.md                          |  41 +--
 pyproject.toml                            |   2 -
 src/voice/_defaults.py                    |  20 ++
 src/voice/comparison/_utils.py            |  32 +++
 src/voice/comparison/comparison.py        | 150 ++++++++++-
 tests/voice/comparison/test_comparison.py | 300 +++++++++++++++++++++-
 8 files changed, 516 insertions(+), 43 deletions(-)

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 0eab24b..74210e4 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -74,16 +74,3 @@ repos:
       - id: pydoclint
         args: ["--config=pyproject.toml"]
         exclude: ^tests/
-
-  # Cognitive complexity limit via flake8-cognitive-complexity (keep for src; exclude tests)
-  - repo: https://github.com/pycqa/flake8
-    rev: 7.3.0
-    hooks:
-      - id: flake8
-        additional_dependencies:
-          - flake8-cognitive-complexity
-        args:
-          - --max-line-length=79
-          - --extend-ignore=E203
-          - --max-cognitive-complexity=10
-        exclude: ^tests/
diff --git a/cspell/project-words.txt b/cspell/project-words.txt
index c377ac6..ecd0a7f 100644
--- a/cspell/project-words.txt
+++ b/cspell/project-words.txt
@@ -19,3 +19,4 @@ hyperparams
 redef
 evals
 submatrix
+vocab
diff --git a/docs/02_evals.md b/docs/02_evals.md
index 871d3ca..564c1a2 100644
--- a/docs/02_evals.md
+++ b/docs/02_evals.md
@@ -54,33 +54,29 @@ VOICE is not concerned with whether two distributions are statistically indistin
 
 ## Uncertainty Quantification
 
-The uncertainty in $\mathcal{S}$ arises from the finite number of completions $n$. It is estimated using jackknife resampling over the completion set, operated directly at the level of tail values so that no analytical propagation through the calibration CDF is required.
+The uncertainty in $\mathcal{S}$ arises from the finite number of completions $n$. It is estimated using jackknife resampling over the completion set, operated directly at the level of tail values so that no analytical propagation through the calibration CDF is required. Operating at the tail value level naturally produces asymmetric confidence intervals that respect the $[0, 1]$ boundary of the score.
 
 ### Jackknife Replicates
 
 For each $i = 1, \dots, n$, remove $c_i$ and its paired reference $d_i$ and run the full pipeline (recomputing the Wasserstein distance and locating it within $\mathcal{W}_0$) for every metric $f$. This yields a matrix of leave-one-out tail values $\{t_{f,-i}\}$, or equivalently $n$ replicate vectors $\mathbf{t}_{-i} \in \mathbb{R}^M$ where $M$ is the total number of metrics across all groups.
 
-### Covariance Matrix
+### Confidence Intervals
 
-The jackknife covariance matrix $\hat{\Sigma} \in \mathbb{R}^{M \times M}$ is estimated from these replicates:
+Per-metric, group, and overall confidence intervals are all read directly from the empirical distribution of the corresponding jackknife replicates.
 
-$$\hat{\Sigma}_{f,f'} = \frac{n-1}{n} \sum_{i=1}^n \left(t_{f,-i} - \bar{t}_{f,-\cdot}\right)\left(t_{f',-i} - \bar{t}_{f',-\cdot}\right)$$
+For an individual metric $f$, the replicates $\{t_{f,-i}\}$ give a distribution of leave-one-out tail values from which percentile intervals are taken directly.
 
-where $\bar{t}_{f,-\cdot} = \frac{1}{n}\sum_i t_{f,-i}$. This matrix captures the full correlation structure across metrics, both within and between groups, without requiring any independence assumptions.
+For a group $g$, the replicate group scores are computed as:
 
-### Propagation to Per-Metric, Group and Overall Uncertainty
+$$\bar{t}_{g,-i} = \frac{1}{|g|} \sum_{f \in g} t_{f,-i}$$
 
-The variance of an individual tail value is the corresponding diagonal entry:
+and percentile intervals are read from $\{\bar{t}_{g,-i}\}$.
 
-$$\hat{\sigma}_{t_f}^2 = \hat{\Sigma}_{ff}$$
+For the overall score, the replicate scores are:
 
-For a group score $\bar{t}_g$, the variance is obtained from the submatrix $\hat{\Sigma}_g$ of $\hat{\Sigma}$ corresponding to metrics within group $g$, with uniform weights $\mathbf{w}_g = \frac{1}{|g|}\mathbf{1}$:
+$$\mathcal{S}_{-i} = \frac{1}{G} \sum_{g=1}^G \bar{t}_{g,-i}$$
 
-$$\hat{\sigma}_{\bar{t}_g}^2 = \mathbf{w}_g^\top \hat{\Sigma}_g \mathbf{w}_g$$
-
-The overall score $\mathcal{S} = \mathbf{w}^\top \mathbf{t}$ is a linear combination of all per-metric tail values, where the weight for metric $f$ in group $g$ is $w_f = \frac{1}{G \cdot |g|}$. Its variance is:
-
-$$\hat{\sigma}_{\mathcal{S}}^2 = \mathbf{w}^\top \hat{\Sigma} \mathbf{w}$$
+and percentile intervals are read from $\{\mathcal{S}_{-i}\}$.
 
 ---
 
@@ -97,16 +93,23 @@ Suppose a model achieves an alignment score of **0.2**. A natural reading is:
 The API exposes intermediate results for deeper inspection:
 
 ```python
-results = make_comparison(completions, true_ds)
+results = make_comparison(completions, true_ds)  # uncertainty=True by default
 
 # Per-metric tail values
-results.metric_tails   # dict[str, float]
+results.metric_tails       # dict[str, float]
 
 # Per-group average tail values
-results.group_tails    # dict[MetricGroup, float]
+results.group_tails        # dict[MetricGroup, float]
 
 # Overall score
-results.score          # float
+results.score              # float
+
+# Jackknife percentile confidence intervals (default 90%)
+results.metric_tail_cis()   # dict[str, tuple[float, float]] | None
+results.group_tail_cis()    # dict[MetricGroup, tuple[float, float]] | None
+results.score_ci()          # tuple[float, float] | None
 ```
 
-Per-metric tails are useful for identifying which stylometric dimensions are misaligned; per-group tails aggregate correlated metrics and correspond directly to the terms summed in the overall score.
\ No newline at end of file
+Per-metric tails are useful for identifying which stylometric dimensions are misaligned; per-group tails aggregate correlated metrics and correspond directly to the terms summed in the overall score.
+
+The confidence interval methods accept a `confidence` keyword (default 0.90, see `UNCERTAINTY_DEFAULTS`). Passing `uncertainty=False` to `make_comparison` skips the jackknife pass, in which case the interval methods return `None`.
diff --git a/pyproject.toml b/pyproject.toml
index 2ec89c9..95ecd18 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -42,8 +42,6 @@ where = ["src"]
 [dependency-groups]
 dev = [
     "detect-secrets>=1.5.0",
-    "flake8>=7.3.0",
-    "flake8-cognitive-complexity>=0.1.0",
     "isort>=7.0.0",
     "mypy>=1.19.1",
     "notebook>=7.5.3",
diff --git a/src/voice/_defaults.py b/src/voice/_defaults.py
index da6b078..d7e9d38 100644
--- a/src/voice/_defaults.py
+++ b/src/voice/_defaults.py
@@ -106,6 +106,26 @@ class CalibrationDefaults:
 CALIBRATION_DEFAULTS: CalibrationDefaults = CalibrationDefaults()
 
 
+@dataclass(frozen=True)
+class UncertaintyDefaults:
+    """
+    Default parameters for uncertainty quantification.
+
+    These defaults control the confidence intervals reported alongside
+    comparison results.
+
+    .. attribute :: confidence
+
+        Confidence level for percentile intervals read from the
+        jackknife replicate distributions.
+    """
+
+    confidence: float = 0.90  # Safe default for validation/test size = 150
+
+
+UNCERTAINTY_DEFAULTS: UncertaintyDefaults = UncertaintyDefaults()
+
+
 @dataclass(frozen=True)
 class PlottingDefaults:
     """
diff --git a/src/voice/comparison/_utils.py b/src/voice/comparison/_utils.py
index a684a69..e1756ea 100644
--- a/src/voice/comparison/_utils.py
+++ b/src/voice/comparison/_utils.py
@@ -35,6 +35,38 @@ def calibrated_percentile(value: float, dist: np.ndarray) -> float:
     return float(np.mean(dist <= value))
 
 
+# -----------------------------------------------------------------------------
+# Percentile interval
+# -----------------------------------------------------------------------------
+
+
+def percentile_interval(
+    values: np.ndarray, confidence: float
+) -> tuple[float, float]:
+    """
+    Compute a central percentile interval from an empirical distribution.
+
+    For a confidence level c, this returns the (alpha/2, 1 - alpha/2)
+    empirical percentiles of `values` with alpha = 1 - c. The interval is
+    asymmetric in general and bounded by the range of `values`.
+
+    :param values: Array of samples from an empirical distribution
+    :param confidence: Confidence level, strictly between 0 and 1
+    :return: (lower, upper) percentile interval
+    :raises ValueError: If `values` is empty or `confidence` is not in [0, 1]
+    """
+    values = np.asarray(values, dtype=float).ravel()
+    if values.size == 0:
+        raise ValueError("values must be non-empty.")
+    if not 0.0 < confidence < 1.0:
+        raise ValueError("confidence must be strictly between 0 and 1.")
+
+    alpha = 1.0 - confidence
+    lower = float(np.percentile(values, 100.0 * alpha / 2.0))
+    upper = float(np.percentile(values, 100.0 * (1.0 - alpha / 2.0)))
+    return lower, upper
+
+
 # -----------------------------------------------------------------------------
 # Bootstrap null distribution
 # -----------------------------------------------------------------------------
diff --git a/src/voice/comparison/comparison.py b/src/voice/comparison/comparison.py
index 7173b68..de11e45 100644
--- a/src/voice/comparison/comparison.py
+++ b/src/voice/comparison/comparison.py
@@ -20,11 +20,13 @@
 
 from voice._defaults import (
     CALIBRATION_DEFAULTS,
+    UNCERTAINTY_DEFAULTS,
     MetricGroup,
 )
 from voice.comparison._utils import (
     bootstrap_null_distribution,
     calibrated_percentile,
+    percentile_interval,
 )
 from voice.datasets import VoiceDataset
 from voice.datasets.dataset import Example
@@ -185,6 +187,11 @@ class ComparisonResults:
     A score of 1 indicates perfect stylistic indistinguishability; 0 indicates
     maximum divergence.
 
+    When uncertainty quantification is enabled, the matrix of jackknife
+    leave-one-out tail-value replicates is attached and per-metric,
+    per-group and overall-score percentile confidence intervals are read
+    directly from its empirical distributions.
+
     .. attribute :: metrics
 
         Tuple of stylometric metric names to run comparison for.
@@ -192,6 +199,7 @@ class ComparisonResults:
 
     metrics: tuple[str, ...]
     _entries: dict[str, ComparisonEntry] = field(default_factory=dict)
+    _replicates: np.ndarray | None = field(default=None, repr=False)
 
     def __post_init__(self) -> None:
         """
@@ -222,9 +230,10 @@ def __repr__(self) -> str:
             )
             + "}"
         )
-        return (
-            f"ComparisonResults(groups={group_part}, score={self.score:.4f})"
-        )
+        score_part = f"{self.score:.4f}"
+        if (ci := self.score_ci()) is not None:
+            score_part += f" [{ci[0]:.4f}, {ci[1]:.4f}]"
+        return f"ComparisonResults(groups={group_part}, score={score_part})"
 
     @property
     def num_metrics(self) -> int:
@@ -267,6 +276,95 @@ def score(self) -> float:
             return 0.0
         return sum(e.avg_tail for e in entries.values()) / len(entries)
 
+    @property
+    def _group_indices(self) -> dict[MetricGroup, list[int]]:
+        """
+        Indices into `metrics` (and replicate matrix columns) per group.
+
+        :return: Mapping of MetricGroup to column indices
+        """
+        registry = get_metrics()
+        indices: dict[MetricGroup, list[int]] = {}
+        for j, metric in enumerate(self.metrics):
+            indices.setdefault(registry[metric].group, []).append(j)
+        return indices
+
+    @property
+    def has_uncertainty(self) -> bool:
+        """Whether jackknife uncertainty estimates are attached."""
+        return self._replicates is not None
+
+    def metric_tail_cis(
+        self,
+        confidence: float = UNCERTAINTY_DEFAULTS.confidence,
+    ) -> dict[str, tuple[float, float]] | None:
+        """
+        Jackknife confidence interval for each per-metric tail value.
+
+        Percentile intervals are read directly from the empirical
+        distribution of leave-one-out tail values for each metric.
+
+        :param confidence: Confidence level, strictly between 0 and 1
+        :return: Mapping of metric name to (lower, upper) interval or None if
+            uncertainty quantification was not run
+        """
+        if self._replicates is None:
+            return None
+        return {
+            m: percentile_interval(self._replicates[:, j], confidence)
+            for j, m in enumerate(self.metrics)
+        }
+
+    def group_tail_cis(
+        self,
+        confidence: float = UNCERTAINTY_DEFAULTS.confidence,
+    ) -> dict[MetricGroup, tuple[float, float]] | None:
+        """
+        Jackknife confidence interval for each group's average tail value.
+
+        Each leave-one-out replicate's tail values are averaged within the
+        group, and the percentile interval is read from the resulting
+        empirical distribution of replicate group scores.
+
+        :param confidence: Confidence level, strictly between 0 and 1
+        :return: Mapping of MetricGroup to (lower, upper) interval or None if
+            uncertainty quantification was not run
+        """
+        if self._replicates is None:
+            return None
+        return {
+            group: percentile_interval(
+                self._replicates[:, idx].mean(axis=1), confidence
+            )
+            for group, idx in self._group_indices.items()
+        }
+
+    def score_ci(
+        self,
+        confidence: float = UNCERTAINTY_DEFAULTS.confidence,
+    ) -> tuple[float, float] | None:
+        """
+        Jackknife confidence interval for the overall alignment score.
+
+        Each leave-one-out replicate yields a replicate score (the mean of
+        its per-group average tail values, groups weighted equally); the
+        percentile interval is read from the empirical distribution of these
+        replicate scores.
+
+        :param confidence: Confidence level, strictly between 0 and 1
+        :return: (lower, upper) interval for the score, or None if
+            uncertainty quantification was not run
+        """
+        if self._replicates is None:
+            return None
+        group_scores = np.stack(
+            [
+                self._replicates[:, idx].mean(axis=1)
+                for idx in self._group_indices.values()
+            ]
+        )
+        return percentile_interval(group_scores.mean(axis=0), confidence)
+
     @property
     def metric_tails(self) -> dict[str, float]:
         """
@@ -356,6 +454,7 @@ def make_comparison(
     true_ds: VoiceDataset,
     *,
     metrics: Sequence[str] | None = None,
+    uncertainty: bool = True,
 ) -> ComparisonResults:
     """
     Compare stylometric distributions of `completions` to the true corpus.
@@ -371,14 +470,25 @@ def make_comparison(
     values, and an overall alignment score equal to the mean of group-level
     average tail values (groups weighted equally).
 
+    When `uncertainty` is True (the default), jackknife resampling over the
+    completion set is used: each completion and its paired reference are
+    left out in turn, the Wasserstein distance is recomputed, and the
+    leave-one-out distance is located within the same
+    self-Wasserstein distribution. Per-metric, per-group and overall score
+    percentile confidence intervals are then available on the returned
+    results, read directly from the replicate distributions.
+
     :param completions: Sequence of examples to analyse
     :param true_ds: VoiceDataset providing the true corpus
     :param metrics: Optional subset of metric names; defaults to all metrics
+    :param uncertainty: Whether to estimate uncertainty
     :return: ComparisonResults populated for the selected metrics
     :raises ValueError:
         - If completions is empty,
         - If sample sizes are invalid
         - If metrics are not registered
+        - If `uncertainty` is True and there are fewer than two completions
+          or the observed split does not match the completions in length
     """
     if not completions:
         raise ValueError("completions must be non-empty.")
@@ -392,20 +502,52 @@ def make_comparison(
     observed_split = completions[0].split
     ds_observed_split = true_ds[observed_split]
 
+    n = len(completions)
+    if uncertainty:
+        if n < 2:
+            raise ValueError(
+                "uncertainty quantification requires at least 2 completions."
+            )
+        if len(ds_observed_split) != n:
+            raise ValueError(
+                "uncertainty quantification requires completions to be "
+                "paired one-to-one with the observed split "
+                f"({n} completions vs {len(ds_observed_split)} references)."
+            )
+
+    distributions: dict[str, tuple[np.ndarray, np.ndarray, np.ndarray]] = {}
+
     for metric in metric_list:
         x = stylometric_distribution(completions, metric)
         y = stylometric_distribution(ds_observed_split, metric)
 
         d = float(wasserstein_distance(x, y))
 
+        # Cached per metric: the jackknife pass below reuses this
+        # distribution rather than re-running the bootstrap per replicate.
         self_dist = self_wasserstein_distribution(
             true_ds,
             metric=metric,
-            sample_size=len(completions),
+            sample_size=n,
         )
 
         p = calibrated_percentile(d, self_dist)
 
         results.add(metric=metric, wasserstein=d, percentile=p)
 
+        if uncertainty:
+            distributions[metric] = (x, y, self_dist)
+
+    if uncertainty:
+        replicates = np.empty((n, len(metric_list)), dtype=float)
+        for i in range(n):
+            mask = np.ones(n, dtype=bool)
+            mask[i] = False
+            for j, metric in enumerate(metric_list):
+                x, y, self_dist = distributions[metric]
+                d_i = float(wasserstein_distance(x[mask], y[mask]))
+                replicates[i, j] = 1.0 - calibrated_percentile(d_i, self_dist)
+
+        results._replicates = replicates
+
     return results
diff --git a/tests/voice/comparison/test_comparison.py b/tests/voice/comparison/test_comparison.py
index b328b72..84f1973 100644
--- a/tests/voice/comparison/test_comparison.py
+++ b/tests/voice/comparison/test_comparison.py
@@ -20,7 +20,10 @@
 from datasets import Dataset
 
 from voice._defaults import MetricGroup
-from voice.comparison._utils import calibrated_percentile
+from voice.comparison._utils import (
+    calibrated_percentile,
+    percentile_interval,
+)
 from voice.comparison.comparison import (
     ComparisonEntry,
     ComparisonResults,
@@ -154,6 +157,42 @@ def test_calibrated_percentile_flattens_non_1d_inputs():
     assert calibrated_percentile(2.0, dist) == 0.5
 
 
+# -----------------------------------------------------------------------------
+# percentile_interval
+# -----------------------------------------------------------------------------
+
+
+def test_percentile_interval_rejects_empty_values():
+    with pytest.raises(ValueError, match=r"values must be non-empty"):
+        percentile_interval(np.array([]), 0.9)
+
+
+@pytest.mark.parametrize("confidence", [0.0, 1.0, -0.1, 1.5])
+def test_percentile_interval_rejects_invalid_confidence(confidence):
+    with pytest.raises(ValueError, match=r"strictly between 0 and 1"):
+        percentile_interval(np.array([0.1, 0.2]), confidence)
+
+
+def test_percentile_interval_hand_computed_example():
+    # 25th/75th percentiles of 5 evenly spaced points (linear interpolation)
+    values = np.array([0.1, 0.2, 0.3, 0.4, 0.5])
+    lo, hi = percentile_interval(values, 0.5)
+    assert lo == pytest.approx(0.2)
+    assert hi == pytest.approx(0.4)
+
+
+def test_percentile_interval_collapses_for_constant_values():
+    lo, hi = percentile_interval(np.full(10, 0.7), 0.9)
+    assert lo == pytest.approx(0.7)
+    assert hi == pytest.approx(0.7)
+
+
+def test_percentile_interval_is_bounded_by_value_range():
+    values = np.array([0.0, 0.0, 0.0, 0.9])
+    lo, hi = percentile_interval(values, 0.99)
+    assert 0.0 <= lo <= hi <= 0.9
+
+
 # -----------------------------------------------------------------------------
 # self_wasserstein_distribution
 # -----------------------------------------------------------------------------
@@ -428,6 +467,92 @@ def test_comparison_results_group_tails(patch_metrics):
     assert tails[MetricGroup.VOCABULARY_RICHNESS] == pytest.approx(0.7)
 
 
+def test_comparison_results_uncertainty_defaults_to_none(patch_metrics):
+    r = ComparisonResults(metrics=("len", "vowels"))
+    assert r.has_uncertainty is False
+    assert r.metric_tail_cis() is None
+    assert r.group_tail_cis() is None
+    assert r.score_ci() is None
+
+
+def test_comparison_results_confidence_intervals(patch_metrics):
+    r = ComparisonResults(metrics=("len", "vowels"))
+    r.add(metric="len", wasserstein=1.0, percentile=0.8)
+    r.add(metric="vowels", wasserstein=0.5, percentile=0.4)
+    r._replicates = np.array(
+        [
+            [0.1, 0.5],
+            [0.2, 0.6],
+            [0.3, 0.7],
+            [0.4, 0.8],
+            [0.5, 0.9],
+        ]
+    )
+
+    assert r.has_uncertainty is True
+
+    cis = r.metric_tail_cis(confidence=0.5)
+    assert cis["len"] == pytest.approx((0.2, 0.4))
+    assert cis["vowels"] == pytest.approx((0.6, 0.8))
+
+    # One metric per group, so group CI equals metric CI
+    g = r.group_tail_cis(confidence=0.5)
+    assert g[MetricGroup.WORD_LENGTH_DISTRIBUTION] == pytest.approx((0.2, 0.4))
+    assert g[MetricGroup.VOCABULARY_RICHNESS] == pytest.approx((0.6, 0.8))
+
+    # Replicate scores are row means over groups: [0.3, 0.4, 0.5, 0.6, 0.7]
+    assert r.score_ci(confidence=0.5) == pytest.approx((0.4, 0.6))
+
+
+def test_comparison_results_cis_average_replicates_within_groups(
+    patch_metrics, monkeypatch
+):
+    registry = {
+        "len": _Metric(
+            fn=lambda s: float(len(s)),
+            group=MetricGroup.WORD_LENGTH_DISTRIBUTION,
+        ),
+        "len2": _Metric(
+            fn=lambda s: float(len(s)),
+            group=MetricGroup.WORD_LENGTH_DISTRIBUTION,
+        ),
+        "vowels": _Metric(
+            fn=lambda s: float(sum(c in "aeiou" for c in s.lower())),
+            group=MetricGroup.VOCABULARY_RICHNESS,
+        ),
+    }
+    import voice.comparison.comparison as comparison
+
+    monkeypatch.setattr(comparison, "get_metrics", lambda: dict(registry))
+
+    r = ComparisonResults(metrics=("len", "len2", "vowels"))
+    r._replicates = np.array(
+        [
+            [0.0, 0.2, 0.9],
+            [0.2, 0.4, 0.9],
+            [0.4, 0.6, 0.9],
+        ]
+    )
+
+    # Group replicates: word_length = [0.1, 0.3, 0.5], vocab = [0.9] * 3
+    g = r.group_tail_cis(confidence=0.5)
+    assert g[MetricGroup.WORD_LENGTH_DISTRIBUTION] == pytest.approx((0.2, 0.4))
+    assert g[MetricGroup.VOCABULARY_RICHNESS] == pytest.approx((0.9, 0.9))
+
+    # Score replicates: mean over groups = [0.5, 0.6, 0.7]
+    assert r.score_ci(confidence=0.5) == pytest.approx((0.55, 0.65))
+
+
+def test_comparison_results_repr_includes_ci_when_available(patch_metrics):
+    r = ComparisonResults(metrics=("len",))
+    r.add(metric="len", wasserstein=1.0, percentile=0.8)
+    assert "[" not in repr(r)
+
+    # 5th/95th percentiles of [0.1, 0.2, 0.3] (default confidence 0.90)
+    r._replicates = np.array([[0.1], [0.2], [0.3]])
+    assert "[0.1100, 0.2900]" in repr(r)
+
+
 # -----------------------------------------------------------------------------
 # make_comparison
 # -----------------------------------------------------------------------------
@@ -464,7 +589,7 @@ def fake_self_dist(*_args, **_kwargs):
 
     comparison.self_wasserstein_distribution = fake_self_dist  # type: ignore[assignment]
 
-    out = make_comparison(completions, true_ds)
+    out = make_comparison(completions, true_ds, uncertainty=False)
 
     assert out.metrics == ("len", "vowels")
     for m in out.metrics:
@@ -492,7 +617,9 @@ def test_make_comparison_respects_subset_metrics(patch_metrics, monkeypatch):
         lambda *_args, **_kwargs: np.array([0.0, 1.0], dtype=float),
     )
 
-    out = make_comparison(completions, true_ds, metrics=["len"])
+    out = make_comparison(
+        completions, true_ds, metrics=["len"], uncertainty=False
+    )
     assert out.metrics == ("len",)
     assert "len" in out.as_dict()
     with pytest.raises(KeyError):
@@ -529,7 +656,9 @@ def recording_getitem(self, key):
 
     monkeypatch.setattr(VoiceDataset, "__getitem__", recording_getitem)
 
-    _ = make_comparison(completions, true_ds, metrics=["len"])
+    _ = make_comparison(
+        completions, true_ds, metrics=["len"], uncertainty=False
+    )
 
     assert Split.TEST in calls
 
@@ -563,6 +692,167 @@ def spy_self_dist(*_args, sample_size: int = 0, **_kwargs):
         comparison, "self_wasserstein_distribution", spy_self_dist
     )
 
-    make_comparison(completions, true_ds, metrics=["len"])
+    make_comparison(completions, true_ds, metrics=["len"], uncertainty=False)
 
     assert all(s == len(completions) for s in captured)
+
+
+# -----------------------------------------------------------------------------
+# make_comparison: uncertainty quantification
+# -----------------------------------------------------------------------------
+
+
+def _paired_setup(n: int = 5) -> tuple[list[Example], VoiceDataset]:
+    """Completions paired one-to-one with the TEST split."""
+    p = _pinned(splits=(Split.TRAIN, Split.TEST))
+    true_ds = VoiceDataset(
+        datasets={
+            Split.TRAIN: _canonical_hf_ds([f"a{i}" for i in range(40)]),
+            Split.TEST: _canonical_hf_ds(
+                ["ref text " * (i + 1) for i in range(n)]
+            ),
+        },
+        spec=p,
+    )
+    completions = [
+        _ex(answer="hello there " * (i + 2), split=Split.TEST)
+        for i in range(n)
+    ]
+    return completions, true_ds
+
+
+def test_make_comparison_uncertainty_populates_cis_and_caches_self_dist(
+    patch_metrics, monkeypatch
+):
+    completions, true_ds = _paired_setup(n=5)
+
+    import voice.comparison.comparison as comparison
+
+    calls: list[str] = []
+
+    def spy_self_dist(*_args, metric: str = "", **_kwargs):
+        calls.append(metric)
+        return np.linspace(0.0, 50.0, 100)
+
+    monkeypatch.setattr(
+        comparison, "self_wasserstein_distribution", spy_self_dist
+    )
+
+    out = make_comparison(completions, true_ds)
+
+    assert out.has_uncertainty is True
+
+    cis = out.metric_tail_cis()
+    assert set(cis) == {"len", "vowels"}
+    for lo, hi in cis.values():
+        assert 0.0 <= lo <= hi <= 1.0
+
+    assert set(out.group_tail_cis()) == {
+        MetricGroup.WORD_LENGTH_DISTRIBUTION,
+        MetricGroup.VOCABULARY_RICHNESS,
+    }
+
+    lo, hi = out.score_ci()
+    assert 0.0 <= lo <= hi <= 1.0
+
+    # A wider confidence level produces a containing interval
+    lo99, hi99 = out.score_ci(confidence=0.99)
+    assert lo99 <= lo
+    assert hi <= hi99
+
+    # Self-Wasserstein bootstrap is cached: computed exactly once per
+    # metric, never per jackknife replicate.
+    assert sorted(calls) == ["len", "vowels"]
+
+
+def test_make_comparison_uncertainty_false_skips_jackknife(
+    patch_metrics, monkeypatch
+):
+    completions, true_ds = _paired_setup(n=5)
+
+    import voice.comparison.comparison as comparison
+
+    monkeypatch.setattr(
+        comparison,
+        "self_wasserstein_distribution",
+        lambda *_args, **_kwargs: np.array([0.0, 1.0], dtype=float),
+    )
+
+    out = make_comparison(completions, true_ds, uncertainty=False)
+
+    assert out.has_uncertainty is False
+    assert out.metric_tail_cis() is None
+    assert out.group_tail_cis() is None
+    assert out.score_ci() is None
+
+
+def test_make_comparison_uncertainty_requires_two_completions(
+    patch_metrics, monkeypatch
+):
+    completions, true_ds = _paired_setup(n=1)
+
+    with pytest.raises(ValueError, match=r"at least 2 completions"):
+        make_comparison(completions, true_ds)
+
+
+def test_make_comparison_uncertainty_requires_paired_split(
+    patch_metrics, monkeypatch
+):
+    completions, true_ds = _paired_setup(n=5)
+
+    with pytest.raises(ValueError, match=r"paired one-to-one"):
+        make_comparison(completions[:3], true_ds)
+
+
+def test_make_comparison_score_ci_matches_metric_ci_for_single_metric(
+    patch_metrics, monkeypatch
+):
+    completions, true_ds = _paired_setup(n=5)
+
+    import voice.comparison.comparison as comparison
+
+    monkeypatch.setattr(
+        comparison,
+        "self_wasserstein_distribution",
+        lambda *_args, **_kwargs: np.linspace(0.0, 50.0, 100),
+    )
+
+    out = make_comparison(completions, true_ds, metrics=["len"])
+
+    assert out.score_ci() == pytest.approx(out.metric_tail_cis()["len"])
+    assert out.score_ci() == pytest.approx(
+        out.group_tail_cis()[MetricGroup.WORD_LENGTH_DISTRIBUTION]
+    )
+
+
+def test_make_comparison_ci_collapses_for_identical_paired_data(
+    patch_metrics, monkeypatch
+):
+    n = 5
+    answers = [f"some answer {i} " * (i + 1) for i in range(n)]
+    p = _pinned(splits=(Split.TRAIN, Split.TEST))
+    true_ds = VoiceDataset(
+        datasets={
+            Split.TRAIN: _canonical_hf_ds([f"a{i}" for i in range(40)]),
+            Split.TEST: _canonical_hf_ds(answers),
+        },
+        spec=p,
+    )
+    completions = [_ex(answer=a, split=Split.TEST) for a in answers]
+
+    import voice.comparison.comparison as comparison
+
+    monkeypatch.setattr(
+        comparison,
+        "self_wasserstein_distribution",
+        lambda *_args, **_kwargs: np.linspace(0.0, 50.0, 100),
+    )
+
+    # x == y for every metric, so every leave-one-out distance is 0 and
+    # all replicates are identical: each interval collapses to a point.
+    out = make_comparison(completions, true_ds)
+
+    lo, hi = out.score_ci()
+    assert lo == pytest.approx(hi)
+    for m_lo, m_hi in out.metric_tail_cis().values():
+        assert m_lo == pytest.approx(m_hi)

From 724c10b6f8cbfae106dc1884e50f7ff66a061d63 Mon Sep 17 00:00:00 2001
From: Fin Griffin <fingriffin2023@outlook.com>
Date: Wed, 10 Jun 2026 12:10:41 +0100
Subject: [PATCH 5/5] feat(finetuning): write confidence intervals to disk via
 fine-tuning orchestrator

---
 src/voice/finetune/_orchestrator.py | 16 ++++++++++++++--
 1 file changed, 14 insertions(+), 2 deletions(-)

diff --git a/src/voice/finetune/_orchestrator.py b/src/voice/finetune/_orchestrator.py
index f06727c..40a74e9 100644
--- a/src/voice/finetune/_orchestrator.py
+++ b/src/voice/finetune/_orchestrator.py
@@ -177,8 +177,11 @@ def _score_split(
 
     Alignment results are written to
     ``alignment/epoch_{n}/{split}.json`` with keys ``score``,
-    ``metric_tails``, and ``group_tails``. Only the overall score is
-    returned; the full tails are available on disk only.
+    ``score_ci``, ``metric_tails``, ``metric_tail_cis``, ``group_tails``,
+    and ``group_tail_cis``. Confidence intervals are jackknife percentile
+    intervals at the default confidence level and are serialised as
+    ``[lower, upper]`` pairs. Only the overall score is returned; the
+    full tails and intervals are available on disk only.
 
     :param run_dir: Run directory.
     :param split: Split name (``"validation"`` or ``"test"``).
@@ -191,16 +194,25 @@ def _score_split(
     completions = _load_completions(run_dir, split, epoch)
     results = make_comparison(completions, reference_ds)
 
+    group_cis = results.group_tail_cis()
+
     out_path = run_dir.alignment_path(split, epoch)
     out_path.parent.mkdir(parents=True, exist_ok=True)
     out_path.write_text(
         json.dumps(
             {
                 "score": results.score,
+                "score_ci": results.score_ci(),
                 "metric_tails": results.metric_tails,
+                "metric_tail_cis": results.metric_tail_cis(),
                 "group_tails": {
                     g.value: v for g, v in results.group_tails.items()
                 },
+                "group_tail_cis": (
+                    None
+                    if group_cis is None
+                    else {g.value: ci for g, ci in group_cis.items()}
+                ),
             },
             indent=2,
         )