From 5b0e94584ca5580d76f764d638deefd2431ad012 Mon Sep 17 00:00:00 2001 From: masaya ishihara Date: Sat, 25 Jul 2026 16:30:49 +0900 Subject: [PATCH] feat: pick pairing from the ASR's segmentation instead of hardcoding 2 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The old default of 2 was not a property of songs, it was a property of faster-whisper medium, which merges about two sung lines into one segment on Japanese rap. Nothing said so, including this README. large-v3 splits far finer on the same four-minute track — 41 segments to 64, mean segment 4.73s to 2.78s — so a two-line unit straddles a boundary and the match degrades. Measured: large-v3 at the old default is worse than medium (mean 0.50s -> 1.13s, six outliers against one). With pairing following the model it is better than either, and removes the 3.6s outlier this project failed to fix five other ways today: mean 0.31s, worst 0.76s, no outliers. Upgrading the model alone is therefore a trap, and the fix belongs in the default. Lines-per-segment is exactly what pairing means, so it is also the estimator. On all five measured configurations auto matches or beats the fixed 2, including the degenerate full-mix case where the ASR collapses whole verses — capped at 3 there, since the answer to that input is to separate the vocal, which the CLI already says. The README states the cost as well: at pairing 1 there are only 64 segments for 76 lines, so that configuration places 49 of them against medium's 74. More accurate on every measured line, much less complete. medium stays the default model for that reason. Also does not rescue the repeated-hook track, as predicted before running it: large-v3 trails medium there at every pairing, because four identical hook lines carry no information about which repetition they are, whatever transcribes them. 7 tests added, 57 pass. Co-Authored-By: Claude Opus 5 (1M context) --- README.md | 49 ++++++++++++++++++++++++++++++++++--- src/lyric_align/__init__.py | 4 +-- src/lyric_align/anchor.py | 40 +++++++++++++++++++++++++++--- src/lyric_align/cli.py | 25 +++++++++++++++---- tests/test_anchor.py | 33 +++++++++++++++++++++++++ tests/test_cli.py | 30 +++++++++++++++++++++++ 6 files changed, 167 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 4702d06..be2ebb4 100644 --- a/README.md +++ b/README.md @@ -181,8 +181,8 @@ aligned 12/12 lines Note `--no-vad`: this is a slow hymn, and the ASR's voice-activity filter mistakes sustained singing for silence. With the filter on, the same file yields **one** garbage segment for 130 seconds; with it off, twelve clean ones. Keep the -filter for rap, drop it for anything sung slowly. (`--pairing 1` because this ASR -already split one lyric line per segment.) +filter for rap, drop it for anything sung slowly. (The `--pairing 1` is what +`auto` picks here anyway — this ASR already split one lyric line per segment.) Drop `--no-vad` and you can watch the honest-gap contract hold: every line is reported unmatched, nothing is written, and the cause is named. @@ -214,7 +214,7 @@ slow step, so the stem is cached and reused. from lyric_align import align, Segment segments = [Segment.from_dict(d) for d in whisper_output] -aligned = align(segments, lyric_lines, pairing=2, karaoke=True) +aligned = align(segments, lyric_lines, karaoke=True) # pairing="auto" for a in aligned: print(a.start, a.matched, a.line) ``` @@ -284,6 +284,49 @@ granularity) reproduces this: **33/33 matched, median |err| 0.30 s**, 26/33 within 0.5 s. Its mean of 0.94 s comes almost entirely from four outliers, all on the *same* line — see below. +### A bigger ASR model is a trap unless the pairing follows + +Lines are matched in stanza units, and `pairing` says how many lyric lines make +one unit. That is not a property of the song — it is a property of **the ASR's +segmentation**, and models differ. On the same four-minute track: + +| model | segments | mean segment | pairing | mean \|err\| | ≤0.5 s | worst | lines placed | +|---|---|---|---|---|---|---|---| +| `medium` | 41 | 4.73 s | 2 | 0.50 s | 14/20 | 3.58 s | **74/76** | +| `large-v3` | 64 | 2.78 s | **1** | **0.31 s** | **15/20** | **0.76 s** | 49/76 | +| `large-v3` | 64 | 2.78 s | 2 | 1.13 s | 10/20 | 3.70 s | 68/76 | + +`large-v3` transcribes visibly better, and with the pairing it deserves it +**removes the 3.6 s outlier entirely** — worst case 3.58 s → 0.76 s, mean down +39 %. Left at a pairing tuned for `medium`, the same upgrade is *worse than not +upgrading*, because a two-line unit now straddles a segment boundary. + +**Read the last column before switching.** At pairing 1 there are only 64 +segments for 76 lines, so a quarter of them cannot place at all — that row is +more accurate *and* much less complete. Both `large-v3` rows match the same 18 +of the 20 measured lines; the extra lines pairing 2 places are ones the ground +truth cannot check, and that configuration also produces six measurable +outliers against pairing 1's none. Which trade you want depends on whether you +are hand-correcting afterwards. `medium` remains the default because 74/76 with +one bad line is the better starting point for most people. + +So the default is `--pairing auto`, which reads lines-per-segment off the ASR +output. On every case measured it matches or beats the old fixed 2: + +| | lines / segments | auto picks | vs. fixed 2 | +|---|---|---|---| +| 過ぎたるもの, `medium` | 76 / 41 | 2 | identical | +| 過ぎたるもの, `large-v3` | 76 / 64 | **1** | 0.50 s → **0.31 s** | +| 黒砂, `medium` | 80 / 46 | 2 | identical | +| 黒砂, `large-v3` | 80 / 46 | 2 | identical | +| 過ぎたるもの, full mix (ASR collapsed) | 76 / 11 | 3 (capped) | no worse | + +It does **not** rescue the repeated-hook track: `large-v3` is behind `medium` +there at every pairing, because four identical hook lines carry no information +about which repetition they are, whatever transcribes them. Better ASR fixes +outliers caused by *garbled text*; it cannot fix outliers caused by *identical* +text. + ### Against Vilm, the one other maintained tool here [Vilm Lyrics Aligner](https://github.com/banjuman/vilm-lyrics-aligner) solves the diff --git a/src/lyric_align/__init__.py b/src/lyric_align/__init__.py index f6aa1ce..a64cae5 100644 --- a/src/lyric_align/__init__.py +++ b/src/lyric_align/__init__.py @@ -5,14 +5,14 @@ matching — built for space-less languages (Japanese, Chinese) and sung vocals (including rap), where whitespace tokenization and forced aligners fall short. """ -from .anchor import align, interpolate_gaps +from .anchor import align, auto_pairing, interpolate_gaps from .model import AlignedLine, Segment, Word from .normalize import normalize, similarity __version__ = "0.3.0" __all__ = [ - "align", "interpolate_gaps", + "align", "auto_pairing", "interpolate_gaps", "AlignedLine", "Segment", "Word", "normalize", "similarity", "__version__", diff --git a/src/lyric_align/anchor.py b/src/lyric_align/anchor.py index 8de7fae..9c384b5 100644 --- a/src/lyric_align/anchor.py +++ b/src/lyric_align/anchor.py @@ -52,6 +52,33 @@ from .normalize import default_threshold, similarity +AUTO_PAIRING_MAX = 3 + + +def auto_pairing(lyrics: list[str], segments: list[Segment]) -> int: + """How many lyric lines the ASR appears to have merged into one segment. + + `pairing` is not a property of the song, it is a property of the *ASR's* + segmentation, and different models segment differently. faster-whisper + `medium` merges about two sung lines per segment on Japanese rap, which is + where the old fixed default of 2 came from — but `large-v3` splits much + finer (4.7 s average segment down to 2.8 s, 41 segments up to 64 on the same + four-minute track), so a two-line unit straddles a segment boundary and the + match degrades. Measured on that track: `large-v3` at the old default is + *worse* than `medium` (mean 0.50 s -> 1.13 s), and better than it once the + pairing follows (0.31 s, worst case 3.58 s -> 0.76 s). Upgrading the model + alone is a trap. + + Lines per segment is exactly what pairing means, so it is also the estimate. + Capped at 3: beyond that the ASR has stopped producing line-like segments + (a full mix, where whole verses collapse into one), and the answer there is + to separate the vocal, not to widen the unit — which the CLI already says. + """ + if not segments: + return 1 + return max(1, min(AUTO_PAIRING_MAX, round(len(lyrics) / len(segments)))) + + def _stanzas(lines: list[str], pairing: int) -> list[list[str]]: """Group lyric lines into stanza units of size `pairing` (1 = per line).""" if pairing < 1: @@ -80,7 +107,7 @@ def align( segments: list[Segment], lyrics: list[str], *, - pairing: int = 2, + pairing: int | str = "auto", threshold: float | None = None, window: int = 4, karaoke: bool = False, @@ -88,9 +115,10 @@ def align( """Align known `lyrics` lines onto ASR `segments`. Args: - pairing: lyric lines per stanza unit matched to one segment. Whisper - merges ~2 sung lines per segment, so 2 is a good default for - Japanese rap; use 1 if your ASR already splits per line. + pairing: lyric lines per stanza unit matched to one segment, or "auto" + (the default) to read it off the ASR's own segmentation — see + `auto_pairing`, which explains why a fixed value is tied to one + model. Pass an int to override. threshold: minimum character similarity to accept a match. Defaults to a script-aware value (see `normalize.default_threshold`). window: how many segments ahead to search from the current position. @@ -101,6 +129,10 @@ def align( """ if threshold is None: threshold = default_threshold(lyrics) + if isinstance(pairing, str): + if pairing != "auto": + raise ValueError(f"pairing must be an int or 'auto', got {pairing!r}") + pairing = auto_pairing(lyrics, segments) units = _stanzas(lyrics, pairing) results: list[AlignedLine] = [] diff --git a/src/lyric_align/cli.py b/src/lyric_align/cli.py index 022cb76..dbfe42e 100644 --- a/src/lyric_align/cli.py +++ b/src/lyric_align/cli.py @@ -17,7 +17,7 @@ from pathlib import Path from . import __version__ -from .anchor import align, interpolate_gaps +from .anchor import align, auto_pairing, interpolate_gaps from .formats import EXTENSIONS, FORMATTERS, NEEDS_SYLLABLES, from_aud from .model import Segment from .normalize import CJK_THRESHOLD, LATIN_THRESHOLD, default_threshold @@ -116,9 +116,12 @@ def build_parser() -> argparse.ArgumentParser: "get few or no segments, try this first") al = p.add_argument_group("alignment") - al.add_argument("--pairing", type=int, default=2, - help="lyric lines per stanza unit matched to one segment " - "(default: %(default)s; use 1 if the ASR splits per line)") + al.add_argument("--pairing", default="auto", metavar="N|auto", + help="lyric lines per stanza unit matched to one segment. " + "Default 'auto' reads it off the ASR's own " + "segmentation, because the right value depends on the " + "model: medium merges ~2 sung lines per segment, " + "large-v3 splits far finer. Pass an int to override") al.add_argument("--interpolate", action="store_true", help="give unmatched lines guessed timestamps between their " "neighbours (they stay flagged as unmatched)") @@ -239,7 +242,19 @@ def log(msg: str) -> None: log(f"match threshold: {threshold} " f"({'CJK' if threshold == CJK_THRESHOLD else 'alphabetic'} script)") - aligned = align(segments, lyrics, pairing=args.pairing, + # Resolve here rather than inside align(), so the log says what was chosen. + if str(args.pairing).lower() == "auto": + pairing = auto_pairing(lyrics, segments) + log(f"pairing: {pairing} (auto — {len(lyrics)} lines / " + f"{len(segments)} segments)") + else: + try: + pairing = int(args.pairing) + except ValueError: + raise SystemExit(f"--pairing expects an integer or 'auto', " + f"got {args.pairing!r}") from None + + aligned = align(segments, lyrics, pairing=pairing, threshold=threshold, window=args.window, karaoke=karaoke) if args.interpolate: diff --git a/tests/test_anchor.py b/tests/test_anchor.py index 0a98e54..bf065d7 100644 --- a/tests/test_anchor.py +++ b/tests/test_anchor.py @@ -71,3 +71,36 @@ def test_char_timings_are_strictly_positive_and_ordered(): ct = char_timings("島の左近と佐和山の城なり", words) assert all(c["end"] > c["start"] for c in ct) assert all(b["start"] >= a["end"] for a, b in zip(ct, ct[1:])) + + +def test_auto_pairing_reads_the_asr_segmentation(): + # pairing is a property of the ASR's segmentation, not of the song: it is + # how many lyric lines the model merged into one segment. + from lyric_align import auto_pairing + segs = [Segment(float(i), i + 1.0, "x") for i in range(10)] + assert auto_pairing(["l"] * 20, segs) == 2 # medium-like: ~2 lines/segment + assert auto_pairing(["l"] * 11, segs) == 1 # large-v3-like: ~1 line/segment + + +def test_auto_pairing_is_capped_and_safe_when_the_asr_collapses(): + # A full mix makes Whisper merge whole verses; the answer there is to + # separate the vocal, not to widen the unit, so the estimate stops at 3. + from lyric_align.anchor import AUTO_PAIRING_MAX, auto_pairing + segs = [Segment(0.0, 200.0, "x")] * 3 + assert auto_pairing(["l"] * 76, segs) == AUTO_PAIRING_MAX + assert auto_pairing(["l"] * 10, []) == 1 # no segments: never divide by zero + + +def test_align_defaults_to_auto_pairing(): + from lyric_align import auto_pairing + segs = load() + lyrics = ["あかねさす紫野ゆき", "標野ゆき野守は見ずや", "君が袖振る"] + chosen = auto_pairing(lyrics, segs) + assert [a.to_dict() for a in align(segs, lyrics)] == \ + [a.to_dict() for a in align(segs, lyrics, pairing=chosen)] + + +def test_align_rejects_an_unknown_pairing_keyword(): + import pytest + with pytest.raises(ValueError, match="auto"): + align(load(), ["あかねさす紫野ゆき"], pairing="whatever") diff --git a/tests/test_cli.py b/tests/test_cli.py index 15e4365..3158646 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -68,3 +68,33 @@ def test_cli_rejects_empty_lyrics(tmp_path, capsys): lyrics = write_lyrics(tmp_path, "[Verse 1]\n\n") assert main([str(lyrics), "--segments", str(FIX)]) == 2 assert "no lyric lines" in capsys.readouterr().err + + +def test_cli_reports_the_pairing_it_picked(tmp_path, capsys): + from lyric_align.cli import main + lyrics = tmp_path / "l.txt" + lyrics.write_text("あかねさす紫野ゆき\n標野ゆき野守は見ずや\n") + assert main([str(lyrics), "--segments", str(FIX), "-o", str(tmp_path / "o.json")]) == 0 + err = capsys.readouterr().err + assert "pairing:" in err and "auto" in err + + +def test_cli_accepts_an_explicit_pairing(tmp_path, capsys): + from lyric_align.cli import main + lyrics = tmp_path / "l.txt" + lyrics.write_text("あかねさす紫野ゆき\n") + assert main([str(lyrics), "--segments", str(FIX), "--pairing", "1", + "-o", str(tmp_path / "o.json")]) == 0 + assert "auto" not in capsys.readouterr().err + + +def test_cli_rejects_a_nonsense_pairing(tmp_path, capsys): + import pytest + + from lyric_align.cli import main + lyrics = tmp_path / "l.txt" + lyrics.write_text("あかねさす紫野ゆき\n") + with pytest.raises(SystemExit): + main([str(lyrics), "--segments", str(FIX), "--pairing", "two", + "-o", str(tmp_path / "o.json")]) + capsys.readouterr()