diff --git a/README.md b/README.md index 4702d06..be2ebb4 100644 --- a/README.md +++ b/README.md @@ -181,8 +181,8 @@ aligned 12/12 lines Note `--no-vad`: this is a slow hymn, and the ASR's voice-activity filter mistakes sustained singing for silence. With the filter on, the same file yields **one** garbage segment for 130 seconds; with it off, twelve clean ones. Keep the -filter for rap, drop it for anything sung slowly. (`--pairing 1` because this ASR -already split one lyric line per segment.) +filter for rap, drop it for anything sung slowly. (The `--pairing 1` is what +`auto` picks here anyway — this ASR already split one lyric line per segment.) Drop `--no-vad` and you can watch the honest-gap contract hold: every line is reported unmatched, nothing is written, and the cause is named. @@ -214,7 +214,7 @@ slow step, so the stem is cached and reused. from lyric_align import align, Segment segments = [Segment.from_dict(d) for d in whisper_output] -aligned = align(segments, lyric_lines, pairing=2, karaoke=True) +aligned = align(segments, lyric_lines, karaoke=True) # pairing="auto" for a in aligned: print(a.start, a.matched, a.line) ``` @@ -284,6 +284,49 @@ granularity) reproduces this: **33/33 matched, median |err| 0.30 s**, 26/33 within 0.5 s. Its mean of 0.94 s comes almost entirely from four outliers, all on the *same* line — see below. +### A bigger ASR model is a trap unless the pairing follows + +Lines are matched in stanza units, and `pairing` says how many lyric lines make +one unit. That is not a property of the song — it is a property of **the ASR's +segmentation**, and models differ. On the same four-minute track: + +| model | segments | mean segment | pairing | mean \|err\| | ≤0.5 s | worst | lines placed | +|---|---|---|---|---|---|---|---| +| `medium` | 41 | 4.73 s | 2 | 0.50 s | 14/20 | 3.58 s | **74/76** | +| `large-v3` | 64 | 2.78 s | **1** | **0.31 s** | **15/20** | **0.76 s** | 49/76 | +| `large-v3` | 64 | 2.78 s | 2 | 1.13 s | 10/20 | 3.70 s | 68/76 | + +`large-v3` transcribes visibly better, and with the pairing it deserves it +**removes the 3.6 s outlier entirely** — worst case 3.58 s → 0.76 s, mean down +39 %. Left at a pairing tuned for `medium`, the same upgrade is *worse than not +upgrading*, because a two-line unit now straddles a segment boundary. + +**Read the last column before switching.** At pairing 1 there are only 64 +segments for 76 lines, so a quarter of them cannot place at all — that row is +more accurate *and* much less complete. Both `large-v3` rows match the same 18 +of the 20 measured lines; the extra lines pairing 2 places are ones the ground +truth cannot check, and that configuration also produces six measurable +outliers against pairing 1's none. Which trade you want depends on whether you +are hand-correcting afterwards. `medium` remains the default because 74/76 with +one bad line is the better starting point for most people. + +So the default is `--pairing auto`, which reads lines-per-segment off the ASR +output. On every case measured it matches or beats the old fixed 2: + +| | lines / segments | auto picks | vs. fixed 2 | +|---|---|---|---| +| 過ぎたるもの, `medium` | 76 / 41 | 2 | identical | +| 過ぎたるもの, `large-v3` | 76 / 64 | **1** | 0.50 s → **0.31 s** | +| 黒砂, `medium` | 80 / 46 | 2 | identical | +| 黒砂, `large-v3` | 80 / 46 | 2 | identical | +| 過ぎたるもの, full mix (ASR collapsed) | 76 / 11 | 3 (capped) | no worse | + +It does **not** rescue the repeated-hook track: `large-v3` is behind `medium` +there at every pairing, because four identical hook lines carry no information +about which repetition they are, whatever transcribes them. Better ASR fixes +outliers caused by *garbled text*; it cannot fix outliers caused by *identical* +text. + ### Against Vilm, the one other maintained tool here [Vilm Lyrics Aligner](https://github.com/banjuman/vilm-lyrics-aligner) solves the diff --git a/src/lyric_align/__init__.py b/src/lyric_align/__init__.py index f6aa1ce..a64cae5 100644 --- a/src/lyric_align/__init__.py +++ b/src/lyric_align/__init__.py @@ -5,14 +5,14 @@ matching — built for space-less languages (Japanese, Chinese) and sung vocals (including rap), where whitespace tokenization and forced aligners fall short. """ -from .anchor import align, interpolate_gaps +from .anchor import align, auto_pairing, interpolate_gaps from .model import AlignedLine, Segment, Word from .normalize import normalize, similarity __version__ = "0.3.0" __all__ = [ - "align", "interpolate_gaps", + "align", "auto_pairing", "interpolate_gaps", "AlignedLine", "Segment", "Word", "normalize", "similarity", "__version__", diff --git a/src/lyric_align/anchor.py b/src/lyric_align/anchor.py index 8de7fae..9c384b5 100644 --- a/src/lyric_align/anchor.py +++ b/src/lyric_align/anchor.py @@ -52,6 +52,33 @@ from .normalize import default_threshold, similarity +AUTO_PAIRING_MAX = 3 + + +def auto_pairing(lyrics: list[str], segments: list[Segment]) -> int: + """How many lyric lines the ASR appears to have merged into one segment. + + `pairing` is not a property of the song, it is a property of the *ASR's* + segmentation, and different models segment differently. faster-whisper + `medium` merges about two sung lines per segment on Japanese rap, which is + where the old fixed default of 2 came from — but `large-v3` splits much + finer (4.7 s average segment down to 2.8 s, 41 segments up to 64 on the same + four-minute track), so a two-line unit straddles a segment boundary and the + match degrades. Measured on that track: `large-v3` at the old default is + *worse* than `medium` (mean 0.50 s -> 1.13 s), and better than it once the + pairing follows (0.31 s, worst case 3.58 s -> 0.76 s). Upgrading the model + alone is a trap. + + Lines per segment is exactly what pairing means, so it is also the estimate. + Capped at 3: beyond that the ASR has stopped producing line-like segments + (a full mix, where whole verses collapse into one), and the answer there is + to separate the vocal, not to widen the unit — which the CLI already says. + """ + if not segments: + return 1 + return max(1, min(AUTO_PAIRING_MAX, round(len(lyrics) / len(segments)))) + + def _stanzas(lines: list[str], pairing: int) -> list[list[str]]: """Group lyric lines into stanza units of size `pairing` (1 = per line).""" if pairing < 1: @@ -80,7 +107,7 @@ def align( segments: list[Segment], lyrics: list[str], *, - pairing: int = 2, + pairing: int | str = "auto", threshold: float | None = None, window: int = 4, karaoke: bool = False, @@ -88,9 +115,10 @@ def align( """Align known `lyrics` lines onto ASR `segments`. Args: - pairing: lyric lines per stanza unit matched to one segment. Whisper - merges ~2 sung lines per segment, so 2 is a good default for - Japanese rap; use 1 if your ASR already splits per line. + pairing: lyric lines per stanza unit matched to one segment, or "auto" + (the default) to read it off the ASR's own segmentation — see + `auto_pairing`, which explains why a fixed value is tied to one + model. Pass an int to override. threshold: minimum character similarity to accept a match. Defaults to a script-aware value (see `normalize.default_threshold`). window: how many segments ahead to search from the current position. @@ -101,6 +129,10 @@ def align( """ if threshold is None: threshold = default_threshold(lyrics) + if isinstance(pairing, str): + if pairing != "auto": + raise ValueError(f"pairing must be an int or 'auto', got {pairing!r}") + pairing = auto_pairing(lyrics, segments) units = _stanzas(lyrics, pairing) results: list[AlignedLine] = [] diff --git a/src/lyric_align/cli.py b/src/lyric_align/cli.py index 022cb76..dbfe42e 100644 --- a/src/lyric_align/cli.py +++ b/src/lyric_align/cli.py @@ -17,7 +17,7 @@ from pathlib import Path from . import __version__ -from .anchor import align, interpolate_gaps +from .anchor import align, auto_pairing, interpolate_gaps from .formats import EXTENSIONS, FORMATTERS, NEEDS_SYLLABLES, from_aud from .model import Segment from .normalize import CJK_THRESHOLD, LATIN_THRESHOLD, default_threshold @@ -116,9 +116,12 @@ def build_parser() -> argparse.ArgumentParser: "get few or no segments, try this first") al = p.add_argument_group("alignment") - al.add_argument("--pairing", type=int, default=2, - help="lyric lines per stanza unit matched to one segment " - "(default: %(default)s; use 1 if the ASR splits per line)") + al.add_argument("--pairing", default="auto", metavar="N|auto", + help="lyric lines per stanza unit matched to one segment. " + "Default 'auto' reads it off the ASR's own " + "segmentation, because the right value depends on the " + "model: medium merges ~2 sung lines per segment, " + "large-v3 splits far finer. Pass an int to override") al.add_argument("--interpolate", action="store_true", help="give unmatched lines guessed timestamps between their " "neighbours (they stay flagged as unmatched)") @@ -239,7 +242,19 @@ def log(msg: str) -> None: log(f"match threshold: {threshold} " f"({'CJK' if threshold == CJK_THRESHOLD else 'alphabetic'} script)") - aligned = align(segments, lyrics, pairing=args.pairing, + # Resolve here rather than inside align(), so the log says what was chosen. + if str(args.pairing).lower() == "auto": + pairing = auto_pairing(lyrics, segments) + log(f"pairing: {pairing} (auto — {len(lyrics)} lines / " + f"{len(segments)} segments)") + else: + try: + pairing = int(args.pairing) + except ValueError: + raise SystemExit(f"--pairing expects an integer or 'auto', " + f"got {args.pairing!r}") from None + + aligned = align(segments, lyrics, pairing=pairing, threshold=threshold, window=args.window, karaoke=karaoke) if args.interpolate: diff --git a/tests/test_anchor.py b/tests/test_anchor.py index 0a98e54..bf065d7 100644 --- a/tests/test_anchor.py +++ b/tests/test_anchor.py @@ -71,3 +71,36 @@ def test_char_timings_are_strictly_positive_and_ordered(): ct = char_timings("島の左近と佐和山の城なり", words) assert all(c["end"] > c["start"] for c in ct) assert all(b["start"] >= a["end"] for a, b in zip(ct, ct[1:])) + + +def test_auto_pairing_reads_the_asr_segmentation(): + # pairing is a property of the ASR's segmentation, not of the song: it is + # how many lyric lines the model merged into one segment. + from lyric_align import auto_pairing + segs = [Segment(float(i), i + 1.0, "x") for i in range(10)] + assert auto_pairing(["l"] * 20, segs) == 2 # medium-like: ~2 lines/segment + assert auto_pairing(["l"] * 11, segs) == 1 # large-v3-like: ~1 line/segment + + +def test_auto_pairing_is_capped_and_safe_when_the_asr_collapses(): + # A full mix makes Whisper merge whole verses; the answer there is to + # separate the vocal, not to widen the unit, so the estimate stops at 3. + from lyric_align.anchor import AUTO_PAIRING_MAX, auto_pairing + segs = [Segment(0.0, 200.0, "x")] * 3 + assert auto_pairing(["l"] * 76, segs) == AUTO_PAIRING_MAX + assert auto_pairing(["l"] * 10, []) == 1 # no segments: never divide by zero + + +def test_align_defaults_to_auto_pairing(): + from lyric_align import auto_pairing + segs = load() + lyrics = ["あかねさす紫野ゆき", "標野ゆき野守は見ずや", "君が袖振る"] + chosen = auto_pairing(lyrics, segs) + assert [a.to_dict() for a in align(segs, lyrics)] == \ + [a.to_dict() for a in align(segs, lyrics, pairing=chosen)] + + +def test_align_rejects_an_unknown_pairing_keyword(): + import pytest + with pytest.raises(ValueError, match="auto"): + align(load(), ["あかねさす紫野ゆき"], pairing="whatever") diff --git a/tests/test_cli.py b/tests/test_cli.py index 15e4365..3158646 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -68,3 +68,33 @@ def test_cli_rejects_empty_lyrics(tmp_path, capsys): lyrics = write_lyrics(tmp_path, "[Verse 1]\n\n") assert main([str(lyrics), "--segments", str(FIX)]) == 2 assert "no lyric lines" in capsys.readouterr().err + + +def test_cli_reports_the_pairing_it_picked(tmp_path, capsys): + from lyric_align.cli import main + lyrics = tmp_path / "l.txt" + lyrics.write_text("あかねさす紫野ゆき\n標野ゆき野守は見ずや\n") + assert main([str(lyrics), "--segments", str(FIX), "-o", str(tmp_path / "o.json")]) == 0 + err = capsys.readouterr().err + assert "pairing:" in err and "auto" in err + + +def test_cli_accepts_an_explicit_pairing(tmp_path, capsys): + from lyric_align.cli import main + lyrics = tmp_path / "l.txt" + lyrics.write_text("あかねさす紫野ゆき\n") + assert main([str(lyrics), "--segments", str(FIX), "--pairing", "1", + "-o", str(tmp_path / "o.json")]) == 0 + assert "auto" not in capsys.readouterr().err + + +def test_cli_rejects_a_nonsense_pairing(tmp_path, capsys): + import pytest + + from lyric_align.cli import main + lyrics = tmp_path / "l.txt" + lyrics.write_text("あかねさす紫野ゆき\n") + with pytest.raises(SystemExit): + main([str(lyrics), "--segments", str(FIX), "--pairing", "two", + "-o", str(tmp_path / "o.json")]) + capsys.readouterr()