From b283bf732be0ffb0d246d92e91df8411d8b54948 Mon Sep 17 00:00:00 2001 From: Thanapat Trachu Date: Thu, 23 Jul 2026 15:52:42 -0400 Subject: [PATCH 1/8] feat(speaker): add backend resolver for espnet vs huggingface speaker models --- test/test_metrics/test_speaker.py | 40 ++++++++++++++++++++++++++++++ versa/utterance_metrics/speaker.py | 36 ++++++++++++++++++++++++++- 2 files changed, 75 insertions(+), 1 deletion(-) create mode 100644 test/test_metrics/test_speaker.py diff --git a/test/test_metrics/test_speaker.py b/test/test_metrics/test_speaker.py new file mode 100644 index 0000000..aee1f99 --- /dev/null +++ b/test/test_metrics/test_speaker.py @@ -0,0 +1,40 @@ +import numpy as np +import pytest + +from versa.utterance_metrics.speaker import resolve_speaker_backend + + +def test_resolve_backend_default_tag_is_espnet(): + assert resolve_speaker_backend(model_tag="default") == "espnet" + + +def test_resolve_backend_espnet_prefix_is_espnet(): + assert resolve_speaker_backend(model_tag="espnet/voxcelebs12_rawnet3") == "espnet" + + +def test_resolve_backend_hf_tag_is_huggingface(): + assert resolve_speaker_backend(model_tag="microsoft/wavlm-base-sv") == "huggingface" + + +def test_resolve_backend_local_espnet_files_win(): + assert ( + resolve_speaker_backend( + model_tag="microsoft/wavlm-base-sv", + model_path="/path/model.pth", + model_config="/path/config.yaml", + ) + == "espnet" + ) + + +def test_resolve_backend_explicit_override(): + assert ( + resolve_speaker_backend(model_tag="default", backend="huggingface") + == "huggingface" + ) + assert resolve_speaker_backend(model_tag="some/tag", backend="espnet") == "espnet" + + +def test_resolve_backend_invalid_backend_raises(): + with pytest.raises(ValueError, match="backend"): + resolve_speaker_backend(model_tag="default", backend="wavlm2000") diff --git a/versa/utterance_metrics/speaker.py b/versa/utterance_metrics/speaker.py index e5abbbf..337bf2f 100644 --- a/versa/utterance_metrics/speaker.py +++ b/versa/utterance_metrics/speaker.py @@ -18,6 +18,40 @@ logger = logging.getLogger(__name__) +ESPNET_DEFAULT_SPEAKER_TAG = "espnet/voxcelebs12_rawnet3" +SPEAKER_BACKENDS = ("espnet", "huggingface") + + +def resolve_speaker_backend( + model_tag="default", backend=None, model_path=None, model_config=None +): + """Resolve which speaker-model backend a configuration refers to. + + Args: + model_tag: Model tag from the config ("default", an ESPnet hub tag + such as "espnet/voxcelebs12_rawnet3", or any other HuggingFace + repo id such as "microsoft/wavlm-base-sv"). + backend: Optional explicit backend override ("espnet" or "huggingface"). + model_path: Optional local ESPnet model checkpoint path. + model_config: Optional local ESPnet train config path. + + Returns: + "espnet" or "huggingface". + """ + if backend is not None: + if backend not in SPEAKER_BACKENDS: + raise ValueError( + "Unknown speaker backend '{}'. Supported backends: {}".format( + backend, SPEAKER_BACKENDS + ) + ) + return backend + if model_path is not None and model_config is not None: + return "espnet" + if model_tag == "default" or model_tag.startswith("espnet/"): + return "espnet" + return "huggingface" + def speaker_model_setup( model_tag="default", @@ -39,7 +73,7 @@ def speaker_model_setup( ) else: if model_tag == "default": - model_tag = "espnet/voxcelebs12_rawnet3" + model_tag = ESPNET_DEFAULT_SPEAKER_TAG if cache_dir is None: model = Speech2Embedding.from_pretrained(model_tag=model_tag, device=device) else: From c9e0486a770b88bfcff6553d01eed2b3650efc81 Mon Sep 17 00:00:00 2001 From: Thanapat Trachu Date: Thu, 23 Jul 2026 15:59:08 -0400 Subject: [PATCH 2/8] feat(speaker): add HuggingFace x-vector backend (WavLM et al.) --- test/test_metrics/test_speaker.py | 48 ++++++++++++++++++++ versa/utterance_metrics/speaker.py | 72 ++++++++++++++++++++++++++++++ 2 files changed, 120 insertions(+) diff --git a/test/test_metrics/test_speaker.py b/test/test_metrics/test_speaker.py index aee1f99..af6c2cb 100644 --- a/test/test_metrics/test_speaker.py +++ b/test/test_metrics/test_speaker.py @@ -38,3 +38,51 @@ def test_resolve_backend_explicit_override(): def test_resolve_backend_invalid_backend_raises(): with pytest.raises(ValueError, match="backend"): resolve_speaker_backend(model_tag="default", backend="wavlm2000") + + +from versa.utterance_metrics.speaker import is_transformers_available + + +def _fixed_audio(freq, duration=1.0, sample_rate=16000): + t = np.linspace(0, duration, int(sample_rate * duration), endpoint=False) + envelope = 0.5 + 0.5 * np.sin(2 * np.pi * 0.5 * t) + return (envelope * np.sin(2 * np.pi * freq * t)).astype(np.float32) + + +@pytest.mark.skipif( + not is_transformers_available(), reason="Transformers not available" +) +def test_hf_speaker_model_embedding_shape(): + from versa.utterance_metrics.speaker import hf_speaker_model_setup + + model = hf_speaker_model_setup(model_tag="microsoft/wavlm-base-sv", use_gpu=False) + embedding = model(_fixed_audio(150)) + assert embedding.dim() == 2 + assert embedding.shape[0] == 1 + assert embedding.shape[1] > 0 + + +@pytest.mark.skipif( + not is_transformers_available(), reason="Transformers not available" +) +def test_hf_speaker_metric_identical_signals(): + from versa.utterance_metrics.speaker import hf_speaker_model_setup, speaker_metric + + model = hf_speaker_model_setup(model_tag="microsoft/wavlm-base-sv", use_gpu=False) + audio = _fixed_audio(150) + result = speaker_metric(model, audio, audio, 16000) + assert "spk_similarity" in result + assert result["spk_similarity"] == pytest.approx(1.0, abs=1e-4) + + +@pytest.mark.skipif( + not is_transformers_available(), reason="Transformers not available" +) +def test_hf_speaker_metric_different_signals(): + from versa.utterance_metrics.speaker import hf_speaker_model_setup, speaker_metric + + model = hf_speaker_model_setup(model_tag="microsoft/wavlm-base-sv", use_gpu=False) + same = speaker_metric(model, _fixed_audio(150), _fixed_audio(150), 16000) + diff = speaker_metric(model, _fixed_audio(150), _fixed_audio(420), 16000) + assert diff["spk_similarity"] < same["spk_similarity"] + assert -1.0 <= diff["spk_similarity"] <= 1.0 diff --git a/versa/utterance_metrics/speaker.py b/versa/utterance_metrics/speaker.py index 337bf2f..7db0975 100644 --- a/versa/utterance_metrics/speaker.py +++ b/versa/utterance_metrics/speaker.py @@ -6,18 +6,35 @@ import logging import numpy as np +import torch from versa.audio_utils import resample_audio +from versa.huggingface_cache import configure_huggingface_cache, get_hf_cache_dir try: from espnet2.bin.spk_inference import Speech2Embedding except ImportError: Speech2Embedding = None +try: + from transformers import AutoFeatureExtractor, AutoModelForAudioXVector + + TRANSFORMERS_AVAILABLE = True +except ImportError: + AutoFeatureExtractor = None + AutoModelForAudioXVector = None + TRANSFORMERS_AVAILABLE = False + from versa.definition import BaseMetric, MetricCategory, MetricMetadata, MetricType logger = logging.getLogger(__name__) + +def is_transformers_available(): + """Check whether transformers is importable for the HuggingFace backend.""" + return TRANSFORMERS_AVAILABLE + + ESPNET_DEFAULT_SPEAKER_TAG = "espnet/voxcelebs12_rawnet3" SPEAKER_BACKENDS = ("espnet", "huggingface") @@ -90,6 +107,61 @@ def speaker_model_setup( return model +class HFSpeakerModel: + """Callable wrapper around a HuggingFace x-vector speaker model. + + Mirrors the call signature of espnet's Speech2Embedding so that + speaker_metric can consume either backend interchangeably. + """ + + def __init__(self, model, feature_extractor, device): + self.model = model + self.feature_extractor = feature_extractor + self.device = device + + def __call__(self, speech): + inputs = self.feature_extractor( + speech, sampling_rate=16000, return_tensors="pt" + ) + inputs = {key: value.to(self.device) for key, value in inputs.items()} + with torch.no_grad(): + outputs = self.model(**inputs) + return outputs.embeddings + + +def hf_speaker_model_setup( + model_tag="microsoft/wavlm-base-sv", use_gpu=False, cache_dir=None +): + """Load a HuggingFace x-vector speaker model (e.g. WavLM-base-sv). + + Works with any AutoModelForAudioXVector checkpoint, including + microsoft/wavlm-base-sv, microsoft/unispeech-sat-base-sv, and other + x-vector fine-tuned speech encoders on the HuggingFace hub. + """ + if not TRANSFORMERS_AVAILABLE: + raise ImportError( + "HuggingFace speaker models require transformers. " + "Please install it with `pip install transformers` " + "(or `pip install versa[ml]`) and retry." + ) + if use_gpu and not torch.cuda.is_available(): + logger.warning("use_gpu requested but CUDA is unavailable; using CPU.") + device = "cuda" if use_gpu and torch.cuda.is_available() else "cpu" + resolved_cache_dir = get_hf_cache_dir(cache_dir) + configure_huggingface_cache(resolved_cache_dir) + feature_extractor = AutoFeatureExtractor.from_pretrained( + model_tag, cache_dir=resolved_cache_dir + ) + model = ( + AutoModelForAudioXVector.from_pretrained( + model_tag, cache_dir=resolved_cache_dir + ) + .to(device) + .eval() + ) + return HFSpeakerModel(model, feature_extractor, device) + + def speaker_metric(model, pred_x, gt_x, fs): # NOTE(jiatong): only work for 16000 Hz if fs < 16000: From 4c52dc757e401a6c90f499b995c73666d9e4c181 Mon Sep 17 00:00:00 2001 From: Thanapat Trachu Date: Thu, 23 Jul 2026 16:07:06 -0400 Subject: [PATCH 3/8] feat(speaker): dispatch SpeakerMetric between espnet and huggingface backends --- test/test_metrics/test_speaker.py | 35 +++++++++++++++++++++++++ versa/utterance_metrics/speaker.py | 41 +++++++++++++++++++++++++----- 2 files changed, 69 insertions(+), 7 deletions(-) diff --git a/test/test_metrics/test_speaker.py b/test/test_metrics/test_speaker.py index af6c2cb..8719216 100644 --- a/test/test_metrics/test_speaker.py +++ b/test/test_metrics/test_speaker.py @@ -86,3 +86,38 @@ def test_hf_speaker_metric_different_signals(): diff = speaker_metric(model, _fixed_audio(150), _fixed_audio(420), 16000) assert diff["spk_similarity"] < same["spk_similarity"] assert -1.0 <= diff["spk_similarity"] <= 1.0 + + +@pytest.mark.skipif( + not is_transformers_available(), reason="Transformers not available" +) +def test_speaker_metric_class_wavlm_backend(): + from versa.utterance_metrics.speaker import SpeakerMetric + + metric = SpeakerMetric({"model_tag": "microsoft/wavlm-base-sv", "use_gpu": False}) + assert metric.backend == "huggingface" + + audio = _fixed_audio(150) + result = metric.compute(audio, audio, metadata={"sample_rate": 16000}) + assert result["spk_similarity"] == pytest.approx(1.0, abs=1e-4) + + +@pytest.mark.skipif( + not is_transformers_available(), reason="Transformers not available" +) +def test_speaker_metric_class_requires_both_signals(): + from versa.utterance_metrics.speaker import SpeakerMetric + + metric = SpeakerMetric({"model_tag": "microsoft/wavlm-base-sv", "use_gpu": False}) + with pytest.raises(ValueError, match="Predicted signal"): + metric.compute(None, _fixed_audio(150), metadata={"sample_rate": 16000}) + with pytest.raises(ValueError, match="Reference signal"): + metric.compute(_fixed_audio(150), None, metadata={"sample_rate": 16000}) + + +def test_speaker_metadata_mentions_both_backends(): + from versa.utterance_metrics.speaker import _speaker_metadata + + metadata = _speaker_metadata() + assert "transformers" in metadata.dependencies + assert "espnet2" in metadata.dependencies diff --git a/versa/utterance_metrics/speaker.py b/versa/utterance_metrics/speaker.py index 7db0975..587d2d8 100644 --- a/versa/utterance_metrics/speaker.py +++ b/versa/utterance_metrics/speaker.py @@ -183,21 +183,44 @@ def speaker_metric(model, pred_x, gt_x, fs): class SpeakerMetric(BaseMetric): - """Speaker embedding cosine similarity.""" + """Speaker embedding cosine similarity. + + Supports ESPnet-SPK models (model_tag "default" or "espnet/...") and + HuggingFace x-vector models (any other model_tag, e.g. + "microsoft/wavlm-base-sv"). The backend is auto-detected from model_tag + and can be forced with the optional "backend" config key + ("espnet" or "huggingface"). + """ def _setup(self): self.model_tag = self.config.get("model_tag", "default") self.model_path = self.config.get("model_path") self.model_config = self.config.get("model_config") self.use_gpu = self.config.get("use_gpu", False) - self.cache_dir = self.config.get("cache_dir", "versa_cache/espnet_model_zoo") - self.model = speaker_model_setup( + self.backend = resolve_speaker_backend( model_tag=self.model_tag, + backend=self.config.get("backend"), model_path=self.model_path, model_config=self.model_config, - use_gpu=self.use_gpu, - cache_dir=self.cache_dir, ) + if self.backend == "huggingface": + self.cache_dir = self.config.get("cache_dir") + self.model = hf_speaker_model_setup( + model_tag=self.model_tag, + use_gpu=self.use_gpu, + cache_dir=self.cache_dir, + ) + else: + self.cache_dir = self.config.get( + "cache_dir", "versa_cache/espnet_model_zoo" + ) + self.model = speaker_model_setup( + model_tag=self.model_tag, + model_path=self.model_path, + model_config=self.model_config, + use_gpu=self.use_gpu, + cache_dir=self.cache_dir, + ) def compute(self, predictions, references=None, metadata=None): if predictions is None: @@ -223,8 +246,12 @@ def _speaker_metadata(): requires_text=False, gpu_compatible=True, auto_install=False, - dependencies=["espnet2", "librosa", "numpy"], - description="Speaker embedding cosine similarity", + dependencies=["espnet2", "transformers", "librosa", "numpy"], + description=( + "Speaker embedding cosine similarity " + "(ESPnet-SPK or HuggingFace x-vector models such as " + "microsoft/wavlm-base-sv)" + ), paper_reference="https://arxiv.org/abs/2401.17230", implementation_source="https://github.com/espnet/espnet", ) From 14bb51ec744ef0025dabf0a19c1ba64977c8aaa4 Mon Sep 17 00:00:00 2001 From: Thanapat Trachu Date: Thu, 23 Jul 2026 16:15:22 -0400 Subject: [PATCH 4/8] feat(speaker): route huggingface speaker models to the hf cache namespace --- test/test_metrics/test_speaker.py | 45 +++++++++++++++++++++++++++++++ versa/scorer_shared.py | 16 ++++++++--- 2 files changed, 58 insertions(+), 3 deletions(-) diff --git a/test/test_metrics/test_speaker.py b/test/test_metrics/test_speaker.py index 8719216..14bbead 100644 --- a/test/test_metrics/test_speaker.py +++ b/test/test_metrics/test_speaker.py @@ -121,3 +121,48 @@ def test_speaker_metadata_mentions_both_backends(): metadata = _speaker_metadata() assert "transformers" in metadata.dependencies assert "espnet2" in metadata.dependencies + + +def test_cache_namespace_speaker_espnet_default(): + from versa.scorer_shared import configure_metric_cache_dirs + + configs = configure_metric_cache_dirs( + [{"name": "speaker", "model_tag": "default"}], cache_folder="/tmp/vc" + ) + assert configs[0]["cache_dir"].endswith("espnet_model_zoo") + + +def test_cache_namespace_speaker_hf_tag(): + from versa.scorer_shared import configure_metric_cache_dirs + + configs = configure_metric_cache_dirs( + [{"name": "speaker", "model_tag": "microsoft/wavlm-base-sv"}], + cache_folder="/tmp/vc", + ) + assert configs[0]["cache_dir"].endswith("huggingface") + + +def test_cache_namespace_speaker_alias_hf_tag(): + from versa.scorer_shared import configure_metric_cache_dirs + + configs = configure_metric_cache_dirs( + [{"name": "spk_similarity", "model_tag": "microsoft/wavlm-base-sv"}], + cache_folder="/tmp/vc", + ) + assert configs[0]["cache_dir"].endswith("huggingface") + + +def test_cache_namespace_explicit_cache_dir_wins(): + from versa.scorer_shared import configure_metric_cache_dirs + + configs = configure_metric_cache_dirs( + [ + { + "name": "speaker", + "model_tag": "microsoft/wavlm-base-sv", + "cache_dir": "/custom/cache", + } + ], + cache_folder="/tmp/vc", + ) + assert configs[0]["cache_dir"] == "/custom/cache" diff --git a/versa/scorer_shared.py b/versa/scorer_shared.py index fc5fe3f..aa13403 100644 --- a/versa/scorer_shared.py +++ b/versa/scorer_shared.py @@ -124,9 +124,20 @@ def load_score_modules( ) -def _metric_cache_namespace(metric_name): +def _metric_cache_namespace(metric_name, metric_config=None): """Return a collision-safe shared namespace for a registered metric.""" name = str(metric_name).lower() + if name in {"speaker", "spk_similarity", "speaker_similarity"}: + from versa.utterance_metrics.speaker import resolve_speaker_backend + + config = metric_config or {} + backend = resolve_speaker_backend( + model_tag=config.get("model_tag", "default"), + backend=config.get("backend"), + model_path=config.get("model_path"), + model_config=config.get("model_config"), + ) + return "huggingface" if backend == "huggingface" else "espnet_model_zoo" if name.startswith(("qwen2_audio_", "qwen_omni_")) or name in { "hubert_wer", "pam", @@ -140,7 +151,6 @@ def _metric_cache_namespace(metric_name): "owsm_lid", "owsm_wer", "se_snr", - "speaker", "universa", }: return "espnet_model_zoo" @@ -173,7 +183,7 @@ def configure_metric_cache_dirs(score_config, cache_folder=None): **config, "cache_dir": config.get( "cache_dir", - str(cache_root / _metric_cache_namespace(config["name"])), + str(cache_root / _metric_cache_namespace(config["name"], config)), ), } for config in score_config From ba636d4701dfa2473feebac522083cbfb8b61057 Mon Sep 17 00:00:00 2001 From: Thanapat Trachu Date: Thu, 23 Jul 2026 16:20:10 -0400 Subject: [PATCH 5/8] docs(speaker): add WavLM speaker similarity example config and docs --- docs/supported_metrics.md | 2 +- egs/separate_metrics/spk_similarity.yaml | 16 ++++++++++------ egs/separate_metrics/spk_similarity_wavlm.yaml | 9 +++++++++ 3 files changed, 20 insertions(+), 7 deletions(-) create mode 100644 egs/separate_metrics/spk_similarity_wavlm.yaml diff --git a/docs/supported_metrics.md b/docs/supported_metrics.md index 4eb70f8..ec00e52 100644 --- a/docs/supported_metrics.md +++ b/docs/supported_metrics.md @@ -112,7 +112,7 @@ We include x mark if the metric is auto-installed in versa. | 7 | | NVIDIA Conformer-Transducer X-Large Speech Recognition-based Error Rate | nemo_wer | nemo_wer |[NeMo](https://github.com/NVIDIA/NeMo) | [paper](https://arxiv.org/abs/2005.08100) | | 8 | x | Facebook Hubert-Large-Finetuned Speech Recognition-based Error Rate | hubert_wer | hubert_wer |[HuBERT](https://github.com/facebookresearch/fairseq/tree/main/examples/hubert) | [paper](https://arxiv.org/abs/2106.07447) | | 9 | | Emotion2vec similarity (emo2vec) | emo2vec_similarity | emotion_similarity | [emo2vec](https://github.com/ftshijt/emotion2vec/tree/main) | [paper](https://arxiv.org/abs/2312.15185) | -| 10 | x | Speaker Embedding Similarity | speaker | spk_similarity | [espnet](https://github.com/espnet/espnet) | [paper](https://arxiv.org/abs/2401.17230) | +| 10 | x | Speaker Embedding Similarity | speaker | spk_similarity | [espnet](https://github.com/espnet/espnet), [transformers x-vector (e.g. WavLM)](https://huggingface.co/microsoft/wavlm-base-sv) | [paper](https://arxiv.org/abs/2401.17230) | | 11 | | NOMAD: Unsupervised Learning of Perceptual Embeddings For Speech Enhancement and Non-Matching Reference Audio Quality Assessment | nomad | nomad |[Nomad](https://github.com/shimhz/nomad/tree/main) | [paper](https://arxiv.org/abs/2309.16284) | | 12 | | Contrastive Language-Audio Pretraining Score (CLAP Score) | clap_score | clap_score | [frechet-audio-distance](https://github.com/gudgud96/frechet-audio-distance) | [paper](https://arxiv.org/abs/2301.12661) | | 13 | | Accompaniment Prompt Adherence (APA) | apa | apa | [Sony-audio-metrics](https://github.com/SonyCSLParis/audio-metrics) | [paper](https://arxiv.org/abs/2404.00775) | diff --git a/egs/separate_metrics/spk_similarity.yaml b/egs/separate_metrics/spk_similarity.yaml index 8e2b0b4..51d0f36 100644 --- a/egs/separate_metrics/spk_similarity.yaml +++ b/egs/separate_metrics/spk_similarity.yaml @@ -1,6 +1,10 @@ -# speaker related metrics -# -- spk_similarity: speaker cosine similarity -# model tag can be any ESPnet-SPK huggingface repo at -# https://huggingface.co/espnet -- name: speaker - model_tag: default \ No newline at end of file +# speaker related metrics +# -- spk_similarity: speaker cosine similarity +# model tag can be: +# - any ESPnet-SPK huggingface repo at +# https://huggingface.co/espnet (default backend) +# - any HuggingFace x-vector speaker model such as +# microsoft/wavlm-base-sv (auto-detected huggingface +# backend, see spk_similarity_wavlm.yaml) +- name: speaker + model_tag: default diff --git a/egs/separate_metrics/spk_similarity_wavlm.yaml b/egs/separate_metrics/spk_similarity_wavlm.yaml new file mode 100644 index 0000000..5d936a4 --- /dev/null +++ b/egs/separate_metrics/spk_similarity_wavlm.yaml @@ -0,0 +1,9 @@ +# speaker related metrics (HuggingFace x-vector backend) +# -- spk_similarity: speaker cosine similarity with microsoft/wavlm-base-sv +# any AutoModelForAudioXVector checkpoint works, e.g. +# microsoft/wavlm-base-sv, microsoft/wavlm-base-plus-sv, +# microsoft/unispeech-sat-base-plus-sv +# the backend is auto-detected from the model tag; +# add `backend: huggingface` to force it explicitly +- name: speaker + model_tag: microsoft/wavlm-base-sv From e94e3baa1e335858ca405b149f94efb9bb6b7363 Mon Sep 17 00:00:00 2001 From: Thanapat Trachu Date: Thu, 23 Jul 2026 16:34:22 -0400 Subject: [PATCH 6/8] fix(speaker): make dependency validation backend-aware and harden resolver --- test/test_metrics/test_speaker.py | 41 ++++++++++++++++++++++++++++++ versa/config_validation.py | 26 ++++++++++++++++++- versa/utterance_metrics/speaker.py | 2 ++ 3 files changed, 68 insertions(+), 1 deletion(-) diff --git a/test/test_metrics/test_speaker.py b/test/test_metrics/test_speaker.py index 14bbead..69fa202 100644 --- a/test/test_metrics/test_speaker.py +++ b/test/test_metrics/test_speaker.py @@ -153,6 +153,11 @@ def test_cache_namespace_speaker_alias_hf_tag(): def test_cache_namespace_explicit_cache_dir_wins(): + """Explicit cache_dir wins at the config-plumbing level. + + Note: at model-load time, get_hf_cache_dir gives the VERSA_HF_CACHE_DIR + environment variable precedence over this per-metric cache_dir. + """ from versa.scorer_shared import configure_metric_cache_dirs configs = configure_metric_cache_dirs( @@ -166,3 +171,39 @@ def test_cache_namespace_explicit_cache_dir_wins(): cache_folder="/tmp/vc", ) assert configs[0]["cache_dir"] == "/custom/cache" + + +def test_cache_namespace_backend_override(): + from versa.scorer_shared import configure_metric_cache_dirs + + configs = configure_metric_cache_dirs( + [{"name": "speaker", "model_tag": "default", "backend": "huggingface"}], + cache_folder="/tmp/vc", + ) + assert configs[0]["cache_dir"].endswith("huggingface") + + +def test_resolve_backend_none_tag_is_espnet(): + assert resolve_speaker_backend(model_tag=None) == "espnet" + + +def test_effective_dependencies_wavlm_excludes_espnet(): + from versa.config_validation import _effective_dependencies + from versa.utterance_metrics.speaker import _speaker_metadata + + deps = _effective_dependencies( + "speaker", _speaker_metadata(), {"model_tag": "microsoft/wavlm-base-sv"} + ) + assert "espnet2" not in deps + assert "transformers" in deps + + +def test_effective_dependencies_default_excludes_transformers(): + from versa.config_validation import _effective_dependencies + from versa.utterance_metrics.speaker import _speaker_metadata + + deps = _effective_dependencies( + "speaker", _speaker_metadata(), {"model_tag": "default"} + ) + assert "transformers" not in deps + assert "espnet2" in deps diff --git a/versa/config_validation.py b/versa/config_validation.py index 2bbdd52..a8b7781 100644 --- a/versa/config_validation.py +++ b/versa/config_validation.py @@ -20,6 +20,30 @@ _GPU_REQUIRED_METRICS = {"audiobox_aesthetics"} +def _effective_dependencies(metric_name, metadata, config): + """Return the dependencies actually required by this metric config. + + The speaker metric bundles both backends in its metadata; only the + backend selected by the config is required at runtime. + """ + name = str(metric_name).lower() + if name in {"speaker", "spk_similarity", "speaker_similarity"}: + from versa.utterance_metrics.speaker import resolve_speaker_backend + + try: + backend = resolve_speaker_backend( + model_tag=config.get("model_tag", "default"), + backend=config.get("backend"), + model_path=config.get("model_path"), + model_config=config.get("model_config"), + ) + except ValueError: + return list(metadata.dependencies) + excluded = "espnet2" if backend == "huggingface" else "transformers" + return [dep for dep in metadata.dependencies if dep != excluded] + return list(metadata.dependencies) + + @dataclass(frozen=True) class ScoreConfigValidationError: """One score configuration validation issue.""" @@ -142,7 +166,7 @@ def validate_score_config( missing_dependencies = [ dependency - for dependency in metadata.dependencies + for dependency in _effective_dependencies(metric_name, metadata, config) if not _dependency_available(dependency) ] if missing_dependencies: diff --git a/versa/utterance_metrics/speaker.py b/versa/utterance_metrics/speaker.py index 587d2d8..26e7846 100644 --- a/versa/utterance_metrics/speaker.py +++ b/versa/utterance_metrics/speaker.py @@ -55,6 +55,8 @@ def resolve_speaker_backend( Returns: "espnet" or "huggingface". """ + if model_tag is None: + model_tag = "default" if backend is not None: if backend not in SPEAKER_BACKENDS: raise ValueError( From b78d3cf3ab93e86a0c5e95a16524f38c021db08f Mon Sep 17 00:00:00 2001 From: Thanapat Trachu Date: Thu, 23 Jul 2026 16:39:21 -0400 Subject: [PATCH 7/8] fix(speaker): coerce non-string model_tag in backend resolver --- test/test_metrics/test_speaker.py | 4 ++++ versa/utterance_metrics/speaker.py | 3 +-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/test/test_metrics/test_speaker.py b/test/test_metrics/test_speaker.py index 69fa202..9e9c381 100644 --- a/test/test_metrics/test_speaker.py +++ b/test/test_metrics/test_speaker.py @@ -187,6 +187,10 @@ def test_resolve_backend_none_tag_is_espnet(): assert resolve_speaker_backend(model_tag=None) == "espnet" +def test_resolve_backend_non_string_tag_does_not_crash(): + assert resolve_speaker_backend(model_tag=123) == "huggingface" + + def test_effective_dependencies_wavlm_excludes_espnet(): from versa.config_validation import _effective_dependencies from versa.utterance_metrics.speaker import _speaker_metadata diff --git a/versa/utterance_metrics/speaker.py b/versa/utterance_metrics/speaker.py index 26e7846..a538ee4 100644 --- a/versa/utterance_metrics/speaker.py +++ b/versa/utterance_metrics/speaker.py @@ -55,8 +55,7 @@ def resolve_speaker_backend( Returns: "espnet" or "huggingface". """ - if model_tag is None: - model_tag = "default" + model_tag = "default" if model_tag is None else str(model_tag) if backend is not None: if backend not in SPEAKER_BACKENDS: raise ValueError( From 6547f2661fbdaf773dfe96925b0f6700fa2a2dc6 Mon Sep 17 00:00:00 2001 From: Thanapat Trachu Date: Thu, 23 Jul 2026 18:31:46 -0400 Subject: [PATCH 8/8] test(speaker): gate real-model tests and add WavLM pipeline test --- test/test_metrics/test_speaker.py | 37 ++++++++++-- test/test_pipeline/test_speaker_wavlm.py | 71 ++++++++++++++++++++++++ 2 files changed, 104 insertions(+), 4 deletions(-) create mode 100644 test/test_pipeline/test_speaker_wavlm.py diff --git a/test/test_metrics/test_speaker.py b/test/test_metrics/test_speaker.py index 9e9c381..5ad0770 100644 --- a/test/test_metrics/test_speaker.py +++ b/test/test_metrics/test_speaker.py @@ -1,7 +1,14 @@ +import os + import numpy as np import pytest -from versa.utterance_metrics.speaker import resolve_speaker_backend +from versa.utterance_metrics.speaker import ( + is_transformers_available, + resolve_speaker_backend, +) + +RUN_REAL_MODEL_TESTS = os.environ.get("VERSA_RUN_REAL_MODEL_TESTS") == "1" def test_resolve_backend_default_tag_is_espnet(): @@ -40,15 +47,17 @@ def test_resolve_backend_invalid_backend_raises(): resolve_speaker_backend(model_tag="default", backend="wavlm2000") -from versa.utterance_metrics.speaker import is_transformers_available - - def _fixed_audio(freq, duration=1.0, sample_rate=16000): t = np.linspace(0, duration, int(sample_rate * duration), endpoint=False) envelope = 0.5 + 0.5 * np.sin(2 * np.pi * 0.5 * t) return (envelope * np.sin(2 * np.pi * freq * t)).astype(np.float32) +@pytest.mark.real_model +@pytest.mark.skipif( + not RUN_REAL_MODEL_TESTS, + reason="Set VERSA_RUN_REAL_MODEL_TESTS=1 to run real model-backed checks", +) @pytest.mark.skipif( not is_transformers_available(), reason="Transformers not available" ) @@ -62,6 +71,11 @@ def test_hf_speaker_model_embedding_shape(): assert embedding.shape[1] > 0 +@pytest.mark.real_model +@pytest.mark.skipif( + not RUN_REAL_MODEL_TESTS, + reason="Set VERSA_RUN_REAL_MODEL_TESTS=1 to run real model-backed checks", +) @pytest.mark.skipif( not is_transformers_available(), reason="Transformers not available" ) @@ -75,6 +89,11 @@ def test_hf_speaker_metric_identical_signals(): assert result["spk_similarity"] == pytest.approx(1.0, abs=1e-4) +@pytest.mark.real_model +@pytest.mark.skipif( + not RUN_REAL_MODEL_TESTS, + reason="Set VERSA_RUN_REAL_MODEL_TESTS=1 to run real model-backed checks", +) @pytest.mark.skipif( not is_transformers_available(), reason="Transformers not available" ) @@ -88,6 +107,11 @@ def test_hf_speaker_metric_different_signals(): assert -1.0 <= diff["spk_similarity"] <= 1.0 +@pytest.mark.real_model +@pytest.mark.skipif( + not RUN_REAL_MODEL_TESTS, + reason="Set VERSA_RUN_REAL_MODEL_TESTS=1 to run real model-backed checks", +) @pytest.mark.skipif( not is_transformers_available(), reason="Transformers not available" ) @@ -102,6 +126,11 @@ def test_speaker_metric_class_wavlm_backend(): assert result["spk_similarity"] == pytest.approx(1.0, abs=1e-4) +@pytest.mark.real_model +@pytest.mark.skipif( + not RUN_REAL_MODEL_TESTS, + reason="Set VERSA_RUN_REAL_MODEL_TESTS=1 to run real model-backed checks", +) @pytest.mark.skipif( not is_transformers_available(), reason="Transformers not available" ) diff --git a/test/test_pipeline/test_speaker_wavlm.py b/test/test_pipeline/test_speaker_wavlm.py new file mode 100644 index 0000000..54fe837 --- /dev/null +++ b/test/test_pipeline/test_speaker_wavlm.py @@ -0,0 +1,71 @@ +import math +import os + +import pytest +import yaml + +from versa.definition import MetricRegistry +from versa.scorer_shared import VersaScorer, compute_summary +from versa.utils_shared import find_files +from versa.utterance_metrics.speaker import ( + is_transformers_available, + register_speaker_metric, +) + +RUN_REAL_MODEL_TESTS = os.environ.get("VERSA_RUN_REAL_MODEL_TESTS") == "1" + + +def _load_wavlm_speaker_config(): + with open( + "egs/separate_metrics/spk_similarity_wavlm.yaml", "r", encoding="utf-8" + ) as f: + return yaml.safe_load(f) + + +def _sample_files(): + gen_path = "test/test_samples/test2" + gt_path = "test/test_samples/test1" + if not os.path.isdir(gen_path) or not os.path.isdir(gt_path): + pytest.skip("Required test sample directories are not available") + return find_files(gen_path), find_files(gt_path) + + +@pytest.mark.real_model +@pytest.mark.skipif( + not RUN_REAL_MODEL_TESTS, + reason="Set VERSA_RUN_REAL_MODEL_TESTS=1 to run real model-backed checks", +) +@pytest.mark.skipif( + not is_transformers_available(), reason="Transformers not available" +) +def test_speaker_wavlm_pipeline_with_real_model(): + """Run the WavLM speaker backend through the registry/scorer path.""" + gen_files, gt_files = _sample_files() + score_config = _load_wavlm_speaker_config() + + registry = MetricRegistry() + register_speaker_metric(registry) + scorer = VersaScorer(registry) + metric_suite = scorer.load_metrics(score_config, use_gt=True, use_gpu=False) + + assert len(score_config) > 0, "no scoring function is provided" + assert set(metric_suite.metrics) == {"speaker"} + + score_info = scorer.score_utterances( + gen_files, + metric_suite, + gt_files=gt_files, + output_file=None, + io="soundfile", + ) + summary = compute_summary(score_info) + + assert "spk_similarity" in summary + assert math.isfinite(summary["spk_similarity"]) + assert -1.0 <= summary["spk_similarity"] <= 1.0 + + +if __name__ == "__main__": + if not RUN_REAL_MODEL_TESTS: + raise SystemExit("Set VERSA_RUN_REAL_MODEL_TESTS=1 to run this check") + pytest.main([__file__, "-q", "-s"])