diff --git a/README.md b/README.md index 208829a..8afe426 100644 --- a/README.md +++ b/README.md @@ -33,6 +33,11 @@ UltraEval-Audio — The world's first open-source framework supporting both spee # Changelog🔥 +- [2026/02/03] + - Support **[Qwen3-TTS](replication/qwen3_tts.md)** evaluation + - GPU parallel acceleration for faster evaluation/inference + - Usage: add `--use_model_pool` and `--workers ` to enable multi-GPU parallel inference, e.g. + - `python audio_evals/main.py --dataset --model --use_model_pool --workers 4` - [2026/01/19] - Support Step-Audio-R1.1 evaluation, with replication report: [Step-Audio-R1.1](replication/step-audio-r1_1.md) - [2025/12/31] @@ -107,8 +112,8 @@ UltraEval-Audio — The world's first open-source framework supporting both spee | **GPT-4o-Realtime** | **51.60** | **69.70** | **74.00** | 70.05 | **98.69** | 4.29\|3.44\|4.26 | **74.00** | | **Qwen3-Omni-30B-A3B-Instruct** | 51.50 | 55.27 | 67.97 | 47.83 | 40.27 | **4.44**\|3.45\|4.12 | 57.15 | | **Qwen2.5-Omni** | 38.89 | 39.94 | 54.00 | **73.72** | 95.65 | 4.23\|**3.48**\|**4.27** | 63.68 | -| **MiniCPM-o 2.6** | 40.00 | 40.20 | 51.00 | 51.37 | 80.68 | 4.12\|3.39\|4.02 | 56.69 | -| **Kimi-Audio-7B-Instruct** | 33.69 | 38.20 | 34.40 | 71.25 | 97.42 | 2.94\|3.22\|3.62 | 56.69 | +| **MiniCPM-o 2.6** | 40.00 | 40.20 | 51.00 | 49.22 | 80.68 | 4.12\|3.39\|4.02 | 56.69 | +| **Kimi-Audio-7B-Instruct** | 33.69 | 38.20 | 34.40 | 66.98 | 97.42 | 2.94\|3.22\|3.62 | 56.69 | | **GLM-4-Voice** | 32.00 | 36.40 | 51.00 | 52.61 | 71.06 | 4.21\|3.46\|4.07 | 53.56 | ## Audio Codec Leaderboard diff --git a/README_zh.md b/README_zh.md index b1be823..a50fd1a 100644 --- a/README_zh.md +++ b/README_zh.md @@ -33,6 +33,11 @@ UltraEval-Audio——全球首个同时支持语音理解和语音生成评估 # 更新日志🔥 +- [2026/02/03] + - 支持 **[Qwen3-TTS](replication/qwen3_tts.md)** 评测 + - GPU 并行加速，提升评测/推理速度 + - 用法：命令行增加 `--use_model_pool` 和 `--workers ` 开启多 GPU 并行推理，例如： + - `python audio_evals/main.py --dataset --model --use_model_pool --workers 4` - [2026/01/19] - 新增对 Step-Audio-R1.1 的评测支持，复现评测见：[Step-Audio-R1.1](replication/step-audio-r1_1.md) - [2025/12/31] diff --git a/audio_evals/dataset/huggingface.py b/audio_evals/dataset/huggingface.py index 3294020..19e15a1 100644 --- a/audio_evals/dataset/huggingface.py +++ b/audio_evals/dataset/huggingface.py @@ -20,6 +20,7 @@ def save_audio_to_local(ds: Dataset, save_path: str): def save_audio(example, index): if "audio" not in example: + logger.error(f"audio not in example: {example}, skip this example") return example audio_array = example["audio"]["array"] output_path = os.path.join(save_path, f"{index}.wav") @@ -28,7 +29,6 @@ def save_audio(example, index): os.makedirs(d, exist_ok=True) if not os.path.exists(output_path): sf.write(output_path, audio_array, example["audio"]["sampling_rate"]) - logger.info(f"save audio to {output_path}") return example ds = ds.map(save_audio, with_indices=True) diff --git a/audio_evals/dataset/minimax_tts.py b/audio_evals/dataset/minimax_tts.py new file mode 100644 index 0000000..b1abae8 --- /dev/null +++ b/audio_evals/dataset/minimax_tts.py @@ -0,0 +1,159 @@ +import os +import logging +from typing import Dict, List +from huggingface_hub import hf_hub_download, list_repo_tree +from audio_evals.dataset.dataset import Dataset as BaseDataset + +logger = logging.getLogger(__name__) + + +class MiniMaxTTSDataset(BaseDataset): + def __init__( + self, + name: str = "MiniMaxAI/TTS-Multilingual-Test-Set", + default_task: str = "tts", + ref_col: str = "text", + col_aliases: Dict[str, str] = None, + language: str = None, + languages: List[str] = None, + ): + super().__init__(default_task, ref_col, col_aliases) + self.name = name + + if language: + self.languages = [language] + elif languages: + self.languages = languages + else: + self.languages = [ + "arabic", "cantonese", "chinese", "czech", "dutch", "english", + "finnish", "french", "german", "greek", "hindi", "indonesian", + "italian", "japanese", "korean", "polish", "portuguese", "romanian", + "russian", "spanish", "thai", "turkish", "ukrainian", "vietnamese" + ] + + def _get_language_from_speaker(self, speaker_label: str) -> str: + """Extract language from speaker label (e.g., 'chinese_female' -> 'chinese')""" + parts = speaker_label.rsplit('_', 1) + return parts[0] if len(parts) > 1 else speaker_label + + def load(self, limit=0) -> List[Dict[str, any]]: + logger.info(f"Loading MiniMax TTS dataset: {self.name} for languages: {self.languages}") + + # Load prompt transcriptions: filename -> prompt_text + prompt_texts = {} + try: + prompt_text_file = hf_hub_download( + repo_id=self.name, + filename="speaker/prompt_text.txt", + repo_type="dataset" + ) + with open(prompt_text_file, "r", encoding="utf-8") as f: + for line in f: + line = line.strip() + if not line or "|" not in line: + continue + parts = line.split("|", 1) + if len(parts) >= 2: + audio_name = parts[0].strip() + p_text = parts[1].strip() + prompt_texts[audio_name] = p_text + except Exception as e: + logger.warning(f"Failed to load prompt_text.txt: {e}") + + # Build speaker map: speaker_label -> (audio_path, prompt_filename) + # Download speaker audio files from the speaker/ folder + speaker_map = {} # speaker_label -> audio_path + speaker_filenames = {} # speaker_label -> filename (for prompt_text lookup) + + # Get all speakers needed from the text files first + speakers_needed = set() + for lang in self.languages: + text_file = hf_hub_download( + repo_id=self.name, + filename=f"text/{lang}.txt", + repo_type="dataset" + ) + with open(text_file, "r", encoding="utf-8") as f: + for line in f: + line = line.strip() + if not line or "|" not in line: + continue + parts = line.split("|", 1) + speaker_label = parts[0].strip() + speakers_needed.add(speaker_label) + if len(speakers_needed) == 1: + with open(text_file, "r", encoding="utf-8") as f: + print(f.read()) + raise ValueError(f"Only one speaker needed for language {lang}") + + # Download prompt audio for each speaker from speaker/{language}/{speaker_label}/ + for speaker_label in speakers_needed: + language = self._get_language_from_speaker(speaker_label) + speaker_folder = f"speaker/{language}/{speaker_label}" + + # List files in the speaker folder to find the prompt audio + files = list(list_repo_tree( + self.name, + path_in_repo=speaker_folder, + repo_type="dataset" + )) + + # Find the mp3 file + for f in files: + if f.path.endswith('.mp3'): + audio_path = hf_hub_download( + repo_id=self.name, + filename=f.path, + repo_type="dataset" + ) + speaker_map[speaker_label] = audio_path + speaker_filenames[speaker_label] = os.path.basename(f.path) + logger.info(f"Downloaded prompt audio for {speaker_label}: {audio_path}") + break + + logger.info(f"Built speaker_map with {len(speaker_map)} speakers: {list(speaker_map.keys())[:5]}...") + + # Load test sentences for each language + results = [] + for lang in self.languages: + text_file = hf_hub_download( + repo_id=self.name, + filename=f"text/{lang}.txt", + repo_type="dataset" + ) + + with open(text_file, "r", encoding="utf-8") as f: + for line in f: + line = line.strip() + if not line or "|" not in line: + continue + parts = line.split("|", 1) + if len(parts) < 2: + continue + speaker_label = parts[0].strip() + text = parts[1].strip() + + if speaker_label not in speaker_map: + logger.warning(f"Speaker '{speaker_label}' not found in speaker_map") + continue + + # Get prompt text using the filename + prompt_filename = speaker_filenames[speaker_label] + prompt_text = prompt_texts[prompt_filename] + + results.append({ + "WavPath": speaker_map[speaker_label], + "prompt_audio": speaker_map[speaker_label], + "prompt_text": prompt_text, + "text": text, + "ans": text, + "language": lang, + "speaker": speaker_label + }) + + if limit > 0: + results = results[:limit] + + logger.info(f"Loaded {len(results)} samples from MiniMax TTS dataset") + return results diff --git a/audio_evals/eval_task.py b/audio_evals/eval_task.py index 80e235f..44af4de 100644 --- a/audio_evals/eval_task.py +++ b/audio_evals/eval_task.py @@ -1,5 +1,6 @@ import json import traceback +import threading from concurrent.futures import as_completed, ThreadPoolExecutor from functools import lru_cache from typing import Dict, List, Tuple, Union @@ -14,7 +15,7 @@ from audio_evals.process.base import Process from audio_evals.prompt.base import Prompt from audio_evals.recorder import Recorder -from audio_evals.utils import merge_data4view +from audio_evals.utils import logger, merge_data4view def extract_score(s: str): @@ -84,15 +85,155 @@ def _run(self, i, doc): print(error_traceback) return i, None, None, 1 + def _inference_only(self, i, doc): + """仅执行推理，不进行评测""" + real_prompt = self.prompt.load(**doc) + try: + self.recorder.add({"type": "prompt", "id": i, "data": {"content": real_prompt}}) + + if "eval_info" in doc and "inference" in doc["eval_info"]: + output = doc["eval_info"]["inference"]["content"] + else: + output = self.predictor.inference(real_prompt) + self.recorder.add({"type": "inference", "id": i, "data": {"content": output}}) + + if "eval_info" in doc and "post_process" in doc["eval_info"]: + output = doc["eval_info"]["post_process"]["content"] + else: + for p in self.post_process: + output = p(output) + self.recorder.add({"type": "post_process", "id": i, "data": {"content": output}}) + + return i, output, doc, 0 + except Exception: + error_traceback = traceback.format_exc() + self.recorder.add({"type": "error", "id": i, "data": {"info": error_traceback}}) + print(error_traceback) + return i, None, doc, 1 + + def _evaluate_only(self, i, output, doc): + """仅执行评测，假设推理已完成""" + try: + reference = doc.get(self.dataset.ref_col, "") + score = self.evaluator(output, reference, **doc) + self.recorder.add({"type": "eval", "id": i, "data": score}) + return i, score, output, 0 + except Exception: + error_traceback = traceback.format_exc() + self.recorder.add({"type": "error", "id": i, "data": {"info": error_traceback}}) + print(error_traceback) + return i, None, output, 1 + + def _release_predictor(self): + """释放推理模型占用的 GPU 显存""" + try: + # 尝试调用模型的释放方法（如果有的话） + if hasattr(self.predictor, 'release') and callable(self.predictor.release): + self.predictor.release() + # 删除模型引用 + del self.predictor + self.predictor = None + print("Predictor released successfully, GPU memory freed.") + else: + predictor_type = type(self.predictor).__name__ + print(f"Predictor ({predictor_type}) does not have a release method, skipping GPU memory release.") + except Exception as e: + print(f"Warning: Failed to release GPU memory: {e}") + + def run_two_phase( + self, limit=None, rand_size=None, max_workers=1 + ) -> Tuple[ScoreUnit, List[ScoreUnit], List[str]]: + """ + 两阶段执行：先并发推理，后串行评测 + :param limit: 限制数据条数 + :param rand_size: 随机采样数量 + :param max_workers: 推理阶段的并发数 + :return: 聚合结果, 各条评分, 各条输出 + """ + quiz = self.dataset.load(limit) + if limit: + quiz = quiz[:limit] + if rand_size: + import random + quiz = random.sample(quiz, rand_size) + + # 第一阶段：并发推理 + print("Phase 1: Running inference...") + inference_results = [None] * len(quiz) + inference_docs = [None] * len(quiz) + inference_error_count = 0 + + with ThreadPoolExecutor(max_workers=max_workers) as executor: + future_to_index = [ + executor.submit(self._inference_only, i, doc) for i, doc in enumerate(quiz) + ] + for future in tqdm(as_completed(future_to_index), total=len(quiz), desc="Inference"): + index, output, doc, has_error = future.result() + inference_error_count += has_error + if output is not None: + inference_results[index] = output + inference_docs[index] = doc + + # 释放模型显存，评测阶段不需要 GPU + print("Releasing model GPU memory...") + self._release_predictor() + + # 第二阶段：串行评测 + print("Phase 2: Running evaluation...") + res = [None] * len(quiz) + answers = [None] * len(quiz) + eval_error_count = 0 + + from audio_evals.registry import registry + self.evaluator = registry.get_evaluator(self.evaluator) + + # 构建需要评测的任务列表 + eval_tasks = [ + (i, inference_results[i], inference_docs[i]) + for i in range(len(quiz)) + if inference_results[i] is not None + ] + + with ThreadPoolExecutor(max_workers=max_workers) as executor: + future_to_index = [ + executor.submit(self._evaluate_only, i, output, doc) + for i, output, doc in eval_tasks + ] + for future in tqdm(as_completed(future_to_index), total=len(eval_tasks), desc="Evaluation"): + index, score, output, has_error = future.result() + eval_error_count += has_error + if score is not None: + res[index] = score + answers[index] = output + + res, answers = [item for item in res if item is not None], [ + item for item in answers if item is not None + ] + merge_data4view( + quiz, self.recorder.name, self.recorder.name.replace(".jsonl", ".xlsx") + ) + final_res = self.agg(res) + total_error = inference_error_count + eval_error_count + final_res["fail_rate(%d)"] = total_error / len(quiz) * 100 + final_res["inference_fail_count"] = inference_error_count + final_res["eval_fail_count"] = eval_error_count + return final_res, res, answers + @lru_cache(maxsize=None) def run( - self, limit=None, rand_size=None, max_workers=1 + self, limit=None, rand_size=None, max_workers=1, two_phase=False ) -> Tuple[ScoreUnit, List[ScoreUnit], List[str]]: """ eval - :param : - :return: + :param limit: 限制数据条数 + :param rand_size: 随机采样数量 + :param max_workers: 并发数 + :param two_phase: 是否使用两阶段模式（先推理后评测） + :return: 聚合结果, 各条评分, 各条输出 """ + if two_phase: + return self.run_two_phase(limit, rand_size, max_workers) + quiz = self.dataset.load(limit) if limit: quiz = quiz[:limit] @@ -104,6 +245,8 @@ def run( res = [None] * len(quiz) answers = [None] * len(quiz) error_count = 0 + from audio_evals.registry import registry + self.evaluator = registry.get_evaluator(self.evaluator) # 使用进程池并控制最大并发量 with ThreadPoolExecutor(max_workers=max_workers) as executor: diff --git a/audio_evals/evaluator/choices.py b/audio_evals/evaluator/choices.py new file mode 100644 index 0000000..78c2b21 --- /dev/null +++ b/audio_evals/evaluator/choices.py @@ -0,0 +1,71 @@ +from .base import Evaluator +import numpy as np +from typing import Dict, List, Optional, Union + + +class ChoicesEval(Evaluator): + def __init__(self, choices_columns: Union[List[str], str], ignore_parse_error: bool = False, constant = False): + self.choices_columns = choices_columns + self.ignore_parse_error = ignore_parse_error + self.constant = constant + + def _eval(self, pred: str, label: str, **kwargs) -> Dict[str, any]: + pred = str(pred).strip() + label = str(label).strip() + + # Valid choices + if isinstance(self.choices_columns, str): + choices = kwargs[self.choices_columns] + choice_letters = [chr(65+i) for i in range(len(choices))] + choices = {choice.lower().strip(): choice_letters[i] for i, choice in enumerate(choices)} + else: + if self.constant: + choice_letters = [chr(65+i) for i in range(len(self.choices_columns))] + choices = {item.lower().strip(): choice_letters[i] for i, item in enumerate(self.choices_columns)} + + else: + choice_letters = [chr(65+i) for i in range(len(self.choices_columns))] + choices = {str(kwargs.get(col, '')).lower().strip().replace('-', ' '): choice_letters[i] for i, col in enumerate(self.choices_columns)} + + + # Extract model prediction from response + model_predict = None + is_format_error = False + + if pred and pred != 'None': + # Check if first character is a valid choice + if pred[0] in choice_letters: + model_predict = pred[0] + # This situation may occur when the answer given by model is "The answer is A." + elif len(pred) > 1: + if pred[-2] in choice_letters: + model_predict = pred[-2] + else: + print(f'Wrong format response: {pred}') + is_format_error = True + else: + print(f'Wrong format response: {pred}') + is_format_error = True + else: + print(f'Wrong format response: {pred}') + is_format_error = True + + # Determine the result: 1 (correct), 0 (incorrect), None (format error) + result: Optional[int] = None + + if is_format_error: + if self.ignore_parse_error: + result = 0 + else: + result = None + elif model_predict: + # Get choices from kwargs + result = 1 if model_predict.lower() == choices[label.lower().strip().replace('-', ' ')].lower() else 0 + + return { + "match": result, # 1, 0, or None + "is_format_error": 1 if is_format_error else 0, # 1 for format error, 0 otherwise + "pred": pred, + "model_predict": model_predict, + "ref": label, + } diff --git a/audio_evals/evaluator/seed_tts_eval_asr_wer.py b/audio_evals/evaluator/seed_tts_eval_asr_wer.py index 081aea9..b8aa6ba 100644 --- a/audio_evals/evaluator/seed_tts_eval_asr_wer.py +++ b/audio_evals/evaluator/seed_tts_eval_asr_wer.py @@ -5,7 +5,9 @@ from jiwer import compute_measures from zhon.hanzi import punctuation import string +import logging +logger = logging.getLogger(__name__) punctuation_all = punctuation + string.punctuation @@ -23,20 +25,22 @@ def process_one(hypo, truth, lang): truth = truth.replace(" ", " ") hypo = hypo.replace(" ", " ") - # Character-based languages (CER) - if lang in ["zh", "ja", "ko"]: - truth = " ".join([x for x in truth]) - hypo = " ".join([x for x in hypo]) + # yue: cantonese, th: thai + if lang in ["zh", "ja", "yue", "th", "ko"]: + truth = " ".join([x for x in truth if x.strip()]) + hypo = " ".join([x for x in hypo if x.strip()]) # Word-based languages (WER) - elif lang in ["en", "de", "es", "fr", "it", "ru"]: + else: truth = truth.lower() hypo = hypo.lower() - else: - raise NotImplementedError - measures = compute_measures(truth, hypo) - ref_list = truth.split(" ") - wer = measures["wer"] + try: + measures = compute_measures(truth, hypo) + wer = measures["wer"] + except Exception as e: + logger.error(f"Error computing measures: {e}. truth: '{truth}', hypo: '{hypo}'") + raise e + return wer @@ -56,8 +60,18 @@ def _eval(self, pred, label, **kwargs) -> Dict[str, any]: ) real_prompt = self.prompt.load(WavPath=pred) - transcription = self.model.inference(real_prompt) - if self.lang == "zh": + + # Pass language to model for non-Chinese languages or if specified + # Whisper model expects language in generate_kwargs + inf_kwargs = {} + if self.lang != "zh": + inf_kwargs["generate_kwargs"] = { + "language": kwargs.get("language", self.lang) + } + + transcription = self.model.inference(real_prompt, **inf_kwargs) + + if self.lang == "zh" or kwargs.get("language") == "chinese": transcription = zhconv.convert(transcription, "zh-cn") res = {"wer%": process_one(transcription, label_text, self.lang) * 100} diff --git a/audio_evals/isolate.py b/audio_evals/isolate.py index 7656d29..6a4466a 100644 --- a/audio_evals/isolate.py +++ b/audio_evals/isolate.py @@ -14,11 +14,21 @@ def decorator(cls): original_init = cls.__init__ @wraps(original_init) - def new_init(self, env_path, requirements_path, *args, **kwargs): + def new_init(self, env_path, requirements_path, *args, gpu_id=None, **kwargs): + """ + Args: + env_path: 虚拟环境路径 + requirements_path: 依赖文件路径 + gpu_id: 指定使用的 GPU ID，如 0, 1, 2。 + 如果为 None，则不设置 CUDA_VISIBLE_DEVICES（使用默认行为） + """ original_init(self, *args, **kwargs) if env_path.endswith("/"): env_path = env_path[:-1] + # 保存 gpu_id 供外部查询 + self._gpu_id = gpu_id + # 创建虚拟环境 if not os.path.exists(env_path): res = subprocess.run(["uv", "venv", env_path, "--python", "3.10"]) @@ -65,9 +75,16 @@ def new_init(self, env_path, requirements_path, *args, **kwargs): ] ) + # 构建 CUDA_VISIBLE_DEVICES 设置 + cuda_env = "" + if gpu_id is not None: + cuda_env = f"export CUDA_VISIBLE_DEVICES={gpu_id} && " + logger.info(f"Setting CUDA_VISIBLE_DEVICES={gpu_id} for isolated process") + # 构建完整命令 command = ( f"source {env_path}/bin/activate && " + f"{cuda_env}" f"export LD_LIBRARY_PATH={lib_path} && " f"{env_path}/bin/python -u {script_path} {args_str}" ) diff --git a/audio_evals/lib/Kimi-Audio/kimia_infer/api/kimia.py b/audio_evals/lib/Kimi-Audio/kimia_infer/api/kimia.py index 3f13ffa..b4058aa 100644 --- a/audio_evals/lib/Kimi-Audio/kimia_infer/api/kimia.py +++ b/audio_evals/lib/Kimi-Audio/kimia_infer/api/kimia.py @@ -202,6 +202,9 @@ def _generate_loop( .numpy() .tolist() ) + # 清理 KV cache 和中间张量，防止显存泄露 + del past_key_values, previous_audio_tokens, text_previous_tokens + del decoder_input_audio_ids, decoder_input_text_ids return return_audio_tokens, return_text_tokens def generate( @@ -259,6 +262,9 @@ def generate( continous_feature=audio_features, output_type=output_type, ) + + # 清理输入张量，释放显存 + del audio_input_ids, text_input_ids, is_continuous_mask, audio_features generated_wav_tokens = [ t for t in generated_wav_tokens if t >= self.kimia_token_offset @@ -275,6 +281,9 @@ def generate( generated_wav = self.detokenize_audio(generated_wav_tokens) else: generated_wav = None + + # 清理中间张量 + del generated_wav_tokens, generated_text_tokens return generated_wav, generated_text diff --git a/audio_evals/lib/Kimi-Audio/main.py b/audio_evals/lib/Kimi-Audio/main.py index 8a6a834..ce910d6 100644 --- a/audio_evals/lib/Kimi-Audio/main.py +++ b/audio_evals/lib/Kimi-Audio/main.py @@ -9,7 +9,7 @@ import soundfile as sf import torch -logging.basicConfig(level=logging.INFO) +logging.basicConfig(level=logging.WARNING) logger = logging.getLogger(__name__) if __name__ == "__main__": @@ -65,9 +65,10 @@ messages = x["messages"] if "messages" in x else x output_type = "both" if config.speech else "text" # 推理 - wav, text = model.generate( - messages, **sampling_params, output_type=output_type - ) + with torch.no_grad(): + wav, text = model.generate( + messages, **sampling_params, output_type=output_type + ) if config.speech: with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as f: @@ -107,8 +108,15 @@ break print("not found close signal, will emit again", flush=True) retry -= 1 + + # 清理显存，防止泄露 + del wav, text + torch.cuda.empty_cache() + except Exception as e: import traceback traceback.print_exc() print(f"Error: {str(e)}", flush=True) + # 异常时也清理显存 + torch.cuda.empty_cache() diff --git a/audio_evals/lib/Qwen3TTS/main.py b/audio_evals/lib/Qwen3TTS/main.py new file mode 100644 index 0000000..5280b2e --- /dev/null +++ b/audio_evals/lib/Qwen3TTS/main.py @@ -0,0 +1,190 @@ +import argparse +import json +import logging +import os +import select +import sys +import tempfile +import time + +import torch +import soundfile as sf +from qwen_tts import Qwen3TTSModel + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + "--path", type=str, required=True, help="Path to Qwen3-TTS model" + ) + parser.add_argument( + "--mode", + type=str, + default="custom_voice", + choices=["custom_voice", "voice_design", "voice_clone"], + help="Generation mode: custom_voice, voice_design, or voice_clone", + ) + parser.add_argument( + "--dtype", + type=str, + default="bfloat16", + choices=["float16", "bfloat16", "float32"], + help="Model dtype", + ) + parser.add_argument( + "--device", + type=str, + default="cuda:0", + help="Device to run the model on", + ) + args = parser.parse_args() + + # Determine dtype + dtype_map = { + "float16": torch.float16, + "bfloat16": torch.bfloat16, + "float32": torch.float32, + } + dtype = dtype_map.get(args.dtype, torch.bfloat16) + + logger.info(f"Loading Qwen3-TTS model from {args.path} with dtype={args.dtype}") + + # Try to use flash attention if available + try: + model = Qwen3TTSModel.from_pretrained( + args.path, + device_map=args.device, + dtype=dtype, + attn_implementation="flash_attention_2", + ) + logger.info("Loaded with flash_attention_2") + except Exception as e: + logger.warning(f"Failed to load with flash_attention_2: {e}, falling back to default") + model = Qwen3TTSModel.from_pretrained( + args.path, + device_map=args.device, + dtype=dtype, + ) + + logger.info(f"Qwen3-TTS model loaded successfully in {args.mode} mode") + + # Get supported speakers and languages for custom voice mode + if args.mode == "custom_voice": + try: + speakers = model.get_supported_speakers() + languages = model.get_supported_languages() + logger.info(f"Supported speakers: {speakers}") + logger.info(f"Supported languages: {languages}") + except Exception as e: + logger.warning(f"Could not get supported speakers/languages: {e}") + + # Enable RTF tracking from environment variable + enable_rtf = int(os.environ.get("ENABLE_RTF", "0")) + logger.info(f"ENABLE_RTF: {enable_rtf}") + + while True: + try: + prompt = input() + anchor = prompt.find("->") + if anchor == -1: + print( + f"Error: Invalid conversation format, must contain '->', but got {prompt}", + flush=True, + ) + continue + + prefix = prompt[:anchor].strip() + "->" + x = json.loads(prompt[anchor + 2:]) + + # Record start time for RTF calculation + torch.cuda.synchronize() + start_time = time.time() + + # Extract common parameters + text = x.pop("text") + language = x.pop("language", "Auto") + + if args.mode == "custom_voice": + # Custom voice generation + speaker = x.pop("speaker", "Vivian") + instruct = x.pop("instruct", None) + generate_kwargs = { + "text": text, + "language": language, + "speaker": speaker, + } + generate_kwargs.update(x) + if instruct: + generate_kwargs["instruct"] = instruct + logger.info(f"generate_custom_voice kwargs: {generate_kwargs}") + wavs, sr = model.generate_custom_voice(**generate_kwargs) + + elif args.mode == "voice_design": + # Voice design generation + instruct = x.pop("instruct", "") + logger.info(f"voice_design: text: {text}, language: {language}, instruct: {instruct}, **x: {x}") + wavs, sr = model.generate_voice_design( + text=text, + language=language, + instruct=instruct, + **x, + ) + + elif args.mode == "voice_clone": + # Voice clone generation + ref_audio = x.pop("prompt_audio") + ref_text = x.pop("prompt_text") + + if ref_audio is None: + raise ValueError("ref_audio is required for voice_clone mode") + logger.info(f"ref_audio: {ref_audio}, ref_text: {ref_text}, text: {text}, language: {language}, **x: {x}") + wavs, sr = model.generate_voice_clone( + text=text, + language=language, + ref_audio=ref_audio, + ref_text=ref_text, + **x, + ) + else: + raise ValueError(f"Unknown mode: {args.mode}") + + # Record end time + torch.cuda.synchronize() + end_time = time.time() + inference_time = end_time - start_time + + # Save output to temporary file + with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as f: + sf.write(f.name, wavs[0], sr) + output_path = f.name + + # Return result with optional RTF + if enable_rtf == 1: + audio_duration = len(wavs[0]) / sr + rtf = inference_time / audio_duration if audio_duration > 0 else 0 + result = json.dumps({"audio": output_path, "RTF": rtf}) + logger.info( + f"RTF: {rtf:.4f} (inference: {inference_time:.2f}s, audio: {audio_duration:.2f}s)" + ) + else: + result = output_path + + # Output result with retry mechanism + retry = 3 + while retry: + retry -= 1 + print(f"{prefix}{result}", flush=True) + rlist, _, _ = select.select([sys.stdin], [], [], 1) + if rlist: + finish = sys.stdin.readline().strip() + if finish == f"{prefix}close": + break + print("not found close signal, will emit again", flush=True) + + except Exception as e: + import traceback + traceback.print_exc() + print(f"Error: {str(e)}", flush=True) diff --git a/audio_evals/lib/Qwen3TTS/requirements.txt b/audio_evals/lib/Qwen3TTS/requirements.txt new file mode 100644 index 0000000..1173d92 --- /dev/null +++ b/audio_evals/lib/Qwen3TTS/requirements.txt @@ -0,0 +1,3 @@ +qwen-tts +torch>=2.0.0 +soundfile diff --git a/audio_evals/lib/StepAudio/requirements.txt b/audio_evals/lib/StepAudio/requirements.txt new file mode 100644 index 0000000..4d9204c --- /dev/null +++ b/audio_evals/lib/StepAudio/requirements.txt @@ -0,0 +1,3 @@ +requests>=2.28.0 +librosa +transformers<5.0.0 \ No newline at end of file diff --git a/audio_evals/lib/StepAudio/serve.py b/audio_evals/lib/StepAudio/serve.py index 5adae11..58d003a 100644 --- a/audio_evals/lib/StepAudio/serve.py +++ b/audio_evals/lib/StepAudio/serve.py @@ -152,7 +152,7 @@ def main(): parser.add_argument( "--max_num_seqs", type=int, - default=32, + default=8, help="Max number of sequences (default: 32)", ) parser.add_argument( diff --git a/audio_evals/lib/StepAudio/stepaudior1vllm.py b/audio_evals/lib/StepAudio/stepaudior1vllm.py index e306d10..feb43a4 100644 --- a/audio_evals/lib/StepAudio/stepaudior1vllm.py +++ b/audio_evals/lib/StepAudio/stepaudior1vllm.py @@ -5,7 +5,7 @@ import re from datetime import datetime from pathlib import Path -from typing import List, Optional, Union +from typing import List, Optional, Tuple, Union import requests from pydub import AudioSegment @@ -153,7 +153,14 @@ def log_request(self, payload): return filename - def stream(self, messages, stream=True, stop=None, **kwargs): + def stream( + self, + messages, + stream: bool = True, + stop=None, + request_timeout: Optional[Union[float, Tuple[float, float]]] = None, + **kwargs, + ): headers = {"Content-Type": "application/json"} payload = kwargs payload["messages"] = self.apply_chat_template(messages) @@ -181,7 +188,11 @@ def stream(self, messages, stream=True, stop=None, **kwargs): # self.log_request(payload) with requests.post( - self.api_url, headers=headers, json=payload, stream=stream + self.api_url, + headers=headers, + json=payload, + stream=stream, + timeout=request_timeout, ) as response: response.raise_for_status() diff --git a/audio_evals/lib/VoxCPM1_5/requirements.txt b/audio_evals/lib/VoxCPM1_5/requirements.txt index 593ce0b..18725d7 100644 --- a/audio_evals/lib/VoxCPM1_5/requirements.txt +++ b/audio_evals/lib/VoxCPM1_5/requirements.txt @@ -1,3 +1,3 @@ -/user/shiqundong/project/Infer/VoxCPM +voxcpm torch==2.5.1 torchcodec diff --git a/audio_evals/lib/qwen2-5omni/main.py b/audio_evals/lib/qwen2-5omni/main.py index 3ea7178..d2f846f 100644 --- a/audio_evals/lib/qwen2-5omni/main.py +++ b/audio_evals/lib/qwen2-5omni/main.py @@ -14,13 +14,24 @@ def load_model(path, **kwargs): - model = Qwen2_5OmniForConditionalGeneration.from_pretrained( - path, - torch_dtype=torch.bfloat16, - device_map=device, - attn_implementation="flash_attention_2", - **kwargs - ) + try: + model = Qwen2_5OmniForConditionalGeneration.from_pretrained( + path, + torch_dtype=torch.bfloat16, + device_map=device, + attn_implementation="flash_attention_2", + **kwargs + ) + except Exception as e: + print(f"Failed to load model with flash_attention_2: {e}, falling back to default") + model = Qwen2_5OmniForConditionalGeneration.from_pretrained( + path, + torch_dtype=torch.bfloat16, + device_map=device, + **kwargs + ) + print(f"Loaded model with default attn_implementation") + print(f"Loaded model successfully") processor = Qwen2_5OmniProcessor.from_pretrained(path) return model, processor diff --git a/audio_evals/lib/qwen2-5omni/requirements.txt b/audio_evals/lib/qwen2-5omni/requirements.txt index ead0767..95947da 100644 --- a/audio_evals/lib/qwen2-5omni/requirements.txt +++ b/audio_evals/lib/qwen2-5omni/requirements.txt @@ -1,5 +1,5 @@ accelerate==1.6.0 -flash-attn==2.7.4.post1 +# flash-attn==2.7.4.post1 qwen-omni-utils[decord]==0.0.4 torch==2.6.0 torchvision==0.21.0 diff --git a/audio_evals/lib/qwen3-omni/main.py b/audio_evals/lib/qwen3-omni/main.py index 1ecbf7a..e0729aa 100644 --- a/audio_evals/lib/qwen3-omni/main.py +++ b/audio_evals/lib/qwen3-omni/main.py @@ -6,14 +6,33 @@ import soundfile as sf import torch +import torch_npu from transformers import Qwen3OmniMoeForConditionalGeneration, Qwen3OmniMoeProcessor from qwen_omni_utils import process_mm_info - +import subprocess +import os device = "cuda" +def load_ascend_env(script_path="/usr/local/Ascend/ascend-toolkit/latest/bin/set_env.sh"): + # 执行 shell 脚本并在执行后通过 env 命令打印所有变量 + command = f"source {script_path} && env" + proc = subprocess.Popen(command, stdout=subprocess.PIPE, shell=True, executable="/bin/bash") + + for line in proc.stdout: + line = line.decode("utf-8").strip() + if "=" in line: + key, value = line.split("=", 1) + # 只有当变量名包含 ASCEND 或涉及库路径时才注入，避免污染 + if "ASCEND" in key or key in ["LD_LIBRARY_PATH", "PYTHONPATH"]: + os.environ[key] = value + +# 在 import torch 之前调用 def load_model(path, **kwargs): + # 1. 设置昇腾底层库路径 (把 libhccl.so 所在的路径加进去) + # 注意：路径需要根据你环境的实际位置微调，通常是 latest/lib64 + load_ascend_env() model = Qwen3OmniMoeForConditionalGeneration.from_pretrained( path, torch_dtype="auto", diff --git a/audio_evals/lib/simo/models_ecapa_tdnn.py b/audio_evals/lib/simo/models_ecapa_tdnn.py index db637d1..3788419 100644 --- a/audio_evals/lib/simo/models_ecapa_tdnn.py +++ b/audio_evals/lib/simo/models_ecapa_tdnn.py @@ -1,6 +1,7 @@ # models_ecapa_tdnn.py # part of the code is borrowed from https://github.com/lawlict/ECAPA-TDNN +import os import torch import torch.nn as nn import torch.nn.functional as F @@ -268,8 +269,13 @@ def __init__( else: if config_path is None: torch.hub._validate_not_a_forked_repo = lambda a, b, c: True - # Todo : tmp put cache file in local ~/.cache/s3prl/download/ - self.feature_extract = torch.hub.load("s3prl/s3prl", "wavlm_large") + # 优先从本地缓存加载，避免联网检查 GitHub + local_cache_path = os.path.expanduser("~/.cache/torch/hub/s3prl_s3prl_main") + if os.path.exists(local_cache_path): + self.feature_extract = torch.hub.load(local_cache_path, "wavlm_large", source='local') + else: + # 本地没有缓存，联网下载 + self.feature_extract = torch.hub.load("s3prl/s3prl", "wavlm_large") # self.feature_extract = torch.hub.load('/data/luoyuanZ/eval_1030/f2d5200177fd6a33b278b7b76b454f25cd8ee866d55c122e69fccf6c7467d37d.wavlm_large.pt') # print(self.feature_extract) else: diff --git a/audio_evals/lib/whisper/cv3.py b/audio_evals/lib/whisper/cv3.py index 3e505b6..43ce7d2 100644 --- a/audio_evals/lib/whisper/cv3.py +++ b/audio_evals/lib/whisper/cv3.py @@ -39,7 +39,7 @@ # Process input logger.info(f"Received input: {x}") - result = model.transcribe(x["audio"], language=x.get("language", "english")) + result = model.transcribe(x["audio"], language=x.get("generate_kwargs", {}).get("language", "english")) transcription = result["text"].strip() result = {"text": transcription} retry = 3 diff --git a/audio_evals/lib/whisper/seed_tts_eval.py b/audio_evals/lib/whisper/seed_tts_eval.py index e8625de..84f1439 100644 --- a/audio_evals/lib/whisper/seed_tts_eval.py +++ b/audio_evals/lib/whisper/seed_tts_eval.py @@ -62,7 +62,7 @@ ).input_features input_features = input_features.to(device) forced_decoder_ids = processor.get_decoder_prompt_ids( - language=x.get("language", "english"), task="transcribe" + language=x.get("generate_kwargs", {}).get("language", "english"), task="transcribe" ) with torch.no_grad(): predicted_ids = model.generate( @@ -80,7 +80,7 @@ ).input_features input_features = input_features.to(device) forced_decoder_ids = processor.get_decoder_prompt_ids( - language=x.get("language", "english"), task="transcribe" + language=x.get("generate_kwargs", {}).get("language", "english"), task="transcribe" ) predicted_ids = model.generate( input_features, forced_decoder_ids=forced_decoder_ids diff --git a/audio_evals/main.py b/audio_evals/main.py index 60ec0cb..5994fdc 100644 --- a/audio_evals/main.py +++ b/audio_evals/main.py @@ -7,6 +7,7 @@ from audio_evals.recorder import Recorder from audio_evals.registry import registry from audio_evals.utils import find_latest_jsonl +from audio_evals.models.model_pool import IsolatedModelPool, get_available_gpu_ids def get_args(): @@ -25,6 +26,13 @@ def get_args(): parser.add_argument("--limit", type=int, default=0) parser.add_argument("--rand", type=int, default=0) parser.add_argument("--workers", type=int, default=1) + parser.add_argument( + "--use_model_pool", + action="store_true", + help="Use IsolatedModelPool for multi-GPU parallel inference. " + "Creates `workers` model instances, GPUs are assigned in round-robin. " + "If workers > num_gpus, multiple instances will share the same GPU.", + ) parser.add_argument( "-r", "--resume", @@ -38,6 +46,12 @@ def get_args(): "a valid file", ) parser.add_argument("--inf_file", type=str, default="") + parser.add_argument( + "--two_phase", + action="store_true", + help="Run in two-phase mode: first parallel inference, then sequential evaluation. " + "Useful when evaluator cannot run concurrently.", + ) args = parser.parse_args() return args @@ -105,16 +119,42 @@ def main(): setattr(task_cfg, attr, getattr(args, attr)) logger.info("task cfg:\n{}".format(task_cfg)) + # 创建 predictor：根据 --use_model_pool 决定是否使用模型池 + if args.use_model_pool: + gpu_ids = get_available_gpu_ids() + num_instances = args.workers if args.workers > 1 else len(gpu_ids) + + # 获取模型配置 + model_spec = registry._model.get(task_cfg.model, {}) + model_kwargs = model_spec.get("args", {}) + + logger.info( + f"Using IsolatedModelPool with {num_instances} instances on GPUs {gpu_ids}" + ) + predictor = IsolatedModelPool( + model_factory=lambda **kw: registry.get_model(task_cfg.model, **kw), + model_kwargs=model_kwargs, + gpu_ids=gpu_ids, + num_instances=num_instances, + ) + else: + predictor = registry.get_model(task_cfg.model) + + # evaluator = registry.get_evaluator(task_cfg.evaluator) + + if args.two_phase: + logger.info("Two-phase mode enabled: parallel inference then sequential evaluation") + t = EvalTask( dataset=dataset, prompt=registry.get_prompt(task_cfg.prompt), - predictor=registry.get_model(task_cfg.model), - evaluator=registry.get_evaluator(task_cfg.evaluator), + predictor=predictor, + evaluator=task_cfg.evaluator, post_process=[registry.get_process(item) for item in task_cfg.post_process], agg=registry.get_agg(task_cfg.agg), recorder=Recorder(args.save), ) - res = t.run(args.limit, args.rand, args.workers) + res = t.run(args.limit, args.rand, args.workers, args.two_phase) with open(overall_save, "w") as f: f.write(str(res[0])) with open(args.save, "r") as f: diff --git a/audio_evals/models/TTS/qwen3_tts.py b/audio_evals/models/TTS/qwen3_tts.py new file mode 100644 index 0000000..b655cb4 --- /dev/null +++ b/audio_evals/models/TTS/qwen3_tts.py @@ -0,0 +1,136 @@ +""" +Qwen3-TTS model wrappers for UltraEval-Audio. + +Supports three modes: +- custom_voice: Use preset speakers with optional style instructions +- voice_design: Create voices from natural language descriptions +- voice_clone: Clone voices from reference audio + +Reference: https://github.com/QwenLM/Qwen3-TTS +""" + +import json +import logging +import os +import select +from typing import Dict, Optional + +from audio_evals.base import PromptStruct +from audio_evals.isolate import isolated +from audio_evals.models.model import OfflineModel + +logger = logging.getLogger(__name__) + + +@isolated("audio_evals/lib/Qwen3TTS/main.py", pre_command="uv pip install -U qwen-tts") +class Qwen3TTS(OfflineModel): + """ + Qwen3-TTS unified model. + + Supports three modes via the `mode` parameter: + - custom_voice: Use preset speakers with optional style control + - voice_design: Create voices from natural language descriptions + - voice_clone: Clone voices from reference audio + + Available speakers (for custom_voice mode): + - Vivian: Bright, slightly edgy young female voice (Chinese) + - Serena: Warm, gentle young female voice (Chinese) + - Uncle_Fu: Seasoned male voice with a low, mellow timbre (Chinese) + - Dylan: Youthful Beijing male voice (Chinese, Beijing Dialect) + - Eric: Lively Chengdu male voice (Chinese, Sichuan Dialect) + - Ryan: Dynamic male voice with strong rhythmic drive (English) + - Aiden: Sunny American male voice (English) + - Ono_Anna: Playful Japanese female voice (Japanese) + - Sohee: Warm Korean female voice (Korean) + """ + + def __init__( + self, + path: str, + mode: str = "custom_voice", + dtype: str = "bfloat16", + device: str = "cuda:0", + sample_params: Optional[Dict] = None, + *args, + **kwargs, + ): + """ + Initialize Qwen3-TTS model. + + Args: + path: Model path or HuggingFace model ID + mode: Generation mode - "custom_voice", "voice_design", or "voice_clone" + dtype: Model dtype - "float16", "bfloat16", or "float32" + device: Device to run on, e.g., "cuda:0" + sample_params: Additional sampling parameters + """ + if mode not in ("custom_voice", "voice_design", "voice_clone"): + raise ValueError(f"Invalid mode: {mode}. Must be one of: custom_voice, voice_design, voice_clone") + + if not os.path.exists(path): + path = self._download_model(path) + + self.command_args = { + "path": path, + "mode": mode, + "dtype": dtype, + "device": device, + } + super().__init__(is_chat=True, sample_params=sample_params) + + def _inference(self, prompt: PromptStruct, **kwargs) -> str: + import uuid + + uid = str(uuid.uuid4()) + prefix = f"{uid}->" + + # Merge prompt dict with kwargs + if isinstance(prompt, dict): + prompt.update(kwargs) + else: + prompt = {"text": prompt, **kwargs} + + while True: + _, wlist, _ = select.select([], [self.process.stdin], [], 180) + if not wlist: + raise RuntimeError("Write timeout after 180 seconds") + try: + self.process.stdin.write( + f"{prefix}{json.dumps(prompt, ensure_ascii=False)}\n" + ) + self.process.stdin.flush() + logger.debug("prompt written to Qwen3-TTS stdin") + break + except BlockingIOError: + continue + + while True: + rlist, _, _ = select.select( + [self.process.stdout, self.process.stderr], [], [], 300 + ) + if not rlist: + err_msg = "Read timeout after 300 seconds" + logger.error(err_msg) + raise RuntimeError(err_msg) + + try: + for stream in rlist: + if stream == self.process.stdout: + result = self.process.stdout.readline().strip() + if not result: + continue + if result.startswith(prefix): + self.process.stdin.write(f"{prefix}close\n") + self.process.stdin.flush() + return result[len(prefix):] + elif result.startswith("Error:"): + raise RuntimeError(f"Qwen3-TTS failed: {result}") + else: + logger.info(result) + elif stream == self.process.stderr: + err = self.process.stderr.readline().strip() + if err: + logger.error(f"Process stderr: {err}") + except BlockingIOError as e: + logger.error(f"BlockingIOError occurred: {str(e)}") + continue diff --git a/audio_evals/models/glm4voice.py b/audio_evals/models/glm4voice.py index 97c2835..38b59ba 100644 --- a/audio_evals/models/glm4voice.py +++ b/audio_evals/models/glm4voice.py @@ -1,6 +1,7 @@ import json +import random import tempfile -from typing import Dict +from typing import Dict, List, Union import requests @@ -66,10 +67,10 @@ def save_audio_response(response, output_file, sample_rate, volume=1.0, cut_gree class GLM4Voice(APIModel): def __init__( - self, url: str, sr: int, volume: float = 1.0, cut_greeting: bool = False, sample_params: Dict[str, any] = None + self, url: Union[str, List[str]], sr: int, volume: float = 1.0, cut_greeting: bool = False, sample_params: Dict[str, any] = None ): super().__init__(True, sample_params) - self.url = url + self.url = url if isinstance(url, list) else [url] self.sr = sr self.volume = volume self.cut_greeting = cut_greeting @@ -92,7 +93,9 @@ def _inference(self, prompt: PromptStruct, **kwargs) -> str: 'prompt': '', 'audio': audio_base64 } - response = requests.post(self.url, headers=headers, data=json.dumps(data), stream=True) + # 随机选择一个 URL + url = random.choice(self.url) + response = requests.post(url, headers=headers, data=json.dumps(data), stream=True) with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as f: audio, text = save_audio_response(response, f.name, self.sr, self.volume, self.cut_greeting) return json.dumps({"audio": audio, "text": text}, ensure_ascii=False) diff --git a/audio_evals/models/model_pool.py b/audio_evals/models/model_pool.py new file mode 100644 index 0000000..170be12 --- /dev/null +++ b/audio_evals/models/model_pool.py @@ -0,0 +1,281 @@ +""" +IsolatedModelPool: 管理多个隔离模型实例，支持多 GPU 并发推理 + +适用于使用 @isolated 装饰器的离线模型，通过 CUDA_VISIBLE_DEVICES 实现 GPU 隔离 +""" +import os +import queue +import logging +from typing import Callable, Any, Dict, List, Optional + +logger = logging.getLogger(__name__) + + +def get_available_gpu_ids() -> List[int]: + """ + 获取可用 GPU 的 ID 列表 + + 优先使用 CUDA_VISIBLE_DEVICES 环境变量中指定的 GPU， + 如果未设置则通过 nvidia-smi 获取所有可用 GPU。 + + Returns: + 可用 GPU ID 列表，如 [0, 1, 2, 3] + """ + # 优先检查 CUDA_VISIBLE_DEVICES 环境变量 + cuda_visible = os.environ.get('CUDA_VISIBLE_DEVICES', '') + if cuda_visible: + try: + gpu_ids = [int(x.strip()) for x in cuda_visible.split(',') if x.strip()] + if gpu_ids: + logger.info(f"Using GPUs from CUDA_VISIBLE_DEVICES: {gpu_ids}") + return gpu_ids + except ValueError: + pass + + # 通过 nvidia-smi 获取所有可用 GPU + try: + import subprocess + result = subprocess.run( + ['nvidia-smi', '--query-gpu=index', '--format=csv,noheader'], + capture_output=True, text=True + ) + if result.returncode == 0: + gpu_ids = [int(x.strip()) for x in result.stdout.strip().split('\n') if x.strip()] + if gpu_ids: + logger.info(f"Detected GPUs via nvidia-smi: {gpu_ids}") + return gpu_ids + except Exception as e: + logger.warning(f"Failed to detect GPUs via nvidia-smi: {e}") + + # 默认返回 GPU 0 + logger.warning("No GPUs detected, using default GPU 0") + return [0] + + +class IsolatedModelPool: + """ + 隔离模型池：创建多个模型实例，支持并发推理 + + 适用于使用 @isolated 装饰器的离线模型。 + 每个模型实例通过 gpu_id 参数指定使用的 GPU， + isolated 装饰器会设置 CUDA_VISIBLE_DEVICES 环境变量实现隔离。 + + GPU 分配策略： + - 当 num_instances >= len(gpu_ids) 时：GPU 循环分配，多个实例可能共用同一个 GPU + - 当 num_instances < len(gpu_ids) 时：每个实例分配多个 GPU（均匀分配） + + Example: + ```python + # 场景1: 4 个 GPU，8 个实例 → 每个 GPU 跑 2 个实例 + pool = IsolatedModelPool( + model_factory=lambda **kw: registry.get_model("qwen3-tts", **kw), + model_kwargs={"path": "/path/to/model"}, + gpu_ids=[0, 1, 2, 3], + num_instances=8, + ) + + # 场景2: 8 个 GPU，2 个实例 → 每个实例分配 4 个 GPU + # 实例 0 使用 GPU 0,1,2,3；实例 1 使用 GPU 4,5,6,7 + pool = IsolatedModelPool( + model_factory=lambda **kw: registry.get_model("qwen3-tts", **kw), + model_kwargs={"path": "/path/to/model"}, + gpu_ids=[0, 1, 2, 3, 4, 5, 6, 7], + num_instances=2, + ) + + # 并发推理时，会自动从池中获取空闲实例 + result = pool.inference(prompt) + ``` + """ + + def __init__( + self, + model_factory: Callable[..., Any], + model_kwargs: Dict[str, Any], + gpu_ids: Optional[List[int]] = None, + num_instances: Optional[int] = None, + ): + """ + Args: + model_factory: 模型创建函数，接受 **kwargs 参数 + model_kwargs: 传给 model_factory 的基础参数 + gpu_ids: 可用 GPU ID 列表，如 [0, 1, 2, 3]。如果为 None，自动检测 + num_instances: 模型实例数量。如果为 None，默认等于 GPU 数量 + """ + if gpu_ids is None: + gpu_ids = get_available_gpu_ids() + + if not gpu_ids: + raise ValueError("gpu_ids cannot be empty") + + if num_instances is None: + num_instances = len(gpu_ids) + + if num_instances < 1: + raise ValueError(f"num_instances must be >= 1, got {num_instances}") + + self.gpu_ids = gpu_ids + self.num_instances = num_instances + self._pool: queue.Queue = queue.Queue() + self._models = [] + + logger.info( + f"Creating IsolatedModelPool with {num_instances} instances on GPUs {gpu_ids}" + ) + + # 创建多个模型实例，分配 GPU + # 计算每个实例的 GPU 分配 + gpu_assignments = self._compute_gpu_assignments(gpu_ids, num_instances) + + for i in range(num_instances): + assigned_gpus = gpu_assignments[i] + # gpu_id 可以是单个 int 或逗号分隔的字符串（多 GPU） + if len(assigned_gpus) == 1: + gpu_id = assigned_gpus[0] + else: + gpu_id = ','.join(map(str, assigned_gpus)) + + kwargs = model_kwargs.copy() + kwargs['gpu_id'] = gpu_id + logger.info(f"Creating model instance {i} on GPU(s) {gpu_id}") + try: + model = model_factory(**kwargs) + self._models.append(model) + self._pool.put(model) + logger.info(f"Model instance {i} on GPU(s) {gpu_id} created successfully") + except Exception as e: + logger.error(f"Failed to create model instance {i} on GPU(s) {gpu_id}: {e}") + # 清理已创建的实例 + self._cleanup() + raise + + logger.info( + f"IsolatedModelPool initialized: {len(self._models)} instances on {len(gpu_ids)} GPUs" + ) + + @staticmethod + def _compute_gpu_assignments(gpu_ids: List[int], num_instances: int) -> List[List[int]]: + """ + 计算每个实例的 GPU 分配 + + - 当 num_instances >= len(gpu_ids) 时：GPU 循环分配，多个实例可能共用同一个 GPU + - 当 num_instances < len(gpu_ids) 时：每个实例分配多个 GPU + + Args: + gpu_ids: 可用 GPU ID 列表 + num_instances: 实例数量 + + Returns: + 每个实例分配的 GPU ID 列表，如 [[0, 1], [2, 3]] 表示实例 0 用 GPU 0,1，实例 1 用 GPU 2,3 + """ + n_gpus = len(gpu_ids) + + if num_instances >= n_gpus: + # GPU 数量不足，每个实例分配一个 GPU（循环使用） + return [[gpu_ids[i % n_gpus]] for i in range(num_instances)] + else: + # GPU 数量充足，每个实例分配多个 GPU + # 计算基础分配数和余数 + base_count = n_gpus // num_instances + remainder = n_gpus % num_instances + + assignments = [] + gpu_idx = 0 + for i in range(num_instances): + # 前 remainder 个实例多分配 1 个 GPU + count = base_count + (1 if i < remainder else 0) + assigned = gpu_ids[gpu_idx:gpu_idx + count] + assignments.append(assigned) + gpu_idx += count + + return assignments + + def _acquire(self, timeout: float = None): + """ + 获取一个空闲的模型实例 + + Args: + timeout: 超时时间（秒），None 表示无限等待 + + Returns: + 空闲的模型实例 + + Raises: + queue.Empty: 超时未获取到实例 + """ + return self._pool.get(timeout=timeout) + + def _release(self, model): + """归还模型实例到池中""" + self._pool.put(model) + + def inference(self, prompt, **kwargs) -> str: + """ + 从池中获取模型实例进行推理，完成后自动归还 + + Args: + prompt: 输入 prompt + **kwargs: 其他推理参数 + + Returns: + 推理结果 + """ + model = self._acquire() + try: + return model.inference(prompt, **kwargs) + finally: + self._release(model) + + def _cleanup(self): + """清理所有模型实例""" + for model in self._models: + try: + # 调用模型自身的释放方法（如果有） + if hasattr(model, 'release') and callable(model.release): + model.release() + elif hasattr(model, 'unload') and callable(model.unload): + model.unload() + + # 处理子进程 + if hasattr(model, 'process') and model.process is not None and model.process.poll() is None: + model.process.terminate() + try: + model.process.wait(timeout=5) + except Exception: + model.process.kill() + except Exception as e: + logger.warning(f"Error cleaning up model: {e}") + self._models.clear() + + # 清空队列 + while not self._pool.empty(): + try: + self._pool.get_nowait() + except queue.Empty: + break + + def release(self): + """ + 释放所有模型实例并清理 GPU 显存 + + 调用此方法后，模型池将不可用，需要重新创建。 + """ + logger.info("Releasing IsolatedModelPool...") + + # 清理所有模型实例 + self._cleanup() + self.num_instances = 0 + logger.info("IsolatedModelPool released.") + + def __del__(self): + """析构时清理所有模型实例""" + self._cleanup() + + def __len__(self): + """返回池中的实例数量""" + return self.num_instances + + @property + def available_count(self) -> int: + """返回当前空闲的实例数量""" + return self._pool.qsize() diff --git a/audio_evals/models/qwen2_5.py b/audio_evals/models/qwen2_5.py index 25923a3..f3cf6fd 100644 --- a/audio_evals/models/qwen2_5.py +++ b/audio_evals/models/qwen2_5.py @@ -54,9 +54,7 @@ def _inference(self, prompt: PromptStruct, **kwargs): "content": [ { "type": "text", - "text": "You are Qwen, a virtual human developed by the Qwen Team, " - "Alibaba Group, capable of perceiving auditory and visual " - "inputs, as well as generating text and speech", + "text": "You are Qwen, a virtual human developed by the Qwen Team, Alibaba Group, capable of perceiving auditory and visual inputs, as well as generating text and speech." } ], } diff --git a/audio_evals/models/step_audio_r1.py b/audio_evals/models/step_audio_r1.py index 3a6cd77..a876d61 100644 --- a/audio_evals/models/step_audio_r1.py +++ b/audio_evals/models/step_audio_r1.py @@ -13,12 +13,10 @@ import os import select import sys +import time from typing import Dict, Any, List -# Add the StepAudio library to path -sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../lib/StepAudio")) - -from stepaudior1vllm import StepAudioR1 as StepAudioR1Client +from audio_evals.lib.StepAudio.stepaudior1vllm import StepAudioR1 as StepAudioR1Client from audio_evals.base import PromptStruct from audio_evals.isolate import isolated @@ -31,8 +29,8 @@ "audio_evals/lib/StepAudio/serve.py", pre_command="mkdir -p ./third_party && " "([ ! -d './third_party/vllm-step-audio' ] && " - "git clone -b step-audio-2-mini https://github.com/stepfun-ai/vllm.git ./third_party/vllm-step-audio && cd ../../) || true && " - "(python -c 'import vllm' 2>/dev/null || VLLM_USE_PRECOMPILED=1 uv pip install -e ./third_party/vllm-step-audio)", + "git clone https://github.com/stepfun-ai/vllm.git ./third_party/vllm-step-audio && cd ../../) || true && " + "(python -c 'import vllm' 2>/dev/null || VLLM_USE_PRECOMPILED=1 uv pip install -e ./third_party/vllm-step-audio && cd ./third_party/vllm-step-audio && git checkout step-audio2-mini origin/step-audio2-mini && cd ../../)", ) class StepAudioR1(OfflineModel): """ @@ -45,7 +43,7 @@ class StepAudioR1(OfflineModel): it back via stdout. Requirements: - - Customized vLLM from https://github.com/stepfun-ai/vllm (step-audio-2-mini branch) + - Customized vLLM from https://github.com/stepfun-ai/vllm (step-audio2-mini branch) - Step-Audio-R1.1 model weights """ @@ -207,20 +205,36 @@ def _inference(self, prompt: PromptStruct, **kwargs) -> str: full_text = "" audio_tokens = [] + timeout_seconds = 180 + # Requests-level timeout: (connect_timeout, read_timeout) + # - connect_timeout prevents "connection miss" hanging forever + # - read_timeout prevents "server never returns any bytes" hanging forever + request_timeout = (10, timeout_seconds) + start_time = time.time() + try: - for _, text, audio in self.client.stream(messages, **api_params): + for _, text, audio in self.client.stream( + messages, request_timeout=request_timeout, **api_params + ): + # Total wall-clock timeout (even if server keeps streaming slowly) + elapsed = time.time() - start_time + if elapsed > timeout_seconds: + raise TimeoutError( + f"StepAudioR1 stream exceeded {timeout_seconds}s (elapsed: {elapsed:.2f}s)" + ) + if text: full_text += text if audio: audio_tokens.extend(audio) + except TimeoutError as e: + logger.error(f"Timeout during API call: {e}") + raise except Exception as e: logger.error(f"Error during API call: {e}") raise text_result = self._extract_response(full_text) if full_text else "" - logger.info( - f"Extracted response: {text_result[:200] if text_result else 'None'}..." - ) if not self.speech: return text_result diff --git a/audio_evals/models/whisper.py b/audio_evals/models/whisper.py index aa5ac2a..1066168 100644 --- a/audio_evals/models/whisper.py +++ b/audio_evals/models/whisper.py @@ -51,7 +51,7 @@ def _inference(self, prompt: PromptStruct, **kwargs) -> str: while True: _, wlist, _ = select.select([], [self.process.stdin], [], 60) if wlist: - prompt["kwargs"] = kwargs + prompt['kwargs'] = kwargs self.process.stdin.write(f"{prefix}{json.dumps(prompt)}\n") self.process.stdin.flush() print("already write in") @@ -122,7 +122,7 @@ def _inference(self, prompt: PromptStruct, **kwargs) -> str: while True: _, wlist, _ = select.select([], [self.process.stdin], [], 60) if wlist: - prompt["kwargs"] = kwargs + prompt.update(kwargs) self.process.stdin.write(f"{prefix}{json.dumps(prompt)}\n") self.process.stdin.flush() print("already write in") @@ -189,7 +189,7 @@ def _inference(self, prompt: PromptStruct, **kwargs) -> str: while True: _, wlist, _ = select.select([], [self.process.stdin], [], 60) if wlist: - prompt["kwargs"] = kwargs + prompt.update(kwargs) self.process.stdin.write(f"{prefix}{json.dumps(prompt)}\n") self.process.stdin.flush() print("already write in") diff --git a/registry/dataset/minimax.yaml b/registry/dataset/minimax.yaml new file mode 100644 index 0000000..83ec009 --- /dev/null +++ b/registry/dataset/minimax.yaml @@ -0,0 +1,192 @@ +minimax_tts_arabic: + class: audio_evals.dataset.minimax_tts.MiniMaxTTSDataset + args: + name: MiniMaxAI/TTS-Multilingual-Test-Set + default_task: minimax_tts_ar + ref_col: WavPath + language: "arabic" + +minimax_tts_cantonese: + class: audio_evals.dataset.minimax_tts.MiniMaxTTSDataset + args: + name: MiniMaxAI/TTS-Multilingual-Test-Set + default_task: minimax_tts_yue + ref_col: WavPath + language: "cantonese" + +minimax_tts_chinese: + class: audio_evals.dataset.minimax_tts.MiniMaxTTSDataset + args: + name: MiniMaxAI/TTS-Multilingual-Test-Set + default_task: minimax_tts_zh + ref_col: WavPath + language: "chinese" + +minimax_tts_czech: + class: audio_evals.dataset.minimax_tts.MiniMaxTTSDataset + args: + name: MiniMaxAI/TTS-Multilingual-Test-Set + default_task: minimax_tts_cs + ref_col: WavPath + language: "czech" + +minimax_tts_dutch: + class: audio_evals.dataset.minimax_tts.MiniMaxTTSDataset + args: + name: MiniMaxAI/TTS-Multilingual-Test-Set + default_task: minimax_tts_nl + ref_col: WavPath + language: "dutch" + +minimax_tts_english: + class: audio_evals.dataset.minimax_tts.MiniMaxTTSDataset + args: + name: MiniMaxAI/TTS-Multilingual-Test-Set + default_task: minimax_tts_en + ref_col: WavPath + language: "english" + +minimax_tts_finnish: + class: audio_evals.dataset.minimax_tts.MiniMaxTTSDataset + args: + name: MiniMaxAI/TTS-Multilingual-Test-Set + default_task: minimax_tts_fi + ref_col: WavPath + language: "finnish" + +minimax_tts_french: + class: audio_evals.dataset.minimax_tts.MiniMaxTTSDataset + args: + name: MiniMaxAI/TTS-Multilingual-Test-Set + default_task: minimax_tts_fr + ref_col: WavPath + language: "french" + +minimax_tts_german: + class: audio_evals.dataset.minimax_tts.MiniMaxTTSDataset + args: + name: MiniMaxAI/TTS-Multilingual-Test-Set + default_task: minimax_tts_de + ref_col: WavPath + language: "german" + +minimax_tts_greek: + class: audio_evals.dataset.minimax_tts.MiniMaxTTSDataset + args: + name: MiniMaxAI/TTS-Multilingual-Test-Set + default_task: minimax_tts_el + ref_col: WavPath + language: "greek" + +minimax_tts_hindi: + class: audio_evals.dataset.minimax_tts.MiniMaxTTSDataset + args: + name: MiniMaxAI/TTS-Multilingual-Test-Set + default_task: minimax_tts_hi + ref_col: WavPath + language: "hindi" + +minimax_tts_indonesian: + class: audio_evals.dataset.minimax_tts.MiniMaxTTSDataset + args: + name: MiniMaxAI/TTS-Multilingual-Test-Set + default_task: minimax_tts_id + ref_col: WavPath + language: "indonesian" + +minimax_tts_italian: + class: audio_evals.dataset.minimax_tts.MiniMaxTTSDataset + args: + name: MiniMaxAI/TTS-Multilingual-Test-Set + default_task: minimax_tts_it + ref_col: WavPath + language: "italian" + +minimax_tts_japanese: + class: audio_evals.dataset.minimax_tts.MiniMaxTTSDataset + args: + name: MiniMaxAI/TTS-Multilingual-Test-Set + default_task: minimax_tts_ja + ref_col: WavPath + language: "japanese" + +minimax_tts_korean: + class: audio_evals.dataset.minimax_tts.MiniMaxTTSDataset + args: + name: MiniMaxAI/TTS-Multilingual-Test-Set + default_task: minimax_tts_ko + ref_col: WavPath + language: "korean" + +minimax_tts_polish: + class: audio_evals.dataset.minimax_tts.MiniMaxTTSDataset + args: + name: MiniMaxAI/TTS-Multilingual-Test-Set + default_task: minimax_tts_pl + ref_col: WavPath + language: "polish" + +minimax_tts_portuguese: + class: audio_evals.dataset.minimax_tts.MiniMaxTTSDataset + args: + name: MiniMaxAI/TTS-Multilingual-Test-Set + default_task: minimax_tts_pt + ref_col: WavPath + language: "portuguese" + +minimax_tts_romanian: + class: audio_evals.dataset.minimax_tts.MiniMaxTTSDataset + args: + name: MiniMaxAI/TTS-Multilingual-Test-Set + default_task: minimax_tts_ro + ref_col: WavPath + language: "romanian" + +minimax_tts_russian: + class: audio_evals.dataset.minimax_tts.MiniMaxTTSDataset + args: + name: MiniMaxAI/TTS-Multilingual-Test-Set + default_task: minimax_tts_ru + ref_col: WavPath + language: "russian" + +minimax_tts_spanish: + class: audio_evals.dataset.minimax_tts.MiniMaxTTSDataset + args: + name: MiniMaxAI/TTS-Multilingual-Test-Set + default_task: minimax_tts_es + ref_col: WavPath + language: "spanish" + +minimax_tts_thai: + class: audio_evals.dataset.minimax_tts.MiniMaxTTSDataset + args: + name: MiniMaxAI/TTS-Multilingual-Test-Set + default_task: minimax_tts_th + ref_col: WavPath + language: "thai" + +minimax_tts_turkish: + class: audio_evals.dataset.minimax_tts.MiniMaxTTSDataset + args: + name: MiniMaxAI/TTS-Multilingual-Test-Set + default_task: minimax_tts_tr + ref_col: WavPath + language: "turkish" + +minimax_tts_ukrainian: + class: audio_evals.dataset.minimax_tts.MiniMaxTTSDataset + args: + name: MiniMaxAI/TTS-Multilingual-Test-Set + default_task: minimax_tts_uk + ref_col: WavPath + language: "ukrainian" + +minimax_tts_vietnamese: + class: audio_evals.dataset.minimax_tts.MiniMaxTTSDataset + args: + name: MiniMaxAI/TTS-Multilingual-Test-Set + default_task: minimax_tts_vi + ref_col: WavPath + language: "vietnamese" + diff --git a/registry/dataset/speech-cmmlu.yaml b/registry/dataset/speech-cmmlu.yaml index 564e40c..41e5c25 100644 --- a/registry/dataset/speech-cmmlu.yaml +++ b/registry/dataset/speech-cmmlu.yaml @@ -5,3 +5,11 @@ speech-cmmlu: name: TwinkStart/speech-CMMLU split: train ref_col: Answer + +speech-cmmlu-s2t: + class: audio_evals.dataset.huggingface.Huggingface + args: + default_task: speech-choice-aqa-zh + name: TwinkStart/speech-CMMLU + split: train + ref_col: Answer diff --git a/registry/eval_task/aqa.yaml b/registry/eval_task/aqa.yaml index 4db683a..2d1155f 100644 --- a/registry/eval_task/aqa.yaml +++ b/registry/eval_task/aqa.yaml @@ -53,6 +53,6 @@ s2t-choice-aqa: dataset: clotho-aqa prompt: direct-aqa model: qwen-audio-chat - post_process: ['first_option'] + post_process: ['extract_text', 'first_option'] evaluator: em agg: acc diff --git a/registry/eval_task/minimax.yaml b/registry/eval_task/minimax.yaml new file mode 100644 index 0000000..06f388d --- /dev/null +++ b/registry/eval_task/minimax.yaml @@ -0,0 +1,240 @@ +minimax_tts_ar: + class: audio_evals.base.EvalTaskCfg + args: + dataset: minimax_tts_arabic + prompt: voice-clone + model: qwen-audio-chat + post_process: ['extract_audio'] + evaluator: minimax-zero-shot-ar + agg: mean + +minimax_tts_yue: + class: audio_evals.base.EvalTaskCfg + args: + dataset: minimax_tts_cantonese + prompt: voice-clone + model: qwen-audio-chat + post_process: ['extract_audio'] + evaluator: minimax-zero-shot-yue + agg: mean + +minimax_tts_zh: + class: audio_evals.base.EvalTaskCfg + args: + dataset: minimax_tts_chinese + prompt: voice-clone + model: qwen-audio-chat + post_process: ['extract_audio'] + evaluator: minimax-zero-shot-zh + agg: mean + +minimax_tts_cs: + class: audio_evals.base.EvalTaskCfg + args: + dataset: minimax_tts_czech + prompt: voice-clone + model: qwen-audio-chat + post_process: ['extract_audio'] + evaluator: minimax-zero-shot-cs + agg: mean + +minimax_tts_nl: + class: audio_evals.base.EvalTaskCfg + args: + dataset: minimax_tts_dutch + prompt: voice-clone + model: qwen-audio-chat + post_process: ['extract_audio'] + evaluator: minimax-zero-shot-nl + agg: mean + +minimax_tts_en: + class: audio_evals.base.EvalTaskCfg + args: + dataset: minimax_tts_english + prompt: voice-clone + model: qwen-audio-chat + post_process: ['extract_audio'] + evaluator: minimax-zero-shot-en + agg: mean + +minimax_tts_fi: + class: audio_evals.base.EvalTaskCfg + args: + dataset: minimax_tts_finnish + prompt: voice-clone + model: qwen-audio-chat + post_process: ['extract_audio'] + evaluator: minimax-zero-shot-fi + agg: mean + +minimax_tts_fr: + class: audio_evals.base.EvalTaskCfg + args: + dataset: minimax_tts_french + prompt: voice-clone + model: qwen-audio-chat + post_process: ['extract_audio'] + evaluator: minimax-zero-shot-fr + agg: mean + +minimax_tts_de: + class: audio_evals.base.EvalTaskCfg + args: + dataset: minimax_tts_german + prompt: voice-clone + model: qwen-audio-chat + post_process: ['extract_audio'] + evaluator: minimax-zero-shot-de + agg: mean + +minimax_tts_el: + class: audio_evals.base.EvalTaskCfg + args: + dataset: minimax_tts_greek + prompt: voice-clone + model: qwen-audio-chat + post_process: ['extract_audio'] + evaluator: minimax-zero-shot-el + agg: mean + +minimax_tts_hi: + class: audio_evals.base.EvalTaskCfg + args: + dataset: minimax_tts_hindi + prompt: voice-clone + model: qwen-audio-chat + post_process: ['extract_audio'] + evaluator: minimax-zero-shot-hi + agg: mean + +minimax_tts_id: + class: audio_evals.base.EvalTaskCfg + args: + dataset: minimax_tts_indonesian + prompt: voice-clone + model: qwen-audio-chat + post_process: ['extract_audio'] + evaluator: minimax-zero-shot-id + agg: mean + +minimax_tts_it: + class: audio_evals.base.EvalTaskCfg + args: + dataset: minimax_tts_italian + prompt: voice-clone + model: qwen-audio-chat + post_process: ['extract_audio'] + evaluator: minimax-zero-shot-it + agg: mean + +minimax_tts_ja: + class: audio_evals.base.EvalTaskCfg + args: + dataset: minimax_tts_japanese + prompt: voice-clone + model: qwen-audio-chat + post_process: ['extract_audio'] + evaluator: minimax-zero-shot-ja + agg: mean + +minimax_tts_ko: + class: audio_evals.base.EvalTaskCfg + args: + dataset: minimax_tts_korean + prompt: voice-clone + model: qwen-audio-chat + post_process: ['extract_audio'] + evaluator: minimax-zero-shot-ko + agg: mean + +minimax_tts_pl: + class: audio_evals.base.EvalTaskCfg + args: + dataset: minimax_tts_polish + prompt: voice-clone + model: qwen-audio-chat + post_process: ['extract_audio'] + evaluator: minimax-zero-shot-pl + agg: mean + +minimax_tts_pt: + class: audio_evals.base.EvalTaskCfg + args: + dataset: minimax_tts_portuguese + prompt: voice-clone + model: qwen-audio-chat + post_process: ['extract_audio'] + evaluator: minimax-zero-shot-pt + agg: mean + +minimax_tts_ro: + class: audio_evals.base.EvalTaskCfg + args: + dataset: minimax_tts_romanian + prompt: voice-clone + model: qwen-audio-chat + post_process: ['extract_audio'] + evaluator: minimax-zero-shot-ro + agg: mean + +minimax_tts_ru: + class: audio_evals.base.EvalTaskCfg + args: + dataset: minimax_tts_russian + prompt: voice-clone + model: qwen-audio-chat + post_process: ['extract_audio'] + evaluator: minimax-zero-shot-ru + agg: mean + +minimax_tts_es: + class: audio_evals.base.EvalTaskCfg + args: + dataset: minimax_tts_spanish + prompt: voice-clone + model: qwen-audio-chat + post_process: ['extract_audio'] + evaluator: minimax-zero-shot-es + agg: mean + +minimax_tts_th: + class: audio_evals.base.EvalTaskCfg + args: + dataset: minimax_tts_thai + prompt: voice-clone + model: qwen-audio-chat + post_process: ['extract_audio'] + evaluator: minimax-zero-shot-th + agg: mean + +minimax_tts_tr: + class: audio_evals.base.EvalTaskCfg + args: + dataset: minimax_tts_turkish + prompt: voice-clone + model: qwen-audio-chat + post_process: ['extract_audio'] + evaluator: minimax-zero-shot-tr + agg: mean + +minimax_tts_uk: + class: audio_evals.base.EvalTaskCfg + args: + dataset: minimax_tts_ukrainian + prompt: voice-clone + model: qwen-audio-chat + post_process: ['extract_audio'] + evaluator: minimax-zero-shot-uk + agg: mean + +minimax_tts_vi: + class: audio_evals.base.EvalTaskCfg + args: + dataset: minimax_tts_vietnamese + prompt: voice-clone + model: qwen-audio-chat + post_process: ['extract_audio'] + evaluator: minimax-zero-shot-vi + agg: mean + diff --git a/registry/eval_task/mmsu.yaml b/registry/eval_task/mmsu.yaml new file mode 100644 index 0000000..a85b464 --- /dev/null +++ b/registry/eval_task/mmsu.yaml @@ -0,0 +1,8 @@ +mmsu: + class: audio_evals.base.EvalTaskCfg + args: + dataset: mmsu + prompt: mmsu + model: qwen-audio-chat + evaluator: mmsu-choices + agg: mean \ No newline at end of file diff --git a/registry/evaluator/adv_exist_match.yaml b/registry/evaluator/adv_exist_match.yaml new file mode 100644 index 0000000..70b3c68 --- /dev/null +++ b/registry/evaluator/adv_exist_match.yaml @@ -0,0 +1,3 @@ +advance_exist_match: + class: audio_evals.evaluator.advance_exist_match.AdvanceExistMatchEvaluator + args: {} diff --git a/registry/evaluator/kimi.yaml b/registry/evaluator/kimi.yaml new file mode 100644 index 0000000..85a1be8 --- /dev/null +++ b/registry/evaluator/kimi.yaml @@ -0,0 +1,18 @@ +kimi-mmau-choices: + class: audio_evals.evaluator.choices.ChoicesEval + args: + choices_columns: choices + ignore_parse_error: true + +kimi-vocalsound-choices: + class: audio_evals.evaluator.choices.ChoicesEval + args: + choices_columns: + - Laughter + - Sigh + - Cough + - Throat clearing + - Sneeze + - Sniff + ignore_parse_error: true + constant: true diff --git a/registry/evaluator/long_tts_eval.yaml b/registry/evaluator/long_tts_eval.yaml index 4ec4632..8ddecf4 100644 --- a/registry/evaluator/long_tts_eval.yaml +++ b/registry/evaluator/long_tts_eval.yaml @@ -11,3 +11,10 @@ long-tts-eval-asr-wer-zh: model_name: long_tts_speech_paraformer-speech_seaco_paraformer_large_asr_nat-zh-cn-16k-common-vocab8404-pytorch prompt_name: simple-asr lang: zh + +fix-long-tts-eval-asr-wer-en: + class: audio_evals.evaluator.long_tts_eval_asr_wer.LongTTSEvalASRWER + args: + model_name: whisper + prompt_name: whisper-asr-en + lang: en diff --git a/registry/evaluator/minimax.yaml b/registry/evaluator/minimax.yaml new file mode 100644 index 0000000..8f5c9fe --- /dev/null +++ b/registry/evaluator/minimax.yaml @@ -0,0 +1,336 @@ +minimax-eval-asr-wer-ar: + class: audio_evals.evaluator.seed_tts_eval_asr_wer.SeedTTSEvalASRWER + args: + model_name: seed-tts-whisper + prompt_name: whisper-asr-ar + lang: ar + +minimax-eval-asr-wer-yue: + class: audio_evals.evaluator.seed_tts_eval_asr_wer.SeedTTSEvalASRWER + args: + model_name: seed-tts-whisper + prompt_name: whisper-asr-yue + lang: yue + +minimax-eval-asr-wer-zh: + class: audio_evals.evaluator.seed_tts_eval_asr_wer.SeedTTSEvalASRWER + args: + model_name: speech_paraformer-speech_seaco_paraformer_large_asr_nat-zh-cn-16k-common-vocab8404-pytorch + prompt_name: simple-asr + lang: zh + +minimax-eval-asr-wer-cs: + class: audio_evals.evaluator.seed_tts_eval_asr_wer.SeedTTSEvalASRWER + args: + model_name: seed-tts-whisper + prompt_name: whisper-asr-cs + lang: cs + +minimax-eval-asr-wer-nl: + class: audio_evals.evaluator.seed_tts_eval_asr_wer.SeedTTSEvalASRWER + args: + model_name: seed-tts-whisper + prompt_name: whisper-asr-nl + lang: nl + +minimax-eval-asr-wer-en: + class: audio_evals.evaluator.seed_tts_eval_asr_wer.SeedTTSEvalASRWER + args: + model_name: seed-tts-whisper + prompt_name: whisper-asr-en + lang: en + +minimax-eval-asr-wer-fi: + class: audio_evals.evaluator.seed_tts_eval_asr_wer.SeedTTSEvalASRWER + args: + model_name: seed-tts-whisper + prompt_name: whisper-asr-fi + lang: fi + +minimax-eval-asr-wer-fr: + class: audio_evals.evaluator.seed_tts_eval_asr_wer.SeedTTSEvalASRWER + args: + model_name: seed-tts-whisper + prompt_name: whisper-asr-fr + lang: fr + +minimax-eval-asr-wer-de: + class: audio_evals.evaluator.seed_tts_eval_asr_wer.SeedTTSEvalASRWER + args: + model_name: seed-tts-whisper + prompt_name: whisper-asr-de + lang: de + +minimax-eval-asr-wer-el: + class: audio_evals.evaluator.seed_tts_eval_asr_wer.SeedTTSEvalASRWER + args: + model_name: seed-tts-whisper + prompt_name: whisper-asr-el + lang: el + +minimax-eval-asr-wer-hi: + class: audio_evals.evaluator.seed_tts_eval_asr_wer.SeedTTSEvalASRWER + args: + model_name: seed-tts-whisper + prompt_name: whisper-asr-hi + lang: hi + +minimax-eval-asr-wer-id: + class: audio_evals.evaluator.seed_tts_eval_asr_wer.SeedTTSEvalASRWER + args: + model_name: seed-tts-whisper + prompt_name: whisper-asr-id + lang: id + +minimax-eval-asr-wer-it: + class: audio_evals.evaluator.seed_tts_eval_asr_wer.SeedTTSEvalASRWER + args: + model_name: seed-tts-whisper + prompt_name: whisper-asr-it + lang: it + +minimax-eval-asr-wer-ja: + class: audio_evals.evaluator.seed_tts_eval_asr_wer.SeedTTSEvalASRWER + args: + model_name: seed-tts-whisper + prompt_name: whisper-asr-ja + lang: ja + +minimax-eval-asr-wer-ko: + class: audio_evals.evaluator.seed_tts_eval_asr_wer.SeedTTSEvalASRWER + args: + model_name: seed-tts-whisper + prompt_name: whisper-asr-ko + lang: ko + +minimax-eval-asr-wer-pl: + class: audio_evals.evaluator.seed_tts_eval_asr_wer.SeedTTSEvalASRWER + args: + model_name: seed-tts-whisper + prompt_name: whisper-asr-pl + lang: pl + +minimax-eval-asr-wer-pt: + class: audio_evals.evaluator.seed_tts_eval_asr_wer.SeedTTSEvalASRWER + args: + model_name: seed-tts-whisper + prompt_name: whisper-asr-pt + lang: pt + +minimax-eval-asr-wer-ro: + class: audio_evals.evaluator.seed_tts_eval_asr_wer.SeedTTSEvalASRWER + args: + model_name: seed-tts-whisper + prompt_name: whisper-asr-ro + lang: ro + +minimax-eval-asr-wer-ru: + class: audio_evals.evaluator.seed_tts_eval_asr_wer.SeedTTSEvalASRWER + args: + model_name: seed-tts-whisper + prompt_name: whisper-asr-ru + lang: ru + +minimax-eval-asr-wer-es: + class: audio_evals.evaluator.seed_tts_eval_asr_wer.SeedTTSEvalASRWER + args: + model_name: seed-tts-whisper + prompt_name: whisper-asr-es + lang: es + +minimax-eval-asr-wer-th: + class: audio_evals.evaluator.seed_tts_eval_asr_wer.SeedTTSEvalASRWER + args: + model_name: seed-tts-whisper + prompt_name: whisper-asr-th + lang: th + +minimax-eval-asr-wer-tr: + class: audio_evals.evaluator.seed_tts_eval_asr_wer.SeedTTSEvalASRWER + args: + model_name: seed-tts-whisper + prompt_name: whisper-asr-tr + lang: tr + +minimax-eval-asr-wer-uk: + class: audio_evals.evaluator.seed_tts_eval_asr_wer.SeedTTSEvalASRWER + args: + model_name: seed-tts-whisper + prompt_name: whisper-asr-uk + lang: uk + +minimax-eval-asr-wer-vi: + class: audio_evals.evaluator.seed_tts_eval_asr_wer.SeedTTSEvalASRWER + args: + model_name: seed-tts-whisper + prompt_name: whisper-asr-vi + lang: vi + +minimax-zero-shot-ar: + class: audio_evals.evaluator.ensemble.Ensemble + args: + components: + - minimax-eval-asr-wer-ar + - simo + +minimax-zero-shot-yue: + class: audio_evals.evaluator.ensemble.Ensemble + args: + components: + - minimax-eval-asr-wer-yue + - simo + +minimax-zero-shot-zh: + class: audio_evals.evaluator.ensemble.Ensemble + args: + components: + - minimax-eval-asr-wer-zh + - simo + +minimax-zero-shot-cs: + class: audio_evals.evaluator.ensemble.Ensemble + args: + components: + - minimax-eval-asr-wer-cs + - simo + +minimax-zero-shot-nl: + class: audio_evals.evaluator.ensemble.Ensemble + args: + components: + - minimax-eval-asr-wer-nl + - simo + +minimax-zero-shot-en: + class: audio_evals.evaluator.ensemble.Ensemble + args: + components: + - minimax-eval-asr-wer-en + - simo + +minimax-zero-shot-fi: + class: audio_evals.evaluator.ensemble.Ensemble + args: + components: + - minimax-eval-asr-wer-fi + - simo + +minimax-zero-shot-fr: + class: audio_evals.evaluator.ensemble.Ensemble + args: + components: + - minimax-eval-asr-wer-fr + - simo + +minimax-zero-shot-de: + class: audio_evals.evaluator.ensemble.Ensemble + args: + components: + - minimax-eval-asr-wer-de + - simo + +minimax-zero-shot-el: + class: audio_evals.evaluator.ensemble.Ensemble + args: + components: + - minimax-eval-asr-wer-el + - simo + +minimax-zero-shot-hi: + class: audio_evals.evaluator.ensemble.Ensemble + args: + components: + - minimax-eval-asr-wer-hi + - simo + +minimax-zero-shot-id: + class: audio_evals.evaluator.ensemble.Ensemble + args: + components: + - minimax-eval-asr-wer-id + - simo + +minimax-zero-shot-it: + class: audio_evals.evaluator.ensemble.Ensemble + args: + components: + - minimax-eval-asr-wer-it + - simo + +minimax-zero-shot-ja: + class: audio_evals.evaluator.ensemble.Ensemble + args: + components: + - minimax-eval-asr-wer-ja + - simo + +minimax-zero-shot-ko: + class: audio_evals.evaluator.ensemble.Ensemble + args: + components: + - minimax-eval-asr-wer-ko + - simo + +minimax-zero-shot-pl: + class: audio_evals.evaluator.ensemble.Ensemble + args: + components: + - minimax-eval-asr-wer-pl + - simo + +minimax-zero-shot-pt: + class: audio_evals.evaluator.ensemble.Ensemble + args: + components: + - minimax-eval-asr-wer-pt + - simo + +minimax-zero-shot-ro: + class: audio_evals.evaluator.ensemble.Ensemble + args: + components: + - minimax-eval-asr-wer-ro + - simo + +minimax-zero-shot-ru: + class: audio_evals.evaluator.ensemble.Ensemble + args: + components: + - minimax-eval-asr-wer-ru + - simo + +minimax-zero-shot-es: + class: audio_evals.evaluator.ensemble.Ensemble + args: + components: + - minimax-eval-asr-wer-es + - simo + +minimax-zero-shot-th: + class: audio_evals.evaluator.ensemble.Ensemble + args: + components: + - minimax-eval-asr-wer-th + - simo + +minimax-zero-shot-tr: + class: audio_evals.evaluator.ensemble.Ensemble + args: + components: + - minimax-eval-asr-wer-tr + - simo + +minimax-zero-shot-uk: + class: audio_evals.evaluator.ensemble.Ensemble + args: + components: + - minimax-eval-asr-wer-uk + - simo + +minimax-zero-shot-vi: + class: audio_evals.evaluator.ensemble.Ensemble + args: + components: + - minimax-eval-asr-wer-vi + - simo + diff --git a/registry/evaluator/mmsu.yaml b/registry/evaluator/mmsu.yaml new file mode 100644 index 0000000..8b630b5 --- /dev/null +++ b/registry/evaluator/mmsu.yaml @@ -0,0 +1,8 @@ +mmsu-choices: + class: audio_evals.evaluator.choices.ChoicesEval + args: + choices_columns: + - choice_a + - choice_b + - choice_c + - choice_d \ No newline at end of file diff --git a/registry/model/qwen3tts.yaml b/registry/model/qwen3tts.yaml new file mode 100644 index 0000000..e6f14a8 --- /dev/null +++ b/registry/model/qwen3tts.yaml @@ -0,0 +1,112 @@ +# Qwen3-TTS Model Configurations +# Reference: https://github.com/QwenLM/Qwen3-TTS + +# ============================================ +# Custom Voice Models (with preset speakers) +# ============================================ + +qwen3-tts-1.7b-custom-voice: + class: audio_evals.models.TTS.qwen3_tts.Qwen3TTS + args: + path: Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice + mode: custom_voice + dtype: bfloat16 + device: cuda:0 + env_path: envs/qwen3_tts + requirements_path: audio_evals/lib/Qwen3TTS/requirements.txt + sample_params: # ref: https://github.com/QwenLM/Qwen3-TTS/blob/main/examples/test_model_12hz_custom_voice.py + max_new_tokens: 2048 + +qwen3-tts-0.6b-custom-voice: + class: audio_evals.models.TTS.qwen3_tts.Qwen3TTS + args: + path: Qwen/Qwen3-TTS-12Hz-0.6B-CustomVoice + mode: custom_voice + dtype: bfloat16 + device: cuda:0 + env_path: envs/qwen3_tts + requirements_path: audio_evals/lib/Qwen3TTS/requirements.txt + sample_params: # ref: https://github.com/QwenLM/Qwen3-TTS/blob/main/examples/test_model_12hz_custom_voice.py + max_new_tokens: 2048 + +# ============================================ +# Voice Design Models (natural language control) +# ============================================ + +qwen3-tts-1.7b-voice-design: + class: audio_evals.models.TTS.qwen3_tts.Qwen3TTS + args: + path: Qwen/Qwen3-TTS-12Hz-1.7B-VoiceDesign + mode: voice_design + dtype: bfloat16 + device: cuda:0 + env_path: envs/qwen3_tts + requirements_path: audio_evals/lib/Qwen3TTS/requirements.txt + +# ============================================ +# Voice Clone Models (Base models) +# ============================================ + +qwen3-tts-1.7b-base: + class: audio_evals.models.TTS.qwen3_tts.Qwen3TTS + args: + path: Qwen/Qwen3-TTS-12Hz-1.7B-Base + mode: voice_clone + dtype: bfloat16 + device: cuda:0 + env_path: envs/qwen3_tts + requirements_path: audio_evals/lib/Qwen3TTS/requirements.txt + sample_params: # ref: https://github.com/QwenLM/Qwen3-TTS/blob/main/examples/test_model_12hz_base.py + max_new_tokens: 2048 + do_sample: true + top_k: 50 + top_p: 1.0 + temperature: 0.9 + repetition_penalty: 1.05 + subtalker_dosample: true + subtalker_top_k: 50 + subtalker_top_p: 1.0 + subtalker_temperature: 0.9 + +qwen3-tts-12hz-1.7b-base-xvec_only: + class: audio_evals.models.TTS.qwen3_tts.Qwen3TTS + args: + path: Qwen/Qwen3-TTS-12Hz-1.7B-Base + mode: voice_clone + dtype: bfloat16 + device: cuda:0 + env_path: envs/qwen3_tts + requirements_path: audio_evals/lib/Qwen3TTS/requirements.txt + sample_params: + max_new_tokens: 2048 + do_sample: true + top_k: 50 + top_p: 1.0 + temperature: 0.9 + repetition_penalty: 1.05 + subtalker_dosample: true + subtalker_top_k: 50 + subtalker_top_p: 1.0 + subtalker_temperature: 0.9 + x_vector_only_mode: true + +qwen3-tts-0.6b-base: + class: audio_evals.models.TTS.qwen3_tts.Qwen3TTS + args: + path: Qwen/Qwen3-TTS-12Hz-0.6B-Base + mode: voice_clone + dtype: bfloat16 + device: cuda:0 + env_path: envs/qwen3_tts + requirements_path: audio_evals/lib/Qwen3TTS/requirements.txt + sample_params: + max_new_tokens: 2048 + do_sample: true + top_k: 50 + top_p: 1.0 + temperature: 0.9 + repetition_penalty: 1.05 + subtalker_dosample: true + subtalker_top_k: 50 + subtalker_top_p: 1.0 + subtalker_temperature: 0.9 diff --git a/registry/model/voxcpm_1_5.yaml b/registry/model/voxcpm_1_5.yaml index 0a08c45..bcbe51d 100644 --- a/registry/model/voxcpm_1_5.yaml +++ b/registry/model/voxcpm_1_5.yaml @@ -1,17 +1,17 @@ voxcpm1_5: class: audio_evals.models.TTS.voxcpm_1_5.VoxCPM args: - path: /user/zhouyixuan/ckpt/VoxCPM-1.5-0.5B-6hz-44khz-20251204/ + path: openbmb/VoxCPM1.5 vc_mode: True denoise: True - denoise_path: /user/shiqundong/project/UltraEval-Audio/init_model/iic/speech_zipenhancer_ans_multiloss_16k_base + denoise_path: iic/speech_zipenhancer_ans_multiloss_16k_base env_path: envs/VoxCPM_1_5 requirements_path: audio_evals/lib/VoxCPM1_5/requirements.txt voxcpm1_5-no-denoise-no-retry-no-normalization: class: audio_evals.models.TTS.voxcpm_1_5.VoxCPM args: - path: /user/zhouyixuan/ckpt/VoxCPM-1.5-0.5B-6hz-44khz-20251204/ + path: openbmb/VoxCPM1.5 vc_mode: True denoise: False env_path: envs/VoxCPM_1_5 diff --git a/registry/prompt/choice.yaml b/registry/prompt/choice.yaml index cb00caa..53c5bb5 100644 --- a/registry/prompt/choice.yaml +++ b/registry/prompt/choice.yaml @@ -21,3 +21,14 @@ single_choice_with_answer: value: "{{WavPath}}" - type: text value: "{{question}} Select one option from the provided choices.\n{{choices}}" + +mmau: + class: audio_evals.prompt.base.Prompt + args: + template: + - role: user + contents: + - type: audio + value: "{{WavPath}}" + - type: text + value: "{{question}} Select one option from the provided choices without explain. {% for choice in choices %}{{ 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'[loop.index0] }}. {{ choice }}\n{% endfor %}" diff --git a/registry/prompt/kimi-audio.yaml b/registry/prompt/kimi-audio.yaml index 77fa684..f693cbb 100644 --- a/registry/prompt/kimi-audio.yaml +++ b/registry/prompt/kimi-audio.yaml @@ -19,3 +19,26 @@ kimi-audio-asr-zh: value: '请把这段语音转录成文本。' - type: audio value: '{{WavPath}}' + +kimi-mmau: + class: audio_evals.prompt.base.Prompt + args: + template: + - role: user + contents: + - type: text + value: "{{question}} \n{% for choice in choices %}({{ 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'[loop.index0] }}) {{ choice }} {% endfor %}" + - type: audio + value: "{{WavPath}}" + + +kimi-cocalsound: + class: audio_evals.prompt.base.Prompt + args: + template: + - role: user + contents: + - type: text + value: "Identify the human vocal sound in the audio.\nOptions:\n(A) Laughter\n(B) Sigh\n(C) Cough\n(D) Throat clearing\n(E) Sneeze\n(F) Sniff\n.Answer with the option's letter from the given choices directly and only give the best option." + - type: audio + value: "{{WavPath}}" diff --git a/registry/prompt/mmsu.yaml b/registry/prompt/mmsu.yaml new file mode 100644 index 0000000..0304eec --- /dev/null +++ b/registry/prompt/mmsu.yaml @@ -0,0 +1,11 @@ +mmsu: + class: audio_evals.prompt.base.Prompt + args: + template: + - role: user + contents: + - type: text + value: "Choose the most suitable answer from options A, B, C, and D to respond the question in next line, **you should only choose A or B or C or D.** Do not provide any additional explanations or content.\n\nQuestion: {{question}}\n\nA. {{choice_a}}\nB. {{choice_b}}\nC. {{ choice_c | default('None') }}\nD. {{ choice_d | default('None') }}" + - type: audio + value: "{{WavPath}}" + \ No newline at end of file diff --git a/registry/prompt/qwen3-omni.yaml b/registry/prompt/qwen3-omni.yaml index b3fd1db..545ec66 100644 --- a/registry/prompt/qwen3-omni.yaml +++ b/registry/prompt/qwen3-omni.yaml @@ -234,7 +234,7 @@ qwen3-omni-tts-en: - role: user contents: - type: text - value: 'Repeat the following text once without adding any other words:\n\n\n {{Text}}\n' + value: 'Repeat the following text once without adding any other words:\n\n\n {{text}}\n' qwen3-omni-tts-zh: @@ -244,4 +244,16 @@ qwen3-omni-tts-zh: - role: user contents: - type: text - value: '重复一遍这段文字不要加其他字:\n\n\n {{Text}}\n' + value: '重复一遍这段文字不要加其他字:\n\n\n {{text}}\n' + +qwen3-mmsu: + class: audio_evals.prompt.base.Prompt + args: + template: + - role: user + contents: + - type: audio + value: "{{WavPath}}" + - type: text + value: "Choose the most suitable answer from options A, B, C, and D to respond the question in next line, **you should only choose A or B or C or D.** Do not provide any additional explanations or content.\n\nQuestion: {{question}}\n\nA. {{choice_a}}\nB. {{choice_b}}\nC. {{ choice_c | default('None') }}\nD. {{ choice_d | default('None') }}" + \ No newline at end of file diff --git a/registry/prompt/qwen3-tts.yaml b/registry/prompt/qwen3-tts.yaml new file mode 100644 index 0000000..aa9e768 --- /dev/null +++ b/registry/prompt/qwen3-tts.yaml @@ -0,0 +1,163 @@ +qwen3-tts-vivian-chinese: + class: audio_evals.prompt.base.Prompt + args: + template: + text: "{{text}}" + language: "Chinese" + speaker: "Vivian" + +qwen3-tts-ryan-english: + class: audio_evals.prompt.base.Prompt + args: + template: + text: "{{text}}" + language: "English" + speaker: "Ryan" + +qwen3-tts-serena-chinese: + class: audio_evals.prompt.base.Prompt + args: + template: + text: "{{text}}" + language: "Chinese" + speaker: "Serena" + +qwen3-tts-uncle-fu-chinese: + class: audio_evals.prompt.base.Prompt + args: + template: + text: "{{text}}" + language: "Chinese" + speaker: "Uncle_Fu" + +qwen3-tts-dylan-chinese: + class: audio_evals.prompt.base.Prompt + args: + template: + text: "{{text}}" + language: "Chinese" + speaker: "Dylan" + +qwen3-tts-eric-chinese: + class: audio_evals.prompt.base.Prompt + args: + template: + text: "{{text}}" + language: "Chinese" + speaker: "Eric" + +qwen3-tts-aiden-english: + class: audio_evals.prompt.base.Prompt + args: + template: + text: "{{text}}" + language: "English" + speaker: "Aiden" + +qwen3-tts-ono-anna-japanese: + class: audio_evals.prompt.base.Prompt + args: + template: + text: "{{text}}" + language: "Japanese" + speaker: "Ono_Anna" + +qwen3-tts-sohee-korean: + class: audio_evals.prompt.base.Prompt + args: + template: + text: "{{text}}" + language: "Korean" + speaker: "Sohee" + +# Qwen3-TTS prompt configurations with language parameter + +qwen3-tts-voice-clone-chinese: + class: audio_evals.prompt.base.Prompt + args: + template: + text: "{{text}}" + prompt_audio: "{{WavPath}}" + prompt_text: "{{prompt_text}}" + language: "Chinese" + +qwen3-tts-voice-clone-english: + class: audio_evals.prompt.base.Prompt + args: + template: + text: "{{text}}" + prompt_audio: "{{WavPath}}" + prompt_text: "{{prompt_text}}" + language: "English" + +qwen3-tts-voice-clone-japanese: + class: audio_evals.prompt.base.Prompt + args: + template: + text: "{{text}}" + prompt_audio: "{{WavPath}}" + prompt_text: "{{prompt_text}}" + language: "Japanese" + +qwen3-tts-voice-clone-korean: + class: audio_evals.prompt.base.Prompt + args: + template: + text: "{{text}}" + prompt_audio: "{{WavPath}}" + prompt_text: "{{prompt_text}}" + language: "Korean" + +qwen3-tts-voice-clone-german: + class: audio_evals.prompt.base.Prompt + args: + template: + text: "{{text}}" + prompt_audio: "{{WavPath}}" + prompt_text: "{{prompt_text}}" + language: "German" + +qwen3-tts-voice-clone-french: + class: audio_evals.prompt.base.Prompt + args: + template: + text: "{{text}}" + prompt_audio: "{{WavPath}}" + prompt_text: "{{prompt_text}}" + language: "French" + +qwen3-tts-voice-clone-russian: + class: audio_evals.prompt.base.Prompt + args: + template: + text: "{{text}}" + prompt_audio: "{{WavPath}}" + prompt_text: "{{prompt_text}}" + language: "Russian" + +qwen3-tts-voice-clone-portuguese: + class: audio_evals.prompt.base.Prompt + args: + template: + text: "{{text}}" + prompt_audio: "{{WavPath}}" + prompt_text: "{{prompt_text}}" + language: "Portuguese" + +qwen3-tts-voice-clone-spanish: + class: audio_evals.prompt.base.Prompt + args: + template: + text: "{{text}}" + prompt_audio: "{{WavPath}}" + prompt_text: "{{prompt_text}}" + language: "Spanish" + +qwen3-tts-voice-clone-italian: + class: audio_evals.prompt.base.Prompt + args: + template: + text: "{{text}}" + prompt_audio: "{{WavPath}}" + prompt_text: "{{prompt_text}}" + language: "Italian" diff --git a/registry/prompt/whisper-pretrain.yaml b/registry/prompt/whisper-pretrain.yaml index bdbd4fc..91166b9 100644 --- a/registry/prompt/whisper-pretrain.yaml +++ b/registry/prompt/whisper-pretrain.yaml @@ -126,3 +126,115 @@ whisper-asr-ru: audio: '{{WavPath}}' generate_kwargs: language: russian + +whisper-asr-ar: + class: audio_evals.prompt.base.Prompt + args: + template: + audio: '{{WavPath}}' + generate_kwargs: + language: arabic + +whisper-asr-cs: + class: audio_evals.prompt.base.Prompt + args: + template: + audio: '{{WavPath}}' + generate_kwargs: + language: czech + +whisper-asr-nl: + class: audio_evals.prompt.base.Prompt + args: + template: + audio: '{{WavPath}}' + generate_kwargs: + language: dutch + +whisper-asr-fi: + class: audio_evals.prompt.base.Prompt + args: + template: + audio: '{{WavPath}}' + generate_kwargs: + language: finnish + +whisper-asr-el: + class: audio_evals.prompt.base.Prompt + args: + template: + audio: '{{WavPath}}' + generate_kwargs: + language: greek + +whisper-asr-hi: + class: audio_evals.prompt.base.Prompt + args: + template: + audio: '{{WavPath}}' + generate_kwargs: + language: hindi + +whisper-asr-id: + class: audio_evals.prompt.base.Prompt + args: + template: + audio: '{{WavPath}}' + generate_kwargs: + language: indonesian + +whisper-asr-pl: + class: audio_evals.prompt.base.Prompt + args: + template: + audio: '{{WavPath}}' + generate_kwargs: + language: polish + +whisper-asr-pt: + class: audio_evals.prompt.base.Prompt + args: + template: + audio: '{{WavPath}}' + generate_kwargs: + language: portuguese + +whisper-asr-ro: + class: audio_evals.prompt.base.Prompt + args: + template: + audio: '{{WavPath}}' + generate_kwargs: + language: romanian + +whisper-asr-th: + class: audio_evals.prompt.base.Prompt + args: + template: + audio: '{{WavPath}}' + generate_kwargs: + language: thai + +whisper-asr-tr: + class: audio_evals.prompt.base.Prompt + args: + template: + audio: '{{WavPath}}' + generate_kwargs: + language: turkish + +whisper-asr-uk: + class: audio_evals.prompt.base.Prompt + args: + template: + audio: '{{WavPath}}' + generate_kwargs: + language: ukrainian + +whisper-asr-vi: + class: audio_evals.prompt.base.Prompt + args: + template: + audio: '{{WavPath}}' + generate_kwargs: + language: vietnamese diff --git a/replication/qwen3_tts.md b/replication/qwen3_tts.md new file mode 100644 index 0000000..c8b1f6a --- /dev/null +++ b/replication/qwen3_tts.md @@ -0,0 +1,55 @@ +# Qwen3-TTS 复现文档与评测结果 + +**模型**: [Qwen3-TTS](../registry/model/qwen3tts.yaml) +**评测日期**: 2026/02 + +**指标说明**: +- **WER⬇️ / CER⬇️**: ASR 识别错误率（越低越好） +- **SIM⬆️**: 说话人相似度（越高越好） +- **DNSMOS⬆️**: 语音质量打分（越高越好，范围 0–5） + +--- + +## Seed-TTS-Eval（Voice Clone）复现结果 + +**Note**: 性能格式为 `reproduced_result(official_result)`，括号内为论文/官方结果（如有）。 + +> 下面命令会自动下载权重到 `init_model/` 并运行 voice clone；首次运行会较慢。 + +| 模型 | SEED-test-en (WER⬇️) | SEED-test-en (SIM⬆️) | SEED-test-zh (CER⬇️) | SEED-test-zh (SIM⬆️) | eval_cli | +|---|---:|---:|---:|---:|---| +| Qwen3-TTS-12Hz-1.7B-Base-official-infer-params | 1.58 (1.24) | 71.24 | 0.87 (0.78) | 76.89 | en:[1] zh:[2] | +| Qwen3-TTS-12Hz-1.7B-Base-official-infer-params-xvec_only | 1.56 (1.24) | 59.61 | 0.78 (0.78) | 72.92 | en:[3] zh:[4] | +| Qwen3-TTS-12Hz-0.6B-Base-official-infer-params | 1.69 (1.32) | 70.55 | 1.01 (0.92) | 76.48 | en:[5] zh:[6] | + +## CV3-Eval（Zero-shot Voice Clone）复现结果 + +> CV3-Eval 在本项目中按 split 分开跑（`cv3_zero_shot_{en,zh}` 与 `cv3_zero_shot_hard_{en,zh}`），表格为汇总展示。 + +| 模型 | zh CER/%⬇️ | en WER/%⬇️ | hard-zh CER/%⬇️ | hard-zh SIM/%⬆️ | hard-zh DNSMOS⬆️ | hard-en WER/%⬇️ | hard-en SIM/%⬆️ | hard-en DNSMOS⬆️ | eval_cli | +|---|---:|---:|---:|---:|---:|---:|---:|---:|---| +| Qwen3-TTS-12Hz-1.7B-Base-official-infer-params | 3.12±0.07 | 3.77±0.19 | 11.33±1.43 | 70.13 | 3.83 | 7.90±1.77 | 66.06 | 3.91 | zh:[8] en:[7] hard-zh:[10] hard-en:[9] | +| Qwen3-TTS-12Hz-0.6B-Base-official-infer-params | 3.40±0.09 | 33.91±13.06 | 10.70±1.06 | 69.72 | 3.82 | 10.70±2.90 | 67.04 | 3.88 | zh:[12] en:[11] hard-zh:[14] hard-en:[13] | + +--- + +## Evaluation Commands + +[1] `python audio_evals/main.py --dataset seed_tts_eval_en --model qwen3-tts-1.7b-base --prompt qwen3-tts-voice-clone-english --use_model_pool --workers 8` +[2] `python audio_evals/main.py --dataset seed_tts_eval_zh --model qwen3-tts-1.7b-base --prompt qwen3-tts-voice-clone-chinese --use_model_pool --workers 8` +[3] `python audio_evals/main.py --dataset seed_tts_eval_en --model qwen3-tts-12hz-1.7b-base-xvec_only --prompt qwen3-tts-voice-clone-english --use_model_pool --workers 8` +[4] `python audio_evals/main.py --dataset seed_tts_eval_zh --model qwen3-tts-12hz-1.7b-base-xvec_only --prompt qwen3-tts-voice-clone-chinese --use_model_pool --workers 8` +[5] `python audio_evals/main.py --dataset seed_tts_eval_en --model qwen3-tts-0.6b-base --prompt qwen3-tts-voice-clone-english --use_model_pool --workers 8` +[6] `python audio_evals/main.py --dataset seed_tts_eval_zh --model qwen3-tts-0.6b-base --prompt qwen3-tts-voice-clone-chinese --use_model_pool --workers 8` + +[7] `python audio_evals/main.py --dataset cv3_zero_shot_en --model qwen3-tts-1.7b-base --prompt qwen3-tts-voice-clone-english --use_model_pool --workers 8` +[8] `python audio_evals/main.py --dataset cv3_zero_shot_zh --model qwen3-tts-1.7b-base --prompt qwen3-tts-voice-clone-chinese --use_model_pool --workers 8` +[9] `python audio_evals/main.py --dataset cv3_zero_shot_hard_en --model qwen3-tts-1.7b-base --prompt qwen3-tts-voice-clone-english --use_model_pool --workers 8` +[10] `python audio_evals/main.py --dataset cv3_zero_shot_hard_zh --model qwen3-tts-1.7b-base --prompt qwen3-tts-voice-clone-chinese --use_model_pool --workers 8` + +[11] `python audio_evals/main.py --dataset cv3_zero_shot_en --model qwen3-tts-0.6b-base --prompt qwen3-tts-voice-clone-english --use_model_pool --workers 8` +[12] `python audio_evals/main.py --dataset cv3_zero_shot_zh --model qwen3-tts-0.6b-base --prompt qwen3-tts-voice-clone-chinese --use_model_pool --workers 8` +[13] `python audio_evals/main.py --dataset cv3_zero_shot_hard_en --model qwen3-tts-0.6b-base --prompt qwen3-tts-voice-clone-english --use_model_pool --workers 8` +[14] `python audio_evals/main.py --dataset cv3_zero_shot_hard_zh --model qwen3-tts-0.6b-base --prompt qwen3-tts-voice-clone-chinese --use_model_pool --workers 8` + + diff --git a/setup_vllm.sh b/setup_vllm.sh new file mode 100644 index 0000000..0a2eb1c --- /dev/null +++ b/setup_vllm.sh @@ -0,0 +1,143 @@ +#!/bin/bash +# set -e # 遇到错误立即退出 +set -x # 打印执行日志 + +# ================= 配置区域 ================= +# 修正路径获取方式 +ALGORITHM_PATH=$(pwd) +# 定义服务端口 +PORT=8630 +# 定义服务名称 +SERVED_MODEL_NAME="qwen3omni" + + +VLLM_MODEL_PATH="/opt/huawei/dataset/data/modelpt/Qwen_omni/Qwen3-Omni-30B-A3B-Instruct" + + + +echo ">>> [Init] Displaying the NPU memory...." +npu-smi info + + +# ================= 环境准备 ================= +echo ">>> [Init] Checking environment..." +#cd /opt/huawei/dataset/Audio_dataset/framework/ms-swift-main/ +#pip install -e . +pip install --upgrade pip +pip install vllm==0.13.0 +pip install vllm-ascend==0.13.0rc1 +pip install transformers==4.57.1 +pip install torchvision torchaudio +pip install accelerate==1.10.1 +pip install deepspeed +pip install qwen_omni_utils +pip install msgspec +pip install urllib3==1.26.0 numpy==1.26.4 requests +pip install "decord" -U + +source /usr/local/Ascend/ascend-toolkit/set_env.sh +source /usr/local/Ascend/nnal/atb/set_env.sh + +pip list | grep transformers +pip list | grep torch +pip list | grep vllm + +# ================= 环境变量设置 (Ascend NPU) ================= +export VLLM_WORKER_MULTIPROC_METHOD=spawn +export HCCL_CONNECT_TIMEOUT=1200 +export HCCL_EXEC_TIMEOUT=1200 +export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True + +# ================= 启动 vLLM 后端 ================= +echo ">>> [Start] Starting vLLM server in background..." + +export USE_VLLM=1 + +# 1. 【核心修复】定义正确的日志目录和文件名 +LOG_DIR="${ALGORITHM_PATH}/log" +mkdir -p "${LOG_DIR}" # 确保目录存在 + +# 使用 SERVED_MODEL_NAME (qwen3vl) 作为文件名，不再使用未定义的 benchmarklist +SERVER_LOG="${LOG_DIR}/vllm_server_${SERVED_MODEL_NAME}.log" +SERVER_JUDGE_LOG="${LOG_DIR}/vllm_judge_server_${SERVED_MODEL_NAME}.log" + +echo ">>> [Log] Log file path: ${SERVER_LOG}" + +touch "${SERVER_LOG}" + +# ================= 启动 vLLM 后端 ================= +echo ">>> [Start] Starting vLLM server in background..." + +# 1. 启动 vLLM，日志依然写入文件 (这样为了保证 SERVER_PID 能抓到正确的 vLLM 进程) +ASCEND_RT_VISIBLE_DEVICES=0,1 \ +nohup vllm serve ${VLLM_MODEL_PATH} \ + --host 0.0.0.0 \ + --port ${PORT} \ + --served-model-name ${SERVED_MODEL_NAME} \ + --enforce-eager \ + --tensor-parallel-size 2 \ + --dtype bfloat16 \ + --max-model-len 8192 \ + --max-num-batched-tokens 8192 \ + --max-num-seqs 2 \ + --trust-remote-code \ + --enable-expert-parallel \ + --gpu-memory-utilization 0.98 > ${SERVER_LOG} 2>&1 & + +# 获取 vLLM 进程 ID +SERVER_PID=$! +echo ">>> [Start] vLLM server started with PID: ${SERVER_PID}" +echo ">>> [Log] Log file is located at: ${SERVER_LOG}" + +# 2. 【核心修改】启动 tail 在后台实时打印日志到屏幕 +echo ">>> [Log] Streaming logs to console..." +tail -f ${SERVER_LOG} & +TAIL_PID=$! # 记录 tail 的 PID，以便稍后关闭 + +# 3. 修改清理函数：退出时同时杀掉 vLLM 和 tail 进程 +cleanup() { + echo ">>> [Exit] Cleaning up..." + kill ${TAIL_PID} 2>/dev/null || true # 先停止打印日志 + kill ${SERVER_PID} 2>/dev/null || true # 再停止服务 +} +trap cleanup EXIT + +# ================= 等待服务就绪 ================= +echo ">>> [Wait] Waiting for vLLM to be ready on port ${PORT}..." + +MAX_RETRIES=600 # 最多等待 60 * 5 = 300秒 +for ((i=1; i<=MAX_RETRIES; i++)); do + # 检查 /health 或 /v1/models 接口 + if curl -s http://localhost:${PORT}/v1/models > /dev/null; then + echo ">>> [Wait] vLLM server is READY!" + break + fi + echo ">>> [Wait] Server not ready yet... (Attempt $i/$MAX_RETRIES). Sleeping 5s..." + sleep 5 +done + + +if ! curl -s http://localhost:${PORT}/v1/models > /dev/null; then + echo ">>> [Error] Server failed to start. Check logs below:" + tail -n 50 ${SERVER_LOG} + exit 1 +fi + + +# ================= 执行测试 ================= +echo ">>> [Test] Running inference test..." +export TEST_IMAGE_PATH="${ALGORITHM_PATH}/test.jpg" +export SERVED_MODEL_NAME="qwen3omni" +export VLLM_PORT=8630 + +python3 run_test.py + +echo ">>> [Done] Script finished successfully." + + +# 检查所有进程是否成功执行 +if [ $? -eq 0 ]; then + echo "所有部署进程成功启动！" +else + echo "某些部署进程失败。" +fi