diff --git a/LICENSE b/LICENSE index d5683e2a..db027d34 100644 --- a/LICENSE +++ b/LICENSE @@ -44,6 +44,7 @@ The project code is MIT. Model weights carry their own licenses: Echo-TTS CC-BY-NC-SA-4.0 No Fish Audio S2 Pro Fish Audio Research License No Dots TTS Apache-2.0 Yes + Audio8 TTS Apache-2.0 Yes DramaBox LTX-2 Community License Conditional OmniVoice Apache-2.0 Yes MOSS-TTS Apache-2.0 Yes diff --git a/PROJECT_INDEX.md b/PROJECT_INDEX.md index 3f15a09f..984dc837 100644 --- a/PROJECT_INDEX.md +++ b/PROJECT_INDEX.md @@ -17,7 +17,7 @@ **Key architectural rules:** - Chunking happens in the **processor**, not the adapter (`generate_single()` on adapter = raw single call) - Runtime routing happens through `ModelLoadConfig.runtime_mode` + `runtime_profile`, not ad-hoc subprocess calls -- Shared runtime workers are currently used for fragile engine families such as VibeVoice, Qwen3-TTS / ASR, Granite forced alignment, and Higgs Audio 2. Engines that support the modern stack run natively in the main Transformers 5 environment. +- Shared runtime workers are currently used for fragile engine families such as VibeVoice, Qwen3-TTS / ASR, Granite forced alignment, Higgs Audio 2, and Audio8 TTS. Engines that support the modern stack run natively in the main Transformers 5 environment. - YAML (`docs/Dev reports/tts_audio_suite_engines.yaml`) is source of truth for engine doc tables → run `python3 scripts/generate_engine_tables.py --readme` to regenerate - Auxiliary YAML (`docs/Dev reports/tts_audio_suite_aux_models.yaml`) is source of truth for helper/post-process model docs → run `python3 scripts/generate_aux_model_docs.py` - All models download to `ComfyUI/models/TTS//` @@ -25,7 +25,7 @@ ## Engines -19 engines follow the pattern above: +20 engines follow the pattern above: | Engine | Adapter | Processor | SRT Processor | Engine Node | |--------|---------|-----------|---------------|-------------| @@ -44,13 +44,14 @@ | Echo-TTS | `echo_tts_adapter.py` | `nodes/echo_tts/echo_tts_processor.py` | `echo_tts_srt_processor.py` | `echo_tts_engine_node.py` | | Fish Audio S2 Pro | `fish_audio_s2_adapter.py` | `nodes/fish_audio_s2/fish_audio_s2_processor.py` | `fish_audio_s2_srt_processor.py` | `fish_audio_s2_engine_node.py` | | Dots TTS | `dots_tts_adapter.py` | `nodes/dots_tts/dots_tts_processor.py` | `dots_tts_srt_processor.py` | `dots_tts_engine_node.py` | +| Audio8 TTS | `audio8_tts_adapter.py` | `nodes/audio8_tts/audio8_tts_processor.py` | `audio8_tts_srt_processor.py` | `audio8_tts_engine_node.py` | | DramaBox | `dramabox_adapter.py` | `nodes/dramabox/dramabox_processor.py` | `dramabox_srt_processor.py` | `dramabox_engine_node.py` | | OmniVoice | `omnivoice_adapter.py` | `nodes/omnivoice/omnivoice_processor.py` | `omnivoice_srt_processor.py` | `omnivoice_engine_node.py` | | MOSS-SoundEffect v2 | `moss_soundeffect_v2_adapter.py` | — | — | `moss_soundeffect_v2_engine_node.py` | | RVC | — | `engines/rvc/` | — | `rvc_engine_node.py` | **Engine implementations live in:** -- `engines/chatterbox/`, `engines/chatterbox_official_23lang/`, `engines/f5tts/`, `engines/higgs_audio/`, `engines/higgs_audio_v3/`, `engines/vibevoice_engine/`, `engines/step_audio_editx/`, `engines/cosyvoice/`, `engines/qwen3_tts/`, `engines/qwen3_asr/`, `engines/moss_tts/`, `engines/moss_soundeffect_v2/`, `engines/granite_asr/`, `engines/echo_tts/`, `engines/fish_audio_s2/`, `engines/dots_tts/`, `engines/dramabox/`, `engines/omnivoice/`, `engines/rvc/` +- `engines/chatterbox/`, `engines/chatterbox_official_23lang/`, `engines/f5tts/`, `engines/higgs_audio/`, `engines/higgs_audio_v3/`, `engines/vibevoice_engine/`, `engines/step_audio_editx/`, `engines/cosyvoice/`, `engines/qwen3_tts/`, `engines/qwen3_asr/`, `engines/moss_tts/`, `engines/moss_soundeffect_v2/`, `engines/granite_asr/`, `engines/echo_tts/`, `engines/fish_audio_s2/`, `engines/dots_tts/`, `engines/audio8_tts/`, `engines/dramabox/`, `engines/omnivoice/`, `engines/rvc/` ## Documentation Files @@ -145,13 +146,14 @@ - `launcher.py` - runtime bootstrap, venv creation, Windows toolchain env setup - `session.py`, `protocol.py` - JSONL worker transport and message protocol - `bootstrap.py` - shared runtime bootstrap helpers -- `vibevoice_proxy.py`, `qwen3_tts_proxy.py`, `qwen3_asr_proxy.py`, `higgs_audio_proxy.py` - parent-process proxies -- `workers/` - worker subprocess entrypoints for VibeVoice, Qwen3-TTS, Qwen3-ASR/aligner, Higgs Audio +- `vibevoice_proxy.py`, `qwen3_tts_proxy.py`, `qwen3_asr_proxy.py`, `higgs_audio_proxy.py`, `audio8_tts_proxy.py` - parent-process proxies +- `workers/` - worker subprocess entrypoints for VibeVoice, Qwen3-TTS, Qwen3-ASR/aligner, Higgs Audio, and Audio8 TTS - Current shared legacy T4 runtime profile is reused by: - VibeVoice / Kugel - Qwen3-TTS - Qwen3-ASR and Granite's optional Qwen forced aligner - Higgs Audio 2 + - Audio8 TTS ### Audio (`utils/audio/`) - `processing.py` - Tensor manipulation, normalization, format conversion diff --git a/README.md b/README.md index a1a4280b..63cadbd4 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ Subtitle workflows are still a core focus: the suite can transcribe to SRT, rebu -## Quick Engine Comparison — 19 Engines +## Quick Engine Comparison — 20 Engines | Engine | Languages | Size | Key Features | |--------|-----------|------|--------------| @@ -41,6 +41,7 @@ Subtitle workflows are still a core focus: the suite can transcribe to SRT, rebu | **Echo-TTS** | 🇺🇸 | ~5.3GB + ~1.8GB | Diffusion-based (~30s best), Force Speaker KV (speaker drift control) | | **Fish Audio S2 Pro** | 🌐 80+ languages | ~10.3GB / ~8.0GB | Free-form sub-word emotion/prosody tags, Native multi-speaker and multi-turn dialogue with dynamic speaker references | | **Dots TTS** | 🇺🇸​🇨🇳​🇩🇪​🇪🇸​🇫🇷​🇮🇹 +13 | ~6GB | Official auto language detect / language control, SOAR and MeanFlow distilled variants | +| **Audio8 TTS** | 🌐 11 recommended languages | ~2.39 GiB | Reference-free TTS + zero-shot/cross-lingual cloning, 44.1kHz output with sampling or greedy decoding | | **DramaBox** | 🇺🇸 | ~16.4GB | Expressive scene prompting and stage directions, Native and SRT-aware duration targeting | | **OmniVoice** | 🌐 600+ languages | ~3.7GB | Inline non-verbal tags and pronunciation overrides, Reference-free voice design | | **MOSS-TTS** | 🇺🇸​🇨🇳​🇩🇪​🇪🇸​🇫🇷​🇮🇹 +18 | ~8.5GB tokenizer + ~6.1GB/17GB/18GB model | Reference-free voice design with MOSS-VoiceGenerator, Native 1-5 speaker TTSD dialogue | @@ -1527,6 +1528,7 @@ For offline/manual setup: | Granite ASR | `ComfyUI/models/TTS/granite_asr/` | ✅ | Granite ASR models; plus adds native diarization/timestamps, optional Qwen forced aligner reused lazily for timestamps/SRT fallback | | Echo-TTS | `ComfyUI/models/TTS/echo-tts-base/` | ✅ | ~7.1GB total (base + dac); CC-BY-NC-SA | | Dots TTS | `ComfyUI/models/TTS/dots_tts/` | ✅ | Official base / soar / mf checkpoints with tokenizer, vocoder, speaker encoder | +| Audio8 TTS | `ComfyUI/models/TTS/audio8_tts/Audio8-TTS-Preview-0.6b/` | ✅ | ~2.39 GiB official 0.6B Preview checkpoint; shared Transformers 4.57.3 runtime required because main 5.10.2 collapses voice cloning | | DramaBox | `ComfyUI/models/TTS/dramabox/DramaBox/` | ✅ | ~16.4GB download; fast mode roughly 24GB VRAM; experimental FP8 peaks on RTX 4090: staged ~15.1GB allocated, sequential ~11.7GB allocated / ~12.4GB reserved; conditional LTX-2 Community License | | Fish Audio S2 Pro | `ComfyUI/models/TTS/fish_audio_s2_pro/` | ✅ | Official BF16 or optional community FP8 checkpoint; the official checkpoint can be quantized on load with BNB INT8/NF4; main T5 environment with process teardown for Clear VRAM; Fish Audio Research License | | OmniVoice | `ComfyUI/models/TTS/omnivoice/` | ✅ | Official OmniVoice model. Voice cloning in this suite requires explicit reference text. | diff --git a/docs/Dev reports/tts_audio_suite_engines.yaml b/docs/Dev reports/tts_audio_suite_engines.yaml index 619768fb..f2ade9bc 100644 --- a/docs/Dev reports/tts_audio_suite_engines.yaml +++ b/docs/Dev reports/tts_audio_suite_engines.yaml @@ -1339,6 +1339,77 @@ engines: speed_performance: { supported: "partial", notes: "Moderate; mf variant is faster" } reference_free_tts: { supported: true, notes: "(default speaker)" } + - id: audio8_tts + name: Audio8 TTS + models: "Audio8 TTS Preview 0.6B" + size: "~2.39 GiB" + license: "Apache-2.0" + commercial: true + language_summary_full: "11 recommended languages (including Cantonese)" + language_summary_compact: "🌐 11 recommended languages" + + capabilities: + tts: true + srt: true + vc: false + asr: false + training: false + + runtime_isolation: + default_mode: "shared_runtime" + main_environment: false + shared_runtime: true + dedicated_runtime: false + runtime_profile: "vibevoice_transformers4_shared" + notes: "Required for feature-complete Audio8 inference: main Transformers 5.10.2 collapsed the official voice-clone demo to 0.325s, while the shared Transformers 4.57.3 runtime generated 10.495s for the published 10.54s demo" + + readme_key_features: + - "Reference-free TTS + zero-shot/cross-lingual cloning" + - "44.1kHz output with sampling or greedy decoding" + + special_features: + - "44.1kHz reference-free TTS and zero-shot/cross-lingual voice cloning" + - "Exact reference transcript required when voice cloning" + - "Greedy or sampled decoding with temperature, top-p, top-k, seed, and token-limit controls" + - "Suite text chunking, character switching, and SRT timing/assembly" + - "No explicit language selector; multilingual behavior is text-driven" + - "No native emotion/style/speed controls, streaming, timestamps, dialogue/multi-speaker mode, or presets" + - "Not compatible with Voice Designer: the official model has no instruction-conditioned voice-design mode" + - "Unified TTS/SRT integration only; no special node or suite training integration" + + model_sources: + - component: "Audio8 TTS Preview 0.6B" + source_name: "Audio8/Audio8-TTS-Preview-0.6b" + source_url: "https://huggingface.co/Audio8/Audio8-TTS-Preview-0.6b" + size: "~2.39 GiB" + auto_download: true + notes: "Official 0.6B Preview checkpoint with bundled 44.1kHz codec; suite uses the shared Transformers 4.57.3 runtime because voice cloning is not compatible with main Transformers 5.10.2" + + languages: + en: { supported: true, flag: "🇺🇸", notes: "" } + zh: { supported: true, flag: "🇨🇳", notes: "(Chinese; upstream separately recommends Cantonese, folded into this matrix row)" } + de: { supported: true, flag: "🇩🇪", notes: "" } + es: { supported: true, flag: "🇪🇸", notes: "" } + fr: { supported: true, flag: "🇫🇷", notes: "" } + it: { supported: true, flag: "🇮🇹", notes: "" } + ja: { supported: true, flag: "🇯🇵", notes: "" } + ko: { supported: true, flag: "🇰🇷", notes: "" } + pl: { supported: true, flag: "🇵🇱", notes: "" } + nl: { supported: true, flag: "🇳🇱", notes: "" } + + features: + voice_cloning: { supported: true, notes: "Zero-shot and cross-lingual; reference audio must be paired with its exact transcript" } + reference_transcript: { requirement: conditional } + native_multi_speaker: { supported: false, notes: "No native dialogue or multi-speaker interface; suite character switching generates speakers as separate segments" } + voice_conversion: { supported: false, notes: "No separate voice-conversion mode" } + asr_transcribe: { supported: false, notes: "" } + emotion_control: { supported: false, notes: "No native emotion, style, or speed controls" } + native_long_form: { supported: false, notes: "(uses suite text chunking and SRT assembly)" } + community_finetunes: { supported: false, notes: "No community model or upstream SFT/training integration in the suite" } + vram_efficient: { supported: "partial", notes: "~1.75 GiB model-plus-codec parameter memory at fp16/bf16 or ~3.5 GiB at float32/CPU, before KV cache, activations, and framework overhead; shared runtime reuses the main environment's PyTorch" } + speed_performance: { supported: "partial", notes: "Live shared-runtime validation: 6.36s reference-free audio in 39.3s including load; 9.99s cloned audio in 36.7s warm; exact cache reuse in 56ms" } + reference_free_tts: { supported: true, notes: "Reference audio is optional" } + - id: dramabox name: DramaBox models: "DramaBox 3.3B" @@ -1903,8 +1974,9 @@ table_notes: reference_transcript: >- Conditional means the transcript is required only for the specific mode: CosyVoice3 zero-shot, Qwen3-TTS full Base cloning, or MOSS-TTSD cloned-speaker - dialogue. Higgs Audio 2, Higgs Audio v3, and Dots TTS accept matching text - when provided but do not require it. + dialogue, and Audio8 TTS zero-shot/cross-lingual cloning. Higgs Audio 2, + Higgs Audio v3, and Dots TTS accept matching text when provided but do not + require it. language_support: - "**CosyVoice3 Chinese**: Includes 18+ dialects (Cantonese, Sichuan, Dongbei, Shanghai, etc.)" - "**Higgs Audio 2**: Trained on EN, ZH (Mandarin), KO, DE, ES (English majority) - 10M hours AudioVerse dataset" @@ -1912,6 +1984,7 @@ table_notes: - "**IndexTTS-2**: Trained on 55K+ hours - ZH, EN, JA primary." - "**MOSS-TTS**: Official list includes ZH, EN, DE, ES, FR, JA, IT, HU, KO, RU, FA, AR, PL, PT, CS, DA, SV, EL, TR; HU/FA/CS are not separate columns in this matrix." - "**OmniVoice**: Official model supports 600+ languages; this matrix only shows the suite's comparison subset." + - "**Audio8 TTS**: The Preview recommends Cantonese, Chinese, Dutch, English, French, German, Italian, Japanese, Korean, Polish, and Spanish. It has no explicit language selector; language behavior is text-driven. Cantonese is folded into the Chinese row in this matrix." - "**RVC**: Language-agnostic voice conversion/post-processing; it is marked supported for every language row." - "Qwen3-TTS: Supports pt-BR only with instruction (voice design or harcoded voices). Base can't do instructions, so it will always ouput pt-PT." @@ -1980,6 +2053,10 @@ readme_model_download_table: primary_model_path: "ComfyUI/models/TTS/dots_tts/" auto_download: "✅" notes: "Official base / soar / mf checkpoints with tokenizer, vocoder, speaker encoder" + - engine: "Audio8 TTS" + primary_model_path: "ComfyUI/models/TTS/audio8_tts/Audio8-TTS-Preview-0.6b/" + auto_download: "✅" + notes: "~2.39 GiB official 0.6B Preview checkpoint; shared Transformers 4.57.3 runtime required because main 5.10.2 collapses voice cloning" - engine: "DramaBox" primary_model_path: "ComfyUI/models/TTS/dramabox/DramaBox/" auto_download: "✅" @@ -2423,6 +2500,22 @@ model_layouts_markdown: | - Native sample rate is 48kHz. - Main-environment support works on Transformers 5; on Windows, `normalize_text` falls back to no-op if `WeTextProcessing` is unavailable. + ## Audio8 TTS + + ```text + ComfyUI/models/TTS/audio8_tts/ + └── Audio8-TTS-Preview-0.6b/ + └── (13 required official runtime files, including codec weights) + ``` + + Notes: + + - The official `Audio8/Audio8-TTS-Preview-0.6b` checkpoint is about 2.39 GiB and includes the 44.1kHz neural codec. + - The 0.6B model plus codec use about 1.75 GiB for parameters at fp16/bf16 or about 3.5 GiB at float32/CPU, before KV cache, activations, and framework overhead. + - The suite uses the existing `vibevoice_transformers4_shared` runtime with Transformers 4.57.3. Main Transformers 5.10.2 is not compatible with Audio8 voice cloning: the official demo collapsed to 0.325s, while the shared runtime generated 10.495s for the published 10.54s demo. + - Live shared-runtime validation produced 6.36s of reference-free audio in 39.3s including model load, 9.99s of cloned audio in 36.7s warm, and exact cache reuse in 56ms. + - The engine has unified TTS and SRT support only: no Voice Designer support, special node, training integration, native streaming/timestamps/dialogue, or built-in presets. + ## OmniVoice ```text diff --git a/docs/ENGINE_COMPARISON.md b/docs/ENGINE_COMPARISON.md index fae62873..dcc5a681 100644 --- a/docs/ENGINE_COMPARISON.md +++ b/docs/ENGINE_COMPARISON.md @@ -18,6 +18,7 @@ | **Echo-TTS** | Main | echo-tts-base + fish-s1-dac-min | ~5.3GB + ~1.8GB | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | CC-BY-NC-SA-4.0 | Diffusion-based (~30s best), Force Speaker KV (speaker drift control) | 1 | | **Fish Audio S2 Pro** | Main | S2 Pro 4B / FP8 | ~10.3GB / ~8.0GB | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | Fish Audio Research License | Free-form sub-word emotion/prosody tags, Native multi-speaker and multi-turn dialogue with dynamic speaker references, Zero-shot voice cloning, Optional per-segment custom character switching, Configurable 4K-32K native context with reduced KV-cache VRAM, Optional community FP8 weight-only checkpoint with BF16 activations, Optional on-the-fly BitsAndBytes INT8/NF4 for the official checkpoint | 80+ languages | | **Dots TTS** | Main | dots.tts-base, dots.tts-soar, dots.tts-mf | ~6GB | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | Apache-2.0 | Official auto language detect / language control, SOAR and MeanFlow distilled variants | 19 | +| **Audio8 TTS** | Shared | Audio8 TTS Preview 0.6B | ~2.39 GiB | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | Apache-2.0 | 44.1kHz reference-free TTS and zero-shot/cross-lingual voice cloning, Exact reference transcript required when voice cloning, Greedy or sampled decoding with temperature, top-p, top-k, seed, and token-limit controls, Suite text chunking, character switching, and SRT timing/assembly, No explicit language selector; multilingual behavior is text-driven, No native emotion/style/speed controls, streaming, timestamps, dialogue/multi-speaker mode, or presets, Not compatible with Voice Designer: the official model has no instruction-conditioned voice-design mode, Unified TTS/SRT integration only; no special node or suite training integration | 11 recommended languages (including Cantonese) | | **DramaBox** | Main | DramaBox 3.3B | ~16.4GB | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | LTX-2 Community License | Expressive scene prompting and stage directions, Native and SRT-aware duration targeting, Official duration-aware long-form chunking with scene-prefix preservation, Optional 10-second zero-shot voice reference, CFG negative prompt with per-segment switching, Explicit generation/reference durations, CFG rescale control, and optional Perth watermark, Experimental staged and sequential strategies for lowering peak VRAM; official LTX FP8-cast storage, Optional official torch.compile path | 1 | | **OmniVoice** | Main | OmniVoice | ~3.7GB | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | Apache-2.0 | Inline non-verbal tags and pronunciation overrides, Reference-free voice design, 600+ language support, Upstream long-form chunk orchestration | 600+ | | **MOSS-TTS** | Main | Local 1.7B, Delay 8B v1.5/1.0, VoiceGenerator 1.7B, SoundEffect 8B v1, TTSD 8B | ~8.5GB tokenizer + ~6.1GB/17GB/18GB model | ✅ | ✅ | ❌ | ❌ | ✅ | ✅ | Apache-2.0 | Reference-free voice design with MOSS-VoiceGenerator, Native 1-5 speaker TTSD dialogue, 31-language generation with MOSS-TTS-v1.5, Prompt-only sound-effect generation with MOSS-SoundEffect v1, Long-form generation (TTSD/Delay), Duration token hint, Local/Delay/TTSD variants, Initial integrated LoRA training workflow (Delay 8B) | 24 | diff --git a/docs/FEATURE_COMPARISON.md b/docs/FEATURE_COMPARISON.md index 27bc6aa5..38c9efb3 100644 --- a/docs/FEATURE_COMPARISON.md +++ b/docs/FEATURE_COMPARISON.md @@ -2,22 +2,22 @@ ## Feature Comparison Matrix -| Feature | F5-TTS | ChatterBox | ChatterBox 23L | VibeVoice | Higgs Audio 2 | Higgs Audio v3 | IndexTTS-2 | CosyVoice3 | Qwen3-TTS | Granite ASR | Step Audio EditX | Echo-TTS | Fish Audio S2 Pro | Dots TTS | DramaBox | OmniVoice | MOSS-TTS | MOSS-SoundEffect v2 | RVC | -|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---| -| **TTS** | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | -| **SRT** | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | -| **Voice Conversion** | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | -| **ASR (Transcribe)** | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | -| **Sound Effects** | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | -| **Training** | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ✅ | -| **Voice Cloning** | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ (Base model) | ❌ | ✅ | ✅ | ✅ Reference audio plus exact transcript | ✅ | ✅ Optional reference audio; configurable 3-30 second window from the beginning of the source, default 10 seconds | ✅ | ✅ | ❌ | ⚠️ (needs training) | -| **Reference Transcript†** | **Required** | Not used | Not used | Not used | Optional | Optional | Not used | Conditional | Conditional | N/A | **Required** | Not used | **Required** | Optional | Not used | **Required** | Conditional | Not used | N/A | -| **Native Multi-Speaker** | ❌ | ❌ | ❌ | ✅ (Base only, Kugel uses fallback) | ✅ | ❌ | ❌ | ❌ | ❌ | ⚠️ (Plus variant speaker attribution / diarization) | ❌ | ❌ | ✅ Native dialogue is default; optional custom mode generates each [Character] segment independently as local speaker 0 | ❌ | ❌ | ❌ | ✅ (TTSD v1.0; 1-5 speakers) | ❌ | ❌ | -| **Emotion Control** | ❌ | ❌ | ⚠️ (v2 tags - doesn't work) | ❌ | ⚠️ (via prompt) | ✅ (native inline tags) | ✅ (8 emotions) | ⚠️ (via instruct) | ⚠️ (via instruct) | ❌ | ✅ (14 emotions) | ❌ | ✅ Free-form inline natural-language tags | ❌ | ✅ Natural-language scene prompt and stage directions | ⚠️ (voice-design instruct + inline non-verbal tags) | ✅ (MOSS-VoiceGenerator instruction-conditioned voice design) | ❌ | ❌ | -| **Native Long-form** | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ Configurable 4K-32K native context; suite text chunking is bypassed | ❌ | ✅ Official duration-aware quote-group chunking; ~37s target / 45s cap | ✅ (uses upstream audio_chunk_duration / audio_chunk_threshold orchestration; bypasses suite char-based chunk splitting) | ✅ (TTSD/Delay long-form; use chunk orchestration for very long inputs) | ❌ | N/A | -| **Community Finetunes** | ✅ | ✅ | ✅ | ✅ KugelAudio, Hindi | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (LoRA adapter inference supported; initial integrated LoRA training support added for MOSS-TTS Delay 8B) | ❌ | ✅ | -| **VRAM Efficient** | ✅ | ✅ | ✅ | ⚠️ (5-18GB) | ⚠️ (9GB) | ⚠️ (~8-10GB) | ⚠️ (9-12GB) | ✅ (5.4GB) | ✅ (3-6GB) | ✅ (~4.6GB) | ⚠️ (7GB) | ⚠️ (~7GB total) | ⚠️ 8K context measured at ~15.2GB BF16, ~11.2GB FP8, ~11.2GB BNB INT8, or ~8.9GB BNB NF4; BF16 codec and activations; BNB is a load-time option for the official checkpoint | ⚠️ (main env works; 2B-class model is not lightweight) | ✅ Fast mode is ~24GB; experimental FP8 peaks on RTX 4090: staged ~15.1GB allocated, sequential ~11.7GB allocated / ~12.4GB reserved | ⚠️ (main env works; 2B-class model is not lightweight) | ⚠️ (Local 1.7B smaller; tokenizer is large) | ⚠️ (runs in the main ComfyUI environment but remains GPU-heavy) | ✅ | -| **Speed/Performance** | ✅ Very Fast | ✅ Fast | ✅ Fast | ⚠️ | ⚠️ | ⚠️ CUDA recommended | ⚠️ | ✅ Fast | ⚠️ | ⚠️ Moderate | ⚠️ | ✅ Fast (diffusion, realtime-capable) | ✅ Main-environment subprocess with reliable teardown; local compile measurements: ~40 it/s BF16, ~11.8 it/s NF4 at ~8.9GB VRAM, and ~3.7 it/s INT8 at ~11.2GB VRAM; quality comparison pending | ⚠️ Moderate; mf variant is faster | ⚠️ Fast favors repeated generation; staged reloads temporary components; sequential also transfers the transformer; optional block-level torch.compile accelerates repeat denoising | ✅ Fast; upstream reports sub-realtime RTF | ✅ Fast with CUDA/FlashAttention | ⚠️ (100 diffusion steps by default) | ✅ Fast | -| **No Narrator Required** | ❌ | ✅ (default speaker) | ✅ (default speaker) | ✅ (zero-shot / default speaker) | ✅ (basic TTS if no narrator/reference is provided) | ✅ (zero-shot) | ❌ | ✅ (cross-lingual or instruct mode) | ✅ (Base default voice or CustomVoice presets) | N/A | ❌ | ❌ | ✅ Reference audio is optional | ✅ (default speaker) | ✅ Voice reference is optional | ✅ (native default voice; instruct also works without narrator) | ✅ (direct TTS and prompt-only generation) | ❌ | N/A | +| Feature | F5-TTS | ChatterBox | ChatterBox 23L | VibeVoice | Higgs Audio 2 | Higgs Audio v3 | IndexTTS-2 | CosyVoice3 | Qwen3-TTS | Granite ASR | Step Audio EditX | Echo-TTS | Fish Audio S2 Pro | Dots TTS | Audio8 TTS | DramaBox | OmniVoice | MOSS-TTS | MOSS-SoundEffect v2 | RVC | +|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---| +| **TTS** | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | +| **SRT** | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | +| **Voice Conversion** | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | +| **ASR (Transcribe)** | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | +| **Sound Effects** | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | +| **Training** | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ✅ | +| **Voice Cloning** | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ (Base model) | ❌ | ✅ | ✅ | ✅ Reference audio plus exact transcript | ✅ | ✅ Zero-shot and cross-lingual; reference audio must be paired with its exact transcript | ✅ Optional reference audio; configurable 3-30 second window from the beginning of the source, default 10 seconds | ✅ | ✅ | ❌ | ⚠️ (needs training) | +| **Reference Transcript†** | **Required** | Not used | Not used | Not used | Optional | Optional | Not used | Conditional | Conditional | N/A | **Required** | Not used | **Required** | Optional | Conditional | Not used | **Required** | Conditional | Not used | N/A | +| **Native Multi-Speaker** | ❌ | ❌ | ❌ | ✅ (Base only, Kugel uses fallback) | ✅ | ❌ | ❌ | ❌ | ❌ | ⚠️ (Plus variant speaker attribution / diarization) | ❌ | ❌ | ✅ Native dialogue is default; optional custom mode generates each [Character] segment independently as local speaker 0 | ❌ | ❌ | ❌ | ❌ | ✅ (TTSD v1.0; 1-5 speakers) | ❌ | ❌ | +| **Emotion Control** | ❌ | ❌ | ⚠️ (v2 tags - doesn't work) | ❌ | ⚠️ (via prompt) | ✅ (native inline tags) | ✅ (8 emotions) | ⚠️ (via instruct) | ⚠️ (via instruct) | ❌ | ✅ (14 emotions) | ❌ | ✅ Free-form inline natural-language tags | ❌ | ❌ | ✅ Natural-language scene prompt and stage directions | ⚠️ (voice-design instruct + inline non-verbal tags) | ✅ (MOSS-VoiceGenerator instruction-conditioned voice design) | ❌ | ❌ | +| **Native Long-form** | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ Configurable 4K-32K native context; suite text chunking is bypassed | ❌ | ❌ | ✅ Official duration-aware quote-group chunking; ~37s target / 45s cap | ✅ (uses upstream audio_chunk_duration / audio_chunk_threshold orchestration; bypasses suite char-based chunk splitting) | ✅ (TTSD/Delay long-form; use chunk orchestration for very long inputs) | ❌ | N/A | +| **Community Finetunes** | ✅ | ✅ | ✅ | ✅ KugelAudio, Hindi | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (LoRA adapter inference supported; initial integrated LoRA training support added for MOSS-TTS Delay 8B) | ❌ | ✅ | +| **VRAM Efficient** | ✅ | ✅ | ✅ | ⚠️ (5-18GB) | ⚠️ (9GB) | ⚠️ (~8-10GB) | ⚠️ (9-12GB) | ✅ (5.4GB) | ✅ (3-6GB) | ✅ (~4.6GB) | ⚠️ (7GB) | ⚠️ (~7GB total) | ⚠️ 8K context measured at ~15.2GB BF16, ~11.2GB FP8, ~11.2GB BNB INT8, or ~8.9GB BNB NF4; BF16 codec and activations; BNB is a load-time option for the official checkpoint | ⚠️ (main env works; 2B-class model is not lightweight) | ⚠️ ~1.75 GiB model-plus-codec parameter memory at fp16/bf16 or ~3.5 GiB at float32/CPU, before KV cache, activations, and framework overhead; shared runtime reuses the main environment's PyTorch | ✅ Fast mode is ~24GB; experimental FP8 peaks on RTX 4090: staged ~15.1GB allocated, sequential ~11.7GB allocated / ~12.4GB reserved | ⚠️ (main env works; 2B-class model is not lightweight) | ⚠️ (Local 1.7B smaller; tokenizer is large) | ⚠️ (runs in the main ComfyUI environment but remains GPU-heavy) | ✅ | +| **Speed/Performance** | ✅ Very Fast | ✅ Fast | ✅ Fast | ⚠️ | ⚠️ | ⚠️ CUDA recommended | ⚠️ | ✅ Fast | ⚠️ | ⚠️ Moderate | ⚠️ | ✅ Fast (diffusion, realtime-capable) | ✅ Main-environment subprocess with reliable teardown; local compile measurements: ~40 it/s BF16, ~11.8 it/s NF4 at ~8.9GB VRAM, and ~3.7 it/s INT8 at ~11.2GB VRAM; quality comparison pending | ⚠️ Moderate; mf variant is faster | ⚠️ Live shared-runtime validation: 6.36s reference-free audio in 39.3s including load; 9.99s cloned audio in 36.7s warm; exact cache reuse in 56ms | ⚠️ Fast favors repeated generation; staged reloads temporary components; sequential also transfers the transformer; optional block-level torch.compile accelerates repeat denoising | ✅ Fast; upstream reports sub-realtime RTF | ✅ Fast with CUDA/FlashAttention | ⚠️ (100 diffusion steps by default) | ✅ Fast | +| **No Narrator Required** | ❌ | ✅ (default speaker) | ✅ (default speaker) | ✅ (zero-shot / default speaker) | ✅ (basic TTS if no narrator/reference is provided) | ✅ (zero-shot) | ❌ | ✅ (cross-lingual or instruct mode) | ✅ (Base default voice or CustomVoice presets) | N/A | ❌ | ❌ | ✅ Reference audio is optional | ✅ (default speaker) | ✅ Reference audio is optional | ✅ Voice reference is optional | ✅ (native default voice; instruct also works without narrator) | ✅ (direct TTS and prompt-only generation) | ❌ | N/A | -† **Reference Transcript:** Conditional means the transcript is required only for the specific mode: CosyVoice3 zero-shot, Qwen3-TTS full Base cloning, or MOSS-TTSD cloned-speaker dialogue. Higgs Audio 2, Higgs Audio v3, and Dots TTS accept matching text when provided but do not require it. \ No newline at end of file +† **Reference Transcript:** Conditional means the transcript is required only for the specific mode: CosyVoice3 zero-shot, Qwen3-TTS full Base cloning, or MOSS-TTSD cloned-speaker dialogue, and Audio8 TTS zero-shot/cross-lingual cloning. Higgs Audio 2, Higgs Audio v3, and Dots TTS accept matching text when provided but do not require it. \ No newline at end of file diff --git a/docs/LANGUAGE_SUPPORT.md b/docs/LANGUAGE_SUPPORT.md index 84a59933..6cbc7a06 100644 --- a/docs/LANGUAGE_SUPPORT.md +++ b/docs/LANGUAGE_SUPPORT.md @@ -2,110 +2,110 @@ ## Language Support by Engine -| Language | Code | F5-TTS | ChatterBox | ChatterBox 23L | VibeVoice | Higgs Audio 2 | Higgs Audio v3 | IndexTTS-2 | CosyVoice3 | Qwen3-TTS | Granite ASR | Step Audio EditX | Echo-TTS | Fish Audio S2 Pro | Dots TTS | DramaBox | OmniVoice | MOSS-TTS | MOSS-SoundEffect v2 | RVC | -|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---| -| 🇺🇸 **English** | EN | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ Tier 1 | ✅ | ✅ Official model is English-only | ✅ | ✅ | ✅ Officially demonstrated prompt language | ✅ | -| 🇨🇳 **Chinese** | ZH | ❌ | ❌ | ✅ | ✅ | ✅ (Mandarin) | ✅ | ✅ | ✅ + 18 dialects | ✅ | ❌ | ✅ (Mandarin + Sichuanese, Cantonese) | ❌ | ✅ Tier 1 | ✅ (Mandarin; official also exposes YUE separately outside this matrix) | ❌ | ✅ | ✅ | ✅ Officially demonstrated prompt language | ✅ | -| 🇩🇪 **German** | DE | ✅ | ✅ (×3) | ✅ | ✅ (Kugel) | ✅ | ✅ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ✅ Tier 2 | ✅ | ❌ | ✅ | ✅ | ❌ | ✅ | -| 🇪🇸 **Spanish** | ES | ✅ | ❌ | ✅ | ✅ (Kugel) | ✅ | ✅ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ✅ Tier 2 | ✅ | ❌ | ✅ | ✅ | ❌ | ✅ | -| 🇫🇷 **French** | FR | ✅ | ✅ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ✅ Tier 2 | ✅ | ❌ | ✅ | ✅ | ❌ | ✅ | -| 🇮🇹 **Italian** | IT | ✅ | ✅ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ✅ | -| 🇯🇵 **Japanese** | JA | ✅ | ✅ | ✅ | ✅ (Kugel) | ❌ | ✅ | ✅ ? | ✅ | ✅ | ✅ (4.1-2b supports Japanese; plus variant does not) | ✅ | ❌ | ✅ Tier 1 | ✅ | ❌ | ✅ | ✅ | ❌ | ✅ | -| 🇰🇷 **Korean** | KO | ❌ | ✅ | ✅ | ✅ (Kugel) | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ✅ | ❌ | ✅ Tier 2 | ✅ | ❌ | ✅ | ✅ | ❌ | ✅ | -| 🇷🇺 **Russian** | RU | ❌ | ✅ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ✅ Tier 2 | ✅ | ❌ | ✅ | ✅ | ❌ | ✅ | -| 🇧🇷 **Portuguese** | PT | ✅ (BR) | ❌ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ✅ (EU/BR*) | ✅ | ❌ | ❌ | ✅ Tier 2 | ✅ (Official PT tag is generic; upstream does not expose separate PT-BR/PT-PT tags and it may lean more European Portuguese than Brazilian Portuguese) | ❌ | ✅ (generic PT; official language space is much broader than this matrix) | ✅ | ❌ | ✅ | -| 🇵🇱 **Polish** | PL | ✅ | ❌ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ✅ | -| 🇮🇳 **Hindi** | HI | ✅ | ❌ | ✅ | ✅ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ (v1.5) | ❌ | ✅ | -| 🇪🇬 **Arabic** | AR | ❌ | ❌ | ✅ (Egyptian) | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ Tier 2 | ✅ | ❌ | ✅ | ✅ | ❌ | ✅ | -| 🇹🇷 **Turkish** | TR | ❌ | ❌ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ✅ | -| 🇹🇭 **Thai** | TH | ✅ | ❌ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ (v1.5) | ❌ | ✅ | -| 🇳🇴 **Norwegian** | NO | ❌ | ✅ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | -| 🇻🇳 **Vietnamese** | VI | ❌ | ❌ | ✅ (Viterbox) | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ (v1.5) | ❌ | ✅ | -| 🇦🇲 **Armenian** | HY | ❌ | ✅ | ❌ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | -| 🇬🇪 **Georgian** | KA | ❌ | ✅ | ❌ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | -| 🇩🇰 **Danish** | DA | ❌ | ❌ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | ✅ | -| 🇫🇮 **Finnish** | FI | ❌ | ❌ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ (v1.5) | ❌ | ✅ | -| 🇬🇷 **Greek** | EL | ❌ | ❌ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ✅ | -| 🇮🇱 **Hebrew** | HE | ❌ | ❌ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ (v1.5) | ❌ | ✅ | -| 🇲🇾 **Malay** | MS | ❌ | ❌ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ (v1.5) | ❌ | ✅ | -| 🇳🇱 **Dutch** | NL | ❌ | ❌ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ (v1.5) | ❌ | ✅ | -| 🇸🇪 **Swedish** | SV | ❌ | ❌ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | ✅ | -| 🇰🇪 **Swahili** | SW | ❌ | ❌ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ (v1.5) | ❌ | ✅ | -| 🇿🇦 **Afrikaans** | AF | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | -| 🇦🇱 **Albanian** | SQ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (usable) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | -| 🇮🇳 **Assamese** | AS | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | -| 🇪🇸 **Asturian** | AST | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | -| 🇦🇿 **Azerbaijani** | AZ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | -| 🇷🇺 **Bashkir** | BA | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | -| 🇪🇸 **Basque** | EU | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | -| 🇧🇾 **Belarusian** | BE | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | -| 🇧🇩 **Bengali** | BN | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | -| 🇧🇦 **Bosnian** | BS | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | -| 🇧🇬 **Bulgarian** | BG | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | -| 🇪🇸 **Catalan** | CA | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | -| 🇵🇭 **Cebuano** | CEB | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | -| 🇮🇶 **Central Kurdish** | CKB | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | -| 🇭🇷 **Croatian** | HR | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | -| 🇨🇿 **Czech** | CS | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | -| 🇷🇺 **Eastern Mari** | MHR | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | -| 🌍 **Esperanto** | EO | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | -| 🇪🇪 **Estonian** | ET | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | -| 🇪🇸 **Galician** | GL | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | -| 🇮🇳 **Gujarati** | GU | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | -| 🇭🇹 **Haitian Creole** | HT | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | -| 🇳🇬 **Hausa** | HA | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | -| 🇭🇺 **Hungarian** | HU | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | -| 🇮🇩 **Indonesian** | ID | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | -| 🇮🇩 **Javanese** | JV | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | -| 🇮🇳 **Kannada** | KN | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | -| 🇰🇿 **Kazakh** | KK | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | -| 🇷🇼 **Kinyarwanda** | RW | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | -| 🇰🇬 **Kyrgyz** | KY | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | -| 🇱🇻 **Latvian** | LV | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | -| 🇨🇩 **Lingala** | LN | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | -| 🇱🇹 **Lithuanian** | LT | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | -| 🇰🇪 **Luo** | LUO | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | -| 🇲🇰 **Macedonian** | MK | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | -| 🇮🇳 **Malayalam** | ML | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | -| 🇲🇹 **Maltese** | MT | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | -| 🇳🇿 **Māori** | MI | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | -| 🇮🇳 **Marathi** | MR | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | -| 🇲🇳 **Mongolian** | MN | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | -| 🇳🇵 **Nepali** | NE | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | -| 🇫🇷 **Occitan** | OC | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | -| 🇮🇷 **Persian** | FA | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | -| 🇷🇴 **Romanian** | RO | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | -| 🇿🇦 **Sepedi** | NSO | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | -| 🇷🇸 **Serbian** | SR | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | -| 🇿🇼 **Shona** | SN | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | -| 🇸🇰 **Slovak** | SK | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | -| 🇸🇮 **Slovene** | SL | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | -| 🇵🇭 **Tagalog** | TL | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | -| 🇹🇯 **Tajik** | TG | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | -| 🇮🇳 **Tamil** | TA | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | -| 🇮🇳 **Telugu** | TE | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | -| 🇺🇦 **Ukrainian** | UK | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | -| 🇵🇰 **Urdu** | UR | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | -| 🇨🇳 **Uyghur** | UG | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | -| 🇺🇿 **Uzbek** | UZ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | -| 🇿🇦 **Xhosa** | XH | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | -| 🇿🇦 **Zulu** | ZU | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | -| 🇲🇼 **Chichewa/Nyanja** | NY | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (usable) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | -| 🇮🇳 **Eastern Punjabi** | PA | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (usable) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | -| 🇺🇬 **Ganda** | LG | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (usable) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | -| 🇮🇸 **Icelandic** | IS | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (usable) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | -| 🇮🇪 **Irish** | GA | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (usable) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | -| 🇩🇿 **Kabyle** | KAB | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (usable) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | -| 🇨🇻 **Kabuverdianu** | KEA | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (usable) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | -| 🇰🇪 **Kamba** | KAM | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (usable) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | -| 🇻🇦 **Latin** | LA | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (usable) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | -| 🇱🇺 **Luxembourgish** | LB | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (usable) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | -| 🇪🇹 **Oromo** | OM | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (usable) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | -| 🇦🇫 **Pashto** | PS | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (usable) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | -| 🇵🇰 **Sindhi** | SD | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (usable) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | -| 🇸🇴 **Somali** | SO | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (usable) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | -| 🇦🇴 **Umbundu** | UMB | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (usable) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | -| 🏴 **Welsh** | CY | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (usable) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | +| Language | Code | F5-TTS | ChatterBox | ChatterBox 23L | VibeVoice | Higgs Audio 2 | Higgs Audio v3 | IndexTTS-2 | CosyVoice3 | Qwen3-TTS | Granite ASR | Step Audio EditX | Echo-TTS | Fish Audio S2 Pro | Dots TTS | Audio8 TTS | DramaBox | OmniVoice | MOSS-TTS | MOSS-SoundEffect v2 | RVC | +|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---| +| 🇺🇸 **English** | EN | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ Tier 1 | ✅ | ✅ | ✅ Official model is English-only | ✅ | ✅ | ✅ Officially demonstrated prompt language | ✅ | +| 🇨🇳 **Chinese** | ZH | ❌ | ❌ | ✅ | ✅ | ✅ (Mandarin) | ✅ | ✅ | ✅ + 18 dialects | ✅ | ❌ | ✅ (Mandarin + Sichuanese, Cantonese) | ❌ | ✅ Tier 1 | ✅ (Mandarin; official also exposes YUE separately outside this matrix) | ✅ (Chinese; upstream separately recommends Cantonese, folded into this matrix row) | ❌ | ✅ | ✅ | ✅ Officially demonstrated prompt language | ✅ | +| 🇩🇪 **German** | DE | ✅ | ✅ (×3) | ✅ | ✅ (Kugel) | ✅ | ✅ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ✅ Tier 2 | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ✅ | +| 🇪🇸 **Spanish** | ES | ✅ | ❌ | ✅ | ✅ (Kugel) | ✅ | ✅ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ✅ Tier 2 | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ✅ | +| 🇫🇷 **French** | FR | ✅ | ✅ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ✅ Tier 2 | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ✅ | +| 🇮🇹 **Italian** | IT | ✅ | ✅ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ✅ | +| 🇯🇵 **Japanese** | JA | ✅ | ✅ | ✅ | ✅ (Kugel) | ❌ | ✅ | ✅ ? | ✅ | ✅ | ✅ (4.1-2b supports Japanese; plus variant does not) | ✅ | ❌ | ✅ Tier 1 | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ✅ | +| 🇰🇷 **Korean** | KO | ❌ | ✅ | ✅ | ✅ (Kugel) | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ✅ | ❌ | ✅ Tier 2 | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ✅ | +| 🇷🇺 **Russian** | RU | ❌ | ✅ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ✅ Tier 2 | ✅ | ❌ | ❌ | ✅ | ✅ | ❌ | ✅ | +| 🇧🇷 **Portuguese** | PT | ✅ (BR) | ❌ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ✅ (EU/BR*) | ✅ | ❌ | ❌ | ✅ Tier 2 | ✅ (Official PT tag is generic; upstream does not expose separate PT-BR/PT-PT tags and it may lean more European Portuguese than Brazilian Portuguese) | ❌ | ❌ | ✅ (generic PT; official language space is much broader than this matrix) | ✅ | ❌ | ✅ | +| 🇵🇱 **Polish** | PL | ✅ | ❌ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ✅ | +| 🇮🇳 **Hindi** | HI | ✅ | ❌ | ✅ | ✅ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | ✅ (v1.5) | ❌ | ✅ | +| 🇪🇬 **Arabic** | AR | ❌ | ❌ | ✅ (Egyptian) | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ Tier 2 | ✅ | ❌ | ❌ | ✅ | ✅ | ❌ | ✅ | +| 🇹🇷 **Turkish** | TR | ❌ | ❌ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | ✅ | ❌ | ✅ | +| 🇹🇭 **Thai** | TH | ✅ | ❌ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | ✅ (v1.5) | ❌ | ✅ | +| 🇳🇴 **Norwegian** | NO | ❌ | ✅ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | +| 🇻🇳 **Vietnamese** | VI | ❌ | ❌ | ✅ (Viterbox) | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | ✅ (v1.5) | ❌ | ✅ | +| 🇦🇲 **Armenian** | HY | ❌ | ✅ | ❌ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | +| 🇬🇪 **Georgian** | KA | ❌ | ✅ | ❌ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | +| 🇩🇰 **Danish** | DA | ❌ | ❌ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | ✅ | +| 🇫🇮 **Finnish** | FI | ❌ | ❌ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | ✅ (v1.5) | ❌ | ✅ | +| 🇬🇷 **Greek** | EL | ❌ | ❌ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | ✅ | ❌ | ✅ | +| 🇮🇱 **Hebrew** | HE | ❌ | ❌ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ (v1.5) | ❌ | ✅ | +| 🇲🇾 **Malay** | MS | ❌ | ❌ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ (v1.5) | ❌ | ✅ | +| 🇳🇱 **Dutch** | NL | ❌ | ❌ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | ✅ | ✅ (v1.5) | ❌ | ✅ | +| 🇸🇪 **Swedish** | SV | ❌ | ❌ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | ✅ | +| 🇰🇪 **Swahili** | SW | ❌ | ❌ | ✅ | ✅ (Kugel) | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ (v1.5) | ❌ | ✅ | +| 🇿🇦 **Afrikaans** | AF | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | +| 🇦🇱 **Albanian** | SQ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (usable) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | +| 🇮🇳 **Assamese** | AS | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | +| 🇪🇸 **Asturian** | AST | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | +| 🇦🇿 **Azerbaijani** | AZ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | +| 🇷🇺 **Bashkir** | BA | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | +| 🇪🇸 **Basque** | EU | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | +| 🇧🇾 **Belarusian** | BE | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | +| 🇧🇩 **Bengali** | BN | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | +| 🇧🇦 **Bosnian** | BS | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | +| 🇧🇬 **Bulgarian** | BG | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | +| 🇪🇸 **Catalan** | CA | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | +| 🇵🇭 **Cebuano** | CEB | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | +| 🇮🇶 **Central Kurdish** | CKB | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | +| 🇭🇷 **Croatian** | HR | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | +| 🇨🇿 **Czech** | CS | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | +| 🇷🇺 **Eastern Mari** | MHR | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | +| 🌍 **Esperanto** | EO | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | +| 🇪🇪 **Estonian** | ET | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | +| 🇪🇸 **Galician** | GL | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | +| 🇮🇳 **Gujarati** | GU | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | +| 🇭🇹 **Haitian Creole** | HT | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | +| 🇳🇬 **Hausa** | HA | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | +| 🇭🇺 **Hungarian** | HU | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | +| 🇮🇩 **Indonesian** | ID | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | +| 🇮🇩 **Javanese** | JV | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | +| 🇮🇳 **Kannada** | KN | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | +| 🇰🇿 **Kazakh** | KK | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | +| 🇷🇼 **Kinyarwanda** | RW | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | +| 🇰🇬 **Kyrgyz** | KY | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | +| 🇱🇻 **Latvian** | LV | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | +| 🇨🇩 **Lingala** | LN | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | +| 🇱🇹 **Lithuanian** | LT | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | +| 🇰🇪 **Luo** | LUO | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | +| 🇲🇰 **Macedonian** | MK | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | +| 🇮🇳 **Malayalam** | ML | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | +| 🇲🇹 **Maltese** | MT | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | +| 🇳🇿 **Māori** | MI | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | +| 🇮🇳 **Marathi** | MR | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | +| 🇲🇳 **Mongolian** | MN | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | +| 🇳🇵 **Nepali** | NE | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | +| 🇫🇷 **Occitan** | OC | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | +| 🇮🇷 **Persian** | FA | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | +| 🇷🇴 **Romanian** | RO | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | +| 🇿🇦 **Sepedi** | NSO | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | +| 🇷🇸 **Serbian** | SR | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | +| 🇿🇼 **Shona** | SN | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | +| 🇸🇰 **Slovak** | SK | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | +| 🇸🇮 **Slovene** | SL | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | +| 🇵🇭 **Tagalog** | TL | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | +| 🇹🇯 **Tajik** | TG | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | +| 🇮🇳 **Tamil** | TA | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | +| 🇮🇳 **Telugu** | TE | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | +| 🇺🇦 **Ukrainian** | UK | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | +| 🇵🇰 **Urdu** | UR | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | +| 🇨🇳 **Uyghur** | UG | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | +| 🇺🇿 **Uzbek** | UZ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | +| 🇿🇦 **Xhosa** | XH | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | +| 🇿🇦 **Zulu** | ZU | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (polished) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | +| 🇲🇼 **Chichewa/Nyanja** | NY | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (usable) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | +| 🇮🇳 **Eastern Punjabi** | PA | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (usable) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | +| 🇺🇬 **Ganda** | LG | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (usable) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | +| 🇮🇸 **Icelandic** | IS | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (usable) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | +| 🇮🇪 **Irish** | GA | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (usable) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | +| 🇩🇿 **Kabyle** | KAB | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (usable) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | +| 🇨🇻 **Kabuverdianu** | KEA | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (usable) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | +| 🇰🇪 **Kamba** | KAM | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (usable) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | +| 🇻🇦 **Latin** | LA | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (usable) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | +| 🇱🇺 **Luxembourgish** | LB | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (usable) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | +| 🇪🇹 **Oromo** | OM | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (usable) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | +| 🇦🇫 **Pashto** | PS | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (usable) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | +| 🇵🇰 **Sindhi** | SD | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (usable) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | +| 🇸🇴 **Somali** | SO | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (usable) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | +| 🇦🇴 **Umbundu** | UMB | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (usable) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | +| 🏴 **Welsh** | CY | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ (usable) | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | **Notes:** @@ -115,5 +115,6 @@ - **IndexTTS-2**: Trained on 55K+ hours - ZH, EN, JA primary. - **MOSS-TTS**: Official list includes ZH, EN, DE, ES, FR, JA, IT, HU, KO, RU, FA, AR, PL, PT, CS, DA, SV, EL, TR; HU/FA/CS are not separate columns in this matrix. - **OmniVoice**: Official model supports 600+ languages; this matrix only shows the suite's comparison subset. +- **Audio8 TTS**: The Preview recommends Cantonese, Chinese, Dutch, English, French, German, Italian, Japanese, Korean, Polish, and Spanish. It has no explicit language selector; language behavior is text-driven. Cantonese is folded into the Chinese row in this matrix. - **RVC**: Language-agnostic voice conversion/post-processing; it is marked supported for every language row. - Qwen3-TTS: Supports pt-BR only with instruction (voice design or harcoded voices). Base can't do instructions, so it will always ouput pt-PT. \ No newline at end of file diff --git a/docs/MODEL_DOWNLOAD_SOURCES.md b/docs/MODEL_DOWNLOAD_SOURCES.md index 01b205e1..e5c39287 100644 --- a/docs/MODEL_DOWNLOAD_SOURCES.md +++ b/docs/MODEL_DOWNLOAD_SOURCES.md @@ -129,6 +129,12 @@ Use this as the canonical list of model repositories/links for offline setup. | dots.tts-soar | [rednote-hilab/dots.tts-soar](https://huggingface.co/rednote-hilab/dots.tts-soar) | ~6GB | ✅ | Official SOAR checkpoint for higher-quality zero-shot cloning | | dots.tts-mf | [rednote-hilab/dots.tts-mf](https://huggingface.co/rednote-hilab/dots.tts-mf) | ~6GB | ✅ | Official MeanFlow-distilled checkpoint for faster inference | +## Audio8 TTS + +| Component | Source | Size | Auto-Download | Notes | +|---|---|---|---|---| +| Audio8 TTS Preview 0.6B | [Audio8/Audio8-TTS-Preview-0.6b](https://huggingface.co/Audio8/Audio8-TTS-Preview-0.6b) | ~2.39 GiB | ✅ | Official 0.6B Preview checkpoint with bundled 44.1kHz codec; suite uses the shared Transformers 4.57.3 runtime because voice cloning is not compatible with main Transformers 5.10.2 | + ## DramaBox | Component | Source | Size | Auto-Download | Notes | diff --git a/docs/MODEL_LAYOUTS.md b/docs/MODEL_LAYOUTS.md index 9cfe8e33..c8a6f188 100644 --- a/docs/MODEL_LAYOUTS.md +++ b/docs/MODEL_LAYOUTS.md @@ -427,6 +427,22 @@ Notes: - Native sample rate is 48kHz. - Main-environment support works on Transformers 5; on Windows, `normalize_text` falls back to no-op if `WeTextProcessing` is unavailable. +## Audio8 TTS + +```text +ComfyUI/models/TTS/audio8_tts/ +└── Audio8-TTS-Preview-0.6b/ + └── (13 required official runtime files, including codec weights) +``` + +Notes: + +- The official `Audio8/Audio8-TTS-Preview-0.6b` checkpoint is about 2.39 GiB and includes the 44.1kHz neural codec. +- The 0.6B model plus codec use about 1.75 GiB for parameters at fp16/bf16 or about 3.5 GiB at float32/CPU, before KV cache, activations, and framework overhead. +- The suite uses the existing `vibevoice_transformers4_shared` runtime with Transformers 4.57.3. Main Transformers 5.10.2 is not compatible with Audio8 voice cloning: the official demo collapsed to 0.325s, while the shared runtime generated 10.495s for the published 10.54s demo. +- Live shared-runtime validation produced 6.36s of reference-free audio in 39.3s including model load, 9.99s of cloned audio in 36.7s warm, and exact cache reuse in 56ms. +- The engine has unified TTS and SRT support only: no Voice Designer support, special node, training integration, native streaming/timestamps/dialogue, or built-in presets. + ## OmniVoice ```text diff --git a/docs/New Engines Guides/NEW_ENGINE_IMPLEMENTATION_GUIDE.md b/docs/New Engines Guides/NEW_ENGINE_IMPLEMENTATION_GUIDE.md index bec6fdbe..ad46c410 100644 --- a/docs/New Engines Guides/NEW_ENGINE_IMPLEMENTATION_GUIDE.md +++ b/docs/New Engines Guides/NEW_ENGINE_IMPLEMENTATION_GUIDE.md @@ -466,7 +466,7 @@ Also to test, requirements and dependencies need to be added. - [ ] Pause tags work with `[pause:1.5s]` - [ ] Caching works (same input = cached output) - [ ] Model auto-download works -- [ ] VRAM management works (model unloads) +- [ ] VRAM management works (unload, switch to another isolated engine, then switch back with cache disabled) - [ ] Different parameter combinations work - [ ] Engine prints a standard `Settings:` summary with the active generation/load parameters - [ ] **Interrupt handling works** - User can stop SRT generation and it stops within ~1 segment diff --git a/docs/New Engines Guides/fails_to_avoid_TTS_Engine_Implementation.md b/docs/New Engines Guides/fails_to_avoid_TTS_Engine_Implementation.md index 9ce1173d..60873675 100644 --- a/docs/New Engines Guides/fails_to_avoid_TTS_Engine_Implementation.md +++ b/docs/New Engines Guides/fails_to_avoid_TTS_Engine_Implementation.md @@ -4,7 +4,7 @@ ### Import Errors - **Missing import**: Always add `import folder_paths` when using `folder_paths.get_temp_directory()` in node files -- **Bundled code imports**: For complex bundled packages with internal cross-imports, add `sys.path.insert(0, impl_dir)` at top of main files instead of converting all imports +- **Bundled code imports**: Never permanently prepend a bundled implementation directory to `sys.path` in the main ComfyUI process. Generic files such as `utils.py` can shadow suite packages and break later isolated workers. Prefer package-relative or vendor-namespaced imports; if upstream imports make path injection unavoidable, confine it to the isolated worker and load suite protocol modules before adding the vendor path. ### Audio Utility Functions - **Temp file creation**: Use `AudioProcessingUtils.save_audio_to_temp_file()` not `save_audio()` (doesn't exist) diff --git a/engines/adapters/__init__.py b/engines/adapters/__init__.py index 769ffd44..c9f3e3ee 100644 --- a/engines/adapters/__init__.py +++ b/engines/adapters/__init__.py @@ -49,6 +49,19 @@ class DotsTTSEngineAdapter: def __init__(self, *args, **kwargs): raise ImportError(f"Dots TTS adapter not available: {e}") +try: + from .audio8_tts_adapter import Audio8TTSEngineAdapter + AUDIO8_TTS_ADAPTER_AVAILABLE = True +except Exception as exc: + AUDIO8_TTS_ADAPTER_AVAILABLE = False + _AUDIO8_TTS_ADAPTER_ERROR = str(exc) + + class Audio8TTSEngineAdapter: + def __init__(self, *args, **kwargs): + raise ImportError( + f"Audio8 TTS adapter not available: {_AUDIO8_TTS_ADAPTER_ERROR}" + ) + try: from .dramabox_adapter import DramaBoxEngineAdapter DRAMABOX_ADAPTER_AVAILABLE = True @@ -96,10 +109,10 @@ def __init__(self, *args, **kwargs): __all__ = [ 'ChatterBoxEngineAdapter', 'F5TTSEngineAdapter', 'CosyVoiceAdapter', 'EchoTTSEngineAdapter', - 'DotsTTSEngineAdapter', 'DramaBoxEngineAdapter', 'OmniVoiceEngineAdapter', + 'DotsTTSEngineAdapter', 'Audio8TTSEngineAdapter', 'DramaBoxEngineAdapter', 'OmniVoiceEngineAdapter', 'MossTTSEngineAdapter', 'HiggsAudioV3EngineAdapter', 'CHATTERBOX_ADAPTER_AVAILABLE', 'F5TTS_ADAPTER_AVAILABLE', 'COSYVOICE_ADAPTER_AVAILABLE', - 'ECHO_TTS_ADAPTER_AVAILABLE', 'DOTS_TTS_ADAPTER_AVAILABLE', + 'ECHO_TTS_ADAPTER_AVAILABLE', 'DOTS_TTS_ADAPTER_AVAILABLE', 'AUDIO8_TTS_ADAPTER_AVAILABLE', 'DRAMABOX_ADAPTER_AVAILABLE', 'OMNIVOICE_ADAPTER_AVAILABLE', 'MOSS_TTS_ADAPTER_AVAILABLE', 'HIGGS_AUDIO_V3_ADAPTER_AVAILABLE', 'MossSoundEffectV2Adapter' diff --git a/engines/adapters/audio8_tts_adapter.py b/engines/adapters/audio8_tts_adapter.py new file mode 100644 index 00000000..e31126c7 --- /dev/null +++ b/engines/adapters/audio8_tts_adapter.py @@ -0,0 +1,277 @@ +"""Adapter between suite processors and the official Audio8 TTS engine.""" + +from __future__ import annotations + +import os +from typing import Any, Dict, Optional, Tuple + +import torch + +from engines.audio8_tts.downloader import Audio8TTSDownloader +from utils.audio.audio_hash import generate_stable_audio_component +from utils.audio.cache import get_audio_cache +from utils.device import resolve_torch_device +from utils.models.factory_config import ModelLoadConfig, RUNTIME_MODE_SHARED +from utils.voice.character_logging import resolved_character_label +from utils.voice.reference import effective_voice_audio + + +class Audio8TTSEngineAdapter: + """Translate unified TTS calls into Audio8's official inference API.""" + + SAMPLE_RATE = 44100 + + def __init__(self, config: Optional[Dict[str, Any]] = None): + self.config = dict(config or {}) + self.audio_cache = get_audio_cache() + self.downloader = Audio8TTSDownloader() + self._last_config: Optional[ModelLoadConfig] = None + self._load_signature: Optional[Tuple[Any, ...]] = None + + def update_config(self, new_config: Dict[str, Any]): + self.config = dict(new_config or {}) + + def _model_selection(self) -> str: + return str( + self.config.get( + "model_variant", + self.config.get( + "model_name", + Audio8TTSDownloader.MODEL_NAME, + ), + ) + or Audio8TTSDownloader.MODEL_NAME + ) + + def _build_load_signature(self) -> Tuple[Any, ...]: + return ( + self._model_selection(), + resolve_torch_device(self.config.get("device", "auto")), + self.config.get("dtype", "auto"), + ) + + def load_model( + self, + model_variant: str, + device: str = "auto", + dtype: str = "auto", + ): + """Resolve the organized model path and load through the unified factory.""" + from utils.models.unified_model_interface import unified_model_interface + + model_path = self.downloader.resolve_model_path(model_variant) + config = ModelLoadConfig( + engine_name="audio8_tts", + model_type="tts", + model_name=model_variant, + model_path=model_path, + device=device, + additional_params={"dtype": dtype}, + runtime_mode=RUNTIME_MODE_SHARED, + runtime_profile="vibevoice_transformers4_shared", + ) + self._last_config = config + return unified_model_interface.load_model(config) + + def _ensure_model_loaded(self): + signature = self._build_load_signature() + if signature != self._load_signature or self._last_config is None: + self.load_model( + model_variant=self._model_selection(), + device=self.config.get("device", "auto"), + dtype=self.config.get("dtype", "auto"), + ) + self._load_signature = signature + + def _get_engine(self): + if self._last_config is None: + self._ensure_model_loaded() + from utils.models.unified_model_interface import unified_model_interface + + return unified_model_interface.load_model(self._last_config) + + @staticmethod + def _mono_waveform(waveform: Any) -> torch.Tensor: + audio = torch.as_tensor(waveform, dtype=torch.float32).detach().cpu() + if audio.ndim == 3: + if audio.shape[0] != 1: + raise ValueError( + "Audio8 TTS reference audio must contain one batch item" + ) + audio = audio[0] + if audio.ndim == 2: + audio = audio.mean(dim=0) + if audio.ndim != 1 or audio.numel() == 0: + raise ValueError( + "Audio8 TTS reference audio must be non-empty mono or " + "channels-first audio" + ) + return audio.contiguous() + + def _in_memory_reference( + self, + waveform: Any, + sample_rate: Any, + ) -> Tuple[Dict[str, Any], str]: + mono = self._mono_waveform(waveform) + sample_rate = int(sample_rate) + comfy_audio = { + "waveform": mono.unsqueeze(0), + "sample_rate": sample_rate, + } + processor_audio = { + "array": mono, + "sampling_rate": sample_rate, + } + return ( + processor_audio, + generate_stable_audio_component(reference_audio=comfy_audio), + ) + + def _extract_voice_reference( + self, + voice_ref: Optional[Dict[str, Any]], + ) -> Tuple[Any, str, str]: + if not isinstance(voice_ref, dict): + return None, "", "default_voice" + + reference_text = str( + voice_ref.get("reference_text") + or voice_ref.get("prompt_text") + or voice_ref.get("text") + or "" + ).strip() + reference_audio = effective_voice_audio(voice_ref) + if reference_audio is None: + return None, reference_text, "default_voice" + + if isinstance(reference_audio, (str, os.PathLike)): + path = os.fspath(reference_audio) + component = generate_stable_audio_component(audio_file_path=path) + return path, reference_text, component + + if isinstance(reference_audio, dict): + waveform = reference_audio.get( + "waveform", + reference_audio.get("array"), + ) + sample_rate = reference_audio.get( + "sample_rate", + reference_audio.get("sampling_rate"), + ) + if waveform is None or sample_rate is None: + raise ValueError( + "Audio8 TTS in-memory reference requires waveform and sample_rate" + ) + normalized, component = self._in_memory_reference( + waveform, + sample_rate, + ) + return normalized, reference_text, component + + if isinstance(reference_audio, (tuple, list)) and len(reference_audio) == 2: + normalized, component = self._in_memory_reference( + reference_audio[0], + reference_audio[1], + ) + return normalized, reference_text, component + + if torch.is_tensor(reference_audio): + normalized, component = self._in_memory_reference( + reference_audio, + voice_ref.get("sample_rate", self.SAMPLE_RATE), + ) + return normalized, reference_text, component + + raise TypeError( + f"Unsupported Audio8 TTS reference type: {type(reference_audio)}" + ) + + def generate_single( + self, + text: str, + voice_ref: Optional[Dict[str, Any]], + seed: int = 0, + enable_audio_cache: bool = True, + character_name: Optional[str] = None, + ) -> torch.Tensor: + """Generate one raw utterance; chunking remains processor-owned.""" + text = str(text or "").strip() + if not text: + return torch.zeros(1, 0, dtype=torch.float32) + + reference_audio, reference_text, audio_component = ( + self._extract_voice_reference(voice_ref) + ) + if reference_audio is not None and not reference_text: + raise ValueError( + "Audio8 TTS voice cloning requires the exact transcript of " + "the reference audio. Add reference text to Character Voices " + "or the narrator voice." + ) + + params = { + "model_variant": self._model_selection(), + "max_new_tokens": int(self.config.get("max_new_tokens", 1024)), + "retry_max_new_tokens": int(self.config.get("retry_max_new_tokens", 2000)), + "temperature": float(self.config.get("temperature", 0.8)), + "top_p": float(self.config.get("top_p", 0.95)), + "top_k": int(self.config.get("top_k", 50)), + "do_sample": bool(self.config.get("do_sample", True)), + "dtype": self.config.get("dtype", "auto"), + "device": resolve_torch_device(self.config.get("device", "auto")), + "seed": int(seed), + } + + cache_key = None + if enable_audio_cache: + cache_key = self.audio_cache.generate_cache_key( + "audio8_tts", + text=text, + audio_component=audio_component, + reference_text=reference_text, + character=character_name or "narrator", + **params, + ) + cached = self.audio_cache.get_cached_audio(cache_key) + if cached: + display_name = resolved_character_label( + character_name or "narrator", + voice_ref, + ) + print( + "💾 Using cached Audio8 TTS audio for " + f"'{display_name}': '{text[:30]}...'" + ) + return cached[0] + + self._ensure_model_loaded() + audio = self._get_engine().generate( + text=text, + reference_audio=reference_audio, + reference_text=reference_text or None, + max_new_tokens=params["max_new_tokens"], + retry_max_new_tokens=params["retry_max_new_tokens"], + temperature=params["temperature"], + top_p=params["top_p"], + top_k=params["top_k"], + do_sample=params["do_sample"], + seed=params["seed"], + ) + if not isinstance(audio, torch.Tensor): + audio = torch.as_tensor(audio, dtype=torch.float32) + audio = audio.detach().float().cpu() + if audio.ndim == 1: + audio = audio.unsqueeze(0) + if audio.ndim != 2: + raise RuntimeError( + f"Audio8 TTS returned invalid audio shape: {tuple(audio.shape)}" + ) + + if cache_key: + self.audio_cache.cache_audio( + cache_key, + audio, + audio.shape[-1] / self.SAMPLE_RATE, + ) + return audio diff --git a/engines/audio8_tts/__init__.py b/engines/audio8_tts/__init__.py new file mode 100644 index 00000000..1bef6408 --- /dev/null +++ b/engines/audio8_tts/__init__.py @@ -0,0 +1,6 @@ +"""Audio8 TTS engine integration.""" + +from .audio8_tts_engine import Audio8TTSEngine +from .downloader import Audio8TTSDownloader + +__all__ = ["Audio8TTSEngine", "Audio8TTSDownloader"] diff --git a/engines/audio8_tts/audio8_tts_engine.py b/engines/audio8_tts/audio8_tts_engine.py new file mode 100644 index 00000000..34b0903b --- /dev/null +++ b/engines/audio8_tts/audio8_tts_engine.py @@ -0,0 +1,375 @@ +"""Wrapper for the official Audio8 TTS remote-code model.""" + +from __future__ import annotations + +import os +import random +import warnings +from typing import Any, Dict, Optional, Tuple + +import torch + +from engines.audio8_tts.downloader import Audio8TTSDownloader +from engines.audio8_tts.progress import build_audio8_stopping_criteria +from utils.device import resolve_torch_device + + +class Audio8TTSEngine: + """ComfyUI-friendly wrapper around the official Audio8 model.""" + + SAMPLE_RATE = 44100 + + def __init__( + self, + model_name: str = Audio8TTSDownloader.MODEL_NAME, + device: str = "auto", + dtype: str = "auto", + model_dir: Optional[str] = None, + ): + self.model_name = model_name + self.device = resolve_torch_device(device) + self.dtype_name = str(dtype or "auto").lower() + self.dtype = self._resolve_dtype(self.dtype_name, self.device) + self.model_dir = model_dir or Audio8TTSDownloader().resolve_model_path( + model_name + ) + self._processor = None + self._model = None + self._codec = None + + @staticmethod + def _resolve_dtype(dtype: str, device: str) -> torch.dtype: + normalized = str(dtype or "auto").lower() + valid = { + "bfloat16": torch.bfloat16, + "float16": torch.float16, + "float32": torch.float32, + } + if normalized not in {"auto", *valid}: + raise ValueError( + "Audio8 TTS dtype must be auto, bfloat16, float16, or float32" + ) + if str(device).startswith("cpu"): + return torch.float32 + if normalized in valid: + return valid[normalized] + if str(device).startswith("cuda") and torch.cuda.is_available(): + major, _minor = torch.cuda.get_device_capability(torch.device(device)) + return torch.bfloat16 if major >= 8 else torch.float16 + if str(device).startswith("xpu"): + return torch.bfloat16 + return torch.float16 + + def _ensure_model_loaded(self): + """Load the processor, model, and bundled codec strictly from local files.""" + if ( + self._processor is not None + and self._model is not None + and self._codec is not None + ): + return self + + if not os.path.isdir(self.model_dir): + raise FileNotFoundError( + f"Audio8 TTS local model directory does not exist: {self.model_dir}" + ) + + try: + from transformers import AutoModel, AutoProcessor + except ImportError as exc: + raise ImportError( + "Audio8 TTS requires the suite's shared Transformers 4 runtime" + ) from exc + + print( + f"📦 Loading Audio8 TTS from {self.model_dir} " + f"on {self.device} ({self.dtype})" + ) + with warnings.catch_warnings(): + warnings.filterwarnings( + "ignore", + message=( + r"`torch\.nn\.utils\.weight_norm` is deprecated in favor of " + r"`torch\.nn\.utils\.parametrizations\.weight_norm`\." + ), + category=FutureWarning, + ) + self._processor = AutoProcessor.from_pretrained( + self.model_dir, + trust_remote_code=True, + local_files_only=True, + ) + self._model = AutoModel.from_pretrained( + self.model_dir, + trust_remote_code=True, + local_files_only=True, + dtype=self.dtype, + ) + self._model = self._model.eval().to(self.device) + + # The official model stores the codec outside nn.Module registration. + # Hold and move it explicitly so model offload/reload cannot strand it. + self._codec = self._model.load_codec( + device=self.device, + dtype=self.dtype, + ) + self._codec.eval() + + sample_rate = int( + getattr(self._model.config, "codec_sample_rate", self.SAMPLE_RATE) + ) + if sample_rate != self.SAMPLE_RATE: + raise RuntimeError( + f"Audio8 TTS checkpoint reports unexpected sample rate " + f"{sample_rate}; expected {self.SAMPLE_RATE}" + ) + print("✅ Audio8 TTS model and codec loaded") + return self + + # Compatibility alias for callers using the shorter established name. + _ensure_loaded = _ensure_model_loaded + + def _clear_kv_caches(self) -> None: + """Drop every static slow/fast attention cache before device movement.""" + if self._model is None: + return + for collection_name in ("layers", "fast_layers"): + for layer in getattr(self._model, collection_name, ()) or (): + attention = getattr(layer, "attention", None) + if attention is not None and hasattr(attention, "kv_cache"): + attention.kv_cache = None + + def to(self, device): + """Move all weights for ComfyUI Clear VRAM and reload handling.""" + target = resolve_torch_device( + str(device) if not isinstance(device, str) else device + ) + self._clear_kv_caches() + self.device = target + if self._model is not None: + self._model = self._model.to(target).eval() + if self._codec is not None: + codec_dtype = torch.float32 if str(target).startswith("cpu") else self.dtype + self._codec = self._codec.to( + device=target, + dtype=codec_dtype, + ).eval() + return self + + def parameters(self): + """Expose all weight tensors to ComfyUI memory accounting.""" + if self._model is not None: + yield from self._model.parameters() + if self._codec is not None: + yield from self._codec.parameters() + + @staticmethod + def _mono_audio(waveform: Any) -> torch.Tensor: + audio = torch.as_tensor(waveform, dtype=torch.float32).detach().cpu() + if audio.ndim == 3: + if audio.shape[0] != 1: + raise ValueError( + "Audio8 TTS reference audio must contain one batch item" + ) + audio = audio[0] + if audio.ndim == 2: + audio = audio.mean(dim=0) + if audio.ndim != 1 or audio.numel() == 0: + raise ValueError( + "Audio8 TTS reference audio must be non-empty mono or " + "channels-first audio" + ) + return audio.contiguous() + + def _normalize_reference_audio( + self, + reference_audio: Any, + ) -> Tuple[Any, Optional[int]]: + if isinstance(reference_audio, (str, os.PathLike)): + return os.fspath(reference_audio), None + if isinstance(reference_audio, dict): + waveform = reference_audio.get( + "waveform", + reference_audio.get("array"), + ) + sample_rate = reference_audio.get( + "sample_rate", + reference_audio.get("sampling_rate"), + ) + if waveform is None or sample_rate is None: + raise ValueError( + "Audio8 TTS in-memory reference audio requires waveform " + "and sample_rate" + ) + return { + "array": self._mono_audio(waveform), + "sampling_rate": int(sample_rate), + }, int(sample_rate) + if isinstance(reference_audio, (tuple, list)) and len(reference_audio) == 2: + waveform, sample_rate = reference_audio + if not isinstance(sample_rate, (int, float)): + raise ValueError( + "Audio8 TTS reference tuple must be (waveform, sample_rate)" + ) + return { + "array": self._mono_audio(waveform), + "sampling_rate": int(sample_rate), + }, int(sample_rate) + raise TypeError( + f"Unsupported Audio8 TTS reference audio type: {type(reference_audio)}" + ) + + def _prepare_inputs( + self, + text: str, + reference_audio: Any, + reference_text: Optional[str], + ) -> Dict[str, torch.Tensor]: + processor_kwargs: Dict[str, Any] = { + "text": [text], + "return_tensors": "pt", + } + if reference_audio is not None: + normalized_audio, sample_rate = self._normalize_reference_audio( + reference_audio + ) + processor_kwargs.update( + reference_audio=[normalized_audio], + reference_text=[reference_text], + ) + if sample_rate is not None: + processor_kwargs["sampling_rate"] = [sample_rate] + + inputs = self._processor(**processor_kwargs) + return {name: value.to(self.device) for name, value in inputs.items()} + + def _make_generator(self, seed: int): + seed = int(seed) + random.seed(seed) + torch.manual_seed(seed) + if torch.cuda.is_available(): + torch.cuda.manual_seed_all(seed) + try: + return torch.Generator(device=torch.device(self.device)).manual_seed(seed) + except Exception: + return None + + def _generate_once( + self, + inputs: Dict[str, torch.Tensor], + *, + max_new_tokens: int, + temperature: float, + top_p: float, + top_k: int, + do_sample: bool, + generator, + ): + criteria, tracker = build_audio8_stopping_criteria(max_new_tokens) + try: + output = self._model.generate( + **inputs, + max_new_tokens=max_new_tokens, + temperature=temperature, + top_p=top_p, + top_k=top_k, + do_sample=do_sample, + generator=generator, + stopping_criteria=criteria, + return_dict_in_generate=True, + ) + except BaseException: + tracker.abort() + raise + tracker.close() + return output + + def generate( + self, + text: str, + reference_audio: Any = None, + reference_text: Optional[str] = None, + *, + max_new_tokens: int = 1024, + retry_max_new_tokens: int = 2000, + temperature: float = 0.8, + top_p: float = 0.95, + top_k: int = 50, + do_sample: bool = True, + seed: int = 42, + ) -> torch.Tensor: + """Generate one utterance as a CPU float tensor ``[channels, samples]``.""" + text = str(text or "").strip() + if not text: + return torch.zeros(1, 0, dtype=torch.float32) + + reference_text = str(reference_text or "").strip() + if reference_audio is not None and not reference_text: + raise ValueError( + "Audio8 TTS voice cloning requires the exact transcript of " + "the reference audio" + ) + if reference_audio is None and reference_text: + raise ValueError( + "Audio8 TTS reference_text requires matching reference audio" + ) + + max_new_tokens = int(max_new_tokens) + retry_max_new_tokens = int(retry_max_new_tokens) + temperature = float(temperature) + top_p = float(top_p) + top_k = int(top_k) + if max_new_tokens < 1: + raise ValueError("Audio8 TTS max_new_tokens must be positive") + if retry_max_new_tokens < max_new_tokens: + raise ValueError( + "Audio8 TTS retry_max_new_tokens must be >= max_new_tokens" + ) + if temperature <= 0: + raise ValueError("Audio8 TTS temperature must be positive") + if not 0 < top_p <= 1: + raise ValueError("Audio8 TTS top_p must be in (0, 1]") + if top_k < 1: + raise ValueError("Audio8 TTS top_k must be at least 1") + + self._ensure_model_loaded() + inputs = self._prepare_inputs(text, reference_audio, reference_text) + generator = self._make_generator(seed) + + output = self._generate_once( + inputs, + max_new_tokens=max_new_tokens, + temperature=temperature, + top_p=top_p, + top_k=top_k, + do_sample=bool(do_sample), + generator=generator, + ) + finished = bool(output.finished[0].item()) + + if not finished and retry_max_new_tokens > max_new_tokens: + print( + "⚠️ Audio8 TTS reached max_new_tokens without EOS; " + f"retrying with {retry_max_new_tokens}" + ) + output = self._generate_once( + inputs, + max_new_tokens=retry_max_new_tokens, + temperature=temperature, + top_p=top_p, + top_k=top_k, + do_sample=bool(do_sample), + generator=generator, + ) + finished = bool(output.finished[0].item()) + + if not finished: + print( + "⚠️ Audio8 TTS generation ended without EOS; returning the " + "valid decoded frames" + ) + + waveforms, waveform_lengths = self._model.decode_audio(output.codes) + waveform_length = int(waveform_lengths[0].item()) + waveform = waveforms[0, :waveform_length].detach().float().cpu() + return waveform.unsqueeze(0).contiguous() diff --git a/engines/audio8_tts/downloader.py b/engines/audio8_tts/downloader.py new file mode 100644 index 00000000..e27528a2 --- /dev/null +++ b/engines/audio8_tts/downloader.py @@ -0,0 +1,180 @@ +"""Organized model download and discovery for official Audio8 TTS.""" + +from __future__ import annotations + +import os +from typing import List, Optional + +import folder_paths + +from utils.downloads.unified_downloader import unified_downloader +from utils.models.extra_paths import ( + get_all_tts_model_paths, + get_preferred_download_path, +) + + +class Audio8TTSDownloader: + """Resolve Audio8 checkpoints without using the Hugging Face model cache.""" + + MODEL_NAME = "Audio8-TTS-Preview-0.6b" + REPO_ID = "Audio8/Audio8-TTS-Preview-0.6b" + REVISION = "1b17c91db5f4dccb6914aa4aa5cb0e56661a6c17" + + # Complete runtime snapshot at the pinned revision. Non-runtime model-card + # assets are deliberately excluded. + REQUIRED_FILES = [ + "codec.pth", + "config.json", + "configuration_arktts.py", + "generation_config.json", + "model.safetensors", + "modeling_arktts.py", + "modeling_arktts_codec.py", + "preprocessor_config.json", + "processing_arktts.py", + "processor_config.json", + "special_tokens_map.json", + "tokenizer.json", + "tokenizer_config.json", + ] + + def __init__(self, base_path: Optional[str] = None): + if base_path is None: + try: + self.base_path = get_preferred_download_path( + model_type="TTS", + engine_name="audio8_tts", + ) + except Exception: + self.base_path = os.path.join( + folder_paths.models_dir, + "TTS", + "audio8_tts", + ) + else: + self.base_path = os.path.abspath(os.fspath(base_path)) + os.makedirs(self.base_path, exist_ok=True) + + def get_available_models(self) -> List[str]: + """Return the canonical model plus complete local installations.""" + models = [self.MODEL_NAME] + canonical_path = os.path.normcase( + os.path.abspath(os.path.join(self.base_path, self.MODEL_NAME)) + ) + for base_path in get_all_tts_model_paths("TTS"): + for folder_name in ("audio8_tts", "Audio8_TTS", ""): + root = ( + os.path.join(base_path, folder_name) if folder_name else base_path + ) + if not os.path.isdir(root): + continue + for item in sorted(os.listdir(root)): + candidate = os.path.join(root, item) + if os.path.normcase(os.path.abspath(candidate)) == canonical_path: + continue + local_name = f"local:{item}" + if local_name not in models and self._is_model_complete(candidate): + models.append(local_name) + return models + + def resolve_model_path( + self, + model_identifier: str = MODEL_NAME, + ) -> str: + """Resolve an absolute path, ``local:`` name, or canonical model name.""" + model_identifier = str(model_identifier or self.MODEL_NAME).strip() + + if os.path.isabs(model_identifier) or os.path.isdir(model_identifier): + candidate = os.path.abspath(model_identifier) + if self._is_model_complete(candidate, verbose=True): + return candidate + raise FileNotFoundError( + f"Audio8 TTS model path is missing required files: {candidate}" + ) + + if model_identifier.startswith("local:"): + local_name = model_identifier[6:].strip() + if not local_name: + raise ValueError("Audio8 TTS local model name must not be empty") + for base_path in get_all_tts_model_paths("TTS"): + for folder_name in ("audio8_tts", "Audio8_TTS", ""): + candidate = ( + os.path.join(base_path, folder_name, local_name) + if folder_name + else os.path.join(base_path, local_name) + ) + if self._is_model_complete(candidate): + print(f"📁 Using local Audio8 TTS model: {candidate}") + return candidate + raise FileNotFoundError( + f"Local Audio8 TTS model not found or incomplete: {local_name}" + ) + + if model_identifier != self.MODEL_NAME: + raise ValueError(f"Unknown Audio8 TTS model: {model_identifier}") + return self.get_model_path() + + def get_model_path(self, model_name: str = MODEL_NAME) -> str: + """Return the organized canonical model path, downloading if needed.""" + if model_name != self.MODEL_NAME: + raise ValueError(f"Unknown Audio8 TTS model: {model_name}") + model_dir = os.path.join(self.base_path, self.MODEL_NAME) + if not self._is_model_complete(model_dir): + return self.download_model(model_dir) + return model_dir + + def download_model( + self, + model_dir: Optional[str] = None, + force: bool = False, + ) -> str: + """Download the pinned official snapshot into the organized model folder.""" + model_dir = model_dir or os.path.join(self.base_path, self.MODEL_NAME) + + print("\n" + "=" * 60) + print("📦 Audio8 TTS Model Download") + print("=" * 60) + print(f"Repository: {self.REPO_ID}") + print(f"Revision: {self.REVISION}") + print(f"Target: {model_dir}") + print("=" * 60 + "\n") + + unified_downloader.download_huggingface_snapshot( + repo_id=self.REPO_ID, + target_dir=model_dir, + revision=self.REVISION, + allow_patterns=self.REQUIRED_FILES, + required_files=self.REQUIRED_FILES, + force_download=bool(force), + description=self.MODEL_NAME, + ) + if not self._is_model_complete(model_dir, verbose=True): + raise RuntimeError( + f"Downloaded Audio8 TTS model is incomplete: {model_dir}" + ) + print(f"✅ Audio8 TTS model ready: {model_dir}") + return model_dir + + def _is_model_complete( + self, + model_dir: str, + *, + verbose: bool = False, + ) -> bool: + if not os.path.isdir(model_dir): + return False + missing = [ + rel_path + for rel_path in self.REQUIRED_FILES + if not os.path.isfile(os.path.join(model_dir, rel_path)) + or os.path.getsize(os.path.join(model_dir, rel_path)) <= 0 + ] + if missing and verbose: + print( + f"❌ Audio8 TTS model incomplete. Missing or empty " + f"{len(missing)} file(s):" + ) + for rel_path in missing: + print(f" - {rel_path}") + return not missing diff --git a/engines/audio8_tts/progress.py b/engines/audio8_tts/progress.py new file mode 100644 index 00000000..7c2e8901 --- /dev/null +++ b/engines/audio8_tts/progress.py @@ -0,0 +1,116 @@ +"""Native generation progress and interruption support for Audio8 TTS.""" + +from __future__ import annotations + +import time +from typing import Any, Optional + +import torch +from transformers import StoppingCriteria, StoppingCriteriaList + + +def _make_comfy_progress_bar(total_steps: int) -> Optional[Any]: + try: + from comfy.utils import ProgressBar + + return ProgressBar(total_steps) + except Exception: + return None + + +class Audio8TTSStoppingCriteria(StoppingCriteria): + """Track native codec-frame generation and honor ComfyUI interruption.""" + + def __init__(self, max_steps: int, progress_bar: Optional[Any] = None): + self.max_steps = max(1, int(max_steps)) + self.progress_bar = progress_bar + self.steps = 0 + self.start_time = time.monotonic() + self.last_print_time = self.start_time + self.last_print_step = 0 + self.rendered = False + + @staticmethod + def _check_interrupted() -> None: + try: + import comfy.model_management as model_management + except Exception: + return + + checker = getattr( + model_management, + "throw_exception_if_processing_interrupted", + None, + ) + if callable(checker): + checker() + elif getattr(model_management, "interrupt_processing", False): + raise InterruptedError("Audio8 TTS generation interrupted by user") + + def __call__(self, input_ids, scores, **kwargs): + del scores, kwargs + self._check_interrupted() + self.steps = min(self.steps + 1, self.max_steps) + if self.progress_bar is not None: + try: + self.progress_bar.update(1) + except Exception: + pass + + now = time.monotonic() + if ( + self.steps == 1 + or self.steps >= self.max_steps + or now - self.last_print_time >= 0.5 + ): + delta_time = max(now - self.last_print_time, 1e-6) + delta_steps = self.steps - self.last_print_step + rate = delta_steps / delta_time + elapsed = now - self.start_time + width = 12 + filled = min( + width, + int(width * self.steps / self.max_steps), + ) + bar = "█" * filled + "░" * (width - filled) + print( + f"\r Audio8: [{bar}] {self.steps}/{self.max_steps} | " + f"{rate:.1f} frames/s | {elapsed:.1f}s ", + end="", + flush=True, + ) + self.last_print_time = now + self.last_print_step = self.steps + self.rendered = True + + # The official model handles EOS itself. Interruption raises above. + batch_size = int(input_ids.shape[0]) if input_ids.ndim else 1 + return torch.zeros( + batch_size, + dtype=torch.bool, + device=input_ids.device, + ) + + def close(self) -> None: + elapsed = max(time.monotonic() - self.start_time, 1e-6) + average = self.steps / elapsed + if self.rendered: + print( + f"\r Audio8 complete: {self.steps} frames in " + f"{elapsed:.1f}s ({average:.1f} frames/s)" + " " * 20 + ) + + def abort(self) -> None: + if self.rendered: + print() + + +def build_audio8_stopping_criteria( + max_steps: int, +) -> tuple[StoppingCriteriaList, Audio8TTSStoppingCriteria]: + """Create the native stopping-criteria list and its progress tracker.""" + tracker = Audio8TTSStoppingCriteria( + max_steps=max_steps, + progress_bar=_make_comfy_progress_bar(max(1, int(max_steps))), + ) + return StoppingCriteriaList([tracker]), tracker diff --git a/nodes.py b/nodes.py index bdf1ec12..f3546650 100644 --- a/nodes.py +++ b/nodes.py @@ -179,6 +179,14 @@ def load_node_module(module_name, file_name): print(f"❌ Dots TTS Engine failed: {e}") DOTS_TTS_ENGINE_AVAILABLE = False +try: + audio8_tts_engine_module = load_node_module("audio8_tts_engine_node", "engines/audio8_tts_engine_node.py") + Audio8TTSEngineNode = audio8_tts_engine_module.Audio8TTSEngineNode + AUDIO8_TTS_ENGINE_AVAILABLE = True +except Exception as e: + print(f"❌ Audio8 TTS Engine failed: {e}") + AUDIO8_TTS_ENGINE_AVAILABLE = False + try: dramabox_engine_module = load_node_module("dramabox_engine_node", "engines/dramabox_engine_node.py") DramaBoxEngineNode = dramabox_engine_module.DramaBoxEngineNode @@ -670,6 +678,10 @@ def error(self, error): NODE_CLASS_MAPPINGS["DotsTTSEngineNode"] = DotsTTSEngineNode NODE_DISPLAY_NAME_MAPPINGS["DotsTTSEngineNode"] = "⚙️ Dots TTS Engine" +if AUDIO8_TTS_ENGINE_AVAILABLE: + NODE_CLASS_MAPPINGS["Audio8TTSEngineNode"] = Audio8TTSEngineNode + NODE_DISPLAY_NAME_MAPPINGS["Audio8TTSEngineNode"] = "⚙️ Audio8 TTS Engine" + if DRAMABOX_ENGINE_AVAILABLE: NODE_CLASS_MAPPINGS["DramaBoxEngineNode"] = DramaBoxEngineNode NODE_DISPLAY_NAME_MAPPINGS["DramaBoxEngineNode"] = "⚙️ DramaBox Engine" diff --git a/nodes/audio8_tts/__init__.py b/nodes/audio8_tts/__init__.py new file mode 100644 index 00000000..f6f5a295 --- /dev/null +++ b/nodes/audio8_tts/__init__.py @@ -0,0 +1 @@ +"""Audio8 TTS text and SRT processors.""" diff --git a/nodes/audio8_tts/audio8_tts_processor.py b/nodes/audio8_tts/audio8_tts_processor.py new file mode 100644 index 00000000..793fb098 --- /dev/null +++ b/nodes/audio8_tts/audio8_tts_processor.py @@ -0,0 +1,403 @@ +"""Suite orchestration for Audio8 TTS text generation.""" + +import os +import re +import sys +from typing import Any, Dict, List, Tuple, Union + +import comfy.model_management as model_management +import torch + +current_dir = os.path.dirname(__file__) +nodes_dir = os.path.dirname(current_dir) +project_root = os.path.dirname(nodes_dir) +if project_root not in sys.path: + sys.path.insert(0, project_root) + +from utils.audio.chunk_timing import ChunkTimingHelper +from utils.audio.edit_post_processor import ( + process_segments as apply_edit_post_processing, +) +from utils.text.character_parser import character_parser +from utils.text.pause_processor import PauseTagProcessor +from utils.text.segment_parameters import ParameterValidator, apply_segment_parameters +from utils.text.step_audio_editx_special_tags import get_edit_tags_for_segment +from utils.voice.character_logging import ( + format_resolved_character_block, + resolved_character_label, +) +from utils.voice.discovery import ( + get_available_characters, + get_character_mapping, + voice_discovery, +) +from utils.voice.reference import effective_voice_audio + + +class Audio8TTSProcessor: + """Handle suite-level tags, voices, chunking, and Audio8 generation.""" + + SAMPLE_RATE = 44100 + + def __init__(self, adapter, engine_config: Dict[str, Any]): + self.adapter = adapter + self.config = engine_config.copy() if engine_config else {} + + def update_config(self, new_config: Dict[str, Any]): + self.config = new_config.copy() if new_config else {} + self.adapter.update_config(self.config) + + @staticmethod + def _check_interrupt(context: str = "") -> None: + if not model_management.interrupt_processing: + return + suffix = f" {context}" if context else "" + raise InterruptedError(f"Audio8 TTS generation interrupted{suffix}") + + def _setup_character_parser(self, text: str) -> None: + character_tags = re.findall(r"\[([^\]]+)\]", text or "") + characters_from_tags = [ + tag.split("|")[0].strip() + for tag in character_tags + if not tag.lower().startswith("pause:") + ] + + all_available = set(get_available_characters() or []) + for alias, target in voice_discovery.get_character_aliases().items(): + all_available.add(alias.lower()) + all_available.add(target.lower()) + all_available.update( + character.lower() for character in characters_from_tags if character + ) + all_available.add("narrator") + + character_parser.set_available_characters(list(all_available)) + character_parser.reset_session_cache() + + @staticmethod + def _as_voice_reference(value: Any) -> Dict[str, Any]: + if isinstance(value, dict): + return value.copy() + if value is None: + return {} + return {"audio": value} + + def _resolve_voice( + self, + character: str, + voice_mapping: Dict[str, Any], + discovered_mapping: Dict[str, Tuple[Any, Any]], + ) -> Dict[str, Any]: + if character in voice_mapping: + mapped_voice = self._as_voice_reference(voice_mapping[character]) + if effective_voice_audio(mapped_voice) is not None or mapped_voice.get( + "reference_text" + ): + return mapped_voice + + if character != "narrator": + audio_path, reference_text = discovered_mapping.get(character, (None, None)) + if audio_path: + return { + "audio_path": audio_path, + "reference_text": reference_text or "", + } + print( + f"⚠️ Audio8 TTS: No voice found for '{character}'; " + "using narrator/no-reference fallback" + ) + + return self._as_voice_reference(voice_mapping.get("narrator")) + + @staticmethod + def _voice_log_note(voice_ref: Dict[str, Any]) -> str: + reference_text = "" + if isinstance(voice_ref, dict): + reference_text = str(voice_ref.get("reference_text") or "").strip() + has_audio = ( + isinstance(voice_ref, dict) and effective_voice_audio(voice_ref) is not None + ) + + if not has_audio and not reference_text: + return " [no reference voice]" + if has_audio and reference_text: + return f" [reference transcript: {len(reference_text)} chars]" + if has_audio: + return " [reference audio has no transcript; adapter will validate]" + return " [reference transcript has no audio; adapter will validate]" + + @staticmethod + def _format_parameter_log( + filtered_params: Dict[str, Any], + current_config: Dict[str, Any], + current_seed: int, + ) -> str: + if not filtered_params: + return "" + + values = [] + for key in ("seed", "temperature", "top_p", "top_k", "max_new_tokens"): + if key not in filtered_params: + continue + value = current_seed if key == "seed" else current_config.get(key) + values.append(f"{key}={value}") + return ", ".join(values) + + @staticmethod + def _validate_segment_config( + config: Dict[str, Any], + filtered_params: Dict[str, Any], + ) -> Dict[str, Any]: + """Apply Audio8-specific bounds and keep retry budgets consistent.""" + if "top_p" in filtered_params and float(config["top_p"]) <= 0: + raise ValueError("Audio8 segment top_p must be greater than 0") + + if "max_new_tokens" in filtered_params: + max_new_tokens = int(config["max_new_tokens"]) + if max_new_tokens > 2048: + raise ValueError("Audio8 segment max_new_tokens must be at most 2048") + config["retry_max_new_tokens"] = max( + max_new_tokens, + int(config.get("retry_max_new_tokens", 2000)), + ) + return config + + def _log_generation( + self, + character: str, + text: str, + voice_ref: Dict[str, Any], + config: Dict[str, Any], + chunk_count: int, + parameter_log: str, + show_text_logging: bool, + ) -> None: + display_name = resolved_character_label(character, voice_ref) + print( + f"🎭 Audio8 TTS - Generating for '{display_name}'" + f"{self._voice_log_note(voice_ref)}" + ) + print( + " Settings: " + f"mode={'Sampling' if config.get('do_sample', True) else 'Greedy'}, " + f"temperature={config.get('temperature', 0.8)}, " + f"top_p={config.get('top_p', 0.95)}, " + f"top_k={config.get('top_k', 50)}, " + f"max_new_tokens={config.get('max_new_tokens', 1024)}" + ) + if parameter_log: + print(f"🎛️ Audio8 TTS segment params: {parameter_log}") + if show_text_logging: + print(format_resolved_character_block(character, text, voice_ref)) + if chunk_count > 1: + print( + f"📝 Audio8 TTS: Chunking '{display_name}' into " + f"{chunk_count} suite chunks" + ) + + @staticmethod + def _normalize_audio(audio: Any) -> torch.Tensor: + if not isinstance(audio, torch.Tensor): + audio = torch.as_tensor(audio, dtype=torch.float32) + audio = audio.detach().to(device="cpu", dtype=torch.float32) + if audio.dim() == 1: + audio = audio.unsqueeze(0) + elif audio.dim() == 3 and audio.shape[0] == 1: + audio = audio.squeeze(0) + if audio.dim() != 2: + raise ValueError( + "Audio8 adapter returned an invalid waveform shape; " + "expected [channels, samples]" + ) + return audio + + def process_text( + self, + text: str, + voice_mapping: Dict[str, Any], + seed: int, + enable_chunking: bool = True, + max_chars_per_chunk: int = 400, + chunk_combination_method: str = "auto", + silence_between_chunks_ms: int = 100, + enable_audio_cache: bool = True, + apply_edit_postprocessing: bool = True, + show_text_logging: bool = True, + **_unused, + ) -> List[Dict[str, Any]]: + del chunk_combination_method, silence_between_chunks_ms + + self._check_interrupt() + self._setup_character_parser(text) + base_config = self.config.copy() + segment_objects = character_parser.parse_text_segments(text) + if not segment_objects: + segment_objects = character_parser.parse_text_segments( + "narrator " + (text or "") + ) + + characters = list( + {segment.character for segment in segment_objects if segment.character} + ) + discovered_mapping = get_character_mapping(characters, engine_type="audio_only") + segment_records: List[Dict[str, Any]] = [] + + try: + for segment_index, segment in enumerate(segment_objects): + character = segment.character or "narrator" + self._check_interrupt( + f"before segment {segment_index + 1}/{len(segment_objects)} " + f"for '{character}'" + ) + segment_text = (segment.text or "").strip() + if not segment_text: + continue + + segment_params = segment.parameters or {} + filtered_params = ParameterValidator.filter_parameters_for_engine( + segment_params, "audio8_tts" + ) + current_config = ( + apply_segment_parameters(base_config, filtered_params, "audio8_tts") + if filtered_params + else base_config.copy() + ) + current_config = self._validate_segment_config( + current_config, filtered_params + ) + current_seed = int(current_config.get("seed", seed)) + parameter_log = self._format_parameter_log( + filtered_params, current_config, current_seed + ) + self.adapter.update_config(current_config) + + voice_ref = self._resolve_voice( + character, voice_mapping, discovered_mapping + ) + seed_offset = 0 + + def generate_chunks(text_content: str, edit_tags: list) -> None: + nonlocal seed_offset + clean_content = (text_content or "").strip() + if not clean_content: + return + + if enable_chunking: + from utils.text.chunking import ImprovedChatterBoxChunker + + max_chars = ImprovedChatterBoxChunker.validate_chunking_params( + max_chars_per_chunk + ) + chunks = ImprovedChatterBoxChunker.split_into_chunks( + clean_content, max_chars=max_chars + ) + else: + chunks = [clean_content] + chunks = [chunk.strip() for chunk in chunks if chunk.strip()] + + self._log_generation( + character=character, + text=clean_content, + voice_ref=voice_ref, + config=current_config, + chunk_count=len(chunks), + parameter_log=parameter_log, + show_text_logging=show_text_logging, + ) + + for chunk_index, chunk in enumerate(chunks): + self._check_interrupt( + f"before chunk {chunk_index + 1}/{len(chunks)} " + f"for '{character}'" + ) + chunk_seed = current_seed + seed_offset + seed_offset += 1 + audio = self.adapter.generate_single( + text=chunk, + voice_ref=voice_ref, + seed=chunk_seed, + enable_audio_cache=enable_audio_cache, + character_name=character, + ) + segment_records.append( + { + "waveform": self._normalize_audio(audio), + "sample_rate": self.SAMPLE_RATE, + "text": chunk, + "edit_tags": (edit_tags if chunk_index == 0 else []), + } + ) + + if PauseTagProcessor.has_pause_tags(segment_text): + pause_segments, _ = PauseTagProcessor.parse_pause_tags(segment_text) + for fragment_type, fragment_content in pause_segments: + self._check_interrupt( + f"while processing segment {segment_index + 1}" + ) + if fragment_type == "text": + clean_text, edit_tags = get_edit_tags_for_segment( + fragment_content + ) + generate_chunks(clean_text, edit_tags) + elif fragment_type == "pause": + silence = PauseTagProcessor.create_silence_segment( + fragment_content, + self.SAMPLE_RATE, + torch.device("cpu"), + torch.float32, + ) + if silence.dim() == 1: + silence = silence.unsqueeze(0) + segment_records.append( + { + "waveform": silence.cpu(), + "sample_rate": self.SAMPLE_RATE, + "text": f"[pause:{fragment_content}s]", + "edit_tags": [], + } + ) + else: + clean_text, edit_tags = get_edit_tags_for_segment(segment_text) + generate_chunks(clean_text, edit_tags) + finally: + self.adapter.update_config(base_config) + + if ( + apply_edit_postprocessing + and segment_records + and any(record.get("edit_tags") for record in segment_records) + ): + self._check_interrupt("before edit post-processing") + segment_records = apply_edit_post_processing( + segment_records, engine_config=base_config + ) + for record in segment_records: + record["waveform"] = self._normalize_audio(record["waveform"]) + + return segment_records + + def combine_audio_segments( + self, + segments: List[Dict[str, Any]], + method: str = "auto", + silence_ms: int = 100, + original_text: str = "", + return_info: bool = False, + ) -> Union[torch.Tensor, Tuple[torch.Tensor, Dict[str, Any]]]: + self._check_interrupt("before audio assembly") + if not segments: + empty = torch.zeros(0, dtype=torch.float32) + return (empty, {}) if return_info else empty + + text_chunks = [segment.get("text", "") for segment in segments] + combined_audio, chunk_info = ChunkTimingHelper.combine_audio_with_timing( + audio_segments=[segment["waveform"] for segment in segments], + combination_method=method, + silence_ms=silence_ms, + crossfade_duration=0.1, + sample_rate=self.SAMPLE_RATE, + text_length=len(" ".join(text_chunks)), + original_text=original_text, + text_chunks=text_chunks, + ) + return (combined_audio, chunk_info) if return_info else combined_audio diff --git a/nodes/audio8_tts/audio8_tts_srt_processor.py b/nodes/audio8_tts/audio8_tts_srt_processor.py new file mode 100644 index 00000000..4d975da0 --- /dev/null +++ b/nodes/audio8_tts/audio8_tts_srt_processor.py @@ -0,0 +1,222 @@ +"""SRT orchestration for Audio8 TTS.""" + +import importlib.util +import os + +import comfy.model_management as model_management +import torch + +from engines.adapters.audio8_tts_adapter import Audio8TTSEngineAdapter +from utils.system.import_manager import import_manager +from utils.timing.assembly import AudioAssemblyEngine +from utils.timing.engine import TimingEngine +from utils.timing.overlap_detection import SRTOverlapHandler +from utils.timing.reporting import SRTReportGenerator + + +class Audio8TTSSRTProcessor: + """Generate Audio8 speech per subtitle and apply shared timing modes.""" + + SAMPLE_RATE = 44100 + + def __init__(self, node_instance, config): + self.node_instance = node_instance + self.config = dict(config or {}) + self.adapter = Audio8TTSEngineAdapter(self.config) + self._processor = None + + success, modules, message = import_manager.import_srt_modules() + if not success: + raise ImportError(f"Audio8 TTS SRT unavailable: {message}") + self.SRTParser = modules["SRTParser"] + + @property + def processor(self): + if self._processor is None: + processor_path = os.path.join( + os.path.dirname(__file__), "audio8_tts_processor.py" + ) + spec = importlib.util.spec_from_file_location( + "audio8_tts_processor_module", processor_path + ) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + self._processor = module.Audio8TTSProcessor(self.adapter, self.config) + return self._processor + + def update_config(self, config): + self.config = dict(config or {}) + self.adapter.update_config(self.config) + if self._processor is not None: + self._processor.update_config(self.config) + + @staticmethod + def _check_interrupt(index=None, total=None): + if not model_management.interrupt_processing: + return + location = f" at subtitle {index + 1}/{total}" if index is not None else "" + raise InterruptedError(f"Audio8 TTS SRT generation interrupted{location}") + + def process_srt_content( + self, + srt_content, + voice_mapping, + seed, + timing_mode, + timing_params, + enable_audio_cache=True, + ): + self._check_interrupt() + subtitles = self.SRTParser().parse_srt_content(srt_content, allow_overlaps=True) + has_overlaps = SRTOverlapHandler.detect_overlaps(subtitles) + active_mode, switched = SRTOverlapHandler.handle_smart_natural_fallback( + timing_mode, has_overlaps, "Audio8 TTS SRT" + ) + + audio_segments = [] + adjustments = [] + for index, subtitle in enumerate(subtitles): + self._check_interrupt(index, len(subtitles)) + text = (subtitle.text or "").strip() + if text: + records = self.processor.process_text( + text=text, + voice_mapping=voice_mapping, + seed=seed + index, + enable_chunking=False, + enable_audio_cache=enable_audio_cache, + apply_edit_postprocessing=True, + ) + audio, _ = self.processor.combine_audio_segments( + records, + method="auto", + silence_ms=0, + original_text=text, + return_info=True, + ) + if audio.dim() == 1: + audio = audio.unsqueeze(0) + else: + audio = torch.zeros(1, int(subtitle.duration * self.SAMPLE_RATE)) + + audio = audio.detach().cpu().float() + audio_segments.append(audio) + natural_duration = audio.shape[-1] / self.SAMPLE_RATE + target_duration = subtitle.duration + stretch_factor = ( + target_duration / natural_duration if natural_duration else 1.0 + ) + adjustments.append( + { + "index": index, + "segment_index": index, + "sequence": subtitle.sequence, + "natural_duration": natural_duration, + "target_start": subtitle.start_time, + "target_end": subtitle.end_time, + "target_duration": target_duration, + "start_time": subtitle.start_time, + "end_time": subtitle.end_time, + "stretch_factor": stretch_factor, + "needs_stretching": (abs(stretch_factor - 1.0) > 0.05), + "stretch_type": ( + "compress" + if stretch_factor < 1.0 + else "expand" + if stretch_factor > 1.0 + else "none" + ), + "adjustment": natural_duration - target_duration, + "adjusted_start": subtitle.start_time, + "adjusted_end": subtitle.end_time, + "adjusted_duration": natural_duration, + } + ) + + self._check_interrupt() + final_audio, replacement, stretch_method = self._assemble( + audio_segments, subtitles, active_mode, timing_params or {} + ) + self._check_interrupt() + if replacement is not None: + adjustments = replacement + + reporter = SRTReportGenerator() + timing_report = reporter.generate_timing_report( + subtitles, + adjustments, + active_mode, + has_overlaps, + switched, + timing_mode if switched else None, + stretch_method, + ) + adjusted_srt = reporter.generate_adjusted_srt_string( + subtitles, adjustments, active_mode + ) + + final_audio = final_audio.detach().cpu().float() + if final_audio.dim() == 1: + final_audio = final_audio.unsqueeze(0).unsqueeze(0) + elif final_audio.dim() == 2: + final_audio = final_audio.unsqueeze(0) + duration = final_audio.shape[-1] / self.SAMPLE_RATE + generation_info = ( + f"Generated {duration:.1f}s Audio8 TTS SRT audio from " + f"{len(subtitles)} subtitles using {active_mode} mode" + ) + return ( + {"waveform": final_audio, "sample_rate": self.SAMPLE_RATE}, + generation_info, + timing_report, + adjusted_srt, + ) + + def _assemble(self, audio_segments, subtitles, mode, params): + if mode == "stretch_to_fit": + from engines.chatterbox.audio_timing import TimedAudioAssembler + + assembler = TimedAudioAssembler(self.SAMPLE_RATE) + audio, method = assembler.assemble_timed_audio( + audio_segments, + [(subtitle.start_time, subtitle.end_time) for subtitle in subtitles], + fade_duration=params.get("fade_for_StretchToFit", 0.01), + ) + return audio, None, method + + assembler = AudioAssemblyEngine(self.SAMPLE_RATE) + if mode == "pad_with_silence": + audio = assembler.assemble_with_overlaps( + audio_segments, subtitles, torch.device("cpu") + ) + return audio, None, None + + timing = TimingEngine(self.SAMPLE_RATE) + if mode == "concatenate": + replacements = timing.calculate_concatenation_adjustments( + audio_segments, subtitles + ) + audio = assembler.assemble_concatenation( + audio_segments, + params.get("fade_for_StretchToFit", 0.01), + ) + return audio, replacements, None + + if mode != "smart_natural": + raise ValueError(f"Unsupported Audio8 SRT timing mode: {mode}") + replacements, processed = timing.calculate_smart_timing_adjustments( + audio_segments, + subtitles, + params.get("timing_tolerance", 2.0), + params.get("max_stretch_ratio", 1.0), + params.get("min_stretch_ratio", 0.5), + torch.device("cpu"), + ) + audio = assembler.assemble_smart_natural( + audio_segments, + processed, + replacements, + subtitles, + torch.device("cpu"), + ) + return audio, replacements, None diff --git a/nodes/engines/audio8_tts_engine_node.py b/nodes/engines/audio8_tts_engine_node.py new file mode 100644 index 00000000..a2431a03 --- /dev/null +++ b/nodes/engines/audio8_tts_engine_node.py @@ -0,0 +1,230 @@ +"""Audio8 TTS Preview engine configuration node.""" + +import importlib.util +import os +import sys + +current_dir = os.path.dirname(__file__) +nodes_dir = os.path.dirname(current_dir) +project_root = os.path.dirname(nodes_dir) +if project_root not in sys.path: + sys.path.insert(0, project_root) + +base_node_path = os.path.join(nodes_dir, "base", "base_node.py") +base_spec = importlib.util.spec_from_file_location("base_node_module", base_node_path) +base_module = importlib.util.module_from_spec(base_spec) +sys.modules["base_node_module"] = base_module +base_spec.loader.exec_module(base_module) +BaseTTSNode = base_module.BaseTTSNode + + +class Audio8TTSEngineNode(BaseTTSNode): + """Configure the official Audio8 TTS Preview checkpoint.""" + + DEFAULT_MODEL = "Audio8-TTS-Preview-0.6b" + + @classmethod + def NAME(cls): + return "⚙️ Audio8 TTS Engine" + + @classmethod + def _get_model_options(cls): + try: + from engines.audio8_tts.downloader import Audio8TTSDownloader + + options = Audio8TTSDownloader().get_available_models() + return options or [cls.DEFAULT_MODEL] + except Exception: + return [cls.DEFAULT_MODEL] + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "model_variant": ( + cls._get_model_options(), + { + "default": cls.DEFAULT_MODEL, + "tooltip": ( + "Official 0.6B Audio8 TTS Preview checkpoint. " + "The canonical option downloads from " + "Audio8/Audio8-TTS-Preview-0.6b; local: options are " + "detected model folders.\n\n" + "Preview scope: multilingual speech and zero-shot " + "voice cloning, with 11 recommended languages. The " + "model infers language from text and has no language " + "selection control.\n\n" + "Voice cloning requires reference audio and its exact " + "matching transcript. Generation without a reference " + "is supported. Audio8 has no instruction-conditioned " + "voice-design mode. The suite runs it in the existing " + "shared Transformers 4 runtime for correct cloning. " + "Code and weights: Apache License 2.0." + ), + }, + ), + "device": ( + ["auto", "cuda", "cpu"], + { + "default": "auto", + "tooltip": ( + "Execution device. CUDA is recommended. CPU inference " + "is supported but slow and runs in float32." + ), + }, + ), + "temperature": ( + "FLOAT", + { + "default": 0.8, + "min": 0.1, + "max": 2.0, + "step": 0.05, + "tooltip": ( + "Official sampling temperature. Used only in Sampling " + "mode; lower values are more conservative." + ), + }, + ), + "top_p": ( + "FLOAT", + { + "default": 0.95, + "min": 0.01, + "max": 1.0, + "step": 0.01, + "tooltip": ( + "Official nucleus-sampling cutoff. Used only in " + "Sampling mode." + ), + }, + ), + "top_k": ( + "INT", + { + "default": 50, + "min": 1, + "max": 100, + "step": 1, + "tooltip": ( + "Official top-k sampling limit. Used only in Sampling mode." + ), + }, + ), + "max_new_tokens": ( + "INT", + { + "default": 1024, + "min": 64, + "max": 2048, + "step": 64, + "tooltip": ( + "Maximum acoustic frames for the first generation " + "attempt. Audio8 emits about 21.5 frames per second. " + "Larger budgets take more time and context memory." + ), + }, + ), + }, + "optional": { + "retry_max_new_tokens": ( + "INT", + { + "default": 2048, + "min": 64, + "max": 2048, + "step": 8, + "tooltip": ( + "Second-attempt frame budget when the first attempt " + "does not emit EOS. Must be at least max_new_tokens. " + "Set it equal to max_new_tokens to disable the larger " + "retry." + ), + }, + ), + "dtype": ( + ["auto", "bfloat16", "float16", "float32"], + { + "default": "auto", + "tooltip": ( + "Model precision. Auto prefers bfloat16 on Ampere-or-" + "newer CUDA GPUs, uses float16 on older CUDA GPUs, and " + "float32 on CPU. CPU coerces reduced-precision choices " + "to float32." + ), + }, + ), + "sampling_mode": ( + ["Sampling", "Greedy"], + { + "default": "Sampling", + "tooltip": ( + "Official decoding mode. Sampling uses temperature, " + "top_p, and top_k. Greedy is deterministic for a fixed " + "prompt and ignores those sampling controls." + ), + }, + ), + }, + } + + RETURN_TYPES = ("TTS_ENGINE",) + RETURN_NAMES = ("TTS_engine",) + FUNCTION = "create_engine_config" + CATEGORY = "TTS Audio Suite/⚙️ Engines" + + def create_engine_config( + self, + model_variant: str, + device: str, + temperature: float, + top_p: float, + top_k: int, + max_new_tokens: int, + retry_max_new_tokens: int = 2048, + dtype: str = "auto", + sampling_mode: str = "Sampling", + ) -> tuple: + if retry_max_new_tokens < max_new_tokens: + raise ValueError( + "Audio8 retry_max_new_tokens must be at least max_new_tokens" + ) + + do_sample = sampling_mode == "Sampling" + config = { + "engine_type": "audio8_tts", + "model_variant": model_variant, + "device": device, + "dtype": dtype, + "max_new_tokens": int(max_new_tokens), + "retry_max_new_tokens": int(retry_max_new_tokens), + "temperature": float(temperature), + "top_p": float(top_p), + "top_k": int(top_k), + "do_sample": do_sample, + } + + print(f"⚙️ Audio8 TTS Preview: {model_variant} on {device} ({dtype})") + print( + " Settings: " + f"mode={sampling_mode}, temperature={temperature}, top_p={top_p}, " + f"top_k={top_k}, max_new_tokens={max_new_tokens}, " + f"retry_max_new_tokens={retry_max_new_tokens}" + ) + print( + " Voice cloning requires reference audio plus its exact matching " + "transcript; no-reference generation is supported." + ) + print( + " Runtime: shared Transformers 4 profile (required for correct cloning)" + ) + print(" Preview checkpoint | Apache License 2.0") + + return ( + { + "engine_type": "audio8_tts", + "config": config, + "adapter_class": "Audio8TTSEngineAdapter", + "capabilities": ["tts"], + }, + ) diff --git a/nodes/shared/character_voices_node.py b/nodes/shared/character_voices_node.py index 5f0026b4..d0ea81d3 100644 --- a/nodes/shared/character_voices_node.py +++ b/nodes/shared/character_voices_node.py @@ -69,10 +69,11 @@ def INPUT_TYPES(cls): "default": "", "tooltip": """Create reference text on-the-fly for connected audio input. -ENGINE REQUIREMENTS: -• F5-TTS: REQUIRES reference text (must match spoken audio exactly) -• Higgs Audio 2: Optional but uses reference text if provided -• ChatterBox/VibeVoice/IndexTTS: Don't use reference text +ENGINE REQUIREMENTS: +• Audio8 TTS: REQUIRES the exact spoken transcript for voice cloning +• F5-TTS: REQUIRES reference text (must match spoken audio exactly) +• Higgs Audio 2: Optional but uses reference text if provided +• ChatterBox/VibeVoice/IndexTTS: Don't use reference text Selecting a library voice loads its transcription here automatically. Edits are temporary workflow overrides and never modify the source .txt file.""" }), diff --git a/nodes/step_audio_editx_special/step_audio_editx_audio_editor_node.py b/nodes/step_audio_editx_special/step_audio_editx_audio_editor_node.py index df210214..b4c9176f 100644 --- a/nodes/step_audio_editx_special/step_audio_editx_audio_editor_node.py +++ b/nodes/step_audio_editx_special/step_audio_editx_audio_editor_node.py @@ -208,13 +208,20 @@ def _get_or_create_engine(self, tts_engine_data=None, inline_tag_precision=None, self._cached_settings = None # Check if engine was deleted (MUST be after settings change check, before early return) - engine_was_deleted = (self._engine is not None and - hasattr(self._engine, '_tts_engine') and - self._engine._tts_engine is None) - - if engine_was_deleted: - print("⚠️ Step Audio EditX engine was deleted, reloading...") - self._engine = None + engine_was_deleted = self._engine is not None and ( + ( + hasattr(self._engine, '_tts_engine') + and self._engine._tts_engine is None + ) + or ( + hasattr(self._engine, '_initialized') + and not self._engine._initialized + ) + ) + + if engine_was_deleted: + print("⚠️ Step Audio EditX engine was unloaded, reloading...") + self._engine = None elif self._engine is not None: # Using cached engine - ensure it's on the correct device from utils.device import resolve_torch_device diff --git a/nodes/unified/tts_srt_node.py b/nodes/unified/tts_srt_node.py index b009edb0..37e4563e 100644 --- a/nodes/unified/tts_srt_node.py +++ b/nodes/unified/tts_srt_node.py @@ -84,7 +84,7 @@ def INPUT_TYPES(cls): }), "narrator_voice": (reference_files, { "default": "none", - "tooltip": "Fallback narrator voice from voice folders. Used when opt_narrator is not connected. Select 'none' for engines that support direct TTS without voice cloning, such as MOSS." + "tooltip": "Fallback narrator voice from voice folders. Used when opt_narrator is not connected. Select 'none' for engines that support direct TTS without voice cloning, such as MOSS or Audio8." }), "seed": ("INT", { "default": 1, "min": 0, "max": 2**32 - 1, @@ -223,6 +223,12 @@ def _create_proper_engine_node_instance(self, engine_data: Dict[str, Any]): stable_params['optimize'] = config.get('optimize', False) stable_params['max_generate_length'] = config.get('max_generate_length', 500) + if engine_type == "audio8_tts": + stable_params['model_variant'] = config.get( + 'model_variant', 'Audio8-TTS-Preview-0.6b' + ) + stable_params['dtype'] = config.get('dtype', 'auto') + if engine_type == "dramabox": stable_params['model_name'] = config.get('model_name', 'DramaBox') stable_params['precision'] = config.get('precision', 'auto') @@ -578,6 +584,40 @@ def check_interrupt(self): } return engine_instance + elif engine_type == "audio8_tts": + processor_path = os.path.join( + nodes_dir, "audio8_tts", "audio8_tts_srt_processor.py" + ) + processor_spec = importlib.util.spec_from_file_location( + "audio8_tts_srt_processor_module", processor_path + ) + processor_module = importlib.util.module_from_spec(processor_spec) + processor_spec.loader.exec_module(processor_module) + Audio8TTSSRTProcessor = processor_module.Audio8TTSSRTProcessor + + class Audio8TTSSRTWrapper: + def __init__(self, cfg): + self.config = cfg.copy() + self.processor = Audio8TTSSRTProcessor(self, self.config) + + def update_config(self, new_config): + self.config = new_config.copy() + self.processor.update_config(new_config) + + def check_interrupt(self): + if model_management.interrupt_processing: + raise InterruptedError( + "Audio8 TTS SRT processing interrupted by user" + ) + + engine_instance = Audio8TTSSRTWrapper(config) + import time + self._cached_engine_instances[cache_key] = { + 'instance': engine_instance, + 'timestamp': time.time() + } + return engine_instance + elif engine_type == "dramabox": processor_path = os.path.join( nodes_dir, "dramabox", "dramabox_srt_processor.py" @@ -987,7 +1027,13 @@ def check_interrupt(self): raise ValueError(f"Unknown engine type: {engine_type}") except Exception as e: - if isinstance(e, InterruptedError): + if isinstance( + e, + ( + InterruptedError, + model_management.InterruptProcessingException, + ), + ): raise print(f"❌ Failed to create engine SRT node instance: {e}") return None @@ -1053,7 +1099,7 @@ def _get_voice_reference(self, opt_narrator, narrator_voice: str): print(f"📺 TTS SRT: Using direct audio input ({character_name})") print( "⚠️ TTS SRT: Direct audio input has no reference text - " - "F5-TTS and OmniVoice cloning will fail" + "Audio8 TTS, F5-TTS, and OmniVoice cloning will fail" ) return None, audio_tensor, reference_text, character_name @@ -1080,7 +1126,13 @@ def _get_voice_reference(self, opt_narrator, narrator_voice: str): return None, None, "", "narrator" except Exception as e: - if isinstance(e, InterruptedError): + if isinstance( + e, + ( + InterruptedError, + model_management.InterruptProcessingException, + ), + ): raise print(f"❌ Voice reference error: {e}") return None, None, "", "narrator" @@ -1175,6 +1227,13 @@ def generate_srt_speech(self, TTS_engine: Dict[str, Any], srt_content: str, narr "OmniVoice voice cloning requires reference text. " "Do not connect raw audio directly. Use Character Voices node or a narrator voice with a matching .reference.txt file." ) + if engine_type == "audio8_tts" and (audio_tensor is not None or audio_path) and not reference_text.strip(): + raise ValueError( + "Audio8 TTS voice cloning requires reference text. " + "Do not connect raw audio directly. Use Character Voices with " + "the exact transcript, or a narrator voice with a matching " + ".reference.txt file." + ) # Create proper engine SRT node instance to preserve ALL functionality engine_instance = self._create_proper_engine_node_instance(TTS_engine) @@ -1375,6 +1434,30 @@ def generate_srt_speech(self, TTS_engine: Dict[str, Any], srt_content: str, narr enable_audio_cache=enable_audio_cache ) + elif engine_type == "audio8_tts": + timing_params = { + 'fade_for_StretchToFit': fade_for_StretchToFit, + 'max_stretch_ratio': max_stretch_ratio, + 'min_stretch_ratio': min_stretch_ratio, + 'timing_tolerance': timing_tolerance, + } + voice_mapping = {} + if audio_tensor is not None or audio_path: + voice_mapping['narrator'] = { + 'audio': audio_tensor, + 'audio_path': audio_path, + 'reference_text': reference_text or '', + 'character_name': character_name or 'narrator', + } + result = engine_instance.processor.process_srt_content( + srt_content=srt_content, + voice_mapping=voice_mapping, + seed=seed, + timing_mode=timing_mode, + timing_params=timing_params, + enable_audio_cache=enable_audio_cache, + ) + elif engine_type == "dramabox": timing_params = { 'fade_for_StretchToFit': fade_for_StretchToFit, @@ -1688,9 +1771,16 @@ def generate_srt_speech(self, TTS_engine: Dict[str, Any], srt_content: str, narr "is a voice-design model and cannot be used with TTS SRT" in msg or "Pause tags are not compatible with force_speaker_kv" in msg or "MOSS-TTSD Native Multi-Speaker Dialogue does not support this SRT input" in msg + or "Audio8 TTS voice cloning requires reference text" in msg ): raise - if isinstance(e, InterruptedError): + if isinstance( + e, + ( + InterruptedError, + model_management.InterruptProcessingException, + ), + ): raise error_msg = f"❌ TTS SRT generation failed: {e}" print(error_msg) diff --git a/nodes/unified/tts_text_node.py b/nodes/unified/tts_text_node.py index 94949bcd..7a41b1bd 100644 --- a/nodes/unified/tts_text_node.py +++ b/nodes/unified/tts_text_node.py @@ -89,7 +89,7 @@ def INPUT_TYPES(cls): }), "narrator_voice": (reference_files, { "default": "none", - "tooltip": "Fallback narrator voice from voice folders. Used when opt_narrator is not connected. Select 'none' for engines that support direct TTS without voice cloning, such as MOSS." + "tooltip": "Fallback narrator voice from voice folders. Used when opt_narrator is not connected. Select 'none' for engines that support direct TTS without voice cloning, such as MOSS or Audio8." }), "seed": ("INT", { "default": 1, "min": 0, "max": 2**32 - 1, @@ -218,6 +218,12 @@ def _create_proper_engine_node_instance(self, engine_data: Dict[str, Any]): stable_params['optimize'] = config.get('optimize', False) stable_params['max_generate_length'] = config.get('max_generate_length', 500) + if engine_type == "audio8_tts": + stable_params['model_variant'] = config.get( + 'model_variant', 'Audio8-TTS-Preview-0.6b' + ) + stable_params['dtype'] = config.get('dtype', 'auto') + if engine_type == "dramabox": stable_params['model_name'] = config.get('model_name', 'DramaBox') stable_params['precision'] = config.get('precision', 'auto') @@ -445,7 +451,13 @@ def generate_tts_audio(self, text, char_audio, char_text, character="narrator", voice_mapping[char] = {"waveform": waveform, "sample_rate": sample_rate} print(f"🎭 VibeVoice: Using character-specific voice for '{char}'") except Exception as e: - if isinstance(e, InterruptedError): + if isinstance( + e, + ( + InterruptedError, + model_management.InterruptProcessingException, + ), + ): raise print(f"⚠️ Failed to load character audio for '{char}': {e}") voice_mapping[char] = char_audio # Fallback to main voice @@ -580,6 +592,38 @@ def update_config(self, new_config): return engine_instance + elif engine_type == "audio8_tts": + from engines.adapters.audio8_tts_adapter import Audio8TTSEngineAdapter + + processor_path = os.path.join( + nodes_dir, "audio8_tts", "audio8_tts_processor.py" + ) + processor_spec = importlib.util.spec_from_file_location( + "audio8_tts_processor_module", processor_path + ) + processor_module = importlib.util.module_from_spec(processor_spec) + processor_spec.loader.exec_module(processor_module) + Audio8TTSProcessor = processor_module.Audio8TTSProcessor + + class Audio8TTSWrapper: + def __init__(self, cfg): + self.config = cfg.copy() + self.adapter = Audio8TTSEngineAdapter(self.config) + self.processor = Audio8TTSProcessor(self.adapter, self.config) + + def update_config(self, new_config): + self.config = new_config.copy() + self.adapter.update_config(new_config) + self.processor.update_config(new_config) + + engine_instance = Audio8TTSWrapper(config) + import time + self._cached_engine_instances[cache_key] = { + 'instance': engine_instance, + 'timestamp': time.time() + } + return engine_instance + elif engine_type == "dramabox": from engines.adapters.dramabox_adapter import DramaBoxEngineAdapter processor_path = os.path.join(nodes_dir, "dramabox", "dramabox_processor.py") @@ -837,7 +881,13 @@ def generate_tts_audio(self, text, char_audio, char_text, character="narrator", raise ValueError(f"Unknown engine type: {engine_type}") except Exception as e: - if isinstance(e, InterruptedError): + if isinstance( + e, + ( + InterruptedError, + model_management.InterruptProcessingException, + ), + ): raise if "MOSS LoRA/base model mismatch" in str(e): raise @@ -906,7 +956,8 @@ def _get_voice_reference(self, opt_narrator, narrator_voice: str): print(f"🎤 TTS Text: Using direct audio input ({character_name})") print( "⚠️ TTS Text: Direct audio input has no reference text - " - "F5-TTS and OmniVoice cloning will fail, Qwen3-TTS will use x_vector_only mode (lower quality)" + "Audio8 TTS, F5-TTS, and OmniVoice cloning will fail; " + "Qwen3-TTS will use x_vector_only mode (lower quality)" ) return None, audio_tensor, reference_text, character_name @@ -952,7 +1003,13 @@ def _get_voice_reference(self, opt_narrator, narrator_voice: str): return None, None, "", "narrator" except Exception as e: - if isinstance(e, InterruptedError): + if isinstance( + e, + ( + InterruptedError, + model_management.InterruptProcessingException, + ), + ): raise print(f"❌ Voice reference error: {e}") return None, None, "", "narrator" @@ -1059,6 +1116,13 @@ def generate_speech(self, TTS_engine: Dict[str, Any], text: str, narrator_voice: "OmniVoice voice cloning requires reference text. " "Do not connect raw audio directly. Use Character Voices node or a narrator voice with a matching .reference.txt file." ) + if engine_type == "audio8_tts" and (audio_tensor is not None or audio_path) and not reference_text.strip(): + raise ValueError( + "Audio8 TTS voice cloning requires reference text. " + "Do not connect raw audio directly. Use Character Voices with " + "the exact transcript, or a narrator voice with a matching " + ".reference.txt file." + ) # Create proper engine node instance to preserve ALL functionality engine_instance = self._create_proper_engine_node_instance(TTS_engine) @@ -1520,6 +1584,59 @@ def __init__(self): formatted_audio = AudioProcessingUtils.format_for_comfyui(combined_audio, 48000) result = (formatted_audio, generation_info) + elif engine_type == "audio8_tts": + import re + from utils.audio.chunk_timing import ChunkTimingHelper + + voice_mapping = {} + if audio_tensor is not None or audio_path: + voice_mapping['narrator'] = { + 'audio': audio_tensor, + 'audio_path': audio_path, + 'reference_text': reference_text or '', + 'character_name': character_name or 'narrator', + } + + segment_records = engine_instance.processor.process_text( + text=text, + voice_mapping=voice_mapping, + seed=seed, + enable_chunking=enable_chunking, + max_chars_per_chunk=max_chars_per_chunk, + chunk_combination_method=chunk_combination_method, + silence_between_chunks_ms=silence_between_chunks_ms, + enable_audio_cache=enable_audio_cache, + ) + combined_audio, chunk_info = engine_instance.processor.combine_audio_segments( + segments=segment_records, + method=chunk_combination_method, + silence_ms=silence_between_chunks_ms, + original_text=text, + return_info=True, + ) + + total_duration = ( + combined_audio.shape[-1] / 44100.0 + if combined_audio.numel() + else 0.0 + ) + clean_text = re.sub(r'\[.*?\]', '', text) + base_info = ( + f"Generated {total_duration:.1f}s audio from {len(clean_text)} characters " + f"(Audio8 TTS, narrator: {char_display})" + ) + base_info += ( + "\n🎭 Reference-free speech, zero-shot cloning, character switching, " + "and pause tags enabled" + ) + generation_info = ChunkTimingHelper.enhance_generation_info( + f"✅ {base_info}", chunk_info + ) + result = ( + AudioProcessingUtils.format_for_comfyui(combined_audio, 44100), + generation_info, + ) + elif engine_type == "dramabox": import re from utils.audio.chunk_timing import ChunkTimingHelper @@ -1900,7 +2017,13 @@ def __init__(self): } print(f"🎭 Qwen3-TTS: Using character-specific voice for '{character}' (ICL mode)") except Exception as e: - if isinstance(e, InterruptedError): + if isinstance( + e, + ( + InterruptedError, + model_management.InterruptProcessingException, + ), + ): raise print(f"⚠️ Failed to load character audio for '{character}': {e}") # Fallback to narrator voice if available @@ -1945,7 +2068,13 @@ def __init__(self): print(f"⚠️⚠️ Qwen3-TTS: Character '{character}' has audio but NO reference text") print(f"⚠️⚠️ Using x_vector_only mode (speaker embedding only) - LOWER QUALITY than ICL mode") except Exception as e: - if isinstance(e, InterruptedError): + if isinstance( + e, + ( + InterruptedError, + model_management.InterruptProcessingException, + ), + ): raise print(f"⚠️ Failed to load character audio for '{character}': {e}") # Fallback to narrator voice if available @@ -2082,9 +2211,17 @@ def __init__(self): raise if "MOSS LoRA/base model mismatch" in str(e): raise + if "Audio8 TTS voice cloning requires reference text" in str(e): + raise if engine_type == "index_tts": raise - if isinstance(e, InterruptedError): + if isinstance( + e, + ( + InterruptedError, + model_management.InterruptProcessingException, + ), + ): raise error_msg = f"❌ TTS Text generation failed: {e}" print(error_msg) diff --git a/tests/unit/test_runtime_launcher.py b/tests/unit/test_runtime_launcher.py index 559c7145..7d7bc3e5 100644 --- a/tests/unit/test_runtime_launcher.py +++ b/tests/unit/test_runtime_launcher.py @@ -1,4 +1,7 @@ +import json +import os from pathlib import Path +import subprocess import sys import pytest @@ -41,3 +44,40 @@ def test_build_env_preserves_valid_pythonhashseed(monkeypatch, value): env = IsolatedRuntimeLauncher().build_env(profile) assert env["PYTHONHASHSEED"] == value + + +@pytest.mark.unit +def test_audio8_worker_prioritizes_project_root_over_shadowing_utils(tmp_path): + shadow_dir = tmp_path / "step_audio_editx_impl" + shadow_dir.mkdir() + (shadow_dir / "utils.py").write_text( + 'raise RuntimeError("shadow utils imported")\n', + encoding="utf-8", + ) + + worker_path = REPO_ROOT / "utils" / "runtimes" / "workers" / "audio8_tts_worker.py" + env = os.environ.copy() + env["PYTHONPATH"] = os.pathsep.join((str(shadow_dir), str(REPO_ROOT))) + completed = subprocess.run( + [sys.executable, str(worker_path)], + input=json.dumps( + { + "action": "ping", + "request_id": "audio8-path-priority-test", + } + ) + + "\n", + capture_output=True, + text=True, + encoding="utf-8", + errors="replace", + env=env, + cwd=str(REPO_ROOT), + timeout=30, + check=False, + ) + + assert completed.returncode == 0, completed.stderr + response = json.loads(completed.stdout.strip()) + assert response["ok"] is True + assert response["result"] == {"pong": True} diff --git a/utils/audio/cache.py b/utils/audio/cache.py index 427b727c..2b968540 100644 --- a/utils/audio/cache.py +++ b/utils/audio/cache.py @@ -577,6 +577,30 @@ def generate_cache_key(self, **params) -> str: return hashlib.md5(cache_string.encode()).hexdigest() +class Audio8TTSCacheKeyGenerator(CacheKeyGenerator): + """Cache key generator for official Audio8 TTS inference.""" + + def generate_cache_key(self, **params) -> str: + cache_data = { + 'text': params.get('text', ''), + 'audio_component': params.get('audio_component', ''), + 'reference_text': params.get('reference_text', ''), + 'model_variant': params.get('model_variant', 'Audio8-TTS-Preview-0.6b'), + 'max_new_tokens': int(params.get('max_new_tokens', 1024)), + 'retry_max_new_tokens': int(params.get('retry_max_new_tokens', 2000)), + 'temperature': round(float(params.get('temperature', 0.8)), 4), + 'top_p': round(float(params.get('top_p', 0.95)), 4), + 'top_k': int(params.get('top_k', 50)), + 'do_sample': bool(params.get('do_sample', True)), + 'dtype': params.get('dtype', 'auto'), + 'device': params.get('device', 'auto'), + 'seed': int(params.get('seed', 42)), + 'character': params.get('character', 'narrator'), + 'engine': 'audio8_tts', + } + return hashlib.md5(str(sorted(cache_data.items())).encode()).hexdigest() + + class DramaBoxCacheKeyGenerator(CacheKeyGenerator): """Cache key generator for official DramaBox inference.""" @@ -710,6 +734,7 @@ def __init__(self): 'cosyvoice': CosyVoiceCacheKeyGenerator(), 'qwen3_tts': Qwen3TTSCacheKeyGenerator(), 'dots_tts': DotsTTSCacheKeyGenerator(), + 'audio8_tts': Audio8TTSCacheKeyGenerator(), 'dramabox': DramaBoxCacheKeyGenerator(), 'fish_audio_s2': FishAudioS2CacheKeyGenerator(), 'omnivoice': OmniVoiceCacheKeyGenerator(), @@ -792,7 +817,9 @@ def _calculate_duration(self, audio_tensor: torch.Tensor, engine_type: str) -> f sample_rate = 24000 elif engine_type in ('index_tts', 'cosyvoice'): sample_rate = 22050 - else: + elif engine_type == 'audio8_tts': + sample_rate = 44100 + else: sample_rate = 44100 return num_samples / sample_rate diff --git a/utils/models/engine_registry.py b/utils/models/engine_registry.py index b5db26d4..b5737949 100644 --- a/utils/models/engine_registry.py +++ b/utils/models/engine_registry.py @@ -157,6 +157,15 @@ def __post_init__(self): fallback_languages=["English"], ), + "audio8_tts": EngineCapabilities( + supports_voice_conversion=False, + multilingual_model_switching=False, + can_corrupt_on_reload=False, + fallback_languages=["English"], + supports_runtime_isolation=True, + default_runtime_profile="vibevoice_transformers4_shared", + ), + "dramabox": EngineCapabilities( supports_voice_conversion=False, multilingual_model_switching=False, diff --git a/utils/models/unified_model_interface.py b/utils/models/unified_model_interface.py index 4c12fc2d..f0acda82 100644 --- a/utils/models/unified_model_interface.py +++ b/utils/models/unified_model_interface.py @@ -11,9 +11,14 @@ from pathlib import Path from utils.models.comfyui_model_wrapper import tts_model_manager, ModelInfo -from utils.models.factory_config import ModelLoadConfig, runtime_uses_isolation +from utils.models.factory_config import ( + ModelLoadConfig, + RUNTIME_MODE_SHARED, + runtime_uses_isolation, +) from utils.models.engine_registry import get_default_runtime_profile from utils.runtimes import ( + build_audio8_tts_isolated_proxy, build_fish_audio_s2_proxy, build_higgs_audio_isolated_proxy, build_qwen3_asr_isolated_proxy, @@ -124,6 +129,10 @@ def load_model(self, config: ModelLoadConfig, force_reload: bool = False) -> Any # Check PyTorch consistency on first model load self._check_pytorch_consistency() + if config.engine_name == "audio8_tts": + config.runtime_mode = RUNTIME_MODE_SHARED + config.runtime_profile = "vibevoice_transformers4_shared" + if runtime_uses_isolation(config.runtime_mode): return self._load_isolated_model(config, force_reload=force_reload) @@ -219,6 +228,13 @@ def _load_isolated_model(self, config: ModelLoadConfig, force_reload: bool = Fal cached = self._isolated_model_cache.get(cache_key) if cached is not None: + ensure_registration = getattr( + cached, + "_ensure_comfy_model_registration", + None, + ) + if callable(ensure_registration): + ensure_registration() return cached profile_name = config.runtime_profile or get_default_runtime_profile(config.engine_name or "") @@ -244,6 +260,11 @@ def _load_isolated_model(self, config: ModelLoadConfig, force_reload: bool = Fal self._isolated_model_cache[cache_key] = proxy return proxy + if config.engine_name == "audio8_tts" and config.model_type == "tts": + proxy = build_audio8_tts_isolated_proxy(config) + self._isolated_model_cache[cache_key] = proxy + return proxy + if config.engine_name == "qwen3_asr" and config.model_type in ("asr", "aligner"): proxy = build_qwen3_asr_isolated_proxy(config) self._isolated_model_cache[cache_key] = proxy @@ -1751,6 +1772,19 @@ def dots_tts_factory(config: ModelLoadConfig): unified_model_interface.register_model_factory("dots_tts", "tts", dots_tts_factory) +def register_audio8_tts_factory(): + """Register Audio8 TTS through its required Transformers 4 runtime.""" + def audio8_tts_factory(config: ModelLoadConfig): + del config + raise RuntimeError( + "Audio8 TTS reached its embedded factory unexpectedly; all " + "Audio8 loads must be routed through the shared Transformers 4 " + "runtime" + ) + + unified_model_interface.register_model_factory("audio8_tts", "tts", audio8_tts_factory) + + def register_dramabox_factory(): """Register official DramaBox in the main Transformers 5 environment.""" def dramabox_factory(config: ModelLoadConfig): @@ -2018,6 +2052,7 @@ def initialize_all_factories(): register_qwen3_tts_factory() register_fish_audio_s2_factory() register_dots_tts_factory() + register_audio8_tts_factory() register_dramabox_factory() register_omnivoice_factory() register_qwen3_asr_factory() diff --git a/utils/runtimes/__init__.py b/utils/runtimes/__init__.py index 124b13c6..18ee08a1 100644 --- a/utils/runtimes/__init__.py +++ b/utils/runtimes/__init__.py @@ -6,6 +6,7 @@ """ from .bootstrap import ensure_runtime, resolve_runtime_dir, resolve_runtime_python +from .audio8_tts_proxy import Audio8TTSIsolatedProxy, build_audio8_tts_isolated_proxy from .higgs_audio_proxy import HiggsAudioIsolatedProxy, build_higgs_audio_isolated_proxy from .fish_audio_s2_proxy import FishAudioS2Proxy, build_fish_audio_s2_proxy from .launcher import IsolatedRuntimeLauncher @@ -20,6 +21,8 @@ "ensure_runtime", "resolve_runtime_dir", "resolve_runtime_python", + "Audio8TTSIsolatedProxy", + "build_audio8_tts_isolated_proxy", "HiggsAudioIsolatedProxy", "IsolatedRuntimeLauncher", "RuntimeProfile", diff --git a/utils/runtimes/audio8_tts_proxy.py b/utils/runtimes/audio8_tts_proxy.py new file mode 100644 index 00000000..2e170b30 --- /dev/null +++ b/utils/runtimes/audio8_tts_proxy.py @@ -0,0 +1,546 @@ +"""Audio8 TTS proxy for the shared official Transformers 4 runtime.""" + +from __future__ import annotations + +import os +import sys +import tempfile +import threading +import uuid +import weakref +from pathlib import Path +from typing import Any, Dict, Optional + +import torch + +from utils.models.factory_config import ModelLoadConfig +from .bootstrap import PROJECT_ROOT, ensure_runtime +from .launcher import IsolatedRuntimeLauncher +from .profiles import RuntimeProfile, get_runtime_profile +from .protocol import RuntimeJobRequest +from .session import JsonLineWorkerSession + + +_INTERRUPT_EVENTS: set[threading.Event] = set() +_INTERRUPT_EVENTS_LOCK = threading.Lock() +_INTERRUPT_DISPATCH_LOCK = threading.Lock() + + +def _install_comfy_interrupt_dispatcher() -> None: + """Forward Comfy's server interrupt edge to active Audio8 requests.""" + server_module = sys.modules.get("server") + comfy_nodes = getattr(server_module, "nodes", None) + if comfy_nodes is None: + comfy_nodes = sys.modules.get("nodes") + if comfy_nodes is None: + return + + with _INTERRUPT_DISPATCH_LOCK: + current_interrupt = getattr(comfy_nodes, "interrupt_processing", None) + if not callable(current_interrupt): + return + if getattr(current_interrupt, "_tts_audio8_dispatcher", False): + if ( + getattr(current_interrupt, "_tts_audio8_events", None) + is _INTERRUPT_EVENTS + ): + return + current_interrupt = getattr( + current_interrupt, + "_tts_audio8_original_interrupt", + current_interrupt, + ) + + def dispatch_interrupt(value=True): + # TTS Audio Suite patch: preserve Comfy's interrupt edge for + # blocking Audio8 subprocess requests without replacing the + # process-wide callback separately for every generation. + result = current_interrupt(value) + if value: + with _INTERRUPT_EVENTS_LOCK: + active_events = tuple(_INTERRUPT_EVENTS) + for active_event in active_events: + active_event.set() + return result + + dispatch_interrupt._tts_audio8_dispatcher = True + dispatch_interrupt._tts_audio8_original_interrupt = current_interrupt + dispatch_interrupt._tts_audio8_events = _INTERRUPT_EVENTS + comfy_nodes.interrupt_processing = dispatch_interrupt + + +def _register_interrupt_event(interrupt_event: threading.Event) -> None: + _install_comfy_interrupt_dispatcher() + with _INTERRUPT_EVENTS_LOCK: + _INTERRUPT_EVENTS.add(interrupt_event) + + +def _unregister_interrupt_event(interrupt_event: threading.Event) -> None: + with _INTERRUPT_EVENTS_LOCK: + _INTERRUPT_EVENTS.discard(interrupt_event) + + +class Audio8TTSIsolatedProxy: + """Expose Audio8's engine API while inference runs under Transformers 4.""" + + SAMPLE_RATE = 44100 + + def __init__(self, config: ModelLoadConfig, profile: RuntimeProfile): + self.config = config + self.profile = profile + self.model_name = config.model_name or "Audio8-TTS-Preview-0.6b" + self.model_path = config.model_path + self.device = str(config.device) + self.current_device = self.device + self.dtype_name = str((config.additional_params or {}).get("dtype", "auto")) + self.dtype = self._resolve_dtype(self.dtype_name) + self.load_device = self.current_loaded_device() + self.offload_device = torch.device("cpu") + self.currently_used = True + self.model = self + self.engine = self + self.parent = None + self.model_options = {} + self.model_keys = set() + estimate_gib = ( + 3.5 + if self.device.startswith("cpu") or self.dtype == torch.float32 + else 1.75 + ) + self._estimated_memory_size = int(estimate_gib * 1024**3) + self._comfy_loaded_model = None + self._comfy_model_management = None + self._initialized = False + + launcher = IsolatedRuntimeLauncher(runtime_root=str(PROJECT_ROOT)) + python_path = ensure_runtime(profile) + worker_script = PROJECT_ROOT / "utils/runtimes/workers/audio8_tts_worker.py" + self._session = JsonLineWorkerSession( + python_path=str(python_path), + worker_script=str(worker_script), + env=launcher.build_env(profile), + ) + try: + self._initialize_remote_engine() + except Exception: + self._session.close() + raise + self._register_with_comfy_model_management() + + def _resolve_dtype(self, dtype_name: str) -> torch.dtype: + normalized = str(dtype_name or "auto").lower() + if not self.device.startswith("cuda"): + return torch.float32 + if normalized == "float32": + return torch.float32 + if normalized == "float16": + return torch.float16 + if normalized == "bfloat16": + return torch.bfloat16 + if torch.cuda.is_available(): + major, _minor = torch.cuda.get_device_capability(torch.device(self.device)) + return torch.bfloat16 if major >= 8 else torch.float16 + return torch.float16 + + def _request(self, action: str, payload: Dict[str, Any]): + response = self._session.request( + RuntimeJobRequest( + engine_name="audio8_tts", + action=action, + model_name=self.model_name, + device=self.device, + runtime_profile=self.profile.name, + payload=payload, + request_id=str(uuid.uuid4()), + ) + ) + if not response.ok: + details = response.error or ( + f"Audio8 TTS isolated action '{action}' failed" + ) + if response.logs: + details = f"{details}\n" + "\n".join(response.logs) + raise RuntimeError(details) + return response + + def _request_generation(self, payload: Dict[str, Any]): + """Run generation while the main process monitors Comfy interruption.""" + try: + import comfy.model_management as model_management + except Exception: + return self._request("generate", payload) + + checker = getattr( + model_management, + "throw_exception_if_processing_interrupted", + None, + ) + if not callable(checker): + return self._request("generate", payload) + + interrupt_event = threading.Event() + _register_interrupt_event(interrupt_event) + result: Dict[str, Any] = {} + request_thread = None + request_started = False + + def run_request() -> None: + try: + result["response"] = self._request("generate", payload) + except BaseException as exc: + result["error"] = exc + + try: + checker() + request_thread = threading.Thread( + target=run_request, + name="audio8-tts-worker-request", + daemon=True, + ) + request_thread.start() + request_started = True + + while request_thread.is_alive(): + request_thread.join(timeout=0.05) + if interrupt_event.is_set(): + raise model_management.InterruptProcessingException() + checker() + except BaseException as exc: + if request_started: + self._initialized = False + try: + self._session.terminate_now() + except Exception: + pass + request_thread.join(timeout=5) + if isinstance( + exc, + model_management.InterruptProcessingException, + ): + interrupt_setter = getattr( + model_management, + "interrupt_current_processing", + None, + ) + if callable(interrupt_setter): + interrupt_setter(False) + raise + finally: + _unregister_interrupt_event(interrupt_event) + + if "error" in result: + raise result["error"] + return result["response"] + + def _initialize_remote_engine(self) -> None: + self._request( + "initialize", + { + "model_path": self.model_path, + "dtype": self.dtype_name, + }, + ) + self._initialized = True + print(f"✅ Audio8 TTS isolated runtime ready ({self.profile.name})") + + def _ensure_remote_engine(self) -> None: + process = getattr(self._session, "_process", None) + if not self._initialized or process is None or process.poll() is not None: + self._initialize_remote_engine() + + @staticmethod + def _serialize_reference_audio( + reference_audio: Any, + bundle_dir: Path, + ) -> Optional[Dict[str, Any]]: + if reference_audio is None: + return None + if isinstance(reference_audio, (str, os.PathLike)): + return { + "kind": "audio_path", + "audio_path": os.fspath(reference_audio), + } + + waveform = None + sample_rate = None + if isinstance(reference_audio, dict): + waveform = reference_audio.get( + "waveform", + reference_audio.get("array"), + ) + sample_rate = reference_audio.get( + "sample_rate", + reference_audio.get("sampling_rate"), + ) + elif isinstance(reference_audio, (tuple, list)) and len(reference_audio) == 2: + waveform, sample_rate = reference_audio + + if waveform is None or sample_rate is None: + raise TypeError( + "Audio8 TTS isolated reference audio must be a path, " + "audio dict, or (waveform, sample_rate) pair" + ) + + tensor_path = bundle_dir / "reference_audio.pt" + waveform_tensor = torch.as_tensor(waveform).detach().float().cpu() + torch.save( + { + "waveform": waveform_tensor, + "sample_rate": int(sample_rate), + }, + tensor_path, + ) + return { + "kind": "tensor_path", + "tensor_path": str(tensor_path), + } + + def generate( + self, + text: str, + reference_audio: Any = None, + reference_text: Optional[str] = None, + *, + max_new_tokens: int = 1024, + retry_max_new_tokens: int = 2000, + temperature: float = 0.8, + top_p: float = 0.95, + top_k: int = 50, + do_sample: bool = True, + seed: int = 42, + ) -> torch.Tensor: + self._ensure_remote_engine() + with tempfile.TemporaryDirectory(prefix="tts_audio8_iso_") as temp_dir: + bundle_dir = Path(temp_dir) + output_path = bundle_dir / "result.pt" + serialized_reference = self._serialize_reference_audio( + reference_audio, + bundle_dir, + ) + self._request_generation( + { + "text": text, + "reference_audio": serialized_reference, + "reference_text": reference_text, + "max_new_tokens": int(max_new_tokens), + "retry_max_new_tokens": int(retry_max_new_tokens), + "temperature": float(temperature), + "top_p": float(top_p), + "top_k": int(top_k), + "do_sample": bool(do_sample), + "seed": int(seed), + "output_path": str(output_path), + } + ) + if not output_path.exists(): + raise RuntimeError( + "Audio8 TTS isolated worker returned no output payload" + ) + result = torch.load(output_path, map_location="cpu") + audio = torch.as_tensor(result["audio"]).detach().float().cpu() + sample_rate = int(result["sample_rate"]) + if sample_rate != self.SAMPLE_RATE: + raise RuntimeError( + f"Audio8 TTS worker returned {sample_rate} Hz; " + f"expected {self.SAMPLE_RATE} Hz" + ) + if audio.ndim == 1: + audio = audio.unsqueeze(0) + if audio.ndim != 2 or audio.shape[0] != 1: + raise RuntimeError( + "Audio8 TTS worker returned invalid audio shape " + f"{tuple(audio.shape)}" + ) + return audio.contiguous() + + def _register_with_comfy_model_management(self) -> None: + self._ensure_comfy_model_registration() + + def _ensure_comfy_model_registration(self) -> None: + try: + import comfy.model_management as model_management + except Exception: + return + if not hasattr(model_management, "LoadedModel") or not hasattr( + model_management, + "current_loaded_models", + ): + return + for loaded_model in model_management.current_loaded_models: + wrapper = getattr(loaded_model, "_tts_wrapper_ref", None) + real_model = getattr(loaded_model, "real_model", None) + resolved_real_model = ( + real_model() + if isinstance(real_model, weakref.ReferenceType) + else real_model + ) + if wrapper is self or resolved_real_model is self: + self._comfy_loaded_model = loaded_model + self._comfy_model_management = model_management + return + try: + loaded_model = model_management.LoadedModel(self) + loaded_model.real_model = weakref.ref(self) + finalizer = getattr( + model_management, + "cleanup_models", + lambda: None, + ) + loaded_model.model_finalizer = weakref.finalize(self, finalizer) + loaded_model._tts_wrapper_ref = self + model_management.current_loaded_models.insert(0, loaded_model) + self._comfy_loaded_model = loaded_model + self._comfy_model_management = model_management + except Exception as exc: + print( + "⚠️ Failed to register isolated Audio8 TTS runtime with " + f"ComfyUI model management: {exc}" + ) + + def _unregister_from_comfy_model_management(self) -> None: + model_management = self._comfy_model_management + loaded_model = self._comfy_loaded_model + if model_management is None or loaded_model is None: + return + try: + if loaded_model in model_management.current_loaded_models: + model_management.current_loaded_models.remove(loaded_model) + except Exception as exc: + print( + "⚠️ Failed to remove isolated Audio8 TTS runtime from " + f"ComfyUI tracking: {exc}" + ) + finally: + self._comfy_loaded_model = None + self._comfy_model_management = None + + def to(self, device): + self.device = str(device) + self.current_device = self.device + self.load_device = self.current_loaded_device() + return self + + def eval(self): + return self + + def loaded_size(self) -> int: + if self._initialized and self.device.startswith("cuda"): + return self._estimated_memory_size + return 0 + + def model_size(self) -> int: + return self._estimated_memory_size + + def model_memory(self) -> int: + return self._estimated_memory_size + + def get_ram_usage(self) -> int: + return self._estimated_memory_size + + def model_offloaded_memory(self) -> int: + return ( + 0 + if self._initialized and self.device.startswith("cuda") + else self._estimated_memory_size + ) + + def model_mmap_residency(self, free: bool = False) -> tuple[int, int]: + del free + return 0, self._estimated_memory_size + + def pinned_memory_size(self) -> int: + return 0 + + def lowvram_patch_counter(self) -> int: + return 0 + + def model_dtype(self): + return self.dtype + + def model_patches_models(self): + return () + + def model_patches_to(self, target) -> None: + if isinstance(target, torch.dtype): + self.dtype = target + elif isinstance(target, torch.device): + self.device = str(target) + + def is_dynamic(self) -> bool: + return False + + def current_loaded_device(self) -> torch.device: + if self.device.startswith("cuda") and torch.cuda.is_available(): + return torch.device(self.device) + return torch.device("cpu") + + def partially_load( + self, + device, + extra_memory, + force_patch_weights=False, + ) -> int: + del extra_memory, force_patch_weights + self.to(device) + return 0 + + def partially_unload_ram(self, ram_to_unload: int) -> int: + return self.partially_unload("cpu", ram_to_unload) + + def partially_unload(self, device, memory_to_free) -> int: + del device, memory_to_free + freed = self.loaded_size() + self.cleanup(unregister=False) + return freed + + def model_unload( + self, + memory_to_free=None, + unpatch_weights=True, + ) -> bool: + del memory_to_free, unpatch_weights + self.cleanup(unregister=False) + return True + + def detach(self, unpatch_weights=True) -> None: + del unpatch_weights + self.cleanup(unregister=False) + + def is_clone(self, other: Any) -> bool: + return other is self + + def cleanup(self, unregister: bool = True) -> None: + if unregister: + self._unregister_from_comfy_model_management() + else: + self._comfy_loaded_model = None + self._comfy_model_management = None + self._initialized = False + if getattr(self, "_session", None) is not None: + self._session.close() + + close = cleanup + + def __del__(self): + try: + self.cleanup() + except Exception: + pass + + +def build_audio8_tts_isolated_proxy( + config: ModelLoadConfig, +) -> Audio8TTSIsolatedProxy: + profile_name = config.runtime_profile or "vibevoice_transformers4_shared" + profile = get_runtime_profile(profile_name) + if profile is None: + raise RuntimeError( + f"Unknown isolated runtime profile '{profile_name}' for Audio8 TTS" + ) + if profile.name != "vibevoice_transformers4_shared": + raise RuntimeError( + "Audio8 TTS requires the shared Transformers 4 runtime profile " + "'vibevoice_transformers4_shared'" + ) + config.runtime_profile = profile.name + return Audio8TTSIsolatedProxy(config=config, profile=profile) diff --git a/utils/runtimes/profiles.py b/utils/runtimes/profiles.py index 21fa1700..b7e1a350 100644 --- a/utils/runtimes/profiles.py +++ b/utils/runtimes/profiles.py @@ -28,7 +28,7 @@ class RuntimeProfile: "numpy>=1.26.4,<2.3.0", "soundfile>=0.12.0", "omegaconf>=2.3.0", - "transformers>=4.51.3,<=4.57.3", + "transformers>=4.57.0,<=4.57.3", "kernels>=0.6.1,<=0.9", "accelerate", "requests", @@ -63,9 +63,12 @@ class RuntimeProfile: RUNTIME_PROFILES: Dict[str, RuntimeProfile] = { "vibevoice_transformers4_shared": RuntimeProfile( name="vibevoice_transformers4_shared", - engine_names=["vibevoice", "step_audio_editx"], + engine_names=["vibevoice", "step_audio_editx", "audio8_tts"], python_path_hint="runtimes/shared_legacy_t4/Scripts/python.exe", - description="Shared legacy Transformers 4 runtime for VibeVoice/Kugel, Step Audio EditX, and similar engines.", + description=( + "Shared legacy Transformers 4 runtime for VibeVoice/Kugel, " + "Step Audio EditX, Audio8 TTS, and similar engines." + ), inherit_base_site_packages=True, pip_packages=list(_VIBEVOICE_T4_PACKAGES), pip_packages_no_deps=[ diff --git a/utils/runtimes/session.py b/utils/runtimes/session.py index 9c8c194a..e5fbc3ef 100644 --- a/utils/runtimes/session.py +++ b/utils/runtimes/session.py @@ -1,9 +1,9 @@ -from __future__ import annotations - """ Persistent JSON-line worker session for isolated runtimes. """ +from __future__ import annotations + import json import subprocess import threading @@ -118,6 +118,24 @@ def close(self) -> None: self._process = None + def terminate_now(self) -> None: + """Stop a worker without waiting for the request lock.""" + process = self._process + if process is None or process.poll() is not None: + return + try: + process.terminate() + except OSError: + return + try: + process.wait(timeout=2) + except subprocess.TimeoutExpired: + try: + process.kill() + process.wait(timeout=2) + except OSError: + pass + def __del__(self): try: self.close() diff --git a/utils/runtimes/workers/audio8_tts_worker.py b/utils/runtimes/workers/audio8_tts_worker.py new file mode 100644 index 00000000..3c357716 --- /dev/null +++ b/utils/runtimes/workers/audio8_tts_worker.py @@ -0,0 +1,191 @@ +"""Persistent Audio8 TTS worker for the shared Transformers 4 runtime.""" + +from __future__ import annotations + +import json +import sys +import traceback +from pathlib import Path +from typing import Any, Dict, Optional + +import torch + +PROJECT_ROOT = Path(__file__).resolve().parents[3] +project_root_str = str(PROJECT_ROOT) +sys.path = [ + entry for entry in sys.path if str(Path(entry).resolve()) != project_root_str +] +sys.path.insert(0, project_root_str) + +from utils.runtimes.protocol import RuntimeJobResponse # noqa: E402 + + +def _emit(protocol_out, response: RuntimeJobResponse) -> None: + protocol_out.write(json.dumps(response.to_dict(), ensure_ascii=True) + "\n") + protocol_out.flush() + + +def _load_reference_audio( + payload: Optional[Dict[str, Any]], +) -> Any: + if payload is None: + return None + kind = payload.get("kind") + if kind == "audio_path": + audio_path = payload.get("audio_path") + if not audio_path: + raise RuntimeError("Audio8 TTS audio_path payload is missing its file path") + return audio_path + if kind == "tensor_path": + tensor_path = payload.get("tensor_path") + if not tensor_path: + raise RuntimeError( + "Audio8 TTS tensor_path payload is missing its file path" + ) + tensor_payload = torch.load(tensor_path, map_location="cpu") + return { + "array": tensor_payload["waveform"], + "sampling_rate": int(tensor_payload["sample_rate"]), + } + raise RuntimeError(f"Unsupported Audio8 TTS reference payload kind: {kind}") + + +def main() -> int: + protocol_out = sys.stdout + sys.stdout = sys.stderr + engine = None + + for line in sys.stdin: + stripped = line.strip() + if not stripped: + continue + + request = None + try: + request = json.loads(stripped) + action = request.get("action") + payload = request.get("payload") or {} + request_id = request.get("request_id") + + if action == "shutdown": + _emit( + protocol_out, + RuntimeJobResponse( + ok=True, + result={"shutdown": True}, + request_id=request_id, + ), + ) + break + + if action == "ping": + _emit( + protocol_out, + RuntimeJobResponse( + ok=True, + result={"pong": True}, + request_id=request_id, + ), + ) + continue + + if action == "initialize": + import transformers + from packaging.version import Version + + transformers_version = Version(transformers.__version__) + if not (Version("4.57.0") <= transformers_version < Version("5.0.0")): + raise RuntimeError( + "Audio8 TTS requires Transformers >=4.57.0,<5; " + f"worker loaded Transformers {transformers.__version__}" + ) + + from engines.audio8_tts.audio8_tts_engine import ( + Audio8TTSEngine, + ) + + engine = Audio8TTSEngine( + model_name=(request.get("model_name") or "Audio8-TTS-Preview-0.6b"), + model_dir=payload.get("model_path"), + device=request.get("device") or "auto", + dtype=payload.get("dtype", "auto"), + ) + engine._ensure_model_loaded() + _emit( + protocol_out, + RuntimeJobResponse( + ok=True, + result={ + "model_name": engine.model_name, + "sample_rate": engine.SAMPLE_RATE, + "transformers_version": (transformers.__version__), + }, + request_id=request_id, + ), + ) + continue + + if engine is None: + raise RuntimeError( + "Audio8 TTS worker received generation before initialization" + ) + if action != "generate": + raise RuntimeError(f"Unsupported Audio8 TTS worker action '{action}'") + + reference_audio = _load_reference_audio(payload.get("reference_audio")) + audio = engine.generate( + text=payload["text"], + reference_audio=reference_audio, + reference_text=payload.get("reference_text"), + max_new_tokens=int(payload.get("max_new_tokens", 1024)), + retry_max_new_tokens=int(payload.get("retry_max_new_tokens", 2000)), + temperature=float(payload.get("temperature", 0.8)), + top_p=float(payload.get("top_p", 0.95)), + top_k=int(payload.get("top_k", 50)), + do_sample=bool(payload.get("do_sample", True)), + seed=int(payload.get("seed", 42)), + ) + audio = torch.as_tensor(audio).detach().float().cpu() + if audio.ndim == 1: + audio = audio.unsqueeze(0) + if audio.ndim != 2 or audio.shape[0] != 1: + raise RuntimeError( + f"Audio8 TTS generated invalid audio shape {tuple(audio.shape)}" + ) + + output_path = Path(payload["output_path"]) + torch.save( + { + "audio": audio.contiguous(), + "sample_rate": engine.SAMPLE_RATE, + }, + output_path, + ) + _emit( + protocol_out, + RuntimeJobResponse( + ok=True, + result={ + "output_path": str(output_path), + "sample_rate": engine.SAMPLE_RATE, + }, + request_id=request_id, + ), + ) + except Exception as exc: + request_id = ( + request.get("request_id") if isinstance(request, dict) else None + ) + _emit( + protocol_out, + RuntimeJobResponse( + ok=False, + error=f"{exc}\n{traceback.format_exc()}", + request_id=request_id, + ), + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/utils/text/segment_parameters.py b/utils/text/segment_parameters.py index 7243e867..fdbef276 100644 --- a/utils/text/segment_parameters.py +++ b/utils/text/segment_parameters.py @@ -115,12 +115,13 @@ 'seed': { 'chatterbox', 'chatterbox_official_23lang', 'f5tts', 'higgs_audio', 'higgs_audio_v3', 'vibevoice', 'index_tts', 'step_audio_editx', 'cosyvoice', 'qwen3_tts', - 'dots_tts', 'fish_audio_s2', 'omnivoice', + 'dots_tts', 'audio8_tts', 'fish_audio_s2', 'omnivoice', 'echo_tts', 'moss_tts', 'moss_soundeffect_v2', 'dramabox' }, 'temperature': { 'chatterbox', 'chatterbox_official_23lang', 'f5tts', 'higgs_audio', - 'higgs_audio_v3', 'vibevoice', 'index_tts', 'step_audio_editx', 'qwen3_tts', 'moss_tts', 'fish_audio_s2' + 'higgs_audio_v3', 'vibevoice', 'index_tts', 'step_audio_editx', 'qwen3_tts', + 'moss_tts', 'fish_audio_s2', 'audio8_tts' }, 'cfg': { 'f5tts', 'vibevoice', 'index_tts', 'chatterbox', 'chatterbox_official_23lang', @@ -214,10 +215,12 @@ 'f5tts', 'cosyvoice', 'omnivoice' }, 'top_p': { - 'higgs_audio', 'higgs_audio_v3', 'vibevoice', 'index_tts', 'qwen3_tts', 'moss_tts', 'fish_audio_s2' + 'higgs_audio', 'higgs_audio_v3', 'vibevoice', 'index_tts', 'qwen3_tts', + 'moss_tts', 'fish_audio_s2', 'audio8_tts' }, 'top_k': { - 'higgs_audio', 'higgs_audio_v3', 'index_tts', 'qwen3_tts', 'moss_tts' + 'higgs_audio', 'higgs_audio_v3', 'index_tts', 'qwen3_tts', 'moss_tts', + 'audio8_tts' }, 'audio_temperature': { 'moss_tts' @@ -238,7 +241,7 @@ 'moss_tts' }, 'max_new_tokens': { - 'higgs_audio_v3', 'moss_tts', 'fish_audio_s2' + 'higgs_audio_v3', 'moss_tts', 'fish_audio_s2', 'audio8_tts' }, 'max_generate_length': { 'dots_tts'