diff --git a/.gitignore b/.gitignore index b88632fe..43adc34f 100644 --- a/.gitignore +++ b/.gitignore @@ -28,3 +28,18 @@ __pycache__/ /*.wav /flutter/ /.cache/ + +# --- portable bundle: distributed manually, never committed --- +/audiocpp-portable/ + +# --- WebUI runtime artifacts: keep the source + empty dirs, drop generated files --- +/webui/output/* +!/webui/output/.gitkeep +/webui/logs/* +!/webui/logs/.gitkeep +/webui/third_party/ +/webui/llm_api_key.txt +# written by the in-UI language picker; per-machine, not a project setting +/webui/configs/ui_language.json +# personal voice recording — stays local, repo is public +/webui/voice/my-record.wav diff --git a/CMakeLists.txt b/CMakeLists.txt index 39d1dfcb..e583d6f3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -568,6 +568,13 @@ target_link_libraries(engine_runtime PUBLIC ggml) target_link_libraries(engine_runtime PRIVATE sentencepiece cjson_vendor yaml_vendor) if (ENGINE_ENABLE_OPENMP) target_link_libraries(engine_runtime PRIVATE OpenMP::OpenMP_CXX) + if (MSVC) + # MSVC's default /openmp implements only OpenMP 2.0 and rejects the + # '#pragma omp simd' directives in longformer_attention.cpp (error C7660). + # /openmp:experimental enables the OpenMP 4.0 SIMD support; it overrides the + # /openmp added by OpenMP::OpenMP_CXX above (harmless D9025 override notice). + target_compile_options(engine_runtime PRIVATE /openmp:experimental) + endif() endif() if (ENGINE_ENABLE_CUDA) diff --git a/_env.bat b/_env.bat new file mode 100644 index 00000000..0912f6f0 --- /dev/null +++ b/_env.bat @@ -0,0 +1,47 @@ +@echo off +REM _env.bat -- shared environment detection for the audio.cpp .bat launchers. +REM Called (not run) by run_webui.bat / run_server.bat / run_cli_tts.bat. It sets +REM common variables and deliberately does NOT use setlocal, so they propagate back +REM to the caller. Change detection logic here only. +REM +REM Exports: ROOT BUNDLE WEBUI_DIR PY HAS_CUDA BACKEND SERVER_EXE CLI_EXE GGUF_EXE + +REM --- ROOT = this script's directory, without the trailing backslash --- +set "ROOT=%~dp0" +if "%ROOT:~-1%"=="\" set "ROOT=%ROOT:~0,-1%" + +REM Dev tree: the repo root doubles as the bundle (models\ live under it; the +REM binaries live under build\). webui.py's own _find_bundle_root handles this. +set "BUNDLE=%ROOT%" +set "WEBUI_DIR=%ROOT%\webui" + +REM --- Python with the deps (gradio/requests/torch/safetensors/opencc/...) --- +REM Order: explicit override, project venv (Scripts\ on Windows), then a bundle venv. +set "PY=" +if defined AUDIOCPP_PYTHON if exist "%AUDIOCPP_PYTHON%" set "PY=%AUDIOCPP_PYTHON%" +if not defined PY if exist "%ROOT%\venv\Scripts\python.exe" set "PY=%ROOT%\venv\Scripts\python.exe" +if not defined PY if exist "%ROOT%\venv\python.exe" set "PY=%ROOT%\venv\python.exe" +if not defined PY if exist "%BUNDLE%\venv\Scripts\python.exe" set "PY=%BUNDLE%\venv\Scripts\python.exe" +if not defined PY if exist "%BUNDLE%\venv\python.exe" set "PY=%BUNDLE%\venv\python.exe" + +REM --- CUDA present? (NVIDIA driver installs nvcuda.dll in System32) --- +set "HAS_CUDA=" +if exist "%SystemRoot%\System32\nvcuda.dll" set "HAS_CUDA=1" + +REM --- Locate the from-source binaries. The default Visual Studio generator nests +REM them in build\bin\Release (multi-config); Ninja/Makefiles use build\bin. --- +set "BIN=" +if exist "%ROOT%\build\bin\Release\audiocpp_server.exe" set "BIN=%ROOT%\build\bin\Release" +if not defined BIN if exist "%ROOT%\build\bin\audiocpp_server.exe" set "BIN=%ROOT%\build\bin" +if defined BIN set "SERVER_EXE=%BIN%\audiocpp_server.exe" +if defined BIN set "CLI_EXE=%BIN%\audiocpp_cli.exe" +if defined BIN set "GGUF_EXE=%BIN%\audiocpp_gguf.exe" + +REM --- BACKEND: read the actual build's GGML_CUDA flag from its CMakeCache, so we +REM never advertise a GPU backend a CPU-only build can't serve. cuda when ON, else cpu. --- +set "BACKEND=cpu" +if exist "%ROOT%\build\CMakeCache.txt" ( + for /f "tokens=2 delims==" %%A in ('findstr /b /c:"GGML_CUDA:BOOL" "%ROOT%\build\CMakeCache.txt" 2^>nul') do ( + if /I "%%A"=="ON" set "BACKEND=cuda" + ) +) diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..aaad0125 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,87 @@ +# audio.cpp -- unified Python requirements (webui + SpeakType + tools) +# Frozen from the project venv on 2026-07-15. +# One env covers every Python layer of the project: +# webui/ gradio UI + realtime pipeline (gradio, numpy, requests; torch for silero VAD) +# SpeakType/ voice dictation demo (sounddevice, pywebview, pywin32, pyperclip; torch for silero VAD) +# tools/ model_manager etc. (requests, huggingface-hub, safetensors, tqdm) +# The portable bundle ships this same env pre-installed as audiocpp-portable\venv. +# Windows-only wheels (pywin32 and the pythonnet/pywebview desktop stack, used by +# SpeakType) carry a sys_platform marker so this file also installs cleanly on Linux. +# Regenerate: venv\Scripts\python.exe -m pip freeze > requirements.txt +#annotated-doc==0.0.4 +annotated-types==0.7.0 +anyio==4.14.1 +bottle==0.13.4 +brotli==1.2.0 +certifi==2026.6.17 +cffi==2.1.0 +charset-normalizer==3.4.7 +click==8.4.2 +clr_loader==0.3.1; sys_platform == "win32" +colorama==0.4.6 +fastapi==0.138.2 +filelock==3.29.4 +fsspec==2026.6.0 +gradio==6.19.0 +gradio_client==2.5.0 +groovy==0.1.2 +h11==0.16.0 +hf-gradio==0.4.1 +hf-xet==1.5.1 +httpcore==1.0.9 +httptools==0.8.0 +httpx==0.28.1 +huggingface_hub==1.21.0 +idna==3.18 +iniconfig==2.3.0 +Jinja2==3.1.6 +markdown-it-py==4.2.0 +MarkupSafe==3.0.3 +mdurl==0.1.2 +mpmath==1.3.0 +networkx==3.6.1 +numpy==2.4.6 +opencc-python-reimplemented==0.1.7 +orjson==3.11.9 +packaging==26.2 +pandas==3.0.3 +pillow==12.2.0 +pluggy==1.6.0 +proxy_tools==0.1.0; sys_platform == "win32" +pycparser==3.0 +pydantic==2.13.4 +pydantic_core==2.46.4 +pydub==0.25.1 +Pygments==2.20.0 +pyperclip==1.11.0 +pytest==9.1.1 +python-dateutil==2.9.0.post0 +python-dotenv==1.2.2 +python-multipart==0.0.32 +pythonnet==3.1.0; sys_platform == "win32" +pytz==2026.2 +pywebview==6.2.1; sys_platform == "win32" +pywin32==312; sys_platform == "win32" +PyYAML==6.0.3 +requests==2.34.2 +rich==15.0.0 +safehttpx==0.1.7 +safetensors==0.8.0 +semantic-version==2.10.0 +shellingham==1.5.4 +six==1.17.0 +sounddevice==0.5.5 +starlette==1.3.1 +sympy==1.14.0 +tomlkit==0.14.0 +torch==2.12.1 +torchaudio==2.11.0 +tqdm==4.68.3 +typer==0.25.1 +typing-inspection==0.4.2 +typing_extensions==4.15.0 +tzdata==2026.2 +urllib3==2.7.0 +uvicorn==0.49.0 +watchfiles==1.2.0 +websockets==16.0 diff --git a/run_webui.bat b/run_webui.bat new file mode 100644 index 00000000..2b8537e2 --- /dev/null +++ b/run_webui.bat @@ -0,0 +1,32 @@ +@echo off +setlocal +chcp 65001 >nul +cd /d "%~dp0" +call "%~dp0_env.bat" + +REM _env.bat auto-detected BACKEND (cuda|cpu) from the NVIDIA driver + bundled exes; +REM hand it to webui.py unless the user already chose via AUDIOCPP_BACKEND. +if not defined AUDIOCPP_BACKEND ( + if /I "%BACKEND%"=="cuda" ( set "AUDIOCPP_BACKEND=gpu" ) else ( set "AUDIOCPP_BACKEND=cpu" ) +) + +REM Python (with gradio/requests/torch/safetensors/...) is located by _env.bat (PY). +if not exist "%PY%" ( + echo [run_webui] no Python with deps found. Looked for: + echo %BUNDLE%\venv\python.exe ^(bundle venv^) + echo %ROOT%\venv\python.exe ^(root venv^) + echo %ROOT%\venv\Scripts\python.exe ^(project venv^) + echo Install into one of them: gradio requests torch safetensors pyyaml huggingface_hub + pause + exit /b 1 +) +echo [run_webui] python: %PY% + +echo [run_webui] the WebUI starts/switches audiocpp_server on demand +echo [run_webui] pick a model in the UI and click "load" (no need to run run_server.bat) +echo [run_webui] backend: %AUDIOCPP_BACKEND% (auto-detected; override with AUDIOCPP_BACKEND=gpu or cpu) +echo [run_webui] UI -^> http://127.0.0.1:7860 +"%PY%" "%WEBUI_DIR%\webui.py" + +endlocal +pause diff --git a/run_webui.sh b/run_webui.sh new file mode 100755 index 00000000..e2eb0fd8 --- /dev/null +++ b/run_webui.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +# Launch the audio.cpp WebUI on Linux/macOS (POSIX counterpart of run_webui.bat). +# +# The WebUI starts/switches audiocpp_server on demand — pick a model in the UI and +# click load; no need to start a server separately. Backend (cuda|cpu) is auto-detected +# by webui.py from nvidia-smi and the available build; override with AUDIOCPP_BACKEND=gpu|cpu. +# UI language: English by default, with 中文 / 中文繁體 selectable from the picker in the +# UI. That pick is saved to webui/configs/ui_language.json and wins on later runs, so +# AUDIOCPP_LANG (en|zh|zh-Hant) only sets the default before anything has been picked. +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +WEBUI_DIR="$ROOT/webui" + +# Locate a Python that has the deps (gradio/requests/torch/safetensors/opencc/...). +PY="" +for cand in \ + "${AUDIOCPP_PYTHON:-}" \ + "$ROOT/venv/bin/python" \ + "$ROOT/.venv/bin/python" \ + "$(command -v python3 || true)" \ + "$(command -v python || true)"; do + if [ -n "$cand" ] && [ -x "$cand" ]; then PY="$cand"; break; fi +done + +if [ -z "$PY" ]; then + echo "[run_webui] no Python found. Create a venv and install deps:" >&2 + echo " python3 -m venv venv && ./venv/bin/pip install -r requirements.txt" >&2 + exit 1 +fi + +echo "[run_webui] python: $PY" +echo "[run_webui] backend: ${AUDIOCPP_BACKEND:-auto} language: ${AUDIOCPP_LANG:-en (unless already picked in the UI)}" +echo "[run_webui] UI -> http://127.0.0.1:7860" +exec "$PY" "$WEBUI_DIR/webui.py" diff --git a/tools/model_manager_webui.py b/tools/model_manager_webui.py new file mode 100644 index 00000000..1e469304 --- /dev/null +++ b/tools/model_manager_webui.py @@ -0,0 +1,412 @@ +#!/usr/bin/env python3 +"""WebUI front-end for ``tools/model_manager.py`` — same CLI, sturdier downloads. + +The Gradio WebUI kicks off model installs as a background subprocess +(``model_manager_webui.py install ``), where two things matter that +the upstream tool intentionally does not do: + +* **Resumable, no-Torch downloads.** Weights land in a ``.part`` sidecar and + resume across runs via HTTP ``Range`` (or the hub client's own resume for + Xet-backed repos), so a dropped multi-GB transfer picks up instead of + restarting. A plain snapshot download must not import Torch either — its native + DLLs can fail to load in some environments (e.g. conda Python on Windows) and + would needlessly block downloads for models whose inference runs in the C++ + server. +* **Windows-tolerant finalization.** Defender/indexers and the WebUI's own + progress scan can briefly hold a directory handle right after the last shard is + written, so promoting a staged directory retries past transient sharing + violations. + +Rather than fork the 2700-line upstream tool, this module imports it and +overrides only the download path, then delegates to its unchanged CLI. Everything +not redefined here — the catalog, argument parsing, ``list``/``info``, converter +installs, all the ``convert_*`` post-processing — comes straight from +``model_manager`` (referenced as ``mm.*`` below, so what is overridden vs. +inherited stays obvious). +""" +from __future__ import annotations + +import os +import re +import shutil +import time +from pathlib import Path +from typing import Iterable +from urllib.error import HTTPError +from urllib.request import Request, urlopen + +import model_manager as mm + +# huggingface_hub is imported lazily (see _ensure_download_deps) so that +# ``list``/``info`` keep working without it, exactly as upstream keeps Torch lazy. +hf_hub_download = None + + +# --- dependency loading ------------------------------------------------------ + +def _ensure_download_deps() -> None: + """Import only what a plain snapshot/file download needs: huggingface_hub + yaml. + + Downloading weights must not drag in Torch (see the module docstring). yaml is + published into the upstream module's namespace because its config/convert + helpers read ``mm.yaml``.""" + global hf_hub_download + if hf_hub_download is not None: + return + from huggingface_hub import hf_hub_download as _hf_hub_download + import yaml as _yaml + + hf_hub_download = _hf_hub_download + mm.yaml = _yaml + + +def _ensure_install_deps() -> None: + """Everything _ensure_download_deps loads, plus Torch/safetensors for conversions. + + Only the composite/converter paths need this; plain snapshot downloads call + _ensure_download_deps() so they stay Torch-free. Torch et al. are populated on + the upstream module (mm.torch, mm.safe_open, ...) because its convert_* helpers + read them from there.""" + _ensure_download_deps() + mm._ensure_install_deps() + + +# --- plain-URL downloads (resumable) ----------------------------------------- + +def download_file( + url: str, + target: Path, + expected_size: int | None, + label: str | None = None, +) -> int: + """Download ``url`` into ``target``, resuming across runs when possible. + + Bytes land in a sidecar ``.part`` file first. If a previous run was + interrupted, an HTTP ``Range`` request fetches only the missing tail instead + of restarting the transfer (critical for multi-GB weights on flaky links); + the ``.part`` file is promoted to ``target`` only once the full length has + arrived. Files already present at the expected size are skipped outright. + """ + name = label or target.name + if expected_size is not None and target.is_file() and target.stat().st_size == expected_size: + print(f"skip {name} (already complete)") + return expected_size + + part = target.with_name(target.name + ".part") + existing = part.stat().st_size if part.is_file() else 0 + if expected_size is not None and existing >= expected_size: + # A finished-but-unpromoted leftover promotes as-is; anything larger than + # expected is corrupt, so discard it and start over. + if existing == expected_size: + part.replace(target) + print(f"skip {name} (already complete)") + return expected_size + part.unlink() + existing = 0 + + headers = mm.http_headers() + mode = "wb" + if existing > 0: + headers["Range"] = f"bytes={existing}-" + mode = "ab" + print(f"resume {name} (from {existing} bytes)") + else: + print(f"download {name}") + + request = Request(url, headers=headers) + try: + response = urlopen(request) + except HTTPError as ex: + if ex.code == 416 and existing > 0: + # Requested range past EOF: the server has nothing more to send. + if expected_size is None or existing == expected_size: + part.replace(target) + return existing + part.unlink(missing_ok=True) + raise + + with response: + status = getattr(response, "status", None) or response.getcode() + if existing > 0 and status != 206: + # Server ignored the Range header (answered 200) — start over. + print(f"restart {name} (server ignored resume)") + existing = 0 + mode = "wb" + written = existing + with part.open(mode) as handle: + while True: + chunk = response.read(1 << 20) + if not chunk: + break + handle.write(chunk) + written += len(chunk) + + if expected_size is not None and written != expected_size: + raise RuntimeError(f"downloaded size mismatch for {target}: {written} != {expected_size}") + part.replace(target) + return written + + +# --- huggingface_hub downloads (Xet-backed repos) ---------------------------- + +def download_hf_file( + source, + relative_path: str, + destination_root: Path, + expected_size: int | None, +) -> None: + """Fetch one repo file into ``destination_root/relative_path`` via huggingface_hub. + + Plain HTTP against the resolve URL cannot be used here: Xet-backed repos redirect + to a CDN that rejects ordinary GETs, so only the hub client (with ``hf_xet``) can + pull their weights. ``local_dir`` writes the file straight to its final location + instead of duplicating it in the shared blob cache, and the hub client does its + own resume, so an interrupted run picks up where it left off just like + ``download_file`` does for the plain-URL callers. + """ + destination = destination_root / relative_path + if expected_size is not None and destination.is_file() and destination.stat().st_size == expected_size: + print(f"skip {relative_path} (already complete)") + return + print(f"download {relative_path}") + hf_hub_download( + repo_id=source.repo_id, + filename=relative_path, + revision=source.revision, + local_dir=destination_root, + token=mm.huggingface_token(), + ) + + +def prune_hf_local_dir_cache(destination_root: Path) -> None: + """Drop the ``.cache/huggingface`` bookkeeping tree hf_hub_download leaves behind. + + With ``local_dir=`` the hub client stores per-file resume metadata under + ``/.cache/huggingface``. It is invisible to validation (which only + checks that required files exist), but without this it would be promoted into + the installed model directory as stray junk. Only safe to call once every file + for this destination has arrived, since removing it discards resume state. + """ + cache_dir = destination_root / ".cache" / "huggingface" + if cache_dir.is_dir(): + shutil.rmtree(cache_dir, ignore_errors=True) + parent = destination_root / ".cache" + if parent.is_dir() and not any(parent.iterdir()): + parent.rmdir() + + +def install_snapshot_into_dir( + source, + destination_root: Path, + required_files: Iterable[str], + *, + validate: bool = True, +) -> None: + files = mm.list_hf_files(source) + for relative, expected_size in files: + destination = destination_root / relative + destination.parent.mkdir(parents=True, exist_ok=True) + download_hf_file(source, relative, destination_root, expected_size) + prune_hf_local_dir_cache(destination_root) + if validate: + mm.validate_required_files_list(required_files, destination_root, source.repo_id) + + +# --- resumable staging + Windows-tolerant finalization ----------------------- + +def staging_dir_name(package) -> str: + """Deterministic staging directory name so an interrupted install resumes + into the same place on the next run (a random ``mkdtemp`` name would strand + the partial downloads).""" + safe = re.sub(r"[^A-Za-z0-9_.-]", "_", package.target_directory) + return f"{safe}.partial" + + +def prune_staging_root(staging_root: Path) -> None: + try: + if staging_root.exists() and not any(staging_root.iterdir()): + staging_root.rmdir() + except OSError: + pass + + +def promote_staging_directory(source: Path, destination: Path) -> None: + """Rename a completed staging directory, tolerating transient Windows locks. + + Defender/indexers and the WebUI progress scan can briefly retain a directory + enumeration handle just after the final shard is written. Windows then reports + access denied/sharing violation even though the ACL and destination are valid. + """ + retry_delays = (0.1, 0.25, 0.5, 1.0, 2.0, 3.0, 5.0) + for attempt, delay in enumerate(retry_delays, start=1): + try: + source.rename(destination) + return + except OSError as error: + if os.name != "nt" or getattr(error, "winerror", None) not in {5, 32}: + raise + print( + f"retry model directory finalization ({attempt}/{len(retry_delays)}): " + f"{source} -> {destination} ({error})" + ) + time.sleep(delay) + source.rename(destination) + + +def install_snapshot(package, source, models_root: Path, overwrite: bool) -> Path: + target_dir = models_root / package.target_directory + staging_root = models_root / ".engine_model_staging" + staging_root.mkdir(parents=True, exist_ok=True) + staging_dir = staging_root / staging_dir_name(package) + staging_dir.mkdir(parents=True, exist_ok=True) + try: + pre_validate_files = tuple(relative for relative in package.required_files if relative != "audiovae.safetensors") + install_snapshot_into_dir(source, staging_dir, pre_validate_files, validate=package.id != "voxcpm2") + if package.id == "voxcpm2": + mm.convert_voxcpm2_audiovae(staging_dir) + mm.validate_required_files(package, staging_dir) + elif package.id == "moss_tts_nano_100m_model": + mm.convert_moss_tts_weights(staging_dir) + if target_dir.exists(): + if not overwrite: + raise RuntimeError(f"model directory already exists: {target_dir}") + shutil.rmtree(target_dir) + target_dir.parent.mkdir(parents=True, exist_ok=True) + promote_staging_directory(staging_dir, target_dir) + return target_dir + finally: + # On success the staging dir was renamed away; on failure it is kept so + # the next run resumes partial downloads. Drop the parent when empty. + prune_staging_root(staging_root) + + +def install_composite_snapshot(package, source, models_root: Path, overwrite: bool) -> Path: + package_root = models_root / package.target_directory + staging_root = models_root / ".engine_model_staging" + staging_root.mkdir(parents=True, exist_ok=True) + staging_bundle = staging_root / staging_dir_name(package) + staging_bundle.mkdir(parents=True, exist_ok=True) + staged_roots: dict[Path, Path] = {} + try: + staged_package_root = staging_bundle / package.target_directory + for placement in source.placements: + destination_root = mm.normalized_join(staged_package_root, placement.target_subdir) + final_root = mm.normalized_join(package_root, placement.target_subdir) + if final_root.exists() and not overwrite: + mm.validate_required_files_list(placement.required_files, final_root, str(final_root)) + continue + destination_root.mkdir(parents=True, exist_ok=True) + install_snapshot_into_dir(placement.source, destination_root, placement.required_files) + staged_roots[final_root] = destination_root + + if package.id == "vevo2": + whisper_root = staging_bundle / "whisper-medium" + mm.install_whisper_medium_dependency(whisper_root) + staged_roots[package_root.parent / "whisper-medium"] = whisper_root + mm.prepare_vevo2_snapshot_layout(staged_package_root) + elif package.id == "ace_step": + mm.convert_ace_step_silence_latent(staged_package_root / "acestep-v15-turbo") + mm.convert_ace_step_silence_latent(staged_package_root / "acestep-v15-base") + elif package.id == "moss_tts_nano_100m": + mm.convert_moss_tts_weights(staged_package_root) + elif package.id in {"irodori_tts_500m_v3", "irodori_tts_600m_v3_voice_design"}: + mm.write_irodori_model_config(staged_package_root) + dacvae_root = staged_package_root.parent / "Semantic-DACVAE-Japanese-32dim" + if dacvae_root.exists(): + mm.convert_irodori_dacvae_weights(dacvae_root) + elif package.id == "outetts_1_0_1b": + dac_root = staged_package_root.parent / "DAC.speech.v1.0" + if dac_root.exists(): + mm.convert_outetts_dac_weights(dac_root) + elif package.id == "vibevoice_asr": + mm.copy_bundled_model_manager_assets( + "vibevoice_1_5b", + staged_package_root, + ("tokenizer.json", "tokenizer_config.json", "vocab.json", "merges.txt"), + ) + elif package.id in {"vibevoice_1_5b", "vibevoice_7b"}: + # VibeVoice 1.5B and 7B share the same Qwen2.5 tokenizer, and neither + # upstream repo ships the tokenizer files, so both reuse one bundle. + mm.copy_bundled_model_manager_assets( + "vibevoice_1_5b", + staged_package_root, + ("tokenizer.json", "tokenizer_config.json", "vocab.json", "merges.txt"), + ) + mm.validate_composite_required_files(package, staged_package_root, package_root) + + top_level_roots: list[Path] = [] + for final_root in sorted(staged_roots.keys(), key=lambda path: len(path.parts)): + if any(final_root.is_relative_to(existing) for existing in top_level_roots): + continue + top_level_roots.append(final_root) + + for final_root in sorted(top_level_roots, key=lambda path: len(path.parts), reverse=True): + if final_root.exists(): + if not overwrite: + raise RuntimeError(f"model directory already exists: {final_root}") + shutil.rmtree(final_root) + + for final_root in sorted(top_level_roots, key=lambda path: len(path.parts)): + destination_root = staged_roots[final_root] + final_root.parent.mkdir(parents=True, exist_ok=True) + promote_staging_directory(destination_root, final_root) + shutil.rmtree(staging_bundle, ignore_errors=True) + return package_root + finally: + # Failed runs keep their staging bundle so partial downloads resume; + # successful runs already removed it above. Drop the parent when empty. + try: + if staging_root.exists() and not any(staging_root.iterdir()): + staging_root.rmdir() + except OSError: + pass + + +# --- install command (chooses download-only vs. full deps) ------------------- + +def command_install(args) -> int: + package = mm.PACKAGE_BY_ID.get(args.package_id) + if package is None: + raise RuntimeError(f"unknown package id: {args.package_id}") + models_root = mm.resolve_path(args.models_root) + models_root.mkdir(parents=True, exist_ok=True) + source = package.source + if isinstance(source, mm.UnsupportedSource): + raise RuntimeError(f"{package.id} is not installable: {source.reason}") + if isinstance(source, mm.SnapshotSource): + # Plain download: huggingface_hub only, no Torch DLLs. + _ensure_download_deps() + install_path = install_snapshot(package, source, models_root, args.overwrite) + elif isinstance(source, mm.CompositeSnapshotSource): + # Composite snapshots may run a Torch post-process step, so bring in full deps. + _ensure_install_deps() + install_path = install_composite_snapshot(package, source, models_root, args.overwrite) + else: + _ensure_install_deps() + install_path = mm.install_converter( + package, + source, + models_root, + args.overwrite, + args.source_file, + args.output_file, + args.source_dir, + args.variant, + ) + print(f"installed {package.id} -> {install_path}") + return 0 + + +# Patch the two entry points the upstream call-graph resolves by name at run time: +# * main() dispatches ``install`` to ``command_install`` looked up in mm's globals; +# * mm's own dependency installers (whisper-medium, demucs, converters) call +# ``download_file`` in mm's globals — routing them through the resumable version +# matches what a single combined module would do. +# Everything else this module overrides is reached only from command_install below, +# so it resolves within this module's own namespace and needs no patching. +mm.download_file = download_file +mm.command_install = command_install + + +if __name__ == "__main__": + raise SystemExit(mm.main()) diff --git a/webui/README.md b/webui/README.md new file mode 100644 index 00000000..3ea25ff2 --- /dev/null +++ b/webui/README.md @@ -0,0 +1,403 @@ +# audio.cpp WebUI 启动脚本说明 + +仓库根目录下的一组 `.bat` 脚本，覆盖 audio.cpp 的三种本地运行方式：命令行单句合成、 +HTTP API 服务、图形界面。所有脚本都可以**双击运行**，也可以在命令行/PowerShell +里带参数调用。 + +| 脚本 | 作用 | 典型命令 | +|---|---|---| +| `run_cli_tts.bat` | 单句/单次命令行 TTS | `run_cli_tts.bat qwen3-tts "你好世界"` | +| `run_server.bat` | OpenAI 兼容 HTTP API 服务 | `run_server.bat qwen3-tts 8080` | +| `run_server_asr.bat` | Qwen3-ASR 服务（`run_server.bat` 的 ASR 预设） | `run_server_asr.bat`（默认 :8081） | +| `run_webui.bat` | Gradio 网页界面（按需起服务） | `run_webui.bat` | +| `_env.bat` | 共享环境探测（**不直接运行**） | 被其它脚本 `call` | + +## Linux / macOS + +本文档其余部分描述 Windows 的 `.bat` 脚本。在 Linux / macOS 上用仓库根目录的 +`run_webui.sh`（`run_webui.bat` 的 POSIX 版本）： + +```bash +python3 -m venv venv && ./venv/bin/pip install -r requirements.txt +./run_webui.sh # UI -> http://127.0.0.1:7860 +``` + +- Python 解释器依次探测 `$AUDIOCPP_PYTHON`、`venv/bin/python`、`.venv/bin/python`、 + `python3`、`python`。 +- 后端（cuda/cpu）自动探测：Windows 看 `nvcuda.dll`，其它平台看 `nvidia-smi`， + 再确认对应的 server 构建存在；用 `AUDIOCPP_BACKEND=gpu|cpu` 覆盖。 +- 二进制既支持 portable 包的 `gpu/`、`cpu/` 目录，也支持从源码构建的 + `build/--/bin`（如 `build/linux-cuda-release/bin`）。 + 直接 `cmake -B build` 产生的 `build/bin` 也能识别——目录名不含后端信息时， + 从 `CMakeCache.txt` 的 `GGML_CUDA` 判断。 +- `requirements.txt` 里 Windows 专用的包（pywin32 及 SpeakType 用到的 + pythonnet/pywebview）带 `sys_platform` 标记，因此在 Linux 上也能直接安装。 + +## 界面语言 / UI language + +界面支持 **English / 中文 / 中文繁體**，默认英文，右上角的语言下拉可随时切换。 +选择会保存到 `webui/configs/ui_language.json`，下次启动沿用。 + +环境变量 `AUDIOCPP_LANG`（`en` | `zh` | `zh-Hant`，也接受 `zh_TW`、`zh-CN` 等写法） +只在**还没有保存过选择时**决定默认语言——否则它会盖掉用户在界面里的明确选择。 +想改回环境变量控制，删掉 `webui/configs/ui_language.json` 即可。 + +> 繁體中文由 OpenCC 从简体字面转换（`opencc-python-reimplemented`），属于**字形**转换， +> 不做台湾/香港的用词替换（例如「软件」→「軟件」而非「軟體」）。 + +> 长文本合成不再需要单独脚本（原 `run_tts_long.bat` 已移除）：WebUI 的 TTS 标签页会自动 +> 把长文本分段（VibeVoice 600 字/段，其它模型 1000 字/段），逐段合成后拼接成一个 wav。 +> 命令行等价物是 `audiocpp_cli` 的 `--batch-text-file --batch-merge-audio concat`。 +> +> 反过来，**VibeVoice 对过短文本（约 <40 个汉字）会整段胡言乱语**——模型特性，与音色/ +> 参数/seed 无关，WebUI 会直接拦截并提示加长文本或改用其它模型；分段后的过短尾段也会 +> 自动并回前一段。短句测试请用 `qwen3-tts` / `voxcpm2` / `pocket-tts`。 + +--- + +## 通用约定 + +- **模型用 catalog id 指定。** 两个合成脚本（cli / server）都用 `configs\models_catalog.json` + 里的 **id** 来指定模型，脚本会自动查出它的 `family` / `task` / 绝对路径，你不用再手写这些。 + 当前已安装的 id：`qwen3-tts`、`qwen3-asr`、`vibevoice`、`omnivoice`、`pocket-tts`。 + 未安装的 id 会提示 “not installed”，可在 WebUI 里下载，或用 + `python tools/model_manager.py install ` 安装（见 `models_catalog.json`）。 +- **后端自动选择：** 检测到 CUDA（NVIDIA 驱动）就用 GPU，否则回退 CPU。 + 想强制某个后端，设环境变量 `AUDIOCPP_BACKEND=gpu`（=cuda）或 `AUDIOCPP_BACKEND=cpu`。 + CLI、server、WebUI 都遵循这一检测（无 N 卡的机器自动落到 CPU 版，速度较慢、部分大模型不实用）。 +- **路径基准：** 脚本内相对路径（如 `voice\demo_01_man.wav`、`output\xxx.wav`）都相对 `webui\` 目录。 +- **可执行文件来源：** 自动定位整合包 `..\audiocpp-portable`（内含 `cpu\ gpu\ models\`）， + 脚本被拷进整合包时也能自识别。 + +--- + +## `_env.bat`（内部共享，不要直接运行） + +被其它脚本 `call`，负责一次性设置好公共变量（故意不用 `setlocal`，这样变量能带回调用方）： + +- `BUNDLE` — 整合包根目录（含 `cpu\ gpu\ models\`） +- `HAS_CUDA` — 是否检测到 CUDA（`nvcuda.dll` 或 `nvidia-smi`） +- `BACKEND` / `CLI_EXE` — 选定的后端（`cuda`/`cpu`）与对应的 `audiocpp_cli.exe` +- `SERVER_EXE` — 按 `BACKEND` 选 `gpu\` 或 `cpu\` 的 `audiocpp_server.exe`（cpu 版缺失时回退 gpu 版） +- `PY` — 带依赖的 Python（供 `run_webui.bat` 用） + +改动探测逻辑只需改这一个文件。 + +--- + +## 1. `run_cli_tts.bat` — 命令行单次 TTS + +一次加载模型、合成一句、输出一个 wav。适合快速测试或脚本化单次生成。 + +``` +用法: run_cli_tts.bat [model_id] ["合成文本"] [voice_ref] [ref_text] +``` + +| 位置参数 | 含义 | 默认 | +|---|---|---| +| 1 `model_id` | catalog 里的模型 id | `qwen3-tts` | +| 2 `"文本"` | 要合成的文本（含空格务必加引号） | 一句英文示例 | +| 3 `voice_ref` | 参考音色 wav（声音克隆用） | `voice\demo_01_man.wav` | +| 4 `ref_text` | 参考音频对应的文本 | 示例台词 | + +- 输出固定到 `output\out_cli.wav`（可在脚本顶部改 `OUT`）。 +- 语言、`max-tokens`、`seed` 等也在脚本顶部可改。 +- 非声音克隆的模型可把 `VOICE_REF` 留空（脚本会自动不带 `--voice-ref`）。 + +**示例** + +```bat +run_cli_tts.bat qwen3-tts "Hello, this is audio dot cpp." +run_cli_tts.bat qwen3-tts "换个音色" voice\demo_02_woman.wav "her reference line." +set AUDIOCPP_BACKEND=cpu & run_cli_tts.bat qwen3-tts "强制用 CPU 跑" +``` + +--- + +## 2. `run_server.bat` — HTTP API 服务 + +启动一个 OpenAI 兼容的 HTTP 服务，供**其它应用**调用。后端自动检测：有 CUDA 用 GPU， +否则用 CPU 版 server（CPU 下自动把 ggml 线程数设为核数-1；速度较慢，部分大模型不实用）。 + +``` +用法: run_server.bat [port] [device] +``` + +| 位置参数 | 含义 | 默认 | +|---|---|---| +| 1 `model_id` | catalog 里的模型 id | （必填） | +| 2 `port` | 监听端口 | `8080` | +| 3 `device` | GPU 设备号 | `0` | + +- 脚本会用该 id 生成一份**单模型、绝对路径**的临时配置 + `%TEMP%\audiocpp_server_.json`（按端口命名，两个实例互不冲突），再启动 server。 +- **同时起两个服务**：在两个窗口分别运行不同 id + 不同端口，例如一个做 TTS、一个做 ASR： + + ```bat + run_server.bat qwen3-tts 8080 :: 窗口 A：TTS + run_server_asr.bat :: 窗口 B：ASR（= run_server.bat qwen3-asr 8081） + ``` + + ⚠️ 两个模型要同时装进显存（8GB 下 0.6B + 0.6B 没问题；两个 1.7B 装不下）。 +- **`run_server_asr.bat`**：`run_server.bat` 的 ASR 预设包装，双击即用。 + 参数为 `[port] [device] [model_id]`，默认 `8081` / `0` / `qwen3-asr`。 +- **局域网访问**：设 `AUDIOCPP_HOST=0.0.0.0` 让其它机器能连（**无鉴权**，仅在可信内网使用）。 + +### API 端点 + +| 方法 | 路径 | 说明 | +|---|---|---| +| GET | `/health` | 就绪状态 + 已配置模型数 | +| GET | `/v1/models` | 列出该实例加载的模型 | +| POST | `/v1/audio/speech` | 文本转语音，默认返回 `audio/wav` | +| POST | `/v1/audio/transcriptions` | 语音转文本（ASR） | +| POST | `/v1/tasks/run` | 通用任务入口（字段同 CLI 请求格式） | + +> **参考音色是每次请求带的**（server 端不预存音色）。声音克隆 TTS 每个请求要带 +> `voice_ref` + `reference_text`。请求里的 `voice_ref` / `audio` 路径是**服务器本机路径**， +> 相对路径以 server 的工作目录（用本脚本启动时即 `webui\`）为基准，也可用绝对路径。 + +### 调用示例 + +TTS（用现成模板 `configs\req_speech.json`，其中含 `input`/`voice_ref`/`reference_text`）： + +```bat +curl http://127.0.0.1:8080/v1/audio/speech -H "Content-Type: application/json" -o output\out_server.wav -d @configs\req_speech.json +``` + +ASR（音频用服务器本机路径）： + +```bat +curl http://127.0.0.1:8081/v1/audio/transcriptions -H "Content-Type: application/json" -d "{\"model\":\"qwen3-asr\",\"audio\":\"D:/path/to/input.wav\"}" +``` + +查看状态： + +```bat +curl http://127.0.0.1:8080/health +curl http://127.0.0.1:8080/v1/models +``` + +--- + +## 3. `run_webui.bat` — 图形界面 + +启动 Gradio 网页界面（`webui.py`），浏览器访问 **http://127.0.0.1:7860**。 + +- **按需加载**：不需要先跑 `run_server.bat`——在界面里选模型点“加载”/“生成”时，WebUI 会自动 + 起/切换底层的 `audiocpp_server`（一次一个模型在显存里，换模型即重启）。 +- 界面里可上传参考音色、下载未安装的模型、填 HF token / 代理等。 +- 后端自动检测（同上：有 CUDA 用 GPU，否则 CPU）；`AUDIOCPP_BACKEND=gpu|cpu` 可强制。 + CPU 模式下 ggml 线程数自动设为核数-1（可用 `AUDIOCPP_THREADS=N` 覆盖），且不再显示显存警告。 + +> 网页界面（7860）是给人用的；要给**其它程序**当 API，请用 `run_server.bat` 起的 **8080** 那个服务， +> 或让 WebUI 起来后直接打它的 8080 端口（见 `run_server.bat` 的端点表）。 + +--- + +## WebUI 高级参数（按模型自动生成控件） + +TTS 标签页「合成设置 → 高级参数」里的控件由 **`configs/model_params.json`** 驱动：选中某个模型后，WebUI 按其 `family` **动态生成对应的滑块/数字框/开关/文本框**（`gr.render`），不用再手写 JSON。控件下方还留了一个可折叠的「其它参数（JSON）」兜底框，用于传配置里没列出的键。通用规则： + +- **只有你改动过的控件值才会随请求发送**（未动的用模型自身默认值）；`options` 合并顺序：家族默认 → 生成的控件 → JSON 框（JSON 覆盖控件）。 +- `seed`、`max_tokens` 已有专用输入框（合成设置），不在此重复。 +- 参考音色用「上传/录制」或「内置参考音色」；参考音频对应的原话用「参考文本」框（等价 `reference_text`）。 +- 填错的值通常被忽略，或由 server 报错——错误显示在**输出音频下方**（不弹卡片）。 +- **Chatterbox** 的克隆参数在模型加载时固定：改动后需重新点『📥 加载模型』才生效（否则会报 “session config is fixed”）。 + +### 自定义控件（编辑 `configs/model_params.json`） + +按 `family` 分组，每项一个控件规格： + +```json +{"name": "guidance_scale", "type": "slider", "label": "guidance_scale", + "default": 1.3, "minimum": 0.0, "maximum": 5.0, "step": 0.1, "info": "CFG 引导强度"} +``` + +- `name`：透传给请求 `options` 的键名。`type`：`slider` / `number`（`precision:0` 表整数）/ `bool` / `text` / `choice`（配 `choices:[...]`）。 +- `default` 应等于模型默认值（已按各 `src/models//*.cpp` 校对）。 +- 改完点界面上的『🔄 刷新列表』即可重新加载本文件，无需重启。 +- 文件路径 / parity 类少见参数（如 `*_noise_file`）未纳入控件，可用「其它参数（JSON）」框传。量化键（如 `vibevoice.weight_type`）见项目根 `README.md`。 + +下表是每个模型 `session.cpp` **实际读取**的完整可用键（控件是其中精选的常用子集；其余键仍可用 JSON 框传）： + +| 模型（family） | 可用键（JSON 框也可传） | 示例 | +|---|---|---| +| **Qwen3-TTS**（qwen3_tts）0.6B / 1.7B / CustomVoice | `do_sample` `temperature` `top_k` `top_p`；CustomVoice 版另有 `speaker` | `{"do_sample": true, "temperature": 0.8, "top_k": 40, "top_p": 0.9}`
CustomVoice 选内置音色：`{"speaker": ""}` | +| **VibeVoice**（vibevoice）1.5B 长文/多说话人 | `num_inference_steps` `guidance_scale` `max_length_times` `do_sample` `temperature` `top_k` `top_p`；多说话人 `voice_samples`（逗号分隔 wav，最多 4，**不能**与参考音色同用） | `{"num_inference_steps": 10, "guidance_scale": 1.3, "max_length_times": 2.0}`
多说话人：`{"voice_samples": "D:/a.wav,D:/b.wav"}` | +| **VoxCPM2**（voxcpm2） | `num_inference_steps` `guidance_scale` `min_tokens` `retry_badcase` `retry_badcase_max_times` `retry_badcase_ratio_threshold`；参考原话 `prompt_text` | `{"num_inference_steps": 10, "guidance_scale": 2.0, "retry_badcase": true}` | +| **MioTTS**（miotts，需 MioCodec） | `temperature` `top_k` `top_p` `repetition_penalty` `presence_penalty` `frequency_penalty` `do_sample` `best_of_n` `best_of_n_enabled` `best_of_n_language` | `{"temperature": 0.9, "top_p": 0.9, "repetition_penalty": 1.1, "best_of_n": 3}` | +| **Chatterbox**（chatterbox，声音克隆） | `exaggeration` `guidance_scale` `temperature` `repetition_penalty` `min_p` `top_p` `s3gen_cfg_rate` `max_new_tokens` `do_sample` `greedy` `stop_on_eos` | `{"exaggeration": 0.5, "guidance_scale": 0.5, "temperature": 0.8, "repetition_penalty": 1.2}` | +| **OmniVoice**（omnivoice） | `instruct`（风格/指令文本）；`reference_text`（一般用「参考文本」框即可） | `{"instruct": "以轻快的语气朗读"}` | +| **Pocket TTS**（pocket_tts） | 无专用高级参数（只需参考音色 + 语言） | — | + +> 键名取自各模型 `src/models//session.cpp` 实际读取的选项；同一键在不同模型里的取值范围/含义可能不同。量化相关键（如 `vibevoice.weight_type`、`voxcpm2.*_weight_type`）见项目根 `README.md` 的量化章节，不是通用默认项。 + +### 音乐生成 / 声音转换参数详解 + +页面上的提示已精简，完整说明集中在这里。 + +**ACE-Step（音乐生成/编辑）** + +- 提示词写风格/乐器/情绪（英文效果最好），可选填歌词；时长填 `-1` 表示自动。 +- `task_route` 操作类型：`text2music`＝纯文生曲（默认，不需要源音频）；`cover`＝换词翻唱 + （原版 Remix 主路线，配合下面两个 cover 滑条）；`cover-nofsq`＝cover 变体（不过 FSQ 量化）； + `remix`＝flow-edit 精细换词；`complete` / `lego` / `extract` / `repaint` 为其它编辑路线。 + **除 text2music 外都需要上传源音频。** +- 上传源音频后建议先点『🔍 分析源音频』：反推源曲描述/歌词/BPM/调性并自动填入高级参数 + （remix/cover 换词前尤其建议；首次需先『📥 加载模型』，1 分钟音频约需几十秒）。 + 分析结果可复现：同一音频每次分析一致（VAE 均值编码；seed=-1 时分析固定用 1234， + 想重抽歌词转写可换一个具体 seed）。 +- 扩散参数：`num_inference_steps` turbo 上限 20，remix 路由不填时默认 16、其他路由默认 8； + `shift`（时间步弯曲）默认 3.0 对齐原版 turbo UI——调回 1.0 会明显劣化 remix 换词咬字。 +- cover 路线两个滑条： + - `audio_cover_strength`（Remix 强度）：多少比例的去噪步参考源曲结构，1=贴近原曲、 + 0=自由发挥；原版 Remix 建议 0.5。仅 cover/cover-nofsq 生效。 + - `cover_noise_strength`（旋律保持）：从源曲部分加噪的起点开始去噪，0=不保旋律、 + 0.1~0.25=推荐区间（保旋律又能换词换风格）、越高越贴原曲。仅 cover 生效。 +- remix（flow-edit）参数： + - `source_caption` / `source_lyrics`：源侧文本条件（源歌曲本来的风格描述 / 原歌词， + 带 `[Verse]` `[Chorus]` 标签）；留空 caption 用主提示词；『🔍 分析』可自动填。 + **新歌词写在主界面『歌词』框。** + - `flow_edit_n_min`（编辑起点）：跳过前面高噪声步的比例，0=从头编辑；调大更保源曲但换词更弱。 + - `flow_edit_n_max`（编辑终点）：1=全程配对编辑；调低到 0.7~0.9 时收尾只朝新歌词去噪—— + **歌词唱不出来时优先调这个**。 + - `flow_edit_n_avg`：每步多次采样取平均（remix 默认 2，更稳），1=最快。 + 注意 remix 默认 16 步 × n_avg 2 ≈ 旧默认（8 步 ×1）4 倍耗时，求快可手动调回。 +- 曲谱参数 `bpm` / `keyscale`（如 `F major`、`c# minor`）/ `timesignature`（如 `4`）： + 0/留空=不指定；『🔍 分析』后自动填。 + +**Stable Audio（音乐/音效）**：提示词**仅支持英文**，不使用歌词；music 版生成音乐、sfx 版生成音效。 +上传源音频可做续写/修补：`audio_input_kind` 选 `init_audio`（配 `init_noise_level` 强度）或 +`inpaint_audio`。 + +**HeartMuLa（歌词+标签生成歌曲）**：高级参数 `tags` 必填（逗号分隔，如 +`pop,bright,drums,female vocals`），『歌词』填唱词。3B 模型，官方 120 秒长歌实测峰值显存 +~25G（docs/memory_saver.md），8G 显卡跑不动；已默认开 mem_saver，长歌曲可开 `infinite_mode`。 + +**Chatterbox VC（语音转换）**：源语音提供内容，目标音色参考提供说话人身份，输出 24kHz +单声道语音。`s3gen_cfg_rate` 控制音色引导强度，`num_inference_steps` 控制生成步数；默认分别为 +0.7 和 10。该入口与 TTS 页的 Chatterbox 声音克隆共用同一套模型文件。 + +**Seed-VC（语音转换）**：源语音 + 目标音色参考（几秒到几十秒干净人声）。`route` 留空按任务默认 +（vc 条目→`v2_vc`，svc 条目→`v1_svc`）；`v1_whisper_bigvgan_vc` / `v1_xlsr_hift_vc` 为 v1 旧路线； +`v1_svc` 只能配 svc 条目。`intelligibility_cfg_rate` / `similarity_cfg_rate` 仅 v2 生效， +`inference_cfg_rate` 仅 v1 生效。 + +**Vevo2（语音转换）**：默认 `route=style_preserved_vc`（保留源语音的说话风格，只换音色）。 +`route` 留空按条目任务默认（vc→style_preserved_vc，svc→style_preserved_svc，s2s→editing）， +且须与所选条目任务匹配；`style_converted_*` / `editing` 需在「其它参数(JSON)」里补 +`style_ref`（服务器本地 wav 路径）/ `style_ref_text` / `target_text`。 +`use_pitch_shift`（按源/目标中位音高差整体移调）留空按路线默认：style_preserved_* 及 +singing 路线默认开，style_converted_vc / editing 默认关。 +长音频按『目标音色时长 + 每段源时长 ≤ 显存预算』自适应分段后拼接，参考音色超过约 10s +自动截短（8G 显存限制）。 + +### 各任务页输入要求详解 + +- **VibeVoice**：多说话人脚本每行 `Speaker N: 内容`（N 从 0 起），只填普通文字会自动包成 + `Speaker 0: ...`。多角色不同音色用高级参数 `voice_samples`（逗号分隔服务器本地 wav，≤4 个）， + 此时**不要**再上传参考音色。 +- **VoxCPM2 / Qwen3-TTS**：上传/选一段干净的单人参考音色并在『参考文本』填该音频的原话， + 否则可能提前截断。长文本自动分段合成后拼接；VoxCPM2 在 8G 显卡默认 q8_0 量化。 +- **Chatterbox**：语言只支持 english / spanish / french / german / italian / portuguese / korean + （无中文/日文/俄文，也没有自动检测）；『留空』=英语。 +- **Qwen3-ASR**：长音频自动在静音处按 ≤60 秒分段转写后拼接。『上下文提示』填人名/术语/背景 + （如：会议讨论 ggml 量化，参会人：张伟、李娜）帮助认出专有名词。对话模式（限 120s）先用 + Sortformer 说话人分离（≤4 人）再逐段转写成带说话人和时间戳的对话稿，需已安装 Sortformer 模型。 +- **音频分析（VAD/分离/对齐）**：WAV 输入自动转 16 kHz 单声道后送模型，结果时间轴按 16 kHz 换算。 + Qwen3 强制对齐单次音频上限约 115 秒。 +- **音源分离**：HTDemucs 输出 drums/bass/other/vocals 四轨（长音频耗时较长）； + Mel-Band RoFormer 输出人声轨 + 伴奏轨（mixture − vocals）。 +- **IndexTTS2**（0.3 新增）：中/英声音克隆，**必须**提供参考音色。情感控制在高级参数： + `emotion_text` 填情感描述（填了会自动开启 `use_emotion_text`）+ `emotion_alpha` 调强度； + 或勾 `use_emotion_text` 从朗读文本自动推断；`emotion_vector`（8 个浮点）走 JSON 兜底框。 +- **Irodori-TTS**（0.3 新增，日语）：500M 默认无参考直接生成，上传参考音色自动切克隆 + （界面替你发 `no_ref=false`）；600M VoiceDesign 在『声音设计』页用日语 caption 描述音色。 + 语言下拉只认 japanese/留空。 +- **MOSS-TTS**（0.3 新增）：Local v1.5 纯文本可生成，克隆时建议配『参考文本』，输出 48kHz + 立体声；Nano 100M 轻量，无参考=续写式生成（音色随机），有参考=克隆。 +- **Supertonic 3**（0.3 新增）：预置音色多语种 TTS（英/日/韩/欧洲语种，**无中文**）， + 高级参数选 `voice`（M1-M5 男 / F1-F5 女）和 `speaking_rate`；不支持参考音频克隆。 +- **模型下载**在后台进行，进度自动刷新，也可点「📊 下载进度」手动查看。 + +### GGUF 转换、检查与加载 + +每个任务页的「模型管理」卡片都提供同一组 GGUF 操作：选择类型（默认 `q8_0`）后点「🧊 转换 GGUF」， +会把结果写为所选模型目录下的 `model.gguf`；已有文件不会被覆盖。点「🔎 检查 GGUF」会在页面上执行 +`audiocpp_gguf.exe --inspect` 并显示包的元数据。对已接入原生 GGUF 的模型，目录存在 `model.gguf` 时，普通 +「📥 加载模型」会自动优先使用 GGUF；点「🗑️ 删除 GGUF」会删除该文件（以及同名残留 `.tmp`），下次普通加载 +即恢复原始权重。 + +- 转换器按顺序查找开发构建的 `build\windows-cuda-release\bin` / `build\windows-cpu-release\bin`，以及整合包的 + `audiocpp-portable\gpu` / `audiocpp-portable\cpu`；也可用 `AUDIOCPP_GGUF` 指向自定义 `audiocpp_gguf.exe`。 +- 页面只会把已接入原生 GGUF 模型规格、且能明确整理转换输入的模型标为「可转换」；存在 `.safetensors` 不代表对应 + C++ 后端已支持 GGUF。支持转换但尚未完整安装的模型会提前显示「可转换，但模型未完整安装」，便于下载前判断； + Stable Audio 当前仍使用原始权重，不会标为可转换。 +- 页面自动处理受支持的单个 `model.safetensors`、分片索引和 Qwen3-TTS 复合权重。其他需要多个命名 + `--input namespace=...` 的复合模型仍应使用命令行，避免 UI 猜错权重命名空间。 +- 仅 audio.cpp-native GGUF 可加载；量化兼容性因模型和推理路线而异。转换成功也应先用短样本检查输出质量。 + +--- + +## 模型 id 速查 + +完整清单见 `configs\models_catalog.json`（每条含 `id` / `family` / `path` / `task` / `download_id`）。 +常用： + +| id | 家族 | 任务 | 说明 | +|---|---|---|---| +| `qwen3-tts` | qwen3_tts | tts | Qwen3-TTS 0.6B（声音克隆） | +| `qwen3-asr` | qwen3_asr | asr | Qwen3-ASR 0.6B | +| `vibevoice` | vibevoice | tts | VibeVoice 1.5B（长文/多说话人，`Speaker N:` 脚本） | +| `omnivoice` | omnivoice | tts | OmniVoice | +| `pocket-tts` | pocket_tts | tts | Pocket TTS（需参考音色） | +| `index-tts2` | index_tts2 | tts | IndexTTS2（中英克隆+情感，需参考音色） | +| `irodori-tts` | irodori_tts | tts | Irodori-TTS 500M（日语） | +| `irodori-tts-vdesign` | irodori_tts | vdes | Irodori-TTS 600M VoiceDesign（日语 caption） | +| `moss-tts-local` | moss_tts_local | tts | MOSS-TTS-Local v1.5（48kHz 立体声） | +| `moss-tts-nano` | moss_tts_nano | tts | MOSS-TTS-Nano 100M（轻量） | +| `supertonic` | supertonic | tts | Supertonic 3（预置音色，无中文） | + +未安装的 id 运行时会提示，可在 WebUI 里点“下载”，或 +`python tools\model_manager.py install --models-root \models`。 + +--- + +## 环境变量 + +| 变量 | 作用 | 适用 | +|---|---|---| +| `AUDIOCPP_BACKEND` | `gpu`(=cuda) / `cpu` 强制后端 | cli / server / webui | +| `AUDIOCPP_HOST` | server 绑定地址（`0.0.0.0` 开放局域网） | server | +| `AUDIOCPP_BUNDLE` | 手动指定整合包根目录 | 全部 | +| `AUDIOCPP_SERVER` | 让 WebUI 连一个已在跑的外部 server | webui | +| `AUDIOCPP_LOAD_TIMEOUT` | WebUI 等待模型加载的秒数（默认 300） | webui | + +--- + +## 常见问题 + +- **`.bat` 双击闪退 / 命令语法错误**：这些脚本必须是 **CRLF** 行尾（LF 会让 cmd 解析出错）， + 编辑后请保持 CRLF。 +- **端口被占用**：`run_server.bat` 和 WebUI 默认都用 8080。要同时用，就给 server 换端口， + 或设 `AUDIOCPP_SERVER` 让 WebUI 复用外部 server。 +- **`model path does not exist` / not installed**：模型没装。用上面的 model_manager 命令或 WebUI 下载。 +- **显存不足**：8GB 下同时跑两个 server 时，两个模型都要装得下；1.7B 建议单开。 +- **声音克隆生成过短（~0.4s 就结束）**：`voice_ref` 音色不干净或缺 `reference_text`；换单一说话人的 + 干净参考音频并配上对应文本。 + +--- + +## API 方式 vs 命令行的性能 + +同一套引擎、同一后端 → **推理本身完全一样**。差别主要在**模型加载的摊销**： + +- `run_cli_tts.bat` **每次调用都要把模型重新装进显存**（每次固定几秒开销）。 +- `run_server.bat` 的服务**只加载一次、常驻**，之后每个请求只花“推理 + 极小的传输”。 + 本机 HTTP + 几 MB 的 wav 传输 ≈ 毫秒级，相对多秒的推理可忽略（建议用默认二进制 wav， + 别用 `response_format:"json"` 的 base64，会大约 +33%）。 +- 网页界面（7860）比直连 8080 多一跳代理；其它程序直接打 8080 就没有这一跳。 + +**结论**：走 API 每次生成几乎没有额外成本，只有一次性的预热被服务端摊掉了——除了“只生成一次”的 +场景，API 方式通常比反复调 CLI **更快**。 diff --git a/webui/configs/model_params.json b/webui/configs/model_params.json new file mode 100644 index 00000000..80adfe1e --- /dev/null +++ b/webui/configs/model_params.json @@ -0,0 +1,149 @@ +{ + "_comment": "WebUI TTS 高级参数控件配置：按模型 family 动态生成控件（gr.render）。每项字段：name=选项键(随请求 options 透传给模型)；type=slider|number|bool|text|choice；label/info=显示文案；default=默认值(应等于模型默认，已按 src/models//*.cpp 校对)；minimum/maximum/step=数值范围；precision=0 表示整数；choices=下拉候选。规则：只有被用户改动过的控件值才会随请求发送；seed/max_tokens 已有专用输入框，勿在此重复；参考文本用『参考文本』框(reference_text)；文件路径/parity 类参数(如 *_noise_file)未纳入，可用『其它参数(JSON)』兜底框传。", + + "qwen3_tts": [ + {"name": "temperature", "type": "slider", "label": "temperature", "default": 0.9, "minimum": 0.0, "maximum": 2.0, "step": 0.05}, + {"name": "top_k", "type": "number", "label": "top_k", "default": 50, "minimum": 0, "step": 1, "precision": 0}, + {"name": "top_p", "type": "slider", "label": "top_p", "default": 1.0, "minimum": 0.0, "maximum": 1.0, "step": 0.01}, + {"name": "repetition_penalty", "type": "slider", "label": "repetition_penalty", "default": 1.05, "minimum": 1.0, "maximum": 2.0, "step": 0.01}, + {"name": "do_sample", "type": "bool", "label": "do_sample", "default": true}, + {"name": "instruct", "type": "text", "label": "instruct（仅 VoiceDesign/CustomVoice）", "default": "", "placeholder": "风格/音色指令，Base 版忽略"}, + {"name": "speaker", "type": "text", "label": "speaker（仅 CustomVoice）", "default": "", "placeholder": "内置音色名，其它版忽略"} + ], + + "vibevoice": [ + {"name": "num_inference_steps", "type": "number", "label": "num_inference_steps", "default": 10, "minimum": 1, "step": 1, "precision": 0, "info": "扩散步数（官方默认 10），越大越慢越稳"}, + {"name": "guidance_scale", "type": "slider", "label": "guidance_scale", "default": 1.3, "minimum": 0.0, "maximum": 5.0, "step": 0.1, "info": "CFG 引导强度"}, + {"name": "max_length_times", "type": "number", "label": "max_length_times", "default": 2.0, "minimum": 0.1, "step": 0.1, "info": "最大输出长度倍数"}, + {"name": "temperature", "type": "slider", "label": "temperature", "default": 1.0, "minimum": 0.05, "maximum": 2.0, "step": 0.05}, + {"name": "top_p", "type": "slider", "label": "top_p", "default": 1.0, "minimum": 0.05, "maximum": 1.0, "step": 0.01}, + {"name": "do_sample", "type": "bool", "label": "do_sample", "default": false}, + {"name": "voice_samples", "type": "text", "label": "voice_samples（多说话人，逗号分隔 wav，≤4）", "default": "", "placeholder": "D:/a.wav,D:/b.wav — 用此项时勿再上传参考音色"} + ], + + "voxcpm2": [ + {"name": "num_inference_steps", "type": "number", "label": "num_inference_steps", "default": 10, "minimum": 1, "step": 1, "precision": 0, "info": "CFM/DiT 步数"}, + {"name": "guidance_scale", "type": "slider", "label": "guidance_scale", "default": 2.0, "minimum": 0.0, "maximum": 5.0, "step": 0.1}, + {"name": "min_tokens", "type": "number", "label": "min_tokens", "default": 2, "minimum": 0, "step": 1, "precision": 0}, + {"name": "retry_badcase", "type": "bool", "label": "retry_badcase（自动重试异常输出）", "default": true} + ], + + "miotts": [ + {"name": "temperature", "type": "slider", "label": "temperature", "default": 0.8, "minimum": 0.0, "maximum": 2.0, "step": 0.05}, + {"name": "top_k", "type": "number", "label": "top_k", "default": 50, "minimum": 0, "step": 1, "precision": 0}, + {"name": "top_p", "type": "slider", "label": "top_p", "default": 1.0, "minimum": 0.0, "maximum": 1.0, "step": 0.01}, + {"name": "repetition_penalty", "type": "slider", "label": "repetition_penalty", "default": 1.0, "minimum": 1.0, "maximum": 1.5, "step": 0.01}, + {"name": "best_of_n", "type": "number", "label": "best_of_n（候选数，>1 自动开启）", "default": 1, "minimum": 1, "maximum": 8, "step": 1, "precision": 0} + ], + + "chatterbox": [ + {"name": "exaggeration", "type": "slider", "label": "exaggeration", "default": 0.5, "minimum": 0.0, "maximum": 2.0, "step": 0.05, "info": "改动后需重新『加载模型』才生效"}, + {"name": "guidance_scale", "type": "slider", "label": "guidance_scale", "default": 0.5, "minimum": 0.0, "maximum": 1.0, "step": 0.05, "info": "改动后需重新『加载模型』才生效"}, + {"name": "temperature", "type": "slider", "label": "temperature", "default": 0.8, "minimum": 0.0, "maximum": 2.0, "step": 0.05}, + {"name": "repetition_penalty", "type": "slider", "label": "repetition_penalty", "default": 1.2, "minimum": 1.0, "maximum": 2.0, "step": 0.01} + ], + + "chatterbox-vc": [ + {"name": "s3gen_cfg_rate", "type": "slider", "label": "s3gen_cfg_rate（音色引导强度）", "label_en": "s3gen_cfg_rate (voice guidance)", "default": 0.7, "minimum": 0.0, "maximum": 2.0, "step": 0.05}, + {"name": "num_inference_steps", "type": "number", "label": "num_inference_steps（生成步数）", "label_en": "num_inference_steps", "default": 10, "minimum": 1, "maximum": 100, "step": 1, "precision": 0} + ], + + "omnivoice": [ + {"name": "num_inference_steps", "type": "number", "label": "num_inference_steps", "default": 32, "minimum": 1, "step": 1, "precision": 0}, + {"name": "guidance_scale", "type": "slider", "label": "guidance_scale", "default": 2.0, "minimum": 0.0, "maximum": 5.0, "step": 0.1}, + {"name": "speed", "type": "slider", "label": "speed", "default": 1.0, "minimum": 0.5, "maximum": 2.0, "step": 0.05}, + {"name": "instruct", "type": "text", "label": "instruct（风格/音色指令）", "default": "", "placeholder": "如：以轻快的语气朗读"} + ], + + "pocket_tts": [ + {"name": "frames_after_eos", "type": "number", "label": "frames_after_eos（-1=自动）", "default": -1, "minimum": -1, "step": 1, "precision": 0} + ], + + "ace_step": [ + {"name": "task_route", "type": "choice", "label": "task_route（操作类型）", "default": "text2music", "choices": ["text2music", "complete", "lego", "extract", "cover", "cover-nofsq", "repaint", "remix"], "info": "cover/remix=换词翻唱，非 text2music 需上传源音频；详见 webui/README.md"}, + {"name": "num_inference_steps", "type": "number", "label": "num_inference_steps", "default": 8, "minimum": 1, "maximum": 20, "step": 1, "precision": 0, "info": "扩散步数（turbo 上限 20）；remix 路由不填时默认 16，其他路由默认 8"}, + {"name": "shift", "type": "slider", "label": "shift（时间步弯曲）", "default": 3.0, "minimum": 1.0, "maximum": 5.0, "step": 0.5, "info": "原版 turbo 默认 3.0；1.0 会明显劣化 remix 换词咬字"}, + {"name": "guidance_scale", "type": "slider", "label": "guidance_scale", "default": 1.0, "minimum": 0.0, "maximum": 5.0, "step": 0.1}, + {"name": "audio_cover_strength", "type": "slider", "label": "【cover】audio_cover_strength", "default": 1.0, "minimum": 0.0, "maximum": 1.0, "step": 0.05, "info": "1=贴近原曲，0=自由发挥；建议 0.5"}, + {"name": "cover_noise_strength", "type": "slider", "label": "【cover】cover_noise_strength", "default": 0.0, "minimum": 0.0, "maximum": 1.0, "step": 0.05, "info": "保旋律强度；推荐 0.1~0.25"}, + {"name": "source_caption", "type": "text", "label": "【remix】source_caption", "default": "", "placeholder": "源歌曲描述；『🔍 分析』自动填"}, + {"name": "source_lyrics", "type": "text", "lines": 4, "label": "【remix】source_lyrics", "default": "", "placeholder": "源歌曲原歌词；『🔍 分析』自动填"}, + {"name": "flow_edit_n_min", "type": "slider", "label": "【remix】flow_edit_n_min", "default": 0.0, "minimum": 0.0, "maximum": 1.0, "step": 0.05, "info": "调大更保源曲、换词更弱"}, + {"name": "flow_edit_n_max", "type": "slider", "label": "【remix】flow_edit_n_max", "default": 1.0, "minimum": 0.0, "maximum": 1.0, "step": 0.05, "info": "唱不出新歌词时降到 0.7~0.9"}, + {"name": "flow_edit_n_avg", "type": "number", "label": "【remix】flow_edit_n_avg", "default": 2, "minimum": 1, "maximum": 4, "step": 1, "precision": 0, "info": "每步多次采样取平均（remix 默认 2）；1=最快"}, + {"name": "bpm", "type": "number", "label": "【曲谱】BPM", "default": 0, "minimum": 0, "step": 1, "precision": 0, "info": "0=不指定"}, + {"name": "keyscale", "type": "text", "label": "【曲谱】keyscale", "default": "", "placeholder": "如 F major"}, + {"name": "timesignature", "type": "text", "label": "【曲谱】timesignature", "default": "", "placeholder": "如 4"} + ], + + "stable_audio": [ + {"name": "num_inference_steps", "type": "number", "label": "num_inference_steps", "default": 8, "minimum": 1, "step": 1, "precision": 0, "info": "RF 扩散步数"}, + {"name": "guidance_scale", "type": "slider", "label": "guidance_scale", "default": 1.0, "minimum": 0.0, "maximum": 5.0, "step": 0.1}, + {"name": "audio_input_kind", "type": "choice", "label": "audio_input_kind（仅上传源音频时生效）", "default": "init_audio", "choices": ["init_audio", "inpaint_audio"]}, + {"name": "init_noise_level", "type": "slider", "label": "init_noise_level（init_audio 强度）", "default": 1.0, "minimum": 0.0, "maximum": 1.0, "step": 0.05} + ], + + "seed_vc": [ + {"name": "route", "type": "choice", "label": "route（转换路径）", "default": "", "choices": ["", "v2_vc", "v1_whisper_bigvgan_vc", "v1_xlsr_hift_vc", "v1_svc"], "info": "留空=按任务默认"}, + {"name": "num_inference_steps", "type": "number", "label": "num_inference_steps", "default": 30, "minimum": 1, "step": 1, "precision": 0, "info": "CFM 扩散步数"}, + {"name": "length_adjust", "type": "slider", "label": "length_adjust（时长伸缩）", "default": 1.0, "minimum": 0.5, "maximum": 2.0, "step": 0.05}, + {"name": "intelligibility_cfg_rate", "type": "slider", "label": "intelligibility_cfg_rate（仅 v2_vc）", "default": 0.7, "minimum": 0.0, "maximum": 1.0, "step": 0.05}, + {"name": "similarity_cfg_rate", "type": "slider", "label": "similarity_cfg_rate（仅 v2_vc）", "default": 0.7, "minimum": 0.0, "maximum": 1.0, "step": 0.05}, + {"name": "inference_cfg_rate", "type": "slider", "label": "inference_cfg_rate（仅 v1 路径）", "default": 0.7, "minimum": 0.0, "maximum": 1.0, "step": 0.05} + ], + + "vevo2": [ + {"name": "route", "type": "choice", "label": "route（任务路线）", "default": "", "choices": ["", "style_preserved_vc", "style_converted_vc", "style_preserved_svc", "style_converted_svc", "singing_style_conversion", "editing"], "info": "留空=按任务默认；详见 webui/README.md"}, + {"name": "num_inference_steps", "type": "number", "label": "num_inference_steps", "default": 32, "minimum": 1, "step": 1, "precision": 0, "info": "流匹配步数"}, + {"name": "use_pitch_shift", "type": "choice", "label": "use_pitch_shift（自动音高对齐）", "default": "", "choices": ["", "true", "false"], "info": "留空=按路线默认"}, + {"name": "temperature", "type": "slider", "label": "temperature（AR 路线用）", "default": 0.7, "minimum": 0.0, "maximum": 2.0, "step": 0.05, "info": "默认取自模型 generation_config.json"}, + {"name": "top_k", "type": "number", "label": "top_k（AR 路线用）", "default": 20, "minimum": 0, "step": 1, "precision": 0, "info": "默认取自模型 generation_config.json"}, + {"name": "top_p", "type": "slider", "label": "top_p（AR 路线用）", "default": 0.8, "minimum": 0.0, "maximum": 1.0, "step": 0.01} + ], + + "heartmula": [ + {"name": "tags", "type": "text", "label": "tags（必填，逗号分隔）", "default": "", "placeholder": "pop,bright,drums,female vocals", "info": "风格/情绪/乐器/人声标签，模型必需"}, + {"name": "temperature", "type": "slider", "label": "temperature", "default": 1.0, "minimum": 0.0, "maximum": 2.0, "step": 0.05}, + {"name": "top_k", "type": "number", "label": "top_k", "default": 50, "minimum": 0, "step": 1, "precision": 0}, + {"name": "guidance_scale", "type": "slider", "label": "guidance_scale（MuLa CFG）", "default": 1.5, "minimum": 0.0, "maximum": 5.0, "step": 0.1}, + {"name": "num_inference_steps", "type": "number", "label": "num_inference_steps（codec 步数）", "default": 10, "minimum": 1, "step": 1, "precision": 0}, + {"name": "infinite_mode", "type": "bool", "label": "infinite_mode（长输出分段生成）", "default": false}, + {"name": "codec_guidance_scale", "type": "slider", "label": "codec_guidance_scale", "default": 1.25, "minimum": 0.0, "maximum": 5.0, "step": 0.05} + ], + + "index_tts2": [ + {"name": "emotion_text", "type": "text", "label": "emotion_text（情绪参考文本）", "label_en": "emotion_text (emotion reference text)", "default": "", "placeholder": "例：你吓死我了！你是鬼吗？", "placeholder_en": "e.g. You scared me to death!", "info": "填写后自动开启情感条件（use_emotion_text）", "info_en": "Setting this enables emotion conditioning."}, + {"name": "emotion_alpha", "type": "slider", "label": "emotion_alpha（情感强度）", "label_en": "emotion_alpha", "default": 1.0, "minimum": 0.0, "maximum": 1.0, "step": 0.05}, + {"name": "use_emotion_text", "type": "bool", "label": "use_emotion_text（从朗读文本推断情感）", "label_en": "use_emotion_text (infer from text)", "default": false}, + {"name": "use_random_emotion", "type": "bool", "label": "use_random_emotion（随机情感）", "label_en": "use_random_emotion", "default": false}, + {"name": "interval_silence_ms", "type": "number", "label": "interval_silence_ms（分段间静音）", "label_en": "interval_silence_ms", "default": 200, "minimum": 0, "step": 50, "precision": 0} + ], + + "irodori_tts": [ + {"name": "num_inference_steps", "type": "number", "label": "num_inference_steps（RF 扩散步数）", "label_en": "num_inference_steps", "default": 40, "minimum": 1, "step": 1, "precision": 0}, + {"name": "duration_seconds", "type": "number", "label": "duration_seconds（0=模型自动预测时长）", "label_en": "duration_seconds (0 = auto)", "default": 0, "minimum": 0, "step": 0.5}, + {"name": "duration_scale", "type": "slider", "label": "duration_scale（语速倒数，越大越慢）", "label_en": "duration_scale", "default": 1.0, "minimum": 0.5, "maximum": 2.0, "step": 0.05} + ], + + "moss_tts_local": [ + {"name": "do_sample", "type": "bool", "label": "do_sample", "default": true}, + {"name": "temperature", "type": "slider", "label": "temperature", "default": 1.7, "minimum": 0.0, "maximum": 2.5, "step": 0.05}, + {"name": "top_p", "type": "slider", "label": "top_p", "default": 0.8, "minimum": 0.0, "maximum": 1.0, "step": 0.01}, + {"name": "top_k", "type": "number", "label": "top_k", "default": 25, "minimum": 0, "step": 1, "precision": 0}, + {"name": "repetition_penalty", "type": "slider", "label": "repetition_penalty", "default": 1.0, "minimum": 1.0, "maximum": 2.0, "step": 0.01} + ], + + "moss_tts_nano": [ + {"name": "do_sample", "type": "bool", "label": "do_sample", "default": true}, + {"name": "temperature", "type": "slider", "label": "temperature", "default": 1.7, "minimum": 0.0, "maximum": 2.5, "step": 0.05}, + {"name": "top_p", "type": "slider", "label": "top_p", "default": 0.8, "minimum": 0.0, "maximum": 1.0, "step": 0.01}, + {"name": "top_k", "type": "number", "label": "top_k", "default": 25, "minimum": 0, "step": 1, "precision": 0}, + {"name": "repetition_penalty", "type": "slider", "label": "repetition_penalty", "default": 1.0, "minimum": 1.0, "maximum": 2.0, "step": 0.01} + ], + + "supertonic": [ + {"name": "voice", "type": "choice", "label": "voice（预置音色：M 男声 / F 女声）", "label_en": "voice (M = male, F = female presets)", "default": "M1", "choices": ["M1", "M2", "M3", "M4", "M5", "F1", "F2", "F3", "F4", "F5"]}, + {"name": "speaking_rate", "type": "slider", "label": "speaking_rate（语速倍率）", "label_en": "speaking_rate", "default": 1.05, "minimum": 0.5, "maximum": 2.0, "step": 0.05}, + {"name": "num_inference_steps", "type": "number", "label": "num_inference_steps（流匹配步数）", "label_en": "num_inference_steps", "default": 8, "minimum": 1, "step": 1, "precision": 0} + ] +} diff --git a/webui/configs/models_catalog.json b/webui/configs/models_catalog.json new file mode 100644 index 00000000..643b7c4c --- /dev/null +++ b/webui/configs/models_catalog.json @@ -0,0 +1,71 @@ +{ + "host": "127.0.0.1", + "port": 8088, + "device": 0, + "threads": 1, + "_comment": "Whitelist of every model family currently enabled in registry.cpp. The WebUI loads ONE model at a time by (re)starting audiocpp_server with a single-model config; paths are relative to the bundle root (where cpu/ gpu/ models/ live). Entries whose directory is missing show as '未安装'; click 下载 in the UI to fetch them in the background (runs `python tools/model_manager.py install --models-root /models`). 'task' must be one of: tts, asr, vad, diar, sep, gen, clon, vc, s2s, align, vdes, spk, svc. UI tab mapping: tts/clon -> TTS 标签页, asr -> ASR, gen -> 音乐生成, vc/svc/s2s -> 声音转换, sep -> 音源分离, vad/diar/align -> 音频分析, vdes -> 声音设计. 'download_id' is the model_manager package id (omit for bundled assets like silero_vad). Optional per-entry keys: input_hint (overrides the family hint in webui.py), default_options, min_vram_gb. 'min_vram_gb' = ESTIMATED minimum CUDA VRAM (GB) to run typical requests at this entry's default precision/session_options — sources: docs/memory_saver.md official peak measurements (heartmula 25.6G@120s, stable-audio-medium 10.4G, chatterbox 13.4G, omnivoice 11.4G, qwen3-tts-1.7B 7.5G, stable-audio-small 3.7G), local RTX 4060 8G measurements (vibevoice 6.9G peak, ace-step w/ q8_0 preset), weight-size extrapolation for the rest. The UI warns when it exceeds detected local VRAM; exceeding means it may still run but will spill into shared memory and slow down badly.", + + "models": [ + { "id": "omnivoice", "display_name": "OmniVoice (tts)", "family": "omnivoice", "path": "models/OmniVoice", "task": "tts", "mode": "offline", "download_id": "omnivoice", "min_vram_gb": 10 }, + { "id": "pocket-tts", "display_name": "Pocket TTS (tts)", "family": "pocket_tts", "path": "models/pocket-tts", "task": "tts", "mode": "offline", "download_id": "pocket_tts", "min_vram_gb": 2 }, + { "id": "qwen3-tts", "display_name": "Qwen3-TTS 0.6B (tts)", "family": "qwen3_tts", "path": "models/Qwen3-TTS-12Hz-0.6B-Base", "task": "tts", "mode": "offline", "download_id": "qwen3_tts_0_6b_base", "min_vram_gb": 5 }, + { "id": "qwen3-tts-1.7b", "display_name": "Qwen3-TTS 1.7B Base (tts)", "family": "qwen3_tts", "path": "models/Qwen3-TTS-12Hz-1.7B-Base", "task": "tts", "mode": "offline", "download_id": "qwen3_tts_1_7b_base", "min_vram_gb": 8 }, + { "id": "qwen3-tts-1.7b-custom", "display_name": "Qwen3-TTS 1.7B CustomVoice (tts)", "family": "qwen3_tts", "path": "models/Qwen3-TTS-12Hz-1.7B-CustomVoice", "task": "tts", "mode": "offline", "download_id": "qwen3_tts_1_7b_custom_voice", "min_vram_gb": 8 }, + { "id": "miotts", "display_name": "MioTTS 1.7B (tts; needs MioCodec)", "family": "miotts", "path": "models/MioTTS-1.7B", "task": "tts", "mode": "offline", "download_id": "miotts_1_7b", "min_vram_gb": 8 }, + { "id": "voxcpm2", "display_name": "VoxCPM2 (tts)", "family": "voxcpm2", "path": "models/VoxCPM2", "task": "tts", "mode": "offline", "download_id": "voxcpm2", "session_options": { "voxcpm2.weight_type": "q8_0" }, "min_vram_gb": 6 }, + { "id": "vibevoice", "display_name": "VibeVoice 1.5B (tts, long-form/multi-speaker)", "family": "vibevoice", "path": "models/VibeVoice-1.5B", "task": "tts", "mode": "offline", "download_id": "vibevoice_1_5b", "min_vram_gb": 7 }, + { "id": "index-tts2", "display_name": "IndexTTS2 (tts 中英克隆+情感)", "display_name_en": "IndexTTS2 (tts, zh/en clone + emotion)", "family": "index_tts2", "path": "models/IndexTTS-2", "task": "tts", "mode": "offline", "download_id": "index_tts2", "min_vram_gb": 8 }, + { "id": "irodori-tts", "display_name": "Irodori-TTS 500M (tts 日语)", "display_name_en": "Irodori-TTS 500M (ja tts)", "family": "irodori_tts", "path": "models/Irodori-TTS-500M-v3", "task": "tts", "mode": "offline", "download_id": "irodori_tts_500m_v3", "min_vram_gb": 4 }, + { "id": "moss-tts-local", "display_name": "MOSS-TTS-Local v1.5 (tts)", "family": "moss_tts_local", "path": "models/MOSS-TTS-Local-Transformer-v1.5", "task": "tts", "mode": "offline", "download_id": "moss_tts_local_v1_5", "min_vram_gb": 8 }, + { "id": "moss-tts-nano", "display_name": "MOSS-TTS-Nano 100M (tts)", "family": "moss_tts_nano", "path": "models/MOSS-TTS-Nano-100M", "task": "tts", "mode": "offline", "download_id": "moss_tts_nano_100m", "min_vram_gb": 2 }, + { "id": "supertonic", "display_name": "Supertonic 3 (tts 预置音色/多语种)", "display_name_en": "Supertonic 3 (tts, preset voices)", "family": "supertonic", "path": "models/supertonic-3", "task": "tts", "mode": "offline", "download_id": "supertonic_3", "min_vram_gb": 2 }, + + { "id": "chatterbox", "display_name": "Chatterbox (voice clone)", "family": "chatterbox", "path": "models/chatterbox", "task": "clon", "mode": "offline", "download_id": "chatterbox", "min_vram_gb": 12 }, + + { "id": "ace-step", "display_name": "ACE-Step 1.5 (music gen)", "family": "ace_step", "path": "models/Ace-Step1.5", "task": "gen", "mode": "offline", "download_id": "ace_step", "session_options": { "ace_step.mem_saver": "true", "ace_step.dit_weight_type": "q8_0", "ace_step.text_encoder_weight_type": "q8_0", "ace_step.planner_weight_type": "q8_0" }, "min_vram_gb": 8 }, + { "id": "stable-audio-small-music","display_name": "Stable Audio 3 Small Music (gen)", "family": "stable_audio", "path": "models/stable-audio-3-small-music", "task": "gen", "mode": "offline", "download_id": "stable_audio_3_small_music", "min_vram_gb": 4 }, + { "id": "stable-audio-small-sfx", "display_name": "Stable Audio 3 Small SFX (gen)", "family": "stable_audio", "path": "models/stable-audio-3-small-sfx", "task": "gen", "mode": "offline", "download_id": "stable_audio_3_small_sfx", "min_vram_gb": 4 }, + { "id": "stable-audio-medium", "display_name": "Stable Audio 3 Medium (gen)", "family": "stable_audio", "path": "models/stable-audio-3-medium", "task": "gen", "mode": "offline", "download_id": "stable_audio_3_medium", "session_options": { "stable_audio.mem_saver": "true" }, "min_vram_gb": 10 }, + { "id": "heartmula", "display_name": "HeartMuLa 3B (music gen)", "family": "heartmula", "path": "models/HeartMuLa", "task": "gen", "mode": "offline", "download_id": "heartmula", "session_options": { "heartmula.mem_saver": "true" }, "min_vram_gb": 24 }, + + { "id": "qwen3-asr", "display_name": "Qwen3-ASR 0.6B (asr)", "family": "qwen3_asr", "path": "models/Qwen3-ASR-0.6B", "task": "asr", "mode": "offline", "download_id": "qwen3_asr_0_6b", "min_vram_gb": 3 }, + { "id": "qwen3-asr-1.7b", "display_name": "Qwen3-ASR 1.7B HF (asr)", "family": "qwen3_asr", "path": "models/Qwen3-ASR-1.7B-hf", "task": "asr", "mode": "offline", "download_id": "qwen3_asr_1_7b_hf", "min_vram_gb": 6, + "input_hint": "**Qwen3-ASR 1.7B**（HF 原生权重，免转换）：精度高于 0.6B；长音频自动分段转写；8G 卡显存偏紧，长音频建议先短段试跑。" }, + { "id": "citrinet-asr", "display_name": "Citrinet ASR (asr)", "family": "citrinet_asr", "path": "models/citrinet", "task": "asr", "mode": "offline", "download_id": "citrinet_asr", "min_vram_gb": 2 }, + { "id": "nemotron-asr", "display_name": "Nemotron 3.5 ASR 0.6B (asr, 100+语种)", "family": "nemotron_asr", "path": "models/nemotron-3.5-asr-streaming-0.6b", "task": "asr", "mode": "offline", "download_id": "nemotron_asr", "min_vram_gb": 4, + "input_hint": "**Nemotron ASR**：100+ 语种，语种码为 BCP-47（如 en-US / zh-CN），留空=auto；模型自带长音频处理。" }, + { "id": "higgs-audio-stt", "display_name": "Higgs Audio v3 STT (asr, 英语)", "family": "higgs_audio_stt", "path": "models/higgs-audio-v3-stt", "task": "asr", "mode": "offline", "download_id": "higgs_audio_stt", "min_vram_gb": 8, + "input_hint": "**Higgs Audio STT**：英语转写；可在文本框填指令（默认相当于 Transcribe the speech.）；离线模式自动切分长音频。" }, + { "id": "hviske-asr", "display_name": "Hviske v5.3 (asr, 丹麦语)", "family": "hviske_asr", "path": "models/hviske-v5.3", "task": "asr", "mode": "offline", "download_id": "hviske_asr", "min_vram_gb": 6, + "input_hint": "**Hviske ASR**：丹麦语专用；模型侧自动分段。" }, + { "id": "vibevoice-asr", "display_name": "VibeVoice ASR (asr, 多语种+说话人分段)", "family": "vibevoice_asr", "path": "models/VibeVoice-ASR", "task": "asr", "mode": "offline", "download_id": "vibevoice_asr", "min_vram_gb": 20, + "input_hint": "**VibeVoice ASR**：自动语种，可输出分段/说话人轮次；文本框可填上下文提示（如 The recording is a meeting conversation.）。权重 17.3G，8G 卡跑不动。" }, + { "id": "voxtral-realtime", "display_name": "Voxtral Mini 4B Realtime (asr, 自动语种+流式)", "display_name_en": "Voxtral Mini 4B Realtime (asr, auto + streaming)", "family": "voxtral_realtime", "path": "models/Voxtral-Mini-4B-Realtime-2602", "task": "asr", "mode": "offline", "download_id": "voxtral_realtime", "session_options": { "voxtral_realtime.weight_type": "q8_0" }, "min_vram_gb": 8 }, + + { "id": "chatterbox-vc", "display_name": "Chatterbox (vc 声音转换)", "display_name_en": "Chatterbox (voice conversion)", "family": "chatterbox", "path": "models/chatterbox", "task": "vc", "mode": "offline", "download_id": "chatterbox", "min_vram_gb": 12, + "input_hint": "**Chatterbox VC**：上传源语音和目标音色参考；模型保留源语音内容，将说话人音色转换为目标音色，输出 24kHz 单声道。", + "input_hint_en": "**Chatterbox VC**: upload source speech and a target-voice reference. It preserves the source content and converts the speaker identity; output is 24 kHz mono." }, + { "id": "vevo2", "display_name": "Vevo2 (vc 语音转换)", "family": "vevo2", "path": "models/Vevo2", "task": "vc", "mode": "offline", "download_id": "vevo2", "min_vram_gb": 6 }, + { "id": "vevo2-svc", "display_name": "Vevo2 (svc 歌声转换)", "family": "vevo2", "path": "models/Vevo2", "task": "svc", "mode": "offline", "download_id": "vevo2", "min_vram_gb": 6, + "input_hint": "**Vevo2 歌声转换 (svc)**：上传源歌声 + 目标歌手参考音色，默认 route=style_preserved_svc。style_converted_svc / singing_style_conversion 等风格转换 route 需在『其它参数(JSON)』里补 `style_ref`（服务器本地 wav 路径）/ `style_ref_text` / `target_text`。" }, + { "id": "vevo2-s2s", "display_name": "Vevo2 (s2s 语音编辑)", "family": "vevo2", "path": "models/Vevo2", "task": "s2s", "mode": "offline", "download_id": "vevo2", "min_vram_gb": 6, + "input_hint": "**Vevo2 语音编辑 (s2s)**：上传要编辑的源语音，并在『其它参数(JSON)』里填 `{\"target_text\": \"替换后的完整句子\"}`（编辑保持原说话人音色，可不上传目标音色）。" }, + { "id": "seed-vc", "display_name": "Seed-VC (vc 语音转换)", "family": "seed_vc", "path": "models/SeedVC-MLX", "task": "vc", "mode": "offline", "download_id": "seed_vc", "min_vram_gb": 4 }, + { "id": "seed-vc-svc", "display_name": "Seed-VC (svc 歌声转换)", "family": "seed_vc", "path": "models/SeedVC-MLX", "task": "svc", "mode": "offline", "download_id": "seed_vc", "min_vram_gb": 4, + "input_hint": "**Seed-VC 歌声转换 (svc)**：上传源歌声 + 目标歌手参考音色，默认 route=v1_svc（带 F0 条件）。可在『其它参数(JSON)』里调 `auto_f0_adjust` / `semi_tone_shift` / `f0_condition`。" }, + { "id": "miocodec", "display_name": "MioCodec (vc; codec dependency)", "family": "miocodec", "path": "models/MioCodec-25Hz-44.1kHz-v2", "task": "vc", "mode": "offline", "download_id": "miocodec_25hz_44k_v2", "min_vram_gb": 3 }, + + { "id": "htdemucs", "display_name": "HTDemucs (sep 音源分离)", "family": "htdemucs", "path": "models/htdemucs", "task": "sep", "mode": "offline", "download_id": "htdemucs", "min_vram_gb": 3 }, + { "id": "mel-band-roformer", "display_name": "Mel-Band RoFormer (sep 人声分离)", "family": "mel_band_roformer", "path": "models/mel-roformer-mlx", "task": "sep", "mode": "offline", "download_id": "mel_band_roformer", "min_vram_gb": 3 }, + + { "id": "silero-vad", "display_name": "Silero VAD (vad, bundled)", "family": "silero_vad", "path": "assets/framework/models/silero_vad", "task": "vad", "mode": "offline", "min_vram_gb": 1 }, + { "id": "marblenet-vad", "display_name": "MarbleNet VAD (vad)", "family": "marblenet_vad", "path": "models/marblenet_vad", "task": "vad", "mode": "offline", "download_id": "marblenet_vad", "min_vram_gb": 1 }, + { "id": "sortformer-diar", "display_name": "Sortformer Diarization 4spk (diar)", "family": "sortformer_diar", "path": "models/diar_sortformer_4spk-v1", "task": "diar", "mode": "offline", "download_id": "sortformer_diar_4spk_v1", "min_vram_gb": 2 }, + { "id": "qwen3-forced-aligner", "display_name": "Qwen3 Forced Aligner (align)", "family": "qwen3_forced_aligner", "path": "models/Qwen3-ForcedAligner-0.6B", "task": "align", "mode": "offline", "download_id": "qwen3_forced_aligner_0_6b", "min_vram_gb": 3 }, + + { "id": "qwen3-tts-1.7b-vdesign", "display_name": "Qwen3-TTS 1.7B VoiceDesign (vdes)", "family": "qwen3_tts", "path": "models/Qwen3-TTS-12Hz-1.7B-VoiceDesign", "task": "vdes", "mode": "offline", "download_id": "qwen3_tts_1_7b_voice_design", "min_vram_gb": 8, + "input_hint": "**Qwen3-TTS VoiceDesign**：在『音色描述』里用文字描述想要的声音（如“低沉磁性的中年男声，语速偏慢”），配上要念的文本即可，无需参考音频。" }, + { "id": "irodori-tts-vdesign", "display_name": "Irodori-TTS 600M VoiceDesign (vdes 日语)", "display_name_en": "Irodori-TTS 600M VoiceDesign (ja vdes)", "family": "irodori_tts", "path": "models/Irodori-TTS-600M-v3-VoiceDesign", "task": "vdes", "mode": "offline", "download_id": "irodori_tts_600m_v3_voice_design", "min_vram_gb": 4, + "input_hint": "**Irodori-TTS VoiceDesign**（日语）：『音色描述』用日语 caption 描述音色（如「落ち着いた大人の男性。深く響く声。」），文本填要念的日语内容，无需参考音频。" } + ] +} diff --git a/webui/configs/req_speech.json b/webui/configs/req_speech.json new file mode 100644 index 00000000..d575a3cd --- /dev/null +++ b/webui/configs/req_speech.json @@ -0,0 +1,8 @@ +{ + "model": "qwen3-tts", + "input": "This audio is generated by the audio dot cpp server over HTTP.", + "voice_ref": "voice/demo_01_man.wav", + "reference_text": "okay, I'm Cemo and what you just heard wasn't a human voice.", + "max_tokens": 1200, + "seed": 1234 +} diff --git a/webui/configs/required_files.json b/webui/configs/required_files.json new file mode 100644 index 00000000..1370be25 --- /dev/null +++ b/webui/configs/required_files.json @@ -0,0 +1,434 @@ +{ + "_comment": "download_id -> files that must exist inside the installed model directory (relative paths, forward slashes). Mirrors ModelPackage.required_files in tools/model_manager.py - the post-install validation list, i.e. the final on-disk layout AFTER conversions (e.g. ace_step silence_latent.pt -> .safetensors). webui.py uses this to tell a complete install apart from a manually copied / interrupted directory. Regenerate after editing the model_manager CATALOG: venv\\Scripts\\python.exe with this repo's tools/ on sys.path, dump {p.id: list(p.required_files) for p in CATALOG}.", + "ace_step": [ + "Qwen3-Embedding-0.6B/model.safetensors", + "acestep-5Hz-lm-1.7B/model.safetensors", + "acestep-v15-base/model.safetensors", + "acestep-v15-base/silence_latent.safetensors", + "acestep-v15-turbo/model.safetensors", + "acestep-v15-turbo/silence_latent.safetensors", + "vae/diffusion_pytorch_model.safetensors" + ], + "kokoro_82m_bf16": [ + "config.json", + "kokoro-v1_0.safetensors", + "voices/af_heart.safetensors" + ], + "moss_tts_nano_100m": [ + "config.json", + "model.safetensors", + "tokenizer.model", + "tokenizer_config.json", + "audio_tokenizer/config.json", + "audio_tokenizer/model-00001-of-00001.safetensors", + "audio_tokenizer/model.safetensors.index.json" + ], + "moss_tts_nano_100m_model": [ + "config.json", + "pytorch_model.bin", + "tokenizer.model", + "tokenizer_config.json" + ], + "moss_audio_tokenizer_nano": [ + "config.json", + "model-00001-of-00001.safetensors", + "model.safetensors.index.json" + ], + "moss_audio_tokenizer_v2": [ + "config.json", + "model.safetensors.index.json", + "model-00001-of-00003.safetensors", + "model-00002-of-00003.safetensors", + "model-00003-of-00003.safetensors" + ], + "moss_tts_local_v1_5": [ + "config.json", + "model.safetensors", + "tokenizer.json", + "tokenizer_config.json", + "vocab.json", + "merges.txt", + "special_tokens_map.json", + "added_tokens.json", + "chat_template.jinja", + "audio_tokenizer/config.json", + "audio_tokenizer/model.safetensors.index.json", + "audio_tokenizer/model-00001-of-00003.safetensors", + "audio_tokenizer/model-00002-of-00003.safetensors", + "audio_tokenizer/model-00003-of-00003.safetensors" + ], + "omnivoice": [ + "config.json", + "model.safetensors", + "tokenizer.json", + "audio_tokenizer/config.json", + "audio_tokenizer/model.safetensors" + ], + "qwen3_asr_0_6b": [ + "config.json", + "generation_config.json", + "model.safetensors", + "preprocessor_config.json", + "tokenizer_config.json", + "vocab.json", + "merges.txt" + ], + "qwen3_asr_1_7b_hf": [ + "config.json", + "generation_config.json", + "model.safetensors", + "processor_config.json", + "tokenizer_config.json", + "tokenizer.json" + ], + "voxtral_realtime": [ + "config.json", + "generation_config.json", + "model.safetensors", + "params.json", + "processor_config.json", + "tekken.json" + ], + "higgs_audio_stt": [ + "config.json", + "generation_config.json", + "model.safetensors.index.json", + "model-00001-of-00002.safetensors", + "model-00002-of-00002.safetensors", + "tokenizer_config.json", + "vocab.json", + "merges.txt", + "../whisper-large-v3/preprocessor_config.json" + ], + "hviske_asr": [ + "config.json", + "generation_config.json", + "model.safetensors", + "tokenizer.model" + ], + "nemotron_asr": [ + "config.json", + "model.safetensors", + "processor_config.json", + "tokenizer.json" + ], + "qwen3_forced_aligner_0_6b": [ + "config.json", + "generation_config.json", + "model.safetensors", + "preprocessor_config.json", + "tokenizer_config.json", + "vocab.json", + "merges.txt" + ], + "qwen3_tts_0_6b_base": [ + "config.json", + "generation_config.json", + "model.safetensors", + "speech_tokenizer/config.json", + "speech_tokenizer/model.safetensors", + "tokenizer_config.json", + "vocab.json", + "merges.txt" + ], + "qwen3_tts_1_7b_base": [ + "config.json", + "generation_config.json", + "model.safetensors", + "speech_tokenizer/config.json", + "speech_tokenizer/model.safetensors", + "tokenizer_config.json", + "vocab.json", + "merges.txt" + ], + "qwen3_tts_1_7b_custom_voice": [ + "config.json", + "generation_config.json", + "model.safetensors", + "speech_tokenizer/config.json", + "speech_tokenizer/model.safetensors", + "tokenizer_config.json", + "vocab.json", + "merges.txt" + ], + "qwen3_tts_1_7b_voice_design": [ + "config.json", + "generation_config.json", + "model.safetensors", + "speech_tokenizer/config.json", + "speech_tokenizer/model.safetensors", + "tokenizer_config.json", + "vocab.json", + "merges.txt" + ], + "qwen3_tts_tokenizer_12hz": [ + "config.json", + "model.safetensors" + ], + "chatterbox": [ + "ve.safetensors", + "t3_cfg.safetensors", + "t3_mtl23ls_v2.safetensors", + "t3_mtl23ls_v3.safetensors", + "s3gen.safetensors", + "tokenizer.json", + "grapheme_mtl_merged_expanded_v1.json", + "Cangjie5_TC.json", + "conds.pt" + ], + "sortformer_diar_4spk_v1": [ + "config.json", + "model.safetensors", + "processor_config.json" + ], + "parakeet_tdt_0_6b_v3": [ + "config.json", + "model.safetensors", + "processor_config.json", + "tokenizer.json" + ], + "pocket_tts": [ + "languages/english/model.safetensors", + "languages/english/tokenizer.model", + "languages/english/embeddings/alba.safetensors" + ], + "miocodec_25hz_44k_v2": [ + "config.yaml", + "model.safetensors", + "wavlm-base-plus-mlx/config.json", + "wavlm-base-plus-mlx/weights.safetensors" + ], + "miotts_1_7b": [ + "config.json", + "generation_config.json", + "tokenizer_config.json", + "tokenizer.json", + "vocab.json", + "merges.txt", + "model.safetensors" + ], + "vibevoice_asr": [ + "config.json", + "model.safetensors.index.json", + "model-00001-of-00008.safetensors", + "model-00002-of-00008.safetensors", + "model-00003-of-00008.safetensors", + "model-00004-of-00008.safetensors", + "model-00005-of-00008.safetensors", + "model-00006-of-00008.safetensors", + "model-00007-of-00008.safetensors", + "model-00008-of-00008.safetensors", + "tokenizer.json", + "tokenizer_config.json", + "vocab.json", + "merges.txt" + ], + "vibevoice_1_5b": [ + "config.json", + "model.safetensors.index.json", + "model-00001-of-00003.safetensors", + "model-00002-of-00003.safetensors", + "model-00003-of-00003.safetensors", + "preprocessor_config.json", + "tokenizer.json", + "tokenizer_config.json", + "vocab.json", + "merges.txt" + ], + "vibevoice_7b": [ + "config.json", + "model.safetensors.index.json", + "model-00001-of-00010.safetensors", + "model-00002-of-00010.safetensors", + "model-00003-of-00010.safetensors", + "model-00004-of-00010.safetensors", + "model-00005-of-00010.safetensors", + "model-00006-of-00010.safetensors", + "model-00007-of-00010.safetensors", + "model-00008-of-00010.safetensors", + "model-00009-of-00010.safetensors", + "model-00010-of-00010.safetensors", + "preprocessor_config.json", + "tokenizer.json", + "tokenizer_config.json", + "vocab.json", + "merges.txt" + ], + "higgs_audio_v3_tts_4b": [ + "chat_template.jinja", + "config.json", + "model.safetensors.index.json", + "model.safetensors", + "tokenizer.json", + "tokenizer_config.json" + ], + "heartmula": [ + "tokenizer.json", + "gen_config.json", + "HeartMuLa-oss-3B/config.json", + "HeartMuLa-oss-3B/model.safetensors.index.json", + "HeartMuLa-oss-3B/model-00001-of-00004.safetensors", + "HeartMuLa-oss-3B/model-00002-of-00004.safetensors", + "HeartMuLa-oss-3B/model-00003-of-00004.safetensors", + "HeartMuLa-oss-3B/model-00004-of-00004.safetensors", + "HeartCodec-oss/config.json", + "HeartCodec-oss/model.safetensors.index.json", + "HeartCodec-oss/model-00001-of-00002.safetensors", + "HeartCodec-oss/model-00002-of-00002.safetensors" + ], + "irodori_tts_500m_v3": [ + "model.safetensors", + "model_config.json", + "../llm-jp-3-150m/tokenizer.json", + "../Semantic-DACVAE-Japanese-32dim/weights.safetensors" + ], + "irodori_tts_600m_v3_voice_design": [ + "model.safetensors", + "model_config.json", + "../llm-jp-3-150m/tokenizer.json", + "../Semantic-DACVAE-Japanese-32dim/weights.safetensors" + ], + "stable_audio_3_small_music": [ + "model_config.json", + "model.safetensors", + "t5gemma-b-b-ul2/config.json", + "t5gemma-b-b-ul2/model.safetensors", + "t5gemma-b-b-ul2/tokenizer.json", + "t5gemma-b-b-ul2/tokenizer.model" + ], + "stable_audio_3_small_sfx": [ + "model_config.json", + "model.safetensors", + "t5gemma-b-b-ul2/config.json", + "t5gemma-b-b-ul2/model.safetensors", + "t5gemma-b-b-ul2/tokenizer.json", + "t5gemma-b-b-ul2/tokenizer.model" + ], + "stable_audio_3_medium": [ + "model_config.json", + "model.safetensors", + "t5gemma-b-b-ul2/config.json", + "t5gemma-b-b-ul2/model.safetensors", + "t5gemma-b-b-ul2/tokenizer.json", + "t5gemma-b-b-ul2/tokenizer.model" + ], + "supertonic_3": [ + "config/tts.json", + "config/unicode_indexer.json", + "ggml/supertonic.safetensors", + "voice_styles/M1.json" + ], + "index_tts2": [ + "config.yaml", + "bpe.model", + "gpt.safetensors", + "s2mel.safetensors", + "feat1.safetensors", + "feat2.safetensors", + "wav2vec2bert_stats.safetensors", + "semantic_codec_model.safetensors", + "campplus.safetensors", + "w2v-bert-2.0/config.json", + "w2v-bert-2.0/preprocessor_config.json", + "w2v-bert-2.0/model.safetensors", + "bigvgan/config.json", + "bigvgan/model.safetensors", + "qwen0.6bemo4-merge/config.json", + "qwen0.6bemo4-merge/generation_config.json", + "qwen0.6bemo4-merge/tokenizer.json", + "qwen0.6bemo4-merge/tokenizer_config.json", + "qwen0.6bemo4-merge/vocab.json", + "qwen0.6bemo4-merge/merges.txt", + "qwen0.6bemo4-merge/model.safetensors" + ], + "mel_band_roformer": [ + "config.json", + "model.safetensors" + ], + "vevo2": [ + "acoustic_modeling/fm_emilia101k_singnet7k_repa/config.json", + "acoustic_modeling/fm_emilia101k_singnet7k_repa/model.safetensors", + "acoustic_modeling/fm_emilia101k_singnet7k_repa/whisper_stats.safetensors", + "acoustic_modeling/fm_emilia101k_singnet7k_repa_text/config.json", + "acoustic_modeling/fm_emilia101k_singnet7k_repa_text/model.safetensors", + "acoustic_modeling/fm_emilia101k_singnet7k_repa_text/whisper_stats.safetensors", + "contentstyle_modeling/posttrained/amphion_config.json", + "contentstyle_modeling/posttrained/config.json", + "contentstyle_modeling/posttrained/generation_config.json", + "contentstyle_modeling/posttrained/merges.txt", + "contentstyle_modeling/posttrained/model.safetensors", + "contentstyle_modeling/posttrained/tokenizer.json", + "contentstyle_modeling/posttrained/tokenizer_config.json", + "contentstyle_modeling/posttrained/vocab.json", + "contentstyle_modeling/pretrained/config.json", + "contentstyle_modeling/pretrained/generation_config.json", + "contentstyle_modeling/pretrained/merges.txt", + "contentstyle_modeling/pretrained/model.safetensors", + "contentstyle_modeling/pretrained/tokenizer.json", + "contentstyle_modeling/pretrained/tokenizer_config.json", + "contentstyle_modeling/pretrained/vocab.json", + "tokenizer/contentstyle_fvq16384_12.5hz/model.safetensors", + "tokenizer/prosody_fvq512_6.25hz/model.safetensors", + "vocoder/config.json", + "vocoder/model.safetensors", + "vocoder/model_1.safetensors", + "vocoder/model_2.safetensors" + ], + "seed_vc": [ + "seed_vc_manifest.json", + "v2/vc_wrapper.json", + "v2/ar.safetensors", + "v2/cfm.safetensors", + "v1/svc.json", + "v1/svc.safetensors", + "v1/whisper_bigvgan.json", + "v1/whisper_bigvgan.safetensors", + "v1/xlsr_hift.json", + "v1/xlsr_hift.safetensors", + "astral/bsq32.json", + "astral/bsq32.safetensors", + "astral/bsq2048.json", + "astral/bsq2048.safetensors", + "campplus/model.safetensors", + "rmvpe/model.safetensors", + "hift/config.json", + "hift/model.safetensors", + "bigvgan/v2_22khz_80band_256x/config.json", + "bigvgan/v2_22khz_80band_256x/model.safetensors", + "bigvgan/v2_44khz_128band_512x/config.json", + "bigvgan/v2_44khz_128band_512x/model.safetensors", + "whisper-small/config.json", + "whisper-small/model.safetensors", + "hubert-large-ll60k/config.json", + "hubert-large-ll60k/model.safetensors", + "wav2vec2-xls-r-300m/config.json", + "wav2vec2-xls-r-300m/model.safetensors" + ], + "citrinet_asr": [ + "citrinet_256.safetensors", + "citrinet_256_config.json", + "citrinet_256_tokenizer.model", + "citrinet_256_vocab.txt" + ], + "marblenet_vad": [ + "marblenet_vad.safetensors", + "marblenet_vad_config.json", + "marblenet_vad_labels.txt" + ], + "voxcpm2": [ + "config.json", + "model.safetensors", + "tokenizer.json", + "tokenizer_config.json", + "audiovae.pth", + "audiovae.safetensors" + ], + "voxcpm2_audiovae": [ + "audiovae.safetensors" + ], + "htdemucs": [ + "manifest.json", + "955717e8/config.json", + "955717e8/model.safetensors" + ] +} diff --git a/webui/logs/.gitkeep b/webui/logs/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/webui/output/.gitkeep b/webui/output/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/webui/realtime_pipeline.py b/webui/realtime_pipeline.py new file mode 100644 index 00000000..23ffb4e3 --- /dev/null +++ b/webui/realtime_pipeline.py @@ -0,0 +1,739 @@ +""" +Realtime voice pipeline for audio.cpp: VAD -> STT -> LLM -> TTS. + +Everything runs in-process inside the standalone realtime server (see +realtime_server.py). Speech detection is Python silero-vad (tiny, CPU); STT and +TTS are HTTP calls to two audio.cpp C++ servers (one ASR model, one TTS model), +so the realtime loop exercises audio.cpp's own ASR + TTS at the same time. The +LLM is any OpenAI-compatible Chat Completions endpoint (DeepSeek by default). + +Topology (URLs/models are per-session, pushed from the browser Settings panel): + + mic PCM16 16k --> [silero VAD] --> speech segment + --> [ASR server /v1/audio/transcriptions] --> text + --> [LLM /chat/completions, streamed] --> reply tokens + --> [TTS server /v1/audio/speech, per sentence] --> WAV + --> resample to 16k PCM16 --> browser playback + +Two independent audio.cpp servers are expected, e.g.: + + run_server.bat qwen3-tts 8080 (TTS) + run_server.bat qwen3-asr 8081 (ASR) + +Both may also be the *same* multi-model server; the pipeline only needs a URL + +model id for each stage, so either topology works. +""" + +from __future__ import annotations + +import io +import json +import logging +import os +import queue +import re +import tempfile +import threading +import wave +from dataclasses import dataclass +from typing import Any, Iterator, Optional + +import numpy as np +import requests +import torch +import httpx + +logger = logging.getLogger("realtime.pipeline") + +# ── constants ────────────────────────────────────────────────────────── +PIPELINE_SR = 16000 +VAD_FRAME = 512 # silero window at 16 kHz (~32 ms) +CHUNK_SAMPLES = 512 # samples per outbound audio chunk at 16 kHz +_SENTENCE_END = re.compile(r"[.!?。！？…\n]") +_CLAUSE_END = re.compile(r"[,;:，、；：]") +_SOFT_BREAK = re.compile(r"\s+") +_MIN_TTS_CHARS = 6 # don't synthesize tiny fragments on their own +_TARGET_TTS_CHARS = 36 # small chunks keep first audio responsive on local GPUs +_MAX_TTS_CHARS = 72 # force a chunk even if the LLM avoids punctuation + + +# ── cancel scope (cooperative cancellation for barge-in) ─────────────── + + +class CancelScope: + """Generation-counter pattern: each barge-in bumps ``generation``; the + in-flight turn checks ``is_stale(gen)`` between steps and aborts early.""" + + def __init__(self) -> None: + self.generation: int = 0 + + def cancel(self) -> None: + self.generation += 1 + + def is_stale(self, gen: int) -> bool: + return gen != self.generation + + +# ── pipeline events (emitted by stages, consumed by the WebSocket layer) ─ + + +@dataclass +class PipelineEvent: + pass + + +@dataclass +class SpeechStarted(PipelineEvent): + pass + + +@dataclass +class SpeechStopped(PipelineEvent): + # Only the VAD->worker copy carries audio; the outbound copy leaves it None. + audio: Optional[np.ndarray] = None + + +@dataclass +class UserTranscript(PipelineEvent): + text: str + partial: bool = False + + +@dataclass +class AssistantText(PipelineEvent): + text: str + + +@dataclass +class AudioChunk(PipelineEvent): + """One chunk of 16-bit PCM audio at 16 kHz.""" + pcm: bytes # int16 LE + + +@dataclass +class ResponseDone(PipelineEvent): + pass + + +@dataclass +class TurnDiscarded(PipelineEvent): + """A VAD turn ended without producing a conversational response.""" + + reason: str + + +# ── audio helpers ─────────────────────────────────────────────────────── + + +def _wav_bytes_to_pcm16_16k(data: bytes) -> bytes: + """Decode a WAV response and return mono PCM16 at 16 kHz. audio.cpp models + emit whatever native rate they run at (e.g. 24 kHz), but the browser + playback worklet is fixed to 16 kHz, so we downmix + resample here.""" + if len(data) < 44 or data[:4] != b"RIFF": + # Assume the server already handed back raw PCM16 @ 16 kHz. + return data + try: + with wave.open(io.BytesIO(data), "rb") as wf: + sr = wf.getframerate() + channels = wf.getnchannels() + sampwidth = wf.getsampwidth() + frames = wf.readframes(wf.getnframes()) + except (wave.Error, EOFError) as exc: + logger.error("TTS WAV decode failed: %r", exc) + return b"" + if sampwidth != 2: + logger.error("TTS returned %d-byte samples; only PCM16 is supported", sampwidth) + return b"" + arr = np.frombuffer(frames, dtype=" 1: + arr = arr.reshape(-1, channels).mean(axis=1) + if sr != PIPELINE_SR and arr.size: + n_out = int(round(arr.size * PIPELINE_SR / sr)) + if n_out > 0: + x_old = np.linspace(0.0, 1.0, num=arr.size, endpoint=False) + x_new = np.linspace(0.0, 1.0, num=n_out, endpoint=False) + arr = np.interp(x_new, x_old, arr.astype(np.float32)) + arr = np.clip(np.round(arr), -32768, 32767).astype(" bytes: + """float32 [-1,1] @ 16 kHz mono -> WAV bytes (for the ASR request).""" + pcm = np.clip(np.round(audio * 32767.0), -32768, 32767).astype(" None: + arr = np.frombuffer(pcm, dtype=np.int16).astype(np.float32) / 32768.0 + self._pending = np.concatenate([self._pending, arr]) if self._pending.size else arr + + def _speech_prob(self, frame: np.ndarray) -> float: + with torch.no_grad(): + out = self._model(torch.from_numpy(frame), PIPELINE_SR) + return float(out.reshape(-1)[0]) + + def process(self) -> None: + """Consume whole 512-sample frames from the pending buffer. Called from + the VAD ticker thread every ~30 ms worth of audio.""" + n = self._pending.size // VAD_FRAME + if n == 0: + return + frames = self._pending[: n * VAD_FRAME].reshape(n, VAD_FRAME) + self._pending = self._pending[n * VAD_FRAME:].copy() + for frame in frames: + voiced = self._speech_prob(frame) >= self._threshold + if not self._speaking: + if voiced: + self._speech_run += 1 + # Accumulate into pre-speech ring buffer so these frames + # aren't lost when speech is confirmed below. + self._pre_speech_buf.append(frame.copy()) + if len(self._pre_speech_buf) > self._pre_speech_max: + self._pre_speech_buf.pop(0) + if self._speech_run >= self._min_speech_frames: + self._speaking = True + self._silence_run = 0 + # Include ALL pre-speech frames + current frame so the + # ASR gets the complete utterance from the very start. + self._segment = list(self._pre_speech_buf) + self._pre_speech_buf = [] + self._out.put(SpeechStarted()) + self._cancel.cancel() # barge-in + else: + self._speech_run = 0 + self._pre_speech_buf = [] + else: + self._segment.append(frame.copy()) + if voiced: + self._silence_run = 0 + else: + self._silence_run += 1 + if self._silence_run >= self._min_silence_frames: + seg = np.concatenate(self._segment) if self._segment else None + self._segment = [] + self._speaking = False + self._speech_run = 0 + self._out.put(SpeechStopped()) + self._turns.put(SpeechStopped(audio=seg)) + + +# ── STT stage (audio.cpp ASR server) ──────────────────────────────────── + + +class _STTStage: + """Transcribe a speech segment via an audio.cpp ASR server.""" + + def __init__(self, server_url: str, model_id: str, language: str = ""): + self._server_url = server_url.rstrip("/") + self._model_id = model_id + self._language = language + self._http = requests.Session() + + def configure(self, server_url: Optional[str] = None, model_id: Optional[str] = None, + language: Optional[str] = None) -> None: + if server_url: + self._server_url = server_url.rstrip("/") + if model_id: + self._model_id = model_id + if language is not None: + self._language = language + + def transcribe(self, audio: np.ndarray) -> str: + wav = _float_to_wav16k(audio) + tmp = tempfile.NamedTemporaryFile(prefix="rt_asr_", suffix=".wav", delete=False) + try: + tmp.write(wav) + tmp.close() + payload: dict[str, Any] = {"model": self._model_id, "audio": tmp.name} + if self._language: + payload["language"] = self._language + r = self._http.post( + f"{self._server_url}/v1/audio/transcriptions", json=payload, timeout=60) + if r.status_code != 200: + logger.error("ASR error %s: %s", r.status_code, r.text[:200]) + return "" + return (r.json().get("text") or "").strip() + except requests.RequestException as exc: + logger.error("ASR server unreachable (%s): %r", self._server_url, exc) + return "" + except (ValueError, KeyError) as exc: + logger.error("ASR bad response: %r", exc) + return "" + finally: + try: + os.unlink(tmp.name) + except OSError: + pass + + +# ── LLM stage ─────────────────────────────────────────────────────────── + + +class _LLMStage: + """OpenAI-compatible Chat Completions (DeepSeek by default).""" + + DEFAULT_SYSTEM = ( + "You are a helpful voice assistant. Keep responses concise — under 3 " + "sentences when possible. Always reply in the same language the user " + "speaks. If the user speaks Chinese, use Simplified Chinese (简体中文), " + "not Traditional Chinese." + ) + + def __init__(self, base_url: str, api_key: str = "", model: str = "deepseek-chat", + instructions: str = ""): + self._base_url = base_url.rstrip("/") + self._api_key = api_key + self._model = model + self._system = instructions.strip() or self.DEFAULT_SYSTEM + self._messages: list[dict[str, Any]] = [{"role": "system", "content": self._system}] + self._http = httpx.Client(timeout=60.0) + + def configure(self, base_url: Optional[str] = None, api_key: Optional[str] = None, + model: Optional[str] = None, instructions: Optional[str] = None) -> None: + if base_url: + self._base_url = base_url.rstrip("/") + if api_key is not None: + self._api_key = api_key + if model: + self._model = model + if instructions is not None: + new_system = instructions.strip() or self.DEFAULT_SYSTEM + if new_system != self._system: + self._system = new_system + self._messages[0] = {"role": "system", "content": self._system} + + def add_user_message(self, text: str) -> None: + self._messages.append({"role": "user", "content": text}) + + def add_assistant_message(self, text: str) -> None: + self._messages.append({"role": "assistant", "content": text}) + if len(self._messages) > 21: # system + last 20 turns + self._messages = [self._messages[0]] + self._messages[-20:] + + def stream(self, cancel_scope: CancelScope, gen: int) -> Iterator[str]: + headers: dict[str, str] = {"Content-Type": "application/json"} + if self._api_key: + headers["Authorization"] = f"Bearer {self._api_key}" + payload: dict[str, Any] = { + "model": self._model, "messages": self._messages, "stream": True, + } + try: + with self._http.stream("POST", f"{self._base_url}/chat/completions", + json=payload, headers=headers) as resp: + if resp.status_code != 200: + resp.read() + logger.error("LLM API error %s: %s", resp.status_code, resp.text[:300]) + return + for line in resp.iter_lines(): + if cancel_scope.is_stale(gen): + return + if not line.startswith("data: "): + continue + data_str = line[6:] + if data_str == "[DONE]": + return + try: + data = json.loads(data_str) + content = data["choices"][0]["delta"].get("content", "") + if content: + yield content + except (json.JSONDecodeError, KeyError, IndexError): + continue + except httpx.RequestError as exc: + logger.error("LLM API unreachable (%s): %r", self._base_url, exc) + + +# ── TTS stage (audio.cpp TTS server) ──────────────────────────────────── + + +class _TTSStage: + """Synthesize speech via an audio.cpp TTS server, resampled to 16 kHz.""" + + def __init__(self, server_url: str): + self._server_url = server_url.rstrip("/") + self._http = requests.Session() + + def configure(self, server_url: Optional[str] = None) -> None: + if server_url: + self._server_url = server_url.rstrip("/") + + def synthesize(self, text: str, model_id: str, voice: str = "", + voice_ref: str = "", reference_text: str = "") -> bytes: + payload: dict[str, Any] = {"model": model_id, "input": text} + if voice: + payload["voice"] = voice + if voice_ref: + payload["voice_ref"] = voice_ref + if reference_text: + payload["reference_text"] = reference_text + try: + r = self._http.post( + f"{self._server_url}/v1/audio/speech", json=payload, timeout=120) + except requests.RequestException as exc: + logger.error("TTS server unreachable (%s): %r", self._server_url, exc) + return b"" + if r.status_code != 200: + logger.error("TTS error %s: %s", r.status_code, r.text[:200]) + return b"" + return _wav_bytes_to_pcm16_16k(r.content) + + +# ── orchestrator ──────────────────────────────────────────────────────── + + +class RealtimePipeline: + """Owns the full VAD->STT->LLM->TTS loop. The WebSocket layer feeds raw mic + audio via ``feed_audio`` and pulls outbound events via ``drain_events``. + Per-session config (server URLs, model ids, voice, LLM key, instructions) is + applied with ``configure`` from the browser's session.update.""" + + def __init__( + self, + tts_server: str = "http://127.0.0.1:8080", + tts_model: str = "qwen3-tts", + tts_voice: str = "", + tts_voice_ref: str = "", + tts_reference_text: str = "", + asr_server: str = "http://127.0.0.1:8081", + asr_model: str = "qwen3-asr", + asr_language: str = "", + llm_base_url: str = "https://api.deepseek.com/v1", + llm_api_key: str = "", + llm_model: str = "deepseek-chat", + instructions: str = "", + ): + self._out_events: "queue.Queue[PipelineEvent]" = queue.Queue() + self._turn_queue: "queue.Queue[SpeechStopped]" = queue.Queue() + self._cancel_scope = CancelScope() + self._vad = _VADStage(self._out_events, self._turn_queue, self._cancel_scope) + self._stt = _STTStage(asr_server, asr_model, asr_language) + self._llm = _LLMStage(llm_base_url, llm_api_key, llm_model, instructions) + self._tts = _TTSStage(tts_server) + self._tts_model = tts_model + self._tts_voice = tts_voice + self._tts_voice_ref = tts_voice_ref + self._tts_reference_text = tts_reference_text + self._cfg_lock = threading.Lock() + self._threads: list[threading.Thread] = [] + self._stop_event = threading.Event() + # TTS worker thread — LLM and TTS run in parallel via queue + self._tts_queue: "queue.Queue[tuple]" = queue.Queue() + self._tts_pending = 0 + self._tts_lock = threading.Lock() + + # ── lifecycle ───────────────────────────────────────────────────── + + def start(self) -> None: + for target, name in ((self._vad_loop, "realtime-vad"), + (self._pipeline_loop, "realtime-pipeline"), + (self._tts_worker, "realtime-tts")): + t = threading.Thread(target=target, daemon=True, name=name) + t.start() + self._threads.append(t) + + def stop(self) -> None: + self._stop_event.set() + self._cancel_scope.cancel() + self._tts_queue.put(None) # wake TTS worker so it can exit + + def feed_audio(self, pcm: bytes) -> None: + self._vad.add_chunk(pcm) + + def cancel(self) -> None: + self._cancel_scope.cancel() + + def drain_events(self) -> list[PipelineEvent]: + events: list[PipelineEvent] = [] + while True: + try: + events.append(self._out_events.get_nowait()) + except queue.Empty: + break + return events + + def configure(self, cfg: dict[str, Any]) -> None: + """Apply per-session config pushed from the browser (session.update). + + Empty-string values are treated as "no override" so the env defaults + (set in run_realtime.bat) survive — e.g. qwen3-tts-Base requires a + voice_ref; an empty browser field must not clobber the env-provided + reference audio path. + """ + def val(key: str) -> Optional[str]: + v = cfg.get(key) + return v if isinstance(v, str) and v else None + + with self._cfg_lock: + self._stt.configure( + server_url=val("asr_server"), + model_id=val("asr_model"), + language=val("asr_language"), + ) + self._tts.configure(server_url=val("tts_server")) + if val("tts_model"): + self._tts_model = val("tts_model") + if val("tts_voice"): + self._tts_voice = val("tts_voice") + if val("tts_voice_ref"): + self._tts_voice_ref = val("tts_voice_ref") + if val("tts_reference_text"): + self._tts_reference_text = val("tts_reference_text") + self._llm.configure( + base_url=val("llm_base_url"), + api_key=val("llm_api_key"), + model=val("llm_model"), + instructions=val("instructions"), + ) + + # ── background loops ────────────────────────────────────────────── + + def _vad_loop(self) -> None: + while not self._stop_event.is_set(): + try: + self._vad.process() + except Exception: # never let a bad frame kill the ticker + logger.exception("VAD process error") + self._stop_event.wait(0.03) + + def _pipeline_loop(self) -> None: + while not self._stop_event.is_set(): + try: + stopped = self._turn_queue.get(timeout=0.2) + except queue.Empty: + continue + audio = stopped.audio + if audio is None or len(audio) < PIPELINE_SR * 0.3: # < 0.3s: ignore + self._out_events.put(TurnDiscarded(reason="too_short")) + continue + try: + self._run_turn(audio) + except Exception: + logger.exception("turn failed") + self._out_events.put(ResponseDone()) + + # ── TTS worker thread ────────────────────────────────────────────── + + def _tts_worker(self) -> None: + """Pull sentences from _tts_queue, synthesize them, push AudioChunk + events. Runs in its own thread so the LLM continues generating while + the TTS server synthesises the previous sentence.""" + while not self._stop_event.is_set(): + try: + item = self._tts_queue.get(timeout=0.2) + except queue.Empty: + continue + if item is None: # shutdown sentinel + break + text, gen, model, voice, voice_ref, reference_text = item + if self._cancel_scope.is_stale(gen): + with self._tts_lock: + self._tts_pending -= 1 + continue + try: + pcm = self._tts.synthesize( + text, model_id=model, voice=voice, + voice_ref=voice_ref, reference_text=reference_text) + except Exception: + logger.exception("TTS synthesis failed") + with self._tts_lock: + self._tts_pending -= 1 + continue + step = CHUNK_SAMPLES * 2 + for i in range(0, len(pcm), step): + if self._cancel_scope.is_stale(gen): + break + chunk = pcm[i:i + step] + if len(chunk) < step: + chunk = chunk.ljust(step, b"\x00") + self._out_events.put(AudioChunk(pcm=chunk)) + with self._tts_lock: + self._tts_pending -= 1 + + # ── one conversational turn ─────────────────────────────────────── + + def _run_turn(self, audio: np.ndarray) -> None: + gen = self._cancel_scope.generation + + # ── STT ── + text = self._stt.transcribe(audio) + if self._cancel_scope.is_stale(gen): + return + if not text: + logger.info("ASR returned no transcript; discarding realtime turn") + self._out_events.put(TurnDiscarded(reason="empty_transcript")) + return + logger.info("USER: %s", text) + self._out_events.put(UserTranscript(text=text, partial=False)) + self._llm.add_user_message(text) + + # ── LLM stream -> sentence-chunked TTS (parallel via queue) ── + with self._cfg_lock: + tts_model, tts_voice = self._tts_model, self._tts_voice + tts_ref, tts_ref_text = self._tts_voice_ref, self._tts_reference_text + pending = "" + full = "" + for token in self._llm.stream(self._cancel_scope, gen): + if self._cancel_scope.is_stale(gen): + # Barge-in: drain queued TTS items that haven't started yet, + # then return immediately — the worker will discard in-flight + # audio when it checks is_stale. + drained = 0 + while True: + try: + self._tts_queue.get_nowait() + drained += 1 + except queue.Empty: + break + with self._tts_lock: + self._tts_pending = max(0, self._tts_pending - drained) + return + full += token + pending += token + while True: + chunk, pending = self._split_speakable(pending) + if chunk is None: + break + self._speak(chunk, gen, tts_model, tts_voice, tts_ref, tts_ref_text) + + if self._cancel_scope.is_stale(gen): + return + if pending.strip(): + self._speak(pending, gen, tts_model, tts_voice, tts_ref, tts_ref_text) + + # Wait for all queued TTS items to finish before marking the turn done. + while not self._stop_event.is_set(): + with self._tts_lock: + if self._tts_pending <= 0: + break + self._stop_event.wait(0.05) + + if full.strip(): + logger.info("ASSISTANT: %s", full.strip()) + self._llm.add_assistant_message(full.strip()) + self._out_events.put(ResponseDone()) + + @staticmethod + def _split_speakable(buf: str) -> tuple[Optional[str], str]: + """Pull one short, speakable chunk from ``buf``. + + audio.cpp's HTTP TTS endpoint returns a whole WAV per request, so we + simulate streaming by feeding it compact clauses instead of waiting for + an entire assistant paragraph. Sentence punctuation wins; clause + punctuation is allowed once the chunk is useful; a long punctuation-free + span is force-split near a word boundary. + """ + text = buf.lstrip() + if not text: + return None, "" + + sentence = _SENTENCE_END.search(text) + if sentence and sentence.end() <= _TARGET_TTS_CHARS: + end = sentence.end() + head, tail = text[:end], text[end:] + if len(head.strip()) >= _MIN_TTS_CHARS or tail.strip(): + return head.strip(), tail + + if len(text) >= _TARGET_TTS_CHARS: + clause = None + for match in _CLAUSE_END.finditer(text): + if match.end() >= _MIN_TTS_CHARS: + clause = match + if match.end() >= _TARGET_TTS_CHARS: + break + if clause and clause.end() <= _MAX_TTS_CHARS: + end = clause.end() + return text[:end].strip(), text[end:] + + if sentence and sentence.end() <= _MAX_TTS_CHARS: + end = sentence.end() + head, tail = text[:end], text[end:] + if len(head.strip()) >= _MIN_TTS_CHARS or tail.strip(): + return head.strip(), tail + + if len(text) < _MAX_TTS_CHARS: + return None, buf + + window = text[:_MAX_TTS_CHARS] + split_at = 0 + for match in _SOFT_BREAK.finditer(window): + if match.end() >= _TARGET_TTS_CHARS: + split_at = match.end() + break + split_at = match.end() + if split_at < _MIN_TTS_CHARS: + split_at = _MAX_TTS_CHARS + return text[:split_at].strip(), text[split_at:] + + def _speak(self, text: str, gen: int, model: str, voice: str, + voice_ref: str, reference_text: str) -> None: + """Queue a sentence for TTS synthesis. The worker thread picks it up + so LLM generation continues without blocking on the HTTP call.""" + text = text.strip() + if not text: + return + # Match the original speech-to-speech architecture: text reaches the + # client as soon as the LLM has a speakable sentence, while TTS works in + # the background. This keeps the side panel from sitting blank during + # local TTS latency. + self._out_events.put(AssistantText(text=text)) + with self._tts_lock: + self._tts_pending += 1 + self._tts_queue.put((text, gen, model, voice, voice_ref, reference_text)) diff --git a/webui/realtime_server.py b/webui/realtime_server.py new file mode 100644 index 00000000..81fa9e36 --- /dev/null +++ b/webui/realtime_server.py @@ -0,0 +1,398 @@ +""" +Background FastAPI server for the realtime voice pipeline. + +Runs in a daemon thread inside the Gradio webui process. Exposes:: + + ws://127.0.0.1:8765/v1/realtime — OpenAI Realtime protocol WebSocket + http://127.0.0.1:8765/realtime/ — static orb frontend + GET /health — liveness check + +These are only *defaults*; the browser Settings panel overrides them per session +via ``session.update`` (see ``_config_from_session``). + +Environment variables:: + + AUDIOCPP_LLM_API_KEY DeepSeek / OpenAI-compatible API key + AUDIOCPP_LLM_BASE_URL Chat Completions endpoint (default: https://api.deepseek.com/v1) + AUDIOCPP_LLM_MODEL Model ID (default: deepseek-chat) + AUDIOCPP_TTS_SERVER C++ TTS server URL (default: http://127.0.0.1:8080) + AUDIOCPP_TTS_MODEL TTS model id loaded in that server (default: qwen3-tts) + AUDIOCPP_TTS_VOICE Named cached voice id (optional; else use voice_ref) + AUDIOCPP_TTS_VOICE_REF Reference audio path for voice cloning + AUDIOCPP_TTS_REF_TEXT Text of the reference audio + AUDIOCPP_ASR_SERVER C++ ASR server URL (default: http://127.0.0.1:8081) + AUDIOCPP_ASR_MODEL ASR model id loaded in that server (default: qwen3-asr) + AUDIOCPP_ASR_LANGUAGE Optional language hint for the ASR model + AUDIOCPP_INSTRUCTIONS System prompt for the assistant + AUDIOCPP_REALTIME_PORT WebSocket server port (default: 8765) +""" + +from __future__ import annotations + +import asyncio +import base64 +import json +import logging +import os +import threading +from typing import Any, Optional + +import uvicorn +from fastapi import FastAPI, WebSocket, WebSocketDisconnect +from fastapi.staticfiles import StaticFiles + +from realtime_pipeline import ( + AssistantText, + AudioChunk, + PipelineEvent, + RealtimePipeline, + ResponseDone, + SpeechStarted, + SpeechStopped, + TurnDiscarded, + UserTranscript, +) + +logger = logging.getLogger("realtime.server") + +HERE = os.path.dirname(os.path.abspath(__file__)) +STATIC_DIR = os.path.join(HERE, "realtime_static") + +# ── helpers ───────────────────────────────────────────────────────────── + + +def _env(key: str, default: str = "") -> str: + return os.environ.get(key, default).strip() + + +def _generate_id(prefix: str = "id") -> str: + import uuid + return f"{prefix}_{uuid.uuid4().hex[:12]}" + + +def _join_transcript(parts: list[str]) -> str: + out = "" + for part in parts: + text = (part or "").strip() + if not text: + continue + if not out: + out = text + elif (out[-1].isspace() or text[0].isspace() + or text[0] in ",.;:!?，。！？、；：）]}》”’" + or out[-1] in "([{（《“‘" + or "\u3400" <= out[-1] <= "\u9fff" + or "\u3400" <= text[0] <= "\u9fff"): + out += text + else: + out += " " + text + return out.strip() + + +def _resolve_voice_ref(path: str) -> str: + """If *path* is relative, resolve it against the webui/ directory so the + C++ TTS server (whose CWD may be anywhere) can open it.""" + if not path: + return path + if os.path.isabs(path): + return path + return os.path.normpath(os.path.join(HERE, path)) + + +def _default_config() -> dict: + """Pipeline config from environment (browser Settings override per session).""" + return { + "tts_server": _env("AUDIOCPP_TTS_SERVER", "http://127.0.0.1:8080"), + "tts_model": _env("AUDIOCPP_TTS_MODEL", "qwen3-tts"), + "tts_voice": _env("AUDIOCPP_TTS_VOICE", ""), + "tts_voice_ref": _resolve_voice_ref(_env("AUDIOCPP_TTS_VOICE_REF", "")), + "tts_reference_text": _env("AUDIOCPP_TTS_REF_TEXT", ""), + "asr_server": _env("AUDIOCPP_ASR_SERVER", "http://127.0.0.1:8081"), + "asr_model": _env("AUDIOCPP_ASR_MODEL", "qwen3-asr"), + "asr_language": _env("AUDIOCPP_ASR_LANGUAGE", ""), + "llm_base_url": _env("AUDIOCPP_LLM_BASE_URL", "https://api.deepseek.com/v1"), + "llm_api_key": _env("AUDIOCPP_LLM_API_KEY", ""), + "llm_model": _env("AUDIOCPP_LLM_MODEL", "deepseek-chat"), + "instructions": _env("AUDIOCPP_INSTRUCTIONS", ""), + } + + +# Keys the browser is allowed to override via session.update -> session.audiocpp. +_OVERRIDABLE_KEYS = { + "tts_server", "tts_model", "tts_voice", "tts_voice_ref", "tts_reference_text", + "asr_server", "asr_model", "asr_language", + "llm_base_url", "llm_api_key", "llm_model", "instructions", +} + + +def _config_from_session(session: dict) -> dict: + """Extract pipeline overrides from a session.update payload. Custom fields + live under ``session.audiocpp``; the standard OpenAI Realtime fields + ``instructions`` and ``audio.output.voice`` are also honored.""" + cfg: dict[str, Any] = {} + audiocpp = session.get("audiocpp") + if isinstance(audiocpp, dict): + for key, value in audiocpp.items(): + if key in _OVERRIDABLE_KEYS: + cfg[key] = value + if isinstance(session.get("instructions"), str): + cfg["instructions"] = session["instructions"] + audio = session.get("audio") + if isinstance(audio, dict): + output = audio.get("output") + if isinstance(output, dict) and isinstance(output.get("voice"), str) and output["voice"]: + cfg["tts_voice"] = output["voice"] + # Resolve relative voice_ref paths against webui/ so the C++ TTS server + # finds them regardless of its own working directory. + if "tts_voice_ref" in cfg: + cfg["tts_voice_ref"] = _resolve_voice_ref(cfg["tts_voice_ref"]) + return cfg + + +# ── FastAPI app factory ───────────────────────────────────────────────── + + +def create_app() -> FastAPI: + app = FastAPI(title="audio.cpp Realtime") + + defaults = _default_config() + + @app.get("/health") + async def health(): + return {"status": "ok"} + + @app.get("/config") + async def config(): + """Return the env-default pipeline config so the browser Settings panel + can pre-fill fields (TTS/ASR/LLM URLs, model ids, voice_ref, etc.). + Do not echo the LLM API key back to the browser; the pipeline can still + use the env-provided key as its server-side default.""" + public_defaults = dict(defaults) + public_defaults["llm_api_key"] = "" + public_defaults["llm_api_key_set"] = bool(defaults.get("llm_api_key")) + return public_defaults + + @app.websocket("/v1/realtime") + async def ws_realtime(ws: WebSocket): + await ws.accept() + session_id = _generate_id("session") + + pipeline = RealtimePipeline(**defaults) + pipeline.start() + + # ── Send session.created ── + await ws.send_json({ + "type": "session.created", + "session": { + "id": session_id, + "type": "realtime", + "audio": { + "input": {"format": {"type": "pcm16", "sample_rate": 16000, "channels": 1}}, + "output": {"format": {"type": "pcm16", "sample_rate": 16000, "channels": 1}}, + }, + }, + }) + + async def send_events(): + """Poll pipeline events and send them over the WebSocket.""" + current_resp_id: Optional[str] = None # stable across one assistant turn + _full_parts: list[str] = [] # committed sentences for response.done + + def _ensure_resp_id() -> str: + nonlocal current_resp_id + if current_resp_id is None: + current_resp_id = _generate_id("resp") + return current_resp_id + + while True: + await asyncio.sleep(0.01) + for event in pipeline.drain_events(): + if isinstance(event, SpeechStarted): + current_resp_id = None # barge-in: new turn, new response + _full_parts = [] + item_id = _generate_id("item") + await ws.send_json({ + "type": "input_audio_buffer.speech_started", + "item_id": item_id, + }) + elif isinstance(event, SpeechStopped): + await ws.send_json({ + "type": "input_audio_buffer.speech_stopped", + }) + elif isinstance(event, TurnDiscarded): + await ws.send_json({ + "type": "input_audio_buffer.turn_discarded", + "reason": event.reason, + }) + elif isinstance(event, UserTranscript): + item_id = _generate_id("item") + await ws.send_json({ + "type": "conversation.item.input_audio_transcription.completed", + "item_id": item_id, + "transcript": event.text, + }) + elif isinstance(event, AssistantText): + # Each delta is now a complete sentence, emitted when + # its TTS audio starts being sent — so the frontend + # shows the text synchronised with audio playback. + rid = _ensure_resp_id() + await ws.send_json({ + "type": "response.output_audio_transcript.delta", + "response_id": rid, + "delta": event.text, + }) + # Immediately commit as a done segment (the pipeline + # already split on sentence boundaries). + _full_parts.append(event.text) + await ws.send_json({ + "type": "response.output_audio_transcript.done", + "response_id": rid, + "transcript": event.text, + }) + elif isinstance(event, AudioChunk): + b64 = base64.b64encode(event.pcm).decode("ascii") + await ws.send_json({ + "type": "response.output_audio.delta", + "response_id": _ensure_resp_id(), + "delta": b64, + }) + elif isinstance(event, ResponseDone): + rid = current_resp_id or _ensure_resp_id() + full_transcript = _join_transcript(_full_parts) + await ws.send_json({ + "type": "response.done", + "response": { + "id": rid, + "status": "completed", + "output": [{ + "type": "message", + "content": [{ + "type": "audio", + "transcript": full_transcript, + }], + }] if full_transcript else [], + }, + }) + current_resp_id = None + _full_parts = [] + + # Start the event sender as a background task + send_task = asyncio.create_task(send_events()) + + try: + while True: + raw = await ws.receive_text() + try: + msg = json.loads(raw) + except json.JSONDecodeError: + continue + + msg_type = msg.get("type", "") + + if msg_type == "session.update": + session = msg.get("session") + if isinstance(session, dict): + cfg = _config_from_session(session) + if cfg: + pipeline.configure(cfg) + await ws.send_json({"type": "session.updated"}) + + elif msg_type == "input_audio_buffer.append": + audio_b64 = msg.get("audio", "") + if audio_b64: + try: + pcm = base64.b64decode(audio_b64) + pipeline.feed_audio(pcm) + except Exception: + pass + + elif msg_type == "response.cancel": + pipeline.cancel() + + elif msg_type == "input_audio_buffer.commit": + pass # VAD handles boundaries automatically + + elif msg_type == "response.create": + pass # LLM is triggered automatically after STT + + except WebSocketDisconnect: + logger.info("WebSocket client disconnected: %s", session_id) + finally: + send_task.cancel() + try: + await send_task + except asyncio.CancelledError: + pass + pipeline.stop() + + # Serve static orb frontend + if os.path.isdir(STATIC_DIR): + app.mount("/realtime", StaticFiles(directory=STATIC_DIR, html=True), name="static") + + return app + + +# ── server launcher (called from webui.py) ────────────────────────────── + + +class RealtimeServerThread: + """Manages a uvicorn server in a background daemon thread.""" + + def __init__(self, port: int = 8765): + self.port = port + self._thread: Optional[threading.Thread] = None + self._server: Optional[uvicorn.Server] = None + + @property + def running(self) -> bool: + return self._server is not None + + def start(self) -> None: + if self._server is not None: + return + app = create_app() + config = uvicorn.Config(app, host="127.0.0.1", port=self.port, log_level="warning") + self._server = uvicorn.Server(config) + + def _run(): + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + loop.run_until_complete(self._server.serve()) # type: ignore[union-attr] + + self._thread = threading.Thread(target=_run, daemon=True, name="realtime-server") + self._thread.start() + logger.info("Realtime server started on ws://127.0.0.1:%d/v1/realtime", self.port) + + def stop(self) -> None: + if self._server is None: + return + self._server.should_exit = True + logger.info("Realtime server stopped") + + +# Module-level singleton +_server: Optional[RealtimeServerThread] = None + + +def get_server(port: int = 8765) -> RealtimeServerThread: + global _server + if _server is None: + _server = RealtimeServerThread(port=port) + return _server + + +# ── direct execution ────────────────────────────────────────────────── +if __name__ == "__main__": + import sys + port = int(sys.argv[1]) if len(sys.argv) > 1 else int(os.environ.get("AUDIOCPP_REALTIME_PORT", "8765")) + server = get_server(port) + server.start() + print(f"Realtime server running at http://127.0.0.1:{port}/realtime/") + print(f"WebSocket at ws://127.0.0.1:{port}/v1/realtime") + try: + import time + while True: + time.sleep(1) + except KeyboardInterrupt: + server.stop() + print("Server stopped.") diff --git a/webui/realtime_static/index.html b/webui/realtime_static/index.html new file mode 100644 index 00000000..715d0595 --- /dev/null +++ b/webui/realtime_static/index.html @@ -0,0 +1,235 @@ + + + + + + audio.cpp Realtime Voice + + + + + + +
+
+
+
+
+ audio.cpp Realtime +
+

Real-time voice chat powered by audio.cpp TTS + LLM.

+
+
+
+ + +
+
+ +
+
+
+ + +
+ + + + +
+ +

Tap to start

+ +
+ + +
+
+
+
+

Conversation

+ +
+
+
+
+ + + + +
+
+ + +
+
+

Settings

+ +
+ +
+
+
+
+

TTS · audio.cpp 语音合成服务

+
+ + +
+
+ + +
+ + +

ASR · audio.cpp 语音识别服务

+
+ + +
+ + +
+
+

LLM · Chat Completions

+
+ + +
+ + + + + + + +
+
+
+
+ +
+ +
+
+ + + + + diff --git a/webui/realtime_static/main.js b/webui/realtime_static/main.js new file mode 100644 index 00000000..e09e1d79 --- /dev/null +++ b/webui/realtime_static/main.js @@ -0,0 +1,580 @@ +// @ts-check +/** + * audio.cpp Realtime Voice — simplified orb frontend. + * Always connects directly to ws://127.0.0.1:8765/v1/realtime. + * No tools, no settings, no LB mode — just the orb. + * + * @typedef {"idle" | "connecting" | "queued" | "your-turn" | + * "listening" | "user-speaking" | "processing" | + * "ai-speaking" | "error"} AppState + */ + +import { S2sWsRealtimeClient } from "./ws/s2s-ws-client.js"; +import { ChatView } from "./ui/chat.js"; +import { $, truncateError, DEBUG } from "./ui/dom.js"; + +// ── Settings (persisted in localStorage) ─────────────────────────────── +const DEFAULT_RT_URL = "ws://127.0.0.1:8765/v1/realtime"; +const DEFAULT_INSTRUCTIONS = + "You are a friendly voice assistant. Keep replies short and spoken. " + + "Always reply in the same language the user speaks. " + + "If the user speaks Chinese, use Simplified Chinese (简体中文), not Traditional Chinese."; + +const STORAGE = { + rtUrl: "audiocpp.rt.url", + ttsUrl: "audiocpp.tts.url", + ttsModel: "audiocpp.tts.model", + ttsVoice: "audiocpp.tts.voice", + ttsVoiceRef: "audiocpp.tts.voiceRef", + ttsRefText: "audiocpp.tts.refText", + asrUrl: "audiocpp.asr.url", + asrModel: "audiocpp.asr.model", + asrLanguage: "audiocpp.asr.language", + llmUrl: "audiocpp.llm.url", + llmModel: "audiocpp.llm.model", + llmKey: "audiocpp.llm.key", + instructions: "audiocpp.llm.instructions", + noiseGate: "audiocpp.noiseGate", +}; + +// Noise gate: slider minimum = off (gate disabled), rest = active threshold dBFS. +const GATE_OFF_DB = -66; +const GATE_MAX_DB = -3; +const GATE_DEFAULT_DB = -50; + +const DEFAULTS = { + rtUrl: DEFAULT_RT_URL, + ttsUrl: "http://127.0.0.1:8080", + ttsModel: "qwen3-tts", + ttsVoice: "", + ttsVoiceRef: "", + ttsRefText: "", + asrUrl: "http://127.0.0.1:8081", + asrModel: "qwen3-asr", + asrLanguage: "", + llmUrl: "https://api.deepseek.com/v1", + llmModel: "deepseek-chat", + llmKey: "", + instructions: DEFAULT_INSTRUCTIONS, + noiseGate: String(GATE_DEFAULT_DB), +}; + +function loadSettings() { + const s = {}; + for (const [k, def] of Object.entries(DEFAULTS)) { + if (k === "llmKey") { + localStorage.removeItem(STORAGE[k]); + s[k] = def; + continue; + } + const v = localStorage.getItem(STORAGE[k]); + s[k] = v === null ? def : v; + } + return s; +} + +/** Env defaults fetched from the realtime server's /config endpoint. Empty + * until the fetch resolves; merged in so the Settings panel pre-fills fields + * the user hasn't customized. localStorage still wins over env defaults. */ +let envDefaults = {}; + +async function refreshEnvDefaults() { + try { + // The realtime page is served at /realtime/, so the backend root is two + // levels up. Fall back to same origin if that fails. + const urls = [ + new URL("../../config", window.location.href).href, + new URL("/config", window.location.origin).href, + ]; + for (const u of urls) { + try { + const res = await fetch(u, { cache: "no-store" }); + if (res.ok) { + envDefaults = await res.json(); + // Apply: any field the user hasn't set in localStorage takes the env value. + for (const k of Object.keys(DEFAULTS)) { + if (envDefaults[k] != null && localStorage.getItem(STORAGE[k]) === null) { + settings[k] = envDefaults[k]; + } + } + return; + } + } catch { /* try next */ } + } + } catch (e) { + console.warn("[main] /config fetch failed:", e); + } +} + +function saveSettings(s) { + for (const k of Object.keys(DEFAULTS)) { + if (k === "llmKey") { + localStorage.removeItem(STORAGE[k]); + continue; + } + localStorage.setItem(STORAGE[k], String(s[k] ?? "")); + } +} + +/** Build the `audiocpp` config block sent inside session.update. */ +function appConfigFromSettings(s) { + return { + tts_server: s.ttsUrl, + tts_model: s.ttsModel, + tts_voice: s.ttsVoice, + tts_voice_ref: s.ttsVoiceRef, + tts_reference_text: s.ttsRefText, + asr_server: s.asrUrl, + asr_model: s.asrModel, + asr_language: s.asrLanguage, + llm_base_url: s.llmUrl, + llm_model: s.llmModel, + llm_api_key: s.llmKey, + instructions: s.instructions, + }; +} + +let settings = loadSettings(); + +// ── State machine ────────────────────────────────────────────────────── +const STATE_CLASS = { + idle: "state-idle", connecting: "state-connecting", + queued: "state-queued", "your-turn": "state-your-turn", + listening: "state-listening", "user-speaking": "state-user-speaking", + processing: "state-processing", "ai-speaking": "state-ai-speaking", + error: "state-error", +}; +const STATE_VIEWS = { + idle: { caption: "点击开始对话", disabled: false }, + connecting: { caption: "连接中", disabled: true }, + queued: { caption: "排队中…", disabled: true }, + "your-turn": { caption: "准备就绪", disabled: true }, + listening: { caption: "", disabled: false }, + "user-speaking": { caption: "", disabled: false }, + processing: { caption: "正在生成回复", disabled: false }, + "ai-speaking": { caption: "正在播放", disabled: false }, + error: { caption: "点击重试", disabled: false }, +}; +const LIVE_STATES = new Set(["listening", "user-speaking", "processing", "ai-speaking"]); + +/** @type {AppState} */ +let currentState = "idle"; +/** @type {S2sWsRealtimeClient | null} */ +let client = null; +/** @type {AudioContext | null} */ +let audioContext = null; +/** @type {MediaStream | null} */ +let micStream = null; +/** @type {ChatView} */ +let chat; + +// DOM refs +const circleBtn = /** @type {HTMLButtonElement} */ ($("#main-circle")); +const captionEl = $("#circle-caption"); +const micBtn = /** @type {HTMLButtonElement} */ ($("#mic-btn")); +const stopBtn = /** @type {HTMLButtonElement} */ ($("#stop-btn")); +const micGate = $("#mic-gate"); +const mgaArc = /** @type {SVGSVGElement} */ (document.querySelector("#mic-gate-arc")); +const mgaTrack = /** @type {SVGPathElement} */ (document.querySelector("#mga-track")); +const mgaFill = /** @type {SVGPathElement} */ (document.querySelector("#mga-fill")); +const mgaHit = /** @type {SVGPathElement} */ (document.querySelector("#mga-hit")); +const mgaHandle = /** @type {SVGCircleElement} */ (document.querySelector("#mga-handle")); +const gateValue = /** @type {HTMLElement} */ ($("#gate-value")); +const gateMeterFill = /** @type {HTMLElement} */ ($("#gate-meter-fill")); +const queueBar = $("#queue-bar"); +const queuePosition = $("#queue-position"); +const queueLeave = /** @type {HTMLButtonElement} */ ($("#queue-leave")); +const queueFunnel = $("#queue-funnel"); +const queueFunnelMsg = $("#queue-funnel-msg"); +const queueFunnelJoin = /** @type {HTMLButtonElement} */ ($("#queue-funnel-join")); +const queueFunnelLeave = /** @type {HTMLButtonElement} */ ($("#queue-funnel-leave")); + +let muted = false; +let queuedTicketId = ""; + +// Settings modal refs +const settingsBtn = $("#settings-btn"); +const settingsModal = $("#settings-modal"); +const settingsForm = settingsModal?.querySelector("form") || null; +const settingsFields = { + rtUrl: $("#rt-url"), + ttsUrl: $("#tts-url"), + ttsModel: $("#tts-model"), + ttsVoice: $("#tts-voice"), + ttsVoiceRef: $("#tts-voice-ref"), + ttsRefText: $("#tts-ref-text"), + asrUrl: $("#asr-url"), + asrModel: $("#asr-model"), + asrLanguage: $("#asr-language"), + llmUrl: $("#llm-url"), + llmModel: $("#llm-model"), + llmKey: $("#llm-key"), + instructions: $("#instructions"), + noiseGate: $("#noise-gate"), +}; + +// ── Radial gate arc (around the mic button) ───────────────────────────── +// A ~200° arc centred on the left so the wide gap faces the orb (right). +const ARC_R = 40; +const ARC_SPAN_DEG = 200; +const ARC_START_DEG = 180 - ARC_SPAN_DEG / 2; + +function dbToFraction(db) { + const clamped = Math.min(GATE_MAX_DB, Math.max(GATE_OFF_DB, db)); + return (clamped - GATE_OFF_DB) / (GATE_MAX_DB - GATE_OFF_DB); +} + +function fractionToDb(f) { + const clamped = Math.min(1, Math.max(0, f)); + return Math.round(GATE_OFF_DB + clamped * (GATE_MAX_DB - GATE_OFF_DB)); +} + +function arcPoint(f, r = ARC_R) { + const deg = ARC_START_DEG + f * ARC_SPAN_DEG; + const rad = (deg * Math.PI) / 180; + return { x: 50 + r * Math.cos(rad), y: 50 + r * Math.sin(rad) }; +} + +function fullArcD() { + const a = arcPoint(0); + const b = arcPoint(1); + const largeArc = ARC_SPAN_DEG > 180 ? 1 : 0; + return `M ${a.x} ${a.y} A ${ARC_R} ${ARC_R} 0 ${largeArc} 1 ${b.x} ${b.y}`; +} + +function initGateArc() { + const d = fullArcD(); + mgaTrack.setAttribute("d", d); + mgaFill.setAttribute("d", d); + mgaHit.setAttribute("d", d); + mgaFill.setAttribute("pathLength", "100"); + mgaFill.style.strokeDasharray = "100 100"; + mgaFill.style.strokeDashoffset = "100"; + renderGateHandle(); +} + +function renderGateHandle() { + const off = Number(settings.noiseGate) <= GATE_OFF_DB; + const p = arcPoint(dbToFraction(Number(settings.noiseGate))); + mgaHandle.setAttribute("cx", String(p.x)); + mgaHandle.setAttribute("cy", String(p.y)); + micGate.classList.toggle("gate-off", off); +} + +function paintInputLevel(rms) { + const db = rms > 0 ? 20 * Math.log10(rms) : GATE_OFF_DB; + const f = dbToFraction(db); + mgaFill.style.strokeDashoffset = String(100 * (1 - f)); + if (settingsModal && settingsModal.open) gateMeterFill.style.width = `${f * 100}%`; + const enabled = Number(settings.noiseGate) > GATE_OFF_DB; + micGate.classList.toggle("gate-open", enabled && f >= dbToFraction(Number(settings.noiseGate))); +} + +function setGateThreshold(db) { + settings.noiseGate = String(Math.min(GATE_MAX_DB, Math.max(GATE_OFF_DB, Math.round(db)))); + const off = Number(settings.noiseGate) <= GATE_OFF_DB; + settingsFields.noiseGate.value = settings.noiseGate; + gateValue.textContent = off ? "Off" : `${settings.noiseGate} dB`; + renderGateHandle(); + localStorage.setItem(STORAGE.noiseGate, settings.noiseGate); + if (client && LIVE_STATES.has(currentState)) { + const thr = Number(settings.noiseGate); + client.setNoiseGate({ enabled: thr > GATE_OFF_DB, thresholdDb: thr }); + } +} + +function syncGateUi() { + settingsFields.noiseGate.value = settings.noiseGate; + const off = Number(settings.noiseGate) <= GATE_OFF_DB; + gateValue.textContent = off ? "Off" : `${settings.noiseGate} dB`; + renderGateHandle(); +} + +// Drag on the arc band to set the threshold +let gateDragging = false; +function gatePointerToDb(e) { + const rect = mgaArc.getBoundingClientRect(); + const cx = rect.left + rect.width / 2; + const cy = rect.top + rect.height / 2; + let deg = (Math.atan2(e.clientY - cy, e.clientX - cx) * 180) / Math.PI; + if (deg < 0) deg += 360; + const f = (deg - ARC_START_DEG) / ARC_SPAN_DEG; + return fractionToDb(f); +} +mgaHit.addEventListener("pointerdown", (e) => { + gateDragging = true; + mgaHit.setPointerCapture(e.pointerId); + setGateThreshold(gatePointerToDb(e)); +}); +mgaHit.addEventListener("pointermove", (e) => { + if (gateDragging) setGateThreshold(gatePointerToDb(e)); +}); +const endGateDrag = (e) => { + if (!gateDragging) return; + gateDragging = false; + try { mgaHit.releasePointerCapture(e.pointerId); } catch {} +}; +mgaHit.addEventListener("pointerup", endGateDrag); +mgaHit.addEventListener("pointercancel", endGateDrag); + +// Settings noise-gate slider also updates the arc handle live +settingsFields.noiseGate?.addEventListener("input", () => { + setGateThreshold(Number(settingsFields.noiseGate.value)); +}); + +// ── State management ─────────────────────────────────────────────────── +function setState(next) { + currentState = next; + const view = STATE_VIEWS[next]; + circleBtn.disabled = view.disabled; + circleBtn.className = `circle ${STATE_CLASS[next]}`; + if (next === "error") setCaption(view.caption, "error"); + else setCaption(view.caption, ""); + // Live indicator: side buttons appear + gate arc lights up + const wrap = circleBtn.closest(".orb-wrap"); + if (wrap) { + if (LIVE_STATES.has(next)) wrap.classList.add("live"); + else wrap.classList.remove("live"); + } + // Show/hide queue UI + queueBar.hidden = next !== "queued"; + queueFunnel.hidden = next !== "your-turn"; +} + +function setCaption(text, kind) { + const trimmed = text.trim(); + captionEl.textContent = trimmed; + captionEl.className = `circle-caption${kind ? ` ${kind}` : ""}${trimmed ? "" : " empty"}`; +} + +// ── Audio ────────────────────────────────────────────────────────────── +const MIC_CONSTRAINTS = { audio: { echoCancellation: true, noiseSuppression: true, autoGainControl: true } }; + +function createResumedAudioContext() { + try { + const Ctx = window.AudioContext || /** @type {any} */ (window).webkitAudioContext; + const ctx = new Ctx({ latencyHint: "interactive" }); + if (ctx.state === "suspended") void ctx.resume().catch(() => {}); + return /** @type {AudioContext} */ (ctx); + } catch (err) { + console.warn("[main] AudioContext init failed:", err); + return null; + } +} + +async function primeMicPermission() { + try { + const s = await navigator.mediaDevices.getUserMedia(MIC_CONSTRAINTS); + for (const track of s.getTracks()) track.stop(); + } catch (err) { + throw new Error(`Microphone access denied${err instanceof Error ? `: ${err.message}` : ""}`); + } +} + +/** @returns {Promise} */ +async function acquireMicStream() { + micStream = await navigator.mediaDevices.getUserMedia(MIC_CONSTRAINTS); + return micStream; +} + +// ── Client lifecycle ─────────────────────────────────────────────────── +function onClientStatus(status) { + switch (status) { + case "idle": setState("idle"); break; + case "creating-session": setState("connecting"); break; + case "queued": setState("queued"); break; + case "your-turn": setState("your-turn"); break; + case "connecting": setState("connecting"); break; + case "connected": setState("listening"); break; + case "user-speaking": setState("user-speaking"); break; + case "processing": setState("processing"); break; + case "ai-speaking": setState("ai-speaking"); break; + case "closed": setState("idle"); break; + case "error": setState("error"); break; + } +} + +function onFatalError(error) { + console.error("[main] fatal:", error); + setState("error"); + const msg = error instanceof Error ? error.message : String(error); + setCaption(truncateError(msg), "error"); +} + +function onQueuePosition(position) { + setState("queued"); + queuePosition.textContent = `排队位置: ${position}`; +} + +async function doStart() { + if (currentState === "idle" || currentState === "error") { + chat.clear(); + setState("connecting"); + setCaption("请求麦克风权限…", "muted"); + if (!audioContext || audioContext.state === "closed") audioContext = createResumedAudioContext(); + try { await primeMicPermission(); } catch (err) { + if (audioContext) void audioContext.close().catch(() => {}); + audioContext = null; + throw err; + } + const gateThreshold = Number(settings.noiseGate) || GATE_DEFAULT_DB; + const c = new S2sWsRealtimeClient({ + directUrl: settings.rtUrl, + instructions: settings.instructions, + voice: settings.ttsVoice, + appConfig: appConfigFromSettings(settings), + acquireMic: acquireMicStream, + noiseGate: { enabled: gateThreshold > GATE_OFF_DB, thresholdDb: gateThreshold }, + ...(audioContext ? { audioContext } : {}), + }); + client = c; + c.addEventListener("queue", (e) => { + const { position, queueId } = /** @type {CustomEvent} */ (e).detail; + if (queueId) queuedTicketId = queueId; + onQueuePosition(position); + }); + c.addEventListener("ready-to-join", () => { + setState("your-turn"); + queueFunnelMsg.textContent = "已就绪！点击加入开始对话。"; + }); + c.addEventListener("status", (e) => onClientStatus(/** @type {CustomEvent} */ (e).detail.status)); + c.addEventListener("transcript", (e) => chat.onTranscript(/** @type {CustomEvent} */ (e).detail)); + c.addEventListener("response-finished", (e) => chat.onResponseFinished(/** @type {CustomEvent} */ (e).detail)); + c.addEventListener("error", (e) => onFatalError(/** @type {CustomEvent} */ (e).detail.error)); + c.addEventListener("server-error", (e) => { + console.warn("[main] server error (non-fatal):", /** @type {CustomEvent} */ (e).detail.error); + }); + c.addEventListener("input-level", (e) => { + paintInputLevel(/** @type {CustomEvent} */ (e).detail.rms); + }); + try { await c.connect(); chat.reset(); } catch (err) { + if (audioContext) void audioContext.close().catch(() => {}); + audioContext = null; + throw err; + } + } +} + +async function doStop() { + const activeClient = client; + client = null; + if (activeClient) { + try { + await activeClient.close(); + } catch (err) { + console.warn("[main] error closing client:", err); + } + } else if (audioContext && audioContext.state !== "closed") { + try { + await audioContext.close(); + } catch { + // ignored + } + } + audioContext = null; + if (micStream) { + for (const track of micStream.getTracks()) track.stop(); + micStream = null; + } + paintInputLevel(0); + muted = false; + micBtn.classList.remove("muted"); + queueBar.hidden = true; + queueFunnel.hidden = true; + queuedTicketId = ""; + setState("idle"); +} + +// ── Events ───────────────────────────────────────────────────────────── +circleBtn.addEventListener("click", async () => { + try { + if (currentState === "idle" || currentState === "error") { + await doStart(); + } + } catch (err) { + onFatalError(err); + } +}); + +stopBtn.addEventListener("click", () => { void doStop(); }); + +micBtn.addEventListener("click", () => { + muted = !muted; + micBtn.classList.toggle("muted", muted); + micBtn.setAttribute("aria-label", muted ? "Unmute" : "Mute"); + client?.setMuted(muted); +}); + +queueLeave.addEventListener("click", () => { void doStop(); }); +queueFunnelLeave?.addEventListener("click", () => { void doStop(); }); +queueFunnelJoin?.addEventListener("click", () => { + queueFunnel.hidden = true; + client?.join(); +}); + +// ── Settings modal ───────────────────────────────────────────────────── +function fillSettingsForm() { + for (const [k, el] of Object.entries(settingsFields)) { + if (el) el.value = settings[k] ?? ""; + } +} + +function readSettingsForm() { + const s = { ...settings }; + for (const [k, el] of Object.entries(settingsFields)) { + if (el) s[k] = el.value; + } + return s; +} + +settingsBtn?.addEventListener("click", () => { + fillSettingsForm(); + syncGateUi(); + settingsModal?.showModal(); +}); + +settingsForm?.addEventListener("submit", (e) => { + // The dialog's submit button (value="save") closes the dialog; we just read + // the form on close. Nothing to do here on submit itself. +}); + +settingsModal?.addEventListener("close", () => { + const rv = settingsModal.returnValue; + if (rv !== "save") return; + const prevRtUrl = settings.rtUrl; + const next = readSettingsForm(); + settings = next; + saveSettings(next); + syncGateUi(); + // Live-apply TTS/ASR/LLM params to an active session; rtUrl needs a reconnect. + if (client) { + if (next.rtUrl !== prevRtUrl) { + // Realtime backend URL changed — must reconnect. + void (async () => { + await doStop(); + setTimeout(() => { void doStart(); }, 150); + })(); + } else { + client.updateAppConfig(appConfigFromSettings(next)); + const thr = Number(settings.noiseGate); + client.setNoiseGate({ enabled: thr > GATE_OFF_DB, thresholdDb: thr }); + } + } +}); + +// Cleanup on page unload +window.addEventListener("beforeunload", () => { void doStop(); }); + +// ── Init ─────────────────────────────────────────────────────────────── +chat = new ChatView(); +initGateArc(); +syncGateUi(); +setState("idle"); +chat.renderEmptyState(); +// Pull env defaults (TTS/ASR/LLM URLs, voice_ref, ...) from the backend so the +// Settings panel pre-fills with what run_realtime.bat configured. Non-blocking; +// if a session is already starting it still works (env defaults are also the +// backend's fallback for empty browser fields). +void refreshEnvDefaults(); +// Strip booting class after first paint +requestAnimationFrame(() => requestAnimationFrame(() => document.body.classList.remove("booting"))); diff --git a/webui/realtime_static/style.css b/webui/realtime_static/style.css new file mode 100644 index 00000000..3af02d96 --- /dev/null +++ b/webui/realtime_static/style.css @@ -0,0 +1,2674 @@ +:root { + --bg: #0a0b10; + --bg-elev: #13151c; + --bg-elev-2: #1b1e29; + --border: rgba(255, 255, 255, 0.08); + --border-strong: rgba(255, 255, 255, 0.16); + --text: #f5f6fa; + --text-dim: rgba(245, 246, 250, 0.65); + --text-faint: rgba(245, 246, 250, 0.42); + + --accent: #8b7dff; + --accent-2: #22d3ee; + --listening: #22d3ee; + --speaking: #8b7dff; + --processing: #f59e0b; + --error: #ff6a75; + --success: #34d399; + + /* Mono is the "machine voice": reserved for system text the app emits — + * the orb's status caption, tool calls, the transport tag. Body/UI stays + * Inter. */ + --font-mono: "Geist Mono", ui-monospace, SFMono-Regular, "SF Mono", Menlo, monospace; + + /* Thesis: color belongs to the voice. The chrome is monochrome; saturated + * hue appears only on the orb and on these tiny role echoes, which mirror + * the orb's own state colors (you listening = cyan, assistant = violet, + * tool = amber) so the transcript reads in the same color language. */ + --voice-user: var(--accent-2); + --voice-assistant: var(--accent); + --voice-tool: var(--processing); + + /* Smoothed mic RMS in [0..1], updated every frame from JS while a session + * is active. Used by audio-reactive circle states. */ + --audio-level: 0; + /* Five log-spaced frequency bands extracted from the mic analyser; + * drive each bar's height independently for a real "spectrum" feel. */ + --bar0: 0; + --bar1: 0; + --bar2: 0; + --bar3: 0; + --bar4: 0; + + --radius-sm: 8px; + --radius-md: 14px; + --radius-lg: 22px; + + --shadow-soft: 0 10px 40px rgba(0, 0, 0, 0.35); + + color-scheme: dark; +} + +* { + box-sizing: border-box; +} + +html, +body { + margin: 0; + padding: 0; + height: 100%; +} + +body { + font-family: "Inter", system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif; + background: radial-gradient(ellipse at 50% 20%, #1a1c28 0%, var(--bg) 60%); + color: var(--text); + min-height: 100vh; + overflow-x: hidden; + -webkit-font-smoothing: antialiased; +} + +button { + font-family: inherit; +} + +a { + color: var(--text); + text-decoration: none; + border-bottom: 1px solid var(--border-strong); +} +a:hover { + border-bottom-color: var(--text); +} + +#app { + display: grid; + grid-template-rows: auto 1fr auto; + min-height: 100vh; +} + +.hidden { + display: none !important; +} + +/* ─── Topbar ──────────────────────────────────────────────────────────── */ + +/* Topbar is just a floating row of controls over the stage - no + * separator line, no background. Same story for the footer. Keeps the + * app feeling like one continuous canvas. */ +.topbar { + display: flex; + /* Top-align so the right control cluster sits up with the title instead of + * floating to the vertical middle of the tall identity block. */ + align-items: flex-start; + justify-content: space-between; + padding: 18px 28px; + /* Default stacking (below the conversation panel, z 200) so the panel drawer + * covers the controls when it's open. */ +} + +.brand { + display: flex; + align-items: center; + gap: 10px; + font-weight: 600; + font-size: 14px; + letter-spacing: 0.01em; + color: var(--text-dim); +} + +/* Transport tag rides the wordmark as a system label, not prose — mono, + * dimmed, the only mono in the topbar. */ +.brand-tag { + font-family: var(--font-mono); + font-weight: 500; + font-size: 11px; + letter-spacing: 0.02em; + opacity: 0.55; +} + +.brand .brand-logo { + width: 26px; + height: 26px; + object-fit: contain; + flex: none; + filter: drop-shadow(0 1px 2px rgba(0, 0, 0, 0.45)); + transition: transform 0.2s ease; +} +.brand .brand-logo:hover { + transform: translateY(-1px) rotate(-2deg); +} + +/* Narrow viewports (typically the mobile shell iframe and phone-sized + browser windows): the full product name pushes the topbar's right + cluster off-screen. Keep the logo as the brand cue and drop the + wordmark - the orb already gives enough context. */ +@media (max-width: 600px) { + /* The full identity stack is too tall for a phone topbar. Show only the + * title + (i); the popup carries the blurb, credits and pipeline. + * `.brand` prefix raises specificity so this beats the later base + * `.ident-meta { display: flex }` rule regardless of source order. */ + .brand .ident-blurb, + .brand .ident-meta { + display: none; + } + .ident-title { + font-size: 18px; + } + /* (i) moves into the right-hand control cluster on phones, leaving the + * title alone on the left. */ + .brand .about-btn { + display: none; + } + .about-btn-mobile { + display: inline-flex; + } + /* Keep phone icons at the current compact size — the desktop bump above + * shouldn't leak into the mobile topbar. */ + .topbar-right .icon-btn { + width: 36px; + height: 36px; + } + .topbar-right .icon-btn svg { + width: 18px; + height: 18px; + } + /* Keep the account control compact on phones: avatar-only chip, shorter pill. */ + .account-chip { + height: 36px; + padding: 0 6px; + } + .account-handle { + display: none; + } + .signin-pill { + height: 36px; + padding: 0 12px; + } +} + +.topbar-right { + display: flex; + align-items: center; + gap: 10px; +} + +/* The HF user pill: compact avatar + handle. Rendered as a flex row so + * the avatar stays aligned with the text baseline even when the text + * wraps on narrow viewports. */ +.hf-user { + display: inline-flex; + align-items: center; + gap: 8px; + font-size: 13px; + color: var(--text-dim); + padding: 4px 10px 4px 4px; + border-radius: 999px; + background: var(--bg-elev); + border: 1px solid var(--border); + line-height: 1; +} + +.hf-avatar { + width: 24px; + height: 24px; + border-radius: 50%; + object-fit: cover; + background: color-mix(in srgb, var(--accent) 25%, var(--bg-elev-2)); + /* Hidden until an actual URL loads (see `setHfAvatar` in main.ts). + * Keeps the initial login flash from showing a broken image icon. */ + opacity: 0; + transition: opacity 0.25s ease; + flex: none; +} +.hf-avatar.loaded { + opacity: 1; +} + +.hf-user-name { + white-space: nowrap; +} + +/* ─── Transport pill ────────────────────────────────────────────── + * Surface the actual network path WebRTC picked for the robot peer + * connection (LAN, direct-through-NAT, or TURN-relayed). Lets the + * user spot at a glance when audio is going through the internet + * instead of staying on-prem. + */ +.transport-pill { + display: inline-flex; + align-items: center; + gap: 6px; + font-size: 12px; + font-weight: 500; + letter-spacing: 0.02em; + padding: 5px 10px; + border-radius: 999px; + background: var(--bg-elev); + border: 1px solid var(--border); + color: var(--text-dim); + user-select: none; + transition: background 0.2s ease, border-color 0.2s ease, color 0.2s ease; +} + +.transport-pill .transport-dot { + width: 7px; + height: 7px; + border-radius: 50%; + background: currentColor; + box-shadow: 0 0 0 3px color-mix(in srgb, currentColor 25%, transparent); + flex: none; +} + +/* Bitrate readout sits after the kind label. Dimmer + monospace so the + * kind (LAN / Direct / Relayed) stays the primary info and the numbers + * don't wiggle the layout as digits change. */ +.transport-pill .transport-bitrate { + font-family: var(--font-mono); + font-size: 11px; + font-weight: 500; + color: color-mix(in srgb, currentColor 75%, var(--text-dim)); + opacity: 0.85; + padding-left: 6px; + border-left: 1px solid color-mix(in srgb, currentColor 25%, transparent); +} +.transport-pill .transport-bitrate:empty { + display: none; +} + +.transport-pill.transport-checking { + color: var(--text-dim); +} +.transport-pill.transport-checking .transport-dot { + animation: transport-blink 1.2s ease-in-out infinite; +} + +.transport-pill.transport-lan { + color: #4ade80; + border-color: color-mix(in srgb, #4ade80 35%, var(--border)); + background: color-mix(in srgb, #4ade80 10%, var(--bg-elev)); +} + +.transport-pill.transport-direct { + color: #60a5fa; + border-color: color-mix(in srgb, #60a5fa 35%, var(--border)); + background: color-mix(in srgb, #60a5fa 10%, var(--bg-elev)); +} + +.transport-pill.transport-relay { + color: #fbbf24; + border-color: color-mix(in srgb, #fbbf24 35%, var(--border)); + background: color-mix(in srgb, #fbbf24 10%, var(--bg-elev)); +} + +@keyframes transport-blink { + 0%, 100% { opacity: 1; } + 50% { opacity: 0.35; } +} + +.icon-btn { + background: var(--bg-elev); + border: 1px solid var(--border); + color: var(--text-dim); + width: 36px; + height: 36px; + border-radius: var(--radius-sm); + display: inline-flex; + align-items: center; + justify-content: center; + cursor: pointer; + touch-action: manipulation; + transition: background 0.15s, color 0.15s, border-color 0.15s; +} +.icon-btn:hover { + background: var(--bg-elev-2); + color: var(--text); + border-color: var(--border-strong); +} +/* Topbar control cluster reads a touch bigger on desktop. Phones keep the + * compact 36px (reset in the mobile media query). */ +.topbar-right .icon-btn { + width: 40px; + height: 40px; +} +.topbar-right .icon-btn svg { + width: 20px; + height: 20px; +} + +/* ─── Account (HF login chip + sign-in pill + popover) ─────────────────────── */ +.account { + position: relative; + display: inline-flex; + align-items: center; +} +/* Signed-out: a compact pill that reads as the primary affordance. */ +.signin-pill { + display: inline-flex; + align-items: center; + gap: 7px; + height: 40px; + padding: 0 14px; + border-radius: var(--radius-sm); + border: 1px solid var(--border-strong); + background: var(--bg-elev); + color: var(--text); + font-size: 13px; + font-weight: 600; + text-decoration: none; + transition: background 0.15s, border-color 0.15s; +} +.signin-pill svg { + width: 16px; + height: 16px; + flex: none; +} +.signin-pill:hover { + background: var(--bg-elev-2); + border-color: var(--text); +} +/* Narrow viewports: collapse the pill to an icon-only square. */ +@media (max-width: 800px) { + .signin-pill { + gap: 0; + width: 40px; + padding: 0; + justify-content: center; + } + .signin-pill span { + display: none; + } +} +/* Signed-in: avatar + handle chip. */ +.account-chip { + display: inline-flex; + align-items: center; + gap: 8px; + height: 40px; + padding: 0 10px 0 6px; + border-radius: var(--radius-sm); + border: 1px solid var(--border); + background: var(--bg-elev); + color: var(--text-dim); + cursor: pointer; + transition: background 0.15s, color 0.15s, border-color 0.15s; +} +.account-chip:hover { + background: var(--bg-elev-2); + color: var(--text); + border-color: var(--border-strong); +} +.account-avatar { + width: 26px; + height: 26px; + border-radius: 50%; + object-fit: cover; + flex: none; +} +.account-avatar-fallback { + display: inline-flex; + align-items: center; + justify-content: center; + background: var(--bg-elev-2); + color: var(--text); + font-size: 12px; + font-weight: 700; +} +.account-handle { + font-size: 13px; + font-weight: 600; + max-width: 12ch; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} +.account-pro { + font-size: 10px; + font-weight: 700; + letter-spacing: 0.04em; + padding: 2px 5px; + border-radius: 5px; + background: var(--text); + color: var(--bg); +} +/* Org "Team" badge: same pill as PRO, accent-coloured so it reads as + * unlimited without claiming a paid PRO subscription. */ +.account-team { + background: var(--accent); + color: var(--bg); +} +.account-pop { + position: absolute; + top: calc(100% + 8px); + right: 0; + min-width: 200px; + padding: 6px; + border-radius: var(--radius-md); + border: 1px solid var(--border-strong); + background: var(--bg-elev); + box-shadow: var(--shadow-soft); + z-index: 50; +} +.account-pop[hidden] { + display: none; +} +.account-pop-row { + padding: 8px 10px; +} +.account-pop-name { + font-weight: 600; + font-size: 13px; +} +.account-pop-meta { + display: flex; + align-items: center; + justify-content: space-between; + gap: 10px; + padding-top: 0; + font-size: 12px; + color: var(--text-dim); +} +.account-tier { + font-weight: 600; + color: var(--text); +} +.account-pop-link { + display: block; + padding: 8px 10px; + border-radius: var(--radius-sm); + color: var(--text-dim); + font-size: 13px; + text-decoration: none; + transition: background 0.15s, color 0.15s; +} +.account-pop-link:hover { + background: var(--bg-elev-2); + color: var(--text); +} +.account-signout { + border-top: 1px solid var(--border); + margin-top: 4px; + padding-top: 10px; + border-radius: 0 0 var(--radius-sm) var(--radius-sm); +} + +/* ─── Daily-limit modal ────────────────────────────────────────────────────── */ +.limit-modal { + width: min(400px, 92vw); +} +.limit-card { + position: relative; + align-items: center; + text-align: center; + gap: 14px; + padding: 34px 28px 26px; +} +.limit-close { + position: absolute; + top: 12px; + right: 12px; +} +/* HF smiling face (brand yellow) on a neutral badge — friendly, not a stop sign. + * The logo is the single pop of colour, so the badge itself stays quiet. */ +.limit-badge { + width: 66px; + height: 66px; + border-radius: 50%; + display: grid; + place-items: center; + margin: 2px auto 2px; + background: var(--bg-elev-2); + border: 1px solid var(--border-strong); + box-shadow: 0 0 0 6px rgba(255, 210, 30, 0.06); +} +.limit-badge .hf-logo { + width: 46px; + height: auto; +} +.limit-title { + margin: 0; + font-size: 20px; + font-weight: 700; + letter-spacing: 0; +} +.limit-msg { + color: var(--text-dim); + font-size: 14px; + line-height: 1.55; + margin: 0; + max-width: 30ch; +} +.limit-cta { + display: inline-flex; + align-items: center; + justify-content: center; + gap: 8px; + margin-top: 4px; +} +.limit-cta .hf-logo { + width: 20px; + height: auto; + flex: none; +} +.limit-note { + margin: 0; + font-size: 12px; + color: var(--text-faint); +} +#limit-cta[hidden] { + display: none; +} + +/* ─── Stage ───────────────────────────────────────────────────────────── */ + +.stage { + display: flex; + flex-direction: column; + align-items: center; + justify-content: center; + padding: 24px 24px 32px; + gap: 20px; +} + +/* ─── Central circle ──────────────────────────────────────────────────── */ + +/* Wraps the orb and its two side controls so they stay aligned on one row. */ +.orb-wrap { + position: relative; + display: flex; + align-items: center; + justify-content: center; + gap: clamp(18px, 3vw, 32px); +} + +.circle { + position: relative; + width: clamp(220px, 38vw, 320px); + aspect-ratio: 1 / 1; + border-radius: 50%; + border: none; + background: transparent; + padding: 0; + cursor: pointer; + outline: none; + display: grid; + place-items: center; + color: var(--glow, var(--accent)); + transition: transform 0.18s ease, filter 0.18s ease; + -webkit-tap-highlight-color: transparent; + /* Treat taps as immediate clicks (no 300ms delay / double-tap-zoom). */ + touch-action: manipulation; +} +.circle:hover { + filter: brightness(1.08); +} +.circle:active { + transform: scale(0.97); + filter: brightness(0.92); +} +.circle:focus-visible .circle-core { + outline: 2px solid var(--glow, var(--accent)); + outline-offset: 6px; +} +.circle[disabled] { + cursor: default; + opacity: 0.75; +} + +.circle-glow { + position: absolute; + inset: 0; + border-radius: 50%; + background: radial-gradient(circle at center, var(--glow, var(--accent)) 0%, transparent 65%); + filter: blur(28px); + opacity: 0.5; + transform: scale(1); + transition: opacity 0.25s, background 0.25s, transform 1.4s ease-in-out; + pointer-events: none; +} + +/* Two nested circular rings: the inner tracks the core edge, the outer + * expands / fades to convey "audio radiating out" during speaking. */ +.circle-ring, +.circle-ring-outer { + position: absolute; + top: 50%; + left: 50%; + border-radius: 50%; + transform: translate(-50%, -50%); + pointer-events: none; + transition: opacity 0.4s ease, border-color 0.4s ease, transform 0.3s ease; +} +.circle-ring { + width: 82%; + height: 82%; + border: 1.5px solid color-mix(in srgb, var(--glow, var(--accent)) 35%, transparent); + opacity: 0.35; +} +.circle-ring-outer { + width: 94%; + height: 94%; + border: 1px solid color-mix(in srgb, var(--glow, var(--accent)) 22%, transparent); + opacity: 0; +} + +.circle-core { + position: relative; + width: 72%; + height: 72%; + border-radius: 50%; + background: radial-gradient( + circle at 35% 28%, + color-mix(in srgb, var(--glow, var(--accent)) 28%, transparent), + color-mix(in srgb, var(--glow, var(--accent)) 10%, transparent) 55%, + color-mix(in srgb, var(--glow, var(--accent)) 5%, transparent) + ); + border: 2px solid color-mix(in srgb, var(--glow, var(--accent)) 40%, transparent); + display: grid; + place-items: center; + box-shadow: + 0 0 32px color-mix(in srgb, var(--glow, var(--accent)) 22%, transparent), + inset 0 0 28px color-mix(in srgb, var(--glow, var(--accent)) 18%, transparent); + transition: background 0.4s ease, border-color 0.4s ease, box-shadow 0.4s ease, transform 0.4s ease; +} + +/* Indicator slot: a single SVG / spinner / bar group is visible at a time, + * driven by the state class on `.circle`. */ +.circle-indicator { + position: relative; + width: 44%; + height: 44%; + display: grid; + place-items: center; + color: var(--glow, var(--accent)); +} +.circle-indicator > .ind { + grid-area: 1 / 1; + opacity: 0; + transform: scale(0.85); + transition: opacity 0.25s ease, transform 0.25s ease; + pointer-events: none; +} +.circle-indicator > svg.ind { + width: 60%; + height: 60%; +} + +/* Spinner: a rotating ring gap, CSS-only. + * + * Note: the base `.circle-indicator > .ind` rule forces `transform: + * scale(.85)` / `scale(1)` on every indicator to drive the show/hide + * transition. If we only rotate here, the browser has to interpolate + * between `scale(1)` and `rotate(360deg)` (two different transform + * functions), which produces a broken, barely-moving animation. So we + * include the scale explicitly in the keyframes and bump specificity + * with `!important` so the spinner always wins over the state rule. */ +.ind-spinner { + width: 48%; + height: 48%; + border: 3px solid currentColor; + border-right-color: transparent; + border-radius: 50%; + opacity: 0; + animation: ind-spin 0.9s linear infinite; +} + +/* Thinking dots: 3 soft pulsing dots while the model is composing a + * response. Apple-style cadence: each dot scales up and brightens in + * turn, staggered by ~160 ms. Per-dot animation is on the child, so + * the parent's scale(.85 → 1) show/hide transform composes cleanly. */ +.ind-thinking { + display: flex; + align-items: center; + justify-content: center; + gap: 8px; + width: 60%; + height: 60%; +} +.ind-thinking .dot { + width: 10px; + height: 10px; + border-radius: 50%; + background: currentColor; + opacity: 0.3; + animation: thinking-dot 1.25s ease-in-out infinite; +} +.ind-thinking .dot:nth-child(1) { animation-delay: 0s; } +.ind-thinking .dot:nth-child(2) { animation-delay: 0.16s; } +.ind-thinking .dot:nth-child(3) { animation-delay: 0.32s; } + +/* Bars: 5 vertical pills driven by --bar0..--bar4 CSS vars. */ +.ind-bars { + display: flex; + align-items: center; + gap: 5px; + height: 42%; +} +.ind-bars .bar { + width: 4px; + min-height: 4px; + border-radius: 3px; + background: currentColor; + opacity: 0.7; + --h: var(--bar0, 0); + height: calc(4px + var(--h) * 36px); + transition: height 0.08s ease-out, opacity 0.08s ease-out; +} +.ind-bars .bar:nth-child(1) { --h: var(--bar0); } +.ind-bars .bar:nth-child(2) { --h: var(--bar1); } +.ind-bars .bar:nth-child(3) { --h: var(--bar2); } +.ind-bars .bar:nth-child(4) { --h: var(--bar3); } +.ind-bars .bar:nth-child(5) { --h: var(--bar4); } +.ind-bars .bar { + opacity: calc(0.55 + 0.45 * var(--h)); +} + +/* Active indicator per state. + * + * Note: `ai-speaking` deliberately has NO indicator here. The orb + * itself becomes the indicator by pulsing on Reachy's voice (see the + * `--ai-audio-level` rules further down), which is a lot clearer and + * less confusing than reusing the mic-bars (which viewers would read + * as "you are speaking"). */ +.state-signed-out .ind-connect, +.state-authenticated .ind-mic, +.state-ready .ind-mic, +.state-connecting .ind-spinner, +.state-connected .ind-spinner, +.state-auto-selecting .ind-spinner, +.state-starting .ind-spinner, +.state-queued .ind-spinner, +.state-processing .ind-thinking, +.state-listening .ind-bars, +.state-user-speaking .ind-bars, +.state-ai-speaking .ind-voice, +.state-error .ind-error { + opacity: 1; + transform: scale(1); +} + +/* AI speaking indicator: speaker + two sound waves. + * + * Each wave pulses outward (opacity + stroke grow) with a quarter- + * beat offset so it reads as sound radiating out. Kept as a pure + * CSS animation so the icon is always visually alive even between + * syllables when --ai-audio-level momentarily dips. */ +.ind-voice .wave { + transform-origin: 50% 50%; + animation: voice-wave 1.35s ease-out infinite; +} +.ind-voice .wave-1 { animation-delay: 0s; } +.ind-voice .wave-2 { animation-delay: 0.35s; } + +/* Idle indicators: a chain-link "connect" glyph for the signed-out + * step (invites the user to authenticate with HF) and a microphone + * once a session is ready. Both picked up by the generic + * `.circle-indicator > svg.ind` sizing rule (60% × 60% of the slot) + * and rely on per-state opacity transitions for show / hide. */ +.ind-connect, +.ind-mic { + color: color-mix(in srgb, var(--glow, var(--accent)) 85%, white); +} + +/* ─── Caption below the circle ────────────────────────────────────────── */ + +/* The caption under the orb is meant to whisper, not shout: micro-label + * vibe, uppercase, letter-spaced, muted. Only appears for actionable / + * transitional states (see STATE_VIEWS). During a live conversation the + * orb alone carries the state so we collapse this row entirely. */ +.circle-caption { + margin: 0; + font-family: var(--font-mono); + font-size: 11px; + font-weight: 500; + letter-spacing: 0.14em; + text-transform: uppercase; + color: var(--text-faint); + min-height: 1.2em; + text-align: center; + opacity: 0.75; + transition: opacity 0.25s ease, color 0.25s ease, transform 0.25s ease, + min-height 0.25s ease; +} +.circle-caption.empty { + opacity: 0; + min-height: 0; + transform: translateY(-4px); + pointer-events: none; +} +.circle-caption.muted { + color: var(--text-faint); + opacity: 0.65; +} +.circle-caption.error { + color: var(--error); + opacity: 1; + letter-spacing: 0.08em; +} + +/* Warm, human line under the mono caption — sentence case, only while queued. + * The caption keeps the terse position; this reassures. */ +.circle-subcaption { + margin: 8px 0 0; + max-width: 32ch; + font-size: 13.5px; + line-height: 1.5; + color: var(--text-dim); + text-align: center; + text-wrap: balance; + transition: opacity 0.25s ease; +} +.circle-subcaption[hidden] { display: none; } + +/* Queue actions sit under the caption: "Join now" (primary, only when it's your + * turn) stacked above the quiet "Leave queue" escape hatch. */ +.queue-actions { + margin-top: 16px; + display: flex; + flex-direction: column; + align-items: center; + gap: 10px; +} +.queue-actions[hidden] { display: none; } + +/* "Join now": the one call to action in the queue flow, so it reads as inviting + * (filled accent) while everything around it stays quiet. */ +.join-queue-btn { + padding: 10px 26px; + font-family: var(--font-mono); + font-size: 12px; + font-weight: 600; + letter-spacing: 0.1em; + text-transform: uppercase; + color: #0b0b10; + background: var(--accent); + border: none; + border-radius: 999px; + cursor: pointer; + font-variant-numeric: tabular-nums; + transition: transform 0.12s ease, filter 0.2s ease; +} +.join-queue-btn:hover { filter: brightness(1.08); } +.join-queue-btn:active { transform: scale(0.97); } +.join-queue-btn:focus-visible { + outline: 2px solid var(--accent); + outline-offset: 3px; +} +.join-queue-btn[hidden] { display: none; } + +/* "Leave queue": a quiet outlined pill, understated so it reads as an escape + * hatch, not a CTA. */ +.leave-queue-btn { + padding: 7px 16px; + font-family: var(--font-mono); + font-size: 11px; + font-weight: 500; + letter-spacing: 0.12em; + text-transform: uppercase; + color: var(--text-faint); + background: transparent; + border: 1px solid color-mix(in srgb, var(--text-faint) 35%, transparent); + border-radius: 999px; + cursor: pointer; + transition: color 0.2s ease, border-color 0.2s ease, background 0.2s ease; +} +.leave-queue-btn:hover { + color: var(--text); + border-color: color-mix(in srgb, var(--text-faint) 60%, transparent); + background: color-mix(in srgb, var(--text-faint) 8%, transparent); +} +.leave-queue-btn:focus-visible { + outline: 2px solid var(--accent); + outline-offset: 2px; +} +.leave-queue-btn[hidden] { display: none; } + +/* ─── Tool-call toaster ───────────────────────────────────────────────── */ + +/* A small, non-interactive pill that appears below the circle when the + * model invokes a tool (move_head, play_move). Sits just under the + * state caption, collapses to zero height when no toast is active. */ +.tool-toast { + display: inline-flex; + align-items: center; + gap: 8px; + margin-top: 10px; + padding: 6px 12px; + border-radius: 999px; + border: 1px solid var(--border-strong); + background: color-mix(in srgb, var(--bg-elev-2) 78%, transparent); + color: var(--text-dim); + font-size: 12px; + font-weight: 500; + letter-spacing: 0.01em; + line-height: 1; + white-space: nowrap; + max-width: 80vw; + overflow: hidden; + text-overflow: ellipsis; + + opacity: 0; + transform: translateY(-4px) scale(0.96); + pointer-events: none; + transition: opacity 0.22s ease, transform 0.22s ease; +} +.tool-toast.visible { + opacity: 1; + transform: translateY(0) scale(1); +} +.tool-toast-icon { + width: 14px; + height: 14px; + flex: none; + color: color-mix(in srgb, var(--voice-tool) 80%, white); + animation: tool-toast-spin 3.2s linear infinite; + animation-play-state: paused; +} +.tool-toast.visible .tool-toast-icon { + animation-play-state: running; +} +.tool-toast-text { + display: inline-block; + overflow: hidden; + text-overflow: ellipsis; +} +@keyframes tool-toast-spin { + from { transform: rotate(0deg); } + to { transform: rotate(360deg); } +} + +/* ─── Side controls (mic / stop) ──────────────────────────────────────── */ + +/* Mic button + its radial noise-gate arc. The wrapper centres the button; the + arc SVG is an absolute overlay larger than the button, revealed with the + live session (it has no layout footprint, so the idle collapse is unaffected). */ +.mic-gate { + position: relative; + flex: none; + display: inline-flex; + align-items: center; + justify-content: center; +} +.mic-gate-arc { + position: absolute; + left: 50%; + top: 50%; + width: 80px; + height: 80px; + transform: translate(-50%, -50%); + pointer-events: none; /* only the hit-path below catches drags */ + opacity: 0; + transition: opacity 0.25s ease; +} +.orb-wrap.live .mic-gate-arc { opacity: 1; } +.mga-track { + stroke: rgba(255, 255, 255, 0.1); /* hairline; recedes until needed */ + stroke-width: 1.5; + stroke-linecap: round; +} +.mga-fill { + stroke: var(--accent-2); + stroke-width: 2; + stroke-linecap: round; + opacity: 0.85; + transition: stroke-dashoffset 0.06s linear; +} +/* Threshold setpoint: a bead riding the ring. White with a thin dark outline so + it stays legible over the moving fill; it recolors to cyan with a soft glow + the moment the live level crosses it (the gate opening). */ +.mga-handle { + fill: var(--text); + stroke: none; + transition: fill 0.2s ease, filter 0.2s ease; +} +.mic-gate.gate-open .mga-handle { + fill: var(--accent-2); + filter: drop-shadow(0 0 3px var(--accent-2)); +} +.mga-hit { + stroke: transparent; + stroke-width: 16; + pointer-events: none; /* enabled only while live (below) */ + cursor: pointer; + touch-action: none; +} +/* Only catch drags during a live call; when idle the arc is hidden and must + not steal clicks near the collapsed mic button / orb. */ +.orb-wrap.live .mga-hit { pointer-events: stroke; } + +.side-btn { + flex: none; + width: 52px; + height: 52px; + border-radius: 50%; + border: 1px solid var(--border-strong); + /* Brighter background so the buttons actually stand out against the + * deep-blue stage gradient; previous var(--bg-elev) was too close to + * the page bg to be readable. */ + background: color-mix(in srgb, var(--bg-elev-2) 80%, #2a2e3c); + color: var(--text); + display: inline-flex; + align-items: center; + justify-content: center; + cursor: pointer; + touch-action: manipulation; + box-shadow: 0 6px 16px rgba(0, 0, 0, 0.35), + inset 0 1px 0 rgba(255, 255, 255, 0.06); + /* Hidden by default: take up no space until the session is live. The + * `width: 0` collapse keeps the orb centered on the idle screen. */ + opacity: 0; + transform: scale(0.55); + width: 0; + padding: 0; + pointer-events: none; + overflow: hidden; + transition: opacity 0.25s ease, transform 0.25s ease, width 0.25s ease, + background 0.15s, color 0.15s, border-color 0.15s; +} +.side-btn:hover { + background: color-mix(in srgb, var(--bg-elev-2) 60%, #323746); + border-color: var(--text-dim); +} +.side-btn svg { + width: 22px; + height: 22px; + flex: none; +} +.side-btn .mic-off { display: none; } +.side-btn.muted { + color: #fff; + border-color: var(--error); + background: color-mix(in srgb, var(--error) 70%, #1a0e13); +} +.side-btn.muted .mic-on { display: none; } +.side-btn.muted .mic-off { display: block; } + +/* Stop button: subtle warm tint so "end" reads as destructive. */ +#stop-btn:hover { + color: #fff; + border-color: color-mix(in srgb, var(--error) 60%, var(--border-strong)); + background: color-mix(in srgb, var(--error) 25%, var(--bg-elev-2)); +} + +/* Reveal when the session is live: buttons flank the orb on a flex row. */ +.orb-wrap.live .side-btn { + opacity: 1; + transform: scale(1); + width: 52px; + pointer-events: auto; +} + +/* Disable every transition / animation during the first paint so the + * orb doesn't fade-and-scale in when the page loads. `main.ts` removes + * the class after one animation frame. */ +body.booting, +body.booting *, +body.booting *::before, +body.booting *::after { + transition: none !important; + animation-duration: 0s !important; + animation-delay: 0s !important; +} + +/* ─── Circle animation keyframes ──────────────────────────────────────── */ + +/* Slow, subtle breathing for "warm idle" states. */ +@keyframes breathe { + 0%, 100% { transform: translate(-50%, -50%) scale(1); opacity: 0.4; } + 50% { transform: translate(-50%, -50%) scale(1.06); opacity: 0.15; } +} + +/* Outer ring expanding and fading - conveys "I am producing audio". */ +@keyframes ring-out { + 0% { transform: translate(-50%, -50%) scale(1); opacity: 0.35; } + 100% { transform: translate(-50%, -50%) scale(1.18); opacity: 0; } +} + +/* Soft inner scale for the core while talking. */ +@keyframes core-breathe { + 0%, 100% { transform: scale(1); } + 50% { transform: scale(1.04); } +} + +/* Subtle glow throb used for "thinking" — dimmer than speaking. */ +@keyframes thinking { + 0%, 100% { transform: scale(1); opacity: 0.7; } + 50% { transform: scale(0.96); opacity: 0.45; } +} + +/* Individual dot pulse for the 3-dot processing indicator. */ +@keyframes thinking-dot { + 0%, 60%, 100% { transform: scale(0.7); opacity: 0.3; } + 30% { transform: scale(1.15); opacity: 1; } +} + +/* Sound-wave pulse: arc fades in, scales up slightly, fades out. + * The transform-origin is the speaker's center (roughly x=12), so + * the waves feel like they're emanating from the cone. */ +@keyframes voice-wave { + 0% { opacity: 0; transform: scale(0.7); } + 30% { opacity: 1; transform: scale(1); } + 70% { opacity: 0.2; transform: scale(1.12); } + 100% { opacity: 0; transform: scale(0.7); } +} + +@keyframes ind-spin { + /* Scale kept at 1 so we don't fight with the indicator's base + * show/hide transform (see `.ind-spinner` for the rationale). */ + from { transform: scale(1) rotate(0deg); } + to { transform: scale(1) rotate(360deg); } +} + +/* ─── State-specific colors ──────────────────────────────────────────── */ + +.circle.state-signed-out { --glow: #8b7dff; } +.circle.state-authenticated, +.circle.state-ready { --glow: #34d399; } +.circle.state-connecting, +.circle.state-connected, +.circle.state-auto-selecting, +.circle.state-starting { --glow: #facc15; } +/* Queued: a calm slate glow, distinct from connecting's active yellow — this is + * waiting, not working. The spinner turns slowly and the ring breathes. */ +.circle.state-queued { --glow: #94a3b8; } +.circle.state-queued .ind-spinner { animation-duration: 2.4s; } +.circle.state-queued .circle-ring { animation: breathe 2.6s ease-in-out infinite; } +/* Your turn: a slot is held for you — the orb warms to the accent and breathes a + * little quicker, an invitation to join. */ +.circle.state-your-turn { --glow: var(--accent); } +.circle.state-your-turn .circle-ring { animation: breathe 1.6s ease-in-out infinite; } +.circle.state-listening, +.circle.state-user-speaking { --glow: var(--listening); } +.circle.state-processing { --glow: var(--processing); } +.circle.state-ai-speaking { --glow: var(--speaking); } +.circle.state-error { --glow: var(--error); } + +/* Idle / ready: gentle breathing of the inner ring. Kept out of the + * `signed-out` state so the very first paint on page load stays quiet + * (the orb now shows the Reachy head silhouette, no need to also pulse). */ +.circle.state-authenticated .circle-ring, +.circle.state-ready .circle-ring { + animation: breathe 2.4s ease-in-out infinite; +} + +/* Connecting flows: subtle glow throb so the orb feels thoughtful + * while the session is being negotiated. Kept off `processing` on + * purpose - the 3 thinking dots already pulse, and layering another + * throb on top competes with them for attention. */ +.circle.state-connecting .circle-core, +.circle.state-connected .circle-core, +.circle.state-auto-selecting .circle-core, +.circle.state-starting .circle-core { + animation: thinking 1.4s ease-in-out infinite; +} + +/* User is speaking: the mic RMS drives scale + opacity via --audio-level. */ +.circle.state-user-speaking .circle-ring { + animation: none; + opacity: calc(0.25 + 0.55 * var(--audio-level)); + transform: translate(-50%, -50%) scale(calc(1 + 0.08 * var(--audio-level))); + transition: transform 0.08s linear, opacity 0.08s linear; +} +.circle.state-user-speaking .circle-ring-outer { + opacity: calc(0.1 + 0.35 * var(--audio-level)); + transform: translate(-50%, -50%) scale(calc(1 + 0.12 * var(--audio-level))); + transition: transform 0.08s linear, opacity 0.08s linear; +} +.circle.state-listening .circle-ring { + animation: breathe 2s ease-in-out infinite; +} + +/* Assistant is speaking: the whole orb breathes in sync with Reachy's + * voice instead of running a fixed timer. `--ai-audio-level` (0-1) is + * updated at display rate by AiLevelMonitor from the OpenAI output + * track, so every syllable visibly moves the core + outer ring. This + * reads instantly as "the orb is the voice" and completely avoids the + * ambiguity of bars-vs-mic the user flagged. */ +.circle.state-ai-speaking .circle-core { + animation: none; + transform: scale(calc(1 + 0.09 * var(--ai-audio-level, 0))); + transition: transform 0.08s linear; +} +.circle.state-ai-speaking .circle-ring { + animation: none; + opacity: calc(0.3 + 0.5 * var(--ai-audio-level, 0)); + transform: translate(-50%, -50%) scale(calc(1 + 0.05 * var(--ai-audio-level, 0))); + transition: transform 0.08s linear, opacity 0.08s linear; +} +.circle.state-ai-speaking .circle-ring-outer { + animation: none; + opacity: calc(0.15 + 0.55 * var(--ai-audio-level, 0)); + transform: translate(-50%, -50%) scale(calc(1 + 0.18 * var(--ai-audio-level, 0))); + transition: transform 0.08s linear, opacity 0.08s linear; +} +.circle.state-ai-speaking .circle-glow { + opacity: calc(0.35 + 0.45 * var(--ai-audio-level, 0)); + transition: opacity 0.08s linear; +} + +/* ─── Robot picker ────────────────────────────────────────────────────── */ + +.robot-picker { + width: min(420px, 92vw); + background: var(--bg-elev); + border: 1px solid var(--border); + border-radius: var(--radius-md); + padding: 14px 16px; + box-shadow: var(--shadow-soft); +} + +.picker-title { + margin: 0 0 10px; + font-size: 11px; + font-weight: 600; + letter-spacing: 0.1em; + text-transform: uppercase; + color: var(--text-faint); +} + +.robot-list { + display: flex; + flex-direction: column; + gap: 8px; +} + +.robot-card { + display: flex; + justify-content: space-between; + align-items: center; + padding: 12px 14px; + border: 1px solid var(--border); + border-radius: var(--radius-sm); + background: var(--bg-elev-2); + cursor: pointer; + transition: border-color 0.15s, background 0.15s; +} +.robot-card:hover { + border-color: var(--border-strong); +} +.robot-card.selected { + border-color: var(--border-strong); + background: var(--bg-elev); +} + +.robot-card .name { + font-weight: 600; + font-size: 14px; +} +.robot-card .id { + font-family: var(--font-mono); + font-size: 12px; + color: var(--text-faint); +} + +.robot-empty { + padding: 14px; + text-align: center; + color: var(--text-faint); + font-size: 13px; +} + +/* ─── Footer ──────────────────────────────────────────────────────────── */ + +.footer { + padding: 12px 28px 18px; + font-size: 11px; + letter-spacing: 0.02em; + color: var(--text-faint); + display: flex; + justify-content: center; + opacity: 0.55; + transition: opacity 0.25s ease; +} +.footer:hover { + opacity: 0.9; +} +.footer a { + color: inherit; + text-decoration: none; + border-bottom: 1px dotted currentColor; +} +.footer a:hover { + color: var(--text-dim); +} +/* While the webcam preview sits bottom-left, push the credit to the + * bottom-right so the two don't collide. */ +body.cam-on .footer { + justify-content: flex-end; +} + +/* ─── Modal ───────────────────────────────────────────────────────────── */ + +.modal { + border: 1px solid var(--border); + border-radius: var(--radius-md); + padding: 0; + background: var(--bg-elev); + color: var(--text); + width: min(500px, 92vw); + max-width: calc(100vw - 32px); + max-height: calc(100vh - 32px); + box-shadow: var(--shadow-soft); +} +.modal::backdrop { + background: rgba(8, 9, 13, 0.65); + backdrop-filter: blur(4px); +} + +.modal-content { + display: flex; + flex-direction: column; + gap: 16px; + padding: 22px 24px 20px; + overflow-x: hidden; +} + +#settings-modal { + width: min(940px, 94vw); +} + +#settings-modal .modal-content { + gap: 12px; + padding: 18px 20px 16px; +} + +.modal-header { + display: flex; + align-items: center; + justify-content: space-between; + margin-bottom: 4px; +} +.modal-header h2 { + margin: 0; + font-size: 16px; + font-weight: 600; + letter-spacing: 0.02em; +} + +.field[hidden] { display: none; } +.field { + display: flex; + flex-direction: column; + gap: 6px; + font-size: 13px; + font-weight: 500; + color: var(--text-dim); +} +.field > span { + color: var(--text); + font-weight: 600; + font-size: 12px; + letter-spacing: 0.04em; + text-transform: uppercase; +} +.field input, +.field select, +.field textarea { + font-family: inherit; + font-size: 14px; + color: var(--text); + background: var(--bg); + border: 1px solid var(--border); + border-radius: var(--radius-sm); + padding: 10px 12px; + outline: none; + transition: border-color 0.15s; + resize: vertical; +} +.field textarea { + min-height: 84px; +} + +#settings-modal .field { + gap: 5px; +} + +#settings-modal .field input, +#settings-modal .field select, +#settings-modal .field textarea { + padding: 8px 10px; +} + +#settings-modal .field textarea { + min-height: 68px; + resize: none; +} +.field input:focus, +.field select:focus, +.field textarea:focus { + border-color: var(--text-dim); +} +.field small { + color: var(--text-faint); + font-size: 12px; + line-height: 1.4; +} +.field small.error { color: var(--error); } +.field small code { + background: var(--bg); + padding: 1px 5px; + border-radius: 4px; + border: 1px solid var(--border); +} + +/* Noise gate: a header with a live value, a level meter, and a range slider + that shares the meter's horizontal (dB) axis. */ +.field-head { + display: flex; + align-items: baseline; + justify-content: space-between; +} +.field-value { + font-weight: 500; + font-size: 12px; + letter-spacing: 0; + text-transform: none; + color: var(--text-dim); +} +/* The slider and the level meter are one widget: the range input is overlaid + on the meter track (its native track is transparent), so the live-level fill + shows through behind the thumb and the thumb itself is the threshold. */ +.gate { + display: flex; + flex-direction: column; + gap: 4px; +} +.gate-track { + position: relative; + height: 14px; + display: flex; + align-items: center; +} +.gate-track::before { + /* the visible meter track */ + content: ""; + position: absolute; + left: 0; + right: 0; + height: 8px; + border-radius: 999px; + background: var(--bg); + border: 1px solid var(--border); +} +.gate-meter-fill { + position: absolute; + left: 1px; + top: 50%; + transform: translateY(-50%); + height: 6px; + width: 0; + border-radius: 999px; + background: var(--accent-2); + transition: width 0.06s linear; + pointer-events: none; +} +.gate-ends { + display: flex; + justify-content: space-between; + font-size: 11px; + color: var(--text-faint); + letter-spacing: 0; + text-transform: none; +} +/* The range input sits transparently on top of the meter track. */ +.gate-track input[type="range"] { + position: relative; + z-index: 1; + width: 100%; + margin: 0; + -webkit-appearance: none; + appearance: none; + padding: 0; + border: none; + background: transparent; + height: 14px; + cursor: pointer; +} +.gate-track input[type="range"]::-webkit-slider-runnable-track { + height: 14px; + background: transparent; +} +.gate-track input[type="range"]::-moz-range-track { + height: 14px; + background: transparent; +} +.gate-track input[type="range"]::-webkit-slider-thumb { + -webkit-appearance: none; + appearance: none; + width: 6px; + height: 18px; + border-radius: 3px; + background: var(--text); + border: 2px solid var(--bg-elev); + box-shadow: 0 0 0 1px var(--border-strong); +} +.gate-track input[type="range"]::-moz-range-thumb { + width: 6px; + height: 18px; + border-radius: 3px; + background: var(--text); + border: 2px solid var(--bg-elev); + box-shadow: 0 0 0 1px var(--border-strong); +} +.gate-track input[type="range"]:focus { border: none; } + +.modal-footer { + display: flex; + justify-content: space-between; + gap: 12px; + padding-top: 6px; +} + +.btn { + padding: 10px 16px; + border-radius: var(--radius-sm); + border: 1px solid var(--border-strong); + background: var(--bg-elev-2); + color: var(--text); + font-weight: 600; + font-size: 13px; + cursor: pointer; + transition: background 0.15s, border-color 0.15s, transform 0.05s; +} +.btn:hover { + border-color: var(--text); +} +.btn:active { + transform: translateY(1px); +} +.btn.primary { + background: var(--text); + border-color: var(--text); + color: var(--bg); +} +.btn.primary:hover { + background: #fff; + border-color: #fff; +} +.btn.ghost { + background: transparent; + border-color: var(--border); + color: var(--text-dim); +} +.btn.wide { + width: 100%; + padding: 12px 16px; +} +.btn[disabled] { + opacity: 0.45; + cursor: not-allowed; +} +.btn[disabled]:hover { + border-color: var(--border-strong); +} + +/* ─── Settings tabs ───────────────────────────────────────────────────── */ + +.tabs { + display: flex; + gap: 4px; + padding: 4px; + background: var(--bg); + border: 1px solid var(--border); + border-radius: var(--radius-sm); +} +.tab { + flex: 1; + padding: 8px 12px; + border: none; + background: transparent; + color: var(--text-dim); + font-family: inherit; + font-size: 13px; + font-weight: 600; + letter-spacing: 0.02em; + border-radius: calc(var(--radius-sm) - 3px); + cursor: pointer; + transition: background 0.15s, color 0.15s; +} +.tab:hover { + color: var(--text); +} +.tab.active { + background: var(--bg-elev-2); + color: var(--text); + box-shadow: 0 1px 2px rgba(0, 0, 0, 0.3); +} + +.tab-panels { + display: flex; + flex-direction: column; +} +.tab-panel { + display: flex; + flex-direction: column; + gap: 16px; +} +.tab-panel[hidden] { + display: none; +} + +/* Horizontal layout for Voice + Model. */ +.field-row { + display: grid; + grid-template-columns: 1fr 1fr; + gap: 12px; +} + +/* Section label inside the Settings panel (TTS / ASR / LLM). */ +.settings-group-title { + margin: 4px 0 -4px; + font-size: 12px; + font-weight: 600; + letter-spacing: 0.04em; + text-transform: uppercase; + color: var(--text-muted, #888); +} + +@media (min-width: 761px) { + #settings-modal .settings-grid { + display: grid; + grid-template-columns: minmax(0, 1fr) minmax(0, 1fr); + gap: 18px; + align-items: start; + } + + #settings-modal .settings-column { + display: flex; + flex-direction: column; + gap: 10px; + min-width: 0; + } + + #settings-modal .settings-group-title { + margin: 0 0 -2px; + } + + #settings-modal .field-row { + gap: 10px; + } + + #settings-modal .modal-footer { + padding-top: 2px; + } +} + +@media (max-width: 760px) { + #settings-modal { + width: min(500px, 92vw); + } + + #settings-modal .settings-grid, + #settings-modal .settings-column { + display: flex; + flex-direction: column; + gap: 16px; + } +} + +/* ─── Chat button + badge ────────────────────────────────────────────── */ + +#chat-btn { + position: relative; +} + +.chat-badge { + position: absolute; + top: 6px; + right: 6px; + width: 7px; + height: 7px; + border-radius: 50%; + background: var(--text); + border: 1.5px solid var(--bg-elev); + opacity: 0; + transform: scale(0); + transition: opacity 0.2s ease, transform 0.2s ease; + pointer-events: none; +} +.chat-badge.visible { + opacity: 1; + transform: scale(1); +} + +/* ─── Ephemeral bubble stack ─────────────────────────────────────────── */ + +.bubble-stack { + position: fixed; + /* Clear the topbar control row so the first bubble doesn't sit at the same + * height as the buttons. */ + top: 96px; + right: 28px; + width: min(300px, calc(100vw - 56px)); + display: flex; + flex-direction: column; + gap: 8px; + pointer-events: none; + z-index: 100; +} + +.bubble { + --bubble-dx: 10px; + pointer-events: auto; + padding: 10px 14px; + border-radius: var(--radius-md); + font-size: 13px; + line-height: 1.5; + border: 1px solid var(--border); + background: var(--bg-elev); + box-shadow: 0 4px 18px rgba(0, 0, 0, 0.32); + max-width: 100%; + word-break: break-word; + opacity: 0; + transform: translateX(var(--bubble-dx)); + transition: opacity 0.22s ease, transform 0.22s ease; +} +.bubble.in { + opacity: 1; + transform: translateX(0); +} +.bubble.out { + opacity: 0; + transform: translateX(var(--bubble-dx)); + pointer-events: none; + transition: opacity 0.3s ease, transform 0.3s ease; +} + +/* Surfaces stay neutral; the side they sit on plus the mono role label + * carry the distinction. No tinted fills — color is the orb's job. */ +.bubble.user { + --bubble-dx: -10px; + align-self: flex-start; +} +.bubble.assistant { + --bubble-dx: 10px; + align-self: flex-end; +} +.bubble.tool { + --bubble-dx: -10px; + align-self: flex-start; + display: flex; + align-items: center; + gap: 8px; + color: var(--text-dim); + font-family: var(--font-mono); + font-size: 12px; +} +.bubble.tool .bubble-tool-icon { + width: 13px; + height: 13px; + flex: none; + color: var(--voice-tool); +} + +.bubble-role { + font-family: var(--font-mono); + font-size: 10px; + font-weight: 500; + letter-spacing: 0.1em; + text-transform: uppercase; + margin-bottom: 4px; + opacity: 0.7; +} +.bubble.user .bubble-role { color: var(--voice-user); } +.bubble.assistant .bubble-role { color: var(--voice-assistant); } + +/* ─── Conversation history panel ─────────────────────────────────────── */ + +.chat-panel { + position: fixed; + inset: 0; + z-index: 200; + pointer-events: none; +} +.chat-panel-backdrop { + position: absolute; + inset: 0; + background: rgba(8, 9, 13, 0.45); + backdrop-filter: blur(6px); + -webkit-backdrop-filter: blur(6px); + opacity: 0; + transition: opacity 0.25s ease; +} +.chat-panel.open .chat-panel-backdrop { + opacity: 1; + pointer-events: auto; +} +.chat-panel-inner { + position: absolute; + top: 0; + right: 0; + bottom: 0; + width: min(360px, 90vw); + background: var(--bg-elev); + border-left: 1px solid var(--border); + display: flex; + flex-direction: column; + transform: translateX(100%); + transition: transform 0.28s cubic-bezier(0.32, 0.72, 0, 1); + pointer-events: auto; + box-shadow: -8px 0 32px rgba(0, 0, 0, 0.35); +} +.chat-panel.open .chat-panel-inner { + transform: translateX(0); +} +.chat-panel-header { + display: flex; + align-items: center; + justify-content: space-between; + padding: 18px 20px; + border-bottom: 1px solid var(--border); + flex: none; +} +.chat-panel-header h3 { + margin: 0; + font-size: 14px; + font-weight: 600; + letter-spacing: 0.01em; +} +.chat-history { + flex: 1; + overflow-y: auto; + padding: 16px 20px; + display: flex; + flex-direction: column; + gap: 10px; + scroll-behavior: smooth; +} +.chat-history::-webkit-scrollbar { width: 3px; } +.chat-history::-webkit-scrollbar-track { background: transparent; } +.chat-history::-webkit-scrollbar-thumb { background: var(--border-strong); border-radius: 3px; } + +.chat-empty { + flex: 1; + display: flex; + flex-direction: column; + align-items: center; + justify-content: center; + gap: 6px; + color: var(--text-faint); + padding: 48px 0; + opacity: 0.65; +} +.chat-empty svg { + margin-bottom: 6px; + opacity: 0.8; +} +.chat-empty-title { + font-family: var(--font-mono); + font-size: 11px; + font-weight: 500; + letter-spacing: 0.14em; + text-transform: uppercase; +} +.chat-empty-hint { + font-size: 12px; + color: var(--text-faint); + opacity: 0.7; +} + +/* ─── History messages ───────────────────────────────────────────────── */ + +.hist-msg { + display: flex; + flex-direction: column; + gap: 3px; + max-width: 88%; +} +.hist-msg.user { align-self: flex-start; } +.hist-msg.assistant { align-self: flex-end; } +.hist-msg.tool { align-self: flex-start; max-width: 100%; } + +.hist-role { + font-family: var(--font-mono); + font-size: 10px; + font-weight: 500; + letter-spacing: 0.1em; + text-transform: uppercase; + color: var(--text-faint); + padding: 0 2px; +} +.hist-msg.user .hist-role { color: color-mix(in srgb, var(--voice-user) 75%, var(--text-faint)); } +.hist-msg.assistant .hist-role { color: color-mix(in srgb, var(--voice-assistant) 75%, var(--text-faint)); } +.hist-msg.tool .hist-role { color: color-mix(in srgb, var(--voice-tool) 75%, var(--text-faint)); } + +.hist-body { + padding: 9px 12px; + border-radius: var(--radius-md); + font-size: 13px; + line-height: 1.5; + border: 1px solid var(--border); + background: var(--bg-elev-2); + word-break: break-word; +} +/* Bodies share one neutral surface; alignment + the mono role label do the + * distinguishing, so the panel reads as one quiet column. */ +.hist-msg.user .hist-body.partial { opacity: 0.65; } + +/* ─── Tool call history item ─────────────────────────────────────────── */ + +.hist-tool-header { + display: flex; + align-items: center; + gap: 8px; + padding: 8px 12px; + border-radius: var(--radius-md) var(--radius-md) 0 0; + background: color-mix(in srgb, var(--processing) 10%, var(--bg-elev-2)); + border: 1px solid color-mix(in srgb, var(--processing) 22%, var(--border)); + border-bottom: 1px solid color-mix(in srgb, var(--processing) 15%, var(--border)); + cursor: pointer; + color: var(--text); + font-family: inherit; + font-size: 13px; + font-weight: 500; + width: 100%; + text-align: left; + transition: background 0.15s; +} +.hist-tool-header:only-child { + border-radius: var(--radius-md); + border-bottom: 1px solid color-mix(in srgb, var(--processing) 22%, var(--border)); +} +.hist-tool-header:hover { + background: color-mix(in srgb, var(--processing) 16%, var(--bg-elev-2)); +} +.hist-tool-icon { + width: 13px; + height: 13px; + flex: none; + color: var(--processing); +} +.hist-tool-name { + font-family: var(--font-mono); + font-size: 12px; + color: var(--voice-tool); + font-weight: 600; +} +.hist-tool-chevron { + margin-left: auto; + width: 13px; + height: 13px; + color: var(--text-faint); + transition: transform 0.2s ease; + flex: none; +} +.hist-tool-header[aria-expanded="true"] .hist-tool-chevron { + transform: rotate(180deg); +} +.hist-tool-body { + padding: 10px 12px 12px; + border-radius: 0 0 var(--radius-md) var(--radius-md); + background: var(--bg); + border: 1px solid color-mix(in srgb, var(--processing) 22%, var(--border)); + border-top: none; + display: none; +} +.hist-tool-body.open { display: block; } +.hist-tool-label { + font-family: var(--font-mono); + font-size: 10px; + letter-spacing: 0.08em; + text-transform: uppercase; + color: var(--text-faint); + margin: 8px 0 4px; +} +.hist-tool-label:first-child { margin-top: 0; } +.hist-tool-block { + font-family: var(--font-mono); + font-size: 11px; + line-height: 1.65; + color: var(--text-dim); + white-space: pre-wrap; + word-break: break-word; + overflow-x: auto; +} +.hist-tool-output { color: var(--text); } + +/* ─── Phone layout ──────────────────────────────────────────────────────── + * On phones the floating bubble stream overlaps the orb and there isn't room + * for it, so we drop it entirely and rely on the conversation panel (opened + * from the top-right chat button) as the single place to read the transcript. + * The badge still pulses there when new messages arrive while it's closed. + * We also stack the mic / stop controls above and below the orb (instead of + * left/right) so the wide live row never overflows, and let the panel take + * the full width. */ +@media (max-width: 600px) { + .bubble-stack { + display: none; + } + + .topbar { + padding: 14px 16px; + } + + .stage { + padding: 16px 12px 24px; + } + + /* Stack vertically: mic above the orb, stop below it (DOM order is + * mic → circle → stop). */ + .orb-wrap { + flex-direction: column; + gap: 14px; + } + + .circle { + width: clamp(170px, 56vw, 240px); + } + + /* In the column layout the side controls must collapse by HEIGHT, not + * width, so they take no vertical space until the session is live. */ + .side-btn { + width: 44px; + height: 0; + transition: opacity 0.25s ease, transform 0.25s ease, height 0.25s ease, + background 0.15s, color 0.15s, border-color 0.15s; + } + .side-btn svg { + width: 19px; + height: 19px; + } + .orb-wrap.live .side-btn { + width: 44px; + height: 44px; + } + .mic-gate-arc { width: 70px; height: 70px; } + + /* Full-screen conversation on phones — feels more deliberate than a + * narrow slide-over. It already spans top-to-bottom (inset 0); make it + * span edge-to-edge too and drop the now-pointless border/shadow. */ + .chat-panel-inner { + width: 100vw; + border-left: none; + box-shadow: none; + } +} + +/* ─── About panel ───────────────────────────────────────────────────────── + * Opened from the (i) by the wordmark. Reuses the .modal shell. + * Strictly monochrome per DESIGN.md — the only "color" is the orb, never + * here. Machine identifiers (model IDs, role tags, usernames) ride Geist + * Mono; everything human stays Inter. */ +.about-modal { + /* Roomier on tablet/desktop; the 92vw cap keeps phones full-width. */ + width: min(680px, 92vw); + max-height: 88vh; +} +.about-modal .modal-content { + max-height: 88vh; + overflow-y: auto; + gap: 20px; + padding: 26px 30px 24px; +} +.about-modal .modal-content::-webkit-scrollbar { width: 3px; } +.about-modal .modal-content::-webkit-scrollbar-track { background: transparent; } +.about-modal .modal-content::-webkit-scrollbar-thumb { background: var(--border-strong); border-radius: 3px; } + +/* Links here are quiet: no permanent underline (the global dotted rule is + * too loud for a dense credit block), brighten + underline on hover only. */ +.about-modal a { + color: var(--text); + border-bottom: none; + text-decoration: none; + transition: color 0.15s ease; +} +.about-modal a:hover { + color: #fff; +} +.about-intro a:hover { + text-decoration: underline; + text-underline-offset: 2px; +} +.about-modal .ext { + width: 12px; + height: 12px; + flex: none; + opacity: 0.5; +} + +/* Title row: the demo name and the (i) read as one unit. */ +.ident-head { + display: flex; + align-items: center; + gap: 9px; +} +/* (i) trigger sits right after the title. A faint outline keeps it + * catchable without turning into a card; it fills in on hover. */ +.about-btn { + flex: none; + width: 34px; + height: 34px; + border-radius: 50%; + background: transparent; + border: 1px solid var(--border-strong); + color: var(--text-dim); + display: inline-flex; + align-items: center; + justify-content: center; + cursor: pointer; + touch-action: manipulation; + transition: background 0.15s ease, color 0.15s ease, border-color 0.15s ease; +} +.about-btn:hover { + background: var(--bg-elev); + border-color: var(--text-dim); + color: var(--text); +} +.about-btn svg { + width: 20px; + height: 20px; +} + +/* The mobile twin of (i) lives in the right-hand control cluster and is + * hidden on desktop (the in-title one shows there instead). */ +.about-btn-mobile { + display: none; +} + +/* ── Popup intro: a plain paragraph on the project + a repo link ── */ +.about-intro p { + margin: 0; + font-size: 14px; + line-height: 1.6; + color: var(--text-dim); +} +.about-repo { + display: inline-flex; + align-items: center; + gap: 5px; + margin-top: 12px; + font-size: 14px; + font-weight: 500; +} + +/* ── Corner identity (replaces the wordmark) ── + * A compact stack in the topbar: title, one-line blurb, two meta rows. + * Monochrome; only the role/name identifiers ride the machine typeface. */ +.ident { + display: flex; + flex-direction: column; + gap: 10px; + min-width: 0; +} +/* Reset the global underlined-anchor styling inside the identity block. */ +.ident a { + border-bottom: none; + color: inherit; + transition: color 0.15s ease; +} +.ident a:hover { + color: var(--text); + text-decoration: underline; + text-underline-offset: 2px; +} +.ident-title { + font-size: 22px; + font-weight: 600; + letter-spacing: 0.005em; + line-height: 1.1; + color: var(--text); +} +.ident-blurb { + margin: 0; + max-width: 48ch; + font-size: 14.5px; + line-height: 1.5; + font-weight: 400; + color: var(--text-dim); +} +.ident-meta { + display: flex; + flex-direction: column; + gap: 7px; + font-size: 14px; + font-weight: 400; + color: var(--text-dim); +} +.ident-row { + display: flex; + flex-wrap: wrap; + align-items: center; + gap: 4px 8px; +} +.ident-label { + font-family: var(--font-mono); + font-size: 11px; + font-weight: 500; + letter-spacing: 0.1em; + text-transform: uppercase; + color: var(--text-faint); +} + +/* Shared credit bits — now used in the corner identity block. */ +.sep { + color: var(--text-faint); + opacity: 0.6; +} +.hf-credit { + display: inline-flex; + align-items: center; + gap: 5px; +} +/* Brand marks keep their own color — a deliberate exception to the + * monochrome rule, for the HF and Cerebras logos only. */ +.hf-mark { + width: 14px; + height: 14px; + flex: none; + color: #ffd21e; +} +.cerebras-credit { + display: inline-flex; + align-items: center; + gap: 5px; +} +.cerebras-mark { + width: 14px; + height: 14px; + flex: none; +} +/* Usernames are identifiers, so they ride the machine typeface. */ +.handle { + font-family: var(--font-mono); + font-size: 13.5px; +} + +/* ── Pipeline (the signal flow) ── */ +.about-pipeline { + border-top: 1px solid var(--border); + padding-top: 16px; +} +.pipeline-title { + margin: 0 0 14px; + font-family: var(--font-mono); + font-size: 11px; + font-weight: 500; + letter-spacing: 0.14em; + text-transform: uppercase; + color: var(--text-faint); +} +.pipeline { + list-style: none; + margin: 0; + padding: 0; + position: relative; +} +/* One continuous rail threading every node, dot centers at x=8.5px. */ +.pipeline::before { + content: ""; + position: absolute; + left: 8px; + top: 7px; + bottom: 7px; + width: 1px; + background: var(--border-strong); +} +.pipeline > li { + position: relative; + padding-left: 30px; +} +/* Stages: solid node. Endpoints (you / orb): hollow node. */ +.pipe-stage::before { + content: ""; + position: absolute; + left: 5px; + top: 4px; + width: 7px; + height: 7px; + border-radius: 50%; + background: var(--text-dim); +} +.pipe-endpoint::before { + content: ""; + position: absolute; + left: 5px; + top: 3px; + width: 7px; + height: 7px; + border-radius: 50%; + border: 1px solid var(--text-faint); + background: var(--bg-elev); +} +.pipe-endpoint { + font-family: var(--font-mono); + font-size: 11px; + font-weight: 500; + letter-spacing: 0.12em; + text-transform: uppercase; + color: var(--text-faint); + padding-bottom: 12px; +} +.pipeline > li.pipe-endpoint:last-child { + padding-bottom: 0; +} +.pipe-stage { + padding-bottom: 16px; +} +.pipe-tag { + font-family: var(--font-mono); + font-size: 12px; + font-weight: 600; + letter-spacing: 0.08em; + color: var(--text); + margin-right: 9px; +} +.pipe-job { + font-size: 14px; + color: var(--text-dim); +} +/* Middot between the job and its model link, matching the separators + * used elsewhere. */ +.pipe-job::after { + content: "·"; + margin: 0 7px; + color: var(--text-faint); +} +.pipe-note { + color: var(--text-faint); +} +/* The Cerebras link stays as quiet as the note; brightens + underlines on hover. */ +.pipe-note a { + color: inherit; +} +.pipe-note a:hover { + color: var(--text); + text-decoration: underline; + text-underline-offset: 2px; +} +.pipe-model { + display: inline-flex; + align-items: center; + gap: 4px; + margin-top: 4px; + font-family: var(--font-mono); + font-size: 12px; + color: var(--text-dim); + word-break: break-all; +} +.pipe-model:hover { + color: #fff; + text-decoration: underline; + text-underline-offset: 2px; +} +.pipe-model .ext { + width: 11px; + height: 11px; +} + +/* "Interrupted" tag on an assistant reply the user barged in on. */ +.hist-note { + font-family: var(--font-mono); + font-size: 10px; + font-weight: 500; + letter-spacing: 0.1em; + text-transform: uppercase; + color: var(--text-faint); + padding: 0 2px; +} +.hist-msg.assistant .hist-note { + align-self: flex-end; +} +.hist-msg.interrupted .hist-body { + opacity: 0.7; +} + +/* Captured webcam frame shown in the transcript (camera tool result). */ +.hist-image { + display: block; + width: 100%; + max-width: 240px; + border-radius: var(--radius-sm); + border: 1px solid var(--border); + margin-top: 2px; +} + +/* ─── Tools panel ────────────────────────────────────────────────────────── + * Reuses the modal/field shell. Switches are monochrome (checked = near-white, + * the same high-contrast treatment as the primary button): color belongs to + * the voice, not the chrome. */ +.tools-intro { + margin: 0; + font-size: 13px; + line-height: 1.5; + color: var(--text-dim); +} +.tool-list { + display: flex; + flex-direction: column; +} +.tool-row { + display: flex; + align-items: center; + justify-content: space-between; + gap: 16px; + padding: 12px 0; +} +.tool-row-sep { + border-top: 1px solid var(--border); + margin-top: 4px; +} +.tool-info { + display: flex; + flex-direction: column; + gap: 3px; + min-width: 0; +} +.tool-name { + font-size: 14px; + font-weight: 600; + color: var(--text); +} +.tool-desc { + font-size: 12.5px; + color: var(--text-dim); +} +.tool-row.disabled .tool-name, +.tool-row.disabled .tool-desc { + opacity: 0.5; +} +.tool-hint { + display: block; + font-size: 12px; + line-height: 1.4; + color: var(--text-faint); +} +.tools-key { + margin: 0 0 4px; +} + +/* Toggle switch */ +.switch { + position: relative; + display: inline-flex; + flex: none; + width: 40px; + height: 24px; + cursor: pointer; +} +.switch input { + position: absolute; + inset: 0; + width: 100%; + height: 100%; + margin: 0; + opacity: 0; + cursor: pointer; +} +.switch-track { + position: absolute; + inset: 0; + border-radius: 999px; + background: var(--bg); + border: 1px solid var(--border-strong); + transition: background 0.15s, border-color 0.15s; +} +.switch-track::after { + content: ""; + position: absolute; + top: 50%; + left: 3px; + transform: translateY(-50%); + width: 16px; + height: 16px; + border-radius: 50%; + background: var(--text-dim); + transition: transform 0.18s ease, background 0.15s; +} +.switch input:checked + .switch-track { + background: var(--text); + border-color: var(--text); +} +.switch input:checked + .switch-track::after { + transform: translate(16px, -50%); + background: var(--bg); +} +.switch input:focus-visible + .switch-track { + outline: 2px solid var(--text-dim); + outline-offset: 2px; +} +.switch input:disabled { + cursor: not-allowed; +} +.switch input:disabled + .switch-track { + opacity: 0.5; +} + +/* ─── Webcam preview (camera tool) ────────────────────────────────────────── + * Floating self-view, bottom-left. Mirrored for the user; the frame sent to + * the model is drawn un-mirrored (see captureSnapshot). */ +.cam-pip { + position: fixed; + left: 20px; + bottom: 20px; + width: 280px; + aspect-ratio: 4 / 3; + border-radius: var(--radius-md); + overflow: hidden; + border: 1px solid var(--border-strong); + background: var(--bg-elev); + box-shadow: var(--shadow-soft); + z-index: 90; + opacity: 0; + transform: translateY(8px) scale(0.96); + pointer-events: none; + transition: opacity 0.22s ease, transform 0.22s ease; +} +.cam-pip.visible { + opacity: 1; + transform: none; +} +.cam-video { + display: block; + width: 100%; + height: 100%; + object-fit: cover; + transform: scaleX(-1); /* mirror the self-view only */ + background: var(--bg); +} +.cam-label { + position: absolute; + left: 8px; + bottom: 6px; + font-family: var(--font-mono); + font-size: 9.5px; + letter-spacing: 0.14em; + text-transform: uppercase; + color: var(--text); + opacity: 0.8; + text-shadow: 0 1px 3px rgba(0, 0, 0, 0.7); +} +.cam-flash { + position: absolute; + inset: 0; + background: #fff; + opacity: 0; + pointer-events: none; +} +.cam-pip.flash .cam-flash { + animation: cam-flash 0.4s ease; +} +@keyframes cam-flash { + 0% { opacity: 0; } + 12% { opacity: 0.85; } + 100% { opacity: 0; } +} +@media (max-width: 600px) { + /* Bottom-centred on phones. Auto margins centre it without touching the + * transform, so the slide/scale-in animation still works. */ + .cam-pip { + left: 0; + right: 0; + margin-inline: auto; + bottom: 16px; + width: min(188px, 52vw); + } + /* The credit would sit under the centred preview, so drop it while the + * camera is on. */ + body.cam-on .footer { + display: none; + } +} +@media (prefers-reduced-motion: reduce) { + .cam-pip { + transition: opacity 0.22s ease; + transform: none; + } + .cam-pip.flash .cam-flash { + animation: none; + } +} + +/* ─── Desktop type scale ──────────────────────────────────────────────────── + * A uniform step up (~+1px, title +2) for all UI text on larger screens. + * Scoped to min-width: 601px so it can't reach phones — the ≤600px layout + * keeps every size exactly as it was. */ +@media (min-width: 601px) { + .cam-label { font-size: 10.5px; } + + .bubble-role, + .hist-role { font-size: 11px; } + + .circle-caption, + .footer, + .chat-empty-title, + .hist-tool-block, + .ident-label, + .pipeline-title, + .pipe-endpoint { font-size: 12px; } + + .field > span, + .field small, + .bubble.tool, + .chat-empty-hint, + .hist-tool-name, + .pipe-tag, + .pipe-model, + .tool-hint { font-size: 13px; } + + .tool-desc { font-size: 13.5px; } + + .btn, + .bubble, + .field, + .hist-body, + .hist-tool-header, + .tools-intro { font-size: 14px; } + + .handle { font-size: 14.5px; } + + .about-intro p, + .about-repo, + .brand, + .chat-panel-header h3, + .field textarea, + .ident-meta, + .pipe-job, + .tool-name { font-size: 15px; } + + .ident-blurb { font-size: 15.5px; } + + .modal-header h2 { font-size: 17px; } + + .ident-title { font-size: 24px; } +} diff --git a/webui/realtime_static/ui/account.js b/webui/realtime_static/ui/account.js new file mode 100644 index 00000000..dc2d5b28 --- /dev/null +++ b/webui/realtime_static/ui/account.js @@ -0,0 +1,189 @@ +// @ts-check +/** + * Account — the HF login chip and the daily-limit modal. + * + * Reads `/api/me` to learn the current tier (anonymous / signed-in / PRO) and + * remaining daily talk-time, renders a sign-in pill or a signed-in chip with a + * small popover (tier, remaining, sign out, upgrade), and shows the limit modal + * when a conversation is refused or cut. The time metering itself lives in + * main.js (heartbeat loop) + the server; this module is just the surface. + * + * Inert unless the deploy is in LB mode (`/api/me` → `{enabled:true}`). + */ + +import { $, escHtml } from "./dom.js"; + +const PRO_URL = "https://huggingface.co/subscribe/pro"; + +// Official multi-color Hugging Face logo, used in the badge + sign-in CTA. +const HF_MARK = ``; + +/** @param {number} sec @returns {string} "m:ss" */ +function fmt(sec) { + const s = Math.max(0, Math.round(sec)); + return `${Math.floor(s / 60)}:${String(s % 60).padStart(2, "0")}`; +} + +export class Account { + constructor() { + /** @type {HTMLElement} */ + this._root = $("#account"); + /** @type {HTMLDialogElement} */ + this._modal = $("#limit-modal"); + this._modalTitle = $("#limit-title"); + this._modalMsg = $("#limit-msg"); + this._modalNote = $("#limit-note"); + /** @type {HTMLAnchorElement} */ + this._modalCta = /** @type {any} */ ($("#limit-cta")); + + /** @type {{enabled:boolean, auth?:boolean, loggedIn?:boolean, username?:string, avatar?:string, tier?:string, remainingSec?:number|null, limitSec?:number|null, loginUrl?:string|null, logoutUrl?:string|null}} */ + this._me = { enabled: false }; + this._popoverOpen = false; + + $("#limit-close").addEventListener("click", () => this._modal.close()); + this._modal.addEventListener("click", (e) => { + if (e.target === this._modal) this._modal.close(); + }); + // Close the popover on an outside click. + document.addEventListener("click", (e) => { + if (this._popoverOpen && !this._root.contains(/** @type {Node} */ (e.target))) { + this._closePopover(); + } + }); + } + + get tier() { + return this._me.tier || "anon"; + } + + /** Fetch `/api/me` and (re)render the chip. Safe to call repeatedly (load, + * after the OAuth redirect, after a conversation ends). */ + async refresh() { + try { + const res = await fetch("api/me"); + this._me = res.ok ? await res.json() : { enabled: false }; + } catch { + this._me = { enabled: false }; + } + this._render(); + } + + _render() { + const me = this._me; + if (!me.enabled) { + this._root.hidden = true; + this._root.innerHTML = ""; + return; + } + this._root.hidden = false; + + if (!me.loggedIn) { + // Signed-out: a sign-in pill (only when OAuth is actually available). + if (me.auth && me.loginUrl) { + this._root.innerHTML = ``; + } else { + this._root.innerHTML = ""; + this._root.hidden = true; + } + return; + } + + // Signed-in: avatar + handle chip that toggles a popover. + const isPro = me.tier === "pro"; + // Org members get unlimited usage too, but aren't PRO — don't brand them so. + const isUnlimited = isPro || me.tier === "org"; + const avatar = me.avatar + ? `` + : ``; + const remaining = + isUnlimited || me.remainingSec == null + ? "Unlimited" + : `${fmt(me.remainingSec)} left today`; + const tierLabel = isPro ? "PRO" : isUnlimited ? "Team" : "Free"; + + this._root.innerHTML = ` + + `; + + const chip = $("#account-chip"); + chip.addEventListener("click", (e) => { + e.stopPropagation(); + this._popoverOpen ? this._closePopover() : this._openPopover(); + }); + } + + _openPopover() { + const pop = document.getElementById("account-pop"); + const chip = document.getElementById("account-chip"); + if (!pop || !chip) return; + pop.hidden = false; + chip.setAttribute("aria-expanded", "true"); + this._popoverOpen = true; + } + + _closePopover() { + const pop = document.getElementById("account-pop"); + const chip = document.getElementById("account-chip"); + if (pop) pop.hidden = true; + if (chip) chip.setAttribute("aria-expanded", "false"); + this._popoverOpen = false; + } + + /** + * Show the limit modal for a tier — used both when a conversation is refused + * at start (402) and when a live one is cut (heartbeat `expired`). + * @param {string} [tier] + */ + showLimit(tier = this.tier) { + const canSignIn = this._me.auth && this._me.loginUrl; + this._modalTitle.textContent = "Thanks for chatting!"; + if (tier === "anon") { + this._modalMsg.textContent = + "Guest conversations run for 5 minutes. Sign in with Hugging Face to get 10 minutes a day for free, and PRO members chat with no limit at all."; + this._modalNote.textContent = "Your free minutes refresh tomorrow."; + if (canSignIn) { + this._modalCta.innerHTML = `${HF_MARK}Sign in with Hugging Face`; + this._modalCta.href = /** @type {string} */ (this._me.loginUrl); + this._modalCta.hidden = false; + } else { + this._modalCta.hidden = true; + } + } else { + // Signed-in, non-PRO. + this._modalMsg.textContent = + "You've enjoyed your 10 minutes for today. Go PRO for unlimited conversations and to support open source AI."; + this._modalNote.textContent = "Or come back tomorrow. Your minutes reset daily."; + this._modalCta.innerHTML = "Upgrade to PRO"; + this._modalCta.href = PRO_URL; + this._modalCta.hidden = false; + } + if (!this._modal.open) this._modal.showModal(); + } + + /** Show a warm "we're at capacity" message when even the waiting line is full. + * Reuses the limit modal shell; no call-to-action, just reassurance. */ + showBusy() { + this._modalTitle.textContent = "Hugged to the limit 🤗"; + this._modalMsg.textContent = + "Every slot and the whole line are full right now. Too much love! Grab a coffee and pop back in a minute."; + this._modalNote.textContent = "A spot usually opens up within a few minutes."; + this._modalCta.hidden = true; + if (!this._modal.open) this._modal.showModal(); + } +} diff --git a/webui/realtime_static/ui/chat.js b/webui/realtime_static/ui/chat.js new file mode 100644 index 00000000..5a674ef9 --- /dev/null +++ b/webui/realtime_static/ui/chat.js @@ -0,0 +1,425 @@ +// @ts-check +/** + * ChatView — owns the whole conversation surface: the slide-in history panel, + * the ephemeral on-orb bubbles, and all the transcript/tool/streaming + * bookkeeping. main.js wires the realtime client's events straight to the + * `on*` methods here and otherwise doesn't touch chat state. + * + * Two parallel surfaces share one shape (see `_buildMessageEl`): + * - ephemeral bubbles (`.bubble` / `.bubble-*`) fade on a timer + * - persistent history (`.hist-msg` / `.hist-*`) the durable panel log + * + * Keying: + * - user transcripts by the server's `item_id` — a speculative continuation + * REUSES it, so both segments land in one row/bubble; deltas are CUMULATIVE + * (each carries the full sentence so far), so we replace text wholesale. + * - assistant transcripts by `response_id`, so a cancelled speculative reply + * can be marked interrupted without erasing what was already shown. + */ + +import { $, escHtml, DEBUG } from "./dom.js"; + +const WRENCH_PATH = ``; +const CHAT_BUBBLE_SVG = ``; +const EMPTY_STATE_HTML = `
${CHAT_BUBBLE_SVG}No messages yetTap the orb and start talking
`; + +export class ChatView { + constructor() { + /** @type {HTMLButtonElement} */ + this._chatBtn = $("#chat-btn"); + /** @type {HTMLSpanElement} */ + this._chatBadge = $("#chat-badge"); + /** @type {HTMLDivElement} */ + this._chatPanel = $("#chat-panel"); + /** @type {HTMLDivElement} */ + this._chatPanelBackdrop = $("#chat-panel-backdrop"); + /** @type {HTMLButtonElement} */ + this._chatPanelClose = $("#chat-panel-close"); + /** @type {HTMLDivElement} */ + this._chatHistory = $("#chat-history"); + /** @type {HTMLDivElement} */ + this._bubbleStack = $("#bubble-stack"); + + this._panelOpen = false; + this._scrollQueued = false; + + // ── User transcript state (keyed by item_id) ─────────────────────────── + /** @type {Map} */ + this._userHistByItem = new Map(); + /** @type {HTMLElement | null} */ + this._activeUserBubble = null; + this._activeUserItemId = ""; + // Monotonic counter for synthesizing unique keys when the server omits an + // item_id / response_id, so id-less messages never collapse onto each other. + this._anonSeq = 0; + + // ── Assistant transcript state (keyed by response_id) ────────────────── + /** @type {Map} */ + this._asstByResp = new Map(); + + // ── Ephemeral bubble auto-dismiss ────────────────────────────────────── + // Per-element expiry (epoch ms). A bubble fades once its expiry passes — + // but only in stack order (see _reapBubbles). Refreshing the expiry keeps a + // bubble alive while it updates (e.g. the live user utterance). + /** @type {WeakMap} */ + this._bubbleExpiry = new WeakMap(); + // Single pending reaper handle: one timer for the whole stack (not one per + // bubble) so dismissal is strictly oldest-first regardless of per-bubble delays. + this._reaperHandle = 0; + + this._chatBtn.addEventListener("click", () => (this._panelOpen ? this._closePanel() : this._openPanel())); + this._chatPanelClose.addEventListener("click", () => this._closePanel()); + this._chatPanelBackdrop.addEventListener("click", () => this._closePanel()); + document.addEventListener("keydown", (e) => { + if (e.key === "Escape" && this._panelOpen) this._closePanel(); + }); + } + + // ── Panel ─────────────────────────────────────────────────────────────── + + _openPanel() { + this._panelOpen = true; + this._chatPanel.classList.add("open"); + this._chatBadge.classList.remove("visible"); + this._scrollToBottom(); + } + + _closePanel() { + this._panelOpen = false; + this._chatPanel.classList.remove("open"); + } + + // Coalesce scroll-to-bottom: a burst of cumulative transcript deltas would + // otherwise queue one rAF per delta, all writing the same scrollTop. + _scrollToBottom() { + if (!this._panelOpen || this._scrollQueued) return; + this._scrollQueued = true; + requestAnimationFrame(() => { + this._scrollQueued = false; + this._chatHistory.scrollTop = this._chatHistory.scrollHeight; + }); + } + + _markUnread() { + if (this._panelOpen) { + this._scrollToBottom(); + return; + } + this._chatBadge.classList.add("visible"); + } + + // ── Shared rendering ────────────────────────────────────────────────────── + + /** + * Build a role-labelled message element. Ephemeral bubbles and persistent + * history rows share the same shape and differ only in their class prefix + * (`bubble`/`bubble-*` vs `hist-msg`/`hist-*`). + * @param {{ container: string, prefix: string, role: "user"|"assistant", text: string, partial?: boolean }} o + * @returns {HTMLElement} + */ + _buildMessageEl({ container, prefix, role, text, partial = false }) { + const el = document.createElement("div"); + el.className = `${container} ${role}`; + const label = role === "user" ? "You" : "Assistant"; + el.innerHTML = `
${label}
${escHtml(text)}
`; + return el; + } + + // ── Ephemeral bubbles ─────────────────────────────────────────────────── + + /** @param {"user"|"assistant"|"tool"} role @param {string} text @returns {HTMLElement} */ + _spawnBubble(role, text) { + let el; + if (role === "tool") { + el = document.createElement("div"); + el.className = "bubble tool"; + el.innerHTML = `${WRENCH_PATH}${escHtml(text)}`; + } else { + el = this._buildMessageEl({ container: "bubble", prefix: "bubble", role, text }); + } + this._bubbleStack.appendChild(el); + // Cap the stack at 3, but never evict the bubble the caller is still + // actively updating (the live user bubble) — drop the next-oldest instead. + const visible = /** @type {HTMLElement[]} */ ([...this._bubbleStack.querySelectorAll(".bubble:not(.out)")]); + if (visible.length > 3) { + this._dismissBubble(visible.find((b) => b !== this._activeUserBubble) ?? visible[0]); + } + requestAnimationFrame(() => el.classList.add("in")); + return el; + } + + /** @param {HTMLElement} el @param {string} text */ + _updateBubbleText(el, text) { + const t = el.querySelector(".bubble-body"); + if (t) t.textContent = text; + } + + /** @param {HTMLElement} el */ + _dismissBubble(el) { + if (!el || el.classList.contains("out")) return; // idempotent + this._bubbleExpiry.delete(el); + el.classList.remove("in"); + el.classList.add("out"); + const remove = () => el.remove(); + el.addEventListener("transitionend", remove, { once: true }); + // Fallback: transitionend never fires if the bubble's visual state didn't + // change (dismissed pre-paint) or the tab is backgrounded. The transition + // is 0.3s, so force removal a little after. + setTimeout(remove, 400); + } + + /** + * Fade bubbles whose expiry has passed — strictly oldest-first. We walk the + * stack top (oldest) to bottom (newest) and stop at the first bubble still + * alive: nothing newer may leave while an older bubble is still on screen. A + * bubble that keeps updating pushes its own expiry forward, so it (and + * everything behind it) stays put until it finally goes quiet. + */ + _reapBubbles() { + this._reaperHandle = 0; + const now = Date.now(); + const visible = /** @type {HTMLElement[]} */ ([...this._bubbleStack.querySelectorAll(".bubble:not(.out)")]); + let nextWake = Infinity; + for (const el of visible) { + const exp = this._bubbleExpiry.get(el) ?? now; // no expiry recorded → treat as due + if (exp <= now) { + this._dismissBubble(el); + } else { + // Oldest survivor isn't due yet; stop so nothing newer leaves before it. + nextWake = exp; + break; + } + } + if (nextWake !== Infinity) { + this._reaperHandle = setTimeout(() => this._reapBubbles(), Math.max(50, nextWake - Date.now())); + } + } + + /** + * (Re)arm a bubble's auto-dismiss by pushing its expiry out by `delay`. + * Calling it again resets the countdown — so a bubble that keeps updating + * stays on screen and only fades once it goes quiet. Removal is ordered by the + * shared reaper, so the oldest bubble always disappears first. + * @param {HTMLElement} el @param {number} [delay] + */ + _bumpDismiss(el, delay = 4000) { + this._bubbleExpiry.set(el, Date.now() + delay); + if (!this._reaperHandle) this._reaperHandle = setTimeout(() => this._reapBubbles(), delay); + } + + // ── History ─────────────────────────────────────────────────────────────── + + /** Render the empty-state placeholder into the history panel. */ + renderEmptyState() { + this._chatHistory.innerHTML = EMPTY_STATE_HTML; + } + + /** Reset the panel to the empty state and clear the unread badge. */ + clear() { + this.renderEmptyState(); + this._chatBadge.classList.remove("visible"); + } + + /** @param {"user"|"assistant"} role @param {string} text @param {boolean} partial @returns {HTMLElement} */ + _appendHistMsg(role, text, partial) { + const empty = this._chatHistory.querySelector(".chat-empty"); + if (empty) empty.remove(); + const el = this._buildMessageEl({ container: "hist-msg", prefix: "hist", role, text, partial }); + this._chatHistory.appendChild(el); + this._scrollToBottom(); + return el; + } + + /** @param {HTMLElement | null} el @param {string} text @param {boolean} partial */ + _updateHistMsg(el, text, partial) { + if (!el) return; + const body = /** @type {HTMLElement | null} */ (el.querySelector(".hist-body")); + if (!body) return; + body.textContent = text; + body.classList.toggle("partial", partial); + this._scrollToBottom(); + } + + /** + * Append a tool-call row to the conversation. We only add it once the tool + * has run, so the expandable toggle carries BOTH the call input and its result. + * @param {string} name @param {string} argsJson @param {string} output + */ + _appendHistTool(name, argsJson, output) { + const empty = this._chatHistory.querySelector(".chat-empty"); + if (empty) empty.remove(); + let pretty = argsJson; + try { pretty = JSON.stringify(JSON.parse(argsJson), null, 2); } catch {} + const el = document.createElement("div"); + el.className = "hist-msg tool"; + el.innerHTML = ` +
Tool call
+ +
+
Input
+
${escHtml(pretty)}
+
Output
+
${escHtml(output || "(no output)")}
+
+ `; + const header = /** @type {HTMLButtonElement} */ (el.querySelector(".hist-tool-header")); + const body = /** @type {HTMLDivElement} */ (el.querySelector(".hist-tool-body")); + header.addEventListener("click", () => { + const expanded = header.getAttribute("aria-expanded") === "true"; + header.setAttribute("aria-expanded", String(!expanded)); + body.classList.toggle("open", !expanded); + }); + this._chatHistory.appendChild(el); + this._scrollToBottom(); + } + + /** Tag an assistant history row as interrupted (user barged in mid-reply). + * @param {HTMLElement | null} hist */ + _markHistInterrupted(hist) { + if (!hist || hist.querySelector(".hist-note")) return; + hist.classList.add("interrupted"); + const note = document.createElement("div"); + note.className = "hist-note"; + note.textContent = "Interrupted"; + hist.appendChild(note); + } + + /** Render a captured webcam frame in the transcript (the camera tool result). + * @param {string} dataUrl */ + _appendHistImage(dataUrl) { + const empty = this._chatHistory.querySelector(".chat-empty"); + if (empty) empty.remove(); + const el = document.createElement("div"); + el.className = "hist-msg tool"; + el.innerHTML = `
Snapshot
Webcam snapshot sent to the model`; + const img = /** @type {HTMLImageElement} */ (el.querySelector("img")); + img.src = dataUrl; + this._chatHistory.appendChild(el); + this._scrollToBottom(); + } + + /** + * Reset all streaming bookkeeping for session start / teardown. Pass + * `dismiss` to also fade any bubbles still on screen. + * @param {{ dismiss?: boolean }} [opts] + */ + reset(opts) { + if (opts?.dismiss) { + if (this._activeUserBubble) this._dismissBubble(this._activeUserBubble); + for (const { bubble } of this._asstByResp.values()) this._dismissBubble(bubble); + } + this._userHistByItem.clear(); + this._activeUserBubble = null; + this._activeUserItemId = ""; + this._asstByResp.clear(); + } + + // ── Client event handlers ───────────────────────────────────────────────── + + /** + * A streamed transcript delta (user or assistant). + * @param {{ role: "user" | "assistant"; text: string; partial: boolean; itemId?: string; responseId?: string }} d + */ + onTranscript(d) { + if (DEBUG) console.debug(`[ui] transcript role=${d.role} partial=${d.partial} item=${d.itemId} resp=${d.responseId} text=${JSON.stringify(d.text)}`); + + if (d.role === "user") { + // Group by item_id: a speculative continuation reuses the same id, so it + // updates the same row/bubble. A missing id falls back to the active item + // (same utterance) or a fresh unique key, never a shared sentinel that + // would collapse distinct turns into one row. + const id = d.itemId || this._activeUserItemId || `_u${++this._anonSeq}`; + const text = d.text; + + let hist = this._userHistByItem.get(id); + if (!hist) { + hist = this._appendHistMsg("user", text, d.partial); + this._userHistByItem.set(id, hist); + } else { + this._updateHistMsg(hist, text, d.partial); + } + + // One ephemeral bubble per active item. Purely timer-based: the timer is + // refreshed on every delta, so it stays while the user keeps talking and + // fades a few seconds after they stop — no dependency on a response ever + // arriving, so it can never get stuck. + if (this._activeUserItemId !== id || !this._activeUserBubble) { + this._activeUserBubble = this._spawnBubble("user", text); + this._activeUserItemId = id; + } else { + this._updateBubbleText(this._activeUserBubble, text); + } + this._bumpDismiss(this._activeUserBubble, 6000); + this._markUnread(); + } else if (d.role === "assistant") { + // Assistant transcript arrives once, as the full text, keyed by + // response_id so a cancelled speculative response can be removed later. A + // missing id gets a unique key so two id-less replies never collide. + const rid = d.responseId || `_a${++this._anonSeq}`; + const entry = this._asstByResp.get(rid); + if (!entry) { + const bubble = this._spawnBubble("assistant", d.text); + this._asstByResp.set(rid, { bubble, hist: this._appendHistMsg("assistant", d.text, false) }); + } else { + this._updateBubbleText(entry.bubble, d.text); + this._updateHistMsg(entry.hist, d.text, false); + } + this._markUnread(); + } + } + + /** + * A response closed (completed or cancelled). + * @param {{ responseId: string; status: string; audible?: boolean; transcript?: string }} detail + */ + onResponseFinished(detail) { + const { responseId, status, audible, transcript } = detail; + if (DEBUG) console.debug(`[ui] response-finished resp=${responseId} status=${status} audible=${audible} known=${this._asstByResp.has(responseId)}`); + // Without an id we can't target a specific response; the bubble will + // auto-dismiss on its own timer regardless. + if (!responseId) return; + const entry = this._asstByResp.get(responseId); + + if (status === "cancelled") { + // Keep every transcript that was received — mark it interrupted rather + // than erasing it. If the `*.transcript.done` never fired, build the row + // from the text carried in response.done. + let hist = entry?.hist ?? null; + if (!hist && transcript) { + hist = this._appendHistMsg("assistant", transcript, false); + } else if (hist && transcript) { + this._updateHistMsg(hist, transcript, false); + } + if (hist) this._markHistInterrupted(hist); + if (entry?.bubble) this._bumpDismiss(entry.bubble, 6000); + this._asstByResp.delete(responseId); + return; + } + + // Any other terminal close (completed / failed / incomplete / …): just + // release the map entry. The bubble already auto-dismisses on its timer and + // the history row persists as the conversation log. Crucially we do NOT + // touch user state here — that lifecycle is fully independent. + if (entry?.bubble) this._bumpDismiss(entry.bubble, 6000); + this._asstByResp.delete(responseId); + } + + /** The model called a tool — show an ephemeral "running" bubble. + * @param {string} name */ + onToolCall(name) { + this._bumpDismiss(this._spawnBubble("tool", name)); + this._markUnread(); + } + + /** The tool finished — append its call+result row (and any captured image). + * @param {string} name @param {string} argsJson @param {string} output @param {string} [image] */ + onToolResult(name, argsJson, output, image) { + this._appendHistTool(name, argsJson, output); + if (image) this._appendHistImage(image); // show the captured frame below the call + this._markUnread(); + } +} diff --git a/webui/realtime_static/ui/dom.js b/webui/realtime_static/ui/dom.js new file mode 100644 index 00000000..d5a645a0 --- /dev/null +++ b/webui/realtime_static/ui/dom.js @@ -0,0 +1,37 @@ +// @ts-check +/** Small shared helpers used across the UI modules: a strict query selector, + * HTML escaping for text we drop into innerHTML, error-string trimming, and + * the opt-in debug flag. */ + +/** Opt-in tracing: `localStorage.setItem("s2s.debug", "1")` then reload. */ +export const DEBUG = (() => { + try { + return localStorage.getItem("s2s.debug") === "1"; + } catch { + return false; + } +})(); + +/** + * Query a single element, throwing if it's missing (so a broken selector fails + * loudly at startup rather than as a later null-deref). + * @template {HTMLElement} T + * @param {string} selector + * @returns {T} + */ +export function $(selector) { + const el = document.querySelector(selector); + if (!el) throw new Error(`Missing element: ${selector}`); + return /** @type {T} */ (el); +} + +/** @param {string} s @returns {string} */ +export function escHtml(s) { + return s.replace(/&/g, "&").replace(//g, ">"); +} + +/** Trim a long error message to fit the orb caption. @param {string} text */ +export function truncateError(text) { + if (text.length <= 90) return text; + return text.slice(0, 87) + "…"; +} diff --git a/webui/realtime_static/worklets/audio-playback.js b/webui/realtime_static/worklets/audio-playback.js new file mode 100644 index 00000000..474020fc --- /dev/null +++ b/webui/realtime_static/worklets/audio-playback.js @@ -0,0 +1,171 @@ +// @ts-check +/** + * AudioWorkletProcessor that plays back Float32 mono samples received from + * the main thread, upsampling whatever incoming rate the server uses + * (typically 24 kHz PCM16) to the AudioContext rate (typically 48 kHz). + * + * Lifecycle / messaging: + * + * main -> worklet: + * { kind: "config", inputRate: 24000 } one-shot at startup + * { kind: "audio", samples: Float32Array } (transferable) per chunk + * { kind: "clear" } wipe queue (barge-in) + * + * worklet -> main: + * { kind: "stats", queuedMs, played } every ~250 ms + * { kind: "underrun" } every time the queue + * runs dry mid-playback + * + * Underrun strategy: output silence. We do NOT hold the last sample (that + * tends to produce audible clicks/buzzes when long gaps appear between + * TTS chunks). A short ramp-out + ramp-in at boundaries would be nicer but + * the server's 30 ms cadence makes underruns visible only at end of turn. + */ + +const STATS_INTERVAL_FRAMES = 12000; +const FADE_FRAMES = 32; + +class AudioPlaybackProcessor extends AudioWorkletProcessor { + constructor() { + super(); + this._inputRate = 24000; + this._stepRatio = this._inputRate / sampleRate; + this._queue = []; + this._readIdx = 0; + this._fracPos = 0; + this._playing = false; + this._framesSinceStats = 0; + this._totalPlayed = 0; + this._fadeIn = 0; + this._fadeOut = 0; + this._lastSample = 0; + + this.port.onmessage = (e) => { + const data = e.data; + if (!data || typeof data !== "object") return; + switch (data.kind) { + case "config": + if (typeof data.inputRate === "number" && data.inputRate > 0) { + this._inputRate = data.inputRate; + this._stepRatio = this._inputRate / sampleRate; + } + break; + case "audio": + if (data.samples instanceof Float32Array && data.samples.length > 0) { + this._queue.push(data.samples); + if (!this._playing) { + this._playing = true; + this._fadeIn = FADE_FRAMES; + this._fadeOut = 0; + } + } + break; + case "clear": + this._queue.length = 0; + this._readIdx = 0; + this._fracPos = 0; + this._fadeOut = FADE_FRAMES; + break; + } + }; + } + + _queuedSamples() { + let total = -this._readIdx; + for (const buf of this._queue) total += buf.length; + return Math.max(0, total); + } + + /** Linear-interp read at the current fractional position. */ + _readInterpolated() { + if (this._queue.length === 0) return null; + const head = this._queue[0]; + const idx = this._readIdx; + const frac = this._fracPos; + + let a = head[idx]; + let b; + if (idx + 1 < head.length) { + b = head[idx + 1]; + } else if (this._queue.length > 1) { + b = this._queue[1][0]; + } else { + b = a; + } + return a + (b - a) * frac; + } + + /** Advance the read position by `stepRatio`; pop consumed buffers. */ + _advance() { + this._fracPos += this._stepRatio; + while (this._fracPos >= 1) { + this._fracPos -= 1; + this._readIdx += 1; + } + while (this._queue.length > 0 && this._readIdx >= this._queue[0].length) { + this._readIdx -= this._queue[0].length; + this._queue.shift(); + } + } + + process(_, outputs) { + const channels = outputs[0]; + if (!channels || channels.length === 0) return true; + const out = channels[0]; + const stereo = channels.length > 1 ? channels[1] : null; + + for (let i = 0; i < out.length; i++) { + let sample = 0; + + if (this._playing) { + const v = this._readInterpolated(); + if (v === null) { + // Underrun: try to ramp out cleanly to avoid clicks. + sample = this._lastSample * Math.max(0, 1 - 1 / FADE_FRAMES); + this._lastSample = sample; + if (Math.abs(sample) < 1e-4) { + this._playing = false; + this._lastSample = 0; + this.port.postMessage({ kind: "underrun" }); + } + } else { + sample = v; + this._lastSample = v; + this._advance(); + } + + if (this._fadeIn > 0) { + const gain = 1 - this._fadeIn / FADE_FRAMES; + sample *= gain; + this._fadeIn -= 1; + } + if (this._fadeOut > 0) { + const gain = this._fadeOut / FADE_FRAMES; + sample *= gain; + this._fadeOut -= 1; + if (this._fadeOut === 0) { + this._playing = false; + this._lastSample = 0; + } + } + + this._totalPlayed += 1; + } + + out[i] = sample; + if (stereo) stereo[i] = sample; + } + + this._framesSinceStats += out.length; + if (this._framesSinceStats >= STATS_INTERVAL_FRAMES) { + this._framesSinceStats = 0; + const queuedSamples = this._queuedSamples(); + const queuedMs = (queuedSamples / this._inputRate) * 1000; + this.port.postMessage({ kind: "stats", queuedMs, played: this._totalPlayed }); + } + + return true; + } +} + +registerProcessor("audio-playback", AudioPlaybackProcessor); diff --git a/webui/realtime_static/worklets/mic-capture.js b/webui/realtime_static/worklets/mic-capture.js new file mode 100644 index 00000000..a48860b2 --- /dev/null +++ b/webui/realtime_static/worklets/mic-capture.js @@ -0,0 +1,159 @@ +// @ts-check +/** + * AudioWorkletProcessor that resamples the AudioContext rate (typically 48 kHz) + * down to 16 kHz, packs the result as little-endian Int16 PCM, and posts it + * back to the main thread in fixed-size chunks. + * + * The Hugging Face speech-to-speech WebSocket route expects the + * `input_audio_buffer.append` payload at 16 kHz PCM16 mono. + * + * Design notes: + * - 48 -> 16 is an exact 3:1 ratio so we use a 3-tap boxcar average as a + * cheap low-pass before decimating. Good enough for voice STT; we lose + * a tiny bit of >8 kHz content which the pipeline discards anyway. + * - Output frames are emitted at the cadence dictated by `chunkMs` + * (default 40 ms = 640 samples = 1280 bytes). The OpenAI Realtime + * server batches incoming audio so the cadence is flexible; 20-100 ms + * is the sweet spot. + * - Float -> Int16 saturates to [-1, 1] before scaling. + * - Optional noise gate: per-chunk RMS decides open/closed against a + * threshold; the gain ramps (fast attack, hold, slow release) so word + * onsets aren't clipped and quiet tails don't click. The gate only + * affects the audio we SEND; the main-thread visualiser taps the raw + * mic separately. We post the chunk RMS up every frame so the Settings + * mic meter can show the live level against the threshold. + */ + +const TARGET_RATE = 16000; +const DEFAULT_CHUNK_MS = 40; +// Gate envelope timing (fixed; only the threshold is user-tunable). +const GATE_ATTACK_MS = 5; // open almost instantly so word onsets survive +const GATE_HOLD_MS = 250; // stay open this long after the level drops back under +const GATE_RELEASE_MS = 80; // then fade closed over this long (no click) + +class MicCaptureProcessor extends AudioWorkletProcessor { + constructor(options) { + super(); + const chunkMs = options?.processorOptions?.chunkMs ?? DEFAULT_CHUNK_MS; + this._inputRate = sampleRate; + this._ratio = this._inputRate / TARGET_RATE; + this._chunkSamples16k = Math.round((TARGET_RATE * chunkMs) / 1000); + this._scratch = new Float32Array(0); + this._decimated = new Float32Array(this._chunkSamples16k); + this._enabled = true; + + // Noise gate state. Disabled by default (pure passthrough). + this._gateEnabled = false; + this._thresholdLin = 0; // linear amplitude; signal RMS must exceed this to open + this._gateGain = 1; // smoothed gain currently applied + this._holdRemaining = 0; // samples left before the gate may start closing + this._attackCoef = Math.exp(-1 / ((GATE_ATTACK_MS / 1000) * TARGET_RATE)); + this._releaseCoef = Math.exp(-1 / ((GATE_RELEASE_MS / 1000) * TARGET_RATE)); + this._holdSamples = Math.round((GATE_HOLD_MS / 1000) * TARGET_RATE); + + this.port.onmessage = (e) => { + const data = e.data; + if (data?.kind === "enable") this._enabled = !!data.value; + else if (data?.kind === "gate") { + this._gateEnabled = !!data.enabled; + // dB -> linear amplitude. When off, threshold 0 keeps the gate open. + this._thresholdLin = data.enabled ? Math.pow(10, data.thresholdDb / 20) : 0; + } + }; + } + + /** + * Append `incoming` to the internal scratch buffer, then emit as many + * full output chunks as we have material for. + * @param {Float32Array} incoming + */ + _ingest(incoming) { + if (incoming.length === 0) return; + const next = new Float32Array(this._scratch.length + incoming.length); + next.set(this._scratch, 0); + next.set(incoming, this._scratch.length); + this._scratch = next; + this._maybeEmit(); + } + + _maybeEmit() { + const r = this._ratio; + const n = this._chunkSamples16k; + const needIn = Math.ceil(n * r); + const dec = this._decimated; + while (this._scratch.length >= needIn) { + // 1. Decimate to 16 kHz floats and accumulate energy for the gate/meter. + let sumSq = 0; + if (Math.abs(r - 3) < 1e-6) { + // 48 kHz -> 16 kHz fast path with boxcar lowpass. + for (let i = 0; i < n; i++) { + const idx = i * 3; + const s = (this._scratch[idx] + this._scratch[idx + 1] + this._scratch[idx + 2]) / 3; + dec[i] = s; + sumSq += s * s; + } + } else { + // Generic path: linear interpolation. Slower but works at any rate + // (e.g. some Windows boxes report sampleRate=44100). + for (let i = 0; i < n; i++) { + const srcPos = i * r; + const idx = Math.floor(srcPos); + const frac = srcPos - idx; + const a = this._scratch[idx]; + const b = this._scratch[idx + 1] ?? a; + const s = a + (b - a) * frac; + dec[i] = s; + sumSq += s * s; + } + } + const rms = Math.sqrt(sumSq / n); + + // 2. Decide the gate target for this chunk, then ramp sample-by-sample. + let target = 1; + if (this._gateEnabled) { + if (rms >= this._thresholdLin) { + this._holdRemaining = this._holdSamples; // re-arm the hold + } else if (this._holdRemaining > 0) { + this._holdRemaining -= n; // coasting through the hold window + } else { + target = 0; + } + } + + // 3. Apply the (smoothed) gain and pack to Int16. + const out = new Int16Array(n); + let gain = this._gateGain; + for (let i = 0; i < n; i++) { + const coef = target > gain ? this._attackCoef : this._releaseCoef; + gain = target + (gain - target) * coef; + const s = dec[i] * gain; + const clamped = s < -1 ? -1 : s > 1 ? 1 : s; + out[i] = clamped < 0 ? clamped * 0x8000 : clamped * 0x7fff; + } + this._gateGain = gain; + + // Shift the scratch buffer to keep only the trailing unused samples. + const consumed = Math.floor(n * r); + this._scratch = this._scratch.slice(consumed); + + // Live input level for the Settings meter (raw RMS, pre-gate). + this.port.postMessage({ kind: "level", rms }); + + if (this._enabled) { + this.port.postMessage(out.buffer, [out.buffer]); + } + // When disabled (mic muted) we silently consume input so the worklet + // stays alive and the buffer never grows unbounded. + } + } + + process(inputs) { + const input = inputs[0]; + if (!input || input.length === 0 || !input[0]) return true; + const mono = input[0]; + if (mono.length > 0) this._ingest(mono); + return true; + } +} + +registerProcessor("mic-capture", MicCaptureProcessor); diff --git a/webui/realtime_static/ws/codec.js b/webui/realtime_static/ws/codec.js new file mode 100644 index 00000000..511ad9e8 --- /dev/null +++ b/webui/realtime_static/ws/codec.js @@ -0,0 +1,57 @@ +// @ts-check +/** + * Pure, stateless helpers for the WebSocket realtime client: base64 <-> PCM + * conversion for the audio frames on the wire, transcript extraction from a + * `response.done` payload, and a tiny URL helper. Kept separate from the client + * so the protocol/state logic stays readable. + */ + +/** @param {string} url */ +export function trimTrailingSlash(url) { + return url.endsWith("/") ? url.slice(0, -1) : url; +} + +/** + * Pull the assistant transcript out of a `response.done` payload. The text + * lives in `response.output[].content[].transcript` (audio) or `.text`. Used as + * the source of truth for interrupted replies, where the dedicated + * `*.transcript.done` event may never arrive. + * @param {any} response + * @returns {string} + */ +export function extractResponseTranscript(response) { + const output = response?.output; + if (!Array.isArray(output)) return ""; + /** @type {string[]} */ + const parts = []; + for (const item of output) { + for (const part of item?.content ?? []) { + const text = part?.transcript ?? part?.text; + if (typeof text === "string" && text.trim()) parts.push(text.trim()); + } + } + return parts.join(" ").trim(); +} + +/** @param {ArrayBuffer} buf */ +export function base64FromArrayBuffer(buf) { + const bytes = new Uint8Array(buf); + // Chunked encoding so we don't blow up the call stack on long buffers. + let binary = ""; + const chunk = 0x8000; + for (let i = 0; i < bytes.length; i += chunk) { + binary += String.fromCharCode.apply(null, /** @type {number[]} */ ( + /** @type {unknown} */ (bytes.subarray(i, i + chunk)) + )); + } + return btoa(binary); +} + +/** @param {string} b64 */ +export function base64ToBytes(b64) { + const binary = atob(b64); + const len = binary.length; + const out = new Uint8Array(len); + for (let i = 0; i < len; i++) out[i] = binary.charCodeAt(i); + return out; +} diff --git a/webui/realtime_static/ws/orb-visualizer.js b/webui/realtime_static/ws/orb-visualizer.js new file mode 100644 index 00000000..082f6012 --- /dev/null +++ b/webui/realtime_static/ws/orb-visualizer.js @@ -0,0 +1,98 @@ +// @ts-check +/** + * Orb spectrum visualiser. Each animation frame it reads two AnalyserNodes (the + * mic input and the TTS output) and maps the low-frequency speech energy onto + * the orb's CSS custom properties: + * - `--bar0`..`--bar4` the 5-band level meter + * - `--ai-audio-level` the global "Reachy talks" glow / scale pulse + * + * The bottom of the FFT is where speech energy lives, so the band edges stay + * low — that keeps the bars dancing on voice rather than on noise. While the AI + * is speaking we source the bars from the OUTPUT analyser so the orb pulses with + * Reachy's voice instead of sitting dead while the user is silent. + */ + +// Exported so the client can size its AnalyserNodes to match our buffer. +export const VIS_FFT_SIZE = 256; +const VIS_BAND_COUNT = 5; +const VIS_BAND_EDGES = [2, 5, 9, 16, 28, 52]; +const VIS_ATTACK = 0.6; // weight for new sample on upswing (snappy) +const VIS_RELEASE = 0.18; // weight for new sample on decay (gentle fade) + +export class OrbVisualiser { + /** + * @param {AnalyserNode} micAnalyser + * @param {AnalyserNode} outAnalyser + * @param {() => boolean} isAiSpeaking Source the bars from the AI output when + * true, otherwise from the mic. + */ + constructor(micAnalyser, outAnalyser, isAiSpeaking) { + this._mic = micAnalyser; + this._out = outAnalyser; + this._isAiSpeaking = isAiSpeaking; + this._buf = new Uint8Array(micAnalyser.frequencyBinCount); + this._bands = new Float32Array(VIS_BAND_COUNT); + this._aiLevel = 0; + /** @type {number | null} */ + this._frame = null; + } + + /** Begin the rAF loop (idempotent). */ + start() { + if (this._frame !== null) return; + const root = document.documentElement; + const tick = () => { + this._frame = requestAnimationFrame(tick); + this._update(root); + }; + this._frame = requestAnimationFrame(tick); + } + + /** Stop the loop and clear the CSS vars so the orb returns to rest. */ + stop() { + if (this._frame !== null) { + cancelAnimationFrame(this._frame); + this._frame = null; + } + const root = document.documentElement; + for (let i = 0; i < VIS_BAND_COUNT; i++) root.style.removeProperty(`--bar${i}`); + root.style.removeProperty("--ai-audio-level"); + } + + /** @param {HTMLElement} root */ + _update(root) { + // Mic bars: split FFT into 5 log-ish bands, smooth, write CSS vars. + const source = this._isAiSpeaking() ? this._out : this._mic; + source.getByteFrequencyData(this._buf); + + for (let b = 0; b < VIS_BAND_COUNT; b++) { + const lo = VIS_BAND_EDGES[b]; + const hi = VIS_BAND_EDGES[b + 1]; + let sum = 0; + let n = 0; + for (let i = lo; i < hi && i < this._buf.length; i++) { + sum += this._buf[i]; + n += 1; + } + const target = n > 0 ? sum / (n * 255) : 0; + const prev = this._bands[b]; + const k = target > prev ? VIS_ATTACK : VIS_RELEASE; + const next = prev + (target - prev) * k; + this._bands[b] = next; + root.style.setProperty(`--bar${b}`, next.toFixed(3)); + } + + // Global AI audio level: peak of the output analyser, used by the CSS to + // make the orb's glow / scale react to Reachy's voice. + this._out.getByteFrequencyData(this._buf); + let peak = 0; + const limit = Math.min(this._buf.length, VIS_BAND_EDGES[VIS_BAND_COUNT]); + for (let i = 0; i < limit; i++) { + if (this._buf[i] > peak) peak = this._buf[i]; + } + const aiTarget = peak / 255; + const k = aiTarget > this._aiLevel ? VIS_ATTACK : VIS_RELEASE; + this._aiLevel = this._aiLevel + (aiTarget - this._aiLevel) * k; + root.style.setProperty("--ai-audio-level", this._aiLevel.toFixed(3)); + } +} diff --git a/webui/realtime_static/ws/s2s-ws-client.js b/webui/realtime_static/ws/s2s-ws-client.js new file mode 100644 index 00000000..2f0f7ed8 --- /dev/null +++ b/webui/realtime_static/ws/s2s-ws-client.js @@ -0,0 +1,1144 @@ +// @ts-check +/** + * Minimal WebSocket client for the Hugging Face speech-to-speech load balancer. + * + * Two-step handshake (same /session route as the WebRTC client): + * + * 1. POST `/session` -> JSON `{ connect_url: wss:///v1/realtime?session_token=, ... }` + * 2. Open a WebSocket directly on `connect_url` (no rewrite, unlike the WebRTC client). + * + * Once the socket is open we follow the OpenAI Realtime GA WebSocket + * protocol: + * + * - Server pushes `session.created` immediately after upgrade. + * - We send `session.update` (GA schema: `session.audio.{input,output}`, + * `session.output_modalities`, ...). + * - We stream mic audio as PCM16 16 kHz mono base64 chunks via + * `input_audio_buffer.append`. + * - The server pushes `response.output_audio.delta` (PCM16 24 kHz mono + * base64) and transcript deltas. + * + * Audio is handled internally via two AudioWorklet processors so the + * client owns the full mic-in / speaker-out pipeline. The main app only + * sees high-level lifecycle events (`status`, `transcript`, `error`, + * `session`), the same shape as the WebRTC client. + * + * @typedef {"idle" | "creating-session" | "queued" | "your-turn" | "connecting" | + * "connected" | "user-speaking" | "processing" | "ai-speaking" | + * "closed" | "error" + * } WsStatus + * + * @typedef {Object} WsSessionInfo + * @property {string} sessionId + * @property {string} connectUrl + * @property {string} websocketUrl + * @property {string} sessionToken + * @property {number} pendingTimeoutS + * @property {string} [tier] Login tier from the session proxy ("anon"|"free"|"pro"). + * @property {boolean} [limited] Whether this session is metered (heartbeat needed). + * @property {number} [heartbeatSec] Suggested heartbeat cadence in seconds. + * @property {number} [remainingSec] Daily budget left after this grant (display). + * + * @typedef {Object} WsClientOptions + * @property {string} [sessionUrl] URL to POST for the session handshake (returns + * `{ connect_url, ... }`). Usually a same-origin proxy like `api/session` so the + * load-balancer address stays server-side. Provide this OR `directUrl`. + * @property {string} [loadBalancerUrl] Load-balancer base URL. Legacy/direct + * alternative to `sessionUrl`: the client POSTs `/session` itself. Prefer + * `sessionUrl` so the LB address isn't exposed to the browser. + * @property {string} [directUrl] Full WebSocket URL of an s2s realtime endpoint + * (e.g. `ws://localhost:8080/v1/realtime`). When set, the client skips the + * session POST and dials it directly — no load balancer in between. + * @property {string} voice + * @property {string} instructions + * @property {MediaStream} [micStream] Live mic stream. Provide this OR `acquireMic`. + * @property {() => Promise} [acquireMic] Lazily obtain the mic stream, + * called only once a session is actually granted (after any queue wait). Lets the + * caller prime mic permission up front but not hold the mic 'in use' indicator on + * while waiting in line. Ignored if `micStream` is already set. + * @property {AudioContext} [audioContext] Pre-created (and resumed) context. + * iOS Safari only lets an AudioContext start from within a user gesture, so + * the caller creates/resumes it synchronously on the orb tap and hands it + * here; otherwise it stays suspended (silent) after the mic/session awaits. + * @property {ToolDef[]} [tools] Function tools declared to the backend in the + * initial `session.update`. The model decides when to call them; the caller + * executes and replies via `sendToolOutput` + `requestResponse`. + * @property {NoiseGate} [noiseGate] Client-side noise gate applied to the mic + * before it's sent. Tunable live via `setNoiseGate`. + * + * @typedef {Object} NoiseGate + * @property {boolean} enabled + * @property {number} thresholdDb Open threshold in dBFS (e.g. -45). + * + * @typedef {Object} ToolDef + * @property {"function"} type + * @property {string} name + * @property {string} description + * @property {object} parameters JSON Schema for the call arguments. + * + * @typedef {Object} TranscriptEvent + * @property {"user" | "assistant"} role + * @property {string} text + * @property {boolean} partial + */ + +import { + base64FromArrayBuffer, + base64ToBytes, + extractResponseTranscript, + trimTrailingSlash, +} from "./codec.js"; +import { OrbVisualiser, VIS_FFT_SIZE } from "./orb-visualizer.js"; + +/** Build an Error carrying a `code` (and optional extra fields) so callers can + * branch on the failure kind: "limit" | "queue-full" | "queue-expired" | "aborted". + * @param {string} message @param {string} code @param {object} [extra] */ +function _codedError(message, code, extra) { + const err = /** @type {Error & { code?: string }} */ (new Error(message)); + err.code = code; + if (extra) Object.assign(err, extra); + return err; +} + +function _joinTranscript(prev, next) { + const a = (prev || "").trim(); + const b = (next || "").trim(); + if (!a) return b; + if (!b) return a; + if (/[\s([{（《“‘]$/.test(a) || /^[\s,.;:!?，。！？、；：）\]}》”’]/.test(b)) { + return `${a}${b}`; + } + if (/[\u3400-\u9fff]$/.test(a) || /^[\u3400-\u9fff]/.test(b)) { + return `${a}${b}`; + } + return `${a} ${b}`; +} + +// The s2s pipeline runs internally at 16 kHz mono PCM. The WebRTC transport +// resamples to 48 kHz for Opus, but the WebSocket transport emits the +// native pipeline rate. We don't (can't) override it via `audio.output.format` +// because the server's pydantic validator rejects the whole `session.update` +// as soon as a sub-field shape it doesn't know about appears. +const OUTPUT_SAMPLE_RATE = 16000; +const MIC_CHUNK_MS = 40; + +export class S2sWsRealtimeClient extends EventTarget { + /** @param {WsClientOptions} options */ + constructor(options) { + super(); + /** @type {WsClientOptions} */ + this.options = options; + /** @type {ToolDef[]} Function tools declared to the backend. */ + this._tools = options.tools ?? []; + /** @type {string} Direct realtime WS URL (set => skip the LB session POST). */ + this._directUrl = options.directUrl ?? ""; + /** @type {string} Where to POST for the session handshake. Prefer the + * explicit `sessionUrl`; fall back to `/session` for callers + * that still pass the LB address directly. */ + this._sessionUrl = options.sessionUrl + ? options.sessionUrl + : options.loadBalancerUrl + ? `${trimTrailingSlash(options.loadBalancerUrl)}/session` + : ""; + /** @type {(() => Promise) | null} Lazy mic acquisition (post-grant). */ + this._acquireMic = options.acquireMic ?? null; + /** @type {boolean} Set by close() to abort a queue wait in progress. */ + this._closed = false; + /** @type {string} The active queue ticket id while waiting (else ""). */ + this._queueId = ""; + /** @type {(() => void) | null} Wakes the queue poll sleep early on close(). */ + this._queueWake = null; + /** @type {ReturnType | 0} */ + this._queueTimer = 0; + // Join gate: after waiting in line the caller must explicitly `join()` before + // we dial, so a slot isn't spent on someone who walked away. Resolved by + // join(), rejected on timeout (the LB reclaims the slot) or close(). + /** @type {(() => void) | null} */ + this._joinResolve = null; + /** @type {((err: Error) => void) | null} */ + this._joinReject = null; + /** @type {ReturnType | 0} */ + this._joinTimer = 0; + /** @type {NoiseGate} Mic noise gate; off by default. */ + this._noiseGate = options.noiseGate ?? { enabled: false, thresholdDb: -45 }; + /** @type {WebSocket | null} */ + this._ws = null; + /** @type {AudioContext | null} */ + this._ctx = null; + /** @type {MediaStreamAudioSourceNode | null} */ + this._micSrc = null; + /** @type {AudioWorkletNode | null} */ + this._captureNode = null; + /** @type {AudioWorkletNode | null} */ + this._playbackNode = null; + /** @type {GainNode | null} */ + this._captureSink = null; + /** @type {AnalyserNode | null} */ + this._micAnalyser = null; + /** @type {AnalyserNode | null} */ + this._outAnalyser = null; + /** @type {OrbVisualiser | null} */ + this._visualiser = null; + /** @type {WsStatus} */ + this._status = "idle"; + this._aiSpeaking = false; + /** @type {Set} response_ids that have actually played audio, so the + * UI can tell a barge-in cut (keep it) from a never-heard speculative + * response (drop it). */ + this._audibleResponses = new Set(); + /** @type {Map} The CURRENT assistant transcript segment per + * response, accumulated from streamed deltas (reset on each segment's done). */ + this._asstTranscriptByResp = new Map(); + /** @type {Map} Completed assistant transcript segments per + * response, space-joined. A single response can emit several + * `*.transcript.done` events; we concatenate them until response.done. */ + this._asstFullByResp = new Map(); + this._muted = false; + // ── Response lock ──────────────────────────────────────────────────── + // The backend allows only ONE response in flight: creating a second while + // one is active fails with `conversation_already_has_active_response`. So + // we serialize response.create. `_openResponses` counts responses the + // server has confirmed (response.created) but not yet finished + // (response.done) — it's cumulative, so every create maps to one done. + // `_createInFlight` covers the window after we send a create but before its + // response.created echo. Any requestResponse() made while locked is queued + // and replayed, one at a time, as each response.done frees the slot. + this._openResponses = 0; + this._createInFlight = false; + /** @type {{ image?: string }[]} Pending response.create payloads, one per + * queued requestResponse(). A payload may carry an image to send just + * before its create (so the frame travels with the create, not eagerly). */ + this._createQueue = []; + /** @type {Promise | null} */ + this._readyPromise = null; + this._sessionConfigured = false; + /** @type {Record | null} audio.cpp per-session config (TTS/ASR/ + * LLM URLs + params) sent inside session.update as `session.audiocpp`. */ + this._appConfig = options.appConfig ?? null; + this._debug = (() => { try { return localStorage.getItem("s2s.debug") === "1"; } catch { return false; } })(); + } + + get status() { + return this._status; + } + + /** @param {WsStatus} status */ + _setStatus(status) { + if (this._status === status) return; + this._status = status; + this.dispatchEvent(new CustomEvent("status", { detail: { status } })); + } + + /** Full assistant transcript so far for a response: the completed segments + * plus the in-progress one, all space-joined. + * @param {string} rid @returns {string} */ + _asstDisplay(rid) { + const full = this._asstFullByResp.get(rid) || ""; + const seg = this._asstTranscriptByResp.get(rid) || ""; + if (!seg) return full; + return full ? `${full} ${seg}` : seg; + } + + _markAudible() { + if (this._status === "ai-speaking") return; + if (this._status === "closed" || this._status === "error") return; + this._setStatus("ai-speaking"); + } + + /** + * Full handshake. Resolves once the WS is open AND the audio pipeline is + * ready to send/receive samples. + * @returns {Promise} + */ + async connect() { + if (this._ws) throw new Error("Already connected"); + + let connectUrl; + if (this._directUrl) { + // Direct mode: no load balancer, no /session POST — dial the realtime + // endpoint straight away (e.g. a local s2s server). + connectUrl = this._directUrl; + this._setStatus("connecting"); + } else { + if (!this._sessionUrl) { + throw new Error("No session endpoint or direct URL configured"); + } + this._setStatus("creating-session"); + const { grant, waited } = await this._createSessionOrQueue(); + if (this._closed) throw _codedError("connect aborted", "aborted"); + // If we waited in line, don't dial until the user explicitly joins — this + // keeps a freed slot from being spent on someone who stepped away, and the + // click is a fresh gesture (re-arms the AudioContext on iOS). + if (waited) { + await this._awaitJoin(grant); + if (this._closed) throw _codedError("connect aborted", "aborted"); + } + this.dispatchEvent(new CustomEvent("session", { detail: { info: grant } })); + connectUrl = grant.connectUrl; + this._setStatus("connecting"); + } + + // Acquire the mic now — only once a slot is actually ours. The caller primed + // permission up front, so this is silent and the 'in use' indicator lights + // only for a real, connecting session (never during a queue wait). + if (!this.options.micStream && this._acquireMic) { + this.options.micStream = await this._acquireMic(); + } + + // Spin up the AudioContext + worklets in parallel with the WS dial. + const audioReady = this._setupAudio(); + const wsReady = this._openWebSocket(connectUrl); + await Promise.all([audioReady, wsReady]); + } + + /** + * POST the session handshake; if the pool is busy, wait in the queue (polling + * position) until a slot is claimed. Resolves to a grant plus whether we had to + * wait (which decides if an explicit join is required before dialing). + * @returns {Promise<{ grant: WsSessionInfo, waited: boolean }>} + */ + async _createSessionOrQueue() { + const first = await this._postSession(); + if (first.state === "queued") { + this._setStatus("queued"); + const grant = await this._pollQueue(first); + return { grant, waited: true }; + } + return { grant: first.grant, waited: false }; + } + + /** + * Hold at the front of the line until the user clicks join (resolves the gate) + * or the grant lapses. Announces "your-turn" + a deadline the UI counts down. + * @param {WsSessionInfo} grant + * @returns {Promise} + */ + _awaitJoin(grant) { + // The LB reclaims an unclaimed slot at its pending timeout; expire the gate a + // touch earlier so we never dial a session the LB just reaped. + const windowS = Math.max(3, (grant.pendingTimeoutS || 60) - 3); + this._setStatus("your-turn"); + this.dispatchEvent( + new CustomEvent("ready-to-join", { detail: { info: grant, expiresSec: windowS } }), + ); + return new Promise((resolve, reject) => { + this._joinResolve = resolve; + this._joinReject = reject; + this._joinTimer = setTimeout(() => { + this._joinResolve = null; + this._joinReject = null; + reject(_codedError("Your spot expired", "join-expired")); + }, windowS * 1000); + }); + } + + /** Accept the held slot and let connect() proceed to dial. Called from the + * "Join now" click, so it's a user gesture: re-resume the AudioContext, which + * iOS may have suspended while we waited. */ + join() { + if (this._joinTimer) { + clearTimeout(this._joinTimer); + this._joinTimer = 0; + } + try { + void this.options.audioContext?.resume(); + } catch { + // best-effort; _setupAudio resumes again + } + const resolve = this._joinResolve; + this._joinResolve = null; + this._joinReject = null; + resolve?.(); + } + + /** + * POST /session once. Returns either a granted session or a queue ticket. + * @returns {Promise<{ state: "granted", grant: WsSessionInfo } | { state: "queued", queueId: string, position: number, pollIntervalS: number }>} + */ + async _postSession() { + const url = this._sessionUrl; + console.log("[ws] POST", url); + const response = await fetch(url, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: "{}", + }); + if (response.status === 402) { + // The session proxy refused: today's per-tier time budget is spent. Surface + // it as a typed error so the UI shows the limit modal, not a crash. + const body = await response.json().catch(() => ({})); + throw _codedError("Daily conversation limit reached", "limit", { tier: body?.tier }); + } + if (response.status === 503) { + const body = await response.json().catch(() => ({})); + if (body?.state === "at_capacity") { + throw _codedError("The queue is full — try again shortly.", "queue-full"); + } + throw new Error("/session failed (503)"); + } + if (!response.ok) { + const text = await response.text().catch(() => ""); + throw new Error(`/session failed (${response.status}): ${text}`); + } + const json = await response.json(); + if (json.state === "queued") { + return { + state: "queued", + queueId: json.queue_id, + position: json.position, + pollIntervalS: json.poll_interval_s, + }; + } + return { state: "granted", grant: this._parseGrant(json) }; + } + + /** + * Poll the waiting queue until this ticket claims a slot. Emits `queue` events + * ({ position }) as the line advances. Throws on limit (402), expiry (404), or + * close(). Transient network/5xx blips are ignored and retried next tick. + * @param {{ queueId: string, position: number, pollIntervalS: number }} ticket + * @returns {Promise} + */ + async _pollQueue(ticket) { + const intervalMs = Math.max(1, ticket.pollIntervalS || 2) * 1000; + this._queueId = ticket.queueId; + this._emitQueue(ticket.position); + + while (true) { + await this._queueSleep(intervalMs); + if (this._closed) throw _codedError("queue wait aborted", "aborted"); + + let response; + try { + response = await fetch(`api/queue/${encodeURIComponent(this._queueId)}`, { + headers: { "Content-Type": "application/json" }, + }); + } catch { + continue; // network blip — keep our place, retry next tick + } + + if (response.status === 402) { + const body = await response.json().catch(() => ({})); + throw _codedError("Daily conversation limit reached", "limit", { tier: body?.tier }); + } + if (response.status === 404) { + throw _codedError("Queue timed out", "queue-expired"); + } + if (!response.ok) continue; // 502/503 — transient, retry + + const json = await response.json().catch(() => null); + if (!json) continue; + if (json.state === "queued") { + this._emitQueue(json.position); + continue; + } + // Reached the front and claimed a slot. + this._queueId = ""; + return this._parseGrant(json); + } + } + + /** @param {number} position */ + _emitQueue(position) { + this.dispatchEvent( + new CustomEvent("queue", { detail: { position, queueId: this._queueId } }), + ); + } + + /** A sleep that close() can cut short so a queued client tears down promptly. + * @param {number} ms */ + _queueSleep(ms) { + return new Promise((resolve) => { + this._queueWake = resolve; + this._queueTimer = setTimeout(() => { + this._queueWake = null; + resolve(); + }, ms); + }); + } + + /** @param {any} json @returns {WsSessionInfo} */ + _parseGrant(json) { + return { + sessionId: json.session_id, + connectUrl: json.connect_url, + websocketUrl: json.websocket_url, + sessionToken: json.session_token, + pendingTimeoutS: json.pending_timeout_s, + tier: json.tier, + limited: json.limited, + heartbeatSec: json.heartbeatSec, + remainingSec: json.remainingSec, + }; + } + + async _setupAudio() { + // Prefer a context the caller already created + resumed inside the tap + // gesture (required on iOS). Fall back to creating one here for callers + // that don't (desktop is lenient about the gesture timing). + // Most desktops give us 48 kHz, mobiles can give 44.1/24/16 kHz; the + // capture worklet handles any rate (linear interp fallback). + const ctx = this.options.audioContext ?? new AudioContext({ latencyHint: "interactive" }); + this._ctx = ctx; + + // Resume if still suspended. This is best-effort here — on iOS the resume + // that actually counts is the one the caller did synchronously on tap. + if (ctx.state === "suspended") { + try { + await ctx.resume(); + } catch (err) { + console.warn("[ws] AudioContext resume failed:", err); + } + } + + // The worklets live at the repo root, one level up from this module. + const base = new URL("../worklets/", import.meta.url); + await ctx.audioWorklet.addModule(new URL("mic-capture.js", base).href); + await ctx.audioWorklet.addModule(new URL("audio-playback.js", base).href); + + const captureNode = new AudioWorkletNode(ctx, "mic-capture", { + numberOfInputs: 1, + numberOfOutputs: 0, + processorOptions: { chunkMs: MIC_CHUNK_MS }, + }); + captureNode.port.onmessage = (e) => { + const data = e.data; + if (data instanceof ArrayBuffer) { + this._onMicChunk(data); + } else if (data?.kind === "level") { + // Raw pre-gate mic RMS for the Settings meter. + this.dispatchEvent(new CustomEvent("input-level", { detail: { rms: data.rms } })); + } + }; + // Push the initial gate config now that the worklet exists. + captureNode.port.postMessage({ kind: "gate", ...this._noiseGate }); + this._captureNode = captureNode; + + const micSrc = ctx.createMediaStreamSource(this.options.micStream); + micSrc.connect(captureNode); + this._micSrc = micSrc; + + // Mic analyser: tap the mic in parallel with the worklet so we get the + // raw (un-resampled, un-clipped) signal for the visualiser. + const micAnalyser = ctx.createAnalyser(); + micAnalyser.fftSize = VIS_FFT_SIZE; + micAnalyser.smoothingTimeConstant = 0; + micSrc.connect(micAnalyser); + this._micAnalyser = micAnalyser; + + const playbackNode = new AudioWorkletNode(ctx, "audio-playback", { + numberOfInputs: 0, + numberOfOutputs: 1, + outputChannelCount: [1], + }); + playbackNode.port.postMessage({ kind: "config", inputRate: OUTPUT_SAMPLE_RATE }); + playbackNode.port.onmessage = (e) => this._onPlaybackMessage(e.data); + + // Output analyser sits between the playback worklet and the speakers. + const outAnalyser = ctx.createAnalyser(); + outAnalyser.fftSize = VIS_FFT_SIZE; + outAnalyser.smoothingTimeConstant = 0.3; + playbackNode.connect(outAnalyser); + outAnalyser.connect(ctx.destination); + this._outAnalyser = outAnalyser; + this._playbackNode = playbackNode; + + this._visualiser = new OrbVisualiser(micAnalyser, outAnalyser, () => this._aiSpeaking); + this._visualiser.start(); + } + + /** @param {string} connectUrl */ + _openWebSocket(connectUrl) { + return new Promise((resolve, reject) => { + const ws = new WebSocket(connectUrl); + ws.binaryType = "arraybuffer"; + this._ws = ws; + + const onceOpen = () => { + ws.removeEventListener("open", onceOpen); + ws.removeEventListener("error", onceErr); + resolve(); + }; + const onceErr = (e) => { + ws.removeEventListener("open", onceOpen); + ws.removeEventListener("error", onceErr); + reject(new Error(`WebSocket failed to open: ${e?.type ?? "error"}`)); + }; + ws.addEventListener("open", onceOpen); + ws.addEventListener("error", onceErr); + + ws.addEventListener("message", (e) => this._onWsMessage(e.data)); + ws.addEventListener("close", (e) => this._onWsClose(e)); + ws.addEventListener("error", (e) => { + console.error("[ws] socket error", e); + }); + }); + } + + /** + * @param {{ kind: string; queuedMs?: number; played?: number }} data + */ + _onPlaybackMessage(data) { + if (data?.kind === "underrun") { + // Server stopped sending audio mid-response. Most likely the turn + // ended cleanly (a response.done usually arrives just before/after + // this). We let the state machine fall back to "connected" via the + // response.done event handler. + } + } + + /** + * Mic worklet just sent us a ~40 ms PCM16 16 kHz mono chunk. + * Base64-encode and forward via the WS. + * @param {ArrayBuffer} pcm16Buffer + */ + _onMicChunk(pcm16Buffer) { + if (!this._ws || this._ws.readyState !== WebSocket.OPEN) return; + if (!this._sessionConfigured) return; // Server rejects audio before session.update. + if (this._muted) return; + const b64 = base64FromArrayBuffer(pcm16Buffer); + this._send({ type: "input_audio_buffer.append", audio: b64 }); + } + + /** + * @param {string | ArrayBuffer | Blob} raw + */ + async _onWsMessage(raw) { + let text; + if (typeof raw === "string") { + text = raw; + } else if (raw instanceof ArrayBuffer) { + text = new TextDecoder("utf-8").decode(raw); + } else if (raw instanceof Blob) { + text = await raw.text(); + } else { + return; + } + + let event; + try { + event = JSON.parse(text); + } catch { + return; + } + + const type = event?.type; + if (typeof type !== "string") return; + // Opt-in event tracing for diagnosing turn/transcript issues. Enable with + // `localStorage.setItem("s2s.debug", "1")` in the browser console. + if (this._debug) { + const extra = type.startsWith("conversation.item.input_audio_transcription") + ? ` item=${event.item_id} ci=${event.content_index} ${event.delta ?? event.transcript ?? ""}` + : type.startsWith("response.") + ? ` resp=${event.response_id ?? event.response?.id ?? ""} status=${event.response?.status ?? ""} ${event.transcript ?? ""}` + : ""; + console.debug(`[ws] ${type}${extra}`); + } + + switch (type) { + case "session.created": + // Server-side defaults for the s2s pipeline are already what we + // want (server_vad, whisper-1 transcription, PCM16 16k in / 24k + // out). We only push the user-tunable bits: voice + instructions. + this._sendSessionUpdate(); + this._sessionConfigured = true; + if (this._status === "connecting") this._setStatus("connected"); + break; + + case "session.updated": + // Acknowledged by server, nothing to do. + break; + + case "input_audio_buffer.speech_started": + // User started speaking — stop any audio still playing OR queued, every + // time. We clear unconditionally (not just when `_aiSpeaking`): after a + // reply or a tool result the worklet's ring buffer can still be draining + // even though we already flipped `_aiSpeaking` off, and that tail would + // otherwise keep playing over the user's barge-in. + this._playbackNode?.port.postMessage({ kind: "clear" }); + this._aiSpeaking = false; + this._setStatus("user-speaking"); + break; + + case "input_audio_buffer.speech_stopped": + if (this._status === "user-speaking") this._setStatus("processing"); + break; + + case "input_audio_buffer.turn_discarded": + // Local VAD produced a short/no-text turn, so no response.done will + // follow. Only clear the matching processing state: a newer utterance + // may already have moved the client back to user-speaking. + if (this._status === "processing") this._setStatus("connected"); + break; + + case "response.created": + // A response now owns the slot — count it and clear our create guard + // (this confirms either our create or a server-initiated one). + this._openResponses++; + this._createInFlight = false; + if (this._status === "connected" || this._status === "user-speaking") { + this._setStatus("processing"); + } + break; + + case "response.output_item.added": + if (this._status === "connected" || this._status === "user-speaking") { + this._setStatus("processing"); + } + break; + + case "response.audio.delta": + case "response.output_audio.delta": { + this._pushAudioDelta(event.delta); + const rid = event.response_id ?? event.response?.id; + if (rid) this._audibleResponses.add(rid); + if (!this._aiSpeaking) { + this._aiSpeaking = true; + this._markAudible(); + } + break; + } + + case "response.content_part.added": { + const part = event.part; + if (part?.type === "audio" || part?.type === "output_audio") { + this._markAudible(); + } + break; + } + + case "response.done": { + this._aiSpeaking = false; + // This response freed the slot (completion OR cancellation both arrive + // as response.done). Decrement and, if a create was waiting, replay it. + this._openResponses = Math.max(0, this._openResponses - 1); + if (this._status === "ai-speaking" || this._status === "processing") { + this._setStatus("connected"); + } + // A response closes here for BOTH normal completion and cancellation + // (the s2s server signals a speculative-turn interrupt as + // `response.done` with status "cancelled" — there is no separate + // `response.cancelled` event). Surface the id + status so the UI can + // drop a cancelled response's transcript and commit a completed one. + const status = event.response?.status ?? "completed"; + const responseId = event.response?.id ?? ""; + // Did this response ever play audio? Distinguishes a barge-in cut (the + // user heard part of it) from a speculative response that never played. + const audible = responseId ? this._audibleResponses.has(responseId) : false; + this._audibleResponses.delete(responseId); + // Pull whatever transcript the response carries, falling back to the + // segments we concatenated from the `*.transcript.done` events (plus any + // in-progress delta). For an interrupted reply the response payload may + // be empty, so this is the last chance to capture the text. + const transcript = + extractResponseTranscript(event.response) || + this._asstDisplay(responseId) || + ""; + // Response finished — clear both transcript accumulators for it. + this._asstTranscriptByResp.delete(responseId); + this._asstFullByResp.delete(responseId); + this.dispatchEvent(new CustomEvent("response-finished", { + detail: { responseId, status, audible, transcript }, + })); + // The slot is free now — replay a queued create (e.g. a tool follow-up + // that arrived while this response was still running). + this._flushQueuedCreate(); + break; + } + + case "response.function_call_arguments.done": { + const name = typeof event.name === "string" ? event.name : ""; + const args = typeof event.arguments === "string" ? event.arguments : "{}"; + const callId = typeof event.call_id === "string" ? event.call_id : ""; + if (name) { + this.dispatchEvent(new CustomEvent("toolcall", { + detail: { name, arguments: args, callId }, + })); + } else { + // A nameless call can't be executed, so no function_call_output is + // ever sent and the model would wait forever for a result. The + // backend shouldn't emit these; warn loudly rather than stall silently. + console.warn(`[ws] function_call_arguments.done with no name (call_id=${callId}); cannot run tool — turn may stall`); + } + break; + } + + case "conversation.item.input_audio_transcription.delta": { + const delta = typeof event.delta === "string" ? event.delta : ""; + if (delta) { + // `itemId` is REUSED across a speculative continuation, so the UI + // groups both segments into one message. The delta carries the full + // cumulative transcript so far (not an increment). + this.dispatchEvent( + new CustomEvent("transcript", { + detail: { + role: "user", + text: delta, + partial: true, + itemId: typeof event.item_id === "string" ? event.item_id : "", + }, + }), + ); + } + break; + } + + case "conversation.item.input_audio_transcription.completed": { + const transcript = typeof event.transcript === "string" ? event.transcript : ""; + if (transcript) { + this.dispatchEvent( + new CustomEvent("transcript", { + detail: { + role: "user", + text: transcript, + partial: false, + itemId: typeof event.item_id === "string" ? event.item_id : "", + }, + }), + ); + } + break; + } + + case "response.audio_transcript.delta": + case "response.output_audio_transcript.delta": { + // Stream the assistant transcript live: accumulate the incremental + // deltas and push the running text to the UI. Every transcribe event we + // receive reaches the conversation, so an interrupted reply already has + // its partial text even if the `.done` never fires. + this._markAudible(); + const rid = typeof event.response_id === "string" ? event.response_id : ""; + const delta = typeof event.delta === "string" ? event.delta : ""; + if (delta) { + this._asstTranscriptByResp.set(rid, (this._asstTranscriptByResp.get(rid) || "") + delta); + // Show completed segments + the segment streaming in right now. + this.dispatchEvent( + new CustomEvent("transcript", { + detail: { role: "assistant", text: this._asstDisplay(rid), partial: true, responseId: rid }, + }), + ); + } + break; + } + + case "response.audio_transcript.done": + case "response.output_audio_transcript.done": { + const rid = typeof event.response_id === "string" ? event.response_id : ""; + // This is ONE completed segment. A response can emit several; concatenate + // them, space-separated, until response.done clears the accumulator. + const segment = + (typeof event.transcript === "string" && event.transcript) || + this._asstTranscriptByResp.get(rid) || + ""; + this._asstTranscriptByResp.delete(rid); // segment finished; next one starts fresh + if (segment) { + const prev = this._asstFullByResp.get(rid) || ""; + this._asstFullByResp.set(rid, _joinTranscript(prev, segment)); + } + const full = this._asstFullByResp.get(rid) || ""; + if (full) { + this.dispatchEvent( + new CustomEvent("transcript", { + detail: { role: "assistant", text: full, partial: false, responseId: rid }, + }), + ); + } + break; + } + + case "error": { + const err = event.error; + console.error("[ws] server error:", err); + // The "another response is already active" race: our optimistic create + // collided with a still-running response. Don't surface it — clear the + // in-flight guard and re-queue, so the create replays on the next + // response.done (never retried immediately, which would just collide + // again). + if (err?.type === "conversation_already_has_active_response" || + err?.code === "conversation_already_has_active_response") { + if (this._createInFlight) { + this._createInFlight = false; + // Re-queue a BARE create: any image on the original payload was + // already sent before this (rejected) create, so don't resend it. + this._createQueue.push({}); + } + break; + } + // Every other server error is non-fatal: surface it for logging but + // NEVER tear the socket down. Only transport failures (close / failed + // open) are fatal, and those come through their own paths. + this.dispatchEvent( + new CustomEvent("server-error", { detail: { error: new Error(err?.message ?? "Server error") } }), + ); + break; + } + } + } + + /** @param {string} b64 */ + _pushAudioDelta(b64) { + if (!this._playbackNode) return; + if (!b64) return; + const bytes = base64ToBytes(b64); + const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); + const samples = new Float32Array(bytes.byteLength / 2); + for (let i = 0; i < samples.length; i++) { + const s = view.getInt16(i * 2, true); + samples[i] = s < 0 ? s / 0x8000 : s / 0x7fff; + } + this._playbackNode.port.postMessage({ kind: "audio", samples }, [samples.buffer]); + } + + /** @param {CloseEvent} ev */ + _onWsClose(ev) { + console.log("[ws] socket closed:", ev.code, ev.reason); + if (this._status === "closed" || this._status === "error") return; + if (ev.code === 1000) { + this._setStatus("closed"); + } else { + this.dispatchEvent( + new CustomEvent("error", { + detail: { error: new Error(`WebSocket closed (${ev.code}) ${ev.reason || ""}`.trim()) }, + }), + ); + this._setStatus("error"); + } + } + + _sendSessionUpdate() { + // Minimal payload: only the bits the user is allowed to configure. + // The s2s server already defaults to server_vad, whisper-1 + // transcription, 16 kHz PCM input and 24 kHz PCM output, so we don't + // need (and must not send) `audio.input.format`, `audio.input.transcription`, + // `audio.input.turn_detection` or `audio.output.format`: the pydantic + // validator on the server rejects the whole event if any unknown or + // future-shaped sub-field shows up. + /** @type {Record} */ + const session = { type: "realtime" }; + // Only send instructions/voice when set, so an empty field can't clobber a + // value carried in the audiocpp block below (the backend reads both). + if (this.options.instructions) session.instructions = this.options.instructions; + if (this.options.voice) session.audio = { output: { voice: this.options.voice } }; + // audio.cpp per-session config: TTS/ASR server URLs, model ids, voice ref, + // LLM endpoint/key. The realtime backend reads this and reconfigures the + // pipeline before the first turn. + if (this._appConfig) session.audiocpp = this._appConfig; + // Tools are declared here; the backend already accepts them in + // session.update and emits response.function_call_arguments.done when the + // model decides to call one. Only include the keys when we actually have + // tools — the server's pydantic validator is strict about shapes. + if (this._tools.length) { + session.tools = this._tools; + session.tool_choice = "auto"; + } + this._send({ type: "session.update", session }); + } + + /** Update the audio.cpp pipeline config (TTS/ASR/LLM) on a live session. + * @param {Record} cfg */ + updateAppConfig(cfg) { + this._appConfig = { ...(this._appConfig || {}), ...cfg }; + this._send({ + type: "session.update", + session: { type: "realtime", audiocpp: this._appConfig }, + }); + } + + /** Update voice/instructions on a live session without tearing down. */ + /** @param {{ voice?: string; instructions?: string }} patch */ + updateSession(patch) { + /** @type {Record} */ + const session = { type: "realtime" }; + if (patch.instructions) session.instructions = patch.instructions; + if (patch.voice) session.audio = { output: { voice: patch.voice } }; + if (Object.keys(session).length > 1) { + this._send({ type: "session.update", session }); + } + } + + /** + * Replace the declared tool set on a live session (e.g. the user flipped a + * tool switch mid-conversation). Always sends `tools` — an empty array + * clears them — so toggling the last tool off actually removes it. + * @param {ToolDef[]} tools + */ + setTools(tools) { + this._tools = tools; + this._send({ + type: "session.update", + session: { type: "realtime", tools, tool_choice: tools.length ? "auto" : "none" }, + }); + } + + /** + * Return a tool's result to the model. Pairs with the `toolcall` event's + * `callId`. Caller follows this with `requestResponse()` so the model speaks. + * @param {string} callId + * @param {string} output Plain text / JSON string the model will read. + */ + sendToolOutput(callId, output) { + if (!callId) return; // Can't target a result without the call id. + this._send({ + type: "conversation.item.create", + item: { type: "function_call_output", call_id: callId, output }, + }); + } + + /** + * Add an image to the conversation as user content, so the vision-language + * model can see it (used by the camera tool). `dataUrl` is a + * `data:image/jpeg;base64,...` string. + * @param {string} dataUrl + */ + sendUserImage(dataUrl) { + this._send({ + type: "conversation.item.create", + item: { + type: "message", + role: "user", + content: [{ type: "input_image", image_url: dataUrl }], + }, + }); + } + + /** + * Ask the model to generate a response now (after feeding tool results). + * Serialized: if a response is already in flight we queue this request and + * replay it once the active response finishes, so we never trip the + * backend's `conversation_already_has_active_response` guard. + * + * @param {{ image?: string }} [opts] Optional `image` (a data URL) sent as a + * user `input_image` immediately before this response.create — so the frame + * travels with the create (and is deferred together with it if queued), + * rather than being added to the conversation eagerly. Used by the camera + * tool so the model sees the snapshot in the response it's about to speak. + */ + requestResponse(opts = {}) { + if (this._responseActive()) { + this._createQueue.push(opts); + if (this._debug) console.debug(`[ws] response.create queued (a response is active); pending=${this._createQueue.length}`); + return; + } + this._createResponseNow(opts); + } + + /** True while a response occupies the single backend slot. */ + _responseActive() { + return this._openResponses > 0 || this._createInFlight; + } + + /** Send a response.create immediately and arm the in-flight guard. Any image + * on the payload is added as user content right before the create. + * @param {{ image?: string }} [opts] */ + _createResponseNow(opts = {}) { + if (!this._ws || this._ws.readyState !== WebSocket.OPEN) return; + if (opts.image) this.sendUserImage(opts.image); + this._createInFlight = true; + this._send({ type: "response.create" }); + } + + /** Replay one queued response.create if the slot is now free. Called on every + * response.done, so queued creates drain one-per-completion. */ + _flushQueuedCreate() { + if (this._createQueue.length > 0 && !this._responseActive()) { + const opts = this._createQueue.shift(); + if (this._debug) console.debug(`[ws] replaying queued response.create; remaining=${this._createQueue.length}`); + this._createResponseNow(opts); + } + } + + /** @param {boolean} muted */ + setMuted(muted) { + this._muted = muted; + } + + /** + * Update the mic noise gate live (the user moved the Settings cursor). + * @param {NoiseGate} gate + */ + setNoiseGate(gate) { + this._noiseGate = gate; + this._captureNode?.port.postMessage({ kind: "gate", ...gate }); + } + + /** @param {Record} event */ + _send(event) { + if (!this._ws || this._ws.readyState !== WebSocket.OPEN) return; + this._ws.send(JSON.stringify(event)); + } + + async close() { + // Abort a queue wait in progress: flag it and wake the poll sleep so + // `_pollQueue` throws "aborted" and connect() unwinds cleanly. + this._closed = true; + if (this._queueWake) { + clearTimeout(this._queueTimer); + const wake = this._queueWake; + this._queueWake = null; + wake(); + } + if (this._joinTimer) { + clearTimeout(this._joinTimer); + this._joinTimer = 0; + } + if (this._joinReject) { + const reject = this._joinReject; + this._joinResolve = null; + this._joinReject = null; + reject(_codedError("join aborted", "aborted")); + } + this._visualiser?.stop(); + this._visualiser = null; + try { + if (this._ws && this._ws.readyState <= WebSocket.OPEN) { + this._ws.close(1000, "client closed"); + } + } catch { + // ignored + } + this._ws = null; + + try { + this._captureNode?.port.close?.(); + } catch { + // ignored + } + try { + this._micSrc?.disconnect(); + } catch { + // ignored + } + try { + this._captureNode?.disconnect(); + } catch { + // ignored + } + try { + this._micAnalyser?.disconnect(); + } catch { + // ignored + } + try { + this._outAnalyser?.disconnect(); + } catch { + // ignored + } + try { + this._playbackNode?.disconnect(); + } catch { + // ignored + } + try { + await this._ctx?.close(); + } catch { + // ignored + } + this._ctx = null; + this._captureNode = null; + this._playbackNode = null; + this._micSrc = null; + this._micAnalyser = null; + this._outAnalyser = null; + this._setStatus("closed"); + } +} diff --git a/webui/test_realtime_pipeline.py b/webui/test_realtime_pipeline.py new file mode 100644 index 00000000..e80edcc2 --- /dev/null +++ b/webui/test_realtime_pipeline.py @@ -0,0 +1,112 @@ +import queue +import threading +import unittest + +import numpy as np + +from realtime_pipeline import ( + RealtimePipeline, + ResponseDone, + SpeechStopped, + TurnDiscarded, + UserTranscript, +) + + +class _CancelScope: + generation = 1 + + def __init__(self, stale: bool = False): + self._stale = stale + + def is_stale(self, generation: int) -> bool: + return self._stale + + +class _STT: + def __init__(self, text: str): + self._text = text + + def transcribe(self, audio: np.ndarray) -> str: + return self._text + + +class _LLM: + def __init__(self): + self.user_messages: list[str] = [] + + def add_user_message(self, text: str) -> None: + self.user_messages.append(text) + + def add_assistant_message(self, text: str) -> None: + pass + + def stream(self, cancel_scope: _CancelScope, generation: int): + return iter(()) + + +class _SingleTurnQueue: + def __init__(self, turn: SpeechStopped, stop_event: threading.Event): + self._turn = turn + self._stop_event = stop_event + + def get(self, timeout: float) -> SpeechStopped: + self._stop_event.set() + return self._turn + + +def _pipeline_for_transcript(text: str) -> RealtimePipeline: + pipeline = RealtimePipeline.__new__(RealtimePipeline) + pipeline._out_events = queue.Queue() + pipeline._cancel_scope = _CancelScope() + pipeline._stt = _STT(text) + pipeline._llm = _LLM() + pipeline._cfg_lock = threading.Lock() + pipeline._tts_model = "unused" + pipeline._tts_voice = "" + pipeline._tts_voice_ref = "" + pipeline._tts_reference_text = "" + pipeline._tts_lock = threading.Lock() + pipeline._tts_pending = 0 + pipeline._stop_event = threading.Event() + return pipeline + + +class RealtimeTurnLifecycleTests(unittest.TestCase): + def test_too_short_segment_discards_turn(self): + pipeline = RealtimePipeline.__new__(RealtimePipeline) + pipeline._out_events = queue.Queue() + pipeline._stop_event = threading.Event() + pipeline._turn_queue = _SingleTurnQueue( + SpeechStopped(audio=np.zeros(100, dtype=np.float32)), + pipeline._stop_event, + ) + + pipeline._pipeline_loop() + + self.assertEqual( + pipeline.drain_events(), + [TurnDiscarded(reason="too_short")], + ) + + def test_empty_transcript_discards_turn_without_starting_llm(self): + pipeline = _pipeline_for_transcript("") + + pipeline._run_turn(np.zeros(16000, dtype=np.float32)) + + events = pipeline.drain_events() + self.assertEqual(events, [TurnDiscarded(reason="empty_transcript")]) + self.assertEqual(pipeline._llm.user_messages, []) + + def test_successful_transcript_keeps_existing_response_lifecycle(self): + pipeline = _pipeline_for_transcript("hello") + + pipeline._run_turn(np.zeros(16000, dtype=np.float32)) + + events = pipeline.drain_events() + self.assertEqual(events, [UserTranscript(text="hello"), ResponseDone()]) + self.assertEqual(pipeline._llm.user_messages, ["hello"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/webui/test_ui_i18n.py b/webui/test_ui_i18n.py new file mode 100644 index 00000000..77eb3066 --- /dev/null +++ b/webui/test_ui_i18n.py @@ -0,0 +1,155 @@ +import json +import os +import tempfile +import unittest +from unittest import mock + +try: + from ui_i18n import ( + DEFAULT_LANGUAGE, + LANGUAGE_CHOICES, + get_language, + load_language, + normalize_language, + param_spec, + save_language, + set_language, + text, + ) +except ImportError: + from webui.ui_i18n import ( + DEFAULT_LANGUAGE, + LANGUAGE_CHOICES, + get_language, + load_language, + normalize_language, + param_spec, + save_language, + set_language, + text, + ) + + +HERE = os.path.dirname(os.path.abspath(__file__)) + + +class UiI18nTests(unittest.TestCase): + def tearDown(self): + set_language(DEFAULT_LANGUAGE) + + def test_english_is_the_default(self): + set_language(DEFAULT_LANGUAGE) + self.assertEqual(get_language(), "en") + self.assertEqual(text("生成语音", "Generate speech"), "Generate speech") + + def test_manual_chinese_switch(self): + set_language("zh") + self.assertEqual(get_language(), "zh") + self.assertEqual(text("生成语音", "Generate speech"), "生成语音") + + def test_language_choices_are_in_requested_order(self): + self.assertEqual(LANGUAGE_CHOICES, [ + ("English", "en"), + ("中文", "zh"), + ("中文繁體", "zh-Hant"), + ]) + + def test_manual_traditional_chinese_switch(self): + set_language("zh-Hant") + self.assertEqual(get_language(), "zh-Hant") + self.assertEqual(text("生成语音", "Generate speech"), "生成語音") + + def test_unknown_language_falls_back_to_english(self): + self.assertEqual(set_language("fr"), "en") + + def test_missing_language_config_defaults_to_english(self): + set_language("zh") + with tempfile.TemporaryDirectory() as directory: + path = os.path.join(directory, "missing.json") + with mock.patch.dict(os.environ, {}, clear=False): + os.environ.pop("AUDIOCPP_LANG", None) + self.assertEqual(load_language(path), "en") + self.assertEqual(get_language(), "en") + + def test_audiocpp_lang_supplies_the_default_when_nothing_saved(self): + with tempfile.TemporaryDirectory() as directory: + path = os.path.join(directory, "missing.json") + with mock.patch.dict(os.environ, {"AUDIOCPP_LANG": "zh_TW"}): + self.assertEqual(load_language(path), "zh-Hant") + + def test_saved_language_beats_audiocpp_lang(self): + """An explicit pick in the UI must survive a stale env var.""" + with tempfile.TemporaryDirectory() as directory: + path = os.path.join(directory, "ui_language.json") + save_language("zh", path) + with mock.patch.dict(os.environ, {"AUDIOCPP_LANG": "en"}): + self.assertEqual(load_language(path), "zh") + + def test_audiocpp_lang_aliases_normalize(self): + for value, expected in ( + ("EN", "en"), ("english", "en"), + ("zh-CN", "zh"), ("zh_Hans", "zh"), ("Chinese", "zh"), + ("zh-Hant", "zh-Hant"), ("zh_TW", "zh-Hant"), ("tw", "zh-Hant"), + (" zh-hk ", "zh-Hant"), + ("klingon", "en"), + ): + with self.subTest(value=value): + self.assertEqual(normalize_language(value), expected) + + def test_saved_language_is_loaded(self): + with tempfile.TemporaryDirectory() as directory: + path = os.path.join(directory, "ui_language.json") + self.assertEqual(save_language("en", path), "en") + set_language("zh") + self.assertEqual(load_language(path), "en") + self.assertEqual(get_language(), "en") + with open(path, "r", encoding="utf-8") as f: + self.assertEqual(json.load(f), {"language": "en"}) + + def test_saved_traditional_chinese_is_loaded(self): + with tempfile.TemporaryDirectory() as directory: + path = os.path.join(directory, "ui_language.json") + self.assertEqual(save_language("zh-Hant", path), "zh-Hant") + set_language("zh") + self.assertEqual(load_language(path), "zh-Hant") + self.assertEqual(get_language(), "zh-Hant") + + def test_english_param_keeps_identifier_and_drops_long_copy(self): + localized = param_spec({ + "name": "guidance_scale", + "label": "guidance_scale（引导强度）", + "info": "这是一段很长的说明", + "placeholder": "中文占位说明", + }, "en") + self.assertEqual(localized["label"], "guidance_scale") + self.assertIsNone(localized["info"]) + self.assertEqual(localized["placeholder"], "") + + def test_traditional_param_converts_visible_copy_only(self): + localized = param_spec({ + "name": "guidance_scale", + "label": "引导强度", + "info": "加载模型后生效", + "placeholder": "请输入文件路径", + "value": "保持不变", + }, "zh-Hant") + self.assertEqual(localized["name"], "guidance_scale") + self.assertEqual(localized["label"], "引導強度") + self.assertEqual(localized["info"], "加載模型後生效") + self.assertEqual(localized["placeholder"], "請輸入文件路徑") + self.assertEqual(localized["value"], "保持不变") + + def test_every_config_control_has_an_english_identifier_label(self): + path = os.path.join(HERE, "configs", "model_params.json") + with open(path, "r", encoding="utf-8") as f: + config = json.load(f) + for family, specs in config.items(): + if family.startswith("_"): + continue + for spec in specs: + with self.subTest(family=family, name=spec.get("name")): + self.assertTrue(param_spec(spec, "en").get("label")) + + +if __name__ == "__main__": + unittest.main() diff --git a/webui/test_voice_library.py b/webui/test_voice_library.py new file mode 100644 index 00000000..18d0e4a1 --- /dev/null +++ b/webui/test_voice_library.py @@ -0,0 +1,89 @@ +import os +import tempfile +import unittest +import wave +from unittest import mock + +try: + from webui import webui as app +except ImportError: + import webui as app + + +class VoiceLibraryTests(unittest.TestCase): + def setUp(self): + self._previous_language = app.get_language() + app.set_language("zh") + + def tearDown(self): + app.set_language(self._previous_language) + + @staticmethod + def _write_wav(path): + with wave.open(path, "wb") as f: + f.setnchannels(1) + f.setsampwidth(2) + f.setframerate(16000) + f.writeframes(b"\0\0" * 160) + + def test_original_upload_name_is_available_before_staging(self): + with mock.patch.object(app, "_stage_upload", return_value="staged.wav"): + staged, name = app._stage_tts_voice_upload( + os.path.join("somewhere", "Original Voice.wav")) + self.assertEqual(staged, "staged.wav") + self.assertEqual(name, "Original Voice") + + def test_save_copies_wav_and_updates_one_prompt_record(self): + with tempfile.TemporaryDirectory() as directory: + voice_dir = os.path.join(directory, "voice") + os.makedirs(voice_dir) + source = os.path.join(directory, "source.wav") + self._write_wav(source) + + with mock.patch.object(app, "PROMPTS_DIR", voice_dir): + _dropdown, message = app.save_builtin_voice( + source, "测试音色", "第一行\n第二行") + app.save_builtin_voice(source, "测试音色.wav", "更新文本") + + self.assertTrue(os.path.isfile(os.path.join(voice_dir, "测试音色.wav"))) + with open(os.path.join(voice_dir, "prompt_text"), + "r", encoding="utf-8") as f: + self.assertEqual(f.read(), "测试音色|更新文本\n") + self.assertIn("测试音色.wav", message) + + def test_save_requires_name_and_audio(self): + _dropdown, message = app.save_builtin_voice(None, "", "") + self.assertIn("填写", message) + _dropdown, message = app.save_builtin_voice(None, "voice", "") + self.assertIn("上传", message) + + def test_delete_removes_wav_and_prompt_record(self): + with tempfile.TemporaryDirectory() as directory: + voice_dir = os.path.join(directory, "voice") + os.makedirs(voice_dir) + source = os.path.join(directory, "source.wav") + self._write_wav(source) + + with mock.patch.object(app, "PROMPTS_DIR", voice_dir): + app.save_builtin_voice(source, "delete_me", "参考文本") + result = app.delete_builtin_voice("delete_me.wav") + + self.assertFalse(os.path.exists( + os.path.join(voice_dir, "delete_me.wav"))) + with open(os.path.join(voice_dir, "prompt_text"), + "r", encoding="utf-8") as f: + self.assertEqual(f.read(), "") + self.assertIn("已删除", result[-1]) + + def test_delete_none_does_nothing(self): + self.assertEqual( + app.delete_builtin_voice("(none)"), + tuple(app.gr.skip() for _ in range(5))) + + def test_voice_name_rejects_paths(self): + with self.assertRaises(ValueError): + app._builtin_voice_filename("../outside") + + +if __name__ == "__main__": + unittest.main() diff --git a/webui/ui_i18n.py b/webui/ui_i18n.py new file mode 100644 index 00000000..94e3b794 --- /dev/null +++ b/webui/ui_i18n.py @@ -0,0 +1,132 @@ +"""Small multilingual helpers for the local Gradio WebUI.""" + +import json +import os + +from opencc import OpenCC + +LANG_ZH = "zh" +LANG_ZH_HANT = "zh-Hant" +LANG_EN = "en" +DEFAULT_LANGUAGE = LANG_EN +LANGUAGE_CHOICES = [ + ("English", LANG_EN), + ("中文", LANG_ZH), + ("中文繁體", LANG_ZH_HANT), +] +LANGUAGE_CONFIG_PATH = os.path.join( + os.path.dirname(os.path.abspath(__file__)), "configs", "ui_language.json") + +# Spellings accepted from AUDIOCPP_LANG, which people write in several ways. +_LANGUAGE_ALIASES = { + "en": LANG_EN, "en-us": LANG_EN, "en_us": LANG_EN, "english": LANG_EN, + "zh": LANG_ZH, "zh-cn": LANG_ZH, "zh_cn": LANG_ZH, "zh-hans": LANG_ZH, + "zh_hans": LANG_ZH, "cn": LANG_ZH, "chinese": LANG_ZH, + "zh-hant": LANG_ZH_HANT, "zh_hant": LANG_ZH_HANT, "zh-tw": LANG_ZH_HANT, + "zh_tw": LANG_ZH_HANT, "zh-hk": LANG_ZH_HANT, "zh_hk": LANG_ZH_HANT, + "tw": LANG_ZH_HANT, "hant": LANG_ZH_HANT, "traditional": LANG_ZH_HANT, +} + +_language = DEFAULT_LANGUAGE +_s2t_converter = OpenCC("s2t") + + +def normalize_language(language): + if language in (LANG_ZH, LANG_ZH_HANT, LANG_EN): + return language + if isinstance(language, str): + alias = _LANGUAGE_ALIASES.get(language.strip().lower()) + if alias: + return alias + return DEFAULT_LANGUAGE + + +def to_traditional(value): + """Convert visible Simplified Chinese copy while preserving non-strings.""" + return _s2t_converter.convert(value) if isinstance(value, str) else value + + +def set_language(language): + global _language + _language = normalize_language(language) + return _language + + +def get_language(): + return _language + + +def load_language(path=LANGUAGE_CONFIG_PATH): + """Resolve the startup UI language. + + A language saved from the in-UI picker wins, so an explicit click keeps + surviving restarts. AUDIOCPP_LANG only supplies the default for a profile + that has never chosen one (first run, or a scripted/headless launch); + otherwise it would silently defeat the picker. Falls back to English. + """ + saved = None + try: + with open(path, "r", encoding="utf-8") as f: + data = json.load(f) + if isinstance(data, dict): + saved = data.get("language") + except (OSError, ValueError, TypeError): + pass + if saved is None: + saved = os.environ.get("AUDIOCPP_LANG") or DEFAULT_LANGUAGE + return set_language(saved) + + +def save_language(language, path=LANGUAGE_CONFIG_PATH): + """Persist the normalized UI language and return it.""" + language = normalize_language(language) + directory = os.path.dirname(path) + if directory: + os.makedirs(directory, exist_ok=True) + temp_path = path + ".tmp" + with open(temp_path, "w", encoding="utf-8") as f: + json.dump({"language": language}, f, ensure_ascii=False, indent=2) + f.write("\n") + os.replace(temp_path, path) + return language + + +def text(zh, en, language=None, **values): + """Choose and format localized UI copy.""" + language = normalize_language(language or _language) + if language == LANG_EN: + template = en + elif language == LANG_ZH_HANT: + template = to_traditional(zh) + else: + template = zh + return template.format(**values) if values else template + + +def param_spec(spec, language=None): + """Return a localized advanced-parameter spec without changing its value. + + Protocol names, choices and defaults must stay stable. In English mode a + missing translated label falls back to the option name, while untranslated + explanatory copy is omitted to keep dynamic control rows compact. + """ + language = normalize_language(language or _language) + if language == LANG_ZH: + return dict(spec) + + if language == LANG_ZH_HANT: + out = dict(spec) + for key in ("label", "info", "placeholder"): + if key in out: + out[key] = to_traditional(out[key]) + return out + + out = dict(spec) + out["label"] = spec.get("label_en") or spec.get("name", "") + out["info"] = spec.get("info_en") or None + placeholder = spec.get("placeholder_en") + if placeholder is None: + original = spec.get("placeholder", "") + placeholder = original if original.isascii() else "" + out["placeholder"] = placeholder + return out diff --git a/webui/voice/demo_01_man.wav b/webui/voice/demo_01_man.wav new file mode 100644 index 00000000..aa3bc422 Binary files /dev/null and b/webui/voice/demo_01_man.wav differ diff --git a/webui/voice/demo_02_woman.wav b/webui/voice/demo_02_woman.wav new file mode 100644 index 00000000..a9096e89 Binary files /dev/null and b/webui/voice/demo_02_woman.wav differ diff --git a/webui/voice/demo_3_man.wav b/webui/voice/demo_3_man.wav new file mode 100644 index 00000000..1b5d8707 Binary files /dev/null and b/webui/voice/demo_3_man.wav differ diff --git a/webui/voice/prompt_text b/webui/voice/prompt_text new file mode 100644 index 00000000..163eddff --- /dev/null +++ b/webui/voice/prompt_text @@ -0,0 +1,6 @@ +demo_01_man|okay,I'm Cemo and what you just heard wasn't a human voice. +demo_02_woman|以前我对这句话一知半解，现在好像有点懂了。因为你我开始留意很多以前不曾关心的事，开始对这个世界有了更多的好奇和善意。 +zh-Bowen_man|简单来说，人工智能是一门致力于让计算机和机器像人一样思考和行动的科学领域。它的目标是模拟、延伸和扩展人的智能，让机器能够胜任通常需要人类智慧才能完成的任务，比如解决问题、感知环境、理解语言，甚至进行创造。AI是一个非常广泛的领域，它包含了计算机科学、语言学、神经科学，甚至哲学和心理学等多个学科的知识。 +my-record|人脸特征点覆盖范围，等等各项参数，大部分情况下，这些参数程序都会自动填好，只有常用的几个设置，需要我们手动微调。 +demo_3_man|它的目标是模拟、延伸和扩展人的智能，让机器能够胜任通常需要人类智慧才能完成的任务 +ref-w1|这都不会啊，麻将牌九掷色子，四色牌你总会一样吧 diff --git a/webui/voice/ref-w1.wav b/webui/voice/ref-w1.wav new file mode 100644 index 00000000..9246d0cb Binary files /dev/null and b/webui/voice/ref-w1.wav differ diff --git a/webui/voice/zh-Bowen_man.wav b/webui/voice/zh-Bowen_man.wav new file mode 100644 index 00000000..dd2a0be2 Binary files /dev/null and b/webui/voice/zh-Bowen_man.wav differ diff --git a/webui/webui.py b/webui/webui.py new file mode 100644 index 00000000..8e4ac35b --- /dev/null +++ b/webui/webui.py @@ -0,0 +1,4639 @@ +""" +audio.cpp WebUI (Route B) — a thin Gradio frontend that proxies to the local +audiocpp_server HTTP API. + +Model loading is on demand: instead of preloading a model at server startup, this +WebUI reads models_catalog.json, lets you pick a model, and (re)starts +audiocpp_server with a single-model config only when you actually load/run it. +One model lives in VRAM at a time — picking a different model swaps it. + +Uploaded files are saved by Gradio to local temp paths, which we pass to the +server as `voice_ref` / `audio` (frontend + server run on the same machine). + +Just launch this (it starts the server for you): + venv\\Scripts\\python audiocpp-portable\\webui.py + +Env overrides: + AUDIOCPP_BACKEND=gpu|cpu which bin dir to launch the server from + (default: auto — gpu when an NVIDIA driver and the + gpu server build are both present, else cpu) + AUDIOCPP_THREADS=N ggml compute threads (default 1; cpu backend + defaults to all cores minus one) + AUDIOCPP_SERVER=http://... talk to an already-running server instead of managing one + AUDIOCPP_LOAD_TIMEOUT=300 seconds to wait for a model to finish loading + AUDIOCPP_NO_BROWSER=1 don't open a browser tab +""" +import atexit +import base64 +import glob +import io +import json +import logging +import os +import random +import re +import shutil +import socket +import subprocess +import sys +import tempfile +import threading +import time +import warnings +import wave +from urllib.parse import urlparse + +import numpy as np +import requests +import gradio as gr + +try: + from ui_i18n import ( + LANGUAGE_CHOICES, + get_language, + load_language, + param_spec as localized_param_spec, + save_language, + set_language, + text as ui_text, + ) +except ImportError as _exc: + # Fall back to the package-qualified name only when ui_i18n itself is not on + # the path (imported as ``webui.webui`` in tests). A missing *transitive* + # dependency of ui_i18n — e.g. opencc — must surface as-is instead of being + # masked by a misleading "no module named webui.ui_i18n". + if _exc.name not in ("ui_i18n", "webui"): + raise + from webui.ui_i18n import ( + LANGUAGE_CHOICES, + get_language, + load_language, + param_spec as localized_param_spec, + save_language, + set_language, + text as ui_text, + ) + +# 降噪：屏蔽 Gradio 内部触发、每次请求都会刷屏的 Starlette 弃用告警。 +warnings.filterwarnings("ignore", message=r".*HTTP_422_UNPROCESSABLE.*") + + +def _silence_proactor_connection_reset(): + """Windows: swallow the benign `ConnectionResetError [WinError 10054]` that + asyncio's proactor prints when a browser/HTTP connection drops abruptly.""" + if sys.platform != "win32": + return + try: + from asyncio.proactor_events import _ProactorBasePipeTransport + except Exception: + return + _orig = _ProactorBasePipeTransport._call_connection_lost + + def _patched(self, exc): + if isinstance(exc, ConnectionResetError): + exc = None # peer reset == normal close; still run the cleanup below + try: + return _orig(self, exc) + except ConnectionResetError: + # _orig's own sock.shutdown() raced a peer reset (WinError 10054), + # which skips the rest of its cleanup — finish it here. + sock = getattr(self, "_sock", None) + if sock is not None: + try: + sock.close() + except OSError: + pass + self._sock = None + server = getattr(self, "_server", None) + if server is not None: + try: + server._detach() + except Exception: + pass + self._server = None + self._called_connection_lost = True + + _ProactorBasePipeTransport._call_connection_lost = _patched + + +def _silence_h11_content_length_race(): + """Large uploads (e.g. a multi-minute reference wav) can have the browser + abort/replace an in-flight preview fetch for the same file while uvicorn + is still streaming its body; h11 then raises LocalProtocolError trying to + close out that half-sent response. It's a benign race — verified the + aborted request doesn't affect the server or any other request, the + browser's follow-up fetch of the same file completes fine — but uvicorn + logs it as a full "Exception in ASGI application" traceback per occurrence. + Drop just that one exception type from uvicorn's logger instead of hiding + all uvicorn.error output.""" + try: + from h11 import LocalProtocolError + except Exception: + return + + class _DropContentLengthRace(logging.Filter): + def filter(self, record): + exc = record.exc_info[1] if record.exc_info else None + msg = str(exc or "") + if isinstance(exc, LocalProtocolError) and "declared Content-Length" in msg: + return False + # uvicorn with httptools raises these RuntimeErrors on the same + # preview race: "shorter" when the browser aborts/replaces the + # fetch, "longer" when gradio sized Content-Length off a large + # upload still being written to disk and the file grew mid-stream. + # The request is already dead / the browser refetches; do not spam + # a full ASGI traceback for either direction. + if isinstance(exc, RuntimeError) and "than Content-Length" in msg: + return False + return True + + logging.getLogger("uvicorn.error").addFilter(_DropContentLengthRace()) + + +def _patch_gradio_render_config_race(): + """Gradio 6.19 上游竞态（gradio-app/gradio#9991 只冻结了 blocks、漏了 fns）： + 本页有 7 个 @gr.render 动态参数区，页面加载/模型切换时多个渲染事件并发， + 一个事件在 get_config 里遍历 session 的 blocks/fns 字典的同时，另一个渲染 + 正往里注册组件，偶发 "RuntimeError: dictionary changed size during + iteration"（前端表现为该次渲染丢失/报错）。get_config 是纯只读操作， + 撞上竞态时稍等重读即可收敛。""" + try: + BlocksConfig = gr.blocks.BlocksConfig + orig = BlocksConfig.get_config + except AttributeError: + return + + def _get_config_retry(self, renderable=None): + for _ in range(10): + try: + return orig(self, renderable) + except RuntimeError: + time.sleep(0.02) + return orig(self, renderable) + + BlocksConfig.get_config = _get_config_retry + + +_silence_proactor_connection_reset() +_silence_h11_content_length_race() +_patch_gradio_render_config_race() + + +def _t(zh, en, language=None, **values): + """Select UI copy in the active language.""" + return ui_text(zh, en, language=language, **values) + +HERE = os.path.dirname(os.path.abspath(__file__)) +PROJECT_ROOT = os.path.dirname(HERE) + +# WebUI-local working dirs, kept next to this file and created on startup. +CONFIG_DIR = os.path.join(HERE, "configs") +OUTPUT_DIR = os.path.join(HERE, "output") +VOICE_DIR = os.path.join(HERE, "voice") +LOG_DIR = os.path.join(HERE, "logs") +for _d in (CONFIG_DIR, OUTPUT_DIR, VOICE_DIR, LOG_DIR): + os.makedirs(_d, exist_ok=True) + +INITIAL_LANGUAGE = load_language() + +PROMPTS_DIR = VOICE_DIR # built-in / reference voices +CATALOG_PATH = os.path.join(CONFIG_DIR, "models_catalog.json") +MODEL_PARAMS_PATH = os.path.join(CONFIG_DIR, "model_params.json") +REQUIRED_FILES_PATH = os.path.join(CONFIG_DIR, "required_files.json") + + +# Executable/name conventions differ per OS: Windows binaries carry a .exe suffix, POSIX ones don't. +EXE_SUFFIX = ".exe" if os.name == "nt" else "" +SERVER_EXE_NAME = "audiocpp_server" + EXE_SUFFIX +GGUF_EXE_NAME = "audiocpp_gguf" + EXE_SUFFIX +# The standalone server launcher, named only in messages telling the user which +# script may be holding the port. +SERVER_LAUNCHER = "run_server.bat" if os.name == "nt" else "run_server.sh" + + +def _cmake_cache_backend(bin_dir): + """gpu/cpu for a build tree whose directory name doesn't say, read from its + CMakeCache.txt (GGML_CUDA:BOOL=ON). Returns None when there is no cache to + read — an installed tree, or a layout we don't recognize.""" + # Walk up from the bin dir to find the build tree's CMakeCache.txt. Single-config + # generators put the exe in build/bin (cache one level up); multi-config ones + # (Visual Studio) nest it in build/bin/Release, so the cache sits two levels up. + cache = None + d = os.path.dirname(bin_dir) + for _ in range(4): + cand = os.path.join(d, "CMakeCache.txt") + if os.path.isfile(cand): + cache = cand + break + parent = os.path.dirname(d) + if parent == d: + break + d = parent + if cache is None: + return None + try: + with open(cache, encoding="utf-8", errors="replace") as fh: + for line in fh: + if line.startswith("GGML_CUDA:"): + return "gpu" if line.rstrip().upper().endswith("ON") else "cpu" + except OSError: + return None + return "cpu" + + +def _discover_dev_bin_dirs(): + """Locate from-source build outputs, newest-wins per backend. + + Two layouts are in the wild. The one README documents is + build/--/bin (windows-cuda-release, linux-cpu-release, …), + where the backend is in the directory name. A plain `cmake -B build` instead + lands in build/bin and says nothing about the backend, so that one is + classified from its CMakeCache.""" + out = {} + for backend, keyword in (("gpu", "*cuda*"), ("cpu", "*cpu*")): + hits = sorted(d for d in glob.glob(os.path.join(PROJECT_ROOT, "build", keyword, "bin")) + if os.path.isfile(os.path.join(d, SERVER_EXE_NAME))) + if hits: + out[backend] = hits[-1] + # A plain `cmake -B build` lands in build/bin with single-config generators + # (Ninja, Makefiles); multi-config generators (Visual Studio, Xcode) nest the + # exe in a per-config subdir, so also look in build/bin/Release and .../Debug. + for plain in (os.path.join(PROJECT_ROOT, "build", "bin"), + os.path.join(PROJECT_ROOT, "build", "bin", "Release"), + os.path.join(PROJECT_ROOT, "build", "bin", "Debug")): + if not os.path.isfile(os.path.join(plain, SERVER_EXE_NAME)): + continue + backend = _cmake_cache_backend(plain) + # Only fill a backend the named-directory scan didn't already find, so an + # explicit build/linux-cuda-release still wins over a stale plain build/. + if backend and backend not in out: + out[backend] = plain + return out + + +DEV_BIN_DIRS = _discover_dev_bin_dirs() + + +def _dev_server_exe(backend): + d = DEV_BIN_DIRS.get(backend) + return os.path.join(d, SERVER_EXE_NAME) if d else "" + + +def _find_bundle_root(): + """Locate the root that holds models/ (and, when packaged, cpu/ gpu/ tools/). + + Three layouts: a from-source dev tree (binaries under build/, models under + PROJECT_ROOT/models), the packaged bundle next to webui/, and webui/ copied + under the bundle for distribution. Override with AUDIOCPP_BUNDLE.""" + env = os.environ.get("AUDIOCPP_BUNDLE") + if env: + return env + for c in (HERE, # webui.py directly in the bundle + PROJECT_ROOT, # webui/ shipped under the bundle + os.path.join(PROJECT_ROOT, "audiocpp-portable")): + if os.path.isdir(os.path.join(c, "gpu")) or os.path.isdir(os.path.join(c, "cpu")): + return c + if any(os.path.isfile(_dev_server_exe(b)) for b in DEV_BIN_DIRS): + return PROJECT_ROOT + return os.path.join(PROJECT_ROOT, "audiocpp-portable") + + +BUNDLE_ROOT = _find_bundle_root() + +def _detect_backend(): + """Which bundle build (gpu/ or cpu/) to launch. AUDIOCPP_BACKEND=gpu|cuda|cpu + wins; otherwise auto-detect: gpu when an NVIDIA driver AND the gpu server + build are both present, else cpu (the server runs fine on the cpu backend, + just slower and with lower model coverage).""" + env = os.environ.get("AUDIOCPP_BACKEND", "").strip().lower() + if env in ("gpu", "cuda"): + return "gpu" + if env: + return env + if os.name == "nt": + has_nvidia = os.path.isfile(os.path.join( + os.environ.get("SystemRoot", r"C:\Windows"), "System32", "nvcuda.dll")) + else: + has_nvidia = shutil.which("nvidia-smi") is not None + if has_nvidia and (os.path.isfile(os.path.join(BUNDLE_ROOT, "gpu", SERVER_EXE_NAME)) + or os.path.isfile(_dev_server_exe("gpu"))): + return "gpu" + if (os.path.isfile(os.path.join(BUNDLE_ROOT, "cpu", SERVER_EXE_NAME)) + or os.path.isfile(_dev_server_exe("cpu"))): + return "cpu" + return "gpu" + + +BACKEND = _detect_backend() +# The server's own backend name: bin dirs are gpu/ vs cpu/, but the server takes +# cuda|cpu|vulkan|metal, and its config default is "cuda" — which a CPU-only +# build rejects at startup, so the temp config must always spell it out. +SERVER_BACKEND = "cuda" if BACKEND == "gpu" else BACKEND +SERVER_EXE = os.path.join(BUNDLE_ROOT, BACKEND, SERVER_EXE_NAME) +if not os.path.isfile(SERVER_EXE) and os.path.isfile(_dev_server_exe(BACKEND)): + SERVER_EXE = _dev_server_exe(BACKEND) +LOG_PATH = os.path.join(LOG_DIR, "audiocpp_server_webui.log") +LOAD_TIMEOUT = int(os.environ.get("AUDIOCPP_LOAD_TIMEOUT", "300")) +GGUF_TYPES = ("orig", "f16", "bf16", "q8_0", "q2_k", "q3_k", "q4_k", "q5_k", "q6_k") + +# Only families with a model package spec can load the runtime GGUF produced by +# audiocpp_gguf. Keep this aligned with model_specs/*.json; a safetensors file +# by itself is not evidence that the corresponding C++ loader supports GGUF. +GGUF_NATIVE_FAMILIES = frozenset({ + "citrinet_asr", + "higgs_audio_stt", + "hviske_asr", + "index_tts2", + "irodori_tts", + "moss_tts_local", + "moss_tts_nano", + "nemotron_asr", + "omnivoice", + "qwen3_asr", + "qwen3_forced_aligner", + "qwen3_tts", + "supertonic", + "vibevoice_asr", +}) + +# These families have input layouts the WebUI can assemble without guessing. +# Other native-GGUF composite packages remain available through the converter +# CLI until their explicit multi-input layout is added here. +GGUF_SIMPLE_MODEL_FAMILIES = frozenset({ + "higgs_audio_stt", + "hviske_asr", + "nemotron_asr", + "qwen3_asr", + "qwen3_forced_aligner", + "vibevoice_asr", +}) +GGUF_WEBUI_CONVERTIBLE_FAMILIES = GGUF_SIMPLE_MODEL_FAMILIES | {"qwen3_tts"} + + +def _find_gguf_exe(): + """Find the converter in a development build or an integrated bundle. + + Keep this separate from SERVER_EXE: developers normally run the executable + directly from their build tree's bin/, whereas portable users have it beside + the server binary under gpu/ or cpu/. Dev locations come from the same + DEV_BIN_DIRS scan the server uses, so non-Windows build-directory names + (linux-cuda-release, a plain build/bin, …) are covered too; the active + backend is tried first, then the other one. + """ + other = "cpu" if BACKEND == "gpu" else "gpu" + candidates = [ + os.environ.get("AUDIOCPP_GGUF"), + *(os.path.join(DEV_BIN_DIRS[b], GGUF_EXE_NAME) + for b in (BACKEND, other) if b in DEV_BIN_DIRS), + os.path.join(BUNDLE_ROOT, BACKEND, GGUF_EXE_NAME), + os.path.join(BUNDLE_ROOT, "gpu", GGUF_EXE_NAME), + os.path.join(BUNDLE_ROOT, "cpu", GGUF_EXE_NAME), + os.path.join(PROJECT_ROOT, "audiocpp-portable", "gpu", GGUF_EXE_NAME), + os.path.join(PROJECT_ROOT, "audiocpp-portable", "cpu", GGUF_EXE_NAME), + ] + seen = set() + for candidate in candidates: + if not candidate: + continue + candidate = os.path.normpath(candidate) + key = os.path.normcase(candidate) + if key in seen: + continue + seen.add(key) + if os.path.isfile(candidate): + return candidate + return None + +# model_manager_webui.py downloads not-yet-installed models in the background. It +# wraps the upstream tools/model_manager.py CLI with resumable, Torch-free +# downloads (falling back to the upstream tool if the wrapper isn't present). +MODELS_ROOT = os.path.join(BUNDLE_ROOT, "models") + + +def _find_model_manager(): + for c in (os.environ.get("AUDIOCPP_MODEL_MANAGER"), + os.path.join(BUNDLE_ROOT, "tools", "model_manager_webui.py"), + os.path.join(PROJECT_ROOT, "tools", "model_manager_webui.py"), + os.path.join(BUNDLE_ROOT, "tools", "model_manager.py"), + os.path.join(PROJECT_ROOT, "tools", "model_manager.py")): + if c and os.path.isfile(c): + return c + return None + + +MODEL_MANAGER = _find_model_manager() + + +def _detect_vram_gb(): + """本机 NVIDIA 显卡显存总量（GB，多卡取最大）；无 nvidia-smi/无 N 卡返回 None。 + 用于对照 catalog 条目的 min_vram_gb 估算值，提示“下载了也可能跑不动”。""" + try: + out = subprocess.run( + ["nvidia-smi", "--query-gpu=memory.total", "--format=csv,noheader,nounits"], + capture_output=True, text=True, timeout=5) + vals = [float(line) for line in out.stdout.split() if line.strip()] + return round(max(vals) / 1024, 1) if vals else None + except Exception: + return None + + +LOCAL_VRAM_GB = _detect_vram_gb() + + +def _vram_shortfall(entry): + """条目估算显存超过本机显存时返回 (需要GB, 本机GB)，否则 None。""" + if BACKEND == "cpu": + return None # CPU 后端跑在系统内存里，显存对照不适用 + need = entry.get("min_vram_gb") + if need and LOCAL_VRAM_GB and float(need) > LOCAL_VRAM_GB: + return float(need), LOCAL_VRAM_GB + return None + +# TTS 语言下拉的语种集合。下拉首位会合成一个 ("Auto", "") 选项（空串=模型默认/ +# 自动检测），所以这里不要再放字面量 "Auto"——带 lang_map 的家族会拒绝它， +# 且界面上会出现两个分不清的 Auto。 +LANGS = ["", "english", "chinese", "french", "german", "italian", + "japanese", "korean", "portuguese", "russian", "spanish"] + +# Which catalog task tokens each tab can drive. do_tts sends text + optional +# reference voice, which fits both plain TTS ("tts") and voice cloning ("clon"). +TTS_TASKS = ("tts", "clon") +ASR_TASKS = ("asr",) +GEN_TASKS = ("gen",) # music/SFX generation, served via the generic /v1/tasks/run route +VC_TASKS = ("vc", "svc", "s2s") # 声音转换：源音频 + 目标音色，走 /v1/tasks/run +SEP_TASKS = ("sep",) # 音源分离：多轨 named_audio_outputs +ANALYZE_TASKS = ("vad", "diar", "align") # 音频分析：segments / speaker_turns / words +VDES_TASKS = ("vdes",) # 声音设计：文字 + 音色描述，走 /v1/audio/speech + +# Per-family behavior for the TTS tab, keyed by catalog `family`. This is how we +# cope with each model wanting different input formats / options: the shared UI +# stays simple, and each family gets its own hint, optional text transform, and +# default request options. A catalog entry can also carry its own `input_hint` / +# `default_options` to override the family profile without editing this file, and +# the "高级参数 (JSON)" box lets you pass ANY model-specific option at run time. +MODEL_PROFILES = { + "vibevoice": { + "input_hint": ( + "**VibeVoice** 多说话人脚本：每行 `Speaker N: 内容`（纯文字自动包装）；" + "多音色用高级参数 `voice_samples`，不要传参考音频。"), + "wrap_speaker_script": True, + # VibeVoice has no internal text chunking. VRAM is bounded since the + # layerwise-prefill/gallocr decode-graph fix, so chunks no longer need to be + # tiny; 600 chars keeps each chunk's generation inside the default + # max_tokens=1200 budget (~1.5 frames/CJK char) and the KV capacity tier + # <= 2048 (~7.1 GB peak on 8 GB GPUs). + "chunk_chars": 600, + # Advanced-parameter widgets are sent only when the user edits them, so an + # untouched num_inference_steps control (displayed 10) silently falls back to + # the model config's ddpm_num_inference_steps=20 server-side. Send 10 (the + # official demo default) explicitly; widgets/JSON still override. + "default_options": {"num_inference_steps": 10}, + }, + "voxcpm2": { + "input_hint": ( + "**VoxCPM2** 声音克隆：上传干净的单人参考音色并填『参考文本』；长文本自动分段；" + "支持⚡流式生成（生成模式选『流式』，边生成边播放）。"), + # 服务端流式（server mode=streaming + /v1/audio/speech stream_format=sse）： + # delta 事件是 base64 的裸 PCM16，不带采样率——只能客户端自带。取值来自模型 + # config.json 的 audio_vae.out_sample_rate（48000）。流式生成要求显式 + # retry_badcase=false（generator.cpp:1259，重试逻辑与已推流的音频冲突）。 + "supports_streaming": True, + "stream_sample_rate": 48000, + # 8G 4060 实测标定（2026-07-05，CLI --log + nvidia-smi 抓峰值）： + # - 峰值 ≈ 固定基线 + audiovae 解码图。基线（权重+KV+生成图）与文本长度/ + # max_tokens 无关：max_tokens 128/300/1200 峰值都是 7634MiB（生成会提前 + # 遇停止符，解码图按实际帧数而非 max_tokens 建）。所以 chunk_chars / max_tokens + # 都压不动基线——真正的杠杆是**权重量化**（catalog session_options 里已设 + # voxcpm2.weight_type=q8_0：bf16 权重 4.6G，峰值 7634→5549MiB）。 + # - audiovae 解码图仍随每段音频长度涨：容量取「≥latent_frames 的最小 2 的幂」 + # (audiovae.cpp:724)，~1.28MB/帧。q8_0 下实测 ~53 字→cap512→6366MiB， + # ~106 字→cap1024→7722MiB（偏紧）。所以每段要短：60 字→cap≤512→~6.4G， + # 留 ~1.7G 给其它占 GPU 的程序。想少接缝可上调，但注意 8G 边界。 + "chunk_chars": 60, + }, + "qwen3_tts": { + "input_hint": ( + "**Qwen3-TTS** 声音克隆：建议上传参考音色并填『参考文本』，否则可能提前截断。"), + }, + "pocket_tts": { + "input_hint": "**PocketTTS**：必须提供参考音色（上传/录制/内置）。", + }, + "chatterbox": { + "input_hint": ( + "**Chatterbox** 声音克隆：必须提供参考音色；语言仅支持英/西/法/德/意/葡/韩" + "（**无中文**），留空=英语。"), + # Chatterbox validates against 2-letter ISO codes (no zh/ja/ru, no auto), + # so the shared dropdown's friendly names are translated here; names not + # listed are genuinely unsupported by the model and rejected up front. + "lang_map": { + "english": "en", "spanish": "es", "french": "fr", "german": "de", + "italian": "it", "portuguese": "pt", "korean": "ko", + }, + }, + "qwen3_asr": { + # Encoder cap: max_source_positions=1500 tokens at 13 tokens/second + # (qwen3_asr_audio_encoder_token_count) -> ~115 s of audio per request. + # 但 8G 卡上先撞显存：thinker prefill 图随时长超线性膨胀（4060 8G 实测 + # 65s 可过、70s 要 13.2GB、75s 要 14.6GB），所以客户端按 max_input_seconds + # =60s 在静音处切段逐段转写再拼接，顺带解决 70~115s 音频原本的 OOM。 + "input_hint": ("**Qwen3-ASR**：长音频自动分段转写；" + "语种/上下文/对话模式见『转写选项』。"), + "max_input_seconds": 60, + }, + "voxtral_realtime": { + "input_hint": ( + "**Voxtral Mini 4B Realtime**：自动语种转写；支持⚡流式转写，" + "勾选后按模型原生音频分块边转边出字；不输出时间戳。"), + "supports_streaming": True, + }, + "ace_step": { + "input_hint": ( + "**ACE-Step** 音乐生成/编辑：提示词写风格/乐器/情绪（英文最佳），可填歌词。" + "编辑类 route 需上传源音频并建议先点『🔍 分析源音频』；参数详解见 webui/README.md。"), + # 原版 turbo UI 默认 shift=3.0（C++ 端默认 1.0，仅 remix/extract 路由自带 3.0）。 + # 控件只发用户改过的项，所以这里显式发送，保证 UI 显示值=实际值。 + "default_options": {"shift": 3.0}, + }, + "stable_audio": { + "input_hint": ( + "**Stable Audio**：提示词**仅英文**，不用歌词；" + "上传源音频可做 init/inpaint（高级参数选 audio_input_kind）。"), + }, + "heartmula": { + "input_hint": ( + "**HeartMuLa**：高级参数 `tags` 必填（逗号分隔），『歌词』填唱词；" + "峰值显存 ~25G，8G 卡跑不动。"), + }, + "vevo2": { + "input_hint": ( + "**Vevo2**：源语音 + 目标音色，默认只换音色（保留说话风格）；" + "风格转换类 route 需 JSON 补 `style_ref` 等，详见 webui/README.md。" + "长音频自动分段，参考音色自动截 ≤10s。"), + # 8G 4060 实测标定（2026-07-04/07-05）：FM 图一次建图，序列长度 = + # 目标音色(prompt) + 源(target) 帧数（均 50fps，见 fm.cpp:782 cond_frames = + # prompt_frames + target_frames）。cond≈25s（源15s+参考10s）就把 8G 吃满、 + # 峰值溢出到共享显存（idle 已占 7.9G/8G）；cond≈20s 勉强、30s 必炸。所以按 + # (预算 − 参考时长) 反推每段源时长，并把参考截到 ≤ ref_max，令 cond 稳定 + # 落在预算内；各段源仍补零到等长以复用缓存图。带 target_text 的编辑类 route + # 不适合分段（会把文本对不上），这类输入本身也放不进显存。 + # 权重加载后约占 5.5-6G，留给图的只有 ~2G；18s 源(cond≈21) 只剩 ~350MB， + # 所以预算取 16（cond≈16，留 ~0.8-1.2G 给峰值/其它占用 GPU 的程序）。 + "vc_chunk_seconds": 15, # 每段源时长上限（会被显存预算进一步压低） + "vc_fm_budget_seconds": 16, # 参考 + 每段源 的总时长预算（8G 安全线） + "vc_ref_max_seconds": 10, # 目标音色参考截断上限 + "vc_min_chunk_seconds": 6, # 每段源时长下限，避免切得过碎 + }, + "seed_vc": { + "input_hint": ( + "**Seed-VC**：源语音 + 目标音色参考（几秒干净人声）；" + "默认 v2 路线，v1 旧路线在高级参数切换。"), + }, + "miocodec": { + "input_hint": "**MioCodec**：codec 重建式转换——源提供内容，参考提供音色。", + }, + "htdemucs": { + "input_hint": "**HTDemucs**：输出 drums / bass / other / vocals 四轨。", + }, + "mel_band_roformer": { + "input_hint": "**Mel-Band RoFormer**：输出人声轨 + 伴奏轨。", + }, + "nemotron_asr": { + "input_hint": ("**Nemotron ASR**：100+ 语种；支持⚡流式转写" + "（勾选后边转边出字，长音频不用干等）。"), + "supports_streaming": True, + }, + "higgs_audio_stt": { + "input_hint": "**Higgs Audio STT**：支持⚡流式转写（勾选后边转边出字）。", + "supports_streaming": True, + }, + "vibevoice_asr": { + "input_hint": "**VibeVoice-ASR**：离线转写，支持自动语种和说话人分段。", + }, + "silero_vad": { + "input_hint": "**Silero VAD**：检测音频中的语音段。", + }, + "marblenet_vad": { + "input_hint": "**MarbleNet VAD**：帧级语音活动检测，输出语音段列表。", + }, + "sortformer_diar": { + "input_hint": "**Sortformer**：说话人分离（谁在何时说话，≤4 人）。", + }, + "qwen3_forced_aligner": { + "input_hint": "**Qwen3 强制对齐**：『对齐文本』填音频原文，输出逐词时间戳（≤115 秒）。", + }, + "index_tts2": { + "input_hint": ( + "**IndexTTS2** 中/英声音克隆：**必须**提供参考音色（上传/录制/内置）；" + "情感控制在『高级参数』：emotion_text 填情绪参考文本（如“你吓死我了！”）+ " + "emotion_alpha 调强度，或 use_emotion_text 从朗读文本自动推断。"), + # 模型只认 zh/en 语种标签（docs/tts.md），共享下拉的其它语言直接拒绝。 + "lang_map": {"chinese": "zh", "english": "en"}, + "require_voice": True, + }, + "irodori_tts": { + "input_hint": ( + "**Irodori-TTS**（日语）：默认无参考直接生成；上传参考音色即自动切换克隆模式。" + "VoiceDesign 版走『声音设计』标签页，用日语 caption 描述音色。"), + "lang_map": {"japanese": "ja"}, + # 会话默认 no_ref=true（忽略参考音频），带参考时必须显式关掉才走克隆路径。 + "no_ref_toggle": True, + # 声音设计标签页的『音色描述』对本家族要发 options.caption（qwen3_tts 走 + # 服务器的 instructions→instruct 映射，irodori session 只读 caption）。 + "vdes_option_key": "caption", + }, + "moss_tts_local": { + "input_hint": ( + "**MOSS-TTS-Local**：纯文本直接生成；克隆时上传参考音色并尽量填『参考文本』。" + "输出 48kHz 立体声；语言下拉可留空（自动）或选择语言作为提示。"), + }, + "moss_tts_nano": { + "input_hint": ( + "**MOSS-TTS-Nano** 100M 轻量：无参考=文本续写式生成（音色随机）；" + "上传参考音色即声音克隆。"), + }, + "supertonic": { + "input_hint": ( + "**Supertonic 3** 预置音色多语种 TTS：在『高级参数』选 voice（M1-M5 男声 / " + "F1-F5 女声）和语速 speaking_rate；支持⚡流式生成；" + "**不支持**参考音频克隆（**无中文**）。"), + # Supertonic 的 C++ 会话支持 mode=streaming，并通过 pull events 按文本段 + # 输出音频。SSE delta 是不带采样率的裸 PCM16，客户端需使用模型的 44.1kHz。 + "supports_streaming": True, + "stream_sample_rate": 44100, + # 模型收 ISO 语种码（en/ko/ja/...），共享下拉的友好名在此转换；chinese 不在 + # 支持列表所以不映射（选中会被 resolve_language 拒绝并提示）。 + "lang_map": { + "english": "en", "french": "fr", "german": "de", "italian": "it", + "japanese": "ja", "korean": "ko", "portuguese": "pt", + "russian": "ru", "spanish": "es", + }, + }, +} + +# Concise English hints. The Chinese catalog/profile copy remains the detailed +# reference; English intentionally keeps these notes short so model cards stay +# aligned on narrower screens. +MODEL_HINTS_EN = { + "vibevoice": "**VibeVoice**: use one `Speaker N:` line per speaker. Use `voice_samples` for multiple voices.", + "voxcpm2": "**VoxCPM2**: upload a clean voice reference and its transcript. Streaming is supported.", + "qwen3_tts": "**Qwen3-TTS**: a voice reference and matching transcript are recommended.", + "pocket_tts": "**PocketTTS** requires a voice reference.", + "chatterbox": "**Chatterbox** requires a voice reference and supports en/es/fr/de/it/pt/ko.", + "qwen3_asr": "**Qwen3-ASR** automatically splits long audio. Language and context are optional.", + "voxtral_realtime": "**Voxtral Mini 4B Realtime** auto-detects language and supports streaming transcription. Timestamps are not exposed.", + "ace_step": "**ACE-Step**: describe style, instruments and mood. Editing routes require source audio.", + "stable_audio": "**Stable Audio** accepts English prompts only. Source audio enables init/inpaint.", + "heartmula": "**HeartMuLa** requires `tags` and lyrics. Estimated peak VRAM is about 25 GB.", + "vevo2": "**Vevo2**: provide source audio and a target voice. Long audio is split automatically.", + "seed_vc": "**Seed-VC**: provide source audio and a clean target voice reference.", + "miocodec": "**MioCodec** reconstructs source content with the reference voice.", + "htdemucs": "**HTDemucs** outputs drums, bass, other and vocals.", + "mel_band_roformer": "**Mel-Band RoFormer** outputs vocals and accompaniment.", + "nemotron_asr": "**Nemotron ASR** supports 100+ languages and streaming transcription.", + "higgs_audio_stt": "**Higgs Audio STT** supports streaming transcription.", + "vibevoice_asr": "**VibeVoice-ASR** uses offline transcription with automatic language and speaker segmentation.", + "silero_vad": "**Silero VAD** detects speech segments.", + "marblenet_vad": "**MarbleNet VAD** detects frame-level speech activity.", + "sortformer_diar": "**Sortformer** identifies who spoke when (up to four speakers).", + "qwen3_forced_aligner": "**Qwen3 Forced Aligner** requires the source transcript and returns word timestamps.", + "index_tts2": "**IndexTTS2** (zh/en) requires a voice reference; emotion controls live in advanced parameters.", + "irodori_tts": "**Irodori-TTS** (Japanese) works without a reference; uploading one enables voice cloning.", + "moss_tts_local": "**MOSS-TTS-Local**: plain text works; add a voice reference and its transcript to clone. 48 kHz stereo output.", + "moss_tts_nano": "**MOSS-TTS-Nano** 100M: continuation mode without a reference, voice clone with one.", + "supertonic": "**Supertonic 3**: preset voices and streaming are supported; no voice cloning, no Chinese.", +} +# Qwen3-ASR 可强制的语种（模型 config.json 的 support_languages，prompt 里用英文名； +# 留空/Auto = 自动检测）。citrinet 等其它 ASR 族忽略该字段。 +QWEN3_ASR_LANGUAGES = [ + ("中文", "Chinese"), ("英语", "English"), ("粤语", "Cantonese"), + ("日语", "Japanese"), ("韩语", "Korean"), ("俄语", "Russian"), + ("法语", "French"), ("德语", "German"), ("西班牙语", "Spanish"), + ("葡萄牙语", "Portuguese"), ("意大利语", "Italian"), ("阿拉伯语", "Arabic"), + ("印尼语", "Indonesian"), ("泰语", "Thai"), ("越南语", "Vietnamese"), + ("土耳其语", "Turkish"), ("印地语", "Hindi"), ("马来语", "Malay"), + ("荷兰语", "Dutch"), ("瑞典语", "Swedish"), ("丹麦语", "Danish"), + ("芬兰语", "Finnish"), ("波兰语", "Polish"), ("捷克语", "Czech"), + ("菲律宾语", "Filipino"), ("波斯语", "Persian"), ("希腊语", "Greek"), + ("罗马尼亚语", "Romanian"), ("匈牙利语", "Hungarian"), ("马其顿语", "Macedonian"), +] + +DEFAULT_PROFILE = {"input_hint": "", "input_hint_en": "", "wrap_speaker_script": False, + "default_options": {}, + # C++ 会话实现了 IStreamingVoiceTaskSession 的家族（server 需以 + # mode=streaming 加载才走流式路由）；见 registry 各家族 loader。 + "supports_streaming": False, + # Families with internal chunking handle long text fine; the client + # split only exists to bound each HTTP request (no 900 s timeout) + # and surface progress, so the budget can stay coarse. + "chunk_chars": 1000} + +# One "Speaker N:" line (any speaker index) is enough to treat text as a script. +_SPEAKER_RE = re.compile(r"^\s*Speaker\s+\d+\s*:", re.IGNORECASE | re.MULTILINE) + + +def profile_for(entry): + prof = {**DEFAULT_PROFILE, **MODEL_PROFILES.get(entry.get("family", ""), {})} + prof["input_hint_en"] = MODEL_HINTS_EN.get(entry.get("family", ""), "") + if entry.get("input_hint"): + prof["input_hint"] = entry["input_hint"] + if entry.get("input_hint_en"): + prof["input_hint_en"] = entry["input_hint_en"] + if entry.get("default_options"): + prof["default_options"] = {**prof.get("default_options", {}), **entry["default_options"]} + return prof + + +def supports_streaming(model_id): + entry = catalog_by_id(model_id) if model_id else None + return bool(entry) and bool(profile_for(entry).get("supports_streaming")) + + +def asr_stream_update(model_id): + """Reset and show the ASR streaming toggle for the selected model.""" + return gr.update(visible=supports_streaming(model_id), value=False) + + +def model_hint_for(model_id, language=None): + entry = catalog_by_id(model_id) if model_id else None + if not entry: + return "" + language = language or get_language() + prof = profile_for(entry) + hint = _t(prof["input_hint"], prof["input_hint_en"], language) + short = _vram_shortfall(entry) + if short: + warn = _t( + "⚠️ **显存提示**：该模型估算需 **≥{need:g}G** 显存，本机为 **{local:g}G**，运行可能很慢{tail}", + "⚠️ **VRAM**: estimated **≥{need:g} GB**, detected **{local:g} GB**. Performance may be poor{tail}", + language, need=short[0], local=short[1], + tail=(_t("，请谨慎下载。", ".", language) if not entry["installed"] else + _t("。", ".", language))) + hint = warn + ("\n\n" + hint if hint else "") + return hint + + +def resolve_language(prof, language): + """Translate the shared language dropdown into what the selected family + expects. Most families (Qwen3-TTS, VibeVoice, …) take the UI's friendly + names as-is, so they have no `lang_map` and the value passes through. A + family with a restricted language set (Chatterbox: 2-letter ISO codes, no + Chinese/Japanese/Russian, no auto-detect) supplies a `lang_map`; its names + are converted and anything it can't do — including "Auto" — is rejected here + with an actionable message instead of a raw server 500.""" + lang = (language or "").strip() + lang_map = prof.get("lang_map") + if not lang_map: + return lang + if not lang: + return "" # 留空 = 用模型默认 + code = lang_map.get(lang.lower()) + if code is not None: + return code + raise gr.Error(_t( + "所选模型不支持语言「{selected}」。请改选：{supported}，或选 Auto 使用模型默认。", + "This model does not support {selected}. Choose {supported}, or choose Auto for the model default.", + selected=language, supported=" / ".join(lang_map))) + + +def _as_speaker_script(text): + """Wrap plain text into `Speaker 0:` lines when it isn't already a script.""" + if _SPEAKER_RE.search(text or ""): + return text + lines = [ln.strip() for ln in (text or "").splitlines() if ln.strip()] + return "\n".join(f"Speaker 0: {ln}" for ln in lines) if lines else text + + +def _vibevoice_punctuate_script(text): + """VibeVoice is much less stable on tiny lines without sentence punctuation.""" + out = [] + for raw in (text or "").splitlines(): + line = raw.strip() + if not line: + continue + m = _SPEAKER_LINE_RE.match(line) + if not m: + out.append(line) + continue + prefix, body = m.group(1), m.group(2).strip() + if body and body[-1] not in "。！？!?.,;；:": + body += "。" + out.append(f"{prefix} {body}" if body else prefix) + return "\n".join(out) if out else text + + +def _vibevoice_text_max_tokens(chunk, ui_default=1200): + """Estimate VibeVoice speech tokens from text, not reference-prompt length.""" + body = re.sub(r"(?im)^\s*Speaker\s+\d+\s*:\s*", "", chunk or "") + cjk = sum(1 for ch in body if "\u4e00" <= ch <= "\u9fff") + non_space = sum(1 for ch in body if not ch.isspace()) + non_cjk = max(0, non_space - cjk) + pauses = sum(1 for ch in body if ch in "，。！？；：,.!?;:") + estimate = int(cjk * 2.2 + non_cjk * 0.45 + pauses * 3.0 + 8) + return max(18, min(int(ui_default), estimate)) + + +# VibeVoice 1.5B 对超短脚本从第 1 帧起整段胡言乱语——模型级缺陷（长播客数据训练， +# 短文本 OOD），与参考音色/CFG/扩散步数/中英文/seed 全部无关（2026-07-08 A/B+ASR +# 转写实测：27 字必炸、40 字逐字正确；按上面估算公式 69 token 炸 / 87 token 过）。 +# 低于阈值直接拦截；生成出来只会是垃圾，调参数救不了。 +_VIBEVOICE_MIN_EST_TOKENS = 85 + + +def _merge_short_vibevoice_tail(chunks): + """分段时留下的过短尾段同样会胡言乱语：并回前一段。前段最多超预算几十字， + 仍在 max_tokens=1200 封顶的显存包络内。""" + while len(chunks) > 1 and ( + _vibevoice_text_max_tokens(chunks[-1]) < _VIBEVOICE_MIN_EST_TOKENS): + tail = chunks.pop() + chunks[-1] = chunks[-1] + "\n" + tail + return chunks + + +# --- client-side long-text chunking ------------------------------------------ +# Long text is synthesized as one HTTP request per chunk and concatenated here. +# That keeps every request bounded (no 900 s timeout, works for families without +# internal chunking) and lets the UI show real per-chunk progress. +_SPEAKER_LINE_RE = re.compile(r"^\s*(Speaker\s+\d+\s*:)\s*(.*)$", re.IGNORECASE) +_SENTENCE_RE = re.compile(r"[^。！？!?；;…]*[。！？!?；;…]+|[^。！？!?；;…]+$") + + +def _split_long_line(line, budget): + """Split one overlong line at sentence ends into pieces of <= budget chars, + re-attaching its `Speaker N:` prefix (if any) to every piece.""" + m = _SPEAKER_LINE_RE.match(line) + prefix, body = (m.group(1) + " ", m.group(2)) if m else ("", line.strip()) + pieces, cur = [], "" + for sent in _SENTENCE_RE.findall(body): + if cur and len(cur) + len(sent) > budget: + pieces.append(prefix + cur) + cur = "" + cur += sent + if cur: + pieces.append(prefix + cur) + return pieces or [line] + + +def _split_tts_chunks(text, budget): + """Group non-empty lines into chunks of <= budget chars. A line is never + split across chunks unless it alone exceeds the budget (then it is split at + sentence boundaries). Returns a list of chunk strings.""" + units = [] + for ln in (text or "").splitlines(): + if not ln.strip(): + continue + units.extend(_split_long_line(ln, budget) if len(ln) > budget else [ln]) + chunks, cur, cur_len = [], [], 0 + for unit in units: + sep = 1 if cur else 0 # the "\n" join separator counts toward the budget + if cur and cur_len + sep + len(unit) > budget: + chunks.append("\n".join(cur)) + cur, cur_len, sep = [], 0, 0 + cur.append(unit) + cur_len += sep + len(unit) + if cur: + chunks.append("\n".join(cur)) + return chunks or ([text] if (text or "").strip() else []) + + +def _concat_wavs(blobs, out_path, keep_ratios=None): + """Concatenate same-format WAV byte blobs into one file at out_path. + keep_ratios[i]：每段只保留前一部分（分段转换把源补零到等长后， + 按有效占比截掉对应输出的尾部静音）。""" + params, frames = None, [] + for i, blob in enumerate(blobs): + with wave.open(io.BytesIO(blob)) as w: + fmt = (w.getnchannels(), w.getsampwidth(), w.getframerate()) + if params is None: + params = fmt + elif fmt != params: + raise gr.Error(_t("分段音频格式不一致：{left} != {right}", + "Audio chunk formats differ: {left} != {right}", + left=fmt, right=params)) + n = w.getnframes() + if keep_ratios is not None: + n = max(1, min(n, int(round(n * keep_ratios[i])))) + frames.append(w.readframes(n)) + with wave.open(out_path, "wb") as w: + w.setnchannels(params[0]) + w.setsampwidth(params[1]) + w.setframerate(params[2]) + for data in frames: + w.writeframes(data) + + +def _audio_duration_seconds(path): + """Duration of a local audio file, or None when not measurable. wave covers + WAV, i.e. Gradio mic recordings and the typical uploads here; other formats + just skip the duration note instead of failing the request.""" + try: + with wave.open(path, "rb") as w: + rate = w.getframerate() + return (w.getnframes() / float(rate)) if rate else None + except Exception: + return None + + +# 已转码文件缓存：gradio 的临时路径按内容哈希命名，同一上传重复运行不重复转码。 +_WAV_CACHE = {} + + +def _find_ffmpeg(): + """转码用的 ffmpeg：随 webui 分发的 ffmpeg（Windows 下为 ffmpeg.exe）优先，其次 PATH。""" + bundled = os.path.join(HERE, "ffmpeg" + EXE_SUFFIX) + if os.path.exists(bundled): + return bundled + found = shutil.which("ffmpeg") + if not found: + raise gr.Error(_t("找不到 ffmpeg，无法转码非 WAV 音频。", + "ffmpeg was not found; non-WAV audio cannot be converted.")) + return found + + +def _ensure_ascii_path(path): + """若路径含非 ASCII 字符，复制到纯 ASCII 临时文件。 + C++ server 在 Windows 上无法打开含中文等字符的路径。""" + try: + path.encode("ascii") + return path + except UnicodeEncodeError: + pass + fd, out = tempfile.mkstemp(prefix="audiocpp_asc_", suffix=".wav") + os.close(fd) + shutil.copy2(path, out) + return out + + +def _wait_file_stable(path, timeout=8.0, interval=0.08, stable_ticks=3): + """Wait until a just-uploaded/recorded file stops changing on disk. + On Windows, the browser can start/restart audio preview fetches while Gradio + is still replacing a large temp file. Waiting for a few equal size samples + before copying avoids half-written preview files.""" + if not path: + return False + deadline = time.time() + timeout + last_size = -1 + ticks = 0 + while time.time() < deadline: + try: + size = os.path.getsize(path) + except OSError: + size = -1 + if size > 0 and size == last_size: + ticks += 1 + if ticks >= stable_ticks: + return True + else: + last_size = size + ticks = 0 + time.sleep(interval) + return os.path.exists(path) + + +def _safe_audio_ext(path): + ext = os.path.splitext(path)[1].lower() + try: + ext.encode("ascii") + if 0 < len(ext) <= 8: + return ext + except Exception: + pass + return ".wav" + + +def _stage_upload(path, force_copy=False): + """上传/录制完成后，把输入音频换成短 ASCII 临时名再交回控件。 + + 关键点：长音频即使路径本身已经是短 ASCII，也强制复制到一个新的稳定文件。 + 否则在同一个 gr.Audio 控件里点 X 清空再上传长音频时，旧预览 fetch 与新预览 + fetch 容易竞态，前端波形会卡住；短音频通常因为读取很快不明显。""" + if not path or not os.path.exists(path): + return path + + _wait_file_stable(path) + + # 已经是我们 staging 过的文件，不要再次复制，避免 .change / .upload 回写后循环。 + base = os.path.basename(path) + if base.startswith(("audiocpp_up_", "audiocpp_rec_", "audiocpp_asc_")): + return path + + try: + size = os.path.getsize(path) + except OSError: + size = 0 + + must_copy = force_copy or size >= 4 * 1024 * 1024 + try: + path.encode("ascii") + ascii_short = len(path) < 180 + except UnicodeEncodeError: + ascii_short = False + must_copy = True + + if ascii_short and not must_copy: + return path + + ext = _safe_audio_ext(path) + fd, out = tempfile.mkstemp(prefix="audiocpp_up_", suffix=ext) + os.close(fd) + shutil.copy2(path, out) + return out + + +def _stage_recording(path): + """麦克风录音结束后总是换成一个新的稳定临时文件。""" + return _stage_upload(path, force_copy=True) + + +def _ensure_wav(path, target_sr=None): + """server 端只有 WAV 读取器（其它格式报 invalid WAV RIFF header），Gradio 上传 + 的 flac/mp3/ogg/m4a 等在这里先用 ffmpeg 转成 16-bit PCM WAV 临时文件再发； + 已是 RIFF/WAVE 的原样透传。target_sr：模型 prepare() 硬校验采样率的族（音源 + 分离要 44.1k）传入目标值，采样率不符的输入（含 WAV）顺带重采样。""" + if not path: + return path + try: + with open(path, "rb") as f: + head = f.read(12) + except OSError as e: + raise gr.Error(_t("读不到音频文件 {path}：{error}", + "Cannot read audio file {path}: {error}", path=path, error=e)) + if head[:4] == b"RIFF" and head[8:12] == b"WAVE": + if target_sr is None: + return _ensure_ascii_path(path) + try: + with wave.open(path, "rb") as w: + if w.getframerate() == target_sr: + return _ensure_ascii_path(path) + except Exception: + pass # 非 PCM WAV：wave 读不了，交给 ffmpeg 重写 + key = (path, target_sr) + cached = _WAV_CACHE.get(key) + if cached and os.path.exists(cached): + return cached + ffmpeg = _find_ffmpeg() + ext = os.path.splitext(path)[1].lower() or "（无扩展名）" + fd, out = tempfile.mkstemp(prefix="audiocpp_in_", suffix=".wav") + os.close(fd) + cmd = [ffmpeg, "-y", "-v", "error", "-i", path, "-map", "0:a:0"] + if target_sr: + cmd += ["-ar", str(target_sr)] + cmd += ["-c:a", "pcm_s16le", out] + proc = subprocess.run(cmd, capture_output=True, text=True) + if proc.returncode != 0 or not os.path.getsize(out): + try: + os.remove(out) + except OSError: + pass + err = (proc.stderr or "").strip()[-300:] + raise gr.Error(_t("ffmpeg 转码 {ext} → wav 失败：{error}", + "ffmpeg conversion {ext} → wav failed: {error}", + ext=ext, error=err or _t("未知错误", "unknown error"))) + note = (_t("，重采样到 {sr}Hz", ", resampled to {sr}Hz", sr=target_sr) + if target_sr else "") + _ui_log(_t("输入转码：{name}（{ext}）→ 16-bit PCM WAV{note}", + "input transcoded: {name} ({ext}) → 16-bit PCM WAV{note}", + name=os.path.basename(path), ext=ext, note=note)) + _WAV_CACHE[key] = out + return out + + +def _to_16k_mono_wav(path, target_sr=16000): + """VAD / 说话人分离 / 强制对齐族要求 16 kHz 单声道输入（Silero、Sortformer 对 + 非 16k 直接报错），这里用 wave+numpy 把 PCM WAV 转换成 16k 单声道临时文件。 + 已是 16k 单声道、或非 PCM WAV（wave 读不了）时原样透传，由 server 决定成败。""" + try: + with wave.open(path, "rb") as w: + sr, ch, sw = w.getframerate(), w.getnchannels(), w.getsampwidth() + raw = w.readframes(w.getnframes()) + except Exception: + return path + if sr == target_sr and ch == 1: + return path + if sw == 2: + data = np.frombuffer(raw, dtype=np.int16).astype(np.float32) / 32768.0 + elif sw == 4: + data = np.frombuffer(raw, dtype=np.int32).astype(np.float32) / 2147483648.0 + elif sw == 3: + b = np.frombuffer(raw, dtype=np.uint8).reshape(-1, 3) + i24 = (b[:, 0].astype(np.int32) | (b[:, 1].astype(np.int32) << 8) | + (b[:, 2].astype(np.int32) << 16)) + i24 -= (i24 & 0x800000) << 1 # sign-extend 24-bit + data = i24.astype(np.float32) / 8388608.0 + elif sw == 1: + data = (np.frombuffer(raw, dtype=np.uint8).astype(np.float32) - 128.0) / 128.0 + else: + return path + if ch > 1: + data = data[: len(data) // ch * ch].reshape(-1, ch).mean(axis=1) + if sr != target_sr and len(data) > 0: + n_out = max(1, int(round(len(data) * target_sr / sr))) + x_old = np.arange(len(data), dtype=np.float64) / sr + x_new = np.arange(n_out, dtype=np.float64) / target_sr + data = np.interp(x_new, x_old, data).astype(np.float32) + fd, out = tempfile.mkstemp(prefix="audiocpp_16k_", suffix=".wav") + os.close(fd) + pcm = np.clip(data * 32767.0, -32768, 32767).astype(np.int16) + with wave.open(out, "wb") as w: + w.setnchannels(1) + w.setsampwidth(2) + w.setframerate(target_sr) + w.writeframes(pcm.tobytes()) + return out + + +def _split_wav_chunks(path, max_seconds, min_search_frac=0.6, win_ms=50, + pad_to_max=True): + """把 PCM WAV 切成若干段临时 wav，返回 [(路径, 有效占比)]。用于 vevo2 歌声 + 转换：FM 图按整段长度一次建图，8G 卡放不下长音频；且同一 server 上不同长度 + 的请求会重建图并叠加占用显存（实测 20s 成功后 17s 反要 9.9GB），而形状相同 + 的请求可复用缓存图（实测显存平稳）。所以**每段都补零到恰好 max_seconds**， + 占比供拼接时截掉补零对应的尾部输出。切点选在 + [起点+max*min_search_frac, 起点+max] 区间内能量最低的 win_ms 窗口中心 + （典型是呼吸/间奏处）；读不了的（非 PCM WAV）原样返回不分段。 + pad_to_max=False：不补零（ASR 分段转写用——补出的尾部静音只会浪费编码器 + token 甚至诱发幻听，转写也不需要各段图形状一致）。""" + try: + with wave.open(path, "rb") as w: + sr, ch, sw = w.getframerate(), w.getnchannels(), w.getsampwidth() + n = w.getnframes() + raw = w.readframes(n) + except Exception: + return [(path, 1.0)] # 非 PCM WAV：不分段，交给 server + if sw != 2 or n <= 0: + return [(path, 1.0)] + data = np.frombuffer(raw, dtype=np.int16) + frames = data.reshape(-1, ch) if ch > 1 else data.reshape(-1, 1) + mono = np.abs(frames.astype(np.float32)).mean(axis=1) + win = max(1, int(sr * win_ms / 1000.0)) + env_len = max(1, len(mono) // win) + env = mono[: env_len * win].reshape(env_len, win).mean(axis=1) + + max_f, lo_f = int(max_seconds * sr), int(max_seconds * min_search_frac * sr) + spans, pos = [], 0 + while n - pos > max_f: + w0 = min((pos + lo_f) // win, env_len - 1) + w1 = min((pos + max_f) // win, env_len) + wi = w0 + int(np.argmin(env[w0:w1])) if w1 > w0 else w1 - 1 + cut = min(n, wi * win + win // 2) + spans.append((pos, cut)) + pos = cut + if pos < n: + spans.append((pos, n)) + outs = [] + for i, (a, b) in enumerate(spans): + fd, out = tempfile.mkstemp(prefix=f"audiocpp_vcseg{i}_", suffix=".wav") + os.close(fd) + seg = frames[a:b] + if pad_to_max and len(seg) < max_f: # 全部段补零到等长，保证图形状一致 + pad = np.zeros((max_f - len(seg), ch), dtype=np.int16) + seg = np.concatenate([seg, pad], axis=0) + with wave.open(out, "wb") as ww: + ww.setnchannels(ch) + ww.setsampwidth(sw) + ww.setframerate(sr) + ww.writeframes(seg.tobytes()) + outs.append((out, (b - a) / float(max_f))) + return outs + + +def _trim_wav_seconds(path, max_seconds): + """把 PCM WAV 截到前 max_seconds 秒。vevo2 的音色参考超过约 10s 对音色几乎 + 没有额外贡献，却会让 FM 图的 prompt 段变长、和源音频一起把显存吃满，所以 + 转换前先截短。短于上限、或非 PCM WAV（wave 读不了）的原样返回。""" + try: + with wave.open(path, "rb") as w: + sr, ch, sw = w.getframerate(), w.getnchannels(), w.getsampwidth() + keep = int(max_seconds * sr) + if keep <= 0 or w.getnframes() <= keep: + return path + raw = w.readframes(keep) + except Exception: + return path + fd, out = tempfile.mkstemp(prefix="audiocpp_ref_", suffix=".wav") + os.close(fd) + with wave.open(out, "wb") as ww: + ww.setnchannels(ch) + ww.setsampwidth(sw) + ww.setframerate(sr) + ww.writeframes(raw) + return out + + +def _parse_adv_options(raw): + raw = (raw or "").strip() + if not raw: + return {} + try: + obj = json.loads(raw) + except Exception as e: + raise gr.Error(_t("高级参数不是合法 JSON：{error}", + "Advanced options are not valid JSON: {error}", error=e)) + if not isinstance(obj, dict): + raise gr.Error(_t('高级参数必须是 JSON 对象，例如 {"num_inference_steps": 10}', + 'Advanced options must be a JSON object, e.g. {"num_inference_steps": 10}')) + return obj + + +# Map known server-error fragments to an actionable Chinese hint, so a raw 500 +# like "requires a session voice via --voice-ref" becomes "请上传参考音色". +# Ordered specific -> generic; server_error() takes the FIRST match. +ERROR_HINTS = [ + (re.compile(r"failed to allocate .{0,40}graph|out of memory|cudaMalloc", re.I), + "🧠 显存不足：这次请求的计算图放不进剩余显存，通常是音频/文本太长。" + "请剪短或分段后重试。"), + (re.compile(r"invalid WAV RIFF header", re.I), + "🎵 server 只支持 WAV 音频：上传的文件 webui 会自动转码，" + "手动填路径的参数（如 voice_samples）请先转成 .wav。"), + (re.compile(r"unsupported Chatterbox language", re.I), + "🌐 Chatterbox 只支持 en/es/fr/de/it/pt/ko（无中文/日文/俄文，也没有自动检测）。" + "请在“语言”里改选受支持的语言，或选 Auto 用默认（英语）。"), + (re.compile(r"Stable Audio.{0,80}(English|prompt)|prompt.{0,80}(English|Stable Audio)", re.I), + "🎵 Stable Audio 的提示词只支持英文。请把“提示词”改成英文后重试。"), + (re.compile(r"max_source_positions", re.I), + "⏱ 音频过长：Qwen3-ASR 编码器上限 1500 token（约 13 token/秒），" + "单次最多约 115 秒。请把音频剪短或分段后再转写。"), + (re.compile(r"exceeds fixed graph capacity|session_len_sec exceeds", re.I), + "📏 音频超过模型的固定图容量（Sortformer 默认 20 秒、上限约 120 秒）。" + "webui 的说话人分离/对话模式会按时长自动重载；仍报错说明超过 120 秒上限，" + "请先剪短音频。"), + (re.compile(r"combine voice_samples|voice_samples.{0,20}voice_ref", re.I), + "🔀 voice_samples 与单个参考音色不能同时用：多说话人时请不要上传参考音色。"), + (re.compile(r"cached voice id", re.I), + "🎤 需要参考音频文件（不是 voice id）：请上传/录制一段参考音色。"), + (re.compile(r"no valid Speaker|Speaker\s+N", re.I), + "📝 需要多说话人脚本：每行写成 `Speaker 0: 内容`（多角色用 Speaker 0/1/…）。"), + (re.compile(r"reference[-_ ]?text", re.I), + "🗒 需要参考文本：在“参考文本”里填参考音频里说的原话。"), + (re.compile(r"voice[-_ ]?ref|voice[-_ ]?id|session voice|speaker reference|" + r"requires .{0,40}voice|requires audio", re.I), + "🎤 该模型需要参考音色：上传/录制一段参考音频，或选一个内置参考音色后重试。"), +] + +ERROR_HINTS_EN = [ + (ERROR_HINTS[0][0], "Not enough VRAM. Shorten or split the input."), + (ERROR_HINTS[1][0], "The server requires WAV audio. Uploaded files are converted automatically."), + (ERROR_HINTS[2][0], "Chatterbox supports en/es/fr/de/it/pt/ko only."), + (ERROR_HINTS[3][0], "Stable Audio prompts must be in English."), + (ERROR_HINTS[4][0], "The audio exceeds the Qwen3-ASR input limit. Split it and retry."), + (ERROR_HINTS[5][0], "The audio exceeds this model's graph capacity."), + (ERROR_HINTS[6][0], "Do not combine voice_samples with a single voice reference."), + (ERROR_HINTS[7][0], "Upload a voice reference file."), + (ERROR_HINTS[8][0], "Use one `Speaker N:` line per speaker."), + (ERROR_HINTS[9][0], "Enter the transcript spoken in the reference audio."), + (ERROR_HINTS[10][0], "This model requires a voice reference."), +] + + +def _extract_server_message(text): + try: + return json.loads(text)["error"]["message"] or text + except Exception: + return (text or "").strip() + + +def server_error(entry, status, text, extra=None): + """Build a friendly gr.Error from a non-200 server response.""" + msg = _extract_server_message(text) + language = get_language() + hints = ERROR_HINTS_EN if language == "en" else ERROR_HINTS + hint = next((h for pat, h in hints if pat.search(msg)), None) + if hint: + hint = _t(hint, hint, language) + parts = [f"❌ server {status}"] + if hint: + parts.append("💡 " + hint) + else: + parts[0] = f"❌ server {status}：{msg[:400]}" + if extra: + parts.append(extra) + if entry and not hint: + prof = profile_for(entry) + ih = prof.get("input_hint_en" if language == "en" else "input_hint") + if ih: + parts.append("ℹ️ " + _t(ih, ih, language)) + return gr.Error("\n\n".join(parts)) + + +def connection_error(error): + return gr.Error(_t( + "无法连接 server @ {server}：{error}\n💡 server 可能已退出，请重新加载模型。", + "Cannot connect to server @ {server}: {error}\nReload the model and retry.", + server=SERVER, error=error)) + + +def _msg_from_error(e): + """Human-readable text from a gr.Error/exception, for inline (non-popup) display.""" + return getattr(e, "message", None) or str(e) or _t("未知错误", "Unknown error") + + +# Fallback catalog if models_catalog.json is missing/unreadable. +DEFAULT_CATALOG = { + "host": "127.0.0.1", "port": 8080, "device": 0, "threads": 1, + "models": [ + {"id": "qwen3-tts", "display_name": "Qwen3-TTS 0.6B (tts)", + "family": "qwen3_tts", "path": "models/Qwen3-TTS-12Hz-0.6B-Base", + "task": "tts", "mode": "offline"}, + {"id": "vibevoice", "display_name": "VibeVoice 1.5B (tts)", + "family": "vibevoice", "path": "models/VibeVoice-1.5B", + "task": "tts", "mode": "offline"}, + {"id": "qwen3-asr", "display_name": "Qwen3-ASR 0.6B (asr)", + "family": "qwen3_asr", "path": "models/Qwen3-ASR-0.6B", + "task": "asr", "mode": "offline"}, + ], +} + + +def _load_catalog(): + if os.path.isfile(CATALOG_PATH): + try: + with open(CATALOG_PATH, "r", encoding="utf-8") as f: + return json.load(f) + except Exception as e: + print(f"[webui] failed to read {CATALOG_PATH}: {e}; using defaults") + return DEFAULT_CATALOG + + +def _load_model_params(): + """Per-model/family advanced-parameter specs (configs/model_params.json). + Catalog id entries override family entries; a missing/broken file -> {}.""" + if os.path.isfile(MODEL_PARAMS_PATH): + try: + with open(MODEL_PARAMS_PATH, "r", encoding="utf-8") as f: + return json.load(f) + except Exception as e: + print(f"[webui] failed to read {MODEL_PARAMS_PATH}: {e}; no param controls") + return {} + + +def _load_required_files(): + """download_id -> 安装完成后模型目录里必须存在的文件清单（configs/required_files.json， + 由 model_manager.py CATALOG 的 required_files 预生成，含 .pt->.safetensors 等转换后 + 的最终布局）。用于把“手动拷贝/下载中断的不完整目录”和“已安装”区分开——不完整目录 + server 端只会报 no registered model loader，用户看不出缺了什么。 + 文件缺失/损坏 -> {}（完整性检查停用，退回“目录存在即已安装”的旧行为）。""" + if os.path.isfile(REQUIRED_FILES_PATH): + try: + with open(REQUIRED_FILES_PATH, "r", encoding="utf-8") as f: + data = json.load(f) + return {k: v for k, v in data.items() if isinstance(v, list)} + except Exception as e: + print(f"[webui] failed to read {REQUIRED_FILES_PATH}: {e}; " + + _t("跳过模型完整性检查", "skipping model integrity check")) + return {} + + +CATALOG = _load_catalog() +MODEL_PARAMS = _load_model_params() +REQUIRED_FILES = _load_required_files() +HOST = CATALOG.get("host", "127.0.0.1") +PORT = int(CATALOG.get("port", 8080)) +DEVICE = int(CATALOG.get("device", 0)) +THREADS = int(os.environ.get("AUDIOCPP_THREADS") or CATALOG.get("threads", 1)) +if BACKEND == "cpu" and THREADS <= 1: + # catalog 里的 threads=1 是按 CUDA 调的（GPU 路径不吃这个值）；CPU 后端的 + # ggml 计算线程数就是它，单线程没法用 —— 默认全核减一，留一个核给 UI/系统。 + THREADS = max(1, (os.cpu_count() or 4) - 1) + +# If the user points us at an existing server, keep everything consistent with it. +_ENV_SERVER = os.environ.get("AUDIOCPP_SERVER") +if _ENV_SERVER: + _u = urlparse(_ENV_SERVER) + HOST = _u.hostname or HOST + PORT = _u.port or PORT + SERVER = _ENV_SERVER.rstrip("/") +else: + SERVER = f"http://{HOST}:{PORT}" + +# --- managed server process state ------------------------------------------ +_proc_lock = threading.Lock() +_server_proc = None # subprocess.Popen we launched, or None +_loaded_id = None # model id our managed server is serving +_loaded_session_options = None # 随本次加载写进 server config 的 session_options +_loaded_mode = None # 本次加载的运行模式（offline/streaming），None=未加载 + + +def _missing_required_files(entry): + """目录已存在但缺失的必须文件（相对模型目录）；目录不存在或无清单时返回 []。""" + req = REQUIRED_FILES.get(entry.get("download_id") or "") + if not req or not os.path.isdir(entry["abs_path"]): + return [] + return [f for f in req if not os.path.isfile(os.path.join(entry["abs_path"], f))] + + +def catalog_models(): + """Catalog entries annotated with abs_path / installed / incomplete / label. + installed 要求目录存在且 required_files 清单齐全；目录在但缺文件记为 + incomplete（missing_files 列出缺什么），加载入口据此给出明确报错。""" + out = [] + for m in CATALOG.get("models", []): + rel = m.get("path", "") + ap = rel if os.path.isabs(rel) else os.path.join(BUNDLE_ROOT, rel) + entry = dict(m) + entry["abs_path"] = os.path.normpath(ap).replace("\\", "/") + entry["missing_files"] = _missing_required_files(entry) + entry["incomplete"] = bool(entry["missing_files"]) + entry["installed"] = os.path.exists(entry["abs_path"]) and not entry["incomplete"] + entry["label"] = m.get("display_name") or m.get("id", "?") + out.append(entry) + return out + + +def catalog_by_id(model_id): + for m in catalog_models(): + if m.get("id") == model_id: + return m + return None + + +def choices_for_tasks(tasks, language=None): + """[(label, id)] for catalog models whose task is in `tasks`; missing ones flagged. + 未安装且估算显存超过本机的条目额外标注最低显存，防止白下载。""" + language = language or get_language() + out = [] + for m in catalog_models(): + if m.get("task") not in tasks: + continue + label = m["label"] + if language == "en": + label = m.get("display_name_en") or (label if label.isascii() else m["id"]) + else: + label = _t(label, label, language) + if m["incomplete"]: + label += _t(" · 目录不完整", " · incomplete", language) + elif not m["installed"]: + label += _t(" · 未安装", " · not installed", language) + short = _vram_shortfall(m) + if short: + label += _t(" ⚠️估算需≥{need:g}G显存", " ⚠️≥{need:g} GB VRAM", language, + need=short[0]) + out.append((label, m["id"])) + return out + + +def builtin_voices(): + if not os.path.isdir(PROMPTS_DIR): + return [] + return sorted(f for f in os.listdir(PROMPTS_DIR) if f.lower().endswith(".wav")) + + +def _voice_name_from_path(path): + """Original upload basename without its extension, for the save-name box.""" + if not path: + return "" + return os.path.splitext(os.path.basename(path))[0] + + +def _stage_tts_voice_upload(path): + """Preserve the uploaded filename before replacing its preview path.""" + return _stage_upload(path, force_copy=True), _voice_name_from_path(path) + + +def _stage_tts_voice_recording(path): + return _stage_recording(path), _voice_name_from_path(path) + + +def _load_voice_texts(): + """Built-in voice basename -> reference transcript, parsed from + voice/prompt_text (each line is '|').""" + texts = {} + try: + with open(os.path.join(PROMPTS_DIR, "prompt_text"), "r", encoding="utf-8") as f: + for line in f: + name, sep, text = line.rstrip("\n").partition("|") + if sep: + texts[name.strip()] = text + except Exception: + pass + return texts + + +def refresh_builtin_voices(current): + """刷新按钮：重新扫描 voice/ 目录的 wav 列表，并按当前选中项重读 + prompt_text 里的参考文本；选中项已被删除时回落到 '(none)'。 + 必须在同一个 handler 里连带输出 wav/参考文本——拆成 .then 链会和 + 下拉更新触发的 .change 并发执行，撞 Gradio get_config 的竞态 + （RuntimeError: dictionary changed size during iteration）。""" + choices = ["(none)"] + builtin_voices() + if current not in choices: + current = "(none)" + wav, ref, voice_name = on_tts_builtin_voice_change(current) + return gr.update(choices=choices, value=current), wav, ref, voice_name + + +def on_builtin_voice_change(name): + """Selecting a built-in voice mirrors its wav into the upload widget and + fills the matching reference text; '(none)' clears both.""" + if not name or name == "(none)": + return None, "" + path = os.path.join(PROMPTS_DIR, name) + ref = _load_voice_texts().get(os.path.splitext(name)[0], "") + return (_ensure_ascii_path(path) if os.path.isfile(path) else None), ref + + +def on_tts_builtin_voice_change(name): + wav, ref = on_builtin_voice_change(name) + voice_name = (os.path.splitext(name)[0] + if name and name != "(none)" else "") + return wav, ref, voice_name + + +def _builtin_voice_filename(voice_name): + """Validate a user-facing voice name and normalize it to a WAV filename.""" + name = (voice_name or "").strip() + if name.lower().endswith(".wav"): + name = name[:-4] + invalid = '<>:"/\\|?*' + if (not name or name in (".", "..") or name.endswith((" ", ".")) + or any(ch in invalid or ord(ch) < 32 for ch in name)): + raise ValueError(_t( + "名称不能为空、不能是路径，也不能包含这些字符：{chars}", + "The name cannot be empty or a path, and cannot contain: {chars}", + chars=invalid)) + return name + ".wav" + + +def _write_voice_prompt(voice_stem, reference_text): + """Insert or replace one '|' prompt_text record.""" + prompt_path = os.path.join(PROMPTS_DIR, "prompt_text") + try: + with open(prompt_path, "r", encoding="utf-8") as f: + lines = f.read().splitlines() + except FileNotFoundError: + lines = [] + + transcript = " ".join((reference_text or "").splitlines()).strip() + record = f"{voice_stem}|{transcript}" + updated = False + output = [] + for line in lines: + key, sep, _text = line.partition("|") + if sep and key.strip() == voice_stem: + if not updated: + output.append(record) + updated = True + continue + output.append(line) + if not updated: + output.append(record) + + _save_voice_prompt_lines(output) + + +def _save_voice_prompt_lines(lines): + """Atomically replace prompt_text with the supplied records.""" + prompt_path = os.path.join(PROMPTS_DIR, "prompt_text") + + fd, temp_path = tempfile.mkstemp( + prefix="prompt_text_", suffix=".tmp", dir=PROMPTS_DIR) + try: + with os.fdopen(fd, "w", encoding="utf-8", newline="\n") as f: + f.write("\n".join(lines) + ("\n" if lines else "")) + os.replace(temp_path, prompt_path) + except Exception: + try: + os.remove(temp_path) + except OSError: + pass + raise + + +def _delete_voice_prompt(voice_stem): + """Remove every prompt_text record for one built-in voice.""" + prompt_path = os.path.join(PROMPTS_DIR, "prompt_text") + try: + with open(prompt_path, "r", encoding="utf-8") as f: + lines = f.read().splitlines() + except FileNotFoundError: + return + output = [ + line for line in lines + if not (line.partition("|")[1] + and line.partition("|")[0].strip() == voice_stem) + ] + if output != lines: + _save_voice_prompt_lines(output) + + +def save_builtin_voice(uploaded_voice, voice_name, reference_text): + """Copy a reference into voice/, update prompt_text, and refresh its list.""" + if not (voice_name or "").strip(): + return gr.skip(), _t( + "❌ 请填写内置参考音色名称。", + "❌ Enter a name for the built-in voice.") + if not uploaded_voice or not os.path.isfile(uploaded_voice): + return gr.skip(), _t( + "❌ 请先上传或录制参考音频。", + "❌ Upload or record a reference audio file first.") + + try: + filename = _builtin_voice_filename(voice_name) + source_wav = _ensure_wav(uploaded_voice) + destination = os.path.join(PROMPTS_DIR, filename) + if os.path.abspath(source_wav) != os.path.abspath(destination): + fd, temp_path = tempfile.mkstemp( + prefix="voice_", suffix=".wav", dir=PROMPTS_DIR) + os.close(fd) + try: + shutil.copy2(source_wav, temp_path) + os.replace(temp_path, destination) + except Exception: + try: + os.remove(temp_path) + except OSError: + pass + raise + _write_voice_prompt(os.path.splitext(filename)[0], reference_text) + choices = ["(none)"] + builtin_voices() + return gr.update(choices=choices, value=filename), _t( + "✅ 参考音色已保存：{filename}", + "✅ Voice reference saved: {filename}", filename=filename) + except Exception as e: + return gr.skip(), _t( + "❌ 保存参考音色失败：{error}", + "❌ Failed to save voice reference: {error}", error=e) + + +def delete_builtin_voice(current): + """Delete the selected built-in WAV and its prompt_text record.""" + if not current or current == "(none)": + return tuple(gr.skip() for _ in range(5)) + try: + if os.path.basename(current) != current or not current.lower().endswith(".wav"): + raise ValueError(_t("无效的内置音色文件名。", + "Invalid built-in voice filename.")) + path = os.path.join(PROMPTS_DIR, current) + if not os.path.isfile(path): + raise FileNotFoundError(_t( + "找不到内置音色文件：{filename}", + "Built-in voice file not found: {filename}", filename=current)) + os.remove(path) + _delete_voice_prompt(os.path.splitext(current)[0]) + choices = ["(none)"] + builtin_voices() + return (gr.update(choices=choices, value="(none)"), None, "", "", + _t("✅ 已删除内置参考音色：{filename}", + "✅ Built-in voice deleted: {filename}", filename=current)) + except Exception as e: + return (*tuple(gr.skip() for _ in range(4)), _t( + "❌ 删除内置参考音色失败：{error}", + "❌ Failed to delete built-in voice: {error}", error=e)) + + +# --- config-driven advanced-parameter controls (TTS tab) ------------------- +def params_for(model_id): + """Advanced-parameter specs: catalog id override, then family fallback.""" + entry = catalog_by_id(model_id) if model_id else None + if not entry: + return [] + specs = MODEL_PARAMS.get(entry.get("id", "")) + if specs is None: + specs = MODEL_PARAMS.get(entry.get("family", ""), []) + return specs if isinstance(specs, list) else [] + + +def _make_param_component(p, language=None): + """Build one Gradio control from a spec (type: slider|number|bool|text|choice). + + interactive=True is forced: inside @gr.render a control that is only wired to + its own .change handler is otherwise inferred as output-only (read-only).""" + p = localized_param_spec(p, language) + t = p.get("type", "number") + label = p.get("label", p.get("name", "")) + info = p.get("info") + if t == "bool": + return gr.Checkbox(label=label, info=info, value=bool(p.get("default", False)), + interactive=True) + if t == "text": + return gr.Textbox(label=label, info=info, value=p.get("default", ""), + placeholder=p.get("placeholder", ""), + lines=int(p.get("lines", 1)), interactive=True) + if t == "choice": + return gr.Dropdown(label=label, info=info, choices=p.get("choices", []), + value=p.get("default"), interactive=True) + if t == "slider": + return gr.Slider(label=label, info=info, + minimum=p.get("minimum", 0), maximum=p.get("maximum", 1), + step=p.get("step", 0.01), value=p.get("default", 0), + interactive=True) + return gr.Number(label=label, info=info, value=p.get("default"), + minimum=p.get("minimum"), maximum=p.get("maximum"), + step=p.get("step"), precision=p.get("precision"), + interactive=True) + + +def _adv_updater(name): + """change-handler that writes one control's value into the shared advanced + options state dict (keyed by the option name).""" + def _fn(state, value): + state = dict(state or {}) + state[name] = value + return state + return _fn + + +# --- server lifecycle ------------------------------------------------------ +def _port_open(host, port, timeout=0.5): + try: + with socket.create_connection((host, int(port)), timeout=timeout): + return True + except OSError: + return False + + +def server_alive(): + try: + requests.get(f"{SERVER}/health", timeout=3).raise_for_status() + return True + except Exception: + return False + + +def loaded_ids(): + try: + r = requests.get(f"{SERVER}/v1/models", timeout=5) + r.raise_for_status() + return [m.get("id") for m in r.json().get("data", [])] + except Exception: + return [] + + +def _write_temp_config(entry): + model = { + "id": entry["id"], + "family": entry["family"], + "path": entry["abs_path"], + "task": entry.get("task", "tts"), + "mode": entry.get("mode", "offline"), + } + for key in ("config", "weight", "load_options", "session_options"): + if entry.get(key) is not None: + model[key] = entry[key] + cfg = {"host": HOST, "port": PORT, "backend": SERVER_BACKEND, "device": DEVICE, + "threads": THREADS, "models": [model]} + fd, path = tempfile.mkstemp(prefix="audiocpp_webui_cfg_", suffix=".json") + with os.fdopen(fd, "w", encoding="utf-8") as f: + json.dump(cfg, f) + return path + + +def _stop_server(): + global _server_proc, _loaded_id, _loaded_session_options, _loaded_mode + _loaded_session_options = None + _loaded_mode = None + proc, _server_proc, _loaded_id = _server_proc, None, None + if proc is not None and proc.poll() is None: + try: + proc.terminate() + try: + proc.wait(timeout=10) + except subprocess.TimeoutExpired: + proc.kill() + proc.wait(timeout=5) + except Exception: + pass + for _ in range(24): # let the OS release the port + if not _port_open(HOST, PORT): + break + time.sleep(0.25) + + +def _read_tail(path, n=30, max_bytes=65536): + """Last n non-empty lines of a file. Reads only the file's tail, and treats + \\r as a line break so tqdm-style progress (one giant \\r-line) shows its + latest state instead of nothing.""" + try: + with open(path, "rb") as f: + f.seek(0, os.SEEK_END) + size = f.tell() + f.seek(max(0, size - max_bytes)) + data = f.read().decode("utf-8", errors="replace") + lines = [ln for ln in data.replace("\r\n", "\n").replace("\r", "\n").split("\n") + if ln.strip()] + return "\n".join(lines[-n:]).strip() + except Exception: + return "" + + +def _log_tail(n=30): + return _read_tail(LOG_PATH, n) + + +# One shared append handle for the WebUI log: the pump thread (server output) +# and _ui_log (webui-side request events) both write through it, so lines +# interleave correctly instead of two handles overwriting each other. +_log_lock = threading.Lock() +_log_fh = None + + +def _open_log_file(truncate=False): + global _log_fh + with _log_lock: + if _log_fh is not None: + try: + _log_fh.close() + except Exception: + pass + _log_fh = open(LOG_PATH, "w" if truncate else "a", + encoding="utf-8", errors="replace") + + +def _log_write(text): + global _log_fh + with _log_lock: + if _log_fh is None: + try: + _log_fh = open(LOG_PATH, "a", encoding="utf-8", errors="replace") + except Exception: + return + try: + _log_fh.write(text) + _log_fh.flush() + except Exception: + pass + + +def _ts(): + return time.strftime("%H:%M:%S") + + +def _emit_log_line(text): + """One already-formatted line to BOTH the console and the WebUI log file.""" + try: + sys.stdout.write(text) + sys.stdout.flush() + except Exception: + pass + _log_write(text) + + +def _ui_log(msg): + """Timestamped webui-side event (request start/finish, model load, ...) so + the console/log show when things began and ended, not just server spam.""" + _emit_log_line(f"[{time.strftime('%Y-%m-%d %H:%M:%S')}] [webui] {msg}\n") + + +def _pump_server_output(proc): + """Tee the server's combined stdout/stderr to console + log file, prefixing + every line with a timestamp and collapsing consecutive duplicate lines + (e.g. the repeated `CUDA graph warmup complete`) into a periodic counter.""" + last, repeats = None, 0 + try: + for line in proc.stdout: + if line == last: + repeats += 1 + if repeats % 50 == 0: + _emit_log_line(f"[{_ts()}] ... " + _t( + "上一行已重复 {count} 次", "previous line repeated {count} times", + count=repeats) + "\n") + continue + if repeats: + _emit_log_line(f"[{_ts()}] ... " + _t( + "（上一行共重复 {count} 次）", "(previous line repeated {count} times total)", + count=repeats) + "\n") + last, repeats = line, 0 + _emit_log_line(f"[{_ts()}] {line}") + except Exception: + pass + finally: + if repeats: + _emit_log_line(f"[{_ts()}] ... " + _t( + "（上一行共重复 {count} 次）", "(previous line repeated {count} times total)", + count=repeats) + "\n") + + +def _start_server(entry): + global _server_proc, _loaded_id + if not os.path.isfile(SERVER_EXE): + raise gr.Error(_t("找不到 server：{path}（可用 AUDIOCPP_BACKEND=gpu|cpu 指定）", + "Server not found: {path}. Set AUDIOCPP_BACKEND=gpu|cpu.", + path=SERVER_EXE)) + cfg = _write_temp_config(entry) + flags = subprocess.CREATE_NEW_PROCESS_GROUP if os.name == "nt" else 0 + _open_log_file(truncate=True) + # --log 会打开 engine 的 [TRACE]/[TIMING] 调试输出，日常太吵，默认关闭； + # 排查推理问题时设 AUDIOCPP_SERVER_DEBUG=1 再启动 webui。 + cmd = [SERVER_EXE, "--config", cfg, "--host", HOST, "--port", str(PORT)] + if os.environ.get("AUDIOCPP_SERVER_DEBUG") == "1": + cmd.append("--log") + _server_proc = subprocess.Popen( + cmd, + cwd=BUNDLE_ROOT, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + creationflags=flags, text=True, encoding="utf-8", errors="replace", bufsize=1, + ) + _loaded_id = entry["id"] + extra = (_t("，threads={threads}", ", threads={threads}", threads=THREADS) + if SERVER_BACKEND == "cpu" else "") + _ui_log(_t("启动 audiocpp_server（backend={backend}{extra}），加载模型 {label} …", + "starting audiocpp_server (backend={backend}{extra}), loading model {label} …", + backend=SERVER_BACKEND, extra=extra, label=entry['label'])) + threading.Thread(target=_pump_server_output, args=(_server_proc,), + daemon=True).start() + + +def _wait_health(timeout): + start = time.time() + while time.time() - start < timeout: + if _server_proc is not None and _server_proc.poll() is not None: + return False # process exited before becoming healthy + if server_alive(): + return True + time.sleep(0.5) + return False + + +def ensure_model_loaded(model_id, expect_tasks=None, session_options=None, mode=None): + """(Re)start the server so `model_id` is loaded. Returns a status string. + session_options：额外写进本次 server config 的 session_options（string→string， + 如 sortformer 的 session_len_sec）；与上次加载不一致时会重启重载。 + mode：覆盖 catalog 条目的运行模式（"streaming"/"offline"），流式转写/生成用； + 与上次加载不一致时同样重启重载。""" + global _loaded_session_options, _loaded_mode + if not model_id: + raise gr.Error(_t("请先选择一个模型", "Select a model first.")) + entry = catalog_by_id(model_id) + if entry is None: + raise gr.Error(_t("catalog 里没有模型 id：{model}", + "Model id not found in catalog: {model}", model=model_id)) + if not entry["installed"]: + if entry["incomplete"]: + missing = entry["missing_files"] + shown = "、".join(missing[:8]) + (f" 等 {len(missing)} 个" if len(missing) > 8 else "") + raise gr.Error(_t( + "模型目录不完整：{path}\n缺少文件：{missing}\n请点『⬇️ 下载模型』重新安装。", + "Model directory is incomplete: {path}\nMissing: {missing}\nClick Download to reinstall.", + path=entry["abs_path"], missing=shown)) + raise gr.Error(_t("模型未安装：{path}", "Model is not installed: {path}", + path=entry["abs_path"])) + if expect_tasks and entry.get("task") not in expect_tasks: + raise gr.Error(_t( + "模型 {model} 的 task 是 {actual}，此处需要 {expected}", + "Model {model} has task {actual}; expected {expected}.", + model=model_id, actual=entry.get("task"), expected="/".join(expect_tasks))) + want_mode = mode or entry.get("mode", "offline") + mode_note = (_t("（流式模式）", " (streaming)") + if want_mode == "streaming" else "") + + with _proc_lock: + managed_alive = _server_proc is not None and _server_proc.poll() is None + if (managed_alive and _loaded_id == model_id and server_alive() + and (session_options or {}) == (_loaded_session_options or {}) + and want_mode == (_loaded_mode or entry.get("mode", "offline"))): + return _t("✅ 已加载：{label}{mode}", "✅ Loaded: {label}{mode}", + label=entry["label"], mode=mode_note) + + if not managed_alive and server_alive(): + # A server we didn't launch is holding the port. + if session_options: + raise gr.Error(_t( + "{host}:{port} 上的外部 server 无法调整 session 配置，请先关闭。", + "The external server at {host}:{port} cannot change session settings. Stop it first.", + host=HOST, port=PORT)) + if mode and mode != entry.get("mode", "offline"): + raise gr.Error(_t( + "{host}:{port} 上的外部 server 无法切换到 {mode} 模式，请先关闭。", + "The external server at {host}:{port} cannot switch to {mode}. Stop it first.", + host=HOST, port=PORT, mode=mode)) + if model_id in loaded_ids(): + return _t("✅ 复用外部 server：{label}", + "✅ Using external server: {label}", label=entry["label"]) + raise gr.Error(_t( + "检测到外部 server 占用 {host}:{port}，请先关闭或设置 AUDIOCPP_SERVER。", + "An external server is using {host}:{port}. Stop it or set AUDIOCPP_SERVER.", + host=HOST, port=PORT)) + + _stop_server() + if session_options or want_mode != entry.get("mode", "offline"): + entry = dict(entry) + entry["mode"] = want_mode + if session_options: + entry["session_options"] = { + **(entry.get("session_options") or {}), **session_options} + t0 = time.time() + _start_server(entry) + _loaded_session_options = dict(session_options) if session_options else None + _loaded_mode = want_mode + if not _wait_health(LOAD_TIMEOUT): + tail = _log_tail() + _stop_server() + _ui_log(_t("模型 {label} 加载失败/超时（{timeout}s）", + "model {label} load failed/timed out ({timeout}s)", + label=entry['label'], timeout=LOAD_TIMEOUT)) + raise gr.Error(_t("加载 {label} 失败/超时（{timeout}s）。\n日志尾部：\n{tail}", + "Loading {label} failed or timed out ({timeout}s).\nLog tail:\n{tail}", + label=entry["label"], timeout=LOAD_TIMEOUT, tail=tail)) + _ui_log(_t("模型 {label} 加载完成{mode_note}，用时 {seconds:.1f}s", + "model {label} loaded{mode_note}, elapsed {seconds:.1f}s", + label=entry['label'], mode_note=mode_note, seconds=time.time() - t0)) + return _t("✅ 已加载：{label}{mode}", "✅ Loaded: {label}{mode}", + label=entry["label"], mode=mode_note) + + +def unload_model(): + """停止本 WebUI 启动的 server，释放全部显存（权重+常驻计算图缓冲）。 + 下次生成/转写时 ensure_model_loaded 会自动重启重载（实测 ~5s），转写速度 + 本身不受影响（计算图本来就按每次请求的音频长度分配/复用）。""" + with _proc_lock: + managed_alive = _server_proc is not None and _server_proc.poll() is None + if managed_alive: + label = _loaded_id or "(unknown)" + _stop_server() + _ui_log(_t("已卸载模型 {label} 并停止 server，显存已释放", + "unloaded model {label} and stopped server, VRAM released", + label=label)) + return (_t("🧹 已卸载模型并释放显存，下次运行时会自动重新加载。", + "🧹 Model unloaded and VRAM released. It will reload on the next run."), + server_status()) + if server_alive(): + return (_t("⚠️ 当前 server 不是本 WebUI 启动的，请在其启动窗口中关闭。", + "⚠️ This server was started externally. Stop it from its own window."), + server_status()) + return _t("⚪ server 未运行，无需释放。", "⚪ Server is not running."), server_status() + + +def server_status(): + if server_alive(): + ids = ", ".join(loaded_ids()) or "(none)" + return f"✅ server @ {SERVER} · backend={BACKEND} · model id={ids}" + return _t("⚪ server 未运行 @ {server} — 选择模型并点『📥 加载模型』", + "⚪ Server is not running @ {server} — select a model and click Load.", + server=SERVER) + + +def _api_usage_md(language=None): + """状态行下方折叠区的第三方调用说明。端点/字段以 app/server/README.md 为准； + URL 取运行时的 SERVER，避免和 AUDIOCPP_SERVER / catalog 配置不一致。""" + language = language or get_language() + if language == "en": + return f""" +Other applications can call the local `audiocpp_server` started by this WebUI. + +- Base URL: `{SERVER}/v1` +- TTS: `POST {SERVER}/v1/audio/speech` +- ASR: `POST {SERVER}/v1/audio/transcriptions` +- Other tasks: `POST {SERVER}/v1/tasks/run` +- `model` must be the currently loaded model id. Check it with `GET {SERVER}/v1/models`. +- Audio paths in API JSON are paths on the server machine, not browser uploads. + +```bash +curl {SERVER}/v1/audio/speech -H "Content-Type: application/json" -o out.wav \\ + -d '{{"model":"qwen3-tts","input":"Hello from audio.cpp."}}' +``` + +The server stays running while the WebUI command window is open. +""" + content = f""" +第三方应用可以直接调用本 WebUI 启动的 `audiocpp_server`（OpenAI 风格 HTTP API），不经过本页面。 + +- **URL 怎么填**：API 地址是 `{SERVER}`；OpenAI 兼容客户端的 Base URL 填 `{SERVER}/v1`。 + 生成语音：`POST {SERVER}/v1/audio/speech` · 音频转写：`POST {SERVER}/v1/audio/transcriptions` +- **模型名称怎么填**：`model` 填模型 id（与本页模型列表一致，如 `qwen3-tts`、`vibevoice`）， + 且必须是**当前已加载**的那个 —— server 同一时刻只驻留一个模型，先在本页点『📥 加载模型』； + 可用 `GET {SERVER}/v1/models` 查看当前可用的 id。 +- **TTS 请求示例**（响应默认是 WAV 音频；加 `"response_format": "json"` 改为返回 base64 的 JSON）： + +```bash +curl {SERVER}/v1/audio/speech -H "Content-Type: application/json" -o out.wav \\ + -d '{{"model": "qwen3-tts", "input": "你好，audio.cpp。", "voice_ref": "D:/voices/ref.wav", "reference_text": "参考音频里的原话", "seed": 1234}}' +``` + +- **ASR 请求示例**：`-d '{{"model": "qwen3-asr", "audio": "D:/audio/in.wav"}}'`； + 可选 `"language"`（强制语种，如 `"Chinese"`）和 `"context"`（人名/术语偏置提示），仅 qwen3-asr 生效。 + 注意 `voice_ref` / `audio` 填的都是 **server 所在机器上的文件路径**（不是浏览器上传）。 +- **音乐生成（gen 模型）**：走通用路由 `POST {SERVER}/v1/tasks/run`，body 形如 + `{{"model": "ace-step", "request": {{"text": "提示词", "lyrics": "歌词", "duration_seconds": 30, + "options": {{"tags": "pop,bright"}}}}}}`，响应 JSON 的 `audio` 字段是 base64 WAV。 +- **其它任务（vc/svc/s2s/sep/vad/diar/align）**：同样走 `POST {SERVER}/v1/tasks/run`，`request` 里用 + `audio`（源音频路径）/ `voice_ref`（目标音色）/ `text`（对齐文本）等字段；分离多轨在响应的 + `named_audio_outputs`，VAD/说话人/对齐结果在 `segments` / `speaker_turns` / `words`。 +- server 的生命周期跟随本 WebUI 的**命令行窗口**：只关浏览器页面不影响，server 仍可被第三方调用； + 关掉命令窗口（webui.py 退出）才会连带关闭它。也可单独启动 server（如 {SERVER_LAUNCHER}） + 供第三方应用调用。 +""" + return _t(content, content, language) + + +# --- background model downloads (via tools/model_manager.py) ---------------- +_dl_lock = threading.Lock() +_downloads = {} # model_id -> {"proc": Popen, "log": path} + + +def _dl_log_path(model_id): + safe = re.sub(r"[^A-Za-z0-9_.-]", "_", model_id) + return os.path.join(LOG_DIR, f"download_{safe}.log") + + +def _dir_size_bytes(path): + total = 0 + for root, _dirs, files in os.walk(path): + for name in files: + try: + total += os.path.getsize(os.path.join(root, name)) + except OSError: + pass + return total + + +def _fmt_bytes(n): + return f"{n / 1e9:.2f} GB" if n >= 1e9 else f"{n / 1e6:.1f} MB" + + +def _download_progress_note(entry): + """Bytes already on disk for a running download. model_manager stages into + models/.engine_model_staging/.partial/ and renames on completion; + fall back to the whole staging root for packages with composite targets.""" + staging_root = os.path.join(MODELS_ROOT, ".engine_model_staging") + base = os.path.basename(entry.get("path", "").rstrip("/\\")) + safe = re.sub(r"[^A-Za-z0-9_.-]", "_", base) + staging = os.path.join(staging_root, safe + ".partial") + probe = staging if os.path.isdir(staging) else staging_root + if not os.path.isdir(probe): + return _t("尚未写入数据（正在连接/解析）", "Waiting for data…") + return _t("已下载 {size}", "Downloaded {size}", + size=_fmt_bytes(_dir_size_bytes(probe))) + + +def hf_token_present(): + """True if model_manager will find an HF token (env or cached login).""" + if os.environ.get("HF_TOKEN") or os.environ.get("HUGGING_FACE_HUB_TOKEN"): + return True + return os.path.isfile(os.path.join(os.path.expanduser("~"), ".cache", "huggingface", "token")) + + +def download_model(model_id, hf_token="", proxy=""): + """Kick off `model_manager.py install ` in the background.""" + if not model_id: + return _t("❌ 请先选择一个模型", "❌ Select a model first.") + entry = catalog_by_id(model_id) + if entry is None: + return _t("❌ catalog 里没有模型 id：{model}", + "❌ Model id not found: {model}", model=model_id) + if entry["installed"]: + return _t("✅ {label} 已安装，无需下载", "✅ {label} is already installed.", + label=entry["label"]) + dl_id = entry.get("download_id") + if not dl_id: + return _t("⚠️ {label} 没有 download_id，请手动安装。", + "⚠️ {label} has no download_id; install it manually.", label=entry["label"]) + if MODEL_MANAGER is None: + return _t("❌ 找不到 tools/model_manager.py", + "❌ tools/model_manager.py was not found.") + + # Pass a token to the child so gated/private HF repos don't 401. + env = os.environ.copy() + env["PYTHONUNBUFFERED"] = "1" # progress lines land in the log immediately + env["PYTHONIOENCODING"] = "utf-8" + tok = (hf_token or "").strip() + if tok: + env["HF_TOKEN"] = tok + env["HUGGING_FACE_HUB_TOKEN"] = tok + # Route the child's downloads through a proxy (urllib reads these env vars). + px = (proxy or "").strip() + proxy_note = "" + if px: + for var in ("HTTP_PROXY", "HTTPS_PROXY", "http_proxy", "https_proxy"): + env[var] = px + proxy_note = _t("🌐 通过代理 {proxy}\n\n", "🌐 Proxy: {proxy}\n\n", proxy=px) + warn = "" if (tok or hf_token_present()) else _t( + "⚠️ 未检测到 HF token，受限模型可能返回 401。\n\n", + "⚠️ No HF token detected; gated models may return 401.\n\n") + short = _vram_shortfall(entry) + if short: + warn = _t("⚠️ **显存不足**：估算需 **≥{need:g}G**，本机为 **{local:g}G**。\n\n", + "⚠️ **Low VRAM**: estimated **≥{need:g} GB**, detected **{local:g} GB**.\n\n", + need=short[0], local=short[1]) + warn + if entry["incomplete"]: + warn = _t("⚠️ {path} 不完整（缺 {count} 个文件），将覆盖重装。\n\n", + "⚠️ {path} is incomplete ({count} files missing); it will be reinstalled.\n\n", + path=entry["abs_path"], count=len(entry["missing_files"])) + warn + + with _dl_lock: + rec = _downloads.get(model_id) + if rec and rec["proc"].poll() is None: + return _t("⏳ {label} 已在后台下载中…\n```\n{tail}\n```", + "⏳ {label} is already downloading…\n```\n{tail}\n```", + label=entry["label"], tail=_read_tail(rec["log"])) + log = _dl_log_path(model_id) + logf = open(log, "w", encoding="utf-8", errors="replace") + proc = subprocess.Popen( + [sys.executable, "-u", MODEL_MANAGER, "install", dl_id, + "--models-root", MODELS_ROOT, "--overwrite"], + cwd=PROJECT_ROOT, stdout=logf, stderr=subprocess.STDOUT, env=env) + _downloads[model_id] = {"proc": proc, "log": log} + _ui_log(_t("开始后台下载 {label}（{dl_id}），日志：{log}", + "started background download {label} ({dl_id}), log: {log}", + label=entry['label'], dl_id=dl_id, log=log)) + return warn + proxy_note + _t( + "⏳ 已开始下载 **{label}**（{download_id}）。完成后刷新列表。\n日志：{log}", + "⏳ Download started: **{label}** ({download_id}). Refresh the list when complete.\nLog: {log}", + label=entry["label"], download_id=dl_id, log=log) + + +def download_status(model_id): + entry = catalog_by_id(model_id) if model_id else None + if entry is None: + return "" + if entry["installed"]: + return _t("✅ {label} 已安装", "✅ {label} is installed.", label=entry["label"]) + rec = _downloads.get(model_id) + if rec is None: + if entry["incomplete"]: + return _t("⚠️ {label} 目录不完整（缺 {count} 个文件），请重新下载。", + "⚠️ {label} is incomplete ({count} files missing). Download it again.", + label=entry["label"], count=len(entry["missing_files"])) + return _t("⚪ {label} 未安装，未开始下载", "⚪ {label} is not installed.", + label=entry["label"]) + code = rec["proc"].poll() + tail = _read_tail(rec["log"], n=12) + if code is None: + return _t("⏳ 正在下载 {label}… {progress} · 更新于 {time}\n```\n{tail}\n```", + "⏳ Downloading {label}… {progress} · {time}\n```\n{tail}\n```", + label=entry["label"], progress=_download_progress_note(entry), + time=_ts(), tail=tail) + if not rec.get("reported"): + rec["reported"] = True + _ui_log(_t("{label} 下载进程结束 (exit {code})", + "{label} download process ended (exit {code})", + label=entry['label'], code=code)) + if code == 0: + return _t("✅ {label} 下载完成，请刷新列表。\n```\n{tail}\n```", + "✅ {label} downloaded. Refresh the model list.\n```\n{tail}\n```", + label=entry["label"], tail=tail) + return _t("❌ {label} 下载失败（exit {code}）。\n```\n{tail}\n```", + "❌ {label} download failed (exit {code}).\n```\n{tail}\n```", + label=entry["label"], code=code, tail=tail) + + +def _download_running(model_id): + rec = _downloads.get(model_id) if model_id else None + return rec is not None and rec["proc"].poll() is None + + +def download_start(model_id, hf_token="", proxy=""): + """Click handler: kick off the download and arm the auto-refresh timer.""" + msg = download_model(model_id, hf_token, proxy) + return msg, gr.Timer(active=_download_running(model_id)) + + +def download_status_tick(model_id): + """Timer tick: refresh status; stop the timer once the download is idle.""" + return download_status(model_id), gr.Timer(active=_download_running(model_id)) + + +def _gguf_entry(model_id, require_installed=True): + if not model_id: + return None, _t("请先选择一个模型", "Select a model first.") + entry = catalog_by_id(model_id) + if entry is None: + return None, _t("catalog 里没有模型 id：{model}", + "Model id not found in catalog: {model}", model=model_id) + if require_installed and not entry["installed"]: + return None, _t("模型未完整安装。", "The model is not fully installed.") + return entry, "" + + +def _gguf_output_path(entry): + model_path = entry["abs_path"] + if os.path.isfile(model_path) and model_path.lower().endswith(".gguf"): + return model_path + root = model_path if os.path.isdir(model_path) else os.path.dirname(model_path) + return os.path.join(root, "model.gguf") + + +def _gguf_tensor_entrypoint(model_dir): + """Return a single-file or sharded safetensors entry point in model_dir.""" + for name in ("model.safetensors.index.json", "model.safetensors"): + candidate = os.path.join(model_dir, name) + if os.path.isfile(candidate): + return candidate + return None + + +def _gguf_conversion_inputs(entry): + """Build the converter's ordered (namespace, weights) input list.""" + if entry["family"] not in GGUF_WEBUI_CONVERTIBLE_FAMILIES: + return [] + + model_path = entry["abs_path"] + if os.path.isfile(model_path): + lower = model_path.lower() + if lower.endswith(".safetensors") or lower.endswith(".safetensors.index.json"): + return [("", model_path)] + return [] + + # Qwen3-TTS is a composite package. Its GGUF package spec requires both + # tensor sources under the exact namespaces below. + if entry["family"] == "qwen3_tts": + model_weights = _gguf_tensor_entrypoint(model_path) + speech_weights = _gguf_tensor_entrypoint(os.path.join(model_path, "speech_tokenizer")) + if model_weights and speech_weights: + return [ + ("model_weights", model_weights), + ("speech_tokenizer_weights", speech_weights), + ] + return [] + + source = _gguf_tensor_entrypoint(model_path) + return [("", source)] if source else [] + + +def _gguf_conversion_unavailable(entry): + family = entry["family"] + if family not in GGUF_NATIVE_FAMILIES: + return _t("当前模型后端暂不支持原生 GGUF。", + "This model backend does not currently support native GGUF.") + if family not in GGUF_WEBUI_CONVERTIBLE_FAMILIES: + return _t("当前复合模型暂不能在 WebUI 自动转换。", + "This composite model cannot yet be converted automatically in the WebUI.") + return "" + + +def gguf_status(model_id): + entry, error = _gguf_entry(model_id, require_installed=False) + if entry is None: + return f"⚪ {error}" + unavailable = _gguf_conversion_unavailable(entry) + if unavailable: + return f"⚠️ {unavailable}" + if not entry["installed"]: + return _t("🧊 可转换，但模型未完整安装。", + "🧊 Convertible, but the model is not fully installed.") + output = _gguf_output_path(entry) + converter = _find_gguf_exe() + if os.path.isfile(output): + return _t("🧊 已有GGUF，将优先加载该模型。", "🧊 GGUF is available and will be loaded first.") + if converter is None: + return _t("⚠️ 找不到转换器。", "⚠️ Converter not found.") + inputs = _gguf_conversion_inputs(entry) + if not inputs: + return _t("⚠️ 未找到可转换的模型权重。", "⚠️ No convertible model weights found.") + return _t("🧊 可转换。", "🧊 Ready to convert.") + + +def _gguf_inspection_summary(output, text): + info, namespaces = {}, [] + for line in (text or "").splitlines(): + key, separator, value = line.partition("=") + if not separator: + continue + if key == "namespace": + namespaces.append(value) + else: + info[key] = value + + yes_no = lambda value: _t("是" if value == "true" else "否", + "Yes" if value == "true" else "No") + rows = [ + _t("✅ 检查完成", "✅ Inspection complete"), + _t("文件：`{name}`", "File: `{name}`", name=os.path.basename(output)), + _t("路径：`{path}`", "Path: `{path}`", path=output), + _t("张量：{count}", "Tensors: {count}", count=info.get("tensors", "-")), + _t("内嵌资源：{value}", "Embedded sidecars: {value}", + value=yes_no(info.get("embedded_sidecars"))), + _t("内嵌模型配置：{value}", "Embedded model spec: {value}", + value=yes_no(info.get("embedded_model_spec"))), + ] + if info.get("model_spec_family"): + rows.append(_t("模型家族：{family}", "Model family: {family}", + family=info["model_spec_family"])) + if namespaces: + rows.append(_t("权重命名空间：{items}", "Weight namespaces: {items}", + items=", ".join(namespaces))) + return " \n".join(rows) + + +def inspect_gguf(model_id): + entry, error = _gguf_entry(model_id) + if entry is None: + return f"❌ {error}" + output = _gguf_output_path(entry) + if not os.path.isfile(output): + return _t("⚠️ 暂无 GGUF。", "⚠️ No GGUF yet.") + converter = _find_gguf_exe() + if converter is None: + return _t("❌ 找不到 `{name}`；已检查开发构建和 portable 的 gpu/cpu 目录。", + "❌ {name} was not found in development or portable gpu/cpu paths.", name=GGUF_EXE_NAME) + try: + result = subprocess.run([converter, "--inspect", output], cwd=PROJECT_ROOT, + capture_output=True, text=True, encoding="utf-8", errors="replace", + timeout=120) + except Exception as exc: + return _t("❌ GGUF 检查无法启动：{error}", "❌ Could not start GGUF inspection: {error}", error=exc) + if result.returncode != 0: + return _t("❌ 检查失败（exit {code}）。", "❌ Inspection failed (exit {code}).", code=result.returncode) + _ui_log(_t("检查 GGUF：{output}", "checking GGUF: {output}", output=output)) + return _gguf_inspection_summary(output, result.stdout) + + +def convert_model_to_gguf(model_id, weight_type, progress=gr.Progress()): + entry, error = _gguf_entry(model_id) + if entry is None: + return f"❌ {error}" + unavailable = _gguf_conversion_unavailable(entry) + if unavailable: + return f"❌ {unavailable}" + output = _gguf_output_path(entry) + if os.path.isfile(output): + return _t("⚠️ GGUF 已存在；请先检查或删除。", "⚠️ GGUF already exists; inspect or delete it first.") + converter = _find_gguf_exe() + if converter is None: + return _t("❌ 找不到 `{name}`；已检查开发构建和 portable 的 gpu/cpu 目录。", + "❌ {name} was not found in development or portable gpu/cpu paths.", name=GGUF_EXE_NAME) + inputs = _gguf_conversion_inputs(entry) + if not inputs: + return _t("❌ 未找到可自动转换的模型权重。", "❌ No convertible model weights found.") + if weight_type not in GGUF_TYPES: + return _t("❌ 不支持的 GGUF 类型：{type}", "❌ Unsupported GGUF type: {type}", type=weight_type) + + root = entry["abs_path"] if os.path.isdir(entry["abs_path"]) else os.path.dirname(inputs[0][1]) + cmd = [converter] + for namespace, source in inputs: + cmd.extend(["--input", f"{namespace}={source}" if namespace else source]) + cmd.extend(["--root", root, "--output", output, + "--type", weight_type, "--family", entry["family"]]) + progress(0, desc=_t("正在转换 GGUF…", "Converting GGUF…")) + _ui_log(_t("开始转换 GGUF：{label} ({weight_type})", + "converting GGUF: {label} ({weight_type})", + label=entry['label'], weight_type=weight_type)) + try: + result = subprocess.run(cmd, cwd=PROJECT_ROOT, capture_output=True, text=True, + encoding="utf-8", errors="replace", timeout=7200) + except subprocess.TimeoutExpired: + return _t("❌ GGUF 转换超过 2 小时，已停止。", "❌ GGUF conversion exceeded two hours and was stopped.") + except Exception as exc: + return _t("❌ 无法启动 GGUF 转换：{error}", "❌ Could not start GGUF conversion: {error}", error=exc) + + if result.returncode != 0 or not os.path.isfile(output): + _ui_log(_t("GGUF 转换失败：{label} (exit {code})", + "GGUF conversion failed: {label} (exit {code})", + label=entry['label'], code=result.returncode)) + stdout = (result.stdout or "").strip() + stderr = (result.stderr or "").strip() + details = [] + if stdout: + details.append(f"stdout:\n{stdout}") + if stderr: + details.append(f"stderr:\n{stderr}") + process_output = "\n\n".join(details) or _t("（转换器没有输出）", "(The converter produced no output.)") + return _t( + "❌ 转换失败（exit {code}）。\n\n命令：\n```text\n{command}\n```\n\n详细信息：\n```text\n{output}\n```", + "❌ Conversion failed (exit {code}).\n\nCommand:\n```text\n{command}\n```\n\nDetails:\n```text\n{output}\n```", + code=result.returncode, command=subprocess.list2cmdline(cmd), output=process_output) + + stopped = False + with _proc_lock: + if _loaded_id == model_id and _server_proc is not None and _server_proc.poll() is None: + _stop_server() + stopped = True + _ui_log(_t("GGUF 转换完成：{label} → {output}", + "GGUF conversion done: {label} → {output}", + label=entry['label'], output=output)) + stop_note = (_t("请点『加载模型』。", "Click Load.") + if stopped else + _t("点『加载模型』即可。", "Click Load to use it.")) + return _t("✅ 转换成功。{note}", "✅ Conversion complete. {note}", note=stop_note) + + +def delete_gguf(model_id): + entry, error = _gguf_entry(model_id) + if entry is None: + return f"❌ {error}", server_status() + output = _gguf_output_path(entry) + if not os.path.isfile(output): + return _t("⚠️ 暂无 GGUF。", "⚠️ No GGUF to delete."), server_status() + with _proc_lock: + if _loaded_id == model_id and _server_proc is not None and _server_proc.poll() is None: + _stop_server() + elif server_alive() and model_id in loaded_ids(): + return _t("⚠️ 外部 server 正在使用该 GGUF；请先关闭它再删除。", + "⚠️ An external server is using this GGUF. Stop it before deleting."), server_status() + temporary = output + ".tmp" + try: + os.remove(output) + if os.path.isfile(temporary): + os.remove(temporary) + except OSError as exc: + return _t("❌ 删除 GGUF 失败：{error}", "❌ Could not delete GGUF: {error}", error=exc), server_status() + _ui_log(_t("删除 GGUF：{output}", "deleting GGUF: {output}", output=output)) + return _t("✅ 已删除：`{path}`", "✅ Deleted: `{path}`", path=output), server_status() + + +# --- task handlers --------------------------------------------------------- +# Task handlers return (output, message): the reminder/status message is shown +# inline under the output widget instead of as a Gradio popup card. +def _merged_options(prof, adv_values, adv_options): + """请求 options 合并：family 默认值 -> 生成控件（仅用户改过的项）-> JSON 兜底框。""" + options = dict(prof.get("default_options", {})) + if isinstance(adv_values, dict): + options.update({k: v for k, v in adv_values.items() + if v is not None and v != ""}) + options.update(_parse_adv_options(adv_options)) + return options + + +def _run_task(entry, model, req, timeout, log_label): + """POST /v1/tasks/run（通用任务路由）并返回响应 JSON； + 连接失败 / 非 200 统一转成带提示的 gr.Error。""" + try: + r = requests.post(f"{SERVER}/v1/tasks/run", + json={"model": model, "request": req}, timeout=timeout) + except requests.RequestException as e: + _ui_log(_t("{log_label}失败：无法连接 server", + "{log_label} failed: cannot connect to server", + log_label=log_label)) + raise connection_error(e) + if r.status_code != 200: + _ui_log(_t("{log_label}失败：server {code}", + "{log_label} failed: server {code}", + log_label=log_label, code=r.status_code)) + raise server_error(entry, r.status_code, r.text) + return r.json() + + +def _resolve_seed(seed): + """seed=-1 → 每次请求随机抽一个。server/C++ 侧 seed 一律按无符号整数解析 + （多数族 u32，seed_vc/stable_audio u64），负数会直接报错，所以 -1 只能在 + 客户端消化；随机范围取 u32 全集，对所有族安全。返回 (seed, 消息后缀)—— + 后缀把实际用的 seed 回显在结果里，方便复现。""" + s = int(seed) + if s != -1: + return s, "" + s = random.randrange(0, 2 ** 32) + return s, f"🎲 seed={s}" + + +def do_tts(model, text, language, uploaded_voice, builtin_voice, + reference_text, seed, max_tokens, adv_values, adv_options, + progress=gr.Progress()): + try: + if not (text or "").strip(): + raise gr.Error(_t("请输入要合成的文字", "Enter text to synthesize.")) + + entry = catalog_by_id(model) + prof = profile_for(entry) if entry else DEFAULT_PROFILE + + if prof.get("wrap_speaker_script"): # e.g. VibeVoice needs Speaker N: lines + text = _as_speaker_script(text) + text = _vibevoice_punctuate_script(text) + + # 超短文本拦截（见 _VIBEVOICE_MIN_EST_TOKENS）——放在模型加载之前， + # 免得为一个注定被拒的请求重启 server。 + is_vibevoice = bool(entry) and entry.get("family") == "vibevoice" + if is_vibevoice and _vibevoice_text_max_tokens(text) < _VIBEVOICE_MIN_EST_TOKENS: + raise gr.Error(_t( + "VibeVoice 是长文模型。请使用 ≥40 个汉字（英文约 ≥35 词），或改用短句模型。", + "VibeVoice is for long-form text. Use at least ~35 English words, or choose a short-text model.")) + + # 必须参考音色的家族（如 IndexTTS2）在加载模型前就拦下来， + # 免得等几十秒加载后才收到 server 报错。 + if (prof.get("require_voice") and not uploaded_voice + and (not builtin_voice or builtin_voice == "(none)")): + raise gr.Error(_t( + "该模型必须提供参考音色：请上传/录制参考音频，或选择内置音色。", + "This model requires a voice reference: upload/record one or pick a built-in voice.")) + + # 加载/模式切换（可能几十秒）单独计时：状态栏的"用时"只含合成本身， + # 不报加载会让它看起来远小于实际等待时间。 + t_load = time.time() + ensure_model_loaded(model, TTS_TASKS) + load_s = time.time() - t_load + load_note = (_t("，含模型加载 {seconds:.1f}s", ", model load {seconds:.1f}s", + seconds=load_s) if load_s >= 1.0 else "") + + # Model-specific knobs travel in a nested "options" object; the server merges + # every key into the request options and each model reads what it understands. + options = _merged_options(prof, adv_values, adv_options) + + voice_path = None + if uploaded_voice: # gradio gives an absolute temp path + voice_path = uploaded_voice + elif builtin_voice and builtin_voice != "(none)": + voice_path = os.path.join(PROMPTS_DIR, builtin_voice) + has_voice_samples = "voice_samples" in options or "vibevoice.voice_samples" in options + if is_vibevoice and voice_path and not has_voice_samples: + options["voice_samples"] = _ensure_wav(voice_path) + voice_path = None + # voice_samples (multi-speaker) can't be combined with a single voice_ref. + if has_voice_samples and voice_path: + voice_path = None + + # 预置音色家族（Supertonic 的 M1-M5/F1-F5）：控件里的 voice 是请求顶层的 + # cached-voice id，不是 options 项，从 options 里挪出去；有参考音频时以参考为准。 + voice_preset = options.pop("voice", None) + # Irodori 会话默认 no_ref=true（无参考直接生成），带参考时须显式关掉才走克隆。 + if prof.get("no_ref_toggle") and voice_path: + options.setdefault("no_ref", False) + # IndexTTS2：emotion_text 只在 use_emotion_text=true 时生效（request.cpp）， + # 填了情绪参考文本却没勾选是最常见的坑，替用户补上。 + if options.get("emotion_text") and "use_emotion_text" not in options: + options["use_emotion_text"] = True + + seed, seed_note = _resolve_seed(seed) + payload = { + "model": model, + "language": resolve_language(prof, language), + "seed": seed, + "max_tokens": int(max_tokens), + } + auto_vibevoice_max_tokens = ( + is_vibevoice and int(max_tokens) == 1200 + ) + if voice_path: + payload["voice_ref"] = _ensure_wav(voice_path) + elif voice_preset: + payload["voice"] = voice_preset + if (reference_text or "").strip(): + payload["reference_text"] = reference_text + if options: + payload["options"] = options + + # Long text goes out as several bounded requests (concatenated below), so a + # whole chapter neither hits the per-request timeout nor runs blind. + chunks = _split_tts_chunks(text, prof.get("chunk_chars", 1000)) + if is_vibevoice: + chunks = _merge_short_vibevoice_tail(chunks) + _ui_log(_t("TTS 开始：model={model}，{count} 段 / 共 {chars} 字", + "TTS started: model={model}, {count} chunks / {chars} chars total", + model=model, count=len(chunks), chars=sum(len(c) for c in chunks))) + t_start = time.time() + blobs = [] + for i, chunk in enumerate(chunks): + if len(chunks) > 1: + progress((i, len(chunks)), desc=_t("合成 {index}/{total} 段…", + "Synthesizing {index}/{total}…", + index=i + 1, total=len(chunks))) + payload["input"] = chunk + if auto_vibevoice_max_tokens: + payload["max_tokens"] = _vibevoice_text_max_tokens(chunk, int(max_tokens)) + t_chunk = time.time() + try: + r = requests.post(f"{SERVER}/v1/audio/speech", json=payload, timeout=900) + except requests.RequestException as e: + _ui_log(_t("TTS 失败：段 {index}/{count} 无法连接 server", + "TTS failed: chunk {index}/{count} cannot connect to server", + index=i + 1, count=len(chunks))) + raise connection_error(e) + if r.status_code != 200: + _ui_log(_t("TTS 失败：段 {index}/{count}，server {code}", + "TTS failed: chunk {index}/{count}, server {code}", + index=i + 1, count=len(chunks), code=r.status_code)) + raise server_error(entry, r.status_code, r.text) + blobs.append(r.content) + _ui_log(_t("TTS 段 {index}/{count} 完成（{chars} 字，{seconds:.1f}s）", + "TTS chunk {index}/{count} done ({chars} chars, {seconds:.1f}s)", + index=i + 1, count=len(chunks), chars=len(chunk), + seconds=time.time() - t_chunk)) + + out = os.path.join(OUTPUT_DIR, f"audiocpp_tts_{int(time.time()*1000)}.wav") + if len(blobs) == 1: + with open(out, "wb") as f: + f.write(blobs[0]) + else: + _concat_wavs(blobs, out) + elapsed = time.time() - t_start + _ui_log(_t("TTS 完成：{out}，总用时 {seconds:.1f}s", + "TTS done: {out}, total {seconds:.1f}s", + out=out, seconds=elapsed)) + parts_note = (_t("（{count} 段）", " ({count} parts)", count=len(blobs)) + if len(blobs) > 1 else "") + return out, _t("✅ 生成完成{parts}，用时 {seconds:.1f}s{load}。{seed}", + "✅ Complete{parts} in {seconds:.1f}s{load}. {seed}", + parts=parts_note, seconds=elapsed, load=load_note, seed=seed_note) + except gr.Error as e: + return None, _msg_from_error(e) + except Exception as e: + return None, _t("❌ 生成失败：{error}", "❌ Generation failed: {error}", error=e) + + +def do_tts_stream(model, text, language, uploaded_voice, builtin_voice, + reference_text, seed, max_tokens, adv_values, adv_options): + """流式 TTS 生成器：产出 (音频增量, 最终文件, 状态)。 + 音频增量是 (sr, np.int16 数组)，喂给 streaming=True 的 gr.Audio 逐段追加 + 播放；结束时把完整音频写成 wav 一并给普通输出组件（可下载/回放）。 + server 端 /v1/audio/speech stream_format=sse 的 delta 是 base64 裸 PCM16。""" + if not (text or "").strip(): + raise gr.Error(_t("请输入要合成的文字", "Enter text to synthesize.")) + entry = catalog_by_id(model) + prof = profile_for(entry) if entry else DEFAULT_PROFILE + if not prof.get("supports_streaming"): + raise gr.Error(_t("模型 {model} 不支持流式生成。", "Model {model} does not support streaming.", + model=model)) + sr = int(prof.get("stream_sample_rate") or 0) + if sr <= 0: + raise gr.Error(_t("模型 {model} 缺少流式采样率配置。", + "Model {model} has no streaming sample-rate setting.", model=model)) + + t_load = time.time() + ensure_model_loaded(model, TTS_TASKS, mode="streaming") + load_s = time.time() - t_load + load_note = (_t("，含模型加载 {seconds:.1f}s", ", model load {seconds:.1f}s", + seconds=load_s) if load_s >= 1.0 else "") + options = _merged_options(prof, adv_values, adv_options) + family = entry.get("family") if entry else "" + if family == "voxcpm2": + # VoxCPM2 流式生成的硬性要求（generator.cpp:1259）：badcase 重试要 + # 重新生成整段，与已经推给播放器的音频冲突，所以流式下强制关闭。 + options["retry_badcase"] = False + options.pop("voxcpm2.retry_badcase", None) + + voice_path = None + if uploaded_voice: + voice_path = uploaded_voice + elif builtin_voice and builtin_voice != "(none)": + voice_path = os.path.join(PROMPTS_DIR, builtin_voice) + # Supertonic 等预置音色家族要求 voice 位于请求顶层，而不是 options。 + voice_preset = options.pop("voice", None) + + seed, seed_note = _resolve_seed(seed) + payload = { + "model": model, + "language": resolve_language(prof, language), + "seed": seed, + "max_tokens": int(max_tokens), + "stream": True, + "stream_format": "sse", + "response_format": "pcm", + "options": options, + } + if voice_path: + payload["voice_ref"] = _ensure_wav(voice_path) + elif voice_preset: + payload["voice"] = voice_preset + if (reference_text or "").strip(): + payload["reference_text"] = reference_text + + chunks = _split_tts_chunks(text, prof.get("chunk_chars", 1000)) + _ui_log(_t("TTS 开始（流式）：model={model}，{count} 段 / 共 {chars} 字", + "TTS started (streaming): model={model}, {count} chunks / {chars} chars total", + model=model, count=len(chunks), chars=sum(len(c) for c in chunks))) + t_start = time.time() + all_parts = [] # 全部 PCM（拼最终 wav） + pending = [] # 未推给播放器的 PCM 增量（凑批再推，免得刷屏） + pending_samples = 0 + min_push = int(sr * 0.4) # 每 ~0.4s 音频推一次播放器 + ttft_note = "" + + def _flush(): + nonlocal pending, pending_samples + if not pending: + return None + arr = pending[0] if len(pending) == 1 else np.concatenate(pending) + pending, pending_samples = [], 0 + return arr + + for i, chunk in enumerate(chunks): + seg_note = (_t("（{index}/{total} 段）", " ({index}/{total})", + index=i + 1, total=len(chunks)) if len(chunks) > 1 else "") + payload["input"] = chunk + t_chunk = time.time() + try: + r = requests.post(f"{SERVER}/v1/audio/speech", json=payload, + stream=True, timeout=900) + except requests.RequestException as e: + _ui_log(_t("TTS 失败（流式）：段 {index}/{count} 无法连接 server", + "TTS failed (streaming): chunk {index}/{count} cannot connect to server", + index=i + 1, count=len(chunks))) + raise connection_error(e) + with r: + if r.status_code != 200: + _ui_log(_t("TTS 失败（流式）：段 {index}/{count}，server {code}", + "TTS failed (streaming): chunk {index}/{count}, server {code}", + index=i + 1, count=len(chunks), code=r.status_code)) + raise server_error(entry, r.status_code, r.text) + for event in _iter_sse_events(r): + etype = event.get("type") + if etype == "speech.audio.delta": + pcm = base64.b64decode(event.get("audio") or "") + if not pcm: + continue + arr = np.frombuffer(pcm, dtype=np.int16) + all_parts.append(arr) + pending.append(arr) + pending_samples += arr.size + if pending_samples >= min_push: + out_arr = _flush() + done_s = sum(a.size for a in all_parts) / sr + yield ((sr, out_arr), None, + _t("⏳ 流式生成中{segment}…已生成 {seconds:.1f}s", + "⏳ Streaming{segment}… {seconds:.1f}s generated", + segment=seg_note, seconds=done_s)) + elif etype == "speech.audio.done": + if i == 0 and not ttft_note: + ttft = (event.get("timing") or {}).get("ttft_ms") + if ttft: + ttft_note = _t("，首包 {seconds:.1f}s", ", first audio {seconds:.1f}s", + seconds=ttft / 1000) + _ui_log(_t("TTS 段 {index}/{count} 完成（流式，{chars} 字，{seconds:.1f}s）", + "TTS chunk {index}/{count} done (streaming, {chars} chars, {seconds:.1f}s)", + index=i + 1, count=len(chunks), chars=len(chunk), + seconds=time.time() - t_chunk)) + + tail = _flush() + if tail is not None: + yield (sr, tail), None, _t("⏳ 流式生成收尾…", "⏳ Finishing stream…") + if not all_parts: + raise gr.Error(_t("流式生成没有产出音频。", "Streaming produced no audio.")) + out = os.path.join(OUTPUT_DIR, f"audiocpp_tts_stream_{int(time.time() * 1000)}.wav") + with wave.open(out, "wb") as w: + w.setnchannels(1) + w.setsampwidth(2) + w.setframerate(sr) + w.writeframes(np.concatenate(all_parts).tobytes()) + elapsed = time.time() - t_start + audio_s = sum(a.size for a in all_parts) / sr + parts_note = (_t("（{count} 段）", " ({count} parts)", count=len(chunks)) + if len(chunks) > 1 else "") + _ui_log(_t("TTS 完成（流式）：{out}，音频 {audio:.1f}s，总用时 {seconds:.1f}s", + "TTS done (streaming): {out}, audio {audio:.1f}s, total {seconds:.1f}s", + out=out, audio=audio_s, seconds=elapsed)) + yield (None, out, _t( + "✅ 流式生成完成{parts}，音频 {audio:.1f}s，用时 {elapsed:.1f}s{ttft}{load}。{seed}", + "✅ Stream complete{parts}: {audio:.1f}s audio in {elapsed:.1f}s{ttft}{load}. {seed}", + parts=parts_note, audio=audio_s, elapsed=elapsed, ttft=ttft_note, + load=load_note, seed=seed_note)) + + +def do_tts_or_stream(model, gen_mode, text, language, uploaded_voice, builtin_voice, + reference_text, seed, max_tokens, adv_values, adv_options, + progress=gr.Progress()): + """TTS 按钮统一入口：离线模式原样走 do_tts（行为不变），流式模式走 + do_tts_stream。输出：(流式播放增量, 输出文件, 状态)。 + 先 yield 一条即时提示——模型未加载/需切换模式时后续要静默等几十秒。""" + yield None, None, _t("⏳ 生成中…模型切换时可能需要重新加载。", + "⏳ Generating… model switches may require a reload.") + if gen_mode == "流式": + try: + yield from do_tts_stream(model, text, language, uploaded_voice, + builtin_voice, reference_text, seed, + max_tokens, adv_values, adv_options) + except gr.Error as e: + yield None, None, _msg_from_error(e) + except Exception as e: + yield None, None, _t("❌ 流式生成失败：{error}", + "❌ Streaming failed: {error}", error=e) + return + # 离线合成本身是阻塞的 do_tts。旧版把它直接绑到按钮（普通函数），Gradio 会在 + # 输出组件上显示原生的“处理中 X.Xs”计时；现在统一入口是 generator，上面那次 + # 即时 yield 会顶掉原生计时器，于是这里自己计时：后台线程跑 do_tts，主生成器 + # 每 ~0.5s 吐一次“已用时 Xs”，把秒数显示找回来（也顺带覆盖模型重载的等待）。 + result = {} + + def _run_offline(): + result["value"] = do_tts(model, text, language, uploaded_voice, builtin_voice, + reference_text, seed, max_tokens, adv_values, adv_options, + progress=progress) + + worker = threading.Thread(target=_run_offline, daemon=True) + t_offline = time.time() + worker.start() + while worker.is_alive(): + worker.join(0.5) + if worker.is_alive(): + yield None, None, _t("⏳ 生成中…已用时 {seconds:.1f}s", + "⏳ Generating… {seconds:.1f}s elapsed", + seconds=time.time() - t_offline) + out, msg = result.get("value", + (None, _t("❌ 生成失败：无返回结果。", "❌ Generation failed: no result."))) + yield None, out, msg + + +def _iter_sse_events(response): + """逐个产出 SSE 事件（dict）。server 每个事件一行 `data: {json}`，事件间空行 + 分隔（write_sse）；`data: [DONE]` 表示流结束。type=error 的事件直接抛错。""" + for line in response.iter_lines(decode_unicode=True): + if not line or not line.startswith("data:"): + continue + data = line[len("data:"):].strip() + if data == "[DONE]": + return + try: + event = json.loads(data) + except ValueError: + continue + if event.get("type") == "error": + err = event.get("error") or {} + raise gr.Error(_t("server 流式错误：{error}", "Server stream error: {error}", + error=err.get("message") or event)) + yield event + + +def _asr_transcribe_stream(model, entry, wav_path, extras, tag="ASR"): + """流式转写一个 WAV（server 需已按 mode=streaming 加载）：产出 + (累计文本, 是否最终, ttft_ms)。整个文件一个请求——增量出字本身就解决了 + 离线路径靠分段缓解的"长音频干等"问题。""" + payload = {"model": model, "audio": wav_path, "stream": True, **extras} + try: + r = requests.post(f"{SERVER}/v1/audio/transcriptions", json=payload, + stream=True, timeout=900) + except requests.RequestException as e: + _ui_log(_t("{tag} 失败：无法连接 server", + "{tag} failed: cannot connect to server", tag=tag)) + raise connection_error(e) + with r: + if r.status_code != 200: + _ui_log(_t("{tag} 失败：server {code}", + "{tag} failed: server {code}", tag=tag, code=r.status_code)) + raise server_error(entry, r.status_code, r.text) + text = "" + for event in _iter_sse_events(r): + etype = event.get("type") + if etype == "transcript.text.delta": + delta = event.get("delta") or "" + if entry.get("family") == "voxtral_realtime": + # Voxtral currently emits the cumulative transcript (and may + # dispatch the same event twice), despite the server exposing + # it as a delta. Replace instead of appending for this family. + if delta == text: + continue + text = delta + else: + text += delta + yield text, False, None + elif etype == "transcript.text.done": + final = (event.get("text") or text).strip() + ttft = (event.get("timing") or {}).get("ttft_ms") + yield final, True, ttft + return + raise gr.Error(_t("流式转写未收到最终结果。", "Streaming transcription returned no final result.")) + + +def _asr_transcribe_wav(model, entry, prof, wav_path, extras, tag="ASR"): + """转写一个 WAV 文件（调用方已加载好 ASR 模型）：超过单次上限（见 qwen3_asr + profile：8G 卡显存实测取 60s）时在静音处切段逐段转写再拼接；短音频/非 PCM + WAV（测不出时长）走单请求。extras 是可选的 language/context 请求字段。 + 返回 (text, dur, 段数)。""" + dur = _audio_duration_seconds(wav_path) + dur_note = f"{dur:.1f}s" if dur is not None else _t("未知", "unknown") + max_s = prof.get("max_input_seconds") + chunks = [wav_path] + if max_s and dur is not None and dur > max_s: + chunks = [p for p, _ in + _split_wav_chunks(wav_path, max_s, pad_to_max=False)] + if len(chunks) > 1: + _ui_log(_t("{tag} 长音频分段：{dur_note} → {count} 段（每段 ≤{max_s:.0f}s，静音处切分）", + "{tag} long-audio split: {dur_note} → {count} chunks (each ≤{max_s:.0f}s, split at silence)", + tag=tag, dur_note=dur_note, count=len(chunks), max_s=max_s)) + texts = [] + for i, chunk in enumerate(chunks): + t_chunk = time.time() + payload = {"model": model, "audio": chunk, **extras} + try: + r = requests.post(f"{SERVER}/v1/audio/transcriptions", json=payload, timeout=900) + except requests.RequestException as e: + _ui_log(_t("{tag} 失败：无法连接 server", + "{tag} failed: cannot connect to server", tag=tag)) + raise connection_error(e) + if r.status_code != 200: + seg_note = (_t("，第 {index}/{total} 段", ", part {index}/{total}", + index=i + 1, total=len(chunks)) if len(chunks) > 1 else "") + _ui_log(_t("{tag} 失败：server {code}（音频 {dur_note}{seg_note}）", + "{tag} failed: server {code} (audio {dur_note}{seg_note})", + tag=tag, code=r.status_code, dur_note=dur_note, seg_note=seg_note)) + extra = (_t("⏱ 本次音频时长约 {seconds:.1f} 秒", "⏱ audio is about {seconds:.1f}s long", + seconds=dur) if dur is not None else None) + raise server_error(entry, r.status_code, r.text, extra=extra) + try: + data = r.json() + texts.append((data.get("text") or str(data)).strip()) + except Exception: + texts.append(r.text) + if len(chunks) > 1: + _ui_log(_t("{tag} 段 {index}/{count} 完成（{seconds:.1f}s）", + "{tag} chunk {index}/{count} done ({seconds:.1f}s)", + tag=tag, index=i + 1, count=len(chunks), + seconds=time.time() - t_chunk)) + text = texts[0] if len(chunks) == 1 else "\n".join(t for t in texts if t) + return text, dur, len(chunks) + + +def do_asr(model, audio_path, language="", context="", dialogue=False, stream=False): + """生成器：非流式路径只 yield 一次最终结果（行为与旧版 return 完全一致—— + Gradio 对生成器处理器逐次刷新输出）；流式路径边收 SSE 增量边 yield。""" + try: + if not audio_path: + raise gr.Error(_t("请上传或录制音频", "Upload or record audio.")) + audio_path = _ensure_wav(audio_path) + # 可选转写参数：留空不发，请求体和原来完全一致（qwen3_asr 从 text_input + # 读 context/language，其它族忽略；server 端 build_openai_transcription_request + # 只在字段存在时才设置 text_input）。 + extras = {} + if (language or "").strip(): + extras["language"] = language.strip() + if (context or "").strip(): + extras["context"] = context.strip() + if dialogue: + # 对话模式走 Sortformer 切段 + 逐段离线转写，与流式互斥（勾了也忽略）。 + yield _asr_dialogue(model, audio_path, extras) + return + entry = catalog_by_id(model) + prof = profile_for(entry) if entry else DEFAULT_PROFILE + + if stream and prof.get("supports_streaming"): + yield "", _t("⏳ 转写中…模型切换时可能需要重新加载。", + "⏳ Transcribing… model switches may require a reload.") + t_load = time.time() + ensure_model_loaded(model, ASR_TASKS, mode="streaming") + load_s = time.time() - t_load + load_note = (_t("，含模型加载 {seconds:.1f}s", ", model load {seconds:.1f}s", + seconds=load_s) if load_s >= 1.0 else "") + extras_note = "".join(f"，{k}={v[:20]}" for k, v in extras.items()) + _ui_log(_t("ASR 开始（流式）：model={model}{extras_note}", + "ASR started (streaming): model={model}{extras_note}", + model=model, extras_note=extras_note)) + t_start = time.time() + dur = _audio_duration_seconds(audio_path) + dur_note = f"{dur:.1f}s" if dur is not None else _t("未知", "unknown") + last_yield = 0.0 + for text, is_final, ttft in _asr_transcribe_stream( + model, entry, audio_path, extras): + if is_final: + elapsed = time.time() - t_start + ttft_note = (_t("，首字 {seconds:.1f}s", ", first text {seconds:.1f}s", + seconds=ttft / 1000) if ttft else "") + _ui_log(_t("ASR 完成（流式）：音频 {dur_note}，用时 {seconds:.1f}s", + "ASR done (streaming): audio {dur_note}, elapsed {seconds:.1f}s", + dur_note=dur_note, seconds=elapsed)) + yield text, _t( + "✅ 流式转写完成（音频 {duration}），用时 {elapsed:.1f}s{ttft}{load}。", + "✅ Streaming transcript complete ({duration}) in {elapsed:.1f}s{ttft}{load}.", + duration=dur_note, elapsed=elapsed, ttft=ttft_note, load=load_note) + return + # 增量刷新节流：delta 可能非常密，0.15s 一次足够"边转边出字"的观感 + now = time.time() + if now - last_yield >= 0.15: + last_yield = now + yield text, _t("⏳ 流式转写中…（音频 {duration}）", + "⏳ Streaming transcript… ({duration})", duration=dur_note) + return + if stream and entry is not None: + raise gr.Error(_t("模型 {model} 不支持流式转写。", + "Model {model} does not support streaming transcription.", + model=model)) + + yield "", _t("⏳ 转写中…模型切换时可能需要重新加载。", + "⏳ Transcribing… model switches may require a reload.") + t_load = time.time() + ensure_model_loaded(model, ASR_TASKS) + load_s = time.time() - t_load + load_note = (_t("，含模型加载 {seconds:.1f}s", ", model load {seconds:.1f}s", + seconds=load_s) if load_s >= 1.0 else "") + extras_note = "".join(f"，{k}={v[:20]}" for k, v in extras.items()) + _ui_log(_t("ASR 开始：model={model}{extras_note}", + "ASR started: model={model}{extras_note}", + model=model, extras_note=extras_note)) + t_start = time.time() + text, dur, n = _asr_transcribe_wav(model, entry, prof, audio_path, extras) + elapsed = time.time() - t_start + dur_note = f"{dur:.1f}s" if dur is not None else _t("未知", "unknown") + parts_note = (_t("，{count} 段", ", {count} parts", count=n) if n > 1 else "") + _ui_log(_t("ASR 完成：音频 {dur_note}{parts_note}，用时 {seconds:.1f}s", + "ASR done: audio {dur_note}{parts_note}, elapsed {seconds:.1f}s", + dur_note=dur_note, parts_note=parts_note, seconds=elapsed)) + yield text, _t("✅ 转写完成（音频 {duration}{parts}），用时 {elapsed:.1f}s{load}。", + "✅ Transcript complete ({duration}{parts}) in {elapsed:.1f}s{load}.", + duration=dur_note, parts=parts_note, elapsed=elapsed, load=load_note) + except gr.Error as e: + yield "", _msg_from_error(e) + except Exception as e: + yield "", _t("❌ 转写失败：{error}", "❌ Transcription failed: {error}", error=e) + + +# 对话模式：同一说话人相邻发言段合并的最大间隔 / 每段前后补的余量（防止 +# Sortformer 边界切掉字头字尾）/ 短于该值的发言段丢弃（多为口头禅、气口）。 +DIALOGUE_MERGE_GAP_S = 1.0 +DIALOGUE_PAD_S = 0.25 +DIALOGUE_MIN_SEG_S = 0.3 + +# Sortformer CUDA 下是固定图容量：session_len_sec 定容量（默认 20s），上限来自 +# tf_encoder.max_source_positions=1500 @ 12.5 编码帧/秒 = 120s。 +SORTFORMER_MAX_SEC = 120 + + +def _diar_session_options(dur): + """说话人分离的动态 session_len_sec：≤20s 用默认容量（不重载）；更长的按 + 30s 步进向上取整重载（避免每个文件都重启 server）；超过 120s 上限报错。""" + if dur is None or dur <= 20: + return None + if dur > SORTFORMER_MAX_SEC: + raise gr.Error(_t("说话人分离最长约 {seconds} 秒，请先剪短音频。", + "Speaker diarization is limited to about {seconds}s. Shorten the audio.", + seconds=SORTFORMER_MAX_SEC)) + return {"session_len_sec": str(min(SORTFORMER_MAX_SEC, + ((int(dur) // 30) + 1) * 30))} + + +def _fmt_mmss(sec): + return f"{int(sec) // 60:02d}:{int(sec) % 60:02d}" + + +def _first_installed_model(task): + for m in catalog_models(): + if m.get("task") == task and m.get("installed"): + return m + return None + + +def _asr_dialogue(model, wav_path, extras): + """对话模式：Sortformer 说话人分离 → 按说话人合并/切段 → 逐段 ASR → + 带说话人标签和时间戳的对话稿。server 一次只驻留一个模型，所以先换载 diar + 再换回 ASR（每次自动换载 ~5s）。""" + diar = _first_installed_model("diar") + if diar is None: + raise gr.Error(_t("对话模式需要 Sortformer，请先在音频分析页下载安装。", + "Dialogue mode requires Sortformer. Install it from Audio analysis.")) + entry = catalog_by_id(model) + prof = profile_for(entry) if entry else DEFAULT_PROFILE + + wav16 = _to_16k_mono_wav(wav_path) + try: + with wave.open(wav16, "rb") as w: + sr = w.getframerate() + raw = w.readframes(w.getnframes()) + except Exception as e: + raise gr.Error(_t("无法读取对话音频：{error}", + "Cannot read dialogue audio: {error}", error=e)) + samples = np.frombuffer(raw, dtype=np.int16) + total_dur = len(samples) / float(sr) + + t_start = time.time() + _ui_log(_t("对话模式：先用 {id} 做说话人分离（音频 {seconds:.1f}s）", + "dialogue mode: first run speaker diarization with {id} (audio {seconds:.1f}s)", + id=diar['id'], seconds=total_dur)) + ensure_model_loaded(diar["id"], ("diar",), + session_options=_diar_session_options(total_dur)) + data = _run_task(diar, diar["id"], {"audio": wav16}, timeout=900, + log_label="说话人分离") + turns = data.get("speaker_turns") or [] + if not turns: + raise gr.Error(_t("没有检测到说话人发言段。", "No speaker turns were detected.")) + + # 合并同一说话人的相邻发言段（间隔 ≤1s 且合并后不超过 ASR 单次上限）， + # 减少请求数并给 ASR 更完整的上下文。 + max_len = (prof.get("max_input_seconds") or 60) * sr + merged = [] # [说话人, start, end] (样本数) + for t in sorted(turns, key=lambda t: t["start_sample"]): + spk, s, e = str(t.get("speaker_id", "?")), t["start_sample"], t["end_sample"] + if (merged and merged[-1][0] == spk + and s - merged[-1][2] <= DIALOGUE_MERGE_GAP_S * sr + and e - merged[-1][1] <= max_len): + merged[-1][2] = max(merged[-1][2], e) + else: + merged.append([spk, s, e]) + merged = [m for m in merged if m[2] - m[1] >= DIALOGUE_MIN_SEG_S * sr] + if not merged: + raise gr.Error(_t("说话人发言段都太短，无法转写。", + "All detected speaker turns are too short to transcribe.")) + _ui_log(_t("说话人分离完成：{turns} 个发言段 → 合并为 {merged} 段；换回 {model} 逐段转写", + "speaker diarization done: {turns} turns → merged into {merged} segments; switching back to {model} for per-segment transcription", + turns=len(turns), merged=len(merged), model=model)) + + ensure_model_loaded(model, ASR_TASKS) + pad = int(DIALOGUE_PAD_S * sr) + speakers, lines = [], [] + for idx, (spk, s, e) in enumerate(merged, 1): + a, b = max(0, s - pad), min(len(samples), e + pad) + fd, seg_path = tempfile.mkstemp(prefix=f"audiocpp_dlg{idx}_", suffix=".wav") + os.close(fd) + with wave.open(seg_path, "wb") as ww: + ww.setnchannels(1) + ww.setsampwidth(2) + ww.setframerate(sr) + ww.writeframes(samples[a:b].tobytes()) + text, _, _ = _asr_transcribe_wav(model, entry, prof, seg_path, extras, + tag=f"对话段 {idx}/{len(merged)}") + if spk not in speakers: + speakers.append(spk) + label = _t("说话人{index}", "Speaker {index}", index=speakers.index(spk) + 1) + if text: + lines.append(f"[{_fmt_mmss(s / sr)}-{_fmt_mmss(e / sr)}] {label}: {text}") + _ui_log(_t("对话段 {index}/{count} 完成（{label}，{seconds:.1f}s）", + "dialogue turn {index}/{count} done ({label}, {seconds:.1f}s)", + index=idx, count=len(merged), label=label, seconds=(e - s) / sr)) + + elapsed = time.time() - t_start + _ui_log(_t("对话转写完成：{merged} 段发言、{speakers} 个说话人，用时 {seconds:.1f}s", + "dialogue transcription done: {merged} turns, {speakers} speakers, elapsed {seconds:.1f}s", + merged=len(merged), speakers=len(speakers), seconds=elapsed)) + if not lines: + return "", _t("⚠️ 检测到发言段，但没有转写出文字。", + "⚠️ Speaker turns were detected, but no text was transcribed.") + return ("\n".join(lines), _t( + "✅ 对话转写完成（音频 {duration:.1f}s，{turns} 段，{speakers} 人），用时 {elapsed:.1f}s。", + "✅ Dialogue transcript complete ({duration:.1f}s, {turns} turns, {speakers} speakers) in {elapsed:.1f}s.", + duration=total_dur, turns=len(merged), speakers=len(speakers), elapsed=elapsed)) + + +def do_music_gen(model, text, lyrics, source_audio, duration, seed, + adv_values, adv_options): + """Music/SFX generation via the generic /v1/tasks/run route. The request + object uses the CLI request-JSON fields (text/lyrics/duration_seconds/ + task_route/audio + an options map); the response carries base64 WAV.""" + try: + if not (text or "").strip(): + raise gr.Error(_t("请输入音乐/音效提示词", "Enter a music or sound prompt.")) + ensure_model_loaded(model, GEN_TASKS) + entry = catalog_by_id(model) + prof = profile_for(entry) if entry else DEFAULT_PROFILE + options = _merged_options(prof, adv_values, adv_options) + + seed, seed_note = _resolve_seed(seed) + req = {"text": text, "seed": seed} + # task_route is a top-level request field (not a model option); the + # generated controls funnel everything through `options`, so lift it out. + route = options.pop("task_route", None) + if route: + req["task_route"] = route + if (lyrics or "").strip(): + req["lyrics"] = lyrics + if duration is not None and float(duration) != 0: + req["duration_seconds"] = float(duration) + if source_audio: + req["audio"] = _ensure_wav(source_audio) + if options: + req["options"] = options + + dur_note = req.get("duration_seconds", _t("自动", "auto")) + _ui_log(_t("音乐生成开始：model={model}，目标时长 {dur_note}s", + "music generation started: model={model}, target duration {dur_note}s", + model=model, dur_note=dur_note)) + t_start = time.time() + data = _run_task(entry, model, req, timeout=1800, log_label="音乐生成") + b64 = data.get("audio") + if not b64 and data.get("named_audio_outputs"): + b64 = data["named_audio_outputs"][0].get("audio") + if not b64: + raise gr.Error(_t("server 没有返回音频数据。", "The server returned no audio.")) + out = os.path.join(OUTPUT_DIR, f"audiocpp_gen_{int(time.time()*1000)}.wav") + with open(out, "wb") as f: + f.write(base64.b64decode(b64)) + elapsed = time.time() - t_start + _ui_log(_t("音乐生成完成：{out}，用时 {seconds:.1f}s", + "music generation done: {out}, elapsed {seconds:.1f}s", + out=out, seconds=elapsed)) + return out, _t("✅ 生成完成，用时 {seconds:.1f}s。{seed}", + "✅ Complete in {seconds:.1f}s. {seed}", seconds=elapsed, seed=seed_note) + except gr.Error as e: + return None, _msg_from_error(e) + except Exception as e: + return None, _t("❌ 生成失败：{error}", "❌ Generation failed: {error}", error=e) + + +# analyze 返回的字段 -> ACE-Step 高级参数（model_params.json 里的 name） +_ANALYZE_FILL_MAP = (("caption", "source_caption"), ("lyrics", "source_lyrics"), + ("bpm", "bpm"), ("keyscale", "keyscale"), + ("timesignature", "timesignature")) + + +def do_music_analyze(model, source_audio, seed, adv_values): + """『🔍 分析源音频』（仅 ACE-Step）：task_route=analyze 把源音频编码成语义 code， + 再用 5Hz LM 反推 caption/歌词/BPM/调性/拍号，回填到高级参数 + （写进 state 并通过 prefill state 触发控件重渲染，让填进去的值可见可改）。 + 返回 (adv_state, prefill, message)。""" + adv_values = dict(adv_values or {}) + try: + if not source_audio: + raise gr.Error(_t("请先上传源音频", "Upload source audio first.")) + entry = catalog_by_id(model) + if not entry or entry.get("family") != "ace_step": + raise gr.Error(_t("只有 ACE-Step 支持源音频分析。", + "Source analysis is available for ACE-Step only.")) + ensure_model_loaded(model, GEN_TASKS) + + # 分析要可复现：seed=-1（随机）时固定为 1234，不跟生成共享随机性； + # 用户显式填的固定 seed 仍然生效（可换 seed 重抽歌词转写）。 + analyze_seed = 1234 if int(seed if seed is not None else -1) == -1 else int(seed) + req = {"text": "analyze", "task_route": "analyze", + "audio": _ensure_wav(source_audio), "seed": analyze_seed} + dur = _audio_duration_seconds(req["audio"]) + dur_note = f"{dur:.1f}s" if dur is not None else _t("未知", "unknown") + _ui_log(_t("源音频分析开始：model={model}，音频 {dur_note}", + "source audio analysis started: model={model}, audio {dur_note}", + model=model, dur_note=dur_note)) + t_start = time.time() + data = _run_task(entry, model, req, timeout=1800, log_label="源音频分析") + raw = data.get("text") or "" + try: + info = json.loads(raw) + except Exception: + raise gr.Error(_t("server 返回的分析结果不是 JSON：{result}", + "Server analysis result is not JSON: {result}", result=raw[:200])) + elapsed = time.time() - t_start + + for src, dst in _ANALYZE_FILL_MAP: + v = info.get(src) + if v is None or v == "" or v == 0: + continue + adv_values[dst] = v + + lines = [_t("✅ 分析完成（音频 {duration}），用时 {elapsed:.1f}s；已回填高级参数。", + "✅ Analysis complete ({duration}) in {elapsed:.1f}s; advanced options updated.", + duration=dur_note, elapsed=elapsed)] + labels = (("caption", _t("描述", "Caption")), ("bpm", "BPM"), + ("keyscale", _t("调性", "Key")), + ("timesignature", _t("拍号", "Time signature")), + ("language", _t("语言", "Language")), + ("genres", _t("流派", "Genres")), + ("duration", _t("时长(s)", "Duration (s)"))) + for key, label in labels: + v = info.get(key) + if v not in (None, "", 0): + lines.append(f"- **{label}**：{v}") + if info.get("lyrics"): + lines.append(_t("- **歌词**：\n```\n{lyrics}\n```", + "- **Lyrics**:\n```\n{lyrics}\n```", lyrics=info["lyrics"])) + _ui_log(_t("源音频分析完成：用时 {seconds:.1f}s", + "source audio analysis done: elapsed {seconds:.1f}s", + seconds=elapsed)) + return adv_values, dict(adv_values), "\n".join(lines) + except gr.Error as e: + return adv_values, gr.skip(), _msg_from_error(e) + except Exception as e: + return adv_values, gr.skip(), _t("❌ 分析失败：{error}", + "❌ Analysis failed: {error}", error=e) + + +def do_vc(model, source_audio, target_upload, builtin_voice, seed, + adv_values, adv_options, progress=gr.Progress()): + """声音/歌声转换（vc/svc/s2s），走通用 /v1/tasks/run 路由：`audio` 是源音频， + `voice_ref` 是目标音色。seed_vc/miocodec 直接用这两个字段；vevo2 也接受它们 + （audio_input/voice speaker 是 source_audio/target_voice 选项的回退）， + 风格转换类 route 的额外字段（style_ref 等）由“其它参数(JSON)”兜底。 + profile 带 vc_chunk_seconds 的族（vevo2 FM 图按整段建，8G 卡长音频必炸） + 超限时按低能量点分段逐段转换，同一 voice_ref 保证各段音色一致，最后拼接。""" + try: + if not source_audio: + raise gr.Error(_t("请上传要转换的源音频", "Upload source audio to convert.")) + ensure_model_loaded(model, VC_TASKS) + entry = catalog_by_id(model) + prof = profile_for(entry) if entry else DEFAULT_PROFILE + options = _merged_options(prof, adv_values, adv_options) + + source_audio = _ensure_wav(source_audio) + # 同一 seed 用于所有分段，保证各段结果一致可复现。 + seed, seed_note = _resolve_seed(seed) + req = {"seed": seed} + voice_path = target_upload or ( + os.path.join(PROMPTS_DIR, builtin_voice) + if builtin_voice and builtin_voice != "(none)" else None) + if voice_path: + ref_wav = _ensure_wav(voice_path) + ref_cap = prof.get("vc_ref_max_seconds") + if ref_cap: # 参考音色截短，别让 prompt 段吃满显存 + ref_wav = _trim_wav_seconds(ref_wav, ref_cap) + req["voice_ref"] = ref_wav + if options: + req["options"] = options + + # vevo2 的 FM 图一次建图，序列长度 = 参考音色(prompt) + 每段源(target)。按 + # 显存预算反推每段源时长（预算 − 参考时长），令 cond 稳定落在 8G 内；没有 + # 显存预算/参考时的族仍按 vc_chunk_seconds 上限或整段发送。 + chunk_cap = prof.get("vc_chunk_seconds") + ref_sec = _audio_duration_seconds(req.get("voice_ref")) or 0.0 + if chunk_cap and prof.get("vc_fm_budget_seconds"): + min_chunk = prof.get("vc_min_chunk_seconds", 6) + budget = prof["vc_fm_budget_seconds"] + chunk_cap = int(max(min_chunk, min(chunk_cap, round(budget - ref_sec)))) + pieces = (_split_wav_chunks(source_audio, chunk_cap) + if chunk_cap else [(source_audio, 1.0)]) + dur = _audio_duration_seconds(source_audio) + dur_note = f"{dur:.1f}s" if dur is not None else _t("未知", "unknown") + seg_note = (_t("，参考 {reference:.0f}s，自适应分 {count} 段（每段 ≤{cap}s）", + ", {reference:.0f}s reference, {count} adaptive parts (≤{cap}s)", + reference=ref_sec, count=len(pieces), cap=chunk_cap) + if len(pieces) > 1 else "") + _ui_log(_t("声音转换开始：model={model}，源音频 {dur_note}{seg_note}", + "voice conversion started: model={model}, source audio {dur_note}{seg_note}", + model=model, dur_note=dur_note, seg_note=seg_note)) + t_start = time.time() + blobs, ratios = [], [] + for i, (piece, ratio) in enumerate(pieces): + if len(pieces) > 1: + progress((i, len(pieces)), desc=_t("转换 {index}/{total} 段…", + "Converting {index}/{total}…", + index=i + 1, total=len(pieces))) + req["audio"] = piece + t_seg = time.time() + data = _run_task(entry, model, req, timeout=1800, log_label="声音转换") + b64 = data.get("audio") + if not b64 and data.get("named_audio_outputs"): + b64 = data["named_audio_outputs"][0].get("audio") + if not b64: + raise gr.Error(_t("server 没有返回音频数据。", "The server returned no audio.")) + blobs.append(base64.b64decode(b64)) + ratios.append(ratio) + if len(pieces) > 1: + _ui_log(_t("声音转换段 {index}/{count} 完成（{seconds:.1f}s）", + "voice conversion segment {index}/{count} done ({seconds:.1f}s)", + index=i + 1, count=len(pieces), seconds=time.time() - t_seg)) + out = os.path.join(OUTPUT_DIR, f"audiocpp_vc_{int(time.time()*1000)}.wav") + if len(blobs) == 1 and ratios[0] >= 1.0: + with open(out, "wb") as f: + f.write(blobs[0]) + else: + _concat_wavs(blobs, out, keep_ratios=ratios) + elapsed = time.time() - t_start + _ui_log(_t("声音转换完成：{out}，用时 {seconds:.1f}s", + "voice conversion done: {out}, elapsed {seconds:.1f}s", + out=out, seconds=elapsed)) + parts_note = (_t("（{count} 段拼接）", " ({count} joined parts)", count=len(blobs)) + if len(blobs) > 1 else "") + return out, _t("✅ 转换完成{parts}，用时 {seconds:.1f}s。{seed}", + "✅ Conversion complete{parts} in {seconds:.1f}s. {seed}", + parts=parts_note, seconds=elapsed, seed=seed_note) + except gr.Error as e: + return None, _msg_from_error(e) + except Exception as e: + return None, _t("❌ 转换失败：{error}", "❌ Conversion failed: {error}", error=e) + + +# 分轨 id -> 中文标签；未收录的 id 原样显示。 +STEM_LABELS = {"vocals": "人声", "drums": "鼓", "bass": "贝斯", "other": "其它", + "instrumental": "伴奏", "accompaniment": "伴奏", "audio": "输出"} +STEM_LABELS_EN = {"vocals": "Vocals", "drums": "Drums", "bass": "Bass", "other": "Other", + "instrumental": "Instrumental", "accompaniment": "Accompaniment", + "audio": "Output"} +MAX_SEP_STEMS = 4 +# 分离族（htdemucs / mel-band-roformer）的 prepare() 硬校验 44.1kHz（模型配置 +# sample_rate），不自己重采样；其它采样率的输入在 webui 侧先转到 44.1k。 +SEP_SR = 44100 + + +def do_sep(model, audio_path): + """音源分离：响应里的 named_audio_outputs 每轨落盘成一个 wav， + 前 MAX_SEP_STEMS 轨直接放进播放器，全部轨放进文件下载列表。""" + empty = [gr.update(value=None, visible=False) for _ in range(MAX_SEP_STEMS)] + try: + if not audio_path: + raise gr.Error(_t("请上传要分离的音频", "Upload audio to separate.")) + audio_path = _ensure_wav(audio_path, target_sr=SEP_SR) + ensure_model_loaded(model, SEP_TASKS) + entry = catalog_by_id(model) + dur = _audio_duration_seconds(audio_path) + dur_note = f"{dur:.1f}s" if dur is not None else _t("未知", "unknown") + _ui_log(_t("音源分离开始：model={model}，音频时长 {dur_note}", + "source separation started: model={model}, audio duration {dur_note}", + model=model, dur_note=dur_note)) + t_start = time.time() + data = _run_task(entry, model, {"audio": audio_path}, + timeout=1800, log_label="音源分离") + stems = data.get("named_audio_outputs") or [] + if not stems and data.get("audio"): + stems = [{"id": "audio", "audio": data["audio"]}] + if not stems: + raise gr.Error(_t("server 没有返回音轨数据。", "The server returned no tracks.")) + ts = int(time.time() * 1000) + paths = [] + for stem in stems: + sid = stem.get("id") or f"stem{len(paths)}" + safe = re.sub(r"[^A-Za-z0-9_.-]", "_", sid) + p = os.path.join(OUTPUT_DIR, f"audiocpp_sep_{ts}_{safe}.wav") + with open(p, "wb") as f: + f.write(base64.b64decode(stem["audio"])) + paths.append((sid, p)) + updates = [] + for i in range(MAX_SEP_STEMS): + if i < len(paths): + sid, p = paths[i] + language = get_language() + labels = STEM_LABELS_EN if language == "en" else STEM_LABELS + label = labels.get(sid) + if label: + label = _t(label, label, language) + updates.append(gr.update( + value=p, visible=True, label=f"{label}（{sid}）" if label else sid)) + else: + updates.append(gr.update(value=None, visible=False)) + elapsed = time.time() - t_start + _ui_log(_t("音源分离完成：{tracks} 轨，用时 {seconds:.1f}s", + "source separation done: {tracks} tracks, elapsed {seconds:.1f}s", + tracks=len(paths), seconds=elapsed)) + note = ("" if len(paths) <= MAX_SEP_STEMS else + _t("，其余 {count} 轨见下载列表", "; {count} more in the download list", + count=len(paths) - MAX_SEP_STEMS)) + return (*updates, [p for _, p in paths], _t( + "✅ 分离完成（{count} 轨）{note}，用时 {elapsed:.1f}s。", + "✅ Separation complete ({count} tracks){note} in {elapsed:.1f}s.", + count=len(paths), note=note, elapsed=elapsed)) + except gr.Error as e: + return (*empty, None, _msg_from_error(e)) + except Exception as e: + return (*empty, None, _t("❌ 分离失败：{error}", + "❌ Separation failed: {error}", error=e)) + + +# VAD/diar/align 输入统一转成 16 kHz 单声道后再发（见 _to_16k_mono_wav）， +# 所以响应里的 start_sample/end_sample 一律按 16000 换算成秒。 +SR_ANALYZE = 16000 + + +def _fmt_ts(samples): + return f"{samples / SR_ANALYZE:.2f}s" + + +def do_analyze(model, audio_path, transcript, language): + """音频分析（vad/diar/align）：格式化 segments / speaker_turns / words 为 + 可读文本，原始 JSON 落盘供下载。""" + try: + if not audio_path: + raise gr.Error(_t("请上传或录制音频", "Upload or record audio.")) + entry = catalog_by_id(model) + task = entry.get("task") if entry else "" + req = {"audio": _to_16k_mono_wav(_ensure_wav(audio_path))} + if task == "align": + if not (transcript or "").strip(): + raise gr.Error(_t("强制对齐需要填写音频原文。", + "Forced alignment requires the source transcript.")) + req["text"] = transcript.strip() + if (language or "").strip(): + req["language"] = language.strip() + dur = _audio_duration_seconds(req["audio"]) + # diar（Sortformer）是固定图容量，>20s 的音频按时长动态重载。 + ensure_model_loaded(model, ANALYZE_TASKS, + session_options=(_diar_session_options(dur) + if task == "diar" else None)) + dur_note = f"{dur:.1f}s" if dur is not None else _t("未知", "unknown") + _ui_log(_t("音频分析开始：model={model}（task={task}），音频 {dur_note}", + "audio analysis started: model={model} (task={task}), audio {dur_note}", + model=model, task=task, dur_note=dur_note)) + t_start = time.time() + data = _run_task(entry, model, req, timeout=900, log_label="音频分析") + + lines = [] + if data.get("segments"): + segs = data["segments"] + lines.append(_t("共 {count} 个语音段：", "{count} speech segments:", count=len(segs))) + speech = 0 + for i, s in enumerate(segs, 1): + lines.append(_t("{index:3d}. {start} → {end}　置信度 {confidence:.2f}", + "{index:3d}. {start} → {end} confidence {confidence:.2f}", + index=i, start=_fmt_ts(s["start_sample"]), + end=_fmt_ts(s["end_sample"]), confidence=s.get("confidence", 0))) + speech += s["end_sample"] - s["start_sample"] + lines.append(_t("语音总时长约 {seconds:.1f}s", "Total speech: {seconds:.1f}s", + seconds=speech / SR_ANALYZE)) + if data.get("speaker_turns"): + turns = data["speaker_turns"] + spk = sorted({t.get("speaker_id", "?") for t in turns}) + lines.append(_t("共 {turns} 个发言段、{speakers} 个说话人（{ids}）：", + "{turns} turns, {speakers} speakers ({ids}):", + turns=len(turns), speakers=len(spk), ids=", ".join(spk))) + for i, t in enumerate(turns, 1): + lines.append(_t("{index:3d}. {start} → {end}　{speaker}　置信度 {confidence:.2f}", + "{index:3d}. {start} → {end} {speaker} confidence {confidence:.2f}", + index=i, start=_fmt_ts(t["start_sample"]), + end=_fmt_ts(t["end_sample"]), speaker=t.get("speaker_id", "?"), + confidence=t.get("confidence", 0))) + if data.get("words"): + lines.append(_t("共 {count} 个词的时间戳：", "Timestamps for {count} words:", + count=len(data["words"]))) + for w in data["words"]: + lines.append(f"{_fmt_ts(w['start_sample'])} → {_fmt_ts(w['end_sample'])}" + f"　{w.get('word', '')}") + if data.get("text"): + lines.append(_t("文本输出：{text}", "Text: {text}", text=data["text"])) + if not lines: + lines.append(_t("（模型没有返回可显示的分析结果）", + "(The model returned no displayable result.)")) + + json_path = os.path.join(OUTPUT_DIR, f"audiocpp_analyze_{int(time.time()*1000)}.json") + with open(json_path, "w", encoding="utf-8") as f: + json.dump(data, f, ensure_ascii=False, indent=2) + elapsed = time.time() - t_start + _ui_log(_t("音频分析完成：用时 {seconds:.1f}s", + "audio analysis done: elapsed {seconds:.1f}s", + seconds=elapsed)) + return ("\n".join(lines), json_path, + _t("✅ 分析完成（音频 {duration}），用时 {elapsed:.1f}s。", + "✅ Analysis complete ({duration}) in {elapsed:.1f}s.", + duration=dur_note, elapsed=elapsed)) + except gr.Error as e: + return "", None, _msg_from_error(e) + except Exception as e: + return "", None, _t("❌ 分析失败：{error}", "❌ Analysis failed: {error}", error=e) + + +def _align_fields_visibility(model_id): + """音频分析页：只有 align 任务的模型才显示『对齐文本/语言』输入。""" + entry = catalog_by_id(model_id) if model_id else None + show = bool(entry) and entry.get("task") == "align" + return gr.update(visible=show), gr.update(visible=show) + + +def do_vdes(model, text, instruct, seed, max_tokens, adv_values, adv_options): + """声音设计（vdes）：文字 + 音色描述走 /v1/audio/speech（instructions 字段 + 映射到模型的 instruct 选项），响应是 WAV 音频。""" + try: + if not (text or "").strip(): + raise gr.Error(_t("请输入要合成的文字", "Enter text to synthesize.")) + if not (instruct or "").strip(): + raise gr.Error(_t("请填写音色描述。", "Enter a voice description.")) + ensure_model_loaded(model, VDES_TASKS) + entry = catalog_by_id(model) + prof = profile_for(entry) if entry else DEFAULT_PROFILE + options = _merged_options(prof, adv_values, adv_options) + + seed, seed_note = _resolve_seed(seed) + payload = {"model": model, "input": text, + "seed": seed, "max_tokens": int(max_tokens)} + # 音色描述的落点按家族区分：qwen3_tts 走服务器的 instructions→instruct 映射； + # irodori 等只读自家 options 键（profile 里的 vdes_option_key，如 caption）。 + vdes_key = prof.get("vdes_option_key") + if vdes_key: + options.setdefault(vdes_key, instruct) + else: + payload["instructions"] = instruct + if options: + payload["options"] = options + _ui_log(_t("声音设计开始：model={model}，{chars} 字", + "voice design started: model={model}, {chars} chars", + model=model, chars=len(text))) + t_start = time.time() + try: + r = requests.post(f"{SERVER}/v1/audio/speech", json=payload, timeout=900) + except requests.RequestException as e: + _ui_log(_t("声音设计失败：无法连接 server", + "voice design failed: cannot connect to server")) + raise connection_error(e) + if r.status_code != 200: + _ui_log(_t("声音设计失败：server {code}", + "voice design failed: server {code}", code=r.status_code)) + raise server_error(entry, r.status_code, r.text) + out = os.path.join(OUTPUT_DIR, f"audiocpp_vdes_{int(time.time()*1000)}.wav") + with open(out, "wb") as f: + f.write(r.content) + elapsed = time.time() - t_start + _ui_log(_t("声音设计完成：{out}，用时 {seconds:.1f}s", + "voice design done: {out}, elapsed {seconds:.1f}s", + out=out, seconds=elapsed)) + return out, _t("✅ 生成完成，用时 {seconds:.1f}s。{seed}", + "✅ Complete in {seconds:.1f}s. {seed}", seconds=elapsed, seed=seed_note) + except gr.Error as e: + return None, _msg_from_error(e) + except Exception as e: + return None, _t("❌ 生成失败：{error}", "❌ Generation failed: {error}", error=e) + + +def _make_load_handler(tasks): + """『📥 加载模型』按钮的处理器工厂：按标签页各自的 task 集合校验并加载。""" + def _load(model): + try: + s = ensure_model_loaded(model, tasks) + except gr.Error as e: + s = _msg_from_error(e) + return s, server_status() + return _load + + +# 标签页顺序（与 refresh() 的输出、_refresh_outputs 列表一一对应）： +# TTS、ASR、音乐生成、声音转换、音源分离、音频分析、声音设计。 +TAB_SPECS = [TTS_TASKS, ASR_TASKS, GEN_TASKS, VC_TASKS, + SEP_TASKS, ANALYZE_TASKS, VDES_TASKS] + + +def refresh(): + """重读 catalog / 参数配置，刷新每个标签页的模型下拉和提示。 + 返回顺序：各页下拉更新（按 TAB_SPECS 顺序）、状态行、各页提示、 + ASR 流式选项。""" + global CATALOG, MODEL_PARAMS, REQUIRED_FILES + CATALOG = _load_catalog() + MODEL_PARAMS = _load_model_params() + REQUIRED_FILES = _load_required_files() + dropdowns, hints = [], [] + for tasks in TAB_SPECS: + choices = choices_for_tasks(tasks) + value = choices[0][1] if choices else None + dropdowns.append(gr.update(choices=choices, value=value)) + hints.append(model_hint_for(value)) + asr_model_id = dropdowns[1]["value"] if len(dropdowns) > 1 else None + return (*dropdowns, server_status(), *hints, asr_stream_update(asr_model_id)) + + +atexit.register(_stop_server) + + +_I18N_COMPONENTS = [] + + +def _localized(component, **props): + """Register translatable Gradio properties as ``prop=(zh, en)``.""" + for prop, pair in props.items(): + setattr(component, prop, _localized_prop_value(prop, pair, INITIAL_LANGUAGE)) + _I18N_COMPONENTS.append((component, props)) + return component + + +def _localized_prop_value(prop, pair, language): + """Localize one component property without changing choice protocol values.""" + if prop == "choices" and language == "zh-Hant": + localized = [] + for choice in pair[0]: + if isinstance(choice, (tuple, list)) and len(choice) == 2: + label, value = choice + else: + label = value = choice + localized.append((_t(label, label, language), value)) + return localized + return _t(pair[0], pair[1], language) + + +def _localized_updates(language): + return [ + gr.update(**{ + prop: _localized_prop_value(prop, pair, language) + for prop, pair in props.items() + }) + for _component, props in _I18N_COMPONENTS + ] + + +CUSTOM_CSS = """ + +.app-header { + align-items: center !important; + justify-content: space-between !important; + gap: 12px !important; + flex-wrap: nowrap !important; + width: 100% !important; +} +.app-title { flex: 1 1 auto !important; min-width: 0 !important; } +.app-title h1 { margin: 0 !important; } +.language-switch { + flex: 0 0 118px !important; + width: 118px !important; + min-width: 118px !important; + max-width: 118px !important; + margin: 0 0 0 auto !important; + padding: 0 !important; + background: transparent !important; + border: 0 !important; + box-shadow: none !important; +} + +.audio-default { border: none !important; box-shadow: none !important; } + +.mm-btn-row { gap: 10px !important; } +.mm-btn-row button { + border-radius: var(--button-large-radius, var(--radius-lg)) !important; + white-space: nowrap !important; +} +.mm-btn-row button span { white-space: nowrap !important; } +.gguf-btn-row { align-items: center !important; } +.gguf-btn-row > * { align-self: center !important; } + +/* 长音频的波形出现横向滚动条时，WaveSurfer 的滚动层（58px 内容 + 滚动条）会 + 溢出 Gradio 固定 58px 的 .waveform-container / #waveform，盖住下方的 + 0:00/总时长标签。放开这两层的高度让标签随内容下移；短音频（无滚动条）时 + min-height 保证布局与原来一致。 */ +.waveform-container, #waveform { height: auto !important; min-height: 58px; } + +.hint-small { opacity: 0.7; font-size: 0.85em; margin-top: 2px; } + +/* 内置参考音色行：Row 本身透明，会在按钮一列露出 gr.Group 的灰色面板底， + 和 secondary 按钮的灰底连成一片。把整行铺成和下拉 block 相同的白色卡片， + 按钮（灰底）落在卡片内自然形成对比；flex-end + margin 让按钮和下拉输入框 + 本体底部对齐（下拉 block 自带 --block-padding 内边距）。 */ +.voice-refresh-row { + align-items: flex-end !important; + background: var(--block-background-fill, #fff); + border-radius: var(--block-radius, 8px); +} +.voice-refresh-row button { + height: var(--size-10, 40px); + /* 字面量，不能用 var(--block-padding)：该主题变量是双值 "10px 12px"， + 代入 margin 简写会让整条声明非法被丢弃。11px = block 下内边距 10px + 边框， + 让按钮和下拉输入框本体上下沿精确对齐。 */ + margin: 0 12px 11px 0; + border-radius: var(--radius-lg) !important; +} +""" + +# Gradio 的 Audio 播放器（WaveSurfer）换音频时复用同一实例，旧的播放进度会 +# 原样带到新音频上（生成完成后进度条停在上一次的位置）。value=None 清空也挡 +# 不住。修法：每个播放器宿主挂一次 loadedmetadata（每次换源触发、用户 seek +# 不触发），换源后开一个 ~3s 守护窗，暂停状态下把 shadow DOM 里 wavesurfer 的 +#