From 7111f1bb2c95b80073746a3312442380b91a8c95 Mon Sep 17 00:00:00 2001 From: Aliaksandr Babrykovich Date: Sun, 12 Jul 2026 23:16:00 +0200 Subject: [PATCH 1/2] Reuse TTS synthesizer in API server The OpenAI-compatible TTS API recreated SpeechSynthesizer and SpokenTextConverter on every request, reloading the ONNX session each time. Cache both at module scope and warm them during Litestar startup, matching the main Glados engine behavior. Co-authored-by: Cursor --- src/glados/api/app.py | 11 +++++++++-- src/glados/api/tts.py | 24 ++++++++++++++++++++---- 2 files changed, 29 insertions(+), 6 deletions(-) diff --git a/src/glados/api/app.py b/src/glados/api/app.py index 28a355dd..646e0322 100644 --- a/src/glados/api/app.py +++ b/src/glados/api/app.py @@ -1,3 +1,4 @@ +from contextlib import asynccontextmanager from dataclasses import dataclass import io from typing import Literal @@ -6,7 +7,7 @@ from litestar.response import Stream from .log import structlog_plugin -from .tts import write_glados_audio_file +from .tts import warm_tts, write_glados_audio_file Voice = Literal["glados"] ResponseFormat = Literal["mp3", "wav", "ogg"] @@ -49,4 +50,10 @@ async def create_speech(data: RequestData) -> Stream: ) -app = Litestar([create_speech], plugins=[structlog_plugin]) +@asynccontextmanager +async def lifespan(app: Litestar): + warm_tts() + yield + + +app = Litestar([create_speech], plugins=[structlog_plugin], lifespan=[lifespan]) diff --git a/src/glados/api/tts.py b/src/glados/api/tts.py index f03120c1..f197dad7 100644 --- a/src/glados/api/tts.py +++ b/src/glados/api/tts.py @@ -1,11 +1,28 @@ import io +from functools import lru_cache import soundfile as sf -from glados.TTS import tts_glados +from glados.TTS import get_speech_synthesizer from glados.utils import spoken_text_converter +@lru_cache(maxsize=1) +def _get_synthesizer(): + return get_speech_synthesizer("glados") + + +@lru_cache(maxsize=1) +def _get_text_converter() -> spoken_text_converter.SpokenTextConverter: + return spoken_text_converter.SpokenTextConverter() + + +def warm_tts() -> None: + """Load ONNX session and text converter at startup.""" + _get_synthesizer() + _get_text_converter() + + def write_glados_audio_file(f: str | io.BytesIO, text: str, *, format: str) -> None: """Generate GLaDOS-style speech audio from text and write to a file. @@ -14,9 +31,8 @@ def write_glados_audio_file(f: str | io.BytesIO, text: str, *, format: str) -> N text: Text to convert to speech format: Audio format (e.g., "mp3", "wav", "ogg") """ - glados_tts = tts_glados.SpeechSynthesizer() - converter = spoken_text_converter.SpokenTextConverter() - converted_text = converter.text_to_spoken(text) + glados_tts = _get_synthesizer() + converted_text = _get_text_converter().text_to_spoken(text) audio = glados_tts.generate_speech_audio(converted_text) sf.write( f, From 6ebf83d4bcb57bf96f717b194bb953206cadd707 Mon Sep 17 00:00:00 2001 From: Aliaksandr Babrykovich Date: Sun, 12 Jul 2026 23:16:13 +0200 Subject: [PATCH 2/2] Add reuse_tts feature flag for API server Introduce ApiConfig (model-only subconfig) and configs/api_config.yaml with reuse_tts defaulting to true, plus GLADOS_API_REUSE_TTS env override. When enabled, cache SpeechSynthesizer and warm it at Litestar startup; when disabled, keep per-request ONNX loading. Update scripts/serve and Docker to use create_app. Co-authored-by: Cursor --- Dockerfile | 2 +- README.md | 13 +++++++++++++ configs/api_config.yaml | 3 +++ scripts/serve | 2 +- src/glados/api/app.py | 35 +++++++++++++++++++++++++++++------ src/glados/api/config.py | 19 +++++++++++++++++++ src/glados/api/tts.py | 27 ++++++++++++++++++++++----- tests/test_api_config.py | 34 ++++++++++++++++++++++++++++++++++ 8 files changed, 122 insertions(+), 13 deletions(-) create mode 100644 configs/api_config.yaml create mode 100644 src/glados/api/config.py create mode 100644 tests/test_api_config.py diff --git a/Dockerfile b/Dockerfile index 4bcefafd..eb93f634 100644 --- a/Dockerfile +++ b/Dockerfile @@ -28,4 +28,4 @@ RUN uv sync --extra api --extra cpu --no-dev \ && uv run glados download EXPOSE 5050 -CMD ["uv", "run", "litestar", "--app", "glados.api.app:app", "run", "--host", "0.0.0.0", "--port", "5050"] +CMD ["uv", "run", "litestar", "--app", "glados.api.app:create_app", "run", "--host", "0.0.0.0", "--port", "5050"] diff --git a/README.md b/README.md index 4da905fa..3f9e03c2 100644 --- a/README.md +++ b/README.md @@ -506,6 +506,19 @@ Or Docker: docker compose up -d --build ``` +By default the API reuses one `SpeechSynthesizer` across requests (`reuse_tts: true` in `configs/api_config.yaml`). Disable it to restore per-request model loading: + +```yaml +Api: + reuse_tts: false +``` + +Environment override: + +```bash +GLADOS_API_REUSE_TTS=false ./scripts/serve +``` + Generate speech: ```bash curl -X POST http://localhost:5050/v1/audio/speech \ diff --git a/configs/api_config.yaml b/configs/api_config.yaml new file mode 100644 index 00000000..8562e5d7 --- /dev/null +++ b/configs/api_config.yaml @@ -0,0 +1,3 @@ +Api: + # Reuse one SpeechSynthesizer across requests instead of reloading ONNX each call. + reuse_tts: true diff --git a/scripts/serve b/scripts/serve index d71315cd..98a14ad9 100755 --- a/scripts/serve +++ b/scripts/serve @@ -2,4 +2,4 @@ # Start the API server. set -e -uv run litestar --app glados.api.app:app run --host 0.0.0.0 --port 5050 --debug --reload +uv run litestar --app glados.api.app:create_app run --host 0.0.0.0 --port 5050 --debug --reload diff --git a/src/glados/api/app.py b/src/glados/api/app.py index 646e0322..d248cce0 100644 --- a/src/glados/api/app.py +++ b/src/glados/api/app.py @@ -1,17 +1,25 @@ +from collections.abc import AsyncIterator from contextlib import asynccontextmanager from dataclasses import dataclass import io +from pathlib import Path from typing import Literal from litestar import Litestar, post from litestar.response import Stream +import yaml +from glados.utils.resources import resource_path + +from .config import ApiConfig from .log import structlog_plugin -from .tts import warm_tts, write_glados_audio_file +from .tts import configure_tts, write_glados_audio_file Voice = Literal["glados"] ResponseFormat = Literal["mp3", "wav", "ogg"] +DEFAULT_API_CONFIG = resource_path("configs/api_config.yaml") + @dataclass class RequestData: @@ -25,6 +33,16 @@ class RequestData: CONTENT_TYPES: dict[ResponseFormat, str] = {"mp3": "audio/mpeg", "wav": "audio/wav", "ogg": "audio/ogg"} +def load_api_config(path: str | Path | None = None) -> ApiConfig: + """Load API settings from configs/api_config.yaml (Api: section).""" + config_path = Path(path) if path is not None else DEFAULT_API_CONFIG + data: dict[str, object] = {} + if config_path.exists(): + raw = yaml.safe_load(config_path.read_text(encoding="utf-8")) or {} + data = raw.get("Api", {}) or {} + return ApiConfig.model_validate(data) + + @post("/v1/audio/speech") async def create_speech(data: RequestData) -> Stream: """ @@ -50,10 +68,15 @@ async def create_speech(data: RequestData) -> Stream: ) -@asynccontextmanager -async def lifespan(app: Litestar): - warm_tts() - yield +def create_app() -> Litestar: + """Create the Litestar application for the TTS API server.""" + + @asynccontextmanager + async def lifespan(app: Litestar) -> AsyncIterator[None]: + configure_tts(load_api_config()) + yield + + return Litestar([create_speech], plugins=[structlog_plugin], lifespan=[lifespan]) -app = Litestar([create_speech], plugins=[structlog_plugin], lifespan=[lifespan]) +app = create_app() diff --git a/src/glados/api/config.py b/src/glados/api/config.py new file mode 100644 index 00000000..ab73addd --- /dev/null +++ b/src/glados/api/config.py @@ -0,0 +1,19 @@ +import os + +from pydantic import BaseModel, Field, model_validator + + +class ApiConfig(BaseModel): + """Configuration for the OpenAI-compatible TTS API server.""" + + reuse_tts: bool = Field( + default=True, + description="Reuse a single SpeechSynthesizer instance across requests instead of reloading ONNX each call.", + ) + + @model_validator(mode="after") + def _apply_env_overrides(self) -> "ApiConfig": + env_reuse_tts = os.environ.get("GLADOS_API_REUSE_TTS") + if env_reuse_tts is not None: + self.reuse_tts = env_reuse_tts.lower() in ("1", "true", "yes") + return self diff --git a/src/glados/api/tts.py b/src/glados/api/tts.py index f197dad7..9a2486e3 100644 --- a/src/glados/api/tts.py +++ b/src/glados/api/tts.py @@ -1,14 +1,26 @@ -import io from functools import lru_cache +import io import soundfile as sf -from glados.TTS import get_speech_synthesizer +from glados.TTS import SpeechSynthesizerProtocol, get_speech_synthesizer from glados.utils import spoken_text_converter +from .config import ApiConfig + +_api_config = ApiConfig() + + +def configure_tts(config: ApiConfig) -> None: + """Apply API TTS settings and warm models when reuse is enabled.""" + global _api_config + _api_config = config + if config.reuse_tts: + warm_tts() + @lru_cache(maxsize=1) -def _get_synthesizer(): +def _get_synthesizer() -> SpeechSynthesizerProtocol: return get_speech_synthesizer("glados") @@ -31,8 +43,13 @@ def write_glados_audio_file(f: str | io.BytesIO, text: str, *, format: str) -> N text: Text to convert to speech format: Audio format (e.g., "mp3", "wav", "ogg") """ - glados_tts = _get_synthesizer() - converted_text = _get_text_converter().text_to_spoken(text) + if _api_config.reuse_tts: + glados_tts = _get_synthesizer() + converted_text = _get_text_converter().text_to_spoken(text) + else: + glados_tts = get_speech_synthesizer("glados") + converted_text = spoken_text_converter.SpokenTextConverter().text_to_spoken(text) + audio = glados_tts.generate_speech_audio(converted_text) sf.write( f, diff --git a/tests/test_api_config.py b/tests/test_api_config.py new file mode 100644 index 00000000..31c42e38 --- /dev/null +++ b/tests/test_api_config.py @@ -0,0 +1,34 @@ +from pathlib import Path + +import pytest + +from glados.api.app import load_api_config +from glados.api.config import ApiConfig + + +def test_api_config_defaults_to_reuse_tts() -> None: + config = ApiConfig() + assert config.reuse_tts is True + + +def test_api_config_yaml_section(tmp_path: Path) -> None: + config_file = tmp_path / "api_config.yaml" + config_file.write_text("Api:\n reuse_tts: false\n", encoding="utf-8") + + config = load_api_config(str(config_file)) + + assert config.reuse_tts is False + + +def test_api_config_env_override(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("GLADOS_API_REUSE_TTS", "false") + + config = ApiConfig.model_validate({}) + + assert config.reuse_tts is False + + +def test_load_api_config_uses_defaults_when_file_missing(tmp_path: Path) -> None: + config = load_api_config(str(tmp_path / "missing.yaml")) + + assert config.reuse_tts is True