diff --git a/.gitignore b/.gitignore index b27e704..c498a61 100644 --- a/.gitignore +++ b/.gitignore @@ -178,3 +178,9 @@ docs_zh/ asr2clip.conf # all audio/video files data/ + +# IDE and AI agent related +.vscode/ +.claude/ +AGENTS.md +CLAUDE.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0c9d13d..b272463 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -43,7 +43,7 @@ asr2clip uses an engine abstraction (`engines/base.py`) for ASR backends. To add 4. Add any new dependencies as optional extras in `pyproject.toml` 5. Submit a PR -See `engines/openai_compat.py` and `engines/sherpa_onnx.py` for reference implementations. +See `engines/openai_compat.py`, `engines/sherpa_onnx.py`, and `engines/whisper_cpp.py` for reference implementations. ## AI-Assisted Contributions diff --git a/asr2clip.conf.example b/asr2clip.conf.example index b7eb835..4d39e1e 100644 --- a/asr2clip.conf.example +++ b/asr2clip.conf.example @@ -15,4 +15,11 @@ model_name: "whisper-1" # or other compatible model # SiliconFlow or other compatible platform # api_base_url: "https://api.siliconflow.com/v1/" # or other compatible API base URL # api_key: "YOUR_API_KEY" # api key for the platform -# model_name: "FunAudioLLM/SenseVoiceSmall" \ No newline at end of file +# model_name: "FunAudioLLM/SenseVoiceSmall" + +# Local whisper.cpp (no API key required) +# engine: whisper_cpp +# binary: ~/whisper.cpp/build/bin/whisper-cli +# model: ~/whisper.cpp/models/ggml-small.en.bin +# vad_model: ~/whisper.cpp/models/ggml-silero-v6.2.0.bin # optional +# num_threads: 4 \ No newline at end of file diff --git a/asr2clip/config.py b/asr2clip/config.py index 5db70f5..289f013 100644 --- a/asr2clip/config.py +++ b/asr2clip/config.py @@ -45,6 +45,13 @@ # api_base_url: "https://api.siliconflow.com/v1/" # or other compatible API base URL # api_key: "YOUR_API_KEY" # api key for the platform # model_name: "FunAudioLLM/SenseVoiceSmall" + +# Local whisper.cpp (no API key required) +# engine: whisper_cpp +# binary: ~/whisper.cpp/build/bin/whisper-cli +# model: ~/whisper.cpp/models/ggml-small.en.bin +# vad_model: ~/whisper.cpp/models/ggml-silero-v6.2.0.bin # optional +# num_threads: 4 """ diff --git a/asr2clip/engines/__init__.py b/asr2clip/engines/__init__.py index ab1eddf..c73ea2e 100644 --- a/asr2clip/engines/__init__.py +++ b/asr2clip/engines/__init__.py @@ -4,6 +4,7 @@ - :class:`OpenAICompatEngine` — any OpenAI-compatible API (cloud or local server) - :class:`SherpaOnnxEngine` — local inference via sherpa-onnx (lazy import) +- :class:`WhisperCppEngine` — local inference via whisper.cpp CLI (lazy import) Use :func:`create_engine` to create an engine from a configuration dict. """ @@ -30,6 +31,7 @@ def create_engine(config: dict, **kwargs) -> BaseEngine: The engine type is determined by the ``engine`` field in config: - Not set or ``"openai_compat"``: :class:`OpenAICompatEngine` - ``"sherpa_onnx"``: :class:`SherpaOnnxEngine` (lazy-imported) + - ``"whisper_cpp"``: :class:`WhisperCppEngine` (lazy-imported) When ``engine`` is not set, behavior is fully backward-compatible with existing config files that only specify ``api_base_url``, @@ -66,8 +68,18 @@ def create_engine(config: dict, **kwargs) -> BaseEngine: model_dir=config.get("model_dir"), num_threads=config.get("num_threads", 4), ) + elif engine_type == "whisper_cpp": + from .whisper_cpp import WhisperCppEngine + + return WhisperCppEngine( + binary=config["binary"], + model=config["model"], + vad_model=config.get("vad_model"), + num_threads=config.get("num_threads", 4), + timeout_multiplier=config.get("timeout_multiplier", 5.0), + ) else: raise ValueError( f"Unknown engine type: {engine_type!r}. " - f"Supported: 'openai_compat', 'sherpa_onnx'" + f"Supported: 'openai_compat', 'sherpa_onnx', 'whisper_cpp'" ) diff --git a/asr2clip/engines/whisper_cpp.py b/asr2clip/engines/whisper_cpp.py new file mode 100644 index 0000000..da03495 --- /dev/null +++ b/asr2clip/engines/whisper_cpp.py @@ -0,0 +1,101 @@ +"""Local ASR engine using whisper.cpp. + +Transcribes audio by running a locally built ``whisper-cli`` binary with a +GGLM model. No extra Python dependencies beyond the base asr2clip install. + +See build instructions at: https://github.com/ggml-org/whisper.cpp +""" + +from __future__ import annotations + +import os +import subprocess +from pathlib import Path + +from .audio_input import AudioInput +from .base import BaseEngine, TranscriptionError, TranscriptionResult + +_MIN_TIMEOUT = 10.0 +_TIMEOUT_MULTIPLIER = 3.0 + + +class WhisperCppEngine(BaseEngine): + """ASR engine using whisper.cpp ``whisper-cli``. + + Audio is passed with :meth:`~asr2clip.engines.AudioInput.as_file` so + file-backed input is not re-encoded. Transcription result is taken direct + from stdout. + + Args: + binary: Path to ``whisper-cli`` (or a compatible binary). + model: Path to a GGML model (``.bin``). + vad_model: Optional Silero VAD model for ``--vad``. + num_threads: Number of threads (``-t``). + timeout_multiplier: Scales the timeout with audio duration. + """ + + def __init__( + self, + binary: str, + model: str, + vad_model: str | None = None, + num_threads: int = 4, + timeout_multiplier: float = _TIMEOUT_MULTIPLIER, + ) -> None: + self._binary = os.path.expanduser(binary) + self._model = os.path.expanduser(model) + self._vad_model = os.path.expanduser(vad_model) if vad_model else None + self._num_threads = num_threads + self._timeout_multiplier = timeout_multiplier + + def transcribe( + self, + audio: AudioInput, + language: str | None = None, + ) -> TranscriptionResult: + samples, sr = audio.as_numpy() + duration = len(samples) / sr if sr else 0.0 + cmd = [ + self._binary, "-m", self._model, "-f", audio.as_file(), "-t", + str(self._num_threads), "-np", "-nt", + ] + if language: + cmd.extend(["-l", language]) + if self._vad_model: + cmd.extend(["--vad", "-vm", self._vad_model]) + + timeout = _MIN_TIMEOUT + duration * self._timeout_multiplier + try: + proc = subprocess.run( + cmd, capture_output=True, text=True, timeout=timeout, check=False + ) + except subprocess.TimeoutExpired as e: + raise TranscriptionError( + f"whisper.cpp timed out after {timeout:.0f}s" + ) from e + except OSError as e: + raise TranscriptionError(f"Failed to run whisper.cpp: {e}") from e + + if proc.returncode != 0: + raise TranscriptionError( + f"whisper.cpp exited with code {proc.returncode}: " + f"{(proc.stderr or proc.stdout or '').strip()}" + ) + + text = proc.stdout.strip() + if not text: + raise TranscriptionError("whisper.cpp produced no transcription text") + return TranscriptionResult(text=text, duration=duration) + + def test(self) -> bool: + if not os.path.isfile(self._binary) or not os.access(self._binary, os.X_OK): + return False + if not os.path.isfile(self._model): + return False + if self._vad_model and not os.path.isfile(self._vad_model): + return False + return True + + @property + def name(self) -> str: + return f"whisper.cpp/{Path(self._model).stem}"