feat: add whisper.cpp engine via CLI wrapper abstraction (and outline path to real-time whisper.cpp with VAD)

.gitignore

@@ -178,3 +178,9 @@ docs_zh/
 asr2clip.conf
 # all audio/video files
 data/
+
+# IDE and AI agent related
+.vscode/
+.claude/
+AGENTS.md
+CLAUDE.md

CONTRIBUTING.md

@@ -43,7 +43,7 @@ asr2clip uses an engine abstraction (`engines/base.py`) for ASR backends. To add
 4. Add any new dependencies as optional extras in `pyproject.toml`
 5. Submit a PR
 
-See `engines/openai_compat.py` and `engines/sherpa_onnx.py` for reference implementations.
+See `engines/openai_compat.py`, `engines/sherpa_onnx.py`, and `engines/whisper_cpp.py` for reference implementations.
 
 ## AI-Assisted Contributions
 

asr2clip.conf.example

@@ -15,4 +15,11 @@ model_name: "whisper-1"                     # or other compatible model
 # SiliconFlow or other compatible platform
 # api_base_url: "https://api.siliconflow.com/v1/"  # or other compatible API base URL
 # api_key: "YOUR_API_KEY"                          # api key for the platform
-# model_name: "FunAudioLLM/SenseVoiceSmall"
+# model_name: "FunAudioLLM/SenseVoiceSmall"
+
+# Local whisper.cpp (no API key required)
+# engine: whisper_cpp
+# binary: ~/whisper.cpp/build/bin/whisper-cli
+# model: ~/whisper.cpp/models/ggml-small.en.bin
+# vad_model: ~/whisper.cpp/models/ggml-silero-v6.2.0.bin  # optional
+# num_threads: 4

asr2clip/config.py

@@ -45,6 +45,13 @@
 # api_base_url: "https://api.siliconflow.com/v1/"  # or other compatible API base URL
 # api_key: "YOUR_API_KEY"                          # api key for the platform
 # model_name: "FunAudioLLM/SenseVoiceSmall"
+
+# Local whisper.cpp (no API key required)
+# engine: whisper_cpp
+# binary: ~/whisper.cpp/build/bin/whisper-cli
+# model: ~/whisper.cpp/models/ggml-small.en.bin
+# vad_model: ~/whisper.cpp/models/ggml-silero-v6.2.0.bin  # optional
+# num_threads: 4
 """
 
 

asr2clip/engines/__init__.py

@@ -4,6 +4,7 @@
 
 - :class:`OpenAICompatEngine` — any OpenAI-compatible API (cloud or local server)
 - :class:`SherpaOnnxEngine` — local inference via sherpa-onnx (lazy import)
+- :class:`WhisperCppEngine` — local inference via whisper.cpp CLI (lazy import)
 
 Use :func:`create_engine` to create an engine from a configuration dict.
 """
@@ -30,6 +31,7 @@ def create_engine(config: dict, **kwargs) -> BaseEngine:
     The engine type is determined by the ``engine`` field in config:
     - Not set or ``"openai_compat"``: :class:`OpenAICompatEngine`
     - ``"sherpa_onnx"``: :class:`SherpaOnnxEngine` (lazy-imported)
+    - ``"whisper_cpp"``: :class:`WhisperCppEngine` (lazy-imported)
 
     When ``engine`` is not set, behavior is fully backward-compatible
     with existing config files that only specify ``api_base_url``,
@@ -66,8 +68,18 @@ def create_engine(config: dict, **kwargs) -> BaseEngine:
             model_dir=config.get("model_dir"),
             num_threads=config.get("num_threads", 4),
         )
+    elif engine_type == "whisper_cpp":
+        from .whisper_cpp import WhisperCppEngine
+
+        return WhisperCppEngine(
+            binary=config["binary"],
+            model=config["model"],
+            vad_model=config.get("vad_model"),
+            num_threads=config.get("num_threads", 4),
+            timeout_multiplier=config.get("timeout_multiplier", 5.0),
+        )
     else:
         raise ValueError(
             f"Unknown engine type: {engine_type!r}. "
-            f"Supported: 'openai_compat', 'sherpa_onnx'"
+            f"Supported: 'openai_compat', 'sherpa_onnx', 'whisper_cpp'"
         )

asr2clip/engines/whisper_cpp.py

@@ -0,0 +1,101 @@
+"""Local ASR engine using whisper.cpp.
+
+Transcribes audio by running a locally built ``whisper-cli`` binary with a
+GGLM model. No extra Python dependencies beyond the base asr2clip install.
+
+See build instructions at: https://github.com/ggml-org/whisper.cpp
+"""
+
+from __future__ import annotations
+
+import os
+import subprocess
+from pathlib import Path
+
+from .audio_input import AudioInput
+from .base import BaseEngine, TranscriptionError, TranscriptionResult
+
+_MIN_TIMEOUT = 10.0
+_TIMEOUT_MULTIPLIER = 3.0
+
+
+class WhisperCppEngine(BaseEngine):
+    """ASR engine using whisper.cpp ``whisper-cli``.
+
+    Audio is passed with :meth:`~asr2clip.engines.AudioInput.as_file` so
+    file-backed input is not re-encoded. Transcription result is taken direct
+    from stdout.
+
+    Args:
+        binary: Path to ``whisper-cli`` (or a compatible binary).
+        model: Path to a GGML model (``.bin``).
+        vad_model: Optional Silero VAD model for ``--vad``.
+        num_threads: Number of threads (``-t``).
+        timeout_multiplier: Scales the timeout with audio duration.
+    """
+
+    def __init__(
+        self,
+        binary: str,
+        model: str,
+        vad_model: str | None = None,
+        num_threads: int = 4,
+        timeout_multiplier: float = _TIMEOUT_MULTIPLIER,
+    ) -> None:
+        self._binary = os.path.expanduser(binary)
+        self._model = os.path.expanduser(model)
+        self._vad_model = os.path.expanduser(vad_model) if vad_model else None
+        self._num_threads = num_threads
+        self._timeout_multiplier = timeout_multiplier
+
+    def transcribe(
+        self,
+        audio: AudioInput,
+        language: str | None = None,
+    ) -> TranscriptionResult:
+        samples, sr = audio.as_numpy()
+        duration = len(samples) / sr if sr else 0.0
+        cmd = [
+            self._binary, "-m", self._model, "-f", audio.as_file(), "-t",
+            str(self._num_threads), "-np", "-nt",
+        ]
+        if language:
+            cmd.extend(["-l", language])
+        if self._vad_model:
+            cmd.extend(["--vad", "-vm", self._vad_model])
+
+        timeout = _MIN_TIMEOUT + duration * self._timeout_multiplier
+        try:
+            proc = subprocess.run(
+                cmd, capture_output=True, text=True, timeout=timeout, check=False
+            )
+        except subprocess.TimeoutExpired as e:
+            raise TranscriptionError(
+                f"whisper.cpp timed out after {timeout:.0f}s"
+            ) from e
+        except OSError as e:
+            raise TranscriptionError(f"Failed to run whisper.cpp: {e}") from e
+
+        if proc.returncode != 0:
+            raise TranscriptionError(
+                f"whisper.cpp exited with code {proc.returncode}: "
+                f"{(proc.stderr or proc.stdout or '').strip()}"
+            )
+
+        text = proc.stdout.strip()
+        if not text:
+            raise TranscriptionError("whisper.cpp produced no transcription text")
+        return TranscriptionResult(text=text, duration=duration)
+
+    def test(self) -> bool:
+        if not os.path.isfile(self._binary) or not os.access(self._binary, os.X_OK):
+            return False
+        if not os.path.isfile(self._model):
+            return False
+        if self._vad_model and not os.path.isfile(self._vad_model):
+            return False
+        return True
+
+    @property
+    def name(self) -> str:
+        return f"whisper.cpp/{Path(self._model).stem}"