From 728e30cd09e2f30de9c76fcbbedbd7230677faf1 Mon Sep 17 00:00:00 2001 From: ogad-tether Date: Tue, 14 Jul 2026 16:17:43 +0100 Subject: [PATCH] speech addons: fast CPU kernels on desktop linux-arm64 (consume ggml-speech 2026-07-14) Desktop linux-arm64 prebuilds compiled ggml at the plain armv8-a baseline: the ARM dotprod/fp16/i8mm kernels were compiled out and both f16 and quantized models ran on the slowest generic paths (Android already builds GGML_CPU_ALL_VARIANTS). The build-flags change lives in the ggml-speech vcpkg port (2026-07-14, qvac-registry-vcpkg PR #249): per-arch runtime-scored CPU MODULE variants (SIGILL-safe on any aarch64 host), static libc++ in the modules, flat .so staging in lib/, SVE tiers excluded (2-4x slower than NEON on the VL=128 cores in the desktop fleet). This PR consumes the new port version in bci-whispercpp, transcription-whispercpp and tts-ggml, and makes the two whisper addons load the dynamically-loaded backends on linux-arm64: - WhisperModel/BCIModel: the ensureBackendsLoadedAndroid gate (now ensureBackendsLoaded) widens from __ANDROID__ to __ANDROID__ || (__linux__ && __aarch64__) -- with GGML_BACKEND_DL=ON no backend is statically registered and the addon must call ggml_backend_load_all_from_path(backendsDir/BACKENDS_SUBDIR) before whisper_init. Kept off linux-x64, which stays on static backends. tts-ggml already threads backends_dir unconditionally through tts-cpp EngineOptions. Verified end-to-end via overlay-port benchmark runs (ubuntu-24.04-arm, mean RTF): whisper base f16 0.3316 -> 0.0970 / q8_0 0.1270 -> 0.0629, small f16 1.3450 -> 0.3426 (run 29333221534); chatterbox q4 2.70 -> 2.28 (run 29333223845); bci green (run 29333225816). All other platforms and Android unchanged. Co-Authored-By: Claude Fable 5 --- packages/bci-whispercpp/CHANGELOG.md | 1 + .../addon/src/model-interface/bci/BCIModel.cpp | 17 +++++++++-------- packages/bci-whispercpp/vcpkg.json | 2 +- packages/transcription-whispercpp/CHANGELOG.md | 1 + .../whisper.cpp/WhisperModel.cpp | 17 +++++++++-------- packages/transcription-whispercpp/vcpkg.json | 2 +- packages/tts-ggml/CHANGELOG.md | 4 ++++ packages/tts-ggml/vcpkg.json | 2 +- 8 files changed, 27 insertions(+), 19 deletions(-) diff --git a/packages/bci-whispercpp/CHANGELOG.md b/packages/bci-whispercpp/CHANGELOG.md index 717e900f5f..0a59547fa2 100644 --- a/packages/bci-whispercpp/CHANGELOG.md +++ b/packages/bci-whispercpp/CHANGELOG.md @@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed - Faster desktop neural-signal preprocessing on the GPU (Vulkan) and CPU paths. Public API and transcription output are unchanged. +- Desktop linux-arm64 prebuilds now ship per-arch ggml CPU variants (`whisper-cpp` override 1.9.1#3, pulling `ggml-speech` 2026-07-14): the previous armv8-a-baseline build compiled out the ARM dotprod/fp16 kernels. The addon now loads the dynamically-loadable ggml backends on linux-arm64 (previously Android-only). ## [0.5.0] - 2026-07-14 diff --git a/packages/bci-whispercpp/addon/src/model-interface/bci/BCIModel.cpp b/packages/bci-whispercpp/addon/src/model-interface/bci/BCIModel.cpp index 1217080bf8..913dfeb7d5 100644 --- a/packages/bci-whispercpp/addon/src/model-interface/bci/BCIModel.cpp +++ b/packages/bci-whispercpp/addon/src/model-interface/bci/BCIModel.cpp @@ -32,9 +32,10 @@ static bool shouldAbortWhisper(void* userData) { cancelRequested->load(std::memory_order_relaxed); } -#if defined(__ANDROID__) +#if defined(__ANDROID__) || (defined(__linux__) && defined(__aarch64__)) namespace { -// Android ships ggml with `GGML_BACKEND_DL=ON`, so no backend is +// Android -- and, since ggml-speech 2026-07-14, desktop linux-arm64 +// prebuilds too -- ship ggml with `GGML_BACKEND_DL=ON`, so no backend is // statically registered. dlopen the per-arch CPU + GPU `.so` modules // once per process; otherwise whisper_init aborts on a NULL CPU // device. Mirrors transcription-whispercpp / llm-llamacpp / @@ -50,13 +51,13 @@ namespace { // `GGML_BACKEND_DL=ON` flag; from that point on this function is // the only thing standing between us and the SIGABRT we already // hit on transcription's PR #2124. -void ensureBackendsLoadedAndroid(const std::string& backendsDir) { +void ensureBackendsLoaded(const std::string& backendsDir) { static std::once_flag flag; std::call_once(flag, [&]() { if (backendsDir.empty()) { QLOG( qvac_lib_inference_addon_cpp::logger::Priority::WARNING, - "Android: configurationParams.backendsDir not set; falling back to " + "configurationParams.backendsDir not set; falling back to " "ggml_backend_load_all() (default search path). CPU / Vulkan / " "OpenCL registration may fail inside an APK with default " "compressed-native-libs packaging."); @@ -73,13 +74,13 @@ void ensureBackendsLoadedAndroid(const std::string& backendsDir) { #endif QLOG( qvac_lib_inference_addon_cpp::logger::Priority::INFO, - std::string("Android: loading ggml backends from: ") + + std::string("loading ggml backends from: ") + variantsDir.string()); ggml_backend_load_all_from_path(variantsDir.string().c_str()); }); } } // namespace -#endif // __ANDROID__ +#endif // __ANDROID__ || linux-arm64 BCIModel::BCIModel(BCIConfig config) : cfg_(std::move(config)), neuralProcessor_() {} @@ -187,8 +188,8 @@ int adrenoOpenclGpuDeviceIndex() { void BCIModel::load() { if (ctx_) return; -#if defined(__ANDROID__) - ensureBackendsLoadedAndroid(cfg_.backendsDir); +#if defined(__ANDROID__) || (defined(__linux__) && defined(__aarch64__)) + ensureBackendsLoaded(cfg_.backendsDir); #endif whisper_context_params contextParams = toWhisperContextParams(cfg_); diff --git a/packages/bci-whispercpp/vcpkg.json b/packages/bci-whispercpp/vcpkg.json index 19f26a1a52..e88ed85694 100644 --- a/packages/bci-whispercpp/vcpkg.json +++ b/packages/bci-whispercpp/vcpkg.json @@ -38,7 +38,7 @@ { "name": "whisper-cpp", "version": "1.9.1", - "port-version": 1 + "port-version": 3 } ] } diff --git a/packages/transcription-whispercpp/CHANGELOG.md b/packages/transcription-whispercpp/CHANGELOG.md index 8b86ae2621..4f99e56029 100644 --- a/packages/transcription-whispercpp/CHANGELOG.md +++ b/packages/transcription-whispercpp/CHANGELOG.md @@ -41,6 +41,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 objects, and split large C++/JS functions (and their inlined loops) into smaller named helpers per the team coding standards. These are internal refactors with no public API change. +- Desktop linux-arm64 prebuilds now ship per-arch ggml CPU variants (`whisper-cpp` override 1.9.1#3, pulling `ggml-speech` 2026-07-14): the previous armv8-a-baseline build compiled out the ARM dotprod/fp16 kernels (base f16 mean RTF 0.332 -> 0.097, base q8_0 0.127 -> 0.063, small f16 1.345 -> 0.343 on ubuntu-24.04-arm). The addon now loads the dynamically-loadable ggml backends on linux-arm64 (previously Android-only). ## [0.12.0] - 2026-07-14 diff --git a/packages/transcription-whispercpp/addon/src/model-interface/whisper.cpp/WhisperModel.cpp b/packages/transcription-whispercpp/addon/src/model-interface/whisper.cpp/WhisperModel.cpp index 2af354a22d..7e0ff38c46 100644 --- a/packages/transcription-whispercpp/addon/src/model-interface/whisper.cpp/WhisperModel.cpp +++ b/packages/transcription-whispercpp/addon/src/model-interface/whisper.cpp/WhisperModel.cpp @@ -73,19 +73,20 @@ auto WhisperModel::formatCaptionOutput(Transcript& transcript) -> void { std::to_string(static_cast(transcript.end)) + "|>"; } -#if defined(__ANDROID__) +#if defined(__ANDROID__) || (defined(__linux__) && defined(__aarch64__)) namespace { -// Android ships ggml with `GGML_BACKEND_DL=ON`, so no backend is +// Android -- and, since ggml-speech 2026-07-14, desktop linux-arm64 +// prebuilds too -- ship ggml with `GGML_BACKEND_DL=ON`, so no backend is // statically registered. dlopen the per-arch CPU + GPU `.so` modules // once per process; otherwise whisper_init aborts on a NULL CPU device. // Mirrors packages/{diffusion-cpp,llm-llamacpp,classification-ggml,…}. -void ensureBackendsLoadedAndroid(const std::string& backendsDir) { +void ensureBackendsLoaded(const std::string& backendsDir) { static std::once_flag flag; std::call_once(flag, [&]() { if (backendsDir.empty()) { QLOG( qvac_lib_inference_addon_cpp::logger::Priority::WARNING, - "Android: configurationParams.backendsDir not set; falling back to " + "configurationParams.backendsDir not set; falling back to " "ggml_backend_load_all() (default search path). CPU/Vulkan/OpenCL " "registration may fail inside an APK."); ggml_backend_load_all(); @@ -101,13 +102,13 @@ void ensureBackendsLoadedAndroid(const std::string& backendsDir) { #endif QLOG( qvac_lib_inference_addon_cpp::logger::Priority::INFO, - std::string("Android: loading ggml backends from: ") + + std::string("loading ggml backends from: ") + variantsDir.string()); ggml_backend_load_all_from_path(variantsDir.string().c_str()); }); } } // namespace -#endif // __ANDROID__ +#endif // __ANDROID__ || linux-arm64 namespace { std::string toLowerCopy(std::string value) { @@ -202,8 +203,8 @@ int adrenoOpenclGpuDeviceIndex() { void WhisperModel::load() { if (!ctx_) { -#if defined(__ANDROID__) - ensureBackendsLoadedAndroid(cfg_.backendsDir); +#if defined(__ANDROID__) || (defined(__linux__) && defined(__aarch64__)) + ensureBackendsLoaded(cfg_.backendsDir); #endif whisper_context_params contextParams = toWhisperContextParams(cfg_); diff --git a/packages/transcription-whispercpp/vcpkg.json b/packages/transcription-whispercpp/vcpkg.json index 9f75f0b586..cfd4951bcd 100644 --- a/packages/transcription-whispercpp/vcpkg.json +++ b/packages/transcription-whispercpp/vcpkg.json @@ -29,7 +29,7 @@ { "name": "whisper-cpp", "version": "1.9.1", - "port-version": 1 + "port-version": 3 } ] } diff --git a/packages/tts-ggml/CHANGELOG.md b/packages/tts-ggml/CHANGELOG.md index 3c18d76168..af8a9abdc1 100644 --- a/packages/tts-ggml/CHANGELOG.md +++ b/packages/tts-ggml/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Changed + +- Desktop linux-arm64 prebuilds now ship per-arch ggml CPU variants (`tts-cpp` >= 2026-07-13#2, pulling `ggml-speech` 2026-07-14): the previous armv8-a-baseline build compiled out the ARM dotprod/fp16 kernels (chatterbox q4 CPU mean RTF 2.70 -> 2.28 on ubuntu-24.04-arm). + ### Added - **Chatterbox S3Gen classifier-free-guidance rate (`cfgRate`) (QVAC-21908).** diff --git a/packages/tts-ggml/vcpkg.json b/packages/tts-ggml/vcpkg.json index 1a8405d678..772a068848 100644 --- a/packages/tts-ggml/vcpkg.json +++ b/packages/tts-ggml/vcpkg.json @@ -10,7 +10,7 @@ }, { "name": "tts-cpp", - "version>=": "2026-07-13#1" + "version>=": "2026-07-13#2" } ], "features": {