Merge pull request #89 from JRufer/claude/windows-gpu-acceleration

Windows GPU acceleration for Moonshine, via WebGPU rather than DirectML

Author: JRufer

.github/workflows/ci.yml

@@ -3,8 +3,15 @@ name: CI
 on:
   push:
     branches: [master]
+  # Every pull request, whatever it targets — deliberately unfiltered.
+  #
+  # This used to be `branches: [master]`, which meant a pull request based on
+  # another branch got no checks at all: not a compile, not a test, nothing. A
+  # stacked change could then be merged down into its base and reach master
+  # having never been built on its own. That is the same shape of hole as
+  # excluding crates from the Windows check — work that looks reviewed because a
+  # PR exists, and was never compiled.
   pull_request:
-    branches: [master]
 
 env:
   CARGO_HTTP_MULTIPLEXING: false
@@ -75,6 +82,19 @@ jobs:
       - name: cargo check (workspace, default features)
         run: cargo check --workspace --all-targets
 
+      # The GPU features are off by default, so the workspace check above never
+      # compiles a line of their execution-provider code — and the first thing
+      # that would is a release build, where a mistake costs a whole run. That
+      # is the same shape of gap that let the Windows backend ship unbuildable,
+      # so each shipped GPU variant gets checked here.
+      #
+      # `moonshine-webgpu` is what the Windows GPU release artifact is built
+      # with; `vulkan` is what the Linux GPU AppImage is built with. Both pull
+      # their own prebuilt ONNX Runtime / compile their own shaders, so this is
+      # not free, but it is far cheaper than finding out at release time.
+      - name: cargo check (moonshine-webgpu)
+        run: cargo check -p voxctrl-app --features moonshine-webgpu
+
   # ── Unit tests ────────────────────────────────────────────────────────────────
   cargo-test:
     name: cargo test

.github/workflows/release.yml

@@ -110,6 +110,22 @@ jobs:
             artifact_label: windows-x86_64
             cuda: false
 
+          # ── Windows GPU ────────────────────────────────────────────────────
+          # Moonshine on the GPU via ONNX Runtime's WebGPU execution provider —
+          # Dawn over D3D12 on Windows, so NVIDIA, AMD and Intel alike, with no
+          # vendor SDK at build time and nothing extra to ship.
+          #
+          # This accelerates Moonshine, not whisper.cpp: ggml's Vulkan backend
+          # is the only GPU path whisper.cpp has here and it does not register
+          # on Windows MSVC static builds (ggml-org/whisper.cpp#3750). Someone
+          # reading "Windows GPU" should know which engine gets the GPU, which
+          # is why the app reports the two separately.
+          - name: windows-webgpu
+            os: windows-2022
+            features: "moonshine-webgpu"
+            artifact_label: windows-x86_64-webgpu
+            cuda: false
+
     steps:
       - uses: actions/checkout@v4
 
@@ -332,9 +348,15 @@ jobs:
       # the MSI is a secondary, unadvertised artifact. Call tauri directly so
       # `--bundles nsis` reaches tauri instead of cargo.
       - name: Build installer (Windows, CPU)
-        if: runner.os == 'Windows'
+        if: runner.os == 'Windows' && matrix.features == ''
         run: npx tauri build --bundles nsis
 
+      # Split from the step above rather than interpolating a possibly-empty
+      # `--features`, which passes tauri a bare flag and fails the build.
+      - name: Build installer (Windows, GPU)
+        if: runner.os == 'Windows' && matrix.features != ''
+        run: npx tauri build --bundles nsis --features ${{ matrix.features }}
+
       # ── Collect and rename Linux artifacts ──────────────────────────────────
       # Search the whole workspace so the path is correct regardless of whether
       # cargo uses a workspace-level or package-level target directory.
@@ -516,16 +538,26 @@ jobs:
             | Linux | `-linux-x86_64.AppImage` | None |
             | Linux (GPU) | `-linux-x86_64-vulkan.AppImage` | Any Vulkan GPU |
             | Windows | `-windows-x86_64.exe` | None |
+            | Windows (GPU) | `-windows-x86_64-webgpu.exe` | Any Direct3D 12 GPU |
 
             **The Vulkan AppImage** is a portable GPU build: it accelerates any
             NVIDIA/AMD/Intel GPU via the host Vulkan driver, and falls back to
             CPU when no GPU is present.
 
             **CPU builds** run on any hardware, including AMD, Intel, and ARM.
 
-            **The Windows build** is an NSIS installer and runs inference on the
-            CPU. It is not code-signed yet, so SmartScreen will warn that the
-            publisher is unknown: choose **More info → Run anyway**.
+            **The Windows builds** are NSIS installers. Neither is code-signed
+            yet, so SmartScreen will warn that the publisher is unknown: choose
+            **More info → Run anyway**.
+
+            **The Windows GPU build** accelerates the **Moonshine** speech
+            engine on any Direct3D 12 GPU — NVIDIA, AMD or Intel — through ONNX
+            Runtime's WebGPU provider. Select Moonshine in **Settings → Engine**;
+            the tab says which backend the build actually has. Whisper.cpp still
+            runs on the CPU on Windows either way, because its only GPU path
+            here is ggml's Vulkan backend, which does not register on Windows
+            MSVC static builds. If your GPU is not usable the app says so in the
+            log and falls back to the CPU rather than pretending.
 
             ### Windows — first run
 

Cargo.lock

@@ -50,14 +50,35 @@ moonshine = ["dep:ort", "dep:tokenizers"]
 # runtime at build *and* run time, which is exactly what the Vulkan AppImage
 # exists to avoid shipping.
 #
-# There is no `moonshine-vulkan` because ONNX Runtime has no Vulkan execution
-# provider. So the standard Vulkan build — where whisper.cpp *does* get the GPU
-# — runs Moonshine on the CPU, and holds its weights in RAM as fp32 rather than
-# in VRAM. `moonshine_gpu_backend()` reports which of these is true so the UI
-# can say so instead of leaving the user to measure it.
+# There is still no `moonshine-vulkan`: ONNX Runtime has no Vulkan execution
+# provider, so a `vulkan` build accelerates whisper.cpp and nothing else. What
+# closes that gap is `moonshine-webgpu` below, which reaches the same
+# vendor-neutral GPUs by a different road. `moonshine_gpu_backend()` reports
+# which of these a build actually has, so the UI can say so instead of leaving
+# the user to measure it.
 moonshine-cuda = ["moonshine", "ort/cuda"]
 moonshine-coreml = ["moonshine", "ort/coreml"]
 
+# Cross-vendor GPU offload for Moonshine, via ONNX Runtime's WebGPU execution
+# provider. This is the Windows counterpart to what `vulkan` does for
+# whisper.cpp: any modern GPU, no vendor SDK at build time, no runtime to ship.
+# On Windows ORT's WebGPU EP runs on Dawn over D3D12, so it covers NVIDIA, AMD
+# and Intel alike.
+#
+# Chosen over DirectML, which is the obvious-looking answer and does not work
+# here. `ort` exposes an `ort::ep::DirectML` and a `directml` feature, but
+# `ort-sys`'s prebuilt table (build/download/dist.txt) ships no DirectML binary
+# for any target, and its `resolve_dist()` never consults the `directml` feature
+# when choosing which prebuilt to fetch. Enabling it therefore downloads the
+# plain CPU runtime, compiles fine, and then fails to register the provider at
+# run time — forever. Reaching real DirectML means switching to `load-dynamic`
+# and shipping Microsoft's own onnxruntime.dll and DirectML.dll, which gives up
+# the self-contained binary the rest of this file is arranged around.
+#
+# `wgpu` is a tag `resolve_dist()` does understand, and
+# `x86_64-pc-windows-msvc+wgpu` is a prebuilt that exists.
+moonshine-webgpu = ["moonshine", "ort/webgpu"]
+
 [dependencies.ort]
 version = "2.0.0-rc.12"
 optional = true

crates/voxctrl-inference/Cargo.toml

@@ -39,15 +39,97 @@ pub fn whisper_gpu_backend() -> Option<&'static str> {
 /// [`whisper_gpu_backend`] rather than inferred from it: they are genuinely
 /// different answers in the build most people run.
 pub fn moonshine_gpu_backend() -> Option<&'static str> {
+    // Same order as `MoonshineBackend::with_gpu` registers them in. The two
+    // being one edit apart is the whole risk here, so a test holds them
+    // together rather than a comment alone.
     if cfg!(feature = "moonshine-cuda") {
         Some("cuda")
     } else if cfg!(feature = "moonshine-coreml") {
         Some("coreml")
+    } else if cfg!(feature = "moonshine-webgpu") {
+        Some("webgpu")
     } else {
         None
     }
 }
 
+/// The same provider, spelled the way ONNX Runtime spells it, for registration
+/// and logging.
+///
+/// A pure mapping over [`moonshine_gpu_backend`] rather than a second `cfg`
+/// cascade: the Engine tab and the session builder then cannot name different
+/// providers, which is the failure mode worth designing out. `with_gpu` still
+/// selects the provider *type* by `cfg`, but on exactly the conditions above.
+// Its only non-test caller is the branch of `with_gpu` that registers a
+// provider, which is compiled out when no GPU feature is on — the `moonshine`
+// feature alone is not enough to reach it, which is what the first version of
+// this attribute got wrong. The tests below still exercise it either way.
+#[cfg_attr(
+    not(any(
+        feature = "moonshine-cuda",
+        feature = "moonshine-coreml",
+        feature = "moonshine-webgpu"
+    )),
+    allow(dead_code)
+)]
+pub(crate) fn moonshine_gpu_provider() -> Option<&'static str> {
+    match moonshine_gpu_backend() {
+        Some("cuda") => Some("CUDA"),
+        Some("coreml") => Some("CoreML"),
+        Some("webgpu") => Some("WebGPU"),
+        _ => None,
+    }
+}
+
+#[cfg(test)]
+mod gpu_backend_tests {
+    use super::*;
+
+    #[test]
+    fn every_reported_backend_has_a_provider_spelling() {
+        // A backend the mapping does not know would make `with_gpu` log a bare
+        // "GPU" while registering something specific — the Engine tab and the
+        // log disagreeing about what this build does. Runs in every feature
+        // configuration CI builds, including the default CPU one.
+        match moonshine_gpu_backend() {
+            Some(backend) => assert!(
+                moonshine_gpu_provider().is_some(),
+                "{backend} is reported to the UI but has no ONNX Runtime spelling"
+            ),
+            None => assert_eq!(
+                moonshine_gpu_provider(),
+                None,
+                "a CPU-only build named a GPU provider"
+            ),
+        }
+    }
+
+    #[test]
+    fn a_build_with_no_gpu_feature_reports_none() {
+        if cfg!(not(any(
+            feature = "moonshine-cuda",
+            feature = "moonshine-coreml",
+            feature = "moonshine-webgpu"
+        ))) {
+            assert_eq!(moonshine_gpu_backend(), None);
+        }
+    }
+
+    #[test]
+    fn the_webgpu_build_reports_webgpu() {
+        // Guards the release artifact: a `moonshine-webgpu` build that reported
+        // None would show "CPU only" in the Engine tab on the very build that
+        // exists to use the GPU.
+        if cfg!(all(
+            feature = "moonshine-webgpu",
+            not(any(feature = "moonshine-cuda", feature = "moonshine-coreml"))
+        )) {
+            assert_eq!(moonshine_gpu_backend(), Some("webgpu"));
+            assert_eq!(moonshine_gpu_provider(), Some("WebGPU"));
+        }
+    }
+}
+
 use std::sync::Arc;
 
 use anyhow::Result;

crates/voxctrl-inference/src/lib.rs

@@ -241,12 +241,34 @@ impl MoonshineBackend {
     /// point. On failure the builder is recovered from the error and returned
     /// unchanged, so a machine with no usable CUDA still loads the model.
     fn with_gpu(builder: SessionBuilder) -> SessionBuilder {
-        #[cfg(any(feature = "moonshine-cuda", feature = "moonshine-coreml"))]
+        #[cfg(any(
+            feature = "moonshine-cuda",
+            feature = "moonshine-coreml",
+            feature = "moonshine-webgpu"
+        ))]
         {
+            // One provider, picked in the order they are worth having: CUDA
+            // beats a generic device where it exists, CoreML beats WebGPU on
+            // Apple hardware, and WebGPU covers everything else. This order is
+            // mirrored by `moonshine_gpu_backend()`, which is where the name
+            // below comes from.
             #[cfg(feature = "moonshine-cuda")]
-            let (name, ep) = ("CUDA", ort::ep::CUDA::default().build().error_on_failure());
+            let ep = ort::ep::CUDA::default().build().error_on_failure();
             #[cfg(all(feature = "moonshine-coreml", not(feature = "moonshine-cuda")))]
-            let (name, ep) = ("CoreML", ort::ep::CoreML::default().build().error_on_failure());
+            let ep = ort::ep::CoreML::default().build().error_on_failure();
+            #[cfg(all(
+                feature = "moonshine-webgpu",
+                not(any(feature = "moonshine-cuda", feature = "moonshine-coreml"))
+            ))]
+            // Defaults deliberately: on Windows ORT's WebGPU EP runs on Dawn,
+            // which selects D3D12 there on its own. Naming a backend type would
+            // only be a way to get it wrong on someone else's machine.
+            let ep = ort::ep::WebGPU::default().build().error_on_failure();
+
+            // Not a second cascade: the name is derived from the same function
+            // that tells the UI which backend this build has, so the log and
+            // the Engine tab cannot name different providers.
+            let name = crate::moonshine_gpu_provider().unwrap_or("GPU");
 
             match builder.with_execution_providers([ep]) {
                 Ok(with_ep) => {
@@ -266,7 +288,11 @@ impl MoonshineBackend {
             }
         }
 
-        #[cfg(not(any(feature = "moonshine-cuda", feature = "moonshine-coreml")))]
+        #[cfg(not(any(
+            feature = "moonshine-cuda",
+            feature = "moonshine-coreml",
+            feature = "moonshine-webgpu"
+        )))]
         builder
     }
 }
@@ -596,6 +622,8 @@ fn threads() -> usize {
 
 #[cfg(test)]
 mod tests {
+
+
     use super::*;
 
     #[test]