diff --git a/.github/scripts/verify-cpu-levels.py b/.github/scripts/verify-cpu-levels.py new file mode 100644 index 00000000000..f4c945ac7af --- /dev/null +++ b/.github/scripts/verify-cpu-levels.py @@ -0,0 +1,442 @@ +#!/usr/bin/env python3 +"""Check whisper-server's builds for processors without AVX2 (OpenWhispr/openwhispr#2356). + +Each build runs under an emulator that implements only a chosen processor: Intel +SDE on Windows, QEMU user mode on Linux. The levels are upstream ggml's CPU +variants: + + ivybridge SSE4.2 + AVX + F16C (Intel Ivy Bridge, AMD Piledriver, Jaguar) + sandybridge SSE4.2 + AVX (Intel Sandy Bridge, AMD Bulldozer) + +1. Each level build starts the way OpenWhispr starts it (a real model, Silero + VAD on), transcribes samples/jfk.wav on its own level, and its system_info + reports exactly its level's instruction sets, so a build compiled above its + level fails here even if the emulator never ran the offending code. +2. The primary (AVX2) build is stopped at startup by the emulator itself on an + Ivy Bridge (SDE's chip check, or SIGILL under QEMU) and, under QEMU, on a + Piledriver-class AMD FX (AVX, F16C and FMA, no AVX2 or BMI2). That proves the + emulator enforces the level, and that the primary dies before serving, which + OpenWhispr's fallback relies on. It rests on ggml's startup code happening to + run a BMI2 instruction, so a ggml update that moves it fails this check. + Caveat: QEMU gates SHLX/SHRX on BMI1, not BMI2, and its Piledriver model + (Opteron_G5) also lacks the BMI1 a real FX has. It matches real hardware only + while the primary's startup code runs no BMI1 instruction (true today: GCC + builds it with -mbmi2 but not -mbmi). +3. Each level build is stopped at startup by its level check + (OPENWHISPR_CPU_LEVEL_CHECK, examples/server/cpu-level-check.cpp) on the + processor below it: ivybridge on a Sandy Bridge, which lacks F16C, and + sandybridge on a Westmere, which lacks AVX. ggml alone would start there and + only fault at the first transcription. + +--with-llama-libraries also starts each level build on its own level with +llama.cpp's libraries beside it, as in OpenWhispr's resources/bin: whisper-server +loads every ggml library in its folder at startup (ggml_backend_load_all). + +Each result prints as soon as its check finishes; an error inside a check is a +FAIL line, not a traceback. --emulator none runs everything natively. It exists +to smoke-test this script locally and can never pass: nothing stops the +rejection checks. +""" + +import argparse +import hashlib +import http.client +import json +import os +import re +import shutil +import signal +import socket +import subprocess +import sys +import tarfile +import time +import urllib.error +import urllib.request +import uuid +import zipfile +from pathlib import Path + +REPO = Path(__file__).resolve().parents[2] +AUDIO = REPO / "samples" / "jfk.wav" +EXPECTED_WORDS = "ask not what your country can do for you" + +DOWNLOADS = { + # models/README.md lists sha1 bd577a113a864445d4c299885e0cb97d4ba92b5f for this file + "model": ( + "https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-tiny.bin", + "be07e048e1e599ad46341c8d2a135645097a538221678b7acdd1b1919c6e1b21", + ), + # The Silero VAD model OpenWhispr ships for its whisper-server + "vad": ( + "https://huggingface.co/ggml-org/whisper-vad/resolve/main/ggml-silero-v5.1.2.bin", + "29940d98d42b91fbd05ce489f3ecf7c72f0a42f027e4875919a28fb4c04ea2cf", + ), + # Intel SDE 9.58. Intel's download mirror refuses non-browser clients; this + # mirror serves the identical file (oven-sh/bun pins the same sha256). + "sde": ( + "https://github.com/petarpetrovt/setup-sde/releases/download/binaries/" + "sde-external-9.58.0-2025-06-16-win.tar.xz", + "ebb8b3b63fcb0b6c1f9721118ba4883703d2aed9e0db2defed4e44fba78d9ca9", + ), + # The llama.cpp CPU builds OpenWhispr bundles (LLAMA_CPP_TAG in its + # scripts/download-llama-server.js); used by --with-llama-libraries + "llama-win32": ( + "https://github.com/ggml-org/llama.cpp/releases/download/b9763/" + "llama-b9763-bin-win-cpu-x64.zip", + "05144ee4d885a778ebaef619f79ca0b8a4edb7f017eaf70086a8781ff003815f", + ), + "llama-linux": ( + "https://github.com/ggml-org/llama.cpp/releases/download/b9763/" + "llama-b9763-bin-ubuntu-x64.tar.gz", + "4bd11fe0cea35223b240496062900ed9493b46f20a08747d431bfdc2252af2d8", + ), +} +DOWNLOAD_ATTEMPTS = 3 + +# The emulators' names for each processor. SDE emulates Intel chips only. +EMULATED_CPUS = { + "ivybridge": {"sde": "-ivb", "qemu": "IvyBridge"}, + "sandybridge": {"sde": "-snb", "qemu": "SandyBridge"}, + "piledriver": {"qemu": "Opteron_G5"}, + "westmere": {"sde": "-wsm", "qemu": "Westmere"}, +} +# The instruction sets each level build's system_info must report +EXPECTED_FEATURES = {"ivybridge": {"AVX", "F16C"}, "sandybridge": {"AVX"}} +# system_info names that are x86 instruction set extensions above SSE4.2 +ISA_FEATURE = re.compile(r"^(AVX\w*|AMX\w*|F16C|FMA|BMI2)$") +# The libraries OpenWhispr copies out of the llama.cpp archive (copyLibraries there) +LIBRARY = re.compile(r"\.(dll|so(\.\d+)*)$") + +SDE_VIOLATION = re.compile(r"SDE-ERROR:.*not valid for specified chip.*", re.IGNORECASE) +# What cpu-level-check.cpp prints before it raises the fault +LEVEL_CHECK = re.compile(r"whisper-server: built for (\w+), which this processor does not support") +STATUS_ILLEGAL_INSTRUCTION = 0xC000001D +# Observed on GitHub's runners: SDE runs every check in about a minute; under +# QEMU a start takes well under a minute and the slowest transcription ~7 min. +READY_TIMEOUT_S = 300 +INFERENCE_TIMEOUT_S = 1500 + +# Talk to the local server directly, whatever proxy the environment configures +LOCAL = urllib.request.build_opener(urllib.request.ProxyHandler({})) + + +def sha256_of(path): + digest = hashlib.sha256() + with open(path, "rb") as f: + for chunk in iter(lambda: f.read(1 << 20), b""): + digest.update(chunk) + return digest.hexdigest() + + +def fetch(name, work): + """Download a pinned file, retrying a failed or corrupt transfer.""" + url, expected = DOWNLOADS[name] + dest = work / url.rsplit("/", 1)[-1] + if dest.exists() and sha256_of(dest) == expected: + return dest + part = dest.with_name(dest.name + ".part") + for attempt in range(1, DOWNLOAD_ATTEMPTS + 1): + print(f"Downloading {url} (attempt {attempt})", flush=True) + try: + with urllib.request.urlopen(url, timeout=120) as response, open(part, "wb") as out: + shutil.copyfileobj(response, out) + except (OSError, http.client.HTTPException) as error: + print(f" failed: {error}", flush=True) + else: + actual = sha256_of(part) + if actual == expected: + part.replace(dest) + return dest + print(f" sha256 {actual}, expected {expected}", flush=True) + if attempt < DOWNLOAD_ATTEMPTS: + time.sleep(10 * attempt) + raise RuntimeError(f"{dest.name}: no download matched sha256 {expected}") + + +def extract(archive, dest): + shutil.rmtree(dest, ignore_errors=True) + if archive.name.endswith(".zip"): + with zipfile.ZipFile(archive) as z: + z.extractall(dest) + return + with tarfile.open(archive) as t: + if hasattr(tarfile, "data_filter"): + t.extractall(dest, filter="data") + else: + t.extractall(dest) + + +def unpack_server(zip_path, dest): + """Extract a release zip and return the one whisper-server executable in it.""" + extract(zip_path, dest) + servers = [p for p in dest.iterdir() if p.name.startswith("whisper-server")] + if len(servers) != 1: + raise SystemExit(f"{zip_path.name}: expected one whisper-server, found {servers}") + servers[0].chmod(0o755) # zipfile drops the executable bit + return servers[0] + + +def with_llama_libraries(server, level, work): + """Copy a build's folder and llama.cpp's libraries into one folder, flat, the + way OpenWhispr lays out resources/bin: whisper.cpp first, then llama.cpp, + whose files win a name clash. Returns the copied server.""" + llama = work / "llama" + if not llama.exists(): + extract(fetch("llama-win32" if os.name == "nt" else "llama-linux", work), llama) + dest = work / f"with-llama-{level}" + shutil.rmtree(dest, ignore_errors=True) + dest.mkdir() + for path in server.parent.iterdir(): + shutil.copy2(path, dest / path.name) + for path in llama.rglob("*"): + if path.is_file() and LIBRARY.search(path.name): + shutil.copy2(path, dest / path.name) + return dest / server.name + + +def emulator(kind, cpu, work): + """Return (command prefix, working directory) that runs a program on `cpu`.""" + if kind == "sde": + root = work / "sde" + if not root.exists(): + extract(fetch("sde", work), root) + sde = next(root.glob("sde-external-*/sde.exe")) + # SDE must run from its own directory to find Pin's DLLs + return [str(sde), EMULATED_CPUS[cpu]["sde"], "--"], sde.parent + if kind == "qemu": + return ["qemu-x86_64", "-cpu", EMULATED_CPUS[cpu]["qemu"]], None + return [], None + + +def free_port(): + with socket.socket() as s: + s.bind(("127.0.0.1", 0)) + return s.getsockname()[1] + + +def start_server(prefix, cwd, server, model, vad_model, log_path): + """Start whisper-server with the arguments OpenWhispr passes (buildWhisperServerArgs).""" + port = free_port() + args = [str(server), "--model", str(model), "--host", "127.0.0.1", "--port", str(port)] + args += ["--language", "auto", "--max-len", "4096"] + if vad_model: + args += ["--vad", "--vad-model", str(vad_model)] + log = open(log_path, "wb") + proc = subprocess.Popen(prefix + args, cwd=cwd, stdout=log, stderr=subprocess.STDOUT) + return proc, log, port + + +def wait_ready(proc, port): + """Wait until the server answers HTTP. Any answer counts, as OpenWhispr's + health check treats it, but only while the process it started is alive.""" + deadline = time.monotonic() + READY_TIMEOUT_S + while time.monotonic() < deadline: + if proc.poll() is not None: + return False + try: + with LOCAL.open(f"http://127.0.0.1:{port}/", timeout=5): + pass + except urllib.error.HTTPError: + pass + except (OSError, http.client.HTTPException): + time.sleep(1) + continue + return proc.poll() is None + return False + + +def transcribe(port): + """POST samples/jfk.wav to /inference with the fields OpenWhispr sends.""" + boundary = uuid.uuid4().hex + fields = { + "language": "auto", + "entropy_thold": "2.8", + "logprob_thold": "-1.25", + "response_format": "json", + } + parts = [ + f'--{boundary}\r\nContent-Disposition: form-data; name="{k}"\r\n\r\n{v}\r\n'.encode() + for k, v in fields.items() + ] + parts.append( + f"--{boundary}\r\nContent-Disposition: form-data; name=\"file\"; filename=\"jfk.wav\"\r\n" + "Content-Type: audio/wav\r\n\r\n".encode() + + AUDIO.read_bytes() + + f"\r\n--{boundary}--\r\n".encode() + ) + request = urllib.request.Request( + f"http://127.0.0.1:{port}/inference", + data=b"".join(parts), + headers={"Content-Type": f"multipart/form-data; boundary={boundary}"}, + ) + with LOCAL.open(request, timeout=INFERENCE_TIMEOUT_S) as response: + return json.loads(response.read())["text"] + + +def stop(proc): + if proc.poll() is None: + if os.name == "nt": + # SDE runs the server as a child process; /T ends the whole tree + subprocess.run(["taskkill", "/PID", str(proc.pid), "/T", "/F"], capture_output=True) + else: + proc.terminate() + try: + proc.wait(timeout=60) + except subprocess.TimeoutExpired: + proc.kill() + proc.wait() + + +def violation(kind, returncode, output): + """How the program was stopped for an instruction the emulated CPU lacks: SDE's + chip check, or an illegal-instruction death (a real fault, or the one a level + check raises after naming what is missing), or None.""" + if kind == "sde": + match = SDE_VIOLATION.search(output) + if match: + return match.group(0).strip() + illegal = (kind == "sde" and returncode == STATUS_ILLEGAL_INSTRUCTION) or ( + kind == "qemu" and returncode == -signal.SIGILL + ) + if not illegal: + return None + refusals = [m.group(0) for m in LEVEL_CHECK.finditer(output)] + return "; ".join(refusals) if refusals else "killed by an illegal instruction" + + +def reported_features(output): + """Instruction sets above SSE4.2 the build says it was compiled for (system_info).""" + for line in output.splitlines(): + if line.startswith("system_info:") and "CPU :" in line: + enabled = re.findall(r"(\w+) = 1", line.split("CPU :", 1)[1]) + return {name for name in enabled if ISA_FEATURE.match(name)} + return None + + +def words(text): + return " ".join(re.sub(r"[^a-z]+", " ", text.lower()).split()) + + +def run(kind, cpu, server, model, vad_model, work, label, transcribe_audio): + """Start `server` on an emulated `cpu`, optionally transcribe, and stop it. + Returns (ready, text, request_error, output, returncode).""" + prefix, cwd = emulator(kind, cpu, work) + log_path = work / f"{label.replace(' ', '-')}.log" + proc, log, port = start_server(prefix, cwd, server, model, vad_model, log_path) + ready, text, request_error = False, None, None + try: + ready = wait_ready(proc, port) + if ready and transcribe_audio: + try: + text = transcribe(port) + except (OSError, http.client.HTTPException, ValueError, KeyError) as error: + # e.g. the server dying mid-request: its exit code tells why + request_error = f"{type(error).__name__}: {error}" + finally: + stop(proc) + log.close() + return ready, text, request_error, log_path.read_text(errors="replace"), proc.returncode + + +def check_transcribes(kind, level, server, model, vad_model, work): + label = f"{server.name} on {level}" + ready, text, request_error, output, returncode = run( + kind, level, server, model, vad_model, work, label, True + ) + hit = violation(kind, returncode, output) + if hit: + return False, f"{label} ran an instruction {level} lacks: {hit}" + if not ready: + return False, f"{label} did not start (exit {returncode}):\n{output[-2000:]}" + if text is None: + return False, f"{label} failed to transcribe ({request_error}, exit {returncode}):\n{output[-2000:]}" + if EXPECTED_WORDS not in words(text): + return False, f"{label} transcribed jfk.wav as {text!r}" + features = reported_features(output) + if features != EXPECTED_FEATURES[level]: + expected = sorted(EXPECTED_FEATURES[level]) + return False, f"{label} reports {sorted(features or [])}, expected {expected}" + return True, f"{label}: transcribed {text.strip()!r}, reports {sorted(features)}" + + +def check_rejected(kind, cpu, server, model, work, refused_feature=None): + """`server` must die at startup on `cpu`: by its level check refusing exactly + `refused_feature`, or, when that is None, by the emulator itself.""" + label = f"{server.name} on {cpu}" + ready, _, _, output, returncode = run(kind, cpu, server, model, None, work, label, False) + if ready: + return False, ( + f"{label} started serving. Either the emulator does not enforce {cpu} (then the " + "transcription checks prove nothing) or the build no longer dies at startup on a " + "processor it does not support (then OpenWhispr's fallback cannot see the crash)" + ) + hit = violation(kind, returncode, output) + if not hit: + return False, f"{label} exited ({returncode}) without an instruction violation" + refused = {m.group(1) for m in LEVEL_CHECK.finditer(output)} + if refused_feature is not None and refused != {refused_feature}: + return False, f"{label} was stopped ({hit}), expected its level check to refuse {refused_feature}" + if refused_feature is None and (refused or (kind == "sde" and not SDE_VIOLATION.search(output))): + return False, f"{label} was stopped ({hit}), expected the emulator to stop it" + return True, f"{label}: stopped at startup as expected ({hit})" + + +def check_starts_with_llama(kind, level, server, model, work): + server = with_llama_libraries(server, level, work) + label = f"{server.name} with llama.cpp libraries on {level}" + ready, _, _, output, returncode = run(kind, level, server, model, None, work, label, False) + hit = violation(kind, returncode, output) + if hit: + return False, f"{label} ran an instruction {level} lacks: {hit}" + if not ready: + return False, f"{label} did not start (exit {returncode}):\n{output[-2000:]}" + return True, f"{label}: started" + + +def main(): + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--emulator", choices=["sde", "qemu", "none"], required=True) + parser.add_argument("--primary-zip", type=Path, required=True) + parser.add_argument("--ivybridge-zip", type=Path, required=True) + parser.add_argument("--sandybridge-zip", type=Path, required=True) + parser.add_argument("--with-llama-libraries", action="store_true") + parser.add_argument("--work", type=Path, required=True) + args = parser.parse_args() + + work = args.work.resolve() + work.mkdir(parents=True, exist_ok=True) + model, vad_model = fetch("model", work), fetch("vad", work) + primary = unpack_server(args.primary_zip.resolve(), work / "primary") + ivybridge = unpack_server(args.ivybridge_zip.resolve(), work / "ivybridge") + sandybridge = unpack_server(args.sandybridge_zip.resolve(), work / "sandybridge") + + kind = args.emulator + checks = [ + ("ivybridge transcribes", lambda: check_transcribes(kind, "ivybridge", ivybridge, model, vad_model, work)), + ("sandybridge transcribes", lambda: check_transcribes(kind, "sandybridge", sandybridge, model, vad_model, work)), + ("primary stopped on ivybridge", lambda: check_rejected(kind, "ivybridge", primary, model, work)), + ("ivybridge refused on sandybridge", lambda: check_rejected(kind, "sandybridge", ivybridge, model, work, "F16C")), + ("sandybridge refused on westmere", lambda: check_rejected(kind, "westmere", sandybridge, model, work, "AVX")), + ] + if kind != "sde": + checks.append(("primary stopped on piledriver", lambda: check_rejected(kind, "piledriver", primary, model, work))) + if args.with_llama_libraries: + checks += [ + ("ivybridge starts beside llama.cpp", lambda: check_starts_with_llama(kind, "ivybridge", ivybridge, model, work)), + ("sandybridge starts beside llama.cpp", lambda: check_starts_with_llama(kind, "sandybridge", sandybridge, model, work)), + ] + + passed = True + for name, check in checks: + try: + ok, message = check() + except Exception as error: # a broken check is a failure, not the end of the run + ok, message = False, f"{name}: {type(error).__name__}: {error}" + passed = passed and ok + print(f"{'PASS' if ok else 'FAIL'}: {message}", flush=True) + sys.exit(0 if passed else 1) + + +if __name__ == "__main__": + main() diff --git a/.github/workflows/build-binaries.yml b/.github/workflows/build-binaries.yml index 5166516e900..14f70276566 100644 --- a/.github/workflows/build-binaries.yml +++ b/.github/workflows/build-binaries.yml @@ -10,9 +10,21 @@ on: push: tags: - 'openwhispr-*' - + # Every PR proves the CPU binaries before a release: it builds them and runs + # verify-cpu-levels. The GPU and macOS builds and the release skip + # pull_request runs (see their `if:`). + pull_request: + branches: [master] + +# A newer push to a PR supersedes its running check; tag and dispatch runs never +# share a group. +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +# Read-only by default. Only create-release writes (it publishes the release). permissions: - contents: write + contents: read env: # GTX 10 series through RTX 50 (Pascal through Blackwell). sm_120 requires @@ -23,6 +35,7 @@ env: jobs: build-macos-arm64: + if: github.event_name != 'pull_request' runs-on: macos-14 # M1 runner steps: - name: Checkout @@ -81,6 +94,7 @@ jobs: path: dist/whisper-server-darwin-arm64.zip build-macos-x64: + if: github.event_name != 'pull_request' runs-on: macos-14 # Cross-compile for x64 on arm64 runner steps: - name: Checkout @@ -178,12 +192,95 @@ jobs: -DCMAKE_MSVC_RUNTIME_LIBRARY=MultiThreaded cmake --build build --config Release -j %NUMBER_OF_PROCESSORS% + # The build above needs AVX2, FMA, F16C and BMI2 (GGML_NATIVE=OFF turns them + # all on) and dies at startup with STATUS_ILLEGAL_INSTRUCTION on Intel before + # Haswell and on AMD FX (OpenWhispr/openwhispr#2356). These two whisper-server + # builds use upstream ggml's ivybridge (SSE4.2 + AVX + F16C) and sandybridge + # (SSE4.2 + AVX) CPU levels; OpenWhispr falls back through them in that order. + # OPENWHISPR_CPU_LEVEL_CHECK makes each stop at startup on a processor below its + # level (examples/server/cpu-level-check.cpp): ggml alone would start and only + # fault at the first transcription, which OpenWhispr's fallback cannot see. + - name: Build whisper-server for Ivy Bridge-level processors + shell: cmd + env: + # MSVC has no switch for F16C without AVX2, which is why upstream ggml has + # no ivybridge variant on MSVC. Defining the macro turns on ggml's F16C code, + # which MSVC compiles from intrinsics under /arch:AVX. CMake reads CFLAGS and + # CXXFLAGS only when it first configures a build tree. + CFLAGS: /D__F16C__ + CXXFLAGS: /D__F16C__ + run: | + cmake -S . -B build-ivybridge -G "Ninja Multi-Config" ^ + -DCMAKE_BUILD_TYPE=Release ^ + -DCMAKE_C_COMPILER_LAUNCHER=sccache ^ + -DCMAKE_CXX_COMPILER_LAUNCHER=sccache ^ + -DBUILD_SHARED_LIBS=OFF ^ + -DGGML_NATIVE=OFF ^ + -DGGML_CUDA=OFF ^ + -DGGML_AVX=ON ^ + -DGGML_AVX2=OFF ^ + -DGGML_BMI2=OFF ^ + -DGGML_AVX512=OFF ^ + -DGGML_AVX_VNNI=OFF ^ + -DOPENWHISPR_CPU_LEVEL_CHECK=ON ^ + -DCMAKE_MSVC_RUNTIME_LIBRARY=MultiThreaded || exit /b 1 + cmake --build build-ivybridge --config Release --target whisper-server -j %NUMBER_OF_PROCESSORS% + + - name: Build whisper-server for Sandy Bridge-level processors + shell: cmd + run: | + cmake -S . -B build-sandybridge -G "Ninja Multi-Config" ^ + -DCMAKE_BUILD_TYPE=Release ^ + -DCMAKE_C_COMPILER_LAUNCHER=sccache ^ + -DCMAKE_CXX_COMPILER_LAUNCHER=sccache ^ + -DBUILD_SHARED_LIBS=OFF ^ + -DGGML_NATIVE=OFF ^ + -DGGML_CUDA=OFF ^ + -DGGML_AVX=ON ^ + -DGGML_AVX2=OFF ^ + -DGGML_BMI2=OFF ^ + -DGGML_AVX512=OFF ^ + -DGGML_AVX_VNNI=OFF ^ + -DOPENWHISPR_CPU_LEVEL_CHECK=ON ^ + -DCMAKE_MSVC_RUNTIME_LIBRARY=MultiThreaded || exit /b 1 + cmake --build build-sandybridge --config Release --target whisper-server -j %NUMBER_OF_PROCESSORS% + + - name: Check each build's CPU level + # Whole lines: "/arch:AVX GGML_AVX" is also the start of what a missing + # GGML_BMI2=OFF produces ("/arch:AVX GGML_AVX;__BMI2__;GGML_BMI2"). Configuring + # a configured tree again prints the line without rebuilding. MSVC prints the + # same line for both levels, so F16C is checked in the cache here (ggml's F16C + # code is in both C and C++ files) and in the binary's own report by + # .github/scripts/verify-cpu-levels.py. + shell: pwsh + run: | + function Assert-CpuVariant([string]$BuildDir, [string]$Expected) { + $found = @(cmake -S . -B $BuildDir | Where-Object { $_ -like '-- Adding CPU backend variant *' }) + if ($LASTEXITCODE -ne 0) { throw "configuring $BuildDir failed" } + Write-Host "${BuildDir}: $($found -join ' | ')" + if ($found.Count -ne 1 -or $found[0] -cne $Expected) { throw "${BuildDir}: expected '$Expected'" } + } + function Assert-F16cDefine([string]$BuildDir, [bool]$Expected) { + foreach ($flags in @("CMAKE_C_FLAGS", "CMAKE_CXX_FLAGS")) { + $found = [bool](Select-String -Path "$BuildDir\CMakeCache.txt" -Pattern "^${flags}:STRING=.*/D__F16C__" -Quiet) + if ($found -ne $Expected) { throw "${BuildDir}: /D__F16C__ in $flags is $found, expected $Expected" } + } + } + Assert-CpuVariant build '-- Adding CPU backend variant ggml-cpu: /arch:AVX2 GGML_AVX2;GGML_FMA;GGML_F16C;__BMI2__;GGML_BMI2' + Assert-CpuVariant build-ivybridge '-- Adding CPU backend variant ggml-cpu: /arch:AVX GGML_AVX' + Assert-CpuVariant build-sandybridge '-- Adding CPU backend variant ggml-cpu: /arch:AVX GGML_AVX' + Assert-F16cDefine build-ivybridge $true + Assert-F16cDefine build-sandybridge $false + - name: Package binaries shell: pwsh run: | mkdir dist Copy-Item build\bin\Release\whisper-cli.exe dist\whisper-cpp-win32-x64-cpu.exe Copy-Item build\bin\Release\whisper-server.exe dist\whisper-server-win32-x64-cpu.exe + foreach ($level in @("ivybridge", "sandybridge")) { + Copy-Item "build-$level\bin\Release\whisper-server.exe" "dist\whisper-server-win32-x64-cpu-$level.exe" + } # Both exes link the dynamic MSVC runtime: CMAKE_MSVC_RUNTIME_LIBRARY=MultiThreaded # is a silent no-op while CMake policy CMP0091 is unset, and MSVC OpenMP links @@ -201,6 +298,9 @@ jobs: Set-Location dist Compress-Archive -Path (@("whisper-cpp-win32-x64-cpu.exe") + $runtimeDlls) -DestinationPath whisper-cpp-win32-x64-cpu.zip Compress-Archive -Path (@("whisper-server-win32-x64-cpu.exe") + $runtimeDlls) -DestinationPath whisper-server-win32-x64-cpu.zip + foreach ($level in @("ivybridge", "sandybridge")) { + Compress-Archive -Path (@("whisper-server-win32-x64-cpu-$level.exe") + $runtimeDlls) -DestinationPath "whisper-server-win32-x64-cpu-$level.zip" + } - name: Verify zips contain the MSVC runtime DLLs shell: pwsh @@ -228,7 +328,20 @@ jobs: name: whisper-server-win32-x64-cpu path: dist/whisper-server-win32-x64-cpu.zip + - name: Upload whisper-server Ivy Bridge-level artifact + uses: actions/upload-artifact@v4 + with: + name: whisper-server-win32-x64-cpu-ivybridge + path: dist/whisper-server-win32-x64-cpu-ivybridge.zip + + - name: Upload whisper-server Sandy Bridge-level artifact + uses: actions/upload-artifact@v4 + with: + name: whisper-server-win32-x64-cpu-sandybridge + path: dist/whisper-server-win32-x64-cpu-sandybridge.zip + build-windows-x64-cuda: + if: github.event_name != 'pull_request' runs-on: windows-2022 steps: - name: Free up disk space @@ -437,6 +550,57 @@ jobs: -DGGML_CUDA=OFF cmake --build build --config Release -j $(nproc) + # The ivybridge and sandybridge whisper-server builds for processors without + # AVX2; see the Windows CPU job. GCC switches F16C and FMA separately, so both + # are set explicitly. + - name: Build whisper-server for Ivy Bridge-level processors + run: | + cmake -B build-ivybridge \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_C_COMPILER_LAUNCHER=ccache \ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ + -DBUILD_SHARED_LIBS=OFF \ + -DGGML_NATIVE=OFF \ + -DGGML_CUDA=OFF \ + -DGGML_SSE42=ON \ + -DGGML_AVX=ON \ + -DGGML_F16C=ON \ + -DGGML_FMA=OFF \ + -DGGML_BMI2=OFF \ + -DGGML_AVX2=OFF \ + -DOPENWHISPR_CPU_LEVEL_CHECK=ON + cmake --build build-ivybridge --config Release --target whisper-server -j $(nproc) + + - name: Build whisper-server for Sandy Bridge-level processors + run: | + cmake -B build-sandybridge \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_C_COMPILER_LAUNCHER=ccache \ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ + -DBUILD_SHARED_LIBS=OFF \ + -DGGML_NATIVE=OFF \ + -DGGML_CUDA=OFF \ + -DGGML_SSE42=ON \ + -DGGML_AVX=ON \ + -DGGML_F16C=OFF \ + -DGGML_FMA=OFF \ + -DGGML_BMI2=OFF \ + -DGGML_AVX2=OFF \ + -DOPENWHISPR_CPU_LEVEL_CHECK=ON + cmake --build build-sandybridge --config Release --target whisper-server -j $(nproc) + + - name: Check each build's CPU level + # Whole lines, not substrings; see the Windows CPU job + run: | + check() { + found=$(cmake -S . -B "$1" | grep -- '^-- Adding CPU backend variant ' || true) + echo "$1: $found" + [ "$found" = "$2" ] || { echo "::error::$1: expected '$2'"; exit 1; } + } + check build '-- Adding CPU backend variant ggml-cpu: -msse4.2;-mf16c;-mfma;-mbmi2;-mavx;-mavx2 GGML_SSE42;GGML_F16C;GGML_FMA;GGML_BMI2;GGML_AVX;GGML_AVX2' + check build-ivybridge '-- Adding CPU backend variant ggml-cpu: -msse4.2;-mf16c;-mavx GGML_SSE42;GGML_F16C;GGML_AVX' + check build-sandybridge '-- Adding CPU backend variant ggml-cpu: -msse4.2;-mavx GGML_SSE42;GGML_AVX' + - name: Package binaries run: | mkdir -p dist @@ -444,9 +608,16 @@ jobs: chmod +x dist/whisper-cpp-linux-x64-cpu cp build/bin/whisper-server dist/whisper-server-linux-x64-cpu chmod +x dist/whisper-server-linux-x64-cpu + for level in ivybridge sandybridge; do + cp "build-$level/bin/whisper-server" "dist/whisper-server-linux-x64-cpu-$level" + chmod +x "dist/whisper-server-linux-x64-cpu-$level" + done cd dist zip whisper-cpp-linux-x64-cpu.zip whisper-cpp-linux-x64-cpu zip whisper-server-linux-x64-cpu.zip whisper-server-linux-x64-cpu + for level in ivybridge sandybridge; do + zip "whisper-server-linux-x64-cpu-$level.zip" "whisper-server-linux-x64-cpu-$level" + done - name: Upload whisper-cli artifact uses: actions/upload-artifact@v4 @@ -460,7 +631,20 @@ jobs: name: whisper-server-linux-x64-cpu path: dist/whisper-server-linux-x64-cpu.zip + - name: Upload whisper-server Ivy Bridge-level artifact + uses: actions/upload-artifact@v4 + with: + name: whisper-server-linux-x64-cpu-ivybridge + path: dist/whisper-server-linux-x64-cpu-ivybridge.zip + + - name: Upload whisper-server Sandy Bridge-level artifact + uses: actions/upload-artifact@v4 + with: + name: whisper-server-linux-x64-cpu-sandybridge + path: dist/whisper-server-linux-x64-cpu-sandybridge.zip + build-linux-x64-cuda: + if: github.event_name != 'pull_request' runs-on: ubuntu-22.04 steps: - name: Free up disk space @@ -544,6 +728,7 @@ jobs: path: dist/whisper-server-linux-x64-cuda.zip build-linux-x64-vulkan: + if: github.event_name != 'pull_request' runs-on: ubuntu-22.04 steps: - name: Free up disk space @@ -610,6 +795,7 @@ jobs: path: dist/whisper-server-linux-x64-vulkan.zip build-windows-x64-vulkan: + if: github.event_name != 'pull_request' runs-on: windows-2022 steps: - name: Free up disk space @@ -708,9 +894,71 @@ jobs: name: whisper-server-win32-x64-vulkan path: dist/whisper-server-win32-x64-vulkan.zip + verify-cpu-levels: + # Runs the ivybridge and sandybridge builds on emulated processors of exactly + # those levels and checks the next build up is stopped at startup there. See + # .github/scripts/verify-cpu-levels.py. create-release waits for this. + needs: [build-windows-x64-cpu, build-linux-x64-cpu] + strategy: + fail-fast: false + matrix: + include: + - platform: win32 + os: windows-2022 + emulator: sde + - platform: linux + # QEMU emulates AVX from 7.2 on; ubuntu-22.04 ships 6.2 + os: ubuntu-24.04 + emulator: qemu + runs-on: ${{ matrix.os }} + timeout-minutes: 120 + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Install QEMU user-mode emulation + if: matrix.emulator == 'qemu' + run: | + sudo apt-get update + sudo apt-get install -y qemu-user + qemu-x86_64 --version + + - name: Download the CPU builds + uses: actions/download-artifact@v4 + with: + pattern: whisper-server-${{ matrix.platform }}-x64-cpu* + path: artifacts + + - name: Run each build on its emulated processor + shell: bash + run: | + base="artifacts/whisper-server-${{ matrix.platform }}-x64-cpu" + python .github/scripts/verify-cpu-levels.py \ + --emulator ${{ matrix.emulator }} \ + --primary-zip "$base/whisper-server-${{ matrix.platform }}-x64-cpu.zip" \ + --ivybridge-zip "$base-ivybridge/whisper-server-${{ matrix.platform }}-x64-cpu-ivybridge.zip" \ + --sandybridge-zip "$base-sandybridge/whisper-server-${{ matrix.platform }}-x64-cpu-sandybridge.zip" \ + --with-llama-libraries \ + --work "$RUNNER_TEMP/verify-cpu-levels" + + - name: Upload the server logs + if: always() + uses: actions/upload-artifact@v4 + with: + name: verify-cpu-levels-${{ matrix.platform }}-logs + path: ${{ runner.temp }}/verify-cpu-levels/*.log + create-release: - needs: [build-macos-arm64, build-macos-x64, build-windows-x64-cpu, build-windows-x64-cuda, build-windows-x64-vulkan, build-linux-x64-cpu, build-linux-x64-cuda, build-linux-x64-vulkan] + if: github.event_name != 'pull_request' + needs: [build-macos-arm64, build-macos-x64, build-windows-x64-cpu, build-windows-x64-cuda, build-windows-x64-vulkan, build-linux-x64-cpu, build-linux-x64-cuda, build-linux-x64-vulkan, verify-cpu-levels] runs-on: ubuntu-latest + permissions: + contents: write steps: - name: Download all artifacts uses: actions/download-artifact@v4 @@ -734,6 +982,10 @@ jobs: mv artifacts/whisper-server-linux-x64-cuda/whisper-server-linux-x64-cuda.zip release/ mv artifacts/whisper-server-win32-x64-vulkan/whisper-server-win32-x64-vulkan.zip release/ mv artifacts/whisper-server-linux-x64-vulkan/whisper-server-linux-x64-vulkan.zip release/ + for level in ivybridge sandybridge; do + mv "artifacts/whisper-server-win32-x64-cpu-$level/whisper-server-win32-x64-cpu-$level.zip" release/ + mv "artifacts/whisper-server-linux-x64-cpu-$level/whisper-server-linux-x64-cpu-$level.zip" release/ + done ls -la release/ - name: Determine version @@ -776,15 +1028,21 @@ jobs: - `whisper-server-linux-x64-cpu.zip` - Linux x64 CPU-only - `whisper-server-linux-x64-cuda.zip` - Linux x64 with CUDA - `whisper-server-linux-x64-vulkan.zip` - Linux x64 with Vulkan (uses system Vulkan loader) + - `whisper-server-win32-x64-cpu-ivybridge.zip`, `whisper-server-linux-x64-cpu-ivybridge.zip` - CPU-only, for processors with AVX and F16C but no AVX2 (e.g. 3rd-gen Intel Core; AMD Piledriver, Steamroller, Jaguar) + - `whisper-server-win32-x64-cpu-sandybridge.zip`, `whisper-server-linux-x64-cpu-sandybridge.zip` - CPU-only, for processors with AVX but no F16C (e.g. 2nd-gen Intel Core; AMD Bulldozer) + + ## CPU requirements (x64) + - Every x64 build without a level suffix needs AVX2, FMA, F16C and BMI2: most Intel Core processors from Haswell (4th gen) on and AMD from Excavator on; not Intel Pentium, Celeron and Atom models without AVX, which includes most made before 2021–2023. + - `-ivybridge` builds need SSE4.2, AVX and F16C. `-sandybridge` builds need SSE4.2 and AVX. Both stop at startup with an illegal-instruction fault on a processor that lacks them. ## Requirements - **Windows CUDA build**: No additional requirements - CUDA and MSVC runtime DLLs are bundled. - **Windows Vulkan build**: Requires an up-to-date GPU driver that ships the Vulkan runtime (`vulkan-1.dll`). All modern NVIDIA/AMD/Intel drivers include it. MSVC runtime DLLs are bundled. - - **Windows CPU build**: No requirements - MSVC runtime DLLs are bundled. + - **Windows CPU builds**: No software requirements - MSVC runtime DLLs are bundled. See CPU requirements. - **Linux CUDA build**: Install latest NVIDIA drivers and CUDA toolkit for GPU acceleration. - **Linux Vulkan build**: Requires the system Vulkan loader (`libvulkan.so.1`, typically from the `libvulkan1` package) and a Vulkan-capable GPU driver (Mesa for AMD/Intel, NVIDIA proprietary driver for NVIDIA). - - **Linux CPU build**: No requirements. - - **All other platforms**: No requirements. + - **Linux CPU builds**: No software requirements. See CPU requirements. + - **All other platforms**: No software requirements. ## Supported NVIDIA GPUs (CUDA builds) Compute capabilities 6.1+ (GTX 10 series and newer) diff --git a/examples/server/CMakeLists.txt b/examples/server/CMakeLists.txt index c082546bdf1..c324591d560 100644 --- a/examples/server/CMakeLists.txt +++ b/examples/server/CMakeLists.txt @@ -4,6 +4,12 @@ set(CMAKE_CXX_STANDARD_REQUIRED ON) set(TARGET whisper-server) add_executable(${TARGET} server.cpp httplib.h) +# OpenWhispr: see cpu-level-check.cpp +option(OPENWHISPR_CPU_LEVEL_CHECK "whisper-server: stop at startup on a processor below ggml's CPU level" OFF) +if (OPENWHISPR_CPU_LEVEL_CHECK) + target_sources(${TARGET} PRIVATE cpu-level-check.cpp) +endif() + include(DefaultTargetOptions) target_link_libraries(${TARGET} PRIVATE common json_cpp whisper ${CMAKE_THREAD_LIBS_INIT}) diff --git a/examples/server/cpu-level-check.cpp b/examples/server/cpu-level-check.cpp new file mode 100644 index 00000000000..c494bc1bb32 --- /dev/null +++ b/examples/server/cpu-level-check.cpp @@ -0,0 +1,95 @@ +// OpenWhispr (OpenWhispr/openwhispr#2356): stop at startup on a processor below +// the x86 level ggml's CPU code was built for. +// +// A static build has no run-time CPU dispatch, and ggml faults only when it +// first runs an instruction the processor lacks. For F16C that is the first +// transcription, after the server is already serving. OpenWhispr falls back to +// the build for the next older level when whisper-server dies at startup with +// an illegal instruction, so raise that fault here, before main(). Enabled by +// OPENWHISPR_CPU_LEVEL_CHECK, for the builds for older processors only. + +#if defined(__x86_64__) || defined(_M_X64) + +#include "ggml-cpu.h" + +#include + +#if defined(_MSC_VER) +#include +#else +#include +#endif + +namespace { + +struct cpuid_result { + unsigned eax, ebx, ecx, edx; +}; + +cpuid_result cpuid(unsigned leaf, unsigned subleaf) { +#if defined(_MSC_VER) + int r[4]; + __cpuidex(r, (int) leaf, (int) subleaf); + return { (unsigned) r[0], (unsigned) r[1], (unsigned) r[2], (unsigned) r[3] }; +#else + cpuid_result r; + __cpuid_count(leaf, subleaf, r.eax, r.ebx, r.ecx, r.edx); + return r; +#endif +} + +// AVX, F16C and FMA also need the OS to save the YMM registers (XCR0 bits 1-2) +bool os_saves_ymm(const cpuid_result & leaf1) { + if (!(leaf1.ecx & (1u << 27))) { // OSXSAVE + return false; + } +#if defined(_MSC_VER) + const unsigned long long xcr0 = _xgetbv(0); +#else + unsigned lo, hi; + __asm__ volatile("xgetbv" : "=a"(lo), "=d"(hi) : "c"(0)); + const unsigned long long xcr0 = ((unsigned long long) hi << 32) | lo; +#endif + return (xcr0 & 0x6) == 0x6; +} + +bool check_cpu_level() { + const cpuid_result leaf1 = cpuid(1, 0); + const cpuid_result leaf7 = cpuid(0, 0).eax >= 7 ? cpuid(7, 0) : cpuid_result{}; + const bool ymm = (leaf1.ecx & (1u << 28)) && os_saves_ymm(leaf1); + + const struct { + const char * name; + bool built; + bool present; + } features[] = { + { "AVX", ggml_cpu_has_avx() != 0, ymm }, + { "F16C", ggml_cpu_has_f16c() != 0, ymm && (leaf1.ecx & (1u << 29)) }, + { "FMA", ggml_cpu_has_fma() != 0, ymm && (leaf1.ecx & (1u << 12)) }, + { "AVX2", ggml_cpu_has_avx2() != 0, ymm && (leaf7.ebx & (1u << 5)) }, + { "BMI2", ggml_cpu_has_bmi2() != 0, (leaf7.ebx & (1u << 8)) != 0 }, + }; + + bool supported = true; + for (const auto & feature : features) { + if (feature.built && !feature.present) { + fprintf(stderr, "whisper-server: built for %s, which this processor does not support\n", feature.name); + supported = false; + } + } + if (!supported) { + fflush(stderr); +#if defined(_MSC_VER) + __ud2(); +#else + __builtin_trap(); +#endif + } + return supported; +} + +[[maybe_unused]] const bool cpu_level_supported = check_cpu_level(); + +} // namespace + +#endif