perf: device-resident stages, on-device DBCache, weight folding, GLU split, ggml v0.20.2

.github/workflows/ci.yml

@@ -121,17 +121,30 @@ jobs:
           - os: ubuntu-latest
             name: linux-x64-cpu
             backend: cpu
-            # CPU-only package built with -march=native for the target simd
-            # (ggml then uses AVX2/AVX-512 etc. via GGML_NATIVE).
-            cmake_extra: "-DGGML_NATIVE=ON"
+            # v0.20.2 constraint: GGML_CPU_ALL_VARIANTS requires
+            # GGML_BACKEND_DL (ISA variants became dlopen MODULE plugins),
+            # but DL mode does NOT link the CPU backend into the ggml
+            # umbrella target, so game.cpp's direct references
+            # (ggml_backend_cpu_init, ggml_threadpool_new, ...) fail to
+            # link.  Non-DL + GGML_NATIVE=OFF builds a portable baseline
+            # x86-64 CPU backend (no -march=native capture of the runner).
+            # SIMD dispatch for the CPU package needs a dlopen-path refactor
+            # of backend.cpp and is tracked separately.
+            cmake_extra: "-DGGML_NATIVE=OFF"
             build_jobs: 4
             pkg_ext: ""
             lib_glob: "libggml*.so*"
 
           - os: ubuntu-22.04
             name: linux-x64-cuda
             backend: cuda
-            cmake_extra: '-DGAME_GGML_CUDA=ON -DGGML_NATIVE=OFF -DCMAKE_CUDA_ARCHITECTURES=75'
+            # CUDA 12.9 (newest Jimver/cuda-toolkit ships; 12.8+ required to
+            # compile Blackwell sm_120).  Arch list: native SASS for the
+            # mainstream generations (Turing→Hopper) plus compute_120 PTX so
+            # RTX 50-series and future GPUs run native-arch JIT instead of
+            # falling back to compute_75 PTX (which works but loses all
+            # newer-architecture kernel optimisations).
+            cmake_extra: '-DGAME_GGML_CUDA=ON -DGGML_NATIVE=OFF -DCMAKE_CUDA_ARCHITECTURES="75;80;86;89;90;120-virtual"'
             build_jobs: 2
             pkg_ext: ""
             lib_glob: "libggml*.so*"
@@ -163,7 +176,9 @@ jobs:
           - os: windows-2022
             name: windows-x64-cuda
             backend: cuda
-            cmake_extra: '-DGAME_GGML_CUDA=ON -DGGML_NATIVE=OFF -DCMAKE_CUDA_ARCHITECTURES=75'
+            # Same arch policy as linux-x64-cuda (see above): CUDA 12.9 +
+            # SASS for Turing→Hopper + Blackwell PTX.
+            cmake_extra: '-DGAME_GGML_CUDA=ON -DGGML_NATIVE=OFF -DCMAKE_CUDA_ARCHITECTURES="75;80;86;89;90;120-virtual"'
             build_jobs: 2
             pkg_ext: ".exe"
             lib_glob: "ggml*.dll"
@@ -189,17 +204,17 @@ jobs:
 
       - name: Install CUDA Toolkit (Linux)
         if: matrix.backend == 'cuda' && runner.os == 'Linux'
-        uses: Jimver/cuda-toolkit@1a3c14e26833ccf292b268f9a790fb47dea7b2da # v0.2.28
+        uses: Jimver/cuda-toolkit@v0.2.29
         with:
-          cuda: "12.6.3"
+          cuda: "12.9.0"
           method: network
           log-file-suffix: "${{ matrix.name }}.txt"
 
       - name: Install CUDA Toolkit (Windows)
         if: matrix.backend == 'cuda' && runner.os == 'Windows'
-        uses: Jimver/cuda-toolkit@b8bf9c6c28f8a92fbb04dcfcaee872e60c57462d # v0.2.36
+        uses: Jimver/cuda-toolkit@v0.2.29
         with:
-          cuda: "12.6.3"
+          cuda: "12.9.0"
           method: network
           sub-packages: '["nvcc", "cudart", "cublas", "cublas_dev", "visual_studio_integration"]'
           log-file-suffix: "${{ matrix.name }}.txt"

.gitignore

@@ -1,5 +1,6 @@
 # Build outputs
 build/
+build-*/
 out/
 cmake-build-*/
 

BUILDING.md

@@ -22,7 +22,7 @@ sudo apt install libvulkan-dev vulkan-tools
 #   sudo apt install glslc-tools   # not available on all distros
 
 # Optional: CUDA backend (NVIDIA GPUs)
-# Install a CUDA Toolkit supported by your compiler and driver. CI uses CUDA 12.6.3.
+# Install a CUDA Toolkit supported by your compiler and driver. CI uses CUDA 12.9.0.
 # The prebuilt CUDA packages target Turing (CC 7.5) and newer GPUs.
 # https://developer.nvidia.com/cuda-downloads
 
@@ -57,9 +57,9 @@ brew install cmake ccache
 # Vulkan SDK (optional, for Vulkan backend)
 #   https://vulkan.lunarg.com/sdk/home
 #
-# CUDA Toolkit 12.6.x (optional, for CUDA backend)
+# CUDA Toolkit 12.9.x (optional, for CUDA backend)
 #   https://developer.nvidia.com/cuda-downloads
-# CUDA 12.6 supports Visual Studio 2022 / MSVC 193x.
+# CUDA 12.9 supports Visual Studio 2019 16.11+ and 2022 (MSVC 192x/193x).
 ```
 
 > **Windows + CUDA toolchain notes (from local builds):**
@@ -73,7 +73,7 @@ brew install cmake ccache
 >   Pin the one you intend to use, e.g. set `CUDA_PATH_V13_0` to the v11.6 path
 >   when building with CUDA 11.6, otherwise nvcc 13 + MSVC 14.29 hits
 >   `__cudaLaunch` macro breakage (`error C4002`).
-> - CI reference: `ubuntu-22.04` / `windows-2022` + CUDA 12.6.3.
+> - CI reference: `ubuntu-22.04` / `windows-2022` + CUDA 12.9.0.
 
 ## Quick start
 
@@ -193,7 +193,7 @@ build/bin/game_ggml_cli serve game_medium.gguf
 ## CUDA compatibility and CI scope
 
 The hosted CI builds Linux x64 and Windows x64 CUDA packages with CUDA Toolkit
-12.6.3 and Visual Studio 2022 on Windows. It verifies Toolkit discovery, CUDA
+12.9.0 and Visual Studio 2022 on Windows. It verifies Toolkit discovery, CUDA
 compilation, linking, and packaging. On Windows, `ggml-cuda.dll` imports
 `nvcuda.dll` (the NVIDIA driver library) at load time, and GitHub-hosted runners
 have no NVIDIA driver, so the CLI cannot start there even for `--version`; the
@@ -203,19 +203,24 @@ dependencies instead, and performs the startup smoke test only when
 provide an NVIDIA GPU, so actual CUDA inference must still be smoke-tested on an
 NVIDIA system.
 
-The release architecture is `75`, which emits both native CC 7.5 SASS and CC
-7.5 PTX. Turing GPUs (for example, GeForce RTX 20 series) use the native image;
-newer Ampere, Ada, and later drivers can JIT the PTX forward-compatible image.
-This keeps the hosted build practical: compiling every ggml CUDA translation
-unit separately for four real architectures was several times slower and used
-substantially more memory.
+The release architecture list is `75;80;86;89;90;120-virtual`:
+native SASS for Turing (CC 7.5), Ampere data-center (CC 8.0), Ampere consumer
+(CC 8.6), Ada (CC 8.9) and Hopper (CC 9.0), plus `compute_120` PTX so RTX
+50-series and future GPUs JIT with a Blackwell-targeted image. A plain `75`
+build also runs on newer GPUs through its bundled `compute_75` PTX, but the
+JIT-ed kernels then miss every newer-architecture optimisation; shipping SASS
+for the mainstream generations removes that penalty. Compiling six
+architectures roughly multiplies the nvcc workload (the CUDA jobs already run
+with `build_jobs: 2` for memory headroom), which is the trade-off for native
+performance on each generation.
 
 Pascal and Volta are not included in the prebuilt package. Source builds that
 need these older GPUs can use CUDA 12.x and add `61-real` and/or `70-real`.
-Source builds that prefer native images for each newer generation may use
+Source builds that want to trim the list back to Turing→Ada may use
 `75-real;80-real;86-real;89-real`, accepting the longer build and larger binary.
 CUDA 13.0 removed NVCC offline compilation for architectures older than CC 7.5;
-use CUDA 12.9 or earlier when maintaining such builds.
+use CUDA 12.9 or earlier when maintaining such builds. sm_100/sm_120 (Blackwell)
+compile targets require CUDA 12.8 or newer.
 
 CUDA 12.x minor-version compatibility requires at least NVIDIA driver
 525.60.13 on Linux or 528.33 on Windows, subject to the limitations documented

README.md

@@ -273,7 +273,7 @@ int main() {
     params.language = 4;   // from lang_map: { "zh": 4 }
     params.seed     = 42;
     // DBCache (segmenter cross-step reuse; affects nsteps>1 only).
-    // -1 = auto (CPU 0.25, GPU off); 0 = off; >0 = explicit threshold.
+    // -1 = auto (0.25 on all backends); 0 = off; >0 = explicit threshold.
     params.db_cache_threshold = 0.25f;
     params.db_cache_fn_blocks = 1;
     params.db_cache_warmup    = 1;
@@ -346,7 +346,7 @@ trees live under `build/_deps/<name>-src/` after the first configure.
 
 | Dependency | Version pin | License | SPDX identifier |
 |---|---|---|---|
-| [ggml](https://github.com/ggerganov/ggml) | `v0.11.0` tag | MIT | MIT |
+| [ggml](https://github.com/ggerganov/ggml) | `v0.20.2` tag | MIT | MIT |
 | [pocketfft](https://gitlab.mpcdf.mpg.de/mtr/pocketfft) | commit `32424d20` on `cpp` branch | BSD-3-Clause | BSD-3-Clause |
 | [dr_libs](https://github.com/mackron/dr_libs) | commit `243e26ff` on `master` | Public Domain / MIT-0 (dual) | `Unlicense OR MIT-0` |
 | [GoogleTest](https://github.com/google/googletest) | `v1.14.0` tag (tests only) | BSD-3-Clause | BSD-3-Clause |

README_CN.md

@@ -235,7 +235,7 @@ int main() {
     params.language = 4;   // 来自 lang_map: { "zh": 4 }
     params.seed     = 42;
     // DBCache（跨步复用，仅 nsteps>1 生效）：
-    // -1 = 自动（CPU 0.25，GPU 关闭）；0 = 关闭；>0 = 显式阈值。
+    // -1 = 自动（所有后端 0.25）；0 = 关闭；>0 = 显式阈值。
     params.db_cache_threshold = 0.25f;
     params.db_cache_fn_blocks = 1;
     params.db_cache_warmup    = 1;
@@ -282,7 +282,7 @@ ctest --test-dir ggml_backend/build --output-on-failure
 
 | 依赖 | 版本 pin | 许可 | SPDX 标识 |
 |---|---|---|---|
-| [ggml](https://github.com/ggerganov/ggml) | `v0.11.0` tag | MIT | MIT |
+| [ggml](https://github.com/ggerganov/ggml) | `v0.20.2` tag | MIT | MIT |
 | [pocketfft](https://gitlab.mpcdf.mpg.de/mtr/pocketfft) | `cpp` 分支 `32424d20` | BSD-3-Clause | BSD-3-Clause |
 | [dr_libs](https://github.com/mackron/dr_libs) | `master` 分支 `243e26ff` | Public Domain / MIT-0（双许可） | `Unlicense OR MIT-0` |
 | [GoogleTest](https://github.com/google/googletest) | `v1.14.0` tag（仅测试） | BSD-3-Clause | BSD-3-Clause |

cmake/Dependencies.cmake

@@ -62,7 +62,7 @@ endfunction()
 # ---------------------------------------------------------------------------
 # SPIRV-Headers shim (Windows Vulkan only)
 #
-# ggml v0.19.0's Vulkan backend hard-requires find_package(SPIRV-Headers CONFIG).
+# ggml v0.20.x's Vulkan backend hard-requires find_package(SPIRV-Headers CONFIG).
 # Windows Vulkan SDKs older than ~1.4.35x ship the headers but not that CMake
 # config file, so the windows-x64-vulkan CI job fails at configure time.
 # Generate a minimal config pointing at the SDK headers when the SDK doesn't
@@ -87,9 +87,18 @@ endif()
 # ---------------------------------------------------------------------------
 FetchContent_Declare(
     ggml
-    GIT_REPOSITORY https://github.com/ggerganov/ggml.git
-    GIT_TAG        v0.19.0
-    GIT_SHALLOW    TRUE
+    # URL archive instead of git: with FETCHCONTENT_UPDATES_DISCONNECTED=ON
+    # the populate gitupdate step must resolve the pinned ref locally, and a
+    # fresh clone cannot — a shallow clone only carries the default-branch
+    # HEAD, and the v0.20.2 tag/commit is not on it — so populate aborts with
+    # "requested git ref ... not present locally".  A URL archive has no git
+    # ref semantics: populate is a plain download+extract (network needed on
+    # first populate / cache miss; the CI _deps cache then makes later runs
+    # offline).  Patches are still applied with `git apply`, which needs no
+    # .git directory.  (No DOWNLOAD_EXTRACT_TIMESTAMP: requires CMake 3.24+,
+    # project minimum is 3.18.)
+    URL      https://github.com/ggerganov/ggml/archive/refs/tags/v0.20.2.tar.gz
+    URL_HASH SHA256=55dfd1ea4e6b6b3e25d9411f9525eb4df1c796c03a244e2321388b30f189cd3d
 )
 
 FetchContent_GetProperties(ggml)

cmake/patches/ggml-metal-binary-archive.md

@@ -27,7 +27,7 @@ Controlled by the same `GGML_METAL_ARCHIVE_PATH` env var; disable with
 
 ## Baseline
 
-Applied against **ggml v0.19.0** (`ggml-metal-device.m`).  Verify with the
+Applied against **ggml v0.20.2** (`ggml-metal-device.m`).  Verify with the
 same command used in `cmake/Dependencies.cmake`:
 
 ```

cmake/patches/ggml-vulkan-pipeline-cache.md

@@ -38,7 +38,7 @@ driver) load precompiled PSO bytes instead of recompiling every shader.
 
 ## Baseline & re-apply
 
-- Applies to **ggml v0.19.0** (`ggml-vulkan.cpp`). Pinned by game.cpp
+- Applies to **ggml v0.20.2** (`ggml-vulkan.cpp`). Pinned by game.cpp
   FetchContent; re-apply per ggml upgrade via `cmake/Dependencies.cmake`
   `game_ggml_apply_patch` (idempotent: skips if already applied).
 

cmake/patches/ggml-vulkan-pipeline-cache.patch

@@ -1,5 +1,5 @@
 diff --git a/src/ggml-vulkan/ggml-vulkan.cpp b/src/ggml-vulkan/ggml-vulkan.cpp
-index a923755..56304ff 100644
+index 585e10d..4f3686a 100644
 --- a/src/ggml-vulkan/ggml-vulkan.cpp
 +++ b/src/ggml-vulkan/ggml-vulkan.cpp
 @@ -50,6 +50,7 @@ typedef struct VkPhysicalDeviceCooperativeMatrixDecodeVectorFeaturesNV {
@@ -192,7 +192,7 @@ index a923755..56304ff 100644
  void vk_command_pool::init(vk_device& device, vk_queue *q_) {
      cmd_buffers.clear();
      q = q_;
-@@ -3025,7 +3180,8 @@ static void ggml_vk_create_pipeline_func(vk_device& device, vk_pipeline& pipelin
+@@ -3026,7 +3181,8 @@ static void ggml_vk_create_pipeline_func(vk_device& device, vk_pipeline& pipelin
  #endif
 
      try {
@@ -202,7 +202,7 @@ index a923755..56304ff 100644
      } catch (const vk::SystemError& e) {
          std::cerr << "ggml_vulkan: Compute pipeline creation failed for " << pipeline->name << std::endl;
          std::cerr << "ggml_vulkan: " << e.what() << std::endl;
-@@ -6910,6 +7066,8 @@ static vk_device ggml_vk_get_device(size_t idx) {
+@@ -6934,6 +7090,8 @@ static vk_device ggml_vk_get_device(size_t idx) {
              .setPEnabledExtensionNames(device_extensions);
          device_create_info.setPNext(&device_features2);
          device->device = device->physical_device.createDevice(device_create_info);
@@ -211,7 +211,7 @@ index a923755..56304ff 100644
 
          if (device->device_fault) {
              device->pfn_vkGetDeviceFaultInfoEXT = (PFN_vkGetDeviceFaultInfoEXT)
-@@ -17372,6 +17530,11 @@ static ggml_status ggml_backend_vk_graph_compute(ggml_backend_t backend, ggml_cg
+@@ -17402,6 +17560,11 @@ static ggml_status ggml_backend_vk_graph_compute(ggml_backend_t backend, ggml_cg
          ggml_vk_synchronize(ctx);
      }