Merge pull request #9 from KakaruHayate/feature/perf-vulkan-pipeline-cache

perf(Vulkan): persistent VkPipelineCache (cold-start fix)

Author: KakaruHayate

.gitattributes

@@ -0,0 +1,5 @@
+# Patches are applied with `git apply` during CMake configure; they must keep
+# LF line endings on every platform (Windows checkout otherwise rewrites LFs to
+# CRLF, making the patch unapplicable).
+*.patch text eol=lf
+*.md     text eol=lf

README.md

@@ -199,9 +199,12 @@ configs produce identical note output):
   is limited.
 - CPU benefits from Q8 for memory, at essentially the same speed — use the
   `-q8` pack on CPU.
-- Cold start: the first inference on a GPU compiles shaders
-  (Vulkan/Metal). NVIDIA's driver caches these across runs; expect a couple
-  of extra seconds on the very first invocation.
+- Cold start: the first inference on a GPU compiles shaders (Vulkan/Metal).
+  Metal uses an in-repo binary-archive PSO cache; **Vulkan now persists a
+  disk-backed `VkPipelineCache`** (`cmake/patches/ggml-vulkan-pipeline-cache.*`,
+  `GGML_VK_PIPELINE_CACHE_PATH`, default under the user cache dir), so later
+  launches load precompiled PSOs instead of recompiling — no reliance on
+  driver-level caches.
 
 ## Reproducing the benchmark
 

cmake/Dependencies.cmake

@@ -104,6 +104,15 @@ if(NOT ggml_POPULATED)
         )
     endif()
 
+    # Vulkan cold-start fix: persist VkPipelineCache to disk.  Apply on every
+    # build (the patch only touches ggml-vulkan.cpp, compiled only when the
+    # Vulkan backend is enabled) so CPU-only and GPU builds share one source.
+    game_ggml_apply_patch(
+        "${ggml_SOURCE_DIR}"
+        "${CMAKE_CURRENT_LIST_DIR}/patches/ggml-vulkan-pipeline-cache.patch"
+        "ggml Vulkan disk-backed VkPipelineCache"
+    )
+
     add_subdirectory("${ggml_SOURCE_DIR}" "${ggml_BINARY_DIR}")
 endif()
 

cmake/patches/ggml-vulkan-pipeline-cache.md

@@ -0,0 +1,58 @@
+# ggml-vulkan-pipeline-cache（game.cpp 自有 patch）
+
+## Purpose
+
+Vulkan 冷启动 Currently every process recompiles the whole compute-pipeline
+set from SPIR-V on first inference（本地实测首次冷启：Vulkan+Q8 达 42s、
+F32 约 7.5s；NVIDIA 驱动级 shader 缓存能缓解，但 AMD/Intel/其它驱动不能
+保证，且无法随包分发）。
+
+This patch persists `VkPipelineCache` to disk — the Vulkan analog of the
+Metal binary-archive PSO cache (`cmake/patches/ggml-metal-binary-archive.patch`).
+After the first run, subsequent launches (or any process on a compatible
+driver) load precompiled PSO bytes instead of recompiling every shader.
+
+## Change
+
+`src/ggml-vulkan/ggml-vulkan.cpp`:
+
+- `vk_device_struct` gains `pipeline_cache` / `pipeline_cache_init` /
+  `pipeline_cache_path` / `pipeline_cache_dirty`.
+- At device init (`ggml_vk_init_pipeline_cache`) load the on-disk cache bytes
+  into `VkPipelineCacheCreateInfo::pInitialData` (stale/invalid bytes are safe
+  — the driver returns `VK_INCOMPLETE` and builds a fresh cache), and create
+  the handle.
+- `ggml_vk_create_pipeline_func` passes the cache handle to
+  `createComputePipeline` and sets `pipeline_cache_dirty = true`.
+- On graph-compute completion (and at device teardown as a fallback),
+  `ggml_vk_save_pipeline_cache_device` flushes `vkGetPipelineCacheData` to
+  disk, throttled by the dirty flag.
+
+## Env / control
+
+- `GGML_VK_PIPELINE_CACHE_PATH` — cache file path.
+- default: `%LOCALAPPDATA%\\game_ggml_vk_pipeline.cache` (Windows) or
+  `$HOME/.cache/game_ggml_vk_pipeline.cache` (POSIX).
+- `GGML_VK_DISABLE_PIPELINE_CACHE` — opt out.
+- `GGML_VK_PIPELINE_CACHE_DEBUG` — print load/save diagnostics.
+
+## Baseline & re-apply
+
+- Applies to **ggml v0.19.0** (`ggml-vulkan.cpp`). Pinned by game.cpp
+  FetchContent; re-apply per ggml upgrade via `cmake/Dependencies.cmake`
+  `game_ggml_apply_patch` (idempotent: skips if already applied).
+
+## Verified (local, RTX 2070, Vulkan)
+
+- Run1 (no cache): builds from scratch, saves ~1.14 MB.
+- Run2 (with cache): logs "loaded 1,140,086 bytes", same note output (33/33),
+  warm total 0.42s.
+
+## Notes
+
+- The dispatch-table C functions (`vkCreatePipelineCache` /
+  `vkGetPipelineCacheData` / `vkDestroyPipelineCache`) are used because this
+  SDK's generated `vulkan.hpp` does not expose the corresponding C++ class
+  methods.
+- Devices are cached for the process lifetime in ggml-vulkan, so the save is
+  driven by graph-compute completion rather than the device destructor.

cmake/patches/ggml-vulkan-pipeline-cache.patch

@@ -0,0 +1,225 @@
+diff --git a/src/ggml-vulkan/ggml-vulkan.cpp b/src/ggml-vulkan/ggml-vulkan.cpp
+index a923755..56304ff 100644
+--- a/src/ggml-vulkan/ggml-vulkan.cpp
++++ b/src/ggml-vulkan/ggml-vulkan.cpp
+@@ -50,6 +50,7 @@ typedef struct VkPhysicalDeviceCooperativeMatrixDecodeVectorFeaturesNV {
+ 
+ #include <algorithm>
+ #include <cmath>
++#include <fstream>
+ #include <iomanip>
+ #include <iostream>
+ #include <tuple>
+@@ -67,6 +68,7 @@ typedef struct VkPhysicalDeviceCooperativeMatrixDecodeVectorFeaturesNV {
+ #include <future>
+ #include <condition_variable>
+ #include <thread>
++#include <string>
+ 
+ #if defined(_MSC_VER)
+ # define NOMINMAX 1
+@@ -792,6 +794,13 @@ struct vk_device_struct {
+     bool pipeline_robustness;
+     bool memory_priority;
+     vk::Device device;
++    // game.cpp: persistent VkPipelineCache (disk-backed, like the Metal
++    // binary-archive PSO cache).  Loaded at device init, passed to every
++    // createComputePipeline, flushed back to disk at teardown.
++    vk::PipelineCache pipeline_cache;
++    std::vector<uint8_t> pipeline_cache_init;   // bytes read from disk (may be empty)
++    std::string pipeline_cache_path;            // empty = disabled/unset
++    bool pipeline_cache_dirty = false;          // a pipeline was created since last save
+     uint32_t vendor_id;
+     vk::DriverId driver_id;
+     vk_device_architecture architecture;
+@@ -1122,6 +1131,48 @@ struct vk_device_struct {
+         compute_queue.reset();
+         transfer_queue.reset();
+ 
++        // game.cpp: flush the persistent pipeline cache back to disk BEFORE
++        // destroying the individual pipelines (destroying a pipeline evicts its
++        // cached PSOs, so vkGetPipelineCacheData would return empty after the
++        // loop).  This SDK's vulkan.hpp does not generate the
++        // vkGetPipelineCacheData/vkDestroyPipelineCache hpp methods, so use the
++        // dispatch-table C functions directly.
++        if (static_cast<VkPipelineCache>(pipeline_cache) != VK_NULL_HANDLE) {
++            try {
++                VkDevice      dev = static_cast<VkDevice>(device);
++                VkPipelineCache pc = static_cast<VkPipelineCache>(pipeline_cache);
++                size_t sz = 0;
++                // The size query returns VK_INCOMPLETE (and fills *pDataSize);
++                // accept both VK_SUCCESS and VK_INCOMPLETE here.
++                const VkResult q = ::vkGetPipelineCacheData(dev, pc, &sz, nullptr);
++                if (std::getenv("GGML_VK_PIPELINE_CACHE_DEBUG")) {
++                    std::cerr << "ggml_vulkan: pipeline cache query q=" << static_cast<int>(q)
++                              << " sz=" << sz << std::endl;
++                }
++                if ((q == VK_SUCCESS || q == VK_INCOMPLETE) && sz > 0) {
++                    std::vector<uint8_t> data(sz);
++                    const VkResult g = ::vkGetPipelineCacheData(dev, pc, &sz, data.data());
++                    if ((g == VK_SUCCESS || g == VK_INCOMPLETE) && !pipeline_cache_path.empty()) {
++                        data.resize(std::min<size_t>(sz, data.size()));
++                        std::ofstream f(pipeline_cache_path, std::ios::binary | std::ios::trunc);
++                        if (f) {
++                            f.write(reinterpret_cast<const char *>(data.data()),
++                                    static_cast<std::streamsize>(data.size()));
++                            if (std::getenv("GGML_VK_PIPELINE_CACHE_DEBUG")) {
++                                std::cerr << "ggml_vulkan: pipeline cache: saved " << data.size()
++                                          << " bytes to " << pipeline_cache_path << std::endl;
++                            }
++                        }
++                    }
++                }
++            } catch (const vk::SystemError & e) {
++                std::cerr << "ggml_vulkan: pipeline cache save failed: " << e.what() << std::endl;
++            }
++            VkPipelineCache pc = static_cast<VkPipelineCache>(pipeline_cache);
++            ::vkDestroyPipelineCache(static_cast<VkDevice>(device), pc, nullptr);
++            pipeline_cache = vk::PipelineCache{};
++        }
++
+         for (auto& pipeline : all_pipelines) {
+             if (pipeline.expired()) {
+                 continue;
+@@ -1138,6 +1189,110 @@ struct vk_device_struct {
+     }
+ };
+ 
++// game.cpp: persistent pipeline cache helpers.
++    // - Path: GGML_VK_PIPELINE_CACHE_PATH or <user-cache>/game_ggml_vk_pipeline.cache
++    // - Disable: GGML_VK_DISABLE_PIPELINE_CACHE
++    // Stale/invalid cache bytes are safe: the driver returns VK_INCOMPLETE and
++    // creates an empty cache, so an outdated/GPU-swapped cache degrades to a
++    // normal cold compile — never a crash.
++    static std::string ggml_vk_pipeline_cache_path() {
++        const char * env = std::getenv("GGML_VK_PIPELINE_CACHE_PATH");
++        if (env && *env) return std::string(env);
++#ifdef _WIN32
++        if (const char * la = std::getenv("LOCALAPPDATA")) {
++            return std::string(la) + "\\game_ggml_vk_pipeline.cache";
++        }
++#else
++        if (const char * home = std::getenv("HOME")) {
++            return std::string(home) + "/.cache/game_ggml_vk_pipeline.cache";
++        }
++#endif
++        return std::string();
++    }
++
++    static void ggml_vk_init_pipeline_cache(vk_device device) {
++        if (std::getenv("GGML_VK_DISABLE_PIPELINE_CACHE")) return;
++        device->pipeline_cache_path = ggml_vk_pipeline_cache_path();
++        if (device->pipeline_cache_path.empty()) return;
++
++        std::ifstream f(device->pipeline_cache_path, std::ios::binary | std::ios::ate);
++        if (f.good()) {
++            const std::streamsize sz = f.tellg();
++            f.seekg(0);
++            device->pipeline_cache_init.resize(static_cast<size_t>(sz));
++            if (sz > 0) {
++                f.read(reinterpret_cast<char *>(device->pipeline_cache_init.data()), sz);
++            }
++        }
++
++        VkPipelineCacheCreateInfo ci{};
++        ci.sType = VK_STRUCTURE_TYPE_PIPELINE_CACHE_CREATE_INFO;
++        if (!device->pipeline_cache_init.empty()) {
++            ci.initialDataSize = device->pipeline_cache_init.size();
++            ci.pInitialData    = device->pipeline_cache_init.data();
++        }
++        try {
++            VkPipelineCache raw = VK_NULL_HANDLE;
++            if (::vkCreatePipelineCache(static_cast<VkDevice>(device->device), &ci, nullptr, &raw) == VK_SUCCESS) {
++                device->pipeline_cache = vk::PipelineCache(raw);
++                if (std::getenv("GGML_VK_PIPELINE_CACHE_DEBUG")) {
++                    if (device->pipeline_cache_init.empty()) {
++                        std::cerr << "ggml_vulkan: pipeline cache: no cached data, will build from scratch" << std::endl;
++                    } else {
++                        std::cerr << "ggml_vulkan: pipeline cache: loaded " << device->pipeline_cache_init.size()
++                                  << " bytes from " << device->pipeline_cache_path << std::endl;
++                    }
++                }
++            } else {
++                std::cerr << "ggml_vulkan: pipeline cache create failed, disabling" << std::endl;
++                device->pipeline_cache_path.clear();
++            }
++        } catch (const vk::SystemError & e) {
++            std::cerr << "ggml_vulkan: pipeline cache create failed, disabling: "
++                      << e.what() << std::endl;
++            device->pipeline_cache = vk::PipelineCache{};
++            device->pipeline_cache_path.clear();
++        }
++    }
++
++    // Persist the pipeline cache to disk if any pipeline was created since the
++    // last save.  Called on graph-compute completion (the devices are cached
++    // for the process lifetime, so the destructor is not a usable hook in CLI
++    // flows).  The save is throttled by the dirty flag: it only fires after a
++    // graph that actually created new pipelines.
++    static void ggml_vk_save_pipeline_cache_device(vk_device device) {
++        if (!device->pipeline_cache_dirty ||
++            static_cast<VkPipelineCache>(device->pipeline_cache) == VK_NULL_HANDLE ||
++            device->pipeline_cache_path.empty()) {
++            return;
++        }
++        device->pipeline_cache_dirty = false;
++        try {
++            VkDevice      dev = static_cast<VkDevice>(device->device);
++            VkPipelineCache pc = static_cast<VkPipelineCache>(device->pipeline_cache);
++            size_t sz = 0;
++            const VkResult q = ::vkGetPipelineCacheData(dev, pc, &sz, nullptr);
++            if ((q == VK_SUCCESS || q == VK_INCOMPLETE) && sz > 0) {
++                std::vector<uint8_t> data(sz);
++                const VkResult g = ::vkGetPipelineCacheData(dev, pc, &sz, data.data());
++                if ((g == VK_SUCCESS || g == VK_INCOMPLETE) && !data.empty()) {
++                    data.resize(std::min<size_t>(sz, data.size()));
++                    std::ofstream f(device->pipeline_cache_path, std::ios::binary | std::ios::trunc);
++                    if (f) {
++                        f.write(reinterpret_cast<const char *>(data.data()),
++                                static_cast<std::streamsize>(data.size()));
++                        if (std::getenv("GGML_VK_PIPELINE_CACHE_DEBUG")) {
++                            std::cerr << "ggml_vulkan: pipeline cache: saved " << data.size()
++                                      << " bytes to " << device->pipeline_cache_path << std::endl;
++                        }
++                    }
++                }
++            }
++        } catch (const vk::SystemError & e) {
++            std::cerr << "ggml_vulkan: pipeline cache save failed: " << e.what() << std::endl;
++        }
++    }
++
+ void vk_command_pool::init(vk_device& device, vk_queue *q_) {
+     cmd_buffers.clear();
+     q = q_;
+@@ -3025,7 +3180,8 @@ static void ggml_vk_create_pipeline_func(vk_device& device, vk_pipeline& pipelin
+ #endif
+ 
+     try {
+-        pipeline->pipeline = device->device.createComputePipeline(VK_NULL_HANDLE, compute_pipeline_create_info).value;
++        pipeline->pipeline = device->device.createComputePipeline(device->pipeline_cache, compute_pipeline_create_info).value;
++        device->pipeline_cache_dirty = true;
+     } catch (const vk::SystemError& e) {
+         std::cerr << "ggml_vulkan: Compute pipeline creation failed for " << pipeline->name << std::endl;
+         std::cerr << "ggml_vulkan: " << e.what() << std::endl;
+@@ -6910,6 +7066,8 @@ static vk_device ggml_vk_get_device(size_t idx) {
+             .setPEnabledExtensionNames(device_extensions);
+         device_create_info.setPNext(&device_features2);
+         device->device = device->physical_device.createDevice(device_create_info);
++        // game.cpp: load/create a disk-backed VkPipelineCache (cold-start fix).
++        ggml_vk_init_pipeline_cache(device);
+ 
+         if (device->device_fault) {
+             device->pfn_vkGetDeviceFaultInfoEXT = (PFN_vkGetDeviceFaultInfoEXT)
+@@ -17372,6 +17530,11 @@ static ggml_status ggml_backend_vk_graph_compute(ggml_backend_t backend, ggml_cg
+         ggml_vk_synchronize(ctx);
+     }
+ 
++    // game.cpp: persist any newly-created pipelines back to the on-disk
++    // VkPipelineCache (dirty-flag throttled; no-op unless a pipeline was
++    // created since the last save).
++    ggml_vk_save_pipeline_cache_device(ctx->device);
++
+     return GGML_STATUS_SUCCESS;
+ 
+     UNUSED(backend);