diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..418dc56 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,5 @@ +# Patches are applied with `git apply` during CMake configure; they must keep +# LF line endings on every platform (Windows checkout otherwise rewrites LFs to +# CRLF, making the patch unapplicable). +*.patch text eol=lf +*.md text eol=lf diff --git a/README.md b/README.md index 545447a..a7a53ad 100644 --- a/README.md +++ b/README.md @@ -199,9 +199,12 @@ configs produce identical note output): is limited. - CPU benefits from Q8 for memory, at essentially the same speed — use the `-q8` pack on CPU. -- Cold start: the first inference on a GPU compiles shaders - (Vulkan/Metal). NVIDIA's driver caches these across runs; expect a couple - of extra seconds on the very first invocation. +- Cold start: the first inference on a GPU compiles shaders (Vulkan/Metal). + Metal uses an in-repo binary-archive PSO cache; **Vulkan now persists a + disk-backed `VkPipelineCache`** (`cmake/patches/ggml-vulkan-pipeline-cache.*`, + `GGML_VK_PIPELINE_CACHE_PATH`, default under the user cache dir), so later + launches load precompiled PSOs instead of recompiling — no reliance on + driver-level caches. ## Reproducing the benchmark diff --git a/cmake/Dependencies.cmake b/cmake/Dependencies.cmake index 8cf0a13..44f56ba 100644 --- a/cmake/Dependencies.cmake +++ b/cmake/Dependencies.cmake @@ -104,6 +104,15 @@ if(NOT ggml_POPULATED) ) endif() + # Vulkan cold-start fix: persist VkPipelineCache to disk. Apply on every + # build (the patch only touches ggml-vulkan.cpp, compiled only when the + # Vulkan backend is enabled) so CPU-only and GPU builds share one source. + game_ggml_apply_patch( + "${ggml_SOURCE_DIR}" + "${CMAKE_CURRENT_LIST_DIR}/patches/ggml-vulkan-pipeline-cache.patch" + "ggml Vulkan disk-backed VkPipelineCache" + ) + add_subdirectory("${ggml_SOURCE_DIR}" "${ggml_BINARY_DIR}") endif() diff --git a/cmake/patches/ggml-vulkan-pipeline-cache.md b/cmake/patches/ggml-vulkan-pipeline-cache.md new file mode 100644 index 0000000..1ad3a3b --- /dev/null +++ b/cmake/patches/ggml-vulkan-pipeline-cache.md @@ -0,0 +1,58 @@ +# ggml-vulkan-pipeline-cache（game.cpp 自有 patch） + +## Purpose + +Vulkan 冷启动 Currently every process recompiles the whole compute-pipeline +set from SPIR-V on first inference（本地实测首次冷启：Vulkan+Q8 达 42s、 +F32 约 7.5s；NVIDIA 驱动级 shader 缓存能缓解，但 AMD/Intel/其它驱动不能 +保证，且无法随包分发）。 + +This patch persists `VkPipelineCache` to disk — the Vulkan analog of the +Metal binary-archive PSO cache (`cmake/patches/ggml-metal-binary-archive.patch`). +After the first run, subsequent launches (or any process on a compatible +driver) load precompiled PSO bytes instead of recompiling every shader. + +## Change + +`src/ggml-vulkan/ggml-vulkan.cpp`: + +- `vk_device_struct` gains `pipeline_cache` / `pipeline_cache_init` / + `pipeline_cache_path` / `pipeline_cache_dirty`. +- At device init (`ggml_vk_init_pipeline_cache`) load the on-disk cache bytes + into `VkPipelineCacheCreateInfo::pInitialData` (stale/invalid bytes are safe + — the driver returns `VK_INCOMPLETE` and builds a fresh cache), and create + the handle. +- `ggml_vk_create_pipeline_func` passes the cache handle to + `createComputePipeline` and sets `pipeline_cache_dirty = true`. +- On graph-compute completion (and at device teardown as a fallback), + `ggml_vk_save_pipeline_cache_device` flushes `vkGetPipelineCacheData` to + disk, throttled by the dirty flag. + +## Env / control + +- `GGML_VK_PIPELINE_CACHE_PATH` — cache file path. +- default: `%LOCALAPPDATA%\\game_ggml_vk_pipeline.cache` (Windows) or + `$HOME/.cache/game_ggml_vk_pipeline.cache` (POSIX). +- `GGML_VK_DISABLE_PIPELINE_CACHE` — opt out. +- `GGML_VK_PIPELINE_CACHE_DEBUG` — print load/save diagnostics. + +## Baseline & re-apply + +- Applies to **ggml v0.19.0** (`ggml-vulkan.cpp`). Pinned by game.cpp + FetchContent; re-apply per ggml upgrade via `cmake/Dependencies.cmake` + `game_ggml_apply_patch` (idempotent: skips if already applied). + +## Verified (local, RTX 2070, Vulkan) + +- Run1 (no cache): builds from scratch, saves ~1.14 MB. +- Run2 (with cache): logs "loaded 1,140,086 bytes", same note output (33/33), + warm total 0.42s. + +## Notes + +- The dispatch-table C functions (`vkCreatePipelineCache` / + `vkGetPipelineCacheData` / `vkDestroyPipelineCache`) are used because this + SDK's generated `vulkan.hpp` does not expose the corresponding C++ class + methods. +- Devices are cached for the process lifetime in ggml-vulkan, so the save is + driven by graph-compute completion rather than the device destructor. diff --git a/cmake/patches/ggml-vulkan-pipeline-cache.patch b/cmake/patches/ggml-vulkan-pipeline-cache.patch new file mode 100644 index 0000000..8cc2dba --- /dev/null +++ b/cmake/patches/ggml-vulkan-pipeline-cache.patch @@ -0,0 +1,225 @@ +diff --git a/src/ggml-vulkan/ggml-vulkan.cpp b/src/ggml-vulkan/ggml-vulkan.cpp +index a923755..56304ff 100644 +--- a/src/ggml-vulkan/ggml-vulkan.cpp ++++ b/src/ggml-vulkan/ggml-vulkan.cpp +@@ -50,6 +50,7 @@ typedef struct VkPhysicalDeviceCooperativeMatrixDecodeVectorFeaturesNV { + + #include + #include ++#include + #include + #include + #include +@@ -67,6 +68,7 @@ typedef struct VkPhysicalDeviceCooperativeMatrixDecodeVectorFeaturesNV { + #include + #include + #include ++#include + + #if defined(_MSC_VER) + # define NOMINMAX 1 +@@ -792,6 +794,13 @@ struct vk_device_struct { + bool pipeline_robustness; + bool memory_priority; + vk::Device device; ++ // game.cpp: persistent VkPipelineCache (disk-backed, like the Metal ++ // binary-archive PSO cache). Loaded at device init, passed to every ++ // createComputePipeline, flushed back to disk at teardown. ++ vk::PipelineCache pipeline_cache; ++ std::vector pipeline_cache_init; // bytes read from disk (may be empty) ++ std::string pipeline_cache_path; // empty = disabled/unset ++ bool pipeline_cache_dirty = false; // a pipeline was created since last save + uint32_t vendor_id; + vk::DriverId driver_id; + vk_device_architecture architecture; +@@ -1122,6 +1131,48 @@ struct vk_device_struct { + compute_queue.reset(); + transfer_queue.reset(); + ++ // game.cpp: flush the persistent pipeline cache back to disk BEFORE ++ // destroying the individual pipelines (destroying a pipeline evicts its ++ // cached PSOs, so vkGetPipelineCacheData would return empty after the ++ // loop). This SDK's vulkan.hpp does not generate the ++ // vkGetPipelineCacheData/vkDestroyPipelineCache hpp methods, so use the ++ // dispatch-table C functions directly. ++ if (static_cast(pipeline_cache) != VK_NULL_HANDLE) { ++ try { ++ VkDevice dev = static_cast(device); ++ VkPipelineCache pc = static_cast(pipeline_cache); ++ size_t sz = 0; ++ // The size query returns VK_INCOMPLETE (and fills *pDataSize); ++ // accept both VK_SUCCESS and VK_INCOMPLETE here. ++ const VkResult q = ::vkGetPipelineCacheData(dev, pc, &sz, nullptr); ++ if (std::getenv("GGML_VK_PIPELINE_CACHE_DEBUG")) { ++ std::cerr << "ggml_vulkan: pipeline cache query q=" << static_cast(q) ++ << " sz=" << sz << std::endl; ++ } ++ if ((q == VK_SUCCESS || q == VK_INCOMPLETE) && sz > 0) { ++ std::vector data(sz); ++ const VkResult g = ::vkGetPipelineCacheData(dev, pc, &sz, data.data()); ++ if ((g == VK_SUCCESS || g == VK_INCOMPLETE) && !pipeline_cache_path.empty()) { ++ data.resize(std::min(sz, data.size())); ++ std::ofstream f(pipeline_cache_path, std::ios::binary | std::ios::trunc); ++ if (f) { ++ f.write(reinterpret_cast(data.data()), ++ static_cast(data.size())); ++ if (std::getenv("GGML_VK_PIPELINE_CACHE_DEBUG")) { ++ std::cerr << "ggml_vulkan: pipeline cache: saved " << data.size() ++ << " bytes to " << pipeline_cache_path << std::endl; ++ } ++ } ++ } ++ } ++ } catch (const vk::SystemError & e) { ++ std::cerr << "ggml_vulkan: pipeline cache save failed: " << e.what() << std::endl; ++ } ++ VkPipelineCache pc = static_cast(pipeline_cache); ++ ::vkDestroyPipelineCache(static_cast(device), pc, nullptr); ++ pipeline_cache = vk::PipelineCache{}; ++ } ++ + for (auto& pipeline : all_pipelines) { + if (pipeline.expired()) { + continue; +@@ -1138,6 +1189,110 @@ struct vk_device_struct { + } + }; + ++// game.cpp: persistent pipeline cache helpers. ++ // - Path: GGML_VK_PIPELINE_CACHE_PATH or /game_ggml_vk_pipeline.cache ++ // - Disable: GGML_VK_DISABLE_PIPELINE_CACHE ++ // Stale/invalid cache bytes are safe: the driver returns VK_INCOMPLETE and ++ // creates an empty cache, so an outdated/GPU-swapped cache degrades to a ++ // normal cold compile — never a crash. ++ static std::string ggml_vk_pipeline_cache_path() { ++ const char * env = std::getenv("GGML_VK_PIPELINE_CACHE_PATH"); ++ if (env && *env) return std::string(env); ++#ifdef _WIN32 ++ if (const char * la = std::getenv("LOCALAPPDATA")) { ++ return std::string(la) + "\\game_ggml_vk_pipeline.cache"; ++ } ++#else ++ if (const char * home = std::getenv("HOME")) { ++ return std::string(home) + "/.cache/game_ggml_vk_pipeline.cache"; ++ } ++#endif ++ return std::string(); ++ } ++ ++ static void ggml_vk_init_pipeline_cache(vk_device device) { ++ if (std::getenv("GGML_VK_DISABLE_PIPELINE_CACHE")) return; ++ device->pipeline_cache_path = ggml_vk_pipeline_cache_path(); ++ if (device->pipeline_cache_path.empty()) return; ++ ++ std::ifstream f(device->pipeline_cache_path, std::ios::binary | std::ios::ate); ++ if (f.good()) { ++ const std::streamsize sz = f.tellg(); ++ f.seekg(0); ++ device->pipeline_cache_init.resize(static_cast(sz)); ++ if (sz > 0) { ++ f.read(reinterpret_cast(device->pipeline_cache_init.data()), sz); ++ } ++ } ++ ++ VkPipelineCacheCreateInfo ci{}; ++ ci.sType = VK_STRUCTURE_TYPE_PIPELINE_CACHE_CREATE_INFO; ++ if (!device->pipeline_cache_init.empty()) { ++ ci.initialDataSize = device->pipeline_cache_init.size(); ++ ci.pInitialData = device->pipeline_cache_init.data(); ++ } ++ try { ++ VkPipelineCache raw = VK_NULL_HANDLE; ++ if (::vkCreatePipelineCache(static_cast(device->device), &ci, nullptr, &raw) == VK_SUCCESS) { ++ device->pipeline_cache = vk::PipelineCache(raw); ++ if (std::getenv("GGML_VK_PIPELINE_CACHE_DEBUG")) { ++ if (device->pipeline_cache_init.empty()) { ++ std::cerr << "ggml_vulkan: pipeline cache: no cached data, will build from scratch" << std::endl; ++ } else { ++ std::cerr << "ggml_vulkan: pipeline cache: loaded " << device->pipeline_cache_init.size() ++ << " bytes from " << device->pipeline_cache_path << std::endl; ++ } ++ } ++ } else { ++ std::cerr << "ggml_vulkan: pipeline cache create failed, disabling" << std::endl; ++ device->pipeline_cache_path.clear(); ++ } ++ } catch (const vk::SystemError & e) { ++ std::cerr << "ggml_vulkan: pipeline cache create failed, disabling: " ++ << e.what() << std::endl; ++ device->pipeline_cache = vk::PipelineCache{}; ++ device->pipeline_cache_path.clear(); ++ } ++ } ++ ++ // Persist the pipeline cache to disk if any pipeline was created since the ++ // last save. Called on graph-compute completion (the devices are cached ++ // for the process lifetime, so the destructor is not a usable hook in CLI ++ // flows). The save is throttled by the dirty flag: it only fires after a ++ // graph that actually created new pipelines. ++ static void ggml_vk_save_pipeline_cache_device(vk_device device) { ++ if (!device->pipeline_cache_dirty || ++ static_cast(device->pipeline_cache) == VK_NULL_HANDLE || ++ device->pipeline_cache_path.empty()) { ++ return; ++ } ++ device->pipeline_cache_dirty = false; ++ try { ++ VkDevice dev = static_cast(device->device); ++ VkPipelineCache pc = static_cast(device->pipeline_cache); ++ size_t sz = 0; ++ const VkResult q = ::vkGetPipelineCacheData(dev, pc, &sz, nullptr); ++ if ((q == VK_SUCCESS || q == VK_INCOMPLETE) && sz > 0) { ++ std::vector data(sz); ++ const VkResult g = ::vkGetPipelineCacheData(dev, pc, &sz, data.data()); ++ if ((g == VK_SUCCESS || g == VK_INCOMPLETE) && !data.empty()) { ++ data.resize(std::min(sz, data.size())); ++ std::ofstream f(device->pipeline_cache_path, std::ios::binary | std::ios::trunc); ++ if (f) { ++ f.write(reinterpret_cast(data.data()), ++ static_cast(data.size())); ++ if (std::getenv("GGML_VK_PIPELINE_CACHE_DEBUG")) { ++ std::cerr << "ggml_vulkan: pipeline cache: saved " << data.size() ++ << " bytes to " << device->pipeline_cache_path << std::endl; ++ } ++ } ++ } ++ } ++ } catch (const vk::SystemError & e) { ++ std::cerr << "ggml_vulkan: pipeline cache save failed: " << e.what() << std::endl; ++ } ++ } ++ + void vk_command_pool::init(vk_device& device, vk_queue *q_) { + cmd_buffers.clear(); + q = q_; +@@ -3025,7 +3180,8 @@ static void ggml_vk_create_pipeline_func(vk_device& device, vk_pipeline& pipelin + #endif + + try { +- pipeline->pipeline = device->device.createComputePipeline(VK_NULL_HANDLE, compute_pipeline_create_info).value; ++ pipeline->pipeline = device->device.createComputePipeline(device->pipeline_cache, compute_pipeline_create_info).value; ++ device->pipeline_cache_dirty = true; + } catch (const vk::SystemError& e) { + std::cerr << "ggml_vulkan: Compute pipeline creation failed for " << pipeline->name << std::endl; + std::cerr << "ggml_vulkan: " << e.what() << std::endl; +@@ -6910,6 +7066,8 @@ static vk_device ggml_vk_get_device(size_t idx) { + .setPEnabledExtensionNames(device_extensions); + device_create_info.setPNext(&device_features2); + device->device = device->physical_device.createDevice(device_create_info); ++ // game.cpp: load/create a disk-backed VkPipelineCache (cold-start fix). ++ ggml_vk_init_pipeline_cache(device); + + if (device->device_fault) { + device->pfn_vkGetDeviceFaultInfoEXT = (PFN_vkGetDeviceFaultInfoEXT) +@@ -17372,6 +17530,11 @@ static ggml_status ggml_backend_vk_graph_compute(ggml_backend_t backend, ggml_cg + ggml_vk_synchronize(ctx); + } + ++ // game.cpp: persist any newly-created pipelines back to the on-disk ++ // VkPipelineCache (dirty-flag throttled; no-op unless a pipeline was ++ // created since the last save). ++ ggml_vk_save_pipeline_cache_device(ctx->device); ++ + return GGML_STATUS_SUCCESS; + + UNUSED(backend);