From 3966c7e1641078d928a5678130537c01353ad274 Mon Sep 17 00:00:00 2001 From: Marcus Edel Date: Mon, 1 Jun 2026 14:26:34 +0000 Subject: [PATCH 01/58] feat: build llm/embed-llamacpp addons against qvac-fabric PR #144 (b9341). Signed-off-by: Marcus Edel --- packages/embed-llamacpp/CMakeLists.txt | 6 +- packages/embed-llamacpp/vcpkg.json | 2 + .../qvac-fabric/android-vulkan-version.cmake | 36 ++++ .../vcpkg/ports/qvac-fabric/portfile.cmake | 166 ++++++++++++++++++ .../vcpkg/ports/qvac-fabric/vcpkg.json | 37 ++++ packages/llm-llamacpp/CMakeLists.txt | 6 +- .../addon/src/model-interface/LlamaModel.cpp | 74 +++++++- .../addon/src/model-interface/LlmContext.hpp | 34 ++++ .../src/model-interface/MtmdLlmContext.cpp | 3 +- .../src/model-interface/TextLlmContext.cpp | 4 +- .../src/model-interface/TextLlmContext.hpp | 20 +++ .../addon/src/utils/ChatTemplateUtils.cpp | 14 +- .../addon/src/utils/ChatTemplateUtils.hpp | 12 +- packages/llm-llamacpp/vcpkg.json | 2 + .../qvac-fabric/android-vulkan-version.cmake | 36 ++++ .../vcpkg/ports/qvac-fabric/portfile.cmake | 162 +++++++++++++++++ .../vcpkg/ports/qvac-fabric/vcpkg.json | 37 ++++ 17 files changed, 640 insertions(+), 11 deletions(-) create mode 100644 packages/embed-llamacpp/vcpkg/ports/qvac-fabric/android-vulkan-version.cmake create mode 100644 packages/embed-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake create mode 100644 packages/embed-llamacpp/vcpkg/ports/qvac-fabric/vcpkg.json create mode 100644 packages/llm-llamacpp/vcpkg/ports/qvac-fabric/android-vulkan-version.cmake create mode 100644 packages/llm-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake create mode 100644 packages/llm-llamacpp/vcpkg/ports/qvac-fabric/vcpkg.json diff --git a/packages/embed-llamacpp/CMakeLists.txt b/packages/embed-llamacpp/CMakeLists.txt index a1da3e729e..e619ac638b 100644 --- a/packages/embed-llamacpp/CMakeLists.txt +++ b/packages/embed-llamacpp/CMakeLists.txt @@ -17,6 +17,8 @@ find_package(cmake-vcpkg REQUIRED PATHS node_modules/cmake-vcpkg) set(VCPKG_OVERLAY_TRIPLETS "${CMAKE_CURRENT_SOURCE_DIR}/vcpkg/triplets;${VCPKG_OVERLAY_TRIPLETS}") +set(VCPKG_OVERLAY_PORTS "${CMAKE_CURRENT_SOURCE_DIR}/vcpkg/ports;${VCPKG_OVERLAY_PORTS}") + project(embed-llamacpp C CXX) if(CMAKE_SYSTEM_NAME STREQUAL "Linux") @@ -56,7 +58,7 @@ if((ANDROID OR UNIX) AND NOT APPLE) foreach(_backend ${GGML_AVAILABLE_BACKENDS}) list(APPEND BACKEND_DL_LIBS INSTALL TARGET ggml::${_backend}) endforeach() -endif() +endif() add_bare_module(embed-llamacpp EXPORTS ${BACKEND_DL_LIBS}) @@ -99,7 +101,7 @@ if(BUILD_TESTING) find_package(GTest CONFIG REQUIRED) include(GoogleTest) enable_testing() - + add_subdirectory(test/unit) endif() diff --git a/packages/embed-llamacpp/vcpkg.json b/packages/embed-llamacpp/vcpkg.json index 336eb46119..6495e5a828 100644 --- a/packages/embed-llamacpp/vcpkg.json +++ b/packages/embed-llamacpp/vcpkg.json @@ -1,4 +1,6 @@ { + "name": "qvac-embed-llamacpp", + "version": "0.16.0", "dependencies": [ { "name": "opencl", diff --git a/packages/embed-llamacpp/vcpkg/ports/qvac-fabric/android-vulkan-version.cmake b/packages/embed-llamacpp/vcpkg/ports/qvac-fabric/android-vulkan-version.cmake new file mode 100644 index 0000000000..485ea51bb9 --- /dev/null +++ b/packages/embed-llamacpp/vcpkg/ports/qvac-fabric/android-vulkan-version.cmake @@ -0,0 +1,36 @@ +# Function to detect Vulkan version from NDK vulkan_core.h +function(detect_ndk_vulkan_version) + string(TOLOWER "${CMAKE_HOST_SYSTEM_NAME}" host_system_name_lower) + + # CMAKE_HOST_SYSTEM_PROCESSOR is unavailable here. Use a glob pattern to complete the folder instead. + file(GLOB host_dirs LIST_DIRECTORIES true "$ENV{ANDROID_NDK_HOME}/toolchains/llvm/prebuilt/${host_system_name_lower}-*") + if(host_dirs) + list(GET host_dirs 0 host_dir) + get_filename_component(host_arch "${host_dir}" NAME) + set(vulkan_core_h "$ENV{ANDROID_NDK_HOME}/toolchains/llvm/prebuilt/${host_arch}/sysroot/usr/include/vulkan/vulkan_core.h") + else() + message(FATAL "Could not find NDK host directory for ${host_system_name_lower}") + endif() + + if(NOT vulkan_core_h) + message(FATAL "vulkan_core.h not found, using default version") + endif() + + file(READ "${vulkan_core_h}" header_content) + string(REGEX MATCH "VK_HEADER_VERSION ([0-9]+)" version_match "${header_content}") + if(version_match) + set(header_version_3 "${CMAKE_MATCH_1}") + else() + message(FATAL "Could not extract VK_HEADER_VERSION from vulkan_core.h, using default: ${vulkan_version}") + endif() + + # Extract major.minor version from VK_HEADER_VERSION_COMPLETE for download URL + string(REGEX MATCH "VK_HEADER_VERSION_COMPLETE VK_MAKE_API_VERSION\\(([0-9]+), ([0-9]+), ([0-9]+)" version_match "${header_content}") + if(version_match) + set(major "${CMAKE_MATCH_2}") + set(minor "${CMAKE_MATCH_3}") + set(vulkan_version "${major}.${minor}.${header_version_3}" PARENT_SCOPE) + else() + message(FATAL "Could not extract major.minor version from vulkan_core.h, using default: ${vulkan_version}") + endif() +endfunction() diff --git a/packages/embed-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake b/packages/embed-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake new file mode 100644 index 0000000000..9d168bdca3 --- /dev/null +++ b/packages/embed-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake @@ -0,0 +1,166 @@ +vcpkg_from_github( + OUT_SOURCE_PATH SOURCE_PATH + REPO zoq/qvac-ext-lib-llama.cpp + REF bc85668ffad0394d4973d9e08eb8373871188d02 + SHA512 0ae829a9d62af533d3803a5a67388fe2d3b6dc39a9b0c80a64f893253a37378365c7b3712ad6f6e1216ee63625f12d0e5b26e2cfaea0f2da1a35e65cb3e17b27 +) + +vcpkg_check_features( + OUT_FEATURE_OPTIONS FEATURE_OPTIONS + FEATURES + force-profiler FORCE_GGML_VK_PERF_LOGGER + llama BUILD_LLAMA +) + +set(_qvac_gpu_backends OFF) +if("gpu-backends" IN_LIST FEATURES) + set(_qvac_gpu_backends ON) +else() + message(STATUS "qvac-fabric: gpu-backends feature OFF — building CPU-only ggml (no Metal/Vulkan/CUDA/OpenCL)") +endif() + +if (VCPKG_TARGET_IS_ANDROID AND _qvac_gpu_backends) + # NDK only comes with C headers. Pull Vulkan and SPIR-V headers from + # Khronos and drop them under ggml/src/ggml-vulkan/vulkan_cpp_wrapper/include, + # which ggml-vulkan/CMakeLists.txt adds to the include path. Idempotent — + # safe to re-run against an existing local SOURCE_PATH checkout. + include(${CMAKE_CURRENT_LIST_DIR}/android-vulkan-version.cmake) + detect_ndk_vulkan_version() + message(STATUS "Using Vulkan C++ wrappers from version: ${vulkan_version}") + + set(_vk_wrapper "${SOURCE_PATH}/ggml/src/ggml-vulkan/vulkan_cpp_wrapper") + if(NOT EXISTS "${_vk_wrapper}/include/vulkan/vulkan.hpp") + file(REMOVE_RECURSE "${_vk_wrapper}" "${SOURCE_PATH}/Vulkan-Headers-${vulkan_version}") + file(DOWNLOAD + "https://github.com/KhronosGroup/Vulkan-Headers/archive/refs/tags/v${vulkan_version}.tar.gz" + "${SOURCE_PATH}/vulkan-sdk-${vulkan_version}.tar.gz" + TLS_VERIFY ON + ) + file(ARCHIVE_EXTRACT + INPUT "${SOURCE_PATH}/vulkan-sdk-${vulkan_version}.tar.gz" + DESTINATION "${SOURCE_PATH}" + PATTERNS "*.hpp" + ) + file(RENAME + "${SOURCE_PATH}/Vulkan-Headers-${vulkan_version}" + "${_vk_wrapper}" + ) + file(REMOVE "${SOURCE_PATH}/vulkan-sdk-${vulkan_version}.tar.gz") + endif() + + set(_spv_version "1.3.290.0") + if(NOT EXISTS "${_vk_wrapper}/include/spirv/unified1/spirv.hpp") + file(REMOVE_RECURSE "${SOURCE_PATH}/SPIRV-Headers-vulkan-sdk-${_spv_version}") + file(DOWNLOAD + "https://github.com/KhronosGroup/SPIRV-Headers/archive/refs/tags/vulkan-sdk-${_spv_version}.tar.gz" + "${SOURCE_PATH}/spirv-headers-${_spv_version}.tar.gz" + TLS_VERIFY ON + ) + file(ARCHIVE_EXTRACT + INPUT "${SOURCE_PATH}/spirv-headers-${_spv_version}.tar.gz" + DESTINATION "${SOURCE_PATH}" + ) + file(COPY "${SOURCE_PATH}/SPIRV-Headers-vulkan-sdk-${_spv_version}/include/spirv" + DESTINATION "${_vk_wrapper}/include") + file(REMOVE_RECURSE "${SOURCE_PATH}/SPIRV-Headers-vulkan-sdk-${_spv_version}") + file(REMOVE "${SOURCE_PATH}/spirv-headers-${_spv_version}.tar.gz") + endif() +endif() + +set(PLATFORM_OPTIONS) + +if(NOT _qvac_gpu_backends) + # Force every GPU backend off explicitly, in case upstream defaults change. + list(APPEND PLATFORM_OPTIONS + -DGGML_METAL=OFF + -DGGML_VULKAN=OFF + -DGGML_CUDA=OFF + -DGGML_OPENCL=OFF + ) + if (VCPKG_TARGET_IS_IOS) + # Same iOS BLAS/Accelerate gating as the GPU-on path; unrelated to the + # CPU-vs-GPU split, an iOS-toolchain workaround for missing frameworks. + list(APPEND PLATFORM_OPTIONS -DGGML_BLAS=OFF -DGGML_ACCELERATE=OFF) + endif() +elseif (VCPKG_TARGET_IS_OSX OR VCPKG_TARGET_IS_IOS) + list(APPEND PLATFORM_OPTIONS -DGGML_METAL=ON) + if (VCPKG_TARGET_IS_IOS) + list(APPEND PLATFORM_OPTIONS -DGGML_BLAS=OFF -DGGML_ACCELERATE=OFF) + endif() +else() + list(APPEND PLATFORM_OPTIONS -DGGML_VULKAN=ON) +endif() + +if(VCPKG_TARGET_IS_ANDROID AND _qvac_gpu_backends) + set(DL_BACKENDS ON) + list(APPEND PLATFORM_OPTIONS + -DGGML_BACKEND_DL=ON + -DGGML_CPU_ALL_VARIANTS=ON + -DGGML_CPU_REPACK=ON + -DGGML_OPENCL=ON + ) +else() + set(DL_BACKENDS OFF) +endif() + +set(LLAMA_OPTIONS) +if("llama" IN_LIST FEATURES) + list(APPEND LLAMA_OPTIONS -DLLAMA_MTMD=ON) +else() + list(APPEND LLAMA_OPTIONS + -DLLAMA_MTMD=OFF + -DLLAMA_BUILD_COMMON=OFF + ) +endif() + +vcpkg_cmake_configure( + SOURCE_PATH "${SOURCE_PATH}" + DISABLE_PARALLEL_CONFIGURE + OPTIONS + -DGGML_NATIVE=OFF + -DGGML_CCACHE=OFF + -DGGML_OPENMP=OFF + -DGGML_LLAMAFILE=OFF + -DLLAMA_CURL=OFF + -DLLAMA_BUILD_TESTS=OFF + -DLLAMA_BUILD_TOOLS=OFF + -DLLAMA_BUILD_EXAMPLES=OFF + -DLLAMA_BUILD_SERVER=OFF + -DLLAMA_ALL_WARNINGS=OFF + ${LLAMA_OPTIONS} + ${PLATFORM_OPTIONS} + ${FEATURE_OPTIONS} +) + +vcpkg_cmake_install() +vcpkg_cmake_config_fixup( + PACKAGE_NAME ggml) + +if(BUILD_LLAMA) + vcpkg_cmake_config_fixup(PACKAGE_NAME llama) +endif() + +vcpkg_copy_pdbs() +vcpkg_fixup_pkgconfig() + + +if(BUILD_LLAMA) + file(MAKE_DIRECTORY "${CURRENT_PACKAGES_DIR}/tools/${PORT}") + file(RENAME "${CURRENT_PACKAGES_DIR}/bin/convert_hf_to_gguf.py" "${CURRENT_PACKAGES_DIR}/tools/${PORT}/convert-hf-to-gguf.py") + file(INSTALL "${SOURCE_PATH}/gguf-py" DESTINATION "${CURRENT_PACKAGES_DIR}/tools/${PORT}") + file(RENAME "${CURRENT_PACKAGES_DIR}/bin/vulkan_profiling_analyzer.py" "${CURRENT_PACKAGES_DIR}/tools/${PORT}/vulkan_profiling_analyzer.py") +endif() + +if (NOT VCPKG_BUILD_TYPE) + file(REMOVE "${CURRENT_PACKAGES_DIR}/debug/bin/convert_hf_to_gguf.py") +endif() + +file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/include") +file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/share") + +if (NOT DL_BACKENDS AND VCPKG_LIBRARY_LINKAGE MATCHES "static") + file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/bin") + file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/bin") +endif() + +vcpkg_install_copyright(FILE_LIST "${SOURCE_PATH}/LICENSE") diff --git a/packages/embed-llamacpp/vcpkg/ports/qvac-fabric/vcpkg.json b/packages/embed-llamacpp/vcpkg/ports/qvac-fabric/vcpkg.json new file mode 100644 index 0000000000..52f2192918 --- /dev/null +++ b/packages/embed-llamacpp/vcpkg/ports/qvac-fabric/vcpkg.json @@ -0,0 +1,37 @@ +{ + "name": "qvac-fabric", + "version": "8189.0.2", + "port-version": 4, + "description": "LLM inference in C/C++", + "homepage": "https://github.com/tetherto/qvac-fabric-llm.cpp", + "license": "MIT", + "dependencies": [ + { + "name": "opencl", + "platform": "android" + }, + { + "name": "vcpkg-cmake", + "host": true + }, + { + "name": "vcpkg-cmake-config", + "host": true + } + ], + "default-features": [ + "gpu-backends", + "llama" + ], + "features": { + "force-profiler": { + "description": "Force vk performance logging in ggml" + }, + "gpu-backends": { + "description": "Build the GPU backends ggml ships per platform: Metal on Apple, Vulkan on Linux/Windows/Android, plus the Android backend-DL hybrid mode and OpenCL. Default-on so existing consumers (llamacpp-llm, llamacpp-embed, nmtcpp, diffusion-cpp) keep their current behaviour with default features. Disable to produce a CPU-only ggml build (useful for consumers like @qvac/classification-ggml that don't need GPU paths and want to skip the vulkan-sdk / metal / opencl build cost). Orthogonal to the 'llama' feature." + }, + "llama": { + "description": "Build llama components." + } + } +} diff --git a/packages/llm-llamacpp/CMakeLists.txt b/packages/llm-llamacpp/CMakeLists.txt index 5236cf2f7c..0f1884248f 100644 --- a/packages/llm-llamacpp/CMakeLists.txt +++ b/packages/llm-llamacpp/CMakeLists.txt @@ -17,6 +17,8 @@ find_package(cmake-vcpkg REQUIRED PATHS node_modules/cmake-vcpkg) set(VCPKG_OVERLAY_TRIPLETS "${CMAKE_CURRENT_SOURCE_DIR}/vcpkg/triplets;${VCPKG_OVERLAY_TRIPLETS}") +set(VCPKG_OVERLAY_PORTS "${CMAKE_CURRENT_SOURCE_DIR}/vcpkg/ports;${VCPKG_OVERLAY_PORTS}") + project(inference-addon-llama C CXX) if(CMAKE_SYSTEM_NAME STREQUAL "Linux") @@ -163,7 +165,7 @@ if(BUILD_CLI) target_include_directories(cli_tool PRIVATE ${QVAC_LIB_INFERENCE_ADDON_CPP_INCLUDE_DIRS}) # Set C++20 standard for std::views support target_compile_features(cli_tool PRIVATE cxx_std_20) - + # Install ggml backends next to the CLI tool binary in build folder if((ANDROID OR UNIX) AND NOT APPLE) foreach(_backend ${GGML_AVAILABLE_BACKENDS}) @@ -181,7 +183,7 @@ if(BUILD_TESTING) find_package(GTest CONFIG REQUIRED) include(GoogleTest) enable_testing() - + # Integration tests for model classes (includes backend selection tests) # Pass ENABLE_COVERAGE option to test subdirectory add_subdirectory(test/unit) diff --git a/packages/llm-llamacpp/addon/src/model-interface/LlamaModel.cpp b/packages/llm-llamacpp/addon/src/model-interface/LlamaModel.cpp index ac072c59a5..9eb3e72a53 100644 --- a/packages/llm-llamacpp/addon/src/model-interface/LlamaModel.cpp +++ b/packages/llm-llamacpp/addon/src/model-interface/LlamaModel.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -650,6 +651,75 @@ std::string LlamaModel::processPromptImpl(const Prompt& prompt) { out = oss.str(); } + // Re-wrap any extracted tool calls as {...} envelopes + // so JS-side tests and SDK consumers see a uniform shape regardless of which + // template/dialect the model used. Skip models that already emit standard + // envelopes natively (e.g. Qwen3): re-wrapping their parsed + // calls would append a duplicate envelope after the one already in the raw + // output. Gemma 4's dialect uses <|tool_call>/ (no literal + // "" substring), so it is unaffected by this guard and still gets + // re-wrapped. + const std::string assistantText = prompt.outputCallback ? oss.str() : out; + if (!resolved.tools.empty() && + assistantText.find("") == std::string::npos) { + common_chat_parser_params syntax; + syntax.format = state_->llmContext_->getLastChatFormat(); + syntax.parse_tool_calls = true; + syntax.generation_prompt = state_->llmContext_->getLastGenerationPrompt(); + // The fabric's chat layer builds a dialect-specific PEG parser at + // templating time and serializes it into common_chat_params::parser. + // Without loading it here, common_chat_parse falls back to a pure-content + // parser and extracts zero tool calls (e.g. Gemma 4's + // <|tool_call>call:NAME{...} dialect goes unparsed). + const std::string& serializedParser = + state_->llmContext_->getLastChatParser(); + if (!serializedParser.empty()) { + syntax.parser.load(serializedParser); + } + try { + common_chat_msg parsed = common_chat_parse( + assistantText, /*is_partial=*/false, syntax); + if (!parsed.tool_calls.empty()) { + const auto normalizeArgsJson = + [](const std::string& src) -> std::string { + static const std::regex unquotedKeyRe( + R"(([{,])(\s*)([A-Za-z_][A-Za-z0-9_]*)(\s*):)", + std::regex::ECMAScript); + return std::regex_replace(src, unquotedKeyRe, R"($1$2"$3"$4:)"); + }; + std::string envelopes; + envelopes.reserve(parsed.tool_calls.size() * 64); + for (const auto& tc : parsed.tool_calls) { + std::string args = tc.arguments.empty() + ? std::string("{}") + : normalizeArgsJson(tc.arguments); + envelopes.append("{\"name\":\""); + envelopes.append(tc.name); + envelopes.append("\",\"arguments\":"); + envelopes.append(args); + envelopes.append("}"); + } + if (prompt.outputCallback) { + prompt.outputCallback(envelopes); + } else { + out.append(envelopes); + } + } + } catch (const std::exception& e) { + QLOG_IF( + Priority::WARNING, + string_format( + "[LlamaModel] common_chat_parse failed for tool extraction: " + "%s\n", + e.what())); + } catch (...) { + QLOG_IF( + Priority::WARNING, + "[LlamaModel] common_chat_parse threw an unknown exception during " + "tool extraction\n"); + } + } + // Post-generation tools trim decision via controller std::string ossStr; if (needsOutputCapture && prompt.outputCallback) { @@ -1097,9 +1167,9 @@ void LlamaModel::commonParamsParse( postprocess_cpu_params(params.cpuparams, nullptr); postprocess_cpu_params(params.cpuparams_batch, ¶ms.cpuparams); - postprocess_cpu_params(params.speculative.cpuparams, ¶ms.cpuparams); + postprocess_cpu_params(params.speculative.draft.cpuparams, ¶ms.cpuparams); postprocess_cpu_params( - params.speculative.cpuparams_batch, ¶ms.cpuparams_batch); + params.speculative.draft.cpuparams_batch, ¶ms.cpuparams_batch); if (!params.kv_overrides.empty()) { params.kv_overrides.emplace_back(); diff --git a/packages/llm-llamacpp/addon/src/model-interface/LlmContext.hpp b/packages/llm-llamacpp/addon/src/model-interface/LlmContext.hpp index 8bbe45008c..c4ad83fe48 100644 --- a/packages/llm-llamacpp/addon/src/model-interface/LlmContext.hpp +++ b/packages/llm-llamacpp/addon/src/model-interface/LlmContext.hpp @@ -294,4 +294,38 @@ class LlmContext { // NOLINT(cppcoreguidelines-special-member-functions) * */ virtual void resetMedia() {}; + + /** + * Returns the chat format selected by the most recent prompt template + * application (e.g. COMMON_CHAT_FORMAT_GEMMA4). Used by post-generation + * tool-call extraction to tell common_chat_parse which dialect the + * assistant text is in. + */ + [[nodiscard]] virtual common_chat_format getLastChatFormat() const { + return COMMON_CHAT_FORMAT_CONTENT_ONLY; + } + + /** + * Returns the serialized PEG parser produced by the most recent prompt + * template application. The fabric's chat layer builds a dialect-specific + * parser at templating time and stores it (serialized) in + * common_chat_params::parser; common_chat_parse needs it loaded into the + * parser arena, otherwise it silently falls back to a pure-content parser + * and extracts zero tool calls. Empty when no parser was produced. + */ + [[nodiscard]] virtual const std::string& getLastChatParser() const { + static const std::string kEmpty; + return kEmpty; + } + + /** + * Returns the generation prompt from the most recent prompt template + * application. common_chat_parse prepends it to the assistant text so the + * parser's `start` rule (which bakes in the generation-prompt prefix at + * build time) aligns with the input. + */ + [[nodiscard]] virtual const std::string& getLastGenerationPrompt() const { + static const std::string kEmpty; + return kEmpty; + } }; diff --git a/packages/llm-llamacpp/addon/src/model-interface/MtmdLlmContext.cpp b/packages/llm-llamacpp/addon/src/model-interface/MtmdLlmContext.cpp index 2c93da52ee..2c3c6e3183 100644 --- a/packages/llm-llamacpp/addon/src/model-interface/MtmdLlmContext.cpp +++ b/packages/llm-llamacpp/addon/src/model-interface/MtmdLlmContext.cpp @@ -209,7 +209,8 @@ void MtmdLlmContext::tokenizeChat( if (!tools.empty()) { inputs.tools = tools; } - formattedChat = getPrompt(tmpls_.get(), inputs, &thinkingForcedOpen_); + formattedChat = getPrompt(tmpls_.get(), inputs, + /*outFormat=*/nullptr, &thinkingForcedOpen_); if (formattedChat.empty()) { std::string errorMsg = string_format( diff --git a/packages/llm-llamacpp/addon/src/model-interface/TextLlmContext.cpp b/packages/llm-llamacpp/addon/src/model-interface/TextLlmContext.cpp index bd71d2e43a..f4b86d10a9 100644 --- a/packages/llm-llamacpp/addon/src/model-interface/TextLlmContext.cpp +++ b/packages/llm-llamacpp/addon/src/model-interface/TextLlmContext.cpp @@ -248,7 +248,9 @@ void TextLlmContext::tokenizeChat( if (!tools.empty()) { inputs.tools = tools; } - prompt = getPrompt(tmpls_.get(), inputs, &thinkingForcedOpen_); + prompt = getPrompt( + tmpls_.get(), inputs, &lastChatFormat_, &thinkingForcedOpen_, + &lastChatParser_, &lastGenerationPrompt_); QLOG_IF( Priority::DEBUG, diff --git a/packages/llm-llamacpp/addon/src/model-interface/TextLlmContext.hpp b/packages/llm-llamacpp/addon/src/model-interface/TextLlmContext.hpp index d68fed570c..28ca2f0a68 100644 --- a/packages/llm-llamacpp/addon/src/model-interface/TextLlmContext.hpp +++ b/packages/llm-llamacpp/addon/src/model-interface/TextLlmContext.hpp @@ -141,6 +141,18 @@ class TextLlmContext : public LlmContext { */ llama_pos removeLastNTokens(llama_pos count) override; + [[nodiscard]] common_chat_format getLastChatFormat() const override { + return lastChatFormat_; + } + + [[nodiscard]] const std::string& getLastChatParser() const override { + return lastChatParser_; + } + + [[nodiscard]] const std::string& getLastGenerationPrompt() const override { + return lastGenerationPrompt_; + } + private: /** * The check antiprompt method. It checks the antiprompt. @@ -207,5 +219,13 @@ class TextLlmContext : public LlmContext { // tags. bool thinkingForcedOpen_ = false; + common_chat_format lastChatFormat_ = COMMON_CHAT_FORMAT_CONTENT_ONLY; + + // Serialized PEG parser and generation prompt from the most recent prompt + // template application, needed by post-generation tool-call extraction (see + // getLastChatParser / getLastGenerationPrompt). + std::string lastChatParser_; + std::string lastGenerationPrompt_; + std::atomic stopGeneration_ = false; }; diff --git a/packages/llm-llamacpp/addon/src/utils/ChatTemplateUtils.cpp b/packages/llm-llamacpp/addon/src/utils/ChatTemplateUtils.cpp index 3307822224..044e447a9d 100644 --- a/packages/llm-llamacpp/addon/src/utils/ChatTemplateUtils.cpp +++ b/packages/llm-llamacpp/addon/src/utils/ChatTemplateUtils.cpp @@ -174,12 +174,22 @@ std::string getChatTemplate( std::string getPrompt( const struct common_chat_templates* tmpls, - struct common_chat_templates_inputs& inputs, - bool* outThinkingForcedOpen) { + struct common_chat_templates_inputs& inputs, common_chat_format* outFormat, + bool* outThinkingForcedOpen, std::string* outParser, + std::string* outGenerationPrompt) { auto exportParams = [&](const common_chat_params& params) { + if (outFormat) { + *outFormat = params.format; + } if (outThinkingForcedOpen) { *outThinkingForcedOpen = params.thinking_forced_open; } + if (outParser) { + *outParser = params.parser; + } + if (outGenerationPrompt) { + *outGenerationPrompt = params.generation_prompt; + } }; try { auto params = common_chat_templates_apply(tmpls, inputs); diff --git a/packages/llm-llamacpp/addon/src/utils/ChatTemplateUtils.hpp b/packages/llm-llamacpp/addon/src/utils/ChatTemplateUtils.hpp index b523e51b98..1d4cc481e1 100644 --- a/packages/llm-llamacpp/addon/src/utils/ChatTemplateUtils.hpp +++ b/packages/llm-llamacpp/addon/src/utils/ChatTemplateUtils.hpp @@ -69,13 +69,23 @@ std::string getChatTemplate( * @brief Applies chat templates to generate a prompt, with fallback handling * for models that don't support tools. * + * @p outFormat (optional) receives the parsed chat format identifier. * @p outThinkingForcedOpen (optional) receives the flag indicating that the * template force-opened the reasoning channel in the prompt suffix. + * @p outParser (optional) receives the serialized PEG parser + * (common_chat_params::parser) needed for post-generation tool-call + * extraction. + * @p outGenerationPrompt (optional) receives the generation prompt + * (common_chat_params::generation_prompt) that the parser expects to be + * prepended to the assistant text. */ std::string getPrompt( const struct common_chat_templates* tmpls, struct common_chat_templates_inputs& inputs, - bool* outThinkingForcedOpen = nullptr); + common_chat_format* outFormat = nullptr, + bool* outThinkingForcedOpen = nullptr, + std::string* outParser = nullptr, + std::string* outGenerationPrompt = nullptr); } // namespace utils } // namespace qvac_lib_inference_addon_llama diff --git a/packages/llm-llamacpp/vcpkg.json b/packages/llm-llamacpp/vcpkg.json index f39bcfcc07..4390f2e7c5 100644 --- a/packages/llm-llamacpp/vcpkg.json +++ b/packages/llm-llamacpp/vcpkg.json @@ -1,4 +1,6 @@ { + "name": "qvac-llm-llamacpp", + "version": "0.20.0", "dependencies": [ { "name": "opencl", diff --git a/packages/llm-llamacpp/vcpkg/ports/qvac-fabric/android-vulkan-version.cmake b/packages/llm-llamacpp/vcpkg/ports/qvac-fabric/android-vulkan-version.cmake new file mode 100644 index 0000000000..485ea51bb9 --- /dev/null +++ b/packages/llm-llamacpp/vcpkg/ports/qvac-fabric/android-vulkan-version.cmake @@ -0,0 +1,36 @@ +# Function to detect Vulkan version from NDK vulkan_core.h +function(detect_ndk_vulkan_version) + string(TOLOWER "${CMAKE_HOST_SYSTEM_NAME}" host_system_name_lower) + + # CMAKE_HOST_SYSTEM_PROCESSOR is unavailable here. Use a glob pattern to complete the folder instead. + file(GLOB host_dirs LIST_DIRECTORIES true "$ENV{ANDROID_NDK_HOME}/toolchains/llvm/prebuilt/${host_system_name_lower}-*") + if(host_dirs) + list(GET host_dirs 0 host_dir) + get_filename_component(host_arch "${host_dir}" NAME) + set(vulkan_core_h "$ENV{ANDROID_NDK_HOME}/toolchains/llvm/prebuilt/${host_arch}/sysroot/usr/include/vulkan/vulkan_core.h") + else() + message(FATAL "Could not find NDK host directory for ${host_system_name_lower}") + endif() + + if(NOT vulkan_core_h) + message(FATAL "vulkan_core.h not found, using default version") + endif() + + file(READ "${vulkan_core_h}" header_content) + string(REGEX MATCH "VK_HEADER_VERSION ([0-9]+)" version_match "${header_content}") + if(version_match) + set(header_version_3 "${CMAKE_MATCH_1}") + else() + message(FATAL "Could not extract VK_HEADER_VERSION from vulkan_core.h, using default: ${vulkan_version}") + endif() + + # Extract major.minor version from VK_HEADER_VERSION_COMPLETE for download URL + string(REGEX MATCH "VK_HEADER_VERSION_COMPLETE VK_MAKE_API_VERSION\\(([0-9]+), ([0-9]+), ([0-9]+)" version_match "${header_content}") + if(version_match) + set(major "${CMAKE_MATCH_2}") + set(minor "${CMAKE_MATCH_3}") + set(vulkan_version "${major}.${minor}.${header_version_3}" PARENT_SCOPE) + else() + message(FATAL "Could not extract major.minor version from vulkan_core.h, using default: ${vulkan_version}") + endif() +endfunction() diff --git a/packages/llm-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake b/packages/llm-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake new file mode 100644 index 0000000000..f40088a333 --- /dev/null +++ b/packages/llm-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake @@ -0,0 +1,162 @@ +vcpkg_from_github( + OUT_SOURCE_PATH SOURCE_PATH + REPO zoq/qvac-ext-lib-llama.cpp + REF bc85668ffad0394d4973d9e08eb8373871188d02 + SHA512 0ae829a9d62af533d3803a5a67388fe2d3b6dc39a9b0c80a64f893253a37378365c7b3712ad6f6e1216ee63625f12d0e5b26e2cfaea0f2da1a35e65cb3e17b27 +) + +vcpkg_check_features( + OUT_FEATURE_OPTIONS FEATURE_OPTIONS + FEATURES + force-profiler FORCE_GGML_VK_PERF_LOGGER + llama BUILD_LLAMA +) + +set(_qvac_gpu_backends OFF) +if("gpu-backends" IN_LIST FEATURES) + set(_qvac_gpu_backends ON) +else() + message(STATUS "qvac-fabric: gpu-backends feature OFF — building CPU-only ggml (no Metal/Vulkan/CUDA/OpenCL)") +endif() + +if (VCPKG_TARGET_IS_ANDROID AND _qvac_gpu_backends) + include(${CMAKE_CURRENT_LIST_DIR}/android-vulkan-version.cmake) + detect_ndk_vulkan_version() + message(STATUS "Using Vulkan C++ wrappers from version: ${vulkan_version}") + + set(_vk_wrapper "${SOURCE_PATH}/ggml/src/ggml-vulkan/vulkan_cpp_wrapper") + if(NOT EXISTS "${_vk_wrapper}/include/vulkan/vulkan.hpp") + file(REMOVE_RECURSE "${_vk_wrapper}" "${SOURCE_PATH}/Vulkan-Headers-${vulkan_version}") + file(DOWNLOAD + "https://github.com/KhronosGroup/Vulkan-Headers/archive/refs/tags/v${vulkan_version}.tar.gz" + "${SOURCE_PATH}/vulkan-sdk-${vulkan_version}.tar.gz" + TLS_VERIFY ON + ) + file(ARCHIVE_EXTRACT + INPUT "${SOURCE_PATH}/vulkan-sdk-${vulkan_version}.tar.gz" + DESTINATION "${SOURCE_PATH}" + PATTERNS "*.hpp" + ) + file(RENAME + "${SOURCE_PATH}/Vulkan-Headers-${vulkan_version}" + "${_vk_wrapper}" + ) + file(REMOVE "${SOURCE_PATH}/vulkan-sdk-${vulkan_version}.tar.gz") + endif() + + set(_spv_version "1.3.290.0") + if(NOT EXISTS "${_vk_wrapper}/include/spirv/unified1/spirv.hpp") + file(REMOVE_RECURSE "${SOURCE_PATH}/SPIRV-Headers-vulkan-sdk-${_spv_version}") + file(DOWNLOAD + "https://github.com/KhronosGroup/SPIRV-Headers/archive/refs/tags/vulkan-sdk-${_spv_version}.tar.gz" + "${SOURCE_PATH}/spirv-headers-${_spv_version}.tar.gz" + TLS_VERIFY ON + ) + file(ARCHIVE_EXTRACT + INPUT "${SOURCE_PATH}/spirv-headers-${_spv_version}.tar.gz" + DESTINATION "${SOURCE_PATH}" + ) + file(COPY "${SOURCE_PATH}/SPIRV-Headers-vulkan-sdk-${_spv_version}/include/spirv" + DESTINATION "${_vk_wrapper}/include") + file(REMOVE_RECURSE "${SOURCE_PATH}/SPIRV-Headers-vulkan-sdk-${_spv_version}") + file(REMOVE "${SOURCE_PATH}/spirv-headers-${_spv_version}.tar.gz") + endif() +endif() + +set(PLATFORM_OPTIONS) + +if(NOT _qvac_gpu_backends) + # Force every GPU backend off explicitly, in case upstream defaults change. + list(APPEND PLATFORM_OPTIONS + -DGGML_METAL=OFF + -DGGML_VULKAN=OFF + -DGGML_CUDA=OFF + -DGGML_OPENCL=OFF + ) + if (VCPKG_TARGET_IS_IOS) + # Same iOS BLAS/Accelerate gating as the GPU-on path; unrelated to the + # CPU-vs-GPU split, an iOS-toolchain workaround for missing frameworks. + list(APPEND PLATFORM_OPTIONS -DGGML_BLAS=OFF -DGGML_ACCELERATE=OFF) + endif() +elseif (VCPKG_TARGET_IS_OSX OR VCPKG_TARGET_IS_IOS) + list(APPEND PLATFORM_OPTIONS -DGGML_METAL=ON) + if (VCPKG_TARGET_IS_IOS) + list(APPEND PLATFORM_OPTIONS -DGGML_BLAS=OFF -DGGML_ACCELERATE=OFF) + endif() +else() + list(APPEND PLATFORM_OPTIONS -DGGML_VULKAN=ON) +endif() + +if(VCPKG_TARGET_IS_ANDROID AND _qvac_gpu_backends) + set(DL_BACKENDS ON) + list(APPEND PLATFORM_OPTIONS + -DGGML_BACKEND_DL=ON + -DGGML_CPU_ALL_VARIANTS=ON + -DGGML_CPU_REPACK=ON + -DGGML_OPENCL=ON + ) +else() + set(DL_BACKENDS OFF) +endif() + +set(LLAMA_OPTIONS) +if("llama" IN_LIST FEATURES) + list(APPEND LLAMA_OPTIONS -DLLAMA_MTMD=ON) +else() + list(APPEND LLAMA_OPTIONS + -DLLAMA_MTMD=OFF + -DLLAMA_BUILD_COMMON=OFF + ) +endif() + +vcpkg_cmake_configure( + SOURCE_PATH "${SOURCE_PATH}" + DISABLE_PARALLEL_CONFIGURE + OPTIONS + -DGGML_NATIVE=OFF + -DGGML_CCACHE=OFF + -DGGML_OPENMP=OFF + -DGGML_LLAMAFILE=OFF + -DLLAMA_CURL=OFF + -DLLAMA_BUILD_TESTS=OFF + -DLLAMA_BUILD_TOOLS=OFF + -DLLAMA_BUILD_EXAMPLES=OFF + -DLLAMA_BUILD_SERVER=OFF + -DLLAMA_ALL_WARNINGS=OFF + ${LLAMA_OPTIONS} + ${PLATFORM_OPTIONS} + ${FEATURE_OPTIONS} +) + +vcpkg_cmake_install() +vcpkg_cmake_config_fixup( + PACKAGE_NAME ggml) + +if(BUILD_LLAMA) + vcpkg_cmake_config_fixup(PACKAGE_NAME llama) +endif() + +vcpkg_copy_pdbs() +vcpkg_fixup_pkgconfig() + + +if(BUILD_LLAMA) + file(MAKE_DIRECTORY "${CURRENT_PACKAGES_DIR}/tools/${PORT}") + file(RENAME "${CURRENT_PACKAGES_DIR}/bin/convert_hf_to_gguf.py" "${CURRENT_PACKAGES_DIR}/tools/${PORT}/convert-hf-to-gguf.py") + file(INSTALL "${SOURCE_PATH}/gguf-py" DESTINATION "${CURRENT_PACKAGES_DIR}/tools/${PORT}") + file(RENAME "${CURRENT_PACKAGES_DIR}/bin/vulkan_profiling_analyzer.py" "${CURRENT_PACKAGES_DIR}/tools/${PORT}/vulkan_profiling_analyzer.py") +endif() + +if (NOT VCPKG_BUILD_TYPE) + file(REMOVE "${CURRENT_PACKAGES_DIR}/debug/bin/convert_hf_to_gguf.py") +endif() + +file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/include") +file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/share") + +if (NOT DL_BACKENDS AND VCPKG_LIBRARY_LINKAGE MATCHES "static") + file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/bin") + file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/bin") +endif() + +vcpkg_install_copyright(FILE_LIST "${SOURCE_PATH}/LICENSE") diff --git a/packages/llm-llamacpp/vcpkg/ports/qvac-fabric/vcpkg.json b/packages/llm-llamacpp/vcpkg/ports/qvac-fabric/vcpkg.json new file mode 100644 index 0000000000..74629f3288 --- /dev/null +++ b/packages/llm-llamacpp/vcpkg/ports/qvac-fabric/vcpkg.json @@ -0,0 +1,37 @@ +{ + "name": "qvac-fabric", + "version": "8189.0.2", + "port-version": 11, + "description": "LLM inference in C/C++", + "homepage": "https://github.com/tetherto/qvac-fabric-llm.cpp", + "license": "MIT", + "dependencies": [ + { + "name": "opencl", + "platform": "android" + }, + { + "name": "vcpkg-cmake", + "host": true + }, + { + "name": "vcpkg-cmake-config", + "host": true + } + ], + "default-features": [ + "gpu-backends", + "llama" + ], + "features": { + "force-profiler": { + "description": "Force vk performance logging in ggml" + }, + "gpu-backends": { + "description": "Build the GPU backends ggml ships per platform: Metal on Apple, Vulkan on Linux/Windows/Android, plus the Android backend-DL hybrid mode and OpenCL. Default-on so existing consumers (llamacpp-llm, llamacpp-embed, nmtcpp, diffusion-cpp) keep their current behaviour with default features. Disable to produce a CPU-only ggml build (useful for consumers like @qvac/classification-ggml that don't need GPU paths and want to skip the vulkan-sdk / metal / opencl build cost). Orthogonal to the 'llama' feature." + }, + "llama": { + "description": "Build llama components." + } + } +} From ce1b9646daac57eee0c67a9c98921ddcc8193a91 Mon Sep 17 00:00:00 2001 From: Marcus Edel Date: Mon, 1 Jun 2026 14:44:46 +0000 Subject: [PATCH 02/58] build: update to the latest portfiles. Signed-off-by: Marcus Edel --- .../vcpkg/ports/qvac-fabric/portfile.cmake | 118 ++++++++++-------- .../vcpkg/ports/qvac-fabric/vcpkg.json | 15 ++- .../vcpkg/ports/qvac-fabric/portfile.cmake | 114 ++++++++++------- .../vcpkg/ports/qvac-fabric/vcpkg.json | 15 ++- 4 files changed, 158 insertions(+), 104 deletions(-) diff --git a/packages/embed-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake b/packages/embed-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake index 9d168bdca3..17c60b33df 100644 --- a/packages/embed-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake +++ b/packages/embed-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake @@ -5,6 +5,7 @@ vcpkg_from_github( SHA512 0ae829a9d62af533d3803a5a67388fe2d3b6dc39a9b0c80a64f893253a37378365c7b3712ad6f6e1216ee63625f12d0e5b26e2cfaea0f2da1a35e65cb3e17b27 ) +# Upstream CMake options only — passed through to vcpkg_cmake_configure. vcpkg_check_features( OUT_FEATURE_OPTIONS FEATURE_OPTIONS FEATURES @@ -12,64 +13,49 @@ vcpkg_check_features( llama BUILD_LLAMA ) -set(_qvac_gpu_backends OFF) -if("gpu-backends" IN_LIST FEATURES) - set(_qvac_gpu_backends ON) -else() +# Portfile-only feature flags (drive PLATFORM_OPTIONS; not upstream cache vars). +vcpkg_check_features( + OUT_FEATURE_OPTIONS _PORTFILE_FEATURE_OPTIONS + FEATURES + gpu-backends BUILD_GPU_BACKENDS + kleidiai BUILD_KLEIDIAI +) + +# gpu-backends is default-on via default-features in vcpkg.json. CPU-only +# consumers (e.g. @qvac/classification-ggml) disable it with +# default-features:false (and re-add 'llama' if needed). +if(NOT BUILD_GPU_BACKENDS) message(STATUS "qvac-fabric: gpu-backends feature OFF — building CPU-only ggml (no Metal/Vulkan/CUDA/OpenCL)") endif() -if (VCPKG_TARGET_IS_ANDROID AND _qvac_gpu_backends) - # NDK only comes with C headers. Pull Vulkan and SPIR-V headers from - # Khronos and drop them under ggml/src/ggml-vulkan/vulkan_cpp_wrapper/include, - # which ggml-vulkan/CMakeLists.txt adds to the include path. Idempotent — - # safe to re-run against an existing local SOURCE_PATH checkout. +if (VCPKG_TARGET_IS_ANDROID AND BUILD_GPU_BACKENDS) + # NDK only comes with C headers. + # Make sure C++ header exists, it will be used by ggml tensor library. + # Need to determine installed vulkan version and download correct headers include(${CMAKE_CURRENT_LIST_DIR}/android-vulkan-version.cmake) detect_ndk_vulkan_version() message(STATUS "Using Vulkan C++ wrappers from version: ${vulkan_version}") + file(DOWNLOAD + "https://github.com/KhronosGroup/Vulkan-Headers/archive/refs/tags/v${vulkan_version}.tar.gz" + "${SOURCE_PATH}/vulkan-sdk-${vulkan_version}.tar.gz" + TLS_VERIFY ON + ) - set(_vk_wrapper "${SOURCE_PATH}/ggml/src/ggml-vulkan/vulkan_cpp_wrapper") - if(NOT EXISTS "${_vk_wrapper}/include/vulkan/vulkan.hpp") - file(REMOVE_RECURSE "${_vk_wrapper}" "${SOURCE_PATH}/Vulkan-Headers-${vulkan_version}") - file(DOWNLOAD - "https://github.com/KhronosGroup/Vulkan-Headers/archive/refs/tags/v${vulkan_version}.tar.gz" - "${SOURCE_PATH}/vulkan-sdk-${vulkan_version}.tar.gz" - TLS_VERIFY ON - ) - file(ARCHIVE_EXTRACT - INPUT "${SOURCE_PATH}/vulkan-sdk-${vulkan_version}.tar.gz" - DESTINATION "${SOURCE_PATH}" - PATTERNS "*.hpp" - ) - file(RENAME - "${SOURCE_PATH}/Vulkan-Headers-${vulkan_version}" - "${_vk_wrapper}" - ) - file(REMOVE "${SOURCE_PATH}/vulkan-sdk-${vulkan_version}.tar.gz") - endif() + file(ARCHIVE_EXTRACT + INPUT "${SOURCE_PATH}/vulkan-sdk-${vulkan_version}.tar.gz" + DESTINATION "${SOURCE_PATH}" + PATTERNS "*.hpp" + ) - set(_spv_version "1.3.290.0") - if(NOT EXISTS "${_vk_wrapper}/include/spirv/unified1/spirv.hpp") - file(REMOVE_RECURSE "${SOURCE_PATH}/SPIRV-Headers-vulkan-sdk-${_spv_version}") - file(DOWNLOAD - "https://github.com/KhronosGroup/SPIRV-Headers/archive/refs/tags/vulkan-sdk-${_spv_version}.tar.gz" - "${SOURCE_PATH}/spirv-headers-${_spv_version}.tar.gz" - TLS_VERIFY ON - ) - file(ARCHIVE_EXTRACT - INPUT "${SOURCE_PATH}/spirv-headers-${_spv_version}.tar.gz" - DESTINATION "${SOURCE_PATH}" - ) - file(COPY "${SOURCE_PATH}/SPIRV-Headers-vulkan-sdk-${_spv_version}/include/spirv" - DESTINATION "${_vk_wrapper}/include") - file(REMOVE_RECURSE "${SOURCE_PATH}/SPIRV-Headers-vulkan-sdk-${_spv_version}") - file(REMOVE "${SOURCE_PATH}/spirv-headers-${_spv_version}.tar.gz") - endif() + file(RENAME + "${SOURCE_PATH}/Vulkan-Headers-${vulkan_version}" + "${SOURCE_PATH}/ggml/src/ggml-vulkan/vulkan_cpp_wrapper" + ) endif() set(PLATFORM_OPTIONS) -if(NOT _qvac_gpu_backends) +if(NOT BUILD_GPU_BACKENDS) # Force every GPU backend off explicitly, in case upstream defaults change. list(APPEND PLATFORM_OPTIONS -DGGML_METAL=OFF @@ -91,18 +77,48 @@ else() list(APPEND PLATFORM_OPTIONS -DGGML_VULKAN=ON) endif() -if(VCPKG_TARGET_IS_ANDROID AND _qvac_gpu_backends) +# Android: always build CPU variants (NEON_DOTPROD, NEON_I8MM, etc.) and CPU +# repacking. These are CPU-only runtime optimizations selected based on the +# device's SIMD capabilities at load time, completely orthogonal to the GPU +# backends. Bundling them is essential for good CPU inference performance on +# the wide range of arm64 devices the addons ship to. Requires GGML_BACKEND_DL +# to dispatch the variants at runtime; the existing #ifdef guard around +# `ggml_backend_load_all_from_path()` in ggml-backend-reg.cpp keeps the search +# scoped to the consumer's own prebuilds dir. +if(VCPKG_TARGET_IS_ANDROID) set(DL_BACKENDS ON) list(APPEND PLATFORM_OPTIONS -DGGML_BACKEND_DL=ON -DGGML_CPU_ALL_VARIANTS=ON - -DGGML_CPU_REPACK=ON - -DGGML_OPENCL=ON - ) + -DGGML_CPU_REPACK=ON) else() set(DL_BACKENDS OFF) endif() +if(VCPKG_TARGET_IS_ANDROID AND BUILD_KLEIDIAI) + message(STATUS "qvac-fabric: kleidiai feature ON — building with ARM KleidiAI optimized kernels") + # ggml only vendors KleidiAI via FetchContent; registry vcpkg-cmake sets + # FETCHCONTENT_FULLY_DISCONNECTED=ON globally, so allow the download here. + list(APPEND PLATFORM_OPTIONS + -DGGML_CPU_KLEIDIAI=ON + -DFETCHCONTENT_FULLY_DISCONNECTED=OFF + ) +endif() + +if (VCPKG_TARGET_IS_ANDROID AND BUILD_GPU_BACKENDS) + list(APPEND PLATFORM_OPTIONS -DGGML_OPENCL=ON) +endif() + +if(BUILD_GPU_BACKENDS AND NOT VCPKG_TARGET_IS_OSX AND NOT VCPKG_TARGET_IS_IOS) + if(VCPKG_TARGET_IS_WINDOWS AND NOT VCPKG_TARGET_IS_MINGW) + string(APPEND VCPKG_C_FLAGS " /I${CURRENT_INSTALLED_DIR}/include") + string(APPEND VCPKG_CXX_FLAGS " /I${CURRENT_INSTALLED_DIR}/include") + else() + string(APPEND VCPKG_C_FLAGS " -isystem ${CURRENT_INSTALLED_DIR}/include") + string(APPEND VCPKG_CXX_FLAGS " -isystem ${CURRENT_INSTALLED_DIR}/include") + endif() +endif() + set(LLAMA_OPTIONS) if("llama" IN_LIST FEATURES) list(APPEND LLAMA_OPTIONS -DLLAMA_MTMD=ON) @@ -158,7 +174,7 @@ endif() file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/include") file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/share") -if (NOT DL_BACKENDS AND VCPKG_LIBRARY_LINKAGE MATCHES "static") +if (VCPKG_LIBRARY_LINKAGE MATCHES "static") file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/bin") file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/bin") endif() diff --git a/packages/embed-llamacpp/vcpkg/ports/qvac-fabric/vcpkg.json b/packages/embed-llamacpp/vcpkg/ports/qvac-fabric/vcpkg.json index 52f2192918..4c0c4fe5ef 100644 --- a/packages/embed-llamacpp/vcpkg/ports/qvac-fabric/vcpkg.json +++ b/packages/embed-llamacpp/vcpkg/ports/qvac-fabric/vcpkg.json @@ -1,7 +1,6 @@ { "name": "qvac-fabric", - "version": "8189.0.2", - "port-version": 4, + "version": "8828.0.2", "description": "LLM inference in C/C++", "homepage": "https://github.com/tetherto/qvac-fabric-llm.cpp", "license": "MIT", @@ -28,7 +27,17 @@ "description": "Force vk performance logging in ggml" }, "gpu-backends": { - "description": "Build the GPU backends ggml ships per platform: Metal on Apple, Vulkan on Linux/Windows/Android, plus the Android backend-DL hybrid mode and OpenCL. Default-on so existing consumers (llamacpp-llm, llamacpp-embed, nmtcpp, diffusion-cpp) keep their current behaviour with default features. Disable to produce a CPU-only ggml build (useful for consumers like @qvac/classification-ggml that don't need GPU paths and want to skip the vulkan-sdk / metal / opencl build cost). Orthogonal to the 'llama' feature." + "description": "Build the GPU backends ggml ships per platform: Metal on Apple, Vulkan on Linux/Windows/Android, plus the Android backend-DL hybrid mode and OpenCL. Default-on so existing consumers (llamacpp-llm, llamacpp-embed, nmtcpp, diffusion-cpp) keep their current behaviour with default features. Disable to produce a CPU-only ggml build (useful for consumers like @qvac/classification-ggml that don't need GPU paths and want to skip the vulkan-sdk / metal / opencl build cost). Orthogonal to the 'llama' feature.", + "dependencies": [ + { + "name": "spirv-headers", + "platform": "!osx & !ios", + "version>=": "1.4.341.0" + } + ] + }, + "kleidiai": { + "description": "Enable ARM KleidiAI optimized kernels on Android." }, "llama": { "description": "Build llama components." diff --git a/packages/llm-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake b/packages/llm-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake index f40088a333..17c60b33df 100644 --- a/packages/llm-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake +++ b/packages/llm-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake @@ -5,6 +5,7 @@ vcpkg_from_github( SHA512 0ae829a9d62af533d3803a5a67388fe2d3b6dc39a9b0c80a64f893253a37378365c7b3712ad6f6e1216ee63625f12d0e5b26e2cfaea0f2da1a35e65cb3e17b27 ) +# Upstream CMake options only — passed through to vcpkg_cmake_configure. vcpkg_check_features( OUT_FEATURE_OPTIONS FEATURE_OPTIONS FEATURES @@ -12,60 +13,49 @@ vcpkg_check_features( llama BUILD_LLAMA ) -set(_qvac_gpu_backends OFF) -if("gpu-backends" IN_LIST FEATURES) - set(_qvac_gpu_backends ON) -else() +# Portfile-only feature flags (drive PLATFORM_OPTIONS; not upstream cache vars). +vcpkg_check_features( + OUT_FEATURE_OPTIONS _PORTFILE_FEATURE_OPTIONS + FEATURES + gpu-backends BUILD_GPU_BACKENDS + kleidiai BUILD_KLEIDIAI +) + +# gpu-backends is default-on via default-features in vcpkg.json. CPU-only +# consumers (e.g. @qvac/classification-ggml) disable it with +# default-features:false (and re-add 'llama' if needed). +if(NOT BUILD_GPU_BACKENDS) message(STATUS "qvac-fabric: gpu-backends feature OFF — building CPU-only ggml (no Metal/Vulkan/CUDA/OpenCL)") endif() -if (VCPKG_TARGET_IS_ANDROID AND _qvac_gpu_backends) +if (VCPKG_TARGET_IS_ANDROID AND BUILD_GPU_BACKENDS) + # NDK only comes with C headers. + # Make sure C++ header exists, it will be used by ggml tensor library. + # Need to determine installed vulkan version and download correct headers include(${CMAKE_CURRENT_LIST_DIR}/android-vulkan-version.cmake) detect_ndk_vulkan_version() message(STATUS "Using Vulkan C++ wrappers from version: ${vulkan_version}") + file(DOWNLOAD + "https://github.com/KhronosGroup/Vulkan-Headers/archive/refs/tags/v${vulkan_version}.tar.gz" + "${SOURCE_PATH}/vulkan-sdk-${vulkan_version}.tar.gz" + TLS_VERIFY ON + ) - set(_vk_wrapper "${SOURCE_PATH}/ggml/src/ggml-vulkan/vulkan_cpp_wrapper") - if(NOT EXISTS "${_vk_wrapper}/include/vulkan/vulkan.hpp") - file(REMOVE_RECURSE "${_vk_wrapper}" "${SOURCE_PATH}/Vulkan-Headers-${vulkan_version}") - file(DOWNLOAD - "https://github.com/KhronosGroup/Vulkan-Headers/archive/refs/tags/v${vulkan_version}.tar.gz" - "${SOURCE_PATH}/vulkan-sdk-${vulkan_version}.tar.gz" - TLS_VERIFY ON - ) - file(ARCHIVE_EXTRACT - INPUT "${SOURCE_PATH}/vulkan-sdk-${vulkan_version}.tar.gz" - DESTINATION "${SOURCE_PATH}" - PATTERNS "*.hpp" - ) - file(RENAME - "${SOURCE_PATH}/Vulkan-Headers-${vulkan_version}" - "${_vk_wrapper}" - ) - file(REMOVE "${SOURCE_PATH}/vulkan-sdk-${vulkan_version}.tar.gz") - endif() + file(ARCHIVE_EXTRACT + INPUT "${SOURCE_PATH}/vulkan-sdk-${vulkan_version}.tar.gz" + DESTINATION "${SOURCE_PATH}" + PATTERNS "*.hpp" + ) - set(_spv_version "1.3.290.0") - if(NOT EXISTS "${_vk_wrapper}/include/spirv/unified1/spirv.hpp") - file(REMOVE_RECURSE "${SOURCE_PATH}/SPIRV-Headers-vulkan-sdk-${_spv_version}") - file(DOWNLOAD - "https://github.com/KhronosGroup/SPIRV-Headers/archive/refs/tags/vulkan-sdk-${_spv_version}.tar.gz" - "${SOURCE_PATH}/spirv-headers-${_spv_version}.tar.gz" - TLS_VERIFY ON - ) - file(ARCHIVE_EXTRACT - INPUT "${SOURCE_PATH}/spirv-headers-${_spv_version}.tar.gz" - DESTINATION "${SOURCE_PATH}" - ) - file(COPY "${SOURCE_PATH}/SPIRV-Headers-vulkan-sdk-${_spv_version}/include/spirv" - DESTINATION "${_vk_wrapper}/include") - file(REMOVE_RECURSE "${SOURCE_PATH}/SPIRV-Headers-vulkan-sdk-${_spv_version}") - file(REMOVE "${SOURCE_PATH}/spirv-headers-${_spv_version}.tar.gz") - endif() + file(RENAME + "${SOURCE_PATH}/Vulkan-Headers-${vulkan_version}" + "${SOURCE_PATH}/ggml/src/ggml-vulkan/vulkan_cpp_wrapper" + ) endif() set(PLATFORM_OPTIONS) -if(NOT _qvac_gpu_backends) +if(NOT BUILD_GPU_BACKENDS) # Force every GPU backend off explicitly, in case upstream defaults change. list(APPEND PLATFORM_OPTIONS -DGGML_METAL=OFF @@ -87,18 +77,48 @@ else() list(APPEND PLATFORM_OPTIONS -DGGML_VULKAN=ON) endif() -if(VCPKG_TARGET_IS_ANDROID AND _qvac_gpu_backends) +# Android: always build CPU variants (NEON_DOTPROD, NEON_I8MM, etc.) and CPU +# repacking. These are CPU-only runtime optimizations selected based on the +# device's SIMD capabilities at load time, completely orthogonal to the GPU +# backends. Bundling them is essential for good CPU inference performance on +# the wide range of arm64 devices the addons ship to. Requires GGML_BACKEND_DL +# to dispatch the variants at runtime; the existing #ifdef guard around +# `ggml_backend_load_all_from_path()` in ggml-backend-reg.cpp keeps the search +# scoped to the consumer's own prebuilds dir. +if(VCPKG_TARGET_IS_ANDROID) set(DL_BACKENDS ON) list(APPEND PLATFORM_OPTIONS -DGGML_BACKEND_DL=ON -DGGML_CPU_ALL_VARIANTS=ON - -DGGML_CPU_REPACK=ON - -DGGML_OPENCL=ON - ) + -DGGML_CPU_REPACK=ON) else() set(DL_BACKENDS OFF) endif() +if(VCPKG_TARGET_IS_ANDROID AND BUILD_KLEIDIAI) + message(STATUS "qvac-fabric: kleidiai feature ON — building with ARM KleidiAI optimized kernels") + # ggml only vendors KleidiAI via FetchContent; registry vcpkg-cmake sets + # FETCHCONTENT_FULLY_DISCONNECTED=ON globally, so allow the download here. + list(APPEND PLATFORM_OPTIONS + -DGGML_CPU_KLEIDIAI=ON + -DFETCHCONTENT_FULLY_DISCONNECTED=OFF + ) +endif() + +if (VCPKG_TARGET_IS_ANDROID AND BUILD_GPU_BACKENDS) + list(APPEND PLATFORM_OPTIONS -DGGML_OPENCL=ON) +endif() + +if(BUILD_GPU_BACKENDS AND NOT VCPKG_TARGET_IS_OSX AND NOT VCPKG_TARGET_IS_IOS) + if(VCPKG_TARGET_IS_WINDOWS AND NOT VCPKG_TARGET_IS_MINGW) + string(APPEND VCPKG_C_FLAGS " /I${CURRENT_INSTALLED_DIR}/include") + string(APPEND VCPKG_CXX_FLAGS " /I${CURRENT_INSTALLED_DIR}/include") + else() + string(APPEND VCPKG_C_FLAGS " -isystem ${CURRENT_INSTALLED_DIR}/include") + string(APPEND VCPKG_CXX_FLAGS " -isystem ${CURRENT_INSTALLED_DIR}/include") + endif() +endif() + set(LLAMA_OPTIONS) if("llama" IN_LIST FEATURES) list(APPEND LLAMA_OPTIONS -DLLAMA_MTMD=ON) @@ -154,7 +174,7 @@ endif() file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/include") file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/share") -if (NOT DL_BACKENDS AND VCPKG_LIBRARY_LINKAGE MATCHES "static") +if (VCPKG_LIBRARY_LINKAGE MATCHES "static") file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/bin") file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/bin") endif() diff --git a/packages/llm-llamacpp/vcpkg/ports/qvac-fabric/vcpkg.json b/packages/llm-llamacpp/vcpkg/ports/qvac-fabric/vcpkg.json index 74629f3288..4c0c4fe5ef 100644 --- a/packages/llm-llamacpp/vcpkg/ports/qvac-fabric/vcpkg.json +++ b/packages/llm-llamacpp/vcpkg/ports/qvac-fabric/vcpkg.json @@ -1,7 +1,6 @@ { "name": "qvac-fabric", - "version": "8189.0.2", - "port-version": 11, + "version": "8828.0.2", "description": "LLM inference in C/C++", "homepage": "https://github.com/tetherto/qvac-fabric-llm.cpp", "license": "MIT", @@ -28,7 +27,17 @@ "description": "Force vk performance logging in ggml" }, "gpu-backends": { - "description": "Build the GPU backends ggml ships per platform: Metal on Apple, Vulkan on Linux/Windows/Android, plus the Android backend-DL hybrid mode and OpenCL. Default-on so existing consumers (llamacpp-llm, llamacpp-embed, nmtcpp, diffusion-cpp) keep their current behaviour with default features. Disable to produce a CPU-only ggml build (useful for consumers like @qvac/classification-ggml that don't need GPU paths and want to skip the vulkan-sdk / metal / opencl build cost). Orthogonal to the 'llama' feature." + "description": "Build the GPU backends ggml ships per platform: Metal on Apple, Vulkan on Linux/Windows/Android, plus the Android backend-DL hybrid mode and OpenCL. Default-on so existing consumers (llamacpp-llm, llamacpp-embed, nmtcpp, diffusion-cpp) keep their current behaviour with default features. Disable to produce a CPU-only ggml build (useful for consumers like @qvac/classification-ggml that don't need GPU paths and want to skip the vulkan-sdk / metal / opencl build cost). Orthogonal to the 'llama' feature.", + "dependencies": [ + { + "name": "spirv-headers", + "platform": "!osx & !ios", + "version>=": "1.4.341.0" + } + ] + }, + "kleidiai": { + "description": "Enable ARM KleidiAI optimized kernels on Android." }, "llama": { "description": "Build llama components." From bc793717536e5daf2e69a633e669cf7e3a45a232 Mon Sep 17 00:00:00 2001 From: Marcus Edel Date: Mon, 1 Jun 2026 15:04:15 +0000 Subject: [PATCH 03/58] build: point to the correct fabric repo. Signed-off-by: Marcus Edel --- packages/embed-llamacpp/CMakeLists.txt | 2 -- packages/embed-llamacpp/vcpkg-configuration.json | 3 +++ packages/embed-llamacpp/vcpkg.json | 2 -- .../embed-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake | 4 ++-- packages/llm-llamacpp/CMakeLists.txt | 2 -- packages/llm-llamacpp/vcpkg-configuration.json | 3 +++ packages/llm-llamacpp/vcpkg.json | 2 -- packages/llm-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake | 4 ++-- 8 files changed, 10 insertions(+), 12 deletions(-) diff --git a/packages/embed-llamacpp/CMakeLists.txt b/packages/embed-llamacpp/CMakeLists.txt index e619ac638b..0e82061701 100644 --- a/packages/embed-llamacpp/CMakeLists.txt +++ b/packages/embed-llamacpp/CMakeLists.txt @@ -17,8 +17,6 @@ find_package(cmake-vcpkg REQUIRED PATHS node_modules/cmake-vcpkg) set(VCPKG_OVERLAY_TRIPLETS "${CMAKE_CURRENT_SOURCE_DIR}/vcpkg/triplets;${VCPKG_OVERLAY_TRIPLETS}") -set(VCPKG_OVERLAY_PORTS "${CMAKE_CURRENT_SOURCE_DIR}/vcpkg/ports;${VCPKG_OVERLAY_PORTS}") - project(embed-llamacpp C CXX) if(CMAKE_SYSTEM_NAME STREQUAL "Linux") diff --git a/packages/embed-llamacpp/vcpkg-configuration.json b/packages/embed-llamacpp/vcpkg-configuration.json index e894485aaa..37d72f7c86 100644 --- a/packages/embed-llamacpp/vcpkg-configuration.json +++ b/packages/embed-llamacpp/vcpkg-configuration.json @@ -4,6 +4,9 @@ "baseline": "803c0d119ea002694963e89237c207ff6ecf47f6", "repository": "https://github.com/tetherto/qvac-registry-vcpkg.git" }, + "overlay-ports": [ + "vcpkg/ports" + ], "registries": [ { "kind": "git", diff --git a/packages/embed-llamacpp/vcpkg.json b/packages/embed-llamacpp/vcpkg.json index 6495e5a828..336eb46119 100644 --- a/packages/embed-llamacpp/vcpkg.json +++ b/packages/embed-llamacpp/vcpkg.json @@ -1,6 +1,4 @@ { - "name": "qvac-embed-llamacpp", - "version": "0.16.0", "dependencies": [ { "name": "opencl", diff --git a/packages/embed-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake b/packages/embed-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake index 17c60b33df..8024dc3b82 100644 --- a/packages/embed-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake +++ b/packages/embed-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake @@ -1,8 +1,8 @@ vcpkg_from_github( OUT_SOURCE_PATH SOURCE_PATH - REPO zoq/qvac-ext-lib-llama.cpp + REPO tetherto/qvac-fabric-llm.cpp REF bc85668ffad0394d4973d9e08eb8373871188d02 - SHA512 0ae829a9d62af533d3803a5a67388fe2d3b6dc39a9b0c80a64f893253a37378365c7b3712ad6f6e1216ee63625f12d0e5b26e2cfaea0f2da1a35e65cb3e17b27 + SHA512 8e07123e6169ed44d942495ab98f884f407b22b598b5035e0b47bdc372118f63a8fa03beecf6e36cd6482bbae877d8963813f85341f19b285882939c1c27ccee ) # Upstream CMake options only — passed through to vcpkg_cmake_configure. diff --git a/packages/llm-llamacpp/CMakeLists.txt b/packages/llm-llamacpp/CMakeLists.txt index 0f1884248f..060e774293 100644 --- a/packages/llm-llamacpp/CMakeLists.txt +++ b/packages/llm-llamacpp/CMakeLists.txt @@ -17,8 +17,6 @@ find_package(cmake-vcpkg REQUIRED PATHS node_modules/cmake-vcpkg) set(VCPKG_OVERLAY_TRIPLETS "${CMAKE_CURRENT_SOURCE_DIR}/vcpkg/triplets;${VCPKG_OVERLAY_TRIPLETS}") -set(VCPKG_OVERLAY_PORTS "${CMAKE_CURRENT_SOURCE_DIR}/vcpkg/ports;${VCPKG_OVERLAY_PORTS}") - project(inference-addon-llama C CXX) if(CMAKE_SYSTEM_NAME STREQUAL "Linux") diff --git a/packages/llm-llamacpp/vcpkg-configuration.json b/packages/llm-llamacpp/vcpkg-configuration.json index 5eab11ff24..0b52e8c01e 100644 --- a/packages/llm-llamacpp/vcpkg-configuration.json +++ b/packages/llm-llamacpp/vcpkg-configuration.json @@ -4,6 +4,9 @@ "baseline": "c57eec31bdc37ce3a98537b312f5a366b554db6a", "repository": "https://github.com/tetherto/qvac-registry-vcpkg.git" }, + "overlay-ports": [ + "vcpkg/ports" + ], "registries": [ { "kind": "git", diff --git a/packages/llm-llamacpp/vcpkg.json b/packages/llm-llamacpp/vcpkg.json index 4390f2e7c5..f39bcfcc07 100644 --- a/packages/llm-llamacpp/vcpkg.json +++ b/packages/llm-llamacpp/vcpkg.json @@ -1,6 +1,4 @@ { - "name": "qvac-llm-llamacpp", - "version": "0.20.0", "dependencies": [ { "name": "opencl", diff --git a/packages/llm-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake b/packages/llm-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake index 17c60b33df..8024dc3b82 100644 --- a/packages/llm-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake +++ b/packages/llm-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake @@ -1,8 +1,8 @@ vcpkg_from_github( OUT_SOURCE_PATH SOURCE_PATH - REPO zoq/qvac-ext-lib-llama.cpp + REPO tetherto/qvac-fabric-llm.cpp REF bc85668ffad0394d4973d9e08eb8373871188d02 - SHA512 0ae829a9d62af533d3803a5a67388fe2d3b6dc39a9b0c80a64f893253a37378365c7b3712ad6f6e1216ee63625f12d0e5b26e2cfaea0f2da1a35e65cb3e17b27 + SHA512 8e07123e6169ed44d942495ab98f884f407b22b598b5035e0b47bdc372118f63a8fa03beecf6e36cd6482bbae877d8963813f85341f19b285882939c1c27ccee ) # Upstream CMake options only — passed through to vcpkg_cmake_configure. From e70bc6fa67cec104175767a2951f0e643abc31be Mon Sep 17 00:00:00 2001 From: Marcus Edel Date: Mon, 1 Jun 2026 20:24:57 +0000 Subject: [PATCH 04/58] feat: wire MTP draft+verify+accept loop in llm-llamacpp addon. Signed-off-by: Marcus Edel --- .../integration-test-llm-llamacpp.yml | 21 +- .../vcpkg/ports/qvac-fabric/portfile.cmake | 9 +- .../addon/src/model-interface/LlamaModel.cpp | 11 + .../src/model-interface/TextLlmContext.cpp | 234 +++++++++++++++++- .../src/model-interface/TextLlmContext.hpp | 26 ++ .../llm-llamacpp/test/integration/mtp.test.js | 111 +++++++++ .../test/mobile/integration.auto.cjs | 5 + .../llm-llamacpp/test/mobile/test-groups.json | 4 +- .../vcpkg/ports/qvac-fabric/portfile.cmake | 9 +- 9 files changed, 410 insertions(+), 20 deletions(-) create mode 100644 packages/llm-llamacpp/test/integration/mtp.test.js diff --git a/.github/workflows/integration-test-llm-llamacpp.yml b/.github/workflows/integration-test-llm-llamacpp.yml index 1f7b318c94..6f6173d200 100644 --- a/.github/workflows/integration-test-llm-llamacpp.yml +++ b/.github/workflows/integration-test-llm-llamacpp.yml @@ -271,6 +271,12 @@ jobs: if: ${{ matrix.platform != 'win32' }} working-directory: ${{ env.WORKDIR }} run: | + set +e + # Stream test output AND capture it so we can inspect the TAP summary + # if the process exits with a non-zero code. Some configurations + # (observed on linux-arm64) crash with SIGSEGV (exit code 139) during + # static-destructor teardown AFTER all tests have passed. Tolerate + # that specific case, but surface any real failure. if [ "${{ inputs.qvac_perf_only }}" = "true" ]; then echo "qvac_perf_only=true: regenerating test/integration/all.js with perf-emitting tests only" npx brittle -r test/integration/all.js \ @@ -280,10 +286,19 @@ jobs: test/integration/image-fruit-plate.test.js \ test/integration/image-high-res-aurora.test.js bare test/integration/all.js --exit 2>&1 | tee test-output.log - exit ${PIPESTATUS[0]} + EXIT=${PIPESTATUS[0]} + else + npm run test:integration 2>&1 | tee test-output.log + EXIT=${PIPESTATUS[0]} + fi + if [ "$EXIT" -eq 0 ]; then + exit 0 + fi + if [ "$EXIT" -eq 139 ] && grep -qE '^# ok$' test-output.log && grep -qE '^# tests = [0-9]+/[0-9]+ pass$' test-output.log; then + echo "::warning::Tests all passed but process crashed at teardown with SIGSEGV (exit 139) on $(uname -srvmo). Known ggml/Vulkan static-destructor issue; treating as pass." + exit 0 fi - npm run test:integration 2>&1 | tee test-output.log - exit ${PIPESTATUS[0]} + exit "$EXIT" shell: bash env: QASE_API_TOKEN: ${{ secrets.QASE_API_TOKEN }} diff --git a/packages/embed-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake b/packages/embed-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake index 8024dc3b82..f0fe353cbe 100644 --- a/packages/embed-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake +++ b/packages/embed-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake @@ -1,8 +1,9 @@ vcpkg_from_github( - OUT_SOURCE_PATH SOURCE_PATH - REPO tetherto/qvac-fabric-llm.cpp - REF bc85668ffad0394d4973d9e08eb8373871188d02 - SHA512 8e07123e6169ed44d942495ab98f884f407b22b598b5035e0b47bdc372118f63a8fa03beecf6e36cd6482bbae877d8963813f85341f19b285882939c1c27ccee + OUT_SOURCE_PATH SOURCE_PATH + REPO tetherto/qvac-fabric-llm.cpp + REF 97165a1a79650db28ec6a7d965b5905defcd5a23 + SHA512 92f0c54f648c1c30a879e31a5de934511aeb7da97ba0bdeb68caca8bff4ad43388b997288eccfdabb97f6a4c549a7f14f004688c52c99a8477108b2888132a73 + HEAD_REF qvac-b9341 ) # Upstream CMake options only — passed through to vcpkg_cmake_configure. diff --git a/packages/llm-llamacpp/addon/src/model-interface/LlamaModel.cpp b/packages/llm-llamacpp/addon/src/model-interface/LlamaModel.cpp index 9eb3e72a53..57e0913e52 100644 --- a/packages/llm-llamacpp/addon/src/model-interface/LlamaModel.cpp +++ b/packages/llm-llamacpp/addon/src/model-interface/LlamaModel.cpp @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include @@ -1061,6 +1062,16 @@ void LlamaModel::commonParamsParse( } } + for (const std::string& key : {"spec-type", "spec_type"}) { + if (auto iter = configFilemap.find(key); iter != configFilemap.end()) { + auto types = common_speculative_types_from_names( + split(iter->second, ',')); + params.speculative.types.insert( + params.speculative.types.end(), types.begin(), types.end()); + configFilemap.erase(iter); + } + } + // transform json config into the format required by llama.cpp for (auto& keyValuePair : configFilemap) { configVector.push_back(std::string("--") + keyValuePair.first); diff --git a/packages/llm-llamacpp/addon/src/model-interface/TextLlmContext.cpp b/packages/llm-llamacpp/addon/src/model-interface/TextLlmContext.cpp index f4b86d10a9..5caeac585d 100644 --- a/packages/llm-llamacpp/addon/src/model-interface/TextLlmContext.cpp +++ b/packages/llm-llamacpp/addon/src/model-interface/TextLlmContext.cpp @@ -13,6 +13,7 @@ #include "addon/LlmErrors.hpp" #include "common/common.h" #include "common/log.h" +#include "common/speculative.h" #include "inference-addon-cpp/Logger.hpp" #include "utils/ChatTemplateUtils.hpp" #include "utils/LoggingMacros.hpp" @@ -47,6 +48,47 @@ TextLlmContext::TextLlmContext( vocab_ = llama_model_get_vocab(model_); + // Multi-Token Prediction (MTP) draft setup. + const bool wantMtpDraft = std::find(params_.speculative.types.begin(), + params_.speculative.types.end(), + COMMON_SPECULATIVE_TYPE_DRAFT_MTP) != + params_.speculative.types.end(); + if (wantMtpDraft) { + try { + auto cparamsMtp = common_context_params_to_llama(params_); + cparamsMtp.ctx_type = LLAMA_CONTEXT_TYPE_MTP; + cparamsMtp.type_k = params_.speculative.draft.cache_type_k; + cparamsMtp.type_v = params_.speculative.draft.cache_type_v; + cparamsMtp.n_rs_seq = 0; + ctxDraft_.reset(llama_init_from_model(model_, cparamsMtp)); + if (!ctxDraft_) { + QLOG_IF( + Priority::WARNING, + "[TextLlm] MTP draft context could not be created for this " + "model; spec-type=draft-mtp will be inert\n"); + } else { + params_.speculative.draft.ctx_tgt = lctx_; + params_.speculative.draft.ctx_dft = ctxDraft_.get(); + spec_.reset(common_speculative_init( + params_.speculative, + std::max(1, params_.n_parallel))); + QLOG_IF( + Priority::INFO, + "[TextLlm] MTP draft context + common_speculative initialized " + "(Phase 1: load-time only; not yet consulted by decode)\n"); + } + } catch (const std::exception& e) { + QLOG_IF( + Priority::WARNING, + string_format( + "[TextLlm] MTP draft setup failed (%s); continuing without " + "speculative decoding\n", + e.what())); + ctxDraft_.reset(); + spec_.reset(); + } + } + isQwen3Model_ = qvac_lib_inference_addon_llama::utils::isQwen3Model(model_); if (isQwen3Model_) { qvac_lib_inference_addon_llama::utils::initializeQwen3ReasoningState( @@ -414,7 +456,7 @@ bool TextLlmContext::evalMessageWithTools( textBatch->logits[textBatch->n_tokens - 1] = static_cast(true); } // NOLINTNEXTLINE(clang-analyzer-core.CallAndMessage) - int ret = llama_decode(lctx_, *textBatch); + int ret = decodeAndSpecProcess(*textBatch); if (ret != 0) { std::string errorMsg = string_format( "[TextLlm] %s: failed to decode input tokens\n", __func__); @@ -471,7 +513,7 @@ void TextLlmContext::handleStopRequestAndAddEot(LlamaBatch& batch) { nPast_++, {0}, true); - if (llama_decode(lctx_, *batch) != 0) { + if (decodeAndSpecProcess(*batch) != 0) { const char* errorMsg = "[TextLlm] failed to decode EOT token\n"; throw qvac_errors::StatusError( ADDON_ID, toString(FailedToDecode), errorMsg); @@ -481,8 +523,20 @@ void TextLlmContext::handleStopRequestAndAddEot(LlamaBatch& batch) { bool TextLlmContext::generateResponse( const std::function& outputCallback) { + // Protocol: common_speculative_begin runs once at the start of generation. + // For MTP it validates that the draft context's KV cache lines up with + // the target's after prefill; passing an empty prompt skips that validation. + if (spec_ && !specBeganGenerate_) { + common_speculative_begin(spec_.get(), /*seq_id=*/0, {}); + specBeganGenerate_ = true; + } + // Carryover from a prior generation must not leak into this one — every + // call starts by sampling fresh from the post-prefill logits. + pendingSampled_ = LLAMA_TOKEN_NULL; + int nRemain = params_.n_predict; - LlamaBatch batch(1, 0, 1); // batch for next token generation + const int kSpecMaxDraft = spec_ ? params_.speculative.draft.n_max : 0; + LlamaBatch batch(1 + kSpecMaxDraft, 0, 1); reasoningState_.inside_reasoning = false; reasoningState_.recent_output_buffer.clear(); @@ -526,8 +580,17 @@ bool TextLlmContext::generateResponse( } applyContextDiscard(); - llama_token tokenId = common_sampler_sample(smpl_.get(), lctx_, -1); - common_sampler_accept(smpl_.get(), tokenId, true); + // In speculative mode, the previous iter's verify pass already sampled + // (and accepted into the sampler) the token that follows the last + // accepted draft. Reuse it instead of re-sampling from the same logits. + llama_token tokenId; + if (pendingSampled_ != LLAMA_TOKEN_NULL) { + tokenId = pendingSampled_; + pendingSampled_ = LLAMA_TOKEN_NULL; + } else { + tokenId = common_sampler_sample(smpl_.get(), lctx_, -1); + common_sampler_accept(smpl_.get(), tokenId, true); + } --nRemain; std::string tokenStr = @@ -580,17 +643,150 @@ bool TextLlmContext::generateResponse( } common_batch_add(*batch, tokenId, nPast_++, {0}, true); + // Speculative draft path: append up to n_max draft tokens onto the same + // ubatch, capped by remaining context room and budget. MTP's draft() + // reads only id_last / n_past / n_max — the .prompt vector is required + // by the API (the wrapper dereferences ->size() for logging) but never + // consulted by the impl, so we hand it the always-empty specDummyPrompt_. + llama_tokens specDraft; + if (spec_ && kSpecMaxDraft > 0) { + const int ctxRoom = + static_cast(llama_n_ctx(lctx_)) - static_cast(nPast_) - 1; + const int budget = nRemain > 0 ? nRemain - 1 : kSpecMaxDraft; + const int nMaxThisIter = std::min({kSpecMaxDraft, ctxRoom, budget}); + if (nMaxThisIter > 0) { + const llama_pos preDraftPast = nPast_ - 1; + const llama_state_seq_flags dftFlags = + LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY | LLAMA_STATE_SEQ_FLAGS_ON_DEVICE; + if (ctxDraft_) { + const size_t snapSize = + llama_state_seq_get_size_ext(ctxDraft_.get(), /*seq_id=*/0, dftFlags); + specCkptDft_.resize(snapSize); + llama_state_seq_get_data_ext( + ctxDraft_.get(), + specCkptDft_.data(), + snapSize, + /*seq_id=*/0, + dftFlags); + } + common_speculative_get_draft_params(spec_.get(), /*seq_id=*/0) = { + /*.drafting = */ true, + /*.n_max = */ nMaxThisIter, + /*.n_past = */ nPast_, + /*.id_last = */ tokenId, + /*.prompt = */ &specDummyPrompt_, + /*.result = */ &specDraft, + }; + common_speculative_draft(spec_.get()); + if (ctxDraft_ && !specDraft.empty()) { + // Restore the recurrent state to its pre-draft snapshot, then + // clear the attn-KV positions draft() AR-decoded onto ctx_dft. + llama_state_seq_set_data_ext( + ctxDraft_.get(), + specCkptDft_.data(), + specCkptDft_.size(), + /*seq_id=*/0, + dftFlags); + llama_memory_seq_rm( + llama_get_memory(ctxDraft_.get()), + /*seq_id=*/0, + preDraftPast + 1, + -1); + } + for (auto d : specDraft) { + common_batch_add(*batch, d, nPast_++, {0}, true); + } + } + } + // NOLINT(clang-analyzer-core.CallAndMessage) - if (llama_decode(lctx_, *batch) != 0) { + if (decodeAndSpecProcess(*batch) != 0) { const char* errorMsg = "[TextLlm] failed to decode next token\n"; throw qvac_errors::StatusError( ADDON_ID, toString(FailedToDecode), errorMsg); } + + // Verify the speculatively-decoded positions: + // sample_and_accept_n walks idxs 0..draft.size(), at each position + // samples from the target's logits and stops at the first draft that + // disagrees. Returns ids[0..n_accepted] where n_accepted = ids.size()-1 + // matched, and ids.back() is the new "real" sample from the position + // after the last accepted draft. + if (!specDraft.empty()) { + if (spec_) { + const auto ids = + common_sampler_sample_and_accept_n(smpl_.get(), lctx_, specDraft); + const size_t nAccepted = ids.size() - 1; + common_speculative_accept( + spec_.get(), /*seq_id=*/0, static_cast(nAccepted)); + + bool earlyStopSpec = false; + for (size_t i = 0; i < nAccepted; ++i) { + const llama_token id = ids[i]; + --nRemain; + + const std::string str = + common_token_to_piece(lctx_, id, params_.special); + if (outputCallback) { + std::string completeChars = utf8Buffer_.addToken(str); + if (!completeChars.empty()) { + outputCallback(completeChars); + } + } + + if (isQwen3Model_) { + qvac_lib_inference_addon_llama::utils::updateQwen3ReasoningBuffer( + str, reasoningState_); + } + + // Accepted-draft path intentionally. + const bool isEos = llama_vocab_is_eog(vocab_, id); + if (isEos || checkAntiprompt()) { + flushPendingUtf8ToCallback(outputCallback); + earlyStopSpec = true; + break; + } + if (nRemain == 0) { + earlyStopSpec = true; + break; + } + } + + // Roll back KV for any rejected draft positions. + if (nAccepted < specDraft.size()) { + const llama_pos newPast = + nPast_ - static_cast(specDraft.size() - nAccepted); + llama_memory_seq_rm( + llama_get_memory(lctx_), /*seq_id=*/0, newPast, -1); + nPast_ = newPast; + } + + if (earlyStopSpec) { + break; + } + + // Carry the verify's "extra" sample to the next iteration's tokenId. + pendingSampled_ = ids.back(); + } else { + // spec_ was reset mid-decode by decodeAndSpecProcess (process() + // returned false). + const llama_pos newPast = + nPast_ - static_cast(specDraft.size()); + llama_memory_seq_rm( + llama_get_memory(lctx_), /*seq_id=*/0, newPast, -1); + nPast_ = newPast; + } + } } if (nRemain == 0) { flushPendingUtf8ToCallback(outputCallback); } + // Surface MTP draft/accept counts in the addon log so tests + perf runs can + // verify speculative decoding is actually doing useful work. + if (spec_) { + common_speculative_print_stats(spec_.get()); + } return true; } @@ -599,6 +795,28 @@ TextLlmContext::applyGenerationParams(const GenerationParams& overrides) { return applyGenerationParamsToContext(params_, smpl_, model_, overrides); } +int TextLlmContext::decodeAndSpecProcess(const llama_batch& batch) { + const int ret = llama_decode(lctx_, batch); + if (ret != 0) { + return ret; + } + if (spec_) { + // MTP head's KV must track the target on every batch (prefill + gen) + // so common_speculative_begin's validation log stays clean and drafts + // are coherent. Soft-fail: log + drop the spec impl + fall back to the + // single-token path for the rest of this model's lifetime. + if (!common_speculative_process(spec_.get(), batch)) { + QLOG_IF( + Priority::WARNING, + "[TextLlm] common_speculative_process failed; disabling spec for " + "the rest of this model lifetime\n"); + spec_.reset(); + ctxDraft_.reset(); + } + } + return ret; +} + void TextLlmContext::stop() { stopGeneration_.store(true); } void TextLlmContext::resetState(bool resetStats) { @@ -719,7 +937,7 @@ bool TextLlmContext::handleQwen3ReasoningEOS( // Decode closing tag common_batch_clear(batch); common_batch_add(batch, tokenId, nPast++, {0}, true); - if (llama_decode(lctx_, batch) != 0) { + if (decodeAndSpecProcess(batch) != 0) { QLOG_IF( Priority::ERROR, "[TextLlm] Failed to decode closing tag during replacement\n"); @@ -732,7 +950,7 @@ bool TextLlmContext::handleQwen3ReasoningEOS( common_batch_add( batch, reasoningState_.cached_newline_token, nPast++, {0}, true); - if (llama_decode(lctx_, batch) != 0) { + if (decodeAndSpecProcess(batch) != 0) { QLOG_IF( Priority::ERROR, "[TextLlm] Failed to decode newline token during forced " diff --git a/packages/llm-llamacpp/addon/src/model-interface/TextLlmContext.hpp b/packages/llm-llamacpp/addon/src/model-interface/TextLlmContext.hpp index 28ca2f0a68..84c09cd060 100644 --- a/packages/llm-llamacpp/addon/src/model-interface/TextLlmContext.hpp +++ b/packages/llm-llamacpp/addon/src/model-interface/TextLlmContext.hpp @@ -2,6 +2,7 @@ #include +#include #include #include "../utils/ChatTemplateUtils.hpp" @@ -10,6 +11,7 @@ #include "LlmContext.hpp" #include "ToolsCompactController.hpp" #include "common/common.h" +#include "common/speculative.h" #include "inference-addon-cpp/Logger.hpp" class TextLlmContext : public LlmContext { @@ -183,6 +185,11 @@ class TextLlmContext : public LlmContext { void applyContextDiscard(); void handleStopRequestAndAddEot(LlamaBatch& batch); + // Wraps llama_decode(lctx_, batch). When MTP/speculative is active, also + // feeds the batch into common_speculative_process so the draft context's + // KV cache tracks the target's on every prefill ubatch + generation step. + int decodeAndSpecProcess(const llama_batch& batch); + ToolsCompactController& tools_; common_init_result_ptr llamaInit_; llama_model* model_; @@ -227,5 +234,24 @@ class TextLlmContext : public LlmContext { std::string lastChatParser_; std::string lastGenerationPrompt_; + // Speculative decoding state for MTP. If non-null, the draft context and spec struct are live + // and common_speculative_process is called on every decode. + llama_context_ptr ctxDraft_; + common_speculative_ptr spec_; + // common_speculative_begin must run once before the first decode of each + // generation; this flag de-duplicates across multi-turn calls. + bool specBeganGenerate_ = false; + // common_speculative_get_draft_params requires a non-null .prompt; the MTP + // impl never reads its contents (only id_last/n_past/n_max), but the wrapper + // dereferences ->size() for logging. + std::vector specDummyPrompt_; + // Sampled-but-not-yet-consumed token from the previous iteration's + // sample_and_accept_n (the "extra" id past the last accepted draft). + // LLAMA_TOKEN_NULL means: sample fresh next iter. + llama_token pendingSampled_ = LLAMA_TOKEN_NULL; + // Snapshot of ctxDraft_'s partial sequence state (SWA + recurrent) taken + // right before common_speculative_draft AR-decodes onto ctx_dft. + std::vector specCkptDft_; + std::atomic stopGeneration_ = false; }; diff --git a/packages/llm-llamacpp/test/integration/mtp.test.js b/packages/llm-llamacpp/test/integration/mtp.test.js new file mode 100644 index 0000000000..9d7fbb5709 --- /dev/null +++ b/packages/llm-llamacpp/test/integration/mtp.test.js @@ -0,0 +1,111 @@ +'use strict' + +// MTP (Multi-Token Prediction) speculative-decoding integration test. + +const test = require('brittle') +const path = require('bare-path') +const LlmLlamacpp = require('../../index.js') +const { ensureModel } = require('./utils') +const os = require('bare-os') + +const platform = os.platform() +const arch = os.arch() +const isDarwinX64 = platform === 'darwin' && arch === 'x64' +const isLinuxArm64 = platform === 'linux' && arch === 'arm64' +const useCpu = isDarwinX64 || isLinuxArm64 + +const MODEL = { + name: 'Qwen3.5-0.8B-Q8_0.gguf', + url: 'https://huggingface.co/unsloth/Qwen3.5-0.8B-GGUF/resolve/main/Qwen3.5-0.8B-Q8_0.gguf' +} + +const PROMPT = [ + { role: 'system', content: 'You are a helpful assistant.' }, + { role: 'user', content: 'What is the capital of France? Answer in one word.' } +] + +function createLogger () { + return { + info: (...args) => console.info(...args), + warn: (...args) => console.warn(...args), + error: (...args) => console.error(...args), + debug: (...args) => console.debug(...args) + } +} + +async function collectResponse (response) { + const chunks = [] + const ticker = setInterval(() => {}, 50) + try { + await response.onUpdate(data => { chunks.push(data) }).await() + } finally { + clearInterval(ticker) + } + return chunks.join('').trim() +} + +async function runOnce ({ withSpec }) { + const [modelName, dirPath] = await ensureModel({ + modelName: MODEL.name, + downloadUrl: MODEL.url + }) + const modelPath = path.join(dirPath, modelName) + + const config = { + device: useCpu ? 'cpu' : 'gpu', + gpu_layers: '999', + ctx_size: '1024', + n_predict: '64', + temp: '0', + seed: '42', + 'reasoning-budget': '0', + verbosity: '2' + } + if (withSpec) { + config['spec-type'] = 'draft-mtp' + } + + const addon = new LlmLlamacpp({ + files: { model: [modelPath] }, + config, + logger: createLogger(), + opts: { stats: true } + }) + + try { + await addon.load() + const response = await addon.run(PROMPT) + const output = await collectResponse(response) + return { output, stats: response.stats } + } finally { + await addon.unload().catch(() => {}) + } +} + +test('Qwen3.5-0.8B with spec-type=draft-mtp drafts + accepts', { + timeout: 600_000 +}, async t => { + const { output, stats } = await runOnce({ withSpec: true }) + t.ok(output.length > 0, `spec run produced output (${output.length} chars)`) + console.log(` spec output: "${output.slice(0, 200)}"`) + t.ok(stats, 'spec run has response.stats') + t.ok( + /paris/i.test(output), + 'spec output names the capital (Paris) in the reply' + ) +}) + +test('Qwen3.5-0.8B without spec-type', { + timeout: 600_000 +}, async t => { + // Sentinel: confirms the addon's existing single-context path still works, + // proving the MTP code added to TextLlmContext is correctly gated behind + // the `spec-type` config check and doesn't fire for default-config loads. + const { output, stats } = await runOnce({ withSpec: false }) + t.ok(output.length > 0, `non-spec run produced output (${output.length} chars)`) + t.ok(stats, 'non-spec run has stats') + t.ok( + /paris/i.test(output), + 'non-spec output names the capital (Paris) in the reply' + ) +}) diff --git a/packages/llm-llamacpp/test/mobile/integration.auto.cjs b/packages/llm-llamacpp/test/mobile/integration.auto.cjs index 6f6d1fd7ec..62535be2c6 100644 --- a/packages/llm-llamacpp/test/mobile/integration.auto.cjs +++ b/packages/llm-llamacpp/test/mobile/integration.auto.cjs @@ -71,6 +71,11 @@ async function runModelLoadingTest (options = {}) { // eslint-disable-line no-un return runIntegrationModule('../integration/model-loading.test.js', options) } +async function runMtpTest (options = {}) { // eslint-disable-line no-unused-vars + if (typeof __shouldRunTest === 'function' && !__shouldRunTest('runMtpTest')) return __FILTERED + return runIntegrationModule('../integration/mtp.test.js', options) +} + async function runMultiGpuTest (options = {}) { // eslint-disable-line no-unused-vars if (typeof __shouldRunTest === 'function' && !__shouldRunTest('runMultiGpuTest')) return __FILTERED return runIntegrationModule('../integration/multi-gpu.test.js', options) diff --git a/packages/llm-llamacpp/test/mobile/test-groups.json b/packages/llm-llamacpp/test/mobile/test-groups.json index 4d30527438..dc69759f28 100644 --- a/packages/llm-llamacpp/test/mobile/test-groups.json +++ b/packages/llm-llamacpp/test/mobile/test-groups.json @@ -11,6 +11,7 @@ "heavy9": ["runGemma4Test"], "heavy10": ["runOcrPaddleTest"], "heavy11": ["runQuantizedKvcacheTest"], + "heavy12": ["runMtpTest"], "lightA": [ "runApiBehaviorTest", "runBitnetTest", @@ -58,7 +59,8 @@ "groupImagesPerf": [ "runImageElephantTest", "runImageFruitPlateTest", - "runImageHighResAuroraTest" + "runImageHighResAuroraTest", + "runMtpTest" ] } } diff --git a/packages/llm-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake b/packages/llm-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake index 8024dc3b82..f0fe353cbe 100644 --- a/packages/llm-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake +++ b/packages/llm-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake @@ -1,8 +1,9 @@ vcpkg_from_github( - OUT_SOURCE_PATH SOURCE_PATH - REPO tetherto/qvac-fabric-llm.cpp - REF bc85668ffad0394d4973d9e08eb8373871188d02 - SHA512 8e07123e6169ed44d942495ab98f884f407b22b598b5035e0b47bdc372118f63a8fa03beecf6e36cd6482bbae877d8963813f85341f19b285882939c1c27ccee + OUT_SOURCE_PATH SOURCE_PATH + REPO tetherto/qvac-fabric-llm.cpp + REF 97165a1a79650db28ec6a7d965b5905defcd5a23 + SHA512 92f0c54f648c1c30a879e31a5de934511aeb7da97ba0bdeb68caca8bff4ad43388b997288eccfdabb97f6a4c549a7f14f004688c52c99a8477108b2888132a73 + HEAD_REF qvac-b9341 ) # Upstream CMake options only — passed through to vcpkg_cmake_configure. From 6a7de8cbad748419fb0506895236a861e0a7c5cf Mon Sep 17 00:00:00 2001 From: Marcus Edel Date: Mon, 1 Jun 2026 22:58:08 +0000 Subject: [PATCH 05/58] fix: fence saveCacheToDisk write before model.unload() in tools-compact session-reload test to close fast-runner race. Signed-off-by: Marcus Edel --- .../test/integration/tools-compact.test.js | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/packages/llm-llamacpp/test/integration/tools-compact.test.js b/packages/llm-llamacpp/test/integration/tools-compact.test.js index 62a84dd779..93cc862b2c 100644 --- a/packages/llm-llamacpp/test/integration/tools-compact.test.js +++ b/packages/llm-llamacpp/test/integration/tools-compact.test.js @@ -190,6 +190,25 @@ async function runAndCollect (model, prompt, runOptions) { } } +// Fence for `saveCacheToDisk: true`: the addon resolves `chain.await()` once +// inference is done, but the session-write hits the disk on a background +// thread. On a fast runner an immediate `model.unload()` can race the flush +// and leave a truncated file; the next model's session-load then blocks +// indefinitely waiting for bytes that never arrive. Poll until size has +// been stable for one tick to make sure the writer has finalized. +async function waitForStableSessionFile (filePath, { maxMs = 5000 } = {}) { + const start = Date.now() + let lastSize = -1 + while (Date.now() - start < maxMs) { + let size = 0 + try { size = fs.statSync(filePath).size } catch { size = 0 } + if (size > 0 && size === lastSize) return size + lastSize = size + await new Promise(resolve => setTimeout(resolve, 50)) + } + return lastSize +} + async function runExpectingInvalidPrompt (t, model, prompt, expectedReason, runOptions) { const response = await model.run(prompt, runOptions) @@ -727,6 +746,9 @@ safeTest('[tools-compact] session save, destroy, reload with different tools', { t.ok(r1.output.length > 0, 'turn 1 produces output') t.ok(r1.stats.CacheTokens > 0, 'turn 1 has cache tokens') + const persistedSize = await waitForStableSessionFile(sessionName) + t.ok(persistedSize > 0, `session file flushed to disk (${persistedSize} bytes)`) + await model1.unload() const { model: model2 } = await setupModel(t) From 41fb616d0b3ab8e575edb89fb4d198beb6acbb62 Mon Sep 17 00:00:00 2001 From: Marcus Edel Date: Tue, 2 Jun 2026 16:04:34 +0000 Subject: [PATCH 06/58] feat: accept positive reasoning-budget N as a token cap on the channel and wire it through to common-sampling's budget sampler. Signed-off-by: Marcus Edel --- .../llm-llamacpp/addon/src/addon/AddonJs.hpp | 18 +++-- .../model-interface/GenerationParamsApply.cpp | 16 ++++ .../addon/src/model-interface/LlamaModel.cpp | 15 ++-- .../src/model-interface/TextLlmContext.cpp | 33 ++++++++ .../test/integration/reasoning.test.js | 75 +++++++++++++++++++ 5 files changed, 143 insertions(+), 14 deletions(-) diff --git a/packages/llm-llamacpp/addon/src/addon/AddonJs.hpp b/packages/llm-llamacpp/addon/src/addon/AddonJs.hpp index d758e9f70a..4ad1b14c01 100644 --- a/packages/llm-llamacpp/addon/src/addon/AddonJs.hpp +++ b/packages/llm-llamacpp/addon/src/addon/AddonJs.hpp @@ -1,5 +1,7 @@ #pragma once +#include #include +#include #include #include @@ -365,16 +367,18 @@ inline js_value_t* runJob(js_env_t* env, js_callback_info_t* info) try { configObj->getOptionalPropertyAs( env, "reasoning_budget"); if (reasoningBudget.has_value()) { - // Validate against the exact double values (0 and -1 are exactly - // representable in IEEE-754), so fractional inputs like 0.5 or -1.1 - // are rejected instead of being silently truncated. - if (*reasoningBudget != 0 && *reasoningBudget != -1) { + // Reject fractional inputs (0.5, -1.1, 32.7) by requiring the value + // to round-trip through int. -1 = unrestricted, 0 = disabled, N>0 + // caps the reasoning channel at N tokens via the budget sampler. + const double v = *reasoningBudget; + if (v < -1 || v != std::floor(v) || + v > static_cast(std::numeric_limits::max())) { throw StatusError( general_error::InvalidArgument, - "generationParams.reasoning_budget must be -1 (unrestricted) " - "or 0 (disabled)"); + "generationParams.reasoning_budget must be -1 (unrestricted), " + "0 (disabled), or a positive integer (token cap)"); } - ov.reasoning_budget = static_cast(*reasoningBudget); + ov.reasoning_budget = static_cast(v); } } diff --git a/packages/llm-llamacpp/addon/src/model-interface/GenerationParamsApply.cpp b/packages/llm-llamacpp/addon/src/model-interface/GenerationParamsApply.cpp index b438012bb3..1654e99576 100644 --- a/packages/llm-llamacpp/addon/src/model-interface/GenerationParamsApply.cpp +++ b/packages/llm-llamacpp/addon/src/model-interface/GenerationParamsApply.cpp @@ -29,6 +29,22 @@ void applyGenerationOverridesToSampling( setIf(overrides.presence_penalty, sampling.penalty_present); setIf(overrides.repeat_penalty, sampling.penalty_repeat); + // Forward reasoning_budget into the budget-sampler's token cap. The + // start/end/forced vectors were tokenized in the TextLlmContext ctor for + // Qwen3 models and survive the copy from params.sampling into the local + // override, so we only need to mutate the count here. + if (overrides.reasoning_budget) { + const int budget = *overrides.reasoning_budget; + if (budget > 0) { + sampling.reasoning_budget_tokens = budget; + } else { + sampling.reasoning_budget_tokens = -1; + sampling.reasoning_budget_start.clear(); + sampling.reasoning_budget_end.clear(); + sampling.reasoning_budget_forced.clear(); + } + } + // `json_schema` and `grammar` are mutually exclusive at the JS boundary // and in `AddonJs::runJob::parseText`, so reaching this branch with both // set means a caller bypassed those checks (most likely the C++ unit diff --git a/packages/llm-llamacpp/addon/src/model-interface/LlamaModel.cpp b/packages/llm-llamacpp/addon/src/model-interface/LlamaModel.cpp index 57e0913e52..635dc3ee21 100644 --- a/packages/llm-llamacpp/addon/src/model-interface/LlamaModel.cpp +++ b/packages/llm-llamacpp/addon/src/model-interface/LlamaModel.cpp @@ -865,22 +865,23 @@ void LlamaModel::commonParamsParse( "embedded chat template is applied\n"); } - // reasoning-budget controls whether the model emits a reasoning - // channel. -1 (default) leaves it on; 0 disables. `std::from_chars` is used - // instead of `std::stoi` because the latter accepts trailing garbage ("0abc" - // → 0) and throws an uncaught `std::out_of_range` on overflow. + // reasoning-budget controls the size of the model's reasoning + // channel: -1 (default) leaves it unrestricted, 0 disables thinking + // entirely, any positive N caps the reasoning channel at N tokens (the + // budget sampler forces once N reasoning tokens have been + // emitted). auto parseReasoningBudget = [](const std::string& raw) { int value = 0; const char* begin = raw.data(); const char* end = begin + raw.size(); const auto [ptr, ec] = std::from_chars(begin, end, value); - if (ec != std::errc{} || ptr != end || - (value != 0 && value != -1)) { + if (ec != std::errc{} || ptr != end || value < -1) { throw qvac_errors::StatusError( ADDON_ID, qvac_errors::general_error::toString( qvac_errors::general_error::InvalidArgument), - "reasoning-budget must be -1 (unrestricted) or 0 (disabled)"); + "reasoning-budget must be -1 (unrestricted), 0 (disabled), or a " + "positive integer (token cap)"); } return value; }; diff --git a/packages/llm-llamacpp/addon/src/model-interface/TextLlmContext.cpp b/packages/llm-llamacpp/addon/src/model-interface/TextLlmContext.cpp index 5caeac585d..cfb19b3eb5 100644 --- a/packages/llm-llamacpp/addon/src/model-interface/TextLlmContext.cpp +++ b/packages/llm-llamacpp/addon/src/model-interface/TextLlmContext.cpp @@ -22,6 +22,35 @@ using namespace qvac_lib_inference_addon_llama::errors; using namespace qvac_lib_inference_addon_cpp::logger; using namespace qvac_lib_inference_addon_llama::utils; + +namespace { + +// Populate params.sampling.reasoning_budget_{start,end,forced,tokens} from +// params.reasoning_budget so common_sampler_init's budget-sampler kicks in. +// +// < 0 - unrestricted (sampler skipped; thinking still enabled). +// 0 - disabled (sampler skipped; inputs.enable_thinking=false at apply +// time keeps the template from inserting ). +// N>0 - token cap; the sampler counts tokens between and +// and forces once N reasoning tokens have been emitted. +void applyReasoningBudgetToSampling( + common_params& params, bool isQwen3Model, llama_context* lctx) { + if (params.reasoning_budget <= 0 || !isQwen3Model || lctx == nullptr) { + return; + } + params.sampling.reasoning_budget_tokens = params.reasoning_budget; + params.sampling.reasoning_budget_start = + common_tokenize(lctx, "", false, true); + params.sampling.reasoning_budget_end = + common_tokenize(lctx, "", false, true); + params.sampling.reasoning_budget_forced = common_tokenize( + lctx, + params.sampling.reasoning_budget_message + "", + false, + true); +} + +} // namespace // NOLINTNEXTLINE(readability-identifier-naming,readability-function-cognitive-complexity) // NOLINTNEXTLINE(readability-function-cognitive-complexity) @@ -117,6 +146,10 @@ TextLlmContext::TextLlmContext( getChatTemplate(model_, params_, tools_.enabled()); tmpls_ = common_chat_templates_init(model_, chatTemplate); + // Wire reasoning_budget > 0 through to the common-sampling layer's + // budget sampler so it caps the channel at N tokens. + applyReasoningBudgetToSampling(params_, isQwen3Model_, lctx_); + smpl_.reset(common_sampler_init(model_, params_.sampling)); if (!smpl_) { std::string errorMsg = string_format( diff --git a/packages/llm-llamacpp/test/integration/reasoning.test.js b/packages/llm-llamacpp/test/integration/reasoning.test.js index 40778cfdad..929ba5f7d6 100644 --- a/packages/llm-llamacpp/test/integration/reasoning.test.js +++ b/packages/llm-llamacpp/test/integration/reasoning.test.js @@ -240,3 +240,78 @@ safeTest('Qwen3 reasoning-budget=0 disables thinking', { t.ok(disabled.length < baseline.length / 4, `disabled (${disabled.length}) should be substantially shorter than baseline (${baseline.length})`) }) + +// reasoning-budget = N caps the channel at N tokens via the budget +// sampler — is force-emitted once N reasoning tokens have streamed. +safeTest('Qwen3 reasoning-budget=N caps reasoning channel below baseline', { + skip: isDarwinX64 || isWindowsX64, + timeout: 600_000 +}, async t => { + const [modelName, dirPath] = await ensureModel({ + modelName: MODEL.name, + downloadUrl: MODEL.url + }) + const modelPath = path.join(dirPath, modelName) + + const baseConfig = { + ctx_size: '4096', + n_predict: '1024', + seed: '50', + gpu_layers: '999', + temp: '0', + top_p: '1', + device: useCpu ? 'cpu' : 'gpu', + verbosity: '0' + } + + async function runOnce (extra) { + const inference = new LlmLlamacpp({ + files: { model: [modelPath] }, + config: { ...baseConfig, ...extra }, + logger: console + }) + try { + await inference.load() + const messages = [ + { role: 'system', content: 'You are a helpful assistant.' }, + { role: 'user', content: 'What is the capital of France? Answer in one word.' } + ] + return await runCompletion(inference, messages) + } finally { + await inference.unload().catch(() => {}) + } + } + + const baseline = await runOnce({}) + const baselineThink = sliceReasoning(baseline) + t.ok(baselineThink !== null, `baseline produced a reasoning channel (${baselineThink ? baselineThink.length : 0} chars)`) + + const BUDGET = 16 + const capped = await runOnce({ 'reasoning-budget': String(BUDGET) }) + const cappedThink = sliceReasoning(capped) + + t.comment(`baseline reasoning (${baselineThink ? baselineThink.length : 0} chars): ${(baselineThink || '').slice(0, 200)}`) + t.comment(`capped reasoning (${cappedThink ? cappedThink.length : 0} chars): ${cappedThink || ''}`) + + t.ok(/paris/i.test(capped), 'capped output still mentions Paris (correctness preserved past forced close)') + t.ok(capped.includes('') && capped.includes(''), + `capped output keeps balanced reasoning tags: "${capped.slice(0, 200)}"`) + t.ok(cappedThink !== null, + 'capped output has an extractable reasoning channel') + // The budget sampler is char-agnostic — it counts tokens, not characters — + // so we assert the reasoning channel is at least *substantially* shorter + // than the unrestricted baseline rather than asserting an exact char count. + t.ok( + cappedThink.length < Math.max(1, baselineThink.length * 0.6), + `capped reasoning (${cappedThink.length}) is substantially shorter than baseline (${baselineThink.length})` + ) +}) + +function sliceReasoning (text) { + const open = text.indexOf('') + const close = text.indexOf('') + if (open === -1 || close === -1 || close < open) { + return null + } + return text.slice(open + ''.length, close) +} From 928948136eb37fbed03566eb36f888f7baeeeda7 Mon Sep 17 00:00:00 2001 From: Marcus Edel Date: Tue, 2 Jun 2026 20:22:44 +0000 Subject: [PATCH 07/58] build: extend qvac-fabric overlay-port to classification-ggml, vla-ggml, and translation-nmtcpp. Signed-off-by: Marcus Edel --- .../vcpkg-configuration.json | 3 + .../qvac-fabric/android-vulkan-version.cmake | 36 ++++ .../vcpkg/ports/qvac-fabric/portfile.cmake | 191 ++++++++++++++++++ .../vcpkg/ports/qvac-fabric/vcpkg.json | 50 +++++ .../qvac-fabric/android-vulkan-version.cmake | 2 +- .../vcpkg/ports/qvac-fabric/portfile.cmake | 20 +- .../vcpkg/ports/qvac-fabric/vcpkg.json | 14 +- .../qvac-fabric/android-vulkan-version.cmake | 2 +- .../vcpkg/ports/qvac-fabric/portfile.cmake | 20 +- .../vcpkg/ports/qvac-fabric/vcpkg.json | 14 +- .../vcpkg-configuration.json | 3 + .../qvac-fabric/android-vulkan-version.cmake | 36 ++++ .../vcpkg/ports/qvac-fabric/portfile.cmake | 191 ++++++++++++++++++ .../vcpkg/ports/qvac-fabric/vcpkg.json | 50 +++++ packages/vla-ggml/vcpkg-configuration.json | 3 + .../qvac-fabric/android-vulkan-version.cmake | 36 ++++ .../vcpkg/ports/qvac-fabric/portfile.cmake | 191 ++++++++++++++++++ .../vcpkg/ports/qvac-fabric/vcpkg.json | 50 +++++ 18 files changed, 888 insertions(+), 24 deletions(-) create mode 100644 packages/classification-ggml/vcpkg/ports/qvac-fabric/android-vulkan-version.cmake create mode 100644 packages/classification-ggml/vcpkg/ports/qvac-fabric/portfile.cmake create mode 100644 packages/classification-ggml/vcpkg/ports/qvac-fabric/vcpkg.json create mode 100644 packages/translation-nmtcpp/vcpkg/ports/qvac-fabric/android-vulkan-version.cmake create mode 100644 packages/translation-nmtcpp/vcpkg/ports/qvac-fabric/portfile.cmake create mode 100644 packages/translation-nmtcpp/vcpkg/ports/qvac-fabric/vcpkg.json create mode 100644 packages/vla-ggml/vcpkg/ports/qvac-fabric/android-vulkan-version.cmake create mode 100644 packages/vla-ggml/vcpkg/ports/qvac-fabric/portfile.cmake create mode 100644 packages/vla-ggml/vcpkg/ports/qvac-fabric/vcpkg.json diff --git a/packages/classification-ggml/vcpkg-configuration.json b/packages/classification-ggml/vcpkg-configuration.json index b685565a08..2cce8881ec 100644 --- a/packages/classification-ggml/vcpkg-configuration.json +++ b/packages/classification-ggml/vcpkg-configuration.json @@ -4,6 +4,9 @@ "baseline": "803c0d119ea002694963e89237c207ff6ecf47f6", "repository": "https://github.com/tetherto/qvac-registry-vcpkg.git" }, + "overlay-ports": [ + "vcpkg/ports" + ], "registries": [ { "kind": "git", diff --git a/packages/classification-ggml/vcpkg/ports/qvac-fabric/android-vulkan-version.cmake b/packages/classification-ggml/vcpkg/ports/qvac-fabric/android-vulkan-version.cmake new file mode 100644 index 0000000000..21fba00feb --- /dev/null +++ b/packages/classification-ggml/vcpkg/ports/qvac-fabric/android-vulkan-version.cmake @@ -0,0 +1,36 @@ +# Function to detect Vulkan version from NDK vulkan_core.h +function(detect_ndk_vulkan_version) + string(TOLOWER "${CMAKE_HOST_SYSTEM_NAME}" host_system_name_lower) + + # CMAKE_HOST_SYSTEM_PROCESSOR is unavailable here. Use a glob pattern to complete the folder instead. + file(GLOB host_dirs LIST_DIRECTORIES true "$ENV{ANDROID_NDK_HOME}/toolchains/llvm/prebuilt/${host_system_name_lower}-*") + if(host_dirs) + list(GET host_dirs 0 host_dir) + get_filename_component(host_arch "${host_dir}" NAME) + set(vulkan_core_h "$ENV{ANDROID_NDK_HOME}/toolchains/llvm/prebuilt/${host_arch}/sysroot/usr/include/vulkan/vulkan_core.h") + else() + message(FATAL "Could not find NDK host directory for ${host_system_name_lower}") + endif() + + if(NOT vulkan_core_h) + message(FATAL "vulkan_core.h not found, using default version") + endif() + + file(READ "${vulkan_core_h}" header_content) + string(REGEX MATCH "VK_HEADER_VERSION ([0-9]+)" version_match "${header_content}") + if(version_match) + set(header_version_3 "${CMAKE_MATCH_1}") + else() + message(FATAL "Could not extract VK_HEADER_VERSION from vulkan_core.h, using default: ${vulkan_version}") + endif() + + # Extract major.minor version from VK_HEADER_VERSION_COMPLETE for download URL + string(REGEX MATCH "VK_HEADER_VERSION_COMPLETE VK_MAKE_API_VERSION\\(([0-9]+), ([0-9]+), ([0-9]+)" version_match "${header_content}") + if(version_match) + set(major "${CMAKE_MATCH_2}") + set(minor "${CMAKE_MATCH_3}") + set(vulkan_version "${major}.${minor}.${header_version_3}" PARENT_SCOPE) + else() + message(FATAL "Could not extract major.minor version from vulkan_core.h, using default: ${vulkan_version}") + endif() +endfunction() diff --git a/packages/classification-ggml/vcpkg/ports/qvac-fabric/portfile.cmake b/packages/classification-ggml/vcpkg/ports/qvac-fabric/portfile.cmake new file mode 100644 index 0000000000..ceb57f396a --- /dev/null +++ b/packages/classification-ggml/vcpkg/ports/qvac-fabric/portfile.cmake @@ -0,0 +1,191 @@ +vcpkg_from_github( + OUT_SOURCE_PATH SOURCE_PATH + REPO tetherto/qvac-fabric-llm.cpp + REF 1d0e4859d6bc97d64359075a1045c241e109abf9 + SHA512 caffa277b0acd45de435634c085e2c4d42ca00ba658dabae0d7ded41c9d83ea1974819c448a399f2849c2b8656b1c098267ea4daefd202fa127159d563ebe841 + HEAD_REF qvac-b9341 +) + +# Upstream CMake options only — passed through to vcpkg_cmake_configure. +vcpkg_check_features( + OUT_FEATURE_OPTIONS FEATURE_OPTIONS + FEATURES + force-profiler FORCE_GGML_VK_PERF_LOGGER + llama BUILD_LLAMA +) + +# Portfile-only feature flags (drive PLATFORM_OPTIONS; not upstream cache vars). +vcpkg_check_features( + OUT_FEATURE_OPTIONS _PORTFILE_FEATURE_OPTIONS + FEATURES + gpu-backends BUILD_GPU_BACKENDS + kleidiai BUILD_KLEIDIAI + openmp BUILD_OPENMP +) + +# gpu-backends is default-on via default-features in vcpkg.json. CPU-only +# consumers (e.g. @qvac/classification-ggml) disable it with +# default-features:false (and re-add 'llama' if needed). +if(NOT BUILD_GPU_BACKENDS) + message(STATUS "qvac-fabric: gpu-backends feature OFF — building CPU-only ggml (no Metal/Vulkan/CUDA/OpenCL)") +endif() + +if (VCPKG_TARGET_IS_ANDROID AND BUILD_GPU_BACKENDS) + # NDK only comes with C headers. + # Make sure C++ header exists, it will be used by ggml tensor library. + # Need to determine installed vulkan version and download correct headers + include(${CMAKE_CURRENT_LIST_DIR}/android-vulkan-version.cmake) + detect_ndk_vulkan_version() + message(STATUS "Using Vulkan C++ wrappers from version: ${vulkan_version}") + file(DOWNLOAD + "https://github.com/KhronosGroup/Vulkan-Headers/archive/refs/tags/v${vulkan_version}.tar.gz" + "${SOURCE_PATH}/vulkan-sdk-${vulkan_version}.tar.gz" + TLS_VERIFY ON + ) + + file(ARCHIVE_EXTRACT + INPUT "${SOURCE_PATH}/vulkan-sdk-${vulkan_version}.tar.gz" + DESTINATION "${SOURCE_PATH}" + PATTERNS "*.hpp" + ) + + file(RENAME + "${SOURCE_PATH}/Vulkan-Headers-${vulkan_version}" + "${SOURCE_PATH}/ggml/src/ggml-vulkan/vulkan_cpp_wrapper" + ) +endif() + +set(PLATFORM_OPTIONS) + +if(NOT BUILD_GPU_BACKENDS) + # Force every GPU backend off explicitly, in case upstream defaults change. + list(APPEND PLATFORM_OPTIONS + -DGGML_METAL=OFF + -DGGML_VULKAN=OFF + -DGGML_CUDA=OFF + -DGGML_OPENCL=OFF + ) + if (VCPKG_TARGET_IS_IOS) + # Same iOS BLAS/Accelerate gating as the GPU-on path; unrelated to the + # CPU-vs-GPU split, an iOS-toolchain workaround for missing frameworks. + list(APPEND PLATFORM_OPTIONS -DGGML_BLAS=OFF -DGGML_ACCELERATE=OFF) + endif() +elseif (VCPKG_TARGET_IS_OSX OR VCPKG_TARGET_IS_IOS) + list(APPEND PLATFORM_OPTIONS -DGGML_METAL=ON) + if (VCPKG_TARGET_IS_IOS) + list(APPEND PLATFORM_OPTIONS -DGGML_BLAS=OFF -DGGML_ACCELERATE=OFF) + endif() +else() + list(APPEND PLATFORM_OPTIONS -DGGML_VULKAN=ON) +endif() + +# Android: always build CPU variants (NEON_DOTPROD, NEON_I8MM, etc.) and CPU +# repacking. These are CPU-only runtime optimizations selected based on the +# device's SIMD capabilities at load time, completely orthogonal to the GPU +# backends. Bundling them is essential for good CPU inference performance on +# the wide range of arm64 devices the addons ship to. Requires GGML_BACKEND_DL +# to dispatch the variants at runtime; the existing #ifdef guard around +# `ggml_backend_load_all_from_path()` in ggml-backend-reg.cpp keeps the search +# scoped to the consumer's own prebuilds dir. +if(VCPKG_TARGET_IS_ANDROID) + set(DL_BACKENDS ON) + list(APPEND PLATFORM_OPTIONS + -DGGML_BACKEND_DL=ON + -DGGML_CPU_ALL_VARIANTS=ON + -DGGML_CPU_REPACK=ON) +else() + set(DL_BACKENDS OFF) +endif() + +if(VCPKG_TARGET_IS_ANDROID AND BUILD_KLEIDIAI) + message(STATUS "qvac-fabric: kleidiai feature ON — building with ARM KleidiAI optimized kernels") + # ggml only vendors KleidiAI via FetchContent; registry vcpkg-cmake sets + # FETCHCONTENT_FULLY_DISCONNECTED=ON globally, so allow the download here. + list(APPEND PLATFORM_OPTIONS + -DGGML_CPU_KLEIDIAI=ON + -DFETCHCONTENT_FULLY_DISCONNECTED=OFF + ) +endif() + +if(VCPKG_TARGET_IS_ANDROID AND BUILD_OPENMP) + message(STATUS "qvac-fabric: OpenMP for Android enabled") + list(APPEND PLATFORM_OPTIONS -DGGML_OPENMP=ON) +else() + message(STATUS "qvac-fabric: OpenMP Disabled") + list(APPEND PLATFORM_OPTIONS -DGGML_OPENMP=OFF) +endif() + +if (VCPKG_TARGET_IS_ANDROID AND BUILD_GPU_BACKENDS) + list(APPEND PLATFORM_OPTIONS -DGGML_OPENCL=ON) +endif() + +if(BUILD_GPU_BACKENDS AND NOT VCPKG_TARGET_IS_OSX AND NOT VCPKG_TARGET_IS_IOS) + if(VCPKG_TARGET_IS_WINDOWS AND NOT VCPKG_TARGET_IS_MINGW) + string(APPEND VCPKG_C_FLAGS " /I${CURRENT_INSTALLED_DIR}/include") + string(APPEND VCPKG_CXX_FLAGS " /I${CURRENT_INSTALLED_DIR}/include") + else() + string(APPEND VCPKG_C_FLAGS " -isystem ${CURRENT_INSTALLED_DIR}/include") + string(APPEND VCPKG_CXX_FLAGS " -isystem ${CURRENT_INSTALLED_DIR}/include") + endif() +endif() + +set(LLAMA_OPTIONS) +if("llama" IN_LIST FEATURES) + list(APPEND LLAMA_OPTIONS -DLLAMA_MTMD=ON) +else() + list(APPEND LLAMA_OPTIONS + -DLLAMA_MTMD=OFF + -DLLAMA_BUILD_COMMON=OFF + ) +endif() + +vcpkg_cmake_configure( + SOURCE_PATH "${SOURCE_PATH}" + DISABLE_PARALLEL_CONFIGURE + OPTIONS + -DGGML_NATIVE=OFF + -DGGML_CCACHE=OFF + -DGGML_LLAMAFILE=OFF + -DLLAMA_CURL=OFF + -DLLAMA_BUILD_TESTS=OFF + -DLLAMA_BUILD_TOOLS=OFF + -DLLAMA_BUILD_EXAMPLES=OFF + -DLLAMA_BUILD_SERVER=OFF + -DLLAMA_ALL_WARNINGS=OFF + ${LLAMA_OPTIONS} + ${PLATFORM_OPTIONS} + ${FEATURE_OPTIONS} +) + +vcpkg_cmake_install() +vcpkg_cmake_config_fixup( + PACKAGE_NAME ggml) + +if(BUILD_LLAMA) + vcpkg_cmake_config_fixup(PACKAGE_NAME llama) +endif() + +vcpkg_copy_pdbs() +vcpkg_fixup_pkgconfig() + + +if(BUILD_LLAMA) + file(MAKE_DIRECTORY "${CURRENT_PACKAGES_DIR}/tools/${PORT}") + file(RENAME "${CURRENT_PACKAGES_DIR}/bin/convert_hf_to_gguf.py" "${CURRENT_PACKAGES_DIR}/tools/${PORT}/convert-hf-to-gguf.py") + file(INSTALL "${SOURCE_PATH}/gguf-py" DESTINATION "${CURRENT_PACKAGES_DIR}/tools/${PORT}") + file(RENAME "${CURRENT_PACKAGES_DIR}/bin/vulkan_profiling_analyzer.py" "${CURRENT_PACKAGES_DIR}/tools/${PORT}/vulkan_profiling_analyzer.py") +endif() + +if (NOT VCPKG_BUILD_TYPE) + file(REMOVE "${CURRENT_PACKAGES_DIR}/debug/bin/convert_hf_to_gguf.py") +endif() + +file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/include") +file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/share") + +if (VCPKG_LIBRARY_LINKAGE MATCHES "static") + file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/bin") + file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/bin") +endif() + +vcpkg_install_copyright(FILE_LIST "${SOURCE_PATH}/LICENSE") diff --git a/packages/classification-ggml/vcpkg/ports/qvac-fabric/vcpkg.json b/packages/classification-ggml/vcpkg/ports/qvac-fabric/vcpkg.json new file mode 100644 index 0000000000..48e7b449b8 --- /dev/null +++ b/packages/classification-ggml/vcpkg/ports/qvac-fabric/vcpkg.json @@ -0,0 +1,50 @@ +{ + "name": "qvac-fabric", + "version": "8828.0.2", + "port-version": 1, + "description": "LLM inference in C/C++", + "homepage": "https://github.com/tetherto/qvac-fabric-llm.cpp", + "license": "MIT", + "dependencies": [ + { + "name": "opencl", + "platform": "android" + }, + { + "name": "vcpkg-cmake", + "host": true + }, + { + "name": "vcpkg-cmake-config", + "host": true + } + ], + "default-features": [ + "gpu-backends", + "llama" + ], + "features": { + "force-profiler": { + "description": "Force vk performance logging in ggml" + }, + "gpu-backends": { + "description": "Build the GPU backends ggml ships per platform: Metal on Apple, Vulkan on Linux/Windows/Android, plus the Android backend-DL hybrid mode and OpenCL. Default-on so existing consumers (llamacpp-llm, llamacpp-embed, nmtcpp, diffusion-cpp) keep their current behaviour with default features. Disable to produce a CPU-only ggml build (useful for consumers like @qvac/classification-ggml that don't need GPU paths and want to skip the vulkan-sdk / metal / opencl build cost). Orthogonal to the 'llama' feature.", + "dependencies": [ + { + "name": "spirv-headers", + "platform": "!osx & !ios", + "version>=": "1.4.341.0" + } + ] + }, + "kleidiai": { + "description": "Enable ARM KleidiAI optimized kernels on Android." + }, + "llama": { + "description": "Build llama components." + }, + "openmp": { + "description": "Enable openmp on Android." + } + } +} diff --git a/packages/embed-llamacpp/vcpkg/ports/qvac-fabric/android-vulkan-version.cmake b/packages/embed-llamacpp/vcpkg/ports/qvac-fabric/android-vulkan-version.cmake index 485ea51bb9..21fba00feb 100644 --- a/packages/embed-llamacpp/vcpkg/ports/qvac-fabric/android-vulkan-version.cmake +++ b/packages/embed-llamacpp/vcpkg/ports/qvac-fabric/android-vulkan-version.cmake @@ -2,7 +2,7 @@ function(detect_ndk_vulkan_version) string(TOLOWER "${CMAKE_HOST_SYSTEM_NAME}" host_system_name_lower) - # CMAKE_HOST_SYSTEM_PROCESSOR is unavailable here. Use a glob pattern to complete the folder instead. + # CMAKE_HOST_SYSTEM_PROCESSOR is unavailable here. Use a glob pattern to complete the folder instead. file(GLOB host_dirs LIST_DIRECTORIES true "$ENV{ANDROID_NDK_HOME}/toolchains/llvm/prebuilt/${host_system_name_lower}-*") if(host_dirs) list(GET host_dirs 0 host_dir) diff --git a/packages/embed-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake b/packages/embed-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake index f0fe353cbe..ceb57f396a 100644 --- a/packages/embed-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake +++ b/packages/embed-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake @@ -1,9 +1,9 @@ vcpkg_from_github( - OUT_SOURCE_PATH SOURCE_PATH - REPO tetherto/qvac-fabric-llm.cpp - REF 97165a1a79650db28ec6a7d965b5905defcd5a23 - SHA512 92f0c54f648c1c30a879e31a5de934511aeb7da97ba0bdeb68caca8bff4ad43388b997288eccfdabb97f6a4c549a7f14f004688c52c99a8477108b2888132a73 - HEAD_REF qvac-b9341 + OUT_SOURCE_PATH SOURCE_PATH + REPO tetherto/qvac-fabric-llm.cpp + REF 1d0e4859d6bc97d64359075a1045c241e109abf9 + SHA512 caffa277b0acd45de435634c085e2c4d42ca00ba658dabae0d7ded41c9d83ea1974819c448a399f2849c2b8656b1c098267ea4daefd202fa127159d563ebe841 + HEAD_REF qvac-b9341 ) # Upstream CMake options only — passed through to vcpkg_cmake_configure. @@ -20,6 +20,7 @@ vcpkg_check_features( FEATURES gpu-backends BUILD_GPU_BACKENDS kleidiai BUILD_KLEIDIAI + openmp BUILD_OPENMP ) # gpu-backends is default-on via default-features in vcpkg.json. CPU-only @@ -106,6 +107,14 @@ if(VCPKG_TARGET_IS_ANDROID AND BUILD_KLEIDIAI) ) endif() +if(VCPKG_TARGET_IS_ANDROID AND BUILD_OPENMP) + message(STATUS "qvac-fabric: OpenMP for Android enabled") + list(APPEND PLATFORM_OPTIONS -DGGML_OPENMP=ON) +else() + message(STATUS "qvac-fabric: OpenMP Disabled") + list(APPEND PLATFORM_OPTIONS -DGGML_OPENMP=OFF) +endif() + if (VCPKG_TARGET_IS_ANDROID AND BUILD_GPU_BACKENDS) list(APPEND PLATFORM_OPTIONS -DGGML_OPENCL=ON) endif() @@ -136,7 +145,6 @@ vcpkg_cmake_configure( OPTIONS -DGGML_NATIVE=OFF -DGGML_CCACHE=OFF - -DGGML_OPENMP=OFF -DGGML_LLAMAFILE=OFF -DLLAMA_CURL=OFF -DLLAMA_BUILD_TESTS=OFF diff --git a/packages/embed-llamacpp/vcpkg/ports/qvac-fabric/vcpkg.json b/packages/embed-llamacpp/vcpkg/ports/qvac-fabric/vcpkg.json index 4c0c4fe5ef..48e7b449b8 100644 --- a/packages/embed-llamacpp/vcpkg/ports/qvac-fabric/vcpkg.json +++ b/packages/embed-llamacpp/vcpkg/ports/qvac-fabric/vcpkg.json @@ -1,6 +1,7 @@ { "name": "qvac-fabric", "version": "8828.0.2", + "port-version": 1, "description": "LLM inference in C/C++", "homepage": "https://github.com/tetherto/qvac-fabric-llm.cpp", "license": "MIT", @@ -29,11 +30,11 @@ "gpu-backends": { "description": "Build the GPU backends ggml ships per platform: Metal on Apple, Vulkan on Linux/Windows/Android, plus the Android backend-DL hybrid mode and OpenCL. Default-on so existing consumers (llamacpp-llm, llamacpp-embed, nmtcpp, diffusion-cpp) keep their current behaviour with default features. Disable to produce a CPU-only ggml build (useful for consumers like @qvac/classification-ggml that don't need GPU paths and want to skip the vulkan-sdk / metal / opencl build cost). Orthogonal to the 'llama' feature.", "dependencies": [ - { - "name": "spirv-headers", - "platform": "!osx & !ios", - "version>=": "1.4.341.0" - } + { + "name": "spirv-headers", + "platform": "!osx & !ios", + "version>=": "1.4.341.0" + } ] }, "kleidiai": { @@ -41,6 +42,9 @@ }, "llama": { "description": "Build llama components." + }, + "openmp": { + "description": "Enable openmp on Android." } } } diff --git a/packages/llm-llamacpp/vcpkg/ports/qvac-fabric/android-vulkan-version.cmake b/packages/llm-llamacpp/vcpkg/ports/qvac-fabric/android-vulkan-version.cmake index 485ea51bb9..21fba00feb 100644 --- a/packages/llm-llamacpp/vcpkg/ports/qvac-fabric/android-vulkan-version.cmake +++ b/packages/llm-llamacpp/vcpkg/ports/qvac-fabric/android-vulkan-version.cmake @@ -2,7 +2,7 @@ function(detect_ndk_vulkan_version) string(TOLOWER "${CMAKE_HOST_SYSTEM_NAME}" host_system_name_lower) - # CMAKE_HOST_SYSTEM_PROCESSOR is unavailable here. Use a glob pattern to complete the folder instead. + # CMAKE_HOST_SYSTEM_PROCESSOR is unavailable here. Use a glob pattern to complete the folder instead. file(GLOB host_dirs LIST_DIRECTORIES true "$ENV{ANDROID_NDK_HOME}/toolchains/llvm/prebuilt/${host_system_name_lower}-*") if(host_dirs) list(GET host_dirs 0 host_dir) diff --git a/packages/llm-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake b/packages/llm-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake index f0fe353cbe..ceb57f396a 100644 --- a/packages/llm-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake +++ b/packages/llm-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake @@ -1,9 +1,9 @@ vcpkg_from_github( - OUT_SOURCE_PATH SOURCE_PATH - REPO tetherto/qvac-fabric-llm.cpp - REF 97165a1a79650db28ec6a7d965b5905defcd5a23 - SHA512 92f0c54f648c1c30a879e31a5de934511aeb7da97ba0bdeb68caca8bff4ad43388b997288eccfdabb97f6a4c549a7f14f004688c52c99a8477108b2888132a73 - HEAD_REF qvac-b9341 + OUT_SOURCE_PATH SOURCE_PATH + REPO tetherto/qvac-fabric-llm.cpp + REF 1d0e4859d6bc97d64359075a1045c241e109abf9 + SHA512 caffa277b0acd45de435634c085e2c4d42ca00ba658dabae0d7ded41c9d83ea1974819c448a399f2849c2b8656b1c098267ea4daefd202fa127159d563ebe841 + HEAD_REF qvac-b9341 ) # Upstream CMake options only — passed through to vcpkg_cmake_configure. @@ -20,6 +20,7 @@ vcpkg_check_features( FEATURES gpu-backends BUILD_GPU_BACKENDS kleidiai BUILD_KLEIDIAI + openmp BUILD_OPENMP ) # gpu-backends is default-on via default-features in vcpkg.json. CPU-only @@ -106,6 +107,14 @@ if(VCPKG_TARGET_IS_ANDROID AND BUILD_KLEIDIAI) ) endif() +if(VCPKG_TARGET_IS_ANDROID AND BUILD_OPENMP) + message(STATUS "qvac-fabric: OpenMP for Android enabled") + list(APPEND PLATFORM_OPTIONS -DGGML_OPENMP=ON) +else() + message(STATUS "qvac-fabric: OpenMP Disabled") + list(APPEND PLATFORM_OPTIONS -DGGML_OPENMP=OFF) +endif() + if (VCPKG_TARGET_IS_ANDROID AND BUILD_GPU_BACKENDS) list(APPEND PLATFORM_OPTIONS -DGGML_OPENCL=ON) endif() @@ -136,7 +145,6 @@ vcpkg_cmake_configure( OPTIONS -DGGML_NATIVE=OFF -DGGML_CCACHE=OFF - -DGGML_OPENMP=OFF -DGGML_LLAMAFILE=OFF -DLLAMA_CURL=OFF -DLLAMA_BUILD_TESTS=OFF diff --git a/packages/llm-llamacpp/vcpkg/ports/qvac-fabric/vcpkg.json b/packages/llm-llamacpp/vcpkg/ports/qvac-fabric/vcpkg.json index 4c0c4fe5ef..48e7b449b8 100644 --- a/packages/llm-llamacpp/vcpkg/ports/qvac-fabric/vcpkg.json +++ b/packages/llm-llamacpp/vcpkg/ports/qvac-fabric/vcpkg.json @@ -1,6 +1,7 @@ { "name": "qvac-fabric", "version": "8828.0.2", + "port-version": 1, "description": "LLM inference in C/C++", "homepage": "https://github.com/tetherto/qvac-fabric-llm.cpp", "license": "MIT", @@ -29,11 +30,11 @@ "gpu-backends": { "description": "Build the GPU backends ggml ships per platform: Metal on Apple, Vulkan on Linux/Windows/Android, plus the Android backend-DL hybrid mode and OpenCL. Default-on so existing consumers (llamacpp-llm, llamacpp-embed, nmtcpp, diffusion-cpp) keep their current behaviour with default features. Disable to produce a CPU-only ggml build (useful for consumers like @qvac/classification-ggml that don't need GPU paths and want to skip the vulkan-sdk / metal / opencl build cost). Orthogonal to the 'llama' feature.", "dependencies": [ - { - "name": "spirv-headers", - "platform": "!osx & !ios", - "version>=": "1.4.341.0" - } + { + "name": "spirv-headers", + "platform": "!osx & !ios", + "version>=": "1.4.341.0" + } ] }, "kleidiai": { @@ -41,6 +42,9 @@ }, "llama": { "description": "Build llama components." + }, + "openmp": { + "description": "Enable openmp on Android." } } } diff --git a/packages/translation-nmtcpp/vcpkg-configuration.json b/packages/translation-nmtcpp/vcpkg-configuration.json index de484dd926..5584d197ff 100644 --- a/packages/translation-nmtcpp/vcpkg-configuration.json +++ b/packages/translation-nmtcpp/vcpkg-configuration.json @@ -4,6 +4,9 @@ "baseline": "c5955445a221df255d2204be964647c08c82c28f", "repository": "https://github.com/tetherto/qvac-registry-vcpkg.git" }, + "overlay-ports": [ + "vcpkg/ports" + ], "registries": [ { "kind": "git", diff --git a/packages/translation-nmtcpp/vcpkg/ports/qvac-fabric/android-vulkan-version.cmake b/packages/translation-nmtcpp/vcpkg/ports/qvac-fabric/android-vulkan-version.cmake new file mode 100644 index 0000000000..21fba00feb --- /dev/null +++ b/packages/translation-nmtcpp/vcpkg/ports/qvac-fabric/android-vulkan-version.cmake @@ -0,0 +1,36 @@ +# Function to detect Vulkan version from NDK vulkan_core.h +function(detect_ndk_vulkan_version) + string(TOLOWER "${CMAKE_HOST_SYSTEM_NAME}" host_system_name_lower) + + # CMAKE_HOST_SYSTEM_PROCESSOR is unavailable here. Use a glob pattern to complete the folder instead. + file(GLOB host_dirs LIST_DIRECTORIES true "$ENV{ANDROID_NDK_HOME}/toolchains/llvm/prebuilt/${host_system_name_lower}-*") + if(host_dirs) + list(GET host_dirs 0 host_dir) + get_filename_component(host_arch "${host_dir}" NAME) + set(vulkan_core_h "$ENV{ANDROID_NDK_HOME}/toolchains/llvm/prebuilt/${host_arch}/sysroot/usr/include/vulkan/vulkan_core.h") + else() + message(FATAL "Could not find NDK host directory for ${host_system_name_lower}") + endif() + + if(NOT vulkan_core_h) + message(FATAL "vulkan_core.h not found, using default version") + endif() + + file(READ "${vulkan_core_h}" header_content) + string(REGEX MATCH "VK_HEADER_VERSION ([0-9]+)" version_match "${header_content}") + if(version_match) + set(header_version_3 "${CMAKE_MATCH_1}") + else() + message(FATAL "Could not extract VK_HEADER_VERSION from vulkan_core.h, using default: ${vulkan_version}") + endif() + + # Extract major.minor version from VK_HEADER_VERSION_COMPLETE for download URL + string(REGEX MATCH "VK_HEADER_VERSION_COMPLETE VK_MAKE_API_VERSION\\(([0-9]+), ([0-9]+), ([0-9]+)" version_match "${header_content}") + if(version_match) + set(major "${CMAKE_MATCH_2}") + set(minor "${CMAKE_MATCH_3}") + set(vulkan_version "${major}.${minor}.${header_version_3}" PARENT_SCOPE) + else() + message(FATAL "Could not extract major.minor version from vulkan_core.h, using default: ${vulkan_version}") + endif() +endfunction() diff --git a/packages/translation-nmtcpp/vcpkg/ports/qvac-fabric/portfile.cmake b/packages/translation-nmtcpp/vcpkg/ports/qvac-fabric/portfile.cmake new file mode 100644 index 0000000000..ceb57f396a --- /dev/null +++ b/packages/translation-nmtcpp/vcpkg/ports/qvac-fabric/portfile.cmake @@ -0,0 +1,191 @@ +vcpkg_from_github( + OUT_SOURCE_PATH SOURCE_PATH + REPO tetherto/qvac-fabric-llm.cpp + REF 1d0e4859d6bc97d64359075a1045c241e109abf9 + SHA512 caffa277b0acd45de435634c085e2c4d42ca00ba658dabae0d7ded41c9d83ea1974819c448a399f2849c2b8656b1c098267ea4daefd202fa127159d563ebe841 + HEAD_REF qvac-b9341 +) + +# Upstream CMake options only — passed through to vcpkg_cmake_configure. +vcpkg_check_features( + OUT_FEATURE_OPTIONS FEATURE_OPTIONS + FEATURES + force-profiler FORCE_GGML_VK_PERF_LOGGER + llama BUILD_LLAMA +) + +# Portfile-only feature flags (drive PLATFORM_OPTIONS; not upstream cache vars). +vcpkg_check_features( + OUT_FEATURE_OPTIONS _PORTFILE_FEATURE_OPTIONS + FEATURES + gpu-backends BUILD_GPU_BACKENDS + kleidiai BUILD_KLEIDIAI + openmp BUILD_OPENMP +) + +# gpu-backends is default-on via default-features in vcpkg.json. CPU-only +# consumers (e.g. @qvac/classification-ggml) disable it with +# default-features:false (and re-add 'llama' if needed). +if(NOT BUILD_GPU_BACKENDS) + message(STATUS "qvac-fabric: gpu-backends feature OFF — building CPU-only ggml (no Metal/Vulkan/CUDA/OpenCL)") +endif() + +if (VCPKG_TARGET_IS_ANDROID AND BUILD_GPU_BACKENDS) + # NDK only comes with C headers. + # Make sure C++ header exists, it will be used by ggml tensor library. + # Need to determine installed vulkan version and download correct headers + include(${CMAKE_CURRENT_LIST_DIR}/android-vulkan-version.cmake) + detect_ndk_vulkan_version() + message(STATUS "Using Vulkan C++ wrappers from version: ${vulkan_version}") + file(DOWNLOAD + "https://github.com/KhronosGroup/Vulkan-Headers/archive/refs/tags/v${vulkan_version}.tar.gz" + "${SOURCE_PATH}/vulkan-sdk-${vulkan_version}.tar.gz" + TLS_VERIFY ON + ) + + file(ARCHIVE_EXTRACT + INPUT "${SOURCE_PATH}/vulkan-sdk-${vulkan_version}.tar.gz" + DESTINATION "${SOURCE_PATH}" + PATTERNS "*.hpp" + ) + + file(RENAME + "${SOURCE_PATH}/Vulkan-Headers-${vulkan_version}" + "${SOURCE_PATH}/ggml/src/ggml-vulkan/vulkan_cpp_wrapper" + ) +endif() + +set(PLATFORM_OPTIONS) + +if(NOT BUILD_GPU_BACKENDS) + # Force every GPU backend off explicitly, in case upstream defaults change. + list(APPEND PLATFORM_OPTIONS + -DGGML_METAL=OFF + -DGGML_VULKAN=OFF + -DGGML_CUDA=OFF + -DGGML_OPENCL=OFF + ) + if (VCPKG_TARGET_IS_IOS) + # Same iOS BLAS/Accelerate gating as the GPU-on path; unrelated to the + # CPU-vs-GPU split, an iOS-toolchain workaround for missing frameworks. + list(APPEND PLATFORM_OPTIONS -DGGML_BLAS=OFF -DGGML_ACCELERATE=OFF) + endif() +elseif (VCPKG_TARGET_IS_OSX OR VCPKG_TARGET_IS_IOS) + list(APPEND PLATFORM_OPTIONS -DGGML_METAL=ON) + if (VCPKG_TARGET_IS_IOS) + list(APPEND PLATFORM_OPTIONS -DGGML_BLAS=OFF -DGGML_ACCELERATE=OFF) + endif() +else() + list(APPEND PLATFORM_OPTIONS -DGGML_VULKAN=ON) +endif() + +# Android: always build CPU variants (NEON_DOTPROD, NEON_I8MM, etc.) and CPU +# repacking. These are CPU-only runtime optimizations selected based on the +# device's SIMD capabilities at load time, completely orthogonal to the GPU +# backends. Bundling them is essential for good CPU inference performance on +# the wide range of arm64 devices the addons ship to. Requires GGML_BACKEND_DL +# to dispatch the variants at runtime; the existing #ifdef guard around +# `ggml_backend_load_all_from_path()` in ggml-backend-reg.cpp keeps the search +# scoped to the consumer's own prebuilds dir. +if(VCPKG_TARGET_IS_ANDROID) + set(DL_BACKENDS ON) + list(APPEND PLATFORM_OPTIONS + -DGGML_BACKEND_DL=ON + -DGGML_CPU_ALL_VARIANTS=ON + -DGGML_CPU_REPACK=ON) +else() + set(DL_BACKENDS OFF) +endif() + +if(VCPKG_TARGET_IS_ANDROID AND BUILD_KLEIDIAI) + message(STATUS "qvac-fabric: kleidiai feature ON — building with ARM KleidiAI optimized kernels") + # ggml only vendors KleidiAI via FetchContent; registry vcpkg-cmake sets + # FETCHCONTENT_FULLY_DISCONNECTED=ON globally, so allow the download here. + list(APPEND PLATFORM_OPTIONS + -DGGML_CPU_KLEIDIAI=ON + -DFETCHCONTENT_FULLY_DISCONNECTED=OFF + ) +endif() + +if(VCPKG_TARGET_IS_ANDROID AND BUILD_OPENMP) + message(STATUS "qvac-fabric: OpenMP for Android enabled") + list(APPEND PLATFORM_OPTIONS -DGGML_OPENMP=ON) +else() + message(STATUS "qvac-fabric: OpenMP Disabled") + list(APPEND PLATFORM_OPTIONS -DGGML_OPENMP=OFF) +endif() + +if (VCPKG_TARGET_IS_ANDROID AND BUILD_GPU_BACKENDS) + list(APPEND PLATFORM_OPTIONS -DGGML_OPENCL=ON) +endif() + +if(BUILD_GPU_BACKENDS AND NOT VCPKG_TARGET_IS_OSX AND NOT VCPKG_TARGET_IS_IOS) + if(VCPKG_TARGET_IS_WINDOWS AND NOT VCPKG_TARGET_IS_MINGW) + string(APPEND VCPKG_C_FLAGS " /I${CURRENT_INSTALLED_DIR}/include") + string(APPEND VCPKG_CXX_FLAGS " /I${CURRENT_INSTALLED_DIR}/include") + else() + string(APPEND VCPKG_C_FLAGS " -isystem ${CURRENT_INSTALLED_DIR}/include") + string(APPEND VCPKG_CXX_FLAGS " -isystem ${CURRENT_INSTALLED_DIR}/include") + endif() +endif() + +set(LLAMA_OPTIONS) +if("llama" IN_LIST FEATURES) + list(APPEND LLAMA_OPTIONS -DLLAMA_MTMD=ON) +else() + list(APPEND LLAMA_OPTIONS + -DLLAMA_MTMD=OFF + -DLLAMA_BUILD_COMMON=OFF + ) +endif() + +vcpkg_cmake_configure( + SOURCE_PATH "${SOURCE_PATH}" + DISABLE_PARALLEL_CONFIGURE + OPTIONS + -DGGML_NATIVE=OFF + -DGGML_CCACHE=OFF + -DGGML_LLAMAFILE=OFF + -DLLAMA_CURL=OFF + -DLLAMA_BUILD_TESTS=OFF + -DLLAMA_BUILD_TOOLS=OFF + -DLLAMA_BUILD_EXAMPLES=OFF + -DLLAMA_BUILD_SERVER=OFF + -DLLAMA_ALL_WARNINGS=OFF + ${LLAMA_OPTIONS} + ${PLATFORM_OPTIONS} + ${FEATURE_OPTIONS} +) + +vcpkg_cmake_install() +vcpkg_cmake_config_fixup( + PACKAGE_NAME ggml) + +if(BUILD_LLAMA) + vcpkg_cmake_config_fixup(PACKAGE_NAME llama) +endif() + +vcpkg_copy_pdbs() +vcpkg_fixup_pkgconfig() + + +if(BUILD_LLAMA) + file(MAKE_DIRECTORY "${CURRENT_PACKAGES_DIR}/tools/${PORT}") + file(RENAME "${CURRENT_PACKAGES_DIR}/bin/convert_hf_to_gguf.py" "${CURRENT_PACKAGES_DIR}/tools/${PORT}/convert-hf-to-gguf.py") + file(INSTALL "${SOURCE_PATH}/gguf-py" DESTINATION "${CURRENT_PACKAGES_DIR}/tools/${PORT}") + file(RENAME "${CURRENT_PACKAGES_DIR}/bin/vulkan_profiling_analyzer.py" "${CURRENT_PACKAGES_DIR}/tools/${PORT}/vulkan_profiling_analyzer.py") +endif() + +if (NOT VCPKG_BUILD_TYPE) + file(REMOVE "${CURRENT_PACKAGES_DIR}/debug/bin/convert_hf_to_gguf.py") +endif() + +file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/include") +file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/share") + +if (VCPKG_LIBRARY_LINKAGE MATCHES "static") + file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/bin") + file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/bin") +endif() + +vcpkg_install_copyright(FILE_LIST "${SOURCE_PATH}/LICENSE") diff --git a/packages/translation-nmtcpp/vcpkg/ports/qvac-fabric/vcpkg.json b/packages/translation-nmtcpp/vcpkg/ports/qvac-fabric/vcpkg.json new file mode 100644 index 0000000000..48e7b449b8 --- /dev/null +++ b/packages/translation-nmtcpp/vcpkg/ports/qvac-fabric/vcpkg.json @@ -0,0 +1,50 @@ +{ + "name": "qvac-fabric", + "version": "8828.0.2", + "port-version": 1, + "description": "LLM inference in C/C++", + "homepage": "https://github.com/tetherto/qvac-fabric-llm.cpp", + "license": "MIT", + "dependencies": [ + { + "name": "opencl", + "platform": "android" + }, + { + "name": "vcpkg-cmake", + "host": true + }, + { + "name": "vcpkg-cmake-config", + "host": true + } + ], + "default-features": [ + "gpu-backends", + "llama" + ], + "features": { + "force-profiler": { + "description": "Force vk performance logging in ggml" + }, + "gpu-backends": { + "description": "Build the GPU backends ggml ships per platform: Metal on Apple, Vulkan on Linux/Windows/Android, plus the Android backend-DL hybrid mode and OpenCL. Default-on so existing consumers (llamacpp-llm, llamacpp-embed, nmtcpp, diffusion-cpp) keep their current behaviour with default features. Disable to produce a CPU-only ggml build (useful for consumers like @qvac/classification-ggml that don't need GPU paths and want to skip the vulkan-sdk / metal / opencl build cost). Orthogonal to the 'llama' feature.", + "dependencies": [ + { + "name": "spirv-headers", + "platform": "!osx & !ios", + "version>=": "1.4.341.0" + } + ] + }, + "kleidiai": { + "description": "Enable ARM KleidiAI optimized kernels on Android." + }, + "llama": { + "description": "Build llama components." + }, + "openmp": { + "description": "Enable openmp on Android." + } + } +} diff --git a/packages/vla-ggml/vcpkg-configuration.json b/packages/vla-ggml/vcpkg-configuration.json index b0ca350e27..67ed4d1b80 100644 --- a/packages/vla-ggml/vcpkg-configuration.json +++ b/packages/vla-ggml/vcpkg-configuration.json @@ -4,6 +4,9 @@ "baseline": "a9eae49a7c95a63755c0b44ecf9d74b0a00e181b", "repository": "https://github.com/tetherto/qvac-registry-vcpkg.git" }, + "overlay-ports": [ + "vcpkg/ports" + ], "registries": [ { "kind": "git", diff --git a/packages/vla-ggml/vcpkg/ports/qvac-fabric/android-vulkan-version.cmake b/packages/vla-ggml/vcpkg/ports/qvac-fabric/android-vulkan-version.cmake new file mode 100644 index 0000000000..21fba00feb --- /dev/null +++ b/packages/vla-ggml/vcpkg/ports/qvac-fabric/android-vulkan-version.cmake @@ -0,0 +1,36 @@ +# Function to detect Vulkan version from NDK vulkan_core.h +function(detect_ndk_vulkan_version) + string(TOLOWER "${CMAKE_HOST_SYSTEM_NAME}" host_system_name_lower) + + # CMAKE_HOST_SYSTEM_PROCESSOR is unavailable here. Use a glob pattern to complete the folder instead. + file(GLOB host_dirs LIST_DIRECTORIES true "$ENV{ANDROID_NDK_HOME}/toolchains/llvm/prebuilt/${host_system_name_lower}-*") + if(host_dirs) + list(GET host_dirs 0 host_dir) + get_filename_component(host_arch "${host_dir}" NAME) + set(vulkan_core_h "$ENV{ANDROID_NDK_HOME}/toolchains/llvm/prebuilt/${host_arch}/sysroot/usr/include/vulkan/vulkan_core.h") + else() + message(FATAL "Could not find NDK host directory for ${host_system_name_lower}") + endif() + + if(NOT vulkan_core_h) + message(FATAL "vulkan_core.h not found, using default version") + endif() + + file(READ "${vulkan_core_h}" header_content) + string(REGEX MATCH "VK_HEADER_VERSION ([0-9]+)" version_match "${header_content}") + if(version_match) + set(header_version_3 "${CMAKE_MATCH_1}") + else() + message(FATAL "Could not extract VK_HEADER_VERSION from vulkan_core.h, using default: ${vulkan_version}") + endif() + + # Extract major.minor version from VK_HEADER_VERSION_COMPLETE for download URL + string(REGEX MATCH "VK_HEADER_VERSION_COMPLETE VK_MAKE_API_VERSION\\(([0-9]+), ([0-9]+), ([0-9]+)" version_match "${header_content}") + if(version_match) + set(major "${CMAKE_MATCH_2}") + set(minor "${CMAKE_MATCH_3}") + set(vulkan_version "${major}.${minor}.${header_version_3}" PARENT_SCOPE) + else() + message(FATAL "Could not extract major.minor version from vulkan_core.h, using default: ${vulkan_version}") + endif() +endfunction() diff --git a/packages/vla-ggml/vcpkg/ports/qvac-fabric/portfile.cmake b/packages/vla-ggml/vcpkg/ports/qvac-fabric/portfile.cmake new file mode 100644 index 0000000000..ceb57f396a --- /dev/null +++ b/packages/vla-ggml/vcpkg/ports/qvac-fabric/portfile.cmake @@ -0,0 +1,191 @@ +vcpkg_from_github( + OUT_SOURCE_PATH SOURCE_PATH + REPO tetherto/qvac-fabric-llm.cpp + REF 1d0e4859d6bc97d64359075a1045c241e109abf9 + SHA512 caffa277b0acd45de435634c085e2c4d42ca00ba658dabae0d7ded41c9d83ea1974819c448a399f2849c2b8656b1c098267ea4daefd202fa127159d563ebe841 + HEAD_REF qvac-b9341 +) + +# Upstream CMake options only — passed through to vcpkg_cmake_configure. +vcpkg_check_features( + OUT_FEATURE_OPTIONS FEATURE_OPTIONS + FEATURES + force-profiler FORCE_GGML_VK_PERF_LOGGER + llama BUILD_LLAMA +) + +# Portfile-only feature flags (drive PLATFORM_OPTIONS; not upstream cache vars). +vcpkg_check_features( + OUT_FEATURE_OPTIONS _PORTFILE_FEATURE_OPTIONS + FEATURES + gpu-backends BUILD_GPU_BACKENDS + kleidiai BUILD_KLEIDIAI + openmp BUILD_OPENMP +) + +# gpu-backends is default-on via default-features in vcpkg.json. CPU-only +# consumers (e.g. @qvac/classification-ggml) disable it with +# default-features:false (and re-add 'llama' if needed). +if(NOT BUILD_GPU_BACKENDS) + message(STATUS "qvac-fabric: gpu-backends feature OFF — building CPU-only ggml (no Metal/Vulkan/CUDA/OpenCL)") +endif() + +if (VCPKG_TARGET_IS_ANDROID AND BUILD_GPU_BACKENDS) + # NDK only comes with C headers. + # Make sure C++ header exists, it will be used by ggml tensor library. + # Need to determine installed vulkan version and download correct headers + include(${CMAKE_CURRENT_LIST_DIR}/android-vulkan-version.cmake) + detect_ndk_vulkan_version() + message(STATUS "Using Vulkan C++ wrappers from version: ${vulkan_version}") + file(DOWNLOAD + "https://github.com/KhronosGroup/Vulkan-Headers/archive/refs/tags/v${vulkan_version}.tar.gz" + "${SOURCE_PATH}/vulkan-sdk-${vulkan_version}.tar.gz" + TLS_VERIFY ON + ) + + file(ARCHIVE_EXTRACT + INPUT "${SOURCE_PATH}/vulkan-sdk-${vulkan_version}.tar.gz" + DESTINATION "${SOURCE_PATH}" + PATTERNS "*.hpp" + ) + + file(RENAME + "${SOURCE_PATH}/Vulkan-Headers-${vulkan_version}" + "${SOURCE_PATH}/ggml/src/ggml-vulkan/vulkan_cpp_wrapper" + ) +endif() + +set(PLATFORM_OPTIONS) + +if(NOT BUILD_GPU_BACKENDS) + # Force every GPU backend off explicitly, in case upstream defaults change. + list(APPEND PLATFORM_OPTIONS + -DGGML_METAL=OFF + -DGGML_VULKAN=OFF + -DGGML_CUDA=OFF + -DGGML_OPENCL=OFF + ) + if (VCPKG_TARGET_IS_IOS) + # Same iOS BLAS/Accelerate gating as the GPU-on path; unrelated to the + # CPU-vs-GPU split, an iOS-toolchain workaround for missing frameworks. + list(APPEND PLATFORM_OPTIONS -DGGML_BLAS=OFF -DGGML_ACCELERATE=OFF) + endif() +elseif (VCPKG_TARGET_IS_OSX OR VCPKG_TARGET_IS_IOS) + list(APPEND PLATFORM_OPTIONS -DGGML_METAL=ON) + if (VCPKG_TARGET_IS_IOS) + list(APPEND PLATFORM_OPTIONS -DGGML_BLAS=OFF -DGGML_ACCELERATE=OFF) + endif() +else() + list(APPEND PLATFORM_OPTIONS -DGGML_VULKAN=ON) +endif() + +# Android: always build CPU variants (NEON_DOTPROD, NEON_I8MM, etc.) and CPU +# repacking. These are CPU-only runtime optimizations selected based on the +# device's SIMD capabilities at load time, completely orthogonal to the GPU +# backends. Bundling them is essential for good CPU inference performance on +# the wide range of arm64 devices the addons ship to. Requires GGML_BACKEND_DL +# to dispatch the variants at runtime; the existing #ifdef guard around +# `ggml_backend_load_all_from_path()` in ggml-backend-reg.cpp keeps the search +# scoped to the consumer's own prebuilds dir. +if(VCPKG_TARGET_IS_ANDROID) + set(DL_BACKENDS ON) + list(APPEND PLATFORM_OPTIONS + -DGGML_BACKEND_DL=ON + -DGGML_CPU_ALL_VARIANTS=ON + -DGGML_CPU_REPACK=ON) +else() + set(DL_BACKENDS OFF) +endif() + +if(VCPKG_TARGET_IS_ANDROID AND BUILD_KLEIDIAI) + message(STATUS "qvac-fabric: kleidiai feature ON — building with ARM KleidiAI optimized kernels") + # ggml only vendors KleidiAI via FetchContent; registry vcpkg-cmake sets + # FETCHCONTENT_FULLY_DISCONNECTED=ON globally, so allow the download here. + list(APPEND PLATFORM_OPTIONS + -DGGML_CPU_KLEIDIAI=ON + -DFETCHCONTENT_FULLY_DISCONNECTED=OFF + ) +endif() + +if(VCPKG_TARGET_IS_ANDROID AND BUILD_OPENMP) + message(STATUS "qvac-fabric: OpenMP for Android enabled") + list(APPEND PLATFORM_OPTIONS -DGGML_OPENMP=ON) +else() + message(STATUS "qvac-fabric: OpenMP Disabled") + list(APPEND PLATFORM_OPTIONS -DGGML_OPENMP=OFF) +endif() + +if (VCPKG_TARGET_IS_ANDROID AND BUILD_GPU_BACKENDS) + list(APPEND PLATFORM_OPTIONS -DGGML_OPENCL=ON) +endif() + +if(BUILD_GPU_BACKENDS AND NOT VCPKG_TARGET_IS_OSX AND NOT VCPKG_TARGET_IS_IOS) + if(VCPKG_TARGET_IS_WINDOWS AND NOT VCPKG_TARGET_IS_MINGW) + string(APPEND VCPKG_C_FLAGS " /I${CURRENT_INSTALLED_DIR}/include") + string(APPEND VCPKG_CXX_FLAGS " /I${CURRENT_INSTALLED_DIR}/include") + else() + string(APPEND VCPKG_C_FLAGS " -isystem ${CURRENT_INSTALLED_DIR}/include") + string(APPEND VCPKG_CXX_FLAGS " -isystem ${CURRENT_INSTALLED_DIR}/include") + endif() +endif() + +set(LLAMA_OPTIONS) +if("llama" IN_LIST FEATURES) + list(APPEND LLAMA_OPTIONS -DLLAMA_MTMD=ON) +else() + list(APPEND LLAMA_OPTIONS + -DLLAMA_MTMD=OFF + -DLLAMA_BUILD_COMMON=OFF + ) +endif() + +vcpkg_cmake_configure( + SOURCE_PATH "${SOURCE_PATH}" + DISABLE_PARALLEL_CONFIGURE + OPTIONS + -DGGML_NATIVE=OFF + -DGGML_CCACHE=OFF + -DGGML_LLAMAFILE=OFF + -DLLAMA_CURL=OFF + -DLLAMA_BUILD_TESTS=OFF + -DLLAMA_BUILD_TOOLS=OFF + -DLLAMA_BUILD_EXAMPLES=OFF + -DLLAMA_BUILD_SERVER=OFF + -DLLAMA_ALL_WARNINGS=OFF + ${LLAMA_OPTIONS} + ${PLATFORM_OPTIONS} + ${FEATURE_OPTIONS} +) + +vcpkg_cmake_install() +vcpkg_cmake_config_fixup( + PACKAGE_NAME ggml) + +if(BUILD_LLAMA) + vcpkg_cmake_config_fixup(PACKAGE_NAME llama) +endif() + +vcpkg_copy_pdbs() +vcpkg_fixup_pkgconfig() + + +if(BUILD_LLAMA) + file(MAKE_DIRECTORY "${CURRENT_PACKAGES_DIR}/tools/${PORT}") + file(RENAME "${CURRENT_PACKAGES_DIR}/bin/convert_hf_to_gguf.py" "${CURRENT_PACKAGES_DIR}/tools/${PORT}/convert-hf-to-gguf.py") + file(INSTALL "${SOURCE_PATH}/gguf-py" DESTINATION "${CURRENT_PACKAGES_DIR}/tools/${PORT}") + file(RENAME "${CURRENT_PACKAGES_DIR}/bin/vulkan_profiling_analyzer.py" "${CURRENT_PACKAGES_DIR}/tools/${PORT}/vulkan_profiling_analyzer.py") +endif() + +if (NOT VCPKG_BUILD_TYPE) + file(REMOVE "${CURRENT_PACKAGES_DIR}/debug/bin/convert_hf_to_gguf.py") +endif() + +file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/include") +file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/share") + +if (VCPKG_LIBRARY_LINKAGE MATCHES "static") + file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/bin") + file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/bin") +endif() + +vcpkg_install_copyright(FILE_LIST "${SOURCE_PATH}/LICENSE") diff --git a/packages/vla-ggml/vcpkg/ports/qvac-fabric/vcpkg.json b/packages/vla-ggml/vcpkg/ports/qvac-fabric/vcpkg.json new file mode 100644 index 0000000000..48e7b449b8 --- /dev/null +++ b/packages/vla-ggml/vcpkg/ports/qvac-fabric/vcpkg.json @@ -0,0 +1,50 @@ +{ + "name": "qvac-fabric", + "version": "8828.0.2", + "port-version": 1, + "description": "LLM inference in C/C++", + "homepage": "https://github.com/tetherto/qvac-fabric-llm.cpp", + "license": "MIT", + "dependencies": [ + { + "name": "opencl", + "platform": "android" + }, + { + "name": "vcpkg-cmake", + "host": true + }, + { + "name": "vcpkg-cmake-config", + "host": true + } + ], + "default-features": [ + "gpu-backends", + "llama" + ], + "features": { + "force-profiler": { + "description": "Force vk performance logging in ggml" + }, + "gpu-backends": { + "description": "Build the GPU backends ggml ships per platform: Metal on Apple, Vulkan on Linux/Windows/Android, plus the Android backend-DL hybrid mode and OpenCL. Default-on so existing consumers (llamacpp-llm, llamacpp-embed, nmtcpp, diffusion-cpp) keep their current behaviour with default features. Disable to produce a CPU-only ggml build (useful for consumers like @qvac/classification-ggml that don't need GPU paths and want to skip the vulkan-sdk / metal / opencl build cost). Orthogonal to the 'llama' feature.", + "dependencies": [ + { + "name": "spirv-headers", + "platform": "!osx & !ios", + "version>=": "1.4.341.0" + } + ] + }, + "kleidiai": { + "description": "Enable ARM KleidiAI optimized kernels on Android." + }, + "llama": { + "description": "Build llama components." + }, + "openmp": { + "description": "Enable openmp on Android." + } + } +} From 5655f6a640f28a7e6f50b8c213fc1d78496149c1 Mon Sep 17 00:00:00 2001 From: Marcus Edel Date: Wed, 3 Jun 2026 16:17:19 +0000 Subject: [PATCH 08/58] fix: widen reasoning_budget TS type to number so positive token-cap values. Signed-off-by: Marcus Edel --- packages/llm-llamacpp/index.d.ts | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/packages/llm-llamacpp/index.d.ts b/packages/llm-llamacpp/index.d.ts index aa8a754831..491e19ab55 100644 --- a/packages/llm-llamacpp/index.d.ts +++ b/packages/llm-llamacpp/index.d.ts @@ -60,8 +60,13 @@ export interface LlamaConfig { 'cache-type-v'?: string /** Writable directory for OpenCL kernel binary cache. Required on Android for fast GPU startup. */ openclCacheDir?: string - /** Reasoning channel budget. `-1` (default) leaves the model's reasoning channel on; `0` disables it. */ - reasoning_budget?: -1 | 0 | '-1' | '0' + /** + * Reasoning channel budget. `-1` (default) leaves the model's reasoning + * channel on; `0` disables it; any positive integer caps the reasoning + * channel at that many tokens (the sampler force-emits `` once + * the budget is exhausted). + */ + reasoning_budget?: number | `${number}` [key: string]: string | number | boolean | string[] | undefined } @@ -136,11 +141,12 @@ export interface GenerationParams { json_schema?: string | Record /** * Per-request reasoning channel budget. `-1` keeps the model's reasoning - * channel on; `0` disables it for this request. Equivalent to the load-time + * channel on; `0` disables it for this request; any positive integer caps + * the reasoning channel at that many tokens. Equivalent to the load-time * `reasoning_budget` config but scoped to a single `run()` call; the prior * value is restored afterwards. */ - reasoning_budget?: -1 | 0 + reasoning_budget?: number } export interface RunOptions { From 8fd7f20e7a10d7a0bea727650136292d5b5dfc99 Mon Sep 17 00:00:00 2001 From: Marcus Edel Date: Wed, 3 Jun 2026 16:19:01 +0000 Subject: [PATCH 09/58] Point to the latest fabric changes. Signed-off-by: Marcus Edel --- .../vcpkg/ports/qvac-fabric/portfile.cmake | 4 ++-- .../embed-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake | 4 ++-- packages/llm-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake | 4 ++-- .../translation-nmtcpp/vcpkg/ports/qvac-fabric/portfile.cmake | 4 ++-- packages/vla-ggml/vcpkg/ports/qvac-fabric/portfile.cmake | 4 ++-- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/packages/classification-ggml/vcpkg/ports/qvac-fabric/portfile.cmake b/packages/classification-ggml/vcpkg/ports/qvac-fabric/portfile.cmake index ceb57f396a..03dc0ab153 100644 --- a/packages/classification-ggml/vcpkg/ports/qvac-fabric/portfile.cmake +++ b/packages/classification-ggml/vcpkg/ports/qvac-fabric/portfile.cmake @@ -1,8 +1,8 @@ vcpkg_from_github( OUT_SOURCE_PATH SOURCE_PATH REPO tetherto/qvac-fabric-llm.cpp - REF 1d0e4859d6bc97d64359075a1045c241e109abf9 - SHA512 caffa277b0acd45de435634c085e2c4d42ca00ba658dabae0d7ded41c9d83ea1974819c448a399f2849c2b8656b1c098267ea4daefd202fa127159d563ebe841 + REF 3329743b43473326baaabd5354ee21f99e964f36 + SHA512 e97ac4cbed42be89051580bdbc19d15c0c811e8a4f470ddbafb10b42ad7990279c27f47d3f59006463a9c5c06d11a1ff70c1611024b53d198b45d33b5ff4f637 HEAD_REF qvac-b9341 ) diff --git a/packages/embed-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake b/packages/embed-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake index ceb57f396a..03dc0ab153 100644 --- a/packages/embed-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake +++ b/packages/embed-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake @@ -1,8 +1,8 @@ vcpkg_from_github( OUT_SOURCE_PATH SOURCE_PATH REPO tetherto/qvac-fabric-llm.cpp - REF 1d0e4859d6bc97d64359075a1045c241e109abf9 - SHA512 caffa277b0acd45de435634c085e2c4d42ca00ba658dabae0d7ded41c9d83ea1974819c448a399f2849c2b8656b1c098267ea4daefd202fa127159d563ebe841 + REF 3329743b43473326baaabd5354ee21f99e964f36 + SHA512 e97ac4cbed42be89051580bdbc19d15c0c811e8a4f470ddbafb10b42ad7990279c27f47d3f59006463a9c5c06d11a1ff70c1611024b53d198b45d33b5ff4f637 HEAD_REF qvac-b9341 ) diff --git a/packages/llm-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake b/packages/llm-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake index ceb57f396a..03dc0ab153 100644 --- a/packages/llm-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake +++ b/packages/llm-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake @@ -1,8 +1,8 @@ vcpkg_from_github( OUT_SOURCE_PATH SOURCE_PATH REPO tetherto/qvac-fabric-llm.cpp - REF 1d0e4859d6bc97d64359075a1045c241e109abf9 - SHA512 caffa277b0acd45de435634c085e2c4d42ca00ba658dabae0d7ded41c9d83ea1974819c448a399f2849c2b8656b1c098267ea4daefd202fa127159d563ebe841 + REF 3329743b43473326baaabd5354ee21f99e964f36 + SHA512 e97ac4cbed42be89051580bdbc19d15c0c811e8a4f470ddbafb10b42ad7990279c27f47d3f59006463a9c5c06d11a1ff70c1611024b53d198b45d33b5ff4f637 HEAD_REF qvac-b9341 ) diff --git a/packages/translation-nmtcpp/vcpkg/ports/qvac-fabric/portfile.cmake b/packages/translation-nmtcpp/vcpkg/ports/qvac-fabric/portfile.cmake index ceb57f396a..03dc0ab153 100644 --- a/packages/translation-nmtcpp/vcpkg/ports/qvac-fabric/portfile.cmake +++ b/packages/translation-nmtcpp/vcpkg/ports/qvac-fabric/portfile.cmake @@ -1,8 +1,8 @@ vcpkg_from_github( OUT_SOURCE_PATH SOURCE_PATH REPO tetherto/qvac-fabric-llm.cpp - REF 1d0e4859d6bc97d64359075a1045c241e109abf9 - SHA512 caffa277b0acd45de435634c085e2c4d42ca00ba658dabae0d7ded41c9d83ea1974819c448a399f2849c2b8656b1c098267ea4daefd202fa127159d563ebe841 + REF 3329743b43473326baaabd5354ee21f99e964f36 + SHA512 e97ac4cbed42be89051580bdbc19d15c0c811e8a4f470ddbafb10b42ad7990279c27f47d3f59006463a9c5c06d11a1ff70c1611024b53d198b45d33b5ff4f637 HEAD_REF qvac-b9341 ) diff --git a/packages/vla-ggml/vcpkg/ports/qvac-fabric/portfile.cmake b/packages/vla-ggml/vcpkg/ports/qvac-fabric/portfile.cmake index ceb57f396a..03dc0ab153 100644 --- a/packages/vla-ggml/vcpkg/ports/qvac-fabric/portfile.cmake +++ b/packages/vla-ggml/vcpkg/ports/qvac-fabric/portfile.cmake @@ -1,8 +1,8 @@ vcpkg_from_github( OUT_SOURCE_PATH SOURCE_PATH REPO tetherto/qvac-fabric-llm.cpp - REF 1d0e4859d6bc97d64359075a1045c241e109abf9 - SHA512 caffa277b0acd45de435634c085e2c4d42ca00ba658dabae0d7ded41c9d83ea1974819c448a399f2849c2b8656b1c098267ea4daefd202fa127159d563ebe841 + REF 3329743b43473326baaabd5354ee21f99e964f36 + SHA512 e97ac4cbed42be89051580bdbc19d15c0c811e8a4f470ddbafb10b42ad7990279c27f47d3f59006463a9c5c06d11a1ff70c1611024b53d198b45d33b5ff4f637 HEAD_REF qvac-b9341 ) From 15ac661a3a29a03137a7c35b2043a347b552c45c Mon Sep 17 00:00:00 2001 From: Marcus Edel Date: Wed, 3 Jun 2026 17:17:46 +0000 Subject: [PATCH 10/58] fix: capture streamed output whenever tools are configured and feed parsed.content into the normalized envelope so the assistant text exposes exactly one representation per invocation. Signed-off-by: Marcus Edel --- .../llm-llamacpp/addon/src/model-interface/LlamaModel.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/packages/llm-llamacpp/addon/src/model-interface/LlamaModel.cpp b/packages/llm-llamacpp/addon/src/model-interface/LlamaModel.cpp index 635dc3ee21..c0bb109b9b 100644 --- a/packages/llm-llamacpp/addon/src/model-interface/LlamaModel.cpp +++ b/packages/llm-llamacpp/addon/src/model-interface/LlamaModel.cpp @@ -630,7 +630,10 @@ std::string LlamaModel::processPromptImpl(const Prompt& prompt) { } std::ostringstream oss; - bool needsOutputCapture = state_->toolsCompact_->enabled(); + // The streamed output must be captured into `oss` whenever any + // post-generation pass needs to inspect the full assistant text. + const bool needsOutputCapture = + state_->toolsCompact_->enabled() || !resolved.tools.empty(); auto callback = prompt.outputCallback; if (!prompt.outputCallback) { callback = [&](const std::string& token) { oss << token; }; @@ -700,9 +703,11 @@ std::string LlamaModel::processPromptImpl(const Prompt& prompt) { envelopes.append(args); envelopes.append("}"); } + if (prompt.outputCallback) { prompt.outputCallback(envelopes); } else { + out = parsed.content; out.append(envelopes); } } From 05a7a770ad58276ccf27a0df74d8319f560bf315 Mon Sep 17 00:00:00 2001 From: Marcus Edel Date: Wed, 3 Jun 2026 19:19:11 +0000 Subject: [PATCH 11/58] Point to the latest fabric changes. Signed-off-by: Marcus Edel --- .../vcpkg/ports/qvac-fabric/portfile.cmake | 4 ++-- .../embed-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake | 4 ++-- packages/llm-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake | 4 ++-- .../translation-nmtcpp/vcpkg/ports/qvac-fabric/portfile.cmake | 4 ++-- packages/vla-ggml/vcpkg/ports/qvac-fabric/portfile.cmake | 4 ++-- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/packages/classification-ggml/vcpkg/ports/qvac-fabric/portfile.cmake b/packages/classification-ggml/vcpkg/ports/qvac-fabric/portfile.cmake index 03dc0ab153..b53fd19eb6 100644 --- a/packages/classification-ggml/vcpkg/ports/qvac-fabric/portfile.cmake +++ b/packages/classification-ggml/vcpkg/ports/qvac-fabric/portfile.cmake @@ -1,8 +1,8 @@ vcpkg_from_github( OUT_SOURCE_PATH SOURCE_PATH REPO tetherto/qvac-fabric-llm.cpp - REF 3329743b43473326baaabd5354ee21f99e964f36 - SHA512 e97ac4cbed42be89051580bdbc19d15c0c811e8a4f470ddbafb10b42ad7990279c27f47d3f59006463a9c5c06d11a1ff70c1611024b53d198b45d33b5ff4f637 + REF bfa5b0a39c0399713f120443fb928500b1f3e6e1 + SHA512 b2f9042951d1abc593e309ca9d93eb1ec585328a0d4f51b07979f6021a0ce4428e82309eb6c9dc6e5775b911c641320d27dbc425abae65ff11af1d037f5999df HEAD_REF qvac-b9341 ) diff --git a/packages/embed-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake b/packages/embed-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake index 03dc0ab153..b53fd19eb6 100644 --- a/packages/embed-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake +++ b/packages/embed-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake @@ -1,8 +1,8 @@ vcpkg_from_github( OUT_SOURCE_PATH SOURCE_PATH REPO tetherto/qvac-fabric-llm.cpp - REF 3329743b43473326baaabd5354ee21f99e964f36 - SHA512 e97ac4cbed42be89051580bdbc19d15c0c811e8a4f470ddbafb10b42ad7990279c27f47d3f59006463a9c5c06d11a1ff70c1611024b53d198b45d33b5ff4f637 + REF bfa5b0a39c0399713f120443fb928500b1f3e6e1 + SHA512 b2f9042951d1abc593e309ca9d93eb1ec585328a0d4f51b07979f6021a0ce4428e82309eb6c9dc6e5775b911c641320d27dbc425abae65ff11af1d037f5999df HEAD_REF qvac-b9341 ) diff --git a/packages/llm-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake b/packages/llm-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake index 03dc0ab153..b53fd19eb6 100644 --- a/packages/llm-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake +++ b/packages/llm-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake @@ -1,8 +1,8 @@ vcpkg_from_github( OUT_SOURCE_PATH SOURCE_PATH REPO tetherto/qvac-fabric-llm.cpp - REF 3329743b43473326baaabd5354ee21f99e964f36 - SHA512 e97ac4cbed42be89051580bdbc19d15c0c811e8a4f470ddbafb10b42ad7990279c27f47d3f59006463a9c5c06d11a1ff70c1611024b53d198b45d33b5ff4f637 + REF bfa5b0a39c0399713f120443fb928500b1f3e6e1 + SHA512 b2f9042951d1abc593e309ca9d93eb1ec585328a0d4f51b07979f6021a0ce4428e82309eb6c9dc6e5775b911c641320d27dbc425abae65ff11af1d037f5999df HEAD_REF qvac-b9341 ) diff --git a/packages/translation-nmtcpp/vcpkg/ports/qvac-fabric/portfile.cmake b/packages/translation-nmtcpp/vcpkg/ports/qvac-fabric/portfile.cmake index 03dc0ab153..b53fd19eb6 100644 --- a/packages/translation-nmtcpp/vcpkg/ports/qvac-fabric/portfile.cmake +++ b/packages/translation-nmtcpp/vcpkg/ports/qvac-fabric/portfile.cmake @@ -1,8 +1,8 @@ vcpkg_from_github( OUT_SOURCE_PATH SOURCE_PATH REPO tetherto/qvac-fabric-llm.cpp - REF 3329743b43473326baaabd5354ee21f99e964f36 - SHA512 e97ac4cbed42be89051580bdbc19d15c0c811e8a4f470ddbafb10b42ad7990279c27f47d3f59006463a9c5c06d11a1ff70c1611024b53d198b45d33b5ff4f637 + REF bfa5b0a39c0399713f120443fb928500b1f3e6e1 + SHA512 b2f9042951d1abc593e309ca9d93eb1ec585328a0d4f51b07979f6021a0ce4428e82309eb6c9dc6e5775b911c641320d27dbc425abae65ff11af1d037f5999df HEAD_REF qvac-b9341 ) diff --git a/packages/vla-ggml/vcpkg/ports/qvac-fabric/portfile.cmake b/packages/vla-ggml/vcpkg/ports/qvac-fabric/portfile.cmake index 03dc0ab153..b53fd19eb6 100644 --- a/packages/vla-ggml/vcpkg/ports/qvac-fabric/portfile.cmake +++ b/packages/vla-ggml/vcpkg/ports/qvac-fabric/portfile.cmake @@ -1,8 +1,8 @@ vcpkg_from_github( OUT_SOURCE_PATH SOURCE_PATH REPO tetherto/qvac-fabric-llm.cpp - REF 3329743b43473326baaabd5354ee21f99e964f36 - SHA512 e97ac4cbed42be89051580bdbc19d15c0c811e8a4f470ddbafb10b42ad7990279c27f47d3f59006463a9c5c06d11a1ff70c1611024b53d198b45d33b5ff4f637 + REF bfa5b0a39c0399713f120443fb928500b1f3e6e1 + SHA512 b2f9042951d1abc593e309ca9d93eb1ec585328a0d4f51b07979f6021a0ce4428e82309eb6c9dc6e5775b911c641320d27dbc425abae65ff11af1d037f5999df HEAD_REF qvac-b9341 ) From a76332ed08a7efafefd0f3df2387d2f21cae5275 Mon Sep 17 00:00:00 2001 From: Marcus Edel Date: Wed, 3 Jun 2026 20:46:43 +0000 Subject: [PATCH 12/58] Point to the latest fabric changes. Signed-off-by: Marcus Edel --- .../vcpkg/ports/qvac-fabric/portfile.cmake | 4 ++-- .../embed-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake | 4 ++-- packages/llm-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake | 4 ++-- .../translation-nmtcpp/vcpkg/ports/qvac-fabric/portfile.cmake | 4 ++-- packages/vla-ggml/vcpkg/ports/qvac-fabric/portfile.cmake | 4 ++-- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/packages/classification-ggml/vcpkg/ports/qvac-fabric/portfile.cmake b/packages/classification-ggml/vcpkg/ports/qvac-fabric/portfile.cmake index b53fd19eb6..f1aa995a13 100644 --- a/packages/classification-ggml/vcpkg/ports/qvac-fabric/portfile.cmake +++ b/packages/classification-ggml/vcpkg/ports/qvac-fabric/portfile.cmake @@ -1,8 +1,8 @@ vcpkg_from_github( OUT_SOURCE_PATH SOURCE_PATH REPO tetherto/qvac-fabric-llm.cpp - REF bfa5b0a39c0399713f120443fb928500b1f3e6e1 - SHA512 b2f9042951d1abc593e309ca9d93eb1ec585328a0d4f51b07979f6021a0ce4428e82309eb6c9dc6e5775b911c641320d27dbc425abae65ff11af1d037f5999df + REF 9bb541a436352bf6171628c04f3ff74bcc7d6d81 + SHA512 d70177f2c002f6df93c1feb782c9f15b9ba882ab2f6156f2c41430190fc7dfaaa286b756306c5db4cc017482a35824354d25a64f9e1e679e322037f647def2b5 HEAD_REF qvac-b9341 ) diff --git a/packages/embed-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake b/packages/embed-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake index b53fd19eb6..f1aa995a13 100644 --- a/packages/embed-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake +++ b/packages/embed-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake @@ -1,8 +1,8 @@ vcpkg_from_github( OUT_SOURCE_PATH SOURCE_PATH REPO tetherto/qvac-fabric-llm.cpp - REF bfa5b0a39c0399713f120443fb928500b1f3e6e1 - SHA512 b2f9042951d1abc593e309ca9d93eb1ec585328a0d4f51b07979f6021a0ce4428e82309eb6c9dc6e5775b911c641320d27dbc425abae65ff11af1d037f5999df + REF 9bb541a436352bf6171628c04f3ff74bcc7d6d81 + SHA512 d70177f2c002f6df93c1feb782c9f15b9ba882ab2f6156f2c41430190fc7dfaaa286b756306c5db4cc017482a35824354d25a64f9e1e679e322037f647def2b5 HEAD_REF qvac-b9341 ) diff --git a/packages/llm-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake b/packages/llm-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake index b53fd19eb6..f1aa995a13 100644 --- a/packages/llm-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake +++ b/packages/llm-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake @@ -1,8 +1,8 @@ vcpkg_from_github( OUT_SOURCE_PATH SOURCE_PATH REPO tetherto/qvac-fabric-llm.cpp - REF bfa5b0a39c0399713f120443fb928500b1f3e6e1 - SHA512 b2f9042951d1abc593e309ca9d93eb1ec585328a0d4f51b07979f6021a0ce4428e82309eb6c9dc6e5775b911c641320d27dbc425abae65ff11af1d037f5999df + REF 9bb541a436352bf6171628c04f3ff74bcc7d6d81 + SHA512 d70177f2c002f6df93c1feb782c9f15b9ba882ab2f6156f2c41430190fc7dfaaa286b756306c5db4cc017482a35824354d25a64f9e1e679e322037f647def2b5 HEAD_REF qvac-b9341 ) diff --git a/packages/translation-nmtcpp/vcpkg/ports/qvac-fabric/portfile.cmake b/packages/translation-nmtcpp/vcpkg/ports/qvac-fabric/portfile.cmake index b53fd19eb6..f1aa995a13 100644 --- a/packages/translation-nmtcpp/vcpkg/ports/qvac-fabric/portfile.cmake +++ b/packages/translation-nmtcpp/vcpkg/ports/qvac-fabric/portfile.cmake @@ -1,8 +1,8 @@ vcpkg_from_github( OUT_SOURCE_PATH SOURCE_PATH REPO tetherto/qvac-fabric-llm.cpp - REF bfa5b0a39c0399713f120443fb928500b1f3e6e1 - SHA512 b2f9042951d1abc593e309ca9d93eb1ec585328a0d4f51b07979f6021a0ce4428e82309eb6c9dc6e5775b911c641320d27dbc425abae65ff11af1d037f5999df + REF 9bb541a436352bf6171628c04f3ff74bcc7d6d81 + SHA512 d70177f2c002f6df93c1feb782c9f15b9ba882ab2f6156f2c41430190fc7dfaaa286b756306c5db4cc017482a35824354d25a64f9e1e679e322037f647def2b5 HEAD_REF qvac-b9341 ) diff --git a/packages/vla-ggml/vcpkg/ports/qvac-fabric/portfile.cmake b/packages/vla-ggml/vcpkg/ports/qvac-fabric/portfile.cmake index b53fd19eb6..f1aa995a13 100644 --- a/packages/vla-ggml/vcpkg/ports/qvac-fabric/portfile.cmake +++ b/packages/vla-ggml/vcpkg/ports/qvac-fabric/portfile.cmake @@ -1,8 +1,8 @@ vcpkg_from_github( OUT_SOURCE_PATH SOURCE_PATH REPO tetherto/qvac-fabric-llm.cpp - REF bfa5b0a39c0399713f120443fb928500b1f3e6e1 - SHA512 b2f9042951d1abc593e309ca9d93eb1ec585328a0d4f51b07979f6021a0ce4428e82309eb6c9dc6e5775b911c641320d27dbc425abae65ff11af1d037f5999df + REF 9bb541a436352bf6171628c04f3ff74bcc7d6d81 + SHA512 d70177f2c002f6df93c1feb782c9f15b9ba882ab2f6156f2c41430190fc7dfaaa286b756306c5db4cc017482a35824354d25a64f9e1e679e322037f647def2b5 HEAD_REF qvac-b9341 ) From 8a6e7e1df95e8babe0b1deb1c889a7137cf9d38b Mon Sep 17 00:00:00 2001 From: Marcus Edel Date: Thu, 4 Jun 2026 14:55:41 +0000 Subject: [PATCH 13/58] feat: default flash attention to on across every non-bitnet, non-finetuning backend (Vulkan / Metal / CUDA / OpenCL / CPU) so consumers get FA-accelerated decode without explicit opt-in. Signed-off-by: Marcus Edel --- .../addon/src/model-interface/LlamaModel.cpp | 7 ++--- .../test/unit/test_tune_config_map.cpp | 28 ++++++++++--------- 2 files changed, 18 insertions(+), 17 deletions(-) diff --git a/packages/llm-llamacpp/addon/src/model-interface/LlamaModel.cpp b/packages/llm-llamacpp/addon/src/model-interface/LlamaModel.cpp index d2a33f2835..5a8f233c30 100644 --- a/packages/llm-llamacpp/addon/src/model-interface/LlamaModel.cpp +++ b/packages/llm-llamacpp/addon/src/model-interface/LlamaModel.cpp @@ -144,13 +144,12 @@ void LlamaModel::tuneConfigMap( QLOG_IF( Priority::INFO, "[LlamaModel] BitNet model detected: disabling flash attention\n"); - } else if (isOpenCl && notUserSet("flash-attn", "flash_attn")) { + } else if (notUserSet("flash-attn", "flash_attn")) { configFilemap.erase("flash_attn"); - configFilemap["flash-attn"] = "off"; + configFilemap["flash-attn"] = "on"; QLOG_IF( Priority::INFO, - "[LlamaModel] OpenCL backend selected: disabling flash attention by " - "default (not reliably supported on OpenCL)\n"); + "[LlamaModel] Enabling flash attention by default\n"); } constexpr int kAdrenoUbatchThreshold = 800; diff --git a/packages/llm-llamacpp/test/unit/test_tune_config_map.cpp b/packages/llm-llamacpp/test/unit/test_tune_config_map.cpp index 5f194d88e3..792b1bd2f8 100644 --- a/packages/llm-llamacpp/test/unit/test_tune_config_map.cpp +++ b/packages/llm-llamacpp/test/unit/test_tune_config_map.cpp @@ -15,32 +15,33 @@ class TuneConfigMapTest : public ::testing::Test { std::unordered_map configFilemap_; }; -// ---- Non-BitNet: no modifications ---- - -TEST_F(TuneConfigMapTest, NonBitnet_NoChanges) { +TEST_F(TuneConfigMapTest, NonBitnet_FlashAttnDefaultsOn) { MockModelMetaData meta(false, "llama"); LlamaModel::tuneConfigMap(configFilemap_, meta, std::nullopt); - EXPECT_EQ(configFilemap_.count("flash-attn"), 0); + ASSERT_EQ(configFilemap_.count("flash-attn"), 1); + EXPECT_EQ(configFilemap_["flash-attn"], "on"); EXPECT_EQ(configFilemap_.count("ubatch-size"), 0); } -TEST_F(TuneConfigMapTest, OneBitButNotBitnetArch_NoChanges) { +TEST_F(TuneConfigMapTest, OneBitButNotBitnetArch_FlashAttnDefaultsOn) { MockModelMetaData meta(true, "llama"); LlamaModel::tuneConfigMap(configFilemap_, meta, 830); - EXPECT_EQ(configFilemap_.count("flash-attn"), 0); + ASSERT_EQ(configFilemap_.count("flash-attn"), 1); + EXPECT_EQ(configFilemap_["flash-attn"], "on"); EXPECT_EQ(configFilemap_.count("ubatch-size"), 0); } -TEST_F(TuneConfigMapTest, BitnetArchButNotOneBit_NoChanges) { +TEST_F(TuneConfigMapTest, BitnetArchButNotOneBit_FlashAttnDefaultsOn) { MockModelMetaData meta(false, "bitnet"); LlamaModel::tuneConfigMap(configFilemap_, meta, 830); - EXPECT_EQ(configFilemap_.count("flash-attn"), 0); + ASSERT_EQ(configFilemap_.count("flash-attn"), 1); + EXPECT_EQ(configFilemap_["flash-attn"], "on"); EXPECT_EQ(configFilemap_.count("ubatch-size"), 0); } @@ -190,16 +191,16 @@ TEST_F(TuneConfigMapTest, Bitnet_Adreno799_UbatchUnchanged) { EXPECT_EQ(configFilemap_.count("ubatch-size"), 0); } -// ---- OpenCL backend: flash-attn disabled by default unless user-set ---- +// ---- OpenCL backend: flash-attn defaulted ON like every other GPU path ---- -TEST_F(TuneConfigMapTest, OpenCl_NonBitnet_FlashAttnDisabledByDefault) { +TEST_F(TuneConfigMapTest, OpenCl_NonBitnet_FlashAttnDefaultsOn) { MockModelMetaData meta(false, "llama"); LlamaModel::tuneConfigMap( configFilemap_, meta, std::nullopt, FtOverrides{}, /*isOpenCl=*/true); ASSERT_EQ(configFilemap_.count("flash-attn"), 1); - EXPECT_EQ(configFilemap_["flash-attn"], "off"); + EXPECT_EQ(configFilemap_["flash-attn"], "on"); } TEST_F(TuneConfigMapTest, OpenCl_UserSetFlashAttnHyphen_Respected) { @@ -223,13 +224,14 @@ TEST_F(TuneConfigMapTest, OpenCl_UserSetFlashAttnUnderscore_Respected) { EXPECT_EQ(configFilemap_["flash_attn"], "on"); } -TEST_F(TuneConfigMapTest, NotOpenCl_NonBitnet_FlashAttnUnchanged) { +TEST_F(TuneConfigMapTest, NotOpenCl_NonBitnet_FlashAttnDefaultsOn) { MockModelMetaData meta(false, "llama"); LlamaModel::tuneConfigMap( configFilemap_, meta, std::nullopt, FtOverrides{}, /*isOpenCl=*/false); - EXPECT_EQ(configFilemap_.count("flash-attn"), 0); + ASSERT_EQ(configFilemap_.count("flash-attn"), 1); + EXPECT_EQ(configFilemap_["flash-attn"], "on"); } // ---- Finetuning: flash-attn disabled for any architecture ---- From 3288b378a2444db2cd367439ad38bab16386ba7c Mon Sep 17 00:00:00 2001 From: Marcus Edel Date: Thu, 4 Jun 2026 17:22:16 +0000 Subject: [PATCH 14/58] Point to the latest fabric changes. Signed-off-by: Marcus Edel --- .../vcpkg/ports/qvac-fabric/portfile.cmake | 4 ++-- .../embed-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake | 4 ++-- packages/llm-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake | 4 ++-- .../translation-nmtcpp/vcpkg/ports/qvac-fabric/portfile.cmake | 4 ++-- packages/vla-ggml/vcpkg/ports/qvac-fabric/portfile.cmake | 4 ++-- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/packages/classification-ggml/vcpkg/ports/qvac-fabric/portfile.cmake b/packages/classification-ggml/vcpkg/ports/qvac-fabric/portfile.cmake index f1aa995a13..9d7f9ab397 100644 --- a/packages/classification-ggml/vcpkg/ports/qvac-fabric/portfile.cmake +++ b/packages/classification-ggml/vcpkg/ports/qvac-fabric/portfile.cmake @@ -1,8 +1,8 @@ vcpkg_from_github( OUT_SOURCE_PATH SOURCE_PATH REPO tetherto/qvac-fabric-llm.cpp - REF 9bb541a436352bf6171628c04f3ff74bcc7d6d81 - SHA512 d70177f2c002f6df93c1feb782c9f15b9ba882ab2f6156f2c41430190fc7dfaaa286b756306c5db4cc017482a35824354d25a64f9e1e679e322037f647def2b5 + REF 261adf3cb074d3a6f14b676975345d0db395f888 + SHA512 40b9d28c6151e28dffedd1be19e2e77c09d3cde3cb9f7644726e15ed25dcf2b21cb458d5b8e25397ffe5b14b7f93c6d55a4083e5dbcc94058f71ccf75b850f1a HEAD_REF qvac-b9341 ) diff --git a/packages/embed-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake b/packages/embed-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake index f1aa995a13..9d7f9ab397 100644 --- a/packages/embed-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake +++ b/packages/embed-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake @@ -1,8 +1,8 @@ vcpkg_from_github( OUT_SOURCE_PATH SOURCE_PATH REPO tetherto/qvac-fabric-llm.cpp - REF 9bb541a436352bf6171628c04f3ff74bcc7d6d81 - SHA512 d70177f2c002f6df93c1feb782c9f15b9ba882ab2f6156f2c41430190fc7dfaaa286b756306c5db4cc017482a35824354d25a64f9e1e679e322037f647def2b5 + REF 261adf3cb074d3a6f14b676975345d0db395f888 + SHA512 40b9d28c6151e28dffedd1be19e2e77c09d3cde3cb9f7644726e15ed25dcf2b21cb458d5b8e25397ffe5b14b7f93c6d55a4083e5dbcc94058f71ccf75b850f1a HEAD_REF qvac-b9341 ) diff --git a/packages/llm-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake b/packages/llm-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake index f1aa995a13..9d7f9ab397 100644 --- a/packages/llm-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake +++ b/packages/llm-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake @@ -1,8 +1,8 @@ vcpkg_from_github( OUT_SOURCE_PATH SOURCE_PATH REPO tetherto/qvac-fabric-llm.cpp - REF 9bb541a436352bf6171628c04f3ff74bcc7d6d81 - SHA512 d70177f2c002f6df93c1feb782c9f15b9ba882ab2f6156f2c41430190fc7dfaaa286b756306c5db4cc017482a35824354d25a64f9e1e679e322037f647def2b5 + REF 261adf3cb074d3a6f14b676975345d0db395f888 + SHA512 40b9d28c6151e28dffedd1be19e2e77c09d3cde3cb9f7644726e15ed25dcf2b21cb458d5b8e25397ffe5b14b7f93c6d55a4083e5dbcc94058f71ccf75b850f1a HEAD_REF qvac-b9341 ) diff --git a/packages/translation-nmtcpp/vcpkg/ports/qvac-fabric/portfile.cmake b/packages/translation-nmtcpp/vcpkg/ports/qvac-fabric/portfile.cmake index f1aa995a13..9d7f9ab397 100644 --- a/packages/translation-nmtcpp/vcpkg/ports/qvac-fabric/portfile.cmake +++ b/packages/translation-nmtcpp/vcpkg/ports/qvac-fabric/portfile.cmake @@ -1,8 +1,8 @@ vcpkg_from_github( OUT_SOURCE_PATH SOURCE_PATH REPO tetherto/qvac-fabric-llm.cpp - REF 9bb541a436352bf6171628c04f3ff74bcc7d6d81 - SHA512 d70177f2c002f6df93c1feb782c9f15b9ba882ab2f6156f2c41430190fc7dfaaa286b756306c5db4cc017482a35824354d25a64f9e1e679e322037f647def2b5 + REF 261adf3cb074d3a6f14b676975345d0db395f888 + SHA512 40b9d28c6151e28dffedd1be19e2e77c09d3cde3cb9f7644726e15ed25dcf2b21cb458d5b8e25397ffe5b14b7f93c6d55a4083e5dbcc94058f71ccf75b850f1a HEAD_REF qvac-b9341 ) diff --git a/packages/vla-ggml/vcpkg/ports/qvac-fabric/portfile.cmake b/packages/vla-ggml/vcpkg/ports/qvac-fabric/portfile.cmake index f1aa995a13..9d7f9ab397 100644 --- a/packages/vla-ggml/vcpkg/ports/qvac-fabric/portfile.cmake +++ b/packages/vla-ggml/vcpkg/ports/qvac-fabric/portfile.cmake @@ -1,8 +1,8 @@ vcpkg_from_github( OUT_SOURCE_PATH SOURCE_PATH REPO tetherto/qvac-fabric-llm.cpp - REF 9bb541a436352bf6171628c04f3ff74bcc7d6d81 - SHA512 d70177f2c002f6df93c1feb782c9f15b9ba882ab2f6156f2c41430190fc7dfaaa286b756306c5db4cc017482a35824354d25a64f9e1e679e322037f647def2b5 + REF 261adf3cb074d3a6f14b676975345d0db395f888 + SHA512 40b9d28c6151e28dffedd1be19e2e77c09d3cde3cb9f7644726e15ed25dcf2b21cb458d5b8e25397ffe5b14b7f93c6d55a4083e5dbcc94058f71ccf75b850f1a HEAD_REF qvac-b9341 ) From 6559b4c96af03e7bb18c294dbbb7379298a3fb20 Mon Sep 17 00:00:00 2001 From: Marcus Edel Date: Thu, 4 Jun 2026 19:58:36 +0000 Subject: [PATCH 15/58] test: test: move runMultiInstanceTest into android.groupImagesPerf to balance the test load. Signed-off-by: Marcus Edel --- packages/llm-llamacpp/test/mobile/test-groups.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/llm-llamacpp/test/mobile/test-groups.json b/packages/llm-llamacpp/test/mobile/test-groups.json index dc69759f28..408dd25e92 100644 --- a/packages/llm-llamacpp/test/mobile/test-groups.json +++ b/packages/llm-llamacpp/test/mobile/test-groups.json @@ -38,7 +38,6 @@ "runFinetuningPauseResumeTest", "runGenerationParamsTest", "runGrammarTest", - "runMultiInstanceTest", "runQuantizedKvcacheTest", "runQwen35Test", "runReasoningTest", @@ -60,7 +59,8 @@ "runImageElephantTest", "runImageFruitPlateTest", "runImageHighResAuroraTest", - "runMtpTest" + "runMtpTest", + "runMultiInstanceTest" ] } } From 13db8b5ca9a2c836f5787fae0cbec2c6b5c71fbc Mon Sep 17 00:00:00 2001 From: Marcus Edel Date: Fri, 5 Jun 2026 19:37:17 +0000 Subject: [PATCH 16/58] Point to the latest fabric changes. Signed-off-by: Marcus Edel --- .../vcpkg/ports/qvac-fabric/portfile.cmake | 4 ++-- .../embed-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake | 4 ++-- packages/llm-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake | 4 ++-- .../translation-nmtcpp/vcpkg/ports/qvac-fabric/portfile.cmake | 4 ++-- packages/vla-ggml/vcpkg/ports/qvac-fabric/portfile.cmake | 4 ++-- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/packages/classification-ggml/vcpkg/ports/qvac-fabric/portfile.cmake b/packages/classification-ggml/vcpkg/ports/qvac-fabric/portfile.cmake index 9d7f9ab397..2e7ab86719 100644 --- a/packages/classification-ggml/vcpkg/ports/qvac-fabric/portfile.cmake +++ b/packages/classification-ggml/vcpkg/ports/qvac-fabric/portfile.cmake @@ -1,8 +1,8 @@ vcpkg_from_github( OUT_SOURCE_PATH SOURCE_PATH REPO tetherto/qvac-fabric-llm.cpp - REF 261adf3cb074d3a6f14b676975345d0db395f888 - SHA512 40b9d28c6151e28dffedd1be19e2e77c09d3cde3cb9f7644726e15ed25dcf2b21cb458d5b8e25397ffe5b14b7f93c6d55a4083e5dbcc94058f71ccf75b850f1a + REF 25af25286b0b32692e2ef1b6896e33d538c2d0de + SHA512 5f6b38a1d7be3ffa46a4ca4672a588313b0e645f40ac4fd0e303df5e410b37897c01eb76a1d67799ba2ca3377e25dada296ec6a5c51cbf4c7025f3392f4d515c HEAD_REF qvac-b9341 ) diff --git a/packages/embed-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake b/packages/embed-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake index 9d7f9ab397..2e7ab86719 100644 --- a/packages/embed-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake +++ b/packages/embed-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake @@ -1,8 +1,8 @@ vcpkg_from_github( OUT_SOURCE_PATH SOURCE_PATH REPO tetherto/qvac-fabric-llm.cpp - REF 261adf3cb074d3a6f14b676975345d0db395f888 - SHA512 40b9d28c6151e28dffedd1be19e2e77c09d3cde3cb9f7644726e15ed25dcf2b21cb458d5b8e25397ffe5b14b7f93c6d55a4083e5dbcc94058f71ccf75b850f1a + REF 25af25286b0b32692e2ef1b6896e33d538c2d0de + SHA512 5f6b38a1d7be3ffa46a4ca4672a588313b0e645f40ac4fd0e303df5e410b37897c01eb76a1d67799ba2ca3377e25dada296ec6a5c51cbf4c7025f3392f4d515c HEAD_REF qvac-b9341 ) diff --git a/packages/llm-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake b/packages/llm-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake index 9d7f9ab397..2e7ab86719 100644 --- a/packages/llm-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake +++ b/packages/llm-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake @@ -1,8 +1,8 @@ vcpkg_from_github( OUT_SOURCE_PATH SOURCE_PATH REPO tetherto/qvac-fabric-llm.cpp - REF 261adf3cb074d3a6f14b676975345d0db395f888 - SHA512 40b9d28c6151e28dffedd1be19e2e77c09d3cde3cb9f7644726e15ed25dcf2b21cb458d5b8e25397ffe5b14b7f93c6d55a4083e5dbcc94058f71ccf75b850f1a + REF 25af25286b0b32692e2ef1b6896e33d538c2d0de + SHA512 5f6b38a1d7be3ffa46a4ca4672a588313b0e645f40ac4fd0e303df5e410b37897c01eb76a1d67799ba2ca3377e25dada296ec6a5c51cbf4c7025f3392f4d515c HEAD_REF qvac-b9341 ) diff --git a/packages/translation-nmtcpp/vcpkg/ports/qvac-fabric/portfile.cmake b/packages/translation-nmtcpp/vcpkg/ports/qvac-fabric/portfile.cmake index 9d7f9ab397..2e7ab86719 100644 --- a/packages/translation-nmtcpp/vcpkg/ports/qvac-fabric/portfile.cmake +++ b/packages/translation-nmtcpp/vcpkg/ports/qvac-fabric/portfile.cmake @@ -1,8 +1,8 @@ vcpkg_from_github( OUT_SOURCE_PATH SOURCE_PATH REPO tetherto/qvac-fabric-llm.cpp - REF 261adf3cb074d3a6f14b676975345d0db395f888 - SHA512 40b9d28c6151e28dffedd1be19e2e77c09d3cde3cb9f7644726e15ed25dcf2b21cb458d5b8e25397ffe5b14b7f93c6d55a4083e5dbcc94058f71ccf75b850f1a + REF 25af25286b0b32692e2ef1b6896e33d538c2d0de + SHA512 5f6b38a1d7be3ffa46a4ca4672a588313b0e645f40ac4fd0e303df5e410b37897c01eb76a1d67799ba2ca3377e25dada296ec6a5c51cbf4c7025f3392f4d515c HEAD_REF qvac-b9341 ) diff --git a/packages/vla-ggml/vcpkg/ports/qvac-fabric/portfile.cmake b/packages/vla-ggml/vcpkg/ports/qvac-fabric/portfile.cmake index 9d7f9ab397..2e7ab86719 100644 --- a/packages/vla-ggml/vcpkg/ports/qvac-fabric/portfile.cmake +++ b/packages/vla-ggml/vcpkg/ports/qvac-fabric/portfile.cmake @@ -1,8 +1,8 @@ vcpkg_from_github( OUT_SOURCE_PATH SOURCE_PATH REPO tetherto/qvac-fabric-llm.cpp - REF 261adf3cb074d3a6f14b676975345d0db395f888 - SHA512 40b9d28c6151e28dffedd1be19e2e77c09d3cde3cb9f7644726e15ed25dcf2b21cb458d5b8e25397ffe5b14b7f93c6d55a4083e5dbcc94058f71ccf75b850f1a + REF 25af25286b0b32692e2ef1b6896e33d538c2d0de + SHA512 5f6b38a1d7be3ffa46a4ca4672a588313b0e645f40ac4fd0e303df5e410b37897c01eb76a1d67799ba2ca3377e25dada296ec6a5c51cbf4c7025f3392f4d515c HEAD_REF qvac-b9341 ) From 520ded0e708d1bc832f2e05772a6783ba2aeec6f Mon Sep 17 00:00:00 2001 From: Marcus Edel Date: Mon, 8 Jun 2026 12:41:34 +0000 Subject: [PATCH 17/58] Revert workflow changes from MTP commit per review feedback. Signed-off-by: Marcus Edel --- .../integration-test-llm-llamacpp.yml | 21 +++---------------- 1 file changed, 3 insertions(+), 18 deletions(-) diff --git a/.github/workflows/integration-test-llm-llamacpp.yml b/.github/workflows/integration-test-llm-llamacpp.yml index fd5dfd57b8..8b3390b270 100644 --- a/.github/workflows/integration-test-llm-llamacpp.yml +++ b/.github/workflows/integration-test-llm-llamacpp.yml @@ -271,12 +271,6 @@ jobs: if: ${{ matrix.platform != 'win32' }} working-directory: ${{ env.WORKDIR }} run: | - set +e - # Stream test output AND capture it so we can inspect the TAP summary - # if the process exits with a non-zero code. Some configurations - # (observed on linux-arm64) crash with SIGSEGV (exit code 139) during - # static-destructor teardown AFTER all tests have passed. Tolerate - # that specific case, but surface any real failure. if [ "${{ inputs.qvac_perf_only }}" = "true" ]; then echo "qvac_perf_only=true: regenerating test/integration/all.js with perf-emitting tests only" npx brittle -r test/integration/all.js \ @@ -286,19 +280,10 @@ jobs: test/integration/image-fruit-plate.test.js \ test/integration/image-high-res-aurora.test.js bare test/integration/all.js --exit 2>&1 | tee test-output.log - EXIT=${PIPESTATUS[0]} - else - npm run test:integration 2>&1 | tee test-output.log - EXIT=${PIPESTATUS[0]} - fi - if [ "$EXIT" -eq 0 ]; then - exit 0 - fi - if [ "$EXIT" -eq 139 ] && grep -qE '^# ok$' test-output.log && grep -qE '^# tests = [0-9]+/[0-9]+ pass$' test-output.log; then - echo "::warning::Tests all passed but process crashed at teardown with SIGSEGV (exit 139) on $(uname -srvmo). Known ggml/Vulkan static-destructor issue; treating as pass." - exit 0 + exit ${PIPESTATUS[0]} fi - exit "$EXIT" + npm run test:integration 2>&1 | tee test-output.log + exit ${PIPESTATUS[0]} shell: bash env: QASE_API_TOKEN: ${{ secrets.QASE_API_TOKEN }} From 0f3bd3235b0b864f8f7e4a01e4d510af7c6f0387 Mon Sep 17 00:00:00 2001 From: gianni-cor Date: Mon, 15 Jun 2026 23:42:53 +0200 Subject: [PATCH 18/58] infra[notask]: bump qvac-fabric port refs Co-authored-by: Cursor --- .../vcpkg/ports/qvac-fabric/portfile.cmake | 4 ++-- .../embed-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake | 4 ++-- packages/llm-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake | 4 ++-- .../translation-nmtcpp/vcpkg/ports/qvac-fabric/portfile.cmake | 4 ++-- packages/vla-ggml/vcpkg/ports/qvac-fabric/portfile.cmake | 4 ++-- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/packages/classification-ggml/vcpkg/ports/qvac-fabric/portfile.cmake b/packages/classification-ggml/vcpkg/ports/qvac-fabric/portfile.cmake index 2e7ab86719..95301ee148 100644 --- a/packages/classification-ggml/vcpkg/ports/qvac-fabric/portfile.cmake +++ b/packages/classification-ggml/vcpkg/ports/qvac-fabric/portfile.cmake @@ -1,8 +1,8 @@ vcpkg_from_github( OUT_SOURCE_PATH SOURCE_PATH REPO tetherto/qvac-fabric-llm.cpp - REF 25af25286b0b32692e2ef1b6896e33d538c2d0de - SHA512 5f6b38a1d7be3ffa46a4ca4672a588313b0e645f40ac4fd0e303df5e410b37897c01eb76a1d67799ba2ca3377e25dada296ec6a5c51cbf4c7025f3392f4d515c + REF 0290f7a70d96888de04ba0adcf5e97d947624de9 + SHA512 f74fc9088bf2b40cec66246a4d440b1ff4f13b96a5bf3144586a4f838a6ffdba86a99c1ecf5079da26b8d5f5fe75d42524c21c131ab0c3b50542f99267c9b5ca HEAD_REF qvac-b9341 ) diff --git a/packages/embed-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake b/packages/embed-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake index 2e7ab86719..95301ee148 100644 --- a/packages/embed-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake +++ b/packages/embed-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake @@ -1,8 +1,8 @@ vcpkg_from_github( OUT_SOURCE_PATH SOURCE_PATH REPO tetherto/qvac-fabric-llm.cpp - REF 25af25286b0b32692e2ef1b6896e33d538c2d0de - SHA512 5f6b38a1d7be3ffa46a4ca4672a588313b0e645f40ac4fd0e303df5e410b37897c01eb76a1d67799ba2ca3377e25dada296ec6a5c51cbf4c7025f3392f4d515c + REF 0290f7a70d96888de04ba0adcf5e97d947624de9 + SHA512 f74fc9088bf2b40cec66246a4d440b1ff4f13b96a5bf3144586a4f838a6ffdba86a99c1ecf5079da26b8d5f5fe75d42524c21c131ab0c3b50542f99267c9b5ca HEAD_REF qvac-b9341 ) diff --git a/packages/llm-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake b/packages/llm-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake index 2e7ab86719..95301ee148 100644 --- a/packages/llm-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake +++ b/packages/llm-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake @@ -1,8 +1,8 @@ vcpkg_from_github( OUT_SOURCE_PATH SOURCE_PATH REPO tetherto/qvac-fabric-llm.cpp - REF 25af25286b0b32692e2ef1b6896e33d538c2d0de - SHA512 5f6b38a1d7be3ffa46a4ca4672a588313b0e645f40ac4fd0e303df5e410b37897c01eb76a1d67799ba2ca3377e25dada296ec6a5c51cbf4c7025f3392f4d515c + REF 0290f7a70d96888de04ba0adcf5e97d947624de9 + SHA512 f74fc9088bf2b40cec66246a4d440b1ff4f13b96a5bf3144586a4f838a6ffdba86a99c1ecf5079da26b8d5f5fe75d42524c21c131ab0c3b50542f99267c9b5ca HEAD_REF qvac-b9341 ) diff --git a/packages/translation-nmtcpp/vcpkg/ports/qvac-fabric/portfile.cmake b/packages/translation-nmtcpp/vcpkg/ports/qvac-fabric/portfile.cmake index 2e7ab86719..95301ee148 100644 --- a/packages/translation-nmtcpp/vcpkg/ports/qvac-fabric/portfile.cmake +++ b/packages/translation-nmtcpp/vcpkg/ports/qvac-fabric/portfile.cmake @@ -1,8 +1,8 @@ vcpkg_from_github( OUT_SOURCE_PATH SOURCE_PATH REPO tetherto/qvac-fabric-llm.cpp - REF 25af25286b0b32692e2ef1b6896e33d538c2d0de - SHA512 5f6b38a1d7be3ffa46a4ca4672a588313b0e645f40ac4fd0e303df5e410b37897c01eb76a1d67799ba2ca3377e25dada296ec6a5c51cbf4c7025f3392f4d515c + REF 0290f7a70d96888de04ba0adcf5e97d947624de9 + SHA512 f74fc9088bf2b40cec66246a4d440b1ff4f13b96a5bf3144586a4f838a6ffdba86a99c1ecf5079da26b8d5f5fe75d42524c21c131ab0c3b50542f99267c9b5ca HEAD_REF qvac-b9341 ) diff --git a/packages/vla-ggml/vcpkg/ports/qvac-fabric/portfile.cmake b/packages/vla-ggml/vcpkg/ports/qvac-fabric/portfile.cmake index 2e7ab86719..95301ee148 100644 --- a/packages/vla-ggml/vcpkg/ports/qvac-fabric/portfile.cmake +++ b/packages/vla-ggml/vcpkg/ports/qvac-fabric/portfile.cmake @@ -1,8 +1,8 @@ vcpkg_from_github( OUT_SOURCE_PATH SOURCE_PATH REPO tetherto/qvac-fabric-llm.cpp - REF 25af25286b0b32692e2ef1b6896e33d538c2d0de - SHA512 5f6b38a1d7be3ffa46a4ca4672a588313b0e645f40ac4fd0e303df5e410b37897c01eb76a1d67799ba2ca3377e25dada296ec6a5c51cbf4c7025f3392f4d515c + REF 0290f7a70d96888de04ba0adcf5e97d947624de9 + SHA512 f74fc9088bf2b40cec66246a4d440b1ff4f13b96a5bf3144586a4f838a6ffdba86a99c1ecf5079da26b8d5f5fe75d42524c21c131ab0c3b50542f99267c9b5ca HEAD_REF qvac-b9341 ) From 91846a0ac42e94e047c135c8c42f77997dbded7c Mon Sep 17 00:00:00 2001 From: gianni-cor Date: Tue, 16 Jun 2026 00:17:19 +0200 Subject: [PATCH 19/58] test[notask]: add continuous batching mobile runner Co-authored-by: Cursor --- .../scripts/generate-mobile-integration-tests.js | 4 ---- packages/llm-llamacpp/scripts/validate-mobile-tests.js | 4 ---- packages/llm-llamacpp/test/mobile/integration.auto.cjs | 5 +++++ packages/llm-llamacpp/test/mobile/test-groups.json | 4 ++++ 4 files changed, 9 insertions(+), 8 deletions(-) diff --git a/packages/llm-llamacpp/scripts/generate-mobile-integration-tests.js b/packages/llm-llamacpp/scripts/generate-mobile-integration-tests.js index e55a53d003..09fefedb00 100644 --- a/packages/llm-llamacpp/scripts/generate-mobile-integration-tests.js +++ b/packages/llm-llamacpp/scripts/generate-mobile-integration-tests.js @@ -8,9 +8,6 @@ const integrationDir = path.join(repoRoot, 'test', 'integration') const mobileDir = path.join(repoRoot, 'test', 'mobile') const outputFile = path.join(mobileDir, 'integration.auto.cjs') const groupsFile = path.join(mobileDir, 'test-groups.json') -const mobileExcludedTests = new Set([ - 'continuous-batching.test.js' -]) function getIntegrationFiles () { if (!fs.existsSync(integrationDir)) { @@ -19,7 +16,6 @@ function getIntegrationFiles () { return fs.readdirSync(integrationDir) .filter(entry => entry.endsWith('.test.js')) - .filter(entry => !mobileExcludedTests.has(entry)) .sort() } diff --git a/packages/llm-llamacpp/scripts/validate-mobile-tests.js b/packages/llm-llamacpp/scripts/validate-mobile-tests.js index 130c073aac..80dab32a07 100644 --- a/packages/llm-llamacpp/scripts/validate-mobile-tests.js +++ b/packages/llm-llamacpp/scripts/validate-mobile-tests.js @@ -7,9 +7,6 @@ const path = require('path') const repoRoot = path.resolve(__dirname, '..') const integrationDir = path.join(repoRoot, 'test', 'integration') const mobileAutoFile = path.join(repoRoot, 'test', 'mobile', 'integration.auto.cjs') -const mobileExcludedTests = new Set([ - 'continuous-batching.test.js' -]) function getIntegrationTestFiles () { if (!fs.existsSync(integrationDir)) { @@ -18,7 +15,6 @@ function getIntegrationTestFiles () { return fs.readdirSync(integrationDir) .filter(f => f.endsWith('.test.js')) - .filter(f => !mobileExcludedTests.has(f)) .sort() } diff --git a/packages/llm-llamacpp/test/mobile/integration.auto.cjs b/packages/llm-llamacpp/test/mobile/integration.auto.cjs index ada42470e1..94842a1bc0 100644 --- a/packages/llm-llamacpp/test/mobile/integration.auto.cjs +++ b/packages/llm-llamacpp/test/mobile/integration.auto.cjs @@ -31,6 +31,11 @@ async function runConfigParametersTest (options = {}) { // eslint-disable-line n return runIntegrationModule('../integration/config-parameters.test.js', options) } +async function runContinuousBatchingTest (options = {}) { // eslint-disable-line no-unused-vars + if (typeof __shouldRunTest === 'function' && !__shouldRunTest('runContinuousBatchingTest')) return __FILTERED + return runIntegrationModule('../integration/continuous-batching.test.js', options) +} + async function runFinetuningPauseResumeTest (options = {}) { // eslint-disable-line no-unused-vars if (typeof __shouldRunTest === 'function' && !__shouldRunTest('runFinetuningPauseResumeTest')) return __FILTERED return runIntegrationModule('../integration/finetuning-pause-resume.test.js', options) diff --git a/packages/llm-llamacpp/test/mobile/test-groups.json b/packages/llm-llamacpp/test/mobile/test-groups.json index 26382470e9..30b75141bc 100644 --- a/packages/llm-llamacpp/test/mobile/test-groups.json +++ b/packages/llm-llamacpp/test/mobile/test-groups.json @@ -11,6 +11,7 @@ "gemma": ["runGemma4Test"], "ocrPaddle": ["runOcrPaddleTest"], "quantizedKvcache": ["runQuantizedKvcacheTest"], + "continuousBatching": ["runContinuousBatchingTest"], "mtp": ["runMtpTest"], "lightA": [ "runApiBehaviorTest", @@ -51,6 +52,9 @@ "cacheStateMachine": [ "runCacheStateMachineTest" ], + "continuousBatching": [ + "runContinuousBatchingTest" + ], "funcShardA": [ "runFinetuningPauseResumeTest", "runSlidingContextTest", From d4f2e6dfcbe4c14a3d6885581a036071fd0903d8 Mon Sep 17 00:00:00 2001 From: gianni-cor Date: Tue, 16 Jun 2026 10:11:21 +0200 Subject: [PATCH 20/58] fix[notask]: capture streamed tool output Assisted-by: GPT-5.5 Co-authored-by: Cursor --- .../llm-llamacpp/addon/src/model-interface/LlamaModel.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/packages/llm-llamacpp/addon/src/model-interface/LlamaModel.cpp b/packages/llm-llamacpp/addon/src/model-interface/LlamaModel.cpp index ed6e16d9d2..3c80dbf628 100644 --- a/packages/llm-llamacpp/addon/src/model-interface/LlamaModel.cpp +++ b/packages/llm-llamacpp/addon/src/model-interface/LlamaModel.cpp @@ -721,6 +721,11 @@ std::string LlamaModel::processPromptImpl(const Prompt& prompt) { auto callback = prompt.outputCallback; if (!prompt.outputCallback) { callback = [&](const std::string& token) { oss << token; }; + } else if (needsOutputCapture) { + callback = [&](const std::string& token) { + oss << token; + prompt.outputCallback(token); + }; } if (!state_->llmContext_->generateResponse(callback)) { From b3d6861c0efe35288ce9bdebe6a344da9419f123 Mon Sep 17 00:00:00 2001 From: gianni-cor Date: Thu, 18 Jun 2026 12:19:53 +0200 Subject: [PATCH 21/58] infra[notask]: bump qvac-fabric port refs Assisted-by: GPT-5.5 Co-authored-by: Cursor --- .../vcpkg/ports/qvac-fabric/portfile.cmake | 6 +++--- .../embed-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake | 6 +++--- .../llm-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake | 6 +++--- .../vcpkg/ports/qvac-fabric/portfile.cmake | 6 +++--- packages/vla-ggml/vcpkg/ports/qvac-fabric/portfile.cmake | 6 +++--- 5 files changed, 15 insertions(+), 15 deletions(-) diff --git a/packages/classification-ggml/vcpkg/ports/qvac-fabric/portfile.cmake b/packages/classification-ggml/vcpkg/ports/qvac-fabric/portfile.cmake index 95301ee148..ead537a0ce 100644 --- a/packages/classification-ggml/vcpkg/ports/qvac-fabric/portfile.cmake +++ b/packages/classification-ggml/vcpkg/ports/qvac-fabric/portfile.cmake @@ -1,9 +1,9 @@ vcpkg_from_github( OUT_SOURCE_PATH SOURCE_PATH REPO tetherto/qvac-fabric-llm.cpp - REF 0290f7a70d96888de04ba0adcf5e97d947624de9 - SHA512 f74fc9088bf2b40cec66246a4d440b1ff4f13b96a5bf3144586a4f838a6ffdba86a99c1ecf5079da26b8d5f5fe75d42524c21c131ab0c3b50542f99267c9b5ca - HEAD_REF qvac-b9341 + REF 6f37291fe979c8f9a66a4c85809f69ff0ebeb743 + SHA512 6e29daed48978cac5fad4f2ed96f6518f0a81eef7a789d29d44196fb7a2a1b8fb7ed75fdcfc4acd370279a6b0b500c75517b79115dd88b2fbee2882e3a5d720d + HEAD_REF temp-9341 ) # Upstream CMake options only — passed through to vcpkg_cmake_configure. diff --git a/packages/embed-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake b/packages/embed-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake index 95301ee148..ead537a0ce 100644 --- a/packages/embed-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake +++ b/packages/embed-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake @@ -1,9 +1,9 @@ vcpkg_from_github( OUT_SOURCE_PATH SOURCE_PATH REPO tetherto/qvac-fabric-llm.cpp - REF 0290f7a70d96888de04ba0adcf5e97d947624de9 - SHA512 f74fc9088bf2b40cec66246a4d440b1ff4f13b96a5bf3144586a4f838a6ffdba86a99c1ecf5079da26b8d5f5fe75d42524c21c131ab0c3b50542f99267c9b5ca - HEAD_REF qvac-b9341 + REF 6f37291fe979c8f9a66a4c85809f69ff0ebeb743 + SHA512 6e29daed48978cac5fad4f2ed96f6518f0a81eef7a789d29d44196fb7a2a1b8fb7ed75fdcfc4acd370279a6b0b500c75517b79115dd88b2fbee2882e3a5d720d + HEAD_REF temp-9341 ) # Upstream CMake options only — passed through to vcpkg_cmake_configure. diff --git a/packages/llm-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake b/packages/llm-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake index 95301ee148..ead537a0ce 100644 --- a/packages/llm-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake +++ b/packages/llm-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake @@ -1,9 +1,9 @@ vcpkg_from_github( OUT_SOURCE_PATH SOURCE_PATH REPO tetherto/qvac-fabric-llm.cpp - REF 0290f7a70d96888de04ba0adcf5e97d947624de9 - SHA512 f74fc9088bf2b40cec66246a4d440b1ff4f13b96a5bf3144586a4f838a6ffdba86a99c1ecf5079da26b8d5f5fe75d42524c21c131ab0c3b50542f99267c9b5ca - HEAD_REF qvac-b9341 + REF 6f37291fe979c8f9a66a4c85809f69ff0ebeb743 + SHA512 6e29daed48978cac5fad4f2ed96f6518f0a81eef7a789d29d44196fb7a2a1b8fb7ed75fdcfc4acd370279a6b0b500c75517b79115dd88b2fbee2882e3a5d720d + HEAD_REF temp-9341 ) # Upstream CMake options only — passed through to vcpkg_cmake_configure. diff --git a/packages/translation-nmtcpp/vcpkg/ports/qvac-fabric/portfile.cmake b/packages/translation-nmtcpp/vcpkg/ports/qvac-fabric/portfile.cmake index 95301ee148..ead537a0ce 100644 --- a/packages/translation-nmtcpp/vcpkg/ports/qvac-fabric/portfile.cmake +++ b/packages/translation-nmtcpp/vcpkg/ports/qvac-fabric/portfile.cmake @@ -1,9 +1,9 @@ vcpkg_from_github( OUT_SOURCE_PATH SOURCE_PATH REPO tetherto/qvac-fabric-llm.cpp - REF 0290f7a70d96888de04ba0adcf5e97d947624de9 - SHA512 f74fc9088bf2b40cec66246a4d440b1ff4f13b96a5bf3144586a4f838a6ffdba86a99c1ecf5079da26b8d5f5fe75d42524c21c131ab0c3b50542f99267c9b5ca - HEAD_REF qvac-b9341 + REF 6f37291fe979c8f9a66a4c85809f69ff0ebeb743 + SHA512 6e29daed48978cac5fad4f2ed96f6518f0a81eef7a789d29d44196fb7a2a1b8fb7ed75fdcfc4acd370279a6b0b500c75517b79115dd88b2fbee2882e3a5d720d + HEAD_REF temp-9341 ) # Upstream CMake options only — passed through to vcpkg_cmake_configure. diff --git a/packages/vla-ggml/vcpkg/ports/qvac-fabric/portfile.cmake b/packages/vla-ggml/vcpkg/ports/qvac-fabric/portfile.cmake index 95301ee148..ead537a0ce 100644 --- a/packages/vla-ggml/vcpkg/ports/qvac-fabric/portfile.cmake +++ b/packages/vla-ggml/vcpkg/ports/qvac-fabric/portfile.cmake @@ -1,9 +1,9 @@ vcpkg_from_github( OUT_SOURCE_PATH SOURCE_PATH REPO tetherto/qvac-fabric-llm.cpp - REF 0290f7a70d96888de04ba0adcf5e97d947624de9 - SHA512 f74fc9088bf2b40cec66246a4d440b1ff4f13b96a5bf3144586a4f838a6ffdba86a99c1ecf5079da26b8d5f5fe75d42524c21c131ab0c3b50542f99267c9b5ca - HEAD_REF qvac-b9341 + REF 6f37291fe979c8f9a66a4c85809f69ff0ebeb743 + SHA512 6e29daed48978cac5fad4f2ed96f6518f0a81eef7a789d29d44196fb7a2a1b8fb7ed75fdcfc4acd370279a6b0b500c75517b79115dd88b2fbee2882e3a5d720d + HEAD_REF temp-9341 ) # Upstream CMake options only — passed through to vcpkg_cmake_configure. From 5c0497c5dcfa2916d82d6b5b63725f6e6c41c201 Mon Sep 17 00:00:00 2001 From: gianni-cor Date: Thu, 18 Jun 2026 14:04:30 +0200 Subject: [PATCH 22/58] fix[notask]: remove incomplete MTP support Assisted-by: GPT-5.5 Co-authored-by: Cursor --- .../addon/src/model-interface/LlamaModel.cpp | 15 ++- .../src/model-interface/TextLlmContext.cpp | 77 +----------- .../src/model-interface/TextLlmContext.hpp | 26 ---- .../llm-llamacpp/test/integration/mtp.test.js | 111 ------------------ .../test/mobile/integration.auto.cjs | 5 - .../llm-llamacpp/test/mobile/test-groups.json | 2 +- 6 files changed, 15 insertions(+), 221 deletions(-) delete mode 100644 packages/llm-llamacpp/test/integration/mtp.test.js diff --git a/packages/llm-llamacpp/addon/src/model-interface/LlamaModel.cpp b/packages/llm-llamacpp/addon/src/model-interface/LlamaModel.cpp index 3c80dbf628..9e52ddc90b 100644 --- a/packages/llm-llamacpp/addon/src/model-interface/LlamaModel.cpp +++ b/packages/llm-llamacpp/addon/src/model-interface/LlamaModel.cpp @@ -19,7 +19,6 @@ #include #include #include -#include #include #include #ifdef __APPLE__ @@ -1309,11 +1308,15 @@ void LlamaModel::commonParamsParse( for (const std::string& key : {"spec-type", "spec_type"}) { if (auto iter = configFilemap.find(key); iter != configFilemap.end()) { - auto types = common_speculative_types_from_names( - split(iter->second, ',')); - params.speculative.types.insert( - params.speculative.types.end(), types.begin(), types.end()); - configFilemap.erase(iter); + const auto requested = split(iter->second, ','); + if (std::find(requested.begin(), requested.end(), "draft-mtp") != + requested.end()) { + throw qvac_errors::StatusError( + ADDON_ID, + qvac_errors::general_error::toString( + qvac_errors::general_error::InvalidArgument), + "spec-type=draft-mtp is not supported"); + } } } diff --git a/packages/llm-llamacpp/addon/src/model-interface/TextLlmContext.cpp b/packages/llm-llamacpp/addon/src/model-interface/TextLlmContext.cpp index 7cff3558f0..e59ce2bfbb 100644 --- a/packages/llm-llamacpp/addon/src/model-interface/TextLlmContext.cpp +++ b/packages/llm-llamacpp/addon/src/model-interface/TextLlmContext.cpp @@ -15,7 +15,6 @@ #include "addon/LlmErrors.hpp" #include "common/common.h" #include "common/log.h" -#include "common/speculative.h" #include "inference-addon-cpp/Logger.hpp" #include "utils/ChatTemplateUtils.hpp" #include "utils/LoggingMacros.hpp" @@ -139,45 +138,6 @@ void TextLlmContext::initializeCommonState() { ADDON_ID, toString(UnableToCreateSamplingSystem), errorMsg); } - const bool wantMtpDraft = std::find( - params_.speculative.types.begin(), - params_.speculative.types.end(), - COMMON_SPECULATIVE_TYPE_DRAFT_MTP) != - params_.speculative.types.end(); - if (wantMtpDraft) { - try { - auto cparamsMtp = common_context_params_to_llama(params_); - cparamsMtp.ctx_type = LLAMA_CONTEXT_TYPE_MTP; - cparamsMtp.type_k = params_.speculative.draft.cache_type_k; - cparamsMtp.type_v = params_.speculative.draft.cache_type_v; - cparamsMtp.n_rs_seq = 0; - ctxDraft_.reset(llama_init_from_model(modelCtx_.model, cparamsMtp)); - if (!ctxDraft_) { - QLOG_IF( - Priority::WARNING, - "[TextLlm] MTP draft context could not be created for this " - "model; spec-type=draft-mtp will be inert\n"); - } else { - params_.speculative.draft.ctx_tgt = modelCtx_.lctx; - params_.speculative.draft.ctx_dft = ctxDraft_.get(); - spec_.reset(common_speculative_init( - params_.speculative, std::max(1, params_.n_parallel))); - QLOG_IF( - Priority::INFO, - "[TextLlm] MTP draft context + common_speculative initialized\n"); - } - } catch (const std::exception& e) { - QLOG_IF( - Priority::WARNING, - string_format( - "[TextLlm] MTP draft setup failed (%s); continuing without " - "speculative decoding\n", - e.what())); - ctxDraft_.reset(); - spec_.reset(); - } - } - if (!llama_model_has_encoder(modelCtx_.model) && llama_vocab_get_add_eos(modelCtx_.vocab)) { throw qvac_errors::StatusError( @@ -460,7 +420,7 @@ bool TextLlmContext::evalMessageWithTools( textBatch->logits[textBatch->n_tokens - 1] = static_cast(true); } // NOLINTNEXTLINE(clang-analyzer-core.CallAndMessage) - int ret = decodeAndSpecProcess(*textBatch); + int ret = llama_decode(modelCtx_.lctx, *textBatch); if (ret != 0) { std::string errorMsg = string_format( "[TextLlm] %s: failed to decode input tokens\n", __func__); @@ -642,7 +602,7 @@ void TextLlmContext::handleStopRequestAndAddEot(LlamaBatch& batch) { nPast_++, {seqId_}, true); - if (decodeAndSpecProcess(*batch) != 0) { + if (llama_decode(modelCtx_.lctx, *batch) != 0) { const char* errorMsg = "[TextLlm] failed to decode EOT token\n"; throw qvac_errors::StatusError( ADDON_ID, toString(FailedToDecode), errorMsg); @@ -652,12 +612,6 @@ void TextLlmContext::handleStopRequestAndAddEot(LlamaBatch& batch) { bool TextLlmContext::generateResponse( const std::function& outputCallback) { - if (spec_ && !specBeganGenerate_) { - common_speculative_begin(spec_.get(), seqId_, {}); - specBeganGenerate_ = true; - } - pendingSampled_ = LLAMA_TOKEN_NULL; - LlamaBatch batch(1, 0, 1); // batch for next token generation unsigned generatedAfterAccept = 0; @@ -711,7 +665,7 @@ bool TextLlmContext::generateResponse( common_batch_add(*batch, step.token, nPast_++, {seqId_}, true); // NOLINT(clang-analyzer-core.CallAndMessage) - if (decodeAndSpecProcess(*batch) != 0) { + if (llama_decode(modelCtx_.lctx, *batch) != 0) { const char* errorMsg = "[TextLlm] failed to decode next token\n"; throw qvac_errors::StatusError( ADDON_ID, toString(FailedToDecode), errorMsg); @@ -918,9 +872,6 @@ bool TextLlmContext::loadCache( if (auto* mem = llama_get_memory(modelCtx_.lctx); mem != nullptr) { llama_memory_seq_rm(mem, seqId_, nPast_, -1); } - if (spec_) { - common_speculative_print_stats(spec_.get()); - } return true; } @@ -1032,24 +983,6 @@ llama_pos TextLlmContext::removeLastNTokens(llama_pos count) { return tokensToRemove; } -int TextLlmContext::decodeAndSpecProcess(const llama_batch& batch) { - const int ret = llama_decode(modelCtx_.lctx, batch); - if (ret != 0) { - return ret; - } - if (spec_) { - if (!common_speculative_process(spec_.get(), batch)) { - QLOG_IF( - Priority::WARNING, - "[TextLlm] common_speculative_process failed; disabling spec for " - "the rest of this model lifetime\n"); - spec_.reset(); - ctxDraft_.reset(); - } - } - return ret; -} - bool TextLlmContext::handleQwen3ReasoningEOS( llama_token& tokenId, std::string& tokenStr, llama_batch& batch, llama_pos& nPast, @@ -1080,7 +1013,7 @@ bool TextLlmContext::handleQwen3ReasoningEOS( // Decode closing tag common_batch_clear(batch); common_batch_add(batch, tokenId, nPast++, {seqId_}, true); - if (decodeAndSpecProcess(batch) != 0) { + if (llama_decode(modelCtx_.lctx, batch) != 0) { QLOG_IF( Priority::ERROR, "[TextLlm] Failed to decode closing tag during replacement\n"); @@ -1093,7 +1026,7 @@ bool TextLlmContext::handleQwen3ReasoningEOS( common_batch_add( batch, reasoningState_.cached_newline_token, nPast++, {seqId_}, true); - if (decodeAndSpecProcess(batch) != 0) { + if (llama_decode(modelCtx_.lctx, batch) != 0) { QLOG_IF( Priority::ERROR, "[TextLlm] Failed to decode newline token during forced " diff --git a/packages/llm-llamacpp/addon/src/model-interface/TextLlmContext.hpp b/packages/llm-llamacpp/addon/src/model-interface/TextLlmContext.hpp index 56b98861a9..02096a2790 100644 --- a/packages/llm-llamacpp/addon/src/model-interface/TextLlmContext.hpp +++ b/packages/llm-llamacpp/addon/src/model-interface/TextLlmContext.hpp @@ -2,7 +2,6 @@ #include -#include #include #include "../utils/ChatTemplateUtils.hpp" @@ -12,7 +11,6 @@ #include "SequenceDriver.hpp" #include "ToolsCompactController.hpp" #include "common/common.h" -#include "common/speculative.h" #include "inference-addon-cpp/Logger.hpp" /// Concrete text-only LLM context. Implements both the legacy @@ -250,11 +248,6 @@ class TextLlmContext : public LlmContext, public SequenceDriver { llama_pos applyContextDiscard(); void handleStopRequestAndAddEot(LlamaBatch& batch); - // Wraps llama_decode(lctx_, batch). When MTP/speculative is active, also - // feeds the batch into common_speculative_process so the draft context's - // KV cache tracks the target's on every prefill ubatch + generation step. - int decodeAndSpecProcess(const llama_batch& batch); - ToolsCompactController& tools_; common_init_result_ptr llamaInit_; LlmModelContext modelCtx_; @@ -302,24 +295,5 @@ class TextLlmContext : public LlmContext, public SequenceDriver { std::string lastChatParser_; std::string lastGenerationPrompt_; - // Speculative decoding state for MTP. If non-null, the draft context and spec struct are live - // and common_speculative_process is called on every decode. - llama_context_ptr ctxDraft_; - common_speculative_ptr spec_; - // common_speculative_begin must run once before the first decode of each - // generation; this flag de-duplicates across multi-turn calls. - bool specBeganGenerate_ = false; - // common_speculative_get_draft_params requires a non-null .prompt; the MTP - // impl never reads its contents (only id_last/n_past/n_max), but the wrapper - // dereferences ->size() for logging. - std::vector specDummyPrompt_; - // Sampled-but-not-yet-consumed token from the previous iteration's - // sample_and_accept_n (the "extra" id past the last accepted draft). - // LLAMA_TOKEN_NULL means: sample fresh next iter. - llama_token pendingSampled_ = LLAMA_TOKEN_NULL; - // Snapshot of ctxDraft_'s partial sequence state (SWA + recurrent) taken - // right before common_speculative_draft AR-decodes onto ctx_dft. - std::vector specCkptDft_; - std::atomic stopGeneration_ = false; }; diff --git a/packages/llm-llamacpp/test/integration/mtp.test.js b/packages/llm-llamacpp/test/integration/mtp.test.js deleted file mode 100644 index 9d7fbb5709..0000000000 --- a/packages/llm-llamacpp/test/integration/mtp.test.js +++ /dev/null @@ -1,111 +0,0 @@ -'use strict' - -// MTP (Multi-Token Prediction) speculative-decoding integration test. - -const test = require('brittle') -const path = require('bare-path') -const LlmLlamacpp = require('../../index.js') -const { ensureModel } = require('./utils') -const os = require('bare-os') - -const platform = os.platform() -const arch = os.arch() -const isDarwinX64 = platform === 'darwin' && arch === 'x64' -const isLinuxArm64 = platform === 'linux' && arch === 'arm64' -const useCpu = isDarwinX64 || isLinuxArm64 - -const MODEL = { - name: 'Qwen3.5-0.8B-Q8_0.gguf', - url: 'https://huggingface.co/unsloth/Qwen3.5-0.8B-GGUF/resolve/main/Qwen3.5-0.8B-Q8_0.gguf' -} - -const PROMPT = [ - { role: 'system', content: 'You are a helpful assistant.' }, - { role: 'user', content: 'What is the capital of France? Answer in one word.' } -] - -function createLogger () { - return { - info: (...args) => console.info(...args), - warn: (...args) => console.warn(...args), - error: (...args) => console.error(...args), - debug: (...args) => console.debug(...args) - } -} - -async function collectResponse (response) { - const chunks = [] - const ticker = setInterval(() => {}, 50) - try { - await response.onUpdate(data => { chunks.push(data) }).await() - } finally { - clearInterval(ticker) - } - return chunks.join('').trim() -} - -async function runOnce ({ withSpec }) { - const [modelName, dirPath] = await ensureModel({ - modelName: MODEL.name, - downloadUrl: MODEL.url - }) - const modelPath = path.join(dirPath, modelName) - - const config = { - device: useCpu ? 'cpu' : 'gpu', - gpu_layers: '999', - ctx_size: '1024', - n_predict: '64', - temp: '0', - seed: '42', - 'reasoning-budget': '0', - verbosity: '2' - } - if (withSpec) { - config['spec-type'] = 'draft-mtp' - } - - const addon = new LlmLlamacpp({ - files: { model: [modelPath] }, - config, - logger: createLogger(), - opts: { stats: true } - }) - - try { - await addon.load() - const response = await addon.run(PROMPT) - const output = await collectResponse(response) - return { output, stats: response.stats } - } finally { - await addon.unload().catch(() => {}) - } -} - -test('Qwen3.5-0.8B with spec-type=draft-mtp drafts + accepts', { - timeout: 600_000 -}, async t => { - const { output, stats } = await runOnce({ withSpec: true }) - t.ok(output.length > 0, `spec run produced output (${output.length} chars)`) - console.log(` spec output: "${output.slice(0, 200)}"`) - t.ok(stats, 'spec run has response.stats') - t.ok( - /paris/i.test(output), - 'spec output names the capital (Paris) in the reply' - ) -}) - -test('Qwen3.5-0.8B without spec-type', { - timeout: 600_000 -}, async t => { - // Sentinel: confirms the addon's existing single-context path still works, - // proving the MTP code added to TextLlmContext is correctly gated behind - // the `spec-type` config check and doesn't fire for default-config loads. - const { output, stats } = await runOnce({ withSpec: false }) - t.ok(output.length > 0, `non-spec run produced output (${output.length} chars)`) - t.ok(stats, 'non-spec run has stats') - t.ok( - /paris/i.test(output), - 'non-spec output names the capital (Paris) in the reply' - ) -}) diff --git a/packages/llm-llamacpp/test/mobile/integration.auto.cjs b/packages/llm-llamacpp/test/mobile/integration.auto.cjs index 94842a1bc0..8648753f49 100644 --- a/packages/llm-llamacpp/test/mobile/integration.auto.cjs +++ b/packages/llm-llamacpp/test/mobile/integration.auto.cjs @@ -96,11 +96,6 @@ async function runMropeSlidingContextTest (options = {}) { // eslint-disable-lin return runIntegrationModule('../integration/mrope-sliding-context.test.js', options) } -async function runMtpTest (options = {}) { // eslint-disable-line no-unused-vars - if (typeof __shouldRunTest === 'function' && !__shouldRunTest('runMtpTest')) return __FILTERED - return runIntegrationModule('../integration/mtp.test.js', options) -} - async function runMultiGpuTest (options = {}) { // eslint-disable-line no-unused-vars if (typeof __shouldRunTest === 'function' && !__shouldRunTest('runMultiGpuTest')) return __FILTERED return runIntegrationModule('../integration/multi-gpu.test.js', options) diff --git a/packages/llm-llamacpp/test/mobile/test-groups.json b/packages/llm-llamacpp/test/mobile/test-groups.json index 58b31085c7..09c68abc93 100644 --- a/packages/llm-llamacpp/test/mobile/test-groups.json +++ b/packages/llm-llamacpp/test/mobile/test-groups.json @@ -8,7 +8,6 @@ "gemma": ["runGemma4Test"], "quantizedKvcache": ["runQuantizedKvcacheTest"], "continuousBatching": ["runContinuousBatchingTest"], - "mtp": ["runMtpTest"], "lightA": [ "runApiBehaviorTest", "runBitnetTest", @@ -29,6 +28,7 @@ ] }, "android": { + "continuousBatching": ["runContinuousBatchingTest"], "funcShardA": [ "runFinetuningPauseResumeTest", "runConfigParametersTest", From fe48d7dfaa70689d0573f720008cf3a3eef2e68e Mon Sep 17 00:00:00 2001 From: gianni-cor Date: Thu, 18 Jun 2026 19:46:07 +0200 Subject: [PATCH 23/58] fix[notask]: remove tool-call envelope wrapping Assisted-by: GPT-5.5 Co-authored-by: Cursor --- .../addon/src/model-interface/LlamaModel.cpp | 75 +------------------ .../addon/src/model-interface/LlmContext.hpp | 34 --------- .../src/model-interface/MtmdLlmContext.cpp | 3 +- .../src/model-interface/TextLlmContext.cpp | 4 +- .../src/model-interface/TextLlmContext.hpp | 20 ----- .../addon/src/utils/ChatTemplateUtils.cpp | 13 +--- .../addon/src/utils/ChatTemplateUtils.hpp | 12 +-- 7 files changed, 5 insertions(+), 156 deletions(-) diff --git a/packages/llm-llamacpp/addon/src/model-interface/LlamaModel.cpp b/packages/llm-llamacpp/addon/src/model-interface/LlamaModel.cpp index 9e52ddc90b..cf7d7da065 100644 --- a/packages/llm-llamacpp/addon/src/model-interface/LlamaModel.cpp +++ b/packages/llm-llamacpp/addon/src/model-interface/LlamaModel.cpp @@ -7,7 +7,6 @@ #include #include #include -#include #include #include #include @@ -715,8 +714,7 @@ std::string LlamaModel::processPromptImpl(const Prompt& prompt) { std::ostringstream oss; // The streamed output must be captured into `oss` whenever any // post-generation pass needs to inspect the full assistant text. - const bool needsOutputCapture = - state_->toolsCompact_->enabled() || !resolved.tools.empty(); + const bool needsOutputCapture = state_->toolsCompact_->enabled(); auto callback = prompt.outputCallback; if (!prompt.outputCallback) { callback = [&](const std::string& token) { oss << token; }; @@ -738,77 +736,6 @@ std::string LlamaModel::processPromptImpl(const Prompt& prompt) { out = oss.str(); } - // Re-wrap any extracted tool calls as {...} envelopes - // so JS-side tests and SDK consumers see a uniform shape regardless of which - // template/dialect the model used. Skip models that already emit standard - // envelopes natively (e.g. Qwen3): re-wrapping their parsed - // calls would append a duplicate envelope after the one already in the raw - // output. Gemma 4's dialect uses <|tool_call>/ (no literal - // "" substring), so it is unaffected by this guard and still gets - // re-wrapped. - const std::string assistantText = prompt.outputCallback ? oss.str() : out; - if (!resolved.tools.empty() && - assistantText.find("") == std::string::npos) { - common_chat_parser_params syntax; - syntax.format = state_->llmContext_->getLastChatFormat(); - syntax.parse_tool_calls = true; - syntax.generation_prompt = state_->llmContext_->getLastGenerationPrompt(); - // The fabric's chat layer builds a dialect-specific PEG parser at - // templating time and serializes it into common_chat_params::parser. - // Without loading it here, common_chat_parse falls back to a pure-content - // parser and extracts zero tool calls (e.g. Gemma 4's - // <|tool_call>call:NAME{...} dialect goes unparsed). - const std::string& serializedParser = - state_->llmContext_->getLastChatParser(); - if (!serializedParser.empty()) { - syntax.parser.load(serializedParser); - } - try { - common_chat_msg parsed = common_chat_parse( - assistantText, /*is_partial=*/false, syntax); - if (!parsed.tool_calls.empty()) { - const auto normalizeArgsJson = - [](const std::string& src) -> std::string { - static const std::regex unquotedKeyRe( - R"(([{,])(\s*)([A-Za-z_][A-Za-z0-9_]*)(\s*):)", - std::regex::ECMAScript); - return std::regex_replace(src, unquotedKeyRe, R"($1$2"$3"$4:)"); - }; - std::string envelopes; - envelopes.reserve(parsed.tool_calls.size() * 64); - for (const auto& tc : parsed.tool_calls) { - std::string args = tc.arguments.empty() - ? std::string("{}") - : normalizeArgsJson(tc.arguments); - envelopes.append("{\"name\":\""); - envelopes.append(tc.name); - envelopes.append("\",\"arguments\":"); - envelopes.append(args); - envelopes.append("}"); - } - - if (prompt.outputCallback) { - prompt.outputCallback(envelopes); - } else { - out = parsed.content; - out.append(envelopes); - } - } - } catch (const std::exception& e) { - QLOG_IF( - Priority::WARNING, - string_format( - "[LlamaModel] common_chat_parse failed for tool extraction: " - "%s\n", - e.what())); - } catch (...) { - QLOG_IF( - Priority::WARNING, - "[LlamaModel] common_chat_parse threw an unknown exception during " - "tool extraction\n"); - } - } - // Post-generation tools trim decision via controller std::string ossStr; if (needsOutputCapture && prompt.outputCallback) { diff --git a/packages/llm-llamacpp/addon/src/model-interface/LlmContext.hpp b/packages/llm-llamacpp/addon/src/model-interface/LlmContext.hpp index 89b40a0c1e..2b18b1e030 100644 --- a/packages/llm-llamacpp/addon/src/model-interface/LlmContext.hpp +++ b/packages/llm-llamacpp/addon/src/model-interface/LlmContext.hpp @@ -337,40 +337,6 @@ class LlmContext { // NOLINT(cppcoreguidelines-special-member-functions) */ virtual void resetMedia() {}; - /** - * Returns the chat format selected by the most recent prompt template - * application (e.g. COMMON_CHAT_FORMAT_GEMMA4). Used by post-generation - * tool-call extraction to tell common_chat_parse which dialect the - * assistant text is in. - */ - [[nodiscard]] virtual common_chat_format getLastChatFormat() const { - return COMMON_CHAT_FORMAT_CONTENT_ONLY; - } - - /** - * Returns the serialized PEG parser produced by the most recent prompt - * template application. The fabric's chat layer builds a dialect-specific - * parser at templating time and stores it (serialized) in - * common_chat_params::parser; common_chat_parse needs it loaded into the - * parser arena, otherwise it silently falls back to a pure-content parser - * and extracts zero tool calls. Empty when no parser was produced. - */ - [[nodiscard]] virtual const std::string& getLastChatParser() const { - static const std::string kEmpty; - return kEmpty; - } - - /** - * Returns the generation prompt from the most recent prompt template - * application. common_chat_parse prepends it to the assistant text so the - * parser's `start` rule (which bakes in the generation-prompt prefix at - * build time) aligns with the input. - */ - [[nodiscard]] virtual const std::string& getLastGenerationPrompt() const { - static const std::string kEmpty; - return kEmpty; - } - /// Validates an incoming prompt against any policy-level constraints /// (size, layout, KV-cache state). Default is a no-op; concrete /// contexts (`TextLlmContext`, `MtmdLlmContext`) override as needed. diff --git a/packages/llm-llamacpp/addon/src/model-interface/MtmdLlmContext.cpp b/packages/llm-llamacpp/addon/src/model-interface/MtmdLlmContext.cpp index 48bfacf0ef..788d376c71 100644 --- a/packages/llm-llamacpp/addon/src/model-interface/MtmdLlmContext.cpp +++ b/packages/llm-llamacpp/addon/src/model-interface/MtmdLlmContext.cpp @@ -216,8 +216,7 @@ void MtmdLlmContext::tokenizeChat( if (!tools.empty()) { inputs.tools = tools; } - formattedChat = getPrompt(tmpls_.get(), inputs, - /*outFormat=*/nullptr, &thinkingForcedOpen_); + formattedChat = getPrompt(tmpls_.get(), inputs, &thinkingForcedOpen_); if (formattedChat.empty()) { std::string errorMsg = string_format( diff --git a/packages/llm-llamacpp/addon/src/model-interface/TextLlmContext.cpp b/packages/llm-llamacpp/addon/src/model-interface/TextLlmContext.cpp index e59ce2bfbb..8366bd5cba 100644 --- a/packages/llm-llamacpp/addon/src/model-interface/TextLlmContext.cpp +++ b/packages/llm-llamacpp/addon/src/model-interface/TextLlmContext.cpp @@ -305,9 +305,7 @@ void TextLlmContext::tokenizeChat( if (!tools.empty()) { inputs.tools = tools; } - prompt = getPrompt( - tmpls_.get(), inputs, &lastChatFormat_, &thinkingForcedOpen_, - &lastChatParser_, &lastGenerationPrompt_); + prompt = getPrompt(tmpls_.get(), inputs, &thinkingForcedOpen_); QLOG_IF( Priority::DEBUG, diff --git a/packages/llm-llamacpp/addon/src/model-interface/TextLlmContext.hpp b/packages/llm-llamacpp/addon/src/model-interface/TextLlmContext.hpp index 02096a2790..d53764c9c0 100644 --- a/packages/llm-llamacpp/addon/src/model-interface/TextLlmContext.hpp +++ b/packages/llm-llamacpp/addon/src/model-interface/TextLlmContext.hpp @@ -160,18 +160,6 @@ class TextLlmContext : public LlmContext, public SequenceDriver { */ llama_pos removeLastNTokens(llama_pos count) override; - [[nodiscard]] common_chat_format getLastChatFormat() const override { - return lastChatFormat_; - } - - [[nodiscard]] const std::string& getLastChatParser() const override { - return lastChatParser_; - } - - [[nodiscard]] const std::string& getLastGenerationPrompt() const override { - return lastGenerationPrompt_; - } - std::vector preparePrefill( const std::vector& chatMsgs, const std::vector& tools, bool isCacheLoaded, @@ -287,13 +275,5 @@ class TextLlmContext : public LlmContext, public SequenceDriver { // tags. bool thinkingForcedOpen_ = false; - common_chat_format lastChatFormat_ = COMMON_CHAT_FORMAT_CONTENT_ONLY; - - // Serialized PEG parser and generation prompt from the most recent prompt - // template application, needed by post-generation tool-call extraction (see - // getLastChatParser / getLastGenerationPrompt). - std::string lastChatParser_; - std::string lastGenerationPrompt_; - std::atomic stopGeneration_ = false; }; diff --git a/packages/llm-llamacpp/addon/src/utils/ChatTemplateUtils.cpp b/packages/llm-llamacpp/addon/src/utils/ChatTemplateUtils.cpp index 044e447a9d..9bc66778b5 100644 --- a/packages/llm-llamacpp/addon/src/utils/ChatTemplateUtils.cpp +++ b/packages/llm-llamacpp/addon/src/utils/ChatTemplateUtils.cpp @@ -174,22 +174,11 @@ std::string getChatTemplate( std::string getPrompt( const struct common_chat_templates* tmpls, - struct common_chat_templates_inputs& inputs, common_chat_format* outFormat, - bool* outThinkingForcedOpen, std::string* outParser, - std::string* outGenerationPrompt) { + struct common_chat_templates_inputs& inputs, bool* outThinkingForcedOpen) { auto exportParams = [&](const common_chat_params& params) { - if (outFormat) { - *outFormat = params.format; - } if (outThinkingForcedOpen) { *outThinkingForcedOpen = params.thinking_forced_open; } - if (outParser) { - *outParser = params.parser; - } - if (outGenerationPrompt) { - *outGenerationPrompt = params.generation_prompt; - } }; try { auto params = common_chat_templates_apply(tmpls, inputs); diff --git a/packages/llm-llamacpp/addon/src/utils/ChatTemplateUtils.hpp b/packages/llm-llamacpp/addon/src/utils/ChatTemplateUtils.hpp index 1d4cc481e1..b523e51b98 100644 --- a/packages/llm-llamacpp/addon/src/utils/ChatTemplateUtils.hpp +++ b/packages/llm-llamacpp/addon/src/utils/ChatTemplateUtils.hpp @@ -69,23 +69,13 @@ std::string getChatTemplate( * @brief Applies chat templates to generate a prompt, with fallback handling * for models that don't support tools. * - * @p outFormat (optional) receives the parsed chat format identifier. * @p outThinkingForcedOpen (optional) receives the flag indicating that the * template force-opened the reasoning channel in the prompt suffix. - * @p outParser (optional) receives the serialized PEG parser - * (common_chat_params::parser) needed for post-generation tool-call - * extraction. - * @p outGenerationPrompt (optional) receives the generation prompt - * (common_chat_params::generation_prompt) that the parser expects to be - * prepended to the assistant text. */ std::string getPrompt( const struct common_chat_templates* tmpls, struct common_chat_templates_inputs& inputs, - common_chat_format* outFormat = nullptr, - bool* outThinkingForcedOpen = nullptr, - std::string* outParser = nullptr, - std::string* outGenerationPrompt = nullptr); + bool* outThinkingForcedOpen = nullptr); } // namespace utils } // namespace qvac_lib_inference_addon_llama From bcf2525ede3ee7f4e78270a4edef617fea0e31f1 Mon Sep 17 00:00:00 2001 From: gianni-cor Date: Thu, 18 Jun 2026 20:24:03 +0200 Subject: [PATCH 24/58] fix[notask]: remove speculative decoding leftovers Assisted-by: GPT-5.5 Co-authored-by: Cursor --- .../addon/src/model-interface/LlamaModel.cpp | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/packages/llm-llamacpp/addon/src/model-interface/LlamaModel.cpp b/packages/llm-llamacpp/addon/src/model-interface/LlamaModel.cpp index cf7d7da065..771e89c3c5 100644 --- a/packages/llm-llamacpp/addon/src/model-interface/LlamaModel.cpp +++ b/packages/llm-llamacpp/addon/src/model-interface/LlamaModel.cpp @@ -1233,20 +1233,6 @@ void LlamaModel::commonParamsParse( } } - for (const std::string& key : {"spec-type", "spec_type"}) { - if (auto iter = configFilemap.find(key); iter != configFilemap.end()) { - const auto requested = split(iter->second, ','); - if (std::find(requested.begin(), requested.end(), "draft-mtp") != - requested.end()) { - throw qvac_errors::StatusError( - ADDON_ID, - qvac_errors::general_error::toString( - qvac_errors::general_error::InvalidArgument), - "spec-type=draft-mtp is not supported"); - } - } - } - // transform json config into the format required by llama.cpp for (auto& keyValuePair : configFilemap) { configVector.push_back(std::string("--") + keyValuePair.first); @@ -1353,10 +1339,6 @@ void LlamaModel::commonParamsParse( postprocess_cpu_params(params.cpuparams, nullptr); postprocess_cpu_params(params.cpuparams_batch, ¶ms.cpuparams); - postprocess_cpu_params(params.speculative.draft.cpuparams, ¶ms.cpuparams); - postprocess_cpu_params( - params.speculative.draft.cpuparams_batch, ¶ms.cpuparams_batch); - if (!params.kv_overrides.empty()) { params.kv_overrides.emplace_back(); params.kv_overrides.back().key[0] = 0; From 706bb0be8f19c1f3f4c6c1d20d46b15e80b69972 Mon Sep 17 00:00:00 2001 From: gianni-cor Date: Thu, 18 Jun 2026 22:54:30 +0200 Subject: [PATCH 25/58] fix[notask]: remove redundant tools compact capture Assisted-by: GPT-5.5 Co-authored-by: Cursor --- .../addon/src/model-interface/LlamaModel.cpp | 28 ------------------- 1 file changed, 28 deletions(-) diff --git a/packages/llm-llamacpp/addon/src/model-interface/LlamaModel.cpp b/packages/llm-llamacpp/addon/src/model-interface/LlamaModel.cpp index 771e89c3c5..c2e32b39e2 100644 --- a/packages/llm-llamacpp/addon/src/model-interface/LlamaModel.cpp +++ b/packages/llm-llamacpp/addon/src/model-interface/LlamaModel.cpp @@ -712,17 +712,9 @@ std::string LlamaModel::processPromptImpl(const Prompt& prompt) { } std::ostringstream oss; - // The streamed output must be captured into `oss` whenever any - // post-generation pass needs to inspect the full assistant text. - const bool needsOutputCapture = state_->toolsCompact_->enabled(); auto callback = prompt.outputCallback; if (!prompt.outputCallback) { callback = [&](const std::string& token) { oss << token; }; - } else if (needsOutputCapture) { - callback = [&](const std::string& token) { - oss << token; - prompt.outputCallback(token); - }; } if (!state_->llmContext_->generateResponse(callback)) { @@ -736,26 +728,6 @@ std::string LlamaModel::processPromptImpl(const Prompt& prompt) { out = oss.str(); } - // Post-generation tools trim decision via controller - std::string ossStr; - if (needsOutputCapture && prompt.outputCallback) { - // Only materialize a second copy when output streamed via callback. - ossStr = oss.str(); - } - const std::string& outputToCheck = - needsOutputCapture ? (prompt.outputCallback ? ossStr : out) : out; - auto decision = state_->toolsCompact_->onGenerationComplete( - outputToCheck, - state_->llmContext_->getNPast(), - state_->llmContext_->getFirstMsgTokens()); - if (decision.trim) { - state_->llmContext_->removeLastNTokens(decision.tokensToRemoveFromTail); - if (decision.clampFirstMsgTokensToNPast && - state_->llmContext_->getFirstMsgTokens() > - state_->llmContext_->getNPast()) { - state_->llmContext_->setFirstMsgTokens(state_->llmContext_->getNPast()); - } - } maybeSaveCacheToDisk(prompt.saveCacheToDisk, state_->cacheManager_); if (resolved.shouldResetAfterInference) { From 88a24d3aafe638a5b707b783aaf179d8e4306333 Mon Sep 17 00:00:00 2001 From: gianni-cor Date: Thu, 18 Jun 2026 23:47:33 +0200 Subject: [PATCH 26/58] fix[notask]: use template reasoning budget tags Assisted-by: GPT-5.5 Co-authored-by: Cursor --- .../model-interface/GenerationParamsApply.cpp | 11 ++- .../addon/src/model-interface/LlmContext.hpp | 11 +-- .../src/model-interface/MtmdLlmContext.cpp | 36 +++++++++- .../src/model-interface/MtmdLlmContext.hpp | 5 +- .../src/model-interface/TextLlmContext.cpp | 60 +++++++++------- .../src/model-interface/TextLlmContext.hpp | 5 +- .../addon/src/utils/ChatTemplateUtils.cpp | 70 ++++++++++++++++++- .../addon/src/utils/ChatTemplateUtils.hpp | 29 +++++++- .../test/integration/reasoning.test.js | 26 +++++-- 9 files changed, 200 insertions(+), 53 deletions(-) diff --git a/packages/llm-llamacpp/addon/src/model-interface/GenerationParamsApply.cpp b/packages/llm-llamacpp/addon/src/model-interface/GenerationParamsApply.cpp index 3db142ce9c..0a509a7969 100644 --- a/packages/llm-llamacpp/addon/src/model-interface/GenerationParamsApply.cpp +++ b/packages/llm-llamacpp/addon/src/model-interface/GenerationParamsApply.cpp @@ -30,9 +30,8 @@ void applyGenerationOverridesToSampling( setIf(overrides.repeat_penalty, sampling.penalty_repeat); // Forward reasoning_budget into the budget-sampler's token cap. The - // start/end/forced vectors were tokenized in the TextLlmContext ctor for - // Qwen3 models and survive the copy from params.sampling into the local - // override, so we only need to mutate the count here. + // template-specific start/end/forced vectors are refreshed after prompt + // formatting, where common_chat_params exposes the thinking tags. if (overrides.reasoning_budget) { const int budget = *overrides.reasoning_budget; if (budget > 0) { @@ -55,7 +54,7 @@ void applyGenerationOverridesToSampling( LOG_WRN( "%s: both generationParams.grammar and generationParams.json_schema " "were provided; ignoring `grammar` and applying `json_schema` " - "(the JS and AddonJs paths reject this combination — this branch " + "(the JS and AddonJs paths reject this combination - this branch " "exists only for direct C++ callers).\n", __func__); } @@ -87,7 +86,7 @@ std::function applyGenerationParamsToContext( // Apply overrides to *local copies* first. Only commit them onto the // live `params` and `smpl` after both the json_schema parse/convert and - // `common_sampler_init` have succeeded — otherwise a partially applied + // `common_sampler_init` have succeeded - otherwise a partially applied // override (e.g. temp/seed already written, then json_schema throws) // would leak into subsequent requests because no restore lambda gets // returned to the caller's `ScopeGuard`. @@ -99,7 +98,7 @@ std::function applyGenerationParamsToContext( applyGenerationOverridesToSampling(nextSampling, nextPredict, overrides); // `common_sampler_init` returns nullptr on bad inputs (most commonly an - // invalid GBNF grammar — `json_schema` is converted to GBNF above and + // invalid GBNF grammar - `json_schema` is converted to GBNF above and // can in principle produce a grammar that the sampler rejects). Build // the new sampler before touching live state so a failure here also // leaves `params` / `smpl` intact. diff --git a/packages/llm-llamacpp/addon/src/model-interface/LlmContext.hpp b/packages/llm-llamacpp/addon/src/model-interface/LlmContext.hpp index 2b18b1e030..cd74778cd3 100644 --- a/packages/llm-llamacpp/addon/src/model-interface/LlmContext.hpp +++ b/packages/llm-llamacpp/addon/src/model-interface/LlmContext.hpp @@ -31,13 +31,14 @@ struct GenerationParams { std::optional grammar; // JSON-Schema applied per request. Converted to GBNF via llama.cpp's // `json_schema_to_grammar()` and applied identically to `grammar`. - // Mutually exclusive with `grammar` — the JS wrapper rejects requests + // Mutually exclusive with `grammar` - the JS wrapper rejects requests // that set both. Mirrors the load-time `--json-schema` flag. std::optional json_schema; - // Reasoning channel budget override. `-1` keeps reasoning on, `0` disables - // it for this request. Mirrors the load-time `reasoning-budget` config; the - // override is applied to `params_.reasoning_budget` for the duration of the - // request and restored on completion. + // Reasoning channel budget override. `-1` keeps reasoning unrestricted, `0` + // disables it, and positive values cap the reasoning channel at that many + // tokens. Mirrors the load-time `reasoning-budget` config; the override is + // applied to `params_.reasoning_budget` for the duration of the request and + // restored on completion. std::optional reasoning_budget; [[nodiscard]] bool hasOverrides() const { diff --git a/packages/llm-llamacpp/addon/src/model-interface/MtmdLlmContext.cpp b/packages/llm-llamacpp/addon/src/model-interface/MtmdLlmContext.cpp index 788d376c71..9e59e5cdc6 100644 --- a/packages/llm-llamacpp/addon/src/model-interface/MtmdLlmContext.cpp +++ b/packages/llm-llamacpp/addon/src/model-interface/MtmdLlmContext.cpp @@ -216,7 +216,20 @@ void MtmdLlmContext::tokenizeChat( if (!tools.empty()) { inputs.tools = tools; } - formattedChat = getPrompt(tmpls_.get(), inputs, &thinkingForcedOpen_); + std::string thinkingStartTag; + std::string thinkingEndTag; + std::string generationPrompt; + formattedChat = getPrompt( + tmpls_.get(), + inputs, + &thinkingForcedOpen_, + &thinkingStartTag, + &thinkingEndTag, + &generationPrompt); + thinkingForcedOpenText_ = + thinkingForcedOpen_ + ? getThinkingForcedOpenText(generationPrompt, thinkingStartTag) + : std::string{}; if (formattedChat.empty()) { std::string errorMsg = string_format( @@ -224,6 +237,21 @@ void MtmdLlmContext::tokenizeChat( throw qvac_errors::StatusError(ADDON_ID, toString(EmptyPrompt), errorMsg); } + if (configureReasoningBudgetSampling( + params_, + modelCtx_.lctx, + thinkingStartTag, + thinkingEndTag, + generationPrompt)) { + smpl_.reset(common_sampler_init(modelCtx_.model, params_.sampling)); + if (!smpl_) { + std::string errorMsg = string_format( + "[MtmdLlm] %s: failed to initialize sampling subsystem\n", __func__); + throw qvac_errors::StatusError( + ADDON_ID, toString(UnableToCreateSamplingSystem), errorMsg); + } + } + QLOG_IF( Priority::DEBUG, string_format("[MtmdLlm] formatted prompt: %s\n", formattedChat.c_str())); @@ -492,10 +520,10 @@ bool MtmdLlmContext::generateResponse( if (thinkingForcedOpen_ && outputCallback) { // MtmdLlmContext doesn't carry a reasoningState_ (no reasoning-aware EOS // replacement on the multimodal path today), so unlike TextLlmContext we - // only prepend the visible "\n" opener and don't flip an + // only prepend the visible reasoning opener and don't flip an // inside_reasoning flag. If reasoning state is added here later, mirror // TextLlmContext::generateResponse and set it true alongside this emit. - outputCallback("\n"); + outputCallback(thinkingForcedOpenText_); } if (stopGeneration_.load()) { @@ -717,6 +745,8 @@ void MtmdLlmContext::resetState(bool resetStats) { // Clear UTF-8 buffer when resetting state utf8Buffer_.clear(); + thinkingForcedOpen_ = false; + thinkingForcedOpenText_.clear(); clearSequenceMemory(modelCtx_.lctx); diff --git a/packages/llm-llamacpp/addon/src/model-interface/MtmdLlmContext.hpp b/packages/llm-llamacpp/addon/src/model-interface/MtmdLlmContext.hpp index 356743c796..dcca7c7379 100644 --- a/packages/llm-llamacpp/addon/src/model-interface/MtmdLlmContext.hpp +++ b/packages/llm-llamacpp/addon/src/model-interface/MtmdLlmContext.hpp @@ -230,10 +230,11 @@ class MtmdLlmContext : public LlmContext { bool isHarmonyModel_ = false; llama_token harmonyCallToken_ = LLAMA_TOKEN_NULL; - // Force-opens the reasoning channel in the prompt suffix to prepend the - // matching "\n" opener to the visible stream so consumers see balanced + // Force-opens the reasoning channel in the prompt suffix. The text mirrors + // the template-specific visible reasoning opener so consumers see balanced // tags. bool thinkingForcedOpen_ = false; + std::string thinkingForcedOpenText_; std::atomic stopGeneration_ = false; }; diff --git a/packages/llm-llamacpp/addon/src/model-interface/TextLlmContext.cpp b/packages/llm-llamacpp/addon/src/model-interface/TextLlmContext.cpp index 8366bd5cba..7c9a60e230 100644 --- a/packages/llm-llamacpp/addon/src/model-interface/TextLlmContext.cpp +++ b/packages/llm-llamacpp/addon/src/model-interface/TextLlmContext.cpp @@ -26,25 +26,6 @@ using namespace qvac_lib_inference_addon_llama::utils; namespace { -// Populate params.sampling.reasoning_budget_{start,end,forced,tokens} from -// params.reasoning_budget so common_sampler_init budget-sampler kicks in. -void applyReasoningBudgetToSampling( - common_params& params, bool isQwen3Model, llama_context* lctx) { - if (params.reasoning_budget <= 0 || !isQwen3Model || lctx == nullptr) { - return; - } - params.sampling.reasoning_budget_tokens = params.reasoning_budget; - params.sampling.reasoning_budget_start = - common_tokenize(lctx, "", false, true); - params.sampling.reasoning_budget_end = - common_tokenize(lctx, "", false, true); - params.sampling.reasoning_budget_forced = common_tokenize( - lctx, - params.sampling.reasoning_budget_message + "", - false, - true); -} - bool isFileInitialized(const std::filesystem::path& path) { std::error_code errorCode; const auto size = std::filesystem::file_size(path, errorCode); @@ -128,8 +109,6 @@ void TextLlmContext::initializeCommonState() { getChatTemplate(modelCtx_.model, params_, tools_.enabled()); tmpls_ = common_chat_templates_init(modelCtx_.model, chatTemplate); - applyReasoningBudgetToSampling(params_, isQwen3Model_, modelCtx_.lctx); - smpl_.reset(common_sampler_init(modelCtx_.model, params_.sampling)); if (!smpl_) { std::string errorMsg = string_format( @@ -305,7 +284,34 @@ void TextLlmContext::tokenizeChat( if (!tools.empty()) { inputs.tools = tools; } - prompt = getPrompt(tmpls_.get(), inputs, &thinkingForcedOpen_); + std::string thinkingStartTag; + std::string thinkingEndTag; + std::string generationPrompt; + prompt = getPrompt( + tmpls_.get(), + inputs, + &thinkingForcedOpen_, + &thinkingStartTag, + &thinkingEndTag, + &generationPrompt); + thinkingForcedOpenText_ = + thinkingForcedOpen_ + ? getThinkingForcedOpenText(generationPrompt, thinkingStartTag) + : std::string{}; + if (configureReasoningBudgetSampling( + params_, + modelCtx_.lctx, + thinkingStartTag, + thinkingEndTag, + generationPrompt)) { + smpl_.reset(common_sampler_init(modelCtx_.model, params_.sampling)); + if (!smpl_) { + std::string errorMsg = string_format( + "[TextLlm] %s: failed to initialize sampling subsystem\n", __func__); + throw qvac_errors::StatusError( + ADDON_ID, toString(UnableToCreateSamplingSystem), errorMsg); + } + } QLOG_IF( Priority::DEBUG, @@ -619,12 +625,10 @@ bool TextLlmContext::generateResponse( assistantOutput_.clear(); generationStarted_ = false; - // The chat template force-opened the reasoning channel in the prompt (e.g. - // Qwen3-style / DeepSeek-R1 templates end with "\n"), so the model - // resumes generation INSIDE the reasoning block. (Gemma4's reasoning channel - // is model-emitted upstream and does not set this flag.) + // The chat template force-opened the reasoning channel in the prompt, so the + // model resumes generation inside the reasoning block. if (thinkingForcedOpen_ && outputCallback) { - outputCallback("\n"); + outputCallback(thinkingForcedOpenText_); reasoningState_.inside_reasoning = true; } @@ -920,6 +924,8 @@ void TextLlmContext::resetState(bool resetStats) { forcedTokens_.clear(); assistantOutput_.clear(); generationStarted_ = false; + thinkingForcedOpen_ = false; + thinkingForcedOpenText_.clear(); clearSequenceMemory(modelCtx_.lctx); diff --git a/packages/llm-llamacpp/addon/src/model-interface/TextLlmContext.hpp b/packages/llm-llamacpp/addon/src/model-interface/TextLlmContext.hpp index d53764c9c0..46d3068a64 100644 --- a/packages/llm-llamacpp/addon/src/model-interface/TextLlmContext.hpp +++ b/packages/llm-llamacpp/addon/src/model-interface/TextLlmContext.hpp @@ -270,10 +270,11 @@ class TextLlmContext : public LlmContext, public SequenceDriver { bool isHarmonyModel_ = false; llama_token harmonyCallToken_ = LLAMA_TOKEN_NULL; - // Force-opens the reasoning channel in the prompt suffix to prepend the - // matching "\n" opener to the visible stream so consumers see balanced + // Force-opens the reasoning channel in the prompt suffix. The text mirrors + // the template-specific visible reasoning opener so consumers see balanced // tags. bool thinkingForcedOpen_ = false; + std::string thinkingForcedOpenText_; std::atomic stopGeneration_ = false; }; diff --git a/packages/llm-llamacpp/addon/src/utils/ChatTemplateUtils.cpp b/packages/llm-llamacpp/addon/src/utils/ChatTemplateUtils.cpp index 9bc66778b5..f1cac8c15e 100644 --- a/packages/llm-llamacpp/addon/src/utils/ChatTemplateUtils.cpp +++ b/packages/llm-llamacpp/addon/src/utils/ChatTemplateUtils.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include @@ -174,11 +175,22 @@ std::string getChatTemplate( std::string getPrompt( const struct common_chat_templates* tmpls, - struct common_chat_templates_inputs& inputs, bool* outThinkingForcedOpen) { + struct common_chat_templates_inputs& inputs, bool* outThinkingForcedOpen, + std::string* outThinkingStartTag, std::string* outThinkingEndTag, + std::string* outGenerationPrompt) { auto exportParams = [&](const common_chat_params& params) { if (outThinkingForcedOpen) { *outThinkingForcedOpen = params.thinking_forced_open; } + if (outThinkingStartTag) { + *outThinkingStartTag = params.thinking_start_tag; + } + if (outThinkingEndTag) { + *outThinkingEndTag = params.thinking_end_tag; + } + if (outGenerationPrompt) { + *outGenerationPrompt = params.generation_prompt; + } }; try { auto params = common_chat_templates_apply(tmpls, inputs); @@ -211,5 +223,61 @@ std::string getPrompt( } } +bool configureReasoningBudgetSampling( + common_params& params, + ::llama_context* lctx, + const std::string& thinkingStartTag, + const std::string& thinkingEndTag, + const std::string& generationPrompt) { + common_params_sampling next = params.sampling; + next.reasoning_budget_tokens = + params.reasoning_budget > 0 ? params.reasoning_budget : -1; + next.reasoning_budget_start.clear(); + next.reasoning_budget_end.clear(); + next.reasoning_budget_forced.clear(); + next.generation_prompt.clear(); + + if (params.reasoning_budget > 0 && lctx != nullptr && + !thinkingEndTag.empty()) { + next.generation_prompt = generationPrompt; + if (!thinkingStartTag.empty()) { + next.reasoning_budget_start = + common_tokenize(lctx, thinkingStartTag, false, true); + } + next.reasoning_budget_end = + common_tokenize(lctx, thinkingEndTag, false, true); + next.reasoning_budget_forced = common_tokenize( + lctx, + params.sampling.reasoning_budget_message + thinkingEndTag, + false, + true); + } + + const bool changed = + params.sampling.reasoning_budget_tokens != + next.reasoning_budget_tokens || + params.sampling.reasoning_budget_start != next.reasoning_budget_start || + params.sampling.reasoning_budget_end != next.reasoning_budget_end || + params.sampling.reasoning_budget_forced != next.reasoning_budget_forced || + params.sampling.generation_prompt != next.generation_prompt; + if (changed) { + params.sampling = std::move(next); + } + return changed; +} + +std::string getThinkingForcedOpenText( + const std::string& generationPrompt, + const std::string& thinkingStartTag) { + if (thinkingStartTag.empty()) { + return {}; + } + const auto start = generationPrompt.rfind(thinkingStartTag); + if (start == std::string::npos) { + return thinkingStartTag; + } + return generationPrompt.substr(start); +} + } // namespace utils } // namespace qvac_lib_inference_addon_llama diff --git a/packages/llm-llamacpp/addon/src/utils/ChatTemplateUtils.hpp b/packages/llm-llamacpp/addon/src/utils/ChatTemplateUtils.hpp index b523e51b98..6d6586017b 100644 --- a/packages/llm-llamacpp/addon/src/utils/ChatTemplateUtils.hpp +++ b/packages/llm-llamacpp/addon/src/utils/ChatTemplateUtils.hpp @@ -71,11 +71,38 @@ std::string getChatTemplate( * * @p outThinkingForcedOpen (optional) receives the flag indicating that the * template force-opened the reasoning channel in the prompt suffix. + * @p outThinkingStartTag (optional) receives the template-specific reasoning + * start tag, when the template exposes one. + * @p outThinkingEndTag (optional) receives the template-specific reasoning + * end tag, when the template exposes one. + * @p outGenerationPrompt (optional) receives the assistant generation prompt + * already appended to the formatted prompt. */ std::string getPrompt( const struct common_chat_templates* tmpls, struct common_chat_templates_inputs& inputs, - bool* outThinkingForcedOpen = nullptr); + bool* outThinkingForcedOpen = nullptr, + std::string* outThinkingStartTag = nullptr, + std::string* outThinkingEndTag = nullptr, + std::string* outGenerationPrompt = nullptr); + +/** + * @brief Configures the common-sampling reasoning-budget fields from + * template-derived thinking tags. + * + * Returns true when the sampling block changed and the caller should recreate + * the common_sampler. + */ +bool configureReasoningBudgetSampling( + common_params& params, + ::llama_context* lctx, + const std::string& thinkingStartTag, + const std::string& thinkingEndTag, + const std::string& generationPrompt); + +std::string getThinkingForcedOpenText( + const std::string& generationPrompt, + const std::string& thinkingStartTag); } // namespace utils } // namespace qvac_lib_inference_addon_llama diff --git a/packages/llm-llamacpp/test/integration/reasoning.test.js b/packages/llm-llamacpp/test/integration/reasoning.test.js index 929ba5f7d6..f04fa3882b 100644 --- a/packages/llm-llamacpp/test/integration/reasoning.test.js +++ b/packages/llm-llamacpp/test/integration/reasoning.test.js @@ -59,8 +59,8 @@ async function setupReasoningModel (t, toolsEnabled) { } // Shared helper: Run a completion and collect response -async function runCompletion (inference, messages) { - const result = await inference.run(messages) +async function runCompletion (inference, messages, runOptions) { + const result = await inference.run(messages, runOptions) let response = '' await result .onUpdate(token => { @@ -192,7 +192,7 @@ safeTest('Qwen3 reasoning-budget=0 disables thinking', { verbosity: '0' } - async function runOnce (extra) { + async function runOnce (extra, runOptions) { const inference = new LlmLlamacpp({ files: { model: [modelPath] }, config: { ...baseConfig, ...extra }, @@ -204,7 +204,7 @@ safeTest('Qwen3 reasoning-budget=0 disables thinking', { { role: 'system', content: 'You are a helpful assistant.' }, { role: 'user', content: 'What is the capital of France? Answer in one word.' } ] - return await runCompletion(inference, messages) + return await runCompletion(inference, messages, runOptions) } finally { await inference.unload().catch(() => {}) } @@ -242,7 +242,7 @@ safeTest('Qwen3 reasoning-budget=0 disables thinking', { }) // reasoning-budget = N caps the channel at N tokens via the budget -// sampler — is force-emitted once N reasoning tokens have streamed. +// sampler - is force-emitted once N reasoning tokens have streamed. safeTest('Qwen3 reasoning-budget=N caps reasoning channel below baseline', { skip: isDarwinX64 || isWindowsX64, timeout: 600_000 @@ -289,22 +289,36 @@ safeTest('Qwen3 reasoning-budget=N caps reasoning channel below baseline', { const BUDGET = 16 const capped = await runOnce({ 'reasoning-budget': String(BUDGET) }) const cappedThink = sliceReasoning(capped) + const perRequestCapped = await runOnce({}, { + generationParams: { reasoning_budget: BUDGET } + }) + const perRequestCappedThink = sliceReasoning(perRequestCapped) t.comment(`baseline reasoning (${baselineThink ? baselineThink.length : 0} chars): ${(baselineThink || '').slice(0, 200)}`) t.comment(`capped reasoning (${cappedThink ? cappedThink.length : 0} chars): ${cappedThink || ''}`) + t.comment(`request reasoning (${perRequestCappedThink ? perRequestCappedThink.length : 0} chars): ${perRequestCappedThink || ''}`) t.ok(/paris/i.test(capped), 'capped output still mentions Paris (correctness preserved past forced close)') + t.ok(/paris/i.test(perRequestCapped), 'per-request capped output still mentions Paris') t.ok(capped.includes('') && capped.includes(''), `capped output keeps balanced reasoning tags: "${capped.slice(0, 200)}"`) + t.ok(perRequestCapped.includes('') && perRequestCapped.includes(''), + `per-request capped output keeps balanced reasoning tags: "${perRequestCapped.slice(0, 200)}"`) t.ok(cappedThink !== null, 'capped output has an extractable reasoning channel') - // The budget sampler is char-agnostic — it counts tokens, not characters — + t.ok(perRequestCappedThink !== null, + 'per-request capped output has an extractable reasoning channel') + // The budget sampler is char-agnostic - it counts tokens, not characters - // so we assert the reasoning channel is at least *substantially* shorter // than the unrestricted baseline rather than asserting an exact char count. t.ok( cappedThink.length < Math.max(1, baselineThink.length * 0.6), `capped reasoning (${cappedThink.length}) is substantially shorter than baseline (${baselineThink.length})` ) + t.ok( + perRequestCappedThink.length < Math.max(1, baselineThink.length * 0.6), + `per-request capped reasoning (${perRequestCappedThink.length}) is substantially shorter than baseline (${baselineThink.length})` + ) }) function sliceReasoning (text) { From 78a28f59661c5fc3ec04afd7f4a1485daace2407 Mon Sep 17 00:00:00 2001 From: gianni-cor Date: Fri, 19 Jun 2026 00:09:56 +0200 Subject: [PATCH 27/58] fix[notask]: restore existing comment punctuation Assisted-by: GPT-5.5 Co-authored-by: Cursor --- .../addon/src/model-interface/GenerationParamsApply.cpp | 6 +++--- .../llm-llamacpp/addon/src/model-interface/LlmContext.hpp | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/llm-llamacpp/addon/src/model-interface/GenerationParamsApply.cpp b/packages/llm-llamacpp/addon/src/model-interface/GenerationParamsApply.cpp index 0a509a7969..fd8271e7b7 100644 --- a/packages/llm-llamacpp/addon/src/model-interface/GenerationParamsApply.cpp +++ b/packages/llm-llamacpp/addon/src/model-interface/GenerationParamsApply.cpp @@ -54,7 +54,7 @@ void applyGenerationOverridesToSampling( LOG_WRN( "%s: both generationParams.grammar and generationParams.json_schema " "were provided; ignoring `grammar` and applying `json_schema` " - "(the JS and AddonJs paths reject this combination - this branch " + "(the JS and AddonJs paths reject this combination — this branch " "exists only for direct C++ callers).\n", __func__); } @@ -86,7 +86,7 @@ std::function applyGenerationParamsToContext( // Apply overrides to *local copies* first. Only commit them onto the // live `params` and `smpl` after both the json_schema parse/convert and - // `common_sampler_init` have succeeded - otherwise a partially applied + // `common_sampler_init` have succeeded — otherwise a partially applied // override (e.g. temp/seed already written, then json_schema throws) // would leak into subsequent requests because no restore lambda gets // returned to the caller's `ScopeGuard`. @@ -98,7 +98,7 @@ std::function applyGenerationParamsToContext( applyGenerationOverridesToSampling(nextSampling, nextPredict, overrides); // `common_sampler_init` returns nullptr on bad inputs (most commonly an - // invalid GBNF grammar - `json_schema` is converted to GBNF above and + // invalid GBNF grammar — `json_schema` is converted to GBNF above and // can in principle produce a grammar that the sampler rejects). Build // the new sampler before touching live state so a failure here also // leaves `params` / `smpl` intact. diff --git a/packages/llm-llamacpp/addon/src/model-interface/LlmContext.hpp b/packages/llm-llamacpp/addon/src/model-interface/LlmContext.hpp index cd74778cd3..474c24b9b7 100644 --- a/packages/llm-llamacpp/addon/src/model-interface/LlmContext.hpp +++ b/packages/llm-llamacpp/addon/src/model-interface/LlmContext.hpp @@ -31,7 +31,7 @@ struct GenerationParams { std::optional grammar; // JSON-Schema applied per request. Converted to GBNF via llama.cpp's // `json_schema_to_grammar()` and applied identically to `grammar`. - // Mutually exclusive with `grammar` - the JS wrapper rejects requests + // Mutually exclusive with `grammar` — the JS wrapper rejects requests // that set both. Mirrors the load-time `--json-schema` flag. std::optional json_schema; // Reasoning channel budget override. `-1` keeps reasoning unrestricted, `0` From 890d92dc9e6489c533eb056177cfcc02ef4fc6dc Mon Sep 17 00:00:00 2001 From: gianni-cor Date: Fri, 19 Jun 2026 07:55:20 +0200 Subject: [PATCH 28/58] test[notask]: exercise per-request reasoning budget Assisted-by: GPT-5.5 Co-authored-by: Cursor --- packages/llm-llamacpp/test/integration/reasoning.test.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/llm-llamacpp/test/integration/reasoning.test.js b/packages/llm-llamacpp/test/integration/reasoning.test.js index f04fa3882b..ed6bb1974f 100644 --- a/packages/llm-llamacpp/test/integration/reasoning.test.js +++ b/packages/llm-llamacpp/test/integration/reasoning.test.js @@ -264,7 +264,7 @@ safeTest('Qwen3 reasoning-budget=N caps reasoning channel below baseline', { verbosity: '0' } - async function runOnce (extra) { + async function runOnce (extra, runOptions) { const inference = new LlmLlamacpp({ files: { model: [modelPath] }, config: { ...baseConfig, ...extra }, @@ -276,7 +276,7 @@ safeTest('Qwen3 reasoning-budget=N caps reasoning channel below baseline', { { role: 'system', content: 'You are a helpful assistant.' }, { role: 'user', content: 'What is the capital of France? Answer in one word.' } ] - return await runCompletion(inference, messages) + return await runCompletion(inference, messages, runOptions) } finally { await inference.unload().catch(() => {}) } From ee8f4fd9efaebd079843c107e18e920deb87247c Mon Sep 17 00:00:00 2001 From: gianni-cor Date: Fri, 19 Jun 2026 08:40:42 +0200 Subject: [PATCH 29/58] test[notask]: cover reasoning budget helpers Assisted-by: GPT-5.5 Co-authored-by: Cursor --- .../llm-llamacpp/test/unit/CMakeLists.txt | 1 + .../test/unit/test_chat_template_utils.cpp | 49 +++++++ .../unit/test_generation_params_apply.cpp | 138 ++++++++++++++++++ 3 files changed, 188 insertions(+) create mode 100644 packages/llm-llamacpp/test/unit/test_generation_params_apply.cpp diff --git a/packages/llm-llamacpp/test/unit/CMakeLists.txt b/packages/llm-llamacpp/test/unit/CMakeLists.txt index 15eafbdcf9..b87728bcb2 100644 --- a/packages/llm-llamacpp/test/unit/CMakeLists.txt +++ b/packages/llm-llamacpp/test/unit/CMakeLists.txt @@ -20,6 +20,7 @@ add_executable( test_reload_cancel_state.cpp test_backend_selection.cpp test_tune_config_map.cpp + test_generation_params_apply.cpp # Placeholder tests (to be implemented) test_mtmd_llm_context.cpp test_llm_context_base.cpp diff --git a/packages/llm-llamacpp/test/unit/test_chat_template_utils.cpp b/packages/llm-llamacpp/test/unit/test_chat_template_utils.cpp index 52a4ebf84f..b7ebce7ac5 100644 --- a/packages/llm-llamacpp/test/unit/test_chat_template_utils.cpp +++ b/packages/llm-llamacpp/test/unit/test_chat_template_utils.cpp @@ -220,3 +220,52 @@ TEST_F( std::string result = getChatTemplate(nullptr, params, false); EXPECT_EQ(result, "my_custom_template"); } + +TEST_F(ChatTemplateUtilsTest, GetPromptExportsQwenThinkingMetadata) { + common_chat_templates_ptr tmpls = + common_chat_templates_init(nullptr, getFixedQwen3Template()); + ASSERT_NE(tmpls, nullptr); + + common_chat_templates_inputs inputs; + inputs.use_jinja = true; + inputs.enable_thinking = true; + inputs.add_generation_prompt = true; + inputs.messages = {common_chat_msg{ + /* role = */ "user", + /* content = */ "What is the capital of France?", + }}; + + bool thinkingForcedOpen = true; + std::string thinkingStartTag; + std::string thinkingEndTag; + std::string generationPrompt; + const std::string prompt = getPrompt( + tmpls.get(), + inputs, + &thinkingForcedOpen, + &thinkingStartTag, + &thinkingEndTag, + &generationPrompt); + + EXPECT_NE(prompt.find("<|im_start|>assistant"), std::string::npos); + EXPECT_EQ(thinkingStartTag, ""); + EXPECT_EQ(thinkingEndTag, ""); + EXPECT_NE(generationPrompt.find("<|im_start|>assistant"), std::string::npos); + EXPECT_FALSE(thinkingForcedOpen); +} + +TEST_F(ChatTemplateUtilsTest, ThinkingForcedOpenTextUsesTemplateSuffix) { + EXPECT_EQ( + getThinkingForcedOpenText( + "<|assistant|>\n\n", ""), + "\n"); +} + +TEST_F(ChatTemplateUtilsTest, ThinkingForcedOpenTextFallsBackToStartTag) { + EXPECT_EQ( + getThinkingForcedOpenText("<|assistant|>\n", ""), ""); +} + +TEST_F(ChatTemplateUtilsTest, ThinkingForcedOpenTextEmptyWithoutStartTag) { + EXPECT_EQ(getThinkingForcedOpenText("<|assistant|>\n", ""), ""); +} diff --git a/packages/llm-llamacpp/test/unit/test_generation_params_apply.cpp b/packages/llm-llamacpp/test/unit/test_generation_params_apply.cpp new file mode 100644 index 0000000000..84142fb164 --- /dev/null +++ b/packages/llm-llamacpp/test/unit/test_generation_params_apply.cpp @@ -0,0 +1,138 @@ +#include + +#include + +#include "model-interface/GenerationParamsApply.hpp" +#include "utils/ChatTemplateUtils.hpp" + +using qvac_lib_inference_addon_llama::utils::configureReasoningBudgetSampling; + +namespace { + +std::vector tokens(std::initializer_list values) { + return std::vector(values); +} + +} // namespace + +TEST(GenerationParamsApplyTest, NoReasoningBudgetOverrideLeavesSamplingState) { + common_params_sampling sampling; + sampling.reasoning_budget_tokens = 12; + sampling.reasoning_budget_start = tokens({1, 2}); + sampling.reasoning_budget_end = tokens({3}); + sampling.reasoning_budget_forced = tokens({4, 5}); + int nPredict = 32; + + GenerationParams overrides; + overrides.temp = 0.25f; + applyGenerationOverridesToSampling(sampling, nPredict, overrides); + + EXPECT_EQ(sampling.reasoning_budget_tokens, 12); + EXPECT_EQ(sampling.reasoning_budget_start, tokens({1, 2})); + EXPECT_EQ(sampling.reasoning_budget_end, tokens({3})); + EXPECT_EQ(sampling.reasoning_budget_forced, tokens({4, 5})); +} + +TEST(GenerationParamsApplyTest, PositiveReasoningBudgetUpdatesOnlyTokenCap) { + common_params_sampling sampling; + sampling.reasoning_budget_tokens = -1; + sampling.reasoning_budget_start = tokens({10}); + sampling.reasoning_budget_end = tokens({11}); + sampling.reasoning_budget_forced = tokens({12}); + int nPredict = 32; + + GenerationParams overrides; + overrides.reasoning_budget = 16; + applyGenerationOverridesToSampling(sampling, nPredict, overrides); + + EXPECT_EQ(sampling.reasoning_budget_tokens, 16); + EXPECT_EQ(sampling.reasoning_budget_start, tokens({10})); + EXPECT_EQ(sampling.reasoning_budget_end, tokens({11})); + EXPECT_EQ(sampling.reasoning_budget_forced, tokens({12})); +} + +TEST(GenerationParamsApplyTest, ZeroReasoningBudgetClearsBudgetSamplerState) { + common_params_sampling sampling; + sampling.reasoning_budget_tokens = 16; + sampling.reasoning_budget_start = tokens({10}); + sampling.reasoning_budget_end = tokens({11}); + sampling.reasoning_budget_forced = tokens({12}); + int nPredict = 32; + + GenerationParams overrides; + overrides.reasoning_budget = 0; + applyGenerationOverridesToSampling(sampling, nPredict, overrides); + + EXPECT_EQ(sampling.reasoning_budget_tokens, -1); + EXPECT_TRUE(sampling.reasoning_budget_start.empty()); + EXPECT_TRUE(sampling.reasoning_budget_end.empty()); + EXPECT_TRUE(sampling.reasoning_budget_forced.empty()); +} + +TEST( + GenerationParamsApplyTest, + UnrestrictedReasoningBudgetClearsBudgetSamplerState) { + common_params_sampling sampling; + sampling.reasoning_budget_tokens = 16; + sampling.reasoning_budget_start = tokens({10}); + sampling.reasoning_budget_end = tokens({11}); + sampling.reasoning_budget_forced = tokens({12}); + int nPredict = 32; + + GenerationParams overrides; + overrides.reasoning_budget = -1; + applyGenerationOverridesToSampling(sampling, nPredict, overrides); + + EXPECT_EQ(sampling.reasoning_budget_tokens, -1); + EXPECT_TRUE(sampling.reasoning_budget_start.empty()); + EXPECT_TRUE(sampling.reasoning_budget_end.empty()); + EXPECT_TRUE(sampling.reasoning_budget_forced.empty()); +} + +TEST( + GenerationParamsApplyTest, + ConfigureReasoningBudgetSamplingClearsStaleStateWhenDisabled) { + common_params params; + params.reasoning_budget = 0; + params.sampling.reasoning_budget_tokens = 16; + params.sampling.reasoning_budget_start = tokens({10}); + params.sampling.reasoning_budget_end = tokens({11}); + params.sampling.reasoning_budget_forced = tokens({12}); + params.sampling.generation_prompt = ""; + + EXPECT_TRUE(configureReasoningBudgetSampling( + params, + nullptr, + "", + "", + "")); + EXPECT_EQ(params.sampling.reasoning_budget_tokens, -1); + EXPECT_TRUE(params.sampling.reasoning_budget_start.empty()); + EXPECT_TRUE(params.sampling.reasoning_budget_end.empty()); + EXPECT_TRUE(params.sampling.reasoning_budget_forced.empty()); + EXPECT_TRUE(params.sampling.generation_prompt.empty()); +} + +TEST( + GenerationParamsApplyTest, + ConfigureReasoningBudgetSamplingKeepsPositiveCapWithoutContext) { + common_params params; + params.reasoning_budget = 8; + params.sampling.reasoning_budget_tokens = -1; + params.sampling.reasoning_budget_start = tokens({10}); + params.sampling.reasoning_budget_end = tokens({11}); + params.sampling.reasoning_budget_forced = tokens({12}); + params.sampling.generation_prompt = ""; + + EXPECT_TRUE(configureReasoningBudgetSampling( + params, + nullptr, + "", + "", + "")); + EXPECT_EQ(params.sampling.reasoning_budget_tokens, 8); + EXPECT_TRUE(params.sampling.reasoning_budget_start.empty()); + EXPECT_TRUE(params.sampling.reasoning_budget_end.empty()); + EXPECT_TRUE(params.sampling.reasoning_budget_forced.empty()); + EXPECT_TRUE(params.sampling.generation_prompt.empty()); +} From 3efda22dbdb9961314dd37e928827f8d08e3a91f Mon Sep 17 00:00:00 2001 From: gianni-cor Date: Fri, 19 Jun 2026 08:45:03 +0200 Subject: [PATCH 30/58] test[notask]: use safeTest for continuous batching Assisted-by: GPT-5.5 Co-authored-by: Cursor --- .../llm-llamacpp/test/integration/continuous-batching.test.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/llm-llamacpp/test/integration/continuous-batching.test.js b/packages/llm-llamacpp/test/integration/continuous-batching.test.js index 2612c3b333..15a03980da 100644 --- a/packages/llm-llamacpp/test/integration/continuous-batching.test.js +++ b/packages/llm-llamacpp/test/integration/continuous-batching.test.js @@ -5,7 +5,7 @@ const path = require('bare-path') const os = require('bare-os') const process = require('bare-process') const LlmLlamacpp = require('../../index.js') -const { ensureModel } = require('./utils') +const { ensureModel, safeTest } = require('./utils') const { attachSpecLogger } = require('./spec-logger') const platform = os.platform() @@ -152,7 +152,7 @@ function logStreamingProgress (response) { // 1B throughput/correctness run is too slow or complicated for mobile and legacy macOS x64. const skipHeavyPlatform = isMobile || isDarwin -test('continuous batching answers 16 prompts correctly and improves Linux GPU TPS', { timeout: 900_000, skip: skipHeavyPlatform }, async t => { +safeTest('continuous batching answers 16 prompts correctly and improves Linux GPU TPS', { timeout: 900_000, skip: skipHeavyPlatform }, async t => { const singleModel = await setupModel(t) const singleNativeTpsValues = [] const singleWallTpsValues = [] From 3287cd8b1e51009c895c5de752bf9f25f625f5ce Mon Sep 17 00:00:00 2001 From: gianni-cor Date: Fri, 19 Jun 2026 09:11:21 +0200 Subject: [PATCH 31/58] fix: satisfy llm CI lint checks Co-authored-by: Cursor --- .../addon/src/model-interface/LlamaModel.cpp | 5 ++- .../src/model-interface/MtmdLlmContext.cpp | 19 ++++------- .../src/model-interface/TextLlmContext.cpp | 18 +++------- .../addon/src/utils/ChatTemplateUtils.cpp | 34 ++++++++----------- .../addon/src/utils/ChatTemplateUtils.hpp | 29 +++++++--------- .../integration/continuous-batching.test.js | 1 - .../test/unit/test_chat_template_utils.cpp | 20 ++++------- .../unit/test_generation_params_apply.cpp | 27 +++++---------- 8 files changed, 56 insertions(+), 97 deletions(-) diff --git a/packages/llm-llamacpp/addon/src/model-interface/LlamaModel.cpp b/packages/llm-llamacpp/addon/src/model-interface/LlamaModel.cpp index c2e32b39e2..103d8e5261 100644 --- a/packages/llm-llamacpp/addon/src/model-interface/LlamaModel.cpp +++ b/packages/llm-llamacpp/addon/src/model-interface/LlamaModel.cpp @@ -146,9 +146,8 @@ void LlamaModel::tuneConfigMap( } else if (notUserSet("flash-attn", "flash_attn")) { configFilemap.erase("flash_attn"); configFilemap["flash-attn"] = "on"; - QLOG_IF( - Priority::INFO, - "[LlamaModel] Enabling flash attention by default\n"); + QLOG_IF(Priority::INFO, + "[LlamaModel] Enabling flash attention by default\n"); } constexpr int kAdrenoUbatchThreshold = 800; diff --git a/packages/llm-llamacpp/addon/src/model-interface/MtmdLlmContext.cpp b/packages/llm-llamacpp/addon/src/model-interface/MtmdLlmContext.cpp index 9e59e5cdc6..f8d405c10d 100644 --- a/packages/llm-llamacpp/addon/src/model-interface/MtmdLlmContext.cpp +++ b/packages/llm-llamacpp/addon/src/model-interface/MtmdLlmContext.cpp @@ -219,13 +219,9 @@ void MtmdLlmContext::tokenizeChat( std::string thinkingStartTag; std::string thinkingEndTag; std::string generationPrompt; - formattedChat = getPrompt( - tmpls_.get(), - inputs, - &thinkingForcedOpen_, - &thinkingStartTag, - &thinkingEndTag, - &generationPrompt); + formattedChat = + getPrompt(tmpls_.get(), inputs, &thinkingForcedOpen_, &thinkingStartTag, + &thinkingEndTag, &generationPrompt); thinkingForcedOpenText_ = thinkingForcedOpen_ ? getThinkingForcedOpenText(generationPrompt, thinkingStartTag) @@ -237,12 +233,9 @@ void MtmdLlmContext::tokenizeChat( throw qvac_errors::StatusError(ADDON_ID, toString(EmptyPrompt), errorMsg); } - if (configureReasoningBudgetSampling( - params_, - modelCtx_.lctx, - thinkingStartTag, - thinkingEndTag, - generationPrompt)) { + if (configureReasoningBudgetSampling(params_, modelCtx_.lctx, + thinkingStartTag, thinkingEndTag, + generationPrompt)) { smpl_.reset(common_sampler_init(modelCtx_.model, params_.sampling)); if (!smpl_) { std::string errorMsg = string_format( diff --git a/packages/llm-llamacpp/addon/src/model-interface/TextLlmContext.cpp b/packages/llm-llamacpp/addon/src/model-interface/TextLlmContext.cpp index 7c9a60e230..51fcb8fd4e 100644 --- a/packages/llm-llamacpp/addon/src/model-interface/TextLlmContext.cpp +++ b/packages/llm-llamacpp/addon/src/model-interface/TextLlmContext.cpp @@ -287,23 +287,15 @@ void TextLlmContext::tokenizeChat( std::string thinkingStartTag; std::string thinkingEndTag; std::string generationPrompt; - prompt = getPrompt( - tmpls_.get(), - inputs, - &thinkingForcedOpen_, - &thinkingStartTag, - &thinkingEndTag, - &generationPrompt); + prompt = getPrompt(tmpls_.get(), inputs, &thinkingForcedOpen_, + &thinkingStartTag, &thinkingEndTag, &generationPrompt); thinkingForcedOpenText_ = thinkingForcedOpen_ ? getThinkingForcedOpenText(generationPrompt, thinkingStartTag) : std::string{}; - if (configureReasoningBudgetSampling( - params_, - modelCtx_.lctx, - thinkingStartTag, - thinkingEndTag, - generationPrompt)) { + if (configureReasoningBudgetSampling(params_, modelCtx_.lctx, + thinkingStartTag, thinkingEndTag, + generationPrompt)) { smpl_.reset(common_sampler_init(modelCtx_.model, params_.sampling)); if (!smpl_) { std::string errorMsg = string_format( diff --git a/packages/llm-llamacpp/addon/src/utils/ChatTemplateUtils.cpp b/packages/llm-llamacpp/addon/src/utils/ChatTemplateUtils.cpp index f1cac8c15e..b2606c3bf9 100644 --- a/packages/llm-llamacpp/addon/src/utils/ChatTemplateUtils.cpp +++ b/packages/llm-llamacpp/addon/src/utils/ChatTemplateUtils.cpp @@ -173,11 +173,12 @@ std::string getChatTemplate( return chatTemplate; } -std::string getPrompt( - const struct common_chat_templates* tmpls, - struct common_chat_templates_inputs& inputs, bool* outThinkingForcedOpen, - std::string* outThinkingStartTag, std::string* outThinkingEndTag, - std::string* outGenerationPrompt) { +std::string getPrompt(const struct common_chat_templates *tmpls, + struct common_chat_templates_inputs &inputs, + bool *outThinkingForcedOpen, + std::string *outThinkingStartTag, + std::string *outThinkingEndTag, + std::string *outGenerationPrompt) { auto exportParams = [&](const common_chat_params& params) { if (outThinkingForcedOpen) { *outThinkingForcedOpen = params.thinking_forced_open; @@ -223,12 +224,11 @@ std::string getPrompt( } } -bool configureReasoningBudgetSampling( - common_params& params, - ::llama_context* lctx, - const std::string& thinkingStartTag, - const std::string& thinkingEndTag, - const std::string& generationPrompt) { +bool configureReasoningBudgetSampling(common_params ¶ms, + ::llama_context *lctx, + const std::string &thinkingStartTag, + const std::string &thinkingEndTag, + const std::string &generationPrompt) { common_params_sampling next = params.sampling; next.reasoning_budget_tokens = params.reasoning_budget > 0 ? params.reasoning_budget : -1; @@ -247,15 +247,12 @@ bool configureReasoningBudgetSampling( next.reasoning_budget_end = common_tokenize(lctx, thinkingEndTag, false, true); next.reasoning_budget_forced = common_tokenize( - lctx, - params.sampling.reasoning_budget_message + thinkingEndTag, - false, + lctx, params.sampling.reasoning_budget_message + thinkingEndTag, false, true); } const bool changed = - params.sampling.reasoning_budget_tokens != - next.reasoning_budget_tokens || + params.sampling.reasoning_budget_tokens != next.reasoning_budget_tokens || params.sampling.reasoning_budget_start != next.reasoning_budget_start || params.sampling.reasoning_budget_end != next.reasoning_budget_end || params.sampling.reasoning_budget_forced != next.reasoning_budget_forced || @@ -266,9 +263,8 @@ bool configureReasoningBudgetSampling( return changed; } -std::string getThinkingForcedOpenText( - const std::string& generationPrompt, - const std::string& thinkingStartTag) { +std::string getThinkingForcedOpenText(const std::string &generationPrompt, + const std::string &thinkingStartTag) { if (thinkingStartTag.empty()) { return {}; } diff --git a/packages/llm-llamacpp/addon/src/utils/ChatTemplateUtils.hpp b/packages/llm-llamacpp/addon/src/utils/ChatTemplateUtils.hpp index 6d6586017b..8f11fa0815 100644 --- a/packages/llm-llamacpp/addon/src/utils/ChatTemplateUtils.hpp +++ b/packages/llm-llamacpp/addon/src/utils/ChatTemplateUtils.hpp @@ -78,13 +78,12 @@ std::string getChatTemplate( * @p outGenerationPrompt (optional) receives the assistant generation prompt * already appended to the formatted prompt. */ -std::string getPrompt( - const struct common_chat_templates* tmpls, - struct common_chat_templates_inputs& inputs, - bool* outThinkingForcedOpen = nullptr, - std::string* outThinkingStartTag = nullptr, - std::string* outThinkingEndTag = nullptr, - std::string* outGenerationPrompt = nullptr); +std::string getPrompt(const struct common_chat_templates *tmpls, + struct common_chat_templates_inputs &inputs, + bool *outThinkingForcedOpen = nullptr, + std::string *outThinkingStartTag = nullptr, + std::string *outThinkingEndTag = nullptr, + std::string *outGenerationPrompt = nullptr); /** * @brief Configures the common-sampling reasoning-budget fields from @@ -93,16 +92,14 @@ std::string getPrompt( * Returns true when the sampling block changed and the caller should recreate * the common_sampler. */ -bool configureReasoningBudgetSampling( - common_params& params, - ::llama_context* lctx, - const std::string& thinkingStartTag, - const std::string& thinkingEndTag, - const std::string& generationPrompt); +bool configureReasoningBudgetSampling(common_params ¶ms, + ::llama_context *lctx, + const std::string &thinkingStartTag, + const std::string &thinkingEndTag, + const std::string &generationPrompt); -std::string getThinkingForcedOpenText( - const std::string& generationPrompt, - const std::string& thinkingStartTag); +std::string getThinkingForcedOpenText(const std::string &generationPrompt, + const std::string &thinkingStartTag); } // namespace utils } // namespace qvac_lib_inference_addon_llama diff --git a/packages/llm-llamacpp/test/integration/continuous-batching.test.js b/packages/llm-llamacpp/test/integration/continuous-batching.test.js index 15a03980da..9928e7081e 100644 --- a/packages/llm-llamacpp/test/integration/continuous-batching.test.js +++ b/packages/llm-llamacpp/test/integration/continuous-batching.test.js @@ -1,6 +1,5 @@ 'use strict' -const test = require('brittle') const path = require('bare-path') const os = require('bare-os') const process = require('bare-process') diff --git a/packages/llm-llamacpp/test/unit/test_chat_template_utils.cpp b/packages/llm-llamacpp/test/unit/test_chat_template_utils.cpp index b7ebce7ac5..b258136492 100644 --- a/packages/llm-llamacpp/test/unit/test_chat_template_utils.cpp +++ b/packages/llm-llamacpp/test/unit/test_chat_template_utils.cpp @@ -239,13 +239,9 @@ TEST_F(ChatTemplateUtilsTest, GetPromptExportsQwenThinkingMetadata) { std::string thinkingStartTag; std::string thinkingEndTag; std::string generationPrompt; - const std::string prompt = getPrompt( - tmpls.get(), - inputs, - &thinkingForcedOpen, - &thinkingStartTag, - &thinkingEndTag, - &generationPrompt); + const std::string prompt = + getPrompt(tmpls.get(), inputs, &thinkingForcedOpen, &thinkingStartTag, + &thinkingEndTag, &generationPrompt); EXPECT_NE(prompt.find("<|im_start|>assistant"), std::string::npos); EXPECT_EQ(thinkingStartTag, ""); @@ -255,15 +251,13 @@ TEST_F(ChatTemplateUtilsTest, GetPromptExportsQwenThinkingMetadata) { } TEST_F(ChatTemplateUtilsTest, ThinkingForcedOpenTextUsesTemplateSuffix) { - EXPECT_EQ( - getThinkingForcedOpenText( - "<|assistant|>\n\n", ""), - "\n"); + EXPECT_EQ(getThinkingForcedOpenText("<|assistant|>\n\n", ""), + "\n"); } TEST_F(ChatTemplateUtilsTest, ThinkingForcedOpenTextFallsBackToStartTag) { - EXPECT_EQ( - getThinkingForcedOpenText("<|assistant|>\n", ""), ""); + EXPECT_EQ(getThinkingForcedOpenText("<|assistant|>\n", ""), + ""); } TEST_F(ChatTemplateUtilsTest, ThinkingForcedOpenTextEmptyWithoutStartTag) { diff --git a/packages/llm-llamacpp/test/unit/test_generation_params_apply.cpp b/packages/llm-llamacpp/test/unit/test_generation_params_apply.cpp index 84142fb164..090a9c065d 100644 --- a/packages/llm-llamacpp/test/unit/test_generation_params_apply.cpp +++ b/packages/llm-llamacpp/test/unit/test_generation_params_apply.cpp @@ -69,9 +69,8 @@ TEST(GenerationParamsApplyTest, ZeroReasoningBudgetClearsBudgetSamplerState) { EXPECT_TRUE(sampling.reasoning_budget_forced.empty()); } -TEST( - GenerationParamsApplyTest, - UnrestrictedReasoningBudgetClearsBudgetSamplerState) { +TEST(GenerationParamsApplyTest, + UnrestrictedReasoningBudgetClearsBudgetSamplerState) { common_params_sampling sampling; sampling.reasoning_budget_tokens = 16; sampling.reasoning_budget_start = tokens({10}); @@ -89,9 +88,8 @@ TEST( EXPECT_TRUE(sampling.reasoning_budget_forced.empty()); } -TEST( - GenerationParamsApplyTest, - ConfigureReasoningBudgetSamplingClearsStaleStateWhenDisabled) { +TEST(GenerationParamsApplyTest, + ConfigureReasoningBudgetSamplingClearsStaleStateWhenDisabled) { common_params params; params.reasoning_budget = 0; params.sampling.reasoning_budget_tokens = 16; @@ -101,11 +99,7 @@ TEST( params.sampling.generation_prompt = ""; EXPECT_TRUE(configureReasoningBudgetSampling( - params, - nullptr, - "", - "", - "")); + params, nullptr, "", "", "")); EXPECT_EQ(params.sampling.reasoning_budget_tokens, -1); EXPECT_TRUE(params.sampling.reasoning_budget_start.empty()); EXPECT_TRUE(params.sampling.reasoning_budget_end.empty()); @@ -113,9 +107,8 @@ TEST( EXPECT_TRUE(params.sampling.generation_prompt.empty()); } -TEST( - GenerationParamsApplyTest, - ConfigureReasoningBudgetSamplingKeepsPositiveCapWithoutContext) { +TEST(GenerationParamsApplyTest, + ConfigureReasoningBudgetSamplingKeepsPositiveCapWithoutContext) { common_params params; params.reasoning_budget = 8; params.sampling.reasoning_budget_tokens = -1; @@ -125,11 +118,7 @@ TEST( params.sampling.generation_prompt = ""; EXPECT_TRUE(configureReasoningBudgetSampling( - params, - nullptr, - "", - "", - "")); + params, nullptr, "", "", "")); EXPECT_EQ(params.sampling.reasoning_budget_tokens, 8); EXPECT_TRUE(params.sampling.reasoning_budget_start.empty()); EXPECT_TRUE(params.sampling.reasoning_budget_end.empty()); From 2247d0c689d8fd1b8a9c82b1fc34ba02c86500c3 Mon Sep 17 00:00:00 2001 From: gianni-cor Date: Fri, 19 Jun 2026 09:17:48 +0200 Subject: [PATCH 32/58] fix: match CI C++ formatting Co-authored-by: Cursor --- .../addon/src/model-interface/LlamaModel.cpp | 4 +-- .../src/model-interface/MtmdLlmContext.cpp | 19 +++++++++---- .../src/model-interface/TextLlmContext.cpp | 18 ++++++++---- .../addon/src/utils/ChatTemplateUtils.cpp | 28 +++++++++---------- .../addon/src/utils/ChatTemplateUtils.hpp | 26 ++++++++--------- .../test/unit/test_chat_template_utils.cpp | 19 ++++++++----- .../unit/test_generation_params_apply.cpp | 15 ++++++---- 7 files changed, 76 insertions(+), 53 deletions(-) diff --git a/packages/llm-llamacpp/addon/src/model-interface/LlamaModel.cpp b/packages/llm-llamacpp/addon/src/model-interface/LlamaModel.cpp index 103d8e5261..c216923a15 100644 --- a/packages/llm-llamacpp/addon/src/model-interface/LlamaModel.cpp +++ b/packages/llm-llamacpp/addon/src/model-interface/LlamaModel.cpp @@ -146,8 +146,8 @@ void LlamaModel::tuneConfigMap( } else if (notUserSet("flash-attn", "flash_attn")) { configFilemap.erase("flash_attn"); configFilemap["flash-attn"] = "on"; - QLOG_IF(Priority::INFO, - "[LlamaModel] Enabling flash attention by default\n"); + QLOG_IF( + Priority::INFO, "[LlamaModel] Enabling flash attention by default\n"); } constexpr int kAdrenoUbatchThreshold = 800; diff --git a/packages/llm-llamacpp/addon/src/model-interface/MtmdLlmContext.cpp b/packages/llm-llamacpp/addon/src/model-interface/MtmdLlmContext.cpp index f8d405c10d..9e59e5cdc6 100644 --- a/packages/llm-llamacpp/addon/src/model-interface/MtmdLlmContext.cpp +++ b/packages/llm-llamacpp/addon/src/model-interface/MtmdLlmContext.cpp @@ -219,9 +219,13 @@ void MtmdLlmContext::tokenizeChat( std::string thinkingStartTag; std::string thinkingEndTag; std::string generationPrompt; - formattedChat = - getPrompt(tmpls_.get(), inputs, &thinkingForcedOpen_, &thinkingStartTag, - &thinkingEndTag, &generationPrompt); + formattedChat = getPrompt( + tmpls_.get(), + inputs, + &thinkingForcedOpen_, + &thinkingStartTag, + &thinkingEndTag, + &generationPrompt); thinkingForcedOpenText_ = thinkingForcedOpen_ ? getThinkingForcedOpenText(generationPrompt, thinkingStartTag) @@ -233,9 +237,12 @@ void MtmdLlmContext::tokenizeChat( throw qvac_errors::StatusError(ADDON_ID, toString(EmptyPrompt), errorMsg); } - if (configureReasoningBudgetSampling(params_, modelCtx_.lctx, - thinkingStartTag, thinkingEndTag, - generationPrompt)) { + if (configureReasoningBudgetSampling( + params_, + modelCtx_.lctx, + thinkingStartTag, + thinkingEndTag, + generationPrompt)) { smpl_.reset(common_sampler_init(modelCtx_.model, params_.sampling)); if (!smpl_) { std::string errorMsg = string_format( diff --git a/packages/llm-llamacpp/addon/src/model-interface/TextLlmContext.cpp b/packages/llm-llamacpp/addon/src/model-interface/TextLlmContext.cpp index 51fcb8fd4e..7c9a60e230 100644 --- a/packages/llm-llamacpp/addon/src/model-interface/TextLlmContext.cpp +++ b/packages/llm-llamacpp/addon/src/model-interface/TextLlmContext.cpp @@ -287,15 +287,23 @@ void TextLlmContext::tokenizeChat( std::string thinkingStartTag; std::string thinkingEndTag; std::string generationPrompt; - prompt = getPrompt(tmpls_.get(), inputs, &thinkingForcedOpen_, - &thinkingStartTag, &thinkingEndTag, &generationPrompt); + prompt = getPrompt( + tmpls_.get(), + inputs, + &thinkingForcedOpen_, + &thinkingStartTag, + &thinkingEndTag, + &generationPrompt); thinkingForcedOpenText_ = thinkingForcedOpen_ ? getThinkingForcedOpenText(generationPrompt, thinkingStartTag) : std::string{}; - if (configureReasoningBudgetSampling(params_, modelCtx_.lctx, - thinkingStartTag, thinkingEndTag, - generationPrompt)) { + if (configureReasoningBudgetSampling( + params_, + modelCtx_.lctx, + thinkingStartTag, + thinkingEndTag, + generationPrompt)) { smpl_.reset(common_sampler_init(modelCtx_.model, params_.sampling)); if (!smpl_) { std::string errorMsg = string_format( diff --git a/packages/llm-llamacpp/addon/src/utils/ChatTemplateUtils.cpp b/packages/llm-llamacpp/addon/src/utils/ChatTemplateUtils.cpp index b2606c3bf9..3e06c1c249 100644 --- a/packages/llm-llamacpp/addon/src/utils/ChatTemplateUtils.cpp +++ b/packages/llm-llamacpp/addon/src/utils/ChatTemplateUtils.cpp @@ -173,12 +173,11 @@ std::string getChatTemplate( return chatTemplate; } -std::string getPrompt(const struct common_chat_templates *tmpls, - struct common_chat_templates_inputs &inputs, - bool *outThinkingForcedOpen, - std::string *outThinkingStartTag, - std::string *outThinkingEndTag, - std::string *outGenerationPrompt) { +std::string getPrompt( + const struct common_chat_templates* tmpls, + struct common_chat_templates_inputs& inputs, bool* outThinkingForcedOpen, + std::string* outThinkingStartTag, std::string* outThinkingEndTag, + std::string* outGenerationPrompt) { auto exportParams = [&](const common_chat_params& params) { if (outThinkingForcedOpen) { *outThinkingForcedOpen = params.thinking_forced_open; @@ -224,11 +223,10 @@ std::string getPrompt(const struct common_chat_templates *tmpls, } } -bool configureReasoningBudgetSampling(common_params ¶ms, - ::llama_context *lctx, - const std::string &thinkingStartTag, - const std::string &thinkingEndTag, - const std::string &generationPrompt) { +bool configureReasoningBudgetSampling( + common_params& params, ::llama_context* lctx, + const std::string& thinkingStartTag, const std::string& thinkingEndTag, + const std::string& generationPrompt) { common_params_sampling next = params.sampling; next.reasoning_budget_tokens = params.reasoning_budget > 0 ? params.reasoning_budget : -1; @@ -247,7 +245,9 @@ bool configureReasoningBudgetSampling(common_params ¶ms, next.reasoning_budget_end = common_tokenize(lctx, thinkingEndTag, false, true); next.reasoning_budget_forced = common_tokenize( - lctx, params.sampling.reasoning_budget_message + thinkingEndTag, false, + lctx, + params.sampling.reasoning_budget_message + thinkingEndTag, + false, true); } @@ -263,8 +263,8 @@ bool configureReasoningBudgetSampling(common_params ¶ms, return changed; } -std::string getThinkingForcedOpenText(const std::string &generationPrompt, - const std::string &thinkingStartTag) { +std::string getThinkingForcedOpenText( + const std::string& generationPrompt, const std::string& thinkingStartTag) { if (thinkingStartTag.empty()) { return {}; } diff --git a/packages/llm-llamacpp/addon/src/utils/ChatTemplateUtils.hpp b/packages/llm-llamacpp/addon/src/utils/ChatTemplateUtils.hpp index 8f11fa0815..3652d91464 100644 --- a/packages/llm-llamacpp/addon/src/utils/ChatTemplateUtils.hpp +++ b/packages/llm-llamacpp/addon/src/utils/ChatTemplateUtils.hpp @@ -78,12 +78,13 @@ std::string getChatTemplate( * @p outGenerationPrompt (optional) receives the assistant generation prompt * already appended to the formatted prompt. */ -std::string getPrompt(const struct common_chat_templates *tmpls, - struct common_chat_templates_inputs &inputs, - bool *outThinkingForcedOpen = nullptr, - std::string *outThinkingStartTag = nullptr, - std::string *outThinkingEndTag = nullptr, - std::string *outGenerationPrompt = nullptr); +std::string getPrompt( + const struct common_chat_templates* tmpls, + struct common_chat_templates_inputs& inputs, + bool* outThinkingForcedOpen = nullptr, + std::string* outThinkingStartTag = nullptr, + std::string* outThinkingEndTag = nullptr, + std::string* outGenerationPrompt = nullptr); /** * @brief Configures the common-sampling reasoning-budget fields from @@ -92,14 +93,13 @@ std::string getPrompt(const struct common_chat_templates *tmpls, * Returns true when the sampling block changed and the caller should recreate * the common_sampler. */ -bool configureReasoningBudgetSampling(common_params ¶ms, - ::llama_context *lctx, - const std::string &thinkingStartTag, - const std::string &thinkingEndTag, - const std::string &generationPrompt); +bool configureReasoningBudgetSampling( + common_params& params, ::llama_context* lctx, + const std::string& thinkingStartTag, const std::string& thinkingEndTag, + const std::string& generationPrompt); -std::string getThinkingForcedOpenText(const std::string &generationPrompt, - const std::string &thinkingStartTag); +std::string getThinkingForcedOpenText( + const std::string& generationPrompt, const std::string& thinkingStartTag); } // namespace utils } // namespace qvac_lib_inference_addon_llama diff --git a/packages/llm-llamacpp/test/unit/test_chat_template_utils.cpp b/packages/llm-llamacpp/test/unit/test_chat_template_utils.cpp index b258136492..5f786806ce 100644 --- a/packages/llm-llamacpp/test/unit/test_chat_template_utils.cpp +++ b/packages/llm-llamacpp/test/unit/test_chat_template_utils.cpp @@ -239,9 +239,13 @@ TEST_F(ChatTemplateUtilsTest, GetPromptExportsQwenThinkingMetadata) { std::string thinkingStartTag; std::string thinkingEndTag; std::string generationPrompt; - const std::string prompt = - getPrompt(tmpls.get(), inputs, &thinkingForcedOpen, &thinkingStartTag, - &thinkingEndTag, &generationPrompt); + const std::string prompt = getPrompt( + tmpls.get(), + inputs, + &thinkingForcedOpen, + &thinkingStartTag, + &thinkingEndTag, + &generationPrompt); EXPECT_NE(prompt.find("<|im_start|>assistant"), std::string::npos); EXPECT_EQ(thinkingStartTag, ""); @@ -251,13 +255,14 @@ TEST_F(ChatTemplateUtilsTest, GetPromptExportsQwenThinkingMetadata) { } TEST_F(ChatTemplateUtilsTest, ThinkingForcedOpenTextUsesTemplateSuffix) { - EXPECT_EQ(getThinkingForcedOpenText("<|assistant|>\n\n", ""), - "\n"); + EXPECT_EQ( + getThinkingForcedOpenText("<|assistant|>\n\n", ""), + "\n"); } TEST_F(ChatTemplateUtilsTest, ThinkingForcedOpenTextFallsBackToStartTag) { - EXPECT_EQ(getThinkingForcedOpenText("<|assistant|>\n", ""), - ""); + EXPECT_EQ( + getThinkingForcedOpenText("<|assistant|>\n", ""), ""); } TEST_F(ChatTemplateUtilsTest, ThinkingForcedOpenTextEmptyWithoutStartTag) { diff --git a/packages/llm-llamacpp/test/unit/test_generation_params_apply.cpp b/packages/llm-llamacpp/test/unit/test_generation_params_apply.cpp index 090a9c065d..155e174e1b 100644 --- a/packages/llm-llamacpp/test/unit/test_generation_params_apply.cpp +++ b/packages/llm-llamacpp/test/unit/test_generation_params_apply.cpp @@ -69,8 +69,9 @@ TEST(GenerationParamsApplyTest, ZeroReasoningBudgetClearsBudgetSamplerState) { EXPECT_TRUE(sampling.reasoning_budget_forced.empty()); } -TEST(GenerationParamsApplyTest, - UnrestrictedReasoningBudgetClearsBudgetSamplerState) { +TEST( + GenerationParamsApplyTest, + UnrestrictedReasoningBudgetClearsBudgetSamplerState) { common_params_sampling sampling; sampling.reasoning_budget_tokens = 16; sampling.reasoning_budget_start = tokens({10}); @@ -88,8 +89,9 @@ TEST(GenerationParamsApplyTest, EXPECT_TRUE(sampling.reasoning_budget_forced.empty()); } -TEST(GenerationParamsApplyTest, - ConfigureReasoningBudgetSamplingClearsStaleStateWhenDisabled) { +TEST( + GenerationParamsApplyTest, + ConfigureReasoningBudgetSamplingClearsStaleStateWhenDisabled) { common_params params; params.reasoning_budget = 0; params.sampling.reasoning_budget_tokens = 16; @@ -107,8 +109,9 @@ TEST(GenerationParamsApplyTest, EXPECT_TRUE(params.sampling.generation_prompt.empty()); } -TEST(GenerationParamsApplyTest, - ConfigureReasoningBudgetSamplingKeepsPositiveCapWithoutContext) { +TEST( + GenerationParamsApplyTest, + ConfigureReasoningBudgetSamplingKeepsPositiveCapWithoutContext) { common_params params; params.reasoning_budget = 8; params.sampling.reasoning_budget_tokens = -1; From 17384301fd011013ac1f7d16ecec5432158d2db9 Mon Sep 17 00:00:00 2001 From: gianni-cor Date: Fri, 19 Jun 2026 10:49:07 +0200 Subject: [PATCH 33/58] test[notask]: match qwen thinking tag metadata Assisted-by: GPT-5.5 Co-authored-by: Cursor --- packages/llm-llamacpp/test/unit/test_chat_template_utils.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/llm-llamacpp/test/unit/test_chat_template_utils.cpp b/packages/llm-llamacpp/test/unit/test_chat_template_utils.cpp index 5f786806ce..4c2dda0dbf 100644 --- a/packages/llm-llamacpp/test/unit/test_chat_template_utils.cpp +++ b/packages/llm-llamacpp/test/unit/test_chat_template_utils.cpp @@ -248,8 +248,8 @@ TEST_F(ChatTemplateUtilsTest, GetPromptExportsQwenThinkingMetadata) { &generationPrompt); EXPECT_NE(prompt.find("<|im_start|>assistant"), std::string::npos); - EXPECT_EQ(thinkingStartTag, ""); - EXPECT_EQ(thinkingEndTag, ""); + EXPECT_EQ(thinkingStartTag, "\n"); + EXPECT_EQ(thinkingEndTag, "\n\n\n"); EXPECT_NE(generationPrompt.find("<|im_start|>assistant"), std::string::npos); EXPECT_FALSE(thinkingForcedOpen); } From 29b3dfe4dd1197deb03f45791abbe101c599bcfb Mon Sep 17 00:00:00 2001 From: gianni-cor Date: Fri, 19 Jun 2026 10:49:07 +0200 Subject: [PATCH 34/58] test[notask]: run q8 mrope cache test on android Assisted-by: GPT-5.5 Co-authored-by: Cursor --- .../test/integration/mrope-sliding-context.test.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/llm-llamacpp/test/integration/mrope-sliding-context.test.js b/packages/llm-llamacpp/test/integration/mrope-sliding-context.test.js index 7862c0b5bd..afde8e0032 100644 --- a/packages/llm-llamacpp/test/integration/mrope-sliding-context.test.js +++ b/packages/llm-llamacpp/test/integration/mrope-sliding-context.test.js @@ -380,8 +380,7 @@ safeTest('[qwen3.5-imrope-sliding-context] multimodal cache survives sliding sav }) safeTest('[qwen3.5-imrope-sliding-context] q8 K-cache shifts multimodal and text tokens', { - timeout: 1_800_000, - skip: isAndroid + timeout: 1_800_000 }, async t => { await runMultimodalSlidingCacheCase(t, { label: 'q8 K-cache multimodal', From 95f82f87608e1c4fe5d7f9f890ec2d91ba2395b2 Mon Sep 17 00:00:00 2001 From: gianni-cor Date: Fri, 19 Jun 2026 16:29:38 +0200 Subject: [PATCH 35/58] test[notask]: check reasoning budget token cap Assisted-by: GPT-5.5 Co-authored-by: Cursor --- .../test/integration/reasoning.test.js | 89 ----------- .../llm-llamacpp/test/unit/CMakeLists.txt | 1 + .../test/unit/test_reasoning_budget.cpp | 141 ++++++++++++++++++ 3 files changed, 142 insertions(+), 89 deletions(-) create mode 100644 packages/llm-llamacpp/test/unit/test_reasoning_budget.cpp diff --git a/packages/llm-llamacpp/test/integration/reasoning.test.js b/packages/llm-llamacpp/test/integration/reasoning.test.js index ed6bb1974f..7d4e540df3 100644 --- a/packages/llm-llamacpp/test/integration/reasoning.test.js +++ b/packages/llm-llamacpp/test/integration/reasoning.test.js @@ -240,92 +240,3 @@ safeTest('Qwen3 reasoning-budget=0 disables thinking', { t.ok(disabled.length < baseline.length / 4, `disabled (${disabled.length}) should be substantially shorter than baseline (${baseline.length})`) }) - -// reasoning-budget = N caps the channel at N tokens via the budget -// sampler - is force-emitted once N reasoning tokens have streamed. -safeTest('Qwen3 reasoning-budget=N caps reasoning channel below baseline', { - skip: isDarwinX64 || isWindowsX64, - timeout: 600_000 -}, async t => { - const [modelName, dirPath] = await ensureModel({ - modelName: MODEL.name, - downloadUrl: MODEL.url - }) - const modelPath = path.join(dirPath, modelName) - - const baseConfig = { - ctx_size: '4096', - n_predict: '1024', - seed: '50', - gpu_layers: '999', - temp: '0', - top_p: '1', - device: useCpu ? 'cpu' : 'gpu', - verbosity: '0' - } - - async function runOnce (extra, runOptions) { - const inference = new LlmLlamacpp({ - files: { model: [modelPath] }, - config: { ...baseConfig, ...extra }, - logger: console - }) - try { - await inference.load() - const messages = [ - { role: 'system', content: 'You are a helpful assistant.' }, - { role: 'user', content: 'What is the capital of France? Answer in one word.' } - ] - return await runCompletion(inference, messages, runOptions) - } finally { - await inference.unload().catch(() => {}) - } - } - - const baseline = await runOnce({}) - const baselineThink = sliceReasoning(baseline) - t.ok(baselineThink !== null, `baseline produced a reasoning channel (${baselineThink ? baselineThink.length : 0} chars)`) - - const BUDGET = 16 - const capped = await runOnce({ 'reasoning-budget': String(BUDGET) }) - const cappedThink = sliceReasoning(capped) - const perRequestCapped = await runOnce({}, { - generationParams: { reasoning_budget: BUDGET } - }) - const perRequestCappedThink = sliceReasoning(perRequestCapped) - - t.comment(`baseline reasoning (${baselineThink ? baselineThink.length : 0} chars): ${(baselineThink || '').slice(0, 200)}`) - t.comment(`capped reasoning (${cappedThink ? cappedThink.length : 0} chars): ${cappedThink || ''}`) - t.comment(`request reasoning (${perRequestCappedThink ? perRequestCappedThink.length : 0} chars): ${perRequestCappedThink || ''}`) - - t.ok(/paris/i.test(capped), 'capped output still mentions Paris (correctness preserved past forced close)') - t.ok(/paris/i.test(perRequestCapped), 'per-request capped output still mentions Paris') - t.ok(capped.includes('') && capped.includes(''), - `capped output keeps balanced reasoning tags: "${capped.slice(0, 200)}"`) - t.ok(perRequestCapped.includes('') && perRequestCapped.includes(''), - `per-request capped output keeps balanced reasoning tags: "${perRequestCapped.slice(0, 200)}"`) - t.ok(cappedThink !== null, - 'capped output has an extractable reasoning channel') - t.ok(perRequestCappedThink !== null, - 'per-request capped output has an extractable reasoning channel') - // The budget sampler is char-agnostic - it counts tokens, not characters - - // so we assert the reasoning channel is at least *substantially* shorter - // than the unrestricted baseline rather than asserting an exact char count. - t.ok( - cappedThink.length < Math.max(1, baselineThink.length * 0.6), - `capped reasoning (${cappedThink.length}) is substantially shorter than baseline (${baselineThink.length})` - ) - t.ok( - perRequestCappedThink.length < Math.max(1, baselineThink.length * 0.6), - `per-request capped reasoning (${perRequestCappedThink.length}) is substantially shorter than baseline (${baselineThink.length})` - ) -}) - -function sliceReasoning (text) { - const open = text.indexOf('') - const close = text.indexOf('') - if (open === -1 || close === -1 || close < open) { - return null - } - return text.slice(open + ''.length, close) -} diff --git a/packages/llm-llamacpp/test/unit/CMakeLists.txt b/packages/llm-llamacpp/test/unit/CMakeLists.txt index b87728bcb2..0d04bd7906 100644 --- a/packages/llm-llamacpp/test/unit/CMakeLists.txt +++ b/packages/llm-llamacpp/test/unit/CMakeLists.txt @@ -43,6 +43,7 @@ add_executable( test_continuous_batching_integration.cpp test_kv_leak_on_admit_failure.cpp test_runtime_stats.cpp + test_reasoning_budget.cpp ${CMAKE_SOURCE_DIR}/addon/src/model-interface/AsyncWeightsLoader.cpp ${CMAKE_SOURCE_DIR}/addon/src/model-interface/MultiRequestBatcher.cpp ${CMAKE_SOURCE_DIR}/addon/src/model-interface/ContinuousBatchScheduler.cpp diff --git a/packages/llm-llamacpp/test/unit/test_reasoning_budget.cpp b/packages/llm-llamacpp/test/unit/test_reasoning_budget.cpp new file mode 100644 index 0000000000..1edc96a2fd --- /dev/null +++ b/packages/llm-llamacpp/test/unit/test_reasoning_budget.cpp @@ -0,0 +1,141 @@ +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "common/common.h" +#include "model-interface/LlamaModel.hpp" +#include "test_common.hpp" + +namespace fs = std::filesystem; + +namespace { + +constexpr int REASONING_BUDGET = 3; +constexpr int PREDICT_TOKENS = 10; +constexpr const char* THINKING_START_TAG = "\n"; +constexpr const char* THINKING_END_TAG = "\n\n\n"; + +std::string sliceReasoning(const std::string& text) { + const size_t open = text.find(THINKING_START_TAG); + if (open == std::string::npos) { + return ""; + } + + const size_t bodyStart = open + std::string(THINKING_START_TAG).size(); + const size_t close = text.find(THINKING_END_TAG, bodyStart); + if (close == std::string::npos || close < bodyStart) { + return ""; + } + + return text.substr(bodyStart, close - bodyStart); +} + +size_t countTokens(llama_context* ctx, const std::string& text) { + return common_tokenize(ctx, text, false, true).size(); +} + +struct RunResult { + std::string output; + size_t reasoningTokens = 0; +}; + +} // namespace + +class ReasoningBudgetModelTest : public ::testing::Test { +protected: + void SetUp() override { + using MP = test_common::TestModelPath; + qwen3Model_ = + MP("Qwen3-0.6B-Q8_0.gguf", + "QWEN3_MODEL_PATH", + MP::OnMissing::Skip, + "https://huggingface.co/Qwen/Qwen3-0.6B-GGUF"); + + config_["ctx_size"] = "4096"; + config_["n_predict"] = std::to_string(PREDICT_TOKENS); + config_["seed"] = "50"; + config_["temp"] = "0"; + config_["top_p"] = "1"; + config_["device"] = test_common::getTestDevice(); + config_["gpu_layers"] = test_common::getTestGpuLayers(); + config_["backendsDir"] = test_common::getTestBackendsDir().string(); + } + + [[nodiscard]] bool hasQwen3Model() const { + return qwen3Model_.found() && fs::exists(qwen3Model_.path); + } + + std::unique_ptr createModel( + std::unordered_map config) { + auto model = std::make_unique( + std::string(qwen3Model_.path), std::string(), std::move(config)); + model->waitForLoadInitialization(); + return model; + } + + RunResult runBudgetedPrompt( + std::unordered_map config, + const GenerationParams& generationParams = {}) { + auto model = createModel(std::move(config)); + if (!model->isLoaded()) { + throw std::runtime_error("Qwen3 model failed to load"); + } + + LlamaModel::Prompt prompt; + prompt.input = + R"([{"role":"system","content":"You are a helpful assistant."},{"role":"user","content":"What is the capital of France? Answer in one word."}])"; + prompt.generationParams = generationParams; + + const std::string output = model->processPrompt(prompt); + const std::string reasoning = sliceReasoning(output); + return {output, countTokens(model->getContext(), reasoning)}; + } + + test_common::TestModelPath qwen3Model_; + std::unordered_map config_; +}; + +TEST_F( + ReasoningBudgetModelTest, + LoadTimeBudgetThreeGeneratesThreeReasoningTokens) { + if (!hasQwen3Model()) { + GTEST_SKIP() << qwen3Model_.missingMessage(); + } + + auto config = config_; + config["reasoning-budget"] = std::to_string(REASONING_BUDGET); + + const RunResult result = runBudgetedPrompt(std::move(config)); + + EXPECT_NE(result.output.find(THINKING_START_TAG), std::string::npos) + << result.output; + EXPECT_NE(result.output.find(THINKING_END_TAG), std::string::npos) + << result.output; + EXPECT_EQ(result.reasoningTokens, REASONING_BUDGET) << result.output; +} + +TEST_F( + ReasoningBudgetModelTest, + PerRequestBudgetThreeGeneratesThreeReasoningTokens) { + if (!hasQwen3Model()) { + GTEST_SKIP() << qwen3Model_.missingMessage(); + } + + GenerationParams generationParams; + generationParams.reasoning_budget = REASONING_BUDGET; + generationParams.n_predict = PREDICT_TOKENS; + + const RunResult result = runBudgetedPrompt(config_, generationParams); + + EXPECT_NE(result.output.find(THINKING_START_TAG), std::string::npos) + << result.output; + EXPECT_NE(result.output.find(THINKING_END_TAG), std::string::npos) + << result.output; + EXPECT_EQ(result.reasoningTokens, REASONING_BUDGET) << result.output; +} From 8ae7d1135fc4603bf7b0a6e5f896845612f9cb3f Mon Sep 17 00:00:00 2001 From: gianni-cor Date: Fri, 19 Jun 2026 17:53:37 +0200 Subject: [PATCH 36/58] test[notask]: skip q8 mrope cache on android Assisted-by: GPT-5.5 Co-authored-by: Cursor --- .../test/integration/mrope-sliding-context.test.js | 3 ++- packages/llm-llamacpp/test/unit/CMakeLists.txt | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/llm-llamacpp/test/integration/mrope-sliding-context.test.js b/packages/llm-llamacpp/test/integration/mrope-sliding-context.test.js index afde8e0032..7862c0b5bd 100644 --- a/packages/llm-llamacpp/test/integration/mrope-sliding-context.test.js +++ b/packages/llm-llamacpp/test/integration/mrope-sliding-context.test.js @@ -380,7 +380,8 @@ safeTest('[qwen3.5-imrope-sliding-context] multimodal cache survives sliding sav }) safeTest('[qwen3.5-imrope-sliding-context] q8 K-cache shifts multimodal and text tokens', { - timeout: 1_800_000 + timeout: 1_800_000, + skip: isAndroid }, async t => { await runMultimodalSlidingCacheCase(t, { label: 'q8 K-cache multimodal', diff --git a/packages/llm-llamacpp/test/unit/CMakeLists.txt b/packages/llm-llamacpp/test/unit/CMakeLists.txt index 0d04bd7906..489fe57f40 100644 --- a/packages/llm-llamacpp/test/unit/CMakeLists.txt +++ b/packages/llm-llamacpp/test/unit/CMakeLists.txt @@ -130,7 +130,7 @@ target_link_libraries( GTest::gmock_main ) -if(NOT WIN32) +if(NOT WIN32 AND NOT APPLE) target_link_libraries( addon-test PRIVATE From cdfdf378af6a9fc037137cef7c81b18a7e11808d Mon Sep 17 00:00:00 2001 From: gianni-cor Date: Fri, 19 Jun 2026 18:25:04 +0200 Subject: [PATCH 37/58] deps[notask]: bump qvac-fabric opencl fix Assisted-by: GPT-5.5 Co-authored-by: Cursor --- packages/llm-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/llm-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake b/packages/llm-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake index ead537a0ce..530fe1f1bb 100644 --- a/packages/llm-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake +++ b/packages/llm-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake @@ -1,8 +1,8 @@ vcpkg_from_github( OUT_SOURCE_PATH SOURCE_PATH REPO tetherto/qvac-fabric-llm.cpp - REF 6f37291fe979c8f9a66a4c85809f69ff0ebeb743 - SHA512 6e29daed48978cac5fad4f2ed96f6518f0a81eef7a789d29d44196fb7a2a1b8fb7ed75fdcfc4acd370279a6b0b500c75517b79115dd88b2fbee2882e3a5d720d + REF e3d047fb09fcfacff3ce792938784046c2854722 + SHA512 73cfa09f8bea8f20be6f783fb7a14e2e23e9d88dbbae0bcf82bbe92c81f2765515d9efde349253fb2c774bf5468ff0f158b4ea284afcda023ee431437fc99f37 HEAD_REF temp-9341 ) From f1222fd28021d049bd55ad1877b3263ea0960eb7 Mon Sep 17 00:00:00 2001 From: gianni-cor Date: Fri, 19 Jun 2026 18:27:22 +0200 Subject: [PATCH 38/58] test[notask]: run q8 mrope cache test on android Assisted-by: GPT-5.5 Co-authored-by: Cursor --- .../test/integration/mrope-sliding-context.test.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/llm-llamacpp/test/integration/mrope-sliding-context.test.js b/packages/llm-llamacpp/test/integration/mrope-sliding-context.test.js index 7862c0b5bd..d1a542fdcf 100644 --- a/packages/llm-llamacpp/test/integration/mrope-sliding-context.test.js +++ b/packages/llm-llamacpp/test/integration/mrope-sliding-context.test.js @@ -380,14 +380,14 @@ safeTest('[qwen3.5-imrope-sliding-context] multimodal cache survives sliding sav }) safeTest('[qwen3.5-imrope-sliding-context] q8 K-cache shifts multimodal and text tokens', { - timeout: 1_800_000, - skip: isAndroid + timeout: 1_800_000 }, async t => { await runMultimodalSlidingCacheCase(t, { label: 'q8 K-cache multimodal', cacheFileName: 'qwen3-5-q8-kcache-multimodal-sliding-cache.bin', extraConfig: { - 'cache-type-k': 'q8_0' + 'cache-type-k': 'q8_0', + 'cache-type-v': 'q8_0' } }) }) From cc94b2b7e28c8413a4146996abd983c9ed9e5305 Mon Sep 17 00:00:00 2001 From: gianni-cor Date: Fri, 19 Jun 2026 18:31:18 +0200 Subject: [PATCH 39/58] test[notask]: format reasoning budget test Assisted-by: GPT-5.5 Co-authored-by: Cursor --- packages/llm-llamacpp/test/unit/test_reasoning_budget.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/llm-llamacpp/test/unit/test_reasoning_budget.cpp b/packages/llm-llamacpp/test/unit/test_reasoning_budget.cpp index 1edc96a2fd..2e94a9ea5f 100644 --- a/packages/llm-llamacpp/test/unit/test_reasoning_budget.cpp +++ b/packages/llm-llamacpp/test/unit/test_reasoning_budget.cpp @@ -71,8 +71,8 @@ class ReasoningBudgetModelTest : public ::testing::Test { return qwen3Model_.found() && fs::exists(qwen3Model_.path); } - std::unique_ptr createModel( - std::unordered_map config) { + std::unique_ptr + createModel(std::unordered_map config) { auto model = std::make_unique( std::string(qwen3Model_.path), std::string(), std::move(config)); model->waitForLoadInitialization(); From c5885c6f71f52e4e3694f5a1e33e669212490d02 Mon Sep 17 00:00:00 2001 From: gianni-cor Date: Fri, 19 Jun 2026 20:25:49 +0200 Subject: [PATCH 40/58] deps[notask]: update qvac-fabric OpenCL fix Assisted-by: GPT-5.5 Co-authored-by: Cursor --- packages/llm-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/llm-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake b/packages/llm-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake index 530fe1f1bb..5ad0cbeccb 100644 --- a/packages/llm-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake +++ b/packages/llm-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake @@ -1,8 +1,8 @@ vcpkg_from_github( OUT_SOURCE_PATH SOURCE_PATH REPO tetherto/qvac-fabric-llm.cpp - REF e3d047fb09fcfacff3ce792938784046c2854722 - SHA512 73cfa09f8bea8f20be6f783fb7a14e2e23e9d88dbbae0bcf82bbe92c81f2765515d9efde349253fb2c774bf5468ff0f158b4ea284afcda023ee431437fc99f37 + REF 1e285f9df65933e8f3ffcdfd85b7f69eb9c0cbf1 + SHA512 d6bce10d288131231ad06e64052a06a6e8dae1f758f852598ec4f19197c593cde0b1c971f51b7adc8648080330b868267bf951351f9f345781fd1ccf50cc12d8 HEAD_REF temp-9341 ) From 8481f8cf0960d91e416a573a331f7bc3af5b5d26 Mon Sep 17 00:00:00 2001 From: gianni-cor Date: Fri, 19 Jun 2026 18:55:26 +0200 Subject: [PATCH 41/58] test[notask]: skip llm integration tests on darwin x64 Assisted-by: GPT-5.5 Co-authored-by: Cursor --- .../llm-llamacpp/test/integration/_vlm-image-perf.js | 12 ++++++------ .../test/integration/cache-state-machine.test.js | 6 +++++- .../integration/gemma4-image-elephant-perf.test.js | 4 ++-- .../gemma4-image-fruit-plate-perf.test.js | 4 ++-- .../gemma4-image-high-res-aurora-perf.test.js | 8 ++++---- .../test/integration/mrope-sliding-context.test.js | 6 +++++- .../llm-llamacpp/test/integration/multi-gpu.test.js | 12 ++++++++---- .../test/integration/multi-instance.test.js | 9 +++++---- .../integration/qwen3-5-image-elephant-perf.test.js | 4 ++-- .../qwen3-5-image-fruit-plate-perf.test.js | 4 ++-- .../qwen3-5-image-high-res-aurora-perf.test.js | 8 ++++---- .../test/integration/sliding-context.test.js | 2 +- 12 files changed, 46 insertions(+), 33 deletions(-) diff --git a/packages/llm-llamacpp/test/integration/_vlm-image-perf.js b/packages/llm-llamacpp/test/integration/_vlm-image-perf.js index 268a2e51ac..a11295c1fe 100644 --- a/packages/llm-llamacpp/test/integration/_vlm-image-perf.js +++ b/packages/llm-llamacpp/test/integration/_vlm-image-perf.js @@ -36,12 +36,11 @@ function _envInt (key, fallback) { const PERF_RUNS = _envInt('QVAC_PERF_RUNS', 1) const PERF_WARMUP_RUNS = _envInt('QVAC_PERF_WARMUP_RUNS', 1) -// QVAC-19368: skip the heaviest image (aurora) only on Android non-benchmark -// on-PR runs where the 30-min Device Farm per-test cap is tight. iOS and -// desktop always run aurora (fast enough). The benchmark (QVAC_PERF_ONLY=true) -// always runs the full set regardless of platform. Uses the explicit -// QVAC_PERF_ONLY flag (already plumbed to the device via the testspec config) -// instead of proxying off PERF_RUNS, per review feedback. +// QVAC-19368: skip the heaviest image (aurora) on Android non-benchmark +// on-PR runs where the 30-min Device Farm per-test cap is tight. The +// benchmark (QVAC_PERF_ONLY=true) runs the full set on supported platforms. +// Uses the explicit QVAC_PERF_ONLY flag (already plumbed to the device via +// the testspec config) instead of proxying off PERF_RUNS, per review feedback. function _envStr (key) { if (typeof os.getEnv === 'function') return os.getEnv(key) || '' if (typeof process !== 'undefined' && process.env) return process.env[key] || '' @@ -216,6 +215,7 @@ module.exports = { IMAGE_CASES, GEMMA4_MODEL, QWEN35_MODEL, + isDarwinX64, skipHeavyImages, runVlmImagePerf } diff --git a/packages/llm-llamacpp/test/integration/cache-state-machine.test.js b/packages/llm-llamacpp/test/integration/cache-state-machine.test.js index 9e919270e5..6cdc37b706 100644 --- a/packages/llm-llamacpp/test/integration/cache-state-machine.test.js +++ b/packages/llm-llamacpp/test/integration/cache-state-machine.test.js @@ -3,7 +3,7 @@ const path = require('bare-path') const fs = require('bare-fs') const LlmLlamacpp = require('../../index.js') -const { cleanupIntegrationCacheFiles, ensureModel, safeTest } = require('./utils') +const { cleanupIntegrationCacheFiles, ensureModel, safeTest: integrationTest } = require('./utils') const { attachSpecLogger } = require('./spec-logger') const os = require('bare-os') @@ -11,6 +11,10 @@ const isDarwinX64 = os.platform() === 'darwin' && os.arch() === 'x64' const isLinuxArm64 = os.platform() === 'linux' && os.arch() === 'arm64' const useCpu = isDarwinX64 || isLinuxArm64 +function safeTest (name, opts, fn) { + integrationTest(name, { ...opts, skip: opts.skip || isDarwinX64 }, fn) +} + const DEFAULT_MODEL = { name: 'Llama-3.2-1B-Instruct-Q4_0.gguf', url: 'https://huggingface.co/bartowski/Llama-3.2-1B-Instruct-GGUF/resolve/main/Llama-3.2-1B-Instruct-Q4_0.gguf' diff --git a/packages/llm-llamacpp/test/integration/gemma4-image-elephant-perf.test.js b/packages/llm-llamacpp/test/integration/gemma4-image-elephant-perf.test.js index eac37c3246..8bd2113ba3 100644 --- a/packages/llm-llamacpp/test/integration/gemma4-image-elephant-perf.test.js +++ b/packages/llm-llamacpp/test/integration/gemma4-image-elephant-perf.test.js @@ -4,9 +4,9 @@ // 30-minute mobile cap. Asserts the elephant keyword + records perf. const test = require('brittle') -const { GEMMA4_MODEL, IMAGE_CASES, runVlmImagePerf } = require('./_vlm-image-perf.js') +const { GEMMA4_MODEL, IMAGE_CASES, isDarwinX64, runVlmImagePerf } = require('./_vlm-image-perf.js') -test('Gemma4-VL image perf [elephant]', { timeout: 1_800_000 }, async t => { +test('Gemma4-VL image perf [elephant]', { timeout: 1_800_000, skip: isDarwinX64 }, async t => { await runVlmImagePerf(t, GEMMA4_MODEL, IMAGE_CASES.elephant) }) diff --git a/packages/llm-llamacpp/test/integration/gemma4-image-fruit-plate-perf.test.js b/packages/llm-llamacpp/test/integration/gemma4-image-fruit-plate-perf.test.js index 1e6f048f0b..8c695d460a 100644 --- a/packages/llm-llamacpp/test/integration/gemma4-image-fruit-plate-perf.test.js +++ b/packages/llm-llamacpp/test/integration/gemma4-image-fruit-plate-perf.test.js @@ -4,9 +4,9 @@ // 30-minute mobile cap. Asserts a fruit keyword + records perf. const test = require('brittle') -const { GEMMA4_MODEL, IMAGE_CASES, runVlmImagePerf } = require('./_vlm-image-perf.js') +const { GEMMA4_MODEL, IMAGE_CASES, isDarwinX64, runVlmImagePerf } = require('./_vlm-image-perf.js') -test('Gemma4-VL image perf [fruit plate]', { timeout: 1_800_000 }, async t => { +test('Gemma4-VL image perf [fruit plate]', { timeout: 1_800_000, skip: isDarwinX64 }, async t => { await runVlmImagePerf(t, GEMMA4_MODEL, IMAGE_CASES['fruit-plate']) }) diff --git a/packages/llm-llamacpp/test/integration/gemma4-image-high-res-aurora-perf.test.js b/packages/llm-llamacpp/test/integration/gemma4-image-high-res-aurora-perf.test.js index 0d05133b8b..ae0666843d 100644 --- a/packages/llm-llamacpp/test/integration/gemma4-image-high-res-aurora-perf.test.js +++ b/packages/llm-llamacpp/test/integration/gemma4-image-high-res-aurora-perf.test.js @@ -4,12 +4,12 @@ // the 30-minute mobile cap. Asserts an aurora keyword + records perf. const test = require('brittle') -const { GEMMA4_MODEL, IMAGE_CASES, skipHeavyImages, runVlmImagePerf } = require('./_vlm-image-perf.js') +const { GEMMA4_MODEL, IMAGE_CASES, isDarwinX64, skipHeavyImages, runVlmImagePerf } = require('./_vlm-image-perf.js') // QVAC-19368: aurora is the heaviest image; skip it on Android on-PR runs -// where the 30-min Device Farm cap is tight. iOS + desktop always run it. -// The benchmark (QVAC_PERF_ONLY=true) runs all 3 images on all platforms. -test('Gemma4-VL image perf [high-res aurora]', { timeout: 1_800_000, skip: skipHeavyImages }, async t => { +// where the 30-min Device Farm cap is tight. Darwin x64 is skipped separately. +// The benchmark (QVAC_PERF_ONLY=true) runs all 3 images on supported platforms. +test('Gemma4-VL image perf [high-res aurora]', { timeout: 1_800_000, skip: isDarwinX64 || skipHeavyImages }, async t => { await runVlmImagePerf(t, GEMMA4_MODEL, IMAGE_CASES['high-res-aurora']) }) diff --git a/packages/llm-llamacpp/test/integration/mrope-sliding-context.test.js b/packages/llm-llamacpp/test/integration/mrope-sliding-context.test.js index d1a542fdcf..3084b2b211 100644 --- a/packages/llm-llamacpp/test/integration/mrope-sliding-context.test.js +++ b/packages/llm-llamacpp/test/integration/mrope-sliding-context.test.js @@ -4,7 +4,7 @@ const fs = require('bare-fs') const os = require('bare-os') const path = require('bare-path') const LlmLlamacpp = require('../../index.js') -const { cleanupIntegrationCacheFiles, ensureModel, getMediaPath, safeTest } = require('./utils') +const { cleanupIntegrationCacheFiles, ensureModel, getMediaPath, safeTest: integrationTest } = require('./utils') const platform = os.platform() const arch = os.arch() @@ -16,6 +16,10 @@ const isLinuxArm64 = platform === 'linux' && arch === 'arm64' const useCpu = isDarwinX64 || isLinuxArm64 const skipTbqPq = isDarwin || isIos || isAndroid +function safeTest (name, opts, fn) { + integrationTest(name, { ...opts, skip: opts.skip || isDarwinX64 }, fn) +} + const QWEN3_5_MODEL = { name: 'Qwen3.5-0.8B-Q8_0.gguf', url: 'https://huggingface.co/unsloth/Qwen3.5-0.8B-GGUF/resolve/main/Qwen3.5-0.8B-Q8_0.gguf' diff --git a/packages/llm-llamacpp/test/integration/multi-gpu.test.js b/packages/llm-llamacpp/test/integration/multi-gpu.test.js index d6a926370e..21ea45b819 100644 --- a/packages/llm-llamacpp/test/integration/multi-gpu.test.js +++ b/packages/llm-llamacpp/test/integration/multi-gpu.test.js @@ -1,11 +1,14 @@ 'use strict' const process = require('bare-process') +const os = require('bare-os') const LlmLlamacpp = require('../../index.js') const { ensureModel, safeTest } = require('./utils') const { attachSpecLogger } = require('./spec-logger') const path = require('bare-path') +const isDarwinX64 = os.platform() === 'darwin' && os.arch() === 'x64' + const MODEL = { name: 'Qwen3-0.6B-Q8_0.gguf', url: 'https://huggingface.co/unsloth/Qwen3-0.6B-GGUF/resolve/main/Qwen3-0.6B-Q8_0.gguf' @@ -32,6 +35,7 @@ async function collectResponse (response) { } const hasMultiGpu = process.env.QVAC_HAS_MULTI_GPU === '1' +const skip = isDarwinX64 const BASE_CONFIG = { device: 'gpu', @@ -92,19 +96,19 @@ function assertSingleDevice (t, devices) { t.ok(devices.size <= 1, `layers should stay on a single device (found: ${[...devices].join(', ')})`) } -safeTest('multi-gpu: split-mode=layer distributes layers across GPUs', { timeout: 600_000 }, async t => { +safeTest('multi-gpu: split-mode=layer distributes layers across GPUs', { timeout: 600_000, skip }, async t => { await runMultiGpuTest(t, { 'split-mode': 'layer' }, assertMultiDevice('layers')) }) -safeTest('multi-gpu: split-mode=row distributes tensors across GPUs', { timeout: 600_000 }, async t => { +safeTest('multi-gpu: split-mode=row distributes tensors across GPUs', { timeout: 600_000, skip }, async t => { await runMultiGpuTest(t, { 'split-mode': 'row' }, assertMultiDevice('tensors')) }) -safeTest('multi-gpu: default (no split-mode) pins layers to a single device', { timeout: 600_000 }, async t => { +safeTest('multi-gpu: default (no split-mode) pins layers to a single device', { timeout: 600_000, skip }, async t => { await runMultiGpuTest(t, {}, assertSingleDevice) }) -safeTest('multi-gpu: split-mode=layer with tensor-split and main-gpu', { timeout: 600_000 }, async t => { +safeTest('multi-gpu: split-mode=layer with tensor-split and main-gpu', { timeout: 600_000, skip }, async t => { await runMultiGpuTest( t, { 'split-mode': 'layer', 'tensor-split': '1,1', 'main-gpu': '0' }, diff --git a/packages/llm-llamacpp/test/integration/multi-instance.test.js b/packages/llm-llamacpp/test/integration/multi-instance.test.js index cc458d9486..839868fcdf 100644 --- a/packages/llm-llamacpp/test/integration/multi-instance.test.js +++ b/packages/llm-llamacpp/test/integration/multi-instance.test.js @@ -11,6 +11,7 @@ const isDarwinX64 = platform === 'darwin' && arch === 'x64' const isLinuxArm64 = platform === 'linux' && arch === 'arm64' const isWindowsX64 = platform === 'win32' && arch === 'x64' const useCpu = isDarwinX64 || isLinuxArm64 +const skip = isDarwinX64 || isWindowsX64 const DEFAULT_MODEL = { name: 'Llama-3.2-1B-Instruct-Q4_0.gguf', @@ -73,7 +74,7 @@ async function collectResponse (response) { safeTest('Two instances can run inference simultaneously', { timeout: 900_000, - skip: isWindowsX64 // TODO: unskip this once we have a new Windows runner with a GPU + skip // TODO: unskip Windows once we have a new Windows runner with a GPU }, async t => { let addon1 = null let addon2 = null @@ -107,7 +108,7 @@ safeTest('Two instances can run inference simultaneously', { safeTest('Repeated load/unload cycles should remain stable', { timeout: 900_000, - skip: isWindowsX64 // TODO: unskip this once we have a new Windows runner with a GPU + skip // TODO: unskip Windows once we have a new Windows runner with a GPU }, async t => { let currentAddon = null try { @@ -142,7 +143,7 @@ safeTest('Repeated load/unload cycles should remain stable', { safeTest('Unloading one instance does not affect another generating instance', { timeout: 900_000, - skip: isWindowsX64 // TODO: unskip this once we have a new Windows runner with a GPU + skip // TODO: unskip Windows once we have a new Windows runner with a GPU }, async t => { let addon1 = null let addon2 = null @@ -204,7 +205,7 @@ safeTest('Unloading one instance does not affect another generating instance', { safeTest('Multiple load/unload cycles on one instance while another generates', { timeout: 900_000, - skip: isWindowsX64 // TODO: unskip this once we have a new Windows runner with a GPU + skip // TODO: unskip Windows once we have a new Windows runner with a GPU }, async t => { let addon1 = null try { diff --git a/packages/llm-llamacpp/test/integration/qwen3-5-image-elephant-perf.test.js b/packages/llm-llamacpp/test/integration/qwen3-5-image-elephant-perf.test.js index ade746925c..7c9c898891 100644 --- a/packages/llm-llamacpp/test/integration/qwen3-5-image-elephant-perf.test.js +++ b/packages/llm-llamacpp/test/integration/qwen3-5-image-elephant-perf.test.js @@ -4,9 +4,9 @@ // 30-minute mobile cap. Asserts the elephant keyword + records perf. const test = require('brittle') -const { QWEN35_MODEL, IMAGE_CASES, runVlmImagePerf } = require('./_vlm-image-perf.js') +const { QWEN35_MODEL, IMAGE_CASES, isDarwinX64, runVlmImagePerf } = require('./_vlm-image-perf.js') -test('Qwen3.5-VL image perf [elephant]', { timeout: 1_800_000 }, async t => { +test('Qwen3.5-VL image perf [elephant]', { timeout: 1_800_000, skip: isDarwinX64 }, async t => { await runVlmImagePerf(t, QWEN35_MODEL, IMAGE_CASES.elephant) }) diff --git a/packages/llm-llamacpp/test/integration/qwen3-5-image-fruit-plate-perf.test.js b/packages/llm-llamacpp/test/integration/qwen3-5-image-fruit-plate-perf.test.js index 534e9dd681..869953af29 100644 --- a/packages/llm-llamacpp/test/integration/qwen3-5-image-fruit-plate-perf.test.js +++ b/packages/llm-llamacpp/test/integration/qwen3-5-image-fruit-plate-perf.test.js @@ -4,9 +4,9 @@ // the 30-minute mobile cap. Asserts a fruit keyword + records perf. const test = require('brittle') -const { QWEN35_MODEL, IMAGE_CASES, runVlmImagePerf } = require('./_vlm-image-perf.js') +const { QWEN35_MODEL, IMAGE_CASES, isDarwinX64, runVlmImagePerf } = require('./_vlm-image-perf.js') -test('Qwen3.5-VL image perf [fruit plate]', { timeout: 1_800_000 }, async t => { +test('Qwen3.5-VL image perf [fruit plate]', { timeout: 1_800_000, skip: isDarwinX64 }, async t => { await runVlmImagePerf(t, QWEN35_MODEL, IMAGE_CASES['fruit-plate']) }) diff --git a/packages/llm-llamacpp/test/integration/qwen3-5-image-high-res-aurora-perf.test.js b/packages/llm-llamacpp/test/integration/qwen3-5-image-high-res-aurora-perf.test.js index 447df2f686..422e38150e 100644 --- a/packages/llm-llamacpp/test/integration/qwen3-5-image-high-res-aurora-perf.test.js +++ b/packages/llm-llamacpp/test/integration/qwen3-5-image-high-res-aurora-perf.test.js @@ -4,12 +4,12 @@ // the 30-minute mobile cap. Asserts an aurora keyword + records perf. const test = require('brittle') -const { QWEN35_MODEL, IMAGE_CASES, skipHeavyImages, runVlmImagePerf } = require('./_vlm-image-perf.js') +const { QWEN35_MODEL, IMAGE_CASES, isDarwinX64, skipHeavyImages, runVlmImagePerf } = require('./_vlm-image-perf.js') // QVAC-19368: aurora is the heaviest image; skip it on Android on-PR runs -// where the 30-min Device Farm cap is tight. iOS + desktop always run it. -// The benchmark (QVAC_PERF_ONLY=true) runs all 3 images on all platforms. -test('Qwen3.5-VL image perf [high-res aurora]', { timeout: 1_800_000, skip: skipHeavyImages }, async t => { +// where the 30-min Device Farm cap is tight. Darwin x64 is skipped separately. +// The benchmark (QVAC_PERF_ONLY=true) runs all 3 images on supported platforms. +test('Qwen3.5-VL image perf [high-res aurora]', { timeout: 1_800_000, skip: isDarwinX64 || skipHeavyImages }, async t => { await runVlmImagePerf(t, QWEN35_MODEL, IMAGE_CASES['high-res-aurora']) }) diff --git a/packages/llm-llamacpp/test/integration/sliding-context.test.js b/packages/llm-llamacpp/test/integration/sliding-context.test.js index 4e007caa89..a39366e296 100644 --- a/packages/llm-llamacpp/test/integration/sliding-context.test.js +++ b/packages/llm-llamacpp/test/integration/sliding-context.test.js @@ -14,7 +14,7 @@ const useCpu = isDarwinX64 || isLinuxArm64 // These are very slow on CI and should be skipped. // TODO: unskip Windows once we have a new Windows runner with a GPU -const skip = isWindowsX64 || isLinuxArm64 +const skip = isDarwinX64 || isWindowsX64 || isLinuxArm64 const DEFAULT_MODEL = { name: 'Llama-3.2-1B-Instruct-Q4_0.gguf', From f38b711615ad8b5b13c594258883f37934a53441 Mon Sep 17 00:00:00 2001 From: gianni-cor Date: Fri, 19 Jun 2026 18:56:03 +0200 Subject: [PATCH 42/58] ci[notask]: use macos 15 xlarge for llm cpp tests Assisted-by: GPT-5.5 Co-authored-by: Cursor --- .github/workflows/cpp-tests-llm.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cpp-tests-llm.yml b/.github/workflows/cpp-tests-llm.yml index b798baadfe..7e2eb69f59 100644 --- a/.github/workflows/cpp-tests-llm.yml +++ b/.github/workflows/cpp-tests-llm.yml @@ -37,7 +37,7 @@ jobs: platform: linux runner: qvac-ubuntu2404-x64 arch: x64 - - os: macos-14 + - os: macos-15-xlarge platform: darwin arch: arm64 - os: windows-2025 From f9438b88a45ca14075e404c5bb3705ddd004f0d6 Mon Sep 17 00:00:00 2001 From: gianni-cor Date: Fri, 19 Jun 2026 21:03:44 +0200 Subject: [PATCH 43/58] test[notask]: re-enable llm integration tests on windows Assisted-by: GPT-5.5 Co-authored-by: Cursor --- .../test/integration/model-loading.test.js | 8 ++++---- .../test/integration/multi-instance.test.js | 11 +++++------ .../llm-llamacpp/test/integration/reasoning.test.js | 7 +++---- .../test/integration/sliding-context.test.js | 3 +-- 4 files changed, 13 insertions(+), 16 deletions(-) diff --git a/packages/llm-llamacpp/test/integration/model-loading.test.js b/packages/llm-llamacpp/test/integration/model-loading.test.js index c1f084450d..989500b3a5 100644 --- a/packages/llm-llamacpp/test/integration/model-loading.test.js +++ b/packages/llm-llamacpp/test/integration/model-loading.test.js @@ -13,6 +13,7 @@ const arch = os.arch() const isDarwinX64 = platform === 'darwin' && arch === 'x64' const isLinuxArm64 = platform === 'linux' && arch === 'arm64' const isLinuxX64 = platform === 'linux' && arch === 'x64' +const isWindowsX64 = platform === 'win32' && arch === 'x64' const useCpu = isDarwinX64 || isLinuxArm64 const DEFAULT_MODEL = { @@ -126,10 +127,9 @@ const SHARDED_MODEL = { baseUrl: 'https://huggingface.co/jmb95/Qwen3-0.6B-UD-IQ1_S-sharded/resolve/main/' } -// This test can take longer to download and execute. To avoid blowing up testing time on all -// platforms, just use Linux for now. C++ tests already have faster coverage for each type -// of load. -test('sharded model can run inference end-to-end', { timeout: 4 * 60 * 1000, skip: !isLinuxX64 }, async t => { +// This test can take longer to download and execute. Keep it on desktop x64 +// runners where the sharded loader has CI coverage. +test('sharded model can run inference end-to-end', { timeout: 4 * 60 * 1000, skip: !(isLinuxX64 || isWindowsX64) }, async t => { const fs = require('bare-fs') const modelDir = path.resolve(__dirname, '../model') fs.mkdirSync(modelDir, { recursive: true }) diff --git a/packages/llm-llamacpp/test/integration/multi-instance.test.js b/packages/llm-llamacpp/test/integration/multi-instance.test.js index 839868fcdf..1d958cbdbf 100644 --- a/packages/llm-llamacpp/test/integration/multi-instance.test.js +++ b/packages/llm-llamacpp/test/integration/multi-instance.test.js @@ -9,9 +9,8 @@ const platform = os.platform() const arch = os.arch() const isDarwinX64 = platform === 'darwin' && arch === 'x64' const isLinuxArm64 = platform === 'linux' && arch === 'arm64' -const isWindowsX64 = platform === 'win32' && arch === 'x64' const useCpu = isDarwinX64 || isLinuxArm64 -const skip = isDarwinX64 || isWindowsX64 +const skip = isDarwinX64 const DEFAULT_MODEL = { name: 'Llama-3.2-1B-Instruct-Q4_0.gguf', @@ -74,7 +73,7 @@ async function collectResponse (response) { safeTest('Two instances can run inference simultaneously', { timeout: 900_000, - skip // TODO: unskip Windows once we have a new Windows runner with a GPU + skip }, async t => { let addon1 = null let addon2 = null @@ -108,7 +107,7 @@ safeTest('Two instances can run inference simultaneously', { safeTest('Repeated load/unload cycles should remain stable', { timeout: 900_000, - skip // TODO: unskip Windows once we have a new Windows runner with a GPU + skip }, async t => { let currentAddon = null try { @@ -143,7 +142,7 @@ safeTest('Repeated load/unload cycles should remain stable', { safeTest('Unloading one instance does not affect another generating instance', { timeout: 900_000, - skip // TODO: unskip Windows once we have a new Windows runner with a GPU + skip }, async t => { let addon1 = null let addon2 = null @@ -205,7 +204,7 @@ safeTest('Unloading one instance does not affect another generating instance', { safeTest('Multiple load/unload cycles on one instance while another generates', { timeout: 900_000, - skip // TODO: unskip Windows once we have a new Windows runner with a GPU + skip }, async t => { let addon1 = null try { diff --git a/packages/llm-llamacpp/test/integration/reasoning.test.js b/packages/llm-llamacpp/test/integration/reasoning.test.js index 7d4e540df3..b421ef3313 100644 --- a/packages/llm-llamacpp/test/integration/reasoning.test.js +++ b/packages/llm-llamacpp/test/integration/reasoning.test.js @@ -7,7 +7,6 @@ const os = require('bare-os') const LlmLlamacpp = require('../../index.js') const isDarwinX64 = os.platform() === 'darwin' && os.arch() === 'x64' -const isWindowsX64 = os.platform() === 'win32' && os.arch() === 'x64' const isLinuxArm64 = os.platform() === 'linux' && os.arch() === 'arm64' const useCpu = isLinuxArm64 @@ -126,7 +125,7 @@ function createFollowUpMessages (initialMessages, previousResponse) { ] } safeTest('reasoning tag EOS replacement works with tools=false', { - skip: isDarwinX64 || isWindowsX64, // TODO: unskip isWindowsX64 once we have GPU, takes too long + skip: isDarwinX64, timeout: 600_000 }, async t => { const { inference } = await setupReasoningModel(t, false) @@ -149,7 +148,7 @@ safeTest('reasoning tag EOS replacement works with tools=false', { }) safeTest('reasoning tag EOS replacement works with tools=true', { - skip: isDarwinX64 || isWindowsX64, // TODO: unskip isWindowsX64 once we have GPU, takes too long + skip: isDarwinX64, timeout: 600_000 }, async t => { const { inference } = await setupReasoningModel(t, true) @@ -172,7 +171,7 @@ safeTest('reasoning tag EOS replacement works with tools=true', { }) safeTest('Qwen3 reasoning-budget=0 disables thinking', { - skip: isDarwinX64 || isWindowsX64, + skip: isDarwinX64, timeout: 600_000 }, async t => { const [modelName, dirPath] = await ensureModel({ diff --git a/packages/llm-llamacpp/test/integration/sliding-context.test.js b/packages/llm-llamacpp/test/integration/sliding-context.test.js index a39366e296..ca4b3d3e91 100644 --- a/packages/llm-llamacpp/test/integration/sliding-context.test.js +++ b/packages/llm-llamacpp/test/integration/sliding-context.test.js @@ -13,8 +13,7 @@ const isWindowsX64 = platform === 'win32' && arch === 'x64' const useCpu = isDarwinX64 || isLinuxArm64 // These are very slow on CI and should be skipped. -// TODO: unskip Windows once we have a new Windows runner with a GPU -const skip = isDarwinX64 || isWindowsX64 || isLinuxArm64 +const skip = isDarwinX64 || isLinuxArm64 const DEFAULT_MODEL = { name: 'Llama-3.2-1B-Instruct-Q4_0.gguf', From 6bd5ae503d7e651af5285b417a27837605a16dec Mon Sep 17 00:00:00 2001 From: gianni-cor Date: Fri, 19 Jun 2026 21:06:13 +0200 Subject: [PATCH 44/58] test[notask]: remove unused reasoning run options Assisted-by: GPT-5.5 Co-authored-by: Cursor --- packages/llm-llamacpp/test/integration/reasoning.test.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/llm-llamacpp/test/integration/reasoning.test.js b/packages/llm-llamacpp/test/integration/reasoning.test.js index b421ef3313..909cc41204 100644 --- a/packages/llm-llamacpp/test/integration/reasoning.test.js +++ b/packages/llm-llamacpp/test/integration/reasoning.test.js @@ -58,8 +58,8 @@ async function setupReasoningModel (t, toolsEnabled) { } // Shared helper: Run a completion and collect response -async function runCompletion (inference, messages, runOptions) { - const result = await inference.run(messages, runOptions) +async function runCompletion (inference, messages) { + const result = await inference.run(messages) let response = '' await result .onUpdate(token => { @@ -191,7 +191,7 @@ safeTest('Qwen3 reasoning-budget=0 disables thinking', { verbosity: '0' } - async function runOnce (extra, runOptions) { + async function runOnce (extra) { const inference = new LlmLlamacpp({ files: { model: [modelPath] }, config: { ...baseConfig, ...extra }, @@ -203,7 +203,7 @@ safeTest('Qwen3 reasoning-budget=0 disables thinking', { { role: 'system', content: 'You are a helpful assistant.' }, { role: 'user', content: 'What is the capital of France? Answer in one word.' } ] - return await runCompletion(inference, messages, runOptions) + return await runCompletion(inference, messages) } finally { await inference.unload().catch(() => {}) } From 4006ad4f27d1701139e894b081032c24320c2da2 Mon Sep 17 00:00:00 2001 From: gianni-cor Date: Fri, 19 Jun 2026 23:44:57 +0200 Subject: [PATCH 45/58] deps[notask]: update qvac-fabric OpenCL copy fix Assisted-by: GPT-5.5 Co-authored-by: Cursor --- packages/llm-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/llm-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake b/packages/llm-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake index 5ad0cbeccb..230a107327 100644 --- a/packages/llm-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake +++ b/packages/llm-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake @@ -1,8 +1,8 @@ vcpkg_from_github( OUT_SOURCE_PATH SOURCE_PATH REPO tetherto/qvac-fabric-llm.cpp - REF 1e285f9df65933e8f3ffcdfd85b7f69eb9c0cbf1 - SHA512 d6bce10d288131231ad06e64052a06a6e8dae1f758f852598ec4f19197c593cde0b1c971f51b7adc8648080330b868267bf951351f9f345781fd1ccf50cc12d8 + REF 3c349dfa44802a6ea7b13208073c083365d70ed9 + SHA512 f0ba8bb4fcf50d9dc149d6f60efd52c37b8bda041f302212c80807d3a31ddb5b78a26a2d76de4fc2dc20fbfb4c29834b191d0f3155c9e753dea9c3ee1a2755f3 HEAD_REF temp-9341 ) From f5250c3c4be6a673552e8ee2f54330cf6c4d8575 Mon Sep 17 00:00:00 2001 From: gianni-cor Date: Fri, 19 Jun 2026 23:50:17 +0200 Subject: [PATCH 46/58] deps[notask]: align qvac-fabric ports Assisted-by: GPT-5.5 Co-authored-by: Cursor --- .../vcpkg/ports/qvac-fabric/portfile.cmake | 4 ++-- .../embed-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake | 4 ++-- .../translation-nmtcpp/vcpkg/ports/qvac-fabric/portfile.cmake | 4 ++-- packages/vla-ggml/vcpkg/ports/qvac-fabric/portfile.cmake | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/packages/classification-ggml/vcpkg/ports/qvac-fabric/portfile.cmake b/packages/classification-ggml/vcpkg/ports/qvac-fabric/portfile.cmake index ead537a0ce..230a107327 100644 --- a/packages/classification-ggml/vcpkg/ports/qvac-fabric/portfile.cmake +++ b/packages/classification-ggml/vcpkg/ports/qvac-fabric/portfile.cmake @@ -1,8 +1,8 @@ vcpkg_from_github( OUT_SOURCE_PATH SOURCE_PATH REPO tetherto/qvac-fabric-llm.cpp - REF 6f37291fe979c8f9a66a4c85809f69ff0ebeb743 - SHA512 6e29daed48978cac5fad4f2ed96f6518f0a81eef7a789d29d44196fb7a2a1b8fb7ed75fdcfc4acd370279a6b0b500c75517b79115dd88b2fbee2882e3a5d720d + REF 3c349dfa44802a6ea7b13208073c083365d70ed9 + SHA512 f0ba8bb4fcf50d9dc149d6f60efd52c37b8bda041f302212c80807d3a31ddb5b78a26a2d76de4fc2dc20fbfb4c29834b191d0f3155c9e753dea9c3ee1a2755f3 HEAD_REF temp-9341 ) diff --git a/packages/embed-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake b/packages/embed-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake index ead537a0ce..230a107327 100644 --- a/packages/embed-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake +++ b/packages/embed-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake @@ -1,8 +1,8 @@ vcpkg_from_github( OUT_SOURCE_PATH SOURCE_PATH REPO tetherto/qvac-fabric-llm.cpp - REF 6f37291fe979c8f9a66a4c85809f69ff0ebeb743 - SHA512 6e29daed48978cac5fad4f2ed96f6518f0a81eef7a789d29d44196fb7a2a1b8fb7ed75fdcfc4acd370279a6b0b500c75517b79115dd88b2fbee2882e3a5d720d + REF 3c349dfa44802a6ea7b13208073c083365d70ed9 + SHA512 f0ba8bb4fcf50d9dc149d6f60efd52c37b8bda041f302212c80807d3a31ddb5b78a26a2d76de4fc2dc20fbfb4c29834b191d0f3155c9e753dea9c3ee1a2755f3 HEAD_REF temp-9341 ) diff --git a/packages/translation-nmtcpp/vcpkg/ports/qvac-fabric/portfile.cmake b/packages/translation-nmtcpp/vcpkg/ports/qvac-fabric/portfile.cmake index ead537a0ce..230a107327 100644 --- a/packages/translation-nmtcpp/vcpkg/ports/qvac-fabric/portfile.cmake +++ b/packages/translation-nmtcpp/vcpkg/ports/qvac-fabric/portfile.cmake @@ -1,8 +1,8 @@ vcpkg_from_github( OUT_SOURCE_PATH SOURCE_PATH REPO tetherto/qvac-fabric-llm.cpp - REF 6f37291fe979c8f9a66a4c85809f69ff0ebeb743 - SHA512 6e29daed48978cac5fad4f2ed96f6518f0a81eef7a789d29d44196fb7a2a1b8fb7ed75fdcfc4acd370279a6b0b500c75517b79115dd88b2fbee2882e3a5d720d + REF 3c349dfa44802a6ea7b13208073c083365d70ed9 + SHA512 f0ba8bb4fcf50d9dc149d6f60efd52c37b8bda041f302212c80807d3a31ddb5b78a26a2d76de4fc2dc20fbfb4c29834b191d0f3155c9e753dea9c3ee1a2755f3 HEAD_REF temp-9341 ) diff --git a/packages/vla-ggml/vcpkg/ports/qvac-fabric/portfile.cmake b/packages/vla-ggml/vcpkg/ports/qvac-fabric/portfile.cmake index ead537a0ce..230a107327 100644 --- a/packages/vla-ggml/vcpkg/ports/qvac-fabric/portfile.cmake +++ b/packages/vla-ggml/vcpkg/ports/qvac-fabric/portfile.cmake @@ -1,8 +1,8 @@ vcpkg_from_github( OUT_SOURCE_PATH SOURCE_PATH REPO tetherto/qvac-fabric-llm.cpp - REF 6f37291fe979c8f9a66a4c85809f69ff0ebeb743 - SHA512 6e29daed48978cac5fad4f2ed96f6518f0a81eef7a789d29d44196fb7a2a1b8fb7ed75fdcfc4acd370279a6b0b500c75517b79115dd88b2fbee2882e3a5d720d + REF 3c349dfa44802a6ea7b13208073c083365d70ed9 + SHA512 f0ba8bb4fcf50d9dc149d6f60efd52c37b8bda041f302212c80807d3a31ddb5b78a26a2d76de4fc2dc20fbfb4c29834b191d0f3155c9e753dea9c3ee1a2755f3 HEAD_REF temp-9341 ) From 19b23eb8e9fe133610fdb23794b4ad188b66df10 Mon Sep 17 00:00:00 2001 From: gianni-cor Date: Fri, 19 Jun 2026 23:57:45 +0200 Subject: [PATCH 47/58] test[notask]: restore macos unit test sanitizer Assisted-by: GPT-5.5 Co-authored-by: Cursor --- packages/llm-llamacpp/test/unit/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/llm-llamacpp/test/unit/CMakeLists.txt b/packages/llm-llamacpp/test/unit/CMakeLists.txt index 489fe57f40..0d04bd7906 100644 --- a/packages/llm-llamacpp/test/unit/CMakeLists.txt +++ b/packages/llm-llamacpp/test/unit/CMakeLists.txt @@ -130,7 +130,7 @@ target_link_libraries( GTest::gmock_main ) -if(NOT WIN32 AND NOT APPLE) +if(NOT WIN32) target_link_libraries( addon-test PRIVATE From f0cd76d31521fd06fc9b15d7596557a334d84432 Mon Sep 17 00:00:00 2001 From: gianni-cor Date: Sat, 20 Jun 2026 08:57:47 +0200 Subject: [PATCH 48/58] deps[notask]: update qvac-fabric OpenCL cache fix Assisted-by: GPT-5.5 Co-authored-by: Cursor --- .../vcpkg/ports/qvac-fabric/portfile.cmake | 4 ++-- .../embed-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake | 4 ++-- packages/llm-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake | 4 ++-- .../translation-nmtcpp/vcpkg/ports/qvac-fabric/portfile.cmake | 4 ++-- packages/vla-ggml/vcpkg/ports/qvac-fabric/portfile.cmake | 4 ++-- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/packages/classification-ggml/vcpkg/ports/qvac-fabric/portfile.cmake b/packages/classification-ggml/vcpkg/ports/qvac-fabric/portfile.cmake index 230a107327..87df6ad118 100644 --- a/packages/classification-ggml/vcpkg/ports/qvac-fabric/portfile.cmake +++ b/packages/classification-ggml/vcpkg/ports/qvac-fabric/portfile.cmake @@ -1,8 +1,8 @@ vcpkg_from_github( OUT_SOURCE_PATH SOURCE_PATH REPO tetherto/qvac-fabric-llm.cpp - REF 3c349dfa44802a6ea7b13208073c083365d70ed9 - SHA512 f0ba8bb4fcf50d9dc149d6f60efd52c37b8bda041f302212c80807d3a31ddb5b78a26a2d76de4fc2dc20fbfb4c29834b191d0f3155c9e753dea9c3ee1a2755f3 + REF 3f4e89f04f9d3f0105ad25c5039e2fff2130a1d8 + SHA512 d813a3be89c4c05a0761f68da4d41a9515d3bb07b1adfd3b9ff27cfd92d8c400aabdf07fa30ba07b82c82354d06cdbc2b89f0574f4c6466344fccba6e18d1e42 HEAD_REF temp-9341 ) diff --git a/packages/embed-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake b/packages/embed-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake index 230a107327..87df6ad118 100644 --- a/packages/embed-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake +++ b/packages/embed-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake @@ -1,8 +1,8 @@ vcpkg_from_github( OUT_SOURCE_PATH SOURCE_PATH REPO tetherto/qvac-fabric-llm.cpp - REF 3c349dfa44802a6ea7b13208073c083365d70ed9 - SHA512 f0ba8bb4fcf50d9dc149d6f60efd52c37b8bda041f302212c80807d3a31ddb5b78a26a2d76de4fc2dc20fbfb4c29834b191d0f3155c9e753dea9c3ee1a2755f3 + REF 3f4e89f04f9d3f0105ad25c5039e2fff2130a1d8 + SHA512 d813a3be89c4c05a0761f68da4d41a9515d3bb07b1adfd3b9ff27cfd92d8c400aabdf07fa30ba07b82c82354d06cdbc2b89f0574f4c6466344fccba6e18d1e42 HEAD_REF temp-9341 ) diff --git a/packages/llm-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake b/packages/llm-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake index 230a107327..87df6ad118 100644 --- a/packages/llm-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake +++ b/packages/llm-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake @@ -1,8 +1,8 @@ vcpkg_from_github( OUT_SOURCE_PATH SOURCE_PATH REPO tetherto/qvac-fabric-llm.cpp - REF 3c349dfa44802a6ea7b13208073c083365d70ed9 - SHA512 f0ba8bb4fcf50d9dc149d6f60efd52c37b8bda041f302212c80807d3a31ddb5b78a26a2d76de4fc2dc20fbfb4c29834b191d0f3155c9e753dea9c3ee1a2755f3 + REF 3f4e89f04f9d3f0105ad25c5039e2fff2130a1d8 + SHA512 d813a3be89c4c05a0761f68da4d41a9515d3bb07b1adfd3b9ff27cfd92d8c400aabdf07fa30ba07b82c82354d06cdbc2b89f0574f4c6466344fccba6e18d1e42 HEAD_REF temp-9341 ) diff --git a/packages/translation-nmtcpp/vcpkg/ports/qvac-fabric/portfile.cmake b/packages/translation-nmtcpp/vcpkg/ports/qvac-fabric/portfile.cmake index 230a107327..87df6ad118 100644 --- a/packages/translation-nmtcpp/vcpkg/ports/qvac-fabric/portfile.cmake +++ b/packages/translation-nmtcpp/vcpkg/ports/qvac-fabric/portfile.cmake @@ -1,8 +1,8 @@ vcpkg_from_github( OUT_SOURCE_PATH SOURCE_PATH REPO tetherto/qvac-fabric-llm.cpp - REF 3c349dfa44802a6ea7b13208073c083365d70ed9 - SHA512 f0ba8bb4fcf50d9dc149d6f60efd52c37b8bda041f302212c80807d3a31ddb5b78a26a2d76de4fc2dc20fbfb4c29834b191d0f3155c9e753dea9c3ee1a2755f3 + REF 3f4e89f04f9d3f0105ad25c5039e2fff2130a1d8 + SHA512 d813a3be89c4c05a0761f68da4d41a9515d3bb07b1adfd3b9ff27cfd92d8c400aabdf07fa30ba07b82c82354d06cdbc2b89f0574f4c6466344fccba6e18d1e42 HEAD_REF temp-9341 ) diff --git a/packages/vla-ggml/vcpkg/ports/qvac-fabric/portfile.cmake b/packages/vla-ggml/vcpkg/ports/qvac-fabric/portfile.cmake index 230a107327..87df6ad118 100644 --- a/packages/vla-ggml/vcpkg/ports/qvac-fabric/portfile.cmake +++ b/packages/vla-ggml/vcpkg/ports/qvac-fabric/portfile.cmake @@ -1,8 +1,8 @@ vcpkg_from_github( OUT_SOURCE_PATH SOURCE_PATH REPO tetherto/qvac-fabric-llm.cpp - REF 3c349dfa44802a6ea7b13208073c083365d70ed9 - SHA512 f0ba8bb4fcf50d9dc149d6f60efd52c37b8bda041f302212c80807d3a31ddb5b78a26a2d76de4fc2dc20fbfb4c29834b191d0f3155c9e753dea9c3ee1a2755f3 + REF 3f4e89f04f9d3f0105ad25c5039e2fff2130a1d8 + SHA512 d813a3be89c4c05a0761f68da4d41a9515d3bb07b1adfd3b9ff27cfd92d8c400aabdf07fa30ba07b82c82354d06cdbc2b89f0574f4c6466344fccba6e18d1e42 HEAD_REF temp-9341 ) From 09755affd2aed4bac9b1c4d0da14c76b2087d32b Mon Sep 17 00:00:00 2001 From: gianni-cor Date: Sat, 20 Jun 2026 10:56:03 +0200 Subject: [PATCH 49/58] test[notask]: skip q8 mrope cache on android Assisted-by: GPT-5.5 Co-authored-by: Cursor --- .../test/integration/mrope-sliding-context.test.js | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/llm-llamacpp/test/integration/mrope-sliding-context.test.js b/packages/llm-llamacpp/test/integration/mrope-sliding-context.test.js index 3084b2b211..4d8567cd53 100644 --- a/packages/llm-llamacpp/test/integration/mrope-sliding-context.test.js +++ b/packages/llm-llamacpp/test/integration/mrope-sliding-context.test.js @@ -15,6 +15,7 @@ const isDarwinX64 = platform === 'darwin' && arch === 'x64' const isLinuxArm64 = platform === 'linux' && arch === 'arm64' const useCpu = isDarwinX64 || isLinuxArm64 const skipTbqPq = isDarwin || isIos || isAndroid +const skipQ8KCache = isAndroid function safeTest (name, opts, fn) { integrationTest(name, { ...opts, skip: opts.skip || isDarwinX64 }, fn) @@ -384,14 +385,14 @@ safeTest('[qwen3.5-imrope-sliding-context] multimodal cache survives sliding sav }) safeTest('[qwen3.5-imrope-sliding-context] q8 K-cache shifts multimodal and text tokens', { - timeout: 1_800_000 + timeout: 1_800_000, + skip: skipQ8KCache }, async t => { await runMultimodalSlidingCacheCase(t, { label: 'q8 K-cache multimodal', cacheFileName: 'qwen3-5-q8-kcache-multimodal-sliding-cache.bin', extraConfig: { - 'cache-type-k': 'q8_0', - 'cache-type-v': 'q8_0' + 'cache-type-k': 'q8_0' } }) }) From 26ab6c377fe2f58355fd425f70ae4f1a57baa8b5 Mon Sep 17 00:00:00 2001 From: gianni-cor Date: Sat, 20 Jun 2026 11:01:58 +0200 Subject: [PATCH 50/58] deps[notask]: point qvac-fabric ports at temp branch Assisted-by: GPT-5.5 Co-authored-by: Cursor --- .../vcpkg/ports/qvac-fabric/portfile.cmake | 4 ++-- .../embed-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake | 4 ++-- packages/llm-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake | 4 ++-- .../translation-nmtcpp/vcpkg/ports/qvac-fabric/portfile.cmake | 4 ++-- packages/vla-ggml/vcpkg/ports/qvac-fabric/portfile.cmake | 4 ++-- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/packages/classification-ggml/vcpkg/ports/qvac-fabric/portfile.cmake b/packages/classification-ggml/vcpkg/ports/qvac-fabric/portfile.cmake index 87df6ad118..ead537a0ce 100644 --- a/packages/classification-ggml/vcpkg/ports/qvac-fabric/portfile.cmake +++ b/packages/classification-ggml/vcpkg/ports/qvac-fabric/portfile.cmake @@ -1,8 +1,8 @@ vcpkg_from_github( OUT_SOURCE_PATH SOURCE_PATH REPO tetherto/qvac-fabric-llm.cpp - REF 3f4e89f04f9d3f0105ad25c5039e2fff2130a1d8 - SHA512 d813a3be89c4c05a0761f68da4d41a9515d3bb07b1adfd3b9ff27cfd92d8c400aabdf07fa30ba07b82c82354d06cdbc2b89f0574f4c6466344fccba6e18d1e42 + REF 6f37291fe979c8f9a66a4c85809f69ff0ebeb743 + SHA512 6e29daed48978cac5fad4f2ed96f6518f0a81eef7a789d29d44196fb7a2a1b8fb7ed75fdcfc4acd370279a6b0b500c75517b79115dd88b2fbee2882e3a5d720d HEAD_REF temp-9341 ) diff --git a/packages/embed-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake b/packages/embed-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake index 87df6ad118..ead537a0ce 100644 --- a/packages/embed-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake +++ b/packages/embed-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake @@ -1,8 +1,8 @@ vcpkg_from_github( OUT_SOURCE_PATH SOURCE_PATH REPO tetherto/qvac-fabric-llm.cpp - REF 3f4e89f04f9d3f0105ad25c5039e2fff2130a1d8 - SHA512 d813a3be89c4c05a0761f68da4d41a9515d3bb07b1adfd3b9ff27cfd92d8c400aabdf07fa30ba07b82c82354d06cdbc2b89f0574f4c6466344fccba6e18d1e42 + REF 6f37291fe979c8f9a66a4c85809f69ff0ebeb743 + SHA512 6e29daed48978cac5fad4f2ed96f6518f0a81eef7a789d29d44196fb7a2a1b8fb7ed75fdcfc4acd370279a6b0b500c75517b79115dd88b2fbee2882e3a5d720d HEAD_REF temp-9341 ) diff --git a/packages/llm-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake b/packages/llm-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake index 87df6ad118..ead537a0ce 100644 --- a/packages/llm-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake +++ b/packages/llm-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake @@ -1,8 +1,8 @@ vcpkg_from_github( OUT_SOURCE_PATH SOURCE_PATH REPO tetherto/qvac-fabric-llm.cpp - REF 3f4e89f04f9d3f0105ad25c5039e2fff2130a1d8 - SHA512 d813a3be89c4c05a0761f68da4d41a9515d3bb07b1adfd3b9ff27cfd92d8c400aabdf07fa30ba07b82c82354d06cdbc2b89f0574f4c6466344fccba6e18d1e42 + REF 6f37291fe979c8f9a66a4c85809f69ff0ebeb743 + SHA512 6e29daed48978cac5fad4f2ed96f6518f0a81eef7a789d29d44196fb7a2a1b8fb7ed75fdcfc4acd370279a6b0b500c75517b79115dd88b2fbee2882e3a5d720d HEAD_REF temp-9341 ) diff --git a/packages/translation-nmtcpp/vcpkg/ports/qvac-fabric/portfile.cmake b/packages/translation-nmtcpp/vcpkg/ports/qvac-fabric/portfile.cmake index 87df6ad118..ead537a0ce 100644 --- a/packages/translation-nmtcpp/vcpkg/ports/qvac-fabric/portfile.cmake +++ b/packages/translation-nmtcpp/vcpkg/ports/qvac-fabric/portfile.cmake @@ -1,8 +1,8 @@ vcpkg_from_github( OUT_SOURCE_PATH SOURCE_PATH REPO tetherto/qvac-fabric-llm.cpp - REF 3f4e89f04f9d3f0105ad25c5039e2fff2130a1d8 - SHA512 d813a3be89c4c05a0761f68da4d41a9515d3bb07b1adfd3b9ff27cfd92d8c400aabdf07fa30ba07b82c82354d06cdbc2b89f0574f4c6466344fccba6e18d1e42 + REF 6f37291fe979c8f9a66a4c85809f69ff0ebeb743 + SHA512 6e29daed48978cac5fad4f2ed96f6518f0a81eef7a789d29d44196fb7a2a1b8fb7ed75fdcfc4acd370279a6b0b500c75517b79115dd88b2fbee2882e3a5d720d HEAD_REF temp-9341 ) diff --git a/packages/vla-ggml/vcpkg/ports/qvac-fabric/portfile.cmake b/packages/vla-ggml/vcpkg/ports/qvac-fabric/portfile.cmake index 87df6ad118..ead537a0ce 100644 --- a/packages/vla-ggml/vcpkg/ports/qvac-fabric/portfile.cmake +++ b/packages/vla-ggml/vcpkg/ports/qvac-fabric/portfile.cmake @@ -1,8 +1,8 @@ vcpkg_from_github( OUT_SOURCE_PATH SOURCE_PATH REPO tetherto/qvac-fabric-llm.cpp - REF 3f4e89f04f9d3f0105ad25c5039e2fff2130a1d8 - SHA512 d813a3be89c4c05a0761f68da4d41a9515d3bb07b1adfd3b9ff27cfd92d8c400aabdf07fa30ba07b82c82354d06cdbc2b89f0574f4c6466344fccba6e18d1e42 + REF 6f37291fe979c8f9a66a4c85809f69ff0ebeb743 + SHA512 6e29daed48978cac5fad4f2ed96f6518f0a81eef7a789d29d44196fb7a2a1b8fb7ed75fdcfc4acd370279a6b0b500c75517b79115dd88b2fbee2882e3a5d720d HEAD_REF temp-9341 ) From b19931a1290d3be74324ba1aefc2f2406e8c1ec1 Mon Sep 17 00:00:00 2001 From: gianni-cor Date: Sat, 20 Jun 2026 11:13:50 +0200 Subject: [PATCH 51/58] test[notask]: simplify android q8 mrope skip Assisted-by: GPT-5.5 Co-authored-by: Cursor --- .../test/integration/mrope-sliding-context.test.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/llm-llamacpp/test/integration/mrope-sliding-context.test.js b/packages/llm-llamacpp/test/integration/mrope-sliding-context.test.js index 4d8567cd53..ae9a8730ea 100644 --- a/packages/llm-llamacpp/test/integration/mrope-sliding-context.test.js +++ b/packages/llm-llamacpp/test/integration/mrope-sliding-context.test.js @@ -15,7 +15,6 @@ const isDarwinX64 = platform === 'darwin' && arch === 'x64' const isLinuxArm64 = platform === 'linux' && arch === 'arm64' const useCpu = isDarwinX64 || isLinuxArm64 const skipTbqPq = isDarwin || isIos || isAndroid -const skipQ8KCache = isAndroid function safeTest (name, opts, fn) { integrationTest(name, { ...opts, skip: opts.skip || isDarwinX64 }, fn) @@ -386,7 +385,7 @@ safeTest('[qwen3.5-imrope-sliding-context] multimodal cache survives sliding sav safeTest('[qwen3.5-imrope-sliding-context] q8 K-cache shifts multimodal and text tokens', { timeout: 1_800_000, - skip: skipQ8KCache + skip: isAndroid }, async t => { await runMultimodalSlidingCacheCase(t, { label: 'q8 K-cache multimodal', From 4746a9d423ee5b54f9cea90f3c4d19e254629f71 Mon Sep 17 00:00:00 2001 From: gianni-cor Date: Sat, 20 Jun 2026 12:18:12 +0200 Subject: [PATCH 52/58] test[notask]: force sliding overflow on windows Assisted-by: GPT-5.5 Co-authored-by: Cursor --- packages/llm-llamacpp/test/integration/sliding-context.test.js | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/llm-llamacpp/test/integration/sliding-context.test.js b/packages/llm-llamacpp/test/integration/sliding-context.test.js index ca4b3d3e91..76854fedd5 100644 --- a/packages/llm-llamacpp/test/integration/sliding-context.test.js +++ b/packages/llm-llamacpp/test/integration/sliding-context.test.js @@ -145,6 +145,7 @@ safeTest('Generation fails with context overflow when sliding disabled', { skip }, async t => { const { model } = await setupModel(t, { + ctx_size: '256', n_predict: String(SLIDE_PREDICT), n_discarded: '0' }) From bcce0d99ebe2b5b915995c6168155f696d651243 Mon Sep 17 00:00:00 2001 From: gianni-cor Date: Mon, 22 Jun 2026 11:01:46 +0200 Subject: [PATCH 53/58] deps[notask]: remove qvac-fabric overlay ports Assisted-by: GPT-5.5 Co-authored-by: Cursor --- .../vcpkg-configuration.json | 3 - .../qvac-fabric/android-vulkan-version.cmake | 36 ---- .../vcpkg/ports/qvac-fabric/portfile.cmake | 191 ------------------ .../vcpkg/ports/qvac-fabric/vcpkg.json | 50 ----- .../embed-llamacpp/vcpkg-configuration.json | 3 - .../qvac-fabric/android-vulkan-version.cmake | 36 ---- .../vcpkg/ports/qvac-fabric/portfile.cmake | 191 ------------------ .../vcpkg/ports/qvac-fabric/vcpkg.json | 50 ----- .../llm-llamacpp/vcpkg-configuration.json | 3 - .../qvac-fabric/android-vulkan-version.cmake | 36 ---- .../vcpkg/ports/qvac-fabric/portfile.cmake | 191 ------------------ .../vcpkg/ports/qvac-fabric/vcpkg.json | 50 ----- .../vcpkg-configuration.json | 3 - .../qvac-fabric/android-vulkan-version.cmake | 36 ---- .../vcpkg/ports/qvac-fabric/portfile.cmake | 191 ------------------ .../vcpkg/ports/qvac-fabric/vcpkg.json | 50 ----- packages/vla-ggml/vcpkg-configuration.json | 3 - .../qvac-fabric/android-vulkan-version.cmake | 36 ---- .../vcpkg/ports/qvac-fabric/portfile.cmake | 191 ------------------ .../vcpkg/ports/qvac-fabric/vcpkg.json | 50 ----- 20 files changed, 1400 deletions(-) delete mode 100644 packages/classification-ggml/vcpkg/ports/qvac-fabric/android-vulkan-version.cmake delete mode 100644 packages/classification-ggml/vcpkg/ports/qvac-fabric/portfile.cmake delete mode 100644 packages/classification-ggml/vcpkg/ports/qvac-fabric/vcpkg.json delete mode 100644 packages/embed-llamacpp/vcpkg/ports/qvac-fabric/android-vulkan-version.cmake delete mode 100644 packages/embed-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake delete mode 100644 packages/embed-llamacpp/vcpkg/ports/qvac-fabric/vcpkg.json delete mode 100644 packages/llm-llamacpp/vcpkg/ports/qvac-fabric/android-vulkan-version.cmake delete mode 100644 packages/llm-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake delete mode 100644 packages/llm-llamacpp/vcpkg/ports/qvac-fabric/vcpkg.json delete mode 100644 packages/translation-nmtcpp/vcpkg/ports/qvac-fabric/android-vulkan-version.cmake delete mode 100644 packages/translation-nmtcpp/vcpkg/ports/qvac-fabric/portfile.cmake delete mode 100644 packages/translation-nmtcpp/vcpkg/ports/qvac-fabric/vcpkg.json delete mode 100644 packages/vla-ggml/vcpkg/ports/qvac-fabric/android-vulkan-version.cmake delete mode 100644 packages/vla-ggml/vcpkg/ports/qvac-fabric/portfile.cmake delete mode 100644 packages/vla-ggml/vcpkg/ports/qvac-fabric/vcpkg.json diff --git a/packages/classification-ggml/vcpkg-configuration.json b/packages/classification-ggml/vcpkg-configuration.json index 2cce8881ec..b685565a08 100644 --- a/packages/classification-ggml/vcpkg-configuration.json +++ b/packages/classification-ggml/vcpkg-configuration.json @@ -4,9 +4,6 @@ "baseline": "803c0d119ea002694963e89237c207ff6ecf47f6", "repository": "https://github.com/tetherto/qvac-registry-vcpkg.git" }, - "overlay-ports": [ - "vcpkg/ports" - ], "registries": [ { "kind": "git", diff --git a/packages/classification-ggml/vcpkg/ports/qvac-fabric/android-vulkan-version.cmake b/packages/classification-ggml/vcpkg/ports/qvac-fabric/android-vulkan-version.cmake deleted file mode 100644 index 21fba00feb..0000000000 --- a/packages/classification-ggml/vcpkg/ports/qvac-fabric/android-vulkan-version.cmake +++ /dev/null @@ -1,36 +0,0 @@ -# Function to detect Vulkan version from NDK vulkan_core.h -function(detect_ndk_vulkan_version) - string(TOLOWER "${CMAKE_HOST_SYSTEM_NAME}" host_system_name_lower) - - # CMAKE_HOST_SYSTEM_PROCESSOR is unavailable here. Use a glob pattern to complete the folder instead. - file(GLOB host_dirs LIST_DIRECTORIES true "$ENV{ANDROID_NDK_HOME}/toolchains/llvm/prebuilt/${host_system_name_lower}-*") - if(host_dirs) - list(GET host_dirs 0 host_dir) - get_filename_component(host_arch "${host_dir}" NAME) - set(vulkan_core_h "$ENV{ANDROID_NDK_HOME}/toolchains/llvm/prebuilt/${host_arch}/sysroot/usr/include/vulkan/vulkan_core.h") - else() - message(FATAL "Could not find NDK host directory for ${host_system_name_lower}") - endif() - - if(NOT vulkan_core_h) - message(FATAL "vulkan_core.h not found, using default version") - endif() - - file(READ "${vulkan_core_h}" header_content) - string(REGEX MATCH "VK_HEADER_VERSION ([0-9]+)" version_match "${header_content}") - if(version_match) - set(header_version_3 "${CMAKE_MATCH_1}") - else() - message(FATAL "Could not extract VK_HEADER_VERSION from vulkan_core.h, using default: ${vulkan_version}") - endif() - - # Extract major.minor version from VK_HEADER_VERSION_COMPLETE for download URL - string(REGEX MATCH "VK_HEADER_VERSION_COMPLETE VK_MAKE_API_VERSION\\(([0-9]+), ([0-9]+), ([0-9]+)" version_match "${header_content}") - if(version_match) - set(major "${CMAKE_MATCH_2}") - set(minor "${CMAKE_MATCH_3}") - set(vulkan_version "${major}.${minor}.${header_version_3}" PARENT_SCOPE) - else() - message(FATAL "Could not extract major.minor version from vulkan_core.h, using default: ${vulkan_version}") - endif() -endfunction() diff --git a/packages/classification-ggml/vcpkg/ports/qvac-fabric/portfile.cmake b/packages/classification-ggml/vcpkg/ports/qvac-fabric/portfile.cmake deleted file mode 100644 index ead537a0ce..0000000000 --- a/packages/classification-ggml/vcpkg/ports/qvac-fabric/portfile.cmake +++ /dev/null @@ -1,191 +0,0 @@ -vcpkg_from_github( - OUT_SOURCE_PATH SOURCE_PATH - REPO tetherto/qvac-fabric-llm.cpp - REF 6f37291fe979c8f9a66a4c85809f69ff0ebeb743 - SHA512 6e29daed48978cac5fad4f2ed96f6518f0a81eef7a789d29d44196fb7a2a1b8fb7ed75fdcfc4acd370279a6b0b500c75517b79115dd88b2fbee2882e3a5d720d - HEAD_REF temp-9341 -) - -# Upstream CMake options only — passed through to vcpkg_cmake_configure. -vcpkg_check_features( - OUT_FEATURE_OPTIONS FEATURE_OPTIONS - FEATURES - force-profiler FORCE_GGML_VK_PERF_LOGGER - llama BUILD_LLAMA -) - -# Portfile-only feature flags (drive PLATFORM_OPTIONS; not upstream cache vars). -vcpkg_check_features( - OUT_FEATURE_OPTIONS _PORTFILE_FEATURE_OPTIONS - FEATURES - gpu-backends BUILD_GPU_BACKENDS - kleidiai BUILD_KLEIDIAI - openmp BUILD_OPENMP -) - -# gpu-backends is default-on via default-features in vcpkg.json. CPU-only -# consumers (e.g. @qvac/classification-ggml) disable it with -# default-features:false (and re-add 'llama' if needed). -if(NOT BUILD_GPU_BACKENDS) - message(STATUS "qvac-fabric: gpu-backends feature OFF — building CPU-only ggml (no Metal/Vulkan/CUDA/OpenCL)") -endif() - -if (VCPKG_TARGET_IS_ANDROID AND BUILD_GPU_BACKENDS) - # NDK only comes with C headers. - # Make sure C++ header exists, it will be used by ggml tensor library. - # Need to determine installed vulkan version and download correct headers - include(${CMAKE_CURRENT_LIST_DIR}/android-vulkan-version.cmake) - detect_ndk_vulkan_version() - message(STATUS "Using Vulkan C++ wrappers from version: ${vulkan_version}") - file(DOWNLOAD - "https://github.com/KhronosGroup/Vulkan-Headers/archive/refs/tags/v${vulkan_version}.tar.gz" - "${SOURCE_PATH}/vulkan-sdk-${vulkan_version}.tar.gz" - TLS_VERIFY ON - ) - - file(ARCHIVE_EXTRACT - INPUT "${SOURCE_PATH}/vulkan-sdk-${vulkan_version}.tar.gz" - DESTINATION "${SOURCE_PATH}" - PATTERNS "*.hpp" - ) - - file(RENAME - "${SOURCE_PATH}/Vulkan-Headers-${vulkan_version}" - "${SOURCE_PATH}/ggml/src/ggml-vulkan/vulkan_cpp_wrapper" - ) -endif() - -set(PLATFORM_OPTIONS) - -if(NOT BUILD_GPU_BACKENDS) - # Force every GPU backend off explicitly, in case upstream defaults change. - list(APPEND PLATFORM_OPTIONS - -DGGML_METAL=OFF - -DGGML_VULKAN=OFF - -DGGML_CUDA=OFF - -DGGML_OPENCL=OFF - ) - if (VCPKG_TARGET_IS_IOS) - # Same iOS BLAS/Accelerate gating as the GPU-on path; unrelated to the - # CPU-vs-GPU split, an iOS-toolchain workaround for missing frameworks. - list(APPEND PLATFORM_OPTIONS -DGGML_BLAS=OFF -DGGML_ACCELERATE=OFF) - endif() -elseif (VCPKG_TARGET_IS_OSX OR VCPKG_TARGET_IS_IOS) - list(APPEND PLATFORM_OPTIONS -DGGML_METAL=ON) - if (VCPKG_TARGET_IS_IOS) - list(APPEND PLATFORM_OPTIONS -DGGML_BLAS=OFF -DGGML_ACCELERATE=OFF) - endif() -else() - list(APPEND PLATFORM_OPTIONS -DGGML_VULKAN=ON) -endif() - -# Android: always build CPU variants (NEON_DOTPROD, NEON_I8MM, etc.) and CPU -# repacking. These are CPU-only runtime optimizations selected based on the -# device's SIMD capabilities at load time, completely orthogonal to the GPU -# backends. Bundling them is essential for good CPU inference performance on -# the wide range of arm64 devices the addons ship to. Requires GGML_BACKEND_DL -# to dispatch the variants at runtime; the existing #ifdef guard around -# `ggml_backend_load_all_from_path()` in ggml-backend-reg.cpp keeps the search -# scoped to the consumer's own prebuilds dir. -if(VCPKG_TARGET_IS_ANDROID) - set(DL_BACKENDS ON) - list(APPEND PLATFORM_OPTIONS - -DGGML_BACKEND_DL=ON - -DGGML_CPU_ALL_VARIANTS=ON - -DGGML_CPU_REPACK=ON) -else() - set(DL_BACKENDS OFF) -endif() - -if(VCPKG_TARGET_IS_ANDROID AND BUILD_KLEIDIAI) - message(STATUS "qvac-fabric: kleidiai feature ON — building with ARM KleidiAI optimized kernels") - # ggml only vendors KleidiAI via FetchContent; registry vcpkg-cmake sets - # FETCHCONTENT_FULLY_DISCONNECTED=ON globally, so allow the download here. - list(APPEND PLATFORM_OPTIONS - -DGGML_CPU_KLEIDIAI=ON - -DFETCHCONTENT_FULLY_DISCONNECTED=OFF - ) -endif() - -if(VCPKG_TARGET_IS_ANDROID AND BUILD_OPENMP) - message(STATUS "qvac-fabric: OpenMP for Android enabled") - list(APPEND PLATFORM_OPTIONS -DGGML_OPENMP=ON) -else() - message(STATUS "qvac-fabric: OpenMP Disabled") - list(APPEND PLATFORM_OPTIONS -DGGML_OPENMP=OFF) -endif() - -if (VCPKG_TARGET_IS_ANDROID AND BUILD_GPU_BACKENDS) - list(APPEND PLATFORM_OPTIONS -DGGML_OPENCL=ON) -endif() - -if(BUILD_GPU_BACKENDS AND NOT VCPKG_TARGET_IS_OSX AND NOT VCPKG_TARGET_IS_IOS) - if(VCPKG_TARGET_IS_WINDOWS AND NOT VCPKG_TARGET_IS_MINGW) - string(APPEND VCPKG_C_FLAGS " /I${CURRENT_INSTALLED_DIR}/include") - string(APPEND VCPKG_CXX_FLAGS " /I${CURRENT_INSTALLED_DIR}/include") - else() - string(APPEND VCPKG_C_FLAGS " -isystem ${CURRENT_INSTALLED_DIR}/include") - string(APPEND VCPKG_CXX_FLAGS " -isystem ${CURRENT_INSTALLED_DIR}/include") - endif() -endif() - -set(LLAMA_OPTIONS) -if("llama" IN_LIST FEATURES) - list(APPEND LLAMA_OPTIONS -DLLAMA_MTMD=ON) -else() - list(APPEND LLAMA_OPTIONS - -DLLAMA_MTMD=OFF - -DLLAMA_BUILD_COMMON=OFF - ) -endif() - -vcpkg_cmake_configure( - SOURCE_PATH "${SOURCE_PATH}" - DISABLE_PARALLEL_CONFIGURE - OPTIONS - -DGGML_NATIVE=OFF - -DGGML_CCACHE=OFF - -DGGML_LLAMAFILE=OFF - -DLLAMA_CURL=OFF - -DLLAMA_BUILD_TESTS=OFF - -DLLAMA_BUILD_TOOLS=OFF - -DLLAMA_BUILD_EXAMPLES=OFF - -DLLAMA_BUILD_SERVER=OFF - -DLLAMA_ALL_WARNINGS=OFF - ${LLAMA_OPTIONS} - ${PLATFORM_OPTIONS} - ${FEATURE_OPTIONS} -) - -vcpkg_cmake_install() -vcpkg_cmake_config_fixup( - PACKAGE_NAME ggml) - -if(BUILD_LLAMA) - vcpkg_cmake_config_fixup(PACKAGE_NAME llama) -endif() - -vcpkg_copy_pdbs() -vcpkg_fixup_pkgconfig() - - -if(BUILD_LLAMA) - file(MAKE_DIRECTORY "${CURRENT_PACKAGES_DIR}/tools/${PORT}") - file(RENAME "${CURRENT_PACKAGES_DIR}/bin/convert_hf_to_gguf.py" "${CURRENT_PACKAGES_DIR}/tools/${PORT}/convert-hf-to-gguf.py") - file(INSTALL "${SOURCE_PATH}/gguf-py" DESTINATION "${CURRENT_PACKAGES_DIR}/tools/${PORT}") - file(RENAME "${CURRENT_PACKAGES_DIR}/bin/vulkan_profiling_analyzer.py" "${CURRENT_PACKAGES_DIR}/tools/${PORT}/vulkan_profiling_analyzer.py") -endif() - -if (NOT VCPKG_BUILD_TYPE) - file(REMOVE "${CURRENT_PACKAGES_DIR}/debug/bin/convert_hf_to_gguf.py") -endif() - -file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/include") -file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/share") - -if (VCPKG_LIBRARY_LINKAGE MATCHES "static") - file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/bin") - file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/bin") -endif() - -vcpkg_install_copyright(FILE_LIST "${SOURCE_PATH}/LICENSE") diff --git a/packages/classification-ggml/vcpkg/ports/qvac-fabric/vcpkg.json b/packages/classification-ggml/vcpkg/ports/qvac-fabric/vcpkg.json deleted file mode 100644 index 48e7b449b8..0000000000 --- a/packages/classification-ggml/vcpkg/ports/qvac-fabric/vcpkg.json +++ /dev/null @@ -1,50 +0,0 @@ -{ - "name": "qvac-fabric", - "version": "8828.0.2", - "port-version": 1, - "description": "LLM inference in C/C++", - "homepage": "https://github.com/tetherto/qvac-fabric-llm.cpp", - "license": "MIT", - "dependencies": [ - { - "name": "opencl", - "platform": "android" - }, - { - "name": "vcpkg-cmake", - "host": true - }, - { - "name": "vcpkg-cmake-config", - "host": true - } - ], - "default-features": [ - "gpu-backends", - "llama" - ], - "features": { - "force-profiler": { - "description": "Force vk performance logging in ggml" - }, - "gpu-backends": { - "description": "Build the GPU backends ggml ships per platform: Metal on Apple, Vulkan on Linux/Windows/Android, plus the Android backend-DL hybrid mode and OpenCL. Default-on so existing consumers (llamacpp-llm, llamacpp-embed, nmtcpp, diffusion-cpp) keep their current behaviour with default features. Disable to produce a CPU-only ggml build (useful for consumers like @qvac/classification-ggml that don't need GPU paths and want to skip the vulkan-sdk / metal / opencl build cost). Orthogonal to the 'llama' feature.", - "dependencies": [ - { - "name": "spirv-headers", - "platform": "!osx & !ios", - "version>=": "1.4.341.0" - } - ] - }, - "kleidiai": { - "description": "Enable ARM KleidiAI optimized kernels on Android." - }, - "llama": { - "description": "Build llama components." - }, - "openmp": { - "description": "Enable openmp on Android." - } - } -} diff --git a/packages/embed-llamacpp/vcpkg-configuration.json b/packages/embed-llamacpp/vcpkg-configuration.json index 37d72f7c86..e894485aaa 100644 --- a/packages/embed-llamacpp/vcpkg-configuration.json +++ b/packages/embed-llamacpp/vcpkg-configuration.json @@ -4,9 +4,6 @@ "baseline": "803c0d119ea002694963e89237c207ff6ecf47f6", "repository": "https://github.com/tetherto/qvac-registry-vcpkg.git" }, - "overlay-ports": [ - "vcpkg/ports" - ], "registries": [ { "kind": "git", diff --git a/packages/embed-llamacpp/vcpkg/ports/qvac-fabric/android-vulkan-version.cmake b/packages/embed-llamacpp/vcpkg/ports/qvac-fabric/android-vulkan-version.cmake deleted file mode 100644 index 21fba00feb..0000000000 --- a/packages/embed-llamacpp/vcpkg/ports/qvac-fabric/android-vulkan-version.cmake +++ /dev/null @@ -1,36 +0,0 @@ -# Function to detect Vulkan version from NDK vulkan_core.h -function(detect_ndk_vulkan_version) - string(TOLOWER "${CMAKE_HOST_SYSTEM_NAME}" host_system_name_lower) - - # CMAKE_HOST_SYSTEM_PROCESSOR is unavailable here. Use a glob pattern to complete the folder instead. - file(GLOB host_dirs LIST_DIRECTORIES true "$ENV{ANDROID_NDK_HOME}/toolchains/llvm/prebuilt/${host_system_name_lower}-*") - if(host_dirs) - list(GET host_dirs 0 host_dir) - get_filename_component(host_arch "${host_dir}" NAME) - set(vulkan_core_h "$ENV{ANDROID_NDK_HOME}/toolchains/llvm/prebuilt/${host_arch}/sysroot/usr/include/vulkan/vulkan_core.h") - else() - message(FATAL "Could not find NDK host directory for ${host_system_name_lower}") - endif() - - if(NOT vulkan_core_h) - message(FATAL "vulkan_core.h not found, using default version") - endif() - - file(READ "${vulkan_core_h}" header_content) - string(REGEX MATCH "VK_HEADER_VERSION ([0-9]+)" version_match "${header_content}") - if(version_match) - set(header_version_3 "${CMAKE_MATCH_1}") - else() - message(FATAL "Could not extract VK_HEADER_VERSION from vulkan_core.h, using default: ${vulkan_version}") - endif() - - # Extract major.minor version from VK_HEADER_VERSION_COMPLETE for download URL - string(REGEX MATCH "VK_HEADER_VERSION_COMPLETE VK_MAKE_API_VERSION\\(([0-9]+), ([0-9]+), ([0-9]+)" version_match "${header_content}") - if(version_match) - set(major "${CMAKE_MATCH_2}") - set(minor "${CMAKE_MATCH_3}") - set(vulkan_version "${major}.${minor}.${header_version_3}" PARENT_SCOPE) - else() - message(FATAL "Could not extract major.minor version from vulkan_core.h, using default: ${vulkan_version}") - endif() -endfunction() diff --git a/packages/embed-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake b/packages/embed-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake deleted file mode 100644 index ead537a0ce..0000000000 --- a/packages/embed-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake +++ /dev/null @@ -1,191 +0,0 @@ -vcpkg_from_github( - OUT_SOURCE_PATH SOURCE_PATH - REPO tetherto/qvac-fabric-llm.cpp - REF 6f37291fe979c8f9a66a4c85809f69ff0ebeb743 - SHA512 6e29daed48978cac5fad4f2ed96f6518f0a81eef7a789d29d44196fb7a2a1b8fb7ed75fdcfc4acd370279a6b0b500c75517b79115dd88b2fbee2882e3a5d720d - HEAD_REF temp-9341 -) - -# Upstream CMake options only — passed through to vcpkg_cmake_configure. -vcpkg_check_features( - OUT_FEATURE_OPTIONS FEATURE_OPTIONS - FEATURES - force-profiler FORCE_GGML_VK_PERF_LOGGER - llama BUILD_LLAMA -) - -# Portfile-only feature flags (drive PLATFORM_OPTIONS; not upstream cache vars). -vcpkg_check_features( - OUT_FEATURE_OPTIONS _PORTFILE_FEATURE_OPTIONS - FEATURES - gpu-backends BUILD_GPU_BACKENDS - kleidiai BUILD_KLEIDIAI - openmp BUILD_OPENMP -) - -# gpu-backends is default-on via default-features in vcpkg.json. CPU-only -# consumers (e.g. @qvac/classification-ggml) disable it with -# default-features:false (and re-add 'llama' if needed). -if(NOT BUILD_GPU_BACKENDS) - message(STATUS "qvac-fabric: gpu-backends feature OFF — building CPU-only ggml (no Metal/Vulkan/CUDA/OpenCL)") -endif() - -if (VCPKG_TARGET_IS_ANDROID AND BUILD_GPU_BACKENDS) - # NDK only comes with C headers. - # Make sure C++ header exists, it will be used by ggml tensor library. - # Need to determine installed vulkan version and download correct headers - include(${CMAKE_CURRENT_LIST_DIR}/android-vulkan-version.cmake) - detect_ndk_vulkan_version() - message(STATUS "Using Vulkan C++ wrappers from version: ${vulkan_version}") - file(DOWNLOAD - "https://github.com/KhronosGroup/Vulkan-Headers/archive/refs/tags/v${vulkan_version}.tar.gz" - "${SOURCE_PATH}/vulkan-sdk-${vulkan_version}.tar.gz" - TLS_VERIFY ON - ) - - file(ARCHIVE_EXTRACT - INPUT "${SOURCE_PATH}/vulkan-sdk-${vulkan_version}.tar.gz" - DESTINATION "${SOURCE_PATH}" - PATTERNS "*.hpp" - ) - - file(RENAME - "${SOURCE_PATH}/Vulkan-Headers-${vulkan_version}" - "${SOURCE_PATH}/ggml/src/ggml-vulkan/vulkan_cpp_wrapper" - ) -endif() - -set(PLATFORM_OPTIONS) - -if(NOT BUILD_GPU_BACKENDS) - # Force every GPU backend off explicitly, in case upstream defaults change. - list(APPEND PLATFORM_OPTIONS - -DGGML_METAL=OFF - -DGGML_VULKAN=OFF - -DGGML_CUDA=OFF - -DGGML_OPENCL=OFF - ) - if (VCPKG_TARGET_IS_IOS) - # Same iOS BLAS/Accelerate gating as the GPU-on path; unrelated to the - # CPU-vs-GPU split, an iOS-toolchain workaround for missing frameworks. - list(APPEND PLATFORM_OPTIONS -DGGML_BLAS=OFF -DGGML_ACCELERATE=OFF) - endif() -elseif (VCPKG_TARGET_IS_OSX OR VCPKG_TARGET_IS_IOS) - list(APPEND PLATFORM_OPTIONS -DGGML_METAL=ON) - if (VCPKG_TARGET_IS_IOS) - list(APPEND PLATFORM_OPTIONS -DGGML_BLAS=OFF -DGGML_ACCELERATE=OFF) - endif() -else() - list(APPEND PLATFORM_OPTIONS -DGGML_VULKAN=ON) -endif() - -# Android: always build CPU variants (NEON_DOTPROD, NEON_I8MM, etc.) and CPU -# repacking. These are CPU-only runtime optimizations selected based on the -# device's SIMD capabilities at load time, completely orthogonal to the GPU -# backends. Bundling them is essential for good CPU inference performance on -# the wide range of arm64 devices the addons ship to. Requires GGML_BACKEND_DL -# to dispatch the variants at runtime; the existing #ifdef guard around -# `ggml_backend_load_all_from_path()` in ggml-backend-reg.cpp keeps the search -# scoped to the consumer's own prebuilds dir. -if(VCPKG_TARGET_IS_ANDROID) - set(DL_BACKENDS ON) - list(APPEND PLATFORM_OPTIONS - -DGGML_BACKEND_DL=ON - -DGGML_CPU_ALL_VARIANTS=ON - -DGGML_CPU_REPACK=ON) -else() - set(DL_BACKENDS OFF) -endif() - -if(VCPKG_TARGET_IS_ANDROID AND BUILD_KLEIDIAI) - message(STATUS "qvac-fabric: kleidiai feature ON — building with ARM KleidiAI optimized kernels") - # ggml only vendors KleidiAI via FetchContent; registry vcpkg-cmake sets - # FETCHCONTENT_FULLY_DISCONNECTED=ON globally, so allow the download here. - list(APPEND PLATFORM_OPTIONS - -DGGML_CPU_KLEIDIAI=ON - -DFETCHCONTENT_FULLY_DISCONNECTED=OFF - ) -endif() - -if(VCPKG_TARGET_IS_ANDROID AND BUILD_OPENMP) - message(STATUS "qvac-fabric: OpenMP for Android enabled") - list(APPEND PLATFORM_OPTIONS -DGGML_OPENMP=ON) -else() - message(STATUS "qvac-fabric: OpenMP Disabled") - list(APPEND PLATFORM_OPTIONS -DGGML_OPENMP=OFF) -endif() - -if (VCPKG_TARGET_IS_ANDROID AND BUILD_GPU_BACKENDS) - list(APPEND PLATFORM_OPTIONS -DGGML_OPENCL=ON) -endif() - -if(BUILD_GPU_BACKENDS AND NOT VCPKG_TARGET_IS_OSX AND NOT VCPKG_TARGET_IS_IOS) - if(VCPKG_TARGET_IS_WINDOWS AND NOT VCPKG_TARGET_IS_MINGW) - string(APPEND VCPKG_C_FLAGS " /I${CURRENT_INSTALLED_DIR}/include") - string(APPEND VCPKG_CXX_FLAGS " /I${CURRENT_INSTALLED_DIR}/include") - else() - string(APPEND VCPKG_C_FLAGS " -isystem ${CURRENT_INSTALLED_DIR}/include") - string(APPEND VCPKG_CXX_FLAGS " -isystem ${CURRENT_INSTALLED_DIR}/include") - endif() -endif() - -set(LLAMA_OPTIONS) -if("llama" IN_LIST FEATURES) - list(APPEND LLAMA_OPTIONS -DLLAMA_MTMD=ON) -else() - list(APPEND LLAMA_OPTIONS - -DLLAMA_MTMD=OFF - -DLLAMA_BUILD_COMMON=OFF - ) -endif() - -vcpkg_cmake_configure( - SOURCE_PATH "${SOURCE_PATH}" - DISABLE_PARALLEL_CONFIGURE - OPTIONS - -DGGML_NATIVE=OFF - -DGGML_CCACHE=OFF - -DGGML_LLAMAFILE=OFF - -DLLAMA_CURL=OFF - -DLLAMA_BUILD_TESTS=OFF - -DLLAMA_BUILD_TOOLS=OFF - -DLLAMA_BUILD_EXAMPLES=OFF - -DLLAMA_BUILD_SERVER=OFF - -DLLAMA_ALL_WARNINGS=OFF - ${LLAMA_OPTIONS} - ${PLATFORM_OPTIONS} - ${FEATURE_OPTIONS} -) - -vcpkg_cmake_install() -vcpkg_cmake_config_fixup( - PACKAGE_NAME ggml) - -if(BUILD_LLAMA) - vcpkg_cmake_config_fixup(PACKAGE_NAME llama) -endif() - -vcpkg_copy_pdbs() -vcpkg_fixup_pkgconfig() - - -if(BUILD_LLAMA) - file(MAKE_DIRECTORY "${CURRENT_PACKAGES_DIR}/tools/${PORT}") - file(RENAME "${CURRENT_PACKAGES_DIR}/bin/convert_hf_to_gguf.py" "${CURRENT_PACKAGES_DIR}/tools/${PORT}/convert-hf-to-gguf.py") - file(INSTALL "${SOURCE_PATH}/gguf-py" DESTINATION "${CURRENT_PACKAGES_DIR}/tools/${PORT}") - file(RENAME "${CURRENT_PACKAGES_DIR}/bin/vulkan_profiling_analyzer.py" "${CURRENT_PACKAGES_DIR}/tools/${PORT}/vulkan_profiling_analyzer.py") -endif() - -if (NOT VCPKG_BUILD_TYPE) - file(REMOVE "${CURRENT_PACKAGES_DIR}/debug/bin/convert_hf_to_gguf.py") -endif() - -file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/include") -file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/share") - -if (VCPKG_LIBRARY_LINKAGE MATCHES "static") - file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/bin") - file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/bin") -endif() - -vcpkg_install_copyright(FILE_LIST "${SOURCE_PATH}/LICENSE") diff --git a/packages/embed-llamacpp/vcpkg/ports/qvac-fabric/vcpkg.json b/packages/embed-llamacpp/vcpkg/ports/qvac-fabric/vcpkg.json deleted file mode 100644 index 48e7b449b8..0000000000 --- a/packages/embed-llamacpp/vcpkg/ports/qvac-fabric/vcpkg.json +++ /dev/null @@ -1,50 +0,0 @@ -{ - "name": "qvac-fabric", - "version": "8828.0.2", - "port-version": 1, - "description": "LLM inference in C/C++", - "homepage": "https://github.com/tetherto/qvac-fabric-llm.cpp", - "license": "MIT", - "dependencies": [ - { - "name": "opencl", - "platform": "android" - }, - { - "name": "vcpkg-cmake", - "host": true - }, - { - "name": "vcpkg-cmake-config", - "host": true - } - ], - "default-features": [ - "gpu-backends", - "llama" - ], - "features": { - "force-profiler": { - "description": "Force vk performance logging in ggml" - }, - "gpu-backends": { - "description": "Build the GPU backends ggml ships per platform: Metal on Apple, Vulkan on Linux/Windows/Android, plus the Android backend-DL hybrid mode and OpenCL. Default-on so existing consumers (llamacpp-llm, llamacpp-embed, nmtcpp, diffusion-cpp) keep their current behaviour with default features. Disable to produce a CPU-only ggml build (useful for consumers like @qvac/classification-ggml that don't need GPU paths and want to skip the vulkan-sdk / metal / opencl build cost). Orthogonal to the 'llama' feature.", - "dependencies": [ - { - "name": "spirv-headers", - "platform": "!osx & !ios", - "version>=": "1.4.341.0" - } - ] - }, - "kleidiai": { - "description": "Enable ARM KleidiAI optimized kernels on Android." - }, - "llama": { - "description": "Build llama components." - }, - "openmp": { - "description": "Enable openmp on Android." - } - } -} diff --git a/packages/llm-llamacpp/vcpkg-configuration.json b/packages/llm-llamacpp/vcpkg-configuration.json index ee683973db..74bbaaec5a 100644 --- a/packages/llm-llamacpp/vcpkg-configuration.json +++ b/packages/llm-llamacpp/vcpkg-configuration.json @@ -4,9 +4,6 @@ "baseline": "c57eec31bdc37ce3a98537b312f5a366b554db6a", "repository": "https://github.com/tetherto/qvac-registry-vcpkg.git" }, - "overlay-ports": [ - "vcpkg/ports" - ], "registries": [ { "kind": "git", diff --git a/packages/llm-llamacpp/vcpkg/ports/qvac-fabric/android-vulkan-version.cmake b/packages/llm-llamacpp/vcpkg/ports/qvac-fabric/android-vulkan-version.cmake deleted file mode 100644 index 21fba00feb..0000000000 --- a/packages/llm-llamacpp/vcpkg/ports/qvac-fabric/android-vulkan-version.cmake +++ /dev/null @@ -1,36 +0,0 @@ -# Function to detect Vulkan version from NDK vulkan_core.h -function(detect_ndk_vulkan_version) - string(TOLOWER "${CMAKE_HOST_SYSTEM_NAME}" host_system_name_lower) - - # CMAKE_HOST_SYSTEM_PROCESSOR is unavailable here. Use a glob pattern to complete the folder instead. - file(GLOB host_dirs LIST_DIRECTORIES true "$ENV{ANDROID_NDK_HOME}/toolchains/llvm/prebuilt/${host_system_name_lower}-*") - if(host_dirs) - list(GET host_dirs 0 host_dir) - get_filename_component(host_arch "${host_dir}" NAME) - set(vulkan_core_h "$ENV{ANDROID_NDK_HOME}/toolchains/llvm/prebuilt/${host_arch}/sysroot/usr/include/vulkan/vulkan_core.h") - else() - message(FATAL "Could not find NDK host directory for ${host_system_name_lower}") - endif() - - if(NOT vulkan_core_h) - message(FATAL "vulkan_core.h not found, using default version") - endif() - - file(READ "${vulkan_core_h}" header_content) - string(REGEX MATCH "VK_HEADER_VERSION ([0-9]+)" version_match "${header_content}") - if(version_match) - set(header_version_3 "${CMAKE_MATCH_1}") - else() - message(FATAL "Could not extract VK_HEADER_VERSION from vulkan_core.h, using default: ${vulkan_version}") - endif() - - # Extract major.minor version from VK_HEADER_VERSION_COMPLETE for download URL - string(REGEX MATCH "VK_HEADER_VERSION_COMPLETE VK_MAKE_API_VERSION\\(([0-9]+), ([0-9]+), ([0-9]+)" version_match "${header_content}") - if(version_match) - set(major "${CMAKE_MATCH_2}") - set(minor "${CMAKE_MATCH_3}") - set(vulkan_version "${major}.${minor}.${header_version_3}" PARENT_SCOPE) - else() - message(FATAL "Could not extract major.minor version from vulkan_core.h, using default: ${vulkan_version}") - endif() -endfunction() diff --git a/packages/llm-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake b/packages/llm-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake deleted file mode 100644 index ead537a0ce..0000000000 --- a/packages/llm-llamacpp/vcpkg/ports/qvac-fabric/portfile.cmake +++ /dev/null @@ -1,191 +0,0 @@ -vcpkg_from_github( - OUT_SOURCE_PATH SOURCE_PATH - REPO tetherto/qvac-fabric-llm.cpp - REF 6f37291fe979c8f9a66a4c85809f69ff0ebeb743 - SHA512 6e29daed48978cac5fad4f2ed96f6518f0a81eef7a789d29d44196fb7a2a1b8fb7ed75fdcfc4acd370279a6b0b500c75517b79115dd88b2fbee2882e3a5d720d - HEAD_REF temp-9341 -) - -# Upstream CMake options only — passed through to vcpkg_cmake_configure. -vcpkg_check_features( - OUT_FEATURE_OPTIONS FEATURE_OPTIONS - FEATURES - force-profiler FORCE_GGML_VK_PERF_LOGGER - llama BUILD_LLAMA -) - -# Portfile-only feature flags (drive PLATFORM_OPTIONS; not upstream cache vars). -vcpkg_check_features( - OUT_FEATURE_OPTIONS _PORTFILE_FEATURE_OPTIONS - FEATURES - gpu-backends BUILD_GPU_BACKENDS - kleidiai BUILD_KLEIDIAI - openmp BUILD_OPENMP -) - -# gpu-backends is default-on via default-features in vcpkg.json. CPU-only -# consumers (e.g. @qvac/classification-ggml) disable it with -# default-features:false (and re-add 'llama' if needed). -if(NOT BUILD_GPU_BACKENDS) - message(STATUS "qvac-fabric: gpu-backends feature OFF — building CPU-only ggml (no Metal/Vulkan/CUDA/OpenCL)") -endif() - -if (VCPKG_TARGET_IS_ANDROID AND BUILD_GPU_BACKENDS) - # NDK only comes with C headers. - # Make sure C++ header exists, it will be used by ggml tensor library. - # Need to determine installed vulkan version and download correct headers - include(${CMAKE_CURRENT_LIST_DIR}/android-vulkan-version.cmake) - detect_ndk_vulkan_version() - message(STATUS "Using Vulkan C++ wrappers from version: ${vulkan_version}") - file(DOWNLOAD - "https://github.com/KhronosGroup/Vulkan-Headers/archive/refs/tags/v${vulkan_version}.tar.gz" - "${SOURCE_PATH}/vulkan-sdk-${vulkan_version}.tar.gz" - TLS_VERIFY ON - ) - - file(ARCHIVE_EXTRACT - INPUT "${SOURCE_PATH}/vulkan-sdk-${vulkan_version}.tar.gz" - DESTINATION "${SOURCE_PATH}" - PATTERNS "*.hpp" - ) - - file(RENAME - "${SOURCE_PATH}/Vulkan-Headers-${vulkan_version}" - "${SOURCE_PATH}/ggml/src/ggml-vulkan/vulkan_cpp_wrapper" - ) -endif() - -set(PLATFORM_OPTIONS) - -if(NOT BUILD_GPU_BACKENDS) - # Force every GPU backend off explicitly, in case upstream defaults change. - list(APPEND PLATFORM_OPTIONS - -DGGML_METAL=OFF - -DGGML_VULKAN=OFF - -DGGML_CUDA=OFF - -DGGML_OPENCL=OFF - ) - if (VCPKG_TARGET_IS_IOS) - # Same iOS BLAS/Accelerate gating as the GPU-on path; unrelated to the - # CPU-vs-GPU split, an iOS-toolchain workaround for missing frameworks. - list(APPEND PLATFORM_OPTIONS -DGGML_BLAS=OFF -DGGML_ACCELERATE=OFF) - endif() -elseif (VCPKG_TARGET_IS_OSX OR VCPKG_TARGET_IS_IOS) - list(APPEND PLATFORM_OPTIONS -DGGML_METAL=ON) - if (VCPKG_TARGET_IS_IOS) - list(APPEND PLATFORM_OPTIONS -DGGML_BLAS=OFF -DGGML_ACCELERATE=OFF) - endif() -else() - list(APPEND PLATFORM_OPTIONS -DGGML_VULKAN=ON) -endif() - -# Android: always build CPU variants (NEON_DOTPROD, NEON_I8MM, etc.) and CPU -# repacking. These are CPU-only runtime optimizations selected based on the -# device's SIMD capabilities at load time, completely orthogonal to the GPU -# backends. Bundling them is essential for good CPU inference performance on -# the wide range of arm64 devices the addons ship to. Requires GGML_BACKEND_DL -# to dispatch the variants at runtime; the existing #ifdef guard around -# `ggml_backend_load_all_from_path()` in ggml-backend-reg.cpp keeps the search -# scoped to the consumer's own prebuilds dir. -if(VCPKG_TARGET_IS_ANDROID) - set(DL_BACKENDS ON) - list(APPEND PLATFORM_OPTIONS - -DGGML_BACKEND_DL=ON - -DGGML_CPU_ALL_VARIANTS=ON - -DGGML_CPU_REPACK=ON) -else() - set(DL_BACKENDS OFF) -endif() - -if(VCPKG_TARGET_IS_ANDROID AND BUILD_KLEIDIAI) - message(STATUS "qvac-fabric: kleidiai feature ON — building with ARM KleidiAI optimized kernels") - # ggml only vendors KleidiAI via FetchContent; registry vcpkg-cmake sets - # FETCHCONTENT_FULLY_DISCONNECTED=ON globally, so allow the download here. - list(APPEND PLATFORM_OPTIONS - -DGGML_CPU_KLEIDIAI=ON - -DFETCHCONTENT_FULLY_DISCONNECTED=OFF - ) -endif() - -if(VCPKG_TARGET_IS_ANDROID AND BUILD_OPENMP) - message(STATUS "qvac-fabric: OpenMP for Android enabled") - list(APPEND PLATFORM_OPTIONS -DGGML_OPENMP=ON) -else() - message(STATUS "qvac-fabric: OpenMP Disabled") - list(APPEND PLATFORM_OPTIONS -DGGML_OPENMP=OFF) -endif() - -if (VCPKG_TARGET_IS_ANDROID AND BUILD_GPU_BACKENDS) - list(APPEND PLATFORM_OPTIONS -DGGML_OPENCL=ON) -endif() - -if(BUILD_GPU_BACKENDS AND NOT VCPKG_TARGET_IS_OSX AND NOT VCPKG_TARGET_IS_IOS) - if(VCPKG_TARGET_IS_WINDOWS AND NOT VCPKG_TARGET_IS_MINGW) - string(APPEND VCPKG_C_FLAGS " /I${CURRENT_INSTALLED_DIR}/include") - string(APPEND VCPKG_CXX_FLAGS " /I${CURRENT_INSTALLED_DIR}/include") - else() - string(APPEND VCPKG_C_FLAGS " -isystem ${CURRENT_INSTALLED_DIR}/include") - string(APPEND VCPKG_CXX_FLAGS " -isystem ${CURRENT_INSTALLED_DIR}/include") - endif() -endif() - -set(LLAMA_OPTIONS) -if("llama" IN_LIST FEATURES) - list(APPEND LLAMA_OPTIONS -DLLAMA_MTMD=ON) -else() - list(APPEND LLAMA_OPTIONS - -DLLAMA_MTMD=OFF - -DLLAMA_BUILD_COMMON=OFF - ) -endif() - -vcpkg_cmake_configure( - SOURCE_PATH "${SOURCE_PATH}" - DISABLE_PARALLEL_CONFIGURE - OPTIONS - -DGGML_NATIVE=OFF - -DGGML_CCACHE=OFF - -DGGML_LLAMAFILE=OFF - -DLLAMA_CURL=OFF - -DLLAMA_BUILD_TESTS=OFF - -DLLAMA_BUILD_TOOLS=OFF - -DLLAMA_BUILD_EXAMPLES=OFF - -DLLAMA_BUILD_SERVER=OFF - -DLLAMA_ALL_WARNINGS=OFF - ${LLAMA_OPTIONS} - ${PLATFORM_OPTIONS} - ${FEATURE_OPTIONS} -) - -vcpkg_cmake_install() -vcpkg_cmake_config_fixup( - PACKAGE_NAME ggml) - -if(BUILD_LLAMA) - vcpkg_cmake_config_fixup(PACKAGE_NAME llama) -endif() - -vcpkg_copy_pdbs() -vcpkg_fixup_pkgconfig() - - -if(BUILD_LLAMA) - file(MAKE_DIRECTORY "${CURRENT_PACKAGES_DIR}/tools/${PORT}") - file(RENAME "${CURRENT_PACKAGES_DIR}/bin/convert_hf_to_gguf.py" "${CURRENT_PACKAGES_DIR}/tools/${PORT}/convert-hf-to-gguf.py") - file(INSTALL "${SOURCE_PATH}/gguf-py" DESTINATION "${CURRENT_PACKAGES_DIR}/tools/${PORT}") - file(RENAME "${CURRENT_PACKAGES_DIR}/bin/vulkan_profiling_analyzer.py" "${CURRENT_PACKAGES_DIR}/tools/${PORT}/vulkan_profiling_analyzer.py") -endif() - -if (NOT VCPKG_BUILD_TYPE) - file(REMOVE "${CURRENT_PACKAGES_DIR}/debug/bin/convert_hf_to_gguf.py") -endif() - -file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/include") -file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/share") - -if (VCPKG_LIBRARY_LINKAGE MATCHES "static") - file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/bin") - file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/bin") -endif() - -vcpkg_install_copyright(FILE_LIST "${SOURCE_PATH}/LICENSE") diff --git a/packages/llm-llamacpp/vcpkg/ports/qvac-fabric/vcpkg.json b/packages/llm-llamacpp/vcpkg/ports/qvac-fabric/vcpkg.json deleted file mode 100644 index 48e7b449b8..0000000000 --- a/packages/llm-llamacpp/vcpkg/ports/qvac-fabric/vcpkg.json +++ /dev/null @@ -1,50 +0,0 @@ -{ - "name": "qvac-fabric", - "version": "8828.0.2", - "port-version": 1, - "description": "LLM inference in C/C++", - "homepage": "https://github.com/tetherto/qvac-fabric-llm.cpp", - "license": "MIT", - "dependencies": [ - { - "name": "opencl", - "platform": "android" - }, - { - "name": "vcpkg-cmake", - "host": true - }, - { - "name": "vcpkg-cmake-config", - "host": true - } - ], - "default-features": [ - "gpu-backends", - "llama" - ], - "features": { - "force-profiler": { - "description": "Force vk performance logging in ggml" - }, - "gpu-backends": { - "description": "Build the GPU backends ggml ships per platform: Metal on Apple, Vulkan on Linux/Windows/Android, plus the Android backend-DL hybrid mode and OpenCL. Default-on so existing consumers (llamacpp-llm, llamacpp-embed, nmtcpp, diffusion-cpp) keep their current behaviour with default features. Disable to produce a CPU-only ggml build (useful for consumers like @qvac/classification-ggml that don't need GPU paths and want to skip the vulkan-sdk / metal / opencl build cost). Orthogonal to the 'llama' feature.", - "dependencies": [ - { - "name": "spirv-headers", - "platform": "!osx & !ios", - "version>=": "1.4.341.0" - } - ] - }, - "kleidiai": { - "description": "Enable ARM KleidiAI optimized kernels on Android." - }, - "llama": { - "description": "Build llama components." - }, - "openmp": { - "description": "Enable openmp on Android." - } - } -} diff --git a/packages/translation-nmtcpp/vcpkg-configuration.json b/packages/translation-nmtcpp/vcpkg-configuration.json index 68b81b23d6..b49381752c 100644 --- a/packages/translation-nmtcpp/vcpkg-configuration.json +++ b/packages/translation-nmtcpp/vcpkg-configuration.json @@ -4,9 +4,6 @@ "baseline": "1b499699dc209ed0fbc17649f6174335ce5a2743", "repository": "https://github.com/tetherto/qvac-registry-vcpkg.git" }, - "overlay-ports": [ - "vcpkg/ports" - ], "registries": [ { "kind": "git", diff --git a/packages/translation-nmtcpp/vcpkg/ports/qvac-fabric/android-vulkan-version.cmake b/packages/translation-nmtcpp/vcpkg/ports/qvac-fabric/android-vulkan-version.cmake deleted file mode 100644 index 21fba00feb..0000000000 --- a/packages/translation-nmtcpp/vcpkg/ports/qvac-fabric/android-vulkan-version.cmake +++ /dev/null @@ -1,36 +0,0 @@ -# Function to detect Vulkan version from NDK vulkan_core.h -function(detect_ndk_vulkan_version) - string(TOLOWER "${CMAKE_HOST_SYSTEM_NAME}" host_system_name_lower) - - # CMAKE_HOST_SYSTEM_PROCESSOR is unavailable here. Use a glob pattern to complete the folder instead. - file(GLOB host_dirs LIST_DIRECTORIES true "$ENV{ANDROID_NDK_HOME}/toolchains/llvm/prebuilt/${host_system_name_lower}-*") - if(host_dirs) - list(GET host_dirs 0 host_dir) - get_filename_component(host_arch "${host_dir}" NAME) - set(vulkan_core_h "$ENV{ANDROID_NDK_HOME}/toolchains/llvm/prebuilt/${host_arch}/sysroot/usr/include/vulkan/vulkan_core.h") - else() - message(FATAL "Could not find NDK host directory for ${host_system_name_lower}") - endif() - - if(NOT vulkan_core_h) - message(FATAL "vulkan_core.h not found, using default version") - endif() - - file(READ "${vulkan_core_h}" header_content) - string(REGEX MATCH "VK_HEADER_VERSION ([0-9]+)" version_match "${header_content}") - if(version_match) - set(header_version_3 "${CMAKE_MATCH_1}") - else() - message(FATAL "Could not extract VK_HEADER_VERSION from vulkan_core.h, using default: ${vulkan_version}") - endif() - - # Extract major.minor version from VK_HEADER_VERSION_COMPLETE for download URL - string(REGEX MATCH "VK_HEADER_VERSION_COMPLETE VK_MAKE_API_VERSION\\(([0-9]+), ([0-9]+), ([0-9]+)" version_match "${header_content}") - if(version_match) - set(major "${CMAKE_MATCH_2}") - set(minor "${CMAKE_MATCH_3}") - set(vulkan_version "${major}.${minor}.${header_version_3}" PARENT_SCOPE) - else() - message(FATAL "Could not extract major.minor version from vulkan_core.h, using default: ${vulkan_version}") - endif() -endfunction() diff --git a/packages/translation-nmtcpp/vcpkg/ports/qvac-fabric/portfile.cmake b/packages/translation-nmtcpp/vcpkg/ports/qvac-fabric/portfile.cmake deleted file mode 100644 index ead537a0ce..0000000000 --- a/packages/translation-nmtcpp/vcpkg/ports/qvac-fabric/portfile.cmake +++ /dev/null @@ -1,191 +0,0 @@ -vcpkg_from_github( - OUT_SOURCE_PATH SOURCE_PATH - REPO tetherto/qvac-fabric-llm.cpp - REF 6f37291fe979c8f9a66a4c85809f69ff0ebeb743 - SHA512 6e29daed48978cac5fad4f2ed96f6518f0a81eef7a789d29d44196fb7a2a1b8fb7ed75fdcfc4acd370279a6b0b500c75517b79115dd88b2fbee2882e3a5d720d - HEAD_REF temp-9341 -) - -# Upstream CMake options only — passed through to vcpkg_cmake_configure. -vcpkg_check_features( - OUT_FEATURE_OPTIONS FEATURE_OPTIONS - FEATURES - force-profiler FORCE_GGML_VK_PERF_LOGGER - llama BUILD_LLAMA -) - -# Portfile-only feature flags (drive PLATFORM_OPTIONS; not upstream cache vars). -vcpkg_check_features( - OUT_FEATURE_OPTIONS _PORTFILE_FEATURE_OPTIONS - FEATURES - gpu-backends BUILD_GPU_BACKENDS - kleidiai BUILD_KLEIDIAI - openmp BUILD_OPENMP -) - -# gpu-backends is default-on via default-features in vcpkg.json. CPU-only -# consumers (e.g. @qvac/classification-ggml) disable it with -# default-features:false (and re-add 'llama' if needed). -if(NOT BUILD_GPU_BACKENDS) - message(STATUS "qvac-fabric: gpu-backends feature OFF — building CPU-only ggml (no Metal/Vulkan/CUDA/OpenCL)") -endif() - -if (VCPKG_TARGET_IS_ANDROID AND BUILD_GPU_BACKENDS) - # NDK only comes with C headers. - # Make sure C++ header exists, it will be used by ggml tensor library. - # Need to determine installed vulkan version and download correct headers - include(${CMAKE_CURRENT_LIST_DIR}/android-vulkan-version.cmake) - detect_ndk_vulkan_version() - message(STATUS "Using Vulkan C++ wrappers from version: ${vulkan_version}") - file(DOWNLOAD - "https://github.com/KhronosGroup/Vulkan-Headers/archive/refs/tags/v${vulkan_version}.tar.gz" - "${SOURCE_PATH}/vulkan-sdk-${vulkan_version}.tar.gz" - TLS_VERIFY ON - ) - - file(ARCHIVE_EXTRACT - INPUT "${SOURCE_PATH}/vulkan-sdk-${vulkan_version}.tar.gz" - DESTINATION "${SOURCE_PATH}" - PATTERNS "*.hpp" - ) - - file(RENAME - "${SOURCE_PATH}/Vulkan-Headers-${vulkan_version}" - "${SOURCE_PATH}/ggml/src/ggml-vulkan/vulkan_cpp_wrapper" - ) -endif() - -set(PLATFORM_OPTIONS) - -if(NOT BUILD_GPU_BACKENDS) - # Force every GPU backend off explicitly, in case upstream defaults change. - list(APPEND PLATFORM_OPTIONS - -DGGML_METAL=OFF - -DGGML_VULKAN=OFF - -DGGML_CUDA=OFF - -DGGML_OPENCL=OFF - ) - if (VCPKG_TARGET_IS_IOS) - # Same iOS BLAS/Accelerate gating as the GPU-on path; unrelated to the - # CPU-vs-GPU split, an iOS-toolchain workaround for missing frameworks. - list(APPEND PLATFORM_OPTIONS -DGGML_BLAS=OFF -DGGML_ACCELERATE=OFF) - endif() -elseif (VCPKG_TARGET_IS_OSX OR VCPKG_TARGET_IS_IOS) - list(APPEND PLATFORM_OPTIONS -DGGML_METAL=ON) - if (VCPKG_TARGET_IS_IOS) - list(APPEND PLATFORM_OPTIONS -DGGML_BLAS=OFF -DGGML_ACCELERATE=OFF) - endif() -else() - list(APPEND PLATFORM_OPTIONS -DGGML_VULKAN=ON) -endif() - -# Android: always build CPU variants (NEON_DOTPROD, NEON_I8MM, etc.) and CPU -# repacking. These are CPU-only runtime optimizations selected based on the -# device's SIMD capabilities at load time, completely orthogonal to the GPU -# backends. Bundling them is essential for good CPU inference performance on -# the wide range of arm64 devices the addons ship to. Requires GGML_BACKEND_DL -# to dispatch the variants at runtime; the existing #ifdef guard around -# `ggml_backend_load_all_from_path()` in ggml-backend-reg.cpp keeps the search -# scoped to the consumer's own prebuilds dir. -if(VCPKG_TARGET_IS_ANDROID) - set(DL_BACKENDS ON) - list(APPEND PLATFORM_OPTIONS - -DGGML_BACKEND_DL=ON - -DGGML_CPU_ALL_VARIANTS=ON - -DGGML_CPU_REPACK=ON) -else() - set(DL_BACKENDS OFF) -endif() - -if(VCPKG_TARGET_IS_ANDROID AND BUILD_KLEIDIAI) - message(STATUS "qvac-fabric: kleidiai feature ON — building with ARM KleidiAI optimized kernels") - # ggml only vendors KleidiAI via FetchContent; registry vcpkg-cmake sets - # FETCHCONTENT_FULLY_DISCONNECTED=ON globally, so allow the download here. - list(APPEND PLATFORM_OPTIONS - -DGGML_CPU_KLEIDIAI=ON - -DFETCHCONTENT_FULLY_DISCONNECTED=OFF - ) -endif() - -if(VCPKG_TARGET_IS_ANDROID AND BUILD_OPENMP) - message(STATUS "qvac-fabric: OpenMP for Android enabled") - list(APPEND PLATFORM_OPTIONS -DGGML_OPENMP=ON) -else() - message(STATUS "qvac-fabric: OpenMP Disabled") - list(APPEND PLATFORM_OPTIONS -DGGML_OPENMP=OFF) -endif() - -if (VCPKG_TARGET_IS_ANDROID AND BUILD_GPU_BACKENDS) - list(APPEND PLATFORM_OPTIONS -DGGML_OPENCL=ON) -endif() - -if(BUILD_GPU_BACKENDS AND NOT VCPKG_TARGET_IS_OSX AND NOT VCPKG_TARGET_IS_IOS) - if(VCPKG_TARGET_IS_WINDOWS AND NOT VCPKG_TARGET_IS_MINGW) - string(APPEND VCPKG_C_FLAGS " /I${CURRENT_INSTALLED_DIR}/include") - string(APPEND VCPKG_CXX_FLAGS " /I${CURRENT_INSTALLED_DIR}/include") - else() - string(APPEND VCPKG_C_FLAGS " -isystem ${CURRENT_INSTALLED_DIR}/include") - string(APPEND VCPKG_CXX_FLAGS " -isystem ${CURRENT_INSTALLED_DIR}/include") - endif() -endif() - -set(LLAMA_OPTIONS) -if("llama" IN_LIST FEATURES) - list(APPEND LLAMA_OPTIONS -DLLAMA_MTMD=ON) -else() - list(APPEND LLAMA_OPTIONS - -DLLAMA_MTMD=OFF - -DLLAMA_BUILD_COMMON=OFF - ) -endif() - -vcpkg_cmake_configure( - SOURCE_PATH "${SOURCE_PATH}" - DISABLE_PARALLEL_CONFIGURE - OPTIONS - -DGGML_NATIVE=OFF - -DGGML_CCACHE=OFF - -DGGML_LLAMAFILE=OFF - -DLLAMA_CURL=OFF - -DLLAMA_BUILD_TESTS=OFF - -DLLAMA_BUILD_TOOLS=OFF - -DLLAMA_BUILD_EXAMPLES=OFF - -DLLAMA_BUILD_SERVER=OFF - -DLLAMA_ALL_WARNINGS=OFF - ${LLAMA_OPTIONS} - ${PLATFORM_OPTIONS} - ${FEATURE_OPTIONS} -) - -vcpkg_cmake_install() -vcpkg_cmake_config_fixup( - PACKAGE_NAME ggml) - -if(BUILD_LLAMA) - vcpkg_cmake_config_fixup(PACKAGE_NAME llama) -endif() - -vcpkg_copy_pdbs() -vcpkg_fixup_pkgconfig() - - -if(BUILD_LLAMA) - file(MAKE_DIRECTORY "${CURRENT_PACKAGES_DIR}/tools/${PORT}") - file(RENAME "${CURRENT_PACKAGES_DIR}/bin/convert_hf_to_gguf.py" "${CURRENT_PACKAGES_DIR}/tools/${PORT}/convert-hf-to-gguf.py") - file(INSTALL "${SOURCE_PATH}/gguf-py" DESTINATION "${CURRENT_PACKAGES_DIR}/tools/${PORT}") - file(RENAME "${CURRENT_PACKAGES_DIR}/bin/vulkan_profiling_analyzer.py" "${CURRENT_PACKAGES_DIR}/tools/${PORT}/vulkan_profiling_analyzer.py") -endif() - -if (NOT VCPKG_BUILD_TYPE) - file(REMOVE "${CURRENT_PACKAGES_DIR}/debug/bin/convert_hf_to_gguf.py") -endif() - -file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/include") -file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/share") - -if (VCPKG_LIBRARY_LINKAGE MATCHES "static") - file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/bin") - file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/bin") -endif() - -vcpkg_install_copyright(FILE_LIST "${SOURCE_PATH}/LICENSE") diff --git a/packages/translation-nmtcpp/vcpkg/ports/qvac-fabric/vcpkg.json b/packages/translation-nmtcpp/vcpkg/ports/qvac-fabric/vcpkg.json deleted file mode 100644 index 48e7b449b8..0000000000 --- a/packages/translation-nmtcpp/vcpkg/ports/qvac-fabric/vcpkg.json +++ /dev/null @@ -1,50 +0,0 @@ -{ - "name": "qvac-fabric", - "version": "8828.0.2", - "port-version": 1, - "description": "LLM inference in C/C++", - "homepage": "https://github.com/tetherto/qvac-fabric-llm.cpp", - "license": "MIT", - "dependencies": [ - { - "name": "opencl", - "platform": "android" - }, - { - "name": "vcpkg-cmake", - "host": true - }, - { - "name": "vcpkg-cmake-config", - "host": true - } - ], - "default-features": [ - "gpu-backends", - "llama" - ], - "features": { - "force-profiler": { - "description": "Force vk performance logging in ggml" - }, - "gpu-backends": { - "description": "Build the GPU backends ggml ships per platform: Metal on Apple, Vulkan on Linux/Windows/Android, plus the Android backend-DL hybrid mode and OpenCL. Default-on so existing consumers (llamacpp-llm, llamacpp-embed, nmtcpp, diffusion-cpp) keep their current behaviour with default features. Disable to produce a CPU-only ggml build (useful for consumers like @qvac/classification-ggml that don't need GPU paths and want to skip the vulkan-sdk / metal / opencl build cost). Orthogonal to the 'llama' feature.", - "dependencies": [ - { - "name": "spirv-headers", - "platform": "!osx & !ios", - "version>=": "1.4.341.0" - } - ] - }, - "kleidiai": { - "description": "Enable ARM KleidiAI optimized kernels on Android." - }, - "llama": { - "description": "Build llama components." - }, - "openmp": { - "description": "Enable openmp on Android." - } - } -} diff --git a/packages/vla-ggml/vcpkg-configuration.json b/packages/vla-ggml/vcpkg-configuration.json index 67ed4d1b80..b0ca350e27 100644 --- a/packages/vla-ggml/vcpkg-configuration.json +++ b/packages/vla-ggml/vcpkg-configuration.json @@ -4,9 +4,6 @@ "baseline": "a9eae49a7c95a63755c0b44ecf9d74b0a00e181b", "repository": "https://github.com/tetherto/qvac-registry-vcpkg.git" }, - "overlay-ports": [ - "vcpkg/ports" - ], "registries": [ { "kind": "git", diff --git a/packages/vla-ggml/vcpkg/ports/qvac-fabric/android-vulkan-version.cmake b/packages/vla-ggml/vcpkg/ports/qvac-fabric/android-vulkan-version.cmake deleted file mode 100644 index 21fba00feb..0000000000 --- a/packages/vla-ggml/vcpkg/ports/qvac-fabric/android-vulkan-version.cmake +++ /dev/null @@ -1,36 +0,0 @@ -# Function to detect Vulkan version from NDK vulkan_core.h -function(detect_ndk_vulkan_version) - string(TOLOWER "${CMAKE_HOST_SYSTEM_NAME}" host_system_name_lower) - - # CMAKE_HOST_SYSTEM_PROCESSOR is unavailable here. Use a glob pattern to complete the folder instead. - file(GLOB host_dirs LIST_DIRECTORIES true "$ENV{ANDROID_NDK_HOME}/toolchains/llvm/prebuilt/${host_system_name_lower}-*") - if(host_dirs) - list(GET host_dirs 0 host_dir) - get_filename_component(host_arch "${host_dir}" NAME) - set(vulkan_core_h "$ENV{ANDROID_NDK_HOME}/toolchains/llvm/prebuilt/${host_arch}/sysroot/usr/include/vulkan/vulkan_core.h") - else() - message(FATAL "Could not find NDK host directory for ${host_system_name_lower}") - endif() - - if(NOT vulkan_core_h) - message(FATAL "vulkan_core.h not found, using default version") - endif() - - file(READ "${vulkan_core_h}" header_content) - string(REGEX MATCH "VK_HEADER_VERSION ([0-9]+)" version_match "${header_content}") - if(version_match) - set(header_version_3 "${CMAKE_MATCH_1}") - else() - message(FATAL "Could not extract VK_HEADER_VERSION from vulkan_core.h, using default: ${vulkan_version}") - endif() - - # Extract major.minor version from VK_HEADER_VERSION_COMPLETE for download URL - string(REGEX MATCH "VK_HEADER_VERSION_COMPLETE VK_MAKE_API_VERSION\\(([0-9]+), ([0-9]+), ([0-9]+)" version_match "${header_content}") - if(version_match) - set(major "${CMAKE_MATCH_2}") - set(minor "${CMAKE_MATCH_3}") - set(vulkan_version "${major}.${minor}.${header_version_3}" PARENT_SCOPE) - else() - message(FATAL "Could not extract major.minor version from vulkan_core.h, using default: ${vulkan_version}") - endif() -endfunction() diff --git a/packages/vla-ggml/vcpkg/ports/qvac-fabric/portfile.cmake b/packages/vla-ggml/vcpkg/ports/qvac-fabric/portfile.cmake deleted file mode 100644 index ead537a0ce..0000000000 --- a/packages/vla-ggml/vcpkg/ports/qvac-fabric/portfile.cmake +++ /dev/null @@ -1,191 +0,0 @@ -vcpkg_from_github( - OUT_SOURCE_PATH SOURCE_PATH - REPO tetherto/qvac-fabric-llm.cpp - REF 6f37291fe979c8f9a66a4c85809f69ff0ebeb743 - SHA512 6e29daed48978cac5fad4f2ed96f6518f0a81eef7a789d29d44196fb7a2a1b8fb7ed75fdcfc4acd370279a6b0b500c75517b79115dd88b2fbee2882e3a5d720d - HEAD_REF temp-9341 -) - -# Upstream CMake options only — passed through to vcpkg_cmake_configure. -vcpkg_check_features( - OUT_FEATURE_OPTIONS FEATURE_OPTIONS - FEATURES - force-profiler FORCE_GGML_VK_PERF_LOGGER - llama BUILD_LLAMA -) - -# Portfile-only feature flags (drive PLATFORM_OPTIONS; not upstream cache vars). -vcpkg_check_features( - OUT_FEATURE_OPTIONS _PORTFILE_FEATURE_OPTIONS - FEATURES - gpu-backends BUILD_GPU_BACKENDS - kleidiai BUILD_KLEIDIAI - openmp BUILD_OPENMP -) - -# gpu-backends is default-on via default-features in vcpkg.json. CPU-only -# consumers (e.g. @qvac/classification-ggml) disable it with -# default-features:false (and re-add 'llama' if needed). -if(NOT BUILD_GPU_BACKENDS) - message(STATUS "qvac-fabric: gpu-backends feature OFF — building CPU-only ggml (no Metal/Vulkan/CUDA/OpenCL)") -endif() - -if (VCPKG_TARGET_IS_ANDROID AND BUILD_GPU_BACKENDS) - # NDK only comes with C headers. - # Make sure C++ header exists, it will be used by ggml tensor library. - # Need to determine installed vulkan version and download correct headers - include(${CMAKE_CURRENT_LIST_DIR}/android-vulkan-version.cmake) - detect_ndk_vulkan_version() - message(STATUS "Using Vulkan C++ wrappers from version: ${vulkan_version}") - file(DOWNLOAD - "https://github.com/KhronosGroup/Vulkan-Headers/archive/refs/tags/v${vulkan_version}.tar.gz" - "${SOURCE_PATH}/vulkan-sdk-${vulkan_version}.tar.gz" - TLS_VERIFY ON - ) - - file(ARCHIVE_EXTRACT - INPUT "${SOURCE_PATH}/vulkan-sdk-${vulkan_version}.tar.gz" - DESTINATION "${SOURCE_PATH}" - PATTERNS "*.hpp" - ) - - file(RENAME - "${SOURCE_PATH}/Vulkan-Headers-${vulkan_version}" - "${SOURCE_PATH}/ggml/src/ggml-vulkan/vulkan_cpp_wrapper" - ) -endif() - -set(PLATFORM_OPTIONS) - -if(NOT BUILD_GPU_BACKENDS) - # Force every GPU backend off explicitly, in case upstream defaults change. - list(APPEND PLATFORM_OPTIONS - -DGGML_METAL=OFF - -DGGML_VULKAN=OFF - -DGGML_CUDA=OFF - -DGGML_OPENCL=OFF - ) - if (VCPKG_TARGET_IS_IOS) - # Same iOS BLAS/Accelerate gating as the GPU-on path; unrelated to the - # CPU-vs-GPU split, an iOS-toolchain workaround for missing frameworks. - list(APPEND PLATFORM_OPTIONS -DGGML_BLAS=OFF -DGGML_ACCELERATE=OFF) - endif() -elseif (VCPKG_TARGET_IS_OSX OR VCPKG_TARGET_IS_IOS) - list(APPEND PLATFORM_OPTIONS -DGGML_METAL=ON) - if (VCPKG_TARGET_IS_IOS) - list(APPEND PLATFORM_OPTIONS -DGGML_BLAS=OFF -DGGML_ACCELERATE=OFF) - endif() -else() - list(APPEND PLATFORM_OPTIONS -DGGML_VULKAN=ON) -endif() - -# Android: always build CPU variants (NEON_DOTPROD, NEON_I8MM, etc.) and CPU -# repacking. These are CPU-only runtime optimizations selected based on the -# device's SIMD capabilities at load time, completely orthogonal to the GPU -# backends. Bundling them is essential for good CPU inference performance on -# the wide range of arm64 devices the addons ship to. Requires GGML_BACKEND_DL -# to dispatch the variants at runtime; the existing #ifdef guard around -# `ggml_backend_load_all_from_path()` in ggml-backend-reg.cpp keeps the search -# scoped to the consumer's own prebuilds dir. -if(VCPKG_TARGET_IS_ANDROID) - set(DL_BACKENDS ON) - list(APPEND PLATFORM_OPTIONS - -DGGML_BACKEND_DL=ON - -DGGML_CPU_ALL_VARIANTS=ON - -DGGML_CPU_REPACK=ON) -else() - set(DL_BACKENDS OFF) -endif() - -if(VCPKG_TARGET_IS_ANDROID AND BUILD_KLEIDIAI) - message(STATUS "qvac-fabric: kleidiai feature ON — building with ARM KleidiAI optimized kernels") - # ggml only vendors KleidiAI via FetchContent; registry vcpkg-cmake sets - # FETCHCONTENT_FULLY_DISCONNECTED=ON globally, so allow the download here. - list(APPEND PLATFORM_OPTIONS - -DGGML_CPU_KLEIDIAI=ON - -DFETCHCONTENT_FULLY_DISCONNECTED=OFF - ) -endif() - -if(VCPKG_TARGET_IS_ANDROID AND BUILD_OPENMP) - message(STATUS "qvac-fabric: OpenMP for Android enabled") - list(APPEND PLATFORM_OPTIONS -DGGML_OPENMP=ON) -else() - message(STATUS "qvac-fabric: OpenMP Disabled") - list(APPEND PLATFORM_OPTIONS -DGGML_OPENMP=OFF) -endif() - -if (VCPKG_TARGET_IS_ANDROID AND BUILD_GPU_BACKENDS) - list(APPEND PLATFORM_OPTIONS -DGGML_OPENCL=ON) -endif() - -if(BUILD_GPU_BACKENDS AND NOT VCPKG_TARGET_IS_OSX AND NOT VCPKG_TARGET_IS_IOS) - if(VCPKG_TARGET_IS_WINDOWS AND NOT VCPKG_TARGET_IS_MINGW) - string(APPEND VCPKG_C_FLAGS " /I${CURRENT_INSTALLED_DIR}/include") - string(APPEND VCPKG_CXX_FLAGS " /I${CURRENT_INSTALLED_DIR}/include") - else() - string(APPEND VCPKG_C_FLAGS " -isystem ${CURRENT_INSTALLED_DIR}/include") - string(APPEND VCPKG_CXX_FLAGS " -isystem ${CURRENT_INSTALLED_DIR}/include") - endif() -endif() - -set(LLAMA_OPTIONS) -if("llama" IN_LIST FEATURES) - list(APPEND LLAMA_OPTIONS -DLLAMA_MTMD=ON) -else() - list(APPEND LLAMA_OPTIONS - -DLLAMA_MTMD=OFF - -DLLAMA_BUILD_COMMON=OFF - ) -endif() - -vcpkg_cmake_configure( - SOURCE_PATH "${SOURCE_PATH}" - DISABLE_PARALLEL_CONFIGURE - OPTIONS - -DGGML_NATIVE=OFF - -DGGML_CCACHE=OFF - -DGGML_LLAMAFILE=OFF - -DLLAMA_CURL=OFF - -DLLAMA_BUILD_TESTS=OFF - -DLLAMA_BUILD_TOOLS=OFF - -DLLAMA_BUILD_EXAMPLES=OFF - -DLLAMA_BUILD_SERVER=OFF - -DLLAMA_ALL_WARNINGS=OFF - ${LLAMA_OPTIONS} - ${PLATFORM_OPTIONS} - ${FEATURE_OPTIONS} -) - -vcpkg_cmake_install() -vcpkg_cmake_config_fixup( - PACKAGE_NAME ggml) - -if(BUILD_LLAMA) - vcpkg_cmake_config_fixup(PACKAGE_NAME llama) -endif() - -vcpkg_copy_pdbs() -vcpkg_fixup_pkgconfig() - - -if(BUILD_LLAMA) - file(MAKE_DIRECTORY "${CURRENT_PACKAGES_DIR}/tools/${PORT}") - file(RENAME "${CURRENT_PACKAGES_DIR}/bin/convert_hf_to_gguf.py" "${CURRENT_PACKAGES_DIR}/tools/${PORT}/convert-hf-to-gguf.py") - file(INSTALL "${SOURCE_PATH}/gguf-py" DESTINATION "${CURRENT_PACKAGES_DIR}/tools/${PORT}") - file(RENAME "${CURRENT_PACKAGES_DIR}/bin/vulkan_profiling_analyzer.py" "${CURRENT_PACKAGES_DIR}/tools/${PORT}/vulkan_profiling_analyzer.py") -endif() - -if (NOT VCPKG_BUILD_TYPE) - file(REMOVE "${CURRENT_PACKAGES_DIR}/debug/bin/convert_hf_to_gguf.py") -endif() - -file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/include") -file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/share") - -if (VCPKG_LIBRARY_LINKAGE MATCHES "static") - file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/bin") - file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/bin") -endif() - -vcpkg_install_copyright(FILE_LIST "${SOURCE_PATH}/LICENSE") diff --git a/packages/vla-ggml/vcpkg/ports/qvac-fabric/vcpkg.json b/packages/vla-ggml/vcpkg/ports/qvac-fabric/vcpkg.json deleted file mode 100644 index 48e7b449b8..0000000000 --- a/packages/vla-ggml/vcpkg/ports/qvac-fabric/vcpkg.json +++ /dev/null @@ -1,50 +0,0 @@ -{ - "name": "qvac-fabric", - "version": "8828.0.2", - "port-version": 1, - "description": "LLM inference in C/C++", - "homepage": "https://github.com/tetherto/qvac-fabric-llm.cpp", - "license": "MIT", - "dependencies": [ - { - "name": "opencl", - "platform": "android" - }, - { - "name": "vcpkg-cmake", - "host": true - }, - { - "name": "vcpkg-cmake-config", - "host": true - } - ], - "default-features": [ - "gpu-backends", - "llama" - ], - "features": { - "force-profiler": { - "description": "Force vk performance logging in ggml" - }, - "gpu-backends": { - "description": "Build the GPU backends ggml ships per platform: Metal on Apple, Vulkan on Linux/Windows/Android, plus the Android backend-DL hybrid mode and OpenCL. Default-on so existing consumers (llamacpp-llm, llamacpp-embed, nmtcpp, diffusion-cpp) keep their current behaviour with default features. Disable to produce a CPU-only ggml build (useful for consumers like @qvac/classification-ggml that don't need GPU paths and want to skip the vulkan-sdk / metal / opencl build cost). Orthogonal to the 'llama' feature.", - "dependencies": [ - { - "name": "spirv-headers", - "platform": "!osx & !ios", - "version>=": "1.4.341.0" - } - ] - }, - "kleidiai": { - "description": "Enable ARM KleidiAI optimized kernels on Android." - }, - "llama": { - "description": "Build llama components." - }, - "openmp": { - "description": "Enable openmp on Android." - } - } -} From 9f77d44b479bcecdd5cd6418a169727e0c925dcd Mon Sep 17 00:00:00 2001 From: gianni-cor Date: Mon, 22 Jun 2026 11:09:25 +0200 Subject: [PATCH 54/58] chore[notask]: restore embed cmake formatting Assisted-by: GPT-5.5 Co-authored-by: Cursor --- packages/embed-llamacpp/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/embed-llamacpp/CMakeLists.txt b/packages/embed-llamacpp/CMakeLists.txt index b86aa41db0..d93fe44c7c 100644 --- a/packages/embed-llamacpp/CMakeLists.txt +++ b/packages/embed-llamacpp/CMakeLists.txt @@ -56,7 +56,7 @@ if((ANDROID OR UNIX) AND NOT APPLE) foreach(_backend ${GGML_AVAILABLE_BACKENDS}) list(APPEND BACKEND_DL_LIBS INSTALL TARGET ggml::${_backend}) endforeach() -endif() +endif() add_bare_module(embed-llamacpp EXPORTS ${BACKEND_DL_LIBS}) @@ -101,7 +101,7 @@ if(BUILD_TESTING) find_package(GTest CONFIG REQUIRED) include(GoogleTest) enable_testing() - + add_subdirectory(test/unit) endif() From 79e769efb792bebdf7244537711b7f62137f8ebf Mon Sep 17 00:00:00 2001 From: gianni-cor Date: Mon, 22 Jun 2026 11:44:34 +0200 Subject: [PATCH 55/58] fix[llm]: reject OpenCL quantized KV caches Assisted-by: GPT-5.5 Co-authored-by: Cursor --- .../addon/src/model-interface/LlamaModel.cpp | 39 ++++++++++++------- .../addon/src/model-interface/LlamaModel.hpp | 5 +-- .../test/unit/test_tune_config_map.cpp | 39 +++++++++++++++++++ 3 files changed, 66 insertions(+), 17 deletions(-) diff --git a/packages/llm-llamacpp/addon/src/model-interface/LlamaModel.cpp b/packages/llm-llamacpp/addon/src/model-interface/LlamaModel.cpp index c216923a15..fc5cbbaeaf 100644 --- a/packages/llm-llamacpp/addon/src/model-interface/LlamaModel.cpp +++ b/packages/llm-llamacpp/addon/src/model-interface/LlamaModel.cpp @@ -211,13 +211,10 @@ void LlamaModel::tuneConfigMap( } } - // TurboQuant / PolarQuant KV-cache types (tbq3_0 / tbq4_0 / pq3_0 / pq4_0, - // ship Vulkan + CPU implementations only. On the OpenCL backend - // the kernels don't exist; on Metal the standalone MUL_MAT pipelines are - // explicitly disabled. Silently letting the user proceed lets llama.cpp - // commit KV-cache tensors to a backend that can't run the ops on them, - // which then aborts in ggml_backend_sched_split_graph at model load. - // Surface a clean error here instead. + // Quantized KV-cache types are fragile on OpenCL: standard q-cache types can + // fail later during cache shifts, while TBQ/PQ kernels are not implemented. + // Surface a clean error here instead of letting llama.cpp commit KV-cache + // tensors to a backend that can't run the required ops. #if defined(__APPLE__) constexpr bool kIsMetal = true; #else @@ -227,6 +224,10 @@ void LlamaModel::tuneConfigMap( auto isTurboQuantKvType = [](const std::string& v) { return v == "tbq3_0" || v == "tbq4_0" || v == "pq3_0" || v == "pq4_0"; }; + auto isQuantizedKvType = [&](const std::string& v) { + return isTurboQuantKvType(v) || v == "q4_0" || v == "q4_1" || + v == "q5_0" || v == "q5_1" || v == "q8_0" || v == "iq4_nl"; + }; auto checkCacheType = [&](const char* hyphenKey, const char* underscoreKey, const char* side) { @@ -235,20 +236,30 @@ void LlamaModel::tuneConfigMap( it = configFilemap.find(underscoreKey); if (it == configFilemap.end()) return; - if (!isTurboQuantKvType(it->second)) + if (isOpenCl) { + if (!isQuantizedKvType(it->second)) + return; + } else if (!isTurboQuantKvType(it->second)) { return; + } const char* backendName = isOpenCl ? "OpenCL" : "Metal"; + const char* typeName = + isTurboQuantKvType(it->second) ? "TurboQuant/PolarQuant" + : "quantized"; + const char* alternatives = + isOpenCl ? "f32/f16/bf16" + : "f32/f16/bf16/q4_0/q4_1/q5_0/q5_1/q8_0/iq4_nl"; throw qvac_errors::StatusError( qvac_errors::general_error::InvalidArgument, string_format( - "[LlamaModel] cache-type-%s=%s is a TurboQuant/PolarQuant type " - "and is not supported on the %s backend (Vulkan and CPU only). " - "Either pick a different cache type (f32/f16/bf16/q4_0/q4_1/" - "q5_0/q5_1/q8_0/iq4_nl) or switch device to a Vulkan GPU or " - "CPU.\n", + "[LlamaModel] cache-type-%s=%s is a %s KV-cache type and is not " + "supported on the %s backend. Either pick a different cache " + "type (%s) or switch device to a Vulkan GPU or CPU.\n", side, it->second.c_str(), - backendName)); + typeName, + backendName, + alternatives)); }; checkCacheType("cache-type-k", "cache_type_k", "k"); checkCacheType("cache-type-v", "cache_type_v", "v"); diff --git a/packages/llm-llamacpp/addon/src/model-interface/LlamaModel.hpp b/packages/llm-llamacpp/addon/src/model-interface/LlamaModel.hpp index adb93e8a1b..f9390af7ff 100644 --- a/packages/llm-llamacpp/addon/src/model-interface/LlamaModel.hpp +++ b/packages/llm-llamacpp/addon/src/model-interface/LlamaModel.hpp @@ -67,9 +67,8 @@ class LlamaModel : public IModel, public IModelAsyncLoad, public IModelCancel { /// @param adrenoVersion Detected Adreno GPU version, if any. /// @param finetuneOverrides If set, finetuning mode is active with these /// context/batch params and GPU caps. - /// @param isOpenCl True when the chosen GPU backend is OpenCL; used to - /// disable flash-attn by default since it is not reliably supported on - /// the OpenCL backend. + /// @param isOpenCl True when the chosen GPU backend is OpenCL; used to reject + /// unsupported quantized KV-cache types. static void tuneConfigMap( std::unordered_map& configFilemap, const ModelMetaData& metadata, const std::optional& adrenoVersion, diff --git a/packages/llm-llamacpp/test/unit/test_tune_config_map.cpp b/packages/llm-llamacpp/test/unit/test_tune_config_map.cpp index 792b1bd2f8..d3d341c980 100644 --- a/packages/llm-llamacpp/test/unit/test_tune_config_map.cpp +++ b/packages/llm-llamacpp/test/unit/test_tune_config_map.cpp @@ -3,6 +3,7 @@ #include #include +#include #include "model-interface/LlamaModel.hpp" #include "test_common.hpp" @@ -234,6 +235,44 @@ TEST_F(TuneConfigMapTest, NotOpenCl_NonBitnet_FlashAttnDefaultsOn) { EXPECT_EQ(configFilemap_["flash-attn"], "on"); } +TEST_F(TuneConfigMapTest, OpenCl_RejectsStandardQuantizedKCache) { + MockModelMetaData meta(false, "llama"); + configFilemap_["cache-type-k"] = "q8_0"; + + EXPECT_THROW( + LlamaModel::tuneConfigMap( + configFilemap_, meta, std::nullopt, FtOverrides{}, /*isOpenCl=*/true), + qvac_errors::StatusError); +} + +TEST_F(TuneConfigMapTest, OpenCl_RejectsStandardQuantizedVCacheUnderscore) { + MockModelMetaData meta(false, "llama"); + configFilemap_["cache_type_v"] = "q4_0"; + + EXPECT_THROW( + LlamaModel::tuneConfigMap( + configFilemap_, meta, std::nullopt, FtOverrides{}, /*isOpenCl=*/true), + qvac_errors::StatusError); +} + +TEST_F(TuneConfigMapTest, OpenCl_AllowsNonQuantizedCacheTypes) { + MockModelMetaData meta(false, "llama"); + configFilemap_["cache-type-k"] = "f16"; + configFilemap_["cache-type-v"] = "bf16"; + + EXPECT_NO_THROW(LlamaModel::tuneConfigMap( + configFilemap_, meta, std::nullopt, FtOverrides{}, /*isOpenCl=*/true)); +} + +TEST_F(TuneConfigMapTest, NotOpenCl_AllowsStandardQuantizedCacheTypes) { + MockModelMetaData meta(false, "llama"); + configFilemap_["cache-type-k"] = "q8_0"; + configFilemap_["cache-type-v"] = "q4_0"; + + EXPECT_NO_THROW(LlamaModel::tuneConfigMap( + configFilemap_, meta, std::nullopt, FtOverrides{}, /*isOpenCl=*/false)); +} + // ---- Finetuning: flash-attn disabled for any architecture ---- TEST_F(TuneConfigMapTest, Finetuning_Gemma3_FlashAttnDisabled) { From fb1be8ddc9c20ac0c827d589a87246bf9c85b03e Mon Sep 17 00:00:00 2001 From: gianni-cor Date: Mon, 22 Jun 2026 12:00:47 +0200 Subject: [PATCH 56/58] fix[llm]: format OpenCL KV cache guard Assisted-by: GPT-5.5 Co-authored-by: Cursor --- .../addon/src/model-interface/LlamaModel.cpp | 6 +++--- .../test/unit/test_tune_config_map.cpp | 18 ++++++++++++++---- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/packages/llm-llamacpp/addon/src/model-interface/LlamaModel.cpp b/packages/llm-llamacpp/addon/src/model-interface/LlamaModel.cpp index fc5cbbaeaf..fbacc0a652 100644 --- a/packages/llm-llamacpp/addon/src/model-interface/LlamaModel.cpp +++ b/packages/llm-llamacpp/addon/src/model-interface/LlamaModel.cpp @@ -243,9 +243,9 @@ void LlamaModel::tuneConfigMap( return; } const char* backendName = isOpenCl ? "OpenCL" : "Metal"; - const char* typeName = - isTurboQuantKvType(it->second) ? "TurboQuant/PolarQuant" - : "quantized"; + const char* typeName = isTurboQuantKvType(it->second) + ? "TurboQuant/PolarQuant" + : "quantized"; const char* alternatives = isOpenCl ? "f32/f16/bf16" : "f32/f16/bf16/q4_0/q4_1/q5_0/q5_1/q8_0/iq4_nl"; diff --git a/packages/llm-llamacpp/test/unit/test_tune_config_map.cpp b/packages/llm-llamacpp/test/unit/test_tune_config_map.cpp index d3d341c980..2bdc491161 100644 --- a/packages/llm-llamacpp/test/unit/test_tune_config_map.cpp +++ b/packages/llm-llamacpp/test/unit/test_tune_config_map.cpp @@ -260,8 +260,13 @@ TEST_F(TuneConfigMapTest, OpenCl_AllowsNonQuantizedCacheTypes) { configFilemap_["cache-type-k"] = "f16"; configFilemap_["cache-type-v"] = "bf16"; - EXPECT_NO_THROW(LlamaModel::tuneConfigMap( - configFilemap_, meta, std::nullopt, FtOverrides{}, /*isOpenCl=*/true)); + EXPECT_NO_THROW( + LlamaModel::tuneConfigMap( + configFilemap_, + meta, + std::nullopt, + FtOverrides{}, + /*isOpenCl=*/true)); } TEST_F(TuneConfigMapTest, NotOpenCl_AllowsStandardQuantizedCacheTypes) { @@ -269,8 +274,13 @@ TEST_F(TuneConfigMapTest, NotOpenCl_AllowsStandardQuantizedCacheTypes) { configFilemap_["cache-type-k"] = "q8_0"; configFilemap_["cache-type-v"] = "q4_0"; - EXPECT_NO_THROW(LlamaModel::tuneConfigMap( - configFilemap_, meta, std::nullopt, FtOverrides{}, /*isOpenCl=*/false)); + EXPECT_NO_THROW( + LlamaModel::tuneConfigMap( + configFilemap_, + meta, + std::nullopt, + FtOverrides{}, + /*isOpenCl=*/false)); } // ---- Finetuning: flash-attn disabled for any architecture ---- From 6bc7bc405c2fafb63958f7a035d6a5c960082c09 Mon Sep 17 00:00:00 2001 From: gianni-cor Date: Mon, 22 Jun 2026 12:55:31 +0200 Subject: [PATCH 57/58] fix[llm]: gate Metal KV cache guard on backend Co-authored-by: Cursor --- .../addon/src/model-interface/LlamaModel.cpp | 16 +++---- .../addon/src/model-interface/LlamaModel.hpp | 4 +- .../test/unit/test_tune_config_map.cpp | 45 +++++++++++++++++++ 3 files changed, 56 insertions(+), 9 deletions(-) diff --git a/packages/llm-llamacpp/addon/src/model-interface/LlamaModel.cpp b/packages/llm-llamacpp/addon/src/model-interface/LlamaModel.cpp index fbacc0a652..3ecf3148d3 100644 --- a/packages/llm-llamacpp/addon/src/model-interface/LlamaModel.cpp +++ b/packages/llm-llamacpp/addon/src/model-interface/LlamaModel.cpp @@ -98,7 +98,8 @@ void LlamaModel::resolveShardPaths( void LlamaModel::tuneConfigMap( std::unordered_map& configFilemap, const ModelMetaData& metadata, const std::optional& adrenoVersion, - const FinetuneConfigOverrides& finetuneOverrides, bool isOpenCl) { + const FinetuneConfigOverrides& finetuneOverrides, bool isOpenCl, + bool isMetal) { const bool isFinetuning = finetuneOverrides.active; @@ -215,12 +216,7 @@ void LlamaModel::tuneConfigMap( // fail later during cache shifts, while TBQ/PQ kernels are not implemented. // Surface a clean error here instead of letting llama.cpp commit KV-cache // tensors to a backend that can't run the required ops. -#if defined(__APPLE__) - constexpr bool kIsMetal = true; -#else - constexpr bool kIsMetal = false; -#endif - if (isOpenCl || kIsMetal) { + if (isOpenCl || isMetal) { auto isTurboQuantKvType = [](const std::string& v) { return v == "tbq3_0" || v == "tbq4_0" || v == "pq3_0" || v == "pq4_0"; }; @@ -1125,6 +1121,7 @@ void LlamaModel::commonParamsParse( } bool isOpenCl = false; + bool isMetal = false; { using namespace backend_selection; const BackendType preferredBackend = @@ -1191,6 +1188,8 @@ void LlamaModel::commonParamsParse( isOpenCl = chosenBackend.first == BackendType::GPU && chosenBackend.second.find("opencl") != std::string::npos; + isMetal = chosenBackend.first == BackendType::GPU && + chosenBackend.second.find("metal") != std::string::npos; } tuneConfigMap( @@ -1198,7 +1197,8 @@ void LlamaModel::commonParamsParse( metadata_, outAdrenoVersion, pendingFinetuneOverrides_, - isOpenCl); + isOpenCl, + isMetal); // Handle both reverse-prompt variants for (const std::string& key : {"reverse-prompt", "reverse_prompt"}) { diff --git a/packages/llm-llamacpp/addon/src/model-interface/LlamaModel.hpp b/packages/llm-llamacpp/addon/src/model-interface/LlamaModel.hpp index f9390af7ff..a7ef25e507 100644 --- a/packages/llm-llamacpp/addon/src/model-interface/LlamaModel.hpp +++ b/packages/llm-llamacpp/addon/src/model-interface/LlamaModel.hpp @@ -69,11 +69,13 @@ class LlamaModel : public IModel, public IModelAsyncLoad, public IModelCancel { /// context/batch params and GPU caps. /// @param isOpenCl True when the chosen GPU backend is OpenCL; used to reject /// unsupported quantized KV-cache types. + /// @param isMetal True when the chosen GPU backend is Metal; used to reject + /// unsupported TurboQuant/PolarQuant KV-cache types. static void tuneConfigMap( std::unordered_map& configFilemap, const ModelMetaData& metadata, const std::optional& adrenoVersion, const FinetuneConfigOverrides& finetuneOverrides = {}, - bool isOpenCl = false); + bool isOpenCl = false, bool isMetal = false); /** * The Constructor for llama model. diff --git a/packages/llm-llamacpp/test/unit/test_tune_config_map.cpp b/packages/llm-llamacpp/test/unit/test_tune_config_map.cpp index 2bdc491161..8262920a13 100644 --- a/packages/llm-llamacpp/test/unit/test_tune_config_map.cpp +++ b/packages/llm-llamacpp/test/unit/test_tune_config_map.cpp @@ -283,6 +283,51 @@ TEST_F(TuneConfigMapTest, NotOpenCl_AllowsStandardQuantizedCacheTypes) { /*isOpenCl=*/false)); } +TEST_F(TuneConfigMapTest, NotOpenClNotMetal_AllowsTurboQuantCacheTypes) { + MockModelMetaData meta(false, "llama"); + configFilemap_["cache-type-k"] = "tbq4_0"; + configFilemap_["cache-type-v"] = "pq4_0"; + + EXPECT_NO_THROW( + LlamaModel::tuneConfigMap( + configFilemap_, + meta, + std::nullopt, + FtOverrides{}, + /*isOpenCl=*/false, + /*isMetal=*/false)); +} + +TEST_F(TuneConfigMapTest, Metal_RejectsTurboQuantCacheTypes) { + MockModelMetaData meta(false, "llama"); + configFilemap_["cache-type-k"] = "tbq4_0"; + + EXPECT_THROW( + LlamaModel::tuneConfigMap( + configFilemap_, + meta, + std::nullopt, + FtOverrides{}, + /*isOpenCl=*/false, + /*isMetal=*/true), + qvac_errors::StatusError); +} + +TEST_F(TuneConfigMapTest, Metal_AllowsStandardQuantizedCacheTypes) { + MockModelMetaData meta(false, "llama"); + configFilemap_["cache-type-k"] = "q8_0"; + configFilemap_["cache-type-v"] = "q4_0"; + + EXPECT_NO_THROW( + LlamaModel::tuneConfigMap( + configFilemap_, + meta, + std::nullopt, + FtOverrides{}, + /*isOpenCl=*/false, + /*isMetal=*/true)); +} + // ---- Finetuning: flash-attn disabled for any architecture ---- TEST_F(TuneConfigMapTest, Finetuning_Gemma3_FlashAttnDisabled) { From bea0be13686bbe8d7037c04f5dceb661a3600ed0 Mon Sep 17 00:00:00 2001 From: gianni-cor Date: Mon, 22 Jun 2026 14:22:26 +0200 Subject: [PATCH 58/58] fix[llm]: detect mtl Metal backend names Co-authored-by: Cursor --- packages/llm-llamacpp/addon/src/model-interface/LlamaModel.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/llm-llamacpp/addon/src/model-interface/LlamaModel.cpp b/packages/llm-llamacpp/addon/src/model-interface/LlamaModel.cpp index 3ecf3148d3..3253145cc7 100644 --- a/packages/llm-llamacpp/addon/src/model-interface/LlamaModel.cpp +++ b/packages/llm-llamacpp/addon/src/model-interface/LlamaModel.cpp @@ -1189,7 +1189,8 @@ void LlamaModel::commonParamsParse( isOpenCl = chosenBackend.first == BackendType::GPU && chosenBackend.second.find("opencl") != std::string::npos; isMetal = chosenBackend.first == BackendType::GPU && - chosenBackend.second.find("metal") != std::string::npos; + (chosenBackend.second.find("metal") != std::string::npos || + chosenBackend.second.rfind("mtl", 0) == 0); } tuneConfigMap(