diff --git a/.github/workflows/cpp-tests-llm.yml b/.github/workflows/cpp-tests-llm.yml index b798baadfe..7e2eb69f59 100644 --- a/.github/workflows/cpp-tests-llm.yml +++ b/.github/workflows/cpp-tests-llm.yml @@ -37,7 +37,7 @@ jobs: platform: linux runner: qvac-ubuntu2404-x64 arch: x64 - - os: macos-14 + - os: macos-15-xlarge platform: darwin arch: arm64 - os: windows-2025 diff --git a/packages/llm-llamacpp/CMakeLists.txt b/packages/llm-llamacpp/CMakeLists.txt index ce1601da75..43f84e529d 100644 --- a/packages/llm-llamacpp/CMakeLists.txt +++ b/packages/llm-llamacpp/CMakeLists.txt @@ -172,7 +172,7 @@ if(BUILD_CLI) target_include_directories(cli_tool PRIVATE ${QVAC_LIB_INFERENCE_ADDON_CPP_INCLUDE_DIRS}) # Set C++20 standard for std::views support target_compile_features(cli_tool PRIVATE cxx_std_20) - + # Install ggml backends next to the CLI tool binary in build folder if((ANDROID OR UNIX) AND NOT APPLE) foreach(_backend ${GGML_AVAILABLE_BACKENDS}) @@ -190,7 +190,7 @@ if(BUILD_TESTING) find_package(GTest CONFIG REQUIRED) include(GoogleTest) enable_testing() - + # Integration tests for model classes (includes backend selection tests) # Pass ENABLE_COVERAGE option to test subdirectory add_subdirectory(test/unit) diff --git a/packages/llm-llamacpp/addon/src/addon/AddonJs.hpp b/packages/llm-llamacpp/addon/src/addon/AddonJs.hpp index 0cfe0b7998..512715034c 100644 --- a/packages/llm-llamacpp/addon/src/addon/AddonJs.hpp +++ b/packages/llm-llamacpp/addon/src/addon/AddonJs.hpp @@ -1,6 +1,8 @@ #pragma once #include +#include #include +#include #include #include #include @@ -371,15 +373,18 @@ inline void parseGenerationParams( auto reasoningBudget = configObj->getOptionalPropertyAs( env, "reasoning_budget"); if (reasoningBudget.has_value()) { - // Exact compare (0 and -1 are representable doubles) so fractional - // inputs like 0.5 are rejected, not silently truncated. - if (*reasoningBudget != 0 && *reasoningBudget != -1) { + // Reject fractional inputs (0.5, -1.1, 32.7) by requiring the value to + // round-trip through int. -1 = unrestricted, 0 = disabled, N>0 caps the + // reasoning channel at N tokens via the budget sampler. + const double value = *reasoningBudget; + if (value < -1 || value != std::floor(value) || + value > static_cast(std::numeric_limits::max())) { throw StatusError( general_error::InvalidArgument, - "generationParams.reasoning_budget must be -1 (unrestricted) " - "or 0 (disabled)"); + "generationParams.reasoning_budget must be -1 (unrestricted), " + "0 (disabled), or a positive integer (token cap)"); } - overrides.reasoning_budget = static_cast(*reasoningBudget); + overrides.reasoning_budget = static_cast(value); } } diff --git a/packages/llm-llamacpp/addon/src/model-interface/GenerationParamsApply.cpp b/packages/llm-llamacpp/addon/src/model-interface/GenerationParamsApply.cpp index 815615b5da..fd8271e7b7 100644 --- a/packages/llm-llamacpp/addon/src/model-interface/GenerationParamsApply.cpp +++ b/packages/llm-llamacpp/addon/src/model-interface/GenerationParamsApply.cpp @@ -29,6 +29,21 @@ void applyGenerationOverridesToSampling( setIf(overrides.presence_penalty, sampling.penalty_present); setIf(overrides.repeat_penalty, sampling.penalty_repeat); + // Forward reasoning_budget into the budget-sampler's token cap. The + // template-specific start/end/forced vectors are refreshed after prompt + // formatting, where common_chat_params exposes the thinking tags. + if (overrides.reasoning_budget) { + const int budget = *overrides.reasoning_budget; + if (budget > 0) { + sampling.reasoning_budget_tokens = budget; + } else { + sampling.reasoning_budget_tokens = -1; + sampling.reasoning_budget_start.clear(); + sampling.reasoning_budget_end.clear(); + sampling.reasoning_budget_forced.clear(); + } + } + // `json_schema` and `grammar` are mutually exclusive at the JS boundary // and in `AddonJs::runJob::parseText`, so reaching this branch with both // set means a caller bypassed those checks (most likely the C++ unit diff --git a/packages/llm-llamacpp/addon/src/model-interface/LlamaModel.cpp b/packages/llm-llamacpp/addon/src/model-interface/LlamaModel.cpp index fe1b8c500a..3253145cc7 100644 --- a/packages/llm-llamacpp/addon/src/model-interface/LlamaModel.cpp +++ b/packages/llm-llamacpp/addon/src/model-interface/LlamaModel.cpp @@ -98,7 +98,8 @@ void LlamaModel::resolveShardPaths( void LlamaModel::tuneConfigMap( std::unordered_map& configFilemap, const ModelMetaData& metadata, const std::optional& adrenoVersion, - const FinetuneConfigOverrides& finetuneOverrides, bool isOpenCl) { + const FinetuneConfigOverrides& finetuneOverrides, bool isOpenCl, + bool isMetal) { const bool isFinetuning = finetuneOverrides.active; @@ -143,13 +144,11 @@ void LlamaModel::tuneConfigMap( QLOG_IF( Priority::INFO, "[LlamaModel] BitNet model detected: disabling flash attention\n"); - } else if (isOpenCl && notUserSet("flash-attn", "flash_attn")) { + } else if (notUserSet("flash-attn", "flash_attn")) { configFilemap.erase("flash_attn"); - configFilemap["flash-attn"] = "off"; + configFilemap["flash-attn"] = "on"; QLOG_IF( - Priority::INFO, - "[LlamaModel] OpenCL backend selected: disabling flash attention by " - "default (not reliably supported on OpenCL)\n"); + Priority::INFO, "[LlamaModel] Enabling flash attention by default\n"); } constexpr int kAdrenoUbatchThreshold = 800; @@ -213,22 +212,18 @@ void LlamaModel::tuneConfigMap( } } - // TurboQuant / PolarQuant KV-cache types (tbq3_0 / tbq4_0 / pq3_0 / pq4_0, - // ship Vulkan + CPU implementations only. On the OpenCL backend - // the kernels don't exist; on Metal the standalone MUL_MAT pipelines are - // explicitly disabled. Silently letting the user proceed lets llama.cpp - // commit KV-cache tensors to a backend that can't run the ops on them, - // which then aborts in ggml_backend_sched_split_graph at model load. - // Surface a clean error here instead. -#if defined(__APPLE__) - constexpr bool kIsMetal = true; -#else - constexpr bool kIsMetal = false; -#endif - if (isOpenCl || kIsMetal) { + // Quantized KV-cache types are fragile on OpenCL: standard q-cache types can + // fail later during cache shifts, while TBQ/PQ kernels are not implemented. + // Surface a clean error here instead of letting llama.cpp commit KV-cache + // tensors to a backend that can't run the required ops. + if (isOpenCl || isMetal) { auto isTurboQuantKvType = [](const std::string& v) { return v == "tbq3_0" || v == "tbq4_0" || v == "pq3_0" || v == "pq4_0"; }; + auto isQuantizedKvType = [&](const std::string& v) { + return isTurboQuantKvType(v) || v == "q4_0" || v == "q4_1" || + v == "q5_0" || v == "q5_1" || v == "q8_0" || v == "iq4_nl"; + }; auto checkCacheType = [&](const char* hyphenKey, const char* underscoreKey, const char* side) { @@ -237,20 +232,30 @@ void LlamaModel::tuneConfigMap( it = configFilemap.find(underscoreKey); if (it == configFilemap.end()) return; - if (!isTurboQuantKvType(it->second)) + if (isOpenCl) { + if (!isQuantizedKvType(it->second)) + return; + } else if (!isTurboQuantKvType(it->second)) { return; + } const char* backendName = isOpenCl ? "OpenCL" : "Metal"; + const char* typeName = isTurboQuantKvType(it->second) + ? "TurboQuant/PolarQuant" + : "quantized"; + const char* alternatives = + isOpenCl ? "f32/f16/bf16" + : "f32/f16/bf16/q4_0/q4_1/q5_0/q5_1/q8_0/iq4_nl"; throw qvac_errors::StatusError( qvac_errors::general_error::InvalidArgument, string_format( - "[LlamaModel] cache-type-%s=%s is a TurboQuant/PolarQuant type " - "and is not supported on the %s backend (Vulkan and CPU only). " - "Either pick a different cache type (f32/f16/bf16/q4_0/q4_1/" - "q5_0/q5_1/q8_0/iq4_nl) or switch device to a Vulkan GPU or " - "CPU.\n", + "[LlamaModel] cache-type-%s=%s is a %s KV-cache type and is not " + "supported on the %s backend. Either pick a different cache " + "type (%s) or switch device to a Vulkan GPU or CPU.\n", side, it->second.c_str(), - backendName)); + typeName, + backendName, + alternatives)); }; checkCacheType("cache-type-k", "cache_type_k", "k"); checkCacheType("cache-type-v", "cache_type_v", "v"); @@ -994,21 +999,23 @@ void LlamaModel::commonParamsParse( "embedded chat template is applied\n"); } - // reasoning-budget controls whether the model emits a reasoning - // channel. -1 (default) leaves it on; 0 disables. `std::from_chars` is used - // instead of `std::stoi` because the latter accepts trailing garbage ("0abc" - // → 0) and throws an uncaught `std::out_of_range` on overflow. + // reasoning-budget controls the size of the model's reasoning + // channel: -1 (default) leaves it unrestricted, 0 disables thinking + // entirely, any positive N caps the reasoning channel at N tokens (the + // budget sampler forces once N reasoning tokens have been + // emitted). auto parseReasoningBudget = [](const std::string& raw) { int value = 0; const char* begin = raw.data(); const char* end = begin + raw.size(); const auto [ptr, ec] = std::from_chars(begin, end, value); - if (ec != std::errc{} || ptr != end || (value != 0 && value != -1)) { + if (ec != std::errc{} || ptr != end || value < -1) { throw qvac_errors::StatusError( ADDON_ID, qvac_errors::general_error::toString( qvac_errors::general_error::InvalidArgument), - "reasoning-budget must be -1 (unrestricted) or 0 (disabled)"); + "reasoning-budget must be -1 (unrestricted), 0 (disabled), or a " + "positive integer (token cap)"); } return value; }; @@ -1114,6 +1121,7 @@ void LlamaModel::commonParamsParse( } bool isOpenCl = false; + bool isMetal = false; { using namespace backend_selection; const BackendType preferredBackend = @@ -1180,6 +1188,9 @@ void LlamaModel::commonParamsParse( isOpenCl = chosenBackend.first == BackendType::GPU && chosenBackend.second.find("opencl") != std::string::npos; + isMetal = chosenBackend.first == BackendType::GPU && + (chosenBackend.second.find("metal") != std::string::npos || + chosenBackend.second.rfind("mtl", 0) == 0); } tuneConfigMap( @@ -1187,7 +1198,8 @@ void LlamaModel::commonParamsParse( metadata_, outAdrenoVersion, pendingFinetuneOverrides_, - isOpenCl); + isOpenCl, + isMetal); // Handle both reverse-prompt variants for (const std::string& key : {"reverse-prompt", "reverse_prompt"}) { @@ -1310,10 +1322,6 @@ void LlamaModel::commonParamsParse( postprocess_cpu_params(params.cpuparams, nullptr); postprocess_cpu_params(params.cpuparams_batch, ¶ms.cpuparams); - postprocess_cpu_params(params.speculative.draft.cpuparams, ¶ms.cpuparams); - postprocess_cpu_params( - params.speculative.draft.cpuparams_batch, ¶ms.cpuparams_batch); - if (!params.kv_overrides.empty()) { params.kv_overrides.emplace_back(); params.kv_overrides.back().key[0] = 0; diff --git a/packages/llm-llamacpp/addon/src/model-interface/LlamaModel.hpp b/packages/llm-llamacpp/addon/src/model-interface/LlamaModel.hpp index adb93e8a1b..a7ef25e507 100644 --- a/packages/llm-llamacpp/addon/src/model-interface/LlamaModel.hpp +++ b/packages/llm-llamacpp/addon/src/model-interface/LlamaModel.hpp @@ -67,14 +67,15 @@ class LlamaModel : public IModel, public IModelAsyncLoad, public IModelCancel { /// @param adrenoVersion Detected Adreno GPU version, if any. /// @param finetuneOverrides If set, finetuning mode is active with these /// context/batch params and GPU caps. - /// @param isOpenCl True when the chosen GPU backend is OpenCL; used to - /// disable flash-attn by default since it is not reliably supported on - /// the OpenCL backend. + /// @param isOpenCl True when the chosen GPU backend is OpenCL; used to reject + /// unsupported quantized KV-cache types. + /// @param isMetal True when the chosen GPU backend is Metal; used to reject + /// unsupported TurboQuant/PolarQuant KV-cache types. static void tuneConfigMap( std::unordered_map& configFilemap, const ModelMetaData& metadata, const std::optional& adrenoVersion, const FinetuneConfigOverrides& finetuneOverrides = {}, - bool isOpenCl = false); + bool isOpenCl = false, bool isMetal = false); /** * The Constructor for llama model. diff --git a/packages/llm-llamacpp/addon/src/model-interface/LlmContext.hpp b/packages/llm-llamacpp/addon/src/model-interface/LlmContext.hpp index 2b18b1e030..474c24b9b7 100644 --- a/packages/llm-llamacpp/addon/src/model-interface/LlmContext.hpp +++ b/packages/llm-llamacpp/addon/src/model-interface/LlmContext.hpp @@ -34,10 +34,11 @@ struct GenerationParams { // Mutually exclusive with `grammar` — the JS wrapper rejects requests // that set both. Mirrors the load-time `--json-schema` flag. std::optional json_schema; - // Reasoning channel budget override. `-1` keeps reasoning on, `0` disables - // it for this request. Mirrors the load-time `reasoning-budget` config; the - // override is applied to `params_.reasoning_budget` for the duration of the - // request and restored on completion. + // Reasoning channel budget override. `-1` keeps reasoning unrestricted, `0` + // disables it, and positive values cap the reasoning channel at that many + // tokens. Mirrors the load-time `reasoning-budget` config; the override is + // applied to `params_.reasoning_budget` for the duration of the request and + // restored on completion. std::optional reasoning_budget; [[nodiscard]] bool hasOverrides() const { diff --git a/packages/llm-llamacpp/addon/src/model-interface/MtmdLlmContext.cpp b/packages/llm-llamacpp/addon/src/model-interface/MtmdLlmContext.cpp index 788d376c71..9e59e5cdc6 100644 --- a/packages/llm-llamacpp/addon/src/model-interface/MtmdLlmContext.cpp +++ b/packages/llm-llamacpp/addon/src/model-interface/MtmdLlmContext.cpp @@ -216,7 +216,20 @@ void MtmdLlmContext::tokenizeChat( if (!tools.empty()) { inputs.tools = tools; } - formattedChat = getPrompt(tmpls_.get(), inputs, &thinkingForcedOpen_); + std::string thinkingStartTag; + std::string thinkingEndTag; + std::string generationPrompt; + formattedChat = getPrompt( + tmpls_.get(), + inputs, + &thinkingForcedOpen_, + &thinkingStartTag, + &thinkingEndTag, + &generationPrompt); + thinkingForcedOpenText_ = + thinkingForcedOpen_ + ? getThinkingForcedOpenText(generationPrompt, thinkingStartTag) + : std::string{}; if (formattedChat.empty()) { std::string errorMsg = string_format( @@ -224,6 +237,21 @@ void MtmdLlmContext::tokenizeChat( throw qvac_errors::StatusError(ADDON_ID, toString(EmptyPrompt), errorMsg); } + if (configureReasoningBudgetSampling( + params_, + modelCtx_.lctx, + thinkingStartTag, + thinkingEndTag, + generationPrompt)) { + smpl_.reset(common_sampler_init(modelCtx_.model, params_.sampling)); + if (!smpl_) { + std::string errorMsg = string_format( + "[MtmdLlm] %s: failed to initialize sampling subsystem\n", __func__); + throw qvac_errors::StatusError( + ADDON_ID, toString(UnableToCreateSamplingSystem), errorMsg); + } + } + QLOG_IF( Priority::DEBUG, string_format("[MtmdLlm] formatted prompt: %s\n", formattedChat.c_str())); @@ -492,10 +520,10 @@ bool MtmdLlmContext::generateResponse( if (thinkingForcedOpen_ && outputCallback) { // MtmdLlmContext doesn't carry a reasoningState_ (no reasoning-aware EOS // replacement on the multimodal path today), so unlike TextLlmContext we - // only prepend the visible "\n" opener and don't flip an + // only prepend the visible reasoning opener and don't flip an // inside_reasoning flag. If reasoning state is added here later, mirror // TextLlmContext::generateResponse and set it true alongside this emit. - outputCallback("\n"); + outputCallback(thinkingForcedOpenText_); } if (stopGeneration_.load()) { @@ -717,6 +745,8 @@ void MtmdLlmContext::resetState(bool resetStats) { // Clear UTF-8 buffer when resetting state utf8Buffer_.clear(); + thinkingForcedOpen_ = false; + thinkingForcedOpenText_.clear(); clearSequenceMemory(modelCtx_.lctx); diff --git a/packages/llm-llamacpp/addon/src/model-interface/MtmdLlmContext.hpp b/packages/llm-llamacpp/addon/src/model-interface/MtmdLlmContext.hpp index 356743c796..dcca7c7379 100644 --- a/packages/llm-llamacpp/addon/src/model-interface/MtmdLlmContext.hpp +++ b/packages/llm-llamacpp/addon/src/model-interface/MtmdLlmContext.hpp @@ -230,10 +230,11 @@ class MtmdLlmContext : public LlmContext { bool isHarmonyModel_ = false; llama_token harmonyCallToken_ = LLAMA_TOKEN_NULL; - // Force-opens the reasoning channel in the prompt suffix to prepend the - // matching "\n" opener to the visible stream so consumers see balanced + // Force-opens the reasoning channel in the prompt suffix. The text mirrors + // the template-specific visible reasoning opener so consumers see balanced // tags. bool thinkingForcedOpen_ = false; + std::string thinkingForcedOpenText_; std::atomic stopGeneration_ = false; }; diff --git a/packages/llm-llamacpp/addon/src/model-interface/TextLlmContext.cpp b/packages/llm-llamacpp/addon/src/model-interface/TextLlmContext.cpp index e7aa4860ee..7c9a60e230 100644 --- a/packages/llm-llamacpp/addon/src/model-interface/TextLlmContext.cpp +++ b/packages/llm-llamacpp/addon/src/model-interface/TextLlmContext.cpp @@ -284,7 +284,34 @@ void TextLlmContext::tokenizeChat( if (!tools.empty()) { inputs.tools = tools; } - prompt = getPrompt(tmpls_.get(), inputs, &thinkingForcedOpen_); + std::string thinkingStartTag; + std::string thinkingEndTag; + std::string generationPrompt; + prompt = getPrompt( + tmpls_.get(), + inputs, + &thinkingForcedOpen_, + &thinkingStartTag, + &thinkingEndTag, + &generationPrompt); + thinkingForcedOpenText_ = + thinkingForcedOpen_ + ? getThinkingForcedOpenText(generationPrompt, thinkingStartTag) + : std::string{}; + if (configureReasoningBudgetSampling( + params_, + modelCtx_.lctx, + thinkingStartTag, + thinkingEndTag, + generationPrompt)) { + smpl_.reset(common_sampler_init(modelCtx_.model, params_.sampling)); + if (!smpl_) { + std::string errorMsg = string_format( + "[TextLlm] %s: failed to initialize sampling subsystem\n", __func__); + throw qvac_errors::StatusError( + ADDON_ID, toString(UnableToCreateSamplingSystem), errorMsg); + } + } QLOG_IF( Priority::DEBUG, @@ -598,12 +625,10 @@ bool TextLlmContext::generateResponse( assistantOutput_.clear(); generationStarted_ = false; - // The chat template force-opened the reasoning channel in the prompt (e.g. - // Qwen3-style / DeepSeek-R1 templates end with "\n"), so the model - // resumes generation INSIDE the reasoning block. (Gemma4's reasoning channel - // is model-emitted upstream and does not set this flag.) + // The chat template force-opened the reasoning channel in the prompt, so the + // model resumes generation inside the reasoning block. if (thinkingForcedOpen_ && outputCallback) { - outputCallback("\n"); + outputCallback(thinkingForcedOpenText_); reasoningState_.inside_reasoning = true; } @@ -899,6 +924,8 @@ void TextLlmContext::resetState(bool resetStats) { forcedTokens_.clear(); assistantOutput_.clear(); generationStarted_ = false; + thinkingForcedOpen_ = false; + thinkingForcedOpenText_.clear(); clearSequenceMemory(modelCtx_.lctx); diff --git a/packages/llm-llamacpp/addon/src/model-interface/TextLlmContext.hpp b/packages/llm-llamacpp/addon/src/model-interface/TextLlmContext.hpp index d53764c9c0..46d3068a64 100644 --- a/packages/llm-llamacpp/addon/src/model-interface/TextLlmContext.hpp +++ b/packages/llm-llamacpp/addon/src/model-interface/TextLlmContext.hpp @@ -270,10 +270,11 @@ class TextLlmContext : public LlmContext, public SequenceDriver { bool isHarmonyModel_ = false; llama_token harmonyCallToken_ = LLAMA_TOKEN_NULL; - // Force-opens the reasoning channel in the prompt suffix to prepend the - // matching "\n" opener to the visible stream so consumers see balanced + // Force-opens the reasoning channel in the prompt suffix. The text mirrors + // the template-specific visible reasoning opener so consumers see balanced // tags. bool thinkingForcedOpen_ = false; + std::string thinkingForcedOpenText_; std::atomic stopGeneration_ = false; }; diff --git a/packages/llm-llamacpp/addon/src/utils/ChatTemplateUtils.cpp b/packages/llm-llamacpp/addon/src/utils/ChatTemplateUtils.cpp index 9bc66778b5..3e06c1c249 100644 --- a/packages/llm-llamacpp/addon/src/utils/ChatTemplateUtils.cpp +++ b/packages/llm-llamacpp/addon/src/utils/ChatTemplateUtils.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include @@ -174,11 +175,22 @@ std::string getChatTemplate( std::string getPrompt( const struct common_chat_templates* tmpls, - struct common_chat_templates_inputs& inputs, bool* outThinkingForcedOpen) { + struct common_chat_templates_inputs& inputs, bool* outThinkingForcedOpen, + std::string* outThinkingStartTag, std::string* outThinkingEndTag, + std::string* outGenerationPrompt) { auto exportParams = [&](const common_chat_params& params) { if (outThinkingForcedOpen) { *outThinkingForcedOpen = params.thinking_forced_open; } + if (outThinkingStartTag) { + *outThinkingStartTag = params.thinking_start_tag; + } + if (outThinkingEndTag) { + *outThinkingEndTag = params.thinking_end_tag; + } + if (outGenerationPrompt) { + *outGenerationPrompt = params.generation_prompt; + } }; try { auto params = common_chat_templates_apply(tmpls, inputs); @@ -211,5 +223,57 @@ std::string getPrompt( } } +bool configureReasoningBudgetSampling( + common_params& params, ::llama_context* lctx, + const std::string& thinkingStartTag, const std::string& thinkingEndTag, + const std::string& generationPrompt) { + common_params_sampling next = params.sampling; + next.reasoning_budget_tokens = + params.reasoning_budget > 0 ? params.reasoning_budget : -1; + next.reasoning_budget_start.clear(); + next.reasoning_budget_end.clear(); + next.reasoning_budget_forced.clear(); + next.generation_prompt.clear(); + + if (params.reasoning_budget > 0 && lctx != nullptr && + !thinkingEndTag.empty()) { + next.generation_prompt = generationPrompt; + if (!thinkingStartTag.empty()) { + next.reasoning_budget_start = + common_tokenize(lctx, thinkingStartTag, false, true); + } + next.reasoning_budget_end = + common_tokenize(lctx, thinkingEndTag, false, true); + next.reasoning_budget_forced = common_tokenize( + lctx, + params.sampling.reasoning_budget_message + thinkingEndTag, + false, + true); + } + + const bool changed = + params.sampling.reasoning_budget_tokens != next.reasoning_budget_tokens || + params.sampling.reasoning_budget_start != next.reasoning_budget_start || + params.sampling.reasoning_budget_end != next.reasoning_budget_end || + params.sampling.reasoning_budget_forced != next.reasoning_budget_forced || + params.sampling.generation_prompt != next.generation_prompt; + if (changed) { + params.sampling = std::move(next); + } + return changed; +} + +std::string getThinkingForcedOpenText( + const std::string& generationPrompt, const std::string& thinkingStartTag) { + if (thinkingStartTag.empty()) { + return {}; + } + const auto start = generationPrompt.rfind(thinkingStartTag); + if (start == std::string::npos) { + return thinkingStartTag; + } + return generationPrompt.substr(start); +} + } // namespace utils } // namespace qvac_lib_inference_addon_llama diff --git a/packages/llm-llamacpp/addon/src/utils/ChatTemplateUtils.hpp b/packages/llm-llamacpp/addon/src/utils/ChatTemplateUtils.hpp index b523e51b98..3652d91464 100644 --- a/packages/llm-llamacpp/addon/src/utils/ChatTemplateUtils.hpp +++ b/packages/llm-llamacpp/addon/src/utils/ChatTemplateUtils.hpp @@ -71,11 +71,35 @@ std::string getChatTemplate( * * @p outThinkingForcedOpen (optional) receives the flag indicating that the * template force-opened the reasoning channel in the prompt suffix. + * @p outThinkingStartTag (optional) receives the template-specific reasoning + * start tag, when the template exposes one. + * @p outThinkingEndTag (optional) receives the template-specific reasoning + * end tag, when the template exposes one. + * @p outGenerationPrompt (optional) receives the assistant generation prompt + * already appended to the formatted prompt. */ std::string getPrompt( const struct common_chat_templates* tmpls, struct common_chat_templates_inputs& inputs, - bool* outThinkingForcedOpen = nullptr); + bool* outThinkingForcedOpen = nullptr, + std::string* outThinkingStartTag = nullptr, + std::string* outThinkingEndTag = nullptr, + std::string* outGenerationPrompt = nullptr); + +/** + * @brief Configures the common-sampling reasoning-budget fields from + * template-derived thinking tags. + * + * Returns true when the sampling block changed and the caller should recreate + * the common_sampler. + */ +bool configureReasoningBudgetSampling( + common_params& params, ::llama_context* lctx, + const std::string& thinkingStartTag, const std::string& thinkingEndTag, + const std::string& generationPrompt); + +std::string getThinkingForcedOpenText( + const std::string& generationPrompt, const std::string& thinkingStartTag); } // namespace utils } // namespace qvac_lib_inference_addon_llama diff --git a/packages/llm-llamacpp/index.d.ts b/packages/llm-llamacpp/index.d.ts index 95abd62d73..6d0caa71cd 100644 --- a/packages/llm-llamacpp/index.d.ts +++ b/packages/llm-llamacpp/index.d.ts @@ -74,8 +74,13 @@ export interface LlamaConfig { 'cache-type-v'?: string /** Writable directory for OpenCL kernel binary cache. Required on Android for fast GPU startup. */ openclCacheDir?: string - /** Reasoning channel budget. `-1` (default) leaves the model's reasoning channel on; `0` disables it. */ - reasoning_budget?: -1 | 0 | '-1' | '0' + /** + * Reasoning channel budget. `-1` (default) leaves the model's reasoning + * channel on; `0` disables it; any positive integer caps the reasoning + * channel at that many tokens (the sampler force-emits `` once + * the budget is exhausted). + */ + reasoning_budget?: number | `${number}` /** * Number of concurrent sequence slots for continuous-batching (`--parallel` / * `n_parallel` in llama.cpp). Values `>= 2` activate the continuous-batch @@ -157,11 +162,12 @@ export interface GenerationParams { json_schema?: string | Record /** * Per-request reasoning channel budget. `-1` keeps the model's reasoning - * channel on; `0` disables it for this request. Equivalent to the load-time + * channel on; `0` disables it for this request; any positive integer caps + * the reasoning channel at that many tokens. Equivalent to the load-time * `reasoning_budget` config but scoped to a single `run()` call; the prior * value is restored afterwards. */ - reasoning_budget?: -1 | 0 + reasoning_budget?: number } export interface RunOptions { diff --git a/packages/llm-llamacpp/scripts/generate-mobile-integration-tests.js b/packages/llm-llamacpp/scripts/generate-mobile-integration-tests.js index 87f1cbec2f..3560e2c6bb 100644 --- a/packages/llm-llamacpp/scripts/generate-mobile-integration-tests.js +++ b/packages/llm-llamacpp/scripts/generate-mobile-integration-tests.js @@ -9,9 +9,6 @@ const integrationDir = path.join(repoRoot, 'test', 'integration') const mobileDir = path.join(repoRoot, 'test', 'mobile') const outputFile = path.join(mobileDir, 'integration.auto.cjs') const groupsFile = path.join(mobileDir, 'test-groups.json') -const mobileExcludedTests = new Set([ - 'continuous-batching.test.js' -]) // The benchmark-perf-*.test.js shards are generated, not committed (see // .gitignore), but the committed integration.auto.cjs references them. Enumerating @@ -40,7 +37,6 @@ function getIntegrationFiles () { return fs.readdirSync(integrationDir) .filter(entry => entry.endsWith('.test.js')) - .filter(entry => !mobileExcludedTests.has(entry)) .sort() } diff --git a/packages/llm-llamacpp/test/integration/_vlm-image-perf.js b/packages/llm-llamacpp/test/integration/_vlm-image-perf.js index 268a2e51ac..a11295c1fe 100644 --- a/packages/llm-llamacpp/test/integration/_vlm-image-perf.js +++ b/packages/llm-llamacpp/test/integration/_vlm-image-perf.js @@ -36,12 +36,11 @@ function _envInt (key, fallback) { const PERF_RUNS = _envInt('QVAC_PERF_RUNS', 1) const PERF_WARMUP_RUNS = _envInt('QVAC_PERF_WARMUP_RUNS', 1) -// QVAC-19368: skip the heaviest image (aurora) only on Android non-benchmark -// on-PR runs where the 30-min Device Farm per-test cap is tight. iOS and -// desktop always run aurora (fast enough). The benchmark (QVAC_PERF_ONLY=true) -// always runs the full set regardless of platform. Uses the explicit -// QVAC_PERF_ONLY flag (already plumbed to the device via the testspec config) -// instead of proxying off PERF_RUNS, per review feedback. +// QVAC-19368: skip the heaviest image (aurora) on Android non-benchmark +// on-PR runs where the 30-min Device Farm per-test cap is tight. The +// benchmark (QVAC_PERF_ONLY=true) runs the full set on supported platforms. +// Uses the explicit QVAC_PERF_ONLY flag (already plumbed to the device via +// the testspec config) instead of proxying off PERF_RUNS, per review feedback. function _envStr (key) { if (typeof os.getEnv === 'function') return os.getEnv(key) || '' if (typeof process !== 'undefined' && process.env) return process.env[key] || '' @@ -216,6 +215,7 @@ module.exports = { IMAGE_CASES, GEMMA4_MODEL, QWEN35_MODEL, + isDarwinX64, skipHeavyImages, runVlmImagePerf } diff --git a/packages/llm-llamacpp/test/integration/cache-state-machine.test.js b/packages/llm-llamacpp/test/integration/cache-state-machine.test.js index 9e919270e5..6cdc37b706 100644 --- a/packages/llm-llamacpp/test/integration/cache-state-machine.test.js +++ b/packages/llm-llamacpp/test/integration/cache-state-machine.test.js @@ -3,7 +3,7 @@ const path = require('bare-path') const fs = require('bare-fs') const LlmLlamacpp = require('../../index.js') -const { cleanupIntegrationCacheFiles, ensureModel, safeTest } = require('./utils') +const { cleanupIntegrationCacheFiles, ensureModel, safeTest: integrationTest } = require('./utils') const { attachSpecLogger } = require('./spec-logger') const os = require('bare-os') @@ -11,6 +11,10 @@ const isDarwinX64 = os.platform() === 'darwin' && os.arch() === 'x64' const isLinuxArm64 = os.platform() === 'linux' && os.arch() === 'arm64' const useCpu = isDarwinX64 || isLinuxArm64 +function safeTest (name, opts, fn) { + integrationTest(name, { ...opts, skip: opts.skip || isDarwinX64 }, fn) +} + const DEFAULT_MODEL = { name: 'Llama-3.2-1B-Instruct-Q4_0.gguf', url: 'https://huggingface.co/bartowski/Llama-3.2-1B-Instruct-GGUF/resolve/main/Llama-3.2-1B-Instruct-Q4_0.gguf' diff --git a/packages/llm-llamacpp/test/integration/continuous-batching.test.js b/packages/llm-llamacpp/test/integration/continuous-batching.test.js index 2612c3b333..9928e7081e 100644 --- a/packages/llm-llamacpp/test/integration/continuous-batching.test.js +++ b/packages/llm-llamacpp/test/integration/continuous-batching.test.js @@ -1,11 +1,10 @@ 'use strict' -const test = require('brittle') const path = require('bare-path') const os = require('bare-os') const process = require('bare-process') const LlmLlamacpp = require('../../index.js') -const { ensureModel } = require('./utils') +const { ensureModel, safeTest } = require('./utils') const { attachSpecLogger } = require('./spec-logger') const platform = os.platform() @@ -152,7 +151,7 @@ function logStreamingProgress (response) { // 1B throughput/correctness run is too slow or complicated for mobile and legacy macOS x64. const skipHeavyPlatform = isMobile || isDarwin -test('continuous batching answers 16 prompts correctly and improves Linux GPU TPS', { timeout: 900_000, skip: skipHeavyPlatform }, async t => { +safeTest('continuous batching answers 16 prompts correctly and improves Linux GPU TPS', { timeout: 900_000, skip: skipHeavyPlatform }, async t => { const singleModel = await setupModel(t) const singleNativeTpsValues = [] const singleWallTpsValues = [] diff --git a/packages/llm-llamacpp/test/integration/gemma4-image-elephant-perf.test.js b/packages/llm-llamacpp/test/integration/gemma4-image-elephant-perf.test.js index eac37c3246..8bd2113ba3 100644 --- a/packages/llm-llamacpp/test/integration/gemma4-image-elephant-perf.test.js +++ b/packages/llm-llamacpp/test/integration/gemma4-image-elephant-perf.test.js @@ -4,9 +4,9 @@ // 30-minute mobile cap. Asserts the elephant keyword + records perf. const test = require('brittle') -const { GEMMA4_MODEL, IMAGE_CASES, runVlmImagePerf } = require('./_vlm-image-perf.js') +const { GEMMA4_MODEL, IMAGE_CASES, isDarwinX64, runVlmImagePerf } = require('./_vlm-image-perf.js') -test('Gemma4-VL image perf [elephant]', { timeout: 1_800_000 }, async t => { +test('Gemma4-VL image perf [elephant]', { timeout: 1_800_000, skip: isDarwinX64 }, async t => { await runVlmImagePerf(t, GEMMA4_MODEL, IMAGE_CASES.elephant) }) diff --git a/packages/llm-llamacpp/test/integration/gemma4-image-fruit-plate-perf.test.js b/packages/llm-llamacpp/test/integration/gemma4-image-fruit-plate-perf.test.js index 1e6f048f0b..8c695d460a 100644 --- a/packages/llm-llamacpp/test/integration/gemma4-image-fruit-plate-perf.test.js +++ b/packages/llm-llamacpp/test/integration/gemma4-image-fruit-plate-perf.test.js @@ -4,9 +4,9 @@ // 30-minute mobile cap. Asserts a fruit keyword + records perf. const test = require('brittle') -const { GEMMA4_MODEL, IMAGE_CASES, runVlmImagePerf } = require('./_vlm-image-perf.js') +const { GEMMA4_MODEL, IMAGE_CASES, isDarwinX64, runVlmImagePerf } = require('./_vlm-image-perf.js') -test('Gemma4-VL image perf [fruit plate]', { timeout: 1_800_000 }, async t => { +test('Gemma4-VL image perf [fruit plate]', { timeout: 1_800_000, skip: isDarwinX64 }, async t => { await runVlmImagePerf(t, GEMMA4_MODEL, IMAGE_CASES['fruit-plate']) }) diff --git a/packages/llm-llamacpp/test/integration/gemma4-image-high-res-aurora-perf.test.js b/packages/llm-llamacpp/test/integration/gemma4-image-high-res-aurora-perf.test.js index 0d05133b8b..ae0666843d 100644 --- a/packages/llm-llamacpp/test/integration/gemma4-image-high-res-aurora-perf.test.js +++ b/packages/llm-llamacpp/test/integration/gemma4-image-high-res-aurora-perf.test.js @@ -4,12 +4,12 @@ // the 30-minute mobile cap. Asserts an aurora keyword + records perf. const test = require('brittle') -const { GEMMA4_MODEL, IMAGE_CASES, skipHeavyImages, runVlmImagePerf } = require('./_vlm-image-perf.js') +const { GEMMA4_MODEL, IMAGE_CASES, isDarwinX64, skipHeavyImages, runVlmImagePerf } = require('./_vlm-image-perf.js') // QVAC-19368: aurora is the heaviest image; skip it on Android on-PR runs -// where the 30-min Device Farm cap is tight. iOS + desktop always run it. -// The benchmark (QVAC_PERF_ONLY=true) runs all 3 images on all platforms. -test('Gemma4-VL image perf [high-res aurora]', { timeout: 1_800_000, skip: skipHeavyImages }, async t => { +// where the 30-min Device Farm cap is tight. Darwin x64 is skipped separately. +// The benchmark (QVAC_PERF_ONLY=true) runs all 3 images on supported platforms. +test('Gemma4-VL image perf [high-res aurora]', { timeout: 1_800_000, skip: isDarwinX64 || skipHeavyImages }, async t => { await runVlmImagePerf(t, GEMMA4_MODEL, IMAGE_CASES['high-res-aurora']) }) diff --git a/packages/llm-llamacpp/test/integration/model-loading.test.js b/packages/llm-llamacpp/test/integration/model-loading.test.js index c1f084450d..989500b3a5 100644 --- a/packages/llm-llamacpp/test/integration/model-loading.test.js +++ b/packages/llm-llamacpp/test/integration/model-loading.test.js @@ -13,6 +13,7 @@ const arch = os.arch() const isDarwinX64 = platform === 'darwin' && arch === 'x64' const isLinuxArm64 = platform === 'linux' && arch === 'arm64' const isLinuxX64 = platform === 'linux' && arch === 'x64' +const isWindowsX64 = platform === 'win32' && arch === 'x64' const useCpu = isDarwinX64 || isLinuxArm64 const DEFAULT_MODEL = { @@ -126,10 +127,9 @@ const SHARDED_MODEL = { baseUrl: 'https://huggingface.co/jmb95/Qwen3-0.6B-UD-IQ1_S-sharded/resolve/main/' } -// This test can take longer to download and execute. To avoid blowing up testing time on all -// platforms, just use Linux for now. C++ tests already have faster coverage for each type -// of load. -test('sharded model can run inference end-to-end', { timeout: 4 * 60 * 1000, skip: !isLinuxX64 }, async t => { +// This test can take longer to download and execute. Keep it on desktop x64 +// runners where the sharded loader has CI coverage. +test('sharded model can run inference end-to-end', { timeout: 4 * 60 * 1000, skip: !(isLinuxX64 || isWindowsX64) }, async t => { const fs = require('bare-fs') const modelDir = path.resolve(__dirname, '../model') fs.mkdirSync(modelDir, { recursive: true }) diff --git a/packages/llm-llamacpp/test/integration/mrope-sliding-context.test.js b/packages/llm-llamacpp/test/integration/mrope-sliding-context.test.js index 7862c0b5bd..ae9a8730ea 100644 --- a/packages/llm-llamacpp/test/integration/mrope-sliding-context.test.js +++ b/packages/llm-llamacpp/test/integration/mrope-sliding-context.test.js @@ -4,7 +4,7 @@ const fs = require('bare-fs') const os = require('bare-os') const path = require('bare-path') const LlmLlamacpp = require('../../index.js') -const { cleanupIntegrationCacheFiles, ensureModel, getMediaPath, safeTest } = require('./utils') +const { cleanupIntegrationCacheFiles, ensureModel, getMediaPath, safeTest: integrationTest } = require('./utils') const platform = os.platform() const arch = os.arch() @@ -16,6 +16,10 @@ const isLinuxArm64 = platform === 'linux' && arch === 'arm64' const useCpu = isDarwinX64 || isLinuxArm64 const skipTbqPq = isDarwin || isIos || isAndroid +function safeTest (name, opts, fn) { + integrationTest(name, { ...opts, skip: opts.skip || isDarwinX64 }, fn) +} + const QWEN3_5_MODEL = { name: 'Qwen3.5-0.8B-Q8_0.gguf', url: 'https://huggingface.co/unsloth/Qwen3.5-0.8B-GGUF/resolve/main/Qwen3.5-0.8B-Q8_0.gguf' diff --git a/packages/llm-llamacpp/test/integration/multi-gpu.test.js b/packages/llm-llamacpp/test/integration/multi-gpu.test.js index d6a926370e..21ea45b819 100644 --- a/packages/llm-llamacpp/test/integration/multi-gpu.test.js +++ b/packages/llm-llamacpp/test/integration/multi-gpu.test.js @@ -1,11 +1,14 @@ 'use strict' const process = require('bare-process') +const os = require('bare-os') const LlmLlamacpp = require('../../index.js') const { ensureModel, safeTest } = require('./utils') const { attachSpecLogger } = require('./spec-logger') const path = require('bare-path') +const isDarwinX64 = os.platform() === 'darwin' && os.arch() === 'x64' + const MODEL = { name: 'Qwen3-0.6B-Q8_0.gguf', url: 'https://huggingface.co/unsloth/Qwen3-0.6B-GGUF/resolve/main/Qwen3-0.6B-Q8_0.gguf' @@ -32,6 +35,7 @@ async function collectResponse (response) { } const hasMultiGpu = process.env.QVAC_HAS_MULTI_GPU === '1' +const skip = isDarwinX64 const BASE_CONFIG = { device: 'gpu', @@ -92,19 +96,19 @@ function assertSingleDevice (t, devices) { t.ok(devices.size <= 1, `layers should stay on a single device (found: ${[...devices].join(', ')})`) } -safeTest('multi-gpu: split-mode=layer distributes layers across GPUs', { timeout: 600_000 }, async t => { +safeTest('multi-gpu: split-mode=layer distributes layers across GPUs', { timeout: 600_000, skip }, async t => { await runMultiGpuTest(t, { 'split-mode': 'layer' }, assertMultiDevice('layers')) }) -safeTest('multi-gpu: split-mode=row distributes tensors across GPUs', { timeout: 600_000 }, async t => { +safeTest('multi-gpu: split-mode=row distributes tensors across GPUs', { timeout: 600_000, skip }, async t => { await runMultiGpuTest(t, { 'split-mode': 'row' }, assertMultiDevice('tensors')) }) -safeTest('multi-gpu: default (no split-mode) pins layers to a single device', { timeout: 600_000 }, async t => { +safeTest('multi-gpu: default (no split-mode) pins layers to a single device', { timeout: 600_000, skip }, async t => { await runMultiGpuTest(t, {}, assertSingleDevice) }) -safeTest('multi-gpu: split-mode=layer with tensor-split and main-gpu', { timeout: 600_000 }, async t => { +safeTest('multi-gpu: split-mode=layer with tensor-split and main-gpu', { timeout: 600_000, skip }, async t => { await runMultiGpuTest( t, { 'split-mode': 'layer', 'tensor-split': '1,1', 'main-gpu': '0' }, diff --git a/packages/llm-llamacpp/test/integration/multi-instance.test.js b/packages/llm-llamacpp/test/integration/multi-instance.test.js index cc458d9486..1d958cbdbf 100644 --- a/packages/llm-llamacpp/test/integration/multi-instance.test.js +++ b/packages/llm-llamacpp/test/integration/multi-instance.test.js @@ -9,8 +9,8 @@ const platform = os.platform() const arch = os.arch() const isDarwinX64 = platform === 'darwin' && arch === 'x64' const isLinuxArm64 = platform === 'linux' && arch === 'arm64' -const isWindowsX64 = platform === 'win32' && arch === 'x64' const useCpu = isDarwinX64 || isLinuxArm64 +const skip = isDarwinX64 const DEFAULT_MODEL = { name: 'Llama-3.2-1B-Instruct-Q4_0.gguf', @@ -73,7 +73,7 @@ async function collectResponse (response) { safeTest('Two instances can run inference simultaneously', { timeout: 900_000, - skip: isWindowsX64 // TODO: unskip this once we have a new Windows runner with a GPU + skip }, async t => { let addon1 = null let addon2 = null @@ -107,7 +107,7 @@ safeTest('Two instances can run inference simultaneously', { safeTest('Repeated load/unload cycles should remain stable', { timeout: 900_000, - skip: isWindowsX64 // TODO: unskip this once we have a new Windows runner with a GPU + skip }, async t => { let currentAddon = null try { @@ -142,7 +142,7 @@ safeTest('Repeated load/unload cycles should remain stable', { safeTest('Unloading one instance does not affect another generating instance', { timeout: 900_000, - skip: isWindowsX64 // TODO: unskip this once we have a new Windows runner with a GPU + skip }, async t => { let addon1 = null let addon2 = null @@ -204,7 +204,7 @@ safeTest('Unloading one instance does not affect another generating instance', { safeTest('Multiple load/unload cycles on one instance while another generates', { timeout: 900_000, - skip: isWindowsX64 // TODO: unskip this once we have a new Windows runner with a GPU + skip }, async t => { let addon1 = null try { diff --git a/packages/llm-llamacpp/test/integration/qwen3-5-image-elephant-perf.test.js b/packages/llm-llamacpp/test/integration/qwen3-5-image-elephant-perf.test.js index ade746925c..7c9c898891 100644 --- a/packages/llm-llamacpp/test/integration/qwen3-5-image-elephant-perf.test.js +++ b/packages/llm-llamacpp/test/integration/qwen3-5-image-elephant-perf.test.js @@ -4,9 +4,9 @@ // 30-minute mobile cap. Asserts the elephant keyword + records perf. const test = require('brittle') -const { QWEN35_MODEL, IMAGE_CASES, runVlmImagePerf } = require('./_vlm-image-perf.js') +const { QWEN35_MODEL, IMAGE_CASES, isDarwinX64, runVlmImagePerf } = require('./_vlm-image-perf.js') -test('Qwen3.5-VL image perf [elephant]', { timeout: 1_800_000 }, async t => { +test('Qwen3.5-VL image perf [elephant]', { timeout: 1_800_000, skip: isDarwinX64 }, async t => { await runVlmImagePerf(t, QWEN35_MODEL, IMAGE_CASES.elephant) }) diff --git a/packages/llm-llamacpp/test/integration/qwen3-5-image-fruit-plate-perf.test.js b/packages/llm-llamacpp/test/integration/qwen3-5-image-fruit-plate-perf.test.js index 534e9dd681..869953af29 100644 --- a/packages/llm-llamacpp/test/integration/qwen3-5-image-fruit-plate-perf.test.js +++ b/packages/llm-llamacpp/test/integration/qwen3-5-image-fruit-plate-perf.test.js @@ -4,9 +4,9 @@ // the 30-minute mobile cap. Asserts a fruit keyword + records perf. const test = require('brittle') -const { QWEN35_MODEL, IMAGE_CASES, runVlmImagePerf } = require('./_vlm-image-perf.js') +const { QWEN35_MODEL, IMAGE_CASES, isDarwinX64, runVlmImagePerf } = require('./_vlm-image-perf.js') -test('Qwen3.5-VL image perf [fruit plate]', { timeout: 1_800_000 }, async t => { +test('Qwen3.5-VL image perf [fruit plate]', { timeout: 1_800_000, skip: isDarwinX64 }, async t => { await runVlmImagePerf(t, QWEN35_MODEL, IMAGE_CASES['fruit-plate']) }) diff --git a/packages/llm-llamacpp/test/integration/qwen3-5-image-high-res-aurora-perf.test.js b/packages/llm-llamacpp/test/integration/qwen3-5-image-high-res-aurora-perf.test.js index 447df2f686..422e38150e 100644 --- a/packages/llm-llamacpp/test/integration/qwen3-5-image-high-res-aurora-perf.test.js +++ b/packages/llm-llamacpp/test/integration/qwen3-5-image-high-res-aurora-perf.test.js @@ -4,12 +4,12 @@ // the 30-minute mobile cap. Asserts an aurora keyword + records perf. const test = require('brittle') -const { QWEN35_MODEL, IMAGE_CASES, skipHeavyImages, runVlmImagePerf } = require('./_vlm-image-perf.js') +const { QWEN35_MODEL, IMAGE_CASES, isDarwinX64, skipHeavyImages, runVlmImagePerf } = require('./_vlm-image-perf.js') // QVAC-19368: aurora is the heaviest image; skip it on Android on-PR runs -// where the 30-min Device Farm cap is tight. iOS + desktop always run it. -// The benchmark (QVAC_PERF_ONLY=true) runs all 3 images on all platforms. -test('Qwen3.5-VL image perf [high-res aurora]', { timeout: 1_800_000, skip: skipHeavyImages }, async t => { +// where the 30-min Device Farm cap is tight. Darwin x64 is skipped separately. +// The benchmark (QVAC_PERF_ONLY=true) runs all 3 images on supported platforms. +test('Qwen3.5-VL image perf [high-res aurora]', { timeout: 1_800_000, skip: isDarwinX64 || skipHeavyImages }, async t => { await runVlmImagePerf(t, QWEN35_MODEL, IMAGE_CASES['high-res-aurora']) }) diff --git a/packages/llm-llamacpp/test/integration/reasoning.test.js b/packages/llm-llamacpp/test/integration/reasoning.test.js index 40778cfdad..909cc41204 100644 --- a/packages/llm-llamacpp/test/integration/reasoning.test.js +++ b/packages/llm-llamacpp/test/integration/reasoning.test.js @@ -7,7 +7,6 @@ const os = require('bare-os') const LlmLlamacpp = require('../../index.js') const isDarwinX64 = os.platform() === 'darwin' && os.arch() === 'x64' -const isWindowsX64 = os.platform() === 'win32' && os.arch() === 'x64' const isLinuxArm64 = os.platform() === 'linux' && os.arch() === 'arm64' const useCpu = isLinuxArm64 @@ -126,7 +125,7 @@ function createFollowUpMessages (initialMessages, previousResponse) { ] } safeTest('reasoning tag EOS replacement works with tools=false', { - skip: isDarwinX64 || isWindowsX64, // TODO: unskip isWindowsX64 once we have GPU, takes too long + skip: isDarwinX64, timeout: 600_000 }, async t => { const { inference } = await setupReasoningModel(t, false) @@ -149,7 +148,7 @@ safeTest('reasoning tag EOS replacement works with tools=false', { }) safeTest('reasoning tag EOS replacement works with tools=true', { - skip: isDarwinX64 || isWindowsX64, // TODO: unskip isWindowsX64 once we have GPU, takes too long + skip: isDarwinX64, timeout: 600_000 }, async t => { const { inference } = await setupReasoningModel(t, true) @@ -172,7 +171,7 @@ safeTest('reasoning tag EOS replacement works with tools=true', { }) safeTest('Qwen3 reasoning-budget=0 disables thinking', { - skip: isDarwinX64 || isWindowsX64, + skip: isDarwinX64, timeout: 600_000 }, async t => { const [modelName, dirPath] = await ensureModel({ diff --git a/packages/llm-llamacpp/test/integration/sliding-context.test.js b/packages/llm-llamacpp/test/integration/sliding-context.test.js index 4e007caa89..76854fedd5 100644 --- a/packages/llm-llamacpp/test/integration/sliding-context.test.js +++ b/packages/llm-llamacpp/test/integration/sliding-context.test.js @@ -13,8 +13,7 @@ const isWindowsX64 = platform === 'win32' && arch === 'x64' const useCpu = isDarwinX64 || isLinuxArm64 // These are very slow on CI and should be skipped. -// TODO: unskip Windows once we have a new Windows runner with a GPU -const skip = isWindowsX64 || isLinuxArm64 +const skip = isDarwinX64 || isLinuxArm64 const DEFAULT_MODEL = { name: 'Llama-3.2-1B-Instruct-Q4_0.gguf', @@ -146,6 +145,7 @@ safeTest('Generation fails with context overflow when sliding disabled', { skip }, async t => { const { model } = await setupModel(t, { + ctx_size: '256', n_predict: String(SLIDE_PREDICT), n_discarded: '0' }) diff --git a/packages/llm-llamacpp/test/integration/tools-compact.test.js b/packages/llm-llamacpp/test/integration/tools-compact.test.js index bd4884462c..dce2826861 100644 --- a/packages/llm-llamacpp/test/integration/tools-compact.test.js +++ b/packages/llm-llamacpp/test/integration/tools-compact.test.js @@ -197,6 +197,25 @@ async function runAndCollect (model, prompt, runOptions) { } } +// Fence for `saveCacheToDisk: true`: the addon resolves `chain.await()` once +// inference is done, but the session-write hits the disk on a background +// thread. On a fast runner an immediate `model.unload()` can race the flush +// and leave a truncated file; the next model's session-load then blocks +// indefinitely waiting for bytes that never arrive. Poll until size has +// been stable for one tick to make sure the writer has finalized. +async function waitForStableSessionFile (filePath, { maxMs = 5000 } = {}) { + const start = Date.now() + let lastSize = -1 + while (Date.now() - start < maxMs) { + let size = 0 + try { size = fs.statSync(filePath).size } catch { size = 0 } + if (size > 0 && size === lastSize) return size + lastSize = size + await new Promise(resolve => setTimeout(resolve, 50)) + } + return lastSize +} + async function runExpectingInvalidPrompt (t, model, prompt, expectedReason, runOptions) { cleanupRunOptionsCache(runOptions) const response = await model.run(prompt, runOptions) @@ -759,6 +778,9 @@ safeTest('[tools-compact] session save, destroy, reload with different tools', { t.ok(r1.output.length > 0, 'turn 1 produces output') t.ok(r1.stats.CacheTokens > 0, 'turn 1 has cache tokens') + const persistedSize = await waitForStableSessionFile(sessionName) + t.ok(persistedSize > 0, `session file flushed to disk (${persistedSize} bytes)`) + await model1.unload() const { model: model2 } = await setupModel(t) diff --git a/packages/llm-llamacpp/test/mobile/integration.auto.cjs b/packages/llm-llamacpp/test/mobile/integration.auto.cjs index 64ec38ab40..886fac0064 100644 --- a/packages/llm-llamacpp/test/mobile/integration.auto.cjs +++ b/packages/llm-llamacpp/test/mobile/integration.auto.cjs @@ -381,6 +381,11 @@ async function runConfigParametersTest (options = {}) { // eslint-disable-line n return runIntegrationModule('../integration/config-parameters.test.js', options) } +async function runContinuousBatchingTest (options = {}) { // eslint-disable-line no-unused-vars + if (typeof __shouldRunTest === 'function' && !__shouldRunTest('runContinuousBatchingTest')) return __FILTERED + return runIntegrationModule('../integration/continuous-batching.test.js', options) +} + async function runFinetuningPauseResumeTest (options = {}) { // eslint-disable-line no-unused-vars if (typeof __shouldRunTest === 'function' && !__shouldRunTest('runFinetuningPauseResumeTest')) return __FILTERED return runIntegrationModule('../integration/finetuning-pause-resume.test.js', options) diff --git a/packages/llm-llamacpp/test/mobile/test-groups.json b/packages/llm-llamacpp/test/mobile/test-groups.json index 6696c8bb6a..09c68abc93 100644 --- a/packages/llm-llamacpp/test/mobile/test-groups.json +++ b/packages/llm-llamacpp/test/mobile/test-groups.json @@ -7,6 +7,7 @@ "imageFruitPlate": ["runImageFruitPlateTest"], "gemma": ["runGemma4Test"], "quantizedKvcache": ["runQuantizedKvcacheTest"], + "continuousBatching": ["runContinuousBatchingTest"], "lightA": [ "runApiBehaviorTest", "runBitnetTest", @@ -27,6 +28,7 @@ ] }, "android": { + "continuousBatching": ["runContinuousBatchingTest"], "funcShardA": [ "runFinetuningPauseResumeTest", "runConfigParametersTest", diff --git a/packages/llm-llamacpp/test/unit/CMakeLists.txt b/packages/llm-llamacpp/test/unit/CMakeLists.txt index 15eafbdcf9..0d04bd7906 100644 --- a/packages/llm-llamacpp/test/unit/CMakeLists.txt +++ b/packages/llm-llamacpp/test/unit/CMakeLists.txt @@ -20,6 +20,7 @@ add_executable( test_reload_cancel_state.cpp test_backend_selection.cpp test_tune_config_map.cpp + test_generation_params_apply.cpp # Placeholder tests (to be implemented) test_mtmd_llm_context.cpp test_llm_context_base.cpp @@ -42,6 +43,7 @@ add_executable( test_continuous_batching_integration.cpp test_kv_leak_on_admit_failure.cpp test_runtime_stats.cpp + test_reasoning_budget.cpp ${CMAKE_SOURCE_DIR}/addon/src/model-interface/AsyncWeightsLoader.cpp ${CMAKE_SOURCE_DIR}/addon/src/model-interface/MultiRequestBatcher.cpp ${CMAKE_SOURCE_DIR}/addon/src/model-interface/ContinuousBatchScheduler.cpp diff --git a/packages/llm-llamacpp/test/unit/test_chat_template_utils.cpp b/packages/llm-llamacpp/test/unit/test_chat_template_utils.cpp index 52a4ebf84f..4c2dda0dbf 100644 --- a/packages/llm-llamacpp/test/unit/test_chat_template_utils.cpp +++ b/packages/llm-llamacpp/test/unit/test_chat_template_utils.cpp @@ -220,3 +220,51 @@ TEST_F( std::string result = getChatTemplate(nullptr, params, false); EXPECT_EQ(result, "my_custom_template"); } + +TEST_F(ChatTemplateUtilsTest, GetPromptExportsQwenThinkingMetadata) { + common_chat_templates_ptr tmpls = + common_chat_templates_init(nullptr, getFixedQwen3Template()); + ASSERT_NE(tmpls, nullptr); + + common_chat_templates_inputs inputs; + inputs.use_jinja = true; + inputs.enable_thinking = true; + inputs.add_generation_prompt = true; + inputs.messages = {common_chat_msg{ + /* role = */ "user", + /* content = */ "What is the capital of France?", + }}; + + bool thinkingForcedOpen = true; + std::string thinkingStartTag; + std::string thinkingEndTag; + std::string generationPrompt; + const std::string prompt = getPrompt( + tmpls.get(), + inputs, + &thinkingForcedOpen, + &thinkingStartTag, + &thinkingEndTag, + &generationPrompt); + + EXPECT_NE(prompt.find("<|im_start|>assistant"), std::string::npos); + EXPECT_EQ(thinkingStartTag, "\n"); + EXPECT_EQ(thinkingEndTag, "\n\n\n"); + EXPECT_NE(generationPrompt.find("<|im_start|>assistant"), std::string::npos); + EXPECT_FALSE(thinkingForcedOpen); +} + +TEST_F(ChatTemplateUtilsTest, ThinkingForcedOpenTextUsesTemplateSuffix) { + EXPECT_EQ( + getThinkingForcedOpenText("<|assistant|>\n\n", ""), + "\n"); +} + +TEST_F(ChatTemplateUtilsTest, ThinkingForcedOpenTextFallsBackToStartTag) { + EXPECT_EQ( + getThinkingForcedOpenText("<|assistant|>\n", ""), ""); +} + +TEST_F(ChatTemplateUtilsTest, ThinkingForcedOpenTextEmptyWithoutStartTag) { + EXPECT_EQ(getThinkingForcedOpenText("<|assistant|>\n", ""), ""); +} diff --git a/packages/llm-llamacpp/test/unit/test_generation_params_apply.cpp b/packages/llm-llamacpp/test/unit/test_generation_params_apply.cpp new file mode 100644 index 0000000000..155e174e1b --- /dev/null +++ b/packages/llm-llamacpp/test/unit/test_generation_params_apply.cpp @@ -0,0 +1,130 @@ +#include + +#include + +#include "model-interface/GenerationParamsApply.hpp" +#include "utils/ChatTemplateUtils.hpp" + +using qvac_lib_inference_addon_llama::utils::configureReasoningBudgetSampling; + +namespace { + +std::vector tokens(std::initializer_list values) { + return std::vector(values); +} + +} // namespace + +TEST(GenerationParamsApplyTest, NoReasoningBudgetOverrideLeavesSamplingState) { + common_params_sampling sampling; + sampling.reasoning_budget_tokens = 12; + sampling.reasoning_budget_start = tokens({1, 2}); + sampling.reasoning_budget_end = tokens({3}); + sampling.reasoning_budget_forced = tokens({4, 5}); + int nPredict = 32; + + GenerationParams overrides; + overrides.temp = 0.25f; + applyGenerationOverridesToSampling(sampling, nPredict, overrides); + + EXPECT_EQ(sampling.reasoning_budget_tokens, 12); + EXPECT_EQ(sampling.reasoning_budget_start, tokens({1, 2})); + EXPECT_EQ(sampling.reasoning_budget_end, tokens({3})); + EXPECT_EQ(sampling.reasoning_budget_forced, tokens({4, 5})); +} + +TEST(GenerationParamsApplyTest, PositiveReasoningBudgetUpdatesOnlyTokenCap) { + common_params_sampling sampling; + sampling.reasoning_budget_tokens = -1; + sampling.reasoning_budget_start = tokens({10}); + sampling.reasoning_budget_end = tokens({11}); + sampling.reasoning_budget_forced = tokens({12}); + int nPredict = 32; + + GenerationParams overrides; + overrides.reasoning_budget = 16; + applyGenerationOverridesToSampling(sampling, nPredict, overrides); + + EXPECT_EQ(sampling.reasoning_budget_tokens, 16); + EXPECT_EQ(sampling.reasoning_budget_start, tokens({10})); + EXPECT_EQ(sampling.reasoning_budget_end, tokens({11})); + EXPECT_EQ(sampling.reasoning_budget_forced, tokens({12})); +} + +TEST(GenerationParamsApplyTest, ZeroReasoningBudgetClearsBudgetSamplerState) { + common_params_sampling sampling; + sampling.reasoning_budget_tokens = 16; + sampling.reasoning_budget_start = tokens({10}); + sampling.reasoning_budget_end = tokens({11}); + sampling.reasoning_budget_forced = tokens({12}); + int nPredict = 32; + + GenerationParams overrides; + overrides.reasoning_budget = 0; + applyGenerationOverridesToSampling(sampling, nPredict, overrides); + + EXPECT_EQ(sampling.reasoning_budget_tokens, -1); + EXPECT_TRUE(sampling.reasoning_budget_start.empty()); + EXPECT_TRUE(sampling.reasoning_budget_end.empty()); + EXPECT_TRUE(sampling.reasoning_budget_forced.empty()); +} + +TEST( + GenerationParamsApplyTest, + UnrestrictedReasoningBudgetClearsBudgetSamplerState) { + common_params_sampling sampling; + sampling.reasoning_budget_tokens = 16; + sampling.reasoning_budget_start = tokens({10}); + sampling.reasoning_budget_end = tokens({11}); + sampling.reasoning_budget_forced = tokens({12}); + int nPredict = 32; + + GenerationParams overrides; + overrides.reasoning_budget = -1; + applyGenerationOverridesToSampling(sampling, nPredict, overrides); + + EXPECT_EQ(sampling.reasoning_budget_tokens, -1); + EXPECT_TRUE(sampling.reasoning_budget_start.empty()); + EXPECT_TRUE(sampling.reasoning_budget_end.empty()); + EXPECT_TRUE(sampling.reasoning_budget_forced.empty()); +} + +TEST( + GenerationParamsApplyTest, + ConfigureReasoningBudgetSamplingClearsStaleStateWhenDisabled) { + common_params params; + params.reasoning_budget = 0; + params.sampling.reasoning_budget_tokens = 16; + params.sampling.reasoning_budget_start = tokens({10}); + params.sampling.reasoning_budget_end = tokens({11}); + params.sampling.reasoning_budget_forced = tokens({12}); + params.sampling.generation_prompt = ""; + + EXPECT_TRUE(configureReasoningBudgetSampling( + params, nullptr, "", "", "")); + EXPECT_EQ(params.sampling.reasoning_budget_tokens, -1); + EXPECT_TRUE(params.sampling.reasoning_budget_start.empty()); + EXPECT_TRUE(params.sampling.reasoning_budget_end.empty()); + EXPECT_TRUE(params.sampling.reasoning_budget_forced.empty()); + EXPECT_TRUE(params.sampling.generation_prompt.empty()); +} + +TEST( + GenerationParamsApplyTest, + ConfigureReasoningBudgetSamplingKeepsPositiveCapWithoutContext) { + common_params params; + params.reasoning_budget = 8; + params.sampling.reasoning_budget_tokens = -1; + params.sampling.reasoning_budget_start = tokens({10}); + params.sampling.reasoning_budget_end = tokens({11}); + params.sampling.reasoning_budget_forced = tokens({12}); + params.sampling.generation_prompt = ""; + + EXPECT_TRUE(configureReasoningBudgetSampling( + params, nullptr, "", "", "")); + EXPECT_EQ(params.sampling.reasoning_budget_tokens, 8); + EXPECT_TRUE(params.sampling.reasoning_budget_start.empty()); + EXPECT_TRUE(params.sampling.reasoning_budget_end.empty()); + EXPECT_TRUE(params.sampling.reasoning_budget_forced.empty()); + EXPECT_TRUE(params.sampling.generation_prompt.empty()); +} diff --git a/packages/llm-llamacpp/test/unit/test_reasoning_budget.cpp b/packages/llm-llamacpp/test/unit/test_reasoning_budget.cpp new file mode 100644 index 0000000000..2e94a9ea5f --- /dev/null +++ b/packages/llm-llamacpp/test/unit/test_reasoning_budget.cpp @@ -0,0 +1,141 @@ +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "common/common.h" +#include "model-interface/LlamaModel.hpp" +#include "test_common.hpp" + +namespace fs = std::filesystem; + +namespace { + +constexpr int REASONING_BUDGET = 3; +constexpr int PREDICT_TOKENS = 10; +constexpr const char* THINKING_START_TAG = "\n"; +constexpr const char* THINKING_END_TAG = "\n\n\n"; + +std::string sliceReasoning(const std::string& text) { + const size_t open = text.find(THINKING_START_TAG); + if (open == std::string::npos) { + return ""; + } + + const size_t bodyStart = open + std::string(THINKING_START_TAG).size(); + const size_t close = text.find(THINKING_END_TAG, bodyStart); + if (close == std::string::npos || close < bodyStart) { + return ""; + } + + return text.substr(bodyStart, close - bodyStart); +} + +size_t countTokens(llama_context* ctx, const std::string& text) { + return common_tokenize(ctx, text, false, true).size(); +} + +struct RunResult { + std::string output; + size_t reasoningTokens = 0; +}; + +} // namespace + +class ReasoningBudgetModelTest : public ::testing::Test { +protected: + void SetUp() override { + using MP = test_common::TestModelPath; + qwen3Model_ = + MP("Qwen3-0.6B-Q8_0.gguf", + "QWEN3_MODEL_PATH", + MP::OnMissing::Skip, + "https://huggingface.co/Qwen/Qwen3-0.6B-GGUF"); + + config_["ctx_size"] = "4096"; + config_["n_predict"] = std::to_string(PREDICT_TOKENS); + config_["seed"] = "50"; + config_["temp"] = "0"; + config_["top_p"] = "1"; + config_["device"] = test_common::getTestDevice(); + config_["gpu_layers"] = test_common::getTestGpuLayers(); + config_["backendsDir"] = test_common::getTestBackendsDir().string(); + } + + [[nodiscard]] bool hasQwen3Model() const { + return qwen3Model_.found() && fs::exists(qwen3Model_.path); + } + + std::unique_ptr + createModel(std::unordered_map config) { + auto model = std::make_unique( + std::string(qwen3Model_.path), std::string(), std::move(config)); + model->waitForLoadInitialization(); + return model; + } + + RunResult runBudgetedPrompt( + std::unordered_map config, + const GenerationParams& generationParams = {}) { + auto model = createModel(std::move(config)); + if (!model->isLoaded()) { + throw std::runtime_error("Qwen3 model failed to load"); + } + + LlamaModel::Prompt prompt; + prompt.input = + R"([{"role":"system","content":"You are a helpful assistant."},{"role":"user","content":"What is the capital of France? Answer in one word."}])"; + prompt.generationParams = generationParams; + + const std::string output = model->processPrompt(prompt); + const std::string reasoning = sliceReasoning(output); + return {output, countTokens(model->getContext(), reasoning)}; + } + + test_common::TestModelPath qwen3Model_; + std::unordered_map config_; +}; + +TEST_F( + ReasoningBudgetModelTest, + LoadTimeBudgetThreeGeneratesThreeReasoningTokens) { + if (!hasQwen3Model()) { + GTEST_SKIP() << qwen3Model_.missingMessage(); + } + + auto config = config_; + config["reasoning-budget"] = std::to_string(REASONING_BUDGET); + + const RunResult result = runBudgetedPrompt(std::move(config)); + + EXPECT_NE(result.output.find(THINKING_START_TAG), std::string::npos) + << result.output; + EXPECT_NE(result.output.find(THINKING_END_TAG), std::string::npos) + << result.output; + EXPECT_EQ(result.reasoningTokens, REASONING_BUDGET) << result.output; +} + +TEST_F( + ReasoningBudgetModelTest, + PerRequestBudgetThreeGeneratesThreeReasoningTokens) { + if (!hasQwen3Model()) { + GTEST_SKIP() << qwen3Model_.missingMessage(); + } + + GenerationParams generationParams; + generationParams.reasoning_budget = REASONING_BUDGET; + generationParams.n_predict = PREDICT_TOKENS; + + const RunResult result = runBudgetedPrompt(config_, generationParams); + + EXPECT_NE(result.output.find(THINKING_START_TAG), std::string::npos) + << result.output; + EXPECT_NE(result.output.find(THINKING_END_TAG), std::string::npos) + << result.output; + EXPECT_EQ(result.reasoningTokens, REASONING_BUDGET) << result.output; +} diff --git a/packages/llm-llamacpp/test/unit/test_tune_config_map.cpp b/packages/llm-llamacpp/test/unit/test_tune_config_map.cpp index 5f194d88e3..8262920a13 100644 --- a/packages/llm-llamacpp/test/unit/test_tune_config_map.cpp +++ b/packages/llm-llamacpp/test/unit/test_tune_config_map.cpp @@ -3,6 +3,7 @@ #include #include +#include #include "model-interface/LlamaModel.hpp" #include "test_common.hpp" @@ -15,32 +16,33 @@ class TuneConfigMapTest : public ::testing::Test { std::unordered_map configFilemap_; }; -// ---- Non-BitNet: no modifications ---- - -TEST_F(TuneConfigMapTest, NonBitnet_NoChanges) { +TEST_F(TuneConfigMapTest, NonBitnet_FlashAttnDefaultsOn) { MockModelMetaData meta(false, "llama"); LlamaModel::tuneConfigMap(configFilemap_, meta, std::nullopt); - EXPECT_EQ(configFilemap_.count("flash-attn"), 0); + ASSERT_EQ(configFilemap_.count("flash-attn"), 1); + EXPECT_EQ(configFilemap_["flash-attn"], "on"); EXPECT_EQ(configFilemap_.count("ubatch-size"), 0); } -TEST_F(TuneConfigMapTest, OneBitButNotBitnetArch_NoChanges) { +TEST_F(TuneConfigMapTest, OneBitButNotBitnetArch_FlashAttnDefaultsOn) { MockModelMetaData meta(true, "llama"); LlamaModel::tuneConfigMap(configFilemap_, meta, 830); - EXPECT_EQ(configFilemap_.count("flash-attn"), 0); + ASSERT_EQ(configFilemap_.count("flash-attn"), 1); + EXPECT_EQ(configFilemap_["flash-attn"], "on"); EXPECT_EQ(configFilemap_.count("ubatch-size"), 0); } -TEST_F(TuneConfigMapTest, BitnetArchButNotOneBit_NoChanges) { +TEST_F(TuneConfigMapTest, BitnetArchButNotOneBit_FlashAttnDefaultsOn) { MockModelMetaData meta(false, "bitnet"); LlamaModel::tuneConfigMap(configFilemap_, meta, 830); - EXPECT_EQ(configFilemap_.count("flash-attn"), 0); + ASSERT_EQ(configFilemap_.count("flash-attn"), 1); + EXPECT_EQ(configFilemap_["flash-attn"], "on"); EXPECT_EQ(configFilemap_.count("ubatch-size"), 0); } @@ -190,16 +192,16 @@ TEST_F(TuneConfigMapTest, Bitnet_Adreno799_UbatchUnchanged) { EXPECT_EQ(configFilemap_.count("ubatch-size"), 0); } -// ---- OpenCL backend: flash-attn disabled by default unless user-set ---- +// ---- OpenCL backend: flash-attn defaulted ON like every other GPU path ---- -TEST_F(TuneConfigMapTest, OpenCl_NonBitnet_FlashAttnDisabledByDefault) { +TEST_F(TuneConfigMapTest, OpenCl_NonBitnet_FlashAttnDefaultsOn) { MockModelMetaData meta(false, "llama"); LlamaModel::tuneConfigMap( configFilemap_, meta, std::nullopt, FtOverrides{}, /*isOpenCl=*/true); ASSERT_EQ(configFilemap_.count("flash-attn"), 1); - EXPECT_EQ(configFilemap_["flash-attn"], "off"); + EXPECT_EQ(configFilemap_["flash-attn"], "on"); } TEST_F(TuneConfigMapTest, OpenCl_UserSetFlashAttnHyphen_Respected) { @@ -223,13 +225,107 @@ TEST_F(TuneConfigMapTest, OpenCl_UserSetFlashAttnUnderscore_Respected) { EXPECT_EQ(configFilemap_["flash_attn"], "on"); } -TEST_F(TuneConfigMapTest, NotOpenCl_NonBitnet_FlashAttnUnchanged) { +TEST_F(TuneConfigMapTest, NotOpenCl_NonBitnet_FlashAttnDefaultsOn) { MockModelMetaData meta(false, "llama"); LlamaModel::tuneConfigMap( configFilemap_, meta, std::nullopt, FtOverrides{}, /*isOpenCl=*/false); - EXPECT_EQ(configFilemap_.count("flash-attn"), 0); + ASSERT_EQ(configFilemap_.count("flash-attn"), 1); + EXPECT_EQ(configFilemap_["flash-attn"], "on"); +} + +TEST_F(TuneConfigMapTest, OpenCl_RejectsStandardQuantizedKCache) { + MockModelMetaData meta(false, "llama"); + configFilemap_["cache-type-k"] = "q8_0"; + + EXPECT_THROW( + LlamaModel::tuneConfigMap( + configFilemap_, meta, std::nullopt, FtOverrides{}, /*isOpenCl=*/true), + qvac_errors::StatusError); +} + +TEST_F(TuneConfigMapTest, OpenCl_RejectsStandardQuantizedVCacheUnderscore) { + MockModelMetaData meta(false, "llama"); + configFilemap_["cache_type_v"] = "q4_0"; + + EXPECT_THROW( + LlamaModel::tuneConfigMap( + configFilemap_, meta, std::nullopt, FtOverrides{}, /*isOpenCl=*/true), + qvac_errors::StatusError); +} + +TEST_F(TuneConfigMapTest, OpenCl_AllowsNonQuantizedCacheTypes) { + MockModelMetaData meta(false, "llama"); + configFilemap_["cache-type-k"] = "f16"; + configFilemap_["cache-type-v"] = "bf16"; + + EXPECT_NO_THROW( + LlamaModel::tuneConfigMap( + configFilemap_, + meta, + std::nullopt, + FtOverrides{}, + /*isOpenCl=*/true)); +} + +TEST_F(TuneConfigMapTest, NotOpenCl_AllowsStandardQuantizedCacheTypes) { + MockModelMetaData meta(false, "llama"); + configFilemap_["cache-type-k"] = "q8_0"; + configFilemap_["cache-type-v"] = "q4_0"; + + EXPECT_NO_THROW( + LlamaModel::tuneConfigMap( + configFilemap_, + meta, + std::nullopt, + FtOverrides{}, + /*isOpenCl=*/false)); +} + +TEST_F(TuneConfigMapTest, NotOpenClNotMetal_AllowsTurboQuantCacheTypes) { + MockModelMetaData meta(false, "llama"); + configFilemap_["cache-type-k"] = "tbq4_0"; + configFilemap_["cache-type-v"] = "pq4_0"; + + EXPECT_NO_THROW( + LlamaModel::tuneConfigMap( + configFilemap_, + meta, + std::nullopt, + FtOverrides{}, + /*isOpenCl=*/false, + /*isMetal=*/false)); +} + +TEST_F(TuneConfigMapTest, Metal_RejectsTurboQuantCacheTypes) { + MockModelMetaData meta(false, "llama"); + configFilemap_["cache-type-k"] = "tbq4_0"; + + EXPECT_THROW( + LlamaModel::tuneConfigMap( + configFilemap_, + meta, + std::nullopt, + FtOverrides{}, + /*isOpenCl=*/false, + /*isMetal=*/true), + qvac_errors::StatusError); +} + +TEST_F(TuneConfigMapTest, Metal_AllowsStandardQuantizedCacheTypes) { + MockModelMetaData meta(false, "llama"); + configFilemap_["cache-type-k"] = "q8_0"; + configFilemap_["cache-type-v"] = "q4_0"; + + EXPECT_NO_THROW( + LlamaModel::tuneConfigMap( + configFilemap_, + meta, + std::nullopt, + FtOverrides{}, + /*isOpenCl=*/false, + /*isMetal=*/true)); } // ---- Finetuning: flash-attn disabled for any architecture ----