diff --git a/conf/mod_ollama_chat.conf.dist b/conf/mod_ollama_chat.conf.dist index b6000322e..acbacf859 100644 --- a/conf/mod_ollama_chat.conf.dist +++ b/conf/mod_ollama_chat.conf.dist @@ -58,6 +58,14 @@ OllamaChat.Model = llama3.2:1b # | 100+ | 70–80+ | Multi-paragraph (not typical) | OllamaChat.NumPredict = 100 +# OllamaChat.SoftStopEnable +# When enabled (1), the Ollama request streams and stops at the first complete-sentence +# boundary once NumPredict tokens have been generated, so replies end on a finished sentence +# instead of being cut mid-word. NumPredict is the soft target (small token headroom past it to +# finish the sentence). No effect when NumPredict = 0 or ThinkModeEnableForModule = 1. 0 = legacy non-streaming. +# Default: 1 +OllamaChat.SoftStopEnable = 1 + # OllamaChat.Temperature # Description: Controls the "creativity" or randomness of the model’s output. # Lower values (e.g., 0.2) = more focused, repetitive, and predictable. diff --git a/src/mod-ollama-chat_api.cpp b/src/mod-ollama-chat_api.cpp index 271077d9d..d349df83b 100644 --- a/src/mod-ollama-chat_api.cpp +++ b/src/mod-ollama-chat_api.cpp @@ -10,6 +10,7 @@ #include #include #include +#include std::string ExtractTextBetweenDoubleQuotes(const std::string& response) { @@ -21,6 +22,140 @@ std::string ExtractTextBetweenDoubleQuotes(const std::string& response) return response; } +static const uint32_t SOFT_STOP_HEADROOM = 32; + +static bool EndsOnSentenceBoundary(const std::string& s) +{ + size_t e = s.size(); + while (e > 0) + { + unsigned char c = (unsigned char)s[e - 1]; + if (c == ' ' || c == '\t' || c == '\n' || c == '\r' || + c == '"' || c == '\'' || c == ')' || c == ']' || c == '*') + { --e; continue; } + break; + } + if (e == 0) + return false; + char c = s[e - 1]; + if (c == '.' || c == '!' || c == '?') + return true; + if (e >= 3 && + (unsigned char)s[e - 3] == 0xE2 && + (unsigned char)s[e - 2] == 0x80 && + (unsigned char)s[e - 1] == 0xA6) + return true; + return false; +} + +struct OllamaStreamAccumulator +{ + std::string lineBuf; + std::string text; + uint32_t tokens = 0; + uint32_t softTarget = 0; + bool done = false; + + explicit OllamaStreamAccumulator(uint32_t target) : softTarget(target) {} + + bool ConsumeLine(const std::string& raw) + { + size_t b = raw.find_first_not_of(" \t\r\n"); + if (b == std::string::npos) + return false; + size_t e = raw.find_last_not_of(" \t\r\n"); + std::string line = raw.substr(b, e - b + 1); + try + { + nlohmann::json j = nlohmann::json::parse(line); + if (j.contains("response")) + { + std::string piece = j["response"].get(); + if (!piece.empty()) + { + text += piece; + ++tokens; + } + } + if (j.contains("done") && j["done"].is_boolean() && j["done"].get()) + done = true; + } + catch (const std::exception&) + { + } + if (done) + return true; + if (softTarget > 0 && tokens >= softTarget && EndsOnSentenceBoundary(text)) + return true; + return false; + } + + bool Feed(const char* data, size_t len) + { + lineBuf.append(data, len); + size_t nl; + while ((nl = lineBuf.find('\n')) != std::string::npos) + { + std::string line = lineBuf.substr(0, nl); + lineBuf.erase(0, nl + 1); + if (ConsumeLine(line)) + return true; + } + return false; + } +}; + +void SoftStopSelfTest() +{ + int passed = 0, total = 0; + + auto feedAll = [](OllamaStreamAccumulator& acc, const std::string& stream) -> bool { + return acc.Feed(stream.data(), stream.size()); + }; + + { + ++total; + OllamaStreamAccumulator acc(2); + std::string s = + "{\"response\":\"Hello\"}\n{\"response\":\" world\"}\n{\"response\":\".\"}\n{\"response\":\" extra\"}\n"; + bool stopped = feedAll(acc, s); + if (stopped && acc.text == "Hello world.") ++passed; + else LOG_ERROR("server.loading", "[Ollama Chat] SoftStop self-test FAIL (stop-at-period) -> stopped={} text=[{}]", stopped, acc.text); + } + + { + ++total; + OllamaStreamAccumulator acc(2); + std::string s = + "{\"response\":\"Hello\"}\n{\"response\":\" world\"}\n{\"response\":\" today\"}\n{\"response\":\"!\"}\n"; + bool stopped = feedAll(acc, s); + if (stopped && acc.text == "Hello world today!") ++passed; + else LOG_ERROR("server.loading", "[Ollama Chat] SoftStop self-test FAIL (wait-for-boundary) -> stopped={} text=[{}]", stopped, acc.text); + } + + { + ++total; + OllamaStreamAccumulator acc(1); + std::string p1 = "{\"respo"; + std::string p2 = "nse\":\"Hi.\"}\n"; + bool s1 = acc.Feed(p1.data(), p1.size()); + bool s2 = acc.Feed(p2.data(), p2.size()); + if (!s1 && s2 && acc.text == "Hi.") ++passed; + else LOG_ERROR("server.loading", "[Ollama Chat] SoftStop self-test FAIL (chunk-split) -> s1={} s2={} text=[{}]", s1, s2, acc.text); + } + + { + ++total; + OllamaStreamAccumulator acc(10); + std::string s = "{\"response\":\"Hi\",\"done\":false}\n{\"response\":\" bye\",\"done\":true}\n"; + bool stopped = feedAll(acc, s); + if (stopped && acc.text == "Hi bye") ++passed; + else LOG_ERROR("server.loading", "[Ollama Chat] SoftStop self-test FAIL (done-flag) -> stopped={} text=[{}]", stopped, acc.text); + } + + LOG_INFO("server.loading", "[Ollama Chat] SoftStop self-test: {}/{} passed", passed, total); +} + // Function to perform the API call. std::string QueryOllamaAPI(const std::string& prompt) { @@ -132,52 +267,81 @@ std::string QueryOllamaAPI(const std::string& prompt) requestData["hidethinking"] = true; } - std::string requestDataStr = requestData.dump(); + std::string botReply; - // Make HTTP POST request using our custom client - std::string responseBuffer = httpClient.Post(url, requestDataStr); + bool useStreaming = (g_SoftStopEnable && g_OllamaNumPredict > 0 && !g_ThinkModeEnableForModule); - if (responseBuffer.empty()) + if (useStreaming) { - LOG_ERROR("server.loading", "[OllamaChat] ERROR: Failed to reach Ollama API at {}. Check URL configuration and network connectivity.", url); - if(g_DebugEnabled) + requestData["stream"] = true; + requestData["options"]["num_predict"] = g_OllamaNumPredict + SOFT_STOP_HEADROOM; + + std::string requestDataStr = requestData.dump(); + + OllamaStreamAccumulator acc(g_OllamaNumPredict); + bool ok = httpClient.PostStreaming(url, requestDataStr, + [&](const char* data, size_t len) -> bool { + return !acc.Feed(data, len); + }); + + if (!ok && acc.text.empty()) { - LOG_INFO("server.loading", "[OllamaChat] Debug: Empty response buffer from HTTP client. Model: {}", model); + LOG_ERROR("server.loading", "[OllamaChat] ERROR: Streaming request to {} produced no output.", url); + return ""; } - return ""; + botReply = acc.text; + + if (g_DebugEnabled) + LOG_INFO("server.loading", "[Ollama Chat] SoftStop streamed {} tokens (soft target {}).", acc.tokens, g_OllamaNumPredict); } + else + { + std::string requestDataStr = requestData.dump(); - std::stringstream ss(responseBuffer); - std::string line; - std::ostringstream extractedResponse; + // Make HTTP POST request using our custom client + std::string responseBuffer = httpClient.Post(url, requestDataStr); - try - { - while (std::getline(ss, line)) + if (responseBuffer.empty()) { - if (line.empty() || std::all_of(line.begin(), line.end(), isspace)) - continue; + LOG_ERROR("server.loading", "[OllamaChat] ERROR: Failed to reach Ollama API at {}. Check URL configuration and network connectivity.", url); + if(g_DebugEnabled) + { + LOG_INFO("server.loading", "[OllamaChat] Debug: Empty response buffer from HTTP client. Model: {}", model); + } + return ""; + } - nlohmann::json jsonResponse = nlohmann::json::parse(line); + std::stringstream ss(responseBuffer); + std::string line; + std::ostringstream extractedResponse; - if (jsonResponse.contains("response") && !jsonResponse["response"].get().empty()) + try + { + while (std::getline(ss, line)) { - extractedResponse << jsonResponse["response"].get(); + if (line.empty() || std::all_of(line.begin(), line.end(), isspace)) + continue; + + nlohmann::json jsonResponse = nlohmann::json::parse(line); + + if (jsonResponse.contains("response") && !jsonResponse["response"].get().empty()) + { + extractedResponse << jsonResponse["response"].get(); + } } } - } - catch (const std::exception& e) - { - LOG_ERROR("server.loading", "[OllamaChat] ERROR: JSON parsing failed. Exception: {}", e.what()); - if(g_DebugEnabled) + catch (const std::exception& e) { - LOG_INFO("server.loading", "[OllamaChat] Debug: Response buffer content: {}", responseBuffer); + LOG_ERROR("server.loading", "[OllamaChat] ERROR: JSON parsing failed. Exception: {}", e.what()); + if(g_DebugEnabled) + { + LOG_INFO("server.loading", "[OllamaChat] Debug: Response buffer content: {}", responseBuffer); + } + return ""; } - return ""; + botReply = extractedResponse.str(); } - std::string botReply = extractedResponse.str(); - botReply = ExtractTextBetweenDoubleQuotes(botReply); // Check for unclosed think tags diff --git a/src/mod-ollama-chat_api.h b/src/mod-ollama-chat_api.h index 058007fc9..50d5fe44d 100644 --- a/src/mod-ollama-chat_api.h +++ b/src/mod-ollama-chat_api.h @@ -7,6 +7,9 @@ std::string QueryOllamaAPI(const std::string& prompt); +// Debug-only sanity check of the streaming soft-stop accumulator (no network). +void SoftStopSelfTest(); + // Checks if an API response is valid (not an error message) bool IsValidAPIResponse(const std::string& response); diff --git a/src/mod-ollama-chat_config.cpp b/src/mod-ollama-chat_config.cpp index 8bc1b1da3..ff10620ad 100644 --- a/src/mod-ollama-chat_config.cpp +++ b/src/mod-ollama-chat_config.cpp @@ -43,6 +43,7 @@ uint32_t g_EventChatterMaxBotsPerPlayer = 2; std::string g_OllamaUrl = "http://localhost:11434/api/generate"; std::string g_OllamaModel = "llama3.2:1b"; uint32_t g_OllamaNumPredict = 40; +bool g_SoftStopEnable = true; float g_OllamaTemperature = 0.8f; float g_OllamaTopP = 0.95f; float g_OllamaRepeatPenalty = 1.1f; @@ -387,6 +388,7 @@ void LoadOllamaChatConfig() g_OllamaUrl = sConfigMgr->GetOption("OllamaChat.Url", "http://localhost:11434/api/generate"); g_OllamaModel = sConfigMgr->GetOption("OllamaChat.Model", "llama3.2:1b"); g_OllamaNumPredict = sConfigMgr->GetOption("OllamaChat.NumPredict", 40); + g_SoftStopEnable = sConfigMgr->GetOption("OllamaChat.SoftStopEnable", true); g_OllamaTemperature = sConfigMgr->GetOption("OllamaChat.Temperature", 0.8f); g_OllamaTopP = sConfigMgr->GetOption("OllamaChat.TopP", 0.95f); g_OllamaRepeatPenalty = sConfigMgr->GetOption("OllamaChat.RepeatPenalty", 1.1f); @@ -645,6 +647,9 @@ void LoadOllamaChatConfig() g_OllamaUrl, g_OllamaModel, g_MaxConcurrentQueries, g_EnableRandomChatter, g_MinRandomInterval, g_MaxRandomInterval, g_RandomChatterRealPlayerDistance, g_RandomChatterBotCommentChance, g_MaxConcurrentQueries, extraBlacklist); + + if (g_DebugEnabled) + SoftStopSelfTest(); } void LoadPersonalityTemplatesFromDB() diff --git a/src/mod-ollama-chat_config.h b/src/mod-ollama-chat_config.h index 37e66e26e..5fada3ac9 100644 --- a/src/mod-ollama-chat_config.h +++ b/src/mod-ollama-chat_config.h @@ -44,6 +44,7 @@ extern uint32_t g_EventChatterMaxBotsPerPlayer; extern std::string g_OllamaUrl; extern std::string g_OllamaModel; extern uint32_t g_OllamaNumPredict; +extern bool g_SoftStopEnable; // stream + stop at a sentence boundary past NumPredict (soft cap) extern float g_OllamaTemperature; extern float g_OllamaTopP; extern float g_OllamaRepeatPenalty; diff --git a/src/mod-ollama-chat_httpclient.cpp b/src/mod-ollama-chat_httpclient.cpp index b2e3294ab..2a26ead6a 100644 --- a/src/mod-ollama-chat_httpclient.cpp +++ b/src/mod-ollama-chat_httpclient.cpp @@ -156,6 +156,98 @@ std::string OllamaHttpClient::Post(const std::string& url, const std::string& js } } +bool OllamaHttpClient::PostStreaming(const std::string& url, const std::string& jsonData, + const std::function& onChunk) +{ + try + { + std::regex urlRegex(R"(^(https?)://([^:/]+)(?::(\d+))?(/.*)?$)"); + std::smatch match; + if (!std::regex_match(url, match, urlRegex)) + { + LOG_INFO("server.loading", "[Ollama Chat] Invalid URL format: {}", url); + return false; + } + + std::string protocol = match[1].str(); + std::string host = match[2].str(); + int port = 11434; + if (match[3].matched) port = std::stoi(match[3].str()); + else if (protocol == "https") port = 443; + else port = 11434; + std::string path = match[4].matched ? match[4].str() : "/"; + + httplib::Headers headers = { + {"Content-Type", "application/json"}, + {"User-Agent", "AzerothCore-OllamaChat/1.0"}, + {"Accept", "application/json"} + }; + if (host.find("ngrok") != std::string::npos) + headers.emplace("ngrok-skip-browser-warning", "true"); + + // Stream the RESPONSE body via a low-level Request + content_receiver. The buffered + // Post(..., ContentReceiver) overload is not present in every bundled httplib version, + // but send() + Request::content_receiver is; a generic-lambda receiver adapts to either + // ContentReceiverWithProgress signature (uint64_t vs size_t offsets) across versions. + httplib::Request req; + req.method = "POST"; + req.path = path; + req.headers = headers; + req.body = jsonData; + req.content_receiver = + [&](const char* data, size_t len, auto /*offset*/, auto /*total*/) -> bool { + return onChunk(data, len); + }; + + httplib::Response resp; + httplib::Error err = httplib::Error::Success; + bool ok = false; + if (protocol == "https") + { +#ifdef CPPHTTPLIB_OPENSSL_SUPPORT + httplib::SSLClient sslClient(host, port); + sslClient.enable_server_certificate_verification(false); + sslClient.set_connection_timeout(m_timeout); + sslClient.set_read_timeout(m_timeout); + sslClient.set_write_timeout(m_timeout); + ok = sslClient.send(req, resp, err); +#else + LOG_ERROR("server.loading", "[Ollama Chat] HTTPS requested but SSL support not available."); + return false; +#endif + } + else + { + httplib::Client client(host, port); + client.set_connection_timeout(m_timeout); + client.set_read_timeout(m_timeout); + client.set_write_timeout(m_timeout); + ok = client.send(req, resp, err); + } + + // A deliberate abort by content_receiver surfaces as Error::Canceled -- that is success + // for our purposes (we got the prefix we wanted and stopped generation). + if (!ok) + { + if (err == httplib::Error::Canceled) + return true; + LOG_ERROR("server.loading", "[Ollama Chat] Streaming request failed (err {}): {}:{}{}", (int)err, host, port, path); + return false; + } + if (resp.status != 200) + { + LOG_ERROR("server.loading", "[Ollama Chat] Streaming request HTTP {} for {}:{}{}", resp.status, host, port, path); + return false; + } + return true; + } + catch (const std::exception& e) + { + LOG_ERROR("server.loading", "[Ollama Chat] Streaming client exception: {}", e.what()); + return false; + } +} + void OllamaHttpClient::SetTimeout(int seconds) { m_timeout = seconds; diff --git a/src/mod-ollama-chat_httpclient.h b/src/mod-ollama-chat_httpclient.h index ec51274e8..e4ed96c7c 100644 --- a/src/mod-ollama-chat_httpclient.h +++ b/src/mod-ollama-chat_httpclient.h @@ -2,6 +2,7 @@ #define OLLAMA_HTTP_CLIENT_H #include +#include class OllamaHttpClient { @@ -11,7 +12,14 @@ class OllamaHttpClient // Make HTTP POST request to Ollama API std::string Post(const std::string& url, const std::string& jsonData); - + + // Streaming POST: invokes onChunk(data, len) for each received body chunk. + // onChunk returns true to keep receiving, false to abort the transfer. + // Returns true if the request completed or was deliberately aborted by onChunk; + // false on a connection/HTTP error. + bool PostStreaming(const std::string& url, const std::string& jsonData, + const std::function& onChunk); + // Set timeout for requests (in seconds) void SetTimeout(int seconds);