feat: stream + stop at first sentence boundary past NumPredict (SoftStop)

conf/mod_ollama_chat.conf.dist

@@ -58,6 +58,14 @@ OllamaChat.Model = llama3.2:1b
 #     |    100+    |   70–80+      | Multi-paragraph (not typical)    |
 OllamaChat.NumPredict = 100
 
+# OllamaChat.SoftStopEnable
+#     When enabled (1), the Ollama request streams and stops at the first complete-sentence
+#     boundary once NumPredict tokens have been generated, so replies end on a finished sentence
+#     instead of being cut mid-word. NumPredict is the soft target (small token headroom past it to
+#     finish the sentence). No effect when NumPredict = 0 or ThinkModeEnableForModule = 1. 0 = legacy non-streaming.
+#     Default: 1
+OllamaChat.SoftStopEnable = 1
+
 # OllamaChat.Temperature
 #     Description: Controls the "creativity" or randomness of the model’s output.
 #                  Lower values (e.g., 0.2) = more focused, repetitive, and predictable.

src/mod-ollama-chat_api.cpp

@@ -10,6 +10,7 @@
 #include <mutex>
 #include <queue>
 #include <future>
+#include <algorithm>
 
 std::string ExtractTextBetweenDoubleQuotes(const std::string& response)
 {
@@ -21,6 +22,140 @@ std::string ExtractTextBetweenDoubleQuotes(const std::string& response)
     return response;
 }
 
+static const uint32_t SOFT_STOP_HEADROOM = 32;
+
+static bool EndsOnSentenceBoundary(const std::string& s)
+{
+    size_t e = s.size();
+    while (e > 0)
+    {
+        unsigned char c = (unsigned char)s[e - 1];
+        if (c == ' ' || c == '\t' || c == '\n' || c == '\r' ||
+            c == '"' || c == '\'' || c == ')' || c == ']' || c == '*')
+        { --e; continue; }
+        break;
+    }
+    if (e == 0)
+        return false;
+    char c = s[e - 1];
+    if (c == '.' || c == '!' || c == '?')
+        return true;
+    if (e >= 3 &&
+        (unsigned char)s[e - 3] == 0xE2 &&
+        (unsigned char)s[e - 2] == 0x80 &&
+        (unsigned char)s[e - 1] == 0xA6)
+        return true;
+    return false;
+}
+
+struct OllamaStreamAccumulator
+{
+    std::string lineBuf;
+    std::string text;
+    uint32_t    tokens = 0;
+    uint32_t    softTarget = 0;
+    bool        done = false;
+
+    explicit OllamaStreamAccumulator(uint32_t target) : softTarget(target) {}
+
+    bool ConsumeLine(const std::string& raw)
+    {
+        size_t b = raw.find_first_not_of(" \t\r\n");
+        if (b == std::string::npos)
+            return false;
+        size_t e = raw.find_last_not_of(" \t\r\n");
+        std::string line = raw.substr(b, e - b + 1);
+        try
+        {
+            nlohmann::json j = nlohmann::json::parse(line);
+            if (j.contains("response"))
+            {
+                std::string piece = j["response"].get<std::string>();
+                if (!piece.empty())
+                {
+                    text += piece;
+                    ++tokens;
+                }
+            }
+            if (j.contains("done") && j["done"].is_boolean() && j["done"].get<bool>())
+                done = true;
+        }
+        catch (const std::exception&)
+        {
+        }
+        if (done)
+            return true;
+        if (softTarget > 0 && tokens >= softTarget && EndsOnSentenceBoundary(text))
+            return true;
+        return false;
+    }
+
+    bool Feed(const char* data, size_t len)
+    {
+        lineBuf.append(data, len);
+        size_t nl;
+        while ((nl = lineBuf.find('\n')) != std::string::npos)
+        {
+            std::string line = lineBuf.substr(0, nl);
+            lineBuf.erase(0, nl + 1);
+            if (ConsumeLine(line))
+                return true;
+        }
+        return false;
+    }
+};
+
+void SoftStopSelfTest()
+{
+    int passed = 0, total = 0;
+
+    auto feedAll = [](OllamaStreamAccumulator& acc, const std::string& stream) -> bool {
+        return acc.Feed(stream.data(), stream.size());
+    };
+
+    {
+        ++total;
+        OllamaStreamAccumulator acc(2);
+        std::string s =
+            "{\"response\":\"Hello\"}\n{\"response\":\" world\"}\n{\"response\":\".\"}\n{\"response\":\" extra\"}\n";
+        bool stopped = feedAll(acc, s);
+        if (stopped && acc.text == "Hello world.") ++passed;
+        else LOG_ERROR("server.loading", "[Ollama Chat] SoftStop self-test FAIL (stop-at-period) -> stopped={} text=[{}]", stopped, acc.text);
+    }
+
+    {
+        ++total;
+        OllamaStreamAccumulator acc(2);
+        std::string s =
+            "{\"response\":\"Hello\"}\n{\"response\":\" world\"}\n{\"response\":\" today\"}\n{\"response\":\"!\"}\n";
+        bool stopped = feedAll(acc, s);
+        if (stopped && acc.text == "Hello world today!") ++passed;
+        else LOG_ERROR("server.loading", "[Ollama Chat] SoftStop self-test FAIL (wait-for-boundary) -> stopped={} text=[{}]", stopped, acc.text);
+    }
+
+    {
+        ++total;
+        OllamaStreamAccumulator acc(1);
+        std::string p1 = "{\"respo";
+        std::string p2 = "nse\":\"Hi.\"}\n";
+        bool s1 = acc.Feed(p1.data(), p1.size());
+        bool s2 = acc.Feed(p2.data(), p2.size());
+        if (!s1 && s2 && acc.text == "Hi.") ++passed;
+        else LOG_ERROR("server.loading", "[Ollama Chat] SoftStop self-test FAIL (chunk-split) -> s1={} s2={} text=[{}]", s1, s2, acc.text);
+    }
+
+    {
+        ++total;
+        OllamaStreamAccumulator acc(10);
+        std::string s = "{\"response\":\"Hi\",\"done\":false}\n{\"response\":\" bye\",\"done\":true}\n";
+        bool stopped = feedAll(acc, s);
+        if (stopped && acc.text == "Hi bye") ++passed;
+        else LOG_ERROR("server.loading", "[Ollama Chat] SoftStop self-test FAIL (done-flag) -> stopped={} text=[{}]", stopped, acc.text);
+    }
+
+    LOG_INFO("server.loading", "[Ollama Chat] SoftStop self-test: {}/{} passed", passed, total);
+}
+
 // Function to perform the API call.
 std::string QueryOllamaAPI(const std::string& prompt)
 {
@@ -132,52 +267,81 @@ std::string QueryOllamaAPI(const std::string& prompt)
         requestData["hidethinking"] = true;
     }
 
-    std::string requestDataStr = requestData.dump();
+    std::string botReply;
 
-    // Make HTTP POST request using our custom client
-    std::string responseBuffer = httpClient.Post(url, requestDataStr);
+    bool useStreaming = (g_SoftStopEnable && g_OllamaNumPredict > 0 && !g_ThinkModeEnableForModule);
 
-    if (responseBuffer.empty())
+    if (useStreaming)
     {
-        LOG_ERROR("server.loading", "[OllamaChat] ERROR: Failed to reach Ollama API at {}. Check URL configuration and network connectivity.", url);
-        if(g_DebugEnabled)
+        requestData["stream"] = true;
+        requestData["options"]["num_predict"] = g_OllamaNumPredict + SOFT_STOP_HEADROOM;
+
+        std::string requestDataStr = requestData.dump();
+
+        OllamaStreamAccumulator acc(g_OllamaNumPredict);
+        bool ok = httpClient.PostStreaming(url, requestDataStr,
+            [&](const char* data, size_t len) -> bool {
+                return !acc.Feed(data, len);
+            });
+
+        if (!ok && acc.text.empty())
         {
-            LOG_INFO("server.loading", "[OllamaChat] Debug: Empty response buffer from HTTP client. Model: {}", model);
+            LOG_ERROR("server.loading", "[OllamaChat] ERROR: Streaming request to {} produced no output.", url);
+            return "";
         }
-        return "";
+        botReply = acc.text;
+
+        if (g_DebugEnabled)
+            LOG_INFO("server.loading", "[Ollama Chat] SoftStop streamed {} tokens (soft target {}).", acc.tokens, g_OllamaNumPredict);
     }
+    else
+    {
+        std::string requestDataStr = requestData.dump();
 
-    std::stringstream ss(responseBuffer);
-    std::string line;
-    std::ostringstream extractedResponse;
+        // Make HTTP POST request using our custom client
+        std::string responseBuffer = httpClient.Post(url, requestDataStr);
 
-    try
-    {
-        while (std::getline(ss, line))
+        if (responseBuffer.empty())
         {
-            if (line.empty() || std::all_of(line.begin(), line.end(), isspace))
-                continue;
+            LOG_ERROR("server.loading", "[OllamaChat] ERROR: Failed to reach Ollama API at {}. Check URL configuration and network connectivity.", url);
+            if(g_DebugEnabled)
+            {
+                LOG_INFO("server.loading", "[OllamaChat] Debug: Empty response buffer from HTTP client. Model: {}", model);
+            }
+            return "";
+        }
 
-            nlohmann::json jsonResponse = nlohmann::json::parse(line);
+        std::stringstream ss(responseBuffer);
+        std::string line;
+        std::ostringstream extractedResponse;
 
-            if (jsonResponse.contains("response") && !jsonResponse["response"].get<std::string>().empty())
+        try
+        {
+            while (std::getline(ss, line))
             {
-                extractedResponse << jsonResponse["response"].get<std::string>();
+                if (line.empty() || std::all_of(line.begin(), line.end(), isspace))
+                    continue;
+
+                nlohmann::json jsonResponse = nlohmann::json::parse(line);
+
+                if (jsonResponse.contains("response") && !jsonResponse["response"].get<std::string>().empty())
+                {
+                    extractedResponse << jsonResponse["response"].get<std::string>();
+                }
             }
         }
-    }
-    catch (const std::exception& e)
-    {
-        LOG_ERROR("server.loading", "[OllamaChat] ERROR: JSON parsing failed. Exception: {}", e.what());
-        if(g_DebugEnabled)
+        catch (const std::exception& e)
         {
-            LOG_INFO("server.loading", "[OllamaChat] Debug: Response buffer content: {}", responseBuffer);
+            LOG_ERROR("server.loading", "[OllamaChat] ERROR: JSON parsing failed. Exception: {}", e.what());
+            if(g_DebugEnabled)
+            {
+                LOG_INFO("server.loading", "[OllamaChat] Debug: Response buffer content: {}", responseBuffer);
+            }
+            return "";
         }
-        return "";
+        botReply = extractedResponse.str();
     }
 
-    std::string botReply = extractedResponse.str();
-
     botReply = ExtractTextBetweenDoubleQuotes(botReply);
 
     // Check for unclosed think tags

src/mod-ollama-chat_api.h

@@ -7,6 +7,9 @@
 
 std::string QueryOllamaAPI(const std::string& prompt);
 
+// Debug-only sanity check of the streaming soft-stop accumulator (no network).
+void SoftStopSelfTest();
+
 // Checks if an API response is valid (not an error message)
 bool IsValidAPIResponse(const std::string& response);
 

src/mod-ollama-chat_config.cpp

@@ -43,6 +43,7 @@ uint32_t   g_EventChatterMaxBotsPerPlayer    = 2;
 std::string g_OllamaUrl        = "http://localhost:11434/api/generate";
 std::string g_OllamaModel      = "llama3.2:1b";
 uint32_t    g_OllamaNumPredict = 40;
+bool        g_SoftStopEnable = true;
 float       g_OllamaTemperature = 0.8f;
 float       g_OllamaTopP = 0.95f;
 float       g_OllamaRepeatPenalty = 1.1f;
@@ -387,6 +388,7 @@ void LoadOllamaChatConfig()
     g_OllamaUrl                       = sConfigMgr->GetOption<std::string>("OllamaChat.Url", "http://localhost:11434/api/generate");
     g_OllamaModel                     = sConfigMgr->GetOption<std::string>("OllamaChat.Model", "llama3.2:1b");
     g_OllamaNumPredict                = sConfigMgr->GetOption<uint32_t>("OllamaChat.NumPredict", 40);
+    g_SoftStopEnable                  = sConfigMgr->GetOption<bool>("OllamaChat.SoftStopEnable", true);
     g_OllamaTemperature               = sConfigMgr->GetOption<float>("OllamaChat.Temperature", 0.8f);
     g_OllamaTopP                      = sConfigMgr->GetOption<float>("OllamaChat.TopP", 0.95f);
     g_OllamaRepeatPenalty             = sConfigMgr->GetOption<float>("OllamaChat.RepeatPenalty", 1.1f);
@@ -645,6 +647,9 @@ void LoadOllamaChatConfig()
              g_OllamaUrl, g_OllamaModel, g_MaxConcurrentQueries,
              g_EnableRandomChatter, g_MinRandomInterval, g_MaxRandomInterval, g_RandomChatterRealPlayerDistance,
              g_RandomChatterBotCommentChance, g_MaxConcurrentQueries, extraBlacklist);
+
+    if (g_DebugEnabled)
+        SoftStopSelfTest();
 }
 
 void LoadPersonalityTemplatesFromDB()

src/mod-ollama-chat_config.h

@@ -44,6 +44,7 @@ extern uint32_t   g_EventChatterMaxBotsPerPlayer;
 extern std::string g_OllamaUrl;
 extern std::string g_OllamaModel;
 extern uint32_t    g_OllamaNumPredict;
+extern bool        g_SoftStopEnable;   // stream + stop at a sentence boundary past NumPredict (soft cap)
 extern float       g_OllamaTemperature;
 extern float       g_OllamaTopP;
 extern float       g_OllamaRepeatPenalty;