FIX: truncation leading to invalid strings when on UTF boundaries

.gitignore

@@ -11,3 +11,4 @@
 
 *.gem
 .rspec_status
+Gemfile.lock

CHANGELOG.md

@@ -1,5 +1,9 @@
 ## [Unreleased]
 
+## [0.3.2] - 2025-12-10
+
+- Fix truncation logic in OpenAiTokenizer could lead to string parsing fails
+
 ## [0.3.1] - 2025-07-07
 
 - Refactor OpenAiO200kTokenizer class to OpenAiTokenizer as primary class name

lib/discourse_ai/tokenizer/open_ai_tokenizer.rb

@@ -19,9 +19,31 @@ def encode(text)
 
         def decode(token_ids)
           tokenizer.decode(token_ids)
-        rescue Tiktoken::UnicodeError => e
-          # Handle invalid token IDs gracefully by returning empty string
-          ""
+        rescue Tiktoken::UnicodeError
+          token_ids = token_ids.dup
+
+          # this easy case, we started with a valid sequnce but truncated it on an invalid boundary
+          # work backwards removing tokens until we can decode again
+          tries = 4
+          while tries > 0
+            begin
+              token_ids.pop
+              return tokenizer.decode(token_ids)
+            rescue Tiktoken::UnicodeError
+              tries -= 1
+            end
+          end
+
+          # at this point we may have a corrupted sequence so just decode what we can
+          token_ids
+            .map do |id|
+              begin
+                tokenizer.decode([id])
+              rescue Tiktoken::UnicodeError
+                ""
+              end
+            end
+            .join
         end
 
         def truncate(text, max_length, strict: false)
@@ -33,12 +55,12 @@ def truncate(text, max_length, strict: false)
 
           # Take tokens up to max_length, decode, then ensure we don't exceed limit
           truncated_tokens = tokenize(text).take(max_length)
-          truncated_text = tokenizer.decode(truncated_tokens)
+          truncated_text = decode(truncated_tokens)
 
           # If re-encoding exceeds the limit, we need to further truncate
           while tokenize(truncated_text).length > max_length
             truncated_tokens = truncated_tokens[0...-1]
-            truncated_text = tokenizer.decode(truncated_tokens)
+            truncated_text = decode(truncated_tokens)
             break if truncated_tokens.empty?
           end
 

lib/discourse_ai/tokenizers/version.rb

@@ -2,6 +2,6 @@
 
 module DiscourseAi
   module Tokenizers
-    VERSION = "0.3.1"
+    VERSION = "0.3.2"
   end
 end

spec/discourse_ai/tokenizers/error_handling_spec.rb

@@ -84,6 +84,25 @@
           expect(decoded).to include("世界")
         end
       end
+
+      it "handles truncation at all token boundaries without raising" do
+        text = "日本語テスト 🎉 中文测试 العربية"
+        token_count = tokenizer_class.size(text)
+
+        (1..token_count).each do |i|
+          expect {
+            tokenizer_class.truncate(text, i, strict: true)
+          }.not_to raise_error
+        end
+      end
+
+      it "returns valid UTF-8 strings when truncating multi-byte characters" do
+        text = "日本語テスト 🎉 中文测试 العربية"
+
+        result = tokenizer_class.truncate(text, 5, strict: true)
+        expect(result).to be_a(String)
+        expect(result.valid_encoding?).to be true
+      end
     end
 
     describe "edge case parameters" do
@@ -155,6 +174,47 @@
   describe DiscourseAi::Tokenizer::OpenAiTokenizer do
     include_examples "tokenizer error handling",
                      DiscourseAi::Tokenizer::OpenAiTokenizer
+
+    describe "truncation correctness" do
+      let(:tokenizer) { DiscourseAi::Tokenizer::OpenAiTokenizer }
+
+      it "truncates simple ASCII text correctly" do
+        text = "Hello world this is a test"
+        result = tokenizer.truncate(text, 3, strict: true)
+
+        expect(result).to eq("Hello world this")
+        expect(tokenizer.size(result)).to be <= 3
+      end
+
+      it "truncates multi-byte UTF-8 text correctly" do
+        text = "a 🎉 a 🎉 a"
+
+        result = tokenizer.truncate(text, 2, strict: true)
+        expect(result).to eq("a")
+
+        result = tokenizer.truncate(text, 3, strict: true)
+        expect(result).to eq("a 🎉")
+
+        result = tokenizer.truncate(text, 5, strict: true)
+        expect(result).to eq("a 🎉 a")
+      end
+
+      it "never exceeds the requested token limit" do
+        text = "日本語テスト 🎉 中文测试 العربية"
+
+        (1..tokenizer.size(text)).each do |limit|
+          result = tokenizer.truncate(text, limit, strict: true)
+          expect(tokenizer.size(result)).to be <= limit
+        end
+      end
+
+      it "preserves text prefix when truncating" do
+        text = "Hello 世界 test"
+        result = tokenizer.truncate(text, 2, strict: true)
+
+        expect(text).to start_with(result)
+      end
+    end
   end
 
   describe DiscourseAi::Tokenizer::AllMpnetBaseV2Tokenizer do