FIX: Normalize non-UTF-8 inputs before tokenization

CHANGELOG.md

@@ -1,4 +1,6 @@
-## [Unreleased]
+## [0.4.2] - 2026-02-27
+
+- Normalize `ASCII-8BIT`/non-UTF-8 string inputs before tokenization to prevent `EncodingError` in `truncate`, `encode`, and `below_limit?`
 
 ## [0.4.1] - 2026-02-26
 

lib/discourse_ai/tokenizer/basic_tokenizer.rb

@@ -21,7 +21,7 @@ def tokenizer
         end
 
         def tokenize(text)
-          tokenizer.encode(text).tokens
+          tokenizer.encode(normalize_text(text)).tokens
         end
 
         def size(text)
@@ -32,38 +32,67 @@ def decode(token_ids)
           tokenizer.decode(token_ids)
         end
 
-        def encode(tokens)
-          tokenizer.encode(tokens).ids
+        def encode(text)
+          tokenizer.encode(normalize_text(text)).ids
         end
 
         def truncate(text, max_length, strict: false)
           return "" if max_length <= 0
 
+          text = normalize_text(text)
+
           # fast track common case, /2 to handle unicode chars
           # than can take more than 1 token per char
           return text if !strict && text.size < max_length / 2
 
           # Take tokens up to max_length, decode, then ensure we don't exceed limit
           truncated_tokens = tokenizer.encode(text).ids.take(max_length)
-          truncated_text = tokenizer.decode(truncated_tokens)
+          truncated_text = normalize_text(tokenizer.decode(truncated_tokens))
 
           # If re-encoding exceeds the limit, we need to further truncate
           while tokenizer.encode(truncated_text).ids.length > max_length
             truncated_tokens = truncated_tokens[0...-1]
-            truncated_text = tokenizer.decode(truncated_tokens)
+            truncated_text = normalize_text(tokenizer.decode(truncated_tokens))
             break if truncated_tokens.empty?
           end
 
-          truncated_text
+          normalize_text(truncated_text)
         end
 
         def below_limit?(text, limit, strict: false)
+          text = normalize_text(text)
+
           # fast track common case, /2 to handle unicode chars
           # than can take more than 1 token per char
           return true if !strict && text.size < limit / 2
 
           tokenizer.encode(text).ids.length < limit
         end
+
+        private
+
+        def normalize_text(text)
+          return text unless text.is_a?(String)
+
+          # Fast path: avoid allocations for the common valid UTF-8 case.
+          if text.encoding == Encoding::UTF_8 && text.valid_encoding?
+            return text
+          end
+
+          if text.encoding == Encoding::ASCII_8BIT
+            normalized = text.dup
+            normalized.force_encoding(Encoding::UTF_8)
+          elsif text.encoding != Encoding::UTF_8
+            normalized = text.encode(Encoding::UTF_8)
+          else
+            normalized = text
+          end
+
+          normalized.valid_encoding? ? normalized : normalized.scrub
+        rescue Encoding::UndefinedConversionError,
+               Encoding::InvalidByteSequenceError
+          text.encode(Encoding::UTF_8, invalid: :replace, undef: :replace)
+        end
       end
     end
   end

lib/discourse_ai/tokenizer/open_ai_tokenizer.rb

@@ -54,25 +54,29 @@ def decode(token_ids)
         def truncate(text, max_length, strict: false)
           return "" if max_length <= 0
 
+          text = normalize_text(text)
+
           # fast track common case, /2 to handle unicode chars
           # than can take more than 1 token per char
           return text if !strict && text.size < max_length / 2
 
           # Take tokens up to max_length, decode, then ensure we don't exceed limit
           truncated_tokens = tokenize(text).take(max_length)
-          truncated_text = decode(truncated_tokens)
+          truncated_text = normalize_text(decode(truncated_tokens))
 
           # If re-encoding exceeds the limit, we need to further truncate
           while tokenize(truncated_text).length > max_length
             truncated_tokens = truncated_tokens[0...-1]
-            truncated_text = decode(truncated_tokens)
+            truncated_text = normalize_text(decode(truncated_tokens))
             break if truncated_tokens.empty?
           end
 
-          truncated_text
+          normalize_text(truncated_text)
         end
 
         def below_limit?(text, limit, strict: false)
+          text = normalize_text(text)
+
           # fast track common case, /2 to handle unicode chars
           # than can take more than 1 token per char
           return true if !strict && text.size < limit / 2
@@ -83,6 +87,8 @@ def below_limit?(text, limit, strict: false)
         private
 
         def safe_encode(text)
+          text = normalize_text(text)
+
           if !text.is_a?(String) || text.size <= SAFE_CHUNK_SIZE
             return tokenizer.encode(text)
           end

lib/discourse_ai/tokenizers/version.rb

@@ -2,6 +2,6 @@
 
 module DiscourseAi
   module Tokenizers
-    VERSION = "0.4.1"
+    VERSION = "0.4.2"
   end
 end

spec/discourse_ai/tokenizers/error_handling_spec.rb

@@ -103,6 +103,132 @@
         expect(result).to be_a(String)
         expect(result.valid_encoding?).to be true
       end
+
+      it "handles ASCII-8BIT text in truncate" do
+        utf8_text = "日本語テスト 🎉 中文测试 العربية"
+        text = utf8_text.dup.force_encoding(Encoding::ASCII_8BIT)
+
+        limit = 5
+        utf8_result = tokenizer_class.truncate(utf8_text, limit, strict: true)
+        result = tokenizer_class.truncate(text, limit, strict: true)
+
+        expect(result).to be_a(String)
+        expect(result).to eq(utf8_result)
+        expect(result.encoding).to eq(Encoding::UTF_8)
+        expect(result.valid_encoding?).to be true
+        expect(tokenizer_class.size(result)).to be <= limit
+      end
+
+      it "handles ASCII-8BIT text in below_limit?" do
+        utf8_text = "日本語テスト 🎉 中文测试 العربية"
+        text = utf8_text.dup.force_encoding(Encoding::ASCII_8BIT)
+        token_count = tokenizer_class.size(utf8_text)
+        limits = [
+          1,
+          [token_count - 1, 1].max,
+          token_count,
+          token_count + 1
+        ].uniq
+
+        limits.each do |limit|
+          expect(tokenizer_class.below_limit?(text, limit, strict: true)).to eq(
+            tokenizer_class.below_limit?(utf8_text, limit, strict: true)
+          )
+          expect(
+            tokenizer_class.below_limit?(text, limit, strict: false)
+          ).to eq(tokenizer_class.below_limit?(utf8_text, limit, strict: false))
+        end
+      end
+
+      it "always returns valid UTF-8 from chained truncation" do
+        invalid_utf8 =
+          "日本語テスト".bytes[0..-2].pack("C*").force_encoding(Encoding::UTF_8)
+        expect(invalid_utf8.valid_encoding?).to be false
+
+        first = tokenizer_class.truncate(invalid_utf8, 5, strict: true)
+        second = tokenizer_class.truncate(first, 3, strict: true)
+
+        [first, second].each do |result|
+          expect(result).to be_a(String)
+          expect(result.encoding).to eq(Encoding::UTF_8)
+          expect(result.valid_encoding?).to be true
+        end
+
+        expect(tokenizer_class.size(first)).to be <= 5
+        expect(tokenizer_class.size(second)).to be <= 3
+      end
+    end
+
+    describe "pathological encoding handling" do
+      let(:valid_utf8_text) { "Cafe 日本語 🎉" }
+      let(:binary_utf8_text) do
+        valid_utf8_text.dup.force_encoding(Encoding::ASCII_8BIT)
+      end
+      let(:invalid_utf8) { "abc\xE2\x82".b.force_encoding(Encoding::UTF_8) }
+      let(:invalid_binary_text) { "bad\xFF\xFEtext\xC3".b }
+      let(:latin1_text) { "caf\xe9".dup.force_encoding(Encoding::ISO_8859_1) }
+      let(:normalized_cases) do
+        [
+          [binary_utf8_text, valid_utf8_text],
+          [invalid_utf8, invalid_utf8.scrub],
+          [
+            invalid_binary_text,
+            invalid_binary_text.dup.force_encoding(Encoding::UTF_8).scrub
+          ],
+          [latin1_text, "café"]
+        ]
+      end
+
+      it "normalizes tokenize, size, and encode inputs" do
+        normalized_cases.each do |input, normalized|
+          expect(tokenizer_class.tokenize(input)).to eq(
+            tokenizer_class.tokenize(normalized)
+          )
+          expect(tokenizer_class.size(input)).to eq(
+            tokenizer_class.size(normalized)
+          )
+          expect(tokenizer_class.encode(input)).to eq(
+            tokenizer_class.encode(normalized)
+          )
+        end
+      end
+
+      it "normalizes truncate and below_limit? inputs" do
+        normalized_cases.each do |input, normalized|
+          token_count = tokenizer_class.size(normalized)
+          limits = [
+            1,
+            [token_count - 1, 1].max,
+            token_count,
+            token_count + 1
+          ].uniq
+
+          strict_truncation =
+            tokenizer_class.truncate(input, [token_count, 1].max, strict: true)
+          expect(strict_truncation.encoding).to eq(Encoding::UTF_8)
+          expect(strict_truncation.valid_encoding?).to be true
+          expect(strict_truncation).to eq(
+            tokenizer_class.truncate(
+              normalized,
+              [token_count, 1].max,
+              strict: true
+            )
+          )
+
+          limits.each do |limit|
+            expect(
+              tokenizer_class.below_limit?(input, limit, strict: true)
+            ).to eq(
+              tokenizer_class.below_limit?(normalized, limit, strict: true)
+            )
+            expect(
+              tokenizer_class.below_limit?(input, limit, strict: false)
+            ).to eq(
+              tokenizer_class.below_limit?(normalized, limit, strict: false)
+            )
+          end
+        end
+      end
     end
 
     describe "edge case parameters" do

spec/discourse_ai/tokenizers/safe_encode_spec.rb

@@ -100,6 +100,64 @@
         expect(result.length).to be < text.length
       end
     end
+
+    describe "invalid UTF-8 handling with chunking" do
+      let(:invalid_binary_text) { ("word \xFF\xFE " * 10_000).b }
+      let(:normalized_text) do
+        invalid_binary_text.dup.force_encoding(Encoding::UTF_8).scrub
+      end
+
+      it "normalizes tokenize and encode inputs before chunking" do
+        expect(invalid_binary_text.size).to be > chunk_size
+        expect(tokenizer_class.tokenize(invalid_binary_text)).to eq(
+          tokenizer_class.tokenize(normalized_text)
+        )
+        expect(tokenizer_class.encode(invalid_binary_text)).to eq(
+          tokenizer_class.encode(normalized_text)
+        )
+      end
+
+      it "normalizes truncate and below_limit? inputs before checks" do
+        expect(invalid_binary_text.size).to be > chunk_size
+
+        token_count = tokenizer_class.size(normalized_text)
+        limit = [token_count, 1].max
+
+        expect(
+          tokenizer_class.truncate(invalid_binary_text, limit, strict: true)
+        ).to eq(tokenizer_class.truncate(normalized_text, limit, strict: true))
+
+        [1, [token_count - 1, 1].max, token_count, token_count + 1].uniq
+          .each do |current_limit|
+          expect(
+            tokenizer_class.below_limit?(
+              invalid_binary_text,
+              current_limit,
+              strict: true
+            )
+          ).to eq(
+            tokenizer_class.below_limit?(
+              normalized_text,
+              current_limit,
+              strict: true
+            )
+          )
+          expect(
+            tokenizer_class.below_limit?(
+              invalid_binary_text,
+              current_limit,
+              strict: false
+            )
+          ).to eq(
+            tokenizer_class.below_limit?(
+              normalized_text,
+              current_limit,
+              strict: false
+            )
+          )
+        end
+      end
+    end
   end
 
   describe DiscourseAi::Tokenizer::OpenAiTokenizer do