From 07168433e6ba99b933a36811119b40f8fe7b7fdb Mon Sep 17 00:00:00 2001 From: Sam Saffron Date: Wed, 10 Dec 2025 15:03:39 +1100 Subject: [PATCH 1/4] Generally gems should not check in gemfile lock, dependencies should be handled via gemspec --- .gitignore | 1 + Gemfile.lock | 150 --------------------------------------------------- 2 files changed, 1 insertion(+), 150 deletions(-) delete mode 100644 Gemfile.lock diff --git a/.gitignore b/.gitignore index 0bd0cd9..bcd02ee 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,4 @@ *.gem .rspec_status +Gemfile.lock diff --git a/Gemfile.lock b/Gemfile.lock deleted file mode 100644 index 7f00acb..0000000 --- a/Gemfile.lock +++ /dev/null @@ -1,150 +0,0 @@ -PATH - remote: . - specs: - discourse_ai-tokenizers (0.3.1) - activesupport (>= 6.0) - tiktoken_ruby (~> 0.0.11.1) - tokenizers (~> 0.5.4) - -GEM - remote: https://rubygems.org/ - specs: - activesupport (8.0.2) - base64 - benchmark (>= 0.3) - bigdecimal - concurrent-ruby (~> 1.0, >= 1.3.1) - connection_pool (>= 2.2.5) - drb - i18n (>= 1.6, < 2) - logger (>= 1.4.2) - minitest (>= 5.1) - securerandom (>= 0.3) - tzinfo (~> 2.0, >= 2.0.5) - uri (>= 0.13.1) - ast (2.4.3) - base64 (0.3.0) - benchmark (0.4.1) - bigdecimal (3.2.2) - concurrent-ruby (1.3.5) - connection_pool (2.5.3) - date (3.4.1) - diff-lcs (1.6.2) - drb (2.2.3) - erb (5.0.1) - i18n (1.14.7) - concurrent-ruby (~> 1.0) - io-console (0.8.0) - irb (1.15.2) - pp (>= 0.6.0) - rdoc (>= 4.0.0) - reline (>= 0.4.2) - json (2.12.2) - language_server-protocol (3.17.0.5) - lint_roller (1.1.0) - logger (1.7.0) - minitest (5.25.5) - parallel (1.27.0) - parser (3.3.8.0) - ast (~> 2.4.1) - racc - pp (0.6.2) - prettyprint - prettier_print (1.2.1) - prettyprint (0.2.0) - prism (1.4.0) - psych (5.2.6) - date - stringio - racc (1.8.1) - rack (3.1.18) - rainbow (3.1.1) - rake (13.3.0) - rdoc (6.14.1) - erb - psych (>= 4.0.0) - regexp_parser (2.10.0) - reline (0.6.1) - io-console (~> 0.5) - rspec (3.13.1) - rspec-core (~> 3.13.0) - rspec-expectations (~> 3.13.0) - rspec-mocks (~> 3.13.0) - rspec-core (3.13.4) - rspec-support (~> 3.13.0) - rspec-expectations (3.13.5) - diff-lcs (>= 1.2.0, < 2.0) - rspec-support (~> 3.13.0) - rspec-mocks (3.13.5) - diff-lcs (>= 1.2.0, < 2.0) - rspec-support (~> 3.13.0) - rspec-support (3.13.4) - rubocop (1.76.1) - json (~> 2.3) - language_server-protocol (~> 3.17.0.2) - lint_roller (~> 1.1.0) - parallel (~> 1.10) - parser (>= 3.3.0.2) - rainbow (>= 2.2.2, < 4.0) - regexp_parser (>= 2.9.3, < 3.0) - rubocop-ast (>= 1.45.0, < 2.0) - ruby-progressbar (~> 1.7) - unicode-display_width (>= 2.4.0, < 4.0) - rubocop-ast (1.45.1) - parser (>= 3.3.7.2) - prism (~> 1.4) - rubocop-capybara (2.22.1) - lint_roller (~> 1.1) - rubocop (~> 1.72, >= 1.72.1) - rubocop-discourse (3.8.1) - activesupport (>= 6.1) - rubocop (>= 1.59.0) - rubocop-capybara (>= 2.0.0) - rubocop-factory_bot (>= 2.0.0) - rubocop-rails (>= 2.25.0) - rubocop-rspec (>= 3.0.1) - rubocop-rspec_rails (>= 2.30.0) - rubocop-factory_bot (2.27.1) - lint_roller (~> 1.1) - rubocop (~> 1.72, >= 1.72.1) - rubocop-rails (2.32.0) - activesupport (>= 4.2.0) - lint_roller (~> 1.1) - rack (>= 1.1) - rubocop (>= 1.75.0, < 2.0) - rubocop-ast (>= 1.44.0, < 2.0) - rubocop-rspec (3.6.0) - lint_roller (~> 1.1) - rubocop (~> 1.72, >= 1.72.1) - rubocop-rspec_rails (2.31.0) - lint_roller (~> 1.1) - rubocop (~> 1.72, >= 1.72.1) - rubocop-rspec (~> 3.5) - ruby-progressbar (1.13.0) - securerandom (0.4.1) - stringio (3.1.7) - syntax_tree (6.2.0) - prettier_print (>= 1.2.0) - tiktoken_ruby (0.0.11.1-x86_64-linux) - tokenizers (0.5.4-x86_64-linux) - tzinfo (2.0.6) - concurrent-ruby (~> 1.0) - unicode-display_width (3.1.4) - unicode-emoji (~> 4.0, >= 4.0.4) - unicode-emoji (4.0.4) - uri (1.0.3) - -PLATFORMS - x86_64-linux - -DEPENDENCIES - discourse_ai-tokenizers! - irb - rake (~> 13.0) - rspec (~> 3.0) - rubocop (~> 1.21) - rubocop-discourse (= 3.8.1) - syntax_tree (~> 6.2.0) - -BUNDLED WITH - 2.6.7 From 62b780618ce6e8acb04f73581f9fbe6667a9d23c Mon Sep 17 00:00:00 2001 From: Sam Saffron Date: Wed, 10 Dec 2025 15:15:38 +1100 Subject: [PATCH 2/4] FIX: tokenizer not handling UTF-8 truncation gracefully --- CHANGELOG.md | 4 ++++ .../tokenizer/open_ai_tokenizer.rb | 22 ++++++++++++++++--- lib/discourse_ai/tokenizers/version.rb | 2 +- .../tokenizers/error_handling_spec.rb | 19 ++++++++++++++++ 4 files changed, 43 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a54c976..a5e0fa6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ ## [Unreleased] +## [0.3.2] - 2025-12-10 + +- Fix truncation logic in OpenAiTokenizer could lead to string parsing fails + ## [0.3.1] - 2025-07-07 - Refactor OpenAiO200kTokenizer class to OpenAiTokenizer as primary class name diff --git a/lib/discourse_ai/tokenizer/open_ai_tokenizer.rb b/lib/discourse_ai/tokenizer/open_ai_tokenizer.rb index b32cb9a..425a758 100644 --- a/lib/discourse_ai/tokenizer/open_ai_tokenizer.rb +++ b/lib/discourse_ai/tokenizer/open_ai_tokenizer.rb @@ -19,11 +19,27 @@ def encode(text) def decode(token_ids) tokenizer.decode(token_ids) - rescue Tiktoken::UnicodeError => e + rescue Tiktoken::UnicodeError # Handle invalid token IDs gracefully by returning empty string "" end + def safe_decode(token_ids) + tokenizer.decode(token_ids) + rescue Tiktoken::UnicodeError + # Token slice created incomplete UTF-8 sequence + # Decode individual tokens and join, skipping any that fail + token_ids + .map do |id| + begin + tokenizer.decode([id]) + rescue StandardError + "" + end + end + .join + end + def truncate(text, max_length, strict: false) return "" if max_length <= 0 @@ -33,12 +49,12 @@ def truncate(text, max_length, strict: false) # Take tokens up to max_length, decode, then ensure we don't exceed limit truncated_tokens = tokenize(text).take(max_length) - truncated_text = tokenizer.decode(truncated_tokens) + truncated_text = safe_decode(truncated_tokens) # If re-encoding exceeds the limit, we need to further truncate while tokenize(truncated_text).length > max_length truncated_tokens = truncated_tokens[0...-1] - truncated_text = tokenizer.decode(truncated_tokens) + truncated_text = safe_decode(truncated_tokens) break if truncated_tokens.empty? end diff --git a/lib/discourse_ai/tokenizers/version.rb b/lib/discourse_ai/tokenizers/version.rb index 6bafc51..ea53740 100644 --- a/lib/discourse_ai/tokenizers/version.rb +++ b/lib/discourse_ai/tokenizers/version.rb @@ -2,6 +2,6 @@ module DiscourseAi module Tokenizers - VERSION = "0.3.1" + VERSION = "0.3.2" end end diff --git a/spec/discourse_ai/tokenizers/error_handling_spec.rb b/spec/discourse_ai/tokenizers/error_handling_spec.rb index 602cb3d..71755fa 100644 --- a/spec/discourse_ai/tokenizers/error_handling_spec.rb +++ b/spec/discourse_ai/tokenizers/error_handling_spec.rb @@ -84,6 +84,25 @@ expect(decoded).to include("世界") end end + + it "handles truncation at all token boundaries without raising" do + text = "日本語テスト 🎉 中文测试 العربية" + token_count = tokenizer_class.size(text) + + (1..token_count).each do |i| + expect { + tokenizer_class.truncate(text, i, strict: true) + }.not_to raise_error + end + end + + it "returns valid UTF-8 strings when truncating multi-byte characters" do + text = "日本語テスト 🎉 中文测试 العربية" + + result = tokenizer_class.truncate(text, 5, strict: true) + expect(result).to be_a(String) + expect(result.valid_encoding?).to be true + end end describe "edge case parameters" do From 8f47a60cee397325afc2ff6d6e4c987fbc5bc3ee Mon Sep 17 00:00:00 2001 From: Sam Saffron Date: Wed, 10 Dec 2025 15:24:23 +1100 Subject: [PATCH 3/4] a few more truncation specs just in case --- .../tokenizers/error_handling_spec.rb | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/spec/discourse_ai/tokenizers/error_handling_spec.rb b/spec/discourse_ai/tokenizers/error_handling_spec.rb index 71755fa..93805a8 100644 --- a/spec/discourse_ai/tokenizers/error_handling_spec.rb +++ b/spec/discourse_ai/tokenizers/error_handling_spec.rb @@ -174,6 +174,46 @@ describe DiscourseAi::Tokenizer::OpenAiTokenizer do include_examples "tokenizer error handling", DiscourseAi::Tokenizer::OpenAiTokenizer + + describe "truncation correctness" do + let(:tokenizer) { DiscourseAi::Tokenizer::OpenAiTokenizer } + + it "truncates simple ASCII text correctly" do + text = "Hello world this is a test" + result = tokenizer.truncate(text, 3, strict: true) + + expect(result).to eq("Hello world this") + expect(tokenizer.size(result)).to be <= 3 + end + + it "truncates multi-byte UTF-8 text correctly" do + text = "日本語テスト 🎉 中文测试 العربية" + + result = tokenizer.truncate(text, 4, strict: true) + expect(result).to eq("日本語テスト") + expect(tokenizer.size(result)).to eq(4) + + result = tokenizer.truncate(text, 7, strict: true) + expect(result).to eq("日本語テスト 🎉 中文") + expect(tokenizer.size(result)).to eq(7) + end + + it "never exceeds the requested token limit" do + text = "日本語テスト 🎉 中文测试 العربية" + + (1..tokenizer.size(text)).each do |limit| + result = tokenizer.truncate(text, limit, strict: true) + expect(tokenizer.size(result)).to be <= limit + end + end + + it "preserves text prefix when truncating" do + text = "Hello 世界 test" + result = tokenizer.truncate(text, 2, strict: true) + + expect(text).to start_with(result) + end + end end describe DiscourseAi::Tokenizer::AllMpnetBaseV2Tokenizer do From 9426e6b299dc96e86d561cce0586b5fd4de58a4c Mon Sep 17 00:00:00 2001 From: Sam Saffron Date: Wed, 10 Dec 2025 15:55:56 +1100 Subject: [PATCH 4/4] improve decoding logic --- .../tokenizer/open_ai_tokenizer.rb | 28 +++++++++++-------- .../tokenizers/error_handling_spec.rb | 15 +++++----- 2 files changed, 25 insertions(+), 18 deletions(-) diff --git a/lib/discourse_ai/tokenizer/open_ai_tokenizer.rb b/lib/discourse_ai/tokenizer/open_ai_tokenizer.rb index 425a758..f43789c 100644 --- a/lib/discourse_ai/tokenizer/open_ai_tokenizer.rb +++ b/lib/discourse_ai/tokenizer/open_ai_tokenizer.rb @@ -20,20 +20,26 @@ def encode(text) def decode(token_ids) tokenizer.decode(token_ids) rescue Tiktoken::UnicodeError - # Handle invalid token IDs gracefully by returning empty string - "" - end + token_ids = token_ids.dup - def safe_decode(token_ids) - tokenizer.decode(token_ids) - rescue Tiktoken::UnicodeError - # Token slice created incomplete UTF-8 sequence - # Decode individual tokens and join, skipping any that fail + # this easy case, we started with a valid sequnce but truncated it on an invalid boundary + # work backwards removing tokens until we can decode again + tries = 4 + while tries > 0 + begin + token_ids.pop + return tokenizer.decode(token_ids) + rescue Tiktoken::UnicodeError + tries -= 1 + end + end + + # at this point we may have a corrupted sequence so just decode what we can token_ids .map do |id| begin tokenizer.decode([id]) - rescue StandardError + rescue Tiktoken::UnicodeError "" end end @@ -49,12 +55,12 @@ def truncate(text, max_length, strict: false) # Take tokens up to max_length, decode, then ensure we don't exceed limit truncated_tokens = tokenize(text).take(max_length) - truncated_text = safe_decode(truncated_tokens) + truncated_text = decode(truncated_tokens) # If re-encoding exceeds the limit, we need to further truncate while tokenize(truncated_text).length > max_length truncated_tokens = truncated_tokens[0...-1] - truncated_text = safe_decode(truncated_tokens) + truncated_text = decode(truncated_tokens) break if truncated_tokens.empty? end diff --git a/spec/discourse_ai/tokenizers/error_handling_spec.rb b/spec/discourse_ai/tokenizers/error_handling_spec.rb index 93805a8..26eb82a 100644 --- a/spec/discourse_ai/tokenizers/error_handling_spec.rb +++ b/spec/discourse_ai/tokenizers/error_handling_spec.rb @@ -187,15 +187,16 @@ end it "truncates multi-byte UTF-8 text correctly" do - text = "日本語テスト 🎉 中文测试 العربية" + text = "a 🎉 a 🎉 a" + + result = tokenizer.truncate(text, 2, strict: true) + expect(result).to eq("a") - result = tokenizer.truncate(text, 4, strict: true) - expect(result).to eq("日本語テスト") - expect(tokenizer.size(result)).to eq(4) + result = tokenizer.truncate(text, 3, strict: true) + expect(result).to eq("a 🎉") - result = tokenizer.truncate(text, 7, strict: true) - expect(result).to eq("日本語テスト 🎉 中文") - expect(tokenizer.size(result)).to eq(7) + result = tokenizer.truncate(text, 5, strict: true) + expect(result).to eq("a 🎉 a") end it "never exceeds the requested token limit" do