FEATURE: Add DeepSeek tokenizer

CHANGELOG.md

@@ -1,3 +1,7 @@
+## [1.0.0] - 2026-08-03
+
+- Add a tokenizer for DeepSeek-V4-Flash-0731
+
 ## [0.4.2] - 2026-02-27
 
 - Normalize `ASCII-8BIT`/non-UTF-8 string inputs before tokenization to prevent `EncodingError` in `truncate`, `encode`, and `below_limit?`

README.md

@@ -6,7 +6,7 @@ A Ruby gem providing unified access to various AI model tokenizers, including bo
 
 - **Unified Interface**: Consistent API across all tokenizers
 - **Multiple Model Support**: Supports tokenizers for various AI models
-- **LLM Tokenizers**: Anthropic, OpenAI, Gemini, Llama3, Qwen, Mistral
+- **LLM Tokenizers**: Anthropic, DeepSeek, OpenAI, Gemini, Llama3, Qwen, Mistral
 - **Embedding Tokenizers**: BERT, AllMpnetBaseV2, BgeLargeEn, BgeM3, MultilingualE5Large
 - **Common Operations**: tokenize, encode, decode, size calculation, truncation
 - **Unicode Support**: Proper handling of emoji and multibyte characters
@@ -68,6 +68,7 @@ DiscourseAi::Tokenizer::OpenAiTokenizer.below_limit?("Short text", 10)
 #### LLM Tokenizers
 
 - `DiscourseAi::Tokenizer::AnthropicTokenizer` - Claude models
+- `DiscourseAi::Tokenizer::DeepSeekTokenizer` - DeepSeek-V4-Flash-0731
 - `DiscourseAi::Tokenizer::OpenAiTokenizer` - GPT models
 - `DiscourseAi::Tokenizer::GeminiTokenizer` - Google Gemini
 - `DiscourseAi::Tokenizer::Llama3Tokenizer` - Meta Llama 3

lib/discourse_ai/tokenizer/basic_tokenizer.rb

@@ -8,6 +8,7 @@ class << self
         def available_llm_tokenizers
           [
             DiscourseAi::Tokenizer::AnthropicTokenizer,
+            DiscourseAi::Tokenizer::DeepSeekTokenizer,
             DiscourseAi::Tokenizer::GeminiTokenizer,
             DiscourseAi::Tokenizer::Llama3Tokenizer,
             DiscourseAi::Tokenizer::MistralTokenizer,

lib/discourse_ai/tokenizer/deep_seek_tokenizer.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module DiscourseAi
+  module Tokenizer
+    # Tokenizer for deepseek-ai/DeepSeek-V4-Flash-0731.
+    class DeepSeekTokenizer < BasicTokenizer
+      def self.tokenizer
+        @tokenizer ||=
+          ::Tokenizers.from_file(
+            DiscourseAi::Tokenizers.vendor_path("deepseek-v4-flash-0731.json")
+          )
+      end
+    end
+  end
+end

lib/discourse_ai/tokenizers.rb

@@ -17,6 +17,7 @@
 require_relative "tokenizer/gemini_tokenizer"
 require_relative "tokenizer/qwen_tokenizer"
 require_relative "tokenizer/mistral_tokenizer"
+require_relative "tokenizer/deep_seek_tokenizer"
 
 module DiscourseAi
   module Tokenizers

lib/discourse_ai/tokenizers/version.rb

@@ -2,6 +2,6 @@
 
 module DiscourseAi
   module Tokenizers
-    VERSION = "0.4.2"
+    VERSION = "1.0.0"
   end
 end

spec/discourse_ai/tokenizers/basic_tokenizer_spec.rb

@@ -13,6 +13,7 @@
     it "includes expected tokenizer classes" do
       tokenizers = described_class.available_llm_tokenizers
       expect(tokenizers).to include(DiscourseAi::Tokenizer::AnthropicTokenizer)
+      expect(tokenizers).to include(DiscourseAi::Tokenizer::DeepSeekTokenizer)
       expect(tokenizers).to include(DiscourseAi::Tokenizer::GeminiTokenizer)
       expect(tokenizers).to include(DiscourseAi::Tokenizer::Llama3Tokenizer)
       expect(tokenizers).to include(DiscourseAi::Tokenizer::OpenAiTokenizer)

spec/discourse_ai/tokenizers/error_handling_spec.rb

@@ -297,6 +297,11 @@
                      DiscourseAi::Tokenizer::AnthropicTokenizer
   end
 
+  describe DiscourseAi::Tokenizer::DeepSeekTokenizer do
+    include_examples "tokenizer error handling",
+                     DiscourseAi::Tokenizer::DeepSeekTokenizer
+  end
+
   describe DiscourseAi::Tokenizer::OpenAiTokenizer do
     include_examples "tokenizer error handling",
                      DiscourseAi::Tokenizer::OpenAiTokenizer

spec/discourse_ai/tokenizers/integration_spec.rb

@@ -123,6 +123,11 @@
                      DiscourseAi::Tokenizer::AnthropicTokenizer
   end
 
+  describe DiscourseAi::Tokenizer::DeepSeekTokenizer do
+    include_examples "integration tests",
+                     DiscourseAi::Tokenizer::DeepSeekTokenizer
+  end
+
   describe DiscourseAi::Tokenizer::OpenAiTokenizer do
     include_examples "integration tests",
                      DiscourseAi::Tokenizer::OpenAiTokenizer
@@ -177,6 +182,7 @@
         DiscourseAi::Tokenizer::BasicTokenizer.available_llm_tokenizers
 
       expect(available).to include(DiscourseAi::Tokenizer::AnthropicTokenizer)
+      expect(available).to include(DiscourseAi::Tokenizer::DeepSeekTokenizer)
       expect(available).to include(DiscourseAi::Tokenizer::GeminiTokenizer)
       expect(available).to include(DiscourseAi::Tokenizer::Llama3Tokenizer)
       expect(available).to include(DiscourseAi::Tokenizer::OpenAiTokenizer)

spec/discourse_ai/tokenizers/method_consistency_spec.rb

@@ -92,6 +92,11 @@
                      DiscourseAi::Tokenizer::AnthropicTokenizer
   end
 
+  describe DiscourseAi::Tokenizer::DeepSeekTokenizer do
+    include_examples "method consistency",
+                     DiscourseAi::Tokenizer::DeepSeekTokenizer
+  end
+
   describe DiscourseAi::Tokenizer::OpenAiTokenizer do
     include_examples "method consistency",
                      DiscourseAi::Tokenizer::OpenAiTokenizer
@@ -220,6 +225,7 @@
         DiscourseAi::Tokenizer::BasicTokenizer.available_llm_tokenizers
 
       # Check for the MistralTokenizer name mismatch
+      expect(available).to include(DiscourseAi::Tokenizer::DeepSeekTokenizer)
       expect(available).to include(DiscourseAi::Tokenizer::MistralTokenizer)
     end