From 8536ce212f542bde7ca500fa7892cc72d703f9e1 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Tue, 23 Jun 2026 16:25:03 +0000
Subject: [PATCH] chore(deps): Bump tokenizers from 0.22.2 to 0.23.1 in
 /Native/pcai_core

Bumps [tokenizers](https://github.com/huggingface/tokenizers) from 0.22.2 to 0.23.1.
- [Release notes](https://github.com/huggingface/tokenizers/releases)
- [Changelog](https://github.com/huggingface/tokenizers/blob/main/RELEASE.md)
- [Commits](https://github.com/huggingface/tokenizers/compare/v0.22.2...v0.23.1)

---
updated-dependencies:
- dependency-name: tokenizers
  dependency-version: 0.23.1
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 Native/pcai_core/Cargo.lock            | 43 ++++++++++++++++++++++++--
 Native/pcai_core/pcai_media/Cargo.toml |  2 +-
 2 files changed, 42 insertions(+), 3 deletions(-)

diff --git a/Native/pcai_core/Cargo.lock b/Native/pcai_core/Cargo.lock
index 41113b4..b22fa54 100644
--- a/Native/pcai_core/Cargo.lock
+++ b/Native/pcai_core/Cargo.lock
@@ -1126,6 +1126,12 @@ dependencies = [
  "libloading 0.9.0",
 ]
 
+[[package]]
+name = "daachorse"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6f55d7153ba3b507595872a3874803f07a8a81d1e888abed8e5db7da0597d6e2"
+
 [[package]]
 name = "darling"
 version = "0.11.0"
@@ -4251,7 +4257,7 @@ dependencies = [
  "serde_json",
  "tempfile",
  "thiserror 2.0.18",
- "tokenizers 0.22.2",
+ "tokenizers 0.23.1",
  "tokio",
  "tower-http",
  "tracing",
@@ -6233,7 +6239,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd"
 dependencies = [
  "fastrand",
- "getrandom 0.3.4",
+ "getrandom 0.4.2",
  "once_cell",
  "rustix",
  "windows-sys 0.61.2",
@@ -6467,6 +6473,39 @@ dependencies = [
  "esaxx-rs",
  "fancy-regex 0.14.0",
  "getrandom 0.3.4",
+ "itertools 0.14.0",
+ "log",
+ "macro_rules_attribute 0.2.2",
+ "monostate",
+ "onig",
+ "paste",
+ "rand 0.9.3",
+ "rayon",
+ "rayon-cond 0.4.0",
+ "regex",
+ "regex-syntax 0.8.8",
+ "serde",
+ "serde_json",
+ "spm_precompiled",
+ "thiserror 2.0.18",
+ "unicode-normalization-alignments",
+ "unicode-segmentation",
+ "unicode_categories",
+]
+
+[[package]]
+name = "tokenizers"
+version = "0.23.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "44e5bea67576e04b6ff8564c5d9e09c2ef0cf476502245f2f120e497769d3112"
+dependencies = [
+ "ahash",
+ "compact_str",
+ "daachorse",
+ "dary_heap",
+ "derive_builder 0.20.2",
+ "esaxx-rs",
+ "getrandom 0.3.4",
  "indicatif 0.18.3",
  "itertools 0.14.0",
  "log",
diff --git a/Native/pcai_core/pcai_media/Cargo.toml b/Native/pcai_core/pcai_media/Cargo.toml
index 05f792d..c6a2912 100644
--- a/Native/pcai_core/pcai_media/Cargo.toml
+++ b/Native/pcai_core/pcai_media/Cargo.toml
@@ -24,7 +24,7 @@ candle-core = { version = "0.9" }
 candle-nn = "0.10"
 candle-transformers = "0.9"
 safetensors = "0.7"
-tokenizers = "0.22"
+tokenizers = "0.23"
 image = "0.25"
 rand = "0.9"
 # Disable ureq (sync HTTP client) to eliminate the ring crate dependency.