From 02315b1495df39a779b034c55001ea3b3b7c3c73 Mon Sep 17 00:00:00 2001
From: Yash Raj Pandey <yashpn62@gmail.com>
Date: Mon, 13 Jul 2026 10:11:56 -0400
Subject: [PATCH] Return ValueError instead of panicking on duplicate
 mergeable_ranks

Constructing an Encoding with duplicate ranks in mergeable_ranks made
CoreBPE::new_internal hit a raw assert!, which surfaces to Python as an
uncatchable pyo3_runtime.PanicException instead of a clean ValueError.

new_internal already returns Result and reports regex errors cleanly via
Regex::new(pattern)?; py.rs maps that Err to PyValueError. Return an Err
on the length mismatch so it flows through the same path.

Refs #87.

Signed-off-by: Yash Raj Pandey <yashpn62@gmail.com>
---
 src/lib.rs                    | 14 ++++++++------
 tests/test_duplicate_ranks.py | 13 +++++++++++++
 2 files changed, 21 insertions(+), 6 deletions(-)
 create mode 100644 tests/test_duplicate_ranks.py

diff --git a/src/lib.rs b/src/lib.rs
index ea54eac8..0315600e 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -633,12 +633,14 @@ impl CoreBPE {
         let decoder: HashMap<Rank, Vec<u8>> =
             encoder.iter().map(|(k, v)| (*v, k.clone())).collect();
 
-        assert!(
-            encoder.len() == decoder.len(),
-            "Encoder and decoder must be of equal length. Encoder length: {}, decoder length: {}.\nMaybe you had duplicate token indices in your encoder?",
-            encoder.len(),
-            decoder.len()
-        );
+        if encoder.len() != decoder.len() {
+            return Err(format!(
+                "Encoder and decoder must be of equal length. Encoder length: {}, decoder length: {}.\nMaybe you had duplicate token indices in your encoder?",
+                encoder.len(),
+                decoder.len()
+            )
+            .into());
+        }
 
         let special_tokens_decoder: HashMap<Rank, Vec<u8>> = special_tokens_encoder
             .iter()
diff --git a/tests/test_duplicate_ranks.py b/tests/test_duplicate_ranks.py
new file mode 100644
index 00000000..306e8171
--- /dev/null
+++ b/tests/test_duplicate_ranks.py
@@ -0,0 +1,13 @@
+import pytest
+
+import tiktoken
+
+
+def test_duplicate_mergeable_ranks_raise_value_error():
+    with pytest.raises(ValueError):
+        tiktoken.Encoding(
+            name="duplicate_ranks",
+            pat_str=r".",
+            mergeable_ranks={b"a": 0, b"b": 0, b"c": 1},
+            special_tokens={},
+        )