From 20a1b24e912680dd948283b377b066586450d29f Mon Sep 17 00:00:00 2001 From: Mike German Date: Wed, 8 Jul 2026 11:22:31 -0400 Subject: [PATCH] fix: apply surrogate-pair fixup in encode_with_unstable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit encode_with_unstable raised UnicodeEncodeError when given text containing surrogate pairs (e.g. "👍"), while encode and encode_ordinary handled the same input correctly by catching UnicodeEncodeError and re-encoding through UTF-16 surrogatepass. Apply the same try/except fixup to encode_with_unstable so all three methods behave consistently for surrogate-pair inputs. Fixes #541 Signed-off-by: Mike German --- tests/test_encoding.py | 15 +++++++++++++++ tiktoken/core.py | 7 ++++++- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/tests/test_encoding.py b/tests/test_encoding.py index b77ca135..4e6bfa1c 100644 --- a/tests/test_encoding.py +++ b/tests/test_encoding.py @@ -110,6 +110,21 @@ def test_encode_surrogate_pairs(): assert enc.encode("\ud83d") == enc.encode("�") +def test_encode_with_unstable_surrogate_pairs(): + enc = tiktoken.get_encoding("cl100k_base") + + # encode_with_unstable must not raise UnicodeEncodeError for surrogate pairs + stable, completions = enc.encode_with_unstable("👍") + stable_via_surrogate, completions_via_surrogate = enc.encode_with_unstable("\ud83d\udc4d") + assert stable == stable_via_surrogate + assert completions == completions_via_surrogate + + # lone surrogate treated the same as the replacement character + stable_lone, _ = enc.encode_with_unstable("\ud83d") + stable_replacement, _ = enc.encode_with_unstable("�") + assert stable_lone == stable_replacement + + @pytest.mark.parametrize("make_enc", ENCODING_FACTORIES) def test_catastrophically_repetitive(make_enc: Callable[[], tiktoken.Encoding]): enc = make_enc() diff --git a/tiktoken/core.py b/tiktoken/core.py index 530f8f59..482e4bf7 100644 --- a/tiktoken/core.py +++ b/tiktoken/core.py @@ -240,7 +240,12 @@ def encode_with_unstable( if match := _special_token_regex(disallowed_special).search(text): raise_disallowed_special_token(match.group()) - return self._core_bpe.encode_with_unstable(text, allowed_special) + try: + return self._core_bpe.encode_with_unstable(text, allowed_special) + except UnicodeEncodeError: + # see comment in encode; same surrogate-pair fixup applied here + text = text.encode("utf-16", "surrogatepass").decode("utf-16", "replace") + return self._core_bpe.encode_with_unstable(text, allowed_special) def encode_single_token(self, text_or_bytes: str | bytes) -> int: """Encodes text corresponding to a single token to its token value.