diff --git a/tests/test_misc.py b/tests/test_misc.py index 0832c8ee..e239a186 100644 --- a/tests/test_misc.py +++ b/tests/test_misc.py @@ -19,6 +19,9 @@ def test_encoding_for_model(): assert enc.name == "o200k_base" enc = tiktoken.encoding_for_model("gpt-oss-120b") assert enc.name == "o200k_harmony" + # gpt-5.1 uses a dotted name and must not require a hyphenated prefix match + assert tiktoken.model.encoding_name_for_model("gpt-5.1") == "o200k_base" + assert tiktoken.model.encoding_name_for_model("gpt-5.1-2025-11-13") == "o200k_base" def test_optional_blobfile_dependency(): diff --git a/tiktoken/model.py b/tiktoken/model.py index 5c669af4..3ddaa63c 100644 --- a/tiktoken/model.py +++ b/tiktoken/model.py @@ -9,6 +9,9 @@ "o3-": "o200k_base", "o4-mini-": "o200k_base", # chat + # gpt-5.1 / gpt-5.2 style names use a dot, not a hyphen, so they do not + # match the "gpt-5-" prefix used for dated gpt-5-* snapshots. + "gpt-5.": "o200k_base", "gpt-5-": "o200k_base", "gpt-4.5-": "o200k_base", "gpt-4.1-": "o200k_base", @@ -33,6 +36,7 @@ "o4-mini": "o200k_base", # chat "gpt-5": "o200k_base", + "gpt-5.1": "o200k_base", "gpt-4.1": "o200k_base", "gpt-4o": "o200k_base", "gpt-4": "cl100k_base",