Add DisallowedSpecialTokenError (ValueError subclass) for disallowed special tokens (#290)

tests/test_encoding.py

@@ -182,7 +182,7 @@ def test_special_token():
 
     text = "<|endoftext|> hello <|fim_prefix|>"
     assert eot not in enc.encode(text, disallowed_special=())
-    with pytest.raises(ValueError):
+    with pytest.raises(tiktoken.DisallowedSpecialTokenError):
         enc.encode(text)
     with pytest.raises(ValueError):
         enc.encode(text, disallowed_special="all")

tiktoken/__init__.py

@@ -1,4 +1,5 @@
 # This is the public API of tiktoken
+from .core import DisallowedSpecialTokenError as DisallowedSpecialTokenError
 from .core import Encoding as Encoding
 from .model import encoding_for_model as encoding_for_model
 from .model import encoding_name_for_model as encoding_name_for_model

tiktoken/core.py

@@ -13,6 +13,10 @@
     import numpy.typing as npt
 
 
+class DisallowedSpecialTokenError(ValueError):
+    """Raised when text contains a special token that was not allowed."""
+
+
 class Encoding:
     def __init__(
         self,
@@ -439,7 +443,7 @@ def _special_token_regex(tokens: frozenset[str]) -> re.Pattern[str]:
 
 
 def raise_disallowed_special_token(token: str) -> NoReturn:
-    raise ValueError(
+    raise DisallowedSpecialTokenError(
         f"Encountered text corresponding to disallowed special token {token!r}.\n"
         "If you want this text to be encoded as a special token, "
         f"pass it to `allowed_special`, e.g. `allowed_special={{{token!r}, ...}}`.\n"