Codex/reject unsupported special options

.github/workflows/build-maturin.yml

@@ -75,7 +75,7 @@ jobs:
         if: ${{ matrix.platform.ft }}
         with:
           target: ${{ matrix.platform.target }}
-          args: --release --out dist -i python3.13t
+          args: --release --out dist -i python3.14t
           sccache: "false"
           manylinux: auto
           before-script-linux: |
@@ -122,7 +122,7 @@ jobs:
         uses: PyO3/maturin-action@v1
         with:
           target: ${{ matrix.platform.target }}
-          args: --release --out dist -i python3.13t
+          args: --release --out dist -i python3.14t
           sccache: "false"
           manylinux: musllinux_1_2
       - name: Upload wheels
@@ -175,7 +175,7 @@ jobs:
         uses: PyO3/maturin-action@v1
         with:
           target: ${{ matrix.platform.target }}
-          args: --release --out dist -i python3.13t
+          args: --release --out dist -i python3.14t
           sccache: "false"
       - name: Upload wheels
         uses: actions/upload-artifact@v4

Cargo.toml

@@ -2,7 +2,7 @@
 members = [".", "python"]
 
 [workspace.package]
-version = "0.2.1"
+version = "0.2.0"
 edition = "2024"
 license = "Apache-2.0"
 repository = "https://github.com/Atero-ai/fastokens"

pyproject.toml

@@ -3,7 +3,7 @@ requires = ["maturin>=1.0,<2.0"]
 build-backend = "maturin"
 
 [project]
-name = "fastokens"
+name = "fastokens-b10"
 dynamic = ["version"]
 requires-python = ">=3.9"
 

python/Cargo.toml

@@ -15,5 +15,7 @@ crate-type = ["cdylib"]
 [dependencies]
 fastokens = { path = ".." }
 pyo3 = {version = "0.25", features = ["extension-module", "abi3-py39"] }
+pyo3-async-runtimes = { version = "0.25", features = ["tokio-runtime"] }
 rayon = "1"
 serde_json = "1"
+tokio = { version = "1", features = ["rt", "rt-multi-thread"] }

python/fastokens/_compat.py

@@ -20,6 +20,24 @@
 _Encoding = Encoding
 
 
+_UNSUPPORTED_SPLIT_SPECIAL_TOKENS = (
+    "split_special_tokens=True / encode_special_tokens=True is not supported by "
+    "this fastokens release"
+)
+
+
+def _reject_split_special_tokens(value: object) -> None:
+    if bool(value):
+        raise NotImplementedError(_UNSUPPORTED_SPLIT_SPECIAL_TOKENS)
+
+
+def _reject_unsupported_encode_kwargs(kwargs: dict) -> None:
+    _reject_split_special_tokens(kwargs.pop("split_special_tokens", False))
+    if kwargs:
+        names = ", ".join(sorted(kwargs))
+        raise TypeError(f"unsupported fastokens encode option(s): {names}")
+
+
 # ---------------------------------------------------------------------------
 # _TokenizerShim
 # ---------------------------------------------------------------------------
@@ -50,6 +68,7 @@ def __init__(self, src) -> None:
                 f"got {type(src).__name__}"
             )
         self._encode_special_tokens: bool = False
+        _reject_split_special_tokens(getattr(src, "encode_special_tokens", False))
 
     # -- Pickle / copy --------------------------------------------------
 
@@ -72,7 +91,7 @@ def __setstate__(self, state) -> None:
                 self._fast.enable_truncation(**trunc)
             if pad is not None:
                 self._fast.enable_padding(**{k: v for k, v in pad.items() if v is not None})
-            self._encode_special_tokens = enc_special
+            self.encode_special_tokens = enc_special
 
     def __deepcopy__(self, memo):
         new = object.__new__(_TokenizerShim)
@@ -140,7 +159,8 @@ def encode_special_tokens(self) -> bool:
 
     @encode_special_tokens.setter
     def encode_special_tokens(self, value: bool) -> None:
-        self._encode_special_tokens = value
+        _reject_split_special_tokens(value)
+        self._encode_special_tokens = False
 
     # -- Truncation / Padding -------------------------------------------
 
@@ -211,6 +231,7 @@ def encode(
         add_special_tokens: bool = True,
         **kwargs,
     ) -> Encoding:
+        _reject_unsupported_encode_kwargs(kwargs)
         if pair is not None:
             raise NotImplementedError("pair encoding is not supported by fastokens")
         if is_pretokenized:
@@ -224,6 +245,7 @@ def encode_batch(
         add_special_tokens: bool = True,
         **kwargs,
     ) -> list[Encoding]:
+        _reject_unsupported_encode_kwargs(kwargs)
         if is_pretokenized or any(isinstance(inp, (list, tuple)) for inp in inputs):
             raise NotImplementedError(
                 "pair/pre-tokenized batch encoding is not supported by fastokens"
@@ -242,6 +264,20 @@ def encode_batch_fast(
             )
         return self._fast.encode_batch(inputs, add_special_tokens=add_special_tokens)
 
+    async def async_encode_batch(
+        self,
+        inputs: list[str],
+        is_pretokenized: bool = False,
+        add_special_tokens: bool = True,
+    ) -> list[Encoding]:
+        if is_pretokenized or any(isinstance(inp, (list, tuple)) for inp in inputs):
+            raise NotImplementedError(
+                "pair/pre-tokenized batch encoding is not supported by fastokens"
+            )
+        return await self._fast.async_encode_batch(
+            inputs, add_special_tokens=add_special_tokens
+        )
+
     # -- Post-processing ------------------------------------------------
 
     def post_process(
@@ -267,6 +303,15 @@ def decode_batch(
     ) -> list[str]:
         return self._fast.decode_batch(sequences, skip_special_tokens=skip_special_tokens)
 
+    async def async_decode_batch(
+        self,
+        sequences: list[list[int]],
+        skip_special_tokens: bool = True,
+    ) -> list[str]:
+        return await self._fast.async_decode_batch(
+            sequences, skip_special_tokens=skip_special_tokens
+        )
+
     # -- Vocabulary -----------------------------------------------------
 
     def id_to_token(self, id: int) -> str | None:

python/fastokens/_native.pyi

@@ -129,12 +129,20 @@ class Tokenizer:
         self, inputs: list[str], add_special_tokens: bool = False
     ) -> list[Encoding]: ...
 
+    async def async_encode_batch(
+        self, inputs: list[str], add_special_tokens: bool = False
+    ) -> list[Encoding]: ...
+
     def decode_tokens(self, tokens: list[str]) -> str: ...
     def decode(self, ids: list[int], skip_special_tokens: bool = False) -> str: ...
 
     def decode_batch(
         self, sentences: list[list[int]], skip_special_tokens: bool = False
     ) -> list[str]: ...
 
+    async def async_decode_batch(
+        self, sentences: list[list[int]], skip_special_tokens: bool = False
+    ) -> list[str]: ...
+
     def token_to_id(self, token: str) -> Optional[int]: ...
     def id_to_token(self, id: int) -> Optional[str]: ...