From eeed581a4a5ed5d7e8180fbb7379376b90708378 Mon Sep 17 00:00:00 2001 From: Michael Feil <63565275+michaelfeil@users.noreply.github.com> Date: Thu, 19 Mar 2026 16:59:07 -0700 Subject: [PATCH 1/7] add fasttokens --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 7373e4a..ad890c6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ requires = ["maturin>=1.0,<2.0"] build-backend = "maturin" [project] -name = "fastokens" +name = "fastokens-b10" dynamic = ["version"] requires-python = ">=3.9" From df01fd771ad64a8938cc46044cb604b1a2f1b81e Mon Sep 17 00:00:00 2001 From: michaelfeil <63565275+michaelfeil@users.noreply.github.com> Date: Fri, 10 Jul 2026 20:08:36 -0700 Subject: [PATCH 2/7] Add async batch Python API --- Cargo.lock | 43 +++++++++ python/Cargo.toml | 2 + python/fastokens/_compat.py | 23 +++++ python/fastokens/_native.pyi | 8 ++ python/src/lib.rs | 156 +++++++++++++++++++++++++------- python/tests/test_async_stub.py | 106 ++++++++++++++++++++++ 6 files changed, 304 insertions(+), 34 deletions(-) create mode 100644 python/tests/test_async_stub.py diff --git a/Cargo.lock b/Cargo.lock index f41b036..89511b7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -498,8 +498,10 @@ version = "0.2.1" dependencies = [ "fastokens", "pyo3", + "pyo3-async-runtimes", "rayon", "serde_json", + "tokio", ] [[package]] @@ -533,6 +535,21 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "futures" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b147ee9d1f6d097cef9ce628cd2ee62288d963e16fb287bd9286455b241382d" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + [[package]] name = "futures-channel" version = "0.3.32" @@ -540,6 +557,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" dependencies = [ "futures-core", + "futures-sink", ] [[package]] @@ -548,6 +566,17 @@ version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" +[[package]] +name = "futures-executor" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + [[package]] name = "futures-io" version = "0.3.32" @@ -583,6 +612,7 @@ version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" dependencies = [ + "futures-channel", "futures-core", "futures-io", "futures-macro", @@ -1221,6 +1251,19 @@ dependencies = [ "unindent", ] +[[package]] +name = "pyo3-async-runtimes" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d73cc6b1b7d8b3cef02101d37390dbdfe7e450dfea14921cae80a9534ba59ef2" +dependencies = [ + "futures", + "once_cell", + "pin-project-lite", + "pyo3", + "tokio", +] + [[package]] name = "pyo3-build-config" version = "0.25.1" diff --git a/python/Cargo.toml b/python/Cargo.toml index 7b91f04..450c4d8 100644 --- a/python/Cargo.toml +++ b/python/Cargo.toml @@ -15,5 +15,7 @@ crate-type = ["cdylib"] [dependencies] fastokens = { path = ".." } pyo3 = {version = "0.25", features = ["extension-module", "abi3-py39"] } +pyo3-async-runtimes = { version = "0.25", features = ["tokio-runtime"] } rayon = "1" serde_json = "1" +tokio = { version = "1", features = ["rt", "rt-multi-thread"] } diff --git a/python/fastokens/_compat.py b/python/fastokens/_compat.py index 6d4dbd6..381a82f 100644 --- a/python/fastokens/_compat.py +++ b/python/fastokens/_compat.py @@ -242,6 +242,20 @@ def encode_batch_fast( ) return self._fast.encode_batch(inputs, add_special_tokens=add_special_tokens) + async def async_encode_batch( + self, + inputs: list[str], + is_pretokenized: bool = False, + add_special_tokens: bool = True, + ) -> list[Encoding]: + if is_pretokenized or any(isinstance(inp, (list, tuple)) for inp in inputs): + raise NotImplementedError( + "pair/pre-tokenized batch encoding is not supported by fastokens" + ) + return await self._fast.async_encode_batch( + inputs, add_special_tokens=add_special_tokens + ) + # -- Post-processing ------------------------------------------------ def post_process( @@ -267,6 +281,15 @@ def decode_batch( ) -> list[str]: return self._fast.decode_batch(sequences, skip_special_tokens=skip_special_tokens) + async def async_decode_batch( + self, + sequences: list[list[int]], + skip_special_tokens: bool = True, + ) -> list[str]: + return await self._fast.async_decode_batch( + sequences, skip_special_tokens=skip_special_tokens + ) + # -- Vocabulary ----------------------------------------------------- def id_to_token(self, id: int) -> str | None: diff --git a/python/fastokens/_native.pyi b/python/fastokens/_native.pyi index 0292a16..8c6c9b2 100644 --- a/python/fastokens/_native.pyi +++ b/python/fastokens/_native.pyi @@ -129,6 +129,10 @@ class Tokenizer: self, inputs: list[str], add_special_tokens: bool = False ) -> list[Encoding]: ... + async def async_encode_batch( + self, inputs: list[str], add_special_tokens: bool = False + ) -> list[Encoding]: ... + def decode_tokens(self, tokens: list[str]) -> str: ... def decode(self, ids: list[int], skip_special_tokens: bool = False) -> str: ... @@ -136,5 +140,9 @@ class Tokenizer: self, sentences: list[list[int]], skip_special_tokens: bool = False ) -> list[str]: ... + async def async_decode_batch( + self, sentences: list[list[int]], skip_special_tokens: bool = False + ) -> list[str]: ... + def token_to_id(self, token: str) -> Optional[int]: ... def id_to_token(self, id: int) -> Optional[str]: ... diff --git a/python/src/lib.rs b/python/src/lib.rs index 45ff6ab..5539ef9 100644 --- a/python/src/lib.rs +++ b/python/src/lib.rs @@ -1,9 +1,21 @@ -use std::sync::RwLock; +use std::sync::{Arc, LazyLock, RwLock}; use pyo3::exceptions::{PyNotImplementedError, PyValueError}; use pyo3::prelude::*; use pyo3::types::{PyDict, PyList}; +use pyo3_async_runtimes::tokio::future_into_py; +use rayon::prelude::*; use serde_json::Value; +use tokio::runtime::Runtime; + +static TOKIO_RUNTIME: LazyLock> = LazyLock::new(|| { + Arc::new( + tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build() + .expect("failed to create global Tokio runtime"), + ) +}); // --------------------------------------------------------------------------- // PyEncoding @@ -315,6 +327,7 @@ impl PyEncoding { // TruncationParams / PaddingParams // --------------------------------------------------------------------------- +#[derive(Clone)] struct TruncationParams { max_length: usize, stride: usize, @@ -322,6 +335,7 @@ struct TruncationParams { direction: String, } +#[derive(Clone)] struct PaddingParams { direction: String, pad_id: u32, @@ -404,6 +418,42 @@ impl TokenizerState { } } + fn encode_batch_encodings( + &self, + inputs: &[String], + add_special_tokens: bool, + ) -> Result, String> { + let mut batch: Vec> = inputs + .par_iter() + .map(|s| { + self.inner + .encode_with_special_tokens(s.as_str(), add_special_tokens) + .map_err(|e| e.to_string()) + }) + .collect::, _>>()?; + + for ids in &mut batch { + self.do_truncate(ids); + } + + let pad_target: Option = self.pad.as_ref().map(|p| { + let max_len = batch.iter().map(|ids| ids.len()).max().unwrap_or(0); + let base = p.length.unwrap_or(max_len).max(max_len); + match p.pad_to_multiple_of { + Some(m) if m > 0 => (base + m - 1) / m * m, + _ => base, + } + }); + + Ok(batch + .into_iter() + .map(|ids| { + let target = pad_target.unwrap_or(ids.len()); + build_encoding(ids, self.pad.as_ref(), target) + }) + .collect()) + } + /// Parse `json`, update the Rust post-processor in place, and cache the JSON. fn update_post_processor_json(&mut self, json: &str) -> PyResult<()> { use fastokens::json_structs::PostProcessorConfig; @@ -424,7 +474,7 @@ impl TokenizerState { /// An LLM tokenizer backed by `tokenizer.json`. #[pyclass(name = "Tokenizer")] struct PyTokenizer { - state: RwLock, + state: Arc>, } impl PyTokenizer { @@ -449,12 +499,12 @@ impl PyTokenizer { .allow_threads(|| fastokens::Tokenizer::from_json(value).map_err(|e| e.to_string())) .map_err(PyValueError::new_err)?; Ok(Self { - state: RwLock::new(TokenizerState { + state: Arc::new(RwLock::new(TokenizerState { inner, trunc: None, pad: None, post_processor_json, - }), + })), }) } } @@ -655,41 +705,50 @@ impl PyTokenizer { add_special_tokens: bool, py: Python<'_>, ) -> PyResult>> { - use rayon::prelude::*; - let state = self.read(); - let mut batch: Vec> = inputs - .par_iter() - .map(|s| { - state - .inner - .encode_with_special_tokens(s.as_str(), add_special_tokens) - .map_err(|e| PyValueError::new_err(e.to_string())) - }) - .collect::>>()?; - - for ids in &mut batch { - state.do_truncate(ids); - } - - let pad_target: Option = state.pad.as_ref().map(|p| { - let max_len = batch.iter().map(|ids| ids.len()).max().unwrap_or(0); - let base = p.length.unwrap_or(max_len).max(max_len); - match p.pad_to_multiple_of { - Some(m) if m > 0 => (base + m - 1) / m * m, - _ => base, - } - }); - - batch + let encodings = state + .encode_batch_encodings(&inputs, add_special_tokens) + .map_err(PyValueError::new_err)?; + encodings .into_iter() - .map(|ids| { - let target = pad_target.unwrap_or(ids.len()); - Py::new(py, build_encoding(ids, state.pad.as_ref(), target)) - }) + .map(|encoding| Py::new(py, encoding)) .collect() } + /// Encode a batch of inputs in parallel and return a Python awaitable. + /// + /// Truncation is applied per-sequence; padding (if enabled) pads the + /// batch to a uniform length. + #[pyo3(signature = (inputs, add_special_tokens = false))] + fn async_encode_batch<'py>( + &self, + py: Python<'py>, + inputs: Vec, + add_special_tokens: bool, + ) -> PyResult> { + let state = Arc::clone(&self.state); + let rt = Arc::clone(&TOKIO_RUNTIME); + + future_into_py(py, async move { + let encodings = rt + .spawn_blocking(move || { + let state = state.read().expect("PyTokenizer state lock poisoned"); + state.encode_batch_encodings(&inputs, add_special_tokens) + }) + .await + .map_err(|e| PyValueError::new_err(e.to_string()))? + .map_err(PyValueError::new_err)?; + + Python::with_gil(|py| { + let encodings: PyResult>> = encodings + .into_iter() + .map(|encoding| Py::new(py, encoding)) + .collect(); + Ok(encodings?.into_pyobject(py)?.unbind().into_any()) + }) + }) + } + // ── Post-processing ─────────────────────────────────────────────── /// Apply the post-processor to an existing encoding. @@ -766,6 +825,35 @@ impl PyTokenizer { .map_err(|e| PyValueError::new_err(e.to_string())) } + /// Decode a batch of token ID sequences and return a Python awaitable. + #[pyo3(signature = (sentences, skip_special_tokens = false))] + fn async_decode_batch<'py>( + &self, + py: Python<'py>, + sentences: Vec>, + skip_special_tokens: bool, + ) -> PyResult> { + let state = Arc::clone(&self.state); + let rt = Arc::clone(&TOKIO_RUNTIME); + + future_into_py(py, async move { + let decoded = rt + .spawn_blocking(move || { + let state = state.read().expect("PyTokenizer state lock poisoned"); + let refs: Vec<&[u32]> = sentences.iter().map(Vec::as_slice).collect(); + state + .inner + .decode_batch(&refs, skip_special_tokens) + .map_err(|e| e.to_string()) + }) + .await + .map_err(|e| PyValueError::new_err(e.to_string()))? + .map_err(PyValueError::new_err)?; + + Python::with_gil(|py| Ok(decoded.into_pyobject(py)?.unbind().into_any())) + }) + } + // ── Vocabulary ──────────────────────────────────────────────────── /// Look up the token ID for a string. diff --git a/python/tests/test_async_stub.py b/python/tests/test_async_stub.py new file mode 100644 index 0000000..0026ec4 --- /dev/null +++ b/python/tests/test_async_stub.py @@ -0,0 +1,106 @@ +import inspect +import unittest + +from fastokens._compat import _TokenizerShim +from fastokens._native import Tokenizer + + +TOKENIZER_JSON = """ +{ + "version": "1.0", + "added_tokens": [], + "normalizer": null, + "pre_tokenizer": null, + "post_processor": null, + "decoder": null, + "model": { + "type": "BPE", + "dropout": null, + "unk_token": null, + "continuing_subword_prefix": "", + "end_of_word_suffix": "", + "fuse_unk": false, + "byte_fallback": false, + "vocab": { + "h": 0, + "e": 1, + "l": 2, + "o": 3, + " ": 4, + "w": 5, + "r": 6, + "d": 7, + "he": 8, + "hel": 9, + "hell": 10, + "hello": 11, + "wo": 12, + "wor": 13, + "worl": 14, + "world": 15 + }, + "merges": [ + "h e", + "he l", + "hel l", + "hell o", + "w o", + "wo r", + "wor l", + "worl d" + ] + } +} +""" + +INPUTS = ["hello", "hello world", "world"] + + +def _ids(encodings): + return [encoding.ids for encoding in encodings] + + +class NativeAsyncStubTests(unittest.IsolatedAsyncioTestCase): + async def test_async_encode_batch_matches_sync(self) -> None: + tokenizer = Tokenizer.from_json_str(TOKENIZER_JSON) + + sync_encodings = tokenizer.encode_batch(INPUTS) + awaitable = tokenizer.async_encode_batch(INPUTS) + async_encodings = await awaitable + + self.assertTrue(inspect.isawaitable(awaitable)) + self.assertEqual(_ids(sync_encodings), _ids(async_encodings)) + self.assertEqual(sync_encodings[1].ids, [11, 4, 15]) + + async def test_async_decode_batch_matches_sync(self) -> None: + tokenizer = Tokenizer.from_json_str(TOKENIZER_JSON) + encoded = tokenizer.encode_batch(INPUTS) + batch_ids = [encoding.ids for encoding in encoded] + + sync_decoded = tokenizer.decode_batch(batch_ids) + awaitable = tokenizer.async_decode_batch(batch_ids) + async_decoded = await awaitable + + self.assertTrue(inspect.isawaitable(awaitable)) + self.assertEqual(sync_decoded, async_decoded) + self.assertEqual(async_decoded, INPUTS) + + +class CompatAsyncStubTests(unittest.IsolatedAsyncioTestCase): + async def test_async_shim_methods_are_usable(self) -> None: + tokenizer = _TokenizerShim.from_str(TOKENIZER_JSON) + + encode_awaitable = tokenizer.async_encode_batch(INPUTS) + async_encodings = await encode_awaitable + batch_ids = [encoding.ids for encoding in async_encodings] + decode_awaitable = tokenizer.async_decode_batch(batch_ids) + async_decoded = await decode_awaitable + + self.assertTrue(inspect.isawaitable(encode_awaitable)) + self.assertTrue(inspect.isawaitable(decode_awaitable)) + self.assertEqual(batch_ids, [[11], [11, 4, 15], [15]]) + self.assertEqual(async_decoded, INPUTS) + + +if __name__ == "__main__": + unittest.main() From 271af524fff7c0ed2c7d6be7794b41cc0c9028af Mon Sep 17 00:00:00 2001 From: michaelfeil <63565275+michaelfeil@users.noreply.github.com> Date: Fri, 10 Jul 2026 20:15:58 -0700 Subject: [PATCH 3/7] release gil during Python encode --- python/src/lib.rs | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/python/src/lib.rs b/python/src/lib.rs index 5539ef9..cebbb05 100644 --- a/python/src/lib.rs +++ b/python/src/lib.rs @@ -683,15 +683,20 @@ impl PyTokenizer { add_special_tokens: bool, py: Python<'_>, ) -> PyResult> { - let state = self.read(); - let mut ids = state - .inner - .encode_with_special_tokens(input, add_special_tokens) - .map_err(|e| PyValueError::new_err(e.to_string()))?; - state.do_truncate(&mut ids); - let target = state.single_pad_target(ids.len()); + let encoding = py + .allow_threads(|| { + let state = self.read(); + let mut ids = state + .inner + .encode_with_special_tokens(input, add_special_tokens) + .map_err(|e| e.to_string())?; + state.do_truncate(&mut ids); + let target = state.single_pad_target(ids.len()); + Ok(build_encoding(ids, state.pad.as_ref(), target)) + }) + .map_err(PyValueError::new_err)?; - Py::new(py, build_encoding(ids, state.pad.as_ref(), target)) + Py::new(py, encoding) } /// Encode a batch of inputs in parallel. @@ -705,9 +710,11 @@ impl PyTokenizer { add_special_tokens: bool, py: Python<'_>, ) -> PyResult>> { - let state = self.read(); - let encodings = state - .encode_batch_encodings(&inputs, add_special_tokens) + let encodings = py + .allow_threads(|| { + let state = self.read(); + state.encode_batch_encodings(&inputs, add_special_tokens) + }) .map_err(PyValueError::new_err)?; encodings .into_iter() From acb2469137d491cc45d1b5a3ff4c47acbfe04121 Mon Sep 17 00:00:00 2001 From: michaelfeil <63565275+michaelfeil@users.noreply.github.com> Date: Fri, 10 Jul 2026 21:23:05 -0700 Subject: [PATCH 4/7] fix encode gil release build inference --- python/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/src/lib.rs b/python/src/lib.rs index cebbb05..b60b81d 100644 --- a/python/src/lib.rs +++ b/python/src/lib.rs @@ -692,7 +692,7 @@ impl PyTokenizer { .map_err(|e| e.to_string())?; state.do_truncate(&mut ids); let target = state.single_pad_target(ids.len()); - Ok(build_encoding(ids, state.pad.as_ref(), target)) + Ok::(build_encoding(ids, state.pad.as_ref(), target)) }) .map_err(PyValueError::new_err)?; From 85f9eaccfb6de32bfffd5cab7758c4616772db4e Mon Sep 17 00:00:00 2001 From: michaelfeil <63565275+michaelfeil@users.noreply.github.com> Date: Fri, 10 Jul 2026 21:27:09 -0700 Subject: [PATCH 5/7] build free-threaded wheels with Python 3.14 --- .github/workflows/build-maturin.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-maturin.yml b/.github/workflows/build-maturin.yml index db63613..23c370a 100644 --- a/.github/workflows/build-maturin.yml +++ b/.github/workflows/build-maturin.yml @@ -75,7 +75,7 @@ jobs: if: ${{ matrix.platform.ft }} with: target: ${{ matrix.platform.target }} - args: --release --out dist -i python3.13t + args: --release --out dist -i python3.14t sccache: "false" manylinux: auto before-script-linux: | @@ -122,7 +122,7 @@ jobs: uses: PyO3/maturin-action@v1 with: target: ${{ matrix.platform.target }} - args: --release --out dist -i python3.13t + args: --release --out dist -i python3.14t sccache: "false" manylinux: musllinux_1_2 - name: Upload wheels @@ -175,7 +175,7 @@ jobs: uses: PyO3/maturin-action@v1 with: target: ${{ matrix.platform.target }} - args: --release --out dist -i python3.13t + args: --release --out dist -i python3.14t sccache: "false" - name: Upload wheels uses: actions/upload-artifact@v4 From e656d08cef605d021d1b4d8cc386bfad1ad0c961 Mon Sep 17 00:00:00 2001 From: michaelfeil <63565275+michaelfeil@users.noreply.github.com> Date: Fri, 10 Jul 2026 21:43:10 -0700 Subject: [PATCH 6/7] release: Bump version to 0.2.0 --- Cargo.lock | 4 ++-- Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 89511b7..621ad4f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -472,7 +472,7 @@ dependencies = [ [[package]] name = "fastokens" -version = "0.2.1" +version = "0.2.0" dependencies = [ "anyhow", "clap", @@ -494,7 +494,7 @@ dependencies = [ [[package]] name = "fastokens-python" -version = "0.2.1" +version = "0.2.0" dependencies = [ "fastokens", "pyo3", diff --git a/Cargo.toml b/Cargo.toml index 304aa30..74c214c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,7 +2,7 @@ members = [".", "python"] [workspace.package] -version = "0.2.1" +version = "0.2.0" edition = "2024" license = "Apache-2.0" repository = "https://github.com/Atero-ai/fastokens" From 8def2de15b5ecca7c2c12435da0d87bbaecc0950 Mon Sep 17 00:00:00 2001 From: michaelfeil <63565275+michaelfeil@users.noreply.github.com> Date: Fri, 10 Jul 2026 21:56:45 -0700 Subject: [PATCH 7/7] Reject unsupported special token options --- python/fastokens/_compat.py | 26 ++++++- .../tests/test_unsupported_encode_options.py | 67 +++++++++++++++++++ 2 files changed, 91 insertions(+), 2 deletions(-) create mode 100644 python/tests/test_unsupported_encode_options.py diff --git a/python/fastokens/_compat.py b/python/fastokens/_compat.py index 381a82f..89ff316 100644 --- a/python/fastokens/_compat.py +++ b/python/fastokens/_compat.py @@ -20,6 +20,24 @@ _Encoding = Encoding +_UNSUPPORTED_SPLIT_SPECIAL_TOKENS = ( + "split_special_tokens=True / encode_special_tokens=True is not supported by " + "this fastokens release" +) + + +def _reject_split_special_tokens(value: object) -> None: + if bool(value): + raise NotImplementedError(_UNSUPPORTED_SPLIT_SPECIAL_TOKENS) + + +def _reject_unsupported_encode_kwargs(kwargs: dict) -> None: + _reject_split_special_tokens(kwargs.pop("split_special_tokens", False)) + if kwargs: + names = ", ".join(sorted(kwargs)) + raise TypeError(f"unsupported fastokens encode option(s): {names}") + + # --------------------------------------------------------------------------- # _TokenizerShim # --------------------------------------------------------------------------- @@ -50,6 +68,7 @@ def __init__(self, src) -> None: f"got {type(src).__name__}" ) self._encode_special_tokens: bool = False + _reject_split_special_tokens(getattr(src, "encode_special_tokens", False)) # -- Pickle / copy -------------------------------------------------- @@ -72,7 +91,7 @@ def __setstate__(self, state) -> None: self._fast.enable_truncation(**trunc) if pad is not None: self._fast.enable_padding(**{k: v for k, v in pad.items() if v is not None}) - self._encode_special_tokens = enc_special + self.encode_special_tokens = enc_special def __deepcopy__(self, memo): new = object.__new__(_TokenizerShim) @@ -140,7 +159,8 @@ def encode_special_tokens(self) -> bool: @encode_special_tokens.setter def encode_special_tokens(self, value: bool) -> None: - self._encode_special_tokens = value + _reject_split_special_tokens(value) + self._encode_special_tokens = False # -- Truncation / Padding ------------------------------------------- @@ -211,6 +231,7 @@ def encode( add_special_tokens: bool = True, **kwargs, ) -> Encoding: + _reject_unsupported_encode_kwargs(kwargs) if pair is not None: raise NotImplementedError("pair encoding is not supported by fastokens") if is_pretokenized: @@ -224,6 +245,7 @@ def encode_batch( add_special_tokens: bool = True, **kwargs, ) -> list[Encoding]: + _reject_unsupported_encode_kwargs(kwargs) if is_pretokenized or any(isinstance(inp, (list, tuple)) for inp in inputs): raise NotImplementedError( "pair/pre-tokenized batch encoding is not supported by fastokens" diff --git a/python/tests/test_unsupported_encode_options.py b/python/tests/test_unsupported_encode_options.py new file mode 100644 index 0000000..8d79ba7 --- /dev/null +++ b/python/tests/test_unsupported_encode_options.py @@ -0,0 +1,67 @@ +import pickle + +import pytest + +from fastokens._compat import _TokenizerShim +from test_async_stub import TOKENIZER_JSON + + +def test_encode_rejects_split_special_tokens_true() -> None: + tokenizer = _TokenizerShim.from_str(TOKENIZER_JSON) + + with pytest.raises(NotImplementedError, match="split_special_tokens=True"): + tokenizer.encode("hello", split_special_tokens=True) + + +def test_encode_batch_rejects_split_special_tokens_true() -> None: + tokenizer = _TokenizerShim.from_str(TOKENIZER_JSON) + + with pytest.raises(NotImplementedError, match="split_special_tokens=True"): + tokenizer.encode_batch(["hello"], split_special_tokens=True) + + +def test_encode_accepts_split_special_tokens_false_as_noop() -> None: + tokenizer = _TokenizerShim.from_str(TOKENIZER_JSON) + + plain = tokenizer.encode("hello") + explicit_false = tokenizer.encode("hello", split_special_tokens=False) + + assert explicit_false.ids == plain.ids + + +def test_encode_rejects_unknown_kwargs() -> None: + tokenizer = _TokenizerShim.from_str(TOKENIZER_JSON) + + with pytest.raises(TypeError, match="return_offsets_mapping"): + tokenizer.encode("hello", return_offsets_mapping=True) + + +def test_encode_special_tokens_true_is_rejected() -> None: + tokenizer = _TokenizerShim.from_str(TOKENIZER_JSON) + + with pytest.raises(NotImplementedError, match="encode_special_tokens=True"): + tokenizer.encode_special_tokens = True + + +def test_encode_special_tokens_false_is_noop() -> None: + tokenizer = _TokenizerShim.from_str(TOKENIZER_JSON) + + tokenizer.encode_special_tokens = False + + assert tokenizer.encode_special_tokens is False + + +def test_pickle_rejects_stored_encode_special_tokens_true() -> None: + tokenizer = _TokenizerShim.from_str(TOKENIZER_JSON) + state = ( + tokenizer.to_str(), + tokenizer.truncation, + tokenizer.padding, + True, + ) + + payload = pickle.dumps(state) + restored = _TokenizerShim.from_str(TOKENIZER_JSON) + + with pytest.raises(NotImplementedError, match="encode_special_tokens=True"): + restored.__setstate__(pickle.loads(payload))