Skip to content

[DIAGNOSTIC - DO NOT MERGE] fix(load_tokenizer): default to use_fast=True #26

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
hallerite wants to merge 3 commits into
base: main
Choose a base branch
from
Open

[DIAGNOSTIC - DO NOT MERGE] fix(load_tokenizer): default to use_fast=True #26

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
f47ae67
fix(load_tokenizer): default to use_fast=True
hallerite May 13, 2026
e656282
test(load_tokenizer): align mock assertions with use_fast=True default
hallerite May 13, 2026
b2930eb
fix(load_tokenizer): apply use_fast=True on Kimi pinned-revision path…
hallerite May 13, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion renderers/base.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -663,8 +663,13 @@ def load_tokenizer(model_name_or_path: str):
model_name_or_path,
trust_remote_code=True,
revision=revision,
use_fast=True,
)
return AutoTokenizer.from_pretrained(model_name_or_path, trust_remote_code=False)
return AutoTokenizer.from_pretrained(
model_name_or_path,
trust_remote_code=False,
use_fast=True,
)


def _populate_registry():
Expand Down
10 changes: 6 additions & 4 deletions tests/test_load_tokenizer.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,11 +50,11 @@ def test_trusted_revisions_are_full_shas():

@patch("transformers.AutoTokenizer.from_pretrained")
def test_unlisted_model_loads_without_remote_code(mock_from_pretrained):
"""Default path: trust_remote_code=False, no revision pin."""
"""Default path: trust_remote_code=False, use_fast=True, no revision pin."""
load_tokenizer("Qwen/Qwen3-0.6B")
args, kwargs = mock_from_pretrained.call_args
assert args == ("Qwen/Qwen3-0.6B",)
assert kwargs == {"trust_remote_code": False}
assert kwargs == {"trust_remote_code": False, "use_fast": True}


@patch("transformers.AutoTokenizer.from_pretrained")
Expand All @@ -67,6 +67,7 @@ def test_kimi_loads_with_pinned_revision(mock_from_pretrained):
assert kwargs == {
"trust_remote_code": True,
"revision": TRUSTED_REVISIONS["moonshotai/Kimi-K2.5"],
"use_fast": True,
}


Expand All @@ -87,8 +88,9 @@ def test_unknown_path_falls_through_to_no_remote_code(mock_from_pretrained):
load_tokenizer(name)
args, kwargs = mock_from_pretrained.call_args
assert args == (name,)
assert kwargs == {"trust_remote_code": False}, (
f"{name}: unlisted path leaked trust_remote_code=True"
assert kwargs == {"trust_remote_code": False, "use_fast": True}, (
f"{name}: unlisted path leaked trust_remote_code=True "
f"or dropped use_fast=True"
)


Expand Down
Loading