Merge pull request #638 from exo-explore/deepseekv3fix

add exception for mlx-community/DeepSeek-R1-3bit and mlx-community/DeepSeek-V3-3bit in tokenizers test
exo-explore · Jan 26, 2025 · 903950f · 903950f
2 parents 9711d63 + a3766f5
commit 903950f
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/test/test_tokenizers.py b/test/test_tokenizers.py
@@ -37,6 +37,6 @@ def test_tokenizer(name, tokenizer, verbose=False):
 for m in models:
     # TODO: figure out why use_fast=False is giving inconsistent behaviour (no spaces decoding invididual tokens) for Mistral-Large-Instruct-2407-4bit
     # test_tokenizer(m, AutoProcessor.from_pretrained(m, use_fast=False), verbose)
-    if m not in ["mlx-community/DeepSeek-R1-4bit", "mlx-community/DeepSeek-V3-4bit"]:
+    if m not in ["mlx-community/DeepSeek-R1-4bit", "mlx-community/DeepSeek-R1-3bit", "mlx-community/DeepSeek-V3-4bit", "mlx-community/DeepSeek-V3-3bit"]:
       test_tokenizer(m, AutoProcessor.from_pretrained(m, use_fast=True, trust_remote_code=True), verbose)
     test_tokenizer(m, AutoTokenizer.from_pretrained(m, trust_remote_code=True), verbose)