Fix tokenizer test failure

mlx_lm/tokenizer_utils.py

@@ -452,12 +452,12 @@ def detokenizer(self):
         """
         Get a stateful streaming detokenizer.
         """
-        return self._detokenizer_class(self)
+        if not hasattr(self, "_detokenizer"):
+            self._detokenizer = self._detokenizer_class(self)
+        return self._detokenizer
 
     def __getattr__(self, attr):
-        if attr == "detokenizer":
-            return self._detokenizer
-        elif attr == "eos_token_ids":
+        if attr == "eos_token_ids":
             return self._eos_token_ids
         elif attr.startswith("_"):
             return self.__getattribute__(attr)

tests/test_server.py

@@ -203,8 +203,10 @@ def test_handle_completions(self):
         self.assertIn("choices", response_body)
         first_text = response_body["choices"][0]["text"]
         self.assertEqual(
-            first_text,
-            json.loads(requests.post(url, json=post_data).text)["choices"][0]["text"],
+            first_text.strip(),
+            json.loads(requests.post(url, json=post_data).text)["choices"][0][
+                "text"
+            ].strip(),
         )
 
     def test_handle_chat_completions(self):

tests/test_tokenizers.py

@@ -65,6 +65,8 @@ def test_tokenizers(self):
                 tokenizer = load_tokenizer(tokenizer_repo)
                 tokenizer.decode([0, 1, 2])
                 self.assertTrue(isinstance(tokenizer.detokenizer, expected_detokenizer))
+                if expected_detokenizer is BPEStreamingDetokenizer:
+                    tokenizer.detokenizer.clean_spaces = False
                 self.check_tokenizer(tokenizer)
 
         # Try one with a naive detokenizer