diff --git a/xtuner/dataset/utils.py b/xtuner/dataset/utils.py index 84336ddb2..c6322152a 100644 --- a/xtuner/dataset/utils.py +++ b/xtuner/dataset/utils.py @@ -14,7 +14,7 @@ def get_bos_eos_token_ids(tokenizer): if tokenizer.__class__.__name__ in [ - 'QWenTokenizer', 'QWen2Tokenizer', 'Qwen2TokenizerFast' + 'QWenTokenizer', 'Qwen2Tokenizer', 'Qwen2TokenizerFast' ]: bos_token_id = [] eos_token_id = tokenizer.eos_token_id