[Fix] Fix clean_up_tokenization_spaces in tokenizer (#1510)

2024-09-24 21:37:33 -07:00
parent 067d8e16fc
commit fb2d0680e0
4 changed files with 12 additions and 9 deletions
--- a/python/sglang/srt/hf_transformers_utils.py
+++ b/python/sglang/srt/hf_transformers_utils.py
@@ -129,6 +129,7 @@ def get_tokenizer(
            *args,
            trust_remote_code=trust_remote_code,
            tokenizer_revision=tokenizer_revision,
+            clean_up_tokenization_spaces=False,
            **kwargs,
        )
    except TypeError as e: