From fb2d0680e0479acd7ea69737cbb09eec21755af9 Mon Sep 17 00:00:00 2001
From: Lianmin Zheng <lianminzheng@gmail.com>
Date: Tue, 24 Sep 2024 21:37:33 -0700
Subject: [PATCH] [Fix] Fix clean_up_tokenization_spaces in tokenizer (#1510)

---
 python/sglang/srt/hf_transformers_utils.py | 1 +
 python/sglang/test/runners.py              | 9 +++------
 scripts/playground/reference_hf.py         | 6 ++++--
 test/srt/models/test_generation_models.py  | 5 ++++-
 4 files changed, 12 insertions(+), 9 deletions(-)

diff --git a/python/sglang/srt/hf_transformers_utils.py b/python/sglang/srt/hf_transformers_utils.py
index f6c414ec3..eb3025421 100644
--- a/python/sglang/srt/hf_transformers_utils.py
+++ b/python/sglang/srt/hf_transformers_utils.py
@@ -129,6 +129,7 @@ def get_tokenizer(
             *args,
             trust_remote_code=trust_remote_code,
             tokenizer_revision=tokenizer_revision,
+            clean_up_tokenization_spaces=False,
             **kwargs,
         )
     except TypeError as e:
diff --git a/python/sglang/test/runners.py b/python/sglang/test/runners.py
index 886015862..60790a31e 100644
--- a/python/sglang/test/runners.py
+++ b/python/sglang/test/runners.py
@@ -21,8 +21,9 @@ from typing import List, Union
 
 import torch
 import torch.nn.functional as F
-from transformers import AutoModelForCausalLM, AutoTokenizer
+from transformers import AutoModelForCausalLM
 
+from sglang.srt.hf_transformers_utils import get_tokenizer
 from sglang.srt.server import Runtime
 from sglang.test.test_utils import DEFAULT_PORT_FOR_SRT_TEST_RUNNER
 
@@ -92,11 +93,7 @@ class HFRunner:
         self.model_proc.start()
 
     def start_model_process(self, in_queue, out_queue, model_path, torch_dtype):
-        self.tokenizer = AutoTokenizer.from_pretrained(
-            model_path,
-            torch_dtype=torch_dtype,
-        )
-
+        self.tokenizer = get_tokenizer(model_path)
         if self.is_generation:
             self.base_model = AutoModelForCausalLM.from_pretrained(
                 model_path,
diff --git a/scripts/playground/reference_hf.py b/scripts/playground/reference_hf.py
index 56c06a174..9354e01d5 100644
--- a/scripts/playground/reference_hf.py
+++ b/scripts/playground/reference_hf.py
@@ -26,12 +26,14 @@ I'm going to the
 import argparse
 
 import torch
-from transformers import AutoModelForCausalLM, AutoTokenizer
+from transformers import AutoModelForCausalLM
+
+from sglang.srt.hf_transformers_utils import get_tokenizer
 
 
 @torch.inference_mode()
 def normal_text(args):
-    t = AutoTokenizer.from_pretrained(args.model_path, trust_remote_code=True)
+    t = get_tokenizer(args.model_path, trust_remote_code=True)
     m = AutoModelForCausalLM.from_pretrained(
         args.model_path,
         torch_dtype=torch.float16,
diff --git a/test/srt/models/test_generation_models.py b/test/srt/models/test_generation_models.py
index 15d7f2c20..67ef363d0 100644
--- a/test/srt/models/test_generation_models.py
+++ b/test/srt/models/test_generation_models.py
@@ -30,7 +30,7 @@ from typing import List
 import torch
 
 from sglang.test.runners import DEFAULT_PROMPTS, HFRunner, SRTRunner
-from sglang.test.test_utils import calculate_rouge_l
+from sglang.test.test_utils import calculate_rouge_l, is_in_ci
 
 
 @dataclasses.dataclass
@@ -132,6 +132,9 @@ class TestGenerationModels(unittest.TestCase):
                 )
 
     def test_others(self):
+        if is_in_ci():
+            return
+
         for model_case in ALL_OTHER_MODELS:
             if (
                 "ONLY_RUN" in os.environ