From 3fdab91912fb271c20642e21c2055df0e23d514e Mon Sep 17 00:00:00 2001 From: Ke Bao Date: Sat, 27 Jul 2024 17:44:46 +0800 Subject: [PATCH] Fix TransformerTokenizer init for chatglm2 & 3 (#761) --- python/sglang/srt/constrained/fsm_cache.py | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/python/sglang/srt/constrained/fsm_cache.py b/python/sglang/srt/constrained/fsm_cache.py index cc0609a58..8c522c036 100644 --- a/python/sglang/srt/constrained/fsm_cache.py +++ b/python/sglang/srt/constrained/fsm_cache.py @@ -21,7 +21,27 @@ class FSMCache(BaseCache): tokenizer = AutoTokenizer.from_pretrained( tokenizer_path, **tokenizer_args_dict ) - self.outlines_tokenizer = TransformerTokenizer(tokenizer) + try: + self.outlines_tokenizer = TransformerTokenizer(tokenizer) + except AttributeError: + # FIXME: tmp fix for chatglm2 & chatglm3 (pad_token_id=0) + origin_pad_token_id = tokenizer.pad_token_id + + def fset(self, value): + self._value = value + + type(tokenizer).pad_token_id = property( + fget=type(tokenizer).pad_token_id.fget, fset=fset + ) + self.outlines_tokenizer = TransformerTokenizer(tokenizer) + self.outlines_tokenizer.tokenizer.pad_token_id = origin_pad_token_id + self.outlines_tokenizer.pad_token_id = origin_pad_token_id + self.outlines_tokenizer.pad_token = ( + self.outlines_tokenizer.tokenizer.pad_token + ) + self.outlines_tokenizer.vocabulary = ( + self.outlines_tokenizer.tokenizer.get_vocab() + ) else: self.outlines_tokenizer = TransformerTokenizer( tokenizer_path, **tokenizer_args_dict