EngineX-Hygon/sglang
5
0
Fork 0
Code Issues Pull Requests Actions 7 Projects Releases Wiki Activity

Update grok.py and tiktoken tokenizer (#9532)

Browse Source
This commit is contained in:
Lianmin Zheng
2025-08-23 05:40:18 -07:00
committed by GitHub
parent 83871aa12d
commit 86d10d220f
10 changed files with 732 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
python/sglang/srt/constrained/xgrammar_backend.py
View File

@@ -162,12 +162,16 @@ class XGrammarGrammarBackend(BaseGrammarBackend):
):
super().__init__()
# Create TokenizerInfo with model's EOS tokens as the authoritative stop tokens
# This ensures consistency between what the model considers EOS and what XGrammar uses
tokenizer_info = TokenizerInfo.from_huggingface(
tokenizer, vocab_size=vocab_size, stop_token_ids=model_eos_token_ids
)
override_stop_tokens = None
if hasattr(tokenizer, "init_xgrammar"):
# For special tokenizer
tokenizer_info, override_stop_tokens = tokenizer.init_xgrammar()
else:
# Create TokenizerInfo with model's EOS tokens as the authoritative stop tokens
# This ensures consistency between what the model considers EOS and what XGrammar uses
tokenizer_info = TokenizerInfo.from_huggingface(
tokenizer, vocab_size=vocab_size, stop_token_ids=model_eos_token_ids
)
override_stop_tokens = None
self.grammar_compiler = GrammarCompiler(tokenizer_info=tokenizer_info)
self.vocab_size = vocab_size