[Auto Sync] Update xgrammar_backend.py (20250913) (#10395)

Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
This commit is contained in:
Lianmin Zheng
2025-09-12 17:46:56 -07:00
committed by GitHub
parent 7c5a0a1b77
commit cdddab056c

View File

@@ -173,6 +173,10 @@ class XGrammarGrammarBackend(BaseGrammarBackend):
if hasattr(tokenizer, "init_xgrammar"):
# For special tokenizer
tokenizer_info, override_stop_tokens = tokenizer.init_xgrammar()
if tokenizer_info is None:
# Not supported tokenizer
return
else:
# Create TokenizerInfo with model's EOS tokens as the authoritative stop tokens
# This ensures consistency between what the model considers EOS and what XGrammar uses