EngineX-Hygon/sglang
5
0
Fork 0
Code Issues Pull Requests Actions 7 Projects Releases Wiki Activity

Fix missing additional_stop_token_ids (#1769)

Browse Source
This commit is contained in:
Lianmin Zheng
2024-10-23 12:18:59 -07:00
committed by GitHub
parent b7d0559496
commit 0d800090b4
3 changed files with 16 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
python/sglang/srt/hf_transformers_utils.py
View File

@@ -164,14 +164,7 @@ def get_tokenizer(
"slowdown. Consider using a fast tokenizer instead."
)
# Special handling for stop token <|eom_id|> generated by llama 3 tool use.
if "<|eom_id|>" in tokenizer.get_added_vocab():
tokenizer.additional_stop_token_ids = set(
[tokenizer.get_added_vocab()["<|eom_id|>"]]
)
else:
tokenizer.additional_stop_token_ids = None
handle_additional_stop_token_ids(tokenizer)
return tokenizer
@@ -190,4 +183,16 @@ def get_processor(
tokenizer_revision=tokenizer_revision,
**kwargs,
)
handle_additional_stop_token_ids(processor.tokenizer)
return processor
def handle_additional_stop_token_ids(tokenizer):
# Special handling for stop token <|eom_id|> generated by llama 3 tool use.
if "<|eom_id|>" in tokenizer.get_added_vocab():
tokenizer.additional_stop_token_ids = set(
[tokenizer.get_added_vocab()["<|eom_id|>"]]
)
else:
tokenizer.additional_stop_token_ids = None