EngineX-Hygon/sglang
5
0
Fork 0
Code Issues Pull Requests Actions 7 Projects Releases Wiki Activity

Optimize retract (#440)

Browse Source
This commit is contained in:
Liangsheng Yin
2024-05-26 00:07:26 +08:00
committed by GitHub
parent 2cea6146d8
commit f06e90c2cf
7 changed files with 298 additions and 113 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
python/sglang/srt/managers/detokenizer_manager.py
View File

@@ -51,11 +51,6 @@ class DetokenizerManager:
# Trim stop str
# TODO(lmzheng): handle the case where multiple stop strs are hit
for i in range(len(output_strs)):
if recv_obj.hit_stop_str[i] is not None:
pos = output_strs[i].find(recv_obj.hit_stop_str[i])
if pos != -1:
output_strs[i] = output_strs[i][:pos]
if len(output_tokens[i]) > 0:
first_token = self.tokenizer.convert_ids_to_tokens(
int(output_tokens[i][0])
@@ -65,9 +60,12 @@ class DetokenizerManager:
if first_token.startswith("▁"):
output_strs[i] = " " + output_strs[i]
output_strs[i] = (
recv_obj.output_and_jump_forward_strs[i] + output_strs[i]
)
output_strs[i] = recv_obj.prev_output_strs[i] + output_strs[i]
if recv_obj.hit_stop_str[i] is not None:
pos = output_strs[i].find(recv_obj.hit_stop_str[i])
if pos != -1:
output_strs[i] = output_strs[i][:pos]
self.send_to_tokenizer.send_pyobj(
BatchStrOut(