EngineX-Hygon/sglang
5
0
Fork 0
Code Issues Pull Requests Actions 7 Projects Releases Wiki Activity

Fix the default arguments of bench_offline_throughput.py & simplify detokenizer manager (#2042)

Browse Source
This commit is contained in:
Lianmin Zheng
2024-11-15 05:02:44 -08:00
committed by GitHub
parent 29ebe3dff4
commit 2558d6a675
5 changed files with 42 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
python/sglang/srt/managers/detokenizer_manager.py
View File

@@ -100,20 +100,6 @@ class DetokenizerManager:
if isinstance(recv_obj, BatchEmbeddingOut):
# If it is embedding model, no detokenization is needed.
self.send_to_tokenizer.send_pyobj(
BatchEmbeddingOut(
rids=recv_obj.rids,
embeddings=recv_obj.embeddings,
meta_info=recv_obj.meta_info,
finished_reason=recv_obj.finished_reason,
)
)
continue
elif isinstance(recv_obj, UpdateWeightReqOutput):
# If it is a weight update request, no detokenization is needed.
self.send_to_tokenizer.send_pyobj(recv_obj)
continue
elif isinstance(recv_obj, GetMemPoolSizeReqOutput):
self.send_to_tokenizer.send_pyobj(recv_obj)
continue
else: