EngineX-Hygon/sglang
5
0
Fork 0
Code Issues Pull Requests Actions 7 Projects Releases Wiki Activity

Improve performance when running with full parallel (#394)

Browse Source
This commit is contained in:
Liangsheng Yin
2024-04-25 17:29:07 +08:00
committed by GitHub
parent da19434c2f
commit 9216b10678
1 changed files with 3 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
python/sglang/srt/managers/router/model_rpc.py
View File

@@ -348,6 +348,7 @@ class ModelRpcServer:
# Undo the insertion # Undo the insertion
delta = self.tree_cache.dec_ref_counter(req.last_node) delta = self.tree_cache.dec_ref_counter(req.last_node)
available_size += delta available_size += delta
break
else: else:
# Add this request to the running batch # Add this request to the running batch
self.token_to_kv_pool.add_refs(req.prefix_indices) self.token_to_kv_pool.add_refs(req.prefix_indices)
@@ -356,7 +357,8 @@ class ModelRpcServer:
req.extend_input_len + req.max_new_tokens() req.extend_input_len + req.max_new_tokens()
) )
new_batch_input_tokens += req.extend_input_len new_batch_input_tokens += req.extend_input_len
else:
break
if len(can_run_list) == 0: if len(can_run_list) == 0:
return None return None