From 9216b10678a036a1797e19693b0445c889016687 Mon Sep 17 00:00:00 2001 From: Liangsheng Yin Date: Thu, 25 Apr 2024 17:29:07 +0800 Subject: [PATCH] Improve performance when running with full parallel (#394) --- python/sglang/srt/managers/router/model_rpc.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/python/sglang/srt/managers/router/model_rpc.py b/python/sglang/srt/managers/router/model_rpc.py index 883bb12f9..0ca46c854 100644 --- a/python/sglang/srt/managers/router/model_rpc.py +++ b/python/sglang/srt/managers/router/model_rpc.py @@ -348,6 +348,7 @@ class ModelRpcServer: # Undo the insertion delta = self.tree_cache.dec_ref_counter(req.last_node) available_size += delta + break else: # Add this request to the running batch self.token_to_kv_pool.add_refs(req.prefix_indices) @@ -356,7 +357,8 @@ class ModelRpcServer: req.extend_input_len + req.max_new_tokens() ) new_batch_input_tokens += req.extend_input_len - + else: + break if len(can_run_list) == 0: return None