Split the overlapped version of TpModelWorkerClient into a separate file (#1726)

2024-10-20 00:29:29 -07:00
parent 593b19f29d
commit b48edff67f
7 changed files with 217 additions and 131 deletions
--- a/python/sglang/srt/model_executor/model_runner.py
+++ b/python/sglang/srt/model_executor/model_runner.py
@@ -461,6 +461,7 @@ class ModelRunner:
            size=max_num_reqs + 1,
            max_context_len=self.model_config.context_len + 4,
            device=self.device,
+            use_records=False,
        )
        if (
            self.model_config.attention_arch == AttentionArch.MLA