Support DP MLA (#1970)

2024-11-16 17:01:43 +08:00
parent 2f2e07439c
commit 976bc302e5
12 changed files with 395 additions and 63 deletions
--- a/python/sglang/srt/managers/tp_worker_overlap_thread.py
+++ b/python/sglang/srt/managers/tp_worker_overlap_thread.py
@@ -88,6 +88,9 @@ class TpModelWorkerClient:
    def get_tp_cpu_group(self):
        return self.worker.get_tp_cpu_group()

+    def get_tp_device_group(self):
+        return self.worker.get_tp_device_group()
+
    def get_memory_pool(self):
        return (
            self.worker.model_runner.req_to_token_pool,