Optimize mem indices mangement (#619)

2024-07-13 23:39:37 -07:00
parent 5d264a90ac
commit 564a898ad9
15 changed files with 251 additions and 178 deletions
--- a/python/sglang/srt/managers/controller/tp_worker.py
+++ b/python/sglang/srt/managers/controller/tp_worker.py
@@ -314,7 +314,9 @@ class ModelTpServer:
        self.forward_queue.append(req)

    def get_new_fill_batch(self) -> Optional[Batch]:
-        running_bs = len(self.running_batch.reqs) if self.running_batch is not None else 0
+        running_bs = (
+            len(self.running_batch.reqs) if self.running_batch is not None else 0
+        )
        if running_bs >= self.max_running_requests:
            return