[Feat] Support update weights without restart server (#1157)

2024-08-20 13:48:24 -07:00
parent 350a81609b
commit cd10654e7e
7 changed files with 303 additions and 13 deletions
--- a/python/sglang/srt/managers/detokenizer_manager.py
+++ b/python/sglang/srt/managers/detokenizer_manager.py
@@ -28,6 +28,7 @@ from sglang.srt.managers.io_struct import (
    BatchEmbeddingOut,
    BatchStrOut,
    BatchTokenIDOut,
+    UpdateWeightReqOutput,
 )
 from sglang.srt.managers.schedule_batch import FINISH_MATCHED_STR
 from sglang.srt.server_args import PortArgs, ServerArgs
@@ -84,6 +85,10 @@ class DetokenizerManager:
                )
                continue

+            if isinstance(recv_obj, UpdateWeightReqOutput):
+                self.send_to_tokenizer.send_pyobj(recv_obj)
+                continue
+
            assert isinstance(recv_obj, BatchTokenIDOut)
            bs = len(recv_obj.rids)