Organize sampling batch info better (#1562)

2024-10-03 18:29:49 -07:00
parent e0b5dbcec1
commit 32eb6e96f2
8 changed files with 43 additions and 35 deletions
--- a/python/sglang/srt/managers/tokenizer_manager.py
+++ b/python/sglang/srt/managers/tokenizer_manager.py
@@ -87,7 +87,9 @@ class TokenizerManager:
        self.recv_from_detokenizer.bind(f"tcp://127.0.0.1:{port_args.tokenizer_port}")

        self.send_to_scheduler = context.socket(zmq.PUSH)
-        self.send_to_scheduler.connect(f"tcp://127.0.0.1:{port_args.scheduler_port}")
+        self.send_to_scheduler.connect(
+            f"tcp://127.0.0.1:{port_args.scheduler_input_port}"
+        )

        # Read model args
        self.model_path = server_args.model_path