Fix max new tokens (#772)

2024-07-27 17:22:18 -07:00
parent f95e661757
commit a036d41980
1 changed files with 5 additions and 1 deletions
--- a/python/sglang/srt/managers/controller/tp_worker.py
+++ b/python/sglang/srt/managers/controller/tp_worker.py
@@ -306,7 +306,11 @@ class ModelTpServer:
            )
            req.origin_input_ids = req.origin_input_ids[: self.max_req_input_len]
        req.sampling_params.max_new_tokens = min(
-            req.sampling_params.max_new_tokens or 1 << 30,
+            (
+                req.sampling_params.max_new_tokens
+                if req.sampling_params.max_new_tokens is not None
+                else 1 << 30
+            ),
            self.max_req_input_len - 1 - len(req.origin_input_ids),
        )
        self.forward_queue.append(req)