From a036d41980b0a2890e28ada25316406025b3a0b4 Mon Sep 17 00:00:00 2001 From: Lianmin Zheng Date: Sat, 27 Jul 2024 17:22:18 -0700 Subject: [PATCH] Fix max new tokens (#772) --- python/sglang/srt/managers/controller/tp_worker.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/python/sglang/srt/managers/controller/tp_worker.py b/python/sglang/srt/managers/controller/tp_worker.py index 8a8cab974..eb6f826cc 100644 --- a/python/sglang/srt/managers/controller/tp_worker.py +++ b/python/sglang/srt/managers/controller/tp_worker.py @@ -306,7 +306,11 @@ class ModelTpServer: ) req.origin_input_ids = req.origin_input_ids[: self.max_req_input_len] req.sampling_params.max_new_tokens = min( - req.sampling_params.max_new_tokens or 1 << 30, + ( + req.sampling_params.max_new_tokens + if req.sampling_params.max_new_tokens is not None + else 1 << 30 + ), self.max_req_input_len - 1 - len(req.origin_input_ids), ) self.forward_queue.append(req)