EngineX-Hygon/sglang
5
0
Fork 0
Code Issues Pull Requests Actions 7 Projects Releases Wiki Activity

Fix max new tokens (#772)

Browse Source
This commit is contained in:
Lianmin Zheng
2024-07-27 17:22:18 -07:00
committed by GitHub
parent f95e661757
commit a036d41980
1 changed files with 5 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
python/sglang/srt/managers/controller/tp_worker.py
View File

@@ -306,7 +306,11 @@ class ModelTpServer:
) )
req.origin_input_ids = req.origin_input_ids[: self.max_req_input_len] req.origin_input_ids = req.origin_input_ids[: self.max_req_input_len]
req.sampling_params.max_new_tokens = min( req.sampling_params.max_new_tokens = min(
req.sampling_params.max_new_tokens or 1 << 30, (
req.sampling_params.max_new_tokens
if req.sampling_params.max_new_tokens is not None
else 1 << 30
),
self.max_req_input_len - 1 - len(req.origin_input_ids), self.max_req_input_len - 1 - len(req.origin_input_ids),
) )
self.forward_queue.append(req) self.forward_queue.append(req)