From f1a408c75f0ca7b21c53e631367351e20a1cdc86 Mon Sep 17 00:00:00 2001 From: project6-dev Date: Wed, 12 Aug 2026 04:06:21 +0000 Subject: [PATCH] =?UTF-8?q?fix:=20cap=20default=5Fmax=5Ftokens=20at=208192?= =?UTF-8?q?=20=E2=80=94=20prevent=20OOM=20kill=20on=20unlimited=20generati?= =?UTF-8?q?on?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit t3_max_tokens_none test sends request without max_tokens. With max_model_len=262144, default_max_tokens was ~260K tokens. Model generates indefinitely at 12 TPS, container gets OOM killed after ~6min. Cap at 8192 prevents memory exhaustion while still allowing long outputs. --- qwen3_6_scripts/serving_chat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qwen3_6_scripts/serving_chat.py b/qwen3_6_scripts/serving_chat.py index 4ed841b..8d62b6e 100644 --- a/qwen3_6_scripts/serving_chat.py +++ b/qwen3_6_scripts/serving_chat.py @@ -394,7 +394,7 @@ class OpenAIServingChat(OpenAIServing): assert prompt_inputs is not None sampling_params: Union[SamplingParams, BeamSearchParams] - default_max_tokens = self.max_model_len - len( + default_max_tokens = min(8192, self.max_model_len - len( prompt_inputs["prompt_token_ids"]) if request.use_beam_search: sampling_params = request.to_beam_search_params(