fix: cap default_max_tokens at 8192 — prevent OOM kill on unlimited generation

2026-08-12 04:06:24 +00:00
parent 06828a459d
commit 6dcf3590d5
1 changed files with 1 additions and 1 deletions
--- a/qwen3_6_scripts/serving_chat.py
+++ b/qwen3_6_scripts/serving_chat.py
@@ -394,7 +394,7 @@ class OpenAIServingChat(OpenAIServing):
            assert prompt_inputs is not None

            sampling_params: Union[SamplingParams, BeamSearchParams]
-            default_max_tokens = self.max_model_len - len(
+            default_max_tokens = min(8192, self.max_model_len - len(
                prompt_inputs["prompt_token_ids"])
            if request.use_beam_search:
                sampling_params = request.to_beam_search_params(