dylanyunlong/project_6
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

fix: cap default_max_tokens at 8192 — prevent OOM kill on unlimited generation

Browse Source
This commit is contained in:
project6-dev
2026-08-12 04:06:24 +00:00
parent 06828a459d
commit 6dcf3590d5
1 changed files with 1 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
qwen3_6_scripts/serving_chat.py
View File

@@ -394,7 +394,7 @@ class OpenAIServingChat(OpenAIServing):
assert prompt_inputs is not None assert prompt_inputs is not None
sampling_params: Union[SamplingParams, BeamSearchParams] sampling_params: Union[SamplingParams, BeamSearchParams]
default_max_tokens = self.max_model_len - len( default_max_tokens = min(8192, self.max_model_len - len(
prompt_inputs["prompt_token_ids"]) prompt_inputs["prompt_token_ids"])
if request.use_beam_search: if request.use_beam_search:
sampling_params = request.to_beam_search_params( sampling_params = request.to_beam_search_params(