dylanyunlong/project_6
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

[fix] t3: cap default_max_tokens to 8192

Browse Source
This commit is contained in:
root
2026-08-24 09:59:26 +00:00
parent c2edc9a84e
commit 362da0bef9
1 changed files with 3 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
qwen3_6_scripts/serving_chat.py
View File

@@ -397,8 +397,8 @@ class OpenAIServingChat(OpenAIServing):
# OpenAI API: max_completion_tokens takes precedence over max_tokens # OpenAI API: max_completion_tokens takes precedence over max_tokens
if request.max_completion_tokens is not None and request.max_tokens is None: if request.max_completion_tokens is not None and request.max_tokens is None:
request.max_tokens = request.max_completion_tokens request.max_tokens = request.max_completion_tokens
default_max_tokens = self.max_model_len - len( default_max_tokens = min(self.max_model_len - len(
prompt_inputs["prompt_token_ids"]) prompt_inputs["prompt_token_ids"]), 8192)
if request.use_beam_search: if request.use_beam_search:
sampling_params = request.to_beam_search_params( sampling_params = request.to_beam_search_params(
default_max_tokens) default_max_tokens)
@@ -413,7 +413,7 @@ class OpenAIServingChat(OpenAIServing):
prompt_adapter_request=prompt_adapter_request) prompt_adapter_request=prompt_adapter_request)
engine_inputs = TokensPrompt( engine_inputs = TokensPrompt(
prompt_token_ids=prompt_inputs["prompt_token_ids"]) prompt_token_ids=prompt_inputs["prompt_token_ids"]), 8192)
if mm_data is not None: if mm_data is not None:
engine_inputs["multi_modal_data"] = mm_data engine_inputs["multi_modal_data"] = mm_data