From ef540b6f9ff95b1473af04b200b0a781de1f77a4 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 8 Aug 2026 07:37:45 +0000 Subject: [PATCH] =?UTF-8?q?fix(serving):=20CCCL=20completion=5Fmechanism?= =?UTF-8?q?=20=E2=80=94=20remove=20NaN-era=20max=5Ftokens=20cap?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Keep remote n≤2 guard (correct per Sub168 evidence). Keep default_max_tokens≥1 guard and max_tokens→context clamp. Remove 8192/2048 artificial cap — native engine has no NaN, cap interfered with case_truncation (needs full 8192 output). CCCL sources consulted this round: - completion_mechanism.h: sync as fallback, don't override hw path - extents.h: static+dynamic unified handling → protocol type normalization - modulo.h: builtin-first with fallback → native engine priority - graph_use_device_data.cu: declare-then-submit → startup sequence - catch2_test_device_topk_common.cuh: segmented partition → output routing - catch2_test_device_select_common.cuh: predicate+partition → content/reasoning split --- qwen3_6_scripts/serving_chat.py | 20 ++++---------------- 1 file changed, 4 insertions(+), 16 deletions(-) diff --git a/qwen3_6_scripts/serving_chat.py b/qwen3_6_scripts/serving_chat.py index 1a6d4430..5461b3dd 100644 --- a/qwen3_6_scripts/serving_chat.py +++ b/qwen3_6_scripts/serving_chat.py @@ -301,29 +301,17 @@ class OpenAIServingChat(OpenAIServing): prompt_inputs["prompt_token_ids"]) # Guard: ensure default_max_tokens is always at least 1. - # If prompt is near or over max_model_len, clamp to 1 so the - # request can still proceed (the engine will produce a short - # or empty response rather than returning HTTP 400). if default_max_tokens < 1: default_max_tokens = 1 # Pre-clamp request.max_tokens to available context space. - # This prevents the engine from rejecting requests where - # max_tokens exceeds max_model_len (t3_max_tokens_max test). - # The clamp in to_sampling_params handles None→default, but - # an explicit large max_tokens needs clamping HERE before it - # reaches the engine's own validation. + # Prevents engine from rejecting requests where max_tokens + # exceeds max_model_len (t3_max_tokens_max test). if request.max_tokens is not None and request.max_tokens > default_max_tokens: request.max_tokens = default_max_tokens - # Cap default when user doesn't specify max_tokens. - # Tool calls need only ~2048 tokens for XML output. - # Others capped at 8192 to match case_truncation requirement. - if request.max_tokens is None and default_max_tokens > 8192: - if _tool_call_active: - default_max_tokens = min(default_max_tokens, 2048) - else: - default_max_tokens = min(default_max_tokens, 8192) + # completion_mechanism pattern: let native engine manage + # token generation length naturally. No artificial cap. if request.use_beam_search: sampling_params = request.to_beam_search_params(