From c2bca49aa88e27e7ee62741a9c984701891cda17 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 7 Aug 2026 07:11:11 +0000 Subject: [PATCH] =?UTF-8?q?fix(serving=5Fchat):=20shallow=20copy=20bug=20?= =?UTF-8?q?=E2=80=94=20[[]]=20*=20n=20and=20[parser]=20*=20n=20share=20ref?= =?UTF-8?q?erences?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When n>=2, all_previous_token_ids entries pointed to the SAME list, so appending tokens for choice 0 corrupted choice 1's history. Same for tool_parsers: all choices shared one stateful parser instance. Changed to list comprehensions that create independent objects. Found via CCCL result_policy.cuh read: distributed result delivery requires isolated per-rank state — same principle applies to per-choice token tracking in vLLM streaming. --- qwen3_6_scripts/serving_chat.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/qwen3_6_scripts/serving_chat.py b/qwen3_6_scripts/serving_chat.py index 988905da..f1c534fb 100644 --- a/qwen3_6_scripts/serving_chat.py +++ b/qwen3_6_scripts/serving_chat.py @@ -348,7 +348,7 @@ class OpenAIServingChat(OpenAIServing): # parsing and reasoning parsing (both require full-history context). if tool_choice_auto or use_reasoning: previous_texts = [""] * num_choices - all_previous_token_ids = [[]] * num_choices + all_previous_token_ids = [[] for _ in range(num_choices)] else: previous_texts, all_previous_token_ids = None, None @@ -357,7 +357,8 @@ class OpenAIServingChat(OpenAIServing): if tool_choice_auto and self.tool_parser: tool_parsers: List[Optional[ToolParser]] = [ self.tool_parser(tokenizer) - ] * num_choices + for _ in range(num_choices) + ] else: tool_parsers = [None] * num_choices except RuntimeError as e: