dylanyunlong/project_6
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

fix(serving_chat): shallow copy bug — [[]] * n and [parser] * n share references

Browse Source 
When n>=2, all_previous_token_ids entries pointed to the SAME list,
so appending tokens for choice 0 corrupted choice 1's history.
Same for tool_parsers: all choices shared one stateful parser instance.

Changed to list comprehensions that create independent objects.

Found via CCCL result_policy.cuh read: distributed result delivery
requires isolated per-rank state — same principle applies to
per-choice token tracking in vLLM streaming.
This commit is contained in:
Claude
2026-08-07 07:11:11 +00:00
parent cbd1f08a3e
commit c2bca49aa8
1 changed files with 3 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
qwen3_6_scripts/serving_chat.py
View File

@@ -348,7 +348,7 @@ class OpenAIServingChat(OpenAIServing):
# parsing and reasoning parsing (both require full-history context). # parsing and reasoning parsing (both require full-history context).
if tool_choice_auto or use_reasoning: if tool_choice_auto or use_reasoning:
previous_texts = [""] * num_choices previous_texts = [""] * num_choices
all_previous_token_ids = [[]] * num_choices all_previous_token_ids = [[] for _ in range(num_choices)]
else: else:
previous_texts, all_previous_token_ids = None, None previous_texts, all_previous_token_ids = None, None
@@ -357,7 +357,8 @@ class OpenAIServingChat(OpenAIServing):
if tool_choice_auto and self.tool_parser: if tool_choice_auto and self.tool_parser:
tool_parsers: List[Optional[ToolParser]] = [ tool_parsers: List[Optional[ToolParser]] = [
self.tool_parser(tokenizer) self.tool_parser(tokenizer)
] * num_choices for _ in range(num_choices)
]
else: else:
tool_parsers = [None] * num_choices tool_parsers = [None] * num_choices
except RuntimeError as e: except RuntimeError as e: