fix(critical): disable thinking for tool_call requests — fixes d03_tool_call FAIL
Root cause: When tool_choice=auto + tools present, the model enters <think>...</think> mode by default. On BI-V100 hardware, decode is slow enough that thinking consumes the entire max_tokens budget, and the model finishes (finish=stop) before ever emitting <tool_call> XML. Sub168 reference: d03 in 2.12s with tools=1, finish=tool_calls Our sub509: d03 in 49.04s with tools=0, finish=stop — FAIL Fix: Two-layer defense: 1. protocol.py normalize_messages: when tools active + tool_choice=auto and thinking not explicitly set, auto-set enable_thinking=False 2. qwen3coder_tool_parser.py adjust_request: same logic as defense-in-depth 3. baseline.muh synced with actual computility-run.yaml
This commit is contained in:
@@ -19,13 +19,10 @@ vllm:
|
||||
max_model_len: 100000
|
||||
gpu_memory_utilization: 0.90
|
||||
tensor_parallel: 4
|
||||
max_num_seqs: 2
|
||||
max_num_batched_tokens: 4096
|
||||
max_seq_len_to_capture: 32768
|
||||
max_num_seqs: 1
|
||||
trust_remote_code: true
|
||||
disable_log_requests: true
|
||||
disable_frontend_multiprocessing: true
|
||||
enable_chunked_prefill: true
|
||||
enable_auto_tool_choice: true
|
||||
tool_call_parser: qwen3_coder
|
||||
reasoning_parser: qwen3
|
||||
|
||||
@@ -421,13 +421,30 @@ class ChatCompletionRequest(OpenAIBaseModel):
|
||||
# The competition evaluator sends thinking={enable:true/false} (OpenAI API).
|
||||
# Qwen3's chat template expects enable_thinking=True/False in kwargs.
|
||||
thinking = data.get("thinking")
|
||||
thinking_explicitly_set = False
|
||||
if isinstance(thinking, dict):
|
||||
enable = thinking.get("enable")
|
||||
if enable is not None:
|
||||
thinking_explicitly_set = True
|
||||
ctk = data.get("chat_template_kwargs") or {}
|
||||
ctk["enable_thinking"] = bool(enable)
|
||||
data["chat_template_kwargs"] = ctk
|
||||
|
||||
# CRITICAL: When tools are present with tool_choice=auto and thinking
|
||||
# is NOT explicitly requested, disable thinking to preserve token budget
|
||||
# for tool call XML generation. Without this, the model spends all
|
||||
# tokens on <think>...</think> and finishes before emitting <tool_call>.
|
||||
# This matches the competition reference (sub168: d03 in 2.12s).
|
||||
if not thinking_explicitly_set:
|
||||
has_tools = data.get("tools") is not None and len(data.get("tools", [])) > 0
|
||||
tc = data.get("tool_choice")
|
||||
tool_choice_active = (tc == "auto" or (tc is None and has_tools)
|
||||
or isinstance(tc, dict))
|
||||
if has_tools and tool_choice_active:
|
||||
ctk = data.get("chat_template_kwargs") or {}
|
||||
ctk["enable_thinking"] = False
|
||||
data["chat_template_kwargs"] = ctk
|
||||
|
||||
messages = data.get("messages")
|
||||
if not isinstance(messages, list):
|
||||
return data
|
||||
|
||||
@@ -77,6 +77,28 @@ class Qwen3CoderToolParser(ToolParser):
|
||||
logger.debug("vLLM Successfully imported tool parser %s !",
|
||||
self.__class__.__name__)
|
||||
|
||||
def adjust_request(
|
||||
self, request: "ChatCompletionRequest") -> "ChatCompletionRequest":
|
||||
"""Disable thinking when tools are active with auto choice.
|
||||
|
||||
On BI-V100 hardware, the model's <think>...</think> phase can consume
|
||||
the entire max_tokens budget, leaving no room for the <tool_call> XML.
|
||||
Competition reference (sub168) completes d03_tool_call in 2.12s with
|
||||
tools=1; our sub509 took 49s with tools=0 because thinking ate the
|
||||
budget. Disabling thinking for tool-call requests ensures the model
|
||||
emits tool XML within the token budget.
|
||||
"""
|
||||
if (request.tools and request.tool_choice in ("auto", None)
|
||||
and not isinstance(request.tool_choice,
|
||||
type(None).__class__)):
|
||||
# Only override if thinking was not explicitly requested
|
||||
ctk = request.chat_template_kwargs or {}
|
||||
if "enable_thinking" not in ctk:
|
||||
ctk = dict(ctk) # shallow copy
|
||||
ctk["enable_thinking"] = False
|
||||
request.chat_template_kwargs = ctk
|
||||
return request
|
||||
|
||||
|
||||
def _generate_tool_call_id(self) -> str:
|
||||
return f"call_{uuid.uuid4().hex[:24]}"
|
||||
|
||||
Reference in New Issue
Block a user