dylanyunlong/project_6
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

fix(critical): disable thinking for tool_call requests — fixes d03_tool_call + d05/t5 FAIL

Browse Source 
Root cause: Model spends all tokens in <think>...</think> instead of emitting
<tool_call> XML. Competitor Sub168 completes d03 in 2.12s; we took 49s and FAIL.

Fix: When tool_choice != 'none' and tools present, inject enable_thinking=False
into chat_template_kwargs before calling apply_hf_chat_template().

Also handles OpenAI-style thinking field and adds competitive analysis doc.
This commit is contained in:
project6
2026-08-07 08:45:15 +00:00
parent 2d5232c5d6
commit bf6ceb0b12
3 changed files with 192 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

40
qwen3_6_scripts/serving_chat.py
View File

@@ -147,6 +147,38 @@ class OpenAIServingChat(OpenAIServing):
prompt: Union[str, List[int]]
is_mistral_tokenizer = isinstance(tokenizer, MistralTokenizer)
# Build effective chat_template_kwargs.
# When tools are active (tool_choice != "none"), disable thinking
# to prevent the model from wasting tokens on <think>...</think>
# before emitting tool call XML. This is the key fix for d03_tool_call.
effective_chat_template_kwargs = dict(
request.chat_template_kwargs or {})
# Determine if thinking should be explicitly disabled for tool calls
_tool_call_active = (
tool_dicts is not None
and request.tool_choice not in (None, "none"))
if _tool_call_active:
# Only override if the user hasn't explicitly set enable_thinking
if "enable_thinking" not in effective_chat_template_kwargs:
effective_chat_template_kwargs["enable_thinking"] = False
logger.info(
"Tool call detected (tool_choice=%s) — injecting "
"enable_thinking=False into chat_template_kwargs",
request.tool_choice)
# Also respect the OpenAI-style `thinking` request field
if request.thinking:
thinking_type = request.thinking.get("type", "enabled")
if thinking_type == "disabled":
effective_chat_template_kwargs["enable_thinking"] = False
elif thinking_type == "enabled":
# Only set True if not already overridden by tool logic
if not _tool_call_active:
effective_chat_template_kwargs.setdefault(
"enable_thinking", True)
if is_mistral_tokenizer:
prompt = apply_mistral_chat_template(
tokenizer,
@@ -156,7 +188,7 @@ class OpenAIServingChat(OpenAIServing):
continue_final_message=request.continue_final_message,
tools=tool_dicts,
documents=request.documents,
**(request.chat_template_kwargs or {}),
**effective_chat_template_kwargs,
)
else:
prompt = apply_hf_chat_template(
@@ -167,8 +199,12 @@ class OpenAIServingChat(OpenAIServing):
continue_final_message=request.continue_final_message,
tools=tool_dicts,
documents=request.documents,
**(request.chat_template_kwargs or {}),
**effective_chat_template_kwargs,
)
# Store effective kwargs back so reasoning parser gets the same
# enable_thinking state.
request.chat_template_kwargs = effective_chat_template_kwargs
except Exception as e:
logger.exception("Error in applying chat template from request")
return self.create_error_response(str(e))