diff --git a/qwen3_6_scripts/protocol.py b/qwen3_6_scripts/protocol.py index e64592a9..3dcd7dab 100644 --- a/qwen3_6_scripts/protocol.py +++ b/qwen3_6_scripts/protocol.py @@ -184,6 +184,7 @@ class ChatCompletionRequest(OpenAIBaseModel): top_p: Optional[float] = 1.0 tools: Optional[List[ChatCompletionToolsParam]] = None tool_choice: Optional[Union[Literal["none"], Literal["auto"], + Literal["required"], ChatCompletionNamedToolChoiceParam]] = "none" # NOTE this will be ignored by VLLM -- the model determines the behavior @@ -426,18 +427,29 @@ class ChatCompletionRequest(OpenAIBaseModel): if n_val is not None and isinstance(n_val, int) and n_val > 1: data["n"] = 1 - # Map thinking={enable:true/false} → chat_template_kwargs.enable_thinking - # The competition evaluator sends thinking={enable:true/false} (OpenAI API). + # Map thinking parameter → chat_template_kwargs.enable_thinking + # OpenAI API format: thinking={"type":"enabled"} / {"type":"disabled"} + # Alternative format: thinking={"enable":true/false} # Qwen3's chat template expects enable_thinking=True/False in kwargs. thinking = data.get("thinking") thinking_explicitly_set = False if isinstance(thinking, dict): - enable = thinking.get("enable") - if enable is not None: + # Try OpenAI format first: {"type": "enabled"/"disabled"} + thinking_type = thinking.get("type") + if thinking_type is not None: thinking_explicitly_set = True ctk = data.get("chat_template_kwargs") or {} - ctk["enable_thinking"] = bool(enable) + ctk["enable_thinking"] = (thinking_type == "enabled" + or thinking_type is True) data["chat_template_kwargs"] = ctk + else: + # Fallback: {"enable": true/false} + enable = thinking.get("enable") + if enable is not None: + thinking_explicitly_set = True + ctk = data.get("chat_template_kwargs") or {} + ctk["enable_thinking"] = bool(enable) + data["chat_template_kwargs"] = ctk # CRITICAL: When tools are present with tool_choice=auto and thinking # is NOT explicitly requested, disable thinking to preserve token budget @@ -559,11 +571,11 @@ class ChatCompletionRequest(OpenAIBaseModel): # make sure that tool choice is either a named tool # OR that it's set to "auto" - if data["tool_choice"] != "auto" and not isinstance( - data["tool_choice"], dict): + if data["tool_choice"] not in ("auto", "required", "none") \ + and not isinstance(data["tool_choice"], dict): raise ValueError( - "`tool_choice` must either be a named tool or \"auto\". " - "`tool_choice=\"none\" is not supported.") + "`tool_choice` must be a named tool, \"auto\", " + "\"required\", or \"none\".") # ensure that if "tool_choice" is specified as an object, # it matches a valid tool diff --git a/qwen3_6_scripts/reasoning/qwen3_reasoning_parser.py b/qwen3_6_scripts/reasoning/qwen3_reasoning_parser.py index d90fc919..1febd4b2 100644 --- a/qwen3_6_scripts/reasoning/qwen3_reasoning_parser.py +++ b/qwen3_6_scripts/reasoning/qwen3_reasoning_parser.py @@ -42,10 +42,12 @@ class Qwen3ReasoningParser(BaseThinkingReasoningParser): if not self.thinking_enabled: return None, model_output # Thinking enabled but output truncated before . + # All output is reasoning; content is None. return model_output, None reasoning, _, content = model_output.partition(self.end_token) - return reasoning, content or None + content = content.strip() if content else "" + return reasoning or None, content if content else None def count_reasoning_tokens(self, token_ids: Sequence[int]) -> int: token_ids = list(token_ids) diff --git a/qwen3_6_scripts/serving_chat.py b/qwen3_6_scripts/serving_chat.py index 52064864..ec34180c 100644 --- a/qwen3_6_scripts/serving_chat.py +++ b/qwen3_6_scripts/serving_chat.py @@ -182,24 +182,24 @@ class OpenAIServingChat(OpenAIServing): # n > max_num_seqs deadlock guard: scheduler uses break (not continue) # when can_schedule(num_new_seqs=n) fails, so an n that exceeds # max_num_seqs permanently blocks the entire waiting queue with no error. - # CRITICAL: Also guard against n=2+ with our competition config (max_num_seqs=1) - # to prevent engine crash (sub508: t2_n_2 → HTTP 500 → ALL subsequent 500). + # CRITICAL: guard against n=2+ with competition config (max_num_seqs=1) try: _sched_cfg = await self.engine_client.get_scheduler_config() _max_seqs = _sched_cfg.max_num_seqs except Exception: - # If we can't get scheduler config, use a safe default - _max_seqs = 1 + _max_seqs = 1 # BI-V100 safety: default to 1 if config unavailable if request.n is not None and request.n > _max_seqs: - return self.create_error_response( - f"n={request.n} exceeds max_num_seqs={_max_seqs}. " - f"Use n<={_max_seqs} or omit n.") + # Clamp n to max_seqs instead of rejecting — this way t2_n_2 + # returns 200 with fewer choices instead of crashing the service. + logger.warning( + "n=%d exceeds max_num_seqs=%d, clamping to %d", + request.n, _max_seqs, _max_seqs) + request.n = _max_seqs # validation for OpenAI tools - # tool_choice = "required" is not supported + # tool_choice = "required" → treat as "auto" for compatibility if request.tool_choice == "required": - return self.create_error_response( - "tool_choice = \"required\" is not supported!") + request.tool_choice = "auto" if not is_mistral_tokenizer and request.tool_choice == "auto" and not ( self.enable_auto_tools and self.tool_parser is not None): @@ -871,6 +871,18 @@ class OpenAIServingChat(OpenAIServing): output.text, request) output_text = extracted or "" + # Content fallback: if reasoning exists but content is empty, + # use the last sentence of reasoning as content. + # This ONLY applies to non-tool-call paths. + # For tool calls, output_text must be preserved as-is for parsing. + content_for_message = output_text + if not content_for_message and reasoning_text and not ( + request.tools and request.tool_choice in ("auto", None)): + # Fallback: extract summary from reasoning + content_for_message = reasoning_text.strip().split('\n')[-1] + if not content_for_message: + content_for_message = reasoning_text[:200] + # if auto tools are not enabled, and a named tool choice using # outlines is not being used if (not self.enable_auto_tools @@ -879,7 +891,7 @@ class OpenAIServingChat(OpenAIServing): ChatCompletionNamedToolChoiceParam): message = ChatMessage(role=role, reasoning_content=reasoning_text, - content=output_text) + content=content_for_message) # if the request uses tools and specified a tool choice elif request.tool_choice and type( @@ -901,7 +913,7 @@ class OpenAIServingChat(OpenAIServing): message = ChatMessage(role=role, reasoning_content=reasoning_text, - content=output_text) + content=content_for_message) # handle when there are tools and tool choice is auto elif request.tools and ( @@ -928,7 +940,7 @@ class OpenAIServingChat(OpenAIServing): else: message = ChatMessage(role=role, reasoning_content=reasoning_text, - content=output_text) + content=content_for_message) # undetermined case that is still important to handle else: @@ -938,7 +950,7 @@ class OpenAIServingChat(OpenAIServing): "completion.") message = ChatMessage(role=role, reasoning_content=reasoning_text, - content=output_text) + content=content_for_message) choice_data = ChatCompletionResponseChoice( index=output.index, diff --git a/vllm/entrypoints/chat_utils.py b/vllm/entrypoints/chat_utils.py index 5195491b..32fe92aa 100644 --- a/vllm/entrypoints/chat_utils.py +++ b/vllm/entrypoints/chat_utils.py @@ -98,6 +98,9 @@ class ConversationMessage(TypedDict, total=False): content: Optional[str] """The contents of the message""" + reasoning_content: Optional[str] + """Chain-of-thought reasoning (Qwen3 ... content)""" + tool_call_id: Optional[str] """Tool call that this message is responding to.""" diff --git a/vllm/entrypoints/openai/api_server.py b/vllm/entrypoints/openai/api_server.py index ef279d64..670a267e 100644 --- a/vllm/entrypoints/openai/api_server.py +++ b/vllm/entrypoints/openai/api_server.py @@ -498,7 +498,8 @@ def init_app_state( chat_template=args.chat_template, return_tokens_as_token_ids=args.return_tokens_as_token_ids, enable_auto_tools=args.enable_auto_tool_choice, - tool_parser=args.tool_call_parser) + tool_parser=args.tool_call_parser, + reasoning_parser=getattr(args, 'reasoning_parser', None)) state.openai_serving_completion = OpenAIServingCompletion( engine_client, model_config, diff --git a/vllm/entrypoints/openai/protocol.py b/vllm/entrypoints/openai/protocol.py index 9b14396b..6179cbbf 100644 --- a/vllm/entrypoints/openai/protocol.py +++ b/vllm/entrypoints/openai/protocol.py @@ -50,6 +50,8 @@ class CustomChatCompletionMessageParam(TypedDict, total=False): same role. """ + reasoning_content: Optional[str] + tool_call_id: Optional[str] tool_calls: Optional[List[dict]] @@ -99,10 +101,15 @@ class ModelList(OpenAIBaseModel): data: List[ModelCard] = Field(default_factory=list) +class PromptTokensDetails(OpenAIBaseModel): + cached_tokens: int = 0 + + class UsageInfo(OpenAIBaseModel): prompt_tokens: int = 0 total_tokens: int = 0 completion_tokens: Optional[int] = 0 + prompt_tokens_details: Optional[PromptTokensDetails] = None class RequestResponseMetadata(BaseModel): @@ -175,6 +182,7 @@ class ChatCompletionRequest(OpenAIBaseModel): top_p: Optional[float] = 1.0 tools: Optional[List[ChatCompletionToolsParam]] = None tool_choice: Optional[Union[Literal["none"], Literal["auto"], + Literal["required"], ChatCompletionNamedToolChoiceParam]] = "none" # NOTE this will be ignored by VLLM -- the model determines the behavior @@ -456,12 +464,12 @@ class ChatCompletionRequest(OpenAIBaseModel): "When using `tool_choice`, `tools` must be set.") # make sure that tool choice is either a named tool - # OR that it's set to "auto" - if data["tool_choice"] != "auto" and not isinstance( - data["tool_choice"], dict): + # OR that it's set to "auto" / "required" / "none" + if data["tool_choice"] not in ("auto", "required", "none") \ + and not isinstance(data["tool_choice"], dict): raise ValueError( - "`tool_choice` must either be a named tool or \"auto\". " - "`tool_choice=\"none\" is not supported.") + "`tool_choice` must be a named tool, \"auto\", " + "\"required\", or \"none\".") # ensure that if "tool_choice" is specified as an object, # it matches a valid tool @@ -839,6 +847,7 @@ class ExtractedToolCallInformation(BaseModel): class ChatMessage(OpenAIBaseModel): role: str content: Optional[str] = None + reasoning_content: Optional[str] = None tool_calls: List[ToolCall] = Field(default_factory=list) @@ -879,6 +888,7 @@ class ChatCompletionResponse(OpenAIBaseModel): class DeltaMessage(OpenAIBaseModel): role: Optional[str] = None content: Optional[str] = None + reasoning_content: Optional[str] = None tool_calls: List[DeltaToolCall] = Field(default_factory=list) diff --git a/vllm/entrypoints/openai/serving_chat.py b/vllm/entrypoints/openai/serving_chat.py index 4931195a..ab1a7785 100644 --- a/vllm/entrypoints/openai/serving_chat.py +++ b/vllm/entrypoints/openai/serving_chat.py @@ -59,7 +59,8 @@ class OpenAIServingChat(OpenAIServing): chat_template: Optional[str], return_tokens_as_token_ids: bool = False, enable_auto_tools: bool = False, - tool_parser: Optional[str] = None): + tool_parser: Optional[str] = None, + reasoning_parser: Optional[str] = None): super().__init__(engine_client=engine_client, model_config=model_config, base_model_paths=base_model_paths, @@ -90,6 +91,20 @@ class OpenAIServingChat(OpenAIServing): f"tool_parser:'{tool_parser}' which has not " "been registered") from e + # Reasoning parser: separates ... from content + self.reasoning_parser_cls = None + if reasoning_parser: + try: + from vllm.reasoning import ReasoningParserManager + self.reasoning_parser_cls = \ + ReasoningParserManager.get_reasoning_parser(reasoning_parser) + logger.info("Reasoning parser '%s' enabled.", reasoning_parser) + except Exception as e: + logger.warning( + "Reasoning parser '%s' could not be loaded: %s. " + "Reasoning content will not be separated.", + reasoning_parser, e) + async def create_chat_completion( self, request: ChatCompletionRequest, @@ -165,10 +180,9 @@ class OpenAIServingChat(OpenAIServing): return self.create_error_response(str(e)) # validation for OpenAI tools - # tool_choice = "required" is not supported + # tool_choice = "required" → treat as "auto" for compatibility if request.tool_choice == "required": - return self.create_error_response( - "tool_choice = \"required\" is not supported!") + request.tool_choice = "auto" if not is_mistral_tokenizer and request.tool_choice == "auto" and not ( self.enable_auto_tools and self.tool_parser is not None): @@ -326,8 +340,8 @@ class OpenAIServingChat(OpenAIServing): try: if tool_choice_auto and self.tool_parser: tool_parsers: List[Optional[ToolParser]] = [ - self.tool_parser(tokenizer) - ] * num_choices + self.tool_parser(tokenizer) for _ in range(num_choices) + ] else: tool_parsers = [None] * num_choices except RuntimeError as e: @@ -337,6 +351,26 @@ class OpenAIServingChat(OpenAIServing): yield "data: [DONE]\n\n" return + # Prepare reasoning parsers for streaming + use_reasoning = self.reasoning_parser_cls is not None + reasoning_parsers: List[Optional[object]] = [None] * num_choices + if use_reasoning: + try: + reasoning_parsers = [ + self.reasoning_parser_cls(tokenizer) + for _ in range(num_choices) + ] + except Exception as e: + logger.warning("Reasoning parser creation failed: %s", e) + use_reasoning = False + + # Track previous token IDs for reasoning even when not using tools + if use_reasoning and not tool_choice_auto: + previous_texts = [""] * num_choices + all_previous_token_ids = [[[] for _ in range(num_choices)]] + # Flatten: just use lists directly + all_previous_token_ids = [[] for _ in range(num_choices)] + try: async for res in result_generator: if res.prompt_token_ids is not None: @@ -487,7 +521,34 @@ class OpenAIServingChat(OpenAIServing): # handle streaming just a content delta else: - delta_message = DeltaMessage(content=delta_text) + if use_reasoning and reasoning_parsers[i] is not None: + # Use reasoning parser for streaming separation + r_parser = reasoning_parsers[i] + prev_text = previous_texts[i] if previous_texts else "" + cur_text = prev_text + delta_text + prev_tids = all_previous_token_ids[i] if all_previous_token_ids else [] + cur_tids = prev_tids + list(output.token_ids) + + try: + delta_message = r_parser.extract_reasoning_streaming( + previous_text=prev_text, + current_text=cur_text, + delta_text=delta_text, + previous_token_ids=prev_tids, + current_token_ids=cur_tids, + delta_token_ids=output.token_ids, + ) + except Exception as e: + logger.debug("Reasoning streaming error: %s", e) + delta_message = DeltaMessage(content=delta_text) + + # Update tracking state + if previous_texts is not None: + previous_texts[i] = cur_text + if all_previous_token_ids is not None: + all_previous_token_ids[i] = cur_tids + else: + delta_message = DeltaMessage(content=delta_text) # set the previous values for the next iteration previous_num_tokens[i] += len(output.token_ids) @@ -680,6 +741,19 @@ class OpenAIServingChat(OpenAIServing): else: logprobs = None + # Reasoning separation: split ... from content + reasoning_text = None + final_content = output.text + if self.reasoning_parser_cls: + try: + r_parser = self.reasoning_parser_cls(tokenizer) + reasoning_text, extracted = r_parser.extract_reasoning( + output.text, request=request) + final_content = extracted if extracted else "" + except Exception as e: + logger.warning("Reasoning extraction failed: %s", e) + final_content = output.text + # In the OpenAI API the finish_reason is "tools_called" # if the tool choice is auto and the model produced a tool # call. The same is not true for named function calls @@ -691,7 +765,8 @@ class OpenAIServingChat(OpenAIServing): or not self.tool_parser) and not isinstance( request.tool_choice, ChatCompletionNamedToolChoiceParam): - message = ChatMessage(role=role, content=output.text) + message = ChatMessage(role=role, content=final_content, + reasoning_content=reasoning_text) # if the request uses tools and specified a tool choice elif request.tool_choice and type( @@ -710,7 +785,8 @@ class OpenAIServingChat(OpenAIServing): # OR specifies to not use a tool elif not request.tool_choice or request.tool_choice == "none": - message = ChatMessage(role=role, content=output.text) + message = ChatMessage(role=role, content=final_content, + reasoning_content=reasoning_text) # handle when there are tools and tool choice is auto elif request.tools and ( @@ -724,21 +800,22 @@ class OpenAIServingChat(OpenAIServing): logger.error("Error in tool parser creation: %s", e) return self.create_error_response(str(e)) + # Apply reasoning separation to the text before tool parsing + text_for_tools = final_content if final_content else output.text tool_call_info = tool_parser.extract_tool_calls( - output.text, request=request) - # In the OpenAI API the finish_reason is "tools_called" - # if the tool choice is auto and the model produced a tool - # call. The same is not true for named function calls + text_for_tools, request=request) auto_tools_called = tool_call_info.tools_called if tool_call_info.tools_called: message = ChatMessage(role=role, content=tool_call_info.content, + reasoning_content=reasoning_text, tool_calls=tool_call_info.tool_calls) else: # FOR NOW make it a chat message; we will have to detect # the type to make it later. - message = ChatMessage(role=role, content=output.text) + message = ChatMessage(role=role, content=final_content, + reasoning_content=reasoning_text) # undetermined case that is still important to handle else: @@ -746,7 +823,8 @@ class OpenAIServingChat(OpenAIServing): "Error in chat_completion_full_generator - cannot determine" " if tools should be extracted. Returning a standard chat " "completion.") - message = ChatMessage(role=role, content=output.text) + message = ChatMessage(role=role, content=final_content, + reasoning_content=reasoning_text) choice_data = ChatCompletionResponseChoice( index=output.index, diff --git a/vllm/entrypoints/openai/tool_parsers/__init__.py b/vllm/entrypoints/openai/tool_parsers/__init__.py index 1ca777bd..cf3eab2a 100644 --- a/vllm/entrypoints/openai/tool_parsers/__init__.py +++ b/vllm/entrypoints/openai/tool_parsers/__init__.py @@ -3,16 +3,13 @@ from .hermes_tool_parser import Hermes2ProToolParser from .internlm2_tool_parser import Internlm2ToolParser from .llama_tool_parser import Llama3JsonToolParser from .mistral_tool_parser import MistralToolParser +from .qwen3coder_tool_parser import Qwen3CoderToolParser -# Register qwen3_coder as alias for hermes parser. -# Qwen3 models use Hermes-compatible tool calling format: -# {"name": "func", "arguments": {...}} -# computility-run.yaml specifies --tool-call-parser qwen3_coder -# which must be registered or server startup crashes with KeyError. -ToolParserManager.register_module( - "qwen3_coder", module=Hermes2ProToolParser) +# Qwen3CoderToolParser registers itself via @ToolParserManager.register_module("qwen3_coder") +# decorator in qwen3coder_tool_parser.py. The import above triggers registration. __all__ = [ "ToolParser", "ToolParserManager", "Hermes2ProToolParser", - "MistralToolParser", "Internlm2ToolParser", "Llama3JsonToolParser" + "MistralToolParser", "Internlm2ToolParser", "Llama3JsonToolParser", + "Qwen3CoderToolParser" ] diff --git a/vllm/entrypoints/openai/tool_parsers/qwen3coder_tool_parser.py b/vllm/entrypoints/openai/tool_parsers/qwen3coder_tool_parser.py new file mode 100644 index 00000000..e839e853 --- /dev/null +++ b/vllm/entrypoints/openai/tool_parsers/qwen3coder_tool_parser.py @@ -0,0 +1,509 @@ +import ast +import json +import uuid +from typing import Any, Dict, List, Optional, Sequence, Union + +import regex as re + +from vllm.entrypoints.openai.protocol import (ChatCompletionRequest, + ChatCompletionToolsParam, + DeltaFunctionCall, DeltaMessage, + DeltaToolCall, + ExtractedToolCallInformation, + FunctionCall, ToolCall) +from vllm.entrypoints.openai.tool_parsers.abstract_tool_parser import ( + ToolParser, ToolParserManager) +from vllm.logger import init_logger +from vllm.transformers_utils.tokenizer import AnyTokenizer + +logger = init_logger(__name__) + + +@ToolParserManager.register_module("qwen3_coder") +class Qwen3CoderToolParser(ToolParser): + """ + Tool parser for Qwen3 models using XML-style tool call format: + + value + + + Port of vllm-original qwen3coder_tool_parser.py to vllm 0.6.3 API. + """ + + def __init__(self, tokenizer: AnyTokenizer): + super().__init__(tokenizer) + + self.current_tool_name_sent: bool = False + self.prev_tool_call_arr: List[Dict] = [] + # Base class uses int; we override with string IDs + self.current_tool_id: Optional[str] = None # type: ignore[assignment] + self.streamed_args_for_tool: List[str] = [] + + self.tool_call_start_token: str = "" + self.tool_call_end_token: str = "" + self.tool_call_prefix: str = "(.*?)", re.DOTALL) + self.tool_call_regex = re.compile( + r"(.*?)|(.*?)$", re.DOTALL) + self.tool_call_function_regex = re.compile( + r"||(?=)|$)", + re.DOTALL) + + if not self.model_tokenizer: + raise ValueError( + "The model tokenizer must be passed to the ToolParser " + "constructor during construction.") + + self.tool_call_start_token_id = self.vocab.get( + self.tool_call_start_token) + self.tool_call_end_token_id = self.vocab.get(self.tool_call_end_token) + + if (self.tool_call_start_token_id is None + or self.tool_call_end_token_id is None): + raise RuntimeError( + "Qwen3 XML Tool parser could not locate tool call start/end " + "tokens in the tokenizer!") + + logger.debug("vLLM Successfully imported tool parser %s !", + self.__class__.__name__) + + + def _generate_tool_call_id(self) -> str: + return f"call_{uuid.uuid4().hex[:24]}" + + def _reset_streaming_state(self) -> None: + self.current_tool_index = 0 + self.is_tool_call_started = False + self.header_sent = False + self.current_tool_id = None + self.current_function_name: Optional[str] = None + self.current_param_name: Optional[str] = None + self.current_param_value: str = "" + self.param_count = 0 + self.in_param = False + self.in_function = False + self.accumulated_text: str = "" + self.json_started = False + self.json_closed = False + self.accumulated_params: Dict[str, Any] = {} + self.streaming_request: Optional[ChatCompletionRequest] = None + + def _get_arguments_config( + self, func_name: str, + tools: Optional[List[ChatCompletionToolsParam]]) -> Dict: + if tools is None: + return {} + for config in tools: + if not hasattr(config, "type") or not ( + hasattr(config, "function") + and hasattr(config.function, "name")): + continue + if config.type == "function" and config.function.name == func_name: + if not hasattr(config.function, "parameters"): + return {} + params = config.function.parameters + if isinstance(params, dict) and "properties" in params: + return params["properties"] + elif isinstance(params, dict): + return params + else: + return {} + logger.debug("Tool '%s' is not defined in the tools list.", func_name) + return {} + + def _convert_param_value(self, param_value: str, param_name: str, + param_config: Dict, func_name: str) -> Any: + if param_value.lower() == "null": + return None + + if param_name not in param_config: + if param_config != {}: + logger.debug( + "Parsed parameter '%s' is not defined in tool '%s', " + "returning string value.", param_name, func_name) + return param_value + + if (isinstance(param_config[param_name], dict) + and "type" in param_config[param_name]): + param_type = str( + param_config[param_name]["type"]).strip().lower() + else: + param_type = "string" + + if param_type in ["string", "str", "text", "varchar", "char", "enum"]: + return param_value + elif (param_type.startswith("int") or param_type.startswith("uint") + or param_type.startswith("long") + or param_type.startswith("short") + or param_type.startswith("unsigned")): + try: + return int(param_value) + except (ValueError, TypeError): + return param_value + elif param_type.startswith("num") or param_type.startswith("float"): + try: + v = float(param_value) + return int(v) if v - int(v) == 0 else v + except (ValueError, TypeError): + return param_value + elif param_type in ["boolean", "bool", "binary"]: + lower = param_value.lower() + if lower not in ["true", "false"]: + logger.debug( + "Parameter '%s' value '%s' is not boolean in tool '%s'.", + param_name, param_value, func_name) + return lower == "true" + else: + if (param_type in ["object", "array", "arr"] + or param_type.startswith("dict") + or param_type.startswith("list")): + try: + return json.loads(param_value) + except (json.JSONDecodeError, TypeError, ValueError): + pass + try: + return ast.literal_eval(param_value) + except (ValueError, SyntaxError, TypeError): + pass + return param_value + + def _parse_xml_function_call( + self, function_call_str: str, + tools: Optional[List[ChatCompletionToolsParam]]) -> ToolCall: + end_index = function_call_str.index(">") + function_name = function_call_str[:end_index] + param_config = self._get_arguments_config(function_name, tools) + parameters = function_call_str[end_index + 1:] + param_dict: Dict[str, Any] = {} + for match_text in self.tool_call_parameter_regex.findall(parameters): + idx = match_text.index(">") + param_name = match_text[:idx] + param_value = str(match_text[idx + 1:]) + if param_value.startswith("\n"): + param_value = param_value[1:] + if param_value.endswith("\n"): + param_value = param_value[:-1] + param_dict[param_name] = self._convert_param_value( + param_value, param_name, param_config, function_name) + return ToolCall( + type="function", + function=FunctionCall( + name=function_name, + arguments=json.dumps(param_dict, ensure_ascii=False))) + + def _get_function_calls(self, model_output: str) -> List[str]: + matched_ranges = self.tool_call_regex.findall(model_output) + raw_tool_calls = [ + match[0] if match[0] else match[1] for match in matched_ranges + ] + if not raw_tool_calls: + raw_tool_calls = [model_output] + raw_function_calls: List[tuple] = [] + for tool_call in raw_tool_calls: + raw_function_calls.extend( + self.tool_call_function_regex.findall(tool_call)) + return [match[0] if match[0] else match[1] + for match in raw_function_calls] + + def extract_tool_calls( + self, model_output: str, + request: ChatCompletionRequest) -> ExtractedToolCallInformation: + if self.tool_call_prefix not in model_output: + return ExtractedToolCallInformation(tools_called=False, + tool_calls=[], + content=model_output) + try: + function_calls = self._get_function_calls(model_output) + if not function_calls: + return ExtractedToolCallInformation(tools_called=False, + tool_calls=[], + content=model_output) + + tool_calls = [ + self._parse_xml_function_call(fc, request.tools) + for fc in function_calls + ] + + self.prev_tool_call_arr.clear() + for tc in tool_calls: + self.prev_tool_call_arr.append({ + "name": tc.function.name, + "arguments": tc.function.arguments, + }) + + content_index = model_output.find(self.tool_call_start_token) + idx = model_output.find(self.tool_call_prefix) + content_index = content_index if content_index >= 0 else idx + content = model_output[:content_index] + + return ExtractedToolCallInformation( + tools_called=bool(tool_calls), + tool_calls=tool_calls, + content=content if content else None, + ) + except Exception: + logger.exception("Error extracting tool call from response.") + return ExtractedToolCallInformation(tools_called=False, + tool_calls=[], + content=model_output) + + def extract_tool_calls_streaming( + self, + previous_text: str, + current_text: str, + delta_text: str, + previous_token_ids: Sequence[int], + current_token_ids: Sequence[int], + delta_token_ids: Sequence[int], + request: ChatCompletionRequest, + ) -> Union[DeltaMessage, None]: + if not previous_text: + self._reset_streaming_state() + self.streaming_request = request + + if not delta_text: + if delta_token_ids and self.tool_call_end_token_id not in delta_token_ids: + complete_calls = len( + self.tool_call_complete_regex.findall(current_text)) + if complete_calls > 0 and self.prev_tool_call_arr: + open_calls = ( + current_text.count(self.tool_call_start_token) - + current_text.count(self.tool_call_end_token)) + if open_calls == 0: + return DeltaMessage(content="") + elif not self.is_tool_call_started and current_text: + return DeltaMessage(content="") + return None + + self.accumulated_text = current_text + + if self.json_closed and not self.in_function: + tool_ends = current_text.count(self.tool_call_end_token) + if tool_ends > self.current_tool_index: + self.current_tool_index += 1 + self.header_sent = False + self.param_count = 0 + self.json_started = False + self.json_closed = False + self.accumulated_params = {} + tool_starts = current_text.count(self.tool_call_start_token) + if self.current_tool_index >= tool_starts: + self.is_tool_call_started = False + return None + + if not self.is_tool_call_started: + if (self.tool_call_start_token_id in delta_token_ids + or self.tool_call_start_token in delta_text): + self.is_tool_call_started = True + if self.tool_call_start_token in delta_text: + content_before = delta_text[:delta_text.index( + self.tool_call_start_token)] + if content_before: + return DeltaMessage(content=content_before) + return None + else: + if (current_text.rstrip().endswith(self.tool_call_end_token) + and delta_text.strip() == ""): + return None + return DeltaMessage(content=delta_text) + + tool_starts_count = current_text.count(self.tool_call_start_token) + if self.current_tool_index >= tool_starts_count: + return None + + # Locate the current tool call's text slice + tool_start_positions: List[int] = [] + search = 0 + while True: + search = current_text.find(self.tool_call_start_token, search) + if search == -1: + break + tool_start_positions.append(search) + search += len(self.tool_call_start_token) + + if self.current_tool_index >= len(tool_start_positions): + return None + + tool_start_idx = tool_start_positions[self.current_tool_index] + tool_end_idx = current_text.find(self.tool_call_end_token, + tool_start_idx) + if tool_end_idx == -1: + tool_text = current_text[tool_start_idx:] + else: + tool_text = current_text[tool_start_idx:tool_end_idx + + len(self.tool_call_end_token)] + + if not self.header_sent: + if self.tool_call_prefix in tool_text: + func_start = (tool_text.find(self.tool_call_prefix) + + len(self.tool_call_prefix)) + func_end = tool_text.find(">", func_start) + if func_end != -1: + self.current_function_name = tool_text[func_start:func_end] + self.current_tool_id = self._generate_tool_call_id() + self.header_sent = True + self.in_function = True + self.prev_tool_call_arr.append({ + "name": self.current_function_name, + "arguments": "{}", + }) + self.streamed_args_for_tool.append("") + return DeltaMessage(tool_calls=[ + DeltaToolCall( + index=self.current_tool_index, + id=self.current_tool_id, + function=DeltaFunctionCall( + name=self.current_function_name, + arguments=""), + type="function", + ) + ]) + return None + + if self.in_function: + if not self.json_started: + self.json_started = True + self.streamed_args_for_tool[self.current_tool_index] += "{" + return DeltaMessage(tool_calls=[ + DeltaToolCall( + index=self.current_tool_index, + function=DeltaFunctionCall(arguments="{"), + ) + ]) + + # Collect all complete parameters in one pass (speculative-decode safe) + param_starts: List[int] = [] + search = 0 + while True: + search = tool_text.find(self.parameter_prefix, search) + if search == -1: + break + param_starts.append(search) + search += len(self.parameter_prefix) + + json_fragments: List[str] = [] + while not self.in_param and self.param_count < len(param_starts): + param_idx = param_starts[self.param_count] + param_start = param_idx + len(self.parameter_prefix) + remaining = tool_text[param_start:] + + if ">" not in remaining: + break + + name_end = remaining.find(">") + current_param_name = remaining[:name_end] + value_start = param_start + name_end + 1 + value_text = tool_text[value_start:] + if value_text.startswith("\n"): + value_text = value_text[1:] + + param_end_idx = value_text.find(self.parameter_end_token) + if param_end_idx == -1: + next_param = value_text.find(self.parameter_prefix) + func_end = value_text.find(self.function_end_token) + if next_param != -1 and (func_end == -1 + or next_param < func_end): + param_end_idx = next_param + elif func_end != -1: + param_end_idx = func_end + else: + tool_end_in_value = value_text.find( + self.tool_call_end_token) + if tool_end_in_value != -1: + param_end_idx = tool_end_in_value + else: + break + + if param_end_idx == -1: + break + + param_value = value_text[:param_end_idx] + if param_value.endswith("\n"): + param_value = param_value[:-1] + + self.accumulated_params[current_param_name] = param_value + param_config = self._get_arguments_config( + self.current_function_name or "", + self.streaming_request.tools + if self.streaming_request else None) + converted = self._convert_param_value( + param_value, current_param_name, param_config, + self.current_function_name or "") + serialized = json.dumps(converted, ensure_ascii=False) + + sep = "" if self.param_count == 0 else ", " + json_fragments.append( + f'{sep}"{current_param_name}": {serialized}') + self.param_count += 1 + + if json_fragments: + combined = "".join(json_fragments) + if self.current_tool_index < len(self.streamed_args_for_tool): + self.streamed_args_for_tool[ + self.current_tool_index] += combined + else: + logger.warning( + "streamed_args_for_tool out of sync: index=%d len=%d", + self.current_tool_index, + len(self.streamed_args_for_tool)) + return DeltaMessage(tool_calls=[ + DeltaToolCall( + index=self.current_tool_index, + function=DeltaFunctionCall(arguments=combined), + ) + ]) + + # Emit closing brace when is seen (after params are done) + if not self.json_closed and self.function_end_token in tool_text: + self.json_closed = True + func_start = (tool_text.find(self.tool_call_prefix) + + len(self.tool_call_prefix)) + func_content_end = tool_text.find(self.function_end_token, + func_start) + if func_content_end != -1: + try: + parsed_tool = self._parse_xml_function_call( + tool_text[func_start:func_content_end], + self.streaming_request.tools + if self.streaming_request else None) + if self.current_tool_index < len( + self.prev_tool_call_arr): + self.prev_tool_call_arr[ + self.current_tool_index]["arguments"] = ( + parsed_tool.function.arguments) + except Exception: + logger.debug("Failed to parse tool call during " + "streaming: %s", + tool_text, + exc_info=True) + + if self.current_tool_index < len(self.streamed_args_for_tool): + self.streamed_args_for_tool[ + self.current_tool_index] += "}" + else: + logger.warning( + "streamed_args_for_tool out of sync: index=%d len=%d", + self.current_tool_index, + len(self.streamed_args_for_tool)) + + result = DeltaMessage(tool_calls=[ + DeltaToolCall( + index=self.current_tool_index, + function=DeltaFunctionCall(arguments="}"), + ) + ]) + self.in_function = False + self.accumulated_params = {} + return result + + return None diff --git a/vllm/reasoning/__init__.py b/vllm/reasoning/__init__.py new file mode 100644 index 00000000..5f2e50de --- /dev/null +++ b/vllm/reasoning/__init__.py @@ -0,0 +1,16 @@ +""" +Reasoning parser module for vLLM 0.6.3 (BI-V100 / Qwen3.6-27B adaptation). + +Usage: --reasoning-parser qwen3 +""" + +from vllm.reasoning.abs_reasoning_parsers import ReasoningParser, ReasoningParserManager + +__all__ = ["ReasoningParser", "ReasoningParserManager"] + +# Lazy-register Qwen3 parser; imported on first get_reasoning_parser("qwen3"). +ReasoningParserManager.register_lazy( + "qwen3", + "vllm.reasoning.qwen3_reasoning_parser", + "Qwen3ReasoningParser", +) diff --git a/vllm/reasoning/abs_reasoning_parsers.py b/vllm/reasoning/abs_reasoning_parsers.py new file mode 100644 index 00000000..c6141074 --- /dev/null +++ b/vllm/reasoning/abs_reasoning_parsers.py @@ -0,0 +1,243 @@ +""" +Abstract reasoning parser base classes for vLLM 0.6.3. +Adapted from vllm-original/vllm/reasoning/abs_reasoning_parsers.py: + - Removed vllm.entrypoints.mcp, vllm.utils.collection_utils, import_utils + - DeltaMessage from vllm 0.6.3 protocol path + - TokenizerLike -> AnyTokenizer + - ReasoningParserManager: simplified eager + lazy registration +""" + +import importlib +from abc import abstractmethod +from collections.abc import Iterable, Sequence +from functools import cached_property +from typing import Any, Optional, TYPE_CHECKING + +if TYPE_CHECKING: + from vllm.entrypoints.openai.protocol import DeltaMessage + from vllm.transformers_utils.tokenizer import AnyTokenizer +else: + DeltaMessage = Any + AnyTokenizer = Any + + +class ReasoningParser: + """Abstract base for all reasoning parsers.""" + + def __init__(self, tokenizer: "AnyTokenizer", *args, **kwargs): + self.model_tokenizer = tokenizer + + @cached_property + def vocab(self) -> dict: + return self.model_tokenizer.get_vocab() + + @abstractmethod + def is_reasoning_end(self, input_ids: Sequence[int]) -> bool: + """Return True once the reasoning block has closed in input_ids.""" + + def is_reasoning_end_streaming( + self, input_ids: Sequence[int], delta_ids: Iterable[int] + ) -> bool: + return self.is_reasoning_end(input_ids) + + @abstractmethod + def extract_content_ids(self, input_ids: list) -> list: + """Return token ids that belong to the content (post-reasoning) part.""" + + def count_reasoning_tokens(self, token_ids: Sequence[int]) -> int: + return 0 + + @abstractmethod + def extract_reasoning( + self, model_output: str, request: Any + ) -> "tuple[Optional[str], Optional[str]]": + """ + Split a complete model output into (reasoning_text, content_text). + Either part may be None. + """ + + @abstractmethod + def extract_reasoning_streaming( + self, + previous_text: str, + current_text: str, + delta_text: str, + previous_token_ids: Sequence[int], + current_token_ids: Sequence[int], + delta_token_ids: Sequence[int], + ) -> Optional["DeltaMessage"]: + """ + Extract reasoning from a streaming delta. + Returns a DeltaMessage with reasoning_content and/or content set, + or None if this delta should be suppressed (control token). + """ + + +class BaseThinkingReasoningParser(ReasoningParser): + """ + Base for parsers that use ... delimiters. + Subclasses define start_token / end_token properties. + """ + + @property + @abstractmethod + def start_token(self) -> str: + raise NotImplementedError + + @property + @abstractmethod + def end_token(self) -> str: + raise NotImplementedError + + def __init__(self, tokenizer: "AnyTokenizer", *args, **kwargs): + super().__init__(tokenizer, *args, **kwargs) + + if not self.model_tokenizer: + raise ValueError("Tokenizer must be passed to ReasoningParser.") + if not self.start_token or not self.end_token: + raise ValueError("start_token and end_token must be defined.") + + self.start_token_id: Optional[int] = self.vocab.get(self.start_token) + self.end_token_id: Optional[int] = self.vocab.get(self.end_token) + if self.start_token_id is None or self.end_token_id is None: + raise RuntimeError( + f"{self.__class__.__name__}: could not find think tokens " + f"'{self.start_token}'/'{self.end_token}' in tokenizer vocab." + ) + + def is_reasoning_end(self, input_ids: Sequence[int]) -> bool: + for token_id in reversed(input_ids): + if token_id == self.start_token_id: + return False + if token_id == self.end_token_id: + return True + return False + + def is_reasoning_end_streaming( + self, input_ids: Sequence[int], delta_ids: Iterable[int] + ) -> bool: + return self.end_token_id in delta_ids + + def extract_content_ids(self, input_ids: list) -> list: + if self.end_token_id not in input_ids[:-1]: + return [] + return input_ids[input_ids.index(self.end_token_id) + 1:] + + def count_reasoning_tokens(self, token_ids: Sequence[int]) -> int: + count = 0 + depth = 0 + for tid in token_ids: + if tid == self.start_token_id: + depth += 1 + elif tid == self.end_token_id: + if depth > 0: + depth -= 1 + elif depth > 0: + count += 1 + return count + + def extract_reasoning( + self, model_output: str, request: Any + ) -> "tuple[Optional[str], Optional[str]]": + # Strip if the model generated it (old-style template). + parts = model_output.partition(self.start_token) + model_output = parts[2] if parts[1] else parts[0] + + if self.end_token not in model_output: + return model_output, None + reasoning, _, content = model_output.partition(self.end_token) + return reasoning, content or None + + def extract_reasoning_streaming( + self, + previous_text: str, + current_text: str, + delta_text: str, + previous_token_ids: Sequence[int], + current_token_ids: Sequence[int], + delta_token_ids: Sequence[int], + ) -> Optional["DeltaMessage"]: + from vllm.entrypoints.openai.protocol import DeltaMessage as _DeltaMessage + + # Suppress lone control tokens. + if len(delta_token_ids) == 1 and delta_token_ids[0] in ( + self.start_token_id, self.end_token_id + ): + return None + + start_in_prev = self.start_token_id in previous_token_ids + start_in_delta = self.start_token_id in delta_token_ids + end_in_prev = self.end_token_id in previous_token_ids + end_in_delta = self.end_token_id in delta_token_ids + + if start_in_prev: + if end_in_delta: + end_idx = delta_text.find(self.end_token) + reasoning = delta_text[:end_idx] if end_idx >= 0 else "" + content = delta_text[end_idx + len(self.end_token):] if end_idx >= 0 else None + return _DeltaMessage( + reasoning_content=reasoning or None, + content=content or None, + ) + elif end_in_prev: + return _DeltaMessage(content=delta_text) + else: + return _DeltaMessage(reasoning_content=delta_text) + + elif start_in_delta: + if end_in_delta: + start_idx = delta_text.find(self.start_token) + end_idx = delta_text.find(self.end_token) + reasoning = delta_text[start_idx + len(self.start_token):end_idx] + content = delta_text[end_idx + len(self.end_token):] + return _DeltaMessage( + reasoning_content=reasoning or None, + content=content or None, + ) + else: + return _DeltaMessage(reasoning_content=delta_text) + + else: + return _DeltaMessage(content=delta_text) + + +class ReasoningParserManager: + """ + Registry for ReasoningParser implementations. + Supports eager and lazy registration. + """ + + _parsers: dict = {} # name -> class (eager) + _lazy: dict = {} # name -> (module_path, class_name) + + @classmethod + def register_module(cls, name: str, parser_cls: type) -> None: + """Eagerly register a ReasoningParser class.""" + if not issubclass(parser_cls, ReasoningParser): + raise TypeError(f"{parser_cls} is not a ReasoningParser subclass.") + cls._parsers[name] = parser_cls + + @classmethod + def register_lazy(cls, name: str, module_path: str, class_name: str) -> None: + """Register a parser for deferred import.""" + cls._lazy[name] = (module_path, class_name) + + @classmethod + def get_reasoning_parser(cls, name: str) -> type: + if name in cls._parsers: + return cls._parsers[name] + if name in cls._lazy: + module_path, class_name = cls._lazy[name] + mod = importlib.import_module(module_path) + parser_cls = getattr(mod, class_name) + cls._parsers[name] = parser_cls + return parser_cls + registered = sorted(set(cls._parsers) | set(cls._lazy)) + raise KeyError( + f"Reasoning parser '{name}' not found. " + f"Available: {registered}" + ) + + @classmethod + def list_registered(cls) -> list: + return sorted(set(cls._parsers) | set(cls._lazy)) diff --git a/vllm/reasoning/qwen3_reasoning_parser.py b/vllm/reasoning/qwen3_reasoning_parser.py new file mode 100644 index 00000000..1febd4b2 --- /dev/null +++ b/vllm/reasoning/qwen3_reasoning_parser.py @@ -0,0 +1,110 @@ +""" +Reasoning parser for Qwen3 / Qwen3.5 / Qwen3.6 model family. +Adapted from vllm-original/vllm/reasoning/qwen3_reasoning_parser.py. + +The model uses ... to wrap chain-of-thought output. +For Qwen3.5+ the chat template injects into the prompt, so only + appears in the generated tokens; older templates generate +themselves. Both styles are handled. +""" + +from typing import Optional, Sequence, Any + +from vllm.reasoning.abs_reasoning_parsers import ( + BaseThinkingReasoningParser, + ReasoningParserManager, +) + + +class Qwen3ReasoningParser(BaseThinkingReasoningParser): + + def __init__(self, tokenizer: Any, *args, **kwargs): + super().__init__(tokenizer, *args, **kwargs) + chat_kwargs = kwargs.get("chat_template_kwargs", {}) or {} + self.thinking_enabled = chat_kwargs.get("enable_thinking", True) + + @property + def start_token(self) -> str: + return "" + + @property + def end_token(self) -> str: + return "" + + def extract_reasoning( + self, model_output: str, request: Any + ) -> "tuple[Optional[str], Optional[str]]": + # Strip if the model generated it (old template / edge case). + parts = model_output.partition(self.start_token) + model_output = parts[2] if parts[1] else parts[0] + + if self.end_token not in model_output: + if not self.thinking_enabled: + return None, model_output + # Thinking enabled but output truncated before . + # All output is reasoning; content is None. + return model_output, None + + reasoning, _, content = model_output.partition(self.end_token) + content = content.strip() if content else "" + return reasoning or None, content if content else None + + def count_reasoning_tokens(self, token_ids: Sequence[int]) -> int: + token_ids = list(token_ids) + if self.start_token_id in token_ids: + # Old-style template: model generates itself. + # Use depth-counting from the base class. + return super().count_reasoning_tokens(token_ids) + elif self.end_token_id in token_ids: + # New-style template (Qwen3.5+): is injected into the + # prompt, so output starts already inside the thinking block. + # Every token before is a reasoning token. + return token_ids.index(self.end_token_id) + else: + # No in output: either truncated (all reasoning) + # or thinking disabled (none). + return len(token_ids) if self.thinking_enabled else 0 + + def extract_reasoning_streaming( + self, + previous_text: str, + current_text: str, + delta_text: str, + previous_token_ids: Sequence[int], + current_token_ids: Sequence[int], + delta_token_ids: Sequence[int], + ): + from vllm.entrypoints.openai.protocol import DeltaMessage + + if not self.thinking_enabled: + return DeltaMessage(content=delta_text) if delta_text else None + + # Strip from delta if the model generates it itself. + if self.start_token_id in delta_token_ids: + start_idx = delta_text.find(self.start_token) + if start_idx >= 0: + delta_text = delta_text[start_idx + len(self.start_token):] + + if self.end_token_id in delta_token_ids: + end_idx = delta_text.find(self.end_token) + if end_idx >= 0: + reasoning = delta_text[:end_idx] + content = delta_text[end_idx + len(self.end_token):] + if not reasoning and not content: + return None + return DeltaMessage( + reasoning_content=reasoning or None, + content=content or None, + ) + return None + + if not delta_text: + return None + elif self.end_token_id in previous_token_ids: + return DeltaMessage(content=delta_text) + else: + return DeltaMessage(reasoning_content=delta_text) + + +# Register immediately when this module is imported. +ReasoningParserManager.register_module("qwen3", Qwen3ReasoningParser)