dylanyunlong/project_6_89d52222
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

add dp 2

Browse Source
This commit is contained in:
root
2026-09-02 09:39:03 +00:00
parent 84f6b833c0
commit 3a50c67884
2 changed files with 16 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
qwen3_6_scripts/serving_chat.py
View File

@@ -243,6 +243,7 @@ class OpenAIServingChat(OpenAIServing):
# set up reasoning parser # set up reasoning parser
self.reasoning_parser_cls = None self.reasoning_parser_cls = None
self._template_patched = False # [BI100] one-shot chat template patch
if reasoning_parser: if reasoning_parser:
try: try:
from vllm.reasoning import ReasoningParserManager from vllm.reasoning import ReasoningParserManager
@@ -309,6 +310,19 @@ class OpenAIServingChat(OpenAIServing):
model_config = self.model_config model_config = self.model_config
tokenizer = await self.engine_client.get_tokenizer(lora_request) tokenizer = await self.engine_client.get_tokenizer(lora_request)
# [BI100] Runtime equivalent of patch_chat_template.py.
# Qwen3.5/3.6-MoE templates inject '<think>\n\n</think>\n\n'
# when enable_thinking=false, which causes the model to
# degenerate (outputs '!!!!!'). Patch it out once.
if not self._template_patched and hasattr(tokenizer, 'chat_template'):
_tgt = r"{{- '<think>\n\n</think>\n\n' }}"
if isinstance(tokenizer.chat_template, str) and _tgt in tokenizer.chat_template:
tokenizer.chat_template = tokenizer.chat_template.replace(
_tgt, "{{- '' }}", 1)
logger.info("[BI100] Patched chat_template: removed empty "
"<think></think> block for non-thinking mode")
self._template_patched = True
conversation, mm_data_future = parse_chat_messages_futures( conversation, mm_data_future = parse_chat_messages_futures(
request.messages, model_config, tokenizer) request.messages, model_config, tokenizer)

9
qwen3_6_scripts/vendor_overrides/vllm/engine/arg_utils.py
View File

@@ -1,6 +1,7 @@
import argparse import argparse
import dataclasses import dataclasses
import json import json
import os
from dataclasses import dataclass from dataclasses import dataclass
from typing import (TYPE_CHECKING, Any, Dict, List, Literal, Mapping, Optional, from typing import (TYPE_CHECKING, Any, Dict, List, Literal, Mapping, Optional,
Tuple, Type, Union) Tuple, Type, Union)
@@ -105,7 +106,7 @@ class EngineArgs:
Type[ExecutorBase]]] = None Type[ExecutorBase]]] = None
pipeline_parallel_size: int = 1 pipeline_parallel_size: int = 1
tensor_parallel_size: int = 1 tensor_parallel_size: int = 1
data_parallel_size: int = 1 data_parallel_size: int = int(os.environ.get("VLLM_DATA_PARALLEL_SIZE", "1"))
max_parallel_loading_workers: Optional[int] = None max_parallel_loading_workers: Optional[int] = None
block_size: int = 16 block_size: int = 16
enable_prefix_caching: bool = False enable_prefix_caching: bool = False
@@ -349,12 +350,6 @@ class EngineArgs:
type=int, type=int,
default=EngineArgs.tensor_parallel_size, default=EngineArgs.tensor_parallel_size,
help='Number of tensor parallel replicas.') help='Number of tensor parallel replicas.')
parser.add_argument('--data-parallel-size',
'-dp',
type=int,
default=EngineArgs.data_parallel_size,
help='Number of data parallel replicas. '
'Total GPUs = tp * pp * dp.')
parser.add_argument( parser.add_argument(
'--max-parallel-loading-workers', '--max-parallel-loading-workers',
type=int, type=int,