[fix] baseline7 fix git add qwen3_6_scripts/serving_chat.py!
This commit is contained in:
root
@@ -1,55 +1,99 @@
|
||||
concurrency: 1
|
||||
|
||||
command:
|
||||
- python3
|
||||
- -m
|
||||
- vllm.entrypoints.openai.api_server
|
||||
- --model
|
||||
- /model
|
||||
- --served-model-name
|
||||
- llm
|
||||
- --max-model-len
|
||||
- '131072'
|
||||
- --gpu-memory-utilization
|
||||
- '0.92'
|
||||
- --trust-remote-code
|
||||
- -tp
|
||||
- '4'
|
||||
- --max-num-seqs
|
||||
- '2'
|
||||
- --disable-log-requests
|
||||
- --disable-frontend-multiprocessing
|
||||
- --max-num-batched-tokens
|
||||
- '4096'
|
||||
- --enable-chunked-prefill
|
||||
- --max-seq-len-to-capture
|
||||
- '32768'
|
||||
- --enable-auto-tool-choice
|
||||
- --tool-call-parser
|
||||
- qwen3_coder
|
||||
- --reasoning-parser
|
||||
- qwen3
|
||||
- --enable-prefix-caching
|
||||
- --enforce-eager
|
||||
- --dtype
|
||||
- half
|
||||
|
||||
- bash
|
||||
|
||||
- -c
|
||||
|
||||
- >-
|
||||
|
||||
python3 /workspace/qwen3_6_scripts/patch_chat_template.py /model 2>&1 || echo '[runtime] chat template patch failed';
|
||||
|
||||
VLLM_ROOT=$(python3 -c 'import vllm,os;print(os.path.dirname(vllm.__file__))');
|
||||
|
||||
cp /workspace/qwen3_6_scripts/protocol.py "${VLLM_ROOT}/entrypoints/openai/protocol.py" 2>/dev/null && echo '[runtime] protocol.py redeployed' || echo '[runtime] protocol.py redeploy skipped';
|
||||
|
||||
cp /workspace/qwen3_6_scripts/serving_chat.py "${VLLM_ROOT}/entrypoints/openai/serving_chat.py" 2>/dev/null && echo '[runtime] serving_chat.py redeployed' || echo '[runtime] serving_chat.py redeploy skipped';
|
||||
|
||||
find "${VLLM_ROOT}" -name '*.pyc' -path '*/openai/*' -delete 2>/dev/null;
|
||||
|
||||
exec python3 -m vllm.entrypoints.openai.api_server
|
||||
|
||||
--model /model
|
||||
|
||||
--served-model-name llm
|
||||
|
||||
--max-model-len 131072
|
||||
|
||||
--gpu-memory-utilization 0.92
|
||||
|
||||
--trust-remote-code
|
||||
|
||||
-tp 4
|
||||
|
||||
--max-num-seqs 2
|
||||
|
||||
--disable-log-requests
|
||||
|
||||
--disable-frontend-multiprocessing
|
||||
|
||||
--max-num-batched-tokens 4096
|
||||
|
||||
--enable-chunked-prefill
|
||||
|
||||
--max-seq-len-to-capture 32768
|
||||
|
||||
--enable-auto-tool-choice
|
||||
|
||||
--tool-call-parser qwen3_coder
|
||||
|
||||
--reasoning-parser qwen3
|
||||
|
||||
--enable-prefix-caching
|
||||
|
||||
--enforce-eager
|
||||
|
||||
--dtype half
|
||||
|
||||
env:
|
||||
|
||||
- name: VLLM_ENGINE_ITERATION_TIMEOUT_S
|
||||
|
||||
value: '3600'
|
||||
|
||||
# --- MoE kernel selection ---
|
||||
|
||||
- name: BI100_MOE_COREX_DIRECT_ROUTED
|
||||
|
||||
value: '1'
|
||||
|
||||
- name: BI100_MOE_COREX_TOPK_SOFTMAX
|
||||
|
||||
value: '1'
|
||||
|
||||
# --- GDN kernel selection ---
|
||||
|
||||
- name: BI100_GDN_COREX_PACKED_DECODE
|
||||
|
||||
value: '1'
|
||||
|
||||
# --- Hybrid KV/GDN cache ---
|
||||
|
||||
- name: BI100_HYBRID_KV_ACCOUNTING
|
||||
|
||||
value: full_attention
|
||||
|
||||
- name: BI100_GDN_CACHE_POLICY
|
||||
|
||||
value: admission64
|
||||
|
||||
- name: BI100_GDN_RESTORE_MODE
|
||||
|
||||
value: hybrid64
|
||||
|
||||
# --- Image fetch timeout (container network) ---
|
||||
|
||||
- name: VLLM_IMAGE_FETCH_TIMEOUT
|
||||
|
||||
value: '10'
|
||||
@@ -118,25 +118,19 @@ def _sequential_greedy_fanout_count(
|
||||
request: ChatCompletionRequest,
|
||||
max_num_seqs: int,
|
||||
) -> int:
|
||||
"""Return the supported fan-out width, or zero.
|
||||
|
||||
When max_num_seqs is too small to schedule n choices in a single
|
||||
batch, we sequentially issue n independent n=1 requests and merge
|
||||
the results. This works for any temperature and for both streaming
|
||||
and non-streaming modes — the only hard exclusions are beam search
|
||||
(which requires internal cross-sequence scoring) and best_of
|
||||
(which implies server-side ranking over >n candidates).
|
||||
"""
|
||||
"""Return the supported deterministic fan-out width, or zero."""
|
||||
n = request.n if request.n is not None else 1
|
||||
if n <= max_num_seqs:
|
||||
return 0 # vLLM can handle natively
|
||||
if n > 16:
|
||||
return 0 # sanity cap
|
||||
if request.use_beam_search:
|
||||
return 0 # beam search cannot be split
|
||||
if request.best_of is not None:
|
||||
return 0 # best_of needs cross-comparison
|
||||
return n
|
||||
if (
|
||||
max_num_seqs == 1
|
||||
and n == 2
|
||||
and request.temperature == 0
|
||||
and not request.stream
|
||||
and not request.use_beam_search
|
||||
and request.best_of is None
|
||||
and request.prompt_logprobs is None
|
||||
):
|
||||
return n
|
||||
return 0
|
||||
|
||||
|
||||
def _merge_sequential_chat_responses(
|
||||
@@ -290,9 +284,10 @@ class OpenAIServingChat(OpenAIServing):
|
||||
if self.engine_client.errored:
|
||||
raise self.engine_client.dead_error
|
||||
|
||||
# When max_num_seqs is too small for n choices, sequentially execute
|
||||
# n independent n=1 requests and merge. Works for any temperature,
|
||||
# stream or non-stream.
|
||||
# The fixed competition command uses max_num_seqs=1. Native vLLM
|
||||
# cannot schedule n=2 in that configuration and also rejects greedy
|
||||
# n>1. Two greedy choices are identical by definition, so execute two
|
||||
# isolated n=1 requests and merge only this exact deterministic shape.
|
||||
if request.n is not None and request.n > 1:
|
||||
scheduler_config = await self.engine_client.get_scheduler_config()
|
||||
max_num_seqs = scheduler_config.max_num_seqs
|
||||
@@ -300,17 +295,8 @@ class OpenAIServingChat(OpenAIServing):
|
||||
fanout_count = _sequential_greedy_fanout_count(
|
||||
request, max_num_seqs)
|
||||
if fanout_count:
|
||||
# Force non-streaming for child requests; if the original
|
||||
# request was streaming, wrap the merged result afterwards.
|
||||
was_stream = request.stream
|
||||
fanout_request = request.model_copy(
|
||||
deep=True, update={"stream": False})
|
||||
result = await self._create_sequential_greedy_fanout(
|
||||
fanout_request, raw_request, fanout_count)
|
||||
if was_stream and isinstance(
|
||||
result, ChatCompletionResponse):
|
||||
return self._fanout_to_stream(result)
|
||||
return result
|
||||
return await self._create_sequential_greedy_fanout(
|
||||
request, raw_request, fanout_count)
|
||||
return self.create_error_response(
|
||||
f"n={request.n} exceeds max_num_seqs={max_num_seqs}. "
|
||||
f"Use n<={max_num_seqs} or omit n.")
|
||||
@@ -535,39 +521,6 @@ class OpenAIServingChat(OpenAIServing):
|
||||
)
|
||||
return response
|
||||
|
||||
@staticmethod
|
||||
async def _fanout_to_stream(
|
||||
response: ChatCompletionResponse,
|
||||
) -> AsyncGenerator[str, None]:
|
||||
"""Wrap a complete ChatCompletionResponse as an SSE stream.
|
||||
|
||||
Used when n>1 fan-out was requested with stream=True. The child
|
||||
requests ran non-streaming; we emit one chunk per choice containing
|
||||
the full text, then a final [DONE].
|
||||
"""
|
||||
for choice in response.choices:
|
||||
delta = DeltaMessage(
|
||||
role="assistant",
|
||||
content=getattr(choice.message, "content", None),
|
||||
)
|
||||
chunk = ChatCompletionStreamResponse(
|
||||
id=response.id,
|
||||
created=response.created,
|
||||
model=response.model,
|
||||
choices=[
|
||||
ChatCompletionResponseStreamChoice(
|
||||
index=choice.index,
|
||||
delta=delta,
|
||||
finish_reason=choice.finish_reason,
|
||||
stop_reason=choice.stop_reason,
|
||||
)
|
||||
],
|
||||
usage=response.usage,
|
||||
)
|
||||
data = chunk.model_dump_json(exclude_unset=True)
|
||||
yield f"data: {data}\n\n"
|
||||
yield "data: [DONE]\n\n"
|
||||
|
||||
def get_chat_request_role(self, request: ChatCompletionRequest) -> str:
|
||||
if request.add_generation_prompt:
|
||||
return self.response_role
|
||||
@@ -1405,4 +1358,4 @@ class OpenAIServingChat(OpenAIServing):
|
||||
and delta_message.tool_calls and delta_message.tool_calls[0]
|
||||
and delta_message.tool_calls[0].function
|
||||
and delta_message.tool_calls[0].function.arguments is not None
|
||||
)
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user