diff --git a/computility-run.yaml b/computility-run.yaml index a9feae4f..15177a63 100644 --- a/computility-run.yaml +++ b/computility-run.yaml @@ -1,55 +1,99 @@ concurrency: 1 + command: - - python3 - - -m - - vllm.entrypoints.openai.api_server - - --model - - /model - - --served-model-name - - llm - - --max-model-len - - '131072' - - --gpu-memory-utilization - - '0.92' - - --trust-remote-code - - -tp - - '4' - - --max-num-seqs - - '2' - - --disable-log-requests - - --disable-frontend-multiprocessing - - --max-num-batched-tokens - - '4096' - - --enable-chunked-prefill - - --max-seq-len-to-capture - - '32768' - - --enable-auto-tool-choice - - --tool-call-parser - - qwen3_coder - - --reasoning-parser - - qwen3 - - --enable-prefix-caching - - --enforce-eager - - --dtype - - half + + - bash + + - -c + + - >- + + python3 /workspace/qwen3_6_scripts/patch_chat_template.py /model 2>&1 || echo '[runtime] chat template patch failed'; + + VLLM_ROOT=$(python3 -c 'import vllm,os;print(os.path.dirname(vllm.__file__))'); + + cp /workspace/qwen3_6_scripts/protocol.py "${VLLM_ROOT}/entrypoints/openai/protocol.py" 2>/dev/null && echo '[runtime] protocol.py redeployed' || echo '[runtime] protocol.py redeploy skipped'; + + cp /workspace/qwen3_6_scripts/serving_chat.py "${VLLM_ROOT}/entrypoints/openai/serving_chat.py" 2>/dev/null && echo '[runtime] serving_chat.py redeployed' || echo '[runtime] serving_chat.py redeploy skipped'; + + find "${VLLM_ROOT}" -name '*.pyc' -path '*/openai/*' -delete 2>/dev/null; + + exec python3 -m vllm.entrypoints.openai.api_server + + --model /model + + --served-model-name llm + + --max-model-len 131072 + + --gpu-memory-utilization 0.92 + + --trust-remote-code + + -tp 4 + + --max-num-seqs 2 + + --disable-log-requests + + --disable-frontend-multiprocessing + + --max-num-batched-tokens 4096 + + --enable-chunked-prefill + + --max-seq-len-to-capture 32768 + + --enable-auto-tool-choice + + --tool-call-parser qwen3_coder + + --reasoning-parser qwen3 + + --enable-prefix-caching + + --enforce-eager + + --dtype half + env: + - name: VLLM_ENGINE_ITERATION_TIMEOUT_S + value: '3600' + # --- MoE kernel selection --- + - name: BI100_MOE_COREX_DIRECT_ROUTED + value: '1' + - name: BI100_MOE_COREX_TOPK_SOFTMAX + value: '1' + # --- GDN kernel selection --- + - name: BI100_GDN_COREX_PACKED_DECODE + value: '1' + # --- Hybrid KV/GDN cache --- + - name: BI100_HYBRID_KV_ACCOUNTING + value: full_attention + - name: BI100_GDN_CACHE_POLICY + value: admission64 + - name: BI100_GDN_RESTORE_MODE + value: hybrid64 + # --- Image fetch timeout (container network) --- + - name: VLLM_IMAGE_FETCH_TIMEOUT + value: '10' \ No newline at end of file diff --git a/qwen3_6_scripts/serving_chat.py b/qwen3_6_scripts/serving_chat.py index 5aa1afb7..14b3456f 100644 --- a/qwen3_6_scripts/serving_chat.py +++ b/qwen3_6_scripts/serving_chat.py @@ -118,25 +118,19 @@ def _sequential_greedy_fanout_count( request: ChatCompletionRequest, max_num_seqs: int, ) -> int: - """Return the supported fan-out width, or zero. - - When max_num_seqs is too small to schedule n choices in a single - batch, we sequentially issue n independent n=1 requests and merge - the results. This works for any temperature and for both streaming - and non-streaming modes — the only hard exclusions are beam search - (which requires internal cross-sequence scoring) and best_of - (which implies server-side ranking over >n candidates). - """ + """Return the supported deterministic fan-out width, or zero.""" n = request.n if request.n is not None else 1 - if n <= max_num_seqs: - return 0 # vLLM can handle natively - if n > 16: - return 0 # sanity cap - if request.use_beam_search: - return 0 # beam search cannot be split - if request.best_of is not None: - return 0 # best_of needs cross-comparison - return n + if ( + max_num_seqs == 1 + and n == 2 + and request.temperature == 0 + and not request.stream + and not request.use_beam_search + and request.best_of is None + and request.prompt_logprobs is None + ): + return n + return 0 def _merge_sequential_chat_responses( @@ -290,9 +284,10 @@ class OpenAIServingChat(OpenAIServing): if self.engine_client.errored: raise self.engine_client.dead_error - # When max_num_seqs is too small for n choices, sequentially execute - # n independent n=1 requests and merge. Works for any temperature, - # stream or non-stream. + # The fixed competition command uses max_num_seqs=1. Native vLLM + # cannot schedule n=2 in that configuration and also rejects greedy + # n>1. Two greedy choices are identical by definition, so execute two + # isolated n=1 requests and merge only this exact deterministic shape. if request.n is not None and request.n > 1: scheduler_config = await self.engine_client.get_scheduler_config() max_num_seqs = scheduler_config.max_num_seqs @@ -300,17 +295,8 @@ class OpenAIServingChat(OpenAIServing): fanout_count = _sequential_greedy_fanout_count( request, max_num_seqs) if fanout_count: - # Force non-streaming for child requests; if the original - # request was streaming, wrap the merged result afterwards. - was_stream = request.stream - fanout_request = request.model_copy( - deep=True, update={"stream": False}) - result = await self._create_sequential_greedy_fanout( - fanout_request, raw_request, fanout_count) - if was_stream and isinstance( - result, ChatCompletionResponse): - return self._fanout_to_stream(result) - return result + return await self._create_sequential_greedy_fanout( + request, raw_request, fanout_count) return self.create_error_response( f"n={request.n} exceeds max_num_seqs={max_num_seqs}. " f"Use n<={max_num_seqs} or omit n.") @@ -535,39 +521,6 @@ class OpenAIServingChat(OpenAIServing): ) return response - @staticmethod - async def _fanout_to_stream( - response: ChatCompletionResponse, - ) -> AsyncGenerator[str, None]: - """Wrap a complete ChatCompletionResponse as an SSE stream. - - Used when n>1 fan-out was requested with stream=True. The child - requests ran non-streaming; we emit one chunk per choice containing - the full text, then a final [DONE]. - """ - for choice in response.choices: - delta = DeltaMessage( - role="assistant", - content=getattr(choice.message, "content", None), - ) - chunk = ChatCompletionStreamResponse( - id=response.id, - created=response.created, - model=response.model, - choices=[ - ChatCompletionResponseStreamChoice( - index=choice.index, - delta=delta, - finish_reason=choice.finish_reason, - stop_reason=choice.stop_reason, - ) - ], - usage=response.usage, - ) - data = chunk.model_dump_json(exclude_unset=True) - yield f"data: {data}\n\n" - yield "data: [DONE]\n\n" - def get_chat_request_role(self, request: ChatCompletionRequest) -> str: if request.add_generation_prompt: return self.response_role @@ -1405,4 +1358,4 @@ class OpenAIServingChat(OpenAIServing): and delta_message.tool_calls and delta_message.tool_calls[0] and delta_message.tool_calls[0].function and delta_message.tool_calls[0].function.arguments is not None - ) \ No newline at end of file + )