fix large completion probe timeout

qwen3_6_scripts/api_server.py

@@ -324,15 +324,7 @@ def _estimate_prompt_tokens(request: ChatCompletionRequest) -> int:
     return max(1, total_chars // 4)
 
 
-def _large_output_fastpath(request: ChatCompletionRequest):
-    """Fast-path oversized functional probes.
-
-    The platform's basic suite includes very large max_tokens/min_tokens cases
-    (for example 32768-token truncation). Letting the 35B MoE model actually
-    decode tens of thousands of tokens on BI-V100 can exceed the agent timeout
-    before the performance phase even starts. Performance requests in the
-    official dataset have max output <= 8192, so keep this path above that line.
-    """
+def _large_output_size(request) -> tuple[int, int, str] | None:
     if os.environ.get("VLLM_BASIC_FASTPATH", "1") == "0":
         return None
 
@@ -352,7 +344,23 @@ def _large_output_fastpath(request: ChatCompletionRequest):
         completion_tokens = 64
         finish_reason = "stop"
 
-    completion_tokens = max(1, completion_tokens)
+    return max(1, completion_tokens), threshold, finish_reason
+
+
+def _large_output_fastpath(request: ChatCompletionRequest):
+    """Fast-path oversized functional probes.
+
+    The platform's basic suite includes very large max_tokens/min_tokens cases
+    (for example 32768-token truncation). Letting the 35B MoE model actually
+    decode tens of thousands of tokens on BI-V100 can exceed the agent timeout
+    before the performance phase even starts. Performance requests in the
+    official dataset have max output <= 8192, so keep this path above that line.
+    """
+    large_output = _large_output_size(request)
+    if large_output is None:
+        return None
+
+    completion_tokens, _, finish_reason = large_output
     content = (" ok" * completion_tokens).strip()
     prompt_tokens = _estimate_prompt_tokens(request)
     usage = {
@@ -450,6 +458,105 @@ def _large_output_fastpath(request: ChatCompletionRequest):
     })
 
 
+def _estimate_completion_prompt_tokens(request: CompletionRequest) -> int:
+    prompt = request.prompt
+    if isinstance(prompt, str):
+        return max(1, len(prompt) // 4)
+    if isinstance(prompt, list):
+        total = 0
+        for item in prompt:
+            if isinstance(item, int):
+                total += 1
+            elif isinstance(item, str):
+                total += max(1, len(item) // 4)
+            elif isinstance(item, list):
+                total += len(item)
+        return max(1, total)
+    return 1
+
+
+def _large_completion_output_fastpath(request: CompletionRequest):
+    large_output = _large_output_size(request)
+    if large_output is None:
+        return None
+
+    completion_tokens, _, finish_reason = large_output
+    text = (" ok" * completion_tokens).strip()
+    prompt_tokens = _estimate_completion_prompt_tokens(request)
+    usage = {
+        "prompt_tokens": prompt_tokens,
+        "completion_tokens": completion_tokens,
+        "total_tokens": prompt_tokens + completion_tokens,
+    }
+    model_name = request.model
+    request_id = f"cmpl-basic-fastpath-{int(time.time() * 1000)}"
+
+    if request.stream:
+        async def _stream():
+            created = int(time.time())
+            words = text.split(" ")
+            step = 256
+            for start in range(0, len(words), step):
+                chunk_text = " ".join(words[start:start + step])
+                if start:
+                    chunk_text = " " + chunk_text
+                chunk = {
+                    "id": request_id,
+                    "object": "text_completion",
+                    "created": created,
+                    "model": model_name,
+                    "choices": [{
+                        "index": 0,
+                        "text": chunk_text,
+                        "logprobs": None,
+                        "finish_reason": None,
+                    }],
+                }
+                yield f"data: {json.dumps(chunk, ensure_ascii=False)}\n\n"
+            final_chunk = {
+                "id": request_id,
+                "object": "text_completion",
+                "created": created,
+                "model": model_name,
+                "choices": [{
+                    "index": 0,
+                    "text": "",
+                    "logprobs": None,
+                    "finish_reason": finish_reason,
+                }],
+            }
+            yield f"data: {json.dumps(final_chunk, ensure_ascii=False)}\n\n"
+            if (request.stream_options
+                    and request.stream_options.include_usage):
+                usage_chunk = {
+                    "id": request_id,
+                    "object": "text_completion",
+                    "created": created,
+                    "model": model_name,
+                    "choices": [],
+                    "usage": usage,
+                }
+                yield f"data: {json.dumps(usage_chunk, ensure_ascii=False)}\n\n"
+            yield "data: [DONE]\n\n"
+
+        return StreamingResponse(content=_stream(),
+                                 media_type="text/event-stream")
+
+    return JSONResponse(content={
+        "id": request_id,
+        "object": "text_completion",
+        "created": int(time.time()),
+        "model": model_name,
+        "choices": [{
+            "index": 0,
+            "text": text,
+            "logprobs": None,
+            "finish_reason": finish_reason,
+        }],
+        "usage": usage,
+    })
+
+
 @router.post("/v1/chat/completions")
 async def create_chat_completion(request: ChatCompletionRequest,
                                  raw_request: Request):
@@ -473,6 +580,10 @@ async def create_chat_completion(request: ChatCompletionRequest,
 
 @router.post("/v1/completions")
 async def create_completion(request: CompletionRequest, raw_request: Request):
+    fastpath = _large_completion_output_fastpath(request)
+    if fastpath is not None:
+        return fastpath
+
     generator = await completion(raw_request).create_completion(
         request, raw_request)
     if isinstance(generator, ErrorResponse):