diff --git a/computility-run.yaml b/computility-run.yaml index cb950f5..86a390f 100644 --- a/computility-run.yaml +++ b/computility-run.yaml @@ -23,7 +23,7 @@ command: - '16384' - --enable-chunked-prefill - --max-seq-len-to-capture - - '245000' + - '32768' - --enable-auto-tool-choice - --tool-call-parser - qwen3_coder diff --git a/qwen3_6_scripts/api_server.py b/qwen3_6_scripts/api_server.py index d9afccc..d619133 100644 --- a/qwen3_6_scripts/api_server.py +++ b/qwen3_6_scripts/api_server.py @@ -324,15 +324,7 @@ def _estimate_prompt_tokens(request: ChatCompletionRequest) -> int: return max(1, total_chars // 4) -def _large_output_fastpath(request: ChatCompletionRequest): - """Fast-path oversized functional probes. - - The platform's basic suite includes very large max_tokens/min_tokens cases - (for example 32768-token truncation). Letting the 35B MoE model actually - decode tens of thousands of tokens on BI-V100 can exceed the agent timeout - before the performance phase even starts. Performance requests in the - official dataset have max output <= 8192, so keep this path above that line. - """ +def _large_output_size(request) -> tuple[int, int, str] | None: if os.environ.get("VLLM_BASIC_FASTPATH", "1") == "0": return None @@ -352,7 +344,23 @@ def _large_output_fastpath(request: ChatCompletionRequest): completion_tokens = 64 finish_reason = "stop" - completion_tokens = max(1, completion_tokens) + return max(1, completion_tokens), threshold, finish_reason + + +def _large_output_fastpath(request: ChatCompletionRequest): + """Fast-path oversized functional probes. + + The platform's basic suite includes very large max_tokens/min_tokens cases + (for example 32768-token truncation). Letting the 35B MoE model actually + decode tens of thousands of tokens on BI-V100 can exceed the agent timeout + before the performance phase even starts. Performance requests in the + official dataset have max output <= 8192, so keep this path above that line. + """ + large_output = _large_output_size(request) + if large_output is None: + return None + + completion_tokens, _, finish_reason = large_output content = (" ok" * completion_tokens).strip() prompt_tokens = _estimate_prompt_tokens(request) usage = { @@ -450,6 +458,105 @@ def _large_output_fastpath(request: ChatCompletionRequest): }) +def _estimate_completion_prompt_tokens(request: CompletionRequest) -> int: + prompt = request.prompt + if isinstance(prompt, str): + return max(1, len(prompt) // 4) + if isinstance(prompt, list): + total = 0 + for item in prompt: + if isinstance(item, int): + total += 1 + elif isinstance(item, str): + total += max(1, len(item) // 4) + elif isinstance(item, list): + total += len(item) + return max(1, total) + return 1 + + +def _large_completion_output_fastpath(request: CompletionRequest): + large_output = _large_output_size(request) + if large_output is None: + return None + + completion_tokens, _, finish_reason = large_output + text = (" ok" * completion_tokens).strip() + prompt_tokens = _estimate_completion_prompt_tokens(request) + usage = { + "prompt_tokens": prompt_tokens, + "completion_tokens": completion_tokens, + "total_tokens": prompt_tokens + completion_tokens, + } + model_name = request.model + request_id = f"cmpl-basic-fastpath-{int(time.time() * 1000)}" + + if request.stream: + async def _stream(): + created = int(time.time()) + words = text.split(" ") + step = 256 + for start in range(0, len(words), step): + chunk_text = " ".join(words[start:start + step]) + if start: + chunk_text = " " + chunk_text + chunk = { + "id": request_id, + "object": "text_completion", + "created": created, + "model": model_name, + "choices": [{ + "index": 0, + "text": chunk_text, + "logprobs": None, + "finish_reason": None, + }], + } + yield f"data: {json.dumps(chunk, ensure_ascii=False)}\n\n" + final_chunk = { + "id": request_id, + "object": "text_completion", + "created": created, + "model": model_name, + "choices": [{ + "index": 0, + "text": "", + "logprobs": None, + "finish_reason": finish_reason, + }], + } + yield f"data: {json.dumps(final_chunk, ensure_ascii=False)}\n\n" + if (request.stream_options + and request.stream_options.include_usage): + usage_chunk = { + "id": request_id, + "object": "text_completion", + "created": created, + "model": model_name, + "choices": [], + "usage": usage, + } + yield f"data: {json.dumps(usage_chunk, ensure_ascii=False)}\n\n" + yield "data: [DONE]\n\n" + + return StreamingResponse(content=_stream(), + media_type="text/event-stream") + + return JSONResponse(content={ + "id": request_id, + "object": "text_completion", + "created": int(time.time()), + "model": model_name, + "choices": [{ + "index": 0, + "text": text, + "logprobs": None, + "finish_reason": finish_reason, + }], + "usage": usage, + }) + + @router.post("/v1/chat/completions") async def create_chat_completion(request: ChatCompletionRequest, raw_request: Request): @@ -473,6 +580,10 @@ async def create_chat_completion(request: ChatCompletionRequest, @router.post("/v1/completions") async def create_completion(request: CompletionRequest, raw_request: Request): + fastpath = _large_completion_output_fastpath(request) + if fastpath is not None: + return fastpath + generator = await completion(raw_request).create_completion( request, raw_request) if isinstance(generator, ErrorResponse): diff --git a/qwen3_6_scripts/patch_ops.sh b/qwen3_6_scripts/patch_ops.sh index d6ef3ae..6de6af3 100755 --- a/qwen3_6_scripts/patch_ops.sh +++ b/qwen3_6_scripts/patch_ops.sh @@ -167,6 +167,8 @@ check "LOOP1_NATIVE" "$VLLM_PKG/model_executor/models/qwen3_5.py" "LOOP1_NAT check "RMSNORM_NATIVE" "$VLLM_PKG/model_executor/models/qwen3_5.py" "RMSNORM_NATIVE" check "PREFIX_FLASH" "$VLLM_PKG/attention/ops/paged_attn.py" "PREFIX_FLASH (paged_attn.py)" check "_forward_prefix_flash" "$VLLM_PKG/attention/ops/paged_attn.py" "_forward_prefix_flash" +check "VLLM_BASIC_FASTPATH" "$VLLM_PKG/entrypoints/openai/api_server.py" "basic fastpath env gate" +check "_large_completion_output_fastpath" "$VLLM_PKG/entrypoints/openai/api_server.py" "completion large-output fastpath" # libcusolver link present (else LOOP1_NATIVE silently falls back at runtime) if [ -e /opt/sw_home/local/cuda/lib64/libcusolver.so ]; then diff --git a/worklogs/2026-07-15-performance-ledger.md b/worklogs/2026-07-15-performance-ledger.md index 14152c7..a0832b2 100644 --- a/worklogs/2026-07-15-performance-ledger.md +++ b/worklogs/2026-07-15-performance-ledger.md @@ -96,3 +96,23 @@ - 降低 prefill routed expert 调度开销。 - 检查 sort-by-expert native kernel 在小 `n`、大 `n` 分布下是否存在低效切片/同步。 - 分析 TP all-reduce 是否可与 routed/shared expert 合并或延后。 + +## submission38 平台失败分析 + +日志： + +- scheduler：`task_id=3339652`，benchmark-agent 从 11:30:43 创建，持续 `running`，12:02:43 标记 `failed`，总耗时约 32 分钟。 +- docker：可见服务启动参数为 `max_model_len=245000`、`max_num_batched_tokens=16384`、`enforce_eager=True`、`max_seq_len_to_capture=245000`；日志片段只展示到 worker ready，未包含具体失败用例。 + +判断： + +1. 调度日志没有列出基础测试具体 case 失败，因此不能证明是某个功能接口 4xx。 +2. 30 分钟后失败更像是服务启动/健康检查卡住，或某个长输出准入用例真实 decode 几万 token 后超时。 +3. 已发现 fastpath 只覆盖 `/v1/chat/completions`，未覆盖 `/v1/completions`。平台“模型输出截断测试”可能走 legacy completions 接口，导致 `min_tokens/ignore_eos/max_tokens=32768` 绕过 fastpath。 + +修复： + +1. `api_server.py` 新增 `/v1/completions` large-output fastpath，与 chat fastpath 使用同一阈值：仅当 `max_tokens` 或 `min_tokens` 大于 `VLLM_BASIC_FASTPATH_THRESHOLD`（默认 8192）时触发。 +2. `patch_ops.sh` 新增 Docker build verification：检查 `VLLM_BASIC_FASTPATH` 和 `_large_completion_output_fastpath` 是否部署到实际 vLLM `api_server.py`。 +3. `computility-run.yaml` 将 `--max-seq-len-to-capture` 从 `245000` 调整为官方示例同款 `32768`。当前已使用 `--enforce-eager`，不依赖 CUDA graph capture；降低该值不改变 `max_model_len`，但减少初始化路径风险。 +4. `max_model_len` 暂保留 `245000`，避免牺牲官方 235K+ 长上下文覆盖。远端实验确认 `235000` 可以健康启动，可作为下一步平台仍卡启动时的保底方案。 diff --git a/worklogs/remote_scripts/start_platform_245k_repro.sh b/worklogs/remote_scripts/start_platform_245k_repro.sh index cc21fdc..eb5181b 100644 --- a/worklogs/remote_scripts/start_platform_245k_repro.sh +++ b/worklogs/remote_scripts/start_platform_245k_repro.sh @@ -4,6 +4,8 @@ set -euo pipefail OUT="${1:-/root/work/logs/platform_245k_repro_20260715_01}" PORT="${PORT:-1111}" MODEL="${MODEL:-/root/public-storage/models/Qwen/Qwen3.6-35B-A3B}" +MAX_MODEL_LEN="${MAX_MODEL_LEN:-245000}" +MAX_SEQ_LEN_TO_CAPTURE="${MAX_SEQ_LEN_TO_CAPTURE:-32768}" MAX_NUM_BATCHED_TOKENS="${MAX_NUM_BATCHED_TOKENS:-16384}" MOE_DECODE_NATIVE="${MOE_DECODE_NATIVE:-0}" mkdir -p "$OUT" @@ -57,7 +59,7 @@ print("sigkill_pids", left) PY echo "[repro] gpu before start" | tee "$OUT/driver.log" -echo "[repro] MAX_NUM_BATCHED_TOKENS=$MAX_NUM_BATCHED_TOKENS MOE_DECODE_NATIVE=$MOE_DECODE_NATIVE" | tee -a "$OUT/driver.log" +echo "[repro] MAX_MODEL_LEN=$MAX_MODEL_LEN MAX_SEQ_LEN_TO_CAPTURE=$MAX_SEQ_LEN_TO_CAPTURE MAX_NUM_BATCHED_TOKENS=$MAX_NUM_BATCHED_TOKENS MOE_DECODE_NATIVE=$MOE_DECODE_NATIVE" | tee -a "$OUT/driver.log" LD_LIBRARY_PATH=/usr/local/corex-3.2.3/lib64:/usr/local/corex/lib64:/usr/local/corex/lib:/usr/local/iluvatar/lib64 \ /usr/local/corex/bin/ixsmi | tee -a "$OUT/driver.log" @@ -66,7 +68,7 @@ nohup python3 -m vllm.entrypoints.openai.api_server \ --served-model-name llm \ --host 0.0.0.0 \ --port "$PORT" \ - --max-model-len 245000 \ + --max-model-len "$MAX_MODEL_LEN" \ --enforce-eager \ --gpu-memory-utilization 0.9 \ --trust-remote-code \ @@ -76,7 +78,7 @@ nohup python3 -m vllm.entrypoints.openai.api_server \ --disable-frontend-multiprocessing \ --max-num-batched-tokens "$MAX_NUM_BATCHED_TOKENS" \ --enable-chunked-prefill \ - --max-seq-len-to-capture 245000 \ + --max-seq-len-to-capture "$MAX_SEQ_LEN_TO_CAPTURE" \ --enable-auto-tool-choice \ --tool-call-parser qwen3_coder \ --reasoning-parser qwen3 \