From 3cf62afaaa312e4d17134217e9c0d1efc3330c5c Mon Sep 17 00:00:00 2001 From: the magician <82004885@qq.com> Date: Wed, 15 Jul 2026 03:23:39 +0800 Subject: [PATCH] stabilize platform launch and record official-like baseline --- computility-run.yaml | 1 + worklogs/decode_analysis_report_2026-07-14.md | 69 ++++++++++ .../eager_c1_r8_t256_cap40k.json | 124 ++++++++++++++++++ .../patch_head_lf_20260715.log | 99 ++++++++++++++ 4 files changed, 293 insertions(+) create mode 100644 worklogs/remote_results/2026-07-15-official-like-baseline/eager_c1_r8_t256_cap40k.json create mode 100644 worklogs/remote_results/2026-07-15-official-like-baseline/patch_head_lf_20260715.log diff --git a/computility-run.yaml b/computility-run.yaml index 16569f3..dba2a3b 100644 --- a/computility-run.yaml +++ b/computility-run.yaml @@ -9,6 +9,7 @@ command: - llm - --max-model-len - '245000' + - --enforce-eager - --gpu-memory-utilization - '0.9' - --trust-remote-code diff --git a/worklogs/decode_analysis_report_2026-07-14.md b/worklogs/decode_analysis_report_2026-07-14.md index e83927e..db2bc1a 100644 --- a/worklogs/decode_analysis_report_2026-07-14.md +++ b/worklogs/decode_analysis_report_2026-07-14.md @@ -880,3 +880,72 @@ Dockerfile text eol=lf 对应远端日志： - `/root/work/logs/patch_head_lf_20260715.log` + +## 2026-07-15：官方相似负载基线与启动稳定性 + +### 配置 + +本轮使用官方相似合成数据集： + +- 远端数据集：`/root/work/logs/synthetic_cumulative_8_s3_cap40k.jsonl` +- 请求数：8 +- 并发：1 +- `max_tokens=256` +- `--max-model-len 100000` +- `--max-num-seqs 1` +- `--max-num-batched-tokens 16384` +- `--enable-chunked-prefill` +- `--enable-prefix-caching` +- `--chat-template /workspace/chat_template_multi_system.jinja` +- `--enforce-eager` +- Native 优化环境：`MOE_NATIVE=1 CHUNK_PARALLEL=1 PREFIX_FLASH=1 RMSNORM_NATIVE=1 LOOP1_NATIVE=1` + +本地归档结果： + +- `worklogs/remote_results/2026-07-15-official-like-baseline/eager_c1_r8_t256_cap40k.json` +- `worklogs/remote_results/2026-07-15-official-like-baseline/patch_head_lf_20260715.log` + +### 结果 + +| 指标 | 当前值 | 目标 | 结论 | +| --- | ---: | ---: | --- | +| Success Rate | 100.00% | >= 99% | 通过 | +| Cache Hit Rate | 74.17% | >= 50% | 通过 | +| TTFT P90 | 9.803s | <= 5s | 未通过 | +| Output TPS P10 | 5.825 | >= 20 | 未通过 | +| Weighted Token Throughput | 883.109 | >= 8000 | 未通过 | +| Aggregate Output TPS | 5.789 | - | 参考 | +| Aggregate Uncached Input TPS | 178.330 | - | 参考 | +| Aggregate Cache TPS | 512.015 | - | 参考 | + +### 观察 + +官方相似负载下，当前瓶颈不再是模板兼容或缓存命中率，而是长上下文 prefill/缓存读取吞吐和 decode routed expert 吞吐都偏低。8 条请求中，短输出 decode 约 5.8-8.5 tok/s；TTFT P90 被第二条长请求拉到 9.8s。 + +不加 `--enforce-eager` 的 CUDA Graph 路径在 BI-V100 当前环境中启动 OOM，典型日志为： + +```text +torch.cuda.OutOfMemoryError: Tried to allocate 256MiB. GPU 1 ... +``` + +因此提交平台配置先加入 `--enforce-eager`，目标是保证镜像启动和评测稳定。后续如果要重新打开 CUDA Graph，需要先解决图捕获阶段额外显存开销，否则平台会直接启动失败。 + +### 异常处理 + +尝试 `--max-num-batched-tokens 32768` 后服务启动 OOM，并留下 GPU1-3 约 30GB 显存残留。`ixsmi` 进程表为空，但 `ps` 发现上一次 worker 残留为 `multiprocessing.spawn` 子进程。清理残留进程后，显存恢复到每卡约 257MiB。 + +结论：后续每次 OOM 后必须先确认： + +```bash +ps -ef | grep -E 'multiprocessing.spawn|resource_tracker|vllm.entrypoints|api_server' +ixsmi +``` + +若存在残留 worker，需清理后再继续实验，否则会把脏显存误判为参数不可行。 + +### 下一步 + +1. 以 `--enforce-eager + max_num_batched_tokens=16384` 作为稳定提交基线。 +2. 在干净显存下重新验证 `max_num_batched_tokens=32768` 是否必然 OOM。 +3. 对官方相似负载做参数矩阵：`max_num_batched_tokens`、`gpu_memory_utilization`、`max_model_len`、`max_num_seqs`。 +4. 若参数调优收益不足，继续深入 `PREFIX_FLASH` 和 MoE routed expert 的 native/kernel 路径 profiling。 diff --git a/worklogs/remote_results/2026-07-15-official-like-baseline/eager_c1_r8_t256_cap40k.json b/worklogs/remote_results/2026-07-15-official-like-baseline/eager_c1_r8_t256_cap40k.json new file mode 100644 index 0000000..a844a67 --- /dev/null +++ b/worklogs/remote_results/2026-07-15-official-like-baseline/eager_c1_r8_t256_cap40k.json @@ -0,0 +1,124 @@ +{ + "created_at": "2026-07-14T18:06:30", + "url": "http://127.0.0.1:1111", + "model": "llm", + "dataset": "/root/work/logs/synthetic_cumulative_8_s3_cap40k.jsonl", + "concurrency": 1, + "max_requests": 8, + "max_tokens": 256, + "wall_sec": 326.64670574851334, + "success_rate": 1.0, + "ttft_p90_sec": 9.802654626592993, + "output_tps_p10_per_request": 5.825013286992466, + "aggregate_output_tps": 5.789129254087408, + "aggregate_input_tps_uncached": 178.3302846006587, + "aggregate_cache_tps": 512.0149600674832, + "cache_hit_rate": 0.7416795639891973, + "weighted_token_throughput": 883.1090591866864, + "totals": { + "requests": 8, + "success": 8, + "prompt_tokens": 225499, + "input_tokens_uncached": 58251, + "cached_tokens": 167248, + "completion_tokens": 1891, + "reasoning_tokens": 1759 + }, + "targets": { + "output_tps_p10_per_request_gte_20": false, + "ttft_p90_lte_5": false, + "cache_hit_rate_gte_50pct": true, + "success_rate_gte_99pct": true, + "weighted_token_throughput_gte_8000": false + }, + "results": [ + { + "ok": true, + "elapsed_sec": 33.95716667920351, + "ttft_sec": 6.443204963579774, + "prompt_tokens": 7498, + "cached_tokens": 0, + "completion_tokens": 234, + "reasoning_tokens": 148, + "output_tps": 8.504773046446585, + "error": null + }, + { + "ok": true, + "elapsed_sec": 34.63446234725416, + "ttft_sec": 14.952531272545457, + "prompt_tokens": 34177, + "cached_tokens": 7488, + "completion_tokens": 121, + "reasoning_tokens": 75, + "output_tps": 6.147770741636481, + "error": null + }, + { + "ok": true, + "elapsed_sec": 45.5024864859879, + "ttft_sec": 3.0004746820777655, + "prompt_tokens": 35384, + "cached_tokens": 34160, + "completion_tokens": 256, + "reasoning_tokens": 256, + "output_tps": 6.02324429208427, + "error": null + }, + { + "ok": true, + "elapsed_sec": 37.82375564984977, + "ttft_sec": 7.5955646354705095, + "prompt_tokens": 19274, + "cached_tokens": 7472, + "completion_tokens": 256, + "reasoning_tokens": 256, + "output_tps": 8.468915651559275, + "error": null + }, + { + "ok": true, + "elapsed_sec": 46.32829406298697, + "ttft_sec": 3.3594408705830574, + "prompt_tokens": 37099, + "cached_tokens": 35376, + "completion_tokens": 256, + "reasoning_tokens": 256, + "output_tps": 5.95780387374304, + "error": null + }, + { + "ok": true, + "elapsed_sec": 34.30666992627084, + "ttft_sec": 4.208318674936891, + "prompt_tokens": 14661, + "cached_tokens": 7472, + "completion_tokens": 256, + "reasoning_tokens": 256, + "output_tps": 8.505449280669623, + "error": null + }, + { + "ok": true, + "elapsed_sec": 46.81570158340037, + "ttft_sec": 3.0013746675103903, + "prompt_tokens": 38199, + "cached_tokens": 37088, + "completion_tokens": 256, + "reasoning_tokens": 256, + "output_tps": 5.842837674796219, + "error": null + }, + { + "ok": true, + "elapsed_sec": 47.27015720307827, + "ttft_sec": 3.0057141836732626, + "prompt_tokens": 39207, + "cached_tokens": 38192, + "completion_tokens": 256, + "reasoning_tokens": 256, + "output_tps": 5.783423048783708, + "error": null + } + ] +} \ No newline at end of file diff --git a/worklogs/remote_results/2026-07-15-official-like-baseline/patch_head_lf_20260715.log b/worklogs/remote_results/2026-07-15-official-like-baseline/patch_head_lf_20260715.log new file mode 100644 index 0000000..9f159d6 --- /dev/null +++ b/worklogs/remote_results/2026-07-15-official-like-baseline/patch_head_lf_20260715.log @@ -0,0 +1,99 @@ +[patch_ops] linked libcusolver.so -> /usr/local/corex/lib64/libcusolver.so (enables LOOP1_NATIVE) +[patch_model_runner] patched Case-1 prefix_cache_hit fix in: /usr/local/corex/lib64/python3/dist-packages/vllm/worker/model_runner.py +Looking in indexes: https://pypi.tuna.tsinghua.edu.cn/simple +Requirement already satisfied: transformers==4.55.3 in /usr/local/lib/python3.10/site-packages (4.55.3) +Requirement already satisfied: filelock in /usr/local/corex-3.2.3/lib64/python3/dist-packages (from transformers==4.55.3) (3.19.1) +Requirement already satisfied: huggingface-hub<1.0,>=0.34.0 in /usr/local/lib/python3.10/site-packages (from transformers==4.55.3) (0.35.0) +Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/site-packages (from transformers==4.55.3) (1.26.4) +Requirement already satisfied: packaging>=20.0 in /usr/local/corex-3.2.3/lib64/python3/dist-packages (from transformers==4.55.3) (25.0) +Requirement already satisfied: pyyaml>=5.1 in /usr/local/corex-3.2.3/lib64/python3/dist-packages (from transformers==4.55.3) (6.0.2) +Requirement already satisfied: regex!=2019.12.17 in /usr/local/corex-3.2.3/lib64/python3/dist-packages (from transformers==4.55.3) (2025.9.1) +Requirement already satisfied: requests in /usr/local/corex-3.2.3/lib64/python3/dist-packages (from transformers==4.55.3) (2.32.5) +Requirement already satisfied: tokenizers<0.22,>=0.21 in /usr/local/lib/python3.10/site-packages (from transformers==4.55.3) (0.21.4) +Requirement already satisfied: safetensors>=0.4.3 in /usr/local/lib/python3.10/site-packages (from transformers==4.55.3) (0.6.2) +Requirement already satisfied: tqdm>=4.27 in /usr/local/corex-3.2.3/lib64/python3/dist-packages (from transformers==4.55.3) (4.67.1) +Requirement already satisfied: fsspec>=2023.5.0 in /usr/local/corex-3.2.3/lib64/python3/dist-packages (from huggingface-hub<1.0,>=0.34.0->transformers==4.55.3) (2025.9.0) +Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/corex-3.2.3/lib64/python3/dist-packages (from huggingface-hub<1.0,>=0.34.0->transformers==4.55.3) (4.15.0) +Requirement already satisfied: hf-xet<2.0.0,>=1.1.3 in /usr/local/lib/python3.10/site-packages (from huggingface-hub<1.0,>=0.34.0->transformers==4.55.3) (1.1.10) +Requirement already satisfied: charset_normalizer<4,>=2 in /usr/local/corex-3.2.3/lib64/python3/dist-packages (from requests->transformers==4.55.3) (3.4.3) +Requirement already satisfied: idna<4,>=2.5 in /usr/local/corex-3.2.3/lib64/python3/dist-packages (from requests->transformers==4.55.3) (3.10) +Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/corex-3.2.3/lib64/python3/dist-packages (from requests->transformers==4.55.3) (2.5.0) +Requirement already satisfied: certifi>=2017.4.17 in /usr/local/corex-3.2.3/lib64/python3/dist-packages (from requests->transformers==4.55.3) (2025.8.3) +WARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager, possibly rendering your system unusable. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv. Use the --root-user-action option if you know what you are doing and want to suppress this warning. + +[notice] A new release of pip is available: 25.2 -> 26.1.2 +[notice] To update, run: pip install --upgrade pip +=== Patching /usr/local/lib/python3.10/site-packages/transformers/models/auto/configuration_auto.py === + [skip] already patched: '("qwen3", "Qwen3Config"),\n ("qwen3_5", "Qwen3_5Config' + [skip] already patched: '("qwen3", "Qwen3Config"),\n ("qwen3_5", "Qwen3_5Config' + [skip] already patched: '("qwen3", "Qwen3"),\n ("qwen3_5", "Qwen3_5"),\n ' + [skip] already patched: '("qwen3", "Qwen3"),\n ("qwen3_5", "Qwen3_5"),\n ' + +=== Patching /usr/local/lib/python3.10/site-packages/transformers/models/__init__.py === + [skip] already patched: 'from .qwen3 import *\n from .qwen3_5 import *\n from .qw' + +=== Verification === + Qwen3_5Config() smoke-test OK (model_type=qwen3_5) + Qwen3_5MoeConfig() smoke-test OK (model_type=qwen3_5_moe) + num_experts=256, top_k=8, shared=512, layers=40 + +Done. +/usr/local/corex/lib64/python3/dist-packages/torch/cuda/__init__.py:51: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you. + import pynvml # type: ignore[import] +=== Patching /usr/local/corex/lib64/python3/dist-packages/vllm/model_executor/models/registry.py === + [skip] already patched: ' "Qwen3ForCausalLM": ("qwen3", "Qwen3ForCausalLM"),\n "Qwen3MoeFo' + +=== Verification === +INFO 07-14 17:43:30 importing.py:10] Triton not installed; certain GPU-related functions will not be available. +2026-07-14 17:43:31.766722: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. +2026-07-14 17:43:31.818158: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. +To enable the following instructions: SSE3 SSE4.1 SSE4.2 AVX AVX2 AVX512F AVX512_VNNI AVX512_BF16 AVX_VNNI AMX_TILE AMX_INT8 AMX_BF16 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. +WARNING:tensorflow:Deprecation warnings have been disabled. Set TF_ENABLE_DEPRECATION_WARNINGS=1 to re-enable them. + Qwen3_5ForCausalLM found: + Qwen3_5MoeForCausalLM found: + +Done. Remember to: + 1. Set config.json 'architectures': ['Qwen3_5ForCausalLM'] or ['Qwen3_5MoEForCausalLM'] + 2. Run patch_transformers_qwen3_5.py if not already done +=== patch_xformers_sdpa_seq (sequential, pure-math) === +Target: /usr/local/corex/lib64/python3/dist-packages/vllm/attention/backends/xformers.py + [skip] _run_sdpa_fallback already present + [warn] dispatch block anchor not found + +=== patch_arg_utils (disable chunked-prefill auto-enable) === +Target: /usr/local/corex/lib64/python3/dist-packages/vllm/engine/arg_utils.py + [skip] chunked-prefill auto-enable already disabled + +=== patch_logits_processor (seq_groups=None guard for chunked prefill) === +Target: /usr/local/corex/lib64/python3/dist-packages/vllm/model_executor/layers/logits_processor.py + [skip] seq_groups=None guard already present + +Done. +=== Patching /usr/local/corex/lib/python3/dist-packages/vllm/entrypoints/openai/tool_parsers/__init__.py === + [skip] already patched: 'from .mistral_tool_parser import MistralToolParser\nfrom .qwen3coder_to' + [skip] already patched: '"MistralToolParser", "Internlm2ToolParser", "Llama3JsonToolParser",\n ' + +=== Verification === + Module spec loaded: qwen3coder_tool_parser + (full import requires torch/vllm runtime — skipping exec) + +Done. Start vLLM server with: + --tool-call-parser qwen3_coder --enable-auto-tool-choice +[patch_ops] deployed multi_system chat template → /workspace/chat_template_multi_system.jinja + +=== patch_ops verification === + [ok] MOE_NATIVE (qwen3_5.py) + [ok] _native_experts_sorted + [ok] CHUNK_PARALLEL + [ok] LOOP1_NATIVE + [ok] RMSNORM_NATIVE + [ok] PREFIX_FLASH (paged_attn.py) + [ok] _forward_prefix_flash + [ok] libcusolver.so link (LOOP1_NATIVE runtime) + [ok] /workspace/chat_template_multi_system.jinja (multi_system fix) + [ok] torch.linalg.solve_triangular callable + => all five optimizations deployed and enabled + +=== patch complete. Start the server with the computility runner using: === + ../computility-run.yaml + (env + command are defined there; run from outside qwen3_6_scripts/)