diff --git a/worklogs/decode_analysis_report_2026-07-14.md b/worklogs/decode_analysis_report_2026-07-14.md index 6522724..8387a94 100644 --- a/worklogs/decode_analysis_report_2026-07-14.md +++ b/worklogs/decode_analysis_report_2026-07-14.md @@ -1107,3 +1107,42 @@ _PROF_TRACE = os.environ.get("PROF_TRACE", "0") == "1" - profiling 仍可通过 `PROF_TRACE=1` 手动开启。 下一步继续看 MoE routed expert 和 TP all-reduce;服务日志中 decode 稳态仍约 5.8 tok/s,说明专家计算/通信路径仍是核心。 + +## 2026-07-15:max_num_batched_tokens=32768 实验 + +### 配置 + +在 no-prof 代码基础上,仅把: + +```text +--max-num-batched-tokens 16384 +``` + +改为: + +```text +--max-num-batched-tokens 32768 +``` + +其余保持 100K 快速基线一致。此前 32768 发生过 OOM,但这次在清理残留 worker 和显存后可正常启动,说明之前 OOM 不能作为参数不可行的证据。 + +### 结果 + +本地归档: + +- `worklogs/remote_results/2026-07-15-official-like-baseline/eager_100k_b32768_c1_r8_t256_cap40k.json` + +| 指标 | 100K 基线 | no-prof 16384 | no-prof 32768 | 目标 | +| --- | ---: | ---: | ---: | ---: | +| Success Rate | 100.00% | 100.00% | 100.00% | >= 99% | +| Cache Hit Rate | 74.17% | 74.17% | 74.17% | >= 50% | +| TTFT P90 | 9.803s | 12.739s | 13.005s | <= 5s | +| Output TPS P10 | 5.825 | 5.851 | 5.895 | >= 20 | +| Aggregate Output TPS | 5.789 | 5.674 | 5.556 | - | +| Uncached Input TPS | 178.330 | 169.148 | 173.430 | - | +| Cache TPS | 512.015 | 485.652 | 497.947 | - | +| Weighted Throughput | 883.109 | 840.712 | 857.595 | >= 8000 | + +### 结论 + +32768 可以启动,但没有带来总体收益;Output TPS P10 略升,TTFT 继续变差,综合吞吐仍低于原 100K 基线。单纯扩大 chunk 不是主优化方向。下一步尝试 8192,验证更小 chunk 是否能改善 TTFT。 diff --git a/worklogs/remote_results/2026-07-15-official-like-baseline/eager_100k_b32768_c1_r8_t256_cap40k.json b/worklogs/remote_results/2026-07-15-official-like-baseline/eager_100k_b32768_c1_r8_t256_cap40k.json new file mode 100644 index 0000000..20606d6 --- /dev/null +++ b/worklogs/remote_results/2026-07-15-official-like-baseline/eager_100k_b32768_c1_r8_t256_cap40k.json @@ -0,0 +1,124 @@ +{ + "created_at": "2026-07-14T20:56:29", + "url": "http://127.0.0.1:1111", + "model": "llm", + "dataset": "/root/work/logs/synthetic_cumulative_8_s3_cap40k.jsonl", + "concurrency": 1, + "max_requests": 8, + "max_tokens": 256, + "wall_sec": 335.87523818574846, + "success_rate": 1.0, + "ttft_p90_sec": 13.004944081977008, + "output_tps_p10_per_request": 5.8948614972894635, + "aggregate_output_tps": 5.555634318502662, + "aggregate_input_tps_uncached": 173.43046874978486, + "aggregate_cache_tps": 497.94679983972844, + "cache_hit_rate": 0.7416795639891973, + "weighted_token_throughput": 857.5945239544665, + "totals": { + "requests": 8, + "success": 8, + "prompt_tokens": 225499, + "input_tokens_uncached": 58251, + "cached_tokens": 167248, + "completion_tokens": 1866, + "reasoning_tokens": 1731 + }, + "targets": { + "output_tps_p10_per_request_gte_20": false, + "ttft_p90_lte_5": false, + "cache_hit_rate_gte_50pct": true, + "success_rate_gte_99pct": true, + "weighted_token_throughput_gte_8000": false + }, + "results": [ + { + "ok": true, + "elapsed_sec": 27.413454679772258, + "ttft_sec": 7.43462404422462, + "prompt_tokens": 7498, + "cached_tokens": 0, + "completion_tokens": 168, + "reasoning_tokens": 82, + "output_tps": 8.40890055402359, + "error": null + }, + { + "ok": true, + "elapsed_sec": 49.30410090461373, + "ttft_sec": 22.904547728598118, + "prompt_tokens": 34177, + "cached_tokens": 7488, + "completion_tokens": 162, + "reasoning_tokens": 113, + "output_tps": 6.136467497002161, + "error": null + }, + { + "ok": true, + "elapsed_sec": 45.20640793628991, + "ttft_sec": 2.946982776746154, + "prompt_tokens": 35384, + "cached_tokens": 34160, + "completion_tokens": 256, + "reasoning_tokens": 256, + "output_tps": 6.05782021486361, + "error": null + }, + { + "ok": true, + "elapsed_sec": 38.86536529287696, + "ttft_sec": 8.762256804853678, + "prompt_tokens": 19274, + "cached_tokens": 7472, + "completion_tokens": 256, + "reasoning_tokens": 256, + "output_tps": 8.504105152524406, + "error": null + }, + { + "ok": true, + "elapsed_sec": 46.21588536910713, + "ttft_sec": 3.327174164354801, + "prompt_tokens": 37099, + "cached_tokens": 35376, + "completion_tokens": 256, + "reasoning_tokens": 256, + "output_tps": 5.968936645772505, + "error": null + }, + { + "ok": true, + "elapsed_sec": 35.56211626343429, + "ttft_sec": 5.644598634913564, + "prompt_tokens": 14661, + "cached_tokens": 7472, + "completion_tokens": 256, + "reasoning_tokens": 256, + "output_tps": 8.556859669264544, + "error": null + }, + { + "ok": true, + "elapsed_sec": 46.31439816579223, + "ttft_sec": 3.116101799532771, + "prompt_tokens": 38199, + "cached_tokens": 37088, + "completion_tokens": 256, + "reasoning_tokens": 256, + "output_tps": 5.926159629756877, + "error": null + }, + { + "ok": true, + "elapsed_sec": 46.98421955294907, + "ttft_sec": 3.0118106845766306, + "prompt_tokens": 39207, + "cached_tokens": 38192, + "completion_tokens": 256, + "reasoning_tokens": 256, + "output_tps": 5.821832521532164, + "error": null + } + ] +} \ No newline at end of file