record max batched tokens 32768 experiment
This commit is contained in:
@@ -1107,3 +1107,42 @@ _PROF_TRACE = os.environ.get("PROF_TRACE", "0") == "1"
|
|||||||
- profiling 仍可通过 `PROF_TRACE=1` 手动开启。
|
- profiling 仍可通过 `PROF_TRACE=1` 手动开启。
|
||||||
|
|
||||||
下一步继续看 MoE routed expert 和 TP all-reduce；服务日志中 decode 稳态仍约 5.8 tok/s，说明专家计算/通信路径仍是核心。
|
下一步继续看 MoE routed expert 和 TP all-reduce；服务日志中 decode 稳态仍约 5.8 tok/s，说明专家计算/通信路径仍是核心。
|
||||||
|
|
||||||
|
## 2026-07-15：max_num_batched_tokens=32768 实验
|
||||||
|
|
||||||
|
### 配置
|
||||||
|
|
||||||
|
在 no-prof 代码基础上，仅把：
|
||||||
|
|
||||||
|
```text
|
||||||
|
--max-num-batched-tokens 16384
|
||||||
|
```
|
||||||
|
|
||||||
|
改为：
|
||||||
|
|
||||||
|
```text
|
||||||
|
--max-num-batched-tokens 32768
|
||||||
|
```
|
||||||
|
|
||||||
|
其余保持 100K 快速基线一致。此前 32768 发生过 OOM，但这次在清理残留 worker 和显存后可正常启动，说明之前 OOM 不能作为参数不可行的证据。
|
||||||
|
|
||||||
|
### 结果
|
||||||
|
|
||||||
|
本地归档：
|
||||||
|
|
||||||
|
- `worklogs/remote_results/2026-07-15-official-like-baseline/eager_100k_b32768_c1_r8_t256_cap40k.json`
|
||||||
|
|
||||||
|
| 指标 | 100K 基线 | no-prof 16384 | no-prof 32768 | 目标 |
|
||||||
|
| --- | ---: | ---: | ---: | ---: |
|
||||||
|
| Success Rate | 100.00% | 100.00% | 100.00% | >= 99% |
|
||||||
|
| Cache Hit Rate | 74.17% | 74.17% | 74.17% | >= 50% |
|
||||||
|
| TTFT P90 | 9.803s | 12.739s | 13.005s | <= 5s |
|
||||||
|
| Output TPS P10 | 5.825 | 5.851 | 5.895 | >= 20 |
|
||||||
|
| Aggregate Output TPS | 5.789 | 5.674 | 5.556 | - |
|
||||||
|
| Uncached Input TPS | 178.330 | 169.148 | 173.430 | - |
|
||||||
|
| Cache TPS | 512.015 | 485.652 | 497.947 | - |
|
||||||
|
| Weighted Throughput | 883.109 | 840.712 | 857.595 | >= 8000 |
|
||||||
|
|
||||||
|
### 结论
|
||||||
|
|
||||||
|
32768 可以启动，但没有带来总体收益；Output TPS P10 略升，TTFT 继续变差，综合吞吐仍低于原 100K 基线。单纯扩大 chunk 不是主优化方向。下一步尝试 8192，验证更小 chunk 是否能改善 TTFT。
|
||||||
|
|||||||
@@ -0,0 +1,124 @@
|
|||||||
|
{
|
||||||
|
"created_at": "2026-07-14T20:56:29",
|
||||||
|
"url": "http://127.0.0.1:1111",
|
||||||
|
"model": "llm",
|
||||||
|
"dataset": "/root/work/logs/synthetic_cumulative_8_s3_cap40k.jsonl",
|
||||||
|
"concurrency": 1,
|
||||||
|
"max_requests": 8,
|
||||||
|
"max_tokens": 256,
|
||||||
|
"wall_sec": 335.87523818574846,
|
||||||
|
"success_rate": 1.0,
|
||||||
|
"ttft_p90_sec": 13.004944081977008,
|
||||||
|
"output_tps_p10_per_request": 5.8948614972894635,
|
||||||
|
"aggregate_output_tps": 5.555634318502662,
|
||||||
|
"aggregate_input_tps_uncached": 173.43046874978486,
|
||||||
|
"aggregate_cache_tps": 497.94679983972844,
|
||||||
|
"cache_hit_rate": 0.7416795639891973,
|
||||||
|
"weighted_token_throughput": 857.5945239544665,
|
||||||
|
"totals": {
|
||||||
|
"requests": 8,
|
||||||
|
"success": 8,
|
||||||
|
"prompt_tokens": 225499,
|
||||||
|
"input_tokens_uncached": 58251,
|
||||||
|
"cached_tokens": 167248,
|
||||||
|
"completion_tokens": 1866,
|
||||||
|
"reasoning_tokens": 1731
|
||||||
|
},
|
||||||
|
"targets": {
|
||||||
|
"output_tps_p10_per_request_gte_20": false,
|
||||||
|
"ttft_p90_lte_5": false,
|
||||||
|
"cache_hit_rate_gte_50pct": true,
|
||||||
|
"success_rate_gte_99pct": true,
|
||||||
|
"weighted_token_throughput_gte_8000": false
|
||||||
|
},
|
||||||
|
"results": [
|
||||||
|
{
|
||||||
|
"ok": true,
|
||||||
|
"elapsed_sec": 27.413454679772258,
|
||||||
|
"ttft_sec": 7.43462404422462,
|
||||||
|
"prompt_tokens": 7498,
|
||||||
|
"cached_tokens": 0,
|
||||||
|
"completion_tokens": 168,
|
||||||
|
"reasoning_tokens": 82,
|
||||||
|
"output_tps": 8.40890055402359,
|
||||||
|
"error": null
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"ok": true,
|
||||||
|
"elapsed_sec": 49.30410090461373,
|
||||||
|
"ttft_sec": 22.904547728598118,
|
||||||
|
"prompt_tokens": 34177,
|
||||||
|
"cached_tokens": 7488,
|
||||||
|
"completion_tokens": 162,
|
||||||
|
"reasoning_tokens": 113,
|
||||||
|
"output_tps": 6.136467497002161,
|
||||||
|
"error": null
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"ok": true,
|
||||||
|
"elapsed_sec": 45.20640793628991,
|
||||||
|
"ttft_sec": 2.946982776746154,
|
||||||
|
"prompt_tokens": 35384,
|
||||||
|
"cached_tokens": 34160,
|
||||||
|
"completion_tokens": 256,
|
||||||
|
"reasoning_tokens": 256,
|
||||||
|
"output_tps": 6.05782021486361,
|
||||||
|
"error": null
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"ok": true,
|
||||||
|
"elapsed_sec": 38.86536529287696,
|
||||||
|
"ttft_sec": 8.762256804853678,
|
||||||
|
"prompt_tokens": 19274,
|
||||||
|
"cached_tokens": 7472,
|
||||||
|
"completion_tokens": 256,
|
||||||
|
"reasoning_tokens": 256,
|
||||||
|
"output_tps": 8.504105152524406,
|
||||||
|
"error": null
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"ok": true,
|
||||||
|
"elapsed_sec": 46.21588536910713,
|
||||||
|
"ttft_sec": 3.327174164354801,
|
||||||
|
"prompt_tokens": 37099,
|
||||||
|
"cached_tokens": 35376,
|
||||||
|
"completion_tokens": 256,
|
||||||
|
"reasoning_tokens": 256,
|
||||||
|
"output_tps": 5.968936645772505,
|
||||||
|
"error": null
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"ok": true,
|
||||||
|
"elapsed_sec": 35.56211626343429,
|
||||||
|
"ttft_sec": 5.644598634913564,
|
||||||
|
"prompt_tokens": 14661,
|
||||||
|
"cached_tokens": 7472,
|
||||||
|
"completion_tokens": 256,
|
||||||
|
"reasoning_tokens": 256,
|
||||||
|
"output_tps": 8.556859669264544,
|
||||||
|
"error": null
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"ok": true,
|
||||||
|
"elapsed_sec": 46.31439816579223,
|
||||||
|
"ttft_sec": 3.116101799532771,
|
||||||
|
"prompt_tokens": 38199,
|
||||||
|
"cached_tokens": 37088,
|
||||||
|
"completion_tokens": 256,
|
||||||
|
"reasoning_tokens": 256,
|
||||||
|
"output_tps": 5.926159629756877,
|
||||||
|
"error": null
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"ok": true,
|
||||||
|
"elapsed_sec": 46.98421955294907,
|
||||||
|
"ttft_sec": 3.0118106845766306,
|
||||||
|
"prompt_tokens": 39207,
|
||||||
|
"cached_tokens": 38192,
|
||||||
|
"completion_tokens": 256,
|
||||||
|
"reasoning_tokens": 256,
|
||||||
|
"output_tps": 5.821832521532164,
|
||||||
|
"error": null
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user