456 lines
14 KiB
Plaintext
456 lines
14 KiB
Plaintext
[matrix] out_dir=/root/work/logs/moe_tiny_matrix_20260715_codex_05
|
|
[matrix] bench_script=/root/work/decode_microbench.py
|
|
[matrix] model_path=/root/public-storage/models/Qwen/Qwen3.6-35B-A3B
|
|
[matrix] starting tokenwise_max4
|
|
[matrix] waiting health: 18s
|
|
[matrix] waiting health: 28s
|
|
[matrix] waiting health: 39s
|
|
[matrix] waiting health: 49s
|
|
[matrix] waiting health: 59s
|
|
[matrix] waiting health: 69s
|
|
[matrix] waiting health: 79s
|
|
[matrix] waiting health: 89s
|
|
[matrix] waiting health: 99s
|
|
[matrix] health ok
|
|
[matrix] warmup tokenwise_max4
|
|
done 1/1 ok=True tps=7.896776108883328
|
|
{
|
|
"created_at": "2026-07-14T17:07:27",
|
|
"label": "tokenwise_max4_warmup",
|
|
"url": "http://127.0.0.1:1111",
|
|
"model": "llm",
|
|
"prompt_mode": "short",
|
|
"with_tools": false,
|
|
"tool_count": 0,
|
|
"concurrency": 1,
|
|
"requests": 1,
|
|
"max_tokens": 32,
|
|
"wall_sec": 8.475353751331568,
|
|
"success_rate": 1.0,
|
|
"ttft_p50_sec": 4.421089859679341,
|
|
"ttft_p90_sec": 4.421089859679341,
|
|
"output_tps_p10_per_request": 7.896776108883328,
|
|
"output_tps_p50_per_request": 7.896776108883328,
|
|
"aggregate_output_tps": 3.7756536115052968,
|
|
"prompt_tokens": 39,
|
|
"cached_tokens": 0,
|
|
"completion_tokens": 32,
|
|
"reasoning_tokens": 32,
|
|
"chars": 111,
|
|
"monitor": null
|
|
}
|
|
RESULT_FILE /root/work/logs/moe_tiny_matrix_20260715_codex_05/tokenwise_max4_warmup.json
|
|
[matrix] benchmark tokenwise_max4
|
|
done 1/4 ok=True tps=6.377104297707733
|
|
done 2/4 ok=True tps=6.377036995390967
|
|
done 3/4 ok=True tps=6.454291150916464
|
|
done 4/4 ok=True tps=6.454309006035291
|
|
{
|
|
"created_at": "2026-07-14T17:08:10",
|
|
"label": "tokenwise_max4_short_c2_t128_r4",
|
|
"url": "http://127.0.0.1:1111",
|
|
"model": "llm",
|
|
"prompt_mode": "short",
|
|
"with_tools": false,
|
|
"tool_count": 0,
|
|
"concurrency": 2,
|
|
"requests": 4,
|
|
"max_tokens": 128,
|
|
"wall_sec": 43.14171364530921,
|
|
"success_rate": 1.0,
|
|
"ttft_p50_sec": 1.6165212774649262,
|
|
"ttft_p90_sec": 1.671257301606238,
|
|
"output_tps_p10_per_request": 6.377057186085997,
|
|
"output_tps_p50_per_request": 6.415697724312098,
|
|
"aggregate_output_tps": 11.867864225547972,
|
|
"prompt_tokens": 156,
|
|
"cached_tokens": 128,
|
|
"completion_tokens": 512,
|
|
"reasoning_tokens": 512,
|
|
"chars": 1811,
|
|
"monitor": {
|
|
"records": 33,
|
|
"parsed_samples": 132,
|
|
"avg_gpu_util_pct": 51.39393939393939,
|
|
"max_gpu_util_pct": 100,
|
|
"avg_mem_mib": 28876.25,
|
|
"max_mem_mib": 28955,
|
|
"avg_power_w": 55.734848484848484,
|
|
"max_power_w": 58
|
|
}
|
|
}
|
|
RESULT_FILE /root/work/logs/moe_tiny_matrix_20260715_codex_05/tokenwise_max4_short_c2_t128_r4.json
|
|
[matrix] starting tokenwise_max8
|
|
[matrix] waiting health: 177s
|
|
[matrix] waiting health: 187s
|
|
[matrix] waiting health: 197s
|
|
[matrix] waiting health: 207s
|
|
[matrix] waiting health: 217s
|
|
[matrix] waiting health: 227s
|
|
[matrix] waiting health: 237s
|
|
[matrix] waiting health: 247s
|
|
[matrix] waiting health: 257s
|
|
[matrix] health ok
|
|
[matrix] warmup tokenwise_max8
|
|
done 1/1 ok=True tps=8.113631034297331
|
|
{
|
|
"created_at": "2026-07-14T17:10:05",
|
|
"label": "tokenwise_max8_warmup",
|
|
"url": "http://127.0.0.1:1111",
|
|
"model": "llm",
|
|
"prompt_mode": "short",
|
|
"with_tools": false,
|
|
"tool_count": 0,
|
|
"concurrency": 1,
|
|
"requests": 1,
|
|
"max_tokens": 32,
|
|
"wall_sec": 8.34307736903429,
|
|
"success_rate": 1.0,
|
|
"ttft_p50_sec": 4.3972604889422655,
|
|
"ttft_p90_sec": 4.3972604889422655,
|
|
"output_tps_p10_per_request": 8.113631034297331,
|
|
"output_tps_p50_per_request": 8.113631034297331,
|
|
"aggregate_output_tps": 3.835515192364085,
|
|
"prompt_tokens": 39,
|
|
"cached_tokens": 0,
|
|
"completion_tokens": 32,
|
|
"reasoning_tokens": 32,
|
|
"chars": 111,
|
|
"monitor": null
|
|
}
|
|
RESULT_FILE /root/work/logs/moe_tiny_matrix_20260715_codex_05/tokenwise_max8_warmup.json
|
|
[matrix] benchmark tokenwise_max8
|
|
done 1/4 ok=True tps=6.347329155987261
|
|
done 2/4 ok=True tps=6.3472829583524115
|
|
done 3/4 ok=True tps=6.366059179012696
|
|
done 4/4 ok=True tps=6.366082294004555
|
|
{
|
|
"created_at": "2026-07-14T17:10:48",
|
|
"label": "tokenwise_max8_short_c2_t128_r4",
|
|
"url": "http://127.0.0.1:1111",
|
|
"model": "llm",
|
|
"prompt_mode": "short",
|
|
"with_tools": false,
|
|
"tool_count": 0,
|
|
"concurrency": 2,
|
|
"requests": 4,
|
|
"max_tokens": 128,
|
|
"wall_sec": 43.433731293305755,
|
|
"success_rate": 1.0,
|
|
"ttft_p50_sec": 1.5782105717808008,
|
|
"ttft_p90_sec": 1.5890035681426526,
|
|
"output_tps_p10_per_request": 6.3472968176428655,
|
|
"output_tps_p50_per_request": 6.356694167499978,
|
|
"aggregate_output_tps": 11.788073111713343,
|
|
"prompt_tokens": 156,
|
|
"cached_tokens": 128,
|
|
"completion_tokens": 512,
|
|
"reasoning_tokens": 512,
|
|
"chars": 1809,
|
|
"monitor": {
|
|
"records": 33,
|
|
"parsed_samples": 132,
|
|
"avg_gpu_util_pct": 54.56818181818182,
|
|
"max_gpu_util_pct": 100,
|
|
"avg_mem_mib": 28876.25,
|
|
"max_mem_mib": 28955,
|
|
"avg_power_w": 55.60606060606061,
|
|
"max_power_w": 57
|
|
}
|
|
}
|
|
RESULT_FILE /root/work/logs/moe_tiny_matrix_20260715_codex_05/tokenwise_max8_short_c2_t128_r4.json
|
|
[matrix] starting tokenwise_max16
|
|
[matrix] waiting health: 335s
|
|
[matrix] waiting health: 345s
|
|
[matrix] waiting health: 356s
|
|
[matrix] waiting health: 366s
|
|
[matrix] waiting health: 376s
|
|
[matrix] waiting health: 386s
|
|
[matrix] waiting health: 396s
|
|
[matrix] waiting health: 406s
|
|
[matrix] waiting health: 416s
|
|
[matrix] health ok
|
|
[matrix] warmup tokenwise_max16
|
|
done 1/1 ok=True tps=8.604076337909753
|
|
{
|
|
"created_at": "2026-07-14T17:12:43",
|
|
"label": "tokenwise_max16_warmup",
|
|
"url": "http://127.0.0.1:1111",
|
|
"model": "llm",
|
|
"prompt_mode": "short",
|
|
"with_tools": false,
|
|
"tool_count": 0,
|
|
"concurrency": 1,
|
|
"requests": 1,
|
|
"max_tokens": 32,
|
|
"wall_sec": 7.908331776037812,
|
|
"success_rate": 1.0,
|
|
"ttft_p50_sec": 4.187222994863987,
|
|
"ttft_p90_sec": 4.187222994863987,
|
|
"output_tps_p10_per_request": 8.604076337909753,
|
|
"output_tps_p50_per_request": 8.604076337909753,
|
|
"aggregate_output_tps": 4.046365391113176,
|
|
"prompt_tokens": 39,
|
|
"cached_tokens": 0,
|
|
"completion_tokens": 32,
|
|
"reasoning_tokens": 32,
|
|
"chars": 111,
|
|
"monitor": null
|
|
}
|
|
RESULT_FILE /root/work/logs/moe_tiny_matrix_20260715_codex_05/tokenwise_max16_warmup.json
|
|
[matrix] benchmark tokenwise_max16
|
|
done 1/4 ok=True tps=6.292255993870769
|
|
done 2/4 ok=True tps=6.292149220757748
|
|
done 3/4 ok=True tps=6.4821665164576485
|
|
done 4/4 ok=True tps=6.482185873169073
|
|
{
|
|
"created_at": "2026-07-14T17:13:26",
|
|
"label": "tokenwise_max16_short_c2_t128_r4",
|
|
"url": "http://127.0.0.1:1111",
|
|
"model": "llm",
|
|
"prompt_mode": "short",
|
|
"with_tools": false,
|
|
"tool_count": 0,
|
|
"concurrency": 2,
|
|
"requests": 4,
|
|
"max_tokens": 128,
|
|
"wall_sec": 42.65718542598188,
|
|
"success_rate": 1.0,
|
|
"ttft_p50_sec": 1.2818848732858896,
|
|
"ttft_p90_sec": 1.3119126919656992,
|
|
"output_tps_p10_per_request": 6.292181252691654,
|
|
"output_tps_p50_per_request": 6.387211255164209,
|
|
"aggregate_output_tps": 12.002667191636796,
|
|
"prompt_tokens": 156,
|
|
"cached_tokens": 128,
|
|
"completion_tokens": 512,
|
|
"reasoning_tokens": 512,
|
|
"chars": 1908,
|
|
"monitor": {
|
|
"records": 32,
|
|
"parsed_samples": 128,
|
|
"avg_gpu_util_pct": 54.7890625,
|
|
"max_gpu_util_pct": 100,
|
|
"avg_mem_mib": 28876.25,
|
|
"max_mem_mib": 28955,
|
|
"avg_power_w": 55.8671875,
|
|
"max_power_w": 58
|
|
}
|
|
}
|
|
RESULT_FILE /root/work/logs/moe_tiny_matrix_20260715_codex_05/tokenwise_max16_short_c2_t128_r4.json
|
|
[matrix] starting bmm_max8
|
|
[matrix] waiting health: 493s
|
|
[matrix] waiting health: 503s
|
|
[matrix] waiting health: 513s
|
|
[matrix] waiting health: 523s
|
|
[matrix] waiting health: 533s
|
|
[matrix] waiting health: 543s
|
|
[matrix] waiting health: 553s
|
|
[matrix] waiting health: 563s
|
|
[matrix] waiting health: 573s
|
|
[matrix] health ok
|
|
[matrix] warmup bmm_max8
|
|
done 1/1 ok=True tps=8.494017912057346
|
|
{
|
|
"created_at": "2026-07-14T17:15:20",
|
|
"label": "bmm_max8_warmup",
|
|
"url": "http://127.0.0.1:1111",
|
|
"model": "llm",
|
|
"prompt_mode": "short",
|
|
"with_tools": false,
|
|
"tool_count": 0,
|
|
"concurrency": 1,
|
|
"requests": 1,
|
|
"max_tokens": 32,
|
|
"wall_sec": 7.85333033464849,
|
|
"success_rate": 1.0,
|
|
"ttft_p50_sec": 4.084131080657244,
|
|
"ttft_p90_sec": 4.084131080657244,
|
|
"output_tps_p10_per_request": 8.494017912057346,
|
|
"output_tps_p50_per_request": 8.494017912057346,
|
|
"aggregate_output_tps": 4.074704442116441,
|
|
"prompt_tokens": 39,
|
|
"cached_tokens": 0,
|
|
"completion_tokens": 32,
|
|
"reasoning_tokens": 32,
|
|
"chars": 111,
|
|
"monitor": null
|
|
}
|
|
RESULT_FILE /root/work/logs/moe_tiny_matrix_20260715_codex_05/bmm_max8_warmup.json
|
|
[matrix] benchmark bmm_max8
|
|
done 1/4 ok=True tps=6.899938193333011
|
|
done 2/4 ok=True tps=6.89988749567448
|
|
done 3/4 ok=True tps=7.013651698722346
|
|
done 4/4 ok=True tps=7.013673245204575
|
|
{
|
|
"created_at": "2026-07-14T17:16:00",
|
|
"label": "bmm_max8_short_c2_t128_r4",
|
|
"url": "http://127.0.0.1:1111",
|
|
"model": "llm",
|
|
"prompt_mode": "short",
|
|
"with_tools": false,
|
|
"tool_count": 0,
|
|
"concurrency": 2,
|
|
"requests": 4,
|
|
"max_tokens": 128,
|
|
"wall_sec": 39.6690470892936,
|
|
"success_rate": 1.0,
|
|
"ttft_p50_sec": 1.4093408677726984,
|
|
"ttft_p90_sec": 1.4237070180475713,
|
|
"output_tps_p10_per_request": 6.899902704972039,
|
|
"output_tps_p50_per_request": 6.956794946027678,
|
|
"aggregate_output_tps": 12.906788480386393,
|
|
"prompt_tokens": 156,
|
|
"cached_tokens": 128,
|
|
"completion_tokens": 512,
|
|
"reasoning_tokens": 512,
|
|
"chars": 1801,
|
|
"monitor": {
|
|
"records": 31,
|
|
"parsed_samples": 124,
|
|
"avg_gpu_util_pct": 54.66129032258065,
|
|
"max_gpu_util_pct": 100,
|
|
"avg_mem_mib": 28876.25,
|
|
"max_mem_mib": 28955,
|
|
"avg_power_w": 56.58064516129032,
|
|
"max_power_w": 58
|
|
}
|
|
}
|
|
RESULT_FILE /root/work/logs/moe_tiny_matrix_20260715_codex_05/bmm_max8_short_c2_t128_r4.json
|
|
[matrix] starting bmm_max16
|
|
[matrix] waiting health: 647s
|
|
[matrix] waiting health: 657s
|
|
[matrix] waiting health: 667s
|
|
[matrix] waiting health: 677s
|
|
[matrix] waiting health: 687s
|
|
[matrix] waiting health: 697s
|
|
[matrix] waiting health: 708s
|
|
[matrix] waiting health: 718s
|
|
[matrix] waiting health: 728s
|
|
[matrix] health ok
|
|
[matrix] warmup bmm_max16
|
|
done 1/1 ok=True tps=8.103515357530624
|
|
{
|
|
"created_at": "2026-07-14T17:17:55",
|
|
"label": "bmm_max16_warmup",
|
|
"url": "http://127.0.0.1:1111",
|
|
"model": "llm",
|
|
"prompt_mode": "short",
|
|
"with_tools": false,
|
|
"tool_count": 0,
|
|
"concurrency": 1,
|
|
"requests": 1,
|
|
"max_tokens": 32,
|
|
"wall_sec": 8.268167091533542,
|
|
"success_rate": 1.0,
|
|
"ttft_p50_sec": 4.3172661755234,
|
|
"ttft_p90_sec": 4.3172661755234,
|
|
"output_tps_p10_per_request": 8.103515357530624,
|
|
"output_tps_p50_per_request": 8.103515357530624,
|
|
"aggregate_output_tps": 3.8702652771455766,
|
|
"prompt_tokens": 39,
|
|
"cached_tokens": 0,
|
|
"completion_tokens": 32,
|
|
"reasoning_tokens": 32,
|
|
"chars": 111,
|
|
"monitor": null
|
|
}
|
|
RESULT_FILE /root/work/logs/moe_tiny_matrix_20260715_codex_05/bmm_max16_warmup.json
|
|
[matrix] benchmark bmm_max16
|
|
done 1/4 ok=True tps=6.244828150361198
|
|
done 2/4 ok=True tps=6.2447879822717285
|
|
done 3/4 ok=True tps=6.516602560962641
|
|
done 4/4 ok=True tps=6.516622061468281
|
|
{
|
|
"created_at": "2026-07-14T17:18:38",
|
|
"label": "bmm_max16_short_c2_t128_r4",
|
|
"url": "http://127.0.0.1:1111",
|
|
"model": "llm",
|
|
"prompt_mode": "short",
|
|
"with_tools": false,
|
|
"tool_count": 0,
|
|
"concurrency": 2,
|
|
"requests": 4,
|
|
"max_tokens": 128,
|
|
"wall_sec": 42.81683264859021,
|
|
"success_rate": 1.0,
|
|
"ttft_p50_sec": 1.3186510931700468,
|
|
"ttft_p90_sec": 1.4008520478382707,
|
|
"output_tps_p10_per_request": 6.244800032698569,
|
|
"output_tps_p50_per_request": 6.38071535566192,
|
|
"aggregate_output_tps": 11.957913940111546,
|
|
"prompt_tokens": 156,
|
|
"cached_tokens": 128,
|
|
"completion_tokens": 512,
|
|
"reasoning_tokens": 512,
|
|
"chars": 1839,
|
|
"monitor": {
|
|
"records": 32,
|
|
"parsed_samples": 128,
|
|
"avg_gpu_util_pct": 53.1953125,
|
|
"max_gpu_util_pct": 100,
|
|
"avg_mem_mib": 29039.0,
|
|
"max_mem_mib": 29123,
|
|
"avg_power_w": 56.03125,
|
|
"max_power_w": 58
|
|
}
|
|
}
|
|
RESULT_FILE /root/work/logs/moe_tiny_matrix_20260715_codex_05/bmm_max16_short_c2_t128_r4.json
|
|
{
|
|
"out_dir": "/root/work/logs/moe_tiny_matrix_20260715_codex_05",
|
|
"rows": [
|
|
{
|
|
"label": "bmm_max16_short_c2_t128_r4",
|
|
"success_rate": 1.0,
|
|
"ttft_p90_sec": 1.4008520478382707,
|
|
"output_tps_p10_per_request": 6.244800032698569,
|
|
"output_tps_p50_per_request": 6.38071535566192,
|
|
"aggregate_output_tps": 11.957913940111546,
|
|
"avg_gpu_util_pct": 53.1953125,
|
|
"max_gpu_util_pct": 100
|
|
},
|
|
{
|
|
"label": "bmm_max8_short_c2_t128_r4",
|
|
"success_rate": 1.0,
|
|
"ttft_p90_sec": 1.4237070180475713,
|
|
"output_tps_p10_per_request": 6.899902704972039,
|
|
"output_tps_p50_per_request": 6.956794946027678,
|
|
"aggregate_output_tps": 12.906788480386393,
|
|
"avg_gpu_util_pct": 54.66129032258065,
|
|
"max_gpu_util_pct": 100
|
|
},
|
|
{
|
|
"label": "tokenwise_max16_short_c2_t128_r4",
|
|
"success_rate": 1.0,
|
|
"ttft_p90_sec": 1.3119126919656992,
|
|
"output_tps_p10_per_request": 6.292181252691654,
|
|
"output_tps_p50_per_request": 6.387211255164209,
|
|
"aggregate_output_tps": 12.002667191636796,
|
|
"avg_gpu_util_pct": 54.7890625,
|
|
"max_gpu_util_pct": 100
|
|
},
|
|
{
|
|
"label": "tokenwise_max4_short_c2_t128_r4",
|
|
"success_rate": 1.0,
|
|
"ttft_p90_sec": 1.671257301606238,
|
|
"output_tps_p10_per_request": 6.377057186085997,
|
|
"output_tps_p50_per_request": 6.415697724312098,
|
|
"aggregate_output_tps": 11.867864225547972,
|
|
"avg_gpu_util_pct": 51.39393939393939,
|
|
"max_gpu_util_pct": 100
|
|
},
|
|
{
|
|
"label": "tokenwise_max8_short_c2_t128_r4",
|
|
"success_rate": 1.0,
|
|
"ttft_p90_sec": 1.5890035681426526,
|
|
"output_tps_p10_per_request": 6.3472968176428655,
|
|
"output_tps_p50_per_request": 6.356694167499978,
|
|
"aggregate_output_tps": 11.788073111713343,
|
|
"avg_gpu_util_pct": 54.56818181818182,
|
|
"max_gpu_util_pct": 100
|
|
}
|
|
]
|
|
}
|
|
[matrix] done
|
|
[matrix] summary=/root/work/logs/moe_tiny_matrix_20260715_codex_05/summary.md
|