icer/qwen36_01
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

add moe bucket profiling and record bottlenecks

Browse Source
This commit is contained in:
the magician
2026-07-15 11:34:59 +08:00
parent 8b4ee2ff8b
commit e374b14bf9
9 changed files with 2184 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

76
qwen3_6_scripts/qwen3_5.py
View File

@@ -51,12 +51,79 @@ import os as _os_prof
import sys as _sys_prof import sys as _sys_prof
import time as _time_prof import time as _time_prof
_PROF_TRACE = _os_prof.environ.get("PROF_TRACE", "0") == "1" _PROF_TRACE = _os_prof.environ.get("PROF_TRACE", "0") == "1"
_PROF_MOE_SUMMARY = _os_prof.environ.get("PROF_MOE_SUMMARY", "0") == "1"
_PROF_MOE_SYNC = _os_prof.environ.get("PROF_MOE_SYNC", "1") != "0"
_PROF_MOE_INTERVAL = int(_os_prof.environ.get("PROF_MOE_INTERVAL", "400"))
_PROF_MOE_STATS = {}
_t0 = _time_prof.time() _t0 = _time_prof.time()
def _prof(msg): def _prof(msg):
if _PROF_TRACE: if _PROF_TRACE:
print(f"[PROF] {_time_prof.time()-_t0:7.2f}s {msg}", file=_sys_prof.stderr, flush=True) print(f"[PROF] {_time_prof.time()-_t0:7.2f}s {msg}", file=_sys_prof.stderr, flush=True)
def _prof_moe_time():
if not _PROF_MOE_SUMMARY:
return None
if _PROF_MOE_SYNC:
try:
torch.cuda.synchronize()
except Exception:
pass
return _time_prof.perf_counter()
def _prof_moe_bucket(tokens):
return "decode" if int(tokens) == 1 else "prefill"
def _prof_moe_stats(bucket):
stats = _PROF_MOE_STATS.get(bucket)
if stats is None:
stats = {
"calls": 0,
"tokens": 0,
"router_gate": 0.0,
"routed": 0.0,
"shared": 0.0,
"all_reduce": 0.0,
"total": 0.0,
}
_PROF_MOE_STATS[bucket] = stats
return stats
def _prof_moe_add(name, start, bucket):
if not _PROF_MOE_SUMMARY or start is None:
return None
end = _prof_moe_time()
_prof_moe_stats(bucket)[name] += end - start
return end
def _prof_moe_finish(total_start, tokens, bucket):
if not _PROF_MOE_SUMMARY or total_start is None:
return
end = _prof_moe_time()
stats = _prof_moe_stats(bucket)
stats["total"] += end - total_start
stats["calls"] += 1
stats["tokens"] += int(tokens)
if stats["calls"] % max(1, _PROF_MOE_INTERVAL) != 0:
return
calls = stats["calls"]
parts = [
f"pid={_os_prof.getpid()}",
f"bucket={bucket}",
f"calls={calls}",
f"tokens={stats['tokens']}",
]
for name in ("router_gate", "routed", "shared", "all_reduce", "total"):
avg_ms = stats[name] * 1000.0 / calls
parts.append(f"{name}_avg_ms={avg_ms:.3f}")
print("[PROF_MOE_SUMMARY] " + " ".join(parts),
file=_sys_prof.stderr, flush=True)
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
# Pure-PyTorch DeltaNet kernels (fallbacks from transformers 5.2.0) # Pure-PyTorch DeltaNet kernels (fallbacks from transformers 5.2.0)
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
@@ -960,8 +1027,14 @@ class Qwen3_5MoeSparseBlock(nn.Module):
return out return out
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
_moe_total_t = _prof_moe_time()
_tokens = hidden_states.shape[0]
_prof_bucket = _prof_moe_bucket(_tokens)
_section_t = _moe_total_t
router_logits, _ = self.gate(hidden_states) router_logits, _ = self.gate(hidden_states)
_section_t = _prof_moe_add("router_gate", _section_t, _prof_bucket)
routed_out = self._pure_pytorch_experts(hidden_states, router_logits) routed_out = self._pure_pytorch_experts(hidden_states, router_logits)
_section_t = _prof_moe_add("routed", _section_t, _prof_bucket)
_prof(f" MoE routed-experts done tokens={hidden_states.shape[0]}") _prof(f" MoE routed-experts done tokens={hidden_states.shape[0]}")
gate_up, _ = self.shared_expert_gate_up(hidden_states) gate_up, _ = self.shared_expert_gate_up(hidden_states)
@@ -970,12 +1043,15 @@ class Qwen3_5MoeSparseBlock(nn.Module):
# Scalar sigmoid gate (Qwen2-MoE / Qwen3.5-MoE style) # Scalar sigmoid gate (Qwen2-MoE / Qwen3.5-MoE style)
gate_score, _ = self.shared_expert_gate(hidden_states) # (T, 1) gate_score, _ = self.shared_expert_gate(hidden_states) # (T, 1)
shared_out = shared_out * torch.sigmoid(gate_score) shared_out = shared_out * torch.sigmoid(gate_score)
_section_t = _prof_moe_add("shared", _section_t, _prof_bucket)
out = routed_out + shared_out out = routed_out + shared_out
if self.experts.tp_size > 1: if self.experts.tp_size > 1:
_prof(f" MoE all_reduce start tp_size={self.experts.tp_size}") _prof(f" MoE all_reduce start tp_size={self.experts.tp_size}")
out = tensor_model_parallel_all_reduce(out) out = tensor_model_parallel_all_reduce(out)
_prof(f" MoE all_reduce done") _prof(f" MoE all_reduce done")
_section_t = _prof_moe_add("all_reduce", _section_t, _prof_bucket)
_prof_moe_finish(_moe_total_t, _tokens, _prof_bucket)
return out return out

51
worklogs/decode_analysis_report_2026-07-14.md
View File

@@ -1194,3 +1194,54 @@ _PROF_TRACE = os.environ.get("PROF_TRACE", "0") == "1"
2. 拆分 `MLP` 时间中 routed expert、shared expert、TP all-reduce 的占比。 2. 拆分 `MLP` 时间中 routed expert、shared expert、TP all-reduce 的占比。
3. 若 all-reduce 占比高，测试 `--disable-custom-all-reduce` 保持关闭/打开的差异和 TP 通信路径。 3. 若 all-reduce 占比高，测试 `--disable-custom-all-reduce` 保持关闭/打开的差异和 TP 通信路径。
4. 若 routed expert 占比高，继续优化 `_native_experts_sorted` 中逐 expert 循环和 `index_add_`。 4. 若 routed expert 占比高，继续优化 `_native_experts_sorted` 中逐 expert 循环和 `index_add_`。
## 2026-07-15：MoE 聚合 profiling
### 改动
在 `qwen3_5.py` 增加默认关闭的聚合 profiling：
- `PROF_MOE_SUMMARY=1` 开启；
- `PROF_MOE_SYNC=1` 时每段前后 `torch.cuda.synchronize()`，用于获得更可信的 GPU 段时间；
- `PROF_MOE_INTERVAL` 控制每多少次 MoE forward 输出一次汇总；
- 分桶输出 `bucket=decode` (`T=1`) 与 `bucket=prefill` (`T>1`)。
默认 benchmark 不开启这些变量，因此生产路径不受影响。
### 归档
- `worklogs/remote_results/2026-07-15-official-like-baseline/eager_100k_b16384_moe_profile_r2_t256_cap40k.json`
- `worklogs/remote_results/2026-07-15-official-like-baseline/driver_moe_profile.log`
- `worklogs/remote_results/2026-07-15-official-like-baseline/moe_summary_profile.log`
- `worklogs/remote_results/2026-07-15-official-like-baseline/eager_100k_b16384_moe_bucket_profile_r2_t256_cap40k.json`
- `worklogs/remote_results/2026-07-15-official-like-baseline/driver_moe_bucket_profile.log`
- `worklogs/remote_results/2026-07-15-official-like-baseline/moe_bucket_summary_profile.log`
### 关键结果
bucket 分桶的最终平均值如下。数值是每个 worker 进程最终 summary 的均值，单位为 ms / MoE layer call。
| Bucket | router gate | routed expert | shared expert | all-reduce | total |
| --- | ---: | ---: | ---: | ---: | ---: |
| decode (`T=1`) | 0.069 | 0.768 | 0.235 | 0.333 | 1.416 |
| prefill (`T>1`) | 0.577 | 77.970 | 2.078 | 4.177 | 84.814 |
占比：
| Bucket | router gate | routed expert | shared expert | all-reduce |
| --- | ---: | ---: | ---: | ---: |
| decode | 4.9% | 54.2% | 16.6% | 23.5% |
| prefill | 0.7% | 91.9% | 2.4% | 4.9% |
### 结论
MoE routed expert 是最明确主瓶颈：
- 对 TTFT/prefill：routed expert 占 MoE 时间约 92%，优先级最高。
- 对 Output TPS/decode：routed expert 仍是最大项，约 54%；all-reduce 约 23%，属于第二优先级。
下一步优先做 routed expert 路径优化：
1. decode `T=1` 路径尝试使用 `ixformer.functions.act_bias_mm + silu_and_mul` 替换 `F.linear + torch` 激活，保留 bmm down projection。
2. prefill `_native_experts_sorted` 重点减少逐 expert Python loop / 小 kernel 串行开销；需要查 ixformer 是否提供 grouped GEMM 类接口。
3. all-reduce 暂时不是第一主因，但 decode 中占 23%，后续可独立测通信路径。

158
worklogs/remote_results/2026-07-15-official-like-baseline/driver_moe_bucket_profile.log Normal file
View File

@@ -0,0 +1,158 @@
[exp] label=moe_bucket_profile_20260715
[exp] max_num_batched_tokens=16384
[exp] out_dir=/root/work/logs/moe_bucket_profile_20260715
Wed Jul 15 03:22:47 UTC 2026
[exp] stopping existing api_server
[exp] gpu before start
Timestamp Wed Jul 15 03:22:55 2026
+-----------------------------------------------------------------------------+
| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |
|-------------------------------+----------------------+----------------------|
| GPU Name | Bus-Id | Clock-SM Clock-Mem |
| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |
|===============================+======================+======================|
| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |
| 0% 29C P0 47W / 250W | 257MiB / 32768MiB | 0% Default |
+-------------------------------+----------------------+----------------------+
| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |
| 0% 32C P0 47W / 250W | 257MiB / 32768MiB | 0% Default |
+-------------------------------+----------------------+----------------------+
| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |
| 0% 31C P0 48W / 250W | 257MiB / 32768MiB | 0% Default |
+-------------------------------+----------------------+----------------------+
| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |
| 0% 30C P0 47W / 250W | 257MiB / 32768MiB | 0% Default |
+-------------------------------+----------------------+----------------------+
+-----------------------------------------------------------------------------+
| Processes: GPU Memory |
| GPU PID Process name Usage(MiB) |
|=============================================================================|
| No running processes found |
+-----------------------------------------------------------------------------+
[exp] starting server
[exp] server_pid=35011
[exp] waiting for health
[exp] health ok after 18 checks
[exp] running benchmark
done 1/2 ok=True
done 2/2 ok=True
{
"created_at": "2026-07-15T03:25:55",
"url": "http://127.0.0.1:1111",
"model": "llm",
"dataset": "/root/work/logs/synthetic_cumulative_8_s3_cap40k.jsonl",
"concurrency": 1,
"max_requests": 2,
"max_tokens": 256,
"wall_sec": 94.4211983308196,
"success_rate": 1.0,
"ttft_p90_sec": 19.735469196736812,
"output_tps_p10_per_request": 5.978430347357755,
"aggregate_output_tps": 4.522289565780959,
"aggregate_input_tps_uncached": 362.06911799848626,
"aggregate_cache_tps": 79.30422545332043,
"cache_hit_rate": 0.1796760647870426,
"weighted_token_throughput": 1133.7982030784794,
"totals": {
"requests": 2,
"success": 2,
"prompt_tokens": 41675,
"input_tokens_uncached": 34187,
"cached_tokens": 7488,
"completion_tokens": 427,
"reasoning_tokens": 341
},
"targets": {
"output_tps_p10_per_request_gte_20": false,
"ttft_p90_lte_5": false,
"cache_hit_rate_gte_50pct": false,
"success_rate_gte_99pct": true,
"weighted_token_throughput_gte_8000": false
}
}
RESULT_FILE /root/work/logs/moe_bucket_profile_20260715/eager_100k_b16384_c1_r2_t256_cap40k.json
[exp] benchmark rc=0
{
"created_at": "2026-07-15T03:25:55",
"url": "http://127.0.0.1:1111",
"model": "llm",
"dataset": "/root/work/logs/synthetic_cumulative_8_s3_cap40k.jsonl",
"concurrency": 1,
"max_requests": 2,
"max_tokens": 256,
"wall_sec": 94.4211983308196,
"success_rate": 1.0,
"ttft_p90_sec": 19.735469196736812,
"output_tps_p10_per_request": 5.978430347357755,
"aggregate_output_tps": 4.522289565780959,
"aggregate_input_tps_uncached": 362.06911799848626,
"aggregate_cache_tps": 79.30422545332043,
"cache_hit_rate": 0.1796760647870426,
"weighted_token_throughput": 1133.7982030784794,
"totals": {
"requests": 2,
"success": 2,
"prompt_tokens": 41675,
"input_tokens_uncached": 34187,
"cached_tokens": 7488,
"completion_tokens": 427,
"reasoning_tokens": 341
},
"targets": {
"output_tps_p10_per_request_gte_20": false,
"ttft_p90_lte_5": false,
"cache_hit_rate_gte_50pct": false,
"success_rate_gte_99pct": true,
"weighted_token_throughput_gte_8000": false
},
"results": [
{
"ok": true,
"elapsed_sec": 28.944867059588432,
"ttft_sec": 7.041499247774482,
"prompt_tokens": 7498,
"cached_tokens": 0,
"completion_tokens": 171,
"reasoning_tokens": 85,
"output_tps": 7.807018604133026,
"error": null
},
{
"ok": true,
"elapsed_sec": 65.47296597249806,
"ttft_sec": 21.14591030217707,
"prompt_tokens": 34177,
"cached_tokens": 7488,
"completion_tokens": 256,
"reasoning_tokens": 256,
"output_tps": 5.775253874382725,
"error": null
}
]
}[exp] gpu after benchmark
Timestamp Wed Jul 15 03:25:56 2026
+-----------------------------------------------------------------------------+
| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |
|-------------------------------+----------------------+----------------------|
| GPU Name | Bus-Id | Clock-SM Clock-Mem |
| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |
|===============================+======================+======================|
| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |
| 0% 33C P0 62W / 250W | 30143MiB / 32768MiB | 0% Default |
+-------------------------------+----------------------+----------------------+
| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |
| 0% 36C P0 61W / 250W | 30082MiB / 32768MiB | 0% Default |
+-------------------------------+----------------------+----------------------+
| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |
| 0% 34C P0 60W / 250W | 30086MiB / 32768MiB | 0% Default |
+-------------------------------+----------------------+----------------------+
| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |
| 0% 34C P0 59W / 250W | 29946MiB / 32768MiB | 0% Default |
+-------------------------------+----------------------+----------------------+
+-----------------------------------------------------------------------------+
| Processes: GPU Memory |
| GPU PID Process name Usage(MiB) |
|=============================================================================|
+-----------------------------------------------------------------------------+

158
worklogs/remote_results/2026-07-15-official-like-baseline/driver_moe_profile.log Normal file
View File

@@ -0,0 +1,158 @@
[exp] label=moe_summary_profile_20260715
[exp] max_num_batched_tokens=16384
[exp] out_dir=/root/work/logs/moe_summary_profile_20260715
Wed Jul 15 02:49:29 UTC 2026
[exp] stopping existing api_server
[exp] gpu before start
Timestamp Wed Jul 15 02:49:37 2026
+-----------------------------------------------------------------------------+
| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |
|-------------------------------+----------------------+----------------------|
| GPU Name | Bus-Id | Clock-SM Clock-Mem |
| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |
|===============================+======================+======================|
| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |
| 0% 29C P0 47W / 250W | 257MiB / 32768MiB | 0% Default |
+-------------------------------+----------------------+----------------------+
| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |
| 0% 32C P0 47W / 250W | 257MiB / 32768MiB | 0% Default |
+-------------------------------+----------------------+----------------------+
| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |
| 0% 31C P0 47W / 250W | 257MiB / 32768MiB | 0% Default |
+-------------------------------+----------------------+----------------------+
| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |
| 0% 30C P0 47W / 250W | 257MiB / 32768MiB | 0% Default |
+-------------------------------+----------------------+----------------------+
+-----------------------------------------------------------------------------+
| Processes: GPU Memory |
| GPU PID Process name Usage(MiB) |
|=============================================================================|
| No running processes found |
+-----------------------------------------------------------------------------+
[exp] starting server
[exp] server_pid=33749
[exp] waiting for health
[exp] health ok after 18 checks
[exp] running benchmark
done 1/2 ok=True
done 2/2 ok=True
{
"created_at": "2026-07-15T02:52:28",
"url": "http://127.0.0.1:1111",
"model": "llm",
"dataset": "/root/work/logs/synthetic_cumulative_8_s3_cap40k.jsonl",
"concurrency": 1,
"max_requests": 2,
"max_tokens": 256,
"wall_sec": 84.45088295079768,
"success_rate": 1.0,
"ttft_p90_sec": 20.33267523739487,
"output_tps_p10_per_request": 5.792271822208015,
"aggregate_output_tps": 4.4641333142679605,
"aggregate_input_tps_uncached": 404.8151873073707,
"aggregate_cache_tps": 88.66692375925328,
"cache_hit_rate": 0.1796760647870426,
"weighted_token_throughput": 1257.7107697249571,
"totals": {
"requests": 2,
"success": 2,
"prompt_tokens": 41675,
"input_tokens_uncached": 34187,
"cached_tokens": 7488,
"completion_tokens": 377,
"reasoning_tokens": 285
},
"targets": {
"output_tps_p10_per_request_gte_20": false,
"ttft_p90_lte_5": false,
"cache_hit_rate_gte_50pct": false,
"success_rate_gte_99pct": true,
"weighted_token_throughput_gte_8000": false
}
}
RESULT_FILE /root/work/logs/moe_summary_profile_20260715/eager_100k_b16384_c1_r2_t256_cap40k.json
[exp] benchmark rc=0
{
"created_at": "2026-07-15T02:52:28",
"url": "http://127.0.0.1:1111",
"model": "llm",
"dataset": "/root/work/logs/synthetic_cumulative_8_s3_cap40k.jsonl",
"concurrency": 1,
"max_requests": 2,
"max_tokens": 256,
"wall_sec": 84.45088295079768,
"success_rate": 1.0,
"ttft_p90_sec": 20.33267523739487,
"output_tps_p10_per_request": 5.792271822208015,
"aggregate_output_tps": 4.4641333142679605,
"aggregate_input_tps_uncached": 404.8151873073707,
"aggregate_cache_tps": 88.66692375925328,
"cache_hit_rate": 0.1796760647870426,
"weighted_token_throughput": 1257.7107697249571,
"totals": {
"requests": 2,
"success": 2,
"prompt_tokens": 41675,
"input_tokens_uncached": 34187,
"cached_tokens": 7488,
"completion_tokens": 377,
"reasoning_tokens": 285
},
"targets": {
"output_tps_p10_per_request_gte_20": false,
"ttft_p90_lte_5": false,
"cache_hit_rate_gte_50pct": false,
"success_rate_gte_99pct": true,
"weighted_token_throughput_gte_8000": false
},
"results": [
{
"ok": true,
"elapsed_sec": 41.03028617054224,
"ttft_sec": 7.251164922490716,
"prompt_tokens": 7498,
"cached_tokens": 0,
"completion_tokens": 256,
"reasoning_tokens": 210,
"output_tps": 7.578645936941501,
"error": null
},
{
"ok": true,
"elapsed_sec": 43.417322823777795,
"ttft_sec": 21.78617638349533,
"prompt_tokens": 34177,
"cached_tokens": 7488,
"completion_tokens": 121,
"reasoning_tokens": 75,
"output_tps": 5.59378580945985,
"error": null
}
]
}[exp] gpu after benchmark
Timestamp Wed Jul 15 02:52:28 2026
+-----------------------------------------------------------------------------+
| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |
|-------------------------------+----------------------+----------------------|
| GPU Name | Bus-Id | Clock-SM Clock-Mem |
| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |
|===============================+======================+======================|
| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |
| 0% 33C P0 61W / 250W | 30143MiB / 32768MiB | 0% Default |
+-------------------------------+----------------------+----------------------+
| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |
| 0% 35C P0 60W / 250W | 30082MiB / 32768MiB | 0% Default |
+-------------------------------+----------------------+----------------------+
| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |
| 0% 35C P0 60W / 250W | 30086MiB / 32768MiB | 0% Default |
+-------------------------------+----------------------+----------------------+
| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |
| 0% 34C P0 58W / 250W | 29946MiB / 32768MiB | 0% Default |
+-------------------------------+----------------------+----------------------+
+-----------------------------------------------------------------------------+
| Processes: GPU Memory |
| GPU PID Process name Usage(MiB) |
|=============================================================================|
+-----------------------------------------------------------------------------+

58
worklogs/remote_results/2026-07-15-official-like-baseline/eager_100k_b16384_moe_bucket_profile_r2_t256_cap40k.json Normal file
View File

@@ -0,0 +1,58 @@
{
"created_at": "2026-07-15T03:25:55",
"url": "http://127.0.0.1:1111",
"model": "llm",
"dataset": "/root/work/logs/synthetic_cumulative_8_s3_cap40k.jsonl",
"concurrency": 1,
"max_requests": 2,
"max_tokens": 256,
"wall_sec": 94.4211983308196,
"success_rate": 1.0,
"ttft_p90_sec": 19.735469196736812,
"output_tps_p10_per_request": 5.978430347357755,
"aggregate_output_tps": 4.522289565780959,
"aggregate_input_tps_uncached": 362.06911799848626,
"aggregate_cache_tps": 79.30422545332043,
"cache_hit_rate": 0.1796760647870426,
"weighted_token_throughput": 1133.7982030784794,
"totals": {
"requests": 2,
"success": 2,
"prompt_tokens": 41675,
"input_tokens_uncached": 34187,
"cached_tokens": 7488,
"completion_tokens": 427,
"reasoning_tokens": 341
},
"targets": {
"output_tps_p10_per_request_gte_20": false,
"ttft_p90_lte_5": false,
"cache_hit_rate_gte_50pct": false,
"success_rate_gte_99pct": true,
"weighted_token_throughput_gte_8000": false
},
"results": [
{
"ok": true,
"elapsed_sec": 28.944867059588432,
"ttft_sec": 7.041499247774482,
"prompt_tokens": 7498,
"cached_tokens": 0,
"completion_tokens": 171,
"reasoning_tokens": 85,
"output_tps": 7.807018604133026,
"error": null
},
{
"ok": true,
"elapsed_sec": 65.47296597249806,
"ttft_sec": 21.14591030217707,
"prompt_tokens": 34177,
"cached_tokens": 7488,
"completion_tokens": 256,
"reasoning_tokens": 256,
"output_tps": 5.775253874382725,
"error": null
}
]
}

58
worklogs/remote_results/2026-07-15-official-like-baseline/eager_100k_b16384_moe_profile_r2_t256_cap40k.json Normal file
View File

@@ -0,0 +1,58 @@
{
"created_at": "2026-07-15T02:52:28",
"url": "http://127.0.0.1:1111",
"model": "llm",
"dataset": "/root/work/logs/synthetic_cumulative_8_s3_cap40k.jsonl",
"concurrency": 1,
"max_requests": 2,
"max_tokens": 256,
"wall_sec": 84.45088295079768,
"success_rate": 1.0,
"ttft_p90_sec": 20.33267523739487,
"output_tps_p10_per_request": 5.792271822208015,
"aggregate_output_tps": 4.4641333142679605,
"aggregate_input_tps_uncached": 404.8151873073707,
"aggregate_cache_tps": 88.66692375925328,
"cache_hit_rate": 0.1796760647870426,
"weighted_token_throughput": 1257.7107697249571,
"totals": {
"requests": 2,
"success": 2,
"prompt_tokens": 41675,
"input_tokens_uncached": 34187,
"cached_tokens": 7488,
"completion_tokens": 377,
"reasoning_tokens": 285
},
"targets": {
"output_tps_p10_per_request_gte_20": false,
"ttft_p90_lte_5": false,
"cache_hit_rate_gte_50pct": false,
"success_rate_gte_99pct": true,
"weighted_token_throughput_gte_8000": false
},
"results": [
{
"ok": true,
"elapsed_sec": 41.03028617054224,
"ttft_sec": 7.251164922490716,
"prompt_tokens": 7498,
"cached_tokens": 0,
"completion_tokens": 256,
"reasoning_tokens": 210,
"output_tps": 7.578645936941501,
"error": null
},
{
"ok": true,
"elapsed_sec": 43.417322823777795,
"ttft_sec": 21.78617638349533,
"prompt_tokens": 34177,
"cached_tokens": 7488,
"completion_tokens": 121,
"reasoning_tokens": 75,
"output_tps": 5.59378580945985,
"error": null
}
]
}

856
worklogs/remote_results/2026-07-15-official-like-baseline/moe_bucket_summary_profile.log Normal file
View File

@@ -0,0 +1,856 @@
[PROF_MOE_SUMMARY] pid=35011 bucket=prefill calls=80 tokens=955280 router_gate_avg_ms=0.540 routed_avg_ms=76.181 shared_avg_ms=2.020 all_reduce_avg_ms=4.074 total_avg_ms=82.827
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=prefill calls=80 tokens=955280 router_gate_avg_ms=0.540 routed_avg_ms=76.152 shared_avg_ms=1.999 all_reduce_avg_ms=4.067 total_avg_ms=82.769
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=prefill calls=80 tokens=955280 router_gate_avg_ms=0.538 routed_avg_ms=76.156 shared_avg_ms=2.001 all_reduce_avg_ms=4.079 total_avg_ms=82.785
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=prefill calls=80 tokens=955280 router_gate_avg_ms=0.539 routed_avg_ms=76.134 shared_avg_ms=2.001 all_reduce_avg_ms=4.133 total_avg_ms=82.819
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=80 tokens=80 router_gate_avg_ms=0.070 routed_avg_ms=0.778 shared_avg_ms=0.241 all_reduce_avg_ms=0.330 total_avg_ms=1.432
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=80 tokens=80 router_gate_avg_ms=0.070 routed_avg_ms=0.776 shared_avg_ms=0.243 all_reduce_avg_ms=0.326 total_avg_ms=1.426
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=80 tokens=80 router_gate_avg_ms=0.070 routed_avg_ms=0.775 shared_avg_ms=0.236 all_reduce_avg_ms=0.343 total_avg_ms=1.437
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=80 tokens=80 router_gate_avg_ms=0.071 routed_avg_ms=0.780 shared_avg_ms=0.240 all_reduce_avg_ms=0.326 total_avg_ms=1.429
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=160 tokens=160 router_gate_avg_ms=0.069 routed_avg_ms=0.772 shared_avg_ms=0.236 all_reduce_avg_ms=0.323 total_avg_ms=1.413
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=160 tokens=160 router_gate_avg_ms=0.069 routed_avg_ms=0.772 shared_avg_ms=0.240 all_reduce_avg_ms=0.315 total_avg_ms=1.408
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=160 tokens=160 router_gate_avg_ms=0.069 routed_avg_ms=0.771 shared_avg_ms=0.233 all_reduce_avg_ms=0.331 total_avg_ms=1.416
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=160 tokens=160 router_gate_avg_ms=0.069 routed_avg_ms=0.774 shared_avg_ms=0.235 all_reduce_avg_ms=0.322 total_avg_ms=1.412
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=240 tokens=240 router_gate_avg_ms=0.069 routed_avg_ms=0.770 shared_avg_ms=0.235 all_reduce_avg_ms=0.324 total_avg_ms=1.410
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=240 tokens=240 router_gate_avg_ms=0.069 routed_avg_ms=0.769 shared_avg_ms=0.231 all_reduce_avg_ms=0.332 total_avg_ms=1.413
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=240 tokens=240 router_gate_avg_ms=0.069 routed_avg_ms=0.772 shared_avg_ms=0.240 all_reduce_avg_ms=0.312 total_avg_ms=1.404
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=240 tokens=240 router_gate_avg_ms=0.069 routed_avg_ms=0.772 shared_avg_ms=0.233 all_reduce_avg_ms=0.324 total_avg_ms=1.410
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=320 tokens=320 router_gate_avg_ms=0.068 routed_avg_ms=0.769 shared_avg_ms=0.233 all_reduce_avg_ms=0.320 total_avg_ms=1.402
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=320 tokens=320 router_gate_avg_ms=0.068 routed_avg_ms=0.768 shared_avg_ms=0.230 all_reduce_avg_ms=0.326 total_avg_ms=1.405
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=320 tokens=320 router_gate_avg_ms=0.068 routed_avg_ms=0.770 shared_avg_ms=0.238 all_reduce_avg_ms=0.309 total_avg_ms=1.397
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=320 tokens=320 router_gate_avg_ms=0.068 routed_avg_ms=0.770 shared_avg_ms=0.232 all_reduce_avg_ms=0.321 total_avg_ms=1.403
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=400 tokens=400 router_gate_avg_ms=0.068 routed_avg_ms=0.768 shared_avg_ms=0.229 all_reduce_avg_ms=0.323 total_avg_ms=1.400
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=400 tokens=400 router_gate_avg_ms=0.068 routed_avg_ms=0.768 shared_avg_ms=0.233 all_reduce_avg_ms=0.317 total_avg_ms=1.397
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=400 tokens=400 router_gate_avg_ms=0.067 routed_avg_ms=0.769 shared_avg_ms=0.237 all_reduce_avg_ms=0.307 total_avg_ms=1.392
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=400 tokens=400 router_gate_avg_ms=0.068 routed_avg_ms=0.769 shared_avg_ms=0.231 all_reduce_avg_ms=0.319 total_avg_ms=1.398
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=480 tokens=480 router_gate_avg_ms=0.067 routed_avg_ms=0.767 shared_avg_ms=0.229 all_reduce_avg_ms=0.330 total_avg_ms=1.405
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=480 tokens=480 router_gate_avg_ms=0.067 routed_avg_ms=0.768 shared_avg_ms=0.230 all_reduce_avg_ms=0.325 total_avg_ms=1.403
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=480 tokens=480 router_gate_avg_ms=0.068 routed_avg_ms=0.771 shared_avg_ms=0.238 all_reduce_avg_ms=0.307 total_avg_ms=1.395
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=480 tokens=480 router_gate_avg_ms=0.068 routed_avg_ms=0.768 shared_avg_ms=0.233 all_reduce_avg_ms=0.320 total_avg_ms=1.401
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=560 tokens=560 router_gate_avg_ms=0.069 routed_avg_ms=0.769 shared_avg_ms=0.233 all_reduce_avg_ms=0.321 total_avg_ms=1.403
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=560 tokens=560 router_gate_avg_ms=0.068 routed_avg_ms=0.767 shared_avg_ms=0.229 all_reduce_avg_ms=0.331 total_avg_ms=1.407
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=560 tokens=560 router_gate_avg_ms=0.069 routed_avg_ms=0.772 shared_avg_ms=0.239 all_reduce_avg_ms=0.309 total_avg_ms=1.401
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=560 tokens=560 router_gate_avg_ms=0.073 routed_avg_ms=0.769 shared_avg_ms=0.231 all_reduce_avg_ms=0.326 total_avg_ms=1.410
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=640 tokens=640 router_gate_avg_ms=0.069 routed_avg_ms=0.769 shared_avg_ms=0.234 all_reduce_avg_ms=0.325 total_avg_ms=1.408
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=640 tokens=640 router_gate_avg_ms=0.069 routed_avg_ms=0.772 shared_avg_ms=0.241 all_reduce_avg_ms=0.310 total_avg_ms=1.405
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=640 tokens=640 router_gate_avg_ms=0.068 routed_avg_ms=0.767 shared_avg_ms=0.231 all_reduce_avg_ms=0.334 total_avg_ms=1.412
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=640 tokens=640 router_gate_avg_ms=0.073 routed_avg_ms=0.771 shared_avg_ms=0.234 all_reduce_avg_ms=0.325 total_avg_ms=1.414
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=720 tokens=720 router_gate_avg_ms=0.070 routed_avg_ms=0.772 shared_avg_ms=0.241 all_reduce_avg_ms=0.311 total_avg_ms=1.405
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=720 tokens=720 router_gate_avg_ms=0.069 routed_avg_ms=0.768 shared_avg_ms=0.234 all_reduce_avg_ms=0.327 total_avg_ms=1.410
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=720 tokens=720 router_gate_avg_ms=0.068 routed_avg_ms=0.767 shared_avg_ms=0.231 all_reduce_avg_ms=0.336 total_avg_ms=1.414
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=720 tokens=720 router_gate_avg_ms=0.073 routed_avg_ms=0.772 shared_avg_ms=0.235 all_reduce_avg_ms=0.323 total_avg_ms=1.414
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=800 tokens=800 router_gate_avg_ms=0.070 routed_avg_ms=0.772 shared_avg_ms=0.240 all_reduce_avg_ms=0.312 total_avg_ms=1.406
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=800 tokens=800 router_gate_avg_ms=0.069 routed_avg_ms=0.768 shared_avg_ms=0.233 all_reduce_avg_ms=0.329 total_avg_ms=1.411
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=800 tokens=800 router_gate_avg_ms=0.068 routed_avg_ms=0.767 shared_avg_ms=0.230 all_reduce_avg_ms=0.337 total_avg_ms=1.415
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=800 tokens=800 router_gate_avg_ms=0.073 routed_avg_ms=0.773 shared_avg_ms=0.235 all_reduce_avg_ms=0.321 total_avg_ms=1.414
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=880 tokens=880 router_gate_avg_ms=0.069 routed_avg_ms=0.768 shared_avg_ms=0.233 all_reduce_avg_ms=0.330 total_avg_ms=1.411
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=880 tokens=880 router_gate_avg_ms=0.068 routed_avg_ms=0.767 shared_avg_ms=0.230 all_reduce_avg_ms=0.339 total_avg_ms=1.415
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=880 tokens=880 router_gate_avg_ms=0.070 routed_avg_ms=0.772 shared_avg_ms=0.240 all_reduce_avg_ms=0.313 total_avg_ms=1.406
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=880 tokens=880 router_gate_avg_ms=0.072 routed_avg_ms=0.773 shared_avg_ms=0.235 all_reduce_avg_ms=0.320 total_avg_ms=1.413
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=960 tokens=960 router_gate_avg_ms=0.070 routed_avg_ms=0.771 shared_avg_ms=0.239 all_reduce_avg_ms=0.313 total_avg_ms=1.405
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=960 tokens=960 router_gate_avg_ms=0.068 routed_avg_ms=0.766 shared_avg_ms=0.230 all_reduce_avg_ms=0.339 total_avg_ms=1.415
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=960 tokens=960 router_gate_avg_ms=0.068 routed_avg_ms=0.768 shared_avg_ms=0.233 all_reduce_avg_ms=0.330 total_avg_ms=1.411
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=960 tokens=960 router_gate_avg_ms=0.072 routed_avg_ms=0.774 shared_avg_ms=0.235 all_reduce_avg_ms=0.319 total_avg_ms=1.412
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=1040 tokens=1040 router_gate_avg_ms=0.070 routed_avg_ms=0.771 shared_avg_ms=0.240 all_reduce_avg_ms=0.313 total_avg_ms=1.406
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=1040 tokens=1040 router_gate_avg_ms=0.068 routed_avg_ms=0.767 shared_avg_ms=0.233 all_reduce_avg_ms=0.332 total_avg_ms=1.412
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=1040 tokens=1040 router_gate_avg_ms=0.068 routed_avg_ms=0.766 shared_avg_ms=0.230 all_reduce_avg_ms=0.340 total_avg_ms=1.415
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=1040 tokens=1040 router_gate_avg_ms=0.072 routed_avg_ms=0.774 shared_avg_ms=0.237 all_reduce_avg_ms=0.318 total_avg_ms=1.412
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=1120 tokens=1120 router_gate_avg_ms=0.070 routed_avg_ms=0.771 shared_avg_ms=0.240 all_reduce_avg_ms=0.314 total_avg_ms=1.406
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=1120 tokens=1120 router_gate_avg_ms=0.068 routed_avg_ms=0.767 shared_avg_ms=0.233 all_reduce_avg_ms=0.332 total_avg_ms=1.412
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=1120 tokens=1120 router_gate_avg_ms=0.068 routed_avg_ms=0.766 shared_avg_ms=0.230 all_reduce_avg_ms=0.340 total_avg_ms=1.416
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=1120 tokens=1120 router_gate_avg_ms=0.072 routed_avg_ms=0.775 shared_avg_ms=0.237 all_reduce_avg_ms=0.317 total_avg_ms=1.412
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=1200 tokens=1200 router_gate_avg_ms=0.070 routed_avg_ms=0.771 shared_avg_ms=0.239 all_reduce_avg_ms=0.315 total_avg_ms=1.407
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=1200 tokens=1200 router_gate_avg_ms=0.068 routed_avg_ms=0.767 shared_avg_ms=0.233 all_reduce_avg_ms=0.334 total_avg_ms=1.414
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=1200 tokens=1200 router_gate_avg_ms=0.068 routed_avg_ms=0.766 shared_avg_ms=0.230 all_reduce_avg_ms=0.342 total_avg_ms=1.417
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=1200 tokens=1200 router_gate_avg_ms=0.072 routed_avg_ms=0.775 shared_avg_ms=0.237 all_reduce_avg_ms=0.318 total_avg_ms=1.413
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=1280 tokens=1280 router_gate_avg_ms=0.070 routed_avg_ms=0.771 shared_avg_ms=0.239 all_reduce_avg_ms=0.318 total_avg_ms=1.411
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=1280 tokens=1280 router_gate_avg_ms=0.068 routed_avg_ms=0.767 shared_avg_ms=0.233 all_reduce_avg_ms=0.338 total_avg_ms=1.418
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=1280 tokens=1280 router_gate_avg_ms=0.072 routed_avg_ms=0.776 shared_avg_ms=0.237 all_reduce_avg_ms=0.320 total_avg_ms=1.417
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=1280 tokens=1280 router_gate_avg_ms=0.068 routed_avg_ms=0.767 shared_avg_ms=0.230 all_reduce_avg_ms=0.343 total_avg_ms=1.421
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=1360 tokens=1360 router_gate_avg_ms=0.070 routed_avg_ms=0.772 shared_avg_ms=0.240 all_reduce_avg_ms=0.322 total_avg_ms=1.416
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=1360 tokens=1360 router_gate_avg_ms=0.068 routed_avg_ms=0.769 shared_avg_ms=0.231 all_reduce_avg_ms=0.345 total_avg_ms=1.426
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=1360 tokens=1360 router_gate_avg_ms=0.072 routed_avg_ms=0.776 shared_avg_ms=0.238 all_reduce_avg_ms=0.325 total_avg_ms=1.422
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=1360 tokens=1360 router_gate_avg_ms=0.069 routed_avg_ms=0.769 shared_avg_ms=0.234 all_reduce_avg_ms=0.339 total_avg_ms=1.423
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=1440 tokens=1440 router_gate_avg_ms=0.069 routed_avg_ms=0.769 shared_avg_ms=0.232 all_reduce_avg_ms=0.349 total_avg_ms=1.430
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=1440 tokens=1440 router_gate_avg_ms=0.072 routed_avg_ms=0.778 shared_avg_ms=0.239 all_reduce_avg_ms=0.325 total_avg_ms=1.426
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=1440 tokens=1440 router_gate_avg_ms=0.069 routed_avg_ms=0.769 shared_avg_ms=0.235 all_reduce_avg_ms=0.343 total_avg_ms=1.427
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=1440 tokens=1440 router_gate_avg_ms=0.071 routed_avg_ms=0.773 shared_avg_ms=0.241 all_reduce_avg_ms=0.322 total_avg_ms=1.419
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=1520 tokens=1520 router_gate_avg_ms=0.069 routed_avg_ms=0.770 shared_avg_ms=0.235 all_reduce_avg_ms=0.342 total_avg_ms=1.428
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=1520 tokens=1520 router_gate_avg_ms=0.069 routed_avg_ms=0.770 shared_avg_ms=0.232 all_reduce_avg_ms=0.349 total_avg_ms=1.431
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=1520 tokens=1520 router_gate_avg_ms=0.071 routed_avg_ms=0.774 shared_avg_ms=0.242 all_reduce_avg_ms=0.323 total_avg_ms=1.421
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=1520 tokens=1520 router_gate_avg_ms=0.072 routed_avg_ms=0.778 shared_avg_ms=0.239 all_reduce_avg_ms=0.326 total_avg_ms=1.427
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=1600 tokens=1600 router_gate_avg_ms=0.069 routed_avg_ms=0.770 shared_avg_ms=0.232 all_reduce_avg_ms=0.351 total_avg_ms=1.434
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=1600 tokens=1600 router_gate_avg_ms=0.071 routed_avg_ms=0.774 shared_avg_ms=0.243 all_reduce_avg_ms=0.322 total_avg_ms=1.423
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=1600 tokens=1600 router_gate_avg_ms=0.069 routed_avg_ms=0.770 shared_avg_ms=0.235 all_reduce_avg_ms=0.344 total_avg_ms=1.430
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=1600 tokens=1600 router_gate_avg_ms=0.072 routed_avg_ms=0.778 shared_avg_ms=0.239 all_reduce_avg_ms=0.328 total_avg_ms=1.429
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=1680 tokens=1680 router_gate_avg_ms=0.069 routed_avg_ms=0.770 shared_avg_ms=0.236 all_reduce_avg_ms=0.347 total_avg_ms=1.433
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=1680 tokens=1680 router_gate_avg_ms=0.069 routed_avg_ms=0.770 shared_avg_ms=0.232 all_reduce_avg_ms=0.354 total_avg_ms=1.437
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=1680 tokens=1680 router_gate_avg_ms=0.072 routed_avg_ms=0.775 shared_avg_ms=0.243 all_reduce_avg_ms=0.322 total_avg_ms=1.425
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=1680 tokens=1680 router_gate_avg_ms=0.072 routed_avg_ms=0.778 shared_avg_ms=0.239 all_reduce_avg_ms=0.331 total_avg_ms=1.432
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=1760 tokens=1760 router_gate_avg_ms=0.070 routed_avg_ms=0.771 shared_avg_ms=0.236 all_reduce_avg_ms=0.348 total_avg_ms=1.436
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=1760 tokens=1760 router_gate_avg_ms=0.069 routed_avg_ms=0.770 shared_avg_ms=0.233 all_reduce_avg_ms=0.357 total_avg_ms=1.441
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=1760 tokens=1760 router_gate_avg_ms=0.072 routed_avg_ms=0.776 shared_avg_ms=0.244 all_reduce_avg_ms=0.324 total_avg_ms=1.428
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=1760 tokens=1760 router_gate_avg_ms=0.072 routed_avg_ms=0.779 shared_avg_ms=0.240 all_reduce_avg_ms=0.333 total_avg_ms=1.436
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=1840 tokens=1840 router_gate_avg_ms=0.070 routed_avg_ms=0.771 shared_avg_ms=0.236 all_reduce_avg_ms=0.349 total_avg_ms=1.437
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=1840 tokens=1840 router_gate_avg_ms=0.072 routed_avg_ms=0.779 shared_avg_ms=0.240 all_reduce_avg_ms=0.335 total_avg_ms=1.437
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=1840 tokens=1840 router_gate_avg_ms=0.069 routed_avg_ms=0.770 shared_avg_ms=0.233 all_reduce_avg_ms=0.358 total_avg_ms=1.442
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=1840 tokens=1840 router_gate_avg_ms=0.072 routed_avg_ms=0.777 shared_avg_ms=0.245 all_reduce_avg_ms=0.324 total_avg_ms=1.430
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=1920 tokens=1920 router_gate_avg_ms=0.072 routed_avg_ms=0.777 shared_avg_ms=0.245 all_reduce_avg_ms=0.323 total_avg_ms=1.429
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=1920 tokens=1920 router_gate_avg_ms=0.070 routed_avg_ms=0.771 shared_avg_ms=0.236 all_reduce_avg_ms=0.348 total_avg_ms=1.437
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=1920 tokens=1920 router_gate_avg_ms=0.072 routed_avg_ms=0.779 shared_avg_ms=0.240 all_reduce_avg_ms=0.334 total_avg_ms=1.436
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=1920 tokens=1920 router_gate_avg_ms=0.069 routed_avg_ms=0.770 shared_avg_ms=0.233 all_reduce_avg_ms=0.357 total_avg_ms=1.441
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=2000 tokens=2000 router_gate_avg_ms=0.072 routed_avg_ms=0.777 shared_avg_ms=0.245 all_reduce_avg_ms=0.324 total_avg_ms=1.429
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=2000 tokens=2000 router_gate_avg_ms=0.070 routed_avg_ms=0.771 shared_avg_ms=0.237 all_reduce_avg_ms=0.347 total_avg_ms=1.436
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=2000 tokens=2000 router_gate_avg_ms=0.069 routed_avg_ms=0.771 shared_avg_ms=0.233 all_reduce_avg_ms=0.356 total_avg_ms=1.441
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=2000 tokens=2000 router_gate_avg_ms=0.072 routed_avg_ms=0.779 shared_avg_ms=0.240 all_reduce_avg_ms=0.334 total_avg_ms=1.436
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=2080 tokens=2080 router_gate_avg_ms=0.070 routed_avg_ms=0.772 shared_avg_ms=0.237 all_reduce_avg_ms=0.346 total_avg_ms=1.436
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=2080 tokens=2080 router_gate_avg_ms=0.069 routed_avg_ms=0.771 shared_avg_ms=0.233 all_reduce_avg_ms=0.355 total_avg_ms=1.441
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=2080 tokens=2080 router_gate_avg_ms=0.072 routed_avg_ms=0.779 shared_avg_ms=0.240 all_reduce_avg_ms=0.334 total_avg_ms=1.437
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=2080 tokens=2080 router_gate_avg_ms=0.072 routed_avg_ms=0.777 shared_avg_ms=0.245 all_reduce_avg_ms=0.324 total_avg_ms=1.429
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=2160 tokens=2160 router_gate_avg_ms=0.072 routed_avg_ms=0.777 shared_avg_ms=0.245 all_reduce_avg_ms=0.324 total_avg_ms=1.430
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=2160 tokens=2160 router_gate_avg_ms=0.072 routed_avg_ms=0.779 shared_avg_ms=0.240 all_reduce_avg_ms=0.335 total_avg_ms=1.437
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=2160 tokens=2160 router_gate_avg_ms=0.069 routed_avg_ms=0.772 shared_avg_ms=0.234 all_reduce_avg_ms=0.354 total_avg_ms=1.441
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=2160 tokens=2160 router_gate_avg_ms=0.070 routed_avg_ms=0.772 shared_avg_ms=0.237 all_reduce_avg_ms=0.345 total_avg_ms=1.437
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=2240 tokens=2240 router_gate_avg_ms=0.072 routed_avg_ms=0.778 shared_avg_ms=0.240 all_reduce_avg_ms=0.335 total_avg_ms=1.437
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=2240 tokens=2240 router_gate_avg_ms=0.072 routed_avg_ms=0.777 shared_avg_ms=0.245 all_reduce_avg_ms=0.324 total_avg_ms=1.429
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=2240 tokens=2240 router_gate_avg_ms=0.070 routed_avg_ms=0.772 shared_avg_ms=0.237 all_reduce_avg_ms=0.345 total_avg_ms=1.436
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=2240 tokens=2240 router_gate_avg_ms=0.070 routed_avg_ms=0.772 shared_avg_ms=0.234 all_reduce_avg_ms=0.353 total_avg_ms=1.440
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=2320 tokens=2320 router_gate_avg_ms=0.072 routed_avg_ms=0.777 shared_avg_ms=0.245 all_reduce_avg_ms=0.323 total_avg_ms=1.428
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=2320 tokens=2320 router_gate_avg_ms=0.072 routed_avg_ms=0.778 shared_avg_ms=0.239 all_reduce_avg_ms=0.335 total_avg_ms=1.436
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=2320 tokens=2320 router_gate_avg_ms=0.070 routed_avg_ms=0.772 shared_avg_ms=0.237 all_reduce_avg_ms=0.345 total_avg_ms=1.436
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=2320 tokens=2320 router_gate_avg_ms=0.070 routed_avg_ms=0.772 shared_avg_ms=0.234 all_reduce_avg_ms=0.352 total_avg_ms=1.439
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=2400 tokens=2400 router_gate_avg_ms=0.070 routed_avg_ms=0.772 shared_avg_ms=0.237 all_reduce_avg_ms=0.347 total_avg_ms=1.437
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=2400 tokens=2400 router_gate_avg_ms=0.072 routed_avg_ms=0.772 shared_avg_ms=0.234 all_reduce_avg_ms=0.351 total_avg_ms=1.440
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=2400 tokens=2400 router_gate_avg_ms=0.072 routed_avg_ms=0.777 shared_avg_ms=0.244 all_reduce_avg_ms=0.325 total_avg_ms=1.430
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=2400 tokens=2400 router_gate_avg_ms=0.071 routed_avg_ms=0.778 shared_avg_ms=0.239 all_reduce_avg_ms=0.337 total_avg_ms=1.437
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=2480 tokens=2480 router_gate_avg_ms=0.070 routed_avg_ms=0.772 shared_avg_ms=0.237 all_reduce_avg_ms=0.346 total_avg_ms=1.436
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=2480 tokens=2480 router_gate_avg_ms=0.072 routed_avg_ms=0.772 shared_avg_ms=0.234 all_reduce_avg_ms=0.350 total_avg_ms=1.439
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=2480 tokens=2480 router_gate_avg_ms=0.072 routed_avg_ms=0.777 shared_avg_ms=0.244 all_reduce_avg_ms=0.324 total_avg_ms=1.429
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=2480 tokens=2480 router_gate_avg_ms=0.071 routed_avg_ms=0.777 shared_avg_ms=0.239 all_reduce_avg_ms=0.338 total_avg_ms=1.437
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=2560 tokens=2560 router_gate_avg_ms=0.072 routed_avg_ms=0.772 shared_avg_ms=0.234 all_reduce_avg_ms=0.350 total_avg_ms=1.439
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=2560 tokens=2560 router_gate_avg_ms=0.072 routed_avg_ms=0.777 shared_avg_ms=0.244 all_reduce_avg_ms=0.324 total_avg_ms=1.429
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=2560 tokens=2560 router_gate_avg_ms=0.071 routed_avg_ms=0.777 shared_avg_ms=0.239 all_reduce_avg_ms=0.338 total_avg_ms=1.437
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=2560 tokens=2560 router_gate_avg_ms=0.070 routed_avg_ms=0.771 shared_avg_ms=0.237 all_reduce_avg_ms=0.346 total_avg_ms=1.436
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=2640 tokens=2640 router_gate_avg_ms=0.072 routed_avg_ms=0.772 shared_avg_ms=0.233 all_reduce_avg_ms=0.350 total_avg_ms=1.439
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=2640 tokens=2640 router_gate_avg_ms=0.072 routed_avg_ms=0.776 shared_avg_ms=0.244 all_reduce_avg_ms=0.324 total_avg_ms=1.429
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=2640 tokens=2640 router_gate_avg_ms=0.071 routed_avg_ms=0.777 shared_avg_ms=0.238 all_reduce_avg_ms=0.338 total_avg_ms=1.436
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=2640 tokens=2640 router_gate_avg_ms=0.070 routed_avg_ms=0.772 shared_avg_ms=0.237 all_reduce_avg_ms=0.345 total_avg_ms=1.435
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=2720 tokens=2720 router_gate_avg_ms=0.071 routed_avg_ms=0.772 shared_avg_ms=0.233 all_reduce_avg_ms=0.350 total_avg_ms=1.438
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=2720 tokens=2720 router_gate_avg_ms=0.070 routed_avg_ms=0.772 shared_avg_ms=0.237 all_reduce_avg_ms=0.344 total_avg_ms=1.435
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=2720 tokens=2720 router_gate_avg_ms=0.072 routed_avg_ms=0.776 shared_avg_ms=0.244 all_reduce_avg_ms=0.324 total_avg_ms=1.428
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=2720 tokens=2720 router_gate_avg_ms=0.071 routed_avg_ms=0.777 shared_avg_ms=0.238 all_reduce_avg_ms=0.338 total_avg_ms=1.435
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=2800 tokens=2800 router_gate_avg_ms=0.070 routed_avg_ms=0.772 shared_avg_ms=0.237 all_reduce_avg_ms=0.343 total_avg_ms=1.433
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=2800 tokens=2800 router_gate_avg_ms=0.071 routed_avg_ms=0.772 shared_avg_ms=0.233 all_reduce_avg_ms=0.349 total_avg_ms=1.437
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=2800 tokens=2800 router_gate_avg_ms=0.072 routed_avg_ms=0.776 shared_avg_ms=0.243 all_reduce_avg_ms=0.324 total_avg_ms=1.427
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=2800 tokens=2800 router_gate_avg_ms=0.071 routed_avg_ms=0.776 shared_avg_ms=0.238 all_reduce_avg_ms=0.337 total_avg_ms=1.434
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=2880 tokens=2880 router_gate_avg_ms=0.070 routed_avg_ms=0.772 shared_avg_ms=0.236 all_reduce_avg_ms=0.343 total_avg_ms=1.432
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=2880 tokens=2880 router_gate_avg_ms=0.071 routed_avg_ms=0.772 shared_avg_ms=0.233 all_reduce_avg_ms=0.347 total_avg_ms=1.435
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=2880 tokens=2880 router_gate_avg_ms=0.072 routed_avg_ms=0.776 shared_avg_ms=0.243 all_reduce_avg_ms=0.323 total_avg_ms=1.425
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=2880 tokens=2880 router_gate_avg_ms=0.071 routed_avg_ms=0.776 shared_avg_ms=0.238 all_reduce_avg_ms=0.336 total_avg_ms=1.433
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=2960 tokens=2960 router_gate_avg_ms=0.071 routed_avg_ms=0.776 shared_avg_ms=0.243 all_reduce_avg_ms=0.323 total_avg_ms=1.425
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=2960 tokens=2960 router_gate_avg_ms=0.071 routed_avg_ms=0.772 shared_avg_ms=0.233 all_reduce_avg_ms=0.347 total_avg_ms=1.435
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=2960 tokens=2960 router_gate_avg_ms=0.070 routed_avg_ms=0.772 shared_avg_ms=0.236 all_reduce_avg_ms=0.342 total_avg_ms=1.431
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=2960 tokens=2960 router_gate_avg_ms=0.071 routed_avg_ms=0.776 shared_avg_ms=0.238 all_reduce_avg_ms=0.336 total_avg_ms=1.432
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=3040 tokens=3040 router_gate_avg_ms=0.071 routed_avg_ms=0.775 shared_avg_ms=0.242 all_reduce_avg_ms=0.323 total_avg_ms=1.424
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=3040 tokens=3040 router_gate_avg_ms=0.071 routed_avg_ms=0.776 shared_avg_ms=0.237 all_reduce_avg_ms=0.335 total_avg_ms=1.431
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=3040 tokens=3040 router_gate_avg_ms=0.070 routed_avg_ms=0.771 shared_avg_ms=0.236 all_reduce_avg_ms=0.342 total_avg_ms=1.430
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=3040 tokens=3040 router_gate_avg_ms=0.071 routed_avg_ms=0.772 shared_avg_ms=0.233 all_reduce_avg_ms=0.346 total_avg_ms=1.433
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=3120 tokens=3120 router_gate_avg_ms=0.071 routed_avg_ms=0.775 shared_avg_ms=0.242 all_reduce_avg_ms=0.324 total_avg_ms=1.424
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=3120 tokens=3120 router_gate_avg_ms=0.071 routed_avg_ms=0.776 shared_avg_ms=0.237 all_reduce_avg_ms=0.336 total_avg_ms=1.431
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=3120 tokens=3120 router_gate_avg_ms=0.070 routed_avg_ms=0.771 shared_avg_ms=0.236 all_reduce_avg_ms=0.342 total_avg_ms=1.431
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=3120 tokens=3120 router_gate_avg_ms=0.071 routed_avg_ms=0.772 shared_avg_ms=0.233 all_reduce_avg_ms=0.346 total_avg_ms=1.434
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=3200 tokens=3200 router_gate_avg_ms=0.071 routed_avg_ms=0.775 shared_avg_ms=0.242 all_reduce_avg_ms=0.324 total_avg_ms=1.424
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=3200 tokens=3200 router_gate_avg_ms=0.070 routed_avg_ms=0.771 shared_avg_ms=0.236 all_reduce_avg_ms=0.342 total_avg_ms=1.430
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=3200 tokens=3200 router_gate_avg_ms=0.071 routed_avg_ms=0.772 shared_avg_ms=0.233 all_reduce_avg_ms=0.345 total_avg_ms=1.433
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=3200 tokens=3200 router_gate_avg_ms=0.071 routed_avg_ms=0.776 shared_avg_ms=0.237 all_reduce_avg_ms=0.335 total_avg_ms=1.431
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=3280 tokens=3280 router_gate_avg_ms=0.071 routed_avg_ms=0.775 shared_avg_ms=0.242 all_reduce_avg_ms=0.324 total_avg_ms=1.424
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=3280 tokens=3280 router_gate_avg_ms=0.071 routed_avg_ms=0.776 shared_avg_ms=0.237 all_reduce_avg_ms=0.334 total_avg_ms=1.430
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=3280 tokens=3280 router_gate_avg_ms=0.069 routed_avg_ms=0.771 shared_avg_ms=0.236 all_reduce_avg_ms=0.342 total_avg_ms=1.430
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=3280 tokens=3280 router_gate_avg_ms=0.071 routed_avg_ms=0.772 shared_avg_ms=0.233 all_reduce_avg_ms=0.344 total_avg_ms=1.432
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=3360 tokens=3360 router_gate_avg_ms=0.071 routed_avg_ms=0.774 shared_avg_ms=0.242 all_reduce_avg_ms=0.325 total_avg_ms=1.423
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=3360 tokens=3360 router_gate_avg_ms=0.070 routed_avg_ms=0.776 shared_avg_ms=0.237 all_reduce_avg_ms=0.333 total_avg_ms=1.429
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=3360 tokens=3360 router_gate_avg_ms=0.069 routed_avg_ms=0.771 shared_avg_ms=0.235 all_reduce_avg_ms=0.342 total_avg_ms=1.429
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=3360 tokens=3360 router_gate_avg_ms=0.071 routed_avg_ms=0.772 shared_avg_ms=0.233 all_reduce_avg_ms=0.343 total_avg_ms=1.431
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=3440 tokens=3440 router_gate_avg_ms=0.071 routed_avg_ms=0.774 shared_avg_ms=0.241 all_reduce_avg_ms=0.325 total_avg_ms=1.423
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=3440 tokens=3440 router_gate_avg_ms=0.069 routed_avg_ms=0.771 shared_avg_ms=0.235 all_reduce_avg_ms=0.342 total_avg_ms=1.429
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=3440 tokens=3440 router_gate_avg_ms=0.070 routed_avg_ms=0.776 shared_avg_ms=0.237 all_reduce_avg_ms=0.333 total_avg_ms=1.429
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=3440 tokens=3440 router_gate_avg_ms=0.071 routed_avg_ms=0.773 shared_avg_ms=0.233 all_reduce_avg_ms=0.343 total_avg_ms=1.431
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=3520 tokens=3520 router_gate_avg_ms=0.071 routed_avg_ms=0.774 shared_avg_ms=0.241 all_reduce_avg_ms=0.326 total_avg_ms=1.423
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=3520 tokens=3520 router_gate_avg_ms=0.069 routed_avg_ms=0.771 shared_avg_ms=0.235 all_reduce_avg_ms=0.342 total_avg_ms=1.429
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=3520 tokens=3520 router_gate_avg_ms=0.071 routed_avg_ms=0.773 shared_avg_ms=0.233 all_reduce_avg_ms=0.343 total_avg_ms=1.431
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=3520 tokens=3520 router_gate_avg_ms=0.070 routed_avg_ms=0.776 shared_avg_ms=0.237 all_reduce_avg_ms=0.333 total_avg_ms=1.429
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=3600 tokens=3600 router_gate_avg_ms=0.069 routed_avg_ms=0.770 shared_avg_ms=0.235 all_reduce_avg_ms=0.342 total_avg_ms=1.428
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=3600 tokens=3600 router_gate_avg_ms=0.071 routed_avg_ms=0.773 shared_avg_ms=0.233 all_reduce_avg_ms=0.342 total_avg_ms=1.430
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=3600 tokens=3600 router_gate_avg_ms=0.071 routed_avg_ms=0.774 shared_avg_ms=0.241 all_reduce_avg_ms=0.326 total_avg_ms=1.422
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=3600 tokens=3600 router_gate_avg_ms=0.070 routed_avg_ms=0.776 shared_avg_ms=0.237 all_reduce_avg_ms=0.332 total_avg_ms=1.428
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=3680 tokens=3680 router_gate_avg_ms=0.071 routed_avg_ms=0.773 shared_avg_ms=0.241 all_reduce_avg_ms=0.326 total_avg_ms=1.422
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=3680 tokens=3680 router_gate_avg_ms=0.069 routed_avg_ms=0.770 shared_avg_ms=0.235 all_reduce_avg_ms=0.342 total_avg_ms=1.428
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=3680 tokens=3680 router_gate_avg_ms=0.071 routed_avg_ms=0.773 shared_avg_ms=0.233 all_reduce_avg_ms=0.341 total_avg_ms=1.430
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=3680 tokens=3680 router_gate_avg_ms=0.070 routed_avg_ms=0.776 shared_avg_ms=0.237 all_reduce_avg_ms=0.332 total_avg_ms=1.427
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=3760 tokens=3760 router_gate_avg_ms=0.070 routed_avg_ms=0.773 shared_avg_ms=0.241 all_reduce_avg_ms=0.326 total_avg_ms=1.422
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=3760 tokens=3760 router_gate_avg_ms=0.070 routed_avg_ms=0.776 shared_avg_ms=0.237 all_reduce_avg_ms=0.331 total_avg_ms=1.427
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=3760 tokens=3760 router_gate_avg_ms=0.069 routed_avg_ms=0.770 shared_avg_ms=0.235 all_reduce_avg_ms=0.342 total_avg_ms=1.428
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=3760 tokens=3760 router_gate_avg_ms=0.071 routed_avg_ms=0.773 shared_avg_ms=0.233 all_reduce_avg_ms=0.341 total_avg_ms=1.430
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=3840 tokens=3840 router_gate_avg_ms=0.070 routed_avg_ms=0.773 shared_avg_ms=0.240 all_reduce_avg_ms=0.327 total_avg_ms=1.422
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=3840 tokens=3840 router_gate_avg_ms=0.069 routed_avg_ms=0.770 shared_avg_ms=0.235 all_reduce_avg_ms=0.342 total_avg_ms=1.428
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=3840 tokens=3840 router_gate_avg_ms=0.071 routed_avg_ms=0.773 shared_avg_ms=0.233 all_reduce_avg_ms=0.341 total_avg_ms=1.429
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=3840 tokens=3840 router_gate_avg_ms=0.070 routed_avg_ms=0.776 shared_avg_ms=0.237 all_reduce_avg_ms=0.332 total_avg_ms=1.427
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=3920 tokens=3920 router_gate_avg_ms=0.070 routed_avg_ms=0.773 shared_avg_ms=0.240 all_reduce_avg_ms=0.327 total_avg_ms=1.422
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=3920 tokens=3920 router_gate_avg_ms=0.070 routed_avg_ms=0.776 shared_avg_ms=0.237 all_reduce_avg_ms=0.332 total_avg_ms=1.427
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=3920 tokens=3920 router_gate_avg_ms=0.071 routed_avg_ms=0.773 shared_avg_ms=0.234 all_reduce_avg_ms=0.340 total_avg_ms=1.429
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=3920 tokens=3920 router_gate_avg_ms=0.069 routed_avg_ms=0.770 shared_avg_ms=0.235 all_reduce_avg_ms=0.342 total_avg_ms=1.427
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=4000 tokens=4000 router_gate_avg_ms=0.070 routed_avg_ms=0.773 shared_avg_ms=0.240 all_reduce_avg_ms=0.327 total_avg_ms=1.422
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=4000 tokens=4000 router_gate_avg_ms=0.070 routed_avg_ms=0.776 shared_avg_ms=0.237 all_reduce_avg_ms=0.332 total_avg_ms=1.427
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=4000 tokens=4000 router_gate_avg_ms=0.069 routed_avg_ms=0.770 shared_avg_ms=0.235 all_reduce_avg_ms=0.342 total_avg_ms=1.428
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=4000 tokens=4000 router_gate_avg_ms=0.071 routed_avg_ms=0.773 shared_avg_ms=0.234 all_reduce_avg_ms=0.339 total_avg_ms=1.429
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=4080 tokens=4080 router_gate_avg_ms=0.070 routed_avg_ms=0.773 shared_avg_ms=0.240 all_reduce_avg_ms=0.327 total_avg_ms=1.422
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=4080 tokens=4080 router_gate_avg_ms=0.069 routed_avg_ms=0.770 shared_avg_ms=0.235 all_reduce_avg_ms=0.342 total_avg_ms=1.428
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=4080 tokens=4080 router_gate_avg_ms=0.071 routed_avg_ms=0.774 shared_avg_ms=0.234 all_reduce_avg_ms=0.339 total_avg_ms=1.429
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=4080 tokens=4080 router_gate_avg_ms=0.070 routed_avg_ms=0.776 shared_avg_ms=0.237 all_reduce_avg_ms=0.332 total_avg_ms=1.427
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=4160 tokens=4160 router_gate_avg_ms=0.070 routed_avg_ms=0.773 shared_avg_ms=0.240 all_reduce_avg_ms=0.328 total_avg_ms=1.422
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=4160 tokens=4160 router_gate_avg_ms=0.069 routed_avg_ms=0.770 shared_avg_ms=0.234 all_reduce_avg_ms=0.342 total_avg_ms=1.427
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=4160 tokens=4160 router_gate_avg_ms=0.071 routed_avg_ms=0.773 shared_avg_ms=0.234 all_reduce_avg_ms=0.339 total_avg_ms=1.429
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=4160 tokens=4160 router_gate_avg_ms=0.070 routed_avg_ms=0.776 shared_avg_ms=0.237 all_reduce_avg_ms=0.332 total_avg_ms=1.427
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=4240 tokens=4240 router_gate_avg_ms=0.070 routed_avg_ms=0.773 shared_avg_ms=0.240 all_reduce_avg_ms=0.328 total_avg_ms=1.422
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=4240 tokens=4240 router_gate_avg_ms=0.070 routed_avg_ms=0.776 shared_avg_ms=0.237 all_reduce_avg_ms=0.331 total_avg_ms=1.426
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=4240 tokens=4240 router_gate_avg_ms=0.069 routed_avg_ms=0.770 shared_avg_ms=0.234 all_reduce_avg_ms=0.342 total_avg_ms=1.427
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=4240 tokens=4240 router_gate_avg_ms=0.071 routed_avg_ms=0.773 shared_avg_ms=0.234 all_reduce_avg_ms=0.339 total_avg_ms=1.428
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=4320 tokens=4320 router_gate_avg_ms=0.070 routed_avg_ms=0.772 shared_avg_ms=0.240 all_reduce_avg_ms=0.328 total_avg_ms=1.421
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=4320 tokens=4320 router_gate_avg_ms=0.070 routed_avg_ms=0.776 shared_avg_ms=0.237 all_reduce_avg_ms=0.331 total_avg_ms=1.426
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=4320 tokens=4320 router_gate_avg_ms=0.069 routed_avg_ms=0.770 shared_avg_ms=0.234 all_reduce_avg_ms=0.342 total_avg_ms=1.427
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=4320 tokens=4320 router_gate_avg_ms=0.071 routed_avg_ms=0.774 shared_avg_ms=0.234 all_reduce_avg_ms=0.338 total_avg_ms=1.428
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=4400 tokens=4400 router_gate_avg_ms=0.070 routed_avg_ms=0.772 shared_avg_ms=0.240 all_reduce_avg_ms=0.328 total_avg_ms=1.421
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=4400 tokens=4400 router_gate_avg_ms=0.070 routed_avg_ms=0.776 shared_avg_ms=0.237 all_reduce_avg_ms=0.331 total_avg_ms=1.426
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=4400 tokens=4400 router_gate_avg_ms=0.069 routed_avg_ms=0.770 shared_avg_ms=0.234 all_reduce_avg_ms=0.342 total_avg_ms=1.427
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=4400 tokens=4400 router_gate_avg_ms=0.071 routed_avg_ms=0.774 shared_avg_ms=0.234 all_reduce_avg_ms=0.338 total_avg_ms=1.428
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=4480 tokens=4480 router_gate_avg_ms=0.069 routed_avg_ms=0.770 shared_avg_ms=0.234 all_reduce_avg_ms=0.342 total_avg_ms=1.426
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=4480 tokens=4480 router_gate_avg_ms=0.070 routed_avg_ms=0.772 shared_avg_ms=0.239 all_reduce_avg_ms=0.328 total_avg_ms=1.421
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=4480 tokens=4480 router_gate_avg_ms=0.070 routed_avg_ms=0.776 shared_avg_ms=0.237 all_reduce_avg_ms=0.332 total_avg_ms=1.426
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=4480 tokens=4480 router_gate_avg_ms=0.071 routed_avg_ms=0.774 shared_avg_ms=0.234 all_reduce_avg_ms=0.337 total_avg_ms=1.427
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=4560 tokens=4560 router_gate_avg_ms=0.070 routed_avg_ms=0.772 shared_avg_ms=0.239 all_reduce_avg_ms=0.328 total_avg_ms=1.421
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=4560 tokens=4560 router_gate_avg_ms=0.070 routed_avg_ms=0.776 shared_avg_ms=0.236 all_reduce_avg_ms=0.332 total_avg_ms=1.425
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=4560 tokens=4560 router_gate_avg_ms=0.069 routed_avg_ms=0.770 shared_avg_ms=0.234 all_reduce_avg_ms=0.342 total_avg_ms=1.426
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=4560 tokens=4560 router_gate_avg_ms=0.071 routed_avg_ms=0.774 shared_avg_ms=0.234 all_reduce_avg_ms=0.336 total_avg_ms=1.427
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=4640 tokens=4640 router_gate_avg_ms=0.070 routed_avg_ms=0.772 shared_avg_ms=0.239 all_reduce_avg_ms=0.328 total_avg_ms=1.421
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=4640 tokens=4640 router_gate_avg_ms=0.070 routed_avg_ms=0.775 shared_avg_ms=0.236 all_reduce_avg_ms=0.332 total_avg_ms=1.425
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=4640 tokens=4640 router_gate_avg_ms=0.069 routed_avg_ms=0.770 shared_avg_ms=0.234 all_reduce_avg_ms=0.342 total_avg_ms=1.426
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=4640 tokens=4640 router_gate_avg_ms=0.071 routed_avg_ms=0.774 shared_avg_ms=0.234 all_reduce_avg_ms=0.336 total_avg_ms=1.426
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=4720 tokens=4720 router_gate_avg_ms=0.070 routed_avg_ms=0.772 shared_avg_ms=0.239 all_reduce_avg_ms=0.328 total_avg_ms=1.421
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=4720 tokens=4720 router_gate_avg_ms=0.069 routed_avg_ms=0.770 shared_avg_ms=0.234 all_reduce_avg_ms=0.342 total_avg_ms=1.426
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=4720 tokens=4720 router_gate_avg_ms=0.071 routed_avg_ms=0.774 shared_avg_ms=0.234 all_reduce_avg_ms=0.336 total_avg_ms=1.426
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=4720 tokens=4720 router_gate_avg_ms=0.070 routed_avg_ms=0.776 shared_avg_ms=0.236 all_reduce_avg_ms=0.331 total_avg_ms=1.425
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=4800 tokens=4800 router_gate_avg_ms=0.070 routed_avg_ms=0.772 shared_avg_ms=0.239 all_reduce_avg_ms=0.329 total_avg_ms=1.421
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=4800 tokens=4800 router_gate_avg_ms=0.069 routed_avg_ms=0.770 shared_avg_ms=0.234 all_reduce_avg_ms=0.342 total_avg_ms=1.426
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=4800 tokens=4800 router_gate_avg_ms=0.070 routed_avg_ms=0.774 shared_avg_ms=0.234 all_reduce_avg_ms=0.336 total_avg_ms=1.426
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=4800 tokens=4800 router_gate_avg_ms=0.070 routed_avg_ms=0.776 shared_avg_ms=0.236 all_reduce_avg_ms=0.331 total_avg_ms=1.425
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=4880 tokens=4880 router_gate_avg_ms=0.070 routed_avg_ms=0.772 shared_avg_ms=0.239 all_reduce_avg_ms=0.329 total_avg_ms=1.421
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=4880 tokens=4880 router_gate_avg_ms=0.070 routed_avg_ms=0.776 shared_avg_ms=0.236 all_reduce_avg_ms=0.330 total_avg_ms=1.425
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=4880 tokens=4880 router_gate_avg_ms=0.069 routed_avg_ms=0.770 shared_avg_ms=0.234 all_reduce_avg_ms=0.342 total_avg_ms=1.426
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=4880 tokens=4880 router_gate_avg_ms=0.070 routed_avg_ms=0.774 shared_avg_ms=0.234 all_reduce_avg_ms=0.335 total_avg_ms=1.426
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=4960 tokens=4960 router_gate_avg_ms=0.070 routed_avg_ms=0.771 shared_avg_ms=0.239 all_reduce_avg_ms=0.330 total_avg_ms=1.421
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=4960 tokens=4960 router_gate_avg_ms=0.070 routed_avg_ms=0.775 shared_avg_ms=0.234 all_reduce_avg_ms=0.336 total_avg_ms=1.427
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=4960 tokens=4960 router_gate_avg_ms=0.070 routed_avg_ms=0.776 shared_avg_ms=0.236 all_reduce_avg_ms=0.331 total_avg_ms=1.425
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=4960 tokens=4960 router_gate_avg_ms=0.069 routed_avg_ms=0.769 shared_avg_ms=0.235 all_reduce_avg_ms=0.342 total_avg_ms=1.426
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=5040 tokens=5040 router_gate_avg_ms=0.070 routed_avg_ms=0.771 shared_avg_ms=0.239 all_reduce_avg_ms=0.331 total_avg_ms=1.422
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=5040 tokens=5040 router_gate_avg_ms=0.070 routed_avg_ms=0.776 shared_avg_ms=0.237 all_reduce_avg_ms=0.332 total_avg_ms=1.426
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=5040 tokens=5040 router_gate_avg_ms=0.069 routed_avg_ms=0.769 shared_avg_ms=0.235 all_reduce_avg_ms=0.343 total_avg_ms=1.427
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=5040 tokens=5040 router_gate_avg_ms=0.070 routed_avg_ms=0.775 shared_avg_ms=0.234 all_reduce_avg_ms=0.336 total_avg_ms=1.427
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=5120 tokens=5120 router_gate_avg_ms=0.069 routed_avg_ms=0.771 shared_avg_ms=0.239 all_reduce_avg_ms=0.331 total_avg_ms=1.422
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=5120 tokens=5120 router_gate_avg_ms=0.070 routed_avg_ms=0.776 shared_avg_ms=0.237 all_reduce_avg_ms=0.331 total_avg_ms=1.426
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=5120 tokens=5120 router_gate_avg_ms=0.069 routed_avg_ms=0.769 shared_avg_ms=0.235 all_reduce_avg_ms=0.343 total_avg_ms=1.427
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=5120 tokens=5120 router_gate_avg_ms=0.070 routed_avg_ms=0.775 shared_avg_ms=0.234 all_reduce_avg_ms=0.336 total_avg_ms=1.427
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=5200 tokens=5200 router_gate_avg_ms=0.069 routed_avg_ms=0.771 shared_avg_ms=0.239 all_reduce_avg_ms=0.332 total_avg_ms=1.422
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=5200 tokens=5200 router_gate_avg_ms=0.069 routed_avg_ms=0.769 shared_avg_ms=0.235 all_reduce_avg_ms=0.343 total_avg_ms=1.427
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=5200 tokens=5200 router_gate_avg_ms=0.070 routed_avg_ms=0.775 shared_avg_ms=0.234 all_reduce_avg_ms=0.335 total_avg_ms=1.427
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=5200 tokens=5200 router_gate_avg_ms=0.070 routed_avg_ms=0.776 shared_avg_ms=0.237 all_reduce_avg_ms=0.331 total_avg_ms=1.426
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=5280 tokens=5280 router_gate_avg_ms=0.069 routed_avg_ms=0.771 shared_avg_ms=0.239 all_reduce_avg_ms=0.332 total_avg_ms=1.422
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=5280 tokens=5280 router_gate_avg_ms=0.070 routed_avg_ms=0.776 shared_avg_ms=0.237 all_reduce_avg_ms=0.331 total_avg_ms=1.426
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=5280 tokens=5280 router_gate_avg_ms=0.069 routed_avg_ms=0.769 shared_avg_ms=0.235 all_reduce_avg_ms=0.343 total_avg_ms=1.427
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=5280 tokens=5280 router_gate_avg_ms=0.070 routed_avg_ms=0.775 shared_avg_ms=0.234 all_reduce_avg_ms=0.335 total_avg_ms=1.427
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=5360 tokens=5360 router_gate_avg_ms=0.069 routed_avg_ms=0.771 shared_avg_ms=0.238 all_reduce_avg_ms=0.332 total_avg_ms=1.422
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=5360 tokens=5360 router_gate_avg_ms=0.069 routed_avg_ms=0.769 shared_avg_ms=0.235 all_reduce_avg_ms=0.343 total_avg_ms=1.427
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=5360 tokens=5360 router_gate_avg_ms=0.070 routed_avg_ms=0.775 shared_avg_ms=0.234 all_reduce_avg_ms=0.335 total_avg_ms=1.427
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=5360 tokens=5360 router_gate_avg_ms=0.070 routed_avg_ms=0.776 shared_avg_ms=0.237 all_reduce_avg_ms=0.331 total_avg_ms=1.425
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=5440 tokens=5440 router_gate_avg_ms=0.069 routed_avg_ms=0.771 shared_avg_ms=0.238 all_reduce_avg_ms=0.332 total_avg_ms=1.422
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=5440 tokens=5440 router_gate_avg_ms=0.069 routed_avg_ms=0.769 shared_avg_ms=0.234 all_reduce_avg_ms=0.343 total_avg_ms=1.427
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=5440 tokens=5440 router_gate_avg_ms=0.070 routed_avg_ms=0.776 shared_avg_ms=0.237 all_reduce_avg_ms=0.331 total_avg_ms=1.425
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=5440 tokens=5440 router_gate_avg_ms=0.070 routed_avg_ms=0.775 shared_avg_ms=0.234 all_reduce_avg_ms=0.335 total_avg_ms=1.427
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=5520 tokens=5520 router_gate_avg_ms=0.069 routed_avg_ms=0.771 shared_avg_ms=0.238 all_reduce_avg_ms=0.332 total_avg_ms=1.422
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=5520 tokens=5520 router_gate_avg_ms=0.070 routed_avg_ms=0.776 shared_avg_ms=0.237 all_reduce_avg_ms=0.330 total_avg_ms=1.425
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=5520 tokens=5520 router_gate_avg_ms=0.069 routed_avg_ms=0.769 shared_avg_ms=0.234 all_reduce_avg_ms=0.343 total_avg_ms=1.427
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=5520 tokens=5520 router_gate_avg_ms=0.070 routed_avg_ms=0.775 shared_avg_ms=0.234 all_reduce_avg_ms=0.335 total_avg_ms=1.426
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=5600 tokens=5600 router_gate_avg_ms=0.069 routed_avg_ms=0.771 shared_avg_ms=0.238 all_reduce_avg_ms=0.333 total_avg_ms=1.423
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=5600 tokens=5600 router_gate_avg_ms=0.070 routed_avg_ms=0.776 shared_avg_ms=0.237 all_reduce_avg_ms=0.330 total_avg_ms=1.425
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=5600 tokens=5600 router_gate_avg_ms=0.069 routed_avg_ms=0.769 shared_avg_ms=0.234 all_reduce_avg_ms=0.343 total_avg_ms=1.427
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=5600 tokens=5600 router_gate_avg_ms=0.070 routed_avg_ms=0.775 shared_avg_ms=0.234 all_reduce_avg_ms=0.335 total_avg_ms=1.427
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=5680 tokens=5680 router_gate_avg_ms=0.069 routed_avg_ms=0.771 shared_avg_ms=0.238 all_reduce_avg_ms=0.333 total_avg_ms=1.423
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=5680 tokens=5680 router_gate_avg_ms=0.070 routed_avg_ms=0.776 shared_avg_ms=0.237 all_reduce_avg_ms=0.330 total_avg_ms=1.425
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=5680 tokens=5680 router_gate_avg_ms=0.070 routed_avg_ms=0.775 shared_avg_ms=0.235 all_reduce_avg_ms=0.335 total_avg_ms=1.427
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=5680 tokens=5680 router_gate_avg_ms=0.069 routed_avg_ms=0.769 shared_avg_ms=0.235 all_reduce_avg_ms=0.343 total_avg_ms=1.427
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=5760 tokens=5760 router_gate_avg_ms=0.069 routed_avg_ms=0.770 shared_avg_ms=0.238 all_reduce_avg_ms=0.334 total_avg_ms=1.424
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=5760 tokens=5760 router_gate_avg_ms=0.069 routed_avg_ms=0.769 shared_avg_ms=0.235 all_reduce_avg_ms=0.343 total_avg_ms=1.428
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=5760 tokens=5760 router_gate_avg_ms=0.070 routed_avg_ms=0.775 shared_avg_ms=0.235 all_reduce_avg_ms=0.335 total_avg_ms=1.428
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=5760 tokens=5760 router_gate_avg_ms=0.071 routed_avg_ms=0.777 shared_avg_ms=0.237 all_reduce_avg_ms=0.330 total_avg_ms=1.426
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=5840 tokens=5840 router_gate_avg_ms=0.069 routed_avg_ms=0.770 shared_avg_ms=0.238 all_reduce_avg_ms=0.335 total_avg_ms=1.424
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=5840 tokens=5840 router_gate_avg_ms=0.069 routed_avg_ms=0.769 shared_avg_ms=0.235 all_reduce_avg_ms=0.344 total_avg_ms=1.429
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=5840 tokens=5840 router_gate_avg_ms=0.070 routed_avg_ms=0.776 shared_avg_ms=0.235 all_reduce_avg_ms=0.336 total_avg_ms=1.428
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=5840 tokens=5840 router_gate_avg_ms=0.071 routed_avg_ms=0.777 shared_avg_ms=0.237 all_reduce_avg_ms=0.330 total_avg_ms=1.426
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=5920 tokens=5920 router_gate_avg_ms=0.069 routed_avg_ms=0.770 shared_avg_ms=0.238 all_reduce_avg_ms=0.335 total_avg_ms=1.425
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=5920 tokens=5920 router_gate_avg_ms=0.069 routed_avg_ms=0.769 shared_avg_ms=0.235 all_reduce_avg_ms=0.345 total_avg_ms=1.429
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=5920 tokens=5920 router_gate_avg_ms=0.070 routed_avg_ms=0.776 shared_avg_ms=0.235 all_reduce_avg_ms=0.336 total_avg_ms=1.429
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=5920 tokens=5920 router_gate_avg_ms=0.071 routed_avg_ms=0.777 shared_avg_ms=0.237 all_reduce_avg_ms=0.330 total_avg_ms=1.427
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=6000 tokens=6000 router_gate_avg_ms=0.069 routed_avg_ms=0.770 shared_avg_ms=0.238 all_reduce_avg_ms=0.336 total_avg_ms=1.425
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=6000 tokens=6000 router_gate_avg_ms=0.069 routed_avg_ms=0.769 shared_avg_ms=0.235 all_reduce_avg_ms=0.345 total_avg_ms=1.429
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=6000 tokens=6000 router_gate_avg_ms=0.070 routed_avg_ms=0.776 shared_avg_ms=0.235 all_reduce_avg_ms=0.337 total_avg_ms=1.429
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=6000 tokens=6000 router_gate_avg_ms=0.071 routed_avg_ms=0.777 shared_avg_ms=0.238 all_reduce_avg_ms=0.329 total_avg_ms=1.427
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=6080 tokens=6080 router_gate_avg_ms=0.069 routed_avg_ms=0.770 shared_avg_ms=0.238 all_reduce_avg_ms=0.337 total_avg_ms=1.426
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=6080 tokens=6080 router_gate_avg_ms=0.069 routed_avg_ms=0.770 shared_avg_ms=0.235 all_reduce_avg_ms=0.345 total_avg_ms=1.430
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=6080 tokens=6080 router_gate_avg_ms=0.070 routed_avg_ms=0.776 shared_avg_ms=0.235 all_reduce_avg_ms=0.337 total_avg_ms=1.430
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=6080 tokens=6080 router_gate_avg_ms=0.071 routed_avg_ms=0.777 shared_avg_ms=0.238 all_reduce_avg_ms=0.330 total_avg_ms=1.428
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=6160 tokens=6160 router_gate_avg_ms=0.069 routed_avg_ms=0.770 shared_avg_ms=0.238 all_reduce_avg_ms=0.337 total_avg_ms=1.427
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=6160 tokens=6160 router_gate_avg_ms=0.071 routed_avg_ms=0.778 shared_avg_ms=0.238 all_reduce_avg_ms=0.330 total_avg_ms=1.428
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=6160 tokens=6160 router_gate_avg_ms=0.069 routed_avg_ms=0.770 shared_avg_ms=0.235 all_reduce_avg_ms=0.345 total_avg_ms=1.431
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=6160 tokens=6160 router_gate_avg_ms=0.070 routed_avg_ms=0.776 shared_avg_ms=0.235 all_reduce_avg_ms=0.337 total_avg_ms=1.431
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=6240 tokens=6240 router_gate_avg_ms=0.069 routed_avg_ms=0.770 shared_avg_ms=0.238 all_reduce_avg_ms=0.338 total_avg_ms=1.427
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=6240 tokens=6240 router_gate_avg_ms=0.069 routed_avg_ms=0.770 shared_avg_ms=0.235 all_reduce_avg_ms=0.346 total_avg_ms=1.431
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=6240 tokens=6240 router_gate_avg_ms=0.070 routed_avg_ms=0.776 shared_avg_ms=0.235 all_reduce_avg_ms=0.337 total_avg_ms=1.431
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=6240 tokens=6240 router_gate_avg_ms=0.071 routed_avg_ms=0.778 shared_avg_ms=0.238 all_reduce_avg_ms=0.330 total_avg_ms=1.428
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=6320 tokens=6320 router_gate_avg_ms=0.069 routed_avg_ms=0.770 shared_avg_ms=0.238 all_reduce_avg_ms=0.338 total_avg_ms=1.427
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=6320 tokens=6320 router_gate_avg_ms=0.069 routed_avg_ms=0.770 shared_avg_ms=0.235 all_reduce_avg_ms=0.345 total_avg_ms=1.431
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=6320 tokens=6320 router_gate_avg_ms=0.070 routed_avg_ms=0.776 shared_avg_ms=0.235 all_reduce_avg_ms=0.337 total_avg_ms=1.431
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=6320 tokens=6320 router_gate_avg_ms=0.071 routed_avg_ms=0.778 shared_avg_ms=0.238 all_reduce_avg_ms=0.330 total_avg_ms=1.428
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=6400 tokens=6400 router_gate_avg_ms=0.071 routed_avg_ms=0.778 shared_avg_ms=0.238 all_reduce_avg_ms=0.329 total_avg_ms=1.428
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=6400 tokens=6400 router_gate_avg_ms=0.069 routed_avg_ms=0.770 shared_avg_ms=0.238 all_reduce_avg_ms=0.338 total_avg_ms=1.427
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=6400 tokens=6400 router_gate_avg_ms=0.069 routed_avg_ms=0.770 shared_avg_ms=0.235 all_reduce_avg_ms=0.345 total_avg_ms=1.430
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=6400 tokens=6400 router_gate_avg_ms=0.070 routed_avg_ms=0.776 shared_avg_ms=0.235 all_reduce_avg_ms=0.337 total_avg_ms=1.430
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=6480 tokens=6480 router_gate_avg_ms=0.069 routed_avg_ms=0.770 shared_avg_ms=0.238 all_reduce_avg_ms=0.338 total_avg_ms=1.426
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=6480 tokens=6480 router_gate_avg_ms=0.069 routed_avg_ms=0.770 shared_avg_ms=0.235 all_reduce_avg_ms=0.345 total_avg_ms=1.430
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=6480 tokens=6480 router_gate_avg_ms=0.070 routed_avg_ms=0.776 shared_avg_ms=0.235 all_reduce_avg_ms=0.336 total_avg_ms=1.430
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=6480 tokens=6480 router_gate_avg_ms=0.071 routed_avg_ms=0.778 shared_avg_ms=0.238 all_reduce_avg_ms=0.329 total_avg_ms=1.427
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=6560 tokens=6560 router_gate_avg_ms=0.071 routed_avg_ms=0.777 shared_avg_ms=0.238 all_reduce_avg_ms=0.329 total_avg_ms=1.427
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=6560 tokens=6560 router_gate_avg_ms=0.069 routed_avg_ms=0.770 shared_avg_ms=0.235 all_reduce_avg_ms=0.345 total_avg_ms=1.430
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=6560 tokens=6560 router_gate_avg_ms=0.070 routed_avg_ms=0.776 shared_avg_ms=0.235 all_reduce_avg_ms=0.336 total_avg_ms=1.430
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=6560 tokens=6560 router_gate_avg_ms=0.069 routed_avg_ms=0.770 shared_avg_ms=0.238 all_reduce_avg_ms=0.338 total_avg_ms=1.426
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=6640 tokens=6640 router_gate_avg_ms=0.069 routed_avg_ms=0.770 shared_avg_ms=0.238 all_reduce_avg_ms=0.338 total_avg_ms=1.426
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=6640 tokens=6640 router_gate_avg_ms=0.069 routed_avg_ms=0.770 shared_avg_ms=0.235 all_reduce_avg_ms=0.345 total_avg_ms=1.430
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=6640 tokens=6640 router_gate_avg_ms=0.070 routed_avg_ms=0.776 shared_avg_ms=0.235 all_reduce_avg_ms=0.336 total_avg_ms=1.429
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=6640 tokens=6640 router_gate_avg_ms=0.071 routed_avg_ms=0.777 shared_avg_ms=0.238 all_reduce_avg_ms=0.329 total_avg_ms=1.427
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=6720 tokens=6720 router_gate_avg_ms=0.069 routed_avg_ms=0.770 shared_avg_ms=0.238 all_reduce_avg_ms=0.338 total_avg_ms=1.426
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=6720 tokens=6720 router_gate_avg_ms=0.071 routed_avg_ms=0.777 shared_avg_ms=0.238 all_reduce_avg_ms=0.329 total_avg_ms=1.427
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=6720 tokens=6720 router_gate_avg_ms=0.069 routed_avg_ms=0.770 shared_avg_ms=0.235 all_reduce_avg_ms=0.345 total_avg_ms=1.429
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=6720 tokens=6720 router_gate_avg_ms=0.070 routed_avg_ms=0.776 shared_avg_ms=0.235 all_reduce_avg_ms=0.336 total_avg_ms=1.429
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=6800 tokens=6800 router_gate_avg_ms=0.069 routed_avg_ms=0.770 shared_avg_ms=0.238 all_reduce_avg_ms=0.337 total_avg_ms=1.426
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=6800 tokens=6800 router_gate_avg_ms=0.071 routed_avg_ms=0.777 shared_avg_ms=0.238 all_reduce_avg_ms=0.329 total_avg_ms=1.426
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=6800 tokens=6800 router_gate_avg_ms=0.069 routed_avg_ms=0.770 shared_avg_ms=0.234 all_reduce_avg_ms=0.345 total_avg_ms=1.429
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=6800 tokens=6800 router_gate_avg_ms=0.070 routed_avg_ms=0.776 shared_avg_ms=0.235 all_reduce_avg_ms=0.335 total_avg_ms=1.429
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=prefill calls=160 tokens=1966480 router_gate_avg_ms=0.576 routed_avg_ms=77.941 shared_avg_ms=2.075 all_reduce_avg_ms=4.212 total_avg_ms=84.816
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=prefill calls=160 tokens=1966480 router_gate_avg_ms=0.578 routed_avg_ms=77.951 shared_avg_ms=2.076 all_reduce_avg_ms=4.210 total_avg_ms=84.827
[PROF_MOE_SUMMARY] pid=35011 bucket=prefill calls=160 tokens=1966480 router_gate_avg_ms=0.578 routed_avg_ms=78.006 shared_avg_ms=2.087 all_reduce_avg_ms=4.126 total_avg_ms=84.809
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=prefill calls=160 tokens=1966480 router_gate_avg_ms=0.577 routed_avg_ms=77.981 shared_avg_ms=2.073 all_reduce_avg_ms=4.159 total_avg_ms=84.802
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=6880 tokens=6880 router_gate_avg_ms=0.069 routed_avg_ms=0.770 shared_avg_ms=0.238 all_reduce_avg_ms=0.337 total_avg_ms=1.426
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=6880 tokens=6880 router_gate_avg_ms=0.071 routed_avg_ms=0.777 shared_avg_ms=0.238 all_reduce_avg_ms=0.329 total_avg_ms=1.426
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=6880 tokens=6880 router_gate_avg_ms=0.069 routed_avg_ms=0.770 shared_avg_ms=0.235 all_reduce_avg_ms=0.344 total_avg_ms=1.429
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=6880 tokens=6880 router_gate_avg_ms=0.070 routed_avg_ms=0.776 shared_avg_ms=0.235 all_reduce_avg_ms=0.335 total_avg_ms=1.429
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=6960 tokens=6960 router_gate_avg_ms=0.069 routed_avg_ms=0.770 shared_avg_ms=0.238 all_reduce_avg_ms=0.337 total_avg_ms=1.425
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=6960 tokens=6960 router_gate_avg_ms=0.070 routed_avg_ms=0.776 shared_avg_ms=0.235 all_reduce_avg_ms=0.336 total_avg_ms=1.429
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=6960 tokens=6960 router_gate_avg_ms=0.071 routed_avg_ms=0.777 shared_avg_ms=0.237 all_reduce_avg_ms=0.329 total_avg_ms=1.426
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=6960 tokens=6960 router_gate_avg_ms=0.069 routed_avg_ms=0.770 shared_avg_ms=0.235 all_reduce_avg_ms=0.344 total_avg_ms=1.429
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=7040 tokens=7040 router_gate_avg_ms=0.069 routed_avg_ms=0.770 shared_avg_ms=0.235 all_reduce_avg_ms=0.344 total_avg_ms=1.429
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=7040 tokens=7040 router_gate_avg_ms=0.070 routed_avg_ms=0.776 shared_avg_ms=0.235 all_reduce_avg_ms=0.336 total_avg_ms=1.429
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=7040 tokens=7040 router_gate_avg_ms=0.069 routed_avg_ms=0.770 shared_avg_ms=0.238 all_reduce_avg_ms=0.337 total_avg_ms=1.425
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=7040 tokens=7040 router_gate_avg_ms=0.071 routed_avg_ms=0.777 shared_avg_ms=0.237 all_reduce_avg_ms=0.329 total_avg_ms=1.426
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=7120 tokens=7120 router_gate_avg_ms=0.069 routed_avg_ms=0.769 shared_avg_ms=0.238 all_reduce_avg_ms=0.337 total_avg_ms=1.425
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=7120 tokens=7120 router_gate_avg_ms=0.069 routed_avg_ms=0.769 shared_avg_ms=0.235 all_reduce_avg_ms=0.344 total_avg_ms=1.428
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=7120 tokens=7120 router_gate_avg_ms=0.070 routed_avg_ms=0.776 shared_avg_ms=0.235 all_reduce_avg_ms=0.336 total_avg_ms=1.428
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=7120 tokens=7120 router_gate_avg_ms=0.071 routed_avg_ms=0.777 shared_avg_ms=0.237 all_reduce_avg_ms=0.329 total_avg_ms=1.426
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=7200 tokens=7200 router_gate_avg_ms=0.069 routed_avg_ms=0.769 shared_avg_ms=0.238 all_reduce_avg_ms=0.337 total_avg_ms=1.425
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=7200 tokens=7200 router_gate_avg_ms=0.071 routed_avg_ms=0.777 shared_avg_ms=0.237 all_reduce_avg_ms=0.329 total_avg_ms=1.425
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=7200 tokens=7200 router_gate_avg_ms=0.069 routed_avg_ms=0.769 shared_avg_ms=0.235 all_reduce_avg_ms=0.344 total_avg_ms=1.428
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=7200 tokens=7200 router_gate_avg_ms=0.070 routed_avg_ms=0.776 shared_avg_ms=0.235 all_reduce_avg_ms=0.336 total_avg_ms=1.428
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=7280 tokens=7280 router_gate_avg_ms=0.069 routed_avg_ms=0.769 shared_avg_ms=0.237 all_reduce_avg_ms=0.337 total_avg_ms=1.424
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=7280 tokens=7280 router_gate_avg_ms=0.070 routed_avg_ms=0.775 shared_avg_ms=0.235 all_reduce_avg_ms=0.335 total_avg_ms=1.428
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=7280 tokens=7280 router_gate_avg_ms=0.071 routed_avg_ms=0.776 shared_avg_ms=0.237 all_reduce_avg_ms=0.329 total_avg_ms=1.425
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=7280 tokens=7280 router_gate_avg_ms=0.069 routed_avg_ms=0.769 shared_avg_ms=0.235 all_reduce_avg_ms=0.343 total_avg_ms=1.428
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=7360 tokens=7360 router_gate_avg_ms=0.069 routed_avg_ms=0.769 shared_avg_ms=0.237 all_reduce_avg_ms=0.337 total_avg_ms=1.424
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=7360 tokens=7360 router_gate_avg_ms=0.071 routed_avg_ms=0.776 shared_avg_ms=0.237 all_reduce_avg_ms=0.329 total_avg_ms=1.425
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=7360 tokens=7360 router_gate_avg_ms=0.069 routed_avg_ms=0.769 shared_avg_ms=0.235 all_reduce_avg_ms=0.343 total_avg_ms=1.427
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=7360 tokens=7360 router_gate_avg_ms=0.070 routed_avg_ms=0.775 shared_avg_ms=0.235 all_reduce_avg_ms=0.335 total_avg_ms=1.427
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=7440 tokens=7440 router_gate_avg_ms=0.069 routed_avg_ms=0.769 shared_avg_ms=0.237 all_reduce_avg_ms=0.337 total_avg_ms=1.424
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=7440 tokens=7440 router_gate_avg_ms=0.071 routed_avg_ms=0.776 shared_avg_ms=0.237 all_reduce_avg_ms=0.329 total_avg_ms=1.425
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=7440 tokens=7440 router_gate_avg_ms=0.069 routed_avg_ms=0.769 shared_avg_ms=0.235 all_reduce_avg_ms=0.343 total_avg_ms=1.427
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=7440 tokens=7440 router_gate_avg_ms=0.070 routed_avg_ms=0.775 shared_avg_ms=0.235 all_reduce_avg_ms=0.335 total_avg_ms=1.427
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=7520 tokens=7520 router_gate_avg_ms=0.069 routed_avg_ms=0.769 shared_avg_ms=0.235 all_reduce_avg_ms=0.343 total_avg_ms=1.427
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=7520 tokens=7520 router_gate_avg_ms=0.070 routed_avg_ms=0.775 shared_avg_ms=0.235 all_reduce_avg_ms=0.335 total_avg_ms=1.427
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=7520 tokens=7520 router_gate_avg_ms=0.069 routed_avg_ms=0.769 shared_avg_ms=0.237 all_reduce_avg_ms=0.337 total_avg_ms=1.424
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=7520 tokens=7520 router_gate_avg_ms=0.071 routed_avg_ms=0.776 shared_avg_ms=0.237 all_reduce_avg_ms=0.329 total_avg_ms=1.425
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=7600 tokens=7600 router_gate_avg_ms=0.069 routed_avg_ms=0.769 shared_avg_ms=0.235 all_reduce_avg_ms=0.343 total_avg_ms=1.427
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=7600 tokens=7600 router_gate_avg_ms=0.069 routed_avg_ms=0.769 shared_avg_ms=0.237 all_reduce_avg_ms=0.337 total_avg_ms=1.424
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=7600 tokens=7600 router_gate_avg_ms=0.070 routed_avg_ms=0.775 shared_avg_ms=0.235 all_reduce_avg_ms=0.335 total_avg_ms=1.427
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=7600 tokens=7600 router_gate_avg_ms=0.071 routed_avg_ms=0.776 shared_avg_ms=0.237 all_reduce_avg_ms=0.329 total_avg_ms=1.425
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=7680 tokens=7680 router_gate_avg_ms=0.069 routed_avg_ms=0.769 shared_avg_ms=0.237 all_reduce_avg_ms=0.336 total_avg_ms=1.423
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=7680 tokens=7680 router_gate_avg_ms=0.069 routed_avg_ms=0.769 shared_avg_ms=0.235 all_reduce_avg_ms=0.343 total_avg_ms=1.427
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=7680 tokens=7680 router_gate_avg_ms=0.070 routed_avg_ms=0.775 shared_avg_ms=0.235 all_reduce_avg_ms=0.335 total_avg_ms=1.427
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=7680 tokens=7680 router_gate_avg_ms=0.071 routed_avg_ms=0.776 shared_avg_ms=0.237 all_reduce_avg_ms=0.329 total_avg_ms=1.425
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=7760 tokens=7760 router_gate_avg_ms=0.070 routed_avg_ms=0.775 shared_avg_ms=0.235 all_reduce_avg_ms=0.335 total_avg_ms=1.427
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=7760 tokens=7760 router_gate_avg_ms=0.071 routed_avg_ms=0.776 shared_avg_ms=0.237 all_reduce_avg_ms=0.329 total_avg_ms=1.425
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=7760 tokens=7760 router_gate_avg_ms=0.069 routed_avg_ms=0.769 shared_avg_ms=0.237 all_reduce_avg_ms=0.336 total_avg_ms=1.423
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=7760 tokens=7760 router_gate_avg_ms=0.069 routed_avg_ms=0.769 shared_avg_ms=0.235 all_reduce_avg_ms=0.343 total_avg_ms=1.427
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=7840 tokens=7840 router_gate_avg_ms=0.070 routed_avg_ms=0.775 shared_avg_ms=0.235 all_reduce_avg_ms=0.336 total_avg_ms=1.427
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=7840 tokens=7840 router_gate_avg_ms=0.069 routed_avg_ms=0.769 shared_avg_ms=0.237 all_reduce_avg_ms=0.336 total_avg_ms=1.424
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=7840 tokens=7840 router_gate_avg_ms=0.071 routed_avg_ms=0.776 shared_avg_ms=0.237 all_reduce_avg_ms=0.329 total_avg_ms=1.425
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=7840 tokens=7840 router_gate_avg_ms=0.069 routed_avg_ms=0.769 shared_avg_ms=0.235 all_reduce_avg_ms=0.343 total_avg_ms=1.427
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=7920 tokens=7920 router_gate_avg_ms=0.069 routed_avg_ms=0.769 shared_avg_ms=0.237 all_reduce_avg_ms=0.337 total_avg_ms=1.424
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=7920 tokens=7920 router_gate_avg_ms=0.071 routed_avg_ms=0.776 shared_avg_ms=0.237 all_reduce_avg_ms=0.329 total_avg_ms=1.425
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=7920 tokens=7920 router_gate_avg_ms=0.069 routed_avg_ms=0.769 shared_avg_ms=0.235 all_reduce_avg_ms=0.342 total_avg_ms=1.427
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=7920 tokens=7920 router_gate_avg_ms=0.070 routed_avg_ms=0.775 shared_avg_ms=0.235 all_reduce_avg_ms=0.336 total_avg_ms=1.427
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=8000 tokens=8000 router_gate_avg_ms=0.069 routed_avg_ms=0.769 shared_avg_ms=0.235 all_reduce_avg_ms=0.342 total_avg_ms=1.427
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=8000 tokens=8000 router_gate_avg_ms=0.070 routed_avg_ms=0.775 shared_avg_ms=0.235 all_reduce_avg_ms=0.335 total_avg_ms=1.427
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=8000 tokens=8000 router_gate_avg_ms=0.069 routed_avg_ms=0.769 shared_avg_ms=0.237 all_reduce_avg_ms=0.336 total_avg_ms=1.423
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=8000 tokens=8000 router_gate_avg_ms=0.071 routed_avg_ms=0.775 shared_avg_ms=0.237 all_reduce_avg_ms=0.329 total_avg_ms=1.424
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=8080 tokens=8080 router_gate_avg_ms=0.069 routed_avg_ms=0.769 shared_avg_ms=0.235 all_reduce_avg_ms=0.342 total_avg_ms=1.426
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=8080 tokens=8080 router_gate_avg_ms=0.070 routed_avg_ms=0.774 shared_avg_ms=0.234 all_reduce_avg_ms=0.335 total_avg_ms=1.426
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=8080 tokens=8080 router_gate_avg_ms=0.069 routed_avg_ms=0.769 shared_avg_ms=0.237 all_reduce_avg_ms=0.336 total_avg_ms=1.423
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=8080 tokens=8080 router_gate_avg_ms=0.071 routed_avg_ms=0.775 shared_avg_ms=0.237 all_reduce_avg_ms=0.329 total_avg_ms=1.424
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=8160 tokens=8160 router_gate_avg_ms=0.069 routed_avg_ms=0.769 shared_avg_ms=0.235 all_reduce_avg_ms=0.342 total_avg_ms=1.426
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=8160 tokens=8160 router_gate_avg_ms=0.070 routed_avg_ms=0.774 shared_avg_ms=0.234 all_reduce_avg_ms=0.335 total_avg_ms=1.426
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=8160 tokens=8160 router_gate_avg_ms=0.071 routed_avg_ms=0.775 shared_avg_ms=0.237 all_reduce_avg_ms=0.329 total_avg_ms=1.424
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=8160 tokens=8160 router_gate_avg_ms=0.069 routed_avg_ms=0.768 shared_avg_ms=0.237 all_reduce_avg_ms=0.336 total_avg_ms=1.422
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=8240 tokens=8240 router_gate_avg_ms=0.069 routed_avg_ms=0.769 shared_avg_ms=0.235 all_reduce_avg_ms=0.341 total_avg_ms=1.425
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=8240 tokens=8240 router_gate_avg_ms=0.070 routed_avg_ms=0.774 shared_avg_ms=0.234 all_reduce_avg_ms=0.335 total_avg_ms=1.426
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=8240 tokens=8240 router_gate_avg_ms=0.069 routed_avg_ms=0.768 shared_avg_ms=0.237 all_reduce_avg_ms=0.336 total_avg_ms=1.422
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=8240 tokens=8240 router_gate_avg_ms=0.071 routed_avg_ms=0.775 shared_avg_ms=0.237 all_reduce_avg_ms=0.329 total_avg_ms=1.423
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=8320 tokens=8320 router_gate_avg_ms=0.070 routed_avg_ms=0.774 shared_avg_ms=0.234 all_reduce_avg_ms=0.335 total_avg_ms=1.425
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=8320 tokens=8320 router_gate_avg_ms=0.069 routed_avg_ms=0.768 shared_avg_ms=0.237 all_reduce_avg_ms=0.335 total_avg_ms=1.422
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=8320 tokens=8320 router_gate_avg_ms=0.071 routed_avg_ms=0.775 shared_avg_ms=0.237 all_reduce_avg_ms=0.329 total_avg_ms=1.423
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=8320 tokens=8320 router_gate_avg_ms=0.069 routed_avg_ms=0.769 shared_avg_ms=0.235 all_reduce_avg_ms=0.341 total_avg_ms=1.425
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=8400 tokens=8400 router_gate_avg_ms=0.069 routed_avg_ms=0.768 shared_avg_ms=0.237 all_reduce_avg_ms=0.335 total_avg_ms=1.422
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=8400 tokens=8400 router_gate_avg_ms=0.069 routed_avg_ms=0.769 shared_avg_ms=0.235 all_reduce_avg_ms=0.341 total_avg_ms=1.425
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=8400 tokens=8400 router_gate_avg_ms=0.070 routed_avg_ms=0.774 shared_avg_ms=0.234 all_reduce_avg_ms=0.335 total_avg_ms=1.425
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=8400 tokens=8400 router_gate_avg_ms=0.071 routed_avg_ms=0.775 shared_avg_ms=0.237 all_reduce_avg_ms=0.329 total_avg_ms=1.423
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=8480 tokens=8480 router_gate_avg_ms=0.069 routed_avg_ms=0.768 shared_avg_ms=0.237 all_reduce_avg_ms=0.335 total_avg_ms=1.422
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=8480 tokens=8480 router_gate_avg_ms=0.069 routed_avg_ms=0.769 shared_avg_ms=0.235 all_reduce_avg_ms=0.341 total_avg_ms=1.425
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=8480 tokens=8480 router_gate_avg_ms=0.070 routed_avg_ms=0.774 shared_avg_ms=0.234 all_reduce_avg_ms=0.335 total_avg_ms=1.425
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=8480 tokens=8480 router_gate_avg_ms=0.071 routed_avg_ms=0.775 shared_avg_ms=0.237 all_reduce_avg_ms=0.329 total_avg_ms=1.423
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=8560 tokens=8560 router_gate_avg_ms=0.069 routed_avg_ms=0.768 shared_avg_ms=0.237 all_reduce_avg_ms=0.335 total_avg_ms=1.421
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=8560 tokens=8560 router_gate_avg_ms=0.070 routed_avg_ms=0.774 shared_avg_ms=0.234 all_reduce_avg_ms=0.335 total_avg_ms=1.425
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=8560 tokens=8560 router_gate_avg_ms=0.069 routed_avg_ms=0.769 shared_avg_ms=0.235 all_reduce_avg_ms=0.341 total_avg_ms=1.425
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=8560 tokens=8560 router_gate_avg_ms=0.071 routed_avg_ms=0.775 shared_avg_ms=0.237 all_reduce_avg_ms=0.329 total_avg_ms=1.423
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=8640 tokens=8640 router_gate_avg_ms=0.069 routed_avg_ms=0.768 shared_avg_ms=0.237 all_reduce_avg_ms=0.335 total_avg_ms=1.421
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=8640 tokens=8640 router_gate_avg_ms=0.069 routed_avg_ms=0.768 shared_avg_ms=0.235 all_reduce_avg_ms=0.340 total_avg_ms=1.424
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=8640 tokens=8640 router_gate_avg_ms=0.070 routed_avg_ms=0.774 shared_avg_ms=0.234 all_reduce_avg_ms=0.335 total_avg_ms=1.425
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=8640 tokens=8640 router_gate_avg_ms=0.071 routed_avg_ms=0.774 shared_avg_ms=0.237 all_reduce_avg_ms=0.329 total_avg_ms=1.422
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=8720 tokens=8720 router_gate_avg_ms=0.070 routed_avg_ms=0.773 shared_avg_ms=0.234 all_reduce_avg_ms=0.335 total_avg_ms=1.424
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=8720 tokens=8720 router_gate_avg_ms=0.069 routed_avg_ms=0.768 shared_avg_ms=0.237 all_reduce_avg_ms=0.335 total_avg_ms=1.420
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=8720 tokens=8720 router_gate_avg_ms=0.071 routed_avg_ms=0.774 shared_avg_ms=0.236 all_reduce_avg_ms=0.329 total_avg_ms=1.422
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=8720 tokens=8720 router_gate_avg_ms=0.069 routed_avg_ms=0.768 shared_avg_ms=0.235 all_reduce_avg_ms=0.340 total_avg_ms=1.424
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=8800 tokens=8800 router_gate_avg_ms=0.069 routed_avg_ms=0.768 shared_avg_ms=0.237 all_reduce_avg_ms=0.335 total_avg_ms=1.420
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=8800 tokens=8800 router_gate_avg_ms=0.071 routed_avg_ms=0.774 shared_avg_ms=0.236 all_reduce_avg_ms=0.329 total_avg_ms=1.422
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=8800 tokens=8800 router_gate_avg_ms=0.069 routed_avg_ms=0.768 shared_avg_ms=0.235 all_reduce_avg_ms=0.340 total_avg_ms=1.424
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=8800 tokens=8800 router_gate_avg_ms=0.070 routed_avg_ms=0.773 shared_avg_ms=0.234 all_reduce_avg_ms=0.335 total_avg_ms=1.424
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=8880 tokens=8880 router_gate_avg_ms=0.070 routed_avg_ms=0.773 shared_avg_ms=0.234 all_reduce_avg_ms=0.335 total_avg_ms=1.424
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=8880 tokens=8880 router_gate_avg_ms=0.069 routed_avg_ms=0.768 shared_avg_ms=0.237 all_reduce_avg_ms=0.335 total_avg_ms=1.420
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=8880 tokens=8880 router_gate_avg_ms=0.071 routed_avg_ms=0.774 shared_avg_ms=0.236 all_reduce_avg_ms=0.329 total_avg_ms=1.422
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=8880 tokens=8880 router_gate_avg_ms=0.069 routed_avg_ms=0.768 shared_avg_ms=0.235 all_reduce_avg_ms=0.340 total_avg_ms=1.423
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=8960 tokens=8960 router_gate_avg_ms=0.069 routed_avg_ms=0.769 shared_avg_ms=0.235 all_reduce_avg_ms=0.339 total_avg_ms=1.424
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=8960 tokens=8960 router_gate_avg_ms=0.070 routed_avg_ms=0.773 shared_avg_ms=0.234 all_reduce_avg_ms=0.335 total_avg_ms=1.424
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=8960 tokens=8960 router_gate_avg_ms=0.071 routed_avg_ms=0.774 shared_avg_ms=0.236 all_reduce_avg_ms=0.329 total_avg_ms=1.422
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=8960 tokens=8960 router_gate_avg_ms=0.069 routed_avg_ms=0.768 shared_avg_ms=0.237 all_reduce_avg_ms=0.335 total_avg_ms=1.420
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=9040 tokens=9040 router_gate_avg_ms=0.069 routed_avg_ms=0.769 shared_avg_ms=0.235 all_reduce_avg_ms=0.339 total_avg_ms=1.424
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=9040 tokens=9040 router_gate_avg_ms=0.069 routed_avg_ms=0.768 shared_avg_ms=0.237 all_reduce_avg_ms=0.335 total_avg_ms=1.420
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=9040 tokens=9040 router_gate_avg_ms=0.070 routed_avg_ms=0.773 shared_avg_ms=0.234 all_reduce_avg_ms=0.335 total_avg_ms=1.424
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=9040 tokens=9040 router_gate_avg_ms=0.071 routed_avg_ms=0.774 shared_avg_ms=0.236 all_reduce_avg_ms=0.329 total_avg_ms=1.422
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=9120 tokens=9120 router_gate_avg_ms=0.069 routed_avg_ms=0.768 shared_avg_ms=0.237 all_reduce_avg_ms=0.334 total_avg_ms=1.420
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=9120 tokens=9120 router_gate_avg_ms=0.070 routed_avg_ms=0.773 shared_avg_ms=0.234 all_reduce_avg_ms=0.335 total_avg_ms=1.424
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=9120 tokens=9120 router_gate_avg_ms=0.069 routed_avg_ms=0.769 shared_avg_ms=0.235 all_reduce_avg_ms=0.339 total_avg_ms=1.423
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=9120 tokens=9120 router_gate_avg_ms=0.071 routed_avg_ms=0.774 shared_avg_ms=0.236 all_reduce_avg_ms=0.329 total_avg_ms=1.422
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=9200 tokens=9200 router_gate_avg_ms=0.069 routed_avg_ms=0.768 shared_avg_ms=0.237 all_reduce_avg_ms=0.335 total_avg_ms=1.420
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=9200 tokens=9200 router_gate_avg_ms=0.069 routed_avg_ms=0.769 shared_avg_ms=0.235 all_reduce_avg_ms=0.339 total_avg_ms=1.423
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=9200 tokens=9200 router_gate_avg_ms=0.070 routed_avg_ms=0.773 shared_avg_ms=0.234 all_reduce_avg_ms=0.335 total_avg_ms=1.424
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=9200 tokens=9200 router_gate_avg_ms=0.071 routed_avg_ms=0.774 shared_avg_ms=0.236 all_reduce_avg_ms=0.329 total_avg_ms=1.421
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=9280 tokens=9280 router_gate_avg_ms=0.069 routed_avg_ms=0.768 shared_avg_ms=0.237 all_reduce_avg_ms=0.334 total_avg_ms=1.420
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=9280 tokens=9280 router_gate_avg_ms=0.069 routed_avg_ms=0.769 shared_avg_ms=0.235 all_reduce_avg_ms=0.339 total_avg_ms=1.423
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=9280 tokens=9280 router_gate_avg_ms=0.070 routed_avg_ms=0.773 shared_avg_ms=0.234 all_reduce_avg_ms=0.335 total_avg_ms=1.423
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=9280 tokens=9280 router_gate_avg_ms=0.071 routed_avg_ms=0.774 shared_avg_ms=0.236 all_reduce_avg_ms=0.329 total_avg_ms=1.421
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=9360 tokens=9360 router_gate_avg_ms=0.069 routed_avg_ms=0.768 shared_avg_ms=0.237 all_reduce_avg_ms=0.334 total_avg_ms=1.420
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=9360 tokens=9360 router_gate_avg_ms=0.071 routed_avg_ms=0.774 shared_avg_ms=0.236 all_reduce_avg_ms=0.329 total_avg_ms=1.421
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=9360 tokens=9360 router_gate_avg_ms=0.069 routed_avg_ms=0.768 shared_avg_ms=0.235 all_reduce_avg_ms=0.339 total_avg_ms=1.423
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=9360 tokens=9360 router_gate_avg_ms=0.070 routed_avg_ms=0.773 shared_avg_ms=0.234 all_reduce_avg_ms=0.335 total_avg_ms=1.423
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=9440 tokens=9440 router_gate_avg_ms=0.069 routed_avg_ms=0.767 shared_avg_ms=0.237 all_reduce_avg_ms=0.334 total_avg_ms=1.420
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=9440 tokens=9440 router_gate_avg_ms=0.069 routed_avg_ms=0.768 shared_avg_ms=0.234 all_reduce_avg_ms=0.339 total_avg_ms=1.423
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=9440 tokens=9440 router_gate_avg_ms=0.070 routed_avg_ms=0.773 shared_avg_ms=0.234 all_reduce_avg_ms=0.335 total_avg_ms=1.423
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=9440 tokens=9440 router_gate_avg_ms=0.071 routed_avg_ms=0.774 shared_avg_ms=0.236 all_reduce_avg_ms=0.329 total_avg_ms=1.421
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=9520 tokens=9520 router_gate_avg_ms=0.069 routed_avg_ms=0.767 shared_avg_ms=0.237 all_reduce_avg_ms=0.335 total_avg_ms=1.420
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=9520 tokens=9520 router_gate_avg_ms=0.069 routed_avg_ms=0.768 shared_avg_ms=0.234 all_reduce_avg_ms=0.339 total_avg_ms=1.423
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=9520 tokens=9520 router_gate_avg_ms=0.070 routed_avg_ms=0.773 shared_avg_ms=0.234 all_reduce_avg_ms=0.335 total_avg_ms=1.423
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=9520 tokens=9520 router_gate_avg_ms=0.071 routed_avg_ms=0.774 shared_avg_ms=0.236 all_reduce_avg_ms=0.329 total_avg_ms=1.421
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=9600 tokens=9600 router_gate_avg_ms=0.069 routed_avg_ms=0.767 shared_avg_ms=0.237 all_reduce_avg_ms=0.334 total_avg_ms=1.420
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=9600 tokens=9600 router_gate_avg_ms=0.069 routed_avg_ms=0.768 shared_avg_ms=0.234 all_reduce_avg_ms=0.340 total_avg_ms=1.423
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=9600 tokens=9600 router_gate_avg_ms=0.070 routed_avg_ms=0.773 shared_avg_ms=0.234 all_reduce_avg_ms=0.335 total_avg_ms=1.423
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=9600 tokens=9600 router_gate_avg_ms=0.071 routed_avg_ms=0.774 shared_avg_ms=0.236 all_reduce_avg_ms=0.328 total_avg_ms=1.421
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=9680 tokens=9680 router_gate_avg_ms=0.069 routed_avg_ms=0.767 shared_avg_ms=0.237 all_reduce_avg_ms=0.334 total_avg_ms=1.420
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=9680 tokens=9680 router_gate_avg_ms=0.069 routed_avg_ms=0.768 shared_avg_ms=0.234 all_reduce_avg_ms=0.340 total_avg_ms=1.423
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=9680 tokens=9680 router_gate_avg_ms=0.070 routed_avg_ms=0.773 shared_avg_ms=0.234 all_reduce_avg_ms=0.335 total_avg_ms=1.423
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=9680 tokens=9680 router_gate_avg_ms=0.071 routed_avg_ms=0.774 shared_avg_ms=0.236 all_reduce_avg_ms=0.328 total_avg_ms=1.421
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=9760 tokens=9760 router_gate_avg_ms=0.069 routed_avg_ms=0.768 shared_avg_ms=0.234 all_reduce_avg_ms=0.340 total_avg_ms=1.423
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=9760 tokens=9760 router_gate_avg_ms=0.070 routed_avg_ms=0.773 shared_avg_ms=0.234 all_reduce_avg_ms=0.335 total_avg_ms=1.423
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=9760 tokens=9760 router_gate_avg_ms=0.069 routed_avg_ms=0.767 shared_avg_ms=0.237 all_reduce_avg_ms=0.334 total_avg_ms=1.420
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=9760 tokens=9760 router_gate_avg_ms=0.071 routed_avg_ms=0.774 shared_avg_ms=0.236 all_reduce_avg_ms=0.328 total_avg_ms=1.421
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=9840 tokens=9840 router_gate_avg_ms=0.069 routed_avg_ms=0.768 shared_avg_ms=0.234 all_reduce_avg_ms=0.340 total_avg_ms=1.423
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=9840 tokens=9840 router_gate_avg_ms=0.069 routed_avg_ms=0.767 shared_avg_ms=0.237 all_reduce_avg_ms=0.334 total_avg_ms=1.419
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=9840 tokens=9840 router_gate_avg_ms=0.070 routed_avg_ms=0.773 shared_avg_ms=0.234 all_reduce_avg_ms=0.335 total_avg_ms=1.423
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=9840 tokens=9840 router_gate_avg_ms=0.071 routed_avg_ms=0.774 shared_avg_ms=0.236 all_reduce_avg_ms=0.328 total_avg_ms=1.421
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=9920 tokens=9920 router_gate_avg_ms=0.069 routed_avg_ms=0.767 shared_avg_ms=0.237 all_reduce_avg_ms=0.334 total_avg_ms=1.419
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=9920 tokens=9920 router_gate_avg_ms=0.069 routed_avg_ms=0.768 shared_avg_ms=0.234 all_reduce_avg_ms=0.340 total_avg_ms=1.423
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=9920 tokens=9920 router_gate_avg_ms=0.070 routed_avg_ms=0.773 shared_avg_ms=0.234 all_reduce_avg_ms=0.335 total_avg_ms=1.423
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=9920 tokens=9920 router_gate_avg_ms=0.071 routed_avg_ms=0.774 shared_avg_ms=0.236 all_reduce_avg_ms=0.328 total_avg_ms=1.421
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=10000 tokens=10000 router_gate_avg_ms=0.069 routed_avg_ms=0.767 shared_avg_ms=0.237 all_reduce_avg_ms=0.334 total_avg_ms=1.419
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=10000 tokens=10000 router_gate_avg_ms=0.070 routed_avg_ms=0.773 shared_avg_ms=0.234 all_reduce_avg_ms=0.335 total_avg_ms=1.423
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=10000 tokens=10000 router_gate_avg_ms=0.071 routed_avg_ms=0.774 shared_avg_ms=0.236 all_reduce_avg_ms=0.328 total_avg_ms=1.421
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=10000 tokens=10000 router_gate_avg_ms=0.069 routed_avg_ms=0.768 shared_avg_ms=0.234 all_reduce_avg_ms=0.340 total_avg_ms=1.423
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=10080 tokens=10080 router_gate_avg_ms=0.069 routed_avg_ms=0.768 shared_avg_ms=0.234 all_reduce_avg_ms=0.340 total_avg_ms=1.423
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=10080 tokens=10080 router_gate_avg_ms=0.069 routed_avg_ms=0.767 shared_avg_ms=0.237 all_reduce_avg_ms=0.334 total_avg_ms=1.419
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=10080 tokens=10080 router_gate_avg_ms=0.070 routed_avg_ms=0.772 shared_avg_ms=0.234 all_reduce_avg_ms=0.336 total_avg_ms=1.423
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=10080 tokens=10080 router_gate_avg_ms=0.071 routed_avg_ms=0.774 shared_avg_ms=0.236 all_reduce_avg_ms=0.328 total_avg_ms=1.421
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=10160 tokens=10160 router_gate_avg_ms=0.069 routed_avg_ms=0.768 shared_avg_ms=0.234 all_reduce_avg_ms=0.341 total_avg_ms=1.423
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=10160 tokens=10160 router_gate_avg_ms=0.070 routed_avg_ms=0.772 shared_avg_ms=0.234 all_reduce_avg_ms=0.336 total_avg_ms=1.424
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=10160 tokens=10160 router_gate_avg_ms=0.069 routed_avg_ms=0.767 shared_avg_ms=0.237 all_reduce_avg_ms=0.334 total_avg_ms=1.419
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=10160 tokens=10160 router_gate_avg_ms=0.071 routed_avg_ms=0.774 shared_avg_ms=0.237 all_reduce_avg_ms=0.328 total_avg_ms=1.421
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=10240 tokens=10240 router_gate_avg_ms=0.069 routed_avg_ms=0.767 shared_avg_ms=0.237 all_reduce_avg_ms=0.334 total_avg_ms=1.419
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=10240 tokens=10240 router_gate_avg_ms=0.069 routed_avg_ms=0.768 shared_avg_ms=0.234 all_reduce_avg_ms=0.341 total_avg_ms=1.423
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=10240 tokens=10240 router_gate_avg_ms=0.070 routed_avg_ms=0.772 shared_avg_ms=0.234 all_reduce_avg_ms=0.336 total_avg_ms=1.423
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=10240 tokens=10240 router_gate_avg_ms=0.071 routed_avg_ms=0.774 shared_avg_ms=0.237 all_reduce_avg_ms=0.328 total_avg_ms=1.421
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=10320 tokens=10320 router_gate_avg_ms=0.069 routed_avg_ms=0.767 shared_avg_ms=0.237 all_reduce_avg_ms=0.334 total_avg_ms=1.419
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=10320 tokens=10320 router_gate_avg_ms=0.069 routed_avg_ms=0.768 shared_avg_ms=0.234 all_reduce_avg_ms=0.340 total_avg_ms=1.423
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=10320 tokens=10320 router_gate_avg_ms=0.070 routed_avg_ms=0.772 shared_avg_ms=0.234 all_reduce_avg_ms=0.336 total_avg_ms=1.423
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=10320 tokens=10320 router_gate_avg_ms=0.071 routed_avg_ms=0.774 shared_avg_ms=0.237 all_reduce_avg_ms=0.328 total_avg_ms=1.421
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=10400 tokens=10400 router_gate_avg_ms=0.069 routed_avg_ms=0.767 shared_avg_ms=0.237 all_reduce_avg_ms=0.334 total_avg_ms=1.419
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=10400 tokens=10400 router_gate_avg_ms=0.070 routed_avg_ms=0.772 shared_avg_ms=0.234 all_reduce_avg_ms=0.336 total_avg_ms=1.423
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=10400 tokens=10400 router_gate_avg_ms=0.069 routed_avg_ms=0.768 shared_avg_ms=0.234 all_reduce_avg_ms=0.340 total_avg_ms=1.423
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=10400 tokens=10400 router_gate_avg_ms=0.071 routed_avg_ms=0.774 shared_avg_ms=0.237 all_reduce_avg_ms=0.328 total_avg_ms=1.421
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=10480 tokens=10480 router_gate_avg_ms=0.069 routed_avg_ms=0.767 shared_avg_ms=0.237 all_reduce_avg_ms=0.334 total_avg_ms=1.419
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=10480 tokens=10480 router_gate_avg_ms=0.071 routed_avg_ms=0.774 shared_avg_ms=0.236 all_reduce_avg_ms=0.328 total_avg_ms=1.420
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=10480 tokens=10480 router_gate_avg_ms=0.069 routed_avg_ms=0.768 shared_avg_ms=0.234 all_reduce_avg_ms=0.340 total_avg_ms=1.423
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=10480 tokens=10480 router_gate_avg_ms=0.070 routed_avg_ms=0.772 shared_avg_ms=0.234 all_reduce_avg_ms=0.336 total_avg_ms=1.423
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=10560 tokens=10560 router_gate_avg_ms=0.070 routed_avg_ms=0.772 shared_avg_ms=0.234 all_reduce_avg_ms=0.335 total_avg_ms=1.423
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=10560 tokens=10560 router_gate_avg_ms=0.069 routed_avg_ms=0.767 shared_avg_ms=0.237 all_reduce_avg_ms=0.334 total_avg_ms=1.419
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=10560 tokens=10560 router_gate_avg_ms=0.071 routed_avg_ms=0.773 shared_avg_ms=0.236 all_reduce_avg_ms=0.328 total_avg_ms=1.420
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=10560 tokens=10560 router_gate_avg_ms=0.069 routed_avg_ms=0.768 shared_avg_ms=0.234 all_reduce_avg_ms=0.340 total_avg_ms=1.423
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=10640 tokens=10640 router_gate_avg_ms=0.069 routed_avg_ms=0.768 shared_avg_ms=0.234 all_reduce_avg_ms=0.340 total_avg_ms=1.422
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=10640 tokens=10640 router_gate_avg_ms=0.069 routed_avg_ms=0.767 shared_avg_ms=0.237 all_reduce_avg_ms=0.334 total_avg_ms=1.419
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=10640 tokens=10640 router_gate_avg_ms=0.071 routed_avg_ms=0.773 shared_avg_ms=0.236 all_reduce_avg_ms=0.328 total_avg_ms=1.420
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=10640 tokens=10640 router_gate_avg_ms=0.070 routed_avg_ms=0.772 shared_avg_ms=0.234 all_reduce_avg_ms=0.335 total_avg_ms=1.423
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=10720 tokens=10720 router_gate_avg_ms=0.069 routed_avg_ms=0.767 shared_avg_ms=0.237 all_reduce_avg_ms=0.334 total_avg_ms=1.419
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=10720 tokens=10720 router_gate_avg_ms=0.069 routed_avg_ms=0.768 shared_avg_ms=0.234 all_reduce_avg_ms=0.340 total_avg_ms=1.422
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=10720 tokens=10720 router_gate_avg_ms=0.070 routed_avg_ms=0.772 shared_avg_ms=0.234 all_reduce_avg_ms=0.335 total_avg_ms=1.423
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=10720 tokens=10720 router_gate_avg_ms=0.071 routed_avg_ms=0.773 shared_avg_ms=0.236 all_reduce_avg_ms=0.328 total_avg_ms=1.420
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=10800 tokens=10800 router_gate_avg_ms=0.069 routed_avg_ms=0.767 shared_avg_ms=0.237 all_reduce_avg_ms=0.334 total_avg_ms=1.419
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=10800 tokens=10800 router_gate_avg_ms=0.071 routed_avg_ms=0.773 shared_avg_ms=0.236 all_reduce_avg_ms=0.328 total_avg_ms=1.420
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=10800 tokens=10800 router_gate_avg_ms=0.069 routed_avg_ms=0.768 shared_avg_ms=0.234 all_reduce_avg_ms=0.340 total_avg_ms=1.422
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=10800 tokens=10800 router_gate_avg_ms=0.070 routed_avg_ms=0.772 shared_avg_ms=0.234 all_reduce_avg_ms=0.335 total_avg_ms=1.423
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=10880 tokens=10880 router_gate_avg_ms=0.069 routed_avg_ms=0.767 shared_avg_ms=0.237 all_reduce_avg_ms=0.334 total_avg_ms=1.419
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=10880 tokens=10880 router_gate_avg_ms=0.069 routed_avg_ms=0.767 shared_avg_ms=0.234 all_reduce_avg_ms=0.340 total_avg_ms=1.422
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=10880 tokens=10880 router_gate_avg_ms=0.070 routed_avg_ms=0.772 shared_avg_ms=0.234 all_reduce_avg_ms=0.335 total_avg_ms=1.423
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=10880 tokens=10880 router_gate_avg_ms=0.071 routed_avg_ms=0.773 shared_avg_ms=0.236 all_reduce_avg_ms=0.328 total_avg_ms=1.420
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=10960 tokens=10960 router_gate_avg_ms=0.069 routed_avg_ms=0.767 shared_avg_ms=0.237 all_reduce_avg_ms=0.334 total_avg_ms=1.419
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=10960 tokens=10960 router_gate_avg_ms=0.071 routed_avg_ms=0.773 shared_avg_ms=0.236 all_reduce_avg_ms=0.328 total_avg_ms=1.420
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=10960 tokens=10960 router_gate_avg_ms=0.069 routed_avg_ms=0.767 shared_avg_ms=0.234 all_reduce_avg_ms=0.340 total_avg_ms=1.422
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=10960 tokens=10960 router_gate_avg_ms=0.070 routed_avg_ms=0.772 shared_avg_ms=0.234 all_reduce_avg_ms=0.335 total_avg_ms=1.423
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=11040 tokens=11040 router_gate_avg_ms=0.069 routed_avg_ms=0.767 shared_avg_ms=0.234 all_reduce_avg_ms=0.340 total_avg_ms=1.422
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=11040 tokens=11040 router_gate_avg_ms=0.070 routed_avg_ms=0.772 shared_avg_ms=0.234 all_reduce_avg_ms=0.335 total_avg_ms=1.422
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=11040 tokens=11040 router_gate_avg_ms=0.069 routed_avg_ms=0.767 shared_avg_ms=0.237 all_reduce_avg_ms=0.334 total_avg_ms=1.419
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=11040 tokens=11040 router_gate_avg_ms=0.071 routed_avg_ms=0.773 shared_avg_ms=0.236 all_reduce_avg_ms=0.328 total_avg_ms=1.420
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=11120 tokens=11120 router_gate_avg_ms=0.069 routed_avg_ms=0.767 shared_avg_ms=0.234 all_reduce_avg_ms=0.340 total_avg_ms=1.422
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=11120 tokens=11120 router_gate_avg_ms=0.070 routed_avg_ms=0.772 shared_avg_ms=0.234 all_reduce_avg_ms=0.335 total_avg_ms=1.422
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=11120 tokens=11120 router_gate_avg_ms=0.069 routed_avg_ms=0.767 shared_avg_ms=0.237 all_reduce_avg_ms=0.334 total_avg_ms=1.418
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=11120 tokens=11120 router_gate_avg_ms=0.071 routed_avg_ms=0.773 shared_avg_ms=0.236 all_reduce_avg_ms=0.328 total_avg_ms=1.420
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=11200 tokens=11200 router_gate_avg_ms=0.069 routed_avg_ms=0.767 shared_avg_ms=0.237 all_reduce_avg_ms=0.334 total_avg_ms=1.418
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=11200 tokens=11200 router_gate_avg_ms=0.069 routed_avg_ms=0.767 shared_avg_ms=0.234 all_reduce_avg_ms=0.340 total_avg_ms=1.422
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=11200 tokens=11200 router_gate_avg_ms=0.070 routed_avg_ms=0.772 shared_avg_ms=0.234 all_reduce_avg_ms=0.335 total_avg_ms=1.422
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=11200 tokens=11200 router_gate_avg_ms=0.071 routed_avg_ms=0.773 shared_avg_ms=0.236 all_reduce_avg_ms=0.328 total_avg_ms=1.420
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=11280 tokens=11280 router_gate_avg_ms=0.069 routed_avg_ms=0.767 shared_avg_ms=0.237 all_reduce_avg_ms=0.334 total_avg_ms=1.418
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=11280 tokens=11280 router_gate_avg_ms=0.070 routed_avg_ms=0.772 shared_avg_ms=0.234 all_reduce_avg_ms=0.335 total_avg_ms=1.422
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=11280 tokens=11280 router_gate_avg_ms=0.071 routed_avg_ms=0.773 shared_avg_ms=0.236 all_reduce_avg_ms=0.328 total_avg_ms=1.420
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=11280 tokens=11280 router_gate_avg_ms=0.069 routed_avg_ms=0.767 shared_avg_ms=0.234 all_reduce_avg_ms=0.340 total_avg_ms=1.422
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=11360 tokens=11360 router_gate_avg_ms=0.069 routed_avg_ms=0.767 shared_avg_ms=0.234 all_reduce_avg_ms=0.340 total_avg_ms=1.422
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=11360 tokens=11360 router_gate_avg_ms=0.070 routed_avg_ms=0.772 shared_avg_ms=0.234 all_reduce_avg_ms=0.335 total_avg_ms=1.422
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=11360 tokens=11360 router_gate_avg_ms=0.071 routed_avg_ms=0.773 shared_avg_ms=0.236 all_reduce_avg_ms=0.328 total_avg_ms=1.419
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=11360 tokens=11360 router_gate_avg_ms=0.069 routed_avg_ms=0.767 shared_avg_ms=0.237 all_reduce_avg_ms=0.333 total_avg_ms=1.418
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=11440 tokens=11440 router_gate_avg_ms=0.070 routed_avg_ms=0.772 shared_avg_ms=0.234 all_reduce_avg_ms=0.335 total_avg_ms=1.422
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=11440 tokens=11440 router_gate_avg_ms=0.069 routed_avg_ms=0.767 shared_avg_ms=0.237 all_reduce_avg_ms=0.333 total_avg_ms=1.418
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=11440 tokens=11440 router_gate_avg_ms=0.071 routed_avg_ms=0.773 shared_avg_ms=0.236 all_reduce_avg_ms=0.328 total_avg_ms=1.419
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=11440 tokens=11440 router_gate_avg_ms=0.069 routed_avg_ms=0.767 shared_avg_ms=0.234 all_reduce_avg_ms=0.340 total_avg_ms=1.422
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=11520 tokens=11520 router_gate_avg_ms=0.069 routed_avg_ms=0.767 shared_avg_ms=0.234 all_reduce_avg_ms=0.340 total_avg_ms=1.422
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=11520 tokens=11520 router_gate_avg_ms=0.070 routed_avg_ms=0.772 shared_avg_ms=0.234 all_reduce_avg_ms=0.335 total_avg_ms=1.422
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=11520 tokens=11520 router_gate_avg_ms=0.069 routed_avg_ms=0.767 shared_avg_ms=0.237 all_reduce_avg_ms=0.333 total_avg_ms=1.418
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=11520 tokens=11520 router_gate_avg_ms=0.071 routed_avg_ms=0.773 shared_avg_ms=0.236 all_reduce_avg_ms=0.328 total_avg_ms=1.420
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=11600 tokens=11600 router_gate_avg_ms=0.069 routed_avg_ms=0.767 shared_avg_ms=0.234 all_reduce_avg_ms=0.340 total_avg_ms=1.422
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=11600 tokens=11600 router_gate_avg_ms=0.070 routed_avg_ms=0.772 shared_avg_ms=0.233 all_reduce_avg_ms=0.335 total_avg_ms=1.422
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=11600 tokens=11600 router_gate_avg_ms=0.071 routed_avg_ms=0.773 shared_avg_ms=0.236 all_reduce_avg_ms=0.328 total_avg_ms=1.419
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=11600 tokens=11600 router_gate_avg_ms=0.069 routed_avg_ms=0.767 shared_avg_ms=0.237 all_reduce_avg_ms=0.333 total_avg_ms=1.418
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=11680 tokens=11680 router_gate_avg_ms=0.069 routed_avg_ms=0.767 shared_avg_ms=0.234 all_reduce_avg_ms=0.340 total_avg_ms=1.422
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=11680 tokens=11680 router_gate_avg_ms=0.070 routed_avg_ms=0.772 shared_avg_ms=0.233 all_reduce_avg_ms=0.335 total_avg_ms=1.422
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=11680 tokens=11680 router_gate_avg_ms=0.069 routed_avg_ms=0.767 shared_avg_ms=0.237 all_reduce_avg_ms=0.333 total_avg_ms=1.418
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=11680 tokens=11680 router_gate_avg_ms=0.071 routed_avg_ms=0.773 shared_avg_ms=0.236 all_reduce_avg_ms=0.329 total_avg_ms=1.420
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=11760 tokens=11760 router_gate_avg_ms=0.069 routed_avg_ms=0.767 shared_avg_ms=0.237 all_reduce_avg_ms=0.333 total_avg_ms=1.417
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=11760 tokens=11760 router_gate_avg_ms=0.069 routed_avg_ms=0.767 shared_avg_ms=0.234 all_reduce_avg_ms=0.340 total_avg_ms=1.422
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=11760 tokens=11760 router_gate_avg_ms=0.070 routed_avg_ms=0.771 shared_avg_ms=0.233 all_reduce_avg_ms=0.335 total_avg_ms=1.421
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=11760 tokens=11760 router_gate_avg_ms=0.071 routed_avg_ms=0.773 shared_avg_ms=0.236 all_reduce_avg_ms=0.328 total_avg_ms=1.419
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=11840 tokens=11840 router_gate_avg_ms=0.069 routed_avg_ms=0.767 shared_avg_ms=0.237 all_reduce_avg_ms=0.333 total_avg_ms=1.417
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=11840 tokens=11840 router_gate_avg_ms=0.071 routed_avg_ms=0.773 shared_avg_ms=0.236 all_reduce_avg_ms=0.328 total_avg_ms=1.419
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=11840 tokens=11840 router_gate_avg_ms=0.069 routed_avg_ms=0.767 shared_avg_ms=0.234 all_reduce_avg_ms=0.340 total_avg_ms=1.422
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=11840 tokens=11840 router_gate_avg_ms=0.070 routed_avg_ms=0.771 shared_avg_ms=0.233 all_reduce_avg_ms=0.335 total_avg_ms=1.421
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=11920 tokens=11920 router_gate_avg_ms=0.069 routed_avg_ms=0.767 shared_avg_ms=0.237 all_reduce_avg_ms=0.333 total_avg_ms=1.417
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=11920 tokens=11920 router_gate_avg_ms=0.069 routed_avg_ms=0.767 shared_avg_ms=0.234 all_reduce_avg_ms=0.340 total_avg_ms=1.421
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=11920 tokens=11920 router_gate_avg_ms=0.070 routed_avg_ms=0.771 shared_avg_ms=0.233 all_reduce_avg_ms=0.335 total_avg_ms=1.421
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=11920 tokens=11920 router_gate_avg_ms=0.071 routed_avg_ms=0.773 shared_avg_ms=0.236 all_reduce_avg_ms=0.328 total_avg_ms=1.419
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=12000 tokens=12000 router_gate_avg_ms=0.069 routed_avg_ms=0.767 shared_avg_ms=0.237 all_reduce_avg_ms=0.333 total_avg_ms=1.417
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=12000 tokens=12000 router_gate_avg_ms=0.069 routed_avg_ms=0.767 shared_avg_ms=0.234 all_reduce_avg_ms=0.340 total_avg_ms=1.421
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=12000 tokens=12000 router_gate_avg_ms=0.070 routed_avg_ms=0.771 shared_avg_ms=0.233 all_reduce_avg_ms=0.335 total_avg_ms=1.421
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=12000 tokens=12000 router_gate_avg_ms=0.071 routed_avg_ms=0.772 shared_avg_ms=0.236 all_reduce_avg_ms=0.328 total_avg_ms=1.419
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=12080 tokens=12080 router_gate_avg_ms=0.069 routed_avg_ms=0.767 shared_avg_ms=0.237 all_reduce_avg_ms=0.333 total_avg_ms=1.417
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=12080 tokens=12080 router_gate_avg_ms=0.069 routed_avg_ms=0.767 shared_avg_ms=0.234 all_reduce_avg_ms=0.340 total_avg_ms=1.421
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=12080 tokens=12080 router_gate_avg_ms=0.070 routed_avg_ms=0.771 shared_avg_ms=0.233 all_reduce_avg_ms=0.335 total_avg_ms=1.421
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=12080 tokens=12080 router_gate_avg_ms=0.071 routed_avg_ms=0.772 shared_avg_ms=0.236 all_reduce_avg_ms=0.328 total_avg_ms=1.419
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=12160 tokens=12160 router_gate_avg_ms=0.070 routed_avg_ms=0.771 shared_avg_ms=0.233 all_reduce_avg_ms=0.335 total_avg_ms=1.421
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=12160 tokens=12160 router_gate_avg_ms=0.069 routed_avg_ms=0.767 shared_avg_ms=0.237 all_reduce_avg_ms=0.333 total_avg_ms=1.417
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=12160 tokens=12160 router_gate_avg_ms=0.069 routed_avg_ms=0.767 shared_avg_ms=0.234 all_reduce_avg_ms=0.340 total_avg_ms=1.421
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=12160 tokens=12160 router_gate_avg_ms=0.071 routed_avg_ms=0.772 shared_avg_ms=0.236 all_reduce_avg_ms=0.328 total_avg_ms=1.419
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=12240 tokens=12240 router_gate_avg_ms=0.069 routed_avg_ms=0.767 shared_avg_ms=0.234 all_reduce_avg_ms=0.339 total_avg_ms=1.421
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=12240 tokens=12240 router_gate_avg_ms=0.070 routed_avg_ms=0.771 shared_avg_ms=0.233 all_reduce_avg_ms=0.335 total_avg_ms=1.421
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=12240 tokens=12240 router_gate_avg_ms=0.071 routed_avg_ms=0.772 shared_avg_ms=0.236 all_reduce_avg_ms=0.328 total_avg_ms=1.419
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=12240 tokens=12240 router_gate_avg_ms=0.069 routed_avg_ms=0.767 shared_avg_ms=0.237 all_reduce_avg_ms=0.332 total_avg_ms=1.417
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=12320 tokens=12320 router_gate_avg_ms=0.069 routed_avg_ms=0.767 shared_avg_ms=0.234 all_reduce_avg_ms=0.339 total_avg_ms=1.421
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=12320 tokens=12320 router_gate_avg_ms=0.070 routed_avg_ms=0.771 shared_avg_ms=0.233 all_reduce_avg_ms=0.335 total_avg_ms=1.420
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=12320 tokens=12320 router_gate_avg_ms=0.069 routed_avg_ms=0.766 shared_avg_ms=0.237 all_reduce_avg_ms=0.332 total_avg_ms=1.416
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=12320 tokens=12320 router_gate_avg_ms=0.071 routed_avg_ms=0.772 shared_avg_ms=0.236 all_reduce_avg_ms=0.328 total_avg_ms=1.418
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=12400 tokens=12400 router_gate_avg_ms=0.069 routed_avg_ms=0.766 shared_avg_ms=0.237 all_reduce_avg_ms=0.332 total_avg_ms=1.416
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=12400 tokens=12400 router_gate_avg_ms=0.070 routed_avg_ms=0.771 shared_avg_ms=0.233 all_reduce_avg_ms=0.335 total_avg_ms=1.420
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=12400 tokens=12400 router_gate_avg_ms=0.069 routed_avg_ms=0.767 shared_avg_ms=0.234 all_reduce_avg_ms=0.339 total_avg_ms=1.420
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=12400 tokens=12400 router_gate_avg_ms=0.071 routed_avg_ms=0.772 shared_avg_ms=0.236 all_reduce_avg_ms=0.328 total_avg_ms=1.418
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=12480 tokens=12480 router_gate_avg_ms=0.069 routed_avg_ms=0.766 shared_avg_ms=0.237 all_reduce_avg_ms=0.332 total_avg_ms=1.416
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=12480 tokens=12480 router_gate_avg_ms=0.069 routed_avg_ms=0.767 shared_avg_ms=0.234 all_reduce_avg_ms=0.339 total_avg_ms=1.420
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=12480 tokens=12480 router_gate_avg_ms=0.070 routed_avg_ms=0.771 shared_avg_ms=0.233 all_reduce_avg_ms=0.335 total_avg_ms=1.420
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=12480 tokens=12480 router_gate_avg_ms=0.071 routed_avg_ms=0.772 shared_avg_ms=0.236 all_reduce_avg_ms=0.328 total_avg_ms=1.418
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=12560 tokens=12560 router_gate_avg_ms=0.069 routed_avg_ms=0.767 shared_avg_ms=0.234 all_reduce_avg_ms=0.339 total_avg_ms=1.420
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=12560 tokens=12560 router_gate_avg_ms=0.070 routed_avg_ms=0.771 shared_avg_ms=0.233 all_reduce_avg_ms=0.335 total_avg_ms=1.420
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=12560 tokens=12560 router_gate_avg_ms=0.069 routed_avg_ms=0.766 shared_avg_ms=0.237 all_reduce_avg_ms=0.332 total_avg_ms=1.416
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=12560 tokens=12560 router_gate_avg_ms=0.071 routed_avg_ms=0.772 shared_avg_ms=0.236 all_reduce_avg_ms=0.327 total_avg_ms=1.418
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=12640 tokens=12640 router_gate_avg_ms=0.070 routed_avg_ms=0.771 shared_avg_ms=0.233 all_reduce_avg_ms=0.334 total_avg_ms=1.420
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=12640 tokens=12640 router_gate_avg_ms=0.069 routed_avg_ms=0.766 shared_avg_ms=0.237 all_reduce_avg_ms=0.332 total_avg_ms=1.416
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=12640 tokens=12640 router_gate_avg_ms=0.071 routed_avg_ms=0.772 shared_avg_ms=0.236 all_reduce_avg_ms=0.327 total_avg_ms=1.418
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=12640 tokens=12640 router_gate_avg_ms=0.069 routed_avg_ms=0.767 shared_avg_ms=0.234 all_reduce_avg_ms=0.339 total_avg_ms=1.420
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=12720 tokens=12720 router_gate_avg_ms=0.071 routed_avg_ms=0.772 shared_avg_ms=0.236 all_reduce_avg_ms=0.328 total_avg_ms=1.418
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=12720 tokens=12720 router_gate_avg_ms=0.069 routed_avg_ms=0.766 shared_avg_ms=0.237 all_reduce_avg_ms=0.332 total_avg_ms=1.416
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=12720 tokens=12720 router_gate_avg_ms=0.069 routed_avg_ms=0.767 shared_avg_ms=0.234 all_reduce_avg_ms=0.339 total_avg_ms=1.420
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=12720 tokens=12720 router_gate_avg_ms=0.070 routed_avg_ms=0.771 shared_avg_ms=0.233 all_reduce_avg_ms=0.334 total_avg_ms=1.420
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=12800 tokens=12800 router_gate_avg_ms=0.069 routed_avg_ms=0.766 shared_avg_ms=0.237 all_reduce_avg_ms=0.332 total_avg_ms=1.416
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=12800 tokens=12800 router_gate_avg_ms=0.071 routed_avg_ms=0.772 shared_avg_ms=0.236 all_reduce_avg_ms=0.328 total_avg_ms=1.418
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=12800 tokens=12800 router_gate_avg_ms=0.069 routed_avg_ms=0.767 shared_avg_ms=0.234 all_reduce_avg_ms=0.339 total_avg_ms=1.420
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=12800 tokens=12800 router_gate_avg_ms=0.070 routed_avg_ms=0.771 shared_avg_ms=0.233 all_reduce_avg_ms=0.334 total_avg_ms=1.420
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=12880 tokens=12880 router_gate_avg_ms=0.069 routed_avg_ms=0.766 shared_avg_ms=0.237 all_reduce_avg_ms=0.332 total_avg_ms=1.415
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=12880 tokens=12880 router_gate_avg_ms=0.070 routed_avg_ms=0.772 shared_avg_ms=0.236 all_reduce_avg_ms=0.328 total_avg_ms=1.417
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=12880 tokens=12880 router_gate_avg_ms=0.069 routed_avg_ms=0.767 shared_avg_ms=0.234 all_reduce_avg_ms=0.339 total_avg_ms=1.419
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=12880 tokens=12880 router_gate_avg_ms=0.070 routed_avg_ms=0.771 shared_avg_ms=0.233 all_reduce_avg_ms=0.334 total_avg_ms=1.419
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=12960 tokens=12960 router_gate_avg_ms=0.071 routed_avg_ms=0.772 shared_avg_ms=0.236 all_reduce_avg_ms=0.328 total_avg_ms=1.418
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=12960 tokens=12960 router_gate_avg_ms=0.070 routed_avg_ms=0.771 shared_avg_ms=0.233 all_reduce_avg_ms=0.334 total_avg_ms=1.420
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=12960 tokens=12960 router_gate_avg_ms=0.069 routed_avg_ms=0.766 shared_avg_ms=0.237 all_reduce_avg_ms=0.332 total_avg_ms=1.416
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=12960 tokens=12960 router_gate_avg_ms=0.069 routed_avg_ms=0.767 shared_avg_ms=0.234 all_reduce_avg_ms=0.339 total_avg_ms=1.420
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=13040 tokens=13040 router_gate_avg_ms=0.069 routed_avg_ms=0.766 shared_avg_ms=0.237 all_reduce_avg_ms=0.332 total_avg_ms=1.416
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=13040 tokens=13040 router_gate_avg_ms=0.070 routed_avg_ms=0.771 shared_avg_ms=0.233 all_reduce_avg_ms=0.334 total_avg_ms=1.420
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=13040 tokens=13040 router_gate_avg_ms=0.071 routed_avg_ms=0.772 shared_avg_ms=0.236 all_reduce_avg_ms=0.328 total_avg_ms=1.418
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=13040 tokens=13040 router_gate_avg_ms=0.069 routed_avg_ms=0.767 shared_avg_ms=0.234 all_reduce_avg_ms=0.339 total_avg_ms=1.420
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=13120 tokens=13120 router_gate_avg_ms=0.069 routed_avg_ms=0.767 shared_avg_ms=0.234 all_reduce_avg_ms=0.339 total_avg_ms=1.420
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=13120 tokens=13120 router_gate_avg_ms=0.070 routed_avg_ms=0.771 shared_avg_ms=0.233 all_reduce_avg_ms=0.334 total_avg_ms=1.420
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=13120 tokens=13120 router_gate_avg_ms=0.069 routed_avg_ms=0.766 shared_avg_ms=0.237 all_reduce_avg_ms=0.332 total_avg_ms=1.416
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=13120 tokens=13120 router_gate_avg_ms=0.070 routed_avg_ms=0.772 shared_avg_ms=0.236 all_reduce_avg_ms=0.328 total_avg_ms=1.418
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=13200 tokens=13200 router_gate_avg_ms=0.070 routed_avg_ms=0.771 shared_avg_ms=0.233 all_reduce_avg_ms=0.334 total_avg_ms=1.420
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=13200 tokens=13200 router_gate_avg_ms=0.069 routed_avg_ms=0.766 shared_avg_ms=0.237 all_reduce_avg_ms=0.332 total_avg_ms=1.416
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=13200 tokens=13200 router_gate_avg_ms=0.070 routed_avg_ms=0.772 shared_avg_ms=0.236 all_reduce_avg_ms=0.328 total_avg_ms=1.418
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=13200 tokens=13200 router_gate_avg_ms=0.069 routed_avg_ms=0.767 shared_avg_ms=0.234 all_reduce_avg_ms=0.339 total_avg_ms=1.420
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=13280 tokens=13280 router_gate_avg_ms=0.070 routed_avg_ms=0.771 shared_avg_ms=0.233 all_reduce_avg_ms=0.334 total_avg_ms=1.419
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=13280 tokens=13280 router_gate_avg_ms=0.069 routed_avg_ms=0.766 shared_avg_ms=0.237 all_reduce_avg_ms=0.332 total_avg_ms=1.416
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=13280 tokens=13280 router_gate_avg_ms=0.070 routed_avg_ms=0.772 shared_avg_ms=0.236 all_reduce_avg_ms=0.328 total_avg_ms=1.418
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=13280 tokens=13280 router_gate_avg_ms=0.069 routed_avg_ms=0.767 shared_avg_ms=0.234 all_reduce_avg_ms=0.339 total_avg_ms=1.420
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=13360 tokens=13360 router_gate_avg_ms=0.070 routed_avg_ms=0.772 shared_avg_ms=0.236 all_reduce_avg_ms=0.328 total_avg_ms=1.417
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=13360 tokens=13360 router_gate_avg_ms=0.069 routed_avg_ms=0.766 shared_avg_ms=0.237 all_reduce_avg_ms=0.332 total_avg_ms=1.415
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=13360 tokens=13360 router_gate_avg_ms=0.069 routed_avg_ms=0.767 shared_avg_ms=0.234 all_reduce_avg_ms=0.339 total_avg_ms=1.419
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=13360 tokens=13360 router_gate_avg_ms=0.070 routed_avg_ms=0.771 shared_avg_ms=0.233 all_reduce_avg_ms=0.334 total_avg_ms=1.419
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=13440 tokens=13440 router_gate_avg_ms=0.070 routed_avg_ms=0.771 shared_avg_ms=0.233 all_reduce_avg_ms=0.334 total_avg_ms=1.419
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=13440 tokens=13440 router_gate_avg_ms=0.069 routed_avg_ms=0.766 shared_avg_ms=0.237 all_reduce_avg_ms=0.332 total_avg_ms=1.415
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=13440 tokens=13440 router_gate_avg_ms=0.070 routed_avg_ms=0.772 shared_avg_ms=0.236 all_reduce_avg_ms=0.328 total_avg_ms=1.417
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=13440 tokens=13440 router_gate_avg_ms=0.069 routed_avg_ms=0.767 shared_avg_ms=0.234 all_reduce_avg_ms=0.339 total_avg_ms=1.419
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=13520 tokens=13520 router_gate_avg_ms=0.069 routed_avg_ms=0.767 shared_avg_ms=0.234 all_reduce_avg_ms=0.338 total_avg_ms=1.419
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=13520 tokens=13520 router_gate_avg_ms=0.070 routed_avg_ms=0.771 shared_avg_ms=0.233 all_reduce_avg_ms=0.334 total_avg_ms=1.419
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=13520 tokens=13520 router_gate_avg_ms=0.069 routed_avg_ms=0.766 shared_avg_ms=0.237 all_reduce_avg_ms=0.332 total_avg_ms=1.415
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=13520 tokens=13520 router_gate_avg_ms=0.070 routed_avg_ms=0.772 shared_avg_ms=0.236 all_reduce_avg_ms=0.328 total_avg_ms=1.417
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=13600 tokens=13600 router_gate_avg_ms=0.069 routed_avg_ms=0.766 shared_avg_ms=0.237 all_reduce_avg_ms=0.332 total_avg_ms=1.415
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=13600 tokens=13600 router_gate_avg_ms=0.070 routed_avg_ms=0.772 shared_avg_ms=0.236 all_reduce_avg_ms=0.328 total_avg_ms=1.418
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=13600 tokens=13600 router_gate_avg_ms=0.069 routed_avg_ms=0.767 shared_avg_ms=0.234 all_reduce_avg_ms=0.338 total_avg_ms=1.419
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=13600 tokens=13600 router_gate_avg_ms=0.070 routed_avg_ms=0.771 shared_avg_ms=0.233 all_reduce_avg_ms=0.334 total_avg_ms=1.419
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=13680 tokens=13680 router_gate_avg_ms=0.070 routed_avg_ms=0.771 shared_avg_ms=0.233 all_reduce_avg_ms=0.334 total_avg_ms=1.419
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=13680 tokens=13680 router_gate_avg_ms=0.070 routed_avg_ms=0.771 shared_avg_ms=0.236 all_reduce_avg_ms=0.328 total_avg_ms=1.418
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=13680 tokens=13680 router_gate_avg_ms=0.069 routed_avg_ms=0.766 shared_avg_ms=0.237 all_reduce_avg_ms=0.332 total_avg_ms=1.415
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=13680 tokens=13680 router_gate_avg_ms=0.069 routed_avg_ms=0.767 shared_avg_ms=0.234 all_reduce_avg_ms=0.338 total_avg_ms=1.419
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=13760 tokens=13760 router_gate_avg_ms=0.070 routed_avg_ms=0.771 shared_avg_ms=0.233 all_reduce_avg_ms=0.334 total_avg_ms=1.419
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=13760 tokens=13760 router_gate_avg_ms=0.070 routed_avg_ms=0.771 shared_avg_ms=0.236 all_reduce_avg_ms=0.329 total_avg_ms=1.417
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=13760 tokens=13760 router_gate_avg_ms=0.069 routed_avg_ms=0.766 shared_avg_ms=0.237 all_reduce_avg_ms=0.332 total_avg_ms=1.415
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=13760 tokens=13760 router_gate_avg_ms=0.069 routed_avg_ms=0.767 shared_avg_ms=0.234 all_reduce_avg_ms=0.338 total_avg_ms=1.419
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=13840 tokens=13840 router_gate_avg_ms=0.069 routed_avg_ms=0.766 shared_avg_ms=0.237 all_reduce_avg_ms=0.332 total_avg_ms=1.415
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=13840 tokens=13840 router_gate_avg_ms=0.069 routed_avg_ms=0.767 shared_avg_ms=0.234 all_reduce_avg_ms=0.338 total_avg_ms=1.419
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=13840 tokens=13840 router_gate_avg_ms=0.070 routed_avg_ms=0.771 shared_avg_ms=0.233 all_reduce_avg_ms=0.334 total_avg_ms=1.419
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=13840 tokens=13840 router_gate_avg_ms=0.070 routed_avg_ms=0.771 shared_avg_ms=0.236 all_reduce_avg_ms=0.329 total_avg_ms=1.418
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=13920 tokens=13920 router_gate_avg_ms=0.069 routed_avg_ms=0.766 shared_avg_ms=0.237 all_reduce_avg_ms=0.332 total_avg_ms=1.415
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=13920 tokens=13920 router_gate_avg_ms=0.070 routed_avg_ms=0.771 shared_avg_ms=0.236 all_reduce_avg_ms=0.329 total_avg_ms=1.418
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=13920 tokens=13920 router_gate_avg_ms=0.069 routed_avg_ms=0.767 shared_avg_ms=0.234 all_reduce_avg_ms=0.338 total_avg_ms=1.419
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=13920 tokens=13920 router_gate_avg_ms=0.070 routed_avg_ms=0.771 shared_avg_ms=0.233 all_reduce_avg_ms=0.334 total_avg_ms=1.419
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=14000 tokens=14000 router_gate_avg_ms=0.069 routed_avg_ms=0.767 shared_avg_ms=0.234 all_reduce_avg_ms=0.338 total_avg_ms=1.419
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=14000 tokens=14000 router_gate_avg_ms=0.069 routed_avg_ms=0.766 shared_avg_ms=0.237 all_reduce_avg_ms=0.332 total_avg_ms=1.416
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=14000 tokens=14000 router_gate_avg_ms=0.070 routed_avg_ms=0.771 shared_avg_ms=0.233 all_reduce_avg_ms=0.334 total_avg_ms=1.419
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=14000 tokens=14000 router_gate_avg_ms=0.070 routed_avg_ms=0.771 shared_avg_ms=0.236 all_reduce_avg_ms=0.329 total_avg_ms=1.418
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=14080 tokens=14080 router_gate_avg_ms=0.069 routed_avg_ms=0.767 shared_avg_ms=0.234 all_reduce_avg_ms=0.338 total_avg_ms=1.419
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=14080 tokens=14080 router_gate_avg_ms=0.070 routed_avg_ms=0.771 shared_avg_ms=0.233 all_reduce_avg_ms=0.334 total_avg_ms=1.419
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=14080 tokens=14080 router_gate_avg_ms=0.069 routed_avg_ms=0.766 shared_avg_ms=0.237 all_reduce_avg_ms=0.332 total_avg_ms=1.415
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=14080 tokens=14080 router_gate_avg_ms=0.070 routed_avg_ms=0.771 shared_avg_ms=0.236 all_reduce_avg_ms=0.329 total_avg_ms=1.418
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=14160 tokens=14160 router_gate_avg_ms=0.069 routed_avg_ms=0.766 shared_avg_ms=0.237 all_reduce_avg_ms=0.333 total_avg_ms=1.415
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=14160 tokens=14160 router_gate_avg_ms=0.069 routed_avg_ms=0.767 shared_avg_ms=0.234 all_reduce_avg_ms=0.338 total_avg_ms=1.419
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=14160 tokens=14160 router_gate_avg_ms=0.070 routed_avg_ms=0.771 shared_avg_ms=0.233 all_reduce_avg_ms=0.334 total_avg_ms=1.419
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=14160 tokens=14160 router_gate_avg_ms=0.070 routed_avg_ms=0.771 shared_avg_ms=0.236 all_reduce_avg_ms=0.329 total_avg_ms=1.418
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=14240 tokens=14240 router_gate_avg_ms=0.069 routed_avg_ms=0.766 shared_avg_ms=0.237 all_reduce_avg_ms=0.333 total_avg_ms=1.415
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=14240 tokens=14240 router_gate_avg_ms=0.069 routed_avg_ms=0.767 shared_avg_ms=0.234 all_reduce_avg_ms=0.338 total_avg_ms=1.419
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=14240 tokens=14240 router_gate_avg_ms=0.070 routed_avg_ms=0.771 shared_avg_ms=0.236 all_reduce_avg_ms=0.328 total_avg_ms=1.417
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=14240 tokens=14240 router_gate_avg_ms=0.070 routed_avg_ms=0.770 shared_avg_ms=0.233 all_reduce_avg_ms=0.334 total_avg_ms=1.419
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=14320 tokens=14320 router_gate_avg_ms=0.070 routed_avg_ms=0.771 shared_avg_ms=0.233 all_reduce_avg_ms=0.334 total_avg_ms=1.419
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=14320 tokens=14320 router_gate_avg_ms=0.069 routed_avg_ms=0.766 shared_avg_ms=0.237 all_reduce_avg_ms=0.333 total_avg_ms=1.415
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=14320 tokens=14320 router_gate_avg_ms=0.070 routed_avg_ms=0.771 shared_avg_ms=0.236 all_reduce_avg_ms=0.329 total_avg_ms=1.418
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=14320 tokens=14320 router_gate_avg_ms=0.069 routed_avg_ms=0.767 shared_avg_ms=0.234 all_reduce_avg_ms=0.338 total_avg_ms=1.419
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=14400 tokens=14400 router_gate_avg_ms=0.069 routed_avg_ms=0.767 shared_avg_ms=0.234 all_reduce_avg_ms=0.338 total_avg_ms=1.419
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=14400 tokens=14400 router_gate_avg_ms=0.069 routed_avg_ms=0.766 shared_avg_ms=0.237 all_reduce_avg_ms=0.333 total_avg_ms=1.416
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=14400 tokens=14400 router_gate_avg_ms=0.070 routed_avg_ms=0.771 shared_avg_ms=0.236 all_reduce_avg_ms=0.329 total_avg_ms=1.418
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=14400 tokens=14400 router_gate_avg_ms=0.070 routed_avg_ms=0.771 shared_avg_ms=0.233 all_reduce_avg_ms=0.334 total_avg_ms=1.419
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=14480 tokens=14480 router_gate_avg_ms=0.069 routed_avg_ms=0.767 shared_avg_ms=0.234 all_reduce_avg_ms=0.338 total_avg_ms=1.419
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=14480 tokens=14480 router_gate_avg_ms=0.070 routed_avg_ms=0.770 shared_avg_ms=0.233 all_reduce_avg_ms=0.334 total_avg_ms=1.419
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=14480 tokens=14480 router_gate_avg_ms=0.070 routed_avg_ms=0.771 shared_avg_ms=0.236 all_reduce_avg_ms=0.329 total_avg_ms=1.418
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=14480 tokens=14480 router_gate_avg_ms=0.069 routed_avg_ms=0.766 shared_avg_ms=0.237 all_reduce_avg_ms=0.333 total_avg_ms=1.415
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=14560 tokens=14560 router_gate_avg_ms=0.070 routed_avg_ms=0.770 shared_avg_ms=0.233 all_reduce_avg_ms=0.334 total_avg_ms=1.419
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=14560 tokens=14560 router_gate_avg_ms=0.070 routed_avg_ms=0.771 shared_avg_ms=0.236 all_reduce_avg_ms=0.329 total_avg_ms=1.418
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=14560 tokens=14560 router_gate_avg_ms=0.069 routed_avg_ms=0.766 shared_avg_ms=0.237 all_reduce_avg_ms=0.333 total_avg_ms=1.415
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=14560 tokens=14560 router_gate_avg_ms=0.069 routed_avg_ms=0.767 shared_avg_ms=0.234 all_reduce_avg_ms=0.338 total_avg_ms=1.419
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=14640 tokens=14640 router_gate_avg_ms=0.069 routed_avg_ms=0.767 shared_avg_ms=0.234 all_reduce_avg_ms=0.338 total_avg_ms=1.419
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=14640 tokens=14640 router_gate_avg_ms=0.070 routed_avg_ms=0.770 shared_avg_ms=0.233 all_reduce_avg_ms=0.334 total_avg_ms=1.419
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=14640 tokens=14640 router_gate_avg_ms=0.069 routed_avg_ms=0.766 shared_avg_ms=0.237 all_reduce_avg_ms=0.333 total_avg_ms=1.415
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=14640 tokens=14640 router_gate_avg_ms=0.070 routed_avg_ms=0.771 shared_avg_ms=0.236 all_reduce_avg_ms=0.329 total_avg_ms=1.417
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=14720 tokens=14720 router_gate_avg_ms=0.069 routed_avg_ms=0.767 shared_avg_ms=0.234 all_reduce_avg_ms=0.338 total_avg_ms=1.419
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=14720 tokens=14720 router_gate_avg_ms=0.070 routed_avg_ms=0.770 shared_avg_ms=0.233 all_reduce_avg_ms=0.334 total_avg_ms=1.419
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=14720 tokens=14720 router_gate_avg_ms=0.069 routed_avg_ms=0.766 shared_avg_ms=0.237 all_reduce_avg_ms=0.332 total_avg_ms=1.415
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=14720 tokens=14720 router_gate_avg_ms=0.070 routed_avg_ms=0.771 shared_avg_ms=0.236 all_reduce_avg_ms=0.329 total_avg_ms=1.417
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=14800 tokens=14800 router_gate_avg_ms=0.069 routed_avg_ms=0.766 shared_avg_ms=0.237 all_reduce_avg_ms=0.332 total_avg_ms=1.415
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=14800 tokens=14800 router_gate_avg_ms=0.070 routed_avg_ms=0.771 shared_avg_ms=0.235 all_reduce_avg_ms=0.329 total_avg_ms=1.417
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=14800 tokens=14800 router_gate_avg_ms=0.069 routed_avg_ms=0.767 shared_avg_ms=0.234 all_reduce_avg_ms=0.338 total_avg_ms=1.419
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=14800 tokens=14800 router_gate_avg_ms=0.070 routed_avg_ms=0.770 shared_avg_ms=0.233 all_reduce_avg_ms=0.334 total_avg_ms=1.418
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=14880 tokens=14880 router_gate_avg_ms=0.069 routed_avg_ms=0.766 shared_avg_ms=0.237 all_reduce_avg_ms=0.333 total_avg_ms=1.415
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=14880 tokens=14880 router_gate_avg_ms=0.069 routed_avg_ms=0.767 shared_avg_ms=0.234 all_reduce_avg_ms=0.338 total_avg_ms=1.419
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=14880 tokens=14880 router_gate_avg_ms=0.070 routed_avg_ms=0.770 shared_avg_ms=0.233 all_reduce_avg_ms=0.334 total_avg_ms=1.419
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=14880 tokens=14880 router_gate_avg_ms=0.070 routed_avg_ms=0.771 shared_avg_ms=0.236 all_reduce_avg_ms=0.329 total_avg_ms=1.417
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=14960 tokens=14960 router_gate_avg_ms=0.069 routed_avg_ms=0.765 shared_avg_ms=0.237 all_reduce_avg_ms=0.333 total_avg_ms=1.415
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=14960 tokens=14960 router_gate_avg_ms=0.069 routed_avg_ms=0.767 shared_avg_ms=0.234 all_reduce_avg_ms=0.338 total_avg_ms=1.419
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=14960 tokens=14960 router_gate_avg_ms=0.070 routed_avg_ms=0.770 shared_avg_ms=0.233 all_reduce_avg_ms=0.334 total_avg_ms=1.419
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=14960 tokens=14960 router_gate_avg_ms=0.070 routed_avg_ms=0.771 shared_avg_ms=0.236 all_reduce_avg_ms=0.329 total_avg_ms=1.417
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=15040 tokens=15040 router_gate_avg_ms=0.069 routed_avg_ms=0.765 shared_avg_ms=0.237 all_reduce_avg_ms=0.333 total_avg_ms=1.415
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=15040 tokens=15040 router_gate_avg_ms=0.069 routed_avg_ms=0.767 shared_avg_ms=0.234 all_reduce_avg_ms=0.338 total_avg_ms=1.419
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=15040 tokens=15040 router_gate_avg_ms=0.070 routed_avg_ms=0.770 shared_avg_ms=0.233 all_reduce_avg_ms=0.334 total_avg_ms=1.419
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=15040 tokens=15040 router_gate_avg_ms=0.070 routed_avg_ms=0.771 shared_avg_ms=0.236 all_reduce_avg_ms=0.329 total_avg_ms=1.417
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=15120 tokens=15120 router_gate_avg_ms=0.069 routed_avg_ms=0.765 shared_avg_ms=0.237 all_reduce_avg_ms=0.333 total_avg_ms=1.415
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=15120 tokens=15120 router_gate_avg_ms=0.069 routed_avg_ms=0.767 shared_avg_ms=0.234 all_reduce_avg_ms=0.338 total_avg_ms=1.419
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=15120 tokens=15120 router_gate_avg_ms=0.070 routed_avg_ms=0.770 shared_avg_ms=0.233 all_reduce_avg_ms=0.334 total_avg_ms=1.419
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=15120 tokens=15120 router_gate_avg_ms=0.070 routed_avg_ms=0.771 shared_avg_ms=0.236 all_reduce_avg_ms=0.329 total_avg_ms=1.417
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=15200 tokens=15200 router_gate_avg_ms=0.069 routed_avg_ms=0.765 shared_avg_ms=0.237 all_reduce_avg_ms=0.333 total_avg_ms=1.415
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=15200 tokens=15200 router_gate_avg_ms=0.069 routed_avg_ms=0.767 shared_avg_ms=0.234 all_reduce_avg_ms=0.338 total_avg_ms=1.419
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=15200 tokens=15200 router_gate_avg_ms=0.070 routed_avg_ms=0.770 shared_avg_ms=0.233 all_reduce_avg_ms=0.334 total_avg_ms=1.419
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=15200 tokens=15200 router_gate_avg_ms=0.070 routed_avg_ms=0.771 shared_avg_ms=0.236 all_reduce_avg_ms=0.329 total_avg_ms=1.417
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=15280 tokens=15280 router_gate_avg_ms=0.069 routed_avg_ms=0.765 shared_avg_ms=0.237 all_reduce_avg_ms=0.333 total_avg_ms=1.415
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=15280 tokens=15280 router_gate_avg_ms=0.069 routed_avg_ms=0.766 shared_avg_ms=0.234 all_reduce_avg_ms=0.338 total_avg_ms=1.419
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=15280 tokens=15280 router_gate_avg_ms=0.070 routed_avg_ms=0.770 shared_avg_ms=0.233 all_reduce_avg_ms=0.334 total_avg_ms=1.419
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=15280 tokens=15280 router_gate_avg_ms=0.070 routed_avg_ms=0.771 shared_avg_ms=0.236 all_reduce_avg_ms=0.329 total_avg_ms=1.417
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=15360 tokens=15360 router_gate_avg_ms=0.069 routed_avg_ms=0.766 shared_avg_ms=0.234 all_reduce_avg_ms=0.338 total_avg_ms=1.419
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=15360 tokens=15360 router_gate_avg_ms=0.070 routed_avg_ms=0.770 shared_avg_ms=0.233 all_reduce_avg_ms=0.334 total_avg_ms=1.419
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=15360 tokens=15360 router_gate_avg_ms=0.070 routed_avg_ms=0.771 shared_avg_ms=0.236 all_reduce_avg_ms=0.329 total_avg_ms=1.417
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=15360 tokens=15360 router_gate_avg_ms=0.069 routed_avg_ms=0.765 shared_avg_ms=0.237 all_reduce_avg_ms=0.333 total_avg_ms=1.415
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=15440 tokens=15440 router_gate_avg_ms=0.069 routed_avg_ms=0.766 shared_avg_ms=0.234 all_reduce_avg_ms=0.338 total_avg_ms=1.419
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=15440 tokens=15440 router_gate_avg_ms=0.070 routed_avg_ms=0.770 shared_avg_ms=0.233 all_reduce_avg_ms=0.334 total_avg_ms=1.419
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=15440 tokens=15440 router_gate_avg_ms=0.069 routed_avg_ms=0.765 shared_avg_ms=0.237 all_reduce_avg_ms=0.333 total_avg_ms=1.415
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=15440 tokens=15440 router_gate_avg_ms=0.070 routed_avg_ms=0.771 shared_avg_ms=0.236 all_reduce_avg_ms=0.329 total_avg_ms=1.417
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=15520 tokens=15520 router_gate_avg_ms=0.069 routed_avg_ms=0.766 shared_avg_ms=0.234 all_reduce_avg_ms=0.338 total_avg_ms=1.419
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=15520 tokens=15520 router_gate_avg_ms=0.070 routed_avg_ms=0.770 shared_avg_ms=0.233 all_reduce_avg_ms=0.334 total_avg_ms=1.419
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=15520 tokens=15520 router_gate_avg_ms=0.070 routed_avg_ms=0.771 shared_avg_ms=0.236 all_reduce_avg_ms=0.329 total_avg_ms=1.417
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=15520 tokens=15520 router_gate_avg_ms=0.069 routed_avg_ms=0.765 shared_avg_ms=0.237 all_reduce_avg_ms=0.333 total_avg_ms=1.415
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=15600 tokens=15600 router_gate_avg_ms=0.070 routed_avg_ms=0.770 shared_avg_ms=0.233 all_reduce_avg_ms=0.334 total_avg_ms=1.419
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=15600 tokens=15600 router_gate_avg_ms=0.069 routed_avg_ms=0.766 shared_avg_ms=0.234 all_reduce_avg_ms=0.338 total_avg_ms=1.419
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=15600 tokens=15600 router_gate_avg_ms=0.069 routed_avg_ms=0.766 shared_avg_ms=0.237 all_reduce_avg_ms=0.332 total_avg_ms=1.415
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=15600 tokens=15600 router_gate_avg_ms=0.070 routed_avg_ms=0.771 shared_avg_ms=0.235 all_reduce_avg_ms=0.329 total_avg_ms=1.417
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=15680 tokens=15680 router_gate_avg_ms=0.069 routed_avg_ms=0.766 shared_avg_ms=0.234 all_reduce_avg_ms=0.338 total_avg_ms=1.419
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=15680 tokens=15680 router_gate_avg_ms=0.070 routed_avg_ms=0.770 shared_avg_ms=0.233 all_reduce_avg_ms=0.334 total_avg_ms=1.418
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=15680 tokens=15680 router_gate_avg_ms=0.070 routed_avg_ms=0.771 shared_avg_ms=0.235 all_reduce_avg_ms=0.329 total_avg_ms=1.417
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=15680 tokens=15680 router_gate_avg_ms=0.069 routed_avg_ms=0.766 shared_avg_ms=0.237 all_reduce_avg_ms=0.332 total_avg_ms=1.415
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=15760 tokens=15760 router_gate_avg_ms=0.069 routed_avg_ms=0.766 shared_avg_ms=0.237 all_reduce_avg_ms=0.332 total_avg_ms=1.415
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=15760 tokens=15760 router_gate_avg_ms=0.070 routed_avg_ms=0.771 shared_avg_ms=0.235 all_reduce_avg_ms=0.329 total_avg_ms=1.417
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=15760 tokens=15760 router_gate_avg_ms=0.069 routed_avg_ms=0.766 shared_avg_ms=0.234 all_reduce_avg_ms=0.338 total_avg_ms=1.419
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=15760 tokens=15760 router_gate_avg_ms=0.070 routed_avg_ms=0.770 shared_avg_ms=0.233 all_reduce_avg_ms=0.334 total_avg_ms=1.418
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=15840 tokens=15840 router_gate_avg_ms=0.069 routed_avg_ms=0.766 shared_avg_ms=0.234 all_reduce_avg_ms=0.339 total_avg_ms=1.419
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=15840 tokens=15840 router_gate_avg_ms=0.070 routed_avg_ms=0.770 shared_avg_ms=0.233 all_reduce_avg_ms=0.334 total_avg_ms=1.419
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=15840 tokens=15840 router_gate_avg_ms=0.070 routed_avg_ms=0.771 shared_avg_ms=0.235 all_reduce_avg_ms=0.329 total_avg_ms=1.417
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=15840 tokens=15840 router_gate_avg_ms=0.069 routed_avg_ms=0.766 shared_avg_ms=0.237 all_reduce_avg_ms=0.332 total_avg_ms=1.415
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=15920 tokens=15920 router_gate_avg_ms=0.069 routed_avg_ms=0.766 shared_avg_ms=0.237 all_reduce_avg_ms=0.332 total_avg_ms=1.415
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=15920 tokens=15920 router_gate_avg_ms=0.069 routed_avg_ms=0.766 shared_avg_ms=0.234 all_reduce_avg_ms=0.339 total_avg_ms=1.419
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=15920 tokens=15920 router_gate_avg_ms=0.069 routed_avg_ms=0.770 shared_avg_ms=0.233 all_reduce_avg_ms=0.334 total_avg_ms=1.419
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=15920 tokens=15920 router_gate_avg_ms=0.070 routed_avg_ms=0.771 shared_avg_ms=0.235 all_reduce_avg_ms=0.329 total_avg_ms=1.417
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=16000 tokens=16000 router_gate_avg_ms=0.069 routed_avg_ms=0.766 shared_avg_ms=0.234 all_reduce_avg_ms=0.339 total_avg_ms=1.419
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=16000 tokens=16000 router_gate_avg_ms=0.069 routed_avg_ms=0.770 shared_avg_ms=0.233 all_reduce_avg_ms=0.334 total_avg_ms=1.419
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=16000 tokens=16000 router_gate_avg_ms=0.069 routed_avg_ms=0.766 shared_avg_ms=0.237 all_reduce_avg_ms=0.332 total_avg_ms=1.415
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=16000 tokens=16000 router_gate_avg_ms=0.070 routed_avg_ms=0.771 shared_avg_ms=0.235 all_reduce_avg_ms=0.329 total_avg_ms=1.417
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=16080 tokens=16080 router_gate_avg_ms=0.069 routed_avg_ms=0.766 shared_avg_ms=0.237 all_reduce_avg_ms=0.332 total_avg_ms=1.414
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=16080 tokens=16080 router_gate_avg_ms=0.069 routed_avg_ms=0.766 shared_avg_ms=0.234 all_reduce_avg_ms=0.339 total_avg_ms=1.419
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=16080 tokens=16080 router_gate_avg_ms=0.069 routed_avg_ms=0.770 shared_avg_ms=0.233 all_reduce_avg_ms=0.334 total_avg_ms=1.418
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=16080 tokens=16080 router_gate_avg_ms=0.070 routed_avg_ms=0.770 shared_avg_ms=0.235 all_reduce_avg_ms=0.329 total_avg_ms=1.417
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=16160 tokens=16160 router_gate_avg_ms=0.069 routed_avg_ms=0.766 shared_avg_ms=0.237 all_reduce_avg_ms=0.332 total_avg_ms=1.414
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=16160 tokens=16160 router_gate_avg_ms=0.069 routed_avg_ms=0.766 shared_avg_ms=0.234 all_reduce_avg_ms=0.339 total_avg_ms=1.418
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=16160 tokens=16160 router_gate_avg_ms=0.069 routed_avg_ms=0.770 shared_avg_ms=0.233 all_reduce_avg_ms=0.334 total_avg_ms=1.418
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=16160 tokens=16160 router_gate_avg_ms=0.070 routed_avg_ms=0.770 shared_avg_ms=0.235 all_reduce_avg_ms=0.329 total_avg_ms=1.417
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=16240 tokens=16240 router_gate_avg_ms=0.069 routed_avg_ms=0.766 shared_avg_ms=0.234 all_reduce_avg_ms=0.339 total_avg_ms=1.418
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=16240 tokens=16240 router_gate_avg_ms=0.069 routed_avg_ms=0.770 shared_avg_ms=0.233 all_reduce_avg_ms=0.334 total_avg_ms=1.418
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=16240 tokens=16240 router_gate_avg_ms=0.069 routed_avg_ms=0.766 shared_avg_ms=0.237 all_reduce_avg_ms=0.332 total_avg_ms=1.414
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=16240 tokens=16240 router_gate_avg_ms=0.070 routed_avg_ms=0.770 shared_avg_ms=0.235 all_reduce_avg_ms=0.329 total_avg_ms=1.417
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=16320 tokens=16320 router_gate_avg_ms=0.069 routed_avg_ms=0.766 shared_avg_ms=0.237 all_reduce_avg_ms=0.332 total_avg_ms=1.414
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=16320 tokens=16320 router_gate_avg_ms=0.069 routed_avg_ms=0.766 shared_avg_ms=0.234 all_reduce_avg_ms=0.339 total_avg_ms=1.419
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=16320 tokens=16320 router_gate_avg_ms=0.069 routed_avg_ms=0.770 shared_avg_ms=0.233 all_reduce_avg_ms=0.334 total_avg_ms=1.418
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=16320 tokens=16320 router_gate_avg_ms=0.070 routed_avg_ms=0.770 shared_avg_ms=0.235 all_reduce_avg_ms=0.329 total_avg_ms=1.417
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=16400 tokens=16400 router_gate_avg_ms=0.069 routed_avg_ms=0.770 shared_avg_ms=0.233 all_reduce_avg_ms=0.335 total_avg_ms=1.418
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=16400 tokens=16400 router_gate_avg_ms=0.070 routed_avg_ms=0.770 shared_avg_ms=0.235 all_reduce_avg_ms=0.329 total_avg_ms=1.417
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=16400 tokens=16400 router_gate_avg_ms=0.069 routed_avg_ms=0.765 shared_avg_ms=0.237 all_reduce_avg_ms=0.332 total_avg_ms=1.414
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=16400 tokens=16400 router_gate_avg_ms=0.069 routed_avg_ms=0.766 shared_avg_ms=0.234 all_reduce_avg_ms=0.339 total_avg_ms=1.418
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=16480 tokens=16480 router_gate_avg_ms=0.069 routed_avg_ms=0.766 shared_avg_ms=0.234 all_reduce_avg_ms=0.339 total_avg_ms=1.418
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=16480 tokens=16480 router_gate_avg_ms=0.069 routed_avg_ms=0.770 shared_avg_ms=0.233 all_reduce_avg_ms=0.335 total_avg_ms=1.418
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=16480 tokens=16480 router_gate_avg_ms=0.070 routed_avg_ms=0.770 shared_avg_ms=0.235 all_reduce_avg_ms=0.329 total_avg_ms=1.416
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=16480 tokens=16480 router_gate_avg_ms=0.069 routed_avg_ms=0.765 shared_avg_ms=0.237 all_reduce_avg_ms=0.332 total_avg_ms=1.414
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=16560 tokens=16560 router_gate_avg_ms=0.069 routed_avg_ms=0.766 shared_avg_ms=0.234 all_reduce_avg_ms=0.339 total_avg_ms=1.418
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=16560 tokens=16560 router_gate_avg_ms=0.069 routed_avg_ms=0.770 shared_avg_ms=0.233 all_reduce_avg_ms=0.335 total_avg_ms=1.418
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=16560 tokens=16560 router_gate_avg_ms=0.069 routed_avg_ms=0.765 shared_avg_ms=0.237 all_reduce_avg_ms=0.332 total_avg_ms=1.414
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=16560 tokens=16560 router_gate_avg_ms=0.070 routed_avg_ms=0.770 shared_avg_ms=0.235 all_reduce_avg_ms=0.329 total_avg_ms=1.416
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=16640 tokens=16640 router_gate_avg_ms=0.069 routed_avg_ms=0.765 shared_avg_ms=0.237 all_reduce_avg_ms=0.331 total_avg_ms=1.414
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=16640 tokens=16640 router_gate_avg_ms=0.069 routed_avg_ms=0.766 shared_avg_ms=0.234 all_reduce_avg_ms=0.338 total_avg_ms=1.418
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=16640 tokens=16640 router_gate_avg_ms=0.069 routed_avg_ms=0.769 shared_avg_ms=0.233 all_reduce_avg_ms=0.334 total_avg_ms=1.418
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=16640 tokens=16640 router_gate_avg_ms=0.070 routed_avg_ms=0.770 shared_avg_ms=0.235 all_reduce_avg_ms=0.329 total_avg_ms=1.416
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=16720 tokens=16720 router_gate_avg_ms=0.069 routed_avg_ms=0.765 shared_avg_ms=0.237 all_reduce_avg_ms=0.331 total_avg_ms=1.414
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=16720 tokens=16720 router_gate_avg_ms=0.069 routed_avg_ms=0.766 shared_avg_ms=0.234 all_reduce_avg_ms=0.339 total_avg_ms=1.418
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=16720 tokens=16720 router_gate_avg_ms=0.069 routed_avg_ms=0.769 shared_avg_ms=0.233 all_reduce_avg_ms=0.335 total_avg_ms=1.418
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=16720 tokens=16720 router_gate_avg_ms=0.070 routed_avg_ms=0.770 shared_avg_ms=0.235 all_reduce_avg_ms=0.329 total_avg_ms=1.416
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=16800 tokens=16800 router_gate_avg_ms=0.069 routed_avg_ms=0.765 shared_avg_ms=0.237 all_reduce_avg_ms=0.331 total_avg_ms=1.414
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=16800 tokens=16800 router_gate_avg_ms=0.069 routed_avg_ms=0.766 shared_avg_ms=0.234 all_reduce_avg_ms=0.339 total_avg_ms=1.418
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=16800 tokens=16800 router_gate_avg_ms=0.069 routed_avg_ms=0.769 shared_avg_ms=0.233 all_reduce_avg_ms=0.335 total_avg_ms=1.418
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=16800 tokens=16800 router_gate_avg_ms=0.070 routed_avg_ms=0.770 shared_avg_ms=0.235 all_reduce_avg_ms=0.328 total_avg_ms=1.416
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=16880 tokens=16880 router_gate_avg_ms=0.069 routed_avg_ms=0.765 shared_avg_ms=0.237 all_reduce_avg_ms=0.331 total_avg_ms=1.414
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=16880 tokens=16880 router_gate_avg_ms=0.069 routed_avg_ms=0.766 shared_avg_ms=0.234 all_reduce_avg_ms=0.339 total_avg_ms=1.418
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=16880 tokens=16880 router_gate_avg_ms=0.069 routed_avg_ms=0.769 shared_avg_ms=0.233 all_reduce_avg_ms=0.335 total_avg_ms=1.418
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=16880 tokens=16880 router_gate_avg_ms=0.070 routed_avg_ms=0.770 shared_avg_ms=0.235 all_reduce_avg_ms=0.328 total_avg_ms=1.416
[PROF_MOE_SUMMARY] pid=35011 bucket=decode calls=16960 tokens=16960 router_gate_avg_ms=0.069 routed_avg_ms=0.765 shared_avg_ms=0.237 all_reduce_avg_ms=0.331 total_avg_ms=1.414
(VllmWorkerProcess pid=35365) [PROF_MOE_SUMMARY] pid=35365 bucket=decode calls=16960 tokens=16960 router_gate_avg_ms=0.069 routed_avg_ms=0.766 shared_avg_ms=0.234 all_reduce_avg_ms=0.339 total_avg_ms=1.418
(VllmWorkerProcess pid=35364) [PROF_MOE_SUMMARY] pid=35364 bucket=decode calls=16960 tokens=16960 router_gate_avg_ms=0.069 routed_avg_ms=0.769 shared_avg_ms=0.233 all_reduce_avg_ms=0.335 total_avg_ms=1.418
(VllmWorkerProcess pid=35363) [PROF_MOE_SUMMARY] pid=35363 bucket=decode calls=16960 tokens=16960 router_gate_avg_ms=0.070 routed_avg_ms=0.770 shared_avg_ms=0.235 all_reduce_avg_ms=0.328 total_avg_ms=1.416

760
worklogs/remote_results/2026-07-15-official-like-baseline/moe_summary_profile.log Normal file
View File

@@ -0,0 +1,760 @@
[PROF_MOE_SUMMARY] pid=33749 calls=80 tokens=955280 router_gate_avg_ms=0.548 routed_avg_ms=79.356 shared_avg_ms=2.011 all_reduce_avg_ms=4.259 total_avg_ms=86.185
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=80 tokens=955280 router_gate_avg_ms=0.548 routed_avg_ms=79.361 shared_avg_ms=2.009 all_reduce_avg_ms=4.262 total_avg_ms=86.192
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=80 tokens=955280 router_gate_avg_ms=0.545 routed_avg_ms=79.146 shared_avg_ms=2.011 all_reduce_avg_ms=4.493 total_avg_ms=86.206
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=80 tokens=955280 router_gate_avg_ms=0.549 routed_avg_ms=79.519 shared_avg_ms=2.020 all_reduce_avg_ms=4.010 total_avg_ms=86.110
[PROF_MOE_SUMMARY] pid=33749 calls=160 tokens=955360 router_gate_avg_ms=0.310 routed_avg_ms=40.081 shared_avg_ms=1.127 all_reduce_avg_ms=2.300 total_avg_ms=43.830
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=160 tokens=955360 router_gate_avg_ms=0.308 routed_avg_ms=39.980 shared_avg_ms=1.131 all_reduce_avg_ms=2.408 total_avg_ms=43.839
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=160 tokens=955360 router_gate_avg_ms=0.310 routed_avg_ms=40.166 shared_avg_ms=1.132 all_reduce_avg_ms=2.173 total_avg_ms=43.792
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=160 tokens=955360 router_gate_avg_ms=0.309 routed_avg_ms=40.086 shared_avg_ms=1.126 all_reduce_avg_ms=2.300 total_avg_ms=43.833
[PROF_MOE_SUMMARY] pid=33749 calls=240 tokens=955440 router_gate_avg_ms=0.229 routed_avg_ms=26.987 shared_avg_ms=0.830 all_reduce_avg_ms=1.661 total_avg_ms=29.719
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=240 tokens=955440 router_gate_avg_ms=0.230 routed_avg_ms=26.996 shared_avg_ms=0.830 all_reduce_avg_ms=1.651 total_avg_ms=29.719
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=240 tokens=955440 router_gate_avg_ms=0.229 routed_avg_ms=26.925 shared_avg_ms=0.837 all_reduce_avg_ms=1.719 total_avg_ms=29.722
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=240 tokens=955440 router_gate_avg_ms=0.230 routed_avg_ms=27.048 shared_avg_ms=0.835 all_reduce_avg_ms=1.567 total_avg_ms=29.693
[PROF_MOE_SUMMARY] pid=33749 calls=320 tokens=955520 router_gate_avg_ms=0.189 routed_avg_ms=20.439 shared_avg_ms=0.681 all_reduce_avg_ms=1.335 total_avg_ms=22.655
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=320 tokens=955520 router_gate_avg_ms=0.190 routed_avg_ms=20.449 shared_avg_ms=0.681 all_reduce_avg_ms=1.321 total_avg_ms=22.653
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=320 tokens=955520 router_gate_avg_ms=0.189 routed_avg_ms=20.395 shared_avg_ms=0.687 all_reduce_avg_ms=1.374 total_avg_ms=22.656
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=320 tokens=955520 router_gate_avg_ms=0.190 routed_avg_ms=20.489 shared_avg_ms=0.687 all_reduce_avg_ms=1.255 total_avg_ms=22.633
[PROF_MOE_SUMMARY] pid=33749 calls=400 tokens=955600 router_gate_avg_ms=0.164 routed_avg_ms=16.509 shared_avg_ms=0.592 all_reduce_avg_ms=1.166 total_avg_ms=18.443
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=400 tokens=955600 router_gate_avg_ms=0.168 routed_avg_ms=16.528 shared_avg_ms=0.600 all_reduce_avg_ms=1.126 total_avg_ms=18.434
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=400 tokens=955600 router_gate_avg_ms=0.167 routed_avg_ms=16.484 shared_avg_ms=0.604 all_reduce_avg_ms=1.170 total_avg_ms=18.437
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=400 tokens=955600 router_gate_avg_ms=0.166 routed_avg_ms=16.554 shared_avg_ms=0.598 all_reduce_avg_ms=1.094 total_avg_ms=18.424
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=480 tokens=955680 router_gate_avg_ms=0.150 routed_avg_ms=13.872 shared_avg_ms=0.544 all_reduce_avg_ms=1.029 total_avg_ms=15.608
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=480 tokens=955680 router_gate_avg_ms=0.150 routed_avg_ms=13.930 shared_avg_ms=0.538 all_reduce_avg_ms=0.968 total_avg_ms=15.598
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=480 tokens=955680 router_gate_avg_ms=0.151 routed_avg_ms=13.906 shared_avg_ms=0.539 all_reduce_avg_ms=0.998 total_avg_ms=15.607
[PROF_MOE_SUMMARY] pid=33749 calls=480 tokens=955680 router_gate_avg_ms=0.148 routed_avg_ms=13.891 shared_avg_ms=0.533 all_reduce_avg_ms=1.030 total_avg_ms=15.613
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=560 tokens=955760 router_gate_avg_ms=0.139 routed_avg_ms=12.005 shared_avg_ms=0.500 all_reduce_avg_ms=0.929 total_avg_ms=13.585
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=560 tokens=955760 router_gate_avg_ms=0.138 routed_avg_ms=12.054 shared_avg_ms=0.495 all_reduce_avg_ms=0.877 total_avg_ms=13.576
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=560 tokens=955760 router_gate_avg_ms=0.139 routed_avg_ms=12.034 shared_avg_ms=0.495 all_reduce_avg_ms=0.903 total_avg_ms=13.583
[PROF_MOE_SUMMARY] pid=33749 calls=560 tokens=955760 router_gate_avg_ms=0.137 routed_avg_ms=12.021 shared_avg_ms=0.491 all_reduce_avg_ms=0.928 total_avg_ms=13.588
[PROF_MOE_SUMMARY] pid=33749 calls=640 tokens=955840 router_gate_avg_ms=0.133 routed_avg_ms=10.621 shared_avg_ms=0.460 all_reduce_avg_ms=0.853 total_avg_ms=12.080
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=640 tokens=955840 router_gate_avg_ms=0.131 routed_avg_ms=10.630 shared_avg_ms=0.462 all_reduce_avg_ms=0.837 total_avg_ms=12.072
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=640 tokens=955840 router_gate_avg_ms=0.131 routed_avg_ms=10.652 shared_avg_ms=0.463 all_reduce_avg_ms=0.814 total_avg_ms=12.072
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=640 tokens=955840 router_gate_avg_ms=0.134 routed_avg_ms=10.606 shared_avg_ms=0.468 all_reduce_avg_ms=0.856 total_avg_ms=12.077
[PROF_MOE_SUMMARY] pid=33749 calls=720 tokens=955920 router_gate_avg_ms=0.126 routed_avg_ms=9.530 shared_avg_ms=0.436 all_reduce_avg_ms=0.800 total_avg_ms=10.903
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=720 tokens=955920 router_gate_avg_ms=0.124 routed_avg_ms=9.537 shared_avg_ms=0.437 all_reduce_avg_ms=0.788 total_avg_ms=10.898
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=720 tokens=955920 router_gate_avg_ms=0.128 routed_avg_ms=9.517 shared_avg_ms=0.443 all_reduce_avg_ms=0.800 total_avg_ms=10.901
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=720 tokens=955920 router_gate_avg_ms=0.125 routed_avg_ms=9.560 shared_avg_ms=0.439 all_reduce_avg_ms=0.760 total_avg_ms=10.895
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=800 tokens=956000 router_gate_avg_ms=0.118 routed_avg_ms=8.662 shared_avg_ms=0.416 all_reduce_avg_ms=0.747 total_avg_ms=9.955
[PROF_MOE_SUMMARY] pid=33749 calls=800 tokens=956000 router_gate_avg_ms=0.121 routed_avg_ms=8.657 shared_avg_ms=0.416 all_reduce_avg_ms=0.754 total_avg_ms=9.959
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=800 tokens=956000 router_gate_avg_ms=0.121 routed_avg_ms=8.645 shared_avg_ms=0.423 all_reduce_avg_ms=0.756 total_avg_ms=9.958
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=800 tokens=956000 router_gate_avg_ms=0.119 routed_avg_ms=8.685 shared_avg_ms=0.419 all_reduce_avg_ms=0.716 total_avg_ms=9.952
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=880 tokens=956080 router_gate_avg_ms=0.113 routed_avg_ms=7.947 shared_avg_ms=0.399 all_reduce_avg_ms=0.714 total_avg_ms=9.184
[PROF_MOE_SUMMARY] pid=33749 calls=880 tokens=956080 router_gate_avg_ms=0.116 routed_avg_ms=7.943 shared_avg_ms=0.400 all_reduce_avg_ms=0.716 total_avg_ms=9.187
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=880 tokens=956080 router_gate_avg_ms=0.117 routed_avg_ms=7.932 shared_avg_ms=0.406 all_reduce_avg_ms=0.720 total_avg_ms=9.186
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=880 tokens=956080 router_gate_avg_ms=0.115 routed_avg_ms=7.970 shared_avg_ms=0.404 all_reduce_avg_ms=0.680 total_avg_ms=9.180
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=960 tokens=956160 router_gate_avg_ms=0.110 routed_avg_ms=7.350 shared_avg_ms=0.385 all_reduce_avg_ms=0.686 total_avg_ms=8.543
[PROF_MOE_SUMMARY] pid=33749 calls=960 tokens=956160 router_gate_avg_ms=0.113 routed_avg_ms=7.348 shared_avg_ms=0.387 all_reduce_avg_ms=0.685 total_avg_ms=8.545
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=960 tokens=956160 router_gate_avg_ms=0.112 routed_avg_ms=7.338 shared_avg_ms=0.392 all_reduce_avg_ms=0.689 total_avg_ms=8.543
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=960 tokens=956160 router_gate_avg_ms=0.113 routed_avg_ms=7.374 shared_avg_ms=0.390 all_reduce_avg_ms=0.650 total_avg_ms=8.538
[PROF_MOE_SUMMARY] pid=33749 calls=1040 tokens=956240 router_gate_avg_ms=0.110 routed_avg_ms=6.845 shared_avg_ms=0.375 all_reduce_avg_ms=0.659 total_avg_ms=8.001
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=1040 tokens=956240 router_gate_avg_ms=0.107 routed_avg_ms=6.846 shared_avg_ms=0.373 all_reduce_avg_ms=0.662 total_avg_ms=8.000
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=1040 tokens=956240 router_gate_avg_ms=0.109 routed_avg_ms=6.835 shared_avg_ms=0.380 all_reduce_avg_ms=0.664 total_avg_ms=8.000
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=1040 tokens=956240 router_gate_avg_ms=0.110 routed_avg_ms=6.869 shared_avg_ms=0.379 all_reduce_avg_ms=0.624 total_avg_ms=7.994
[PROF_MOE_SUMMARY] pid=33749 calls=1120 tokens=956320 router_gate_avg_ms=0.107 routed_avg_ms=6.413 shared_avg_ms=0.366 all_reduce_avg_ms=0.635 total_avg_ms=7.534
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=1120 tokens=956320 router_gate_avg_ms=0.106 routed_avg_ms=6.404 shared_avg_ms=0.370 all_reduce_avg_ms=0.640 total_avg_ms=7.533
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=1120 tokens=956320 router_gate_avg_ms=0.107 routed_avg_ms=6.436 shared_avg_ms=0.369 all_reduce_avg_ms=0.604 total_avg_ms=7.528
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=1120 tokens=956320 router_gate_avg_ms=0.104 routed_avg_ms=6.415 shared_avg_ms=0.363 all_reduce_avg_ms=0.639 total_avg_ms=7.533
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=1200 tokens=956400 router_gate_avg_ms=0.104 routed_avg_ms=6.061 shared_avg_ms=0.360 all_reduce_avg_ms=0.586 total_avg_ms=7.123
[PROF_MOE_SUMMARY] pid=33749 calls=1200 tokens=956400 router_gate_avg_ms=0.105 routed_avg_ms=6.039 shared_avg_ms=0.357 all_reduce_avg_ms=0.615 total_avg_ms=7.128
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=1200 tokens=956400 router_gate_avg_ms=0.102 routed_avg_ms=6.041 shared_avg_ms=0.355 all_reduce_avg_ms=0.618 total_avg_ms=7.128
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=1200 tokens=956400 router_gate_avg_ms=0.104 routed_avg_ms=6.031 shared_avg_ms=0.361 all_reduce_avg_ms=0.620 total_avg_ms=7.127
[PROF_MOE_SUMMARY] pid=33749 calls=1280 tokens=956480 router_gate_avg_ms=0.103 routed_avg_ms=5.711 shared_avg_ms=0.350 all_reduce_avg_ms=0.599 total_avg_ms=6.774
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=1280 tokens=956480 router_gate_avg_ms=0.102 routed_avg_ms=5.704 shared_avg_ms=0.354 all_reduce_avg_ms=0.602 total_avg_ms=6.774
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=1280 tokens=956480 router_gate_avg_ms=0.102 routed_avg_ms=5.733 shared_avg_ms=0.353 all_reduce_avg_ms=0.570 total_avg_ms=6.770
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=1280 tokens=956480 router_gate_avg_ms=0.100 routed_avg_ms=5.714 shared_avg_ms=0.348 all_reduce_avg_ms=0.601 total_avg_ms=6.774
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=1360 tokens=956560 router_gate_avg_ms=0.100 routed_avg_ms=5.416 shared_avg_ms=0.348 all_reduce_avg_ms=0.586 total_avg_ms=6.461
[PROF_MOE_SUMMARY] pid=33749 calls=1360 tokens=956560 router_gate_avg_ms=0.101 routed_avg_ms=5.423 shared_avg_ms=0.344 all_reduce_avg_ms=0.583 total_avg_ms=6.462
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=1360 tokens=956560 router_gate_avg_ms=0.100 routed_avg_ms=5.442 shared_avg_ms=0.346 all_reduce_avg_ms=0.557 total_avg_ms=6.458
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=1360 tokens=956560 router_gate_avg_ms=0.098 routed_avg_ms=5.425 shared_avg_ms=0.341 all_reduce_avg_ms=0.586 total_avg_ms=6.462
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=1440 tokens=956640 router_gate_avg_ms=0.099 routed_avg_ms=5.185 shared_avg_ms=0.340 all_reduce_avg_ms=0.545 total_avg_ms=6.180
[PROF_MOE_SUMMARY] pid=33749 calls=1440 tokens=956640 router_gate_avg_ms=0.099 routed_avg_ms=5.166 shared_avg_ms=0.338 all_reduce_avg_ms=0.569 total_avg_ms=6.184
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=1440 tokens=956640 router_gate_avg_ms=0.097 routed_avg_ms=5.168 shared_avg_ms=0.335 all_reduce_avg_ms=0.573 total_avg_ms=6.184
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=1440 tokens=956640 router_gate_avg_ms=0.098 routed_avg_ms=5.160 shared_avg_ms=0.342 all_reduce_avg_ms=0.571 total_avg_ms=6.183
[PROF_MOE_SUMMARY] pid=33749 calls=1520 tokens=956720 router_gate_avg_ms=0.098 routed_avg_ms=4.936 shared_avg_ms=0.333 all_reduce_avg_ms=0.556 total_avg_ms=5.934
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=1520 tokens=956720 router_gate_avg_ms=0.097 routed_avg_ms=4.931 shared_avg_ms=0.336 all_reduce_avg_ms=0.558 total_avg_ms=5.934
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=1520 tokens=956720 router_gate_avg_ms=0.097 routed_avg_ms=4.954 shared_avg_ms=0.334 all_reduce_avg_ms=0.534 total_avg_ms=5.931
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=1520 tokens=956720 router_gate_avg_ms=0.095 routed_avg_ms=4.938 shared_avg_ms=0.330 all_reduce_avg_ms=0.560 total_avg_ms=5.935
[PROF_MOE_SUMMARY] pid=33749 calls=1600 tokens=956800 router_gate_avg_ms=0.096 routed_avg_ms=4.729 shared_avg_ms=0.328 all_reduce_avg_ms=0.545 total_avg_ms=5.710
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=1600 tokens=956800 router_gate_avg_ms=0.094 routed_avg_ms=4.732 shared_avg_ms=0.325 all_reduce_avg_ms=0.548 total_avg_ms=5.711
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=1600 tokens=956800 router_gate_avg_ms=0.095 routed_avg_ms=4.725 shared_avg_ms=0.332 all_reduce_avg_ms=0.546 total_avg_ms=5.709
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=1600 tokens=956800 router_gate_avg_ms=0.096 routed_avg_ms=4.746 shared_avg_ms=0.330 all_reduce_avg_ms=0.524 total_avg_ms=5.707
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=1680 tokens=956880 router_gate_avg_ms=0.094 routed_avg_ms=4.559 shared_avg_ms=0.325 all_reduce_avg_ms=0.515 total_avg_ms=5.505
[PROF_MOE_SUMMARY] pid=33749 calls=1680 tokens=956880 router_gate_avg_ms=0.095 routed_avg_ms=4.542 shared_avg_ms=0.323 all_reduce_avg_ms=0.535 total_avg_ms=5.507
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=1680 tokens=956880 router_gate_avg_ms=0.093 routed_avg_ms=4.544 shared_avg_ms=0.321 all_reduce_avg_ms=0.539 total_avg_ms=5.508
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=1680 tokens=956880 router_gate_avg_ms=0.094 routed_avg_ms=4.538 shared_avg_ms=0.327 all_reduce_avg_ms=0.535 total_avg_ms=5.507
[PROF_MOE_SUMMARY] pid=33749 calls=1760 tokens=956960 router_gate_avg_ms=0.094 routed_avg_ms=4.372 shared_avg_ms=0.320 all_reduce_avg_ms=0.526 total_avg_ms=5.323
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=1760 tokens=956960 router_gate_avg_ms=0.093 routed_avg_ms=4.368 shared_avg_ms=0.323 all_reduce_avg_ms=0.525 total_avg_ms=5.322
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=1760 tokens=956960 router_gate_avg_ms=0.093 routed_avg_ms=4.388 shared_avg_ms=0.321 all_reduce_avg_ms=0.507 total_avg_ms=5.320
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=1760 tokens=956960 router_gate_avg_ms=0.091 routed_avg_ms=4.374 shared_avg_ms=0.317 all_reduce_avg_ms=0.530 total_avg_ms=5.324
[PROF_MOE_SUMMARY] pid=33749 calls=1840 tokens=957040 router_gate_avg_ms=0.094 routed_avg_ms=4.217 shared_avg_ms=0.316 all_reduce_avg_ms=0.517 total_avg_ms=5.156
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=1840 tokens=957040 router_gate_avg_ms=0.093 routed_avg_ms=4.213 shared_avg_ms=0.320 all_reduce_avg_ms=0.517 total_avg_ms=5.155
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=1840 tokens=957040 router_gate_avg_ms=0.090 routed_avg_ms=4.219 shared_avg_ms=0.313 all_reduce_avg_ms=0.522 total_avg_ms=5.156
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=1840 tokens=957040 router_gate_avg_ms=0.092 routed_avg_ms=4.233 shared_avg_ms=0.318 all_reduce_avg_ms=0.499 total_avg_ms=5.153
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=1920 tokens=957120 router_gate_avg_ms=0.092 routed_avg_ms=4.071 shared_avg_ms=0.317 all_reduce_avg_ms=0.511 total_avg_ms=5.002
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=1920 tokens=957120 router_gate_avg_ms=0.091 routed_avg_ms=4.090 shared_avg_ms=0.314 all_reduce_avg_ms=0.494 total_avg_ms=5.000
[PROF_MOE_SUMMARY] pid=33749 calls=1920 tokens=957120 router_gate_avg_ms=0.093 routed_avg_ms=4.075 shared_avg_ms=0.313 all_reduce_avg_ms=0.510 total_avg_ms=5.003
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=1920 tokens=957120 router_gate_avg_ms=0.090 routed_avg_ms=4.077 shared_avg_ms=0.310 all_reduce_avg_ms=0.515 total_avg_ms=5.003
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=2000 tokens=957200 router_gate_avg_ms=0.091 routed_avg_ms=3.941 shared_avg_ms=0.313 all_reduce_avg_ms=0.504 total_avg_ms=4.861
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=2000 tokens=957200 router_gate_avg_ms=0.090 routed_avg_ms=3.958 shared_avg_ms=0.311 all_reduce_avg_ms=0.488 total_avg_ms=4.859
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=2000 tokens=957200 router_gate_avg_ms=0.089 routed_avg_ms=3.946 shared_avg_ms=0.307 all_reduce_avg_ms=0.509 total_avg_ms=4.862
[PROF_MOE_SUMMARY] pid=33749 calls=2000 tokens=957200 router_gate_avg_ms=0.092 routed_avg_ms=3.945 shared_avg_ms=0.311 all_reduce_avg_ms=0.502 total_avg_ms=4.861
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=2080 tokens=957280 router_gate_avg_ms=0.089 routed_avg_ms=3.837 shared_avg_ms=0.308 all_reduce_avg_ms=0.482 total_avg_ms=4.728
[PROF_MOE_SUMMARY] pid=33749 calls=2080 tokens=957280 router_gate_avg_ms=0.091 routed_avg_ms=3.824 shared_avg_ms=0.308 all_reduce_avg_ms=0.495 total_avg_ms=4.730
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=2080 tokens=957280 router_gate_avg_ms=0.088 routed_avg_ms=3.825 shared_avg_ms=0.304 all_reduce_avg_ms=0.502 total_avg_ms=4.731
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=2080 tokens=957280 router_gate_avg_ms=0.090 routed_avg_ms=3.820 shared_avg_ms=0.311 all_reduce_avg_ms=0.497 total_avg_ms=4.729
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=2160 tokens=957360 router_gate_avg_ms=0.089 routed_avg_ms=3.725 shared_avg_ms=0.305 all_reduce_avg_ms=0.476 total_avg_ms=4.607
[PROF_MOE_SUMMARY] pid=33749 calls=2160 tokens=957360 router_gate_avg_ms=0.090 routed_avg_ms=3.712 shared_avg_ms=0.305 all_reduce_avg_ms=0.489 total_avg_ms=4.608
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=2160 tokens=957360 router_gate_avg_ms=0.087 routed_avg_ms=3.713 shared_avg_ms=0.302 all_reduce_avg_ms=0.496 total_avg_ms=4.609
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=2160 tokens=957360 router_gate_avg_ms=0.089 routed_avg_ms=3.708 shared_avg_ms=0.308 all_reduce_avg_ms=0.491 total_avg_ms=4.608
[PROF_MOE_SUMMARY] pid=33749 calls=2240 tokens=957440 router_gate_avg_ms=0.090 routed_avg_ms=3.608 shared_avg_ms=0.303 all_reduce_avg_ms=0.484 total_avg_ms=4.496
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=2240 tokens=957440 router_gate_avg_ms=0.089 routed_avg_ms=3.604 shared_avg_ms=0.305 all_reduce_avg_ms=0.486 total_avg_ms=4.496
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=2240 tokens=957440 router_gate_avg_ms=0.088 routed_avg_ms=3.620 shared_avg_ms=0.303 all_reduce_avg_ms=0.471 total_avg_ms=4.494
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=2240 tokens=957440 router_gate_avg_ms=0.087 routed_avg_ms=3.609 shared_avg_ms=0.299 all_reduce_avg_ms=0.489 total_avg_ms=4.496
[PROF_MOE_SUMMARY] pid=33749 calls=2320 tokens=957520 router_gate_avg_ms=0.089 routed_avg_ms=3.511 shared_avg_ms=0.301 all_reduce_avg_ms=0.479 total_avg_ms=4.391
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=2320 tokens=957520 router_gate_avg_ms=0.088 routed_avg_ms=3.507 shared_avg_ms=0.303 all_reduce_avg_ms=0.481 total_avg_ms=4.391
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=2320 tokens=957520 router_gate_avg_ms=0.087 routed_avg_ms=3.523 shared_avg_ms=0.301 all_reduce_avg_ms=0.466 total_avg_ms=4.389
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=2320 tokens=957520 router_gate_avg_ms=0.086 routed_avg_ms=3.513 shared_avg_ms=0.298 all_reduce_avg_ms=0.483 total_avg_ms=4.392
[PROF_MOE_SUMMARY] pid=33749 calls=2400 tokens=957600 router_gate_avg_ms=0.088 routed_avg_ms=3.420 shared_avg_ms=0.299 all_reduce_avg_ms=0.475 total_avg_ms=4.294
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=2400 tokens=957600 router_gate_avg_ms=0.087 routed_avg_ms=3.417 shared_avg_ms=0.301 all_reduce_avg_ms=0.476 total_avg_ms=4.293
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=2400 tokens=957600 router_gate_avg_ms=0.087 routed_avg_ms=3.433 shared_avg_ms=0.299 all_reduce_avg_ms=0.461 total_avg_ms=4.292
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=2400 tokens=957600 router_gate_avg_ms=0.086 routed_avg_ms=3.423 shared_avg_ms=0.296 all_reduce_avg_ms=0.478 total_avg_ms=4.294
[PROF_MOE_SUMMARY] pid=33749 calls=2480 tokens=957680 router_gate_avg_ms=0.087 routed_avg_ms=3.335 shared_avg_ms=0.297 all_reduce_avg_ms=0.471 total_avg_ms=4.202
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=2480 tokens=957680 router_gate_avg_ms=0.087 routed_avg_ms=3.333 shared_avg_ms=0.299 all_reduce_avg_ms=0.471 total_avg_ms=4.202
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=2480 tokens=957680 router_gate_avg_ms=0.086 routed_avg_ms=3.348 shared_avg_ms=0.297 all_reduce_avg_ms=0.457 total_avg_ms=4.200
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=2480 tokens=957680 router_gate_avg_ms=0.085 routed_avg_ms=3.338 shared_avg_ms=0.294 all_reduce_avg_ms=0.474 total_avg_ms=4.202
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=2560 tokens=957760 router_gate_avg_ms=0.086 routed_avg_ms=3.254 shared_avg_ms=0.297 all_reduce_avg_ms=0.467 total_avg_ms=4.116
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=2560 tokens=957760 router_gate_avg_ms=0.086 routed_avg_ms=3.268 shared_avg_ms=0.295 all_reduce_avg_ms=0.454 total_avg_ms=4.114
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=2560 tokens=957760 router_gate_avg_ms=0.085 routed_avg_ms=3.259 shared_avg_ms=0.292 all_reduce_avg_ms=0.469 total_avg_ms=4.116
[PROF_MOE_SUMMARY] pid=33749 calls=2560 tokens=957760 router_gate_avg_ms=0.087 routed_avg_ms=3.256 shared_avg_ms=0.295 all_reduce_avg_ms=0.467 total_avg_ms=4.116
[PROF_MOE_SUMMARY] pid=33749 calls=2640 tokens=957840 router_gate_avg_ms=0.086 routed_avg_ms=3.182 shared_avg_ms=0.293 all_reduce_avg_ms=0.463 total_avg_ms=4.035
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=2640 tokens=957840 router_gate_avg_ms=0.086 routed_avg_ms=3.179 shared_avg_ms=0.295 all_reduce_avg_ms=0.463 total_avg_ms=4.035
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=2640 tokens=957840 router_gate_avg_ms=0.085 routed_avg_ms=3.194 shared_avg_ms=0.293 all_reduce_avg_ms=0.450 total_avg_ms=4.033
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=2640 tokens=957840 router_gate_avg_ms=0.084 routed_avg_ms=3.185 shared_avg_ms=0.290 all_reduce_avg_ms=0.465 total_avg_ms=4.035
[PROF_MOE_SUMMARY] pid=33749 calls=2720 tokens=957920 router_gate_avg_ms=0.085 routed_avg_ms=3.112 shared_avg_ms=0.291 all_reduce_avg_ms=0.459 total_avg_ms=3.959
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=2720 tokens=957920 router_gate_avg_ms=0.085 routed_avg_ms=3.109 shared_avg_ms=0.294 all_reduce_avg_ms=0.458 total_avg_ms=3.958
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=2720 tokens=957920 router_gate_avg_ms=0.085 routed_avg_ms=3.123 shared_avg_ms=0.292 all_reduce_avg_ms=0.446 total_avg_ms=3.957
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=2720 tokens=957920 router_gate_avg_ms=0.084 routed_avg_ms=3.114 shared_avg_ms=0.289 all_reduce_avg_ms=0.460 total_avg_ms=3.958
[PROF_MOE_SUMMARY] pid=33749 calls=2800 tokens=958000 router_gate_avg_ms=0.085 routed_avg_ms=3.045 shared_avg_ms=0.290 all_reduce_avg_ms=0.456 total_avg_ms=3.888
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=2800 tokens=958000 router_gate_avg_ms=0.085 routed_avg_ms=3.043 shared_avg_ms=0.292 all_reduce_avg_ms=0.456 total_avg_ms=3.887
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=2800 tokens=958000 router_gate_avg_ms=0.084 routed_avg_ms=3.057 shared_avg_ms=0.290 all_reduce_avg_ms=0.444 total_avg_ms=3.886
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=2800 tokens=958000 router_gate_avg_ms=0.083 routed_avg_ms=3.049 shared_avg_ms=0.287 all_reduce_avg_ms=0.456 total_avg_ms=3.887
[PROF_MOE_SUMMARY] pid=33749 calls=2880 tokens=958080 router_gate_avg_ms=0.084 routed_avg_ms=2.983 shared_avg_ms=0.288 all_reduce_avg_ms=0.453 total_avg_ms=3.820
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=2880 tokens=958080 router_gate_avg_ms=0.084 routed_avg_ms=2.994 shared_avg_ms=0.288 all_reduce_avg_ms=0.440 total_avg_ms=3.818
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=2880 tokens=958080 router_gate_avg_ms=0.083 routed_avg_ms=2.987 shared_avg_ms=0.286 all_reduce_avg_ms=0.452 total_avg_ms=3.819
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=2880 tokens=958080 router_gate_avg_ms=0.084 routed_avg_ms=2.981 shared_avg_ms=0.291 all_reduce_avg_ms=0.452 total_avg_ms=3.820
[PROF_MOE_SUMMARY] pid=33749 calls=2960 tokens=958160 router_gate_avg_ms=0.084 routed_avg_ms=2.924 shared_avg_ms=0.287 all_reduce_avg_ms=0.449 total_avg_ms=3.755
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=2960 tokens=958160 router_gate_avg_ms=0.084 routed_avg_ms=2.922 shared_avg_ms=0.289 all_reduce_avg_ms=0.448 total_avg_ms=3.755
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=2960 tokens=958160 router_gate_avg_ms=0.083 routed_avg_ms=2.935 shared_avg_ms=0.287 all_reduce_avg_ms=0.437 total_avg_ms=3.754
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=2960 tokens=958160 router_gate_avg_ms=0.082 routed_avg_ms=2.928 shared_avg_ms=0.284 all_reduce_avg_ms=0.449 total_avg_ms=3.755
[PROF_MOE_SUMMARY] pid=33749 calls=3040 tokens=958240 router_gate_avg_ms=0.083 routed_avg_ms=2.868 shared_avg_ms=0.285 all_reduce_avg_ms=0.446 total_avg_ms=3.694
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=3040 tokens=958240 router_gate_avg_ms=0.082 routed_avg_ms=2.872 shared_avg_ms=0.283 all_reduce_avg_ms=0.445 total_avg_ms=3.693
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=3040 tokens=958240 router_gate_avg_ms=0.083 routed_avg_ms=2.867 shared_avg_ms=0.288 all_reduce_avg_ms=0.444 total_avg_ms=3.693
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=3040 tokens=958240 router_gate_avg_ms=0.083 routed_avg_ms=2.879 shared_avg_ms=0.285 all_reduce_avg_ms=0.434 total_avg_ms=3.692
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=3120 tokens=958320 router_gate_avg_ms=0.082 routed_avg_ms=2.825 shared_avg_ms=0.284 all_reduce_avg_ms=0.431 total_avg_ms=3.634
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=3120 tokens=958320 router_gate_avg_ms=0.082 routed_avg_ms=2.819 shared_avg_ms=0.282 all_reduce_avg_ms=0.441 total_avg_ms=3.635
[PROF_MOE_SUMMARY] pid=33749 calls=3120 tokens=958320 router_gate_avg_ms=0.083 routed_avg_ms=2.815 shared_avg_ms=0.284 all_reduce_avg_ms=0.443 total_avg_ms=3.636
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=3120 tokens=958320 router_gate_avg_ms=0.083 routed_avg_ms=2.814 shared_avg_ms=0.287 all_reduce_avg_ms=0.441 total_avg_ms=3.635
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=3200 tokens=958400 router_gate_avg_ms=0.081 routed_avg_ms=2.769 shared_avg_ms=0.281 all_reduce_avg_ms=0.439 total_avg_ms=3.581
[PROF_MOE_SUMMARY] pid=33749 calls=3200 tokens=958400 router_gate_avg_ms=0.083 routed_avg_ms=2.764 shared_avg_ms=0.283 all_reduce_avg_ms=0.440 total_avg_ms=3.582
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=3200 tokens=958400 router_gate_avg_ms=0.082 routed_avg_ms=2.763 shared_avg_ms=0.285 all_reduce_avg_ms=0.438 total_avg_ms=3.581
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=3200 tokens=958400 router_gate_avg_ms=0.082 routed_avg_ms=2.775 shared_avg_ms=0.283 all_reduce_avg_ms=0.429 total_avg_ms=3.580
[PROF_MOE_SUMMARY] pid=33749 calls=3280 tokens=958480 router_gate_avg_ms=0.082 routed_avg_ms=2.716 shared_avg_ms=0.282 all_reduce_avg_ms=0.438 total_avg_ms=3.530
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=3280 tokens=958480 router_gate_avg_ms=0.081 routed_avg_ms=2.720 shared_avg_ms=0.279 all_reduce_avg_ms=0.437 total_avg_ms=3.529
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=3280 tokens=958480 router_gate_avg_ms=0.082 routed_avg_ms=2.715 shared_avg_ms=0.284 all_reduce_avg_ms=0.436 total_avg_ms=3.529
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=3280 tokens=958480 router_gate_avg_ms=0.082 routed_avg_ms=2.727 shared_avg_ms=0.282 all_reduce_avg_ms=0.426 total_avg_ms=3.528
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=3360 tokens=958560 router_gate_avg_ms=0.082 routed_avg_ms=2.670 shared_avg_ms=0.283 all_reduce_avg_ms=0.434 total_avg_ms=3.480
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=3360 tokens=958560 router_gate_avg_ms=0.081 routed_avg_ms=2.681 shared_avg_ms=0.281 all_reduce_avg_ms=0.423 total_avg_ms=3.479
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=3360 tokens=958560 router_gate_avg_ms=0.081 routed_avg_ms=2.675 shared_avg_ms=0.278 all_reduce_avg_ms=0.435 total_avg_ms=3.480
[PROF_MOE_SUMMARY] pid=33749 calls=3360 tokens=958560 router_gate_avg_ms=0.082 routed_avg_ms=2.670 shared_avg_ms=0.281 all_reduce_avg_ms=0.436 total_avg_ms=3.480
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=3440 tokens=958640 router_gate_avg_ms=0.080 routed_avg_ms=2.631 shared_avg_ms=0.277 all_reduce_avg_ms=0.433 total_avg_ms=3.433
[PROF_MOE_SUMMARY] pid=33749 calls=3440 tokens=958640 router_gate_avg_ms=0.081 routed_avg_ms=2.627 shared_avg_ms=0.280 all_reduce_avg_ms=0.433 total_avg_ms=3.433
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=3440 tokens=958640 router_gate_avg_ms=0.081 routed_avg_ms=2.626 shared_avg_ms=0.282 all_reduce_avg_ms=0.431 total_avg_ms=3.433
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=3440 tokens=958640 router_gate_avg_ms=0.081 routed_avg_ms=2.638 shared_avg_ms=0.280 all_reduce_avg_ms=0.421 total_avg_ms=3.431
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=3520 tokens=958720 router_gate_avg_ms=0.080 routed_avg_ms=2.589 shared_avg_ms=0.276 all_reduce_avg_ms=0.430 total_avg_ms=3.387
[PROF_MOE_SUMMARY] pid=33749 calls=3520 tokens=958720 router_gate_avg_ms=0.081 routed_avg_ms=2.585 shared_avg_ms=0.279 all_reduce_avg_ms=0.431 total_avg_ms=3.388
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=3520 tokens=958720 router_gate_avg_ms=0.081 routed_avg_ms=2.585 shared_avg_ms=0.281 all_reduce_avg_ms=0.429 total_avg_ms=3.387
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=3520 tokens=958720 router_gate_avg_ms=0.081 routed_avg_ms=2.596 shared_avg_ms=0.279 all_reduce_avg_ms=0.418 total_avg_ms=3.386
[PROF_MOE_SUMMARY] pid=33749 calls=3600 tokens=958800 router_gate_avg_ms=0.081 routed_avg_ms=2.546 shared_avg_ms=0.278 all_reduce_avg_ms=0.428 total_avg_ms=3.344
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=3600 tokens=958800 router_gate_avg_ms=0.081 routed_avg_ms=2.545 shared_avg_ms=0.280 all_reduce_avg_ms=0.426 total_avg_ms=3.344
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=3600 tokens=958800 router_gate_avg_ms=0.081 routed_avg_ms=2.557 shared_avg_ms=0.278 all_reduce_avg_ms=0.416 total_avg_ms=3.342
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=3600 tokens=958800 router_gate_avg_ms=0.080 routed_avg_ms=2.550 shared_avg_ms=0.275 all_reduce_avg_ms=0.428 total_avg_ms=3.344
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=3680 tokens=958880 router_gate_avg_ms=0.079 routed_avg_ms=2.512 shared_avg_ms=0.274 all_reduce_avg_ms=0.425 total_avg_ms=3.302
[PROF_MOE_SUMMARY] pid=33749 calls=3680 tokens=958880 router_gate_avg_ms=0.081 routed_avg_ms=2.508 shared_avg_ms=0.277 all_reduce_avg_ms=0.426 total_avg_ms=3.302
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=3680 tokens=958880 router_gate_avg_ms=0.081 routed_avg_ms=2.507 shared_avg_ms=0.279 all_reduce_avg_ms=0.423 total_avg_ms=3.302
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=3680 tokens=958880 router_gate_avg_ms=0.080 routed_avg_ms=2.518 shared_avg_ms=0.277 all_reduce_avg_ms=0.413 total_avg_ms=3.301
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=3760 tokens=958960 router_gate_avg_ms=0.080 routed_avg_ms=2.482 shared_avg_ms=0.276 all_reduce_avg_ms=0.411 total_avg_ms=3.261
[PROF_MOE_SUMMARY] pid=33749 calls=3760 tokens=958960 router_gate_avg_ms=0.080 routed_avg_ms=2.471 shared_avg_ms=0.276 all_reduce_avg_ms=0.423 total_avg_ms=3.262
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=3760 tokens=958960 router_gate_avg_ms=0.080 routed_avg_ms=2.471 shared_avg_ms=0.278 all_reduce_avg_ms=0.421 total_avg_ms=3.262
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=3760 tokens=958960 router_gate_avg_ms=0.079 routed_avg_ms=2.475 shared_avg_ms=0.273 all_reduce_avg_ms=0.423 total_avg_ms=3.262
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=3840 tokens=959040 router_gate_avg_ms=0.080 routed_avg_ms=2.447 shared_avg_ms=0.275 all_reduce_avg_ms=0.410 total_avg_ms=3.223
[PROF_MOE_SUMMARY] pid=33749 calls=3840 tokens=959040 router_gate_avg_ms=0.080 routed_avg_ms=2.437 shared_avg_ms=0.275 all_reduce_avg_ms=0.422 total_avg_ms=3.225
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=3840 tokens=959040 router_gate_avg_ms=0.079 routed_avg_ms=2.440 shared_avg_ms=0.272 all_reduce_avg_ms=0.421 total_avg_ms=3.224
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=3840 tokens=959040 router_gate_avg_ms=0.080 routed_avg_ms=2.436 shared_avg_ms=0.277 all_reduce_avg_ms=0.419 total_avg_ms=3.224
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=3920 tokens=959120 router_gate_avg_ms=0.080 routed_avg_ms=2.413 shared_avg_ms=0.275 all_reduce_avg_ms=0.408 total_avg_ms=3.187
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=3920 tokens=959120 router_gate_avg_ms=0.079 routed_avg_ms=2.407 shared_avg_ms=0.272 all_reduce_avg_ms=0.419 total_avg_ms=3.188
[PROF_MOE_SUMMARY] pid=33749 calls=3920 tokens=959120 router_gate_avg_ms=0.080 routed_avg_ms=2.403 shared_avg_ms=0.274 all_reduce_avg_ms=0.419 total_avg_ms=3.188
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=3920 tokens=959120 router_gate_avg_ms=0.080 routed_avg_ms=2.403 shared_avg_ms=0.277 all_reduce_avg_ms=0.417 total_avg_ms=3.188
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=4000 tokens=959200 router_gate_avg_ms=0.079 routed_avg_ms=2.381 shared_avg_ms=0.274 all_reduce_avg_ms=0.407 total_avg_ms=3.152
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=4000 tokens=959200 router_gate_avg_ms=0.078 routed_avg_ms=2.375 shared_avg_ms=0.271 all_reduce_avg_ms=0.418 total_avg_ms=3.153
[PROF_MOE_SUMMARY] pid=33749 calls=4000 tokens=959200 router_gate_avg_ms=0.079 routed_avg_ms=2.371 shared_avg_ms=0.274 all_reduce_avg_ms=0.418 total_avg_ms=3.154
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=4000 tokens=959200 router_gate_avg_ms=0.080 routed_avg_ms=2.371 shared_avg_ms=0.276 all_reduce_avg_ms=0.415 total_avg_ms=3.153
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=4080 tokens=959280 router_gate_avg_ms=0.079 routed_avg_ms=2.350 shared_avg_ms=0.273 all_reduce_avg_ms=0.405 total_avg_ms=3.119
[PROF_MOE_SUMMARY] pid=33749 calls=4080 tokens=959280 router_gate_avg_ms=0.079 routed_avg_ms=2.340 shared_avg_ms=0.273 all_reduce_avg_ms=0.417 total_avg_ms=3.121
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=4080 tokens=959280 router_gate_avg_ms=0.078 routed_avg_ms=2.344 shared_avg_ms=0.270 all_reduce_avg_ms=0.416 total_avg_ms=3.120
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=4080 tokens=959280 router_gate_avg_ms=0.080 routed_avg_ms=2.340 shared_avg_ms=0.275 all_reduce_avg_ms=0.413 total_avg_ms=3.120
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=4160 tokens=959360 router_gate_avg_ms=0.079 routed_avg_ms=2.320 shared_avg_ms=0.272 all_reduce_avg_ms=0.404 total_avg_ms=3.087
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=4160 tokens=959360 router_gate_avg_ms=0.078 routed_avg_ms=2.314 shared_avg_ms=0.270 all_reduce_avg_ms=0.415 total_avg_ms=3.088
[PROF_MOE_SUMMARY] pid=33749 calls=4160 tokens=959360 router_gate_avg_ms=0.079 routed_avg_ms=2.311 shared_avg_ms=0.272 all_reduce_avg_ms=0.415 total_avg_ms=3.088
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=4160 tokens=959360 router_gate_avg_ms=0.079 routed_avg_ms=2.311 shared_avg_ms=0.275 all_reduce_avg_ms=0.411 total_avg_ms=3.088
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=4240 tokens=959440 router_gate_avg_ms=0.079 routed_avg_ms=2.292 shared_avg_ms=0.272 all_reduce_avg_ms=0.402 total_avg_ms=3.056
[PROF_MOE_SUMMARY] pid=33749 calls=4240 tokens=959440 router_gate_avg_ms=0.079 routed_avg_ms=2.282 shared_avg_ms=0.272 all_reduce_avg_ms=0.414 total_avg_ms=3.057
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=4240 tokens=959440 router_gate_avg_ms=0.078 routed_avg_ms=2.286 shared_avg_ms=0.269 all_reduce_avg_ms=0.413 total_avg_ms=3.057
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=4240 tokens=959440 router_gate_avg_ms=0.079 routed_avg_ms=2.282 shared_avg_ms=0.274 all_reduce_avg_ms=0.410 total_avg_ms=3.057
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=4320 tokens=959520 router_gate_avg_ms=0.079 routed_avg_ms=2.264 shared_avg_ms=0.271 all_reduce_avg_ms=0.402 total_avg_ms=3.027
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=4320 tokens=959520 router_gate_avg_ms=0.078 routed_avg_ms=2.258 shared_avg_ms=0.268 all_reduce_avg_ms=0.412 total_avg_ms=3.027
[PROF_MOE_SUMMARY] pid=33749 calls=4320 tokens=959520 router_gate_avg_ms=0.079 routed_avg_ms=2.255 shared_avg_ms=0.271 all_reduce_avg_ms=0.412 total_avg_ms=3.028
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=4320 tokens=959520 router_gate_avg_ms=0.079 routed_avg_ms=2.255 shared_avg_ms=0.273 all_reduce_avg_ms=0.408 total_avg_ms=3.027
[PROF_MOE_SUMMARY] pid=33749 calls=4400 tokens=959600 router_gate_avg_ms=0.078 routed_avg_ms=2.228 shared_avg_ms=0.270 all_reduce_avg_ms=0.411 total_avg_ms=2.999
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=4400 tokens=959600 router_gate_avg_ms=0.078 routed_avg_ms=2.232 shared_avg_ms=0.268 all_reduce_avg_ms=0.411 total_avg_ms=2.999
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=4400 tokens=959600 router_gate_avg_ms=0.079 routed_avg_ms=2.229 shared_avg_ms=0.273 all_reduce_avg_ms=0.407 total_avg_ms=2.998
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=4400 tokens=959600 router_gate_avg_ms=0.078 routed_avg_ms=2.237 shared_avg_ms=0.270 all_reduce_avg_ms=0.401 total_avg_ms=2.998
[PROF_MOE_SUMMARY] pid=33749 calls=4480 tokens=959680 router_gate_avg_ms=0.078 routed_avg_ms=2.203 shared_avg_ms=0.270 all_reduce_avg_ms=0.409 total_avg_ms=2.971
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=4480 tokens=959680 router_gate_avg_ms=0.077 routed_avg_ms=2.206 shared_avg_ms=0.267 all_reduce_avg_ms=0.409 total_avg_ms=2.971
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=4480 tokens=959680 router_gate_avg_ms=0.079 routed_avg_ms=2.203 shared_avg_ms=0.272 all_reduce_avg_ms=0.405 total_avg_ms=2.971
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=4480 tokens=959680 router_gate_avg_ms=0.078 routed_avg_ms=2.212 shared_avg_ms=0.270 all_reduce_avg_ms=0.399 total_avg_ms=2.970
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=4560 tokens=959760 router_gate_avg_ms=0.077 routed_avg_ms=2.181 shared_avg_ms=0.266 all_reduce_avg_ms=0.408 total_avg_ms=2.945
[PROF_MOE_SUMMARY] pid=33749 calls=4560 tokens=959760 router_gate_avg_ms=0.078 routed_avg_ms=2.178 shared_avg_ms=0.269 all_reduce_avg_ms=0.408 total_avg_ms=2.945
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=4560 tokens=959760 router_gate_avg_ms=0.078 routed_avg_ms=2.187 shared_avg_ms=0.269 all_reduce_avg_ms=0.398 total_avg_ms=2.944
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=4560 tokens=959760 router_gate_avg_ms=0.079 routed_avg_ms=2.179 shared_avg_ms=0.272 all_reduce_avg_ms=0.404 total_avg_ms=2.944
[PROF_MOE_SUMMARY] pid=33749 calls=4640 tokens=959840 router_gate_avg_ms=0.078 routed_avg_ms=2.154 shared_avg_ms=0.268 all_reduce_avg_ms=0.407 total_avg_ms=2.919
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=4640 tokens=959840 router_gate_avg_ms=0.078 routed_avg_ms=2.155 shared_avg_ms=0.271 all_reduce_avg_ms=0.402 total_avg_ms=2.918
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=4640 tokens=959840 router_gate_avg_ms=0.078 routed_avg_ms=2.164 shared_avg_ms=0.269 all_reduce_avg_ms=0.396 total_avg_ms=2.918
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=4640 tokens=959840 router_gate_avg_ms=0.077 routed_avg_ms=2.157 shared_avg_ms=0.266 all_reduce_avg_ms=0.407 total_avg_ms=2.919
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=4720 tokens=959920 router_gate_avg_ms=0.078 routed_avg_ms=2.140 shared_avg_ms=0.268 all_reduce_avg_ms=0.395 total_avg_ms=2.893
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=4720 tokens=959920 router_gate_avg_ms=0.077 routed_avg_ms=2.134 shared_avg_ms=0.265 all_reduce_avg_ms=0.406 total_avg_ms=2.893
[PROF_MOE_SUMMARY] pid=33749 calls=4720 tokens=959920 router_gate_avg_ms=0.078 routed_avg_ms=2.131 shared_avg_ms=0.268 all_reduce_avg_ms=0.405 total_avg_ms=2.894
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=4720 tokens=959920 router_gate_avg_ms=0.078 routed_avg_ms=2.132 shared_avg_ms=0.271 all_reduce_avg_ms=0.400 total_avg_ms=2.893
[PROF_MOE_SUMMARY] pid=33749 calls=4800 tokens=960000 router_gate_avg_ms=0.077 routed_avg_ms=2.109 shared_avg_ms=0.267 all_reduce_avg_ms=0.404 total_avg_ms=2.870
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=4800 tokens=960000 router_gate_avg_ms=0.078 routed_avg_ms=2.110 shared_avg_ms=0.270 all_reduce_avg_ms=0.399 total_avg_ms=2.869
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=4800 tokens=960000 router_gate_avg_ms=0.077 routed_avg_ms=2.118 shared_avg_ms=0.267 all_reduce_avg_ms=0.394 total_avg_ms=2.869
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=4800 tokens=960000 router_gate_avg_ms=0.077 routed_avg_ms=2.113 shared_avg_ms=0.265 all_reduce_avg_ms=0.404 total_avg_ms=2.869
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=4880 tokens=960080 router_gate_avg_ms=0.078 routed_avg_ms=2.089 shared_avg_ms=0.269 all_reduce_avg_ms=0.398 total_avg_ms=2.846
[PROF_MOE_SUMMARY] pid=33749 calls=4880 tokens=960080 router_gate_avg_ms=0.077 routed_avg_ms=2.088 shared_avg_ms=0.267 all_reduce_avg_ms=0.403 total_avg_ms=2.847
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=4880 tokens=960080 router_gate_avg_ms=0.077 routed_avg_ms=2.097 shared_avg_ms=0.267 all_reduce_avg_ms=0.393 total_avg_ms=2.846
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=4880 tokens=960080 router_gate_avg_ms=0.077 routed_avg_ms=2.091 shared_avg_ms=0.264 all_reduce_avg_ms=0.403 total_avg_ms=2.846
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=4960 tokens=960160 router_gate_avg_ms=0.078 routed_avg_ms=2.068 shared_avg_ms=0.269 all_reduce_avg_ms=0.397 total_avg_ms=2.823
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=4960 tokens=960160 router_gate_avg_ms=0.077 routed_avg_ms=2.076 shared_avg_ms=0.267 all_reduce_avg_ms=0.392 total_avg_ms=2.823
[PROF_MOE_SUMMARY] pid=33749 calls=4960 tokens=960160 router_gate_avg_ms=0.077 routed_avg_ms=2.067 shared_avg_ms=0.267 all_reduce_avg_ms=0.402 total_avg_ms=2.824
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=4960 tokens=960160 router_gate_avg_ms=0.076 routed_avg_ms=2.070 shared_avg_ms=0.264 all_reduce_avg_ms=0.402 total_avg_ms=2.824
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=5040 tokens=960240 router_gate_avg_ms=0.078 routed_avg_ms=2.048 shared_avg_ms=0.269 all_reduce_avg_ms=0.396 total_avg_ms=2.801
[PROF_MOE_SUMMARY] pid=33749 calls=5040 tokens=960240 router_gate_avg_ms=0.077 routed_avg_ms=2.047 shared_avg_ms=0.266 all_reduce_avg_ms=0.400 total_avg_ms=2.802
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=5040 tokens=960240 router_gate_avg_ms=0.077 routed_avg_ms=2.056 shared_avg_ms=0.266 all_reduce_avg_ms=0.391 total_avg_ms=2.801
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=5040 tokens=960240 router_gate_avg_ms=0.076 routed_avg_ms=2.050 shared_avg_ms=0.263 all_reduce_avg_ms=0.400 total_avg_ms=2.802
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=5120 tokens=960320 router_gate_avg_ms=0.077 routed_avg_ms=2.028 shared_avg_ms=0.268 all_reduce_avg_ms=0.394 total_avg_ms=2.780
[PROF_MOE_SUMMARY] pid=33749 calls=5120 tokens=960320 router_gate_avg_ms=0.077 routed_avg_ms=2.028 shared_avg_ms=0.266 all_reduce_avg_ms=0.399 total_avg_ms=2.781
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=5120 tokens=960320 router_gate_avg_ms=0.076 routed_avg_ms=2.031 shared_avg_ms=0.263 all_reduce_avg_ms=0.399 total_avg_ms=2.780
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=5120 tokens=960320 router_gate_avg_ms=0.077 routed_avg_ms=2.036 shared_avg_ms=0.266 all_reduce_avg_ms=0.390 total_avg_ms=2.780
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=5200 tokens=960400 router_gate_avg_ms=0.077 routed_avg_ms=2.017 shared_avg_ms=0.265 all_reduce_avg_ms=0.389 total_avg_ms=2.759
[PROF_MOE_SUMMARY] pid=33749 calls=5200 tokens=960400 router_gate_avg_ms=0.077 routed_avg_ms=2.009 shared_avg_ms=0.265 all_reduce_avg_ms=0.398 total_avg_ms=2.760
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=5200 tokens=960400 router_gate_avg_ms=0.076 routed_avg_ms=2.012 shared_avg_ms=0.263 all_reduce_avg_ms=0.398 total_avg_ms=2.760
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=5200 tokens=960400 router_gate_avg_ms=0.077 routed_avg_ms=2.009 shared_avg_ms=0.268 all_reduce_avg_ms=0.393 total_avg_ms=2.759
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=5280 tokens=960480 router_gate_avg_ms=0.077 routed_avg_ms=1.999 shared_avg_ms=0.265 all_reduce_avg_ms=0.389 total_avg_ms=2.740
[PROF_MOE_SUMMARY] pid=33749 calls=5280 tokens=960480 router_gate_avg_ms=0.076 routed_avg_ms=1.990 shared_avg_ms=0.265 all_reduce_avg_ms=0.397 total_avg_ms=2.740
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=5280 tokens=960480 router_gate_avg_ms=0.076 routed_avg_ms=1.994 shared_avg_ms=0.262 all_reduce_avg_ms=0.397 total_avg_ms=2.740
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=5280 tokens=960480 router_gate_avg_ms=0.077 routed_avg_ms=1.991 shared_avg_ms=0.267 all_reduce_avg_ms=0.392 total_avg_ms=2.739
[PROF_MOE_SUMMARY] pid=33749 calls=5360 tokens=960560 router_gate_avg_ms=0.076 routed_avg_ms=1.973 shared_avg_ms=0.265 all_reduce_avg_ms=0.396 total_avg_ms=2.721
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=5360 tokens=960560 router_gate_avg_ms=0.077 routed_avg_ms=1.973 shared_avg_ms=0.267 all_reduce_avg_ms=0.391 total_avg_ms=2.720
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=5360 tokens=960560 router_gate_avg_ms=0.076 routed_avg_ms=1.981 shared_avg_ms=0.264 all_reduce_avg_ms=0.388 total_avg_ms=2.720
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=5360 tokens=960560 router_gate_avg_ms=0.076 routed_avg_ms=1.976 shared_avg_ms=0.262 all_reduce_avg_ms=0.396 total_avg_ms=2.721
[PROF_MOE_SUMMARY] pid=33749 calls=5440 tokens=960640 router_gate_avg_ms=0.076 routed_avg_ms=1.955 shared_avg_ms=0.264 all_reduce_avg_ms=0.395 total_avg_ms=2.702
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=5440 tokens=960640 router_gate_avg_ms=0.077 routed_avg_ms=1.956 shared_avg_ms=0.267 all_reduce_avg_ms=0.390 total_avg_ms=2.701
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=5440 tokens=960640 router_gate_avg_ms=0.076 routed_avg_ms=1.963 shared_avg_ms=0.264 all_reduce_avg_ms=0.387 total_avg_ms=2.702
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=5440 tokens=960640 router_gate_avg_ms=0.076 routed_avg_ms=1.959 shared_avg_ms=0.262 all_reduce_avg_ms=0.394 total_avg_ms=2.702
[PROF_MOE_SUMMARY] pid=33749 calls=5520 tokens=960720 router_gate_avg_ms=0.076 routed_avg_ms=1.939 shared_avg_ms=0.264 all_reduce_avg_ms=0.394 total_avg_ms=2.684
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=5520 tokens=960720 router_gate_avg_ms=0.077 routed_avg_ms=1.939 shared_avg_ms=0.266 all_reduce_avg_ms=0.389 total_avg_ms=2.683
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=5520 tokens=960720 router_gate_avg_ms=0.076 routed_avg_ms=1.946 shared_avg_ms=0.263 all_reduce_avg_ms=0.387 total_avg_ms=2.684
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=5520 tokens=960720 router_gate_avg_ms=0.076 routed_avg_ms=1.942 shared_avg_ms=0.261 all_reduce_avg_ms=0.393 total_avg_ms=2.684
[PROF_MOE_SUMMARY] pid=33749 calls=5600 tokens=960800 router_gate_avg_ms=0.076 routed_avg_ms=1.922 shared_avg_ms=0.263 all_reduce_avg_ms=0.393 total_avg_ms=2.666
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=5600 tokens=960800 router_gate_avg_ms=0.076 routed_avg_ms=1.930 shared_avg_ms=0.263 all_reduce_avg_ms=0.386 total_avg_ms=2.666
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=5600 tokens=960800 router_gate_avg_ms=0.077 routed_avg_ms=1.923 shared_avg_ms=0.266 all_reduce_avg_ms=0.388 total_avg_ms=2.665
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=5600 tokens=960800 router_gate_avg_ms=0.076 routed_avg_ms=1.926 shared_avg_ms=0.261 all_reduce_avg_ms=0.392 total_avg_ms=2.666
[PROF_MOE_SUMMARY] pid=33749 calls=5680 tokens=960880 router_gate_avg_ms=0.076 routed_avg_ms=1.907 shared_avg_ms=0.263 all_reduce_avg_ms=0.392 total_avg_ms=2.649
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=5680 tokens=960880 router_gate_avg_ms=0.076 routed_avg_ms=1.914 shared_avg_ms=0.263 all_reduce_avg_ms=0.385 total_avg_ms=2.649
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=5680 tokens=960880 router_gate_avg_ms=0.075 routed_avg_ms=1.910 shared_avg_ms=0.260 all_reduce_avg_ms=0.391 total_avg_ms=2.649
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=5680 tokens=960880 router_gate_avg_ms=0.077 routed_avg_ms=1.907 shared_avg_ms=0.265 all_reduce_avg_ms=0.388 total_avg_ms=2.648
[PROF_MOE_SUMMARY] pid=33749 calls=5760 tokens=960960 router_gate_avg_ms=0.076 routed_avg_ms=1.891 shared_avg_ms=0.263 all_reduce_avg_ms=0.392 total_avg_ms=2.633
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=5760 tokens=960960 router_gate_avg_ms=0.075 routed_avg_ms=1.895 shared_avg_ms=0.260 all_reduce_avg_ms=0.391 total_avg_ms=2.632
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=5760 tokens=960960 router_gate_avg_ms=0.076 routed_avg_ms=1.899 shared_avg_ms=0.262 all_reduce_avg_ms=0.384 total_avg_ms=2.632
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=5760 tokens=960960 router_gate_avg_ms=0.076 routed_avg_ms=1.892 shared_avg_ms=0.265 all_reduce_avg_ms=0.387 total_avg_ms=2.632
[PROF_MOE_SUMMARY] pid=33749 calls=5840 tokens=961040 router_gate_avg_ms=0.076 routed_avg_ms=1.876 shared_avg_ms=0.262 all_reduce_avg_ms=0.391 total_avg_ms=2.616
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=5840 tokens=961040 router_gate_avg_ms=0.076 routed_avg_ms=1.877 shared_avg_ms=0.265 all_reduce_avg_ms=0.386 total_avg_ms=2.615
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=5840 tokens=961040 router_gate_avg_ms=0.076 routed_avg_ms=1.884 shared_avg_ms=0.262 all_reduce_avg_ms=0.383 total_avg_ms=2.616
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=5840 tokens=961040 router_gate_avg_ms=0.075 routed_avg_ms=1.880 shared_avg_ms=0.260 all_reduce_avg_ms=0.390 total_avg_ms=2.616
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=5920 tokens=961120 router_gate_avg_ms=0.076 routed_avg_ms=1.869 shared_avg_ms=0.261 all_reduce_avg_ms=0.383 total_avg_ms=2.600
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=5920 tokens=961120 router_gate_avg_ms=0.075 routed_avg_ms=1.865 shared_avg_ms=0.259 all_reduce_avg_ms=0.389 total_avg_ms=2.600
[PROF_MOE_SUMMARY] pid=33749 calls=5920 tokens=961120 router_gate_avg_ms=0.075 routed_avg_ms=1.862 shared_avg_ms=0.262 all_reduce_avg_ms=0.390 total_avg_ms=2.600
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=5920 tokens=961120 router_gate_avg_ms=0.076 routed_avg_ms=1.863 shared_avg_ms=0.264 all_reduce_avg_ms=0.385 total_avg_ms=2.600
[PROF_MOE_SUMMARY] pid=33749 calls=6000 tokens=961200 router_gate_avg_ms=0.075 routed_avg_ms=1.847 shared_avg_ms=0.261 all_reduce_avg_ms=0.389 total_avg_ms=2.585
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=6000 tokens=961200 router_gate_avg_ms=0.076 routed_avg_ms=1.848 shared_avg_ms=0.264 all_reduce_avg_ms=0.384 total_avg_ms=2.584
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=6000 tokens=961200 router_gate_avg_ms=0.075 routed_avg_ms=1.855 shared_avg_ms=0.261 all_reduce_avg_ms=0.382 total_avg_ms=2.584
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=6000 tokens=961200 router_gate_avg_ms=0.075 routed_avg_ms=1.851 shared_avg_ms=0.259 all_reduce_avg_ms=0.388 total_avg_ms=2.584
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=6080 tokens=961280 router_gate_avg_ms=0.075 routed_avg_ms=1.841 shared_avg_ms=0.261 all_reduce_avg_ms=0.381 total_avg_ms=2.569
[PROF_MOE_SUMMARY] pid=33749 calls=6080 tokens=961280 router_gate_avg_ms=0.075 routed_avg_ms=1.834 shared_avg_ms=0.261 all_reduce_avg_ms=0.388 total_avg_ms=2.569
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=6080 tokens=961280 router_gate_avg_ms=0.075 routed_avg_ms=1.838 shared_avg_ms=0.259 all_reduce_avg_ms=0.387 total_avg_ms=2.569
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=6080 tokens=961280 router_gate_avg_ms=0.076 routed_avg_ms=1.835 shared_avg_ms=0.264 all_reduce_avg_ms=0.383 total_avg_ms=2.569
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=6160 tokens=961360 router_gate_avg_ms=0.075 routed_avg_ms=1.827 shared_avg_ms=0.260 all_reduce_avg_ms=0.380 total_avg_ms=2.554
[PROF_MOE_SUMMARY] pid=33749 calls=6160 tokens=961360 router_gate_avg_ms=0.075 routed_avg_ms=1.820 shared_avg_ms=0.261 all_reduce_avg_ms=0.387 total_avg_ms=2.555
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=6160 tokens=961360 router_gate_avg_ms=0.075 routed_avg_ms=1.824 shared_avg_ms=0.258 all_reduce_avg_ms=0.386 total_avg_ms=2.554
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=6160 tokens=961360 router_gate_avg_ms=0.076 routed_avg_ms=1.821 shared_avg_ms=0.263 all_reduce_avg_ms=0.382 total_avg_ms=2.554
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=6240 tokens=961440 router_gate_avg_ms=0.075 routed_avg_ms=1.811 shared_avg_ms=0.258 all_reduce_avg_ms=0.385 total_avg_ms=2.540
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=6240 tokens=961440 router_gate_avg_ms=0.075 routed_avg_ms=1.814 shared_avg_ms=0.260 all_reduce_avg_ms=0.380 total_avg_ms=2.540
[PROF_MOE_SUMMARY] pid=33749 calls=6240 tokens=961440 router_gate_avg_ms=0.075 routed_avg_ms=1.807 shared_avg_ms=0.260 all_reduce_avg_ms=0.387 total_avg_ms=2.541
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=6240 tokens=961440 router_gate_avg_ms=0.076 routed_avg_ms=1.808 shared_avg_ms=0.263 all_reduce_avg_ms=0.381 total_avg_ms=2.540
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=6320 tokens=961520 router_gate_avg_ms=0.076 routed_avg_ms=1.795 shared_avg_ms=0.263 all_reduce_avg_ms=0.380 total_avg_ms=2.526
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=6320 tokens=961520 router_gate_avg_ms=0.075 routed_avg_ms=1.801 shared_avg_ms=0.259 all_reduce_avg_ms=0.379 total_avg_ms=2.526
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=6320 tokens=961520 router_gate_avg_ms=0.075 routed_avg_ms=1.798 shared_avg_ms=0.258 all_reduce_avg_ms=0.384 total_avg_ms=2.526
[PROF_MOE_SUMMARY] pid=33749 calls=6320 tokens=961520 router_gate_avg_ms=0.075 routed_avg_ms=1.794 shared_avg_ms=0.260 all_reduce_avg_ms=0.386 total_avg_ms=2.527
[PROF_MOE_SUMMARY] pid=33749 calls=6400 tokens=961600 router_gate_avg_ms=0.075 routed_avg_ms=1.782 shared_avg_ms=0.260 all_reduce_avg_ms=0.385 total_avg_ms=2.513
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=6400 tokens=961600 router_gate_avg_ms=0.075 routed_avg_ms=1.789 shared_avg_ms=0.259 all_reduce_avg_ms=0.379 total_avg_ms=2.513
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=6400 tokens=961600 router_gate_avg_ms=0.075 routed_avg_ms=1.786 shared_avg_ms=0.257 all_reduce_avg_ms=0.383 total_avg_ms=2.512
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=6400 tokens=961600 router_gate_avg_ms=0.076 routed_avg_ms=1.783 shared_avg_ms=0.262 all_reduce_avg_ms=0.380 total_avg_ms=2.512
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=6480 tokens=961680 router_gate_avg_ms=0.074 routed_avg_ms=1.774 shared_avg_ms=0.257 all_reduce_avg_ms=0.383 total_avg_ms=2.499
[PROF_MOE_SUMMARY] pid=33749 calls=6480 tokens=961680 router_gate_avg_ms=0.075 routed_avg_ms=1.770 shared_avg_ms=0.259 all_reduce_avg_ms=0.384 total_avg_ms=2.500
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=6480 tokens=961680 router_gate_avg_ms=0.076 routed_avg_ms=1.771 shared_avg_ms=0.262 all_reduce_avg_ms=0.379 total_avg_ms=2.499
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=6480 tokens=961680 router_gate_avg_ms=0.075 routed_avg_ms=1.777 shared_avg_ms=0.259 all_reduce_avg_ms=0.378 total_avg_ms=2.500
[PROF_MOE_SUMMARY] pid=33749 calls=6560 tokens=961760 router_gate_avg_ms=0.075 routed_avg_ms=1.758 shared_avg_ms=0.259 all_reduce_avg_ms=0.384 total_avg_ms=2.487
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=6560 tokens=961760 router_gate_avg_ms=0.074 routed_avg_ms=1.762 shared_avg_ms=0.257 all_reduce_avg_ms=0.382 total_avg_ms=2.487
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=6560 tokens=961760 router_gate_avg_ms=0.076 routed_avg_ms=1.759 shared_avg_ms=0.262 all_reduce_avg_ms=0.379 total_avg_ms=2.487
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=6560 tokens=961760 router_gate_avg_ms=0.075 routed_avg_ms=1.765 shared_avg_ms=0.259 all_reduce_avg_ms=0.378 total_avg_ms=2.487
[PROF_MOE_SUMMARY] pid=33749 calls=6640 tokens=961840 router_gate_avg_ms=0.075 routed_avg_ms=1.747 shared_avg_ms=0.259 all_reduce_avg_ms=0.385 total_avg_ms=2.476
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=6640 tokens=961840 router_gate_avg_ms=0.075 routed_avg_ms=1.753 shared_avg_ms=0.259 all_reduce_avg_ms=0.378 total_avg_ms=2.476
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=6640 tokens=961840 router_gate_avg_ms=0.074 routed_avg_ms=1.751 shared_avg_ms=0.257 all_reduce_avg_ms=0.382 total_avg_ms=2.475
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=6640 tokens=961840 router_gate_avg_ms=0.076 routed_avg_ms=1.748 shared_avg_ms=0.262 all_reduce_avg_ms=0.379 total_avg_ms=2.476
[PROF_MOE_SUMMARY] pid=33749 calls=6720 tokens=961920 router_gate_avg_ms=0.074 routed_avg_ms=1.735 shared_avg_ms=0.259 all_reduce_avg_ms=0.384 total_avg_ms=2.464
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=6720 tokens=961920 router_gate_avg_ms=0.075 routed_avg_ms=1.737 shared_avg_ms=0.261 all_reduce_avg_ms=0.379 total_avg_ms=2.464
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=6720 tokens=961920 router_gate_avg_ms=0.075 routed_avg_ms=1.742 shared_avg_ms=0.259 all_reduce_avg_ms=0.377 total_avg_ms=2.464
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=6720 tokens=961920 router_gate_avg_ms=0.074 routed_avg_ms=1.739 shared_avg_ms=0.257 all_reduce_avg_ms=0.381 total_avg_ms=2.463
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=6800 tokens=962000 router_gate_avg_ms=0.075 routed_avg_ms=1.725 shared_avg_ms=0.261 all_reduce_avg_ms=0.378 total_avg_ms=2.451
[PROF_MOE_SUMMARY] pid=33749 calls=6800 tokens=962000 router_gate_avg_ms=0.074 routed_avg_ms=1.724 shared_avg_ms=0.259 all_reduce_avg_ms=0.383 total_avg_ms=2.452
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=6800 tokens=962000 router_gate_avg_ms=0.075 routed_avg_ms=1.731 shared_avg_ms=0.258 all_reduce_avg_ms=0.377 total_avg_ms=2.452
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=6800 tokens=962000 router_gate_avg_ms=0.074 routed_avg_ms=1.728 shared_avg_ms=0.256 all_reduce_avg_ms=0.381 total_avg_ms=2.451
[PROF_MOE_SUMMARY] pid=33749 calls=6880 tokens=962080 router_gate_avg_ms=0.074 routed_avg_ms=1.714 shared_avg_ms=0.258 all_reduce_avg_ms=0.382 total_avg_ms=2.440
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=6880 tokens=962080 router_gate_avg_ms=0.074 routed_avg_ms=1.720 shared_avg_ms=0.258 all_reduce_avg_ms=0.376 total_avg_ms=2.440
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=6880 tokens=962080 router_gate_avg_ms=0.074 routed_avg_ms=1.718 shared_avg_ms=0.256 all_reduce_avg_ms=0.380 total_avg_ms=2.439
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=6880 tokens=962080 router_gate_avg_ms=0.075 routed_avg_ms=1.715 shared_avg_ms=0.261 all_reduce_avg_ms=0.377 total_avg_ms=2.440
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=6960 tokens=962160 router_gate_avg_ms=0.074 routed_avg_ms=1.710 shared_avg_ms=0.258 all_reduce_avg_ms=0.376 total_avg_ms=2.429
[PROF_MOE_SUMMARY] pid=33749 calls=6960 tokens=962160 router_gate_avg_ms=0.074 routed_avg_ms=1.703 shared_avg_ms=0.258 all_reduce_avg_ms=0.381 total_avg_ms=2.428
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=6960 tokens=962160 router_gate_avg_ms=0.074 routed_avg_ms=1.707 shared_avg_ms=0.256 all_reduce_avg_ms=0.379 total_avg_ms=2.428
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=6960 tokens=962160 router_gate_avg_ms=0.075 routed_avg_ms=1.704 shared_avg_ms=0.261 all_reduce_avg_ms=0.377 total_avg_ms=2.428
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=7040 tokens=962240 router_gate_avg_ms=0.074 routed_avg_ms=1.699 shared_avg_ms=0.257 all_reduce_avg_ms=0.376 total_avg_ms=2.418
[PROF_MOE_SUMMARY] pid=33749 calls=7040 tokens=962240 router_gate_avg_ms=0.074 routed_avg_ms=1.693 shared_avg_ms=0.258 all_reduce_avg_ms=0.381 total_avg_ms=2.418
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=7040 tokens=962240 router_gate_avg_ms=0.074 routed_avg_ms=1.697 shared_avg_ms=0.256 all_reduce_avg_ms=0.379 total_avg_ms=2.417
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=7040 tokens=962240 router_gate_avg_ms=0.075 routed_avg_ms=1.694 shared_avg_ms=0.260 all_reduce_avg_ms=0.376 total_avg_ms=2.417
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=7120 tokens=962320 router_gate_avg_ms=0.074 routed_avg_ms=1.689 shared_avg_ms=0.257 all_reduce_avg_ms=0.376 total_avg_ms=2.407
[PROF_MOE_SUMMARY] pid=33749 calls=7120 tokens=962320 router_gate_avg_ms=0.074 routed_avg_ms=1.683 shared_avg_ms=0.258 all_reduce_avg_ms=0.381 total_avg_ms=2.407
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=7120 tokens=962320 router_gate_avg_ms=0.074 routed_avg_ms=1.687 shared_avg_ms=0.255 all_reduce_avg_ms=0.378 total_avg_ms=2.406
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=7120 tokens=962320 router_gate_avg_ms=0.075 routed_avg_ms=1.684 shared_avg_ms=0.260 all_reduce_avg_ms=0.375 total_avg_ms=2.406
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=7200 tokens=962400 router_gate_avg_ms=0.074 routed_avg_ms=1.679 shared_avg_ms=0.257 all_reduce_avg_ms=0.375 total_avg_ms=2.397
[PROF_MOE_SUMMARY] pid=33749 calls=7200 tokens=962400 router_gate_avg_ms=0.074 routed_avg_ms=1.673 shared_avg_ms=0.258 all_reduce_avg_ms=0.380 total_avg_ms=2.396
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=7200 tokens=962400 router_gate_avg_ms=0.075 routed_avg_ms=1.674 shared_avg_ms=0.260 all_reduce_avg_ms=0.375 total_avg_ms=2.396
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=7200 tokens=962400 router_gate_avg_ms=0.074 routed_avg_ms=1.677 shared_avg_ms=0.255 all_reduce_avg_ms=0.378 total_avg_ms=2.395
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=7280 tokens=962480 router_gate_avg_ms=0.074 routed_avg_ms=1.669 shared_avg_ms=0.256 all_reduce_avg_ms=0.375 total_avg_ms=2.386
[PROF_MOE_SUMMARY] pid=33749 calls=7280 tokens=962480 router_gate_avg_ms=0.074 routed_avg_ms=1.664 shared_avg_ms=0.257 all_reduce_avg_ms=0.379 total_avg_ms=2.386
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=7280 tokens=962480 router_gate_avg_ms=0.075 routed_avg_ms=1.665 shared_avg_ms=0.260 all_reduce_avg_ms=0.374 total_avg_ms=2.385
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=7280 tokens=962480 router_gate_avg_ms=0.074 routed_avg_ms=1.668 shared_avg_ms=0.255 all_reduce_avg_ms=0.377 total_avg_ms=2.385
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=7360 tokens=962560 router_gate_avg_ms=0.075 routed_avg_ms=1.655 shared_avg_ms=0.260 all_reduce_avg_ms=0.374 total_avg_ms=2.375
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=7360 tokens=962560 router_gate_avg_ms=0.074 routed_avg_ms=1.660 shared_avg_ms=0.256 all_reduce_avg_ms=0.375 total_avg_ms=2.376
[PROF_MOE_SUMMARY] pid=33749 calls=7360 tokens=962560 router_gate_avg_ms=0.074 routed_avg_ms=1.654 shared_avg_ms=0.257 all_reduce_avg_ms=0.379 total_avg_ms=2.375
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=7360 tokens=962560 router_gate_avg_ms=0.074 routed_avg_ms=1.658 shared_avg_ms=0.255 all_reduce_avg_ms=0.377 total_avg_ms=2.375
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=7440 tokens=962640 router_gate_avg_ms=0.074 routed_avg_ms=1.651 shared_avg_ms=0.256 all_reduce_avg_ms=0.375 total_avg_ms=2.367
[PROF_MOE_SUMMARY] pid=33749 calls=7440 tokens=962640 router_gate_avg_ms=0.074 routed_avg_ms=1.645 shared_avg_ms=0.257 all_reduce_avg_ms=0.379 total_avg_ms=2.366
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=7440 tokens=962640 router_gate_avg_ms=0.074 routed_avg_ms=1.649 shared_avg_ms=0.255 all_reduce_avg_ms=0.376 total_avg_ms=2.365
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=7440 tokens=962640 router_gate_avg_ms=0.075 routed_avg_ms=1.646 shared_avg_ms=0.259 all_reduce_avg_ms=0.373 total_avg_ms=2.365
[PROF_MOE_SUMMARY] pid=33749 calls=7520 tokens=962720 router_gate_avg_ms=0.074 routed_avg_ms=1.636 shared_avg_ms=0.257 all_reduce_avg_ms=0.378 total_avg_ms=2.356
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=7520 tokens=962720 router_gate_avg_ms=0.074 routed_avg_ms=1.642 shared_avg_ms=0.256 all_reduce_avg_ms=0.375 total_avg_ms=2.357
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=7520 tokens=962720 router_gate_avg_ms=0.074 routed_avg_ms=1.640 shared_avg_ms=0.254 all_reduce_avg_ms=0.376 total_avg_ms=2.355
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=7520 tokens=962720 router_gate_avg_ms=0.075 routed_avg_ms=1.637 shared_avg_ms=0.259 all_reduce_avg_ms=0.373 total_avg_ms=2.356
[PROF_MOE_SUMMARY] pid=33749 calls=7600 tokens=962800 router_gate_avg_ms=0.074 routed_avg_ms=1.627 shared_avg_ms=0.257 all_reduce_avg_ms=0.377 total_avg_ms=2.347
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=7600 tokens=962800 router_gate_avg_ms=0.074 routed_avg_ms=1.631 shared_avg_ms=0.254 all_reduce_avg_ms=0.375 total_avg_ms=2.346
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=7600 tokens=962800 router_gate_avg_ms=0.075 routed_avg_ms=1.629 shared_avg_ms=0.259 all_reduce_avg_ms=0.372 total_avg_ms=2.346
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=7600 tokens=962800 router_gate_avg_ms=0.074 routed_avg_ms=1.633 shared_avg_ms=0.255 all_reduce_avg_ms=0.374 total_avg_ms=2.348
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=7680 tokens=962880 router_gate_avg_ms=0.075 routed_avg_ms=1.620 shared_avg_ms=0.259 all_reduce_avg_ms=0.372 total_avg_ms=2.337
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=7680 tokens=962880 router_gate_avg_ms=0.074 routed_avg_ms=1.624 shared_avg_ms=0.255 all_reduce_avg_ms=0.374 total_avg_ms=2.338
[PROF_MOE_SUMMARY] pid=33749 calls=7680 tokens=962880 router_gate_avg_ms=0.074 routed_avg_ms=1.619 shared_avg_ms=0.256 all_reduce_avg_ms=0.377 total_avg_ms=2.337
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=7680 tokens=962880 router_gate_avg_ms=0.074 routed_avg_ms=1.623 shared_avg_ms=0.254 all_reduce_avg_ms=0.375 total_avg_ms=2.337
[PROF_MOE_SUMMARY] pid=33749 calls=7760 tokens=962960 router_gate_avg_ms=0.073 routed_avg_ms=1.610 shared_avg_ms=0.256 all_reduce_avg_ms=0.376 total_avg_ms=2.328
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=7760 tokens=962960 router_gate_avg_ms=0.075 routed_avg_ms=1.611 shared_avg_ms=0.259 all_reduce_avg_ms=0.372 total_avg_ms=2.328
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=7760 tokens=962960 router_gate_avg_ms=0.074 routed_avg_ms=1.614 shared_avg_ms=0.254 all_reduce_avg_ms=0.375 total_avg_ms=2.327
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=7760 tokens=962960 router_gate_avg_ms=0.074 routed_avg_ms=1.616 shared_avg_ms=0.255 all_reduce_avg_ms=0.373 total_avg_ms=2.329
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=7840 tokens=963040 router_gate_avg_ms=0.075 routed_avg_ms=1.603 shared_avg_ms=0.258 all_reduce_avg_ms=0.372 total_avg_ms=2.319
[PROF_MOE_SUMMARY] pid=33749 calls=7840 tokens=963040 router_gate_avg_ms=0.073 routed_avg_ms=1.602 shared_avg_ms=0.256 all_reduce_avg_ms=0.376 total_avg_ms=2.319
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=7840 tokens=963040 router_gate_avg_ms=0.073 routed_avg_ms=1.606 shared_avg_ms=0.254 all_reduce_avg_ms=0.374 total_avg_ms=2.318
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=7840 tokens=963040 router_gate_avg_ms=0.074 routed_avg_ms=1.608 shared_avg_ms=0.255 all_reduce_avg_ms=0.373 total_avg_ms=2.320
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=7920 tokens=963120 router_gate_avg_ms=0.074 routed_avg_ms=1.595 shared_avg_ms=0.258 all_reduce_avg_ms=0.371 total_avg_ms=2.310
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=7920 tokens=963120 router_gate_avg_ms=0.074 routed_avg_ms=1.599 shared_avg_ms=0.255 all_reduce_avg_ms=0.372 total_avg_ms=2.311
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=7920 tokens=963120 router_gate_avg_ms=0.073 routed_avg_ms=1.598 shared_avg_ms=0.253 all_reduce_avg_ms=0.374 total_avg_ms=2.310
[PROF_MOE_SUMMARY] pid=33749 calls=7920 tokens=963120 router_gate_avg_ms=0.073 routed_avg_ms=1.594 shared_avg_ms=0.256 all_reduce_avg_ms=0.375 total_avg_ms=2.310
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=8000 tokens=963200 router_gate_avg_ms=0.074 routed_avg_ms=1.587 shared_avg_ms=0.258 all_reduce_avg_ms=0.371 total_avg_ms=2.302
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=8000 tokens=963200 router_gate_avg_ms=0.073 routed_avg_ms=1.591 shared_avg_ms=0.254 all_reduce_avg_ms=0.372 total_avg_ms=2.303
[PROF_MOE_SUMMARY] pid=33749 calls=8000 tokens=963200 router_gate_avg_ms=0.073 routed_avg_ms=1.586 shared_avg_ms=0.256 all_reduce_avg_ms=0.375 total_avg_ms=2.302
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=8000 tokens=963200 router_gate_avg_ms=0.073 routed_avg_ms=1.590 shared_avg_ms=0.253 all_reduce_avg_ms=0.373 total_avg_ms=2.301
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=8080 tokens=963280 router_gate_avg_ms=0.074 routed_avg_ms=1.579 shared_avg_ms=0.258 all_reduce_avg_ms=0.371 total_avg_ms=2.293
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=8080 tokens=963280 router_gate_avg_ms=0.073 routed_avg_ms=1.584 shared_avg_ms=0.254 all_reduce_avg_ms=0.372 total_avg_ms=2.294
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=8080 tokens=963280 router_gate_avg_ms=0.073 routed_avg_ms=1.582 shared_avg_ms=0.253 all_reduce_avg_ms=0.373 total_avg_ms=2.293
[PROF_MOE_SUMMARY] pid=33749 calls=8080 tokens=963280 router_gate_avg_ms=0.073 routed_avg_ms=1.578 shared_avg_ms=0.256 all_reduce_avg_ms=0.374 total_avg_ms=2.293
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=8160 tokens=963360 router_gate_avg_ms=0.074 routed_avg_ms=1.571 shared_avg_ms=0.257 all_reduce_avg_ms=0.371 total_avg_ms=2.285
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=8160 tokens=963360 router_gate_avg_ms=0.073 routed_avg_ms=1.576 shared_avg_ms=0.254 all_reduce_avg_ms=0.371 total_avg_ms=2.286
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=8160 tokens=963360 router_gate_avg_ms=0.073 routed_avg_ms=1.574 shared_avg_ms=0.253 all_reduce_avg_ms=0.373 total_avg_ms=2.284
[PROF_MOE_SUMMARY] pid=33749 calls=8160 tokens=963360 router_gate_avg_ms=0.073 routed_avg_ms=1.571 shared_avg_ms=0.255 all_reduce_avg_ms=0.374 total_avg_ms=2.285
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=8240 tokens=963440 router_gate_avg_ms=0.074 routed_avg_ms=1.564 shared_avg_ms=0.257 all_reduce_avg_ms=0.370 total_avg_ms=2.277
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=8240 tokens=963440 router_gate_avg_ms=0.073 routed_avg_ms=1.568 shared_avg_ms=0.254 all_reduce_avg_ms=0.371 total_avg_ms=2.278
[PROF_MOE_SUMMARY] pid=33749 calls=8240 tokens=963440 router_gate_avg_ms=0.073 routed_avg_ms=1.563 shared_avg_ms=0.255 all_reduce_avg_ms=0.373 total_avg_ms=2.277
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=8240 tokens=963440 router_gate_avg_ms=0.073 routed_avg_ms=1.567 shared_avg_ms=0.253 all_reduce_avg_ms=0.372 total_avg_ms=2.276
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=8320 tokens=963520 router_gate_avg_ms=0.073 routed_avg_ms=1.560 shared_avg_ms=0.252 all_reduce_avg_ms=0.372 total_avg_ms=2.268
[PROF_MOE_SUMMARY] pid=33749 calls=8320 tokens=963520 router_gate_avg_ms=0.073 routed_avg_ms=1.556 shared_avg_ms=0.255 all_reduce_avg_ms=0.373 total_avg_ms=2.269
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=8320 tokens=963520 router_gate_avg_ms=0.074 routed_avg_ms=1.556 shared_avg_ms=0.257 all_reduce_avg_ms=0.370 total_avg_ms=2.269
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=8320 tokens=963520 router_gate_avg_ms=0.073 routed_avg_ms=1.561 shared_avg_ms=0.254 all_reduce_avg_ms=0.370 total_avg_ms=2.270
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=8400 tokens=963600 router_gate_avg_ms=0.073 routed_avg_ms=1.552 shared_avg_ms=0.252 all_reduce_avg_ms=0.372 total_avg_ms=2.261
[PROF_MOE_SUMMARY] pid=33749 calls=8400 tokens=963600 router_gate_avg_ms=0.073 routed_avg_ms=1.549 shared_avg_ms=0.255 all_reduce_avg_ms=0.372 total_avg_ms=2.261
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=8400 tokens=963600 router_gate_avg_ms=0.074 routed_avg_ms=1.549 shared_avg_ms=0.257 all_reduce_avg_ms=0.370 total_avg_ms=2.261
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=8400 tokens=963600 router_gate_avg_ms=0.073 routed_avg_ms=1.554 shared_avg_ms=0.254 all_reduce_avg_ms=0.370 total_avg_ms=2.262
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=8480 tokens=963680 router_gate_avg_ms=0.073 routed_avg_ms=1.545 shared_avg_ms=0.252 all_reduce_avg_ms=0.372 total_avg_ms=2.253
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=8480 tokens=963680 router_gate_avg_ms=0.074 routed_avg_ms=1.542 shared_avg_ms=0.257 all_reduce_avg_ms=0.370 total_avg_ms=2.254
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=8480 tokens=963680 router_gate_avg_ms=0.073 routed_avg_ms=1.547 shared_avg_ms=0.254 all_reduce_avg_ms=0.370 total_avg_ms=2.255
[PROF_MOE_SUMMARY] pid=33749 calls=8480 tokens=963680 router_gate_avg_ms=0.073 routed_avg_ms=1.542 shared_avg_ms=0.255 all_reduce_avg_ms=0.372 total_avg_ms=2.254
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=8560 tokens=963760 router_gate_avg_ms=0.074 routed_avg_ms=1.535 shared_avg_ms=0.256 all_reduce_avg_ms=0.370 total_avg_ms=2.247
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=8560 tokens=963760 router_gate_avg_ms=0.073 routed_avg_ms=1.540 shared_avg_ms=0.253 all_reduce_avg_ms=0.369 total_avg_ms=2.247
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=8560 tokens=963760 router_gate_avg_ms=0.073 routed_avg_ms=1.538 shared_avg_ms=0.252 all_reduce_avg_ms=0.372 total_avg_ms=2.246
[PROF_MOE_SUMMARY] pid=33749 calls=8560 tokens=963760 router_gate_avg_ms=0.073 routed_avg_ms=1.535 shared_avg_ms=0.255 all_reduce_avg_ms=0.372 total_avg_ms=2.246
[PROF_MOE_SUMMARY] pid=33749 calls=8640 tokens=963840 router_gate_avg_ms=0.073 routed_avg_ms=1.528 shared_avg_ms=0.255 all_reduce_avg_ms=0.371 total_avg_ms=2.239
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=8640 tokens=963840 router_gate_avg_ms=0.073 routed_avg_ms=1.531 shared_avg_ms=0.252 all_reduce_avg_ms=0.371 total_avg_ms=2.239
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=8640 tokens=963840 router_gate_avg_ms=0.074 routed_avg_ms=1.528 shared_avg_ms=0.256 all_reduce_avg_ms=0.370 total_avg_ms=2.239
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=8640 tokens=963840 router_gate_avg_ms=0.073 routed_avg_ms=1.533 shared_avg_ms=0.253 all_reduce_avg_ms=0.369 total_avg_ms=2.240
[PROF_MOE_SUMMARY] pid=33749 calls=8720 tokens=963920 router_gate_avg_ms=0.073 routed_avg_ms=1.521 shared_avg_ms=0.254 all_reduce_avg_ms=0.371 total_avg_ms=2.232
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=8720 tokens=963920 router_gate_avg_ms=0.074 routed_avg_ms=1.522 shared_avg_ms=0.256 all_reduce_avg_ms=0.369 total_avg_ms=2.232
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=8720 tokens=963920 router_gate_avg_ms=0.073 routed_avg_ms=1.525 shared_avg_ms=0.252 all_reduce_avg_ms=0.371 total_avg_ms=2.232
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=8720 tokens=963920 router_gate_avg_ms=0.073 routed_avg_ms=1.527 shared_avg_ms=0.253 all_reduce_avg_ms=0.368 total_avg_ms=2.233
[PROF_MOE_SUMMARY] pid=33749 calls=8800 tokens=964000 router_gate_avg_ms=0.073 routed_avg_ms=1.515 shared_avg_ms=0.254 all_reduce_avg_ms=0.371 total_avg_ms=2.225
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=8800 tokens=964000 router_gate_avg_ms=0.073 routed_avg_ms=1.518 shared_avg_ms=0.252 all_reduce_avg_ms=0.371 total_avg_ms=2.225
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=8800 tokens=964000 router_gate_avg_ms=0.074 routed_avg_ms=1.515 shared_avg_ms=0.256 all_reduce_avg_ms=0.369 total_avg_ms=2.225
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=8800 tokens=964000 router_gate_avg_ms=0.073 routed_avg_ms=1.520 shared_avg_ms=0.253 all_reduce_avg_ms=0.368 total_avg_ms=2.226
[PROF_MOE_SUMMARY] pid=33749 calls=8880 tokens=964080 router_gate_avg_ms=0.073 routed_avg_ms=1.509 shared_avg_ms=0.254 all_reduce_avg_ms=0.370 total_avg_ms=2.218
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=8880 tokens=964080 router_gate_avg_ms=0.073 routed_avg_ms=1.512 shared_avg_ms=0.251 all_reduce_avg_ms=0.371 total_avg_ms=2.218
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=8880 tokens=964080 router_gate_avg_ms=0.074 routed_avg_ms=1.508 shared_avg_ms=0.256 all_reduce_avg_ms=0.369 total_avg_ms=2.218
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=8880 tokens=964080 router_gate_avg_ms=0.073 routed_avg_ms=1.514 shared_avg_ms=0.253 all_reduce_avg_ms=0.367 total_avg_ms=2.219
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=8960 tokens=964160 router_gate_avg_ms=0.074 routed_avg_ms=1.502 shared_avg_ms=0.255 all_reduce_avg_ms=0.369 total_avg_ms=2.212
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=8960 tokens=964160 router_gate_avg_ms=0.073 routed_avg_ms=1.508 shared_avg_ms=0.253 all_reduce_avg_ms=0.367 total_avg_ms=2.212
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=8960 tokens=964160 router_gate_avg_ms=0.073 routed_avg_ms=1.505 shared_avg_ms=0.251 all_reduce_avg_ms=0.371 total_avg_ms=2.211
[PROF_MOE_SUMMARY] pid=33749 calls=8960 tokens=964160 router_gate_avg_ms=0.073 routed_avg_ms=1.502 shared_avg_ms=0.254 all_reduce_avg_ms=0.370 total_avg_ms=2.211
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=9040 tokens=964240 router_gate_avg_ms=0.073 routed_avg_ms=1.501 shared_avg_ms=0.253 all_reduce_avg_ms=0.367 total_avg_ms=2.205
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=9040 tokens=964240 router_gate_avg_ms=0.073 routed_avg_ms=1.499 shared_avg_ms=0.251 all_reduce_avg_ms=0.370 total_avg_ms=2.205
[PROF_MOE_SUMMARY] pid=33749 calls=9040 tokens=964240 router_gate_avg_ms=0.073 routed_avg_ms=1.496 shared_avg_ms=0.254 all_reduce_avg_ms=0.370 total_avg_ms=2.204
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=9040 tokens=964240 router_gate_avg_ms=0.074 routed_avg_ms=1.496 shared_avg_ms=0.255 all_reduce_avg_ms=0.369 total_avg_ms=2.205
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=9120 tokens=964320 router_gate_avg_ms=0.074 routed_avg_ms=1.490 shared_avg_ms=0.255 all_reduce_avg_ms=0.369 total_avg_ms=2.199
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=9120 tokens=964320 router_gate_avg_ms=0.073 routed_avg_ms=1.495 shared_avg_ms=0.253 all_reduce_avg_ms=0.367 total_avg_ms=2.199
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=9120 tokens=964320 router_gate_avg_ms=0.073 routed_avg_ms=1.493 shared_avg_ms=0.251 all_reduce_avg_ms=0.370 total_avg_ms=2.198
[PROF_MOE_SUMMARY] pid=33749 calls=9120 tokens=964320 router_gate_avg_ms=0.073 routed_avg_ms=1.490 shared_avg_ms=0.254 all_reduce_avg_ms=0.369 total_avg_ms=2.198
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=9200 tokens=964400 router_gate_avg_ms=0.074 routed_avg_ms=1.484 shared_avg_ms=0.255 all_reduce_avg_ms=0.369 total_avg_ms=2.192
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=9200 tokens=964400 router_gate_avg_ms=0.073 routed_avg_ms=1.489 shared_avg_ms=0.253 all_reduce_avg_ms=0.366 total_avg_ms=2.193
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=9200 tokens=964400 router_gate_avg_ms=0.073 routed_avg_ms=1.487 shared_avg_ms=0.251 all_reduce_avg_ms=0.370 total_avg_ms=2.192
[PROF_MOE_SUMMARY] pid=33749 calls=9200 tokens=964400 router_gate_avg_ms=0.073 routed_avg_ms=1.484 shared_avg_ms=0.254 all_reduce_avg_ms=0.369 total_avg_ms=2.191
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=9280 tokens=964480 router_gate_avg_ms=0.074 routed_avg_ms=1.478 shared_avg_ms=0.255 all_reduce_avg_ms=0.369 total_avg_ms=2.186
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=9280 tokens=964480 router_gate_avg_ms=0.073 routed_avg_ms=1.483 shared_avg_ms=0.252 all_reduce_avg_ms=0.366 total_avg_ms=2.186
[PROF_MOE_SUMMARY] pid=33749 calls=9280 tokens=964480 router_gate_avg_ms=0.073 routed_avg_ms=1.478 shared_avg_ms=0.254 all_reduce_avg_ms=0.368 total_avg_ms=2.185
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=9280 tokens=964480 router_gate_avg_ms=0.073 routed_avg_ms=1.481 shared_avg_ms=0.251 all_reduce_avg_ms=0.370 total_avg_ms=2.186
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=9360 tokens=964560 router_gate_avg_ms=0.074 routed_avg_ms=1.472 shared_avg_ms=0.255 all_reduce_avg_ms=0.368 total_avg_ms=2.180
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=9360 tokens=964560 router_gate_avg_ms=0.073 routed_avg_ms=1.478 shared_avg_ms=0.252 all_reduce_avg_ms=0.366 total_avg_ms=2.180
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=9360 tokens=964560 router_gate_avg_ms=0.073 routed_avg_ms=1.475 shared_avg_ms=0.251 all_reduce_avg_ms=0.370 total_avg_ms=2.179
[PROF_MOE_SUMMARY] pid=33749 calls=9360 tokens=964560 router_gate_avg_ms=0.073 routed_avg_ms=1.473 shared_avg_ms=0.254 all_reduce_avg_ms=0.368 total_avg_ms=2.179
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=9440 tokens=964640 router_gate_avg_ms=0.073 routed_avg_ms=1.466 shared_avg_ms=0.254 all_reduce_avg_ms=0.368 total_avg_ms=2.174
[PROF_MOE_SUMMARY] pid=33749 calls=9440 tokens=964640 router_gate_avg_ms=0.073 routed_avg_ms=1.467 shared_avg_ms=0.254 all_reduce_avg_ms=0.368 total_avg_ms=2.173
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=9440 tokens=964640 router_gate_avg_ms=0.073 routed_avg_ms=1.472 shared_avg_ms=0.252 all_reduce_avg_ms=0.366 total_avg_ms=2.174
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=9440 tokens=964640 router_gate_avg_ms=0.073 routed_avg_ms=1.470 shared_avg_ms=0.250 all_reduce_avg_ms=0.369 total_avg_ms=2.173
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=9520 tokens=964720 router_gate_avg_ms=0.073 routed_avg_ms=1.461 shared_avg_ms=0.254 all_reduce_avg_ms=0.368 total_avg_ms=2.168
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=9520 tokens=964720 router_gate_avg_ms=0.073 routed_avg_ms=1.466 shared_avg_ms=0.252 all_reduce_avg_ms=0.366 total_avg_ms=2.168
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=9520 tokens=964720 router_gate_avg_ms=0.073 routed_avg_ms=1.464 shared_avg_ms=0.250 all_reduce_avg_ms=0.369 total_avg_ms=2.168
[PROF_MOE_SUMMARY] pid=33749 calls=9520 tokens=964720 router_gate_avg_ms=0.073 routed_avg_ms=1.462 shared_avg_ms=0.254 all_reduce_avg_ms=0.367 total_avg_ms=2.167
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=9600 tokens=964800 router_gate_avg_ms=0.073 routed_avg_ms=1.455 shared_avg_ms=0.254 all_reduce_avg_ms=0.368 total_avg_ms=2.162
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=9600 tokens=964800 router_gate_avg_ms=0.073 routed_avg_ms=1.461 shared_avg_ms=0.252 all_reduce_avg_ms=0.366 total_avg_ms=2.163
[PROF_MOE_SUMMARY] pid=33749 calls=9600 tokens=964800 router_gate_avg_ms=0.073 routed_avg_ms=1.456 shared_avg_ms=0.254 all_reduce_avg_ms=0.367 total_avg_ms=2.161
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=9600 tokens=964800 router_gate_avg_ms=0.073 routed_avg_ms=1.459 shared_avg_ms=0.250 all_reduce_avg_ms=0.369 total_avg_ms=2.162
[PROF_MOE_SUMMARY] pid=33749 calls=9680 tokens=964880 router_gate_avg_ms=0.073 routed_avg_ms=1.451 shared_avg_ms=0.253 all_reduce_avg_ms=0.367 total_avg_ms=2.156
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=9680 tokens=964880 router_gate_avg_ms=0.073 routed_avg_ms=1.453 shared_avg_ms=0.250 all_reduce_avg_ms=0.369 total_avg_ms=2.156
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=9680 tokens=964880 router_gate_avg_ms=0.073 routed_avg_ms=1.450 shared_avg_ms=0.254 all_reduce_avg_ms=0.368 total_avg_ms=2.157
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=9680 tokens=964880 router_gate_avg_ms=0.073 routed_avg_ms=1.455 shared_avg_ms=0.252 all_reduce_avg_ms=0.365 total_avg_ms=2.157
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=9760 tokens=964960 router_gate_avg_ms=0.072 routed_avg_ms=1.448 shared_avg_ms=0.250 all_reduce_avg_ms=0.369 total_avg_ms=2.151
[PROF_MOE_SUMMARY] pid=33749 calls=9760 tokens=964960 router_gate_avg_ms=0.073 routed_avg_ms=1.445 shared_avg_ms=0.253 all_reduce_avg_ms=0.367 total_avg_ms=2.150
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=9760 tokens=964960 router_gate_avg_ms=0.073 routed_avg_ms=1.444 shared_avg_ms=0.254 all_reduce_avg_ms=0.368 total_avg_ms=2.151
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=9760 tokens=964960 router_gate_avg_ms=0.073 routed_avg_ms=1.450 shared_avg_ms=0.252 all_reduce_avg_ms=0.365 total_avg_ms=2.151
[PROF_MOE_SUMMARY] pid=33749 calls=9840 tokens=965040 router_gate_avg_ms=0.073 routed_avg_ms=1.440 shared_avg_ms=0.253 all_reduce_avg_ms=0.367 total_avg_ms=2.144
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=9840 tokens=965040 router_gate_avg_ms=0.072 routed_avg_ms=1.443 shared_avg_ms=0.250 all_reduce_avg_ms=0.369 total_avg_ms=2.145
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=9840 tokens=965040 router_gate_avg_ms=0.073 routed_avg_ms=1.439 shared_avg_ms=0.254 all_reduce_avg_ms=0.368 total_avg_ms=2.146
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=9840 tokens=965040 router_gate_avg_ms=0.073 routed_avg_ms=1.445 shared_avg_ms=0.252 all_reduce_avg_ms=0.365 total_avg_ms=2.146
[PROF_MOE_SUMMARY] pid=33749 calls=9920 tokens=965120 router_gate_avg_ms=0.073 routed_avg_ms=1.435 shared_avg_ms=0.253 all_reduce_avg_ms=0.367 total_avg_ms=2.139
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=9920 tokens=965120 router_gate_avg_ms=0.072 routed_avg_ms=1.438 shared_avg_ms=0.250 all_reduce_avg_ms=0.369 total_avg_ms=2.140
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=9920 tokens=965120 router_gate_avg_ms=0.073 routed_avg_ms=1.434 shared_avg_ms=0.254 all_reduce_avg_ms=0.367 total_avg_ms=2.140
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=9920 tokens=965120 router_gate_avg_ms=0.073 routed_avg_ms=1.440 shared_avg_ms=0.252 all_reduce_avg_ms=0.364 total_avg_ms=2.140
[PROF_MOE_SUMMARY] pid=33749 calls=10000 tokens=965200 router_gate_avg_ms=0.073 routed_avg_ms=1.430 shared_avg_ms=0.253 all_reduce_avg_ms=0.367 total_avg_ms=2.135
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=10000 tokens=965200 router_gate_avg_ms=0.073 routed_avg_ms=1.435 shared_avg_ms=0.252 all_reduce_avg_ms=0.364 total_avg_ms=2.135
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=10000 tokens=965200 router_gate_avg_ms=0.072 routed_avg_ms=1.433 shared_avg_ms=0.250 all_reduce_avg_ms=0.369 total_avg_ms=2.135
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=10000 tokens=965200 router_gate_avg_ms=0.073 routed_avg_ms=1.429 shared_avg_ms=0.254 all_reduce_avg_ms=0.368 total_avg_ms=2.136
[PROF_MOE_SUMMARY] pid=33749 calls=10080 tokens=965280 router_gate_avg_ms=0.073 routed_avg_ms=1.425 shared_avg_ms=0.253 all_reduce_avg_ms=0.367 total_avg_ms=2.130
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=10080 tokens=965280 router_gate_avg_ms=0.073 routed_avg_ms=1.424 shared_avg_ms=0.254 all_reduce_avg_ms=0.368 total_avg_ms=2.131
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=10080 tokens=965280 router_gate_avg_ms=0.072 routed_avg_ms=1.428 shared_avg_ms=0.250 all_reduce_avg_ms=0.369 total_avg_ms=2.130
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=10080 tokens=965280 router_gate_avg_ms=0.073 routed_avg_ms=1.430 shared_avg_ms=0.252 all_reduce_avg_ms=0.364 total_avg_ms=2.130
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=10160 tokens=965360 router_gate_avg_ms=0.072 routed_avg_ms=1.423 shared_avg_ms=0.250 all_reduce_avg_ms=0.369 total_avg_ms=2.126
[PROF_MOE_SUMMARY] pid=33749 calls=10160 tokens=965360 router_gate_avg_ms=0.073 routed_avg_ms=1.420 shared_avg_ms=0.253 all_reduce_avg_ms=0.368 total_avg_ms=2.125
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=10160 tokens=965360 router_gate_avg_ms=0.073 routed_avg_ms=1.426 shared_avg_ms=0.252 all_reduce_avg_ms=0.364 total_avg_ms=2.126
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=10160 tokens=965360 router_gate_avg_ms=0.073 routed_avg_ms=1.420 shared_avg_ms=0.254 all_reduce_avg_ms=0.367 total_avg_ms=2.126
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=10240 tokens=965440 router_gate_avg_ms=0.072 routed_avg_ms=1.418 shared_avg_ms=0.250 all_reduce_avg_ms=0.369 total_avg_ms=2.121
[PROF_MOE_SUMMARY] pid=33749 calls=10240 tokens=965440 router_gate_avg_ms=0.073 routed_avg_ms=1.415 shared_avg_ms=0.253 all_reduce_avg_ms=0.368 total_avg_ms=2.121
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=10240 tokens=965440 router_gate_avg_ms=0.073 routed_avg_ms=1.421 shared_avg_ms=0.252 all_reduce_avg_ms=0.364 total_avg_ms=2.121
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=10240 tokens=965440 router_gate_avg_ms=0.073 routed_avg_ms=1.415 shared_avg_ms=0.254 all_reduce_avg_ms=0.367 total_avg_ms=2.121
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=10320 tokens=1321320 router_gate_avg_ms=0.074 routed_avg_ms=1.664 shared_avg_ms=0.255 all_reduce_avg_ms=0.380 total_avg_ms=2.384
[PROF_MOE_SUMMARY] pid=33749 calls=10320 tokens=1321320 router_gate_avg_ms=0.075 routed_avg_ms=1.660 shared_avg_ms=0.258 all_reduce_avg_ms=0.378 total_avg_ms=2.383
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=10320 tokens=1321320 router_gate_avg_ms=0.075 routed_avg_ms=1.666 shared_avg_ms=0.257 all_reduce_avg_ms=0.374 total_avg_ms=2.384
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=10320 tokens=1321320 router_gate_avg_ms=0.075 routed_avg_ms=1.660 shared_avg_ms=0.260 all_reduce_avg_ms=0.378 total_avg_ms=2.384
[PROF_MOE_SUMMARY] pid=33749 calls=10400 tokens=2033040 router_gate_avg_ms=0.078 routed_avg_ms=2.134 shared_avg_ms=0.269 all_reduce_avg_ms=0.400 total_avg_ms=2.891
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=10400 tokens=2033040 router_gate_avg_ms=0.078 routed_avg_ms=2.141 shared_avg_ms=0.268 all_reduce_avg_ms=0.395 total_avg_ms=2.893
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=10400 tokens=2033040 router_gate_avg_ms=0.078 routed_avg_ms=2.133 shared_avg_ms=0.270 all_reduce_avg_ms=0.400 total_avg_ms=2.893
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=10400 tokens=2033040 router_gate_avg_ms=0.077 routed_avg_ms=2.138 shared_avg_ms=0.265 all_reduce_avg_ms=0.401 total_avg_ms=2.892
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=10480 tokens=2033120 router_gate_avg_ms=0.078 routed_avg_ms=2.123 shared_avg_ms=0.270 all_reduce_avg_ms=0.400 total_avg_ms=2.882
[PROF_MOE_SUMMARY] pid=33749 calls=10480 tokens=2033120 router_gate_avg_ms=0.077 routed_avg_ms=2.124 shared_avg_ms=0.269 all_reduce_avg_ms=0.399 total_avg_ms=2.881
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=10480 tokens=2033120 router_gate_avg_ms=0.077 routed_avg_ms=2.128 shared_avg_ms=0.265 all_reduce_avg_ms=0.401 total_avg_ms=2.882
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=10480 tokens=2033120 router_gate_avg_ms=0.077 routed_avg_ms=2.131 shared_avg_ms=0.268 all_reduce_avg_ms=0.395 total_avg_ms=2.882
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=10560 tokens=2033200 router_gate_avg_ms=0.077 routed_avg_ms=2.120 shared_avg_ms=0.267 all_reduce_avg_ms=0.395 total_avg_ms=2.871
[PROF_MOE_SUMMARY] pid=33749 calls=10560 tokens=2033200 router_gate_avg_ms=0.077 routed_avg_ms=2.114 shared_avg_ms=0.268 all_reduce_avg_ms=0.399 total_avg_ms=2.870
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=10560 tokens=2033200 router_gate_avg_ms=0.077 routed_avg_ms=2.118 shared_avg_ms=0.265 all_reduce_avg_ms=0.400 total_avg_ms=2.871
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=10560 tokens=2033200 router_gate_avg_ms=0.078 routed_avg_ms=2.113 shared_avg_ms=0.270 all_reduce_avg_ms=0.399 total_avg_ms=2.871
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=10640 tokens=2033280 router_gate_avg_ms=0.078 routed_avg_ms=2.103 shared_avg_ms=0.269 all_reduce_avg_ms=0.399 total_avg_ms=2.861
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=10640 tokens=2033280 router_gate_avg_ms=0.077 routed_avg_ms=2.110 shared_avg_ms=0.267 all_reduce_avg_ms=0.395 total_avg_ms=2.861
[PROF_MOE_SUMMARY] pid=33749 calls=10640 tokens=2033280 router_gate_avg_ms=0.077 routed_avg_ms=2.104 shared_avg_ms=0.268 all_reduce_avg_ms=0.398 total_avg_ms=2.859
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=10640 tokens=2033280 router_gate_avg_ms=0.077 routed_avg_ms=2.108 shared_avg_ms=0.265 all_reduce_avg_ms=0.400 total_avg_ms=2.861
[PROF_MOE_SUMMARY] pid=33749 calls=10720 tokens=2033360 router_gate_avg_ms=0.077 routed_avg_ms=2.094 shared_avg_ms=0.268 all_reduce_avg_ms=0.398 total_avg_ms=2.849
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=10720 tokens=2033360 router_gate_avg_ms=0.077 routed_avg_ms=2.098 shared_avg_ms=0.265 all_reduce_avg_ms=0.400 total_avg_ms=2.851
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=10720 tokens=2033360 router_gate_avg_ms=0.078 routed_avg_ms=2.093 shared_avg_ms=0.269 all_reduce_avg_ms=0.399 total_avg_ms=2.851
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=10720 tokens=2033360 router_gate_avg_ms=0.077 routed_avg_ms=2.101 shared_avg_ms=0.267 all_reduce_avg_ms=0.395 total_avg_ms=2.851
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=10800 tokens=2033440 router_gate_avg_ms=0.078 routed_avg_ms=2.084 shared_avg_ms=0.269 all_reduce_avg_ms=0.398 total_avg_ms=2.840
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=10800 tokens=2033440 router_gate_avg_ms=0.077 routed_avg_ms=2.091 shared_avg_ms=0.267 all_reduce_avg_ms=0.394 total_avg_ms=2.841
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=10800 tokens=2033440 router_gate_avg_ms=0.077 routed_avg_ms=2.088 shared_avg_ms=0.265 all_reduce_avg_ms=0.399 total_avg_ms=2.840
[PROF_MOE_SUMMARY] pid=33749 calls=10800 tokens=2033440 router_gate_avg_ms=0.077 routed_avg_ms=2.085 shared_avg_ms=0.268 all_reduce_avg_ms=0.398 total_avg_ms=2.839
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=10880 tokens=2033520 router_gate_avg_ms=0.077 routed_avg_ms=2.079 shared_avg_ms=0.264 all_reduce_avg_ms=0.399 total_avg_ms=2.830
[PROF_MOE_SUMMARY] pid=33749 calls=10880 tokens=2033520 router_gate_avg_ms=0.077 routed_avg_ms=2.075 shared_avg_ms=0.268 all_reduce_avg_ms=0.397 total_avg_ms=2.829
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=10880 tokens=2033520 router_gate_avg_ms=0.078 routed_avg_ms=2.074 shared_avg_ms=0.269 all_reduce_avg_ms=0.398 total_avg_ms=2.830
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=10880 tokens=2033520 router_gate_avg_ms=0.077 routed_avg_ms=2.082 shared_avg_ms=0.267 all_reduce_avg_ms=0.394 total_avg_ms=2.831
[PROF_MOE_SUMMARY] pid=33749 calls=10960 tokens=2033600 router_gate_avg_ms=0.077 routed_avg_ms=2.066 shared_avg_ms=0.268 all_reduce_avg_ms=0.396 total_avg_ms=2.819
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=10960 tokens=2033600 router_gate_avg_ms=0.078 routed_avg_ms=2.065 shared_avg_ms=0.269 all_reduce_avg_ms=0.398 total_avg_ms=2.820
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=10960 tokens=2033600 router_gate_avg_ms=0.077 routed_avg_ms=2.072 shared_avg_ms=0.266 all_reduce_avg_ms=0.394 total_avg_ms=2.821
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=10960 tokens=2033600 router_gate_avg_ms=0.077 routed_avg_ms=2.070 shared_avg_ms=0.264 all_reduce_avg_ms=0.399 total_avg_ms=2.820
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=11040 tokens=2033680 router_gate_avg_ms=0.078 routed_avg_ms=2.056 shared_avg_ms=0.268 all_reduce_avg_ms=0.397 total_avg_ms=2.810
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=11040 tokens=2033680 router_gate_avg_ms=0.077 routed_avg_ms=2.063 shared_avg_ms=0.266 all_reduce_avg_ms=0.393 total_avg_ms=2.811
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=11040 tokens=2033680 router_gate_avg_ms=0.077 routed_avg_ms=2.060 shared_avg_ms=0.264 all_reduce_avg_ms=0.398 total_avg_ms=2.810
[PROF_MOE_SUMMARY] pid=33749 calls=11040 tokens=2033680 router_gate_avg_ms=0.077 routed_avg_ms=2.057 shared_avg_ms=0.268 all_reduce_avg_ms=0.396 total_avg_ms=2.809
[PROF_MOE_SUMMARY] pid=33749 calls=11120 tokens=2033760 router_gate_avg_ms=0.077 routed_avg_ms=2.048 shared_avg_ms=0.267 all_reduce_avg_ms=0.395 total_avg_ms=2.799
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=11120 tokens=2033760 router_gate_avg_ms=0.077 routed_avg_ms=2.051 shared_avg_ms=0.264 all_reduce_avg_ms=0.398 total_avg_ms=2.801
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=11120 tokens=2033760 router_gate_avg_ms=0.077 routed_avg_ms=2.046 shared_avg_ms=0.268 all_reduce_avg_ms=0.397 total_avg_ms=2.801
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=11120 tokens=2033760 router_gate_avg_ms=0.077 routed_avg_ms=2.054 shared_avg_ms=0.266 all_reduce_avg_ms=0.393 total_avg_ms=2.801
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=11200 tokens=2033840 router_gate_avg_ms=0.076 routed_avg_ms=2.042 shared_avg_ms=0.264 all_reduce_avg_ms=0.397 total_avg_ms=2.791
[PROF_MOE_SUMMARY] pid=33749 calls=11200 tokens=2033840 router_gate_avg_ms=0.077 routed_avg_ms=2.039 shared_avg_ms=0.267 all_reduce_avg_ms=0.395 total_avg_ms=2.790
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=11200 tokens=2033840 router_gate_avg_ms=0.077 routed_avg_ms=2.037 shared_avg_ms=0.268 all_reduce_avg_ms=0.397 total_avg_ms=2.791
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=11200 tokens=2033840 router_gate_avg_ms=0.077 routed_avg_ms=2.045 shared_avg_ms=0.266 all_reduce_avg_ms=0.393 total_avg_ms=2.791
[PROF_MOE_SUMMARY] pid=33749 calls=11280 tokens=2033920 router_gate_avg_ms=0.077 routed_avg_ms=2.030 shared_avg_ms=0.267 all_reduce_avg_ms=0.395 total_avg_ms=2.780
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=11280 tokens=2033920 router_gate_avg_ms=0.076 routed_avg_ms=2.034 shared_avg_ms=0.263 all_reduce_avg_ms=0.397 total_avg_ms=2.782
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=11280 tokens=2033920 router_gate_avg_ms=0.077 routed_avg_ms=2.029 shared_avg_ms=0.268 all_reduce_avg_ms=0.397 total_avg_ms=2.782
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=11280 tokens=2033920 router_gate_avg_ms=0.077 routed_avg_ms=2.036 shared_avg_ms=0.266 all_reduce_avg_ms=0.392 total_avg_ms=2.782
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=11360 tokens=2034000 router_gate_avg_ms=0.077 routed_avg_ms=2.020 shared_avg_ms=0.268 all_reduce_avg_ms=0.396 total_avg_ms=2.773
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=11360 tokens=2034000 router_gate_avg_ms=0.077 routed_avg_ms=2.027 shared_avg_ms=0.265 all_reduce_avg_ms=0.392 total_avg_ms=2.773
[PROF_MOE_SUMMARY] pid=33749 calls=11360 tokens=2034000 router_gate_avg_ms=0.077 routed_avg_ms=2.021 shared_avg_ms=0.267 all_reduce_avg_ms=0.394 total_avg_ms=2.771
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=11360 tokens=2034000 router_gate_avg_ms=0.076 routed_avg_ms=2.025 shared_avg_ms=0.263 all_reduce_avg_ms=0.397 total_avg_ms=2.772
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=11440 tokens=2034080 router_gate_avg_ms=0.077 routed_avg_ms=2.019 shared_avg_ms=0.265 all_reduce_avg_ms=0.391 total_avg_ms=2.764
[PROF_MOE_SUMMARY] pid=33749 calls=11440 tokens=2034080 router_gate_avg_ms=0.077 routed_avg_ms=2.013 shared_avg_ms=0.267 all_reduce_avg_ms=0.394 total_avg_ms=2.762
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=11440 tokens=2034080 router_gate_avg_ms=0.076 routed_avg_ms=2.016 shared_avg_ms=0.263 all_reduce_avg_ms=0.396 total_avg_ms=2.763
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=11440 tokens=2034080 router_gate_avg_ms=0.077 routed_avg_ms=2.012 shared_avg_ms=0.268 all_reduce_avg_ms=0.396 total_avg_ms=2.763
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=11520 tokens=2034160 router_gate_avg_ms=0.077 routed_avg_ms=2.003 shared_avg_ms=0.267 all_reduce_avg_ms=0.395 total_avg_ms=2.754
[PROF_MOE_SUMMARY] pid=33749 calls=11520 tokens=2034160 router_gate_avg_ms=0.077 routed_avg_ms=2.004 shared_avg_ms=0.266 all_reduce_avg_ms=0.394 total_avg_ms=2.753
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=11520 tokens=2034160 router_gate_avg_ms=0.077 routed_avg_ms=2.010 shared_avg_ms=0.265 all_reduce_avg_ms=0.391 total_avg_ms=2.754
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=11520 tokens=2034160 router_gate_avg_ms=0.076 routed_avg_ms=2.008 shared_avg_ms=0.263 all_reduce_avg_ms=0.396 total_avg_ms=2.754
[PROF_MOE_SUMMARY] pid=33749 calls=11600 tokens=2034240 router_gate_avg_ms=0.077 routed_avg_ms=1.996 shared_avg_ms=0.266 all_reduce_avg_ms=0.393 total_avg_ms=2.744
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=11600 tokens=2034240 router_gate_avg_ms=0.077 routed_avg_ms=2.002 shared_avg_ms=0.265 all_reduce_avg_ms=0.391 total_avg_ms=2.745
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=11600 tokens=2034240 router_gate_avg_ms=0.077 routed_avg_ms=1.995 shared_avg_ms=0.267 all_reduce_avg_ms=0.395 total_avg_ms=2.745
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=11600 tokens=2034240 router_gate_avg_ms=0.076 routed_avg_ms=1.999 shared_avg_ms=0.263 all_reduce_avg_ms=0.396 total_avg_ms=2.745
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=11680 tokens=2034320 router_gate_avg_ms=0.076 routed_avg_ms=1.991 shared_avg_ms=0.262 all_reduce_avg_ms=0.395 total_avg_ms=2.736
[PROF_MOE_SUMMARY] pid=33749 calls=11680 tokens=2034320 router_gate_avg_ms=0.077 routed_avg_ms=1.988 shared_avg_ms=0.266 all_reduce_avg_ms=0.393 total_avg_ms=2.735
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=11680 tokens=2034320 router_gate_avg_ms=0.077 routed_avg_ms=1.986 shared_avg_ms=0.267 all_reduce_avg_ms=0.395 total_avg_ms=2.736
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=11680 tokens=2034320 router_gate_avg_ms=0.077 routed_avg_ms=1.994 shared_avg_ms=0.265 all_reduce_avg_ms=0.390 total_avg_ms=2.737
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=11760 tokens=2034400 router_gate_avg_ms=0.077 routed_avg_ms=1.978 shared_avg_ms=0.267 all_reduce_avg_ms=0.394 total_avg_ms=2.728
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=11760 tokens=2034400 router_gate_avg_ms=0.077 routed_avg_ms=1.986 shared_avg_ms=0.264 all_reduce_avg_ms=0.390 total_avg_ms=2.728
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=11760 tokens=2034400 router_gate_avg_ms=0.076 routed_avg_ms=1.983 shared_avg_ms=0.262 all_reduce_avg_ms=0.395 total_avg_ms=2.728
[PROF_MOE_SUMMARY] pid=33749 calls=11760 tokens=2034400 router_gate_avg_ms=0.077 routed_avg_ms=1.980 shared_avg_ms=0.266 all_reduce_avg_ms=0.392 total_avg_ms=2.726
[PROF_MOE_SUMMARY] pid=33749 calls=11840 tokens=2034480 router_gate_avg_ms=0.077 routed_avg_ms=1.972 shared_avg_ms=0.266 all_reduce_avg_ms=0.392 total_avg_ms=2.718
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=11840 tokens=2034480 router_gate_avg_ms=0.077 routed_avg_ms=1.970 shared_avg_ms=0.267 all_reduce_avg_ms=0.394 total_avg_ms=2.719
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=11840 tokens=2034480 router_gate_avg_ms=0.076 routed_avg_ms=1.978 shared_avg_ms=0.264 all_reduce_avg_ms=0.390 total_avg_ms=2.720
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=11840 tokens=2034480 router_gate_avg_ms=0.076 routed_avg_ms=1.975 shared_avg_ms=0.262 all_reduce_avg_ms=0.395 total_avg_ms=2.719
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=11920 tokens=2034560 router_gate_avg_ms=0.077 routed_avg_ms=1.962 shared_avg_ms=0.266 all_reduce_avg_ms=0.394 total_avg_ms=2.711
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=11920 tokens=2034560 router_gate_avg_ms=0.076 routed_avg_ms=1.970 shared_avg_ms=0.264 all_reduce_avg_ms=0.390 total_avg_ms=2.711
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=11920 tokens=2034560 router_gate_avg_ms=0.076 routed_avg_ms=1.967 shared_avg_ms=0.262 all_reduce_avg_ms=0.394 total_avg_ms=2.711
[PROF_MOE_SUMMARY] pid=33749 calls=11920 tokens=2034560 router_gate_avg_ms=0.077 routed_avg_ms=1.964 shared_avg_ms=0.266 all_reduce_avg_ms=0.391 total_avg_ms=2.709
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=12000 tokens=2034640 router_gate_avg_ms=0.076 routed_avg_ms=1.959 shared_avg_ms=0.262 all_reduce_avg_ms=0.394 total_avg_ms=2.702
[PROF_MOE_SUMMARY] pid=33749 calls=12000 tokens=2034640 router_gate_avg_ms=0.076 routed_avg_ms=1.956 shared_avg_ms=0.265 all_reduce_avg_ms=0.391 total_avg_ms=2.700
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=12000 tokens=2034640 router_gate_avg_ms=0.077 routed_avg_ms=1.955 shared_avg_ms=0.266 all_reduce_avg_ms=0.393 total_avg_ms=2.702
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=12000 tokens=2034640 router_gate_avg_ms=0.076 routed_avg_ms=1.962 shared_avg_ms=0.264 all_reduce_avg_ms=0.389 total_avg_ms=2.702
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=12080 tokens=2034720 router_gate_avg_ms=0.077 routed_avg_ms=1.947 shared_avg_ms=0.266 all_reduce_avg_ms=0.393 total_avg_ms=2.694
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=12080 tokens=2034720 router_gate_avg_ms=0.076 routed_avg_ms=1.954 shared_avg_ms=0.264 all_reduce_avg_ms=0.389 total_avg_ms=2.694
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=12080 tokens=2034720 router_gate_avg_ms=0.076 routed_avg_ms=1.951 shared_avg_ms=0.261 all_reduce_avg_ms=0.394 total_avg_ms=2.694
[PROF_MOE_SUMMARY] pid=33749 calls=12080 tokens=2034720 router_gate_avg_ms=0.076 routed_avg_ms=1.948 shared_avg_ms=0.265 all_reduce_avg_ms=0.390 total_avg_ms=2.692
[PROF_MOE_SUMMARY] pid=33749 calls=12160 tokens=2034800 router_gate_avg_ms=0.076 routed_avg_ms=1.941 shared_avg_ms=0.265 all_reduce_avg_ms=0.390 total_avg_ms=2.684
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=12160 tokens=2034800 router_gate_avg_ms=0.076 routed_avg_ms=1.944 shared_avg_ms=0.261 all_reduce_avg_ms=0.393 total_avg_ms=2.686
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=12160 tokens=2034800 router_gate_avg_ms=0.077 routed_avg_ms=1.939 shared_avg_ms=0.266 all_reduce_avg_ms=0.392 total_avg_ms=2.686
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=12160 tokens=2034800 router_gate_avg_ms=0.076 routed_avg_ms=1.946 shared_avg_ms=0.264 all_reduce_avg_ms=0.388 total_avg_ms=2.686
[PROF_MOE_SUMMARY] pid=33749 calls=12240 tokens=2034880 router_gate_avg_ms=0.076 routed_avg_ms=1.933 shared_avg_ms=0.265 all_reduce_avg_ms=0.390 total_avg_ms=2.676
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=12240 tokens=2034880 router_gate_avg_ms=0.077 routed_avg_ms=1.932 shared_avg_ms=0.266 all_reduce_avg_ms=0.392 total_avg_ms=2.678
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=12240 tokens=2034880 router_gate_avg_ms=0.076 routed_avg_ms=1.939 shared_avg_ms=0.263 all_reduce_avg_ms=0.388 total_avg_ms=2.678
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=12240 tokens=2034880 router_gate_avg_ms=0.076 routed_avg_ms=1.936 shared_avg_ms=0.261 all_reduce_avg_ms=0.393 total_avg_ms=2.678
[PROF_MOE_SUMMARY] pid=33749 calls=12320 tokens=2034960 router_gate_avg_ms=0.076 routed_avg_ms=1.926 shared_avg_ms=0.265 all_reduce_avg_ms=0.389 total_avg_ms=2.668
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=12320 tokens=2034960 router_gate_avg_ms=0.077 routed_avg_ms=1.925 shared_avg_ms=0.266 all_reduce_avg_ms=0.392 total_avg_ms=2.670
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=12320 tokens=2034960 router_gate_avg_ms=0.076 routed_avg_ms=1.932 shared_avg_ms=0.263 all_reduce_avg_ms=0.388 total_avg_ms=2.670
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=12320 tokens=2034960 router_gate_avg_ms=0.076 routed_avg_ms=1.929 shared_avg_ms=0.261 all_reduce_avg_ms=0.393 total_avg_ms=2.670
[PROF_MOE_SUMMARY] pid=33749 calls=12400 tokens=2035040 router_gate_avg_ms=0.076 routed_avg_ms=1.919 shared_avg_ms=0.265 all_reduce_avg_ms=0.389 total_avg_ms=2.660
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=12400 tokens=2035040 router_gate_avg_ms=0.077 routed_avg_ms=1.917 shared_avg_ms=0.265 all_reduce_avg_ms=0.391 total_avg_ms=2.662
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=12400 tokens=2035040 router_gate_avg_ms=0.076 routed_avg_ms=1.924 shared_avg_ms=0.263 all_reduce_avg_ms=0.387 total_avg_ms=2.662
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=12400 tokens=2035040 router_gate_avg_ms=0.076 routed_avg_ms=1.922 shared_avg_ms=0.261 all_reduce_avg_ms=0.392 total_avg_ms=2.662
[PROF_MOE_SUMMARY] pid=33749 calls=12480 tokens=2035120 router_gate_avg_ms=0.076 routed_avg_ms=1.911 shared_avg_ms=0.265 all_reduce_avg_ms=0.388 total_avg_ms=2.652
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=12480 tokens=2035120 router_gate_avg_ms=0.077 routed_avg_ms=1.910 shared_avg_ms=0.265 all_reduce_avg_ms=0.391 total_avg_ms=2.654
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=12480 tokens=2035120 router_gate_avg_ms=0.076 routed_avg_ms=1.915 shared_avg_ms=0.261 all_reduce_avg_ms=0.392 total_avg_ms=2.654
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=12480 tokens=2035120 router_gate_avg_ms=0.076 routed_avg_ms=1.917 shared_avg_ms=0.263 all_reduce_avg_ms=0.387 total_avg_ms=2.654
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=12560 tokens=2035200 router_gate_avg_ms=0.076 routed_avg_ms=1.907 shared_avg_ms=0.261 all_reduce_avg_ms=0.391 total_avg_ms=2.646
[PROF_MOE_SUMMARY] pid=33749 calls=12560 tokens=2035200 router_gate_avg_ms=0.076 routed_avg_ms=1.904 shared_avg_ms=0.264 all_reduce_avg_ms=0.388 total_avg_ms=2.645
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=12560 tokens=2035200 router_gate_avg_ms=0.076 routed_avg_ms=1.903 shared_avg_ms=0.265 all_reduce_avg_ms=0.390 total_avg_ms=2.646
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=12560 tokens=2035200 router_gate_avg_ms=0.076 routed_avg_ms=1.910 shared_avg_ms=0.263 all_reduce_avg_ms=0.387 total_avg_ms=2.647
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=12640 tokens=2035280 router_gate_avg_ms=0.076 routed_avg_ms=1.903 shared_avg_ms=0.263 all_reduce_avg_ms=0.386 total_avg_ms=2.639
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=12640 tokens=2035280 router_gate_avg_ms=0.076 routed_avg_ms=1.900 shared_avg_ms=0.260 all_reduce_avg_ms=0.391 total_avg_ms=2.639
[PROF_MOE_SUMMARY] pid=33749 calls=12640 tokens=2035280 router_gate_avg_ms=0.076 routed_avg_ms=1.897 shared_avg_ms=0.264 all_reduce_avg_ms=0.388 total_avg_ms=2.637
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=12640 tokens=2035280 router_gate_avg_ms=0.076 routed_avg_ms=1.896 shared_avg_ms=0.265 all_reduce_avg_ms=0.390 total_avg_ms=2.639
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=12720 tokens=2035360 router_gate_avg_ms=0.076 routed_avg_ms=1.896 shared_avg_ms=0.262 all_reduce_avg_ms=0.386 total_avg_ms=2.632
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=12720 tokens=2035360 router_gate_avg_ms=0.076 routed_avg_ms=1.893 shared_avg_ms=0.260 all_reduce_avg_ms=0.391 total_avg_ms=2.632
[PROF_MOE_SUMMARY] pid=33749 calls=12720 tokens=2035360 router_gate_avg_ms=0.076 routed_avg_ms=1.890 shared_avg_ms=0.264 all_reduce_avg_ms=0.387 total_avg_ms=2.630
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=12720 tokens=2035360 router_gate_avg_ms=0.076 routed_avg_ms=1.889 shared_avg_ms=0.265 all_reduce_avg_ms=0.390 total_avg_ms=2.632
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=12800 tokens=2035440 router_gate_avg_ms=0.076 routed_avg_ms=1.882 shared_avg_ms=0.265 all_reduce_avg_ms=0.390 total_avg_ms=2.624
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=12800 tokens=2035440 router_gate_avg_ms=0.076 routed_avg_ms=1.889 shared_avg_ms=0.262 all_reduce_avg_ms=0.386 total_avg_ms=2.624
[PROF_MOE_SUMMARY] pid=33749 calls=12800 tokens=2035440 router_gate_avg_ms=0.076 routed_avg_ms=1.884 shared_avg_ms=0.264 all_reduce_avg_ms=0.387 total_avg_ms=2.622
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=12800 tokens=2035440 router_gate_avg_ms=0.076 routed_avg_ms=1.887 shared_avg_ms=0.260 all_reduce_avg_ms=0.391 total_avg_ms=2.624
[PROF_MOE_SUMMARY] pid=33749 calls=12880 tokens=2035520 router_gate_avg_ms=0.076 routed_avg_ms=1.877 shared_avg_ms=0.264 all_reduce_avg_ms=0.387 total_avg_ms=2.615
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=12880 tokens=2035520 router_gate_avg_ms=0.076 routed_avg_ms=1.880 shared_avg_ms=0.260 all_reduce_avg_ms=0.390 total_avg_ms=2.617
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=12880 tokens=2035520 router_gate_avg_ms=0.076 routed_avg_ms=1.875 shared_avg_ms=0.265 all_reduce_avg_ms=0.389 total_avg_ms=2.617
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=12880 tokens=2035520 router_gate_avg_ms=0.076 routed_avg_ms=1.882 shared_avg_ms=0.262 all_reduce_avg_ms=0.385 total_avg_ms=2.617
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=12960 tokens=2035600 router_gate_avg_ms=0.076 routed_avg_ms=1.876 shared_avg_ms=0.262 all_reduce_avg_ms=0.385 total_avg_ms=2.610
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=12960 tokens=2035600 router_gate_avg_ms=0.076 routed_avg_ms=1.873 shared_avg_ms=0.260 all_reduce_avg_ms=0.390 total_avg_ms=2.610
[PROF_MOE_SUMMARY] pid=33749 calls=12960 tokens=2035600 router_gate_avg_ms=0.076 routed_avg_ms=1.870 shared_avg_ms=0.264 all_reduce_avg_ms=0.387 total_avg_ms=2.608
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=12960 tokens=2035600 router_gate_avg_ms=0.076 routed_avg_ms=1.869 shared_avg_ms=0.264 all_reduce_avg_ms=0.389 total_avg_ms=2.610
[PROF_MOE_SUMMARY] pid=33749 calls=13040 tokens=2035680 router_gate_avg_ms=0.076 routed_avg_ms=1.864 shared_avg_ms=0.264 all_reduce_avg_ms=0.387 total_avg_ms=2.602
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=13040 tokens=2035680 router_gate_avg_ms=0.076 routed_avg_ms=1.867 shared_avg_ms=0.260 all_reduce_avg_ms=0.390 total_avg_ms=2.604
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=13040 tokens=2035680 router_gate_avg_ms=0.076 routed_avg_ms=1.863 shared_avg_ms=0.264 all_reduce_avg_ms=0.389 total_avg_ms=2.604
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=13040 tokens=2035680 router_gate_avg_ms=0.076 routed_avg_ms=1.869 shared_avg_ms=0.263 all_reduce_avg_ms=0.385 total_avg_ms=2.604
[PROF_MOE_SUMMARY] pid=33749 calls=13120 tokens=2035760 router_gate_avg_ms=0.076 routed_avg_ms=1.857 shared_avg_ms=0.264 all_reduce_avg_ms=0.387 total_avg_ms=2.595
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=13120 tokens=2035760 router_gate_avg_ms=0.076 routed_avg_ms=1.860 shared_avg_ms=0.260 all_reduce_avg_ms=0.390 total_avg_ms=2.597
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=13120 tokens=2035760 router_gate_avg_ms=0.076 routed_avg_ms=1.863 shared_avg_ms=0.262 all_reduce_avg_ms=0.385 total_avg_ms=2.597
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=13120 tokens=2035760 router_gate_avg_ms=0.076 routed_avg_ms=1.857 shared_avg_ms=0.264 all_reduce_avg_ms=0.388 total_avg_ms=2.597
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=13200 tokens=2035840 router_gate_avg_ms=0.076 routed_avg_ms=1.856 shared_avg_ms=0.262 all_reduce_avg_ms=0.385 total_avg_ms=2.590
[PROF_MOE_SUMMARY] pid=33749 calls=13200 tokens=2035840 router_gate_avg_ms=0.076 routed_avg_ms=1.851 shared_avg_ms=0.264 all_reduce_avg_ms=0.386 total_avg_ms=2.588
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=13200 tokens=2035840 router_gate_avg_ms=0.076 routed_avg_ms=1.854 shared_avg_ms=0.260 all_reduce_avg_ms=0.389 total_avg_ms=2.590
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=13200 tokens=2035840 router_gate_avg_ms=0.076 routed_avg_ms=1.850 shared_avg_ms=0.264 all_reduce_avg_ms=0.388 total_avg_ms=2.590
[PROF_MOE_SUMMARY] pid=33749 calls=13280 tokens=2035920 router_gate_avg_ms=0.076 routed_avg_ms=1.844 shared_avg_ms=0.264 all_reduce_avg_ms=0.386 total_avg_ms=2.582
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=13280 tokens=2035920 router_gate_avg_ms=0.075 routed_avg_ms=1.847 shared_avg_ms=0.260 all_reduce_avg_ms=0.389 total_avg_ms=2.583
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=13280 tokens=2035920 router_gate_avg_ms=0.076 routed_avg_ms=1.844 shared_avg_ms=0.264 all_reduce_avg_ms=0.388 total_avg_ms=2.583
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=13280 tokens=2035920 router_gate_avg_ms=0.076 routed_avg_ms=1.850 shared_avg_ms=0.262 all_reduce_avg_ms=0.385 total_avg_ms=2.584
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=13360 tokens=2036000 router_gate_avg_ms=0.076 routed_avg_ms=1.838 shared_avg_ms=0.264 all_reduce_avg_ms=0.388 total_avg_ms=2.577
[PROF_MOE_SUMMARY] pid=33749 calls=13360 tokens=2036000 router_gate_avg_ms=0.076 routed_avg_ms=1.838 shared_avg_ms=0.264 all_reduce_avg_ms=0.385 total_avg_ms=2.575
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=13360 tokens=2036000 router_gate_avg_ms=0.076 routed_avg_ms=1.843 shared_avg_ms=0.262 all_reduce_avg_ms=0.384 total_avg_ms=2.577
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=13360 tokens=2036000 router_gate_avg_ms=0.075 routed_avg_ms=1.841 shared_avg_ms=0.260 all_reduce_avg_ms=0.389 total_avg_ms=2.576
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=13440 tokens=2036080 router_gate_avg_ms=0.075 routed_avg_ms=1.835 shared_avg_ms=0.260 all_reduce_avg_ms=0.388 total_avg_ms=2.570
[PROF_MOE_SUMMARY] pid=33749 calls=13440 tokens=2036080 router_gate_avg_ms=0.076 routed_avg_ms=1.832 shared_avg_ms=0.264 all_reduce_avg_ms=0.385 total_avg_ms=2.568
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=13440 tokens=2036080 router_gate_avg_ms=0.076 routed_avg_ms=1.831 shared_avg_ms=0.264 all_reduce_avg_ms=0.387 total_avg_ms=2.570
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=13440 tokens=2036080 router_gate_avg_ms=0.076 routed_avg_ms=1.837 shared_avg_ms=0.262 all_reduce_avg_ms=0.384 total_avg_ms=2.570
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=13520 tokens=2036160 router_gate_avg_ms=0.076 routed_avg_ms=1.825 shared_avg_ms=0.264 all_reduce_avg_ms=0.387 total_avg_ms=2.563
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=13520 tokens=2036160 router_gate_avg_ms=0.076 routed_avg_ms=1.831 shared_avg_ms=0.262 all_reduce_avg_ms=0.384 total_avg_ms=2.563
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=13520 tokens=2036160 router_gate_avg_ms=0.075 routed_avg_ms=1.829 shared_avg_ms=0.260 all_reduce_avg_ms=0.388 total_avg_ms=2.563
[PROF_MOE_SUMMARY] pid=33749 calls=13520 tokens=2036160 router_gate_avg_ms=0.076 routed_avg_ms=1.826 shared_avg_ms=0.264 all_reduce_avg_ms=0.385 total_avg_ms=2.561
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=13600 tokens=2036240 router_gate_avg_ms=0.076 routed_avg_ms=1.825 shared_avg_ms=0.262 all_reduce_avg_ms=0.384 total_avg_ms=2.557
[PROF_MOE_SUMMARY] pid=33749 calls=13600 tokens=2036240 router_gate_avg_ms=0.076 routed_avg_ms=1.820 shared_avg_ms=0.264 all_reduce_avg_ms=0.384 total_avg_ms=2.555
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=13600 tokens=2036240 router_gate_avg_ms=0.075 routed_avg_ms=1.823 shared_avg_ms=0.259 all_reduce_avg_ms=0.388 total_avg_ms=2.557
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=13600 tokens=2036240 router_gate_avg_ms=0.076 routed_avg_ms=1.819 shared_avg_ms=0.263 all_reduce_avg_ms=0.387 total_avg_ms=2.557
[PROF_MOE_SUMMARY] pid=33749 calls=13680 tokens=2036320 router_gate_avg_ms=0.076 routed_avg_ms=1.814 shared_avg_ms=0.263 all_reduce_avg_ms=0.384 total_avg_ms=2.549
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=13680 tokens=2036320 router_gate_avg_ms=0.075 routed_avg_ms=1.817 shared_avg_ms=0.259 all_reduce_avg_ms=0.388 total_avg_ms=2.551
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=13680 tokens=2036320 router_gate_avg_ms=0.076 routed_avg_ms=1.813 shared_avg_ms=0.263 all_reduce_avg_ms=0.387 total_avg_ms=2.551
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=13680 tokens=2036320 router_gate_avg_ms=0.076 routed_avg_ms=1.819 shared_avg_ms=0.262 all_reduce_avg_ms=0.384 total_avg_ms=2.551
[PROF_MOE_SUMMARY] pid=33749 calls=13760 tokens=2036400 router_gate_avg_ms=0.076 routed_avg_ms=1.808 shared_avg_ms=0.263 all_reduce_avg_ms=0.384 total_avg_ms=2.543
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=13760 tokens=2036400 router_gate_avg_ms=0.076 routed_avg_ms=1.807 shared_avg_ms=0.263 all_reduce_avg_ms=0.387 total_avg_ms=2.545
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=13760 tokens=2036400 router_gate_avg_ms=0.076 routed_avg_ms=1.813 shared_avg_ms=0.261 all_reduce_avg_ms=0.383 total_avg_ms=2.545
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=13760 tokens=2036400 router_gate_avg_ms=0.075 routed_avg_ms=1.811 shared_avg_ms=0.259 all_reduce_avg_ms=0.388 total_avg_ms=2.545
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=13840 tokens=2036480 router_gate_avg_ms=0.076 routed_avg_ms=1.801 shared_avg_ms=0.263 all_reduce_avg_ms=0.386 total_avg_ms=2.538
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=13840 tokens=2036480 router_gate_avg_ms=0.076 routed_avg_ms=1.807 shared_avg_ms=0.261 all_reduce_avg_ms=0.383 total_avg_ms=2.539
[PROF_MOE_SUMMARY] pid=33749 calls=13840 tokens=2036480 router_gate_avg_ms=0.076 routed_avg_ms=1.802 shared_avg_ms=0.263 all_reduce_avg_ms=0.384 total_avg_ms=2.536
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=13840 tokens=2036480 router_gate_avg_ms=0.075 routed_avg_ms=1.805 shared_avg_ms=0.259 all_reduce_avg_ms=0.387 total_avg_ms=2.538
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=13920 tokens=2036560 router_gate_avg_ms=0.075 routed_avg_ms=1.799 shared_avg_ms=0.259 all_reduce_avg_ms=0.387 total_avg_ms=2.532
[PROF_MOE_SUMMARY] pid=33749 calls=13920 tokens=2036560 router_gate_avg_ms=0.076 routed_avg_ms=1.796 shared_avg_ms=0.263 all_reduce_avg_ms=0.384 total_avg_ms=2.530
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=13920 tokens=2036560 router_gate_avg_ms=0.076 routed_avg_ms=1.796 shared_avg_ms=0.263 all_reduce_avg_ms=0.386 total_avg_ms=2.532
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=13920 tokens=2036560 router_gate_avg_ms=0.076 routed_avg_ms=1.802 shared_avg_ms=0.261 all_reduce_avg_ms=0.383 total_avg_ms=2.532
[PROF_MOE_SUMMARY] pid=33749 calls=14000 tokens=2036640 router_gate_avg_ms=0.076 routed_avg_ms=1.791 shared_avg_ms=0.263 all_reduce_avg_ms=0.383 total_avg_ms=2.524
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=14000 tokens=2036640 router_gate_avg_ms=0.075 routed_avg_ms=1.794 shared_avg_ms=0.259 all_reduce_avg_ms=0.387 total_avg_ms=2.526
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=14000 tokens=2036640 router_gate_avg_ms=0.076 routed_avg_ms=1.790 shared_avg_ms=0.263 all_reduce_avg_ms=0.386 total_avg_ms=2.526
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=14000 tokens=2036640 router_gate_avg_ms=0.076 routed_avg_ms=1.796 shared_avg_ms=0.261 all_reduce_avg_ms=0.383 total_avg_ms=2.527
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=14080 tokens=2036720 router_gate_avg_ms=0.076 routed_avg_ms=1.790 shared_avg_ms=0.261 all_reduce_avg_ms=0.382 total_avg_ms=2.520
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=14080 tokens=2036720 router_gate_avg_ms=0.076 routed_avg_ms=1.784 shared_avg_ms=0.263 all_reduce_avg_ms=0.386 total_avg_ms=2.520
[PROF_MOE_SUMMARY] pid=33749 calls=14080 tokens=2036720 router_gate_avg_ms=0.076 routed_avg_ms=1.785 shared_avg_ms=0.263 all_reduce_avg_ms=0.383 total_avg_ms=2.518
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=14080 tokens=2036720 router_gate_avg_ms=0.075 routed_avg_ms=1.788 shared_avg_ms=0.259 all_reduce_avg_ms=0.387 total_avg_ms=2.520
[PROF_MOE_SUMMARY] pid=33749 calls=14160 tokens=2036800 router_gate_avg_ms=0.076 routed_avg_ms=1.779 shared_avg_ms=0.263 all_reduce_avg_ms=0.383 total_avg_ms=2.513
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=14160 tokens=2036800 router_gate_avg_ms=0.075 routed_avg_ms=1.785 shared_avg_ms=0.261 all_reduce_avg_ms=0.382 total_avg_ms=2.515
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=14160 tokens=2036800 router_gate_avg_ms=0.076 routed_avg_ms=1.779 shared_avg_ms=0.263 all_reduce_avg_ms=0.386 total_avg_ms=2.515
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=14160 tokens=2036800 router_gate_avg_ms=0.075 routed_avg_ms=1.782 shared_avg_ms=0.259 all_reduce_avg_ms=0.387 total_avg_ms=2.514
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=14240 tokens=2036880 router_gate_avg_ms=0.075 routed_avg_ms=1.779 shared_avg_ms=0.261 all_reduce_avg_ms=0.382 total_avg_ms=2.509
[PROF_MOE_SUMMARY] pid=33749 calls=14240 tokens=2036880 router_gate_avg_ms=0.076 routed_avg_ms=1.774 shared_avg_ms=0.263 all_reduce_avg_ms=0.383 total_avg_ms=2.507
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=14240 tokens=2036880 router_gate_avg_ms=0.075 routed_avg_ms=1.777 shared_avg_ms=0.259 all_reduce_avg_ms=0.386 total_avg_ms=2.508
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=14240 tokens=2036880 router_gate_avg_ms=0.076 routed_avg_ms=1.773 shared_avg_ms=0.263 all_reduce_avg_ms=0.386 total_avg_ms=2.509
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=14320 tokens=2036960 router_gate_avg_ms=0.075 routed_avg_ms=1.774 shared_avg_ms=0.261 all_reduce_avg_ms=0.382 total_avg_ms=2.503
[PROF_MOE_SUMMARY] pid=33749 calls=14320 tokens=2036960 router_gate_avg_ms=0.076 routed_avg_ms=1.768 shared_avg_ms=0.263 all_reduce_avg_ms=0.383 total_avg_ms=2.501
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=14320 tokens=2036960 router_gate_avg_ms=0.075 routed_avg_ms=1.772 shared_avg_ms=0.259 all_reduce_avg_ms=0.386 total_avg_ms=2.502
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=14320 tokens=2036960 router_gate_avg_ms=0.076 routed_avg_ms=1.768 shared_avg_ms=0.262 all_reduce_avg_ms=0.385 total_avg_ms=2.503
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=14400 tokens=2037040 router_gate_avg_ms=0.075 routed_avg_ms=1.766 shared_avg_ms=0.258 all_reduce_avg_ms=0.386 total_avg_ms=2.497
[PROF_MOE_SUMMARY] pid=33749 calls=14400 tokens=2037040 router_gate_avg_ms=0.076 routed_avg_ms=1.763 shared_avg_ms=0.262 all_reduce_avg_ms=0.383 total_avg_ms=2.495
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=14400 tokens=2037040 router_gate_avg_ms=0.076 routed_avg_ms=1.762 shared_avg_ms=0.262 all_reduce_avg_ms=0.385 total_avg_ms=2.497
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=14400 tokens=2037040 router_gate_avg_ms=0.075 routed_avg_ms=1.768 shared_avg_ms=0.260 all_reduce_avg_ms=0.382 total_avg_ms=2.497
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=14480 tokens=2037120 router_gate_avg_ms=0.075 routed_avg_ms=1.761 shared_avg_ms=0.258 all_reduce_avg_ms=0.385 total_avg_ms=2.491
[PROF_MOE_SUMMARY] pid=33749 calls=14480 tokens=2037120 router_gate_avg_ms=0.075 routed_avg_ms=1.758 shared_avg_ms=0.262 all_reduce_avg_ms=0.382 total_avg_ms=2.489
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=14480 tokens=2037120 router_gate_avg_ms=0.076 routed_avg_ms=1.757 shared_avg_ms=0.262 all_reduce_avg_ms=0.385 total_avg_ms=2.491
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=14480 tokens=2037120 router_gate_avg_ms=0.075 routed_avg_ms=1.763 shared_avg_ms=0.260 all_reduce_avg_ms=0.381 total_avg_ms=2.491
[PROF_MOE_SUMMARY] pid=33749 calls=14560 tokens=2037200 router_gate_avg_ms=0.075 routed_avg_ms=1.752 shared_avg_ms=0.262 all_reduce_avg_ms=0.382 total_avg_ms=2.484
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=14560 tokens=2037200 router_gate_avg_ms=0.075 routed_avg_ms=1.755 shared_avg_ms=0.258 all_reduce_avg_ms=0.385 total_avg_ms=2.485
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=14560 tokens=2037200 router_gate_avg_ms=0.076 routed_avg_ms=1.752 shared_avg_ms=0.262 all_reduce_avg_ms=0.385 total_avg_ms=2.486
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=14560 tokens=2037200 router_gate_avg_ms=0.075 routed_avg_ms=1.758 shared_avg_ms=0.260 all_reduce_avg_ms=0.381 total_avg_ms=2.486
[PROF_MOE_SUMMARY] pid=33749 calls=14640 tokens=2037280 router_gate_avg_ms=0.075 routed_avg_ms=1.747 shared_avg_ms=0.262 all_reduce_avg_ms=0.382 total_avg_ms=2.478
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=14640 tokens=2037280 router_gate_avg_ms=0.076 routed_avg_ms=1.747 shared_avg_ms=0.262 all_reduce_avg_ms=0.384 total_avg_ms=2.480
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=14640 tokens=2037280 router_gate_avg_ms=0.075 routed_avg_ms=1.752 shared_avg_ms=0.260 all_reduce_avg_ms=0.381 total_avg_ms=2.480
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=14640 tokens=2037280 router_gate_avg_ms=0.075 routed_avg_ms=1.750 shared_avg_ms=0.258 all_reduce_avg_ms=0.385 total_avg_ms=2.480
[PROF_MOE_SUMMARY] pid=33749 calls=14720 tokens=2037360 router_gate_avg_ms=0.075 routed_avg_ms=1.742 shared_avg_ms=0.262 all_reduce_avg_ms=0.382 total_avg_ms=2.473
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=14720 tokens=2037360 router_gate_avg_ms=0.076 routed_avg_ms=1.741 shared_avg_ms=0.262 all_reduce_avg_ms=0.384 total_avg_ms=2.475
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=14720 tokens=2037360 router_gate_avg_ms=0.075 routed_avg_ms=1.747 shared_avg_ms=0.260 all_reduce_avg_ms=0.381 total_avg_ms=2.475
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=14720 tokens=2037360 router_gate_avg_ms=0.075 routed_avg_ms=1.745 shared_avg_ms=0.258 all_reduce_avg_ms=0.385 total_avg_ms=2.474
[PROF_MOE_SUMMARY] pid=33749 calls=14800 tokens=2037440 router_gate_avg_ms=0.075 routed_avg_ms=1.737 shared_avg_ms=0.262 all_reduce_avg_ms=0.382 total_avg_ms=2.467
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=14800 tokens=2037440 router_gate_avg_ms=0.076 routed_avg_ms=1.736 shared_avg_ms=0.262 all_reduce_avg_ms=0.384 total_avg_ms=2.469
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=14800 tokens=2037440 router_gate_avg_ms=0.075 routed_avg_ms=1.742 shared_avg_ms=0.260 all_reduce_avg_ms=0.381 total_avg_ms=2.469
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=14800 tokens=2037440 router_gate_avg_ms=0.075 routed_avg_ms=1.740 shared_avg_ms=0.258 all_reduce_avg_ms=0.384 total_avg_ms=2.469
[PROF_MOE_SUMMARY] pid=33749 calls=14880 tokens=2037520 router_gate_avg_ms=0.075 routed_avg_ms=1.732 shared_avg_ms=0.262 all_reduce_avg_ms=0.382 total_avg_ms=2.462
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=14880 tokens=2037520 router_gate_avg_ms=0.076 routed_avg_ms=1.731 shared_avg_ms=0.262 all_reduce_avg_ms=0.384 total_avg_ms=2.464
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=14880 tokens=2037520 router_gate_avg_ms=0.075 routed_avg_ms=1.737 shared_avg_ms=0.260 all_reduce_avg_ms=0.381 total_avg_ms=2.464
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=14880 tokens=2037520 router_gate_avg_ms=0.075 routed_avg_ms=1.735 shared_avg_ms=0.258 all_reduce_avg_ms=0.384 total_avg_ms=2.463
[PROF_MOE_SUMMARY] pid=33749 calls=14960 tokens=2037600 router_gate_avg_ms=0.075 routed_avg_ms=1.727 shared_avg_ms=0.261 all_reduce_avg_ms=0.382 total_avg_ms=2.457
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=14960 tokens=2037600 router_gate_avg_ms=0.076 routed_avg_ms=1.726 shared_avg_ms=0.262 all_reduce_avg_ms=0.384 total_avg_ms=2.458
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=14960 tokens=2037600 router_gate_avg_ms=0.075 routed_avg_ms=1.732 shared_avg_ms=0.260 all_reduce_avg_ms=0.381 total_avg_ms=2.459
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=14960 tokens=2037600 router_gate_avg_ms=0.075 routed_avg_ms=1.730 shared_avg_ms=0.258 all_reduce_avg_ms=0.384 total_avg_ms=2.458
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=15040 tokens=2037680 router_gate_avg_ms=0.075 routed_avg_ms=1.727 shared_avg_ms=0.259 all_reduce_avg_ms=0.381 total_avg_ms=2.453
[PROF_MOE_SUMMARY] pid=33749 calls=15040 tokens=2037680 router_gate_avg_ms=0.075 routed_avg_ms=1.722 shared_avg_ms=0.261 all_reduce_avg_ms=0.382 total_avg_ms=2.452
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=15040 tokens=2037680 router_gate_avg_ms=0.075 routed_avg_ms=1.725 shared_avg_ms=0.258 all_reduce_avg_ms=0.383 total_avg_ms=2.453
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=15040 tokens=2037680 router_gate_avg_ms=0.076 routed_avg_ms=1.721 shared_avg_ms=0.262 all_reduce_avg_ms=0.383 total_avg_ms=2.453
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=15120 tokens=2037760 router_gate_avg_ms=0.075 routed_avg_ms=1.722 shared_avg_ms=0.259 all_reduce_avg_ms=0.381 total_avg_ms=2.448
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=15120 tokens=2037760 router_gate_avg_ms=0.075 routed_avg_ms=1.720 shared_avg_ms=0.258 all_reduce_avg_ms=0.383 total_avg_ms=2.447
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=15120 tokens=2037760 router_gate_avg_ms=0.076 routed_avg_ms=1.716 shared_avg_ms=0.261 all_reduce_avg_ms=0.383 total_avg_ms=2.448
[PROF_MOE_SUMMARY] pid=33749 calls=15120 tokens=2037760 router_gate_avg_ms=0.075 routed_avg_ms=1.717 shared_avg_ms=0.261 all_reduce_avg_ms=0.381 total_avg_ms=2.446
(VllmWorkerProcess pid=34103) [PROF_MOE_SUMMARY] pid=34103 calls=15200 tokens=2037840 router_gate_avg_ms=0.076 routed_avg_ms=1.712 shared_avg_ms=0.261 all_reduce_avg_ms=0.383 total_avg_ms=2.443
(VllmWorkerProcess pid=34102) [PROF_MOE_SUMMARY] pid=34102 calls=15200 tokens=2037840 router_gate_avg_ms=0.075 routed_avg_ms=1.717 shared_avg_ms=0.259 all_reduce_avg_ms=0.380 total_avg_ms=2.443
[PROF_MOE_SUMMARY] pid=33749 calls=15200 tokens=2037840 router_gate_avg_ms=0.075 routed_avg_ms=1.712 shared_avg_ms=0.261 all_reduce_avg_ms=0.381 total_avg_ms=2.441
(VllmWorkerProcess pid=34101) [PROF_MOE_SUMMARY] pid=34101 calls=15200 tokens=2037840 router_gate_avg_ms=0.075 routed_avg_ms=1.716 shared_avg_ms=0.258 all_reduce_avg_ms=0.383 total_avg_ms=2.442

13
worklogs/remote_scripts/run_batched_tokens_experiment.sh
View File

@@ -7,7 +7,9 @@ LABEL="${2:?usage: run_batched_tokens_experiment.sh MAX_BATCHED_TOKENS LABEL}"
OUT_DIR="/root/work/logs/${LABEL}" OUT_DIR="/root/work/logs/${LABEL}"
MODEL_PATH="/root/public-storage/models/Qwen/Qwen3.6-35B-A3B" MODEL_PATH="/root/public-storage/models/Qwen/Qwen3.6-35B-A3B"
DATASET="/root/work/logs/synthetic_cumulative_8_s3_cap40k.jsonl" DATASET="/root/work/logs/synthetic_cumulative_8_s3_cap40k.jsonl"
RESULT_JSON="${OUT_DIR}/eager_100k_b${MAX_BATCHED_TOKENS}_c1_r8_t256_cap40k.json" MAX_REQUESTS="${MAX_REQUESTS:-8}"
MAX_TOKENS="${MAX_TOKENS:-256}"
RESULT_JSON="${OUT_DIR}/eager_100k_b${MAX_BATCHED_TOKENS}_c1_r${MAX_REQUESTS}_t${MAX_TOKENS}_cap40k.json"
SERVER_LOG="${OUT_DIR}/server.log" SERVER_LOG="${OUT_DIR}/server.log"
DRIVER_LOG="${OUT_DIR}/driver.log" DRIVER_LOG="${OUT_DIR}/driver.log"
@@ -35,7 +37,10 @@ export CHUNK_PARALLEL=1
export PREFIX_FLASH=1 export PREFIX_FLASH=1
export RMSNORM_NATIVE=1 export RMSNORM_NATIVE=1
export LOOP1_NATIVE=1 export LOOP1_NATIVE=1
export PROF_TRACE=0 export PROF_TRACE="${PROF_TRACE:-0}"
export PROF_MOE_SUMMARY="${PROF_MOE_SUMMARY:-0}"
export PROF_MOE_SYNC="${PROF_MOE_SYNC:-1}"
export PROF_MOE_INTERVAL="${PROF_MOE_INTERVAL:-400}"
export VLLM_ENGINE_ITERATION_TIMEOUT_S=3600 export VLLM_ENGINE_ITERATION_TIMEOUT_S=3600
echo "[exp] gpu before start" echo "[exp] gpu before start"
@@ -96,8 +101,8 @@ python3 /root/work/formal_perf_bench.py \
--model llm \ --model llm \
--dataset "${DATASET}" \ --dataset "${DATASET}" \
--concurrency 1 \ --concurrency 1 \
--max-requests 8 \ --max-requests "${MAX_REQUESTS}" \
--max-tokens 256 \ --max-tokens "${MAX_TOKENS}" \
--timeout 1800 \ --timeout 1800 \
--out "${RESULT_JSON}" --out "${RESULT_JSON}"
BENCH_RC="$?" BENCH_RC="$?"