diff --git a/worklogs/decode_analysis_report_2026-07-14.md b/worklogs/decode_analysis_report_2026-07-14.md index 3bd015c..57d85c2 100644 --- a/worklogs/decode_analysis_report_2026-07-14.md +++ b/worklogs/decode_analysis_report_2026-07-14.md @@ -765,3 +765,63 @@ ENGINEX_MOE_TINY_MAX=4 ### 影响 这次优先目标是恢复平台可构建、可启动、可进入 benchmark-agent 验证流程。为降低平台构建风险，直接采用成功案例中已验证的一组补丁文件；此前本地 MoE tiny-batch 实验代码可能被成功案例的 native MoE/GDN 优化实现覆盖，后续如果平台构建通过，再基于这套可提交基线继续做性能优化分支。 + +## 2026-07-15：MoE tiny-batch 矩阵实验 + +### 实验过程 + +远端实验目录： + +- `/root/work/logs/moe_tiny_matrix_20260715_codex_05` + +本地归档： + +- `worklogs/remote_results/2026-07-15-moe-tiny-matrix/` + +前两次启动没有形成有效结果： + +- `codex_03`：未加 `--enforce-eager`，服务加载后卡在 CUDA Graph capture，`/health` 一直拒绝连接。 +- `codex_04`：已加 `--enforce-eager`，但旧的 `codex_03` 服务仍占用显存，导致新服务加载 MoE 权重时 OOM。 + +有效实验为 `codex_05`，先清理旧进程，再用 `--enforce-eager` 重跑 5 组矩阵。 + +固定压测口径： + +- `prompt_mode=short` +- `concurrency=2` +- `requests=4` +- `max_tokens=128` +- `max_num_seqs=2` +- `max_model_len=100000` +- `tp=4` +- `enable_prefix_caching` +- `enable_chunked_prefill` + +### 结果 + +| case | success | TTFT P90 | Output TPS P10 | Output TPS P50 | Aggregate Output TPS | Avg GPU util | Max GPU util | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| `bmm_max16` | 100.00% | 1.401 | 6.245 | 6.381 | 11.958 | 53.2 | 100.0 | +| `bmm_max8` | 100.00% | 1.424 | 6.900 | 6.957 | 12.907 | 54.7 | 100.0 | +| `tokenwise_max16` | 100.00% | 1.312 | 6.292 | 6.387 | 12.003 | 54.8 | 100.0 | +| `tokenwise_max4` | 100.00% | 1.671 | 6.377 | 6.416 | 11.868 | 51.4 | 100.0 | +| `tokenwise_max8` | 100.00% | 1.589 | 6.347 | 6.357 | 11.788 | 54.6 | 100.0 | + +### 结论 + +本轮最佳结果是： + +- `ENGINEX_MOE_TINY_IMPL=bmm` +- `ENGINEX_MOE_TINY_MAX=8` +- Aggregate Output TPS = `12.907` +- Output TPS P10 = `6.900` + +相对上一轮 `tokenwise_max4` 的有效结果 `11.28`，本轮 `bmm_max8` 聚合吞吐约提升 `14.4%`。但要注意两轮测试的启动环境不同：本轮强制 `--enforce-eager`，且经过多次重启，不能把这个提升完全归因于 `bmm` 实现。 + +更重要的是，这个矩阵**没有真正验证 `T > TINY_MAX` 后按块拆分是否有收益**。原因是本轮压测固定 `max_num_seqs=2`，decode 阶段 `T` 基本是 1-2，因此 `TINY_MAX=4/8/16` 都会命中同一 tiny fast path。也就是说： + +- `tokenwise_max4/8/16` 之间的差异主要是重启和运行噪声。 +- `bmm_max8/16` 之间的差异也主要是噪声。 +- 本轮更有价值的对比是 `tokenwise` 与 `bmm` 两种 tiny implementation。 + +从本轮结果看，在 `T≈2` 的短输出 decode 场景里，`bmm` 路径不差于 `tokenwise`，并且本次最佳是 `bmm_max8`。下一步如果要验证“`T` 大于阈值后分块计算”的思路，需要把 `max_num_seqs` 提到 8 或 16，或构造能让单步 decode 聚合更多 token 的压测，否则 `TINY_MAX` 参数不会真正生效。 diff --git a/worklogs/remote_results/2026-07-15-moe-tiny-matrix/bmm_max16_ixsmi.json b/worklogs/remote_results/2026-07-15-moe-tiny-matrix/bmm_max16_ixsmi.json new file mode 100644 index 0000000..e1b090e --- /dev/null +++ b/worklogs/remote_results/2026-07-15-moe-tiny-matrix/bmm_max16_ixsmi.json @@ -0,0 +1,1122 @@ +[ + { + "ts": "2026-07-14T17:17:55", + "ok": true, + "raw": "Timestamp Tue Jul 14 17:17:55 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 30C P0 53W / 250W | 28955MiB / 32768MiB | 0% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 53W / 250W | 28894MiB / 32768MiB | 0% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 53W / 250W | 28898MiB / 32768MiB | 0% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 52W / 250W | 28758MiB / 32768MiB | 0% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n", + "parsed": [ + { + "power_w": 53, + "power_cap_w": 250, + "mem_mib": 28955, + "mem_total_mib": 32768, + "util_pct": 0 + }, + { + "power_w": 53, + "power_cap_w": 250, + "mem_mib": 28894, + "mem_total_mib": 32768, + "util_pct": 0 + }, + { + "power_w": 53, + "power_cap_w": 250, + "mem_mib": 28898, + "mem_total_mib": 32768, + "util_pct": 0 + }, + { + "power_w": 52, + "power_cap_w": 250, + "mem_mib": 28758, + "mem_total_mib": 32768, + "util_pct": 0 + } + ] + }, + { + "ts": "2026-07-14T17:17:56", + "ok": true, + "raw": "Timestamp Tue Jul 14 17:17:57 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 54W / 250W | 29123MiB / 32768MiB | 42% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 34C P0 55W / 250W | 29062MiB / 32768MiB | 50% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 33C P0 56W / 250W | 29066MiB / 32768MiB | 61% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 55W / 250W | 28926MiB / 32768MiB | 82% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n", + "parsed": [ + { + "power_w": 54, + "power_cap_w": 250, + "mem_mib": 29123, + "mem_total_mib": 32768, + "util_pct": 42 + }, + { + "power_w": 55, + "power_cap_w": 250, + "mem_mib": 29062, + "mem_total_mib": 32768, + "util_pct": 50 + }, + { + "power_w": 56, + "power_cap_w": 250, + "mem_mib": 29066, + "mem_total_mib": 32768, + "util_pct": 61 + }, + { + "power_w": 55, + "power_cap_w": 250, + "mem_mib": 28926, + "mem_total_mib": 32768, + "util_pct": 82 + } + ] + }, + { + "ts": "2026-07-14T17:17:58", + "ok": true, + "raw": "Timestamp Tue Jul 14 17:17:58 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 30C P0 56W / 250W | 29123MiB / 32768MiB | 77% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 33C P0 56W / 250W | 29062MiB / 32768MiB | 51% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 56W / 250W | 29066MiB / 32768MiB | 79% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 55W / 250W | 28926MiB / 32768MiB | 37% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n", + "parsed": [ + { + "power_w": 56, + "power_cap_w": 250, + "mem_mib": 29123, + "mem_total_mib": 32768, + "util_pct": 77 + }, + { + "power_w": 56, + "power_cap_w": 250, + "mem_mib": 29062, + "mem_total_mib": 32768, + "util_pct": 51 + }, + { + "power_w": 56, + "power_cap_w": 250, + "mem_mib": 29066, + "mem_total_mib": 32768, + "util_pct": 79 + }, + { + "power_w": 55, + "power_cap_w": 250, + "mem_mib": 28926, + "mem_total_mib": 32768, + "util_pct": 37 + } + ] + }, + { + "ts": "2026-07-14T17:17:59", + "ok": true, + "raw": "Timestamp Tue Jul 14 17:17:59 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 30C P0 56W / 250W | 29123MiB / 32768MiB | 69% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 33C P0 56W / 250W | 29062MiB / 32768MiB | 59% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 56W / 250W | 29066MiB / 32768MiB | 60% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 56W / 250W | 28926MiB / 32768MiB | 34% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n", + "parsed": [ + { + "power_w": 56, + "power_cap_w": 250, + "mem_mib": 29123, + "mem_total_mib": 32768, + "util_pct": 69 + }, + { + "power_w": 56, + "power_cap_w": 250, + "mem_mib": 29062, + "mem_total_mib": 32768, + "util_pct": 59 + }, + { + "power_w": 56, + "power_cap_w": 250, + "mem_mib": 29066, + "mem_total_mib": 32768, + "util_pct": 60 + }, + { + "power_w": 56, + "power_cap_w": 250, + "mem_mib": 28926, + "mem_total_mib": 32768, + "util_pct": 34 + } + ] + }, + { + "ts": "2026-07-14T17:18:01", + "ok": true, + "raw": "Timestamp Tue Jul 14 17:18:01 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 55W / 250W | 29123MiB / 32768MiB | 43% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 33C P0 55W / 250W | 29062MiB / 32768MiB | 47% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 33C P0 55W / 250W | 29066MiB / 32768MiB | 59% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 54W / 250W | 28926MiB / 32768MiB | 52% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n", + "parsed": [ + { + "power_w": 55, + "power_cap_w": 250, + "mem_mib": 29123, + "mem_total_mib": 32768, + "util_pct": 43 + }, + { + "power_w": 55, + "power_cap_w": 250, + "mem_mib": 29062, + "mem_total_mib": 32768, + "util_pct": 47 + }, + { + "power_w": 55, + "power_cap_w": 250, + "mem_mib": 29066, + "mem_total_mib": 32768, + "util_pct": 59 + }, + { + "power_w": 54, + "power_cap_w": 250, + "mem_mib": 28926, + "mem_total_mib": 32768, + "util_pct": 52 + } + ] + }, + { + "ts": "2026-07-14T17:18:02", + "ok": true, + "raw": "Timestamp Tue Jul 14 17:18:02 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 56W / 250W | 29123MiB / 32768MiB | 56% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 33C P0 56W / 250W | 29062MiB / 32768MiB | 28% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 57W / 250W | 29066MiB / 32768MiB | 54% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 55W / 250W | 28926MiB / 32768MiB | 25% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n", + "parsed": [ + { + "power_w": 56, + "power_cap_w": 250, + "mem_mib": 29123, + "mem_total_mib": 32768, + "util_pct": 56 + }, + { + "power_w": 56, + "power_cap_w": 250, + "mem_mib": 29062, + "mem_total_mib": 32768, + "util_pct": 28 + }, + { + "power_w": 57, + "power_cap_w": 250, + "mem_mib": 29066, + "mem_total_mib": 32768, + "util_pct": 54 + }, + { + "power_w": 55, + "power_cap_w": 250, + "mem_mib": 28926, + "mem_total_mib": 32768, + "util_pct": 25 + } + ] + }, + { + "ts": "2026-07-14T17:18:03", + "ok": true, + "raw": "Timestamp Tue Jul 14 17:18:03 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 55W / 250W | 29123MiB / 32768MiB | 46% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 33C P0 55W / 250W | 29062MiB / 32768MiB | 26% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 56W / 250W | 29066MiB / 32768MiB | 83% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 55W / 250W | 28926MiB / 32768MiB | 31% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n", + "parsed": [ + { + "power_w": 55, + "power_cap_w": 250, + "mem_mib": 29123, + "mem_total_mib": 32768, + "util_pct": 46 + }, + { + "power_w": 55, + "power_cap_w": 250, + "mem_mib": 29062, + "mem_total_mib": 32768, + "util_pct": 26 + }, + { + "power_w": 56, + "power_cap_w": 250, + "mem_mib": 29066, + "mem_total_mib": 32768, + "util_pct": 83 + }, + { + "power_w": 55, + "power_cap_w": 250, + "mem_mib": 28926, + "mem_total_mib": 32768, + "util_pct": 31 + } + ] + }, + { + "ts": "2026-07-14T17:18:05", + "ok": true, + "raw": "Timestamp Tue Jul 14 17:18:05 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 56W / 250W | 29123MiB / 32768MiB | 33% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 33C P0 56W / 250W | 29062MiB / 32768MiB | 53% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 56W / 250W | 29066MiB / 32768MiB | 89% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 55W / 250W | 28926MiB / 32768MiB | 100% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n", + "parsed": [ + { + "power_w": 56, + "power_cap_w": 250, + "mem_mib": 29123, + "mem_total_mib": 32768, + "util_pct": 33 + }, + { + "power_w": 56, + "power_cap_w": 250, + "mem_mib": 29062, + "mem_total_mib": 32768, + "util_pct": 53 + }, + { + "power_w": 56, + "power_cap_w": 250, + "mem_mib": 29066, + "mem_total_mib": 32768, + "util_pct": 89 + }, + { + "power_w": 55, + "power_cap_w": 250, + "mem_mib": 28926, + "mem_total_mib": 32768, + "util_pct": 100 + } + ] + }, + { + "ts": "2026-07-14T17:18:06", + "ok": true, + "raw": "Timestamp Tue Jul 14 17:18:06 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 56W / 250W | 29123MiB / 32768MiB | 42% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 33C P0 56W / 250W | 29062MiB / 32768MiB | 41% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 56W / 250W | 29066MiB / 32768MiB | 68% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 56W / 250W | 28926MiB / 32768MiB | 63% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n", + "parsed": [ + { + "power_w": 56, + "power_cap_w": 250, + "mem_mib": 29123, + "mem_total_mib": 32768, + "util_pct": 42 + }, + { + "power_w": 56, + "power_cap_w": 250, + "mem_mib": 29062, + "mem_total_mib": 32768, + "util_pct": 41 + }, + { + "power_w": 56, + "power_cap_w": 250, + "mem_mib": 29066, + "mem_total_mib": 32768, + "util_pct": 68 + }, + { + "power_w": 56, + "power_cap_w": 250, + "mem_mib": 28926, + "mem_total_mib": 32768, + "util_pct": 63 + } + ] + }, + { + "ts": "2026-07-14T17:18:08", + "ok": true, + "raw": "Timestamp Tue Jul 14 17:18:08 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 56W / 250W | 29123MiB / 32768MiB | 62% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 34C P0 57W / 250W | 29062MiB / 32768MiB | 67% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 33C P0 57W / 250W | 29066MiB / 32768MiB | 51% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 56W / 250W | 28926MiB / 32768MiB | 50% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n", + "parsed": [ + { + "power_w": 56, + "power_cap_w": 250, + "mem_mib": 29123, + "mem_total_mib": 32768, + "util_pct": 62 + }, + { + "power_w": 57, + "power_cap_w": 250, + "mem_mib": 29062, + "mem_total_mib": 32768, + "util_pct": 67 + }, + { + "power_w": 57, + "power_cap_w": 250, + "mem_mib": 29066, + "mem_total_mib": 32768, + "util_pct": 51 + }, + { + "power_w": 56, + "power_cap_w": 250, + "mem_mib": 28926, + "mem_total_mib": 32768, + "util_pct": 50 + } + ] + }, + { + "ts": "2026-07-14T17:18:09", + "ok": true, + "raw": "Timestamp Tue Jul 14 17:18:09 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 57W / 250W | 29123MiB / 32768MiB | 84% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 33C P0 56W / 250W | 29062MiB / 32768MiB | 67% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 57W / 250W | 29066MiB / 32768MiB | 69% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 56W / 250W | 28926MiB / 32768MiB | 30% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n", + "parsed": [ + { + "power_w": 57, + "power_cap_w": 250, + "mem_mib": 29123, + "mem_total_mib": 32768, + "util_pct": 84 + }, + { + "power_w": 56, + "power_cap_w": 250, + "mem_mib": 29062, + "mem_total_mib": 32768, + "util_pct": 67 + }, + { + "power_w": 57, + "power_cap_w": 250, + "mem_mib": 29066, + "mem_total_mib": 32768, + "util_pct": 69 + }, + { + "power_w": 56, + "power_cap_w": 250, + "mem_mib": 28926, + "mem_total_mib": 32768, + "util_pct": 30 + } + ] + }, + { + "ts": "2026-07-14T17:18:10", + "ok": true, + "raw": "Timestamp Tue Jul 14 17:18:10 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 56W / 250W | 29123MiB / 32768MiB | 37% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 34C P0 56W / 250W | 29062MiB / 32768MiB | 77% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 56W / 250W | 29066MiB / 32768MiB | 65% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 55W / 250W | 28926MiB / 32768MiB | 46% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n", + "parsed": [ + { + "power_w": 56, + "power_cap_w": 250, + "mem_mib": 29123, + "mem_total_mib": 32768, + "util_pct": 37 + }, + { + "power_w": 56, + "power_cap_w": 250, + "mem_mib": 29062, + "mem_total_mib": 32768, + "util_pct": 77 + }, + { + "power_w": 56, + "power_cap_w": 250, + "mem_mib": 29066, + "mem_total_mib": 32768, + "util_pct": 65 + }, + { + "power_w": 55, + "power_cap_w": 250, + "mem_mib": 28926, + "mem_total_mib": 32768, + "util_pct": 46 + } + ] + }, + { + "ts": "2026-07-14T17:18:12", + "ok": true, + "raw": "Timestamp Tue Jul 14 17:18:12 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 56W / 250W | 29123MiB / 32768MiB | 36% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 34C P0 56W / 250W | 29062MiB / 32768MiB | 19% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 57W / 250W | 29066MiB / 32768MiB | 49% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 56W / 250W | 28926MiB / 32768MiB | 100% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n", + "parsed": [ + { + "power_w": 56, + "power_cap_w": 250, + "mem_mib": 29123, + "mem_total_mib": 32768, + "util_pct": 36 + }, + { + "power_w": 56, + "power_cap_w": 250, + "mem_mib": 29062, + "mem_total_mib": 32768, + "util_pct": 19 + }, + { + "power_w": 57, + "power_cap_w": 250, + "mem_mib": 29066, + "mem_total_mib": 32768, + "util_pct": 49 + }, + { + "power_w": 56, + "power_cap_w": 250, + "mem_mib": 28926, + "mem_total_mib": 32768, + "util_pct": 100 + } + ] + }, + { + "ts": "2026-07-14T17:18:13", + "ok": true, + "raw": "Timestamp Tue Jul 14 17:18:13 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 56W / 250W | 29123MiB / 32768MiB | 20% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 33C P0 56W / 250W | 29062MiB / 32768MiB | 56% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 33C P0 56W / 250W | 29066MiB / 32768MiB | 61% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 56W / 250W | 28926MiB / 32768MiB | 47% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n", + "parsed": [ + { + "power_w": 56, + "power_cap_w": 250, + "mem_mib": 29123, + "mem_total_mib": 32768, + "util_pct": 20 + }, + { + "power_w": 56, + "power_cap_w": 250, + "mem_mib": 29062, + "mem_total_mib": 32768, + "util_pct": 56 + }, + { + "power_w": 56, + "power_cap_w": 250, + "mem_mib": 29066, + "mem_total_mib": 32768, + "util_pct": 61 + }, + { + "power_w": 56, + "power_cap_w": 250, + "mem_mib": 28926, + "mem_total_mib": 32768, + "util_pct": 47 + } + ] + }, + { + "ts": "2026-07-14T17:18:14", + "ok": true, + "raw": "Timestamp Tue Jul 14 17:18:14 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 57W / 250W | 29123MiB / 32768MiB | 76% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 34C P0 57W / 250W | 29062MiB / 32768MiB | 53% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 33C P0 57W / 250W | 29066MiB / 32768MiB | 78% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 57W / 250W | 28926MiB / 32768MiB | 49% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n", + "parsed": [ + { + "power_w": 57, + "power_cap_w": 250, + "mem_mib": 29123, + "mem_total_mib": 32768, + "util_pct": 76 + }, + { + "power_w": 57, + "power_cap_w": 250, + "mem_mib": 29062, + "mem_total_mib": 32768, + "util_pct": 53 + }, + { + "power_w": 57, + "power_cap_w": 250, + "mem_mib": 29066, + "mem_total_mib": 32768, + "util_pct": 78 + }, + { + "power_w": 57, + "power_cap_w": 250, + "mem_mib": 28926, + "mem_total_mib": 32768, + "util_pct": 49 + } + ] + }, + { + "ts": "2026-07-14T17:18:16", + "ok": true, + "raw": "Timestamp Tue Jul 14 17:18:16 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 57W / 250W | 29123MiB / 32768MiB | 36% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 34C P0 56W / 250W | 29062MiB / 32768MiB | 76% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 33C P0 57W / 250W | 29066MiB / 32768MiB | 49% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 56W / 250W | 28926MiB / 32768MiB | 87% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n", + "parsed": [ + { + "power_w": 57, + "power_cap_w": 250, + "mem_mib": 29123, + "mem_total_mib": 32768, + "util_pct": 36 + }, + { + "power_w": 56, + "power_cap_w": 250, + "mem_mib": 29062, + "mem_total_mib": 32768, + "util_pct": 76 + }, + { + "power_w": 57, + "power_cap_w": 250, + "mem_mib": 29066, + "mem_total_mib": 32768, + "util_pct": 49 + }, + { + "power_w": 56, + "power_cap_w": 250, + "mem_mib": 28926, + "mem_total_mib": 32768, + "util_pct": 87 + } + ] + }, + { + "ts": "2026-07-14T17:18:17", + "ok": true, + "raw": "Timestamp Tue Jul 14 17:18:17 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 30C P0 56W / 250W | 29123MiB / 32768MiB | 0% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 34C P0 55W / 250W | 29062MiB / 32768MiB | 0% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 33C P0 56W / 250W | 29066MiB / 32768MiB | 0% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 54W / 250W | 28926MiB / 32768MiB | 0% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n", + "parsed": [ + { + "power_w": 56, + "power_cap_w": 250, + "mem_mib": 29123, + "mem_total_mib": 32768, + "util_pct": 0 + }, + { + "power_w": 55, + "power_cap_w": 250, + "mem_mib": 29062, + "mem_total_mib": 32768, + "util_pct": 0 + }, + { + "power_w": 56, + "power_cap_w": 250, + "mem_mib": 29066, + "mem_total_mib": 32768, + "util_pct": 0 + }, + { + "power_w": 54, + "power_cap_w": 250, + "mem_mib": 28926, + "mem_total_mib": 32768, + "util_pct": 0 + } + ] + }, + { + "ts": "2026-07-14T17:18:18", + "ok": true, + "raw": "Timestamp Tue Jul 14 17:18:19 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 57W / 250W | 29123MiB / 32768MiB | 51% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 34C P0 57W / 250W | 29062MiB / 32768MiB | 50% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 33C P0 58W / 250W | 29066MiB / 32768MiB | 66% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 57W / 250W | 28926MiB / 32768MiB | 55% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n", + "parsed": [ + { + "power_w": 57, + "power_cap_w": 250, + "mem_mib": 29123, + "mem_total_mib": 32768, + "util_pct": 51 + }, + { + "power_w": 57, + "power_cap_w": 250, + "mem_mib": 29062, + "mem_total_mib": 32768, + "util_pct": 50 + }, + { + "power_w": 58, + "power_cap_w": 250, + "mem_mib": 29066, + "mem_total_mib": 32768, + "util_pct": 66 + }, + { + "power_w": 57, + "power_cap_w": 250, + "mem_mib": 28926, + "mem_total_mib": 32768, + "util_pct": 55 + } + ] + }, + { + "ts": "2026-07-14T17:18:20", + "ok": true, + "raw": "Timestamp Tue Jul 14 17:18:20 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 57W / 250W | 29123MiB / 32768MiB | 32% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 34C P0 57W / 250W | 29062MiB / 32768MiB | 46% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 57W / 250W | 29066MiB / 32768MiB | 39% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 56W / 250W | 28926MiB / 32768MiB | 71% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n", + "parsed": [ + { + "power_w": 57, + "power_cap_w": 250, + "mem_mib": 29123, + "mem_total_mib": 32768, + "util_pct": 32 + }, + { + "power_w": 57, + "power_cap_w": 250, + "mem_mib": 29062, + "mem_total_mib": 32768, + "util_pct": 46 + }, + { + "power_w": 57, + "power_cap_w": 250, + "mem_mib": 29066, + "mem_total_mib": 32768, + "util_pct": 39 + }, + { + "power_w": 56, + "power_cap_w": 250, + "mem_mib": 28926, + "mem_total_mib": 32768, + "util_pct": 71 + } + ] + }, + { + "ts": "2026-07-14T17:18:21", + "ok": true, + "raw": "Timestamp Tue Jul 14 17:18:21 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 56W / 250W | 29123MiB / 32768MiB | 66% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 34C P0 56W / 250W | 29062MiB / 32768MiB | 62% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 33C P0 57W / 250W | 29066MiB / 32768MiB | 98% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 56W / 250W | 28926MiB / 32768MiB | 69% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n", + "parsed": [ + { + "power_w": 56, + "power_cap_w": 250, + "mem_mib": 29123, + "mem_total_mib": 32768, + "util_pct": 66 + }, + { + "power_w": 56, + "power_cap_w": 250, + "mem_mib": 29062, + "mem_total_mib": 32768, + "util_pct": 62 + }, + { + "power_w": 57, + "power_cap_w": 250, + "mem_mib": 29066, + "mem_total_mib": 32768, + "util_pct": 98 + }, + { + "power_w": 56, + "power_cap_w": 250, + "mem_mib": 28926, + "mem_total_mib": 32768, + "util_pct": 69 + } + ] + }, + { + "ts": "2026-07-14T17:18:23", + "ok": true, + "raw": "Timestamp Tue Jul 14 17:18:23 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 57W / 250W | 29123MiB / 32768MiB | 68% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 34C P0 57W / 250W | 29062MiB / 32768MiB | 41% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 33C P0 57W / 250W | 29066MiB / 32768MiB | 67% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 56W / 250W | 28926MiB / 32768MiB | 49% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n", + "parsed": [ + { + "power_w": 57, + "power_cap_w": 250, + "mem_mib": 29123, + "mem_total_mib": 32768, + "util_pct": 68 + }, + { + "power_w": 57, + "power_cap_w": 250, + "mem_mib": 29062, + "mem_total_mib": 32768, + "util_pct": 41 + }, + { + "power_w": 57, + "power_cap_w": 250, + "mem_mib": 29066, + "mem_total_mib": 32768, + "util_pct": 67 + }, + { + "power_w": 56, + "power_cap_w": 250, + "mem_mib": 28926, + "mem_total_mib": 32768, + "util_pct": 49 + } + ] + }, + { + "ts": "2026-07-14T17:18:24", + "ok": true, + "raw": "Timestamp Tue Jul 14 17:18:24 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 56W / 250W | 29123MiB / 32768MiB | 84% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 34C P0 57W / 250W | 29062MiB / 32768MiB | 65% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 33C P0 57W / 250W | 29066MiB / 32768MiB | 43% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 56W / 250W | 28926MiB / 32768MiB | 54% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n", + "parsed": [ + { + "power_w": 56, + "power_cap_w": 250, + "mem_mib": 29123, + "mem_total_mib": 32768, + "util_pct": 84 + }, + { + "power_w": 57, + "power_cap_w": 250, + "mem_mib": 29062, + "mem_total_mib": 32768, + "util_pct": 65 + }, + { + "power_w": 57, + "power_cap_w": 250, + "mem_mib": 29066, + "mem_total_mib": 32768, + "util_pct": 43 + }, + { + "power_w": 56, + "power_cap_w": 250, + "mem_mib": 28926, + "mem_total_mib": 32768, + "util_pct": 54 + } + ] + }, + { + "ts": "2026-07-14T17:18:25", + "ok": true, + "raw": "Timestamp Tue Jul 14 17:18:25 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 57W / 250W | 29123MiB / 32768MiB | 60% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 34C P0 57W / 250W | 29062MiB / 32768MiB | 88% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 33C P0 57W / 250W | 29066MiB / 32768MiB | 56% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 56W / 250W | 28926MiB / 32768MiB | 62% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n", + "parsed": [ + { + "power_w": 57, + "power_cap_w": 250, + "mem_mib": 29123, + "mem_total_mib": 32768, + "util_pct": 60 + }, + { + "power_w": 57, + "power_cap_w": 250, + "mem_mib": 29062, + "mem_total_mib": 32768, + "util_pct": 88 + }, + { + "power_w": 57, + "power_cap_w": 250, + "mem_mib": 29066, + "mem_total_mib": 32768, + "util_pct": 56 + }, + { + "power_w": 56, + "power_cap_w": 250, + "mem_mib": 28926, + "mem_total_mib": 32768, + "util_pct": 62 + } + ] + }, + { + "ts": "2026-07-14T17:18:27", + "ok": true, + "raw": "Timestamp Tue Jul 14 17:18:27 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 57W / 250W | 29123MiB / 32768MiB | 76% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 34C P0 57W / 250W | 29062MiB / 32768MiB | 56% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 33C P0 57W / 250W | 29066MiB / 32768MiB | 64% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 56W / 250W | 28926MiB / 32768MiB | 37% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n", + "parsed": [ + { + "power_w": 57, + "power_cap_w": 250, + "mem_mib": 29123, + "mem_total_mib": 32768, + "util_pct": 76 + }, + { + "power_w": 57, + "power_cap_w": 250, + "mem_mib": 29062, + "mem_total_mib": 32768, + "util_pct": 56 + }, + { + "power_w": 57, + "power_cap_w": 250, + "mem_mib": 29066, + "mem_total_mib": 32768, + "util_pct": 64 + }, + { + "power_w": 56, + "power_cap_w": 250, + "mem_mib": 28926, + "mem_total_mib": 32768, + "util_pct": 37 + } + ] + }, + { + "ts": "2026-07-14T17:18:28", + "ok": true, + "raw": "Timestamp Tue Jul 14 17:18:28 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 57W / 250W | 29123MiB / 32768MiB | 27% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 34C P0 56W / 250W | 29062MiB / 32768MiB | 58% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 33C P0 56W / 250W | 29066MiB / 32768MiB | 49% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 56W / 250W | 28926MiB / 32768MiB | 78% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n", + "parsed": [ + { + "power_w": 57, + "power_cap_w": 250, + "mem_mib": 29123, + "mem_total_mib": 32768, + "util_pct": 27 + }, + { + "power_w": 56, + "power_cap_w": 250, + "mem_mib": 29062, + "mem_total_mib": 32768, + "util_pct": 58 + }, + { + "power_w": 56, + "power_cap_w": 250, + "mem_mib": 29066, + "mem_total_mib": 32768, + "util_pct": 49 + }, + { + "power_w": 56, + "power_cap_w": 250, + "mem_mib": 28926, + "mem_total_mib": 32768, + "util_pct": 78 + } + ] + }, + { + "ts": "2026-07-14T17:18:29", + "ok": true, + "raw": "Timestamp Tue Jul 14 17:18:29 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 57W / 250W | 29123MiB / 32768MiB | 52% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 34C P0 57W / 250W | 29062MiB / 32768MiB | 14% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 33C P0 57W / 250W | 29066MiB / 32768MiB | 57% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 56W / 250W | 28926MiB / 32768MiB | 52% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n", + "parsed": [ + { + "power_w": 57, + "power_cap_w": 250, + "mem_mib": 29123, + "mem_total_mib": 32768, + "util_pct": 52 + }, + { + "power_w": 57, + "power_cap_w": 250, + "mem_mib": 29062, + "mem_total_mib": 32768, + "util_pct": 14 + }, + { + "power_w": 57, + "power_cap_w": 250, + "mem_mib": 29066, + "mem_total_mib": 32768, + "util_pct": 57 + }, + { + "power_w": 56, + "power_cap_w": 250, + "mem_mib": 28926, + "mem_total_mib": 32768, + "util_pct": 52 + } + ] + }, + { + "ts": "2026-07-14T17:18:31", + "ok": true, + "raw": "Timestamp Tue Jul 14 17:18:31 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 55W / 250W | 29123MiB / 32768MiB | 82% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 34C P0 55W / 250W | 29062MiB / 32768MiB | 100% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 33C P0 56W / 250W | 29066MiB / 32768MiB | 76% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 55W / 250W | 28926MiB / 32768MiB | 39% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n", + "parsed": [ + { + "power_w": 55, + "power_cap_w": 250, + "mem_mib": 29123, + "mem_total_mib": 32768, + "util_pct": 82 + }, + { + "power_w": 55, + "power_cap_w": 250, + "mem_mib": 29062, + "mem_total_mib": 32768, + "util_pct": 100 + }, + { + "power_w": 56, + "power_cap_w": 250, + "mem_mib": 29066, + "mem_total_mib": 32768, + "util_pct": 76 + }, + { + "power_w": 55, + "power_cap_w": 250, + "mem_mib": 28926, + "mem_total_mib": 32768, + "util_pct": 39 + } + ] + }, + { + "ts": "2026-07-14T17:18:32", + "ok": true, + "raw": "Timestamp Tue Jul 14 17:18:32 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 57W / 250W | 29123MiB / 32768MiB | 50% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 34C P0 57W / 250W | 29062MiB / 32768MiB | 77% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 33C P0 57W / 250W | 29066MiB / 32768MiB | 95% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 56W / 250W | 28926MiB / 32768MiB | 50% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n", + "parsed": [ + { + "power_w": 57, + "power_cap_w": 250, + "mem_mib": 29123, + "mem_total_mib": 32768, + "util_pct": 50 + }, + { + "power_w": 57, + "power_cap_w": 250, + "mem_mib": 29062, + "mem_total_mib": 32768, + "util_pct": 77 + }, + { + "power_w": 57, + "power_cap_w": 250, + "mem_mib": 29066, + "mem_total_mib": 32768, + "util_pct": 95 + }, + { + "power_w": 56, + "power_cap_w": 250, + "mem_mib": 28926, + "mem_total_mib": 32768, + "util_pct": 50 + } + ] + }, + { + "ts": "2026-07-14T17:18:33", + "ok": true, + "raw": "Timestamp Tue Jul 14 17:18:33 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 55W / 250W | 29123MiB / 32768MiB | 62% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 34C P0 55W / 250W | 29062MiB / 32768MiB | 55% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 56W / 250W | 29066MiB / 32768MiB | 59% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 55W / 250W | 28926MiB / 32768MiB | 56% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n", + "parsed": [ + { + "power_w": 55, + "power_cap_w": 250, + "mem_mib": 29123, + "mem_total_mib": 32768, + "util_pct": 62 + }, + { + "power_w": 55, + "power_cap_w": 250, + "mem_mib": 29062, + "mem_total_mib": 32768, + "util_pct": 55 + }, + { + "power_w": 56, + "power_cap_w": 250, + "mem_mib": 29066, + "mem_total_mib": 32768, + "util_pct": 59 + }, + { + "power_w": 55, + "power_cap_w": 250, + "mem_mib": 28926, + "mem_total_mib": 32768, + "util_pct": 56 + } + ] + }, + { + "ts": "2026-07-14T17:18:35", + "ok": true, + "raw": "Timestamp Tue Jul 14 17:18:35 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 57W / 250W | 29123MiB / 32768MiB | 72% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 34C P0 57W / 250W | 29062MiB / 32768MiB | 38% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 33C P0 58W / 250W | 29066MiB / 32768MiB | 62% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 57W / 250W | 28926MiB / 32768MiB | 47% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n", + "parsed": [ + { + "power_w": 57, + "power_cap_w": 250, + "mem_mib": 29123, + "mem_total_mib": 32768, + "util_pct": 72 + }, + { + "power_w": 57, + "power_cap_w": 250, + "mem_mib": 29062, + "mem_total_mib": 32768, + "util_pct": 38 + }, + { + "power_w": 58, + "power_cap_w": 250, + "mem_mib": 29066, + "mem_total_mib": 32768, + "util_pct": 62 + }, + { + "power_w": 57, + "power_cap_w": 250, + "mem_mib": 28926, + "mem_total_mib": 32768, + "util_pct": 47 + } + ] + }, + { + "ts": "2026-07-14T17:18:36", + "ok": true, + "raw": "Timestamp Tue Jul 14 17:18:36 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 57W / 250W | 29123MiB / 32768MiB | 49% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 34C P0 57W / 250W | 29062MiB / 32768MiB | 48% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 33C P0 57W / 250W | 29066MiB / 32768MiB | 58% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 56W / 250W | 28926MiB / 32768MiB | 72% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n", + "parsed": [ + { + "power_w": 57, + "power_cap_w": 250, + "mem_mib": 29123, + "mem_total_mib": 32768, + "util_pct": 49 + }, + { + "power_w": 57, + "power_cap_w": 250, + "mem_mib": 29062, + "mem_total_mib": 32768, + "util_pct": 48 + }, + { + "power_w": 57, + "power_cap_w": 250, + "mem_mib": 29066, + "mem_total_mib": 32768, + "util_pct": 58 + }, + { + "power_w": 56, + "power_cap_w": 250, + "mem_mib": 28926, + "mem_total_mib": 32768, + "util_pct": 72 + } + ] + }, + { + "ts": "2026-07-14T17:18:37", + "ok": true, + "raw": "Timestamp Tue Jul 14 17:18:38 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 55W / 250W | 29123MiB / 32768MiB | 61% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 34C P0 55W / 250W | 29062MiB / 32768MiB | 44% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 33C P0 55W / 250W | 29066MiB / 32768MiB | 58% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 54W / 250W | 28926MiB / 32768MiB | 0% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n", + "parsed": [ + { + "power_w": 55, + "power_cap_w": 250, + "mem_mib": 29123, + "mem_total_mib": 32768, + "util_pct": 61 + }, + { + "power_w": 55, + "power_cap_w": 250, + "mem_mib": 29062, + "mem_total_mib": 32768, + "util_pct": 44 + }, + { + "power_w": 55, + "power_cap_w": 250, + "mem_mib": 29066, + "mem_total_mib": 32768, + "util_pct": 58 + }, + { + "power_w": 54, + "power_cap_w": 250, + "mem_mib": 28926, + "mem_total_mib": 32768, + "util_pct": 0 + } + ] + } +] \ No newline at end of file diff --git a/worklogs/remote_results/2026-07-15-moe-tiny-matrix/bmm_max16_server.log b/worklogs/remote_results/2026-07-15-moe-tiny-matrix/bmm_max16_server.log new file mode 100644 index 0000000..29e858a --- /dev/null +++ b/worklogs/remote_results/2026-07-15-moe-tiny-matrix/bmm_max16_server.log @@ -0,0 +1,146 @@ +/usr/local/corex/lib/python3/dist-packages/torch/cuda/__init__.py:51: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you. + import pynvml # type: ignore[import] +INFO 07-14 17:16:18 importing.py:10] Triton not installed; certain GPU-related functions will not be available. +2026-07-14 17:16:20.381623: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. +2026-07-14 17:16:20.432980: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. +To enable the following instructions: SSE3 SSE4.1 SSE4.2 AVX AVX2 AVX512F AVX512_VNNI AVX512_BF16 AVX_VNNI AMX_TILE AMX_INT8 AMX_BF16 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags. +WARNING:tensorflow:Deprecation warnings have been disabled. Set TF_ENABLE_DEPRECATION_WARNINGS=1 to re-enable them. +INFO 07-14 17:16:25 api_server.py:530] vLLM API server version 0.6.3 +INFO 07-14 17:16:25 api_server.py:531] args: Namespace(host='0.0.0.0', port=1111, uvicorn_log_level='info', allow_credentials=False, allowed_origins=['*'], allowed_methods=['*'], allowed_headers=['*'], api_key=None, lora_modules=None, prompt_adapters=None, chat_template=None, response_role='assistant', ssl_keyfile=None, ssl_certfile=None, ssl_ca_certs=None, ssl_cert_reqs=0, root_path=None, middleware=[], return_tokens_as_token_ids=False, disable_frontend_multiprocessing=True, enable_auto_tool_choice=True, tool_call_parser='qwen3_coder', tool_parser_plugin='', reasoning_parser='qwen3', model='/root/public-storage/models/Qwen/Qwen3.6-35B-A3B', tokenizer=None, skip_tokenizer_init=False, revision=None, code_revision=None, tokenizer_revision=None, tokenizer_mode='auto', trust_remote_code=True, download_dir=None, load_format='auto', config_format='auto', dtype='auto', kv_cache_dtype='auto', quantization_param_path=None, max_model_len=100000, guided_decoding_backend='outlines', distributed_executor_backend=None, worker_use_ray=False, pipeline_parallel_size=1, tensor_parallel_size=4, max_parallel_loading_workers=None, ray_workers_use_nsight=False, block_size=16, enable_prefix_caching=True, disable_sliding_window=False, use_v2_block_manager=True, num_lookahead_slots=0, seed=0, swap_space=4, cpu_offload_gb=0, gpu_memory_utilization=0.9, num_gpu_blocks_override=None, max_num_batched_tokens=8192, max_num_seqs=2, max_logprobs=20, disable_log_stats=False, quantization=None, rope_scaling=None, rope_theta=None, enforce_eager=True, max_context_len_to_capture=None, max_seq_len_to_capture=8192, disable_custom_all_reduce=False, tokenizer_pool_size=0, tokenizer_pool_type='ray', tokenizer_pool_extra_config=None, limit_mm_per_prompt=None, mm_processor_kwargs=None, enable_lora=False, max_loras=1, max_lora_rank=16, lora_extra_vocab_size=256, lora_dtype='auto', long_lora_scaling_factors=None, max_cpu_loras=None, fully_sharded_loras=False, enable_prompt_adapter=False, max_prompt_adapters=1, max_prompt_adapter_token=0, device='auto', num_scheduler_steps=1, multi_step_stream_outputs=True, scheduler_delay_factor=0.0, enable_chunked_prefill=True, speculative_model=None, speculative_model_quantization=None, num_speculative_tokens=None, speculative_disable_mqa_scorer=False, speculative_draft_tensor_parallel_size=None, speculative_max_model_len=None, speculative_disable_by_batch_size=None, ngram_prompt_lookup_max=None, ngram_prompt_lookup_min=None, spec_decoding_acceptance_method='rejection_sampler', typical_acceptance_sampler_posterior_threshold=None, typical_acceptance_sampler_posterior_alpha=None, disable_logprobs_during_spec_decoding=None, model_loader_extra_config=None, ignore_patterns=[], preemption_mode=None, served_model_name=['llm'], qlora_adapter_name_or_path=None, otlp_traces_endpoint=None, collect_detailed_traces=None, disable_async_output_proc=False, override_neuron_config=None, scheduling_policy='fcfs', disable_log_requests=True, max_log_len=None, disable_fastapi_docs=False) +INFO 07-14 17:16:25 config.py:1670] Downcasting torch.float32 to torch.float16. +INFO 07-14 17:16:37 config.py:887] Defaulting to use mp for distributed inference +INFO 07-14 17:16:37 config.py:1005] Chunked prefill is enabled with max_num_batched_tokens=8192. +WARNING 07-14 17:16:37 config.py:380] To see benefits of async output processing, enable CUDA graph. Since, enforce-eager is enabled, async output processor cannot be used +INFO 07-14 17:16:37 llm_engine.py:237] Initializing an LLM engine (v0.6.3) with config: model='/root/public-storage/models/Qwen/Qwen3.6-35B-A3B', speculative_config=None, tokenizer='/root/public-storage/models/Qwen/Qwen3.6-35B-A3B', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, override_neuron_config=None, rope_scaling=None, rope_theta=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.float16, max_seq_len=100000, download_dir=None, load_format=LoadFormat.AUTO, tensor_parallel_size=4, pipeline_parallel_size=1, disable_custom_all_reduce=False, quantization=None, enforce_eager=True, kv_cache_dtype=auto, quantization_param_path=None, device_config=cuda, decoding_config=DecodingConfig(guided_decoding_backend='outlines'), observability_config=ObservabilityConfig(otlp_traces_endpoint=None, collect_model_forward_time=False, collect_model_execute_time=False), seed=0, served_model_name=llm, use_v2_block_manager=True, num_scheduler_steps=1, chunked_prefill_enabled=True multi_step_stream_outputs=True, enable_prefix_caching=True, use_async_output_proc=False, use_cached_outputs=False, mm_processor_kwargs=None) +WARNING 07-14 17:16:38 multiproc_gpu_executor.py:53] Reducing Torch parallelism from 64 threads to 1 to avoid unnecessary CPU contention. Set OMP_NUM_THREADS in the external environment to tune this value as needed. +INFO 07-14 17:16:38 custom_cache_manager.py:17] Setting Triton cache manager to: vllm.triton_utils.custom_cache_manager:CustomCacheManager +INFO 07-14 17:16:38 selector.py:266] Cannot use FlashAttention-2 backend because the vllm.vllm_flash_attn package is not found. Make sure that vllm_flash_attn was built and installed (on by default). +INFO 07-14 17:16:38 selector.py:115] Using XFormers backend. +/usr/local/corex/lib/python3/dist-packages/torch/cuda/__init__.py:51: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you. + import pynvml # type: ignore[import] +/usr/local/corex/lib/python3/dist-packages/torch/cuda/__init__.py:51: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you. + import pynvml # type: ignore[import] +/usr/local/corex/lib/python3/dist-packages/torch/cuda/__init__.py:51: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you. + import pynvml # type: ignore[import] +INFO 07-14 17:16:40 importing.py:10] Triton not installed; certain GPU-related functions will not be available. +INFO 07-14 17:16:40 importing.py:10] Triton not installed; certain GPU-related functions will not be available. +INFO 07-14 17:16:40 importing.py:10] Triton not installed; certain GPU-related functions will not be available. +WARNING:tensorflow:Deprecation warnings have been disabled. Set TF_ENABLE_DEPRECATION_WARNINGS=1 to re-enable them. +WARNING:tensorflow:Deprecation warnings have been disabled. Set TF_ENABLE_DEPRECATION_WARNINGS=1 to re-enable them. +WARNING:tensorflow:Deprecation warnings have been disabled. Set TF_ENABLE_DEPRECATION_WARNINGS=1 to re-enable them. +(VllmWorkerProcess pid=23518) INFO 07-14 17:16:47 selector.py:266] Cannot use FlashAttention-2 backend because the vllm.vllm_flash_attn package is not found. Make sure that vllm_flash_attn was built and installed (on by default). +(VllmWorkerProcess pid=23518) INFO 07-14 17:16:47 selector.py:115] Using XFormers backend. +(VllmWorkerProcess pid=23517) INFO 07-14 17:16:47 selector.py:266] Cannot use FlashAttention-2 backend because the vllm.vllm_flash_attn package is not found. Make sure that vllm_flash_attn was built and installed (on by default). +(VllmWorkerProcess pid=23517) INFO 07-14 17:16:47 selector.py:115] Using XFormers backend. +(VllmWorkerProcess pid=23519) INFO 07-14 17:16:47 selector.py:266] Cannot use FlashAttention-2 backend because the vllm.vllm_flash_attn package is not found. Make sure that vllm_flash_attn was built and installed (on by default). +(VllmWorkerProcess pid=23519) INFO 07-14 17:16:47 selector.py:115] Using XFormers backend. +(VllmWorkerProcess pid=23517) INFO 07-14 17:16:47 multiproc_worker_utils.py:216] Worker ready; awaiting tasks +(VllmWorkerProcess pid=23518) INFO 07-14 17:16:47 multiproc_worker_utils.py:216] Worker ready; awaiting tasks +(VllmWorkerProcess pid=23519) INFO 07-14 17:16:47 multiproc_worker_utils.py:216] Worker ready; awaiting tasks +INFO 07-14 17:16:48 shm_broadcast.py:242] vLLM message queue communication handle: Handle(connect_ip='127.0.0.1', local_reader_ranks=[1, 2, 3], buffer=, local_subscribe_port=39903, remote_subscribe_port=None) +INFO 07-14 17:16:48 model_runner.py:1112] Starting to load model /root/public-storage/models/Qwen/Qwen3.6-35B-A3B... +(VllmWorkerProcess pid=23517) INFO 07-14 17:16:48 model_runner.py:1112] Starting to load model /root/public-storage/models/Qwen/Qwen3.6-35B-A3B... +(VllmWorkerProcess pid=23518) INFO 07-14 17:16:48 model_runner.py:1112] Starting to load model /root/public-storage/models/Qwen/Qwen3.6-35B-A3B... +(VllmWorkerProcess pid=23519) INFO 07-14 17:16:48 model_runner.py:1112] Starting to load model /root/public-storage/models/Qwen/Qwen3.6-35B-A3B... +(VllmWorkerProcess pid=23517) INFO 07-14 17:16:48 selector.py:266] Cannot use FlashAttention-2 backend because the vllm.vllm_flash_attn package is not found. Make sure that vllm_flash_attn was built and installed (on by default). +(VllmWorkerProcess pid=23517) INFO 07-14 17:16:48 selector.py:115] Using XFormers backend. +(VllmWorkerProcess pid=23519) INFO 07-14 17:16:48 selector.py:266] Cannot use FlashAttention-2 backend because the vllm.vllm_flash_attn package is not found. Make sure that vllm_flash_attn was built and installed (on by default). +(VllmWorkerProcess pid=23519) INFO 07-14 17:16:48 selector.py:115] Using XFormers backend. +(VllmWorkerProcess pid=23518) INFO 07-14 17:16:48 selector.py:266] Cannot use FlashAttention-2 backend because the vllm.vllm_flash_attn package is not found. Make sure that vllm_flash_attn was built and installed (on by default). +(VllmWorkerProcess pid=23518) INFO 07-14 17:16:48 selector.py:115] Using XFormers backend. +INFO 07-14 17:16:48 selector.py:266] Cannot use FlashAttention-2 backend because the vllm.vllm_flash_attn package is not found. Make sure that vllm_flash_attn was built and installed (on by default). +INFO 07-14 17:16:48 selector.py:115] Using XFormers backend. + Loading safetensors checkpoint shards: 0% Completed | 0/26 [00:00, local_subscribe_port=56475, remote_subscribe_port=None) +INFO 07-14 17:14:14 model_runner.py:1112] Starting to load model /root/public-storage/models/Qwen/Qwen3.6-35B-A3B... +(VllmWorkerProcess pid=22419) INFO 07-14 17:14:14 model_runner.py:1112] Starting to load model /root/public-storage/models/Qwen/Qwen3.6-35B-A3B... +(VllmWorkerProcess pid=22420) INFO 07-14 17:14:14 model_runner.py:1112] Starting to load model /root/public-storage/models/Qwen/Qwen3.6-35B-A3B... +(VllmWorkerProcess pid=22421) INFO 07-14 17:14:14 model_runner.py:1112] Starting to load model /root/public-storage/models/Qwen/Qwen3.6-35B-A3B... +INFO 07-14 17:14:14 selector.py:266] Cannot use FlashAttention-2 backend because the vllm.vllm_flash_attn package is not found. Make sure that vllm_flash_attn was built and installed (on by default). +INFO 07-14 17:14:14 selector.py:115] Using XFormers backend. +(VllmWorkerProcess pid=22421) INFO 07-14 17:14:14 selector.py:266] Cannot use FlashAttention-2 backend because the vllm.vllm_flash_attn package is not found. Make sure that vllm_flash_attn was built and installed (on by default). +(VllmWorkerProcess pid=22421) INFO 07-14 17:14:14 selector.py:115] Using XFormers backend. +(VllmWorkerProcess pid=22419) INFO 07-14 17:14:14 selector.py:266] Cannot use FlashAttention-2 backend because the vllm.vllm_flash_attn package is not found. Make sure that vllm_flash_attn was built and installed (on by default). +(VllmWorkerProcess pid=22419) INFO 07-14 17:14:14 selector.py:115] Using XFormers backend. +(VllmWorkerProcess pid=22420) INFO 07-14 17:14:14 selector.py:266] Cannot use FlashAttention-2 backend because the vllm.vllm_flash_attn package is not found. Make sure that vllm_flash_attn was built and installed (on by default). +(VllmWorkerProcess pid=22420) INFO 07-14 17:14:14 selector.py:115] Using XFormers backend. + Loading safetensors checkpoint shards: 0% Completed | 0/26 [00:00, local_subscribe_port=41787, remote_subscribe_port=None) +INFO 07-14 17:11:36 model_runner.py:1112] Starting to load model /root/public-storage/models/Qwen/Qwen3.6-35B-A3B... +(VllmWorkerProcess pid=21302) INFO 07-14 17:11:36 model_runner.py:1112] Starting to load model /root/public-storage/models/Qwen/Qwen3.6-35B-A3B... +(VllmWorkerProcess pid=21301) INFO 07-14 17:11:36 model_runner.py:1112] Starting to load model /root/public-storage/models/Qwen/Qwen3.6-35B-A3B... +(VllmWorkerProcess pid=21303) INFO 07-14 17:11:36 model_runner.py:1112] Starting to load model /root/public-storage/models/Qwen/Qwen3.6-35B-A3B... +INFO 07-14 17:11:36 selector.py:266] Cannot use FlashAttention-2 backend because the vllm.vllm_flash_attn package is not found. Make sure that vllm_flash_attn was built and installed (on by default). +INFO 07-14 17:11:36 selector.py:115] Using XFormers backend. +(VllmWorkerProcess pid=21302) INFO 07-14 17:11:36 selector.py:266] Cannot use FlashAttention-2 backend because the vllm.vllm_flash_attn package is not found. Make sure that vllm_flash_attn was built and installed (on by default). +(VllmWorkerProcess pid=21302) INFO 07-14 17:11:36 selector.py:115] Using XFormers backend. +(VllmWorkerProcess pid=21301) INFO 07-14 17:11:36 selector.py:266] Cannot use FlashAttention-2 backend because the vllm.vllm_flash_attn package is not found. Make sure that vllm_flash_attn was built and installed (on by default). +(VllmWorkerProcess pid=21301) INFO 07-14 17:11:36 selector.py:115] Using XFormers backend. +(VllmWorkerProcess pid=21303) INFO 07-14 17:11:36 selector.py:266] Cannot use FlashAttention-2 backend because the vllm.vllm_flash_attn package is not found. Make sure that vllm_flash_attn was built and installed (on by default). +(VllmWorkerProcess pid=21303) INFO 07-14 17:11:36 selector.py:115] Using XFormers backend. + Loading safetensors checkpoint shards: 0% Completed | 0/26 [00:00, local_subscribe_port=47013, remote_subscribe_port=None) +(VllmWorkerProcess pid=19083) INFO 07-14 17:06:19 model_runner.py:1112] Starting to load model /root/public-storage/models/Qwen/Qwen3.6-35B-A3B... +INFO 07-14 17:06:19 model_runner.py:1112] Starting to load model /root/public-storage/models/Qwen/Qwen3.6-35B-A3B... +(VllmWorkerProcess pid=19084) INFO 07-14 17:06:19 model_runner.py:1112] Starting to load model /root/public-storage/models/Qwen/Qwen3.6-35B-A3B... +(VllmWorkerProcess pid=19085) INFO 07-14 17:06:19 model_runner.py:1112] Starting to load model /root/public-storage/models/Qwen/Qwen3.6-35B-A3B... +INFO 07-14 17:06:19 selector.py:266] Cannot use FlashAttention-2 backend because the vllm.vllm_flash_attn package is not found. Make sure that vllm_flash_attn was built and installed (on by default). +INFO 07-14 17:06:19 selector.py:115] Using XFormers backend. +(VllmWorkerProcess pid=19083) INFO 07-14 17:06:19 selector.py:266] Cannot use FlashAttention-2 backend because the vllm.vllm_flash_attn package is not found. Make sure that vllm_flash_attn was built and installed (on by default). +(VllmWorkerProcess pid=19085) INFO 07-14 17:06:19 selector.py:266] Cannot use FlashAttention-2 backend because the vllm.vllm_flash_attn package is not found. Make sure that vllm_flash_attn was built and installed (on by default). +(VllmWorkerProcess pid=19083) INFO 07-14 17:06:19 selector.py:115] Using XFormers backend. +(VllmWorkerProcess pid=19085) INFO 07-14 17:06:19 selector.py:115] Using XFormers backend. +(VllmWorkerProcess pid=19084) INFO 07-14 17:06:19 selector.py:266] Cannot use FlashAttention-2 backend because the vllm.vllm_flash_attn package is not found. Make sure that vllm_flash_attn was built and installed (on by default). +(VllmWorkerProcess pid=19084) INFO 07-14 17:06:19 selector.py:115] Using XFormers backend. + Loading safetensors checkpoint shards: 0% Completed | 0/26 [00:00, local_subscribe_port=34695, remote_subscribe_port=None) +INFO 07-14 17:08:59 model_runner.py:1112] Starting to load model /root/public-storage/models/Qwen/Qwen3.6-35B-A3B... +(VllmWorkerProcess pid=20202) INFO 07-14 17:08:59 model_runner.py:1112] Starting to load model /root/public-storage/models/Qwen/Qwen3.6-35B-A3B... +(VllmWorkerProcess pid=20201) INFO 07-14 17:08:59 model_runner.py:1112] Starting to load model /root/public-storage/models/Qwen/Qwen3.6-35B-A3B... +(VllmWorkerProcess pid=20203) INFO 07-14 17:08:59 model_runner.py:1112] Starting to load model /root/public-storage/models/Qwen/Qwen3.6-35B-A3B... +(VllmWorkerProcess pid=20202) INFO 07-14 17:08:59 selector.py:266] Cannot use FlashAttention-2 backend because the vllm.vllm_flash_attn package is not found. Make sure that vllm_flash_attn was built and installed (on by default). +(VllmWorkerProcess pid=20202) INFO 07-14 17:08:59 selector.py:115] Using XFormers backend. +INFO 07-14 17:08:59 selector.py:266] Cannot use FlashAttention-2 backend because the vllm.vllm_flash_attn package is not found. Make sure that vllm_flash_attn was built and installed (on by default). +INFO 07-14 17:08:59 selector.py:115] Using XFormers backend. +(VllmWorkerProcess pid=20203) INFO 07-14 17:08:59 selector.py:266] Cannot use FlashAttention-2 backend because the vllm.vllm_flash_attn package is not found. Make sure that vllm_flash_attn was built and installed (on by default). +(VllmWorkerProcess pid=20203) INFO 07-14 17:08:59 selector.py:115] Using XFormers backend. +(VllmWorkerProcess pid=20201) INFO 07-14 17:08:59 selector.py:266] Cannot use FlashAttention-2 backend because the vllm.vllm_flash_attn package is not found. Make sure that vllm_flash_attn was built and installed (on by default). +(VllmWorkerProcess pid=20201) INFO 07-14 17:08:59 selector.py:115] Using XFormers backend. + Loading safetensors checkpoint shards: 0% Completed | 0/26 [00:00/dev/null || true + pkill -f "VllmWorkerProcess" 2>/dev/null || true + sleep 8 +} + +wait_health() { + local deadline=$((SECONDS + 1200)) + while (( SECONDS < deadline )); do + if python3 - <<'PY' +import urllib.request +try: + urllib.request.urlopen("http://127.0.0.1:1111/health", timeout=5) +except Exception: + raise SystemExit(1) +PY + then + echo "[matrix] health ok" + return 0 + fi + sleep 10 + echo "[matrix] waiting health: ${SECONDS}s" + done + echo "[matrix] ERROR: service did not become healthy" >&2 + return 1 +} + +start_server() { + local impl="$1" + local max_t="$2" + local label="$3" + local server_log="$OUT_DIR/${label}_server.log" + echo "[matrix] starting $label" + ( + cd /root + ENGINEX_MOE_TINY_IMPL="$impl" \ + ENGINEX_MOE_TINY_MAX="$max_t" \ + VLLM_ENGINE_ITERATION_TIMEOUT_S=3600 \ + PYTHONPATH="/usr/local/corex/lib/python3/dist-packages:/usr/local/corex/lib64/python3/dist-packages:${PYTHONPATH:-}" \ + LD_LIBRARY_PATH="/usr/local/openmpi/lib:/usr/local/corex-3.2.3/lib64:/usr/local/corex/lib64:/usr/local/corex/lib:/usr/local/iluvatar/lib64:${LD_LIBRARY_PATH:-}" \ + nohup python3 -m vllm.entrypoints.openai.api_server \ + --model "$MODEL_PATH" \ + --host 0.0.0.0 \ + --port 1111 \ + --served-model-name llm \ + --max-model-len 100000 \ + --enforce-eager \ + --trust-remote-code \ + -tp 4 \ + --gpu-memory-utilization 0.90 \ + --max-num-seqs 2 \ + --disable-log-requests \ + --disable-frontend-multiprocessing \ + --max-num-batched-tokens 8192 \ + --enable-chunked-prefill \ + --enable-prefix-caching \ + --enable-auto-tool-choice \ + --tool-call-parser qwen3_coder \ + --reasoning-parser qwen3 \ + > "$server_log" 2>&1 & + echo $! > "$OUT_DIR/${label}_server.pid" + ) + wait_health +} + +run_case() { + local impl="$1" + local max_t="$2" + local label="${impl}_max${max_t}" + local result="$OUT_DIR/${label}_short_c2_t128_r4.json" + local monitor="$OUT_DIR/${label}_ixsmi.json" + local warmup="$OUT_DIR/${label}_warmup.json" + + stop_server + start_server "$impl" "$max_t" "$label" + + echo "[matrix] warmup $label" + python3 "$BENCH_SCRIPT" \ + --url "$BASE_URL" \ + --model llm \ + --label "${label}_warmup" \ + --prompt-mode short \ + --concurrency 1 \ + --requests 1 \ + --max-tokens 32 \ + --timeout 900 \ + --out "$warmup" + + echo "[matrix] benchmark $label" + python3 "$BENCH_SCRIPT" \ + --url "$BASE_URL" \ + --model llm \ + --label "${label}_short_c2_t128_r4" \ + --prompt-mode short \ + --concurrency 2 \ + --requests 4 \ + --max-tokens 128 \ + --timeout 900 \ + --monitor-out "$monitor" \ + --monitor-interval 1.0 \ + --out "$result" + + stop_server +} + +run_case tokenwise 4 +run_case tokenwise 8 +run_case tokenwise 16 +run_case bmm 8 +run_case bmm 16 + +python3 - "$OUT_DIR" <<'PY' +import json +import sys +from pathlib import Path + +out_dir = Path(sys.argv[1]) +rows = [] +for path in sorted(out_dir.glob("*_short_c2_t128_r4.json")): + data = json.loads(path.read_text(encoding="utf-8")) + rows.append({ + "label": data.get("label"), + "success_rate": data.get("success_rate"), + "ttft_p90_sec": data.get("ttft_p90_sec"), + "output_tps_p10_per_request": data.get("output_tps_p10_per_request"), + "output_tps_p50_per_request": data.get("output_tps_p50_per_request"), + "aggregate_output_tps": data.get("aggregate_output_tps"), + "avg_gpu_util_pct": (data.get("monitor") or {}).get("avg_gpu_util_pct"), + "max_gpu_util_pct": (data.get("monitor") or {}).get("max_gpu_util_pct"), + }) + +summary = { + "out_dir": str(out_dir), + "rows": rows, +} +(out_dir / "summary.json").write_text( + json.dumps(summary, ensure_ascii=False, indent=2), + encoding="utf-8", +) + +lines = [ + "# MoE tiny-batch matrix", + "", + "| case | success | TTFT P90 | Output TPS P10 | Output TPS P50 | Aggregate Output TPS | Avg GPU util | Max GPU util |", + "| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: |", +] +for r in rows: + lines.append( + f"| {r['label']} | {r['success_rate']:.2%} | " + f"{r['ttft_p90_sec']:.3f} | {r['output_tps_p10_per_request']:.3f} | " + f"{r['output_tps_p50_per_request']:.3f} | {r['aggregate_output_tps']:.3f} | " + f"{r['avg_gpu_util_pct']:.1f} | {r['max_gpu_util_pct']:.1f} |" + ) +(out_dir / "summary.md").write_text("\n".join(lines) + "\n", encoding="utf-8") +print(json.dumps(summary, ensure_ascii=False, indent=2)) +PY + +echo "[matrix] done" +echo "[matrix] summary=$OUT_DIR/summary.md"