Files
qwen36_01/worklogs/remote_results/2026-07-15-moe-tiny-matrix/tokenwise_max8_ixsmi.json

1157 lines
84 KiB
JSON
Raw Normal View History

2026-07-15 01:28:06 +08:00
[
{
"ts": "2026-07-14T17:10:05",
"ok": true,
"raw": "Timestamp Tue Jul 14 17:10:05 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 30C P0 53W / 250W | 28955MiB / 32768MiB | 0% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 52W / 250W | 28894MiB / 32768MiB | 0% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 52W / 250W | 28898MiB / 32768MiB | 0% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 51W / 250W | 28758MiB / 32768MiB | 0% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n",
"parsed": [
{
"power_w": 53,
"power_cap_w": 250,
"mem_mib": 28955,
"mem_total_mib": 32768,
"util_pct": 0
},
{
"power_w": 52,
"power_cap_w": 250,
"mem_mib": 28894,
"mem_total_mib": 32768,
"util_pct": 0
},
{
"power_w": 52,
"power_cap_w": 250,
"mem_mib": 28898,
"mem_total_mib": 32768,
"util_pct": 0
},
{
"power_w": 51,
"power_cap_w": 250,
"mem_mib": 28758,
"mem_total_mib": 32768,
"util_pct": 0
}
]
},
{
"ts": "2026-07-14T17:10:06",
"ok": true,
"raw": "Timestamp Tue Jul 14 17:10:06 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 30C P0 49W / 250W | 28955MiB / 32768MiB | 31% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 33C P0 49W / 250W | 28894MiB / 32768MiB | 55% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 50W / 250W | 28898MiB / 32768MiB | 53% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 49W / 250W | 28758MiB / 32768MiB | 73% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n",
"parsed": [
{
"power_w": 49,
"power_cap_w": 250,
"mem_mib": 28955,
"mem_total_mib": 32768,
"util_pct": 31
},
{
"power_w": 49,
"power_cap_w": 250,
"mem_mib": 28894,
"mem_total_mib": 32768,
"util_pct": 55
},
{
"power_w": 50,
"power_cap_w": 250,
"mem_mib": 28898,
"mem_total_mib": 32768,
"util_pct": 53
},
{
"power_w": 49,
"power_cap_w": 250,
"mem_mib": 28758,
"mem_total_mib": 32768,
"util_pct": 73
}
]
},
{
"ts": "2026-07-14T17:10:08",
"ok": true,
"raw": "Timestamp Tue Jul 14 17:10:08 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 55W / 250W | 28955MiB / 32768MiB | 74% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 33C P0 55W / 250W | 28894MiB / 32768MiB | 50% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 56W / 250W | 28898MiB / 32768MiB | 52% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 55W / 250W | 28758MiB / 32768MiB | 73% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n",
"parsed": [
{
"power_w": 55,
"power_cap_w": 250,
"mem_mib": 28955,
"mem_total_mib": 32768,
"util_pct": 74
},
{
"power_w": 55,
"power_cap_w": 250,
"mem_mib": 28894,
"mem_total_mib": 32768,
"util_pct": 50
},
{
"power_w": 56,
"power_cap_w": 250,
"mem_mib": 28898,
"mem_total_mib": 32768,
"util_pct": 52
},
{
"power_w": 55,
"power_cap_w": 250,
"mem_mib": 28758,
"mem_total_mib": 32768,
"util_pct": 73
}
]
},
{
"ts": "2026-07-14T17:10:09",
"ok": true,
"raw": "Timestamp Tue Jul 14 17:10:09 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 57W / 250W | 28955MiB / 32768MiB | 87% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 33C P0 57W / 250W | 28894MiB / 32768MiB | 60% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 57W / 250W | 28898MiB / 32768MiB | 61% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 56W / 250W | 28758MiB / 32768MiB | 60% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n",
"parsed": [
{
"power_w": 57,
"power_cap_w": 250,
"mem_mib": 28955,
"mem_total_mib": 32768,
"util_pct": 87
},
{
"power_w": 57,
"power_cap_w": 250,
"mem_mib": 28894,
"mem_total_mib": 32768,
"util_pct": 60
},
{
"power_w": 57,
"power_cap_w": 250,
"mem_mib": 28898,
"mem_total_mib": 32768,
"util_pct": 61
},
{
"power_w": 56,
"power_cap_w": 250,
"mem_mib": 28758,
"mem_total_mib": 32768,
"util_pct": 60
}
]
},
{
"ts": "2026-07-14T17:10:10",
"ok": true,
"raw": "Timestamp Tue Jul 14 17:10:10 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 56W / 250W | 28955MiB / 32768MiB | 63% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 33C P0 56W / 250W | 28894MiB / 32768MiB | 56% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 56W / 250W | 28898MiB / 32768MiB | 67% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 55W / 250W | 28758MiB / 32768MiB | 62% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n",
"parsed": [
{
"power_w": 56,
"power_cap_w": 250,
"mem_mib": 28955,
"mem_total_mib": 32768,
"util_pct": 63
},
{
"power_w": 56,
"power_cap_w": 250,
"mem_mib": 28894,
"mem_total_mib": 32768,
"util_pct": 56
},
{
"power_w": 56,
"power_cap_w": 250,
"mem_mib": 28898,
"mem_total_mib": 32768,
"util_pct": 67
},
{
"power_w": 55,
"power_cap_w": 250,
"mem_mib": 28758,
"mem_total_mib": 32768,
"util_pct": 62
}
]
},
{
"ts": "2026-07-14T17:10:12",
"ok": true,
"raw": "Timestamp Tue Jul 14 17:10:12 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 56W / 250W | 28955MiB / 32768MiB | 46% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 33C P0 56W / 250W | 28894MiB / 32768MiB | 52% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 56W / 250W | 28898MiB / 32768MiB | 54% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 55W / 250W | 28758MiB / 32768MiB | 59% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n",
"parsed": [
{
"power_w": 56,
"power_cap_w": 250,
"mem_mib": 28955,
"mem_total_mib": 32768,
"util_pct": 46
},
{
"power_w": 56,
"power_cap_w": 250,
"mem_mib": 28894,
"mem_total_mib": 32768,
"util_pct": 52
},
{
"power_w": 56,
"power_cap_w": 250,
"mem_mib": 28898,
"mem_total_mib": 32768,
"util_pct": 54
},
{
"power_w": 55,
"power_cap_w": 250,
"mem_mib": 28758,
"mem_total_mib": 32768,
"util_pct": 59
}
]
},
{
"ts": "2026-07-14T17:10:13",
"ok": true,
"raw": "Timestamp Tue Jul 14 17:10:13 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 57W / 250W | 28955MiB / 32768MiB | 70% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 33C P0 57W / 250W | 28894MiB / 32768MiB | 66% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 57W / 250W | 28898MiB / 32768MiB | 74% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 56W / 250W | 28758MiB / 32768MiB | 70% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n",
"parsed": [
{
"power_w": 57,
"power_cap_w": 250,
"mem_mib": 28955,
"mem_total_mib": 32768,
"util_pct": 70
},
{
"power_w": 57,
"power_cap_w": 250,
"mem_mib": 28894,
"mem_total_mib": 32768,
"util_pct": 66
},
{
"power_w": 57,
"power_cap_w": 250,
"mem_mib": 28898,
"mem_total_mib": 32768,
"util_pct": 74
},
{
"power_w": 56,
"power_cap_w": 250,
"mem_mib": 28758,
"mem_total_mib": 32768,
"util_pct": 70
}
]
},
{
"ts": "2026-07-14T17:10:14",
"ok": true,
"raw": "Timestamp Tue Jul 14 17:10:14 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 57W / 250W | 28955MiB / 32768MiB | 50% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 33C P0 56W / 250W | 28894MiB / 32768MiB | 52% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 57W / 250W | 28898MiB / 32768MiB | 41% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 56W / 250W | 28758MiB / 32768MiB | 65% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n",
"parsed": [
{
"power_w": 57,
"power_cap_w": 250,
"mem_mib": 28955,
"mem_total_mib": 32768,
"util_pct": 50
},
{
"power_w": 56,
"power_cap_w": 250,
"mem_mib": 28894,
"mem_total_mib": 32768,
"util_pct": 52
},
{
"power_w": 57,
"power_cap_w": 250,
"mem_mib": 28898,
"mem_total_mib": 32768,
"util_pct": 41
},
{
"power_w": 56,
"power_cap_w": 250,
"mem_mib": 28758,
"mem_total_mib": 32768,
"util_pct": 65
}
]
},
{
"ts": "2026-07-14T17:10:16",
"ok": true,
"raw": "Timestamp Tue Jul 14 17:10:16 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 56W / 250W | 28955MiB / 32768MiB | 47% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 33C P0 56W / 250W | 28894MiB / 32768MiB | 48% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 57W / 250W | 28898MiB / 32768MiB | 64% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 56W / 250W | 28758MiB / 32768MiB | 46% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n",
"parsed": [
{
"power_w": 56,
"power_cap_w": 250,
"mem_mib": 28955,
"mem_total_mib": 32768,
"util_pct": 47
},
{
"power_w": 56,
"power_cap_w": 250,
"mem_mib": 28894,
"mem_total_mib": 32768,
"util_pct": 48
},
{
"power_w": 57,
"power_cap_w": 250,
"mem_mib": 28898,
"mem_total_mib": 32768,
"util_pct": 64
},
{
"power_w": 56,
"power_cap_w": 250,
"mem_mib": 28758,
"mem_total_mib": 32768,
"util_pct": 46
}
]
},
{
"ts": "2026-07-14T17:10:17",
"ok": true,
"raw": "Timestamp Tue Jul 14 17:10:17 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 57W / 250W | 28955MiB / 32768MiB | 64% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 33C P0 56W / 250W | 28894MiB / 32768MiB | 64% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 57W / 250W | 28898MiB / 32768MiB | 64% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 56W / 250W | 28758MiB / 32768MiB | 55% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n",
"parsed": [
{
"power_w": 57,
"power_cap_w": 250,
"mem_mib": 28955,
"mem_total_mib": 32768,
"util_pct": 64
},
{
"power_w": 56,
"power_cap_w": 250,
"mem_mib": 28894,
"mem_total_mib": 32768,
"util_pct": 64
},
{
"power_w": 57,
"power_cap_w": 250,
"mem_mib": 28898,
"mem_total_mib": 32768,
"util_pct": 64
},
{
"power_w": 56,
"power_cap_w": 250,
"mem_mib": 28758,
"mem_total_mib": 32768,
"util_pct": 55
}
]
},
{
"ts": "2026-07-14T17:10:18",
"ok": true,
"raw": "Timestamp Tue Jul 14 17:10:18 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 57W / 250W | 28955MiB / 32768MiB | 60% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 33C P0 56W / 250W | 28894MiB / 32768MiB | 69% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 57W / 250W | 28898MiB / 32768MiB | 67% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 56W / 250W | 28758MiB / 32768MiB | 36% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n",
"parsed": [
{
"power_w": 57,
"power_cap_w": 250,
"mem_mib": 28955,
"mem_total_mib": 32768,
"util_pct": 60
},
{
"power_w": 56,
"power_cap_w": 250,
"mem_mib": 28894,
"mem_total_mib": 32768,
"util_pct": 69
},
{
"power_w": 57,
"power_cap_w": 250,
"mem_mib": 28898,
"mem_total_mib": 32768,
"util_pct": 67
},
{
"power_w": 56,
"power_cap_w": 250,
"mem_mib": 28758,
"mem_total_mib": 32768,
"util_pct": 36
}
]
},
{
"ts": "2026-07-14T17:10:19",
"ok": true,
"raw": "Timestamp Tue Jul 14 17:10:20 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 57W / 250W | 28955MiB / 32768MiB | 52% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 33C P0 57W / 250W | 28894MiB / 32768MiB | 68% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 57W / 250W | 28898MiB / 32768MiB | 45% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 56W / 250W | 28758MiB / 32768MiB | 49% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n",
"parsed": [
{
"power_w": 57,
"power_cap_w": 250,
"mem_mib": 28955,
"mem_total_mib": 32768,
"util_pct": 52
},
{
"power_w": 57,
"power_cap_w": 250,
"mem_mib": 28894,
"mem_total_mib": 32768,
"util_pct": 68
},
{
"power_w": 57,
"power_cap_w": 250,
"mem_mib": 28898,
"mem_total_mib": 32768,
"util_pct": 45
},
{
"power_w": 56,
"power_cap_w": 250,
"mem_mib": 28758,
"mem_total_mib": 32768,
"util_pct": 49
}
]
},
{
"ts": "2026-07-14T17:10:21",
"ok": true,
"raw": "Timestamp Tue Jul 14 17:10:21 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 56W / 250W | 28955MiB / 32768MiB | 49% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 33C P0 56W / 250W | 28894MiB / 32768MiB | 74% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 56W / 250W | 28898MiB / 32768MiB | 61% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 56W / 250W | 28758MiB / 32768MiB | 51% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n",
"parsed": [
{
"power_w": 56,
"power_cap_w": 250,
"mem_mib": 28955,
"mem_total_mib": 32768,
"util_pct": 49
},
{
"power_w": 56,
"power_cap_w": 250,
"mem_mib": 28894,
"mem_total_mib": 32768,
"util_pct": 74
},
{
"power_w": 56,
"power_cap_w": 250,
"mem_mib": 28898,
"mem_total_mib": 32768,
"util_pct": 61
},
{
"power_w": 56,
"power_cap_w": 250,
"mem_mib": 28758,
"mem_total_mib": 32768,
"util_pct": 51
}
]
},
{
"ts": "2026-07-14T17:10:22",
"ok": true,
"raw": "Timestamp Tue Jul 14 17:10:22 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 56W / 250W | 28955MiB / 32768MiB | 58% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 33C P0 56W / 250W | 28894MiB / 32768MiB | 55% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 57W / 250W | 28898MiB / 32768MiB | 39% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 55W / 250W | 28758MiB / 32768MiB | 58% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n",
"parsed": [
{
"power_w": 56,
"power_cap_w": 250,
"mem_mib": 28955,
"mem_total_mib": 32768,
"util_pct": 58
},
{
"power_w": 56,
"power_cap_w": 250,
"mem_mib": 28894,
"mem_total_mib": 32768,
"util_pct": 55
},
{
"power_w": 57,
"power_cap_w": 250,
"mem_mib": 28898,
"mem_total_mib": 32768,
"util_pct": 39
},
{
"power_w": 55,
"power_cap_w": 250,
"mem_mib": 28758,
"mem_total_mib": 32768,
"util_pct": 58
}
]
},
{
"ts": "2026-07-14T17:10:23",
"ok": true,
"raw": "Timestamp Tue Jul 14 17:10:23 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 56W / 250W | 28955MiB / 32768MiB | 63% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 33C P0 56W / 250W | 28894MiB / 32768MiB | 61% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 56W / 250W | 28898MiB / 32768MiB | 57% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 55W / 250W | 28758MiB / 32768MiB | 41% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n",
"parsed": [
{
"power_w": 56,
"power_cap_w": 250,
"mem_mib": 28955,
"mem_total_mib": 32768,
"util_pct": 63
},
{
"power_w": 56,
"power_cap_w": 250,
"mem_mib": 28894,
"mem_total_mib": 32768,
"util_pct": 61
},
{
"power_w": 56,
"power_cap_w": 250,
"mem_mib": 28898,
"mem_total_mib": 32768,
"util_pct": 57
},
{
"power_w": 55,
"power_cap_w": 250,
"mem_mib": 28758,
"mem_total_mib": 32768,
"util_pct": 41
}
]
},
{
"ts": "2026-07-14T17:10:25",
"ok": true,
"raw": "Timestamp Tue Jul 14 17:10:25 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 56W / 250W | 28955MiB / 32768MiB | 53% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 34C P0 56W / 250W | 28894MiB / 32768MiB | 54% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 57W / 250W | 28898MiB / 32768MiB | 22% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 56W / 250W | 28758MiB / 32768MiB | 68% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n",
"parsed": [
{
"power_w": 56,
"power_cap_w": 250,
"mem_mib": 28955,
"mem_total_mib": 32768,
"util_pct": 53
},
{
"power_w": 56,
"power_cap_w": 250,
"mem_mib": 28894,
"mem_total_mib": 32768,
"util_pct": 54
},
{
"power_w": 57,
"power_cap_w": 250,
"mem_mib": 28898,
"mem_total_mib": 32768,
"util_pct": 22
},
{
"power_w": 56,
"power_cap_w": 250,
"mem_mib": 28758,
"mem_total_mib": 32768,
"util_pct": 68
}
]
},
{
"ts": "2026-07-14T17:10:26",
"ok": true,
"raw": "Timestamp Tue Jul 14 17:10:26 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 57W / 250W | 28955MiB / 32768MiB | 60% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 34C P0 57W / 250W | 28894MiB / 32768MiB | 79% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 57W / 250W | 28898MiB / 32768MiB | 54% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 56W / 250W | 28758MiB / 32768MiB | 38% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n",
"parsed": [
{
"power_w": 57,
"power_cap_w": 250,
"mem_mib": 28955,
"mem_total_mib": 32768,
"util_pct": 60
},
{
"power_w": 57,
"power_cap_w": 250,
"mem_mib": 28894,
"mem_total_mib": 32768,
"util_pct": 79
},
{
"power_w": 57,
"power_cap_w": 250,
"mem_mib": 28898,
"mem_total_mib": 32768,
"util_pct": 54
},
{
"power_w": 56,
"power_cap_w": 250,
"mem_mib": 28758,
"mem_total_mib": 32768,
"util_pct": 38
}
]
},
{
"ts": "2026-07-14T17:10:27",
"ok": true,
"raw": "Timestamp Tue Jul 14 17:10:28 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 30C P0 50W / 250W | 28955MiB / 32768MiB | 25% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 33C P0 50W / 250W | 28894MiB / 32768MiB | 31% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 50W / 250W | 28898MiB / 32768MiB | 41% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 49W / 250W | 28758MiB / 32768MiB | 24% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n",
"parsed": [
{
"power_w": 50,
"power_cap_w": 250,
"mem_mib": 28955,
"mem_total_mib": 32768,
"util_pct": 25
},
{
"power_w": 50,
"power_cap_w": 250,
"mem_mib": 28894,
"mem_total_mib": 32768,
"util_pct": 31
},
{
"power_w": 50,
"power_cap_w": 250,
"mem_mib": 28898,
"mem_total_mib": 32768,
"util_pct": 41
},
{
"power_w": 49,
"power_cap_w": 250,
"mem_mib": 28758,
"mem_total_mib": 32768,
"util_pct": 24
}
]
},
{
"ts": "2026-07-14T17:10:29",
"ok": true,
"raw": "Timestamp Tue Jul 14 17:10:29 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 54W / 250W | 28955MiB / 32768MiB | 45% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 33C P0 54W / 250W | 28894MiB / 32768MiB | 100% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 54W / 250W | 28898MiB / 32768MiB | 63% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 54W / 250W | 28758MiB / 32768MiB | 54% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n",
"parsed": [
{
"power_w": 54,
"power_cap_w": 250,
"mem_mib": 28955,
"mem_total_mib": 32768,
"util_pct": 45
},
{
"power_w": 54,
"power_cap_w": 250,
"mem_mib": 28894,
"mem_total_mib": 32768,
"util_pct": 100
},
{
"power_w": 54,
"power_cap_w": 250,
"mem_mib": 28898,
"mem_total_mib": 32768,
"util_pct": 63
},
{
"power_w": 54,
"power_cap_w": 250,
"mem_mib": 28758,
"mem_total_mib": 32768,
"util_pct": 54
}
]
},
{
"ts": "2026-07-14T17:10:30",
"ok": true,
"raw": "Timestamp Tue Jul 14 17:10:30 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 56W / 250W | 28955MiB / 32768MiB | 60% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 34C P0 56W / 250W | 28894MiB / 32768MiB | 57% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 56W / 250W | 28898MiB / 32768MiB | 51% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 56W / 250W | 28758MiB / 32768MiB | 61% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n",
"parsed": [
{
"power_w": 56,
"power_cap_w": 250,
"mem_mib": 28955,
"mem_total_mib": 32768,
"util_pct": 60
},
{
"power_w": 56,
"power_cap_w": 250,
"mem_mib": 28894,
"mem_total_mib": 32768,
"util_pct": 57
},
{
"power_w": 56,
"power_cap_w": 250,
"mem_mib": 28898,
"mem_total_mib": 32768,
"util_pct": 51
},
{
"power_w": 56,
"power_cap_w": 250,
"mem_mib": 28758,
"mem_total_mib": 32768,
"util_pct": 61
}
]
},
{
"ts": "2026-07-14T17:10:32",
"ok": true,
"raw": "Timestamp Tue Jul 14 17:10:32 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 56W / 250W | 28955MiB / 32768MiB | 81% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 34C P0 56W / 250W | 28894MiB / 32768MiB | 64% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 56W / 250W | 28898MiB / 32768MiB | 50% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 55W / 250W | 28758MiB / 32768MiB | 69% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n",
"parsed": [
{
"power_w": 56,
"power_cap_w": 250,
"mem_mib": 28955,
"mem_total_mib": 32768,
"util_pct": 81
},
{
"power_w": 56,
"power_cap_w": 250,
"mem_mib": 28894,
"mem_total_mib": 32768,
"util_pct": 64
},
{
"power_w": 56,
"power_cap_w": 250,
"mem_mib": 28898,
"mem_total_mib": 32768,
"util_pct": 50
},
{
"power_w": 55,
"power_cap_w": 250,
"mem_mib": 28758,
"mem_total_mib": 32768,
"util_pct": 69
}
]
},
{
"ts": "2026-07-14T17:10:33",
"ok": true,
"raw": "Timestamp Tue Jul 14 17:10:33 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 56W / 250W | 28955MiB / 32768MiB | 66% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 34C P0 56W / 250W | 28894MiB / 32768MiB | 41% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 56W / 250W | 28898MiB / 32768MiB | 49% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 55W / 250W | 28758MiB / 32768MiB | 89% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n",
"parsed": [
{
"power_w": 56,
"power_cap_w": 250,
"mem_mib": 28955,
"mem_total_mib": 32768,
"util_pct": 66
},
{
"power_w": 56,
"power_cap_w": 250,
"mem_mib": 28894,
"mem_total_mib": 32768,
"util_pct": 41
},
{
"power_w": 56,
"power_cap_w": 250,
"mem_mib": 28898,
"mem_total_mib": 32768,
"util_pct": 49
},
{
"power_w": 55,
"power_cap_w": 250,
"mem_mib": 28758,
"mem_total_mib": 32768,
"util_pct": 89
}
]
},
{
"ts": "2026-07-14T17:10:34",
"ok": true,
"raw": "Timestamp Tue Jul 14 17:10:34 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 57W / 250W | 28955MiB / 32768MiB | 34% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 34C P0 57W / 250W | 28894MiB / 32768MiB | 60% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 33C P0 57W / 250W | 28898MiB / 32768MiB | 60% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 57W / 250W | 28758MiB / 32768MiB | 60% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n",
"parsed": [
{
"power_w": 57,
"power_cap_w": 250,
"mem_mib": 28955,
"mem_total_mib": 32768,
"util_pct": 34
},
{
"power_w": 57,
"power_cap_w": 250,
"mem_mib": 28894,
"mem_total_mib": 32768,
"util_pct": 60
},
{
"power_w": 57,
"power_cap_w": 250,
"mem_mib": 28898,
"mem_total_mib": 32768,
"util_pct": 60
},
{
"power_w": 57,
"power_cap_w": 250,
"mem_mib": 28758,
"mem_total_mib": 32768,
"util_pct": 60
}
]
},
{
"ts": "2026-07-14T17:10:35",
"ok": true,
"raw": "Timestamp Tue Jul 14 17:10:35 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 57W / 250W | 28955MiB / 32768MiB | 46% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 34C P0 57W / 250W | 28894MiB / 32768MiB | 44% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 33C P0 57W / 250W | 28898MiB / 32768MiB | 63% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 56W / 250W | 28758MiB / 32768MiB | 69% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n",
"parsed": [
{
"power_w": 57,
"power_cap_w": 250,
"mem_mib": 28955,
"mem_total_mib": 32768,
"util_pct": 46
},
{
"power_w": 57,
"power_cap_w": 250,
"mem_mib": 28894,
"mem_total_mib": 32768,
"util_pct": 44
},
{
"power_w": 57,
"power_cap_w": 250,
"mem_mib": 28898,
"mem_total_mib": 32768,
"util_pct": 63
},
{
"power_w": 56,
"power_cap_w": 250,
"mem_mib": 28758,
"mem_total_mib": 32768,
"util_pct": 69
}
]
},
{
"ts": "2026-07-14T17:10:37",
"ok": true,
"raw": "Timestamp Tue Jul 14 17:10:37 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 57W / 250W | 28955MiB / 32768MiB | 64% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 34C P0 57W / 250W | 28894MiB / 32768MiB | 47% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 33C P0 57W / 250W | 28898MiB / 32768MiB | 42% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 56W / 250W | 28758MiB / 32768MiB | 69% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n",
"parsed": [
{
"power_w": 57,
"power_cap_w": 250,
"mem_mib": 28955,
"mem_total_mib": 32768,
"util_pct": 64
},
{
"power_w": 57,
"power_cap_w": 250,
"mem_mib": 28894,
"mem_total_mib": 32768,
"util_pct": 47
},
{
"power_w": 57,
"power_cap_w": 250,
"mem_mib": 28898,
"mem_total_mib": 32768,
"util_pct": 42
},
{
"power_w": 56,
"power_cap_w": 250,
"mem_mib": 28758,
"mem_total_mib": 32768,
"util_pct": 69
}
]
},
{
"ts": "2026-07-14T17:10:38",
"ok": true,
"raw": "Timestamp Tue Jul 14 17:10:38 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 56W / 250W | 28955MiB / 32768MiB | 64% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 34C P0 56W / 250W | 28894MiB / 32768MiB | 71% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 57W / 250W | 28898MiB / 32768MiB | 74% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 55W / 250W | 28758MiB / 32768MiB | 49% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n",
"parsed": [
{
"power_w": 56,
"power_cap_w": 250,
"mem_mib": 28955,
"mem_total_mib": 32768,
"util_pct": 64
},
{
"power_w": 56,
"power_cap_w": 250,
"mem_mib": 28894,
"mem_total_mib": 32768,
"util_pct": 71
},
{
"power_w": 57,
"power_cap_w": 250,
"mem_mib": 28898,
"mem_total_mib": 32768,
"util_pct": 74
},
{
"power_w": 55,
"power_cap_w": 250,
"mem_mib": 28758,
"mem_total_mib": 32768,
"util_pct": 49
}
]
},
{
"ts": "2026-07-14T17:10:39",
"ok": true,
"raw": "Timestamp Tue Jul 14 17:10:39 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 56W / 250W | 28955MiB / 32768MiB | 70% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 34C P0 56W / 250W | 28894MiB / 32768MiB | 45% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 56W / 250W | 28898MiB / 32768MiB | 71% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 55W / 250W | 28758MiB / 32768MiB | 55% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n",
"parsed": [
{
"power_w": 56,
"power_cap_w": 250,
"mem_mib": 28955,
"mem_total_mib": 32768,
"util_pct": 70
},
{
"power_w": 56,
"power_cap_w": 250,
"mem_mib": 28894,
"mem_total_mib": 32768,
"util_pct": 45
},
{
"power_w": 56,
"power_cap_w": 250,
"mem_mib": 28898,
"mem_total_mib": 32768,
"util_pct": 71
},
{
"power_w": 55,
"power_cap_w": 250,
"mem_mib": 28758,
"mem_total_mib": 32768,
"util_pct": 55
}
]
},
{
"ts": "2026-07-14T17:10:41",
"ok": true,
"raw": "Timestamp Tue Jul 14 17:10:41 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 57W / 250W | 28955MiB / 32768MiB | 64% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 34C P0 57W / 250W | 28894MiB / 32768MiB | 54% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 57W / 250W | 28898MiB / 32768MiB | 40% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 56W / 250W | 28758MiB / 32768MiB | 59% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n",
"parsed": [
{
"power_w": 57,
"power_cap_w": 250,
"mem_mib": 28955,
"mem_total_mib": 32768,
"util_pct": 64
},
{
"power_w": 57,
"power_cap_w": 250,
"mem_mib": 28894,
"mem_total_mib": 32768,
"util_pct": 54
},
{
"power_w": 57,
"power_cap_w": 250,
"mem_mib": 28898,
"mem_total_mib": 32768,
"util_pct": 40
},
{
"power_w": 56,
"power_cap_w": 250,
"mem_mib": 28758,
"mem_total_mib": 32768,
"util_pct": 59
}
]
},
{
"ts": "2026-07-14T17:10:42",
"ok": true,
"raw": "Timestamp Tue Jul 14 17:10:42 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 56W / 250W | 28955MiB / 32768MiB | 54% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 33C P0 55W / 250W | 28894MiB / 32768MiB | 59% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 55W / 250W | 28898MiB / 32768MiB | 57% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 55W / 250W | 28758MiB / 32768MiB | 54% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n",
"parsed": [
{
"power_w": 56,
"power_cap_w": 250,
"mem_mib": 28955,
"mem_total_mib": 32768,
"util_pct": 54
},
{
"power_w": 55,
"power_cap_w": 250,
"mem_mib": 28894,
"mem_total_mib": 32768,
"util_pct": 59
},
{
"power_w": 55,
"power_cap_w": 250,
"mem_mib": 28898,
"mem_total_mib": 32768,
"util_pct": 57
},
{
"power_w": 55,
"power_cap_w": 250,
"mem_mib": 28758,
"mem_total_mib": 32768,
"util_pct": 54
}
]
},
{
"ts": "2026-07-14T17:10:43",
"ok": true,
"raw": "Timestamp Tue Jul 14 17:10:43 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 57W / 250W | 28955MiB / 32768MiB | 42% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 34C P0 56W / 250W | 28894MiB / 32768MiB | 75% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 57W / 250W | 28898MiB / 32768MiB | 51% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 56W / 250W | 28758MiB / 32768MiB | 53% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n",
"parsed": [
{
"power_w": 57,
"power_cap_w": 250,
"mem_mib": 28955,
"mem_total_mib": 32768,
"util_pct": 42
},
{
"power_w": 56,
"power_cap_w": 250,
"mem_mib": 28894,
"mem_total_mib": 32768,
"util_pct": 75
},
{
"power_w": 57,
"power_cap_w": 250,
"mem_mib": 28898,
"mem_total_mib": 32768,
"util_pct": 51
},
{
"power_w": 56,
"power_cap_w": 250,
"mem_mib": 28758,
"mem_total_mib": 32768,
"util_pct": 53
}
]
},
{
"ts": "2026-07-14T17:10:45",
"ok": true,
"raw": "Timestamp Tue Jul 14 17:10:45 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 55W / 250W | 28955MiB / 32768MiB | 64% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 34C P0 55W / 250W | 28894MiB / 32768MiB | 65% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 54W / 250W | 28898MiB / 32768MiB | 46% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 54W / 250W | 28758MiB / 32768MiB | 50% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n",
"parsed": [
{
"power_w": 55,
"power_cap_w": 250,
"mem_mib": 28955,
"mem_total_mib": 32768,
"util_pct": 64
},
{
"power_w": 55,
"power_cap_w": 250,
"mem_mib": 28894,
"mem_total_mib": 32768,
"util_pct": 65
},
{
"power_w": 54,
"power_cap_w": 250,
"mem_mib": 28898,
"mem_total_mib": 32768,
"util_pct": 46
},
{
"power_w": 54,
"power_cap_w": 250,
"mem_mib": 28758,
"mem_total_mib": 32768,
"util_pct": 50
}
]
},
{
"ts": "2026-07-14T17:10:46",
"ok": true,
"raw": "Timestamp Tue Jul 14 17:10:46 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 57W / 250W | 28955MiB / 32768MiB | 30% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 34C P0 57W / 250W | 28894MiB / 32768MiB | 44% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 33C P0 57W / 250W | 28898MiB / 32768MiB | 36% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 57W / 250W | 28758MiB / 32768MiB | 44% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n",
"parsed": [
{
"power_w": 57,
"power_cap_w": 250,
"mem_mib": 28955,
"mem_total_mib": 32768,
"util_pct": 30
},
{
"power_w": 57,
"power_cap_w": 250,
"mem_mib": 28894,
"mem_total_mib": 32768,
"util_pct": 44
},
{
"power_w": 57,
"power_cap_w": 250,
"mem_mib": 28898,
"mem_total_mib": 32768,
"util_pct": 36
},
{
"power_w": 57,
"power_cap_w": 250,
"mem_mib": 28758,
"mem_total_mib": 32768,
"util_pct": 44
}
]
},
{
"ts": "2026-07-14T17:10:47",
"ok": true,
"raw": "Timestamp Tue Jul 14 17:10:48 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 57W / 250W | 28955MiB / 32768MiB | 57% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 34C P0 57W / 250W | 28894MiB / 32768MiB | 55% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 33C P0 57W / 250W | 28898MiB / 32768MiB | 54% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 56W / 250W | 28758MiB / 32768MiB | 49% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n",
"parsed": [
{
"power_w": 57,
"power_cap_w": 250,
"mem_mib": 28955,
"mem_total_mib": 32768,
"util_pct": 57
},
{
"power_w": 57,
"power_cap_w": 250,
"mem_mib": 28894,
"mem_total_mib": 32768,
"util_pct": 55
},
{
"power_w": 57,
"power_cap_w": 250,
"mem_mib": 28898,
"mem_total_mib": 32768,
"util_pct": 54
},
{
"power_w": 56,
"power_cap_w": 250,
"mem_mib": 28758,
"mem_total_mib": 32768,
"util_pct": 49
}
]
}
]