1122 lines
81 KiB
JSON
1122 lines
81 KiB
JSON
|
[
|
||
|
|
{
|
||
|
|
"ts": "2026-07-14T17:17:55",
|
||
|
|
"ok": true,
|
||
|
|
"raw": "Timestamp Tue Jul 14 17:17:55 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 30C P0 53W / 250W | 28955MiB / 32768MiB | 0% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 53W / 250W | 28894MiB / 32768MiB | 0% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 53W / 250W | 28898MiB / 32768MiB | 0% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 52W / 250W | 28758MiB / 32768MiB | 0% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n",
|
||
|
|
"parsed": [
|
||
|
|
{
|
||
|
|
"power_w": 53,
|
||
|
|
"power_cap_w": 250,
|
||
|
|
"mem_mib": 28955,
|
||
|
|
"mem_total_mib": 32768,
|
||
|
|
"util_pct": 0
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"power_w": 53,
|
||
|
|
"power_cap_w": 250,
|
||
|
|
"mem_mib": 28894,
|
||
|
|
"mem_total_mib": 32768,
|
||
|
|
"util_pct": 0
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"power_w": 53,
|
||
|
|
"power_cap_w": 250,
|
||
|
|
"mem_mib": 28898,
|
||
|
|
"mem_total_mib": 32768,
|
||
|
|
"util_pct": 0
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"power_w": 52,
|
||
|
|
"power_cap_w": 250,
|
||
|
|
"mem_mib": 28758,
|
||
|
|
"mem_total_mib": 32768,
|
||
|
|
"util_pct": 0
|
||
|
|
}
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"ts": "2026-07-14T17:17:56",
|
||
|
|
"ok": true,
|
||
|
|
"raw": "Timestamp Tue Jul 14 17:17:57 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 54W / 250W | 29123MiB / 32768MiB | 42% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 34C P0 55W / 250W | 29062MiB / 32768MiB | 50% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 33C P0 56W / 250W | 29066MiB / 32768MiB | 61% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 55W / 250W | 28926MiB / 32768MiB | 82% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n",
|
||
|
|
"parsed": [
|
||
|
|
{
|
||
|
|
"power_w": 54,
|
||
|
|
"power_cap_w": 250,
|
||
|
|
"mem_mib": 29123,
|
||
|
|
"mem_total_mib": 32768,
|
||
|
|
"util_pct": 42
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"power_w": 55,
|
||
|
|
"power_cap_w": 250,
|
||
|
|
"mem_mib": 29062,
|
||
|
|
"mem_total_mib": 32768,
|
||
|
|
"util_pct": 50
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"power_w": 56,
|
||
|
|
"power_cap_w": 250,
|
||
|
|
"mem_mib": 29066,
|
||
|
|
"mem_total_mib": 32768,
|
||
|
|
"util_pct": 61
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"power_w": 55,
|
||
|
|
"power_cap_w": 250,
|
||
|
|
"mem_mib": 28926,
|
||
|
|
"mem_total_mib": 32768,
|
||
|
|
"util_pct": 82
|
||
|
|
}
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"ts": "2026-07-14T17:17:58",
|
||
|
|
"ok": true,
|
||
|
|
"raw": "Timestamp Tue Jul 14 17:17:58 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 30C P0 56W / 250W | 29123MiB / 32768MiB | 77% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 33C P0 56W / 250W | 29062MiB / 32768MiB | 51% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 56W / 250W | 29066MiB / 32768MiB | 79% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 55W / 250W | 28926MiB / 32768MiB | 37% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n",
|
||
|
|
"parsed": [
|
||
|
|
{
|
||
|
|
"power_w": 56,
|
||
|
|
"power_cap_w": 250,
|
||
|
|
"mem_mib": 29123,
|
||
|
|
"mem_total_mib": 32768,
|
||
|
|
"util_pct": 77
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"power_w": 56,
|
||
|
|
"power_cap_w": 250,
|
||
|
|
"mem_mib": 29062,
|
||
|
|
"mem_total_mib": 32768,
|
||
|
|
"util_pct": 51
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"power_w": 56,
|
||
|
|
"power_cap_w": 250,
|
||
|
|
"mem_mib": 29066,
|
||
|
|
"mem_total_mib": 32768,
|
||
|
|
"util_pct": 79
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"power_w": 55,
|
||
|
|
"power_cap_w": 250,
|
||
|
|
"mem_mib": 28926,
|
||
|
|
"mem_total_mib": 32768,
|
||
|
|
"util_pct": 37
|
||
|
|
}
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"ts": "2026-07-14T17:17:59",
|
||
|
|
"ok": true,
|
||
|
|
"raw": "Timestamp Tue Jul 14 17:17:59 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 30C P0 56W / 250W | 29123MiB / 32768MiB | 69% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 33C P0 56W / 250W | 29062MiB / 32768MiB | 59% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 56W / 250W | 29066MiB / 32768MiB | 60% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 56W / 250W | 28926MiB / 32768MiB | 34% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n",
|
||
|
|
"parsed": [
|
||
|
|
{
|
||
|
|
"power_w": 56,
|
||
|
|
"power_cap_w": 250,
|
||
|
|
"mem_mib": 29123,
|
||
|
|
"mem_total_mib": 32768,
|
||
|
|
"util_pct": 69
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"power_w": 56,
|
||
|
|
"power_cap_w": 250,
|
||
|
|
"mem_mib": 29062,
|
||
|
|
"mem_total_mib": 32768,
|
||
|
|
"util_pct": 59
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"power_w": 56,
|
||
|
|
"power_cap_w": 250,
|
||
|
|
"mem_mib": 29066,
|
||
|
|
"mem_total_mib": 32768,
|
||
|
|
"util_pct": 60
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"power_w": 56,
|
||
|
|
"power_cap_w": 250,
|
||
|
|
"mem_mib": 28926,
|
||
|
|
"mem_total_mib": 32768,
|
||
|
|
"util_pct": 34
|
||
|
|
}
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"ts": "2026-07-14T17:18:01",
|
||
|
|
"ok": true,
|
||
|
|
"raw": "Timestamp Tue Jul 14 17:18:01 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 55W / 250W | 29123MiB / 32768MiB | 43% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 33C P0 55W / 250W | 29062MiB / 32768MiB | 47% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 33C P0 55W / 250W | 29066MiB / 32768MiB | 59% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 54W / 250W | 28926MiB / 32768MiB | 52% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n",
|
||
|
|
"parsed": [
|
||
|
|
{
|
||
|
|
"power_w": 55,
|
||
|
|
"power_cap_w": 250,
|
||
|
|
"mem_mib": 29123,
|
||
|
|
"mem_total_mib": 32768,
|
||
|
|
"util_pct": 43
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"power_w": 55,
|
||
|
|
"power_cap_w": 250,
|
||
|
|
"mem_mib": 29062,
|
||
|
|
"mem_total_mib": 32768,
|
||
|
|
"util_pct": 47
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"power_w": 55,
|
||
|
|
"power_cap_w": 250,
|
||
|
|
"mem_mib": 29066,
|
||
|
|
"mem_total_mib": 32768,
|
||
|
|
"util_pct": 59
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"power_w": 54,
|
||
|
|
"power_cap_w": 250,
|
||
|
|
"mem_mib": 28926,
|
||
|
|
"mem_total_mib": 32768,
|
||
|
|
"util_pct": 52
|
||
|
|
}
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"ts": "2026-07-14T17:18:02",
|
||
|
|
"ok": true,
|
||
|
|
"raw": "Timestamp Tue Jul 14 17:18:02 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 56W / 250W | 29123MiB / 32768MiB | 56% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 33C P0 56W / 250W | 29062MiB / 32768MiB | 28% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 57W / 250W | 29066MiB / 32768MiB | 54% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 55W / 250W | 28926MiB / 32768MiB | 25% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n",
|
||
|
|
"parsed": [
|
||
|
|
{
|
||
|
|
"power_w": 56,
|
||
|
|
"power_cap_w": 250,
|
||
|
|
"mem_mib": 29123,
|
||
|
|
"mem_total_mib": 32768,
|
||
|
|
"util_pct": 56
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"power_w": 56,
|
||
|
|
"power_cap_w": 250,
|
||
|
|
"mem_mib": 29062,
|
||
|
|
"mem_total_mib": 32768,
|
||
|
|
"util_pct": 28
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"power_w": 57,
|
||
|
|
"power_cap_w": 250,
|
||
|
|
"mem_mib": 29066,
|
||
|
|
"mem_total_mib": 32768,
|
||
|
|
"util_pct": 54
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"power_w": 55,
|
||
|
|
"power_cap_w": 250,
|
||
|
|
"mem_mib": 28926,
|
||
|
|
"mem_total_mib": 32768,
|
||
|
|
"util_pct": 25
|
||
|
|
}
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"ts": "2026-07-14T17:18:03",
|
||
|
|
"ok": true,
|
||
|
|
"raw": "Timestamp Tue Jul 14 17:18:03 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 55W / 250W | 29123MiB / 32768MiB | 46% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 33C P0 55W / 250W | 29062MiB / 32768MiB | 26% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 56W / 250W | 29066MiB / 32768MiB | 83% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 55W / 250W | 28926MiB / 32768MiB | 31% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n",
|
||
|
|
"parsed": [
|
||
|
|
{
|
||
|
|
"power_w": 55,
|
||
|
|
"power_cap_w": 250,
|
||
|
|
"mem_mib": 29123,
|
||
|
|
"mem_total_mib": 32768,
|
||
|
|
"util_pct": 46
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"power_w": 55,
|
||
|
|
"power_cap_w": 250,
|
||
|
|
"mem_mib": 29062,
|
||
|
|
"mem_total_mib": 32768,
|
||
|
|
"util_pct": 26
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"power_w": 56,
|
||
|
|
"power_cap_w": 250,
|
||
|
|
"mem_mib": 29066,
|
||
|
|
"mem_total_mib": 32768,
|
||
|
|
"util_pct": 83
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"power_w": 55,
|
||
|
|
"power_cap_w": 250,
|
||
|
|
"mem_mib": 28926,
|
||
|
|
"mem_total_mib": 32768,
|
||
|
|
"util_pct": 31
|
||
|
|
}
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"ts": "2026-07-14T17:18:05",
|
||
|
|
"ok": true,
|
||
|
|
"raw": "Timestamp Tue Jul 14 17:18:05 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 56W / 250W | 29123MiB / 32768MiB | 33% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 33C P0 56W / 250W | 29062MiB / 32768MiB | 53% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 56W / 250W | 29066MiB / 32768MiB | 89% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 55W / 250W | 28926MiB / 32768MiB | 100% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n",
|
||
|
|
"parsed": [
|
||
|
|
{
|
||
|
|
"power_w": 56,
|
||
|
|
"power_cap_w": 250,
|
||
|
|
"mem_mib": 29123,
|
||
|
|
"mem_total_mib": 32768,
|
||
|
|
"util_pct": 33
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"power_w": 56,
|
||
|
|
"power_cap_w": 250,
|
||
|
|
"mem_mib": 29062,
|
||
|
|
"mem_total_mib": 32768,
|
||
|
|
"util_pct": 53
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"power_w": 56,
|
||
|
|
"power_cap_w": 250,
|
||
|
|
"mem_mib": 29066,
|
||
|
|
"mem_total_mib": 32768,
|
||
|
|
"util_pct": 89
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"power_w": 55,
|
||
|
|
"power_cap_w": 250,
|
||
|
|
"mem_mib": 28926,
|
||
|
|
"mem_total_mib": 32768,
|
||
|
|
"util_pct": 100
|
||
|
|
}
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"ts": "2026-07-14T17:18:06",
|
||
|
|
"ok": true,
|
||
|
|
"raw": "Timestamp Tue Jul 14 17:18:06 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 56W / 250W | 29123MiB / 32768MiB | 42% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 33C P0 56W / 250W | 29062MiB / 32768MiB | 41% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 56W / 250W | 29066MiB / 32768MiB | 68% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 56W / 250W | 28926MiB / 32768MiB | 63% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n",
|
||
|
|
"parsed": [
|
||
|
|
{
|
||
|
|
"power_w": 56,
|
||
|
|
"power_cap_w": 250,
|
||
|
|
"mem_mib": 29123,
|
||
|
|
"mem_total_mib": 32768,
|
||
|
|
"util_pct": 42
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"power_w": 56,
|
||
|
|
"power_cap_w": 250,
|
||
|
|
"mem_mib": 29062,
|
||
|
|
"mem_total_mib": 32768,
|
||
|
|
"util_pct": 41
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"power_w": 56,
|
||
|
|
"power_cap_w": 250,
|
||
|
|
"mem_mib": 29066,
|
||
|
|
"mem_total_mib": 32768,
|
||
|
|
"util_pct": 68
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"power_w": 56,
|
||
|
|
"power_cap_w": 250,
|
||
|
|
"mem_mib": 28926,
|
||
|
|
"mem_total_mib": 32768,
|
||
|
|
"util_pct": 63
|
||
|
|
}
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"ts": "2026-07-14T17:18:08",
|
||
|
|
"ok": true,
|
||
|
|
"raw": "Timestamp Tue Jul 14 17:18:08 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 56W / 250W | 29123MiB / 32768MiB | 62% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 34C P0 57W / 250W | 29062MiB / 32768MiB | 67% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 33C P0 57W / 250W | 29066MiB / 32768MiB | 51% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 56W / 250W | 28926MiB / 32768MiB | 50% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n",
|
||
|
|
"parsed": [
|
||
|
|
{
|
||
|
|
"power_w": 56,
|
||
|
|
"power_cap_w": 250,
|
||
|
|
"mem_mib": 29123,
|
||
|
|
"mem_total_mib": 32768,
|
||
|
|
"util_pct": 62
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"power_w": 57,
|
||
|
|
"power_cap_w": 250,
|
||
|
|
"mem_mib": 29062,
|
||
|
|
"mem_total_mib": 32768,
|
||
|
|
"util_pct": 67
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"power_w": 57,
|
||
|
|
"power_cap_w": 250,
|
||
|
|
"mem_mib": 29066,
|
||
|
|
"mem_total_mib": 32768,
|
||
|
|
"util_pct": 51
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"power_w": 56,
|
||
|
|
"power_cap_w": 250,
|
||
|
|
"mem_mib": 28926,
|
||
|
|
"mem_total_mib": 32768,
|
||
|
|
"util_pct": 50
|
||
|
|
}
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"ts": "2026-07-14T17:18:09",
|
||
|
|
"ok": true,
|
||
|
|
"raw": "Timestamp Tue Jul 14 17:18:09 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 57W / 250W | 29123MiB / 32768MiB | 84% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 33C P0 56W / 250W | 29062MiB / 32768MiB | 67% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 57W / 250W | 29066MiB / 32768MiB | 69% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 56W / 250W | 28926MiB / 32768MiB | 30% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n",
|
||
|
|
"parsed": [
|
||
|
|
{
|
||
|
|
"power_w": 57,
|
||
|
|
"power_cap_w": 250,
|
||
|
|
"mem_mib": 29123,
|
||
|
|
"mem_total_mib": 32768,
|
||
|
|
"util_pct": 84
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"power_w": 56,
|
||
|
|
"power_cap_w": 250,
|
||
|
|
"mem_mib": 29062,
|
||
|
|
"mem_total_mib": 32768,
|
||
|
|
"util_pct": 67
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"power_w": 57,
|
||
|
|
"power_cap_w": 250,
|
||
|
|
"mem_mib": 29066,
|
||
|
|
"mem_total_mib": 32768,
|
||
|
|
"util_pct": 69
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"power_w": 56,
|
||
|
|
"power_cap_w": 250,
|
||
|
|
"mem_mib": 28926,
|
||
|
|
"mem_total_mib": 32768,
|
||
|
|
"util_pct": 30
|
||
|
|
}
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"ts": "2026-07-14T17:18:10",
|
||
|
|
"ok": true,
|
||
|
|
"raw": "Timestamp Tue Jul 14 17:18:10 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 56W / 250W | 29123MiB / 32768MiB | 37% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 34C P0 56W / 250W | 29062MiB / 32768MiB | 77% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 56W / 250W | 29066MiB / 32768MiB | 65% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 55W / 250W | 28926MiB / 32768MiB | 46% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n",
|
||
|
|
"parsed": [
|
||
|
|
{
|
||
|
|
"power_w": 56,
|
||
|
|
"power_cap_w": 250,
|
||
|
|
"mem_mib": 29123,
|
||
|
|
"mem_total_mib": 32768,
|
||
|
|
"util_pct": 37
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"power_w": 56,
|
||
|
|
"power_cap_w": 250,
|
||
|
|
"mem_mib": 29062,
|
||
|
|
"mem_total_mib": 32768,
|
||
|
|
"util_pct": 77
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"power_w": 56,
|
||
|
|
"power_cap_w": 250,
|
||
|
|
"mem_mib": 29066,
|
||
|
|
"mem_total_mib": 32768,
|
||
|
|
"util_pct": 65
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"power_w": 55,
|
||
|
|
"power_cap_w": 250,
|
||
|
|
"mem_mib": 28926,
|
||
|
|
"mem_total_mib": 32768,
|
||
|
|
"util_pct": 46
|
||
|
|
}
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"ts": "2026-07-14T17:18:12",
|
||
|
|
"ok": true,
|
||
|
|
"raw": "Timestamp Tue Jul 14 17:18:12 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 56W / 250W | 29123MiB / 32768MiB | 36% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 34C P0 56W / 250W | 29062MiB / 32768MiB | 19% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 57W / 250W | 29066MiB / 32768MiB | 49% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 56W / 250W | 28926MiB / 32768MiB | 100% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n",
|
||
|
|
"parsed": [
|
||
|
|
{
|
||
|
|
"power_w": 56,
|
||
|
|
"power_cap_w": 250,
|
||
|
|
"mem_mib": 29123,
|
||
|
|
"mem_total_mib": 32768,
|
||
|
|
"util_pct": 36
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"power_w": 56,
|
||
|
|
"power_cap_w": 250,
|
||
|
|
"mem_mib": 29062,
|
||
|
|
"mem_total_mib": 32768,
|
||
|
|
"util_pct": 19
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"power_w": 57,
|
||
|
|
"power_cap_w": 250,
|
||
|
|
"mem_mib": 29066,
|
||
|
|
"mem_total_mib": 32768,
|
||
|
|
"util_pct": 49
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"power_w": 56,
|
||
|
|
"power_cap_w": 250,
|
||
|
|
"mem_mib": 28926,
|
||
|
|
"mem_total_mib": 32768,
|
||
|
|
"util_pct": 100
|
||
|
|
}
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"ts": "2026-07-14T17:18:13",
|
||
|
|
"ok": true,
|
||
|
|
"raw": "Timestamp Tue Jul 14 17:18:13 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 56W / 250W | 29123MiB / 32768MiB | 20% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 33C P0 56W / 250W | 29062MiB / 32768MiB | 56% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 33C P0 56W / 250W | 29066MiB / 32768MiB | 61% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 56W / 250W | 28926MiB / 32768MiB | 47% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n",
|
||
|
|
"parsed": [
|
||
|
|
{
|
||
|
|
"power_w": 56,
|
||
|
|
"power_cap_w": 250,
|
||
|
|
"mem_mib": 29123,
|
||
|
|
"mem_total_mib": 32768,
|
||
|
|
"util_pct": 20
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"power_w": 56,
|
||
|
|
"power_cap_w": 250,
|
||
|
|
"mem_mib": 29062,
|
||
|
|
"mem_total_mib": 32768,
|
||
|
|
"util_pct": 56
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"power_w": 56,
|
||
|
|
"power_cap_w": 250,
|
||
|
|
"mem_mib": 29066,
|
||
|
|
"mem_total_mib": 32768,
|
||
|
|
"util_pct": 61
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"power_w": 56,
|
||
|
|
"power_cap_w": 250,
|
||
|
|
"mem_mib": 28926,
|
||
|
|
"mem_total_mib": 32768,
|
||
|
|
"util_pct": 47
|
||
|
|
}
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"ts": "2026-07-14T17:18:14",
|
||
|
|
"ok": true,
|
||
|
|
"raw": "Timestamp Tue Jul 14 17:18:14 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 57W / 250W | 29123MiB / 32768MiB | 76% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 34C P0 57W / 250W | 29062MiB / 32768MiB | 53% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 33C P0 57W / 250W | 29066MiB / 32768MiB | 78% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 57W / 250W | 28926MiB / 32768MiB | 49% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n",
|
||
|
|
"parsed": [
|
||
|
|
{
|
||
|
|
"power_w": 57,
|
||
|
|
"power_cap_w": 250,
|
||
|
|
"mem_mib": 29123,
|
||
|
|
"mem_total_mib": 32768,
|
||
|
|
"util_pct": 76
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"power_w": 57,
|
||
|
|
"power_cap_w": 250,
|
||
|
|
"mem_mib": 29062,
|
||
|
|
"mem_total_mib": 32768,
|
||
|
|
"util_pct": 53
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"power_w": 57,
|
||
|
|
"power_cap_w": 250,
|
||
|
|
"mem_mib": 29066,
|
||
|
|
"mem_total_mib": 32768,
|
||
|
|
"util_pct": 78
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"power_w": 57,
|
||
|
|
"power_cap_w": 250,
|
||
|
|
"mem_mib": 28926,
|
||
|
|
"mem_total_mib": 32768,
|
||
|
|
"util_pct": 49
|
||
|
|
}
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"ts": "2026-07-14T17:18:16",
|
||
|
|
"ok": true,
|
||
|
|
"raw": "Timestamp Tue Jul 14 17:18:16 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 57W / 250W | 29123MiB / 32768MiB | 36% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 34C P0 56W / 250W | 29062MiB / 32768MiB | 76% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 33C P0 57W / 250W | 29066MiB / 32768MiB | 49% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 56W / 250W | 28926MiB / 32768MiB | 87% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n",
|
||
|
|
"parsed": [
|
||
|
|
{
|
||
|
|
"power_w": 57,
|
||
|
|
"power_cap_w": 250,
|
||
|
|
"mem_mib": 29123,
|
||
|
|
"mem_total_mib": 32768,
|
||
|
|
"util_pct": 36
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"power_w": 56,
|
||
|
|
"power_cap_w": 250,
|
||
|
|
"mem_mib": 29062,
|
||
|
|
"mem_total_mib": 32768,
|
||
|
|
"util_pct": 76
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"power_w": 57,
|
||
|
|
"power_cap_w": 250,
|
||
|
|
"mem_mib": 29066,
|
||
|
|
"mem_total_mib": 32768,
|
||
|
|
"util_pct": 49
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"power_w": 56,
|
||
|
|
"power_cap_w": 250,
|
||
|
|
"mem_mib": 28926,
|
||
|
|
"mem_total_mib": 32768,
|
||
|
|
"util_pct": 87
|
||
|
|
}
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"ts": "2026-07-14T17:18:17",
|
||
|
|
"ok": true,
|
||
|
|
"raw": "Timestamp Tue Jul 14 17:18:17 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 30C P0 56W / 250W | 29123MiB / 32768MiB | 0% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 34C P0 55W / 250W | 29062MiB / 32768MiB | 0% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 33C P0 56W / 250W | 29066MiB / 32768MiB | 0% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 54W / 250W | 28926MiB / 32768MiB | 0% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n",
|
||
|
|
"parsed": [
|
||
|
|
{
|
||
|
|
"power_w": 56,
|
||
|
|
"power_cap_w": 250,
|
||
|
|
"mem_mib": 29123,
|
||
|
|
"mem_total_mib": 32768,
|
||
|
|
"util_pct": 0
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"power_w": 55,
|
||
|
|
"power_cap_w": 250,
|
||
|
|
"mem_mib": 29062,
|
||
|
|
"mem_total_mib": 32768,
|
||
|
|
"util_pct": 0
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"power_w": 56,
|
||
|
|
"power_cap_w": 250,
|
||
|
|
"mem_mib": 29066,
|
||
|
|
"mem_total_mib": 32768,
|
||
|
|
"util_pct": 0
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"power_w": 54,
|
||
|
|
"power_cap_w": 250,
|
||
|
|
"mem_mib": 28926,
|
||
|
|
"mem_total_mib": 32768,
|
||
|
|
"util_pct": 0
|
||
|
|
}
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"ts": "2026-07-14T17:18:18",
|
||
|
|
"ok": true,
|
||
|
|
"raw": "Timestamp Tue Jul 14 17:18:19 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 57W / 250W | 29123MiB / 32768MiB | 51% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 34C P0 57W / 250W | 29062MiB / 32768MiB | 50% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 33C P0 58W / 250W | 29066MiB / 32768MiB | 66% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 57W / 250W | 28926MiB / 32768MiB | 55% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n",
|
||
|
|
"parsed": [
|
||
|
|
{
|
||
|
|
"power_w": 57,
|
||
|
|
"power_cap_w": 250,
|
||
|
|
"mem_mib": 29123,
|
||
|
|
"mem_total_mib": 32768,
|
||
|
|
"util_pct": 51
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"power_w": 57,
|
||
|
|
"power_cap_w": 250,
|
||
|
|
"mem_mib": 29062,
|
||
|
|
"mem_total_mib": 32768,
|
||
|
|
"util_pct": 50
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"power_w": 58,
|
||
|
|
"power_cap_w": 250,
|
||
|
|
"mem_mib": 29066,
|
||
|
|
"mem_total_mib": 32768,
|
||
|
|
"util_pct": 66
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"power_w": 57,
|
||
|
|
"power_cap_w": 250,
|
||
|
|
"mem_mib": 28926,
|
||
|
|
"mem_total_mib": 32768,
|
||
|
|
"util_pct": 55
|
||
|
|
}
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"ts": "2026-07-14T17:18:20",
|
||
|
|
"ok": true,
|
||
|
|
"raw": "Timestamp Tue Jul 14 17:18:20 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 57W / 250W | 29123MiB / 32768MiB | 32% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 34C P0 57W / 250W | 29062MiB / 32768MiB | 46% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 57W / 250W | 29066MiB / 32768MiB | 39% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 56W / 250W | 28926MiB / 32768MiB | 71% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n",
|
||
|
|
"parsed": [
|
||
|
|
{
|
||
|
|
"power_w": 57,
|
||
|
|
"power_cap_w": 250,
|
||
|
|
"mem_mib": 29123,
|
||
|
|
"mem_total_mib": 32768,
|
||
|
|
"util_pct": 32
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"power_w": 57,
|
||
|
|
"power_cap_w": 250,
|
||
|
|
"mem_mib": 29062,
|
||
|
|
"mem_total_mib": 32768,
|
||
|
|
"util_pct": 46
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"power_w": 57,
|
||
|
|
"power_cap_w": 250,
|
||
|
|
"mem_mib": 29066,
|
||
|
|
"mem_total_mib": 32768,
|
||
|
|
"util_pct": 39
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"power_w": 56,
|
||
|
|
"power_cap_w": 250,
|
||
|
|
"mem_mib": 28926,
|
||
|
|
"mem_total_mib": 32768,
|
||
|
|
"util_pct": 71
|
||
|
|
}
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"ts": "2026-07-14T17:18:21",
|
||
|
|
"ok": true,
|
||
|
|
"raw": "Timestamp Tue Jul 14 17:18:21 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 56W / 250W | 29123MiB / 32768MiB | 66% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 34C P0 56W / 250W | 29062MiB / 32768MiB | 62% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 33C P0 57W / 250W | 29066MiB / 32768MiB | 98% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 56W / 250W | 28926MiB / 32768MiB | 69% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n",
|
||
|
|
"parsed": [
|
||
|
|
{
|
||
|
|
"power_w": 56,
|
||
|
|
"power_cap_w": 250,
|
||
|
|
"mem_mib": 29123,
|
||
|
|
"mem_total_mib": 32768,
|
||
|
|
"util_pct": 66
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"power_w": 56,
|
||
|
|
"power_cap_w": 250,
|
||
|
|
"mem_mib": 29062,
|
||
|
|
"mem_total_mib": 32768,
|
||
|
|
"util_pct": 62
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"power_w": 57,
|
||
|
|
"power_cap_w": 250,
|
||
|
|
"mem_mib": 29066,
|
||
|
|
"mem_total_mib": 32768,
|
||
|
|
"util_pct": 98
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"power_w": 56,
|
||
|
|
"power_cap_w": 250,
|
||
|
|
"mem_mib": 28926,
|
||
|
|
"mem_total_mib": 32768,
|
||
|
|
"util_pct": 69
|
||
|
|
}
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"ts": "2026-07-14T17:18:23",
|
||
|
|
"ok": true,
|
||
|
|
"raw": "Timestamp Tue Jul 14 17:18:23 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 57W / 250W | 29123MiB / 32768MiB | 68% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 34C P0 57W / 250W | 29062MiB / 32768MiB | 41% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 33C P0 57W / 250W | 29066MiB / 32768MiB | 67% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 56W / 250W | 28926MiB / 32768MiB | 49% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n",
|
||
|
|
"parsed": [
|
||
|
|
{
|
||
|
|
"power_w": 57,
|
||
|
|
"power_cap_w": 250,
|
||
|
|
"mem_mib": 29123,
|
||
|
|
"mem_total_mib": 32768,
|
||
|
|
"util_pct": 68
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"power_w": 57,
|
||
|
|
"power_cap_w": 250,
|
||
|
|
"mem_mib": 29062,
|
||
|
|
"mem_total_mib": 32768,
|
||
|
|
"util_pct": 41
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"power_w": 57,
|
||
|
|
"power_cap_w": 250,
|
||
|
|
"mem_mib": 29066,
|
||
|
|
"mem_total_mib": 32768,
|
||
|
|
"util_pct": 67
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"power_w": 56,
|
||
|
|
"power_cap_w": 250,
|
||
|
|
"mem_mib": 28926,
|
||
|
|
"mem_total_mib": 32768,
|
||
|
|
"util_pct": 49
|
||
|
|
}
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"ts": "2026-07-14T17:18:24",
|
||
|
|
"ok": true,
|
||
|
|
"raw": "Timestamp Tue Jul 14 17:18:24 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 56W / 250W | 29123MiB / 32768MiB | 84% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 34C P0 57W / 250W | 29062MiB / 32768MiB | 65% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 33C P0 57W / 250W | 29066MiB / 32768MiB | 43% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 56W / 250W | 28926MiB / 32768MiB | 54% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n",
|
||
|
|
"parsed": [
|
||
|
|
{
|
||
|
|
"power_w": 56,
|
||
|
|
"power_cap_w": 250,
|
||
|
|
"mem_mib": 29123,
|
||
|
|
"mem_total_mib": 32768,
|
||
|
|
"util_pct": 84
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"power_w": 57,
|
||
|
|
"power_cap_w": 250,
|
||
|
|
"mem_mib": 29062,
|
||
|
|
"mem_total_mib": 32768,
|
||
|
|
"util_pct": 65
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"power_w": 57,
|
||
|
|
"power_cap_w": 250,
|
||
|
|
"mem_mib": 29066,
|
||
|
|
"mem_total_mib": 32768,
|
||
|
|
"util_pct": 43
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"power_w": 56,
|
||
|
|
"power_cap_w": 250,
|
||
|
|
"mem_mib": 28926,
|
||
|
|
"mem_total_mib": 32768,
|
||
|
|
"util_pct": 54
|
||
|
|
}
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"ts": "2026-07-14T17:18:25",
|
||
|
|
"ok": true,
|
||
|
|
"raw": "Timestamp Tue Jul 14 17:18:25 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 57W / 250W | 29123MiB / 32768MiB | 60% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 34C P0 57W / 250W | 29062MiB / 32768MiB | 88% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 33C P0 57W / 250W | 29066MiB / 32768MiB | 56% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 56W / 250W | 28926MiB / 32768MiB | 62% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n",
|
||
|
|
"parsed": [
|
||
|
|
{
|
||
|
|
"power_w": 57,
|
||
|
|
"power_cap_w": 250,
|
||
|
|
"mem_mib": 29123,
|
||
|
|
"mem_total_mib": 32768,
|
||
|
|
"util_pct": 60
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"power_w": 57,
|
||
|
|
"power_cap_w": 250,
|
||
|
|
"mem_mib": 29062,
|
||
|
|
"mem_total_mib": 32768,
|
||
|
|
"util_pct": 88
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"power_w": 57,
|
||
|
|
"power_cap_w": 250,
|
||
|
|
"mem_mib": 29066,
|
||
|
|
"mem_total_mib": 32768,
|
||
|
|
"util_pct": 56
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"power_w": 56,
|
||
|
|
"power_cap_w": 250,
|
||
|
|
"mem_mib": 28926,
|
||
|
|
"mem_total_mib": 32768,
|
||
|
|
"util_pct": 62
|
||
|
|
}
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"ts": "2026-07-14T17:18:27",
|
||
|
|
"ok": true,
|
||
|
|
"raw": "Timestamp Tue Jul 14 17:18:27 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 57W / 250W | 29123MiB / 32768MiB | 76% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 34C P0 57W / 250W | 29062MiB / 32768MiB | 56% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 33C P0 57W / 250W | 29066MiB / 32768MiB | 64% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 56W / 250W | 28926MiB / 32768MiB | 37% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n",
|
||
|
|
"parsed": [
|
||
|
|
{
|
||
|
|
"power_w": 57,
|
||
|
|
"power_cap_w": 250,
|
||
|
|
"mem_mib": 29123,
|
||
|
|
"mem_total_mib": 32768,
|
||
|
|
"util_pct": 76
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"power_w": 57,
|
||
|
|
"power_cap_w": 250,
|
||
|
|
"mem_mib": 29062,
|
||
|
|
"mem_total_mib": 32768,
|
||
|
|
"util_pct": 56
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"power_w": 57,
|
||
|
|
"power_cap_w": 250,
|
||
|
|
"mem_mib": 29066,
|
||
|
|
"mem_total_mib": 32768,
|
||
|
|
"util_pct": 64
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"power_w": 56,
|
||
|
|
"power_cap_w": 250,
|
||
|
|
"mem_mib": 28926,
|
||
|
|
"mem_total_mib": 32768,
|
||
|
|
"util_pct": 37
|
||
|
|
}
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"ts": "2026-07-14T17:18:28",
|
||
|
|
"ok": true,
|
||
|
|
"raw": "Timestamp Tue Jul 14 17:18:28 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 57W / 250W | 29123MiB / 32768MiB | 27% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 34C P0 56W / 250W | 29062MiB / 32768MiB | 58% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 33C P0 56W / 250W | 29066MiB / 32768MiB | 49% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 56W / 250W | 28926MiB / 32768MiB | 78% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n",
|
||
|
|
"parsed": [
|
||
|
|
{
|
||
|
|
"power_w": 57,
|
||
|
|
"power_cap_w": 250,
|
||
|
|
"mem_mib": 29123,
|
||
|
|
"mem_total_mib": 32768,
|
||
|
|
"util_pct": 27
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"power_w": 56,
|
||
|
|
"power_cap_w": 250,
|
||
|
|
"mem_mib": 29062,
|
||
|
|
"mem_total_mib": 32768,
|
||
|
|
"util_pct": 58
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"power_w": 56,
|
||
|
|
"power_cap_w": 250,
|
||
|
|
"mem_mib": 29066,
|
||
|
|
"mem_total_mib": 32768,
|
||
|
|
"util_pct": 49
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"power_w": 56,
|
||
|
|
"power_cap_w": 250,
|
||
|
|
"mem_mib": 28926,
|
||
|
|
"mem_total_mib": 32768,
|
||
|
|
"util_pct": 78
|
||
|
|
}
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"ts": "2026-07-14T17:18:29",
|
||
|
|
"ok": true,
|
||
|
|
"raw": "Timestamp Tue Jul 14 17:18:29 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 57W / 250W | 29123MiB / 32768MiB | 52% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 34C P0 57W / 250W | 29062MiB / 32768MiB | 14% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 33C P0 57W / 250W | 29066MiB / 32768MiB | 57% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 56W / 250W | 28926MiB / 32768MiB | 52% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n",
|
||
|
|
"parsed": [
|
||
|
|
{
|
||
|
|
"power_w": 57,
|
||
|
|
"power_cap_w": 250,
|
||
|
|
"mem_mib": 29123,
|
||
|
|
"mem_total_mib": 32768,
|
||
|
|
"util_pct": 52
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"power_w": 57,
|
||
|
|
"power_cap_w": 250,
|
||
|
|
"mem_mib": 29062,
|
||
|
|
"mem_total_mib": 32768,
|
||
|
|
"util_pct": 14
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"power_w": 57,
|
||
|
|
"power_cap_w": 250,
|
||
|
|
"mem_mib": 29066,
|
||
|
|
"mem_total_mib": 32768,
|
||
|
|
"util_pct": 57
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"power_w": 56,
|
||
|
|
"power_cap_w": 250,
|
||
|
|
"mem_mib": 28926,
|
||
|
|
"mem_total_mib": 32768,
|
||
|
|
"util_pct": 52
|
||
|
|
}
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"ts": "2026-07-14T17:18:31",
|
||
|
|
"ok": true,
|
||
|
|
"raw": "Timestamp Tue Jul 14 17:18:31 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 55W / 250W | 29123MiB / 32768MiB | 82% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 34C P0 55W / 250W | 29062MiB / 32768MiB | 100% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 33C P0 56W / 250W | 29066MiB / 32768MiB | 76% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 55W / 250W | 28926MiB / 32768MiB | 39% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n",
|
||
|
|
"parsed": [
|
||
|
|
{
|
||
|
|
"power_w": 55,
|
||
|
|
"power_cap_w": 250,
|
||
|
|
"mem_mib": 29123,
|
||
|
|
"mem_total_mib": 32768,
|
||
|
|
"util_pct": 82
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"power_w": 55,
|
||
|
|
"power_cap_w": 250,
|
||
|
|
"mem_mib": 29062,
|
||
|
|
"mem_total_mib": 32768,
|
||
|
|
"util_pct": 100
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"power_w": 56,
|
||
|
|
"power_cap_w": 250,
|
||
|
|
"mem_mib": 29066,
|
||
|
|
"mem_total_mib": 32768,
|
||
|
|
"util_pct": 76
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"power_w": 55,
|
||
|
|
"power_cap_w": 250,
|
||
|
|
"mem_mib": 28926,
|
||
|
|
"mem_total_mib": 32768,
|
||
|
|
"util_pct": 39
|
||
|
|
}
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"ts": "2026-07-14T17:18:32",
|
||
|
|
"ok": true,
|
||
|
|
"raw": "Timestamp Tue Jul 14 17:18:32 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 57W / 250W | 29123MiB / 32768MiB | 50% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 34C P0 57W / 250W | 29062MiB / 32768MiB | 77% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 33C P0 57W / 250W | 29066MiB / 32768MiB | 95% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 56W / 250W | 28926MiB / 32768MiB | 50% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n",
|
||
|
|
"parsed": [
|
||
|
|
{
|
||
|
|
"power_w": 57,
|
||
|
|
"power_cap_w": 250,
|
||
|
|
"mem_mib": 29123,
|
||
|
|
"mem_total_mib": 32768,
|
||
|
|
"util_pct": 50
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"power_w": 57,
|
||
|
|
"power_cap_w": 250,
|
||
|
|
"mem_mib": 29062,
|
||
|
|
"mem_total_mib": 32768,
|
||
|
|
"util_pct": 77
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"power_w": 57,
|
||
|
|
"power_cap_w": 250,
|
||
|
|
"mem_mib": 29066,
|
||
|
|
"mem_total_mib": 32768,
|
||
|
|
"util_pct": 95
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"power_w": 56,
|
||
|
|
"power_cap_w": 250,
|
||
|
|
"mem_mib": 28926,
|
||
|
|
"mem_total_mib": 32768,
|
||
|
|
"util_pct": 50
|
||
|
|
}
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"ts": "2026-07-14T17:18:33",
|
||
|
|
"ok": true,
|
||
|
|
"raw": "Timestamp Tue Jul 14 17:18:33 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 55W / 250W | 29123MiB / 32768MiB | 62% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 34C P0 55W / 250W | 29062MiB / 32768MiB | 55% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 56W / 250W | 29066MiB / 32768MiB | 59% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 55W / 250W | 28926MiB / 32768MiB | 56% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n",
|
||
|
|
"parsed": [
|
||
|
|
{
|
||
|
|
"power_w": 55,
|
||
|
|
"power_cap_w": 250,
|
||
|
|
"mem_mib": 29123,
|
||
|
|
"mem_total_mib": 32768,
|
||
|
|
"util_pct": 62
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"power_w": 55,
|
||
|
|
"power_cap_w": 250,
|
||
|
|
"mem_mib": 29062,
|
||
|
|
"mem_total_mib": 32768,
|
||
|
|
"util_pct": 55
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"power_w": 56,
|
||
|
|
"power_cap_w": 250,
|
||
|
|
"mem_mib": 29066,
|
||
|
|
"mem_total_mib": 32768,
|
||
|
|
"util_pct": 59
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"power_w": 55,
|
||
|
|
"power_cap_w": 250,
|
||
|
|
"mem_mib": 28926,
|
||
|
|
"mem_total_mib": 32768,
|
||
|
|
"util_pct": 56
|
||
|
|
}
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"ts": "2026-07-14T17:18:35",
|
||
|
|
"ok": true,
|
||
|
|
"raw": "Timestamp Tue Jul 14 17:18:35 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 57W / 250W | 29123MiB / 32768MiB | 72% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 34C P0 57W / 250W | 29062MiB / 32768MiB | 38% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 33C P0 58W / 250W | 29066MiB / 32768MiB | 62% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 57W / 250W | 28926MiB / 32768MiB | 47% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n",
|
||
|
|
"parsed": [
|
||
|
|
{
|
||
|
|
"power_w": 57,
|
||
|
|
"power_cap_w": 250,
|
||
|
|
"mem_mib": 29123,
|
||
|
|
"mem_total_mib": 32768,
|
||
|
|
"util_pct": 72
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"power_w": 57,
|
||
|
|
"power_cap_w": 250,
|
||
|
|
"mem_mib": 29062,
|
||
|
|
"mem_total_mib": 32768,
|
||
|
|
"util_pct": 38
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"power_w": 58,
|
||
|
|
"power_cap_w": 250,
|
||
|
|
"mem_mib": 29066,
|
||
|
|
"mem_total_mib": 32768,
|
||
|
|
"util_pct": 62
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"power_w": 57,
|
||
|
|
"power_cap_w": 250,
|
||
|
|
"mem_mib": 28926,
|
||
|
|
"mem_total_mib": 32768,
|
||
|
|
"util_pct": 47
|
||
|
|
}
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"ts": "2026-07-14T17:18:36",
|
||
|
|
"ok": true,
|
||
|
|
"raw": "Timestamp Tue Jul 14 17:18:36 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 57W / 250W | 29123MiB / 32768MiB | 49% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 34C P0 57W / 250W | 29062MiB / 32768MiB | 48% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 33C P0 57W / 250W | 29066MiB / 32768MiB | 58% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 56W / 250W | 28926MiB / 32768MiB | 72% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n",
|
||
|
|
"parsed": [
|
||
|
|
{
|
||
|
|
"power_w": 57,
|
||
|
|
"power_cap_w": 250,
|
||
|
|
"mem_mib": 29123,
|
||
|
|
"mem_total_mib": 32768,
|
||
|
|
"util_pct": 49
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"power_w": 57,
|
||
|
|
"power_cap_w": 250,
|
||
|
|
"mem_mib": 29062,
|
||
|
|
"mem_total_mib": 32768,
|
||
|
|
"util_pct": 48
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"power_w": 57,
|
||
|
|
"power_cap_w": 250,
|
||
|
|
"mem_mib": 29066,
|
||
|
|
"mem_total_mib": 32768,
|
||
|
|
"util_pct": 58
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"power_w": 56,
|
||
|
|
"power_cap_w": 250,
|
||
|
|
"mem_mib": 28926,
|
||
|
|
"mem_total_mib": 32768,
|
||
|
|
"util_pct": 72
|
||
|
|
}
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"ts": "2026-07-14T17:18:37",
|
||
|
|
"ok": true,
|
||
|
|
"raw": "Timestamp Tue Jul 14 17:18:38 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 55W / 250W | 29123MiB / 32768MiB | 61% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 34C P0 55W / 250W | 29062MiB / 32768MiB | 44% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 33C P0 55W / 250W | 29066MiB / 32768MiB | 58% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 54W / 250W | 28926MiB / 32768MiB | 0% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n",
|
||
|
|
"parsed": [
|
||
|
|
{
|
||
|
|
"power_w": 55,
|
||
|
|
"power_cap_w": 250,
|
||
|
|
"mem_mib": 29123,
|
||
|
|
"mem_total_mib": 32768,
|
||
|
|
"util_pct": 61
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"power_w": 55,
|
||
|
|
"power_cap_w": 250,
|
||
|
|
"mem_mib": 29062,
|
||
|
|
"mem_total_mib": 32768,
|
||
|
|
"util_pct": 44
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"power_w": 55,
|
||
|
|
"power_cap_w": 250,
|
||
|
|
"mem_mib": 29066,
|
||
|
|
"mem_total_mib": 32768,
|
||
|
|
"util_pct": 58
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"power_w": 54,
|
||
|
|
"power_cap_w": 250,
|
||
|
|
"mem_mib": 28926,
|
||
|
|
"mem_total_mib": 32768,
|
||
|
|
"util_pct": 0
|
||
|
|
}
|
||
|
|
]
|
||
|
|
}
|
||
|
|
]
|