[ { "ts": "2026-07-14T17:12:43", "ok": true, "raw": "Timestamp Tue Jul 14 17:12:43 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 30C P0 53W / 250W | 28955MiB / 32768MiB | 0% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 33C P0 53W / 250W | 28894MiB / 32768MiB | 0% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 53W / 250W | 28898MiB / 32768MiB | 0% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 52W / 250W | 28758MiB / 32768MiB | 0% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n", "parsed": [ { "power_w": 53, "power_cap_w": 250, "mem_mib": 28955, "mem_total_mib": 32768, "util_pct": 0 }, { "power_w": 53, "power_cap_w": 250, "mem_mib": 28894, "mem_total_mib": 32768, "util_pct": 0 }, { "power_w": 53, "power_cap_w": 250, "mem_mib": 28898, "mem_total_mib": 32768, "util_pct": 0 }, { "power_w": 52, "power_cap_w": 250, "mem_mib": 28758, "mem_total_mib": 32768, "util_pct": 0 } ] }, { "ts": "2026-07-14T17:12:44", "ok": true, "raw": "Timestamp Tue Jul 14 17:12:44 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 55W / 250W | 28955MiB / 32768MiB | 54% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 34C P0 55W / 250W | 28894MiB / 32768MiB | 57% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 56W / 250W | 28898MiB / 32768MiB | 50% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 55W / 250W | 28758MiB / 32768MiB | 68% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n", "parsed": [ { "power_w": 55, "power_cap_w": 250, "mem_mib": 28955, "mem_total_mib": 32768, "util_pct": 54 }, { "power_w": 55, "power_cap_w": 250, "mem_mib": 28894, "mem_total_mib": 32768, "util_pct": 57 }, { "power_w": 56, "power_cap_w": 250, "mem_mib": 28898, "mem_total_mib": 32768, "util_pct": 50 }, { "power_w": 55, "power_cap_w": 250, "mem_mib": 28758, "mem_total_mib": 32768, "util_pct": 68 } ] }, { "ts": "2026-07-14T17:12:46", "ok": true, "raw": "Timestamp Tue Jul 14 17:12:46 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 55W / 250W | 28955MiB / 32768MiB | 55% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 33C P0 55W / 250W | 28894MiB / 32768MiB | 73% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 56W / 250W | 28898MiB / 32768MiB | 37% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 55W / 250W | 28758MiB / 32768MiB | 73% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n", "parsed": [ { "power_w": 55, "power_cap_w": 250, "mem_mib": 28955, "mem_total_mib": 32768, "util_pct": 55 }, { "power_w": 55, "power_cap_w": 250, "mem_mib": 28894, "mem_total_mib": 32768, "util_pct": 73 }, { "power_w": 56, "power_cap_w": 250, "mem_mib": 28898, "mem_total_mib": 32768, "util_pct": 37 }, { "power_w": 55, "power_cap_w": 250, "mem_mib": 28758, "mem_total_mib": 32768, "util_pct": 73 } ] }, { "ts": "2026-07-14T17:12:47", "ok": true, "raw": "Timestamp Tue Jul 14 17:12:47 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 56W / 250W | 28955MiB / 32768MiB | 57% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 33C P0 56W / 250W | 28894MiB / 32768MiB | 57% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 57W / 250W | 28898MiB / 32768MiB | 53% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 56W / 250W | 28758MiB / 32768MiB | 41% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n", "parsed": [ { "power_w": 56, "power_cap_w": 250, "mem_mib": 28955, "mem_total_mib": 32768, "util_pct": 57 }, { "power_w": 56, "power_cap_w": 250, "mem_mib": 28894, "mem_total_mib": 32768, "util_pct": 57 }, { "power_w": 57, "power_cap_w": 250, "mem_mib": 28898, "mem_total_mib": 32768, "util_pct": 53 }, { "power_w": 56, "power_cap_w": 250, "mem_mib": 28758, "mem_total_mib": 32768, "util_pct": 41 } ] }, { "ts": "2026-07-14T17:12:48", "ok": true, "raw": "Timestamp Tue Jul 14 17:12:49 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 57W / 250W | 28955MiB / 32768MiB | 94% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 33C P0 57W / 250W | 28894MiB / 32768MiB | 43% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 57W / 250W | 28898MiB / 32768MiB | 70% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 56W / 250W | 28758MiB / 32768MiB | 34% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n", "parsed": [ { "power_w": 57, "power_cap_w": 250, "mem_mib": 28955, "mem_total_mib": 32768, "util_pct": 94 }, { "power_w": 57, "power_cap_w": 250, "mem_mib": 28894, "mem_total_mib": 32768, "util_pct": 43 }, { "power_w": 57, "power_cap_w": 250, "mem_mib": 28898, "mem_total_mib": 32768, "util_pct": 70 }, { "power_w": 56, "power_cap_w": 250, "mem_mib": 28758, "mem_total_mib": 32768, "util_pct": 34 } ] }, { "ts": "2026-07-14T17:12:50", "ok": true, "raw": "Timestamp Tue Jul 14 17:12:50 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 56W / 250W | 28955MiB / 32768MiB | 32% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 33C P0 56W / 250W | 28894MiB / 32768MiB | 57% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 56W / 250W | 28898MiB / 32768MiB | 70% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 55W / 250W | 28758MiB / 32768MiB | 81% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n", "parsed": [ { "power_w": 56, "power_cap_w": 250, "mem_mib": 28955, "mem_total_mib": 32768, "util_pct": 32 }, { "power_w": 56, "power_cap_w": 250, "mem_mib": 28894, "mem_total_mib": 32768, "util_pct": 57 }, { "power_w": 56, "power_cap_w": 250, "mem_mib": 28898, "mem_total_mib": 32768, "util_pct": 70 }, { "power_w": 55, "power_cap_w": 250, "mem_mib": 28758, "mem_total_mib": 32768, "util_pct": 81 } ] }, { "ts": "2026-07-14T17:12:51", "ok": true, "raw": "Timestamp Tue Jul 14 17:12:51 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 30C P0 55W / 250W | 28955MiB / 32768MiB | 63% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 33C P0 55W / 250W | 28894MiB / 32768MiB | 61% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 56W / 250W | 28898MiB / 32768MiB | 81% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 55W / 250W | 28758MiB / 32768MiB | 47% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n", "parsed": [ { "power_w": 55, "power_cap_w": 250, "mem_mib": 28955, "mem_total_mib": 32768, "util_pct": 63 }, { "power_w": 55, "power_cap_w": 250, "mem_mib": 28894, "mem_total_mib": 32768, "util_pct": 61 }, { "power_w": 56, "power_cap_w": 250, "mem_mib": 28898, "mem_total_mib": 32768, "util_pct": 81 }, { "power_w": 55, "power_cap_w": 250, "mem_mib": 28758, "mem_total_mib": 32768, "util_pct": 47 } ] }, { "ts": "2026-07-14T17:12:53", "ok": true, "raw": "Timestamp Tue Jul 14 17:12:53 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 57W / 250W | 28955MiB / 32768MiB | 63% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 34C P0 57W / 250W | 28894MiB / 32768MiB | 52% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 33C P0 57W / 250W | 28898MiB / 32768MiB | 42% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 57W / 250W | 28758MiB / 32768MiB | 70% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n", "parsed": [ { "power_w": 57, "power_cap_w": 250, "mem_mib": 28955, "mem_total_mib": 32768, "util_pct": 63 }, { "power_w": 57, "power_cap_w": 250, "mem_mib": 28894, "mem_total_mib": 32768, "util_pct": 52 }, { "power_w": 57, "power_cap_w": 250, "mem_mib": 28898, "mem_total_mib": 32768, "util_pct": 42 }, { "power_w": 57, "power_cap_w": 250, "mem_mib": 28758, "mem_total_mib": 32768, "util_pct": 70 } ] }, { "ts": "2026-07-14T17:12:54", "ok": true, "raw": "Timestamp Tue Jul 14 17:12:54 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 57W / 250W | 28955MiB / 32768MiB | 41% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 34C P0 57W / 250W | 28894MiB / 32768MiB | 65% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 33C P0 57W / 250W | 28898MiB / 32768MiB | 75% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 57W / 250W | 28758MiB / 32768MiB | 59% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n", "parsed": [ { "power_w": 57, "power_cap_w": 250, "mem_mib": 28955, "mem_total_mib": 32768, "util_pct": 41 }, { "power_w": 57, "power_cap_w": 250, "mem_mib": 28894, "mem_total_mib": 32768, "util_pct": 65 }, { "power_w": 57, "power_cap_w": 250, "mem_mib": 28898, "mem_total_mib": 32768, "util_pct": 75 }, { "power_w": 57, "power_cap_w": 250, "mem_mib": 28758, "mem_total_mib": 32768, "util_pct": 59 } ] }, { "ts": "2026-07-14T17:12:55", "ok": true, "raw": "Timestamp Tue Jul 14 17:12:55 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 57W / 250W | 28955MiB / 32768MiB | 48% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 34C P0 56W / 250W | 28894MiB / 32768MiB | 33% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 56W / 250W | 28898MiB / 32768MiB | 100% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 55W / 250W | 28758MiB / 32768MiB | 52% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n", "parsed": [ { "power_w": 57, "power_cap_w": 250, "mem_mib": 28955, "mem_total_mib": 32768, "util_pct": 48 }, { "power_w": 56, "power_cap_w": 250, "mem_mib": 28894, "mem_total_mib": 32768, "util_pct": 33 }, { "power_w": 56, "power_cap_w": 250, "mem_mib": 28898, "mem_total_mib": 32768, "util_pct": 100 }, { "power_w": 55, "power_cap_w": 250, "mem_mib": 28758, "mem_total_mib": 32768, "util_pct": 52 } ] }, { "ts": "2026-07-14T17:12:57", "ok": true, "raw": "Timestamp Tue Jul 14 17:12:57 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 56W / 250W | 28955MiB / 32768MiB | 53% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 34C P0 56W / 250W | 28894MiB / 32768MiB | 74% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 57W / 250W | 28898MiB / 32768MiB | 91% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 56W / 250W | 28758MiB / 32768MiB | 78% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n", "parsed": [ { "power_w": 56, "power_cap_w": 250, "mem_mib": 28955, "mem_total_mib": 32768, "util_pct": 53 }, { "power_w": 56, "power_cap_w": 250, "mem_mib": 28894, "mem_total_mib": 32768, "util_pct": 74 }, { "power_w": 57, "power_cap_w": 250, "mem_mib": 28898, "mem_total_mib": 32768, "util_pct": 91 }, { "power_w": 56, "power_cap_w": 250, "mem_mib": 28758, "mem_total_mib": 32768, "util_pct": 78 } ] }, { "ts": "2026-07-14T17:12:58", "ok": true, "raw": "Timestamp Tue Jul 14 17:12:58 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 56W / 250W | 28955MiB / 32768MiB | 63% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 33C P0 56W / 250W | 28894MiB / 32768MiB | 40% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 56W / 250W | 28898MiB / 32768MiB | 42% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 55W / 250W | 28758MiB / 32768MiB | 63% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n", "parsed": [ { "power_w": 56, "power_cap_w": 250, "mem_mib": 28955, "mem_total_mib": 32768, "util_pct": 63 }, { "power_w": 56, "power_cap_w": 250, "mem_mib": 28894, "mem_total_mib": 32768, "util_pct": 40 }, { "power_w": 56, "power_cap_w": 250, "mem_mib": 28898, "mem_total_mib": 32768, "util_pct": 42 }, { "power_w": 55, "power_cap_w": 250, "mem_mib": 28758, "mem_total_mib": 32768, "util_pct": 63 } ] }, { "ts": "2026-07-14T17:12:59", "ok": true, "raw": "Timestamp Tue Jul 14 17:12:59 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 56W / 250W | 28955MiB / 32768MiB | 52% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 34C P0 56W / 250W | 28894MiB / 32768MiB | 48% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 56W / 250W | 28898MiB / 32768MiB | 58% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 56W / 250W | 28758MiB / 32768MiB | 61% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n", "parsed": [ { "power_w": 56, "power_cap_w": 250, "mem_mib": 28955, "mem_total_mib": 32768, "util_pct": 52 }, { "power_w": 56, "power_cap_w": 250, "mem_mib": 28894, "mem_total_mib": 32768, "util_pct": 48 }, { "power_w": 56, "power_cap_w": 250, "mem_mib": 28898, "mem_total_mib": 32768, "util_pct": 58 }, { "power_w": 56, "power_cap_w": 250, "mem_mib": 28758, "mem_total_mib": 32768, "util_pct": 61 } ] }, { "ts": "2026-07-14T17:13:01", "ok": true, "raw": "Timestamp Tue Jul 14 17:13:01 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 56W / 250W | 28955MiB / 32768MiB | 60% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 34C P0 55W / 250W | 28894MiB / 32768MiB | 76% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 56W / 250W | 28898MiB / 32768MiB | 63% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 55W / 250W | 28758MiB / 32768MiB | 41% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n", "parsed": [ { "power_w": 56, "power_cap_w": 250, "mem_mib": 28955, "mem_total_mib": 32768, "util_pct": 60 }, { "power_w": 55, "power_cap_w": 250, "mem_mib": 28894, "mem_total_mib": 32768, "util_pct": 76 }, { "power_w": 56, "power_cap_w": 250, "mem_mib": 28898, "mem_total_mib": 32768, "util_pct": 63 }, { "power_w": 55, "power_cap_w": 250, "mem_mib": 28758, "mem_total_mib": 32768, "util_pct": 41 } ] }, { "ts": "2026-07-14T17:13:02", "ok": true, "raw": "Timestamp Tue Jul 14 17:13:02 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 56W / 250W | 28955MiB / 32768MiB | 81% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 33C P0 57W / 250W | 28894MiB / 32768MiB | 75% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 33C P0 57W / 250W | 28898MiB / 32768MiB | 75% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 56W / 250W | 28758MiB / 32768MiB | 69% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n", "parsed": [ { "power_w": 56, "power_cap_w": 250, "mem_mib": 28955, "mem_total_mib": 32768, "util_pct": 81 }, { "power_w": 57, "power_cap_w": 250, "mem_mib": 28894, "mem_total_mib": 32768, "util_pct": 75 }, { "power_w": 57, "power_cap_w": 250, "mem_mib": 28898, "mem_total_mib": 32768, "util_pct": 75 }, { "power_w": 56, "power_cap_w": 250, "mem_mib": 28758, "mem_total_mib": 32768, "util_pct": 69 } ] }, { "ts": "2026-07-14T17:13:03", "ok": true, "raw": "Timestamp Tue Jul 14 17:13:03 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 56W / 250W | 28955MiB / 32768MiB | 65% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 34C P0 56W / 250W | 28894MiB / 32768MiB | 48% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 33C P0 56W / 250W | 28898MiB / 32768MiB | 63% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 55W / 250W | 28758MiB / 32768MiB | 63% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n", "parsed": [ { "power_w": 56, "power_cap_w": 250, "mem_mib": 28955, "mem_total_mib": 32768, "util_pct": 65 }, { "power_w": 56, "power_cap_w": 250, "mem_mib": 28894, "mem_total_mib": 32768, "util_pct": 48 }, { "power_w": 56, "power_cap_w": 250, "mem_mib": 28898, "mem_total_mib": 32768, "util_pct": 63 }, { "power_w": 55, "power_cap_w": 250, "mem_mib": 28758, "mem_total_mib": 32768, "util_pct": 63 } ] }, { "ts": "2026-07-14T17:13:05", "ok": true, "raw": "Timestamp Tue Jul 14 17:13:05 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 55W / 250W | 28955MiB / 32768MiB | 0% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 34C P0 54W / 250W | 28894MiB / 32768MiB | 0% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 53W / 250W | 28898MiB / 32768MiB | 0% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 53W / 250W | 28758MiB / 32768MiB | 0% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n", "parsed": [ { "power_w": 55, "power_cap_w": 250, "mem_mib": 28955, "mem_total_mib": 32768, "util_pct": 0 }, { "power_w": 54, "power_cap_w": 250, "mem_mib": 28894, "mem_total_mib": 32768, "util_pct": 0 }, { "power_w": 53, "power_cap_w": 250, "mem_mib": 28898, "mem_total_mib": 32768, "util_pct": 0 }, { "power_w": 53, "power_cap_w": 250, "mem_mib": 28758, "mem_total_mib": 32768, "util_pct": 0 } ] }, { "ts": "2026-07-14T17:13:06", "ok": true, "raw": "Timestamp Tue Jul 14 17:13:06 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 55W / 250W | 28955MiB / 32768MiB | 78% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 34C P0 55W / 250W | 28894MiB / 32768MiB | 66% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 56W / 250W | 28898MiB / 32768MiB | 62% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 55W / 250W | 28758MiB / 32768MiB | 53% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n", "parsed": [ { "power_w": 55, "power_cap_w": 250, "mem_mib": 28955, "mem_total_mib": 32768, "util_pct": 78 }, { "power_w": 55, "power_cap_w": 250, "mem_mib": 28894, "mem_total_mib": 32768, "util_pct": 66 }, { "power_w": 56, "power_cap_w": 250, "mem_mib": 28898, "mem_total_mib": 32768, "util_pct": 62 }, { "power_w": 55, "power_cap_w": 250, "mem_mib": 28758, "mem_total_mib": 32768, "util_pct": 53 } ] }, { "ts": "2026-07-14T17:13:07", "ok": true, "raw": "Timestamp Tue Jul 14 17:13:07 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 57W / 250W | 28955MiB / 32768MiB | 65% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 34C P0 57W / 250W | 28894MiB / 32768MiB | 77% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 57W / 250W | 28898MiB / 32768MiB | 100% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 56W / 250W | 28758MiB / 32768MiB | 68% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n", "parsed": [ { "power_w": 57, "power_cap_w": 250, "mem_mib": 28955, "mem_total_mib": 32768, "util_pct": 65 }, { "power_w": 57, "power_cap_w": 250, "mem_mib": 28894, "mem_total_mib": 32768, "util_pct": 77 }, { "power_w": 57, "power_cap_w": 250, "mem_mib": 28898, "mem_total_mib": 32768, "util_pct": 100 }, { "power_w": 56, "power_cap_w": 250, "mem_mib": 28758, "mem_total_mib": 32768, "util_pct": 68 } ] }, { "ts": "2026-07-14T17:13:09", "ok": true, "raw": "Timestamp Tue Jul 14 17:13:09 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 57W / 250W | 28955MiB / 32768MiB | 59% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 34C P0 57W / 250W | 28894MiB / 32768MiB | 43% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 33C P0 57W / 250W | 28898MiB / 32768MiB | 72% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 56W / 250W | 28758MiB / 32768MiB | 48% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n", "parsed": [ { "power_w": 57, "power_cap_w": 250, "mem_mib": 28955, "mem_total_mib": 32768, "util_pct": 59 }, { "power_w": 57, "power_cap_w": 250, "mem_mib": 28894, "mem_total_mib": 32768, "util_pct": 43 }, { "power_w": 57, "power_cap_w": 250, "mem_mib": 28898, "mem_total_mib": 32768, "util_pct": 72 }, { "power_w": 56, "power_cap_w": 250, "mem_mib": 28758, "mem_total_mib": 32768, "util_pct": 48 } ] }, { "ts": "2026-07-14T17:13:10", "ok": true, "raw": "Timestamp Tue Jul 14 17:13:10 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 56W / 250W | 28955MiB / 32768MiB | 41% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 34C P0 55W / 250W | 28894MiB / 32768MiB | 61% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 33C P0 56W / 250W | 28898MiB / 32768MiB | 61% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 55W / 250W | 28758MiB / 32768MiB | 61% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n", "parsed": [ { "power_w": 56, "power_cap_w": 250, "mem_mib": 28955, "mem_total_mib": 32768, "util_pct": 41 }, { "power_w": 55, "power_cap_w": 250, "mem_mib": 28894, "mem_total_mib": 32768, "util_pct": 61 }, { "power_w": 56, "power_cap_w": 250, "mem_mib": 28898, "mem_total_mib": 32768, "util_pct": 61 }, { "power_w": 55, "power_cap_w": 250, "mem_mib": 28758, "mem_total_mib": 32768, "util_pct": 61 } ] }, { "ts": "2026-07-14T17:13:11", "ok": true, "raw": "Timestamp Tue Jul 14 17:13:11 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 56W / 250W | 28955MiB / 32768MiB | 42% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 34C P0 56W / 250W | 28894MiB / 32768MiB | 53% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 33C P0 56W / 250W | 28898MiB / 32768MiB | 77% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 56W / 250W | 28758MiB / 32768MiB | 59% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n", "parsed": [ { "power_w": 56, "power_cap_w": 250, "mem_mib": 28955, "mem_total_mib": 32768, "util_pct": 42 }, { "power_w": 56, "power_cap_w": 250, "mem_mib": 28894, "mem_total_mib": 32768, "util_pct": 53 }, { "power_w": 56, "power_cap_w": 250, "mem_mib": 28898, "mem_total_mib": 32768, "util_pct": 77 }, { "power_w": 56, "power_cap_w": 250, "mem_mib": 28758, "mem_total_mib": 32768, "util_pct": 59 } ] }, { "ts": "2026-07-14T17:13:13", "ok": true, "raw": "Timestamp Tue Jul 14 17:13:13 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 56W / 250W | 28955MiB / 32768MiB | 25% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 34C P0 56W / 250W | 28894MiB / 32768MiB | 50% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 33C P0 56W / 250W | 28898MiB / 32768MiB | 47% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 55W / 250W | 28758MiB / 32768MiB | 61% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n", "parsed": [ { "power_w": 56, "power_cap_w": 250, "mem_mib": 28955, "mem_total_mib": 32768, "util_pct": 25 }, { "power_w": 56, "power_cap_w": 250, "mem_mib": 28894, "mem_total_mib": 32768, "util_pct": 50 }, { "power_w": 56, "power_cap_w": 250, "mem_mib": 28898, "mem_total_mib": 32768, "util_pct": 47 }, { "power_w": 55, "power_cap_w": 250, "mem_mib": 28758, "mem_total_mib": 32768, "util_pct": 61 } ] }, { "ts": "2026-07-14T17:13:14", "ok": true, "raw": "Timestamp Tue Jul 14 17:13:14 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 55W / 250W | 28955MiB / 32768MiB | 30% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 34C P0 55W / 250W | 28894MiB / 32768MiB | 90% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 33C P0 56W / 250W | 28898MiB / 32768MiB | 72% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 55W / 250W | 28758MiB / 32768MiB | 34% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n", "parsed": [ { "power_w": 55, "power_cap_w": 250, "mem_mib": 28955, "mem_total_mib": 32768, "util_pct": 30 }, { "power_w": 55, "power_cap_w": 250, "mem_mib": 28894, "mem_total_mib": 32768, "util_pct": 90 }, { "power_w": 56, "power_cap_w": 250, "mem_mib": 28898, "mem_total_mib": 32768, "util_pct": 72 }, { "power_w": 55, "power_cap_w": 250, "mem_mib": 28758, "mem_total_mib": 32768, "util_pct": 34 } ] }, { "ts": "2026-07-14T17:13:15", "ok": true, "raw": "Timestamp Tue Jul 14 17:13:15 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 57W / 250W | 28955MiB / 32768MiB | 63% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 34C P0 57W / 250W | 28894MiB / 32768MiB | 68% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 33C P0 57W / 250W | 28898MiB / 32768MiB | 57% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 57W / 250W | 28758MiB / 32768MiB | 42% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n", "parsed": [ { "power_w": 57, "power_cap_w": 250, "mem_mib": 28955, "mem_total_mib": 32768, "util_pct": 63 }, { "power_w": 57, "power_cap_w": 250, "mem_mib": 28894, "mem_total_mib": 32768, "util_pct": 68 }, { "power_w": 57, "power_cap_w": 250, "mem_mib": 28898, "mem_total_mib": 32768, "util_pct": 57 }, { "power_w": 57, "power_cap_w": 250, "mem_mib": 28758, "mem_total_mib": 32768, "util_pct": 42 } ] }, { "ts": "2026-07-14T17:13:17", "ok": true, "raw": "Timestamp Tue Jul 14 17:13:17 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 56W / 250W | 28955MiB / 32768MiB | 58% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 34C P0 56W / 250W | 28894MiB / 32768MiB | 50% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 33C P0 56W / 250W | 28898MiB / 32768MiB | 40% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 55W / 250W | 28758MiB / 32768MiB | 63% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n", "parsed": [ { "power_w": 56, "power_cap_w": 250, "mem_mib": 28955, "mem_total_mib": 32768, "util_pct": 58 }, { "power_w": 56, "power_cap_w": 250, "mem_mib": 28894, "mem_total_mib": 32768, "util_pct": 50 }, { "power_w": 56, "power_cap_w": 250, "mem_mib": 28898, "mem_total_mib": 32768, "util_pct": 40 }, { "power_w": 55, "power_cap_w": 250, "mem_mib": 28758, "mem_total_mib": 32768, "util_pct": 63 } ] }, { "ts": "2026-07-14T17:13:18", "ok": true, "raw": "Timestamp Tue Jul 14 17:13:18 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 56W / 250W | 28955MiB / 32768MiB | 66% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 34C P0 56W / 250W | 28894MiB / 32768MiB | 43% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 55W / 250W | 28898MiB / 32768MiB | 25% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 54W / 250W | 28758MiB / 32768MiB | 52% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n", "parsed": [ { "power_w": 56, "power_cap_w": 250, "mem_mib": 28955, "mem_total_mib": 32768, "util_pct": 66 }, { "power_w": 56, "power_cap_w": 250, "mem_mib": 28894, "mem_total_mib": 32768, "util_pct": 43 }, { "power_w": 55, "power_cap_w": 250, "mem_mib": 28898, "mem_total_mib": 32768, "util_pct": 25 }, { "power_w": 54, "power_cap_w": 250, "mem_mib": 28758, "mem_total_mib": 32768, "util_pct": 52 } ] }, { "ts": "2026-07-14T17:13:19", "ok": true, "raw": "Timestamp Tue Jul 14 17:13:19 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 56W / 250W | 28955MiB / 32768MiB | 43% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 34C P0 57W / 250W | 28894MiB / 32768MiB | 48% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 33C P0 57W / 250W | 28898MiB / 32768MiB | 51% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 56W / 250W | 28758MiB / 32768MiB | 51% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n", "parsed": [ { "power_w": 56, "power_cap_w": 250, "mem_mib": 28955, "mem_total_mib": 32768, "util_pct": 43 }, { "power_w": 57, "power_cap_w": 250, "mem_mib": 28894, "mem_total_mib": 32768, "util_pct": 48 }, { "power_w": 57, "power_cap_w": 250, "mem_mib": 28898, "mem_total_mib": 32768, "util_pct": 51 }, { "power_w": 56, "power_cap_w": 250, "mem_mib": 28758, "mem_total_mib": 32768, "util_pct": 51 } ] }, { "ts": "2026-07-14T17:13:21", "ok": true, "raw": "Timestamp Tue Jul 14 17:13:21 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 56W / 250W | 28955MiB / 32768MiB | 57% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 34C P0 56W / 250W | 28894MiB / 32768MiB | 69% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 56W / 250W | 28898MiB / 32768MiB | 38% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 55W / 250W | 28758MiB / 32768MiB | 66% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n", "parsed": [ { "power_w": 56, "power_cap_w": 250, "mem_mib": 28955, "mem_total_mib": 32768, "util_pct": 57 }, { "power_w": 56, "power_cap_w": 250, "mem_mib": 28894, "mem_total_mib": 32768, "util_pct": 69 }, { "power_w": 56, "power_cap_w": 250, "mem_mib": 28898, "mem_total_mib": 32768, "util_pct": 38 }, { "power_w": 55, "power_cap_w": 250, "mem_mib": 28758, "mem_total_mib": 32768, "util_pct": 66 } ] }, { "ts": "2026-07-14T17:13:22", "ok": true, "raw": "Timestamp Tue Jul 14 17:13:22 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 56W / 250W | 28955MiB / 32768MiB | 61% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 34C P0 56W / 250W | 28894MiB / 32768MiB | 63% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 33C P0 56W / 250W | 28898MiB / 32768MiB | 45% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 56W / 250W | 28758MiB / 32768MiB | 66% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n", "parsed": [ { "power_w": 56, "power_cap_w": 250, "mem_mib": 28955, "mem_total_mib": 32768, "util_pct": 61 }, { "power_w": 56, "power_cap_w": 250, "mem_mib": 28894, "mem_total_mib": 32768, "util_pct": 63 }, { "power_w": 56, "power_cap_w": 250, "mem_mib": 28898, "mem_total_mib": 32768, "util_pct": 45 }, { "power_w": 56, "power_cap_w": 250, "mem_mib": 28758, "mem_total_mib": 32768, "util_pct": 66 } ] }, { "ts": "2026-07-14T17:13:23", "ok": true, "raw": "Timestamp Tue Jul 14 17:13:23 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 57W / 250W | 28955MiB / 32768MiB | 67% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 34C P0 56W / 250W | 28894MiB / 32768MiB | 57% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 33C P0 56W / 250W | 28898MiB / 32768MiB | 54% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 56W / 250W | 28758MiB / 32768MiB | 45% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n", "parsed": [ { "power_w": 57, "power_cap_w": 250, "mem_mib": 28955, "mem_total_mib": 32768, "util_pct": 67 }, { "power_w": 56, "power_cap_w": 250, "mem_mib": 28894, "mem_total_mib": 32768, "util_pct": 57 }, { "power_w": 56, "power_cap_w": 250, "mem_mib": 28898, "mem_total_mib": 32768, "util_pct": 54 }, { "power_w": 56, "power_cap_w": 250, "mem_mib": 28758, "mem_total_mib": 32768, "util_pct": 45 } ] }, { "ts": "2026-07-14T17:13:25", "ok": true, "raw": "Timestamp Tue Jul 14 17:13:25 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 57W / 250W | 28955MiB / 32768MiB | 65% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 34C P0 57W / 250W | 28894MiB / 32768MiB | 36% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 33C P0 58W / 250W | 28898MiB / 32768MiB | 70% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 57W / 250W | 28758MiB / 32768MiB | 69% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n", "parsed": [ { "power_w": 57, "power_cap_w": 250, "mem_mib": 28955, "mem_total_mib": 32768, "util_pct": 65 }, { "power_w": 57, "power_cap_w": 250, "mem_mib": 28894, "mem_total_mib": 32768, "util_pct": 36 }, { "power_w": 58, "power_cap_w": 250, "mem_mib": 28898, "mem_total_mib": 32768, "util_pct": 70 }, { "power_w": 57, "power_cap_w": 250, "mem_mib": 28758, "mem_total_mib": 32768, "util_pct": 69 } ] } ]