From 465a9c89952e7efb96fdd7f94bd6814bde99e3c3 Mon Sep 17 00:00:00 2001 From: the magician <82004885@qq.com> Date: Tue, 14 Jul 2026 17:11:28 +0800 Subject: [PATCH] add decode throughput analysis --- worklogs/2026-07-13-initial-run.md | 24 + worklogs/decode_analysis_report_2026-07-14.md | 299 +++ worklogs/decode_microbench.py | 314 +++ .../decode_full_parser_short_c1_t256_r3.json | 66 + .../decode_full_parser_tool_c1_t128_r3.json | 63 + .../decode_noparser_short_c1_t128_r4.json | 75 + .../decode_noparser_short_c1_t256_r3.json | 63 + .../decode_noparser_short_c2_t128_r4.json | 75 + ...ode_noparser_short_c2_t128_r4_monitor.json | 84 + .../decode_noparser_short_c4_t128_r4.json | 75 + ...smi_noparser_short_c2_t128_r4_monitor.json | 1822 +++++++++++++++++ 11 files changed, 2960 insertions(+) create mode 100644 worklogs/decode_analysis_report_2026-07-14.md create mode 100644 worklogs/decode_microbench.py create mode 100644 worklogs/remote_results/2026-07-14-decode/decode_full_parser_short_c1_t256_r3.json create mode 100644 worklogs/remote_results/2026-07-14-decode/decode_full_parser_tool_c1_t128_r3.json create mode 100644 worklogs/remote_results/2026-07-14-decode/decode_noparser_short_c1_t128_r4.json create mode 100644 worklogs/remote_results/2026-07-14-decode/decode_noparser_short_c1_t256_r3.json create mode 100644 worklogs/remote_results/2026-07-14-decode/decode_noparser_short_c2_t128_r4.json create mode 100644 worklogs/remote_results/2026-07-14-decode/decode_noparser_short_c2_t128_r4_monitor.json create mode 100644 worklogs/remote_results/2026-07-14-decode/decode_noparser_short_c4_t128_r4.json create mode 100644 worklogs/remote_results/2026-07-14-decode/ixsmi_noparser_short_c2_t128_r4_monitor.json diff --git a/worklogs/2026-07-13-initial-run.md b/worklogs/2026-07-13-initial-run.md index dca0b9d..5225c76 100644 --- a/worklogs/2026-07-13-initial-run.md +++ b/worklogs/2026-07-13-initial-run.md @@ -244,3 +244,27 @@ - Interpretation: - Multi-step scheduling is not available with the current xFormers + chunked-prefill path. - Next decode optimization should focus on either enabling a compatible attention backend, checking whether chunked prefill can be disabled for a decode-focused variant, or optimizing parser/reasoning/tool overhead before deeper code changes. + +## 2026-07-14 Decode Microbenchmark Round + +- Added local and remote script: + - Local: `worklogs/decode_microbench.py` + - Remote: `/root/work/decode_microbench.py` +- Synced raw result JSON files to: + - `worklogs/remote_results/2026-07-14-decode/` +- Wrote Chinese analysis report: + - `worklogs/decode_analysis_report_2026-07-14.md` +- Key results: + - Full parser, short prompt, concurrency 1, 256 output tokens: Output TPS P10 `8.74 tok/s`, TTFT P90 `1.85s`. + - Full parser with 29 tools, concurrency 1, 128 output tokens: Output TPS P10 `8.70 tok/s`, TTFT P90 `4.98s`. + - Parser-off, short prompt, concurrency 1, 256 output tokens: Output TPS P10 `8.30 tok/s`, TTFT P90 `3.18s`. + - Parser-off concurrency curve with 128 output tokens: + - c1: aggregate Output TPS `8.00`, per-request P10 `8.30`. + - c2: aggregate Output TPS `7.23`, per-request P10 `3.68`. + - c4: aggregate Output TPS `5.08`, per-request P10 `2.57`, TTFT P90 `51.60s`. + - c2 with `ixsmi` sampling: average GPU utilization `28.91%`, max GPU utilization `100%`, average power `49.67W`. +- Interpretation: + - Tool/reasoning parser is not the decode throughput bottleneck. + - Increasing concurrency does not improve aggregate decode throughput and worsens per-request TPS. + - Low average GPU utilization suggests scheduling, TP synchronization, MoE kernel, paged attention, or xFormers backend limitations before raw compute saturation. + - Next focused experiment: disable chunked prefill and retry `--num-scheduler-steps 4` as a decode-only diagnostic variant. diff --git a/worklogs/decode_analysis_report_2026-07-14.md b/worklogs/decode_analysis_report_2026-07-14.md new file mode 100644 index 0000000..d11b239 --- /dev/null +++ b/worklogs/decode_analysis_report_2026-07-14.md @@ -0,0 +1,299 @@ +# Decode 吞吐专项实验分析报告 + +日期：2026-07-14 + +## 一、结论摘要 + +本轮实验确认：当前服务的主要短板是 decode 阶段吞吐，而不是 prompt prefill、tool parser 或 reasoning parser。 + +核心证据： + +- 短 prompt、单并发、256 token 输出时，Output TPS P10 只有 `8.3-8.7 tok/s`，明显低于官方目标 `>=20 tok/s`。 +- 去掉 `--enable-auto-tool-choice`、`--tool-call-parser`、`--reasoning-parser` 后，decode TPS 没有提升，反而略低。 +- 加并发后没有获得 batch 增益：并发 1 聚合约 `8.00 tok/s`，并发 2 聚合约 `7.23 tok/s`，并发 4 聚合约 `5.08 tok/s`。 +- 并发 2 时硬件采样显示平均 GPU-Util 只有 `28.9%`，平均功耗约 `49.7W / 250W`，说明 GPU 没有被持续打满。 + +初步判断：瓶颈更像是 decode 路径中的调度、TP 同步、MoE kernel/专家路由、paged attention/kernel launch 开销，或 xFormers 后端限制，而不是 API parser 层。 + +## 二、实验环境 + +远端服务： + +- 模型：`/root/public-storage/models/Qwen/Qwen3.6-35B-A3B` +- API：`vllm.entrypoints.openai.api_server` +- GPU：4 x Iluvatar BI-V100, 32GB +- 当前主要实验配置： + - `-tp 4` + - `--gpu-memory-utilization 0.95` + - `--max-num-seqs 2` + - `--max-num-batched-tokens 8192` + - `--enable-chunked-prefill` + - `--enable-prefix-caching` + +本地脚本： + +- `worklogs/decode_microbench.py` + +远端脚本： + +- `/root/work/decode_microbench.py` + +本地原始结果： + +- `worklogs/remote_results/2026-07-14-decode/` + +## 三、实验结果 + +### 1. 纯短 prompt decode 基线 + +完整 parser 配置，短 prompt，单并发，3 请求，每请求 `max_tokens=256`。 + +结果文件： + +- 远端：`/root/work/logs/decode_full_parser_short_c1_t256_r3.json` +- 本地：`worklogs/remote_results/2026-07-14-decode/decode_full_parser_short_c1_t256_r3.json` + +指标： + +| 指标 | 数值 | +|---|---:| +| 成功率 | 100% | +| TTFT P90 | 1.85s | +| Output TPS P10 | 8.74 tok/s | +| Output TPS P50 | 8.74 tok/s | +| 聚合 Output TPS | 8.34 tok/s | +| 总 completion tokens | 768 | +| reasoning tokens | 768 | + +解释： + +短 prompt 下 TTFT 已经较低，但 decode 速度仍只有约 `8.7 tok/s`。这说明长上下文不是唯一问题，短输出生成本身就偏慢。 + +### 2. Tool/parser 触发场景 + +完整 parser 配置，携带 29 个 tools，单并发，3 请求，每请求 `max_tokens=128`。 + +结果文件： + +- 远端：`/root/work/logs/decode_full_parser_tool_c1_t128_r3.json` +- 本地：`worklogs/remote_results/2026-07-14-decode/decode_full_parser_tool_c1_t128_r3.json` + +指标： + +| 指标 | 数值 | +|---|---:| +| 成功率 | 100% | +| TTFT P90 | 4.98s | +| Output TPS P10 | 8.70 tok/s | +| Output TPS P50 | 8.70 tok/s | +| 聚合 Output TPS | 7.38 tok/s | +| prompt tokens | 6639 | +| cached tokens | 4416 | +| completion tokens | 384 | + +单请求细节： + +- 第 1 个请求：TTFT `5.97s`，cached tokens `0` +- 第 2/3 个请求：TTFT 约 `1.0s`，cached tokens `2208` + +解释： + +工具 schema 会明显影响首次 prefill/TTFT，但前缀缓存命中后 TTFT 恢复。Output TPS 仍约 `8.7 tok/s`，与纯短 prompt 基本一致，所以 tool parser 不是 decode TPS 主瓶颈。 + +### 3. Parser-off 对照 + +重启服务，去掉： + +- `--enable-auto-tool-choice` +- `--tool-call-parser qwen3_coder` +- `--reasoning-parser qwen3` + +其余参数保持一致。 + +短 prompt，单并发，3 请求，每请求 `max_tokens=256`。 + +结果文件： + +- 远端：`/root/work/logs/decode_noparser_short_c1_t256_r3.json` +- 本地：`worklogs/remote_results/2026-07-14-decode/decode_noparser_short_c1_t256_r3.json` + +指标： + +| 指标 | 完整 parser | parser-off | +|---|---:|---:| +| 成功率 | 100% | 100% | +| TTFT P90 | 1.85s | 3.18s | +| Output TPS P10 | 8.74 | 8.30 | +| Output TPS P50 | 8.74 | 8.33 | +| 聚合 Output TPS | 8.34 | 7.87 | + +解释： + +关闭 parser 没有改善 decode。parser/reasoning/tool 相关启动项不是当前 decode 吞吐低的主因。 + +### 4. 并发曲线 + +parser-off 服务，短 prompt，每请求 `max_tokens=128`。 + +结果文件： + +- `decode_noparser_short_c1_t128_r4.json` +- `decode_noparser_short_c2_t128_r4.json` +- `decode_noparser_short_c4_t128_r4.json` + +指标： + +| 并发 | 请求数 | 成功率 | TTFT P90 | Output TPS P10/请求 | 聚合 Output TPS | +|---:|---:|---:|---:|---:|---:| +| 1 | 4 | 100% | 0.79s | 8.30 | 8.00 | +| 2 | 4 | 100% | 0.92s | 3.68 | 7.23 | +| 4 | 4 | 100% | 51.60s | 2.57 | 5.08 | + +解释： + +并发提高后没有形成有效 batch 增益。并发 2 时单请求 TPS 近似减半，聚合 TPS 也略降；并发 4 时出现明显排队，TTFT P90 被拉到 `51.6s`。 + +这说明当前配置下 decode 并发能力很弱。由于服务参数 `--max-num-seqs=2`，并发 4 的后两个请求排队符合预期；但并发 2 聚合吞吐仍不提升，说明 decode 内部没有把双请求 batch 变成更高硬件利用率。 + +### 5. 硬件利用率采样 + +parser-off 服务，并发 2，4 请求，每请求 `max_tokens=128`，同时采样 `ixsmi`。 + +结果文件： + +- 远端：`/root/work/logs/decode_noparser_short_c2_t128_r4_monitor.json` +- 远端：`/root/work/logs/ixsmi_noparser_short_c2_t128_r4_monitor.json` +- 本地同名文件位于：`worklogs/remote_results/2026-07-14-decode/` + +指标： + +| 指标 | 数值 | +|---|---:| +| 成功率 | 100% | +| TTFT P90 | 0.92s | +| Output TPS P10 | 3.79 tok/s | +| 聚合 Output TPS | 7.40 tok/s | +| ixsmi records | 52 | +| parsed GPU samples | 208 | +| 平均 GPU-Util | 28.91% | +| 最大 GPU-Util | 100% | +| 平均显存 | 30477 MiB | +| 最大显存 | 30555 MiB | +| 平均功耗 | 49.67 W | +| 最大功耗 | 51 W | + +解释： + +GPU 利用率和功耗都偏低。虽然瞬时 GPU-Util 能到 100%，但平均只有约 29%，功耗长期接近空载到轻载水平。这说明 decode 过程中存在大量空泡、同步等待或小 kernel 启动开销，GPU 算力没有被持续喂满。 + +## 四、瓶颈判断 + +当前最可能的瓶颈排序： + +1. Decode 调度/后端限制 + - `--num-scheduler-steps 4` 曾尝试失败。 + - 报错：`Multi-Step + Chunked-Prefill not supported for attention backend: xformers`。 + - 当前服务日志显示使用 `XFormers backend`。 + +2. TP 通信或同步开销 + - 模型使用 `-tp 4`。 + - 短 decode 每步都可能涉及多卡同步。 + - 并发 2 聚合 TPS 不升反降，符合小 batch 多卡同步效率差的特征。 + +3. MoE decode kernel/专家路由效率 + - Qwen3.6-35B-A3B 是 MoE 模型。 + - decode batch 小时，专家路由和 fused MoE kernel 可能难以形成高利用率。 + +4. Paged attention / attention backend 每 token 开销 + - 当前 attention 后端为 xFormers。 + - multi-step decode 被 xFormers + chunked prefill 组合限制。 + +5. API/parser 层 + - 本轮实验基本排除其为主瓶颈。 + - tool/schema 影响首次 TTFT，但不显著影响 decode TPS。 + +## 五、下一步建议 + +建议下一步不要直接改大段 kernel，而是先做两个能明确指向代码修改方向的服务变体实验。 + +### 实验 A：decode-only multi-step 变体 + +目的：验证 multi-step scheduling 是否能明显提升 decode。 + +做法： + +- 暂时关闭 `--enable-chunked-prefill` +- 加回 `--num-scheduler-steps 4` +- 保持 `max_num_seqs=2` +- 跑短 prompt decode 并发 1/2 曲线 + +判断： + +- 如果 Output TPS 明显提升，说明 decode 调度是关键方向。 +- 后续要研究如何让 multi-step 与长上下文 chunked prefill 共存，或按场景切换。 + +风险： + +- 官方长上下文负载仍需要 chunked prefill，所以这不是最终配置，只是定位实验。 + +### 实验 B：attention backend 变体 + +目的：确认是否可以启用兼容 multi-step 的 attention backend。 + +做法： + +- 尝试设置 `VLLM_ATTENTION_BACKEND=FLASH_ATTN` +- 或检查 CoreX/xFormers flash attention 能否被 vLLM selector 选中 +- 如果能启动，再测试 multi-step + chunked prefill + +判断： + +- 如果 flash attention 能启动且 multi-step 可用，优先走 backend 配置/适配路线。 +- 如果不能启动，需要看 `vllm/attention/selector.py`、`vllm/attention/backends/*` 和 CoreX xFormers 补丁。 + +### 实验 C：代码级 profiling + +目的：把低 GPU 利用率归因到具体模块。 + +建议插桩位置： + +- `vllm/worker/model_runner.py` +- `vllm/worker/multi_step_model_runner.py` +- `vllm/model_executor/models/qwen3_moe.py` +- `vllm/model_executor/layers/fused_moe/*` +- `attention.py` +- `paged_attn.py` + +记录每步： + +- model forward 耗时 +- attention 耗时 +- MoE/MLP 耗时 +- sampler 耗时 +- 每步前后同步耗时 + +### 实验 D：服务参数小网格 + +目的：确认当前 `max_num_seqs=2` 是否已经是最优。 + +建议组合： + +| 参数 | 候选 | +|---|---| +| `max_num_seqs` | 1, 2, 4 | +| `max_num_batched_tokens` | 4096, 8192 | +| `chunked_prefill` | on/off | +| `num_scheduler_steps` | 1, 4 | + +优先只在短 prompt decode 上跑，快速筛掉无效组合。 + +## 六、当前推荐行动 + +下一步优先跑： + +1. `chunked_prefill=off + num_scheduler_steps=4` +2. 若能启动，跑 decode c1/c2/c4 曲线和 ixsmi 采样 +3. 若 decode TPS 明显提升，再研究如何兼容官方长上下文 prefill +4. 若没有提升，进入 qwen3_moe / fused_moe / attention 的代码级 profiling + +本轮最重要的事实是：GPU 平均利用率只有约 `29%`，所以先不要把问题简单归因为“卡算不动”。更像是当前 decode 执行路径没有把四张卡持续喂满。 diff --git a/worklogs/decode_microbench.py b/worklogs/decode_microbench.py new file mode 100644 index 0000000..748461c --- /dev/null +++ b/worklogs/decode_microbench.py @@ -0,0 +1,314 @@ +import argparse +import concurrent.futures +import json +import math +import os +import re +import subprocess +import threading +import time +import urllib.error +import urllib.request +from dataclasses import asdict, dataclass +from datetime import datetime +from pathlib import Path + + +@dataclass +class RequestResult: + ok: bool + elapsed_sec: float + ttft_sec: float | None + completion_tokens: int + prompt_tokens: int + cached_tokens: int + reasoning_tokens: int + output_tps: float | None + chars: int + error: str | None = None + + +def percentile(values, pct): + values = sorted(v for v in values if v is not None) + if not values: + return None + if len(values) == 1: + return values[0] + pos = (len(values) - 1) * pct / 100.0 + lo = math.floor(pos) + hi = math.ceil(pos) + if lo == hi: + return values[lo] + return values[lo] * (hi - pos) + values[hi] * (pos - lo) + + +def make_tools(count): + tools = [] + for i in range(count): + tools.append({ + "type": "function", + "function": { + "name": f"tool_{i:02d}_exec", + "description": "Run a deterministic diagnostic action.", + "parameters": { + "type": "object", + "properties": { + "command": {"type": "string"}, + "path": {"type": "string"}, + }, + "required": ["command"], + }, + }, + }) + return tools + + +def make_request(args, idx): + if args.prompt_mode == "short": + user = ( + "Do not explain. Output a comma-separated sequence of four digit " + "numbers starting at 0001. Continue until the token limit stops you." + ) + messages = [{"role": "user", "content": user}] + elif args.prompt_mode == "tool": + messages = [ + {"role": "system", "content": "You are a coding agent. Return concise tool-call-like JSON text."}, + {"role": "user", "content": "Create a shell command to list Python files and print the answer as JSON."}, + ] + else: + raise ValueError(f"unknown prompt_mode: {args.prompt_mode}") + + payload = { + "model": args.model, + "messages": messages, + "max_tokens": args.max_tokens, + "temperature": 0, + "stream": True, + "stream_options": {"include_usage": True}, + } + if args.with_tools: + payload["tools"] = make_tools(args.tool_count) + payload["tool_choice"] = "auto" + return payload + + +def post_stream(url, payload, timeout): + req = urllib.request.Request( + url.rstrip("/") + "/v1/chat/completions", + data=json.dumps(payload, ensure_ascii=False).encode("utf-8"), + headers={"Content-Type": "application/json"}, + method="POST", + ) + start = time.perf_counter() + first_at = None + usage = {} + chars = 0 + try: + with urllib.request.urlopen(req, timeout=timeout) as resp: + for raw in resp: + line = raw.decode("utf-8", errors="replace").strip() + if not line or not line.startswith("data:"): + continue + data = line[5:].strip() + if data == "[DONE]": + break + obj = json.loads(data) + if obj.get("usage"): + usage = obj["usage"] + for choice in obj.get("choices") or []: + delta = choice.get("delta") or {} + parts = [ + delta.get("content") or "", + delta.get("reasoning_content") or "", + ] + for tool_call in delta.get("tool_calls") or []: + fn = tool_call.get("function") or {} + parts.append(fn.get("name") or "") + parts.append(fn.get("arguments") or "") + added = sum(len(p) for p in parts) + if added and first_at is None: + first_at = time.perf_counter() + chars += added + elapsed = time.perf_counter() - start + ttft = first_at - start if first_at is not None else None + details = usage.get("prompt_tokens_details") or {} + completion_tokens = int(usage.get("completion_tokens") or 0) + decode_sec = elapsed - ttft if ttft is not None else elapsed + return RequestResult( + ok=True, + elapsed_sec=elapsed, + ttft_sec=ttft, + completion_tokens=completion_tokens, + prompt_tokens=int(usage.get("prompt_tokens") or 0), + cached_tokens=int(details.get("cached_tokens") or 0), + reasoning_tokens=int(usage.get("reasoning_tokens") or 0), + output_tps=completion_tokens / decode_sec if completion_tokens and decode_sec > 0 else None, + chars=chars, + ) + except urllib.error.HTTPError as exc: + body = exc.read().decode("utf-8", errors="replace")[:1000] + return RequestResult(False, time.perf_counter() - start, None, 0, 0, 0, 0, None, chars, f"HTTP {exc.code}: {body}") + except Exception as exc: + return RequestResult(False, time.perf_counter() - start, None, 0, 0, 0, 0, None, chars, f"{type(exc).__name__}: {exc}") + + +def parse_ixsmi(raw): + samples = [] + for line in raw.splitlines(): + m = re.search(r"\|\s*\d+%\s+\d+C\s+\S+\s+(\d+)W\s*/\s*(\d+)W\s*\|\s*(\d+)MiB\s*/\s*(\d+)MiB\s*\|\s*(\d+)%", line) + if m: + samples.append({ + "power_w": int(m.group(1)), + "power_cap_w": int(m.group(2)), + "mem_mib": int(m.group(3)), + "mem_total_mib": int(m.group(4)), + "util_pct": int(m.group(5)), + }) + return samples + + +def monitor_ixsmi(stop_event, out_path, interval): + records = [] + env = dict(os.environ) + env["LD_LIBRARY_PATH"] = ":".join([ + "/usr/local/corex/lib64", + "/usr/local/corex/lib", + "/usr/local/iluvatar/lib64", + env.get("LD_LIBRARY_PATH", ""), + ]) + candidates = [ + "/usr/local/corex/bin/ixsmi", + "/usr/local/iluvatar/bin/ixsmi", + "ixsmi", + ] + while not stop_event.is_set(): + ts = datetime.now().isoformat(timespec="seconds") + try: + last_error = None + raw = None + for binary in candidates: + try: + raw = subprocess.check_output([binary], text=True, stderr=subprocess.STDOUT, timeout=10, env=env) + break + except Exception as exc: + last_error = exc + if raw is None: + raise last_error or RuntimeError("ixsmi not found") + records.append({"ts": ts, "ok": True, "raw": raw, "parsed": parse_ixsmi(raw)}) + except Exception as exc: + records.append({"ts": ts, "ok": False, "error": repr(exc)}) + stop_event.wait(interval) + Path(out_path).write_text(json.dumps(records, ensure_ascii=False, indent=2), encoding="utf-8") + + +def summarize_monitor(path): + if not path or not Path(path).exists(): + return None + records = json.loads(Path(path).read_text(encoding="utf-8")) + util = [] + mem = [] + power = [] + for rec in records: + for gpu in rec.get("parsed") or []: + util.append(gpu["util_pct"]) + mem.append(gpu["mem_mib"]) + power.append(gpu["power_w"]) + if not util: + return {"records": len(records), "parsed_samples": 0} + return { + "records": len(records), + "parsed_samples": len(util), + "avg_gpu_util_pct": sum(util) / len(util), + "max_gpu_util_pct": max(util), + "avg_mem_mib": sum(mem) / len(mem), + "max_mem_mib": max(mem), + "avg_power_w": sum(power) / len(power), + "max_power_w": max(power), + } + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--url", default="http://127.0.0.1:1111") + parser.add_argument("--model", default="llm") + parser.add_argument("--label", required=True) + parser.add_argument("--prompt-mode", choices=["short", "tool"], default="short") + parser.add_argument("--with-tools", action="store_true") + parser.add_argument("--tool-count", type=int, default=16) + parser.add_argument("--concurrency", type=int, default=1) + parser.add_argument("--requests", type=int, default=4) + parser.add_argument("--max-tokens", type=int, default=256) + parser.add_argument("--timeout", type=int, default=900) + parser.add_argument("--monitor-out") + parser.add_argument("--monitor-interval", type=float, default=1.0) + parser.add_argument("--out", required=True) + args = parser.parse_args() + + stop_event = threading.Event() + monitor_thread = None + if args.monitor_out: + monitor_thread = threading.Thread( + target=monitor_ixsmi, + args=(stop_event, args.monitor_out, args.monitor_interval), + daemon=True, + ) + monitor_thread.start() + + started = time.perf_counter() + results = [] + try: + with concurrent.futures.ThreadPoolExecutor(max_workers=args.concurrency) as pool: + futures = [ + pool.submit(post_stream, args.url, make_request(args, i), args.timeout) + for i in range(args.requests) + ] + for fut in concurrent.futures.as_completed(futures): + result = fut.result() + results.append(result) + print(f"done {len(results)}/{args.requests} ok={result.ok} tps={result.output_tps}", flush=True) + finally: + stop_event.set() + if monitor_thread: + monitor_thread.join(timeout=15) + + wall = time.perf_counter() - started + ok = [r for r in results if r.ok] + tps_values = [r.output_tps for r in ok if r.output_tps is not None] + ttft_values = [r.ttft_sec for r in ok if r.ttft_sec is not None] + completion = sum(r.completion_tokens for r in ok) + prompt = sum(r.prompt_tokens for r in ok) + cached = sum(r.cached_tokens for r in ok) + summary = { + "created_at": datetime.now().isoformat(timespec="seconds"), + "label": args.label, + "url": args.url, + "model": args.model, + "prompt_mode": args.prompt_mode, + "with_tools": args.with_tools, + "tool_count": args.tool_count if args.with_tools else 0, + "concurrency": args.concurrency, + "requests": args.requests, + "max_tokens": args.max_tokens, + "wall_sec": wall, + "success_rate": len(ok) / len(results) if results else 0, + "ttft_p50_sec": percentile(ttft_values, 50), + "ttft_p90_sec": percentile(ttft_values, 90), + "output_tps_p10_per_request": percentile(tps_values, 10), + "output_tps_p50_per_request": percentile(tps_values, 50), + "aggregate_output_tps": completion / wall if wall > 0 else 0, + "prompt_tokens": prompt, + "cached_tokens": cached, + "completion_tokens": completion, + "reasoning_tokens": sum(r.reasoning_tokens for r in ok), + "chars": sum(r.chars for r in ok), + "monitor": summarize_monitor(args.monitor_out) if args.monitor_out else None, + "results": [asdict(r) for r in results], + } + Path(args.out).parent.mkdir(parents=True, exist_ok=True) + Path(args.out).write_text(json.dumps(summary, ensure_ascii=False, indent=2), encoding="utf-8") + print(json.dumps({k: v for k, v in summary.items() if k != "results"}, ensure_ascii=False, indent=2)) + print("RESULT_FILE", args.out) + + +if __name__ == "__main__": + main() diff --git a/worklogs/remote_results/2026-07-14-decode/decode_full_parser_short_c1_t256_r3.json b/worklogs/remote_results/2026-07-14-decode/decode_full_parser_short_c1_t256_r3.json new file mode 100644 index 0000000..e77c3e0 --- /dev/null +++ b/worklogs/remote_results/2026-07-14-decode/decode_full_parser_short_c1_t256_r3.json @@ -0,0 +1,66 @@ +{ + "created_at": "2026-07-14T08:00:25", + "label": "full_parser_short_c1_t256_r3", + "url": "http://127.0.0.1:1111", + "model": "llm", + "prompt_mode": "short", + "with_tools": false, + "tool_count": 0, + "concurrency": 1, + "requests": 3, + "max_tokens": 256, + "wall_sec": 92.0443243663758, + "success_rate": 1.0, + "ttft_p50_sec": 1.0909903924912214, + "ttft_p90_sec": 1.8471774261444807, + "output_tps_p10_per_request": 8.736438485385, + "output_tps_p50_per_request": 8.738006408510326, + "aggregate_output_tps": 8.343806153033741, + "prompt_tokens": 117, + "cached_tokens": 64, + "completion_tokens": 768, + "reasoning_tokens": 768, + "chars": 2635, + "monitor": { + "records": 92, + "parsed_samples": 0 + }, + "results": [ + { + "ok": true, + "elapsed_sec": 31.292513709515333, + "ttft_sec": 2.0362241845577955, + "completion_tokens": 256, + "prompt_tokens": 39, + "cached_tokens": 0, + "reasoning_tokens": 256, + "output_tps": 8.750255215433768, + "chars": 916, + "error": null + }, + { + "ok": true, + "elapsed_sec": 30.353378538042307, + "ttft_sec": 1.0560779832303524, + "completion_tokens": 256, + "prompt_tokens": 39, + "cached_tokens": 32, + "reasoning_tokens": 256, + "output_tps": 8.738006408510326, + "chars": 856, + "error": null + }, + { + "ok": true, + "elapsed_sec": 30.394863702356815, + "ttft_sec": 1.0909903924912214, + "completion_tokens": 256, + "prompt_tokens": 39, + "cached_tokens": 32, + "reasoning_tokens": 256, + "output_tps": 8.736046504603667, + "chars": 863, + "error": null + } + ] +} \ No newline at end of file diff --git a/worklogs/remote_results/2026-07-14-decode/decode_full_parser_tool_c1_t128_r3.json b/worklogs/remote_results/2026-07-14-decode/decode_full_parser_tool_c1_t128_r3.json new file mode 100644 index 0000000..d43d114 --- /dev/null +++ b/worklogs/remote_results/2026-07-14-decode/decode_full_parser_tool_c1_t128_r3.json @@ -0,0 +1,63 @@ +{ + "created_at": "2026-07-14T08:11:04", + "label": "full_parser_tool_c1_t128_r3", + "url": "http://127.0.0.1:1111", + "model": "llm", + "prompt_mode": "tool", + "with_tools": true, + "tool_count": 29, + "concurrency": 1, + "requests": 3, + "max_tokens": 128, + "wall_sec": 52.042631950229406, + "success_rate": 1.0, + "ttft_p50_sec": 1.0516742002218962, + "ttft_p90_sec": 4.98421496860683, + "output_tps_p10_per_request": 8.700721568035489, + "output_tps_p50_per_request": 8.704939531751927, + "aggregate_output_tps": 7.378566102637461, + "prompt_tokens": 6639, + "cached_tokens": 4416, + "completion_tokens": 384, + "reasoning_tokens": 384, + "chars": 1309, + "monitor": null, + "results": [ + { + "ok": true, + "elapsed_sec": 20.671645294874907, + "ttft_sec": 5.967350160703063, + "completion_tokens": 128, + "prompt_tokens": 2213, + "cached_tokens": 0, + "reasoning_tokens": 128, + "output_tps": 8.704939531751927, + "chars": 445, + "error": null + }, + { + "ok": true, + "elapsed_sec": 15.636568604037166, + "ttft_sec": 1.0516742002218962, + "completion_tokens": 128, + "prompt_tokens": 2213, + "cached_tokens": 2208, + "reasoning_tokens": 128, + "output_tps": 8.776203409914055, + "chars": 456, + "error": null + }, + { + "ok": true, + "elapsed_sec": 15.731618992984295, + "ttft_sec": 1.018412284553051, + "completion_tokens": 128, + "prompt_tokens": 2213, + "cached_tokens": 2208, + "reasoning_tokens": 128, + "output_tps": 8.699667077106378, + "chars": 408, + "error": null + } + ] +} \ No newline at end of file diff --git a/worklogs/remote_results/2026-07-14-decode/decode_noparser_short_c1_t128_r4.json b/worklogs/remote_results/2026-07-14-decode/decode_noparser_short_c1_t128_r4.json new file mode 100644 index 0000000..b246051 --- /dev/null +++ b/worklogs/remote_results/2026-07-14-decode/decode_noparser_short_c1_t128_r4.json @@ -0,0 +1,75 @@ +{ + "created_at": "2026-07-14T08:34:26", + "label": "noparser_short_c1_t128_r4", + "url": "http://127.0.0.1:1111", + "model": "llm", + "prompt_mode": "short", + "with_tools": false, + "tool_count": 0, + "concurrency": 1, + "requests": 4, + "max_tokens": 128, + "wall_sec": 64.02519051544368, + "success_rate": 1.0, + "ttft_p50_sec": 0.7723927367478609, + "ttft_p90_sec": 0.7913718132302165, + "output_tps_p10_per_request": 8.304385542911493, + "output_tps_p50_per_request": 8.42998946937195, + "aggregate_output_tps": 7.99685242446095, + "prompt_tokens": 156, + "cached_tokens": 128, + "completion_tokens": 512, + "reasoning_tokens": 0, + "chars": 1827, + "monitor": null, + "results": [ + { + "ok": true, + "elapsed_sec": 16.272426838055253, + "ttft_sec": 0.7992097493261099, + "completion_tokens": 128, + "prompt_tokens": 39, + "cached_tokens": 32, + "reasoning_tokens": 0, + "output_tps": 8.272358570683828, + "chars": 474, + "error": null + }, + { + "ok": true, + "elapsed_sec": 15.848933763802052, + "ttft_sec": 0.7730832956731319, + "completion_tokens": 128, + "prompt_tokens": 39, + "cached_tokens": 32, + "reasoning_tokens": 0, + "output_tps": 8.490399945966447, + "chars": 449, + "error": null + }, + { + "ok": true, + "elapsed_sec": 16.047778205946088, + "ttft_sec": 0.7717021778225899, + "completion_tokens": 128, + "prompt_tokens": 39, + "cached_tokens": 32, + "reasoning_tokens": 0, + "output_tps": 8.379115144776051, + "chars": 452, + "error": null + }, + { + "ok": true, + "elapsed_sec": 15.853784639388323, + "ttft_sec": 0.7609824072569609, + "completion_tokens": 128, + "prompt_tokens": 39, + "cached_tokens": 32, + "reasoning_tokens": 0, + "output_tps": 8.480863793967849, + "chars": 452, + "error": null + } + ] +} \ No newline at end of file diff --git a/worklogs/remote_results/2026-07-14-decode/decode_noparser_short_c1_t256_r3.json b/worklogs/remote_results/2026-07-14-decode/decode_noparser_short_c1_t256_r3.json new file mode 100644 index 0000000..f02aa87 --- /dev/null +++ b/worklogs/remote_results/2026-07-14-decode/decode_noparser_short_c1_t256_r3.json @@ -0,0 +1,63 @@ +{ + "created_at": "2026-07-14T08:23:01", + "label": "noparser_short_c1_t256_r3", + "url": "http://127.0.0.1:1111", + "model": "llm", + "prompt_mode": "short", + "with_tools": false, + "tool_count": 0, + "concurrency": 1, + "requests": 3, + "max_tokens": 256, + "wall_sec": 97.52940320037305, + "success_rate": 1.0, + "ttft_p50_sec": 0.8136040437966585, + "ttft_p90_sec": 3.1817516405135393, + "output_tps_p10_per_request": 8.295711129981452, + "output_tps_p50_per_request": 8.327121469734012, + "aggregate_output_tps": 7.874548339254703, + "prompt_tokens": 117, + "cached_tokens": 64, + "completion_tokens": 768, + "reasoning_tokens": 0, + "chars": 2635, + "monitor": null, + "results": [ + { + "ok": true, + "elapsed_sec": 34.66234661638737, + "ttft_sec": 3.7737885396927595, + "completion_tokens": 256, + "prompt_tokens": 39, + "cached_tokens": 0, + "reasoning_tokens": 0, + "output_tps": 8.287858545043312, + "chars": 916, + "error": null + }, + { + "ok": true, + "elapsed_sec": 31.533973263576627, + "ttft_sec": 0.7910567671060562, + "completion_tokens": 256, + "prompt_tokens": 39, + "cached_tokens": 32, + "reasoning_tokens": 0, + "output_tps": 8.327121469734012, + "chars": 856, + "error": null + }, + { + "ok": true, + "elapsed_sec": 31.330975525081158, + "ttft_sec": 0.8136040437966585, + "completion_tokens": 256, + "prompt_tokens": 39, + "cached_tokens": 32, + "reasoning_tokens": 0, + "output_tps": 8.388664802176624, + "chars": 863, + "error": null + } + ] +} \ No newline at end of file diff --git a/worklogs/remote_results/2026-07-14-decode/decode_noparser_short_c2_t128_r4.json b/worklogs/remote_results/2026-07-14-decode/decode_noparser_short_c2_t128_r4.json new file mode 100644 index 0000000..7642002 --- /dev/null +++ b/worklogs/remote_results/2026-07-14-decode/decode_noparser_short_c2_t128_r4.json @@ -0,0 +1,75 @@ +{ + "created_at": "2026-07-14T08:35:37", + "label": "noparser_short_c2_t128_r4", + "url": "http://127.0.0.1:1111", + "model": "llm", + "prompt_mode": "short", + "with_tools": false, + "tool_count": 0, + "concurrency": 2, + "requests": 4, + "max_tokens": 128, + "wall_sec": 70.80830597691238, + "success_rate": 1.0, + "ttft_p50_sec": 0.9245951194316149, + "ttft_p90_sec": 0.9249729935079813, + "output_tps_p10_per_request": 3.6832897295816034, + "output_tps_p50_per_request": 3.7126825201316764, + "aggregate_output_tps": 7.230790130284175, + "prompt_tokens": 156, + "cached_tokens": 128, + "completion_tokens": 512, + "reasoning_tokens": 0, + "chars": 1816, + "monitor": null, + "results": [ + { + "ok": true, + "elapsed_sec": 35.12894960306585, + "ttft_sec": 0.9234285913407803, + "completion_tokens": 128, + "prompt_tokens": 39, + "cached_tokens": 32, + "reasoning_tokens": 0, + "output_tps": 3.742085961974495, + "chars": 454, + "error": null + }, + { + "ok": true, + "elapsed_sec": 35.12996072135866, + "ttft_sec": 0.9242945611476898, + "completion_tokens": 128, + "prompt_tokens": 39, + "cached_tokens": 32, + "reasoning_tokens": 0, + "output_tps": 3.7420700827891884, + "chars": 454, + "error": null + }, + { + "ok": true, + "elapsed_sec": 35.67656988278031, + "ttft_sec": 0.925006128847599, + "completion_tokens": 128, + "prompt_tokens": 39, + "cached_tokens": 32, + "reasoning_tokens": 0, + "output_tps": 3.683287489056221, + "chars": 454, + "error": null + }, + { + "ok": true, + "elapsed_sec": 35.67638896778226, + "ttft_sec": 0.9248956777155399, + "completion_tokens": 128, + "prompt_tokens": 39, + "cached_tokens": 32, + "reasoning_tokens": 0, + "output_tps": 3.683294957474164, + "chars": 454, + "error": null + } + ] +} \ No newline at end of file diff --git a/worklogs/remote_results/2026-07-14-decode/decode_noparser_short_c2_t128_r4_monitor.json b/worklogs/remote_results/2026-07-14-decode/decode_noparser_short_c2_t128_r4_monitor.json new file mode 100644 index 0000000..d11ad2e --- /dev/null +++ b/worklogs/remote_results/2026-07-14-decode/decode_noparser_short_c2_t128_r4_monitor.json @@ -0,0 +1,84 @@ +{ + "created_at": "2026-07-14T08:56:17", + "label": "noparser_short_c2_t128_r4_monitor", + "url": "http://127.0.0.1:1111", + "model": "llm", + "prompt_mode": "short", + "with_tools": false, + "tool_count": 0, + "concurrency": 2, + "requests": 4, + "max_tokens": 128, + "wall_sec": 69.160331716761, + "success_rate": 1.0, + "ttft_p50_sec": 0.9177342765033245, + "ttft_p90_sec": 0.9243953077122569, + "output_tps_p10_per_request": 3.7938414669033445, + "output_tps_p50_per_request": 3.8028299895382536, + "aggregate_output_tps": 7.4030876846693445, + "prompt_tokens": 156, + "cached_tokens": 128, + "completion_tokens": 512, + "reasoning_tokens": 0, + "chars": 1816, + "monitor": { + "records": 52, + "parsed_samples": 208, + "avg_gpu_util_pct": 28.91346153846154, + "max_gpu_util_pct": 100, + "avg_mem_mib": 30477.0, + "max_mem_mib": 30555, + "avg_power_w": 49.67307692307692, + "max_power_w": 51 + }, + "results": [ + { + "ok": true, + "elapsed_sec": 34.663240656256676, + "ttft_sec": 0.9245693255215883, + "completion_tokens": 128, + "prompt_tokens": 39, + "cached_tokens": 32, + "reasoning_tokens": 0, + "output_tps": 3.7938660578905252, + "chars": 454, + "error": null + }, + { + "ok": true, + "elapsed_sec": 34.66297300904989, + "ttft_sec": 0.9239892661571503, + "completion_tokens": 128, + "prompt_tokens": 39, + "cached_tokens": 32, + "reasoning_tokens": 0, + "output_tps": 3.793830927908839, + "chars": 454, + "error": null + }, + { + "ok": true, + "elapsed_sec": 34.49104499258101, + "ttft_sec": 0.9110554531216621, + "completion_tokens": 128, + "prompt_tokens": 39, + "cached_tokens": 32, + "reasoning_tokens": 0, + "output_tps": 3.811793921185982, + "chars": 454, + "error": null + }, + { + "ok": true, + "elapsed_sec": 34.491438234224916, + "ttft_sec": 0.9114792868494987, + "completion_tokens": 128, + "prompt_tokens": 39, + "cached_tokens": 32, + "reasoning_tokens": 0, + "output_tps": 3.811797393814395, + "chars": 454, + "error": null + } + ] +} \ No newline at end of file diff --git a/worklogs/remote_results/2026-07-14-decode/decode_noparser_short_c4_t128_r4.json b/worklogs/remote_results/2026-07-14-decode/decode_noparser_short_c4_t128_r4.json new file mode 100644 index 0000000..bf5333f --- /dev/null +++ b/worklogs/remote_results/2026-07-14-decode/decode_noparser_short_c4_t128_r4.json @@ -0,0 +1,75 @@ +{ + "created_at": "2026-07-14T08:37:18", + "label": "noparser_short_c4_t128_r4", + "url": "http://127.0.0.1:1111", + "model": "llm", + "prompt_mode": "short", + "with_tools": false, + "tool_count": 0, + "concurrency": 4, + "requests": 4, + "max_tokens": 128, + "wall_sec": 100.87516433186829, + "success_rate": 1.0, + "ttft_p50_sec": 26.261823972687125, + "ttft_p90_sec": 51.59956250209361, + "output_tps_p10_per_request": 2.5736346925937763, + "output_tps_p50_per_request": 2.585721667979742, + "aggregate_output_tps": 5.075580331305096, + "prompt_tokens": 156, + "cached_tokens": 128, + "completion_tokens": 512, + "reasoning_tokens": 0, + "chars": 1846, + "monitor": null, + "results": [ + { + "ok": true, + "elapsed_sec": 50.6585742700845, + "ttft_sec": 0.9234585296362638, + "completion_tokens": 128, + "prompt_tokens": 39, + "cached_tokens": 32, + "reasoning_tokens": 0, + "output_tps": 2.573634304341249, + "chars": 454, + "error": null + }, + { + "ok": true, + "elapsed_sec": 50.65938341990113, + "ttft_sec": 0.9242926891893148, + "completion_tokens": 128, + "prompt_tokens": 39, + "cached_tokens": 32, + "reasoning_tokens": 0, + "output_tps": 2.5736355985163404, + "chars": 486, + "error": null + }, + { + "ok": true, + "elapsed_sec": 100.87196587957442, + "ttft_sec": 51.59965132176876, + "completion_tokens": 128, + "prompt_tokens": 39, + "cached_tokens": 32, + "reasoning_tokens": 0, + "output_tps": 2.597807737443144, + "chars": 454, + "error": null + }, + { + "ok": true, + "elapsed_sec": 100.8716309145093, + "ttft_sec": 51.599355256184936, + "completion_tokens": 128, + "prompt_tokens": 39, + "cached_tokens": 32, + "reasoning_tokens": 0, + "output_tps": 2.597809788360666, + "chars": 452, + "error": null + } + ] +} \ No newline at end of file diff --git a/worklogs/remote_results/2026-07-14-decode/ixsmi_noparser_short_c2_t128_r4_monitor.json b/worklogs/remote_results/2026-07-14-decode/ixsmi_noparser_short_c2_t128_r4_monitor.json new file mode 100644 index 0000000..e86cfb4 --- /dev/null +++ b/worklogs/remote_results/2026-07-14-decode/ixsmi_noparser_short_c2_t128_r4_monitor.json @@ -0,0 +1,1822 @@ +[ + { + "ts": "2026-07-14T08:55:08", + "ok": true, + "raw": "Timestamp Tue Jul 14 08:55:08 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 29C P0 47W / 250W | 30555MiB / 32768MiB | 40% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 47W / 250W | 30495MiB / 32768MiB | 24% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 48W / 250W | 30499MiB / 32768MiB | 42% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 30C P0 47W / 250W | 30359MiB / 32768MiB | 36% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n", + "parsed": [ + { + "power_w": 47, + "power_cap_w": 250, + "mem_mib": 30555, + "mem_total_mib": 32768, + "util_pct": 40 + }, + { + "power_w": 47, + "power_cap_w": 250, + "mem_mib": 30495, + "mem_total_mib": 32768, + "util_pct": 24 + }, + { + "power_w": 48, + "power_cap_w": 250, + "mem_mib": 30499, + "mem_total_mib": 32768, + "util_pct": 42 + }, + { + "power_w": 47, + "power_cap_w": 250, + "mem_mib": 30359, + "mem_total_mib": 32768, + "util_pct": 36 + } + ] + }, + { + "ts": "2026-07-14T08:55:10", + "ok": true, + "raw": "Timestamp Tue Jul 14 08:55:10 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 30C P0 50W / 250W | 30555MiB / 32768MiB | 29% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 50W / 250W | 30495MiB / 32768MiB | 42% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 51W / 250W | 30499MiB / 32768MiB | 7% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 30C P0 50W / 250W | 30359MiB / 32768MiB | 33% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n", + "parsed": [ + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30555, + "mem_total_mib": 32768, + "util_pct": 29 + }, + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30495, + "mem_total_mib": 32768, + "util_pct": 42 + }, + { + "power_w": 51, + "power_cap_w": 250, + "mem_mib": 30499, + "mem_total_mib": 32768, + "util_pct": 7 + }, + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30359, + "mem_total_mib": 32768, + "util_pct": 33 + } + ] + }, + { + "ts": "2026-07-14T08:55:11", + "ok": true, + "raw": "Timestamp Tue Jul 14 08:55:11 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 30C P0 50W / 250W | 30555MiB / 32768MiB | 27% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 50W / 250W | 30495MiB / 32768MiB | 39% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 50W / 250W | 30499MiB / 32768MiB | 39% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 30C P0 49W / 250W | 30359MiB / 32768MiB | 46% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n", + "parsed": [ + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30555, + "mem_total_mib": 32768, + "util_pct": 27 + }, + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30495, + "mem_total_mib": 32768, + "util_pct": 39 + }, + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30499, + "mem_total_mib": 32768, + "util_pct": 39 + }, + { + "power_w": 49, + "power_cap_w": 250, + "mem_mib": 30359, + "mem_total_mib": 32768, + "util_pct": 46 + } + ] + }, + { + "ts": "2026-07-14T08:55:12", + "ok": true, + "raw": "Timestamp Tue Jul 14 08:55:12 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 29C P0 49W / 250W | 30555MiB / 32768MiB | 15% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 49W / 250W | 30495MiB / 32768MiB | 15% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 50W / 250W | 30499MiB / 32768MiB | 21% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 30C P0 49W / 250W | 30359MiB / 32768MiB | 24% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n", + "parsed": [ + { + "power_w": 49, + "power_cap_w": 250, + "mem_mib": 30555, + "mem_total_mib": 32768, + "util_pct": 15 + }, + { + "power_w": 49, + "power_cap_w": 250, + "mem_mib": 30495, + "mem_total_mib": 32768, + "util_pct": 15 + }, + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30499, + "mem_total_mib": 32768, + "util_pct": 21 + }, + { + "power_w": 49, + "power_cap_w": 250, + "mem_mib": 30359, + "mem_total_mib": 32768, + "util_pct": 24 + } + ] + }, + { + "ts": "2026-07-14T08:55:14", + "ok": true, + "raw": "Timestamp Tue Jul 14 08:55:14 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 30C P0 50W / 250W | 30555MiB / 32768MiB | 44% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 50W / 250W | 30495MiB / 32768MiB | 20% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 50W / 250W | 30499MiB / 32768MiB | 12% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 30C P0 49W / 250W | 30359MiB / 32768MiB | 34% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n", + "parsed": [ + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30555, + "mem_total_mib": 32768, + "util_pct": 44 + }, + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30495, + "mem_total_mib": 32768, + "util_pct": 20 + }, + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30499, + "mem_total_mib": 32768, + "util_pct": 12 + }, + { + "power_w": 49, + "power_cap_w": 250, + "mem_mib": 30359, + "mem_total_mib": 32768, + "util_pct": 34 + } + ] + }, + { + "ts": "2026-07-14T08:55:15", + "ok": true, + "raw": "Timestamp Tue Jul 14 08:55:15 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 30C P0 50W / 250W | 30555MiB / 32768MiB | 13% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 50W / 250W | 30495MiB / 32768MiB | 20% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 50W / 250W | 30499MiB / 32768MiB | 14% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 30C P0 49W / 250W | 30359MiB / 32768MiB | 21% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n", + "parsed": [ + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30555, + "mem_total_mib": 32768, + "util_pct": 13 + }, + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30495, + "mem_total_mib": 32768, + "util_pct": 20 + }, + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30499, + "mem_total_mib": 32768, + "util_pct": 14 + }, + { + "power_w": 49, + "power_cap_w": 250, + "mem_mib": 30359, + "mem_total_mib": 32768, + "util_pct": 21 + } + ] + }, + { + "ts": "2026-07-14T08:55:16", + "ok": true, + "raw": "Timestamp Tue Jul 14 08:55:16 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 30C P0 50W / 250W | 30555MiB / 32768MiB | 20% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 50W / 250W | 30495MiB / 32768MiB | 39% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 50W / 250W | 30499MiB / 32768MiB | 15% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 30C P0 50W / 250W | 30359MiB / 32768MiB | 45% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n", + "parsed": [ + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30555, + "mem_total_mib": 32768, + "util_pct": 20 + }, + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30495, + "mem_total_mib": 32768, + "util_pct": 39 + }, + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30499, + "mem_total_mib": 32768, + "util_pct": 15 + }, + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30359, + "mem_total_mib": 32768, + "util_pct": 45 + } + ] + }, + { + "ts": "2026-07-14T08:55:18", + "ok": true, + "raw": "Timestamp Tue Jul 14 08:55:18 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 30C P0 50W / 250W | 30555MiB / 32768MiB | 18% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 50W / 250W | 30495MiB / 32768MiB | 20% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 50W / 250W | 30499MiB / 32768MiB | 100% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 30C P0 49W / 250W | 30359MiB / 32768MiB | 21% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n", + "parsed": [ + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30555, + "mem_total_mib": 32768, + "util_pct": 18 + }, + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30495, + "mem_total_mib": 32768, + "util_pct": 20 + }, + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30499, + "mem_total_mib": 32768, + "util_pct": 100 + }, + { + "power_w": 49, + "power_cap_w": 250, + "mem_mib": 30359, + "mem_total_mib": 32768, + "util_pct": 21 + } + ] + }, + { + "ts": "2026-07-14T08:55:19", + "ok": true, + "raw": "Timestamp Tue Jul 14 08:55:19 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 30C P0 50W / 250W | 30555MiB / 32768MiB | 21% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 50W / 250W | 30495MiB / 32768MiB | 27% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 50W / 250W | 30499MiB / 32768MiB | 22% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 30C P0 49W / 250W | 30359MiB / 32768MiB | 23% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n", + "parsed": [ + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30555, + "mem_total_mib": 32768, + "util_pct": 21 + }, + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30495, + "mem_total_mib": 32768, + "util_pct": 27 + }, + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30499, + "mem_total_mib": 32768, + "util_pct": 22 + }, + { + "power_w": 49, + "power_cap_w": 250, + "mem_mib": 30359, + "mem_total_mib": 32768, + "util_pct": 23 + } + ] + }, + { + "ts": "2026-07-14T08:55:20", + "ok": true, + "raw": "Timestamp Tue Jul 14 08:55:20 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 30C P0 50W / 250W | 30555MiB / 32768MiB | 27% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 50W / 250W | 30495MiB / 32768MiB | 44% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 50W / 250W | 30499MiB / 32768MiB | 35% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 30C P0 49W / 250W | 30359MiB / 32768MiB | 24% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n", + "parsed": [ + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30555, + "mem_total_mib": 32768, + "util_pct": 27 + }, + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30495, + "mem_total_mib": 32768, + "util_pct": 44 + }, + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30499, + "mem_total_mib": 32768, + "util_pct": 35 + }, + { + "power_w": 49, + "power_cap_w": 250, + "mem_mib": 30359, + "mem_total_mib": 32768, + "util_pct": 24 + } + ] + }, + { + "ts": "2026-07-14T08:55:22", + "ok": true, + "raw": "Timestamp Tue Jul 14 08:55:22 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 30C P0 50W / 250W | 30555MiB / 32768MiB | 15% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 50W / 250W | 30495MiB / 32768MiB | 20% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 50W / 250W | 30499MiB / 32768MiB | 18% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 49W / 250W | 30359MiB / 32768MiB | 45% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n", + "parsed": [ + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30555, + "mem_total_mib": 32768, + "util_pct": 15 + }, + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30495, + "mem_total_mib": 32768, + "util_pct": 20 + }, + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30499, + "mem_total_mib": 32768, + "util_pct": 18 + }, + { + "power_w": 49, + "power_cap_w": 250, + "mem_mib": 30359, + "mem_total_mib": 32768, + "util_pct": 45 + } + ] + }, + { + "ts": "2026-07-14T08:55:23", + "ok": true, + "raw": "Timestamp Tue Jul 14 08:55:23 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 30C P0 49W / 250W | 30555MiB / 32768MiB | 13% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 50W / 250W | 30495MiB / 32768MiB | 15% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 50W / 250W | 30499MiB / 32768MiB | 49% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 49W / 250W | 30359MiB / 32768MiB | 52% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n", + "parsed": [ + { + "power_w": 49, + "power_cap_w": 250, + "mem_mib": 30555, + "mem_total_mib": 32768, + "util_pct": 13 + }, + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30495, + "mem_total_mib": 32768, + "util_pct": 15 + }, + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30499, + "mem_total_mib": 32768, + "util_pct": 49 + }, + { + "power_w": 49, + "power_cap_w": 250, + "mem_mib": 30359, + "mem_total_mib": 32768, + "util_pct": 52 + } + ] + }, + { + "ts": "2026-07-14T08:55:24", + "ok": true, + "raw": "Timestamp Tue Jul 14 08:55:24 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 30C P0 50W / 250W | 30555MiB / 32768MiB | 21% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 50W / 250W | 30495MiB / 32768MiB | 29% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 50W / 250W | 30499MiB / 32768MiB | 37% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 49W / 250W | 30359MiB / 32768MiB | 15% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n", + "parsed": [ + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30555, + "mem_total_mib": 32768, + "util_pct": 21 + }, + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30495, + "mem_total_mib": 32768, + "util_pct": 29 + }, + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30499, + "mem_total_mib": 32768, + "util_pct": 37 + }, + { + "power_w": 49, + "power_cap_w": 250, + "mem_mib": 30359, + "mem_total_mib": 32768, + "util_pct": 15 + } + ] + }, + { + "ts": "2026-07-14T08:55:26", + "ok": true, + "raw": "Timestamp Tue Jul 14 08:55:26 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 30C P0 50W / 250W | 30555MiB / 32768MiB | 24% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 50W / 250W | 30495MiB / 32768MiB | 18% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 50W / 250W | 30499MiB / 32768MiB | 23% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 49W / 250W | 30359MiB / 32768MiB | 16% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n", + "parsed": [ + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30555, + "mem_total_mib": 32768, + "util_pct": 24 + }, + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30495, + "mem_total_mib": 32768, + "util_pct": 18 + }, + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30499, + "mem_total_mib": 32768, + "util_pct": 23 + }, + { + "power_w": 49, + "power_cap_w": 250, + "mem_mib": 30359, + "mem_total_mib": 32768, + "util_pct": 16 + } + ] + }, + { + "ts": "2026-07-14T08:55:27", + "ok": true, + "raw": "Timestamp Tue Jul 14 08:55:27 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 30C P0 50W / 250W | 30555MiB / 32768MiB | 13% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 50W / 250W | 30495MiB / 32768MiB | 76% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 50W / 250W | 30499MiB / 32768MiB | 50% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 50W / 250W | 30359MiB / 32768MiB | 15% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n", + "parsed": [ + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30555, + "mem_total_mib": 32768, + "util_pct": 13 + }, + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30495, + "mem_total_mib": 32768, + "util_pct": 76 + }, + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30499, + "mem_total_mib": 32768, + "util_pct": 50 + }, + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30359, + "mem_total_mib": 32768, + "util_pct": 15 + } + ] + }, + { + "ts": "2026-07-14T08:55:28", + "ok": true, + "raw": "Timestamp Tue Jul 14 08:55:28 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 30C P0 50W / 250W | 30555MiB / 32768MiB | 24% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 50W / 250W | 30495MiB / 32768MiB | 45% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 50W / 250W | 30499MiB / 32768MiB | 38% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 30C P0 50W / 250W | 30359MiB / 32768MiB | 17% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n", + "parsed": [ + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30555, + "mem_total_mib": 32768, + "util_pct": 24 + }, + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30495, + "mem_total_mib": 32768, + "util_pct": 45 + }, + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30499, + "mem_total_mib": 32768, + "util_pct": 38 + }, + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30359, + "mem_total_mib": 32768, + "util_pct": 17 + } + ] + }, + { + "ts": "2026-07-14T08:55:30", + "ok": true, + "raw": "Timestamp Tue Jul 14 08:55:30 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 30C P0 50W / 250W | 30555MiB / 32768MiB | 19% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 50W / 250W | 30495MiB / 32768MiB | 27% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 50W / 250W | 30499MiB / 32768MiB | 42% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 49W / 250W | 30359MiB / 32768MiB | 34% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n", + "parsed": [ + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30555, + "mem_total_mib": 32768, + "util_pct": 19 + }, + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30495, + "mem_total_mib": 32768, + "util_pct": 27 + }, + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30499, + "mem_total_mib": 32768, + "util_pct": 42 + }, + { + "power_w": 49, + "power_cap_w": 250, + "mem_mib": 30359, + "mem_total_mib": 32768, + "util_pct": 34 + } + ] + }, + { + "ts": "2026-07-14T08:55:31", + "ok": true, + "raw": "Timestamp Tue Jul 14 08:55:31 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 30C P0 49W / 250W | 30555MiB / 32768MiB | 19% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 50W / 250W | 30495MiB / 32768MiB | 38% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 50W / 250W | 30499MiB / 32768MiB | 40% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 30C P0 49W / 250W | 30359MiB / 32768MiB | 37% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n", + "parsed": [ + { + "power_w": 49, + "power_cap_w": 250, + "mem_mib": 30555, + "mem_total_mib": 32768, + "util_pct": 19 + }, + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30495, + "mem_total_mib": 32768, + "util_pct": 38 + }, + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30499, + "mem_total_mib": 32768, + "util_pct": 40 + }, + { + "power_w": 49, + "power_cap_w": 250, + "mem_mib": 30359, + "mem_total_mib": 32768, + "util_pct": 37 + } + ] + }, + { + "ts": "2026-07-14T08:55:32", + "ok": true, + "raw": "Timestamp Tue Jul 14 08:55:32 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 30C P0 50W / 250W | 30555MiB / 32768MiB | 16% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 50W / 250W | 30495MiB / 32768MiB | 47% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 50W / 250W | 30499MiB / 32768MiB | 22% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 49W / 250W | 30359MiB / 32768MiB | 21% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n", + "parsed": [ + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30555, + "mem_total_mib": 32768, + "util_pct": 16 + }, + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30495, + "mem_total_mib": 32768, + "util_pct": 47 + }, + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30499, + "mem_total_mib": 32768, + "util_pct": 22 + }, + { + "power_w": 49, + "power_cap_w": 250, + "mem_mib": 30359, + "mem_total_mib": 32768, + "util_pct": 21 + } + ] + }, + { + "ts": "2026-07-14T08:55:34", + "ok": true, + "raw": "Timestamp Tue Jul 14 08:55:34 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 30C P0 50W / 250W | 30555MiB / 32768MiB | 31% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 50W / 250W | 30495MiB / 32768MiB | 47% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 50W / 250W | 30499MiB / 32768MiB | 26% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 49W / 250W | 30359MiB / 32768MiB | 27% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n", + "parsed": [ + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30555, + "mem_total_mib": 32768, + "util_pct": 31 + }, + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30495, + "mem_total_mib": 32768, + "util_pct": 47 + }, + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30499, + "mem_total_mib": 32768, + "util_pct": 26 + }, + { + "power_w": 49, + "power_cap_w": 250, + "mem_mib": 30359, + "mem_total_mib": 32768, + "util_pct": 27 + } + ] + }, + { + "ts": "2026-07-14T08:55:35", + "ok": true, + "raw": "Timestamp Tue Jul 14 08:55:35 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 30C P0 49W / 250W | 30555MiB / 32768MiB | 39% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 49W / 250W | 30495MiB / 32768MiB | 31% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 50W / 250W | 30499MiB / 32768MiB | 26% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 49W / 250W | 30359MiB / 32768MiB | 32% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n", + "parsed": [ + { + "power_w": 49, + "power_cap_w": 250, + "mem_mib": 30555, + "mem_total_mib": 32768, + "util_pct": 39 + }, + { + "power_w": 49, + "power_cap_w": 250, + "mem_mib": 30495, + "mem_total_mib": 32768, + "util_pct": 31 + }, + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30499, + "mem_total_mib": 32768, + "util_pct": 26 + }, + { + "power_w": 49, + "power_cap_w": 250, + "mem_mib": 30359, + "mem_total_mib": 32768, + "util_pct": 32 + } + ] + }, + { + "ts": "2026-07-14T08:55:36", + "ok": true, + "raw": "Timestamp Tue Jul 14 08:55:37 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 30C P0 49W / 250W | 30555MiB / 32768MiB | 26% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 50W / 250W | 30495MiB / 32768MiB | 24% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 50W / 250W | 30499MiB / 32768MiB | 24% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 49W / 250W | 30359MiB / 32768MiB | 43% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n", + "parsed": [ + { + "power_w": 49, + "power_cap_w": 250, + "mem_mib": 30555, + "mem_total_mib": 32768, + "util_pct": 26 + }, + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30495, + "mem_total_mib": 32768, + "util_pct": 24 + }, + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30499, + "mem_total_mib": 32768, + "util_pct": 24 + }, + { + "power_w": 49, + "power_cap_w": 250, + "mem_mib": 30359, + "mem_total_mib": 32768, + "util_pct": 43 + } + ] + }, + { + "ts": "2026-07-14T08:55:38", + "ok": true, + "raw": "Timestamp Tue Jul 14 08:55:38 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 30C P0 50W / 250W | 30555MiB / 32768MiB | 32% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 50W / 250W | 30495MiB / 32768MiB | 38% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 50W / 250W | 30499MiB / 32768MiB | 39% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 49W / 250W | 30359MiB / 32768MiB | 23% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n", + "parsed": [ + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30555, + "mem_total_mib": 32768, + "util_pct": 32 + }, + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30495, + "mem_total_mib": 32768, + "util_pct": 38 + }, + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30499, + "mem_total_mib": 32768, + "util_pct": 39 + }, + { + "power_w": 49, + "power_cap_w": 250, + "mem_mib": 30359, + "mem_total_mib": 32768, + "util_pct": 23 + } + ] + }, + { + "ts": "2026-07-14T08:55:39", + "ok": true, + "raw": "Timestamp Tue Jul 14 08:55:39 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 30C P0 50W / 250W | 30555MiB / 32768MiB | 24% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 50W / 250W | 30495MiB / 32768MiB | 16% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 50W / 250W | 30499MiB / 32768MiB | 27% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 49W / 250W | 30359MiB / 32768MiB | 27% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n", + "parsed": [ + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30555, + "mem_total_mib": 32768, + "util_pct": 24 + }, + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30495, + "mem_total_mib": 32768, + "util_pct": 16 + }, + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30499, + "mem_total_mib": 32768, + "util_pct": 27 + }, + { + "power_w": 49, + "power_cap_w": 250, + "mem_mib": 30359, + "mem_total_mib": 32768, + "util_pct": 27 + } + ] + }, + { + "ts": "2026-07-14T08:55:40", + "ok": true, + "raw": "Timestamp Tue Jul 14 08:55:40 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 30C P0 50W / 250W | 30555MiB / 32768MiB | 31% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 50W / 250W | 30495MiB / 32768MiB | 34% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 50W / 250W | 30499MiB / 32768MiB | 37% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 49W / 250W | 30359MiB / 32768MiB | 25% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n", + "parsed": [ + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30555, + "mem_total_mib": 32768, + "util_pct": 31 + }, + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30495, + "mem_total_mib": 32768, + "util_pct": 34 + }, + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30499, + "mem_total_mib": 32768, + "util_pct": 37 + }, + { + "power_w": 49, + "power_cap_w": 250, + "mem_mib": 30359, + "mem_total_mib": 32768, + "util_pct": 25 + } + ] + }, + { + "ts": "2026-07-14T08:55:42", + "ok": true, + "raw": "Timestamp Tue Jul 14 08:55:42 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 30C P0 50W / 250W | 30555MiB / 32768MiB | 21% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 50W / 250W | 30495MiB / 32768MiB | 21% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 50W / 250W | 30499MiB / 32768MiB | 19% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 49W / 250W | 30359MiB / 32768MiB | 23% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n", + "parsed": [ + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30555, + "mem_total_mib": 32768, + "util_pct": 21 + }, + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30495, + "mem_total_mib": 32768, + "util_pct": 21 + }, + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30499, + "mem_total_mib": 32768, + "util_pct": 19 + }, + { + "power_w": 49, + "power_cap_w": 250, + "mem_mib": 30359, + "mem_total_mib": 32768, + "util_pct": 23 + } + ] + }, + { + "ts": "2026-07-14T08:55:43", + "ok": true, + "raw": "Timestamp Tue Jul 14 08:55:43 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 30C P0 50W / 250W | 30555MiB / 32768MiB | 28% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 50W / 250W | 30495MiB / 32768MiB | 6% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 50W / 250W | 30499MiB / 32768MiB | 49% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 49W / 250W | 30359MiB / 32768MiB | 25% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n", + "parsed": [ + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30555, + "mem_total_mib": 32768, + "util_pct": 28 + }, + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30495, + "mem_total_mib": 32768, + "util_pct": 6 + }, + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30499, + "mem_total_mib": 32768, + "util_pct": 49 + }, + { + "power_w": 49, + "power_cap_w": 250, + "mem_mib": 30359, + "mem_total_mib": 32768, + "util_pct": 25 + } + ] + }, + { + "ts": "2026-07-14T08:55:44", + "ok": true, + "raw": "Timestamp Tue Jul 14 08:55:44 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 30C P0 50W / 250W | 30555MiB / 32768MiB | 47% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 50W / 250W | 30495MiB / 32768MiB | 22% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 51W / 250W | 30499MiB / 32768MiB | 28% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 30C P0 50W / 250W | 30359MiB / 32768MiB | 24% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n", + "parsed": [ + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30555, + "mem_total_mib": 32768, + "util_pct": 47 + }, + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30495, + "mem_total_mib": 32768, + "util_pct": 22 + }, + { + "power_w": 51, + "power_cap_w": 250, + "mem_mib": 30499, + "mem_total_mib": 32768, + "util_pct": 28 + }, + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30359, + "mem_total_mib": 32768, + "util_pct": 24 + } + ] + }, + { + "ts": "2026-07-14T08:55:46", + "ok": true, + "raw": "Timestamp Tue Jul 14 08:55:46 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 30C P0 49W / 250W | 30555MiB / 32768MiB | 47% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 49W / 250W | 30495MiB / 32768MiB | 28% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 50W / 250W | 30499MiB / 32768MiB | 47% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 30C P0 49W / 250W | 30359MiB / 32768MiB | 28% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n", + "parsed": [ + { + "power_w": 49, + "power_cap_w": 250, + "mem_mib": 30555, + "mem_total_mib": 32768, + "util_pct": 47 + }, + { + "power_w": 49, + "power_cap_w": 250, + "mem_mib": 30495, + "mem_total_mib": 32768, + "util_pct": 28 + }, + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30499, + "mem_total_mib": 32768, + "util_pct": 47 + }, + { + "power_w": 49, + "power_cap_w": 250, + "mem_mib": 30359, + "mem_total_mib": 32768, + "util_pct": 28 + } + ] + }, + { + "ts": "2026-07-14T08:55:47", + "ok": true, + "raw": "Timestamp Tue Jul 14 08:55:47 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 30C P0 50W / 250W | 30555MiB / 32768MiB | 8% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 49W / 250W | 30495MiB / 32768MiB | 53% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 50W / 250W | 30499MiB / 32768MiB | 59% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 49W / 250W | 30359MiB / 32768MiB | 30% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n", + "parsed": [ + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30555, + "mem_total_mib": 32768, + "util_pct": 8 + }, + { + "power_w": 49, + "power_cap_w": 250, + "mem_mib": 30495, + "mem_total_mib": 32768, + "util_pct": 53 + }, + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30499, + "mem_total_mib": 32768, + "util_pct": 59 + }, + { + "power_w": 49, + "power_cap_w": 250, + "mem_mib": 30359, + "mem_total_mib": 32768, + "util_pct": 30 + } + ] + }, + { + "ts": "2026-07-14T08:55:48", + "ok": true, + "raw": "Timestamp Tue Jul 14 08:55:48 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 30C P0 50W / 250W | 30555MiB / 32768MiB | 25% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 50W / 250W | 30495MiB / 32768MiB | 35% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 50W / 250W | 30499MiB / 32768MiB | 24% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 49W / 250W | 30359MiB / 32768MiB | 27% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n", + "parsed": [ + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30555, + "mem_total_mib": 32768, + "util_pct": 25 + }, + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30495, + "mem_total_mib": 32768, + "util_pct": 35 + }, + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30499, + "mem_total_mib": 32768, + "util_pct": 24 + }, + { + "power_w": 49, + "power_cap_w": 250, + "mem_mib": 30359, + "mem_total_mib": 32768, + "util_pct": 27 + } + ] + }, + { + "ts": "2026-07-14T08:55:50", + "ok": true, + "raw": "Timestamp Tue Jul 14 08:55:50 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 30C P0 50W / 250W | 30555MiB / 32768MiB | 62% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 49W / 250W | 30495MiB / 32768MiB | 40% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 50W / 250W | 30499MiB / 32768MiB | 38% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 49W / 250W | 30359MiB / 32768MiB | 34% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n", + "parsed": [ + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30555, + "mem_total_mib": 32768, + "util_pct": 62 + }, + { + "power_w": 49, + "power_cap_w": 250, + "mem_mib": 30495, + "mem_total_mib": 32768, + "util_pct": 40 + }, + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30499, + "mem_total_mib": 32768, + "util_pct": 38 + }, + { + "power_w": 49, + "power_cap_w": 250, + "mem_mib": 30359, + "mem_total_mib": 32768, + "util_pct": 34 + } + ] + }, + { + "ts": "2026-07-14T08:55:51", + "ok": true, + "raw": "Timestamp Tue Jul 14 08:55:51 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 30C P0 49W / 250W | 30555MiB / 32768MiB | 12% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 49W / 250W | 30495MiB / 32768MiB | 28% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 50W / 250W | 30499MiB / 32768MiB | 26% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 49W / 250W | 30359MiB / 32768MiB | 14% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n", + "parsed": [ + { + "power_w": 49, + "power_cap_w": 250, + "mem_mib": 30555, + "mem_total_mib": 32768, + "util_pct": 12 + }, + { + "power_w": 49, + "power_cap_w": 250, + "mem_mib": 30495, + "mem_total_mib": 32768, + "util_pct": 28 + }, + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30499, + "mem_total_mib": 32768, + "util_pct": 26 + }, + { + "power_w": 49, + "power_cap_w": 250, + "mem_mib": 30359, + "mem_total_mib": 32768, + "util_pct": 14 + } + ] + }, + { + "ts": "2026-07-14T08:55:52", + "ok": true, + "raw": "Timestamp Tue Jul 14 08:55:53 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 30C P0 50W / 250W | 30555MiB / 32768MiB | 9% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 50W / 250W | 30495MiB / 32768MiB | 61% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 50W / 250W | 30499MiB / 32768MiB | 28% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 49W / 250W | 30359MiB / 32768MiB | 12% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n", + "parsed": [ + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30555, + "mem_total_mib": 32768, + "util_pct": 9 + }, + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30495, + "mem_total_mib": 32768, + "util_pct": 61 + }, + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30499, + "mem_total_mib": 32768, + "util_pct": 28 + }, + { + "power_w": 49, + "power_cap_w": 250, + "mem_mib": 30359, + "mem_total_mib": 32768, + "util_pct": 12 + } + ] + }, + { + "ts": "2026-07-14T08:55:54", + "ok": true, + "raw": "Timestamp Tue Jul 14 08:55:54 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 30C P0 50W / 250W | 30555MiB / 32768MiB | 8% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 50W / 250W | 30495MiB / 32768MiB | 36% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 50W / 250W | 30499MiB / 32768MiB | 15% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 49W / 250W | 30359MiB / 32768MiB | 44% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n", + "parsed": [ + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30555, + "mem_total_mib": 32768, + "util_pct": 8 + }, + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30495, + "mem_total_mib": 32768, + "util_pct": 36 + }, + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30499, + "mem_total_mib": 32768, + "util_pct": 15 + }, + { + "power_w": 49, + "power_cap_w": 250, + "mem_mib": 30359, + "mem_total_mib": 32768, + "util_pct": 44 + } + ] + }, + { + "ts": "2026-07-14T08:55:55", + "ok": true, + "raw": "Timestamp Tue Jul 14 08:55:55 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 30C P0 50W / 250W | 30555MiB / 32768MiB | 33% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 50W / 250W | 30495MiB / 32768MiB | 22% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 50W / 250W | 30499MiB / 32768MiB | 29% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 50W / 250W | 30359MiB / 32768MiB | 52% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n", + "parsed": [ + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30555, + "mem_total_mib": 32768, + "util_pct": 33 + }, + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30495, + "mem_total_mib": 32768, + "util_pct": 22 + }, + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30499, + "mem_total_mib": 32768, + "util_pct": 29 + }, + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30359, + "mem_total_mib": 32768, + "util_pct": 52 + } + ] + }, + { + "ts": "2026-07-14T08:55:56", + "ok": true, + "raw": "Timestamp Tue Jul 14 08:55:56 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 30C P0 50W / 250W | 30555MiB / 32768MiB | 29% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 49W / 250W | 30495MiB / 32768MiB | 32% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 50W / 250W | 30499MiB / 32768MiB | 29% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 49W / 250W | 30359MiB / 32768MiB | 16% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n", + "parsed": [ + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30555, + "mem_total_mib": 32768, + "util_pct": 29 + }, + { + "power_w": 49, + "power_cap_w": 250, + "mem_mib": 30495, + "mem_total_mib": 32768, + "util_pct": 32 + }, + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30499, + "mem_total_mib": 32768, + "util_pct": 29 + }, + { + "power_w": 49, + "power_cap_w": 250, + "mem_mib": 30359, + "mem_total_mib": 32768, + "util_pct": 16 + } + ] + }, + { + "ts": "2026-07-14T08:55:58", + "ok": true, + "raw": "Timestamp Tue Jul 14 08:55:58 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 30C P0 50W / 250W | 30555MiB / 32768MiB | 30% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 50W / 250W | 30495MiB / 32768MiB | 48% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 50W / 250W | 30499MiB / 32768MiB | 19% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 49W / 250W | 30359MiB / 32768MiB | 10% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n", + "parsed": [ + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30555, + "mem_total_mib": 32768, + "util_pct": 30 + }, + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30495, + "mem_total_mib": 32768, + "util_pct": 48 + }, + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30499, + "mem_total_mib": 32768, + "util_pct": 19 + }, + { + "power_w": 49, + "power_cap_w": 250, + "mem_mib": 30359, + "mem_total_mib": 32768, + "util_pct": 10 + } + ] + }, + { + "ts": "2026-07-14T08:55:59", + "ok": true, + "raw": "Timestamp Tue Jul 14 08:55:59 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 30C P0 50W / 250W | 30555MiB / 32768MiB | 28% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 50W / 250W | 30495MiB / 32768MiB | 25% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 50W / 250W | 30499MiB / 32768MiB | 26% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 49W / 250W | 30359MiB / 32768MiB | 29% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n", + "parsed": [ + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30555, + "mem_total_mib": 32768, + "util_pct": 28 + }, + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30495, + "mem_total_mib": 32768, + "util_pct": 25 + }, + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30499, + "mem_total_mib": 32768, + "util_pct": 26 + }, + { + "power_w": 49, + "power_cap_w": 250, + "mem_mib": 30359, + "mem_total_mib": 32768, + "util_pct": 29 + } + ] + }, + { + "ts": "2026-07-14T08:56:00", + "ok": true, + "raw": "Timestamp Tue Jul 14 08:56:01 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 30C P0 50W / 250W | 30555MiB / 32768MiB | 17% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 50W / 250W | 30495MiB / 32768MiB | 31% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 50W / 250W | 30499MiB / 32768MiB | 33% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 30C P0 50W / 250W | 30359MiB / 32768MiB | 28% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n", + "parsed": [ + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30555, + "mem_total_mib": 32768, + "util_pct": 17 + }, + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30495, + "mem_total_mib": 32768, + "util_pct": 31 + }, + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30499, + "mem_total_mib": 32768, + "util_pct": 33 + }, + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30359, + "mem_total_mib": 32768, + "util_pct": 28 + } + ] + }, + { + "ts": "2026-07-14T08:56:02", + "ok": true, + "raw": "Timestamp Tue Jul 14 08:56:02 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 30C P0 50W / 250W | 30555MiB / 32768MiB | 15% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 50W / 250W | 30495MiB / 32768MiB | 35% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 50W / 250W | 30499MiB / 32768MiB | 11% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 50W / 250W | 30359MiB / 32768MiB | 32% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n", + "parsed": [ + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30555, + "mem_total_mib": 32768, + "util_pct": 15 + }, + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30495, + "mem_total_mib": 32768, + "util_pct": 35 + }, + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30499, + "mem_total_mib": 32768, + "util_pct": 11 + }, + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30359, + "mem_total_mib": 32768, + "util_pct": 32 + } + ] + }, + { + "ts": "2026-07-14T08:56:03", + "ok": true, + "raw": "Timestamp Tue Jul 14 08:56:03 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 30C P0 50W / 250W | 30555MiB / 32768MiB | 16% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 50W / 250W | 30495MiB / 32768MiB | 32% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 50W / 250W | 30499MiB / 32768MiB | 18% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 50W / 250W | 30359MiB / 32768MiB | 31% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n", + "parsed": [ + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30555, + "mem_total_mib": 32768, + "util_pct": 16 + }, + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30495, + "mem_total_mib": 32768, + "util_pct": 32 + }, + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30499, + "mem_total_mib": 32768, + "util_pct": 18 + }, + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30359, + "mem_total_mib": 32768, + "util_pct": 31 + } + ] + }, + { + "ts": "2026-07-14T08:56:04", + "ok": true, + "raw": "Timestamp Tue Jul 14 08:56:04 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 30C P0 50W / 250W | 30555MiB / 32768MiB | 54% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 50W / 250W | 30495MiB / 32768MiB | 17% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 50W / 250W | 30499MiB / 32768MiB | 41% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 49W / 250W | 30359MiB / 32768MiB | 9% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n", + "parsed": [ + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30555, + "mem_total_mib": 32768, + "util_pct": 54 + }, + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30495, + "mem_total_mib": 32768, + "util_pct": 17 + }, + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30499, + "mem_total_mib": 32768, + "util_pct": 41 + }, + { + "power_w": 49, + "power_cap_w": 250, + "mem_mib": 30359, + "mem_total_mib": 32768, + "util_pct": 9 + } + ] + }, + { + "ts": "2026-07-14T08:56:06", + "ok": true, + "raw": "Timestamp Tue Jul 14 08:56:06 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 30C P0 49W / 250W | 30555MiB / 32768MiB | 23% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 49W / 250W | 30495MiB / 32768MiB | 31% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 50W / 250W | 30499MiB / 32768MiB | 30% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 49W / 250W | 30359MiB / 32768MiB | 13% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n", + "parsed": [ + { + "power_w": 49, + "power_cap_w": 250, + "mem_mib": 30555, + "mem_total_mib": 32768, + "util_pct": 23 + }, + { + "power_w": 49, + "power_cap_w": 250, + "mem_mib": 30495, + "mem_total_mib": 32768, + "util_pct": 31 + }, + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30499, + "mem_total_mib": 32768, + "util_pct": 30 + }, + { + "power_w": 49, + "power_cap_w": 250, + "mem_mib": 30359, + "mem_total_mib": 32768, + "util_pct": 13 + } + ] + }, + { + "ts": "2026-07-14T08:56:07", + "ok": true, + "raw": "Timestamp Tue Jul 14 08:56:07 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 30C P0 50W / 250W | 30555MiB / 32768MiB | 33% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 50W / 250W | 30495MiB / 32768MiB | 15% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 50W / 250W | 30499MiB / 32768MiB | 25% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 49W / 250W | 30359MiB / 32768MiB | 12% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n", + "parsed": [ + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30555, + "mem_total_mib": 32768, + "util_pct": 33 + }, + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30495, + "mem_total_mib": 32768, + "util_pct": 15 + }, + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30499, + "mem_total_mib": 32768, + "util_pct": 25 + }, + { + "power_w": 49, + "power_cap_w": 250, + "mem_mib": 30359, + "mem_total_mib": 32768, + "util_pct": 12 + } + ] + }, + { + "ts": "2026-07-14T08:56:09", + "ok": true, + "raw": "Timestamp Tue Jul 14 08:56:09 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 30C P0 49W / 250W | 30555MiB / 32768MiB | 28% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 50W / 250W | 30495MiB / 32768MiB | 12% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 50W / 250W | 30499MiB / 32768MiB | 9% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 49W / 250W | 30359MiB / 32768MiB | 26% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n", + "parsed": [ + { + "power_w": 49, + "power_cap_w": 250, + "mem_mib": 30555, + "mem_total_mib": 32768, + "util_pct": 28 + }, + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30495, + "mem_total_mib": 32768, + "util_pct": 12 + }, + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30499, + "mem_total_mib": 32768, + "util_pct": 9 + }, + { + "power_w": 49, + "power_cap_w": 250, + "mem_mib": 30359, + "mem_total_mib": 32768, + "util_pct": 26 + } + ] + }, + { + "ts": "2026-07-14T08:56:10", + "ok": true, + "raw": "Timestamp Tue Jul 14 08:56:10 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 30C P0 50W / 250W | 30555MiB / 32768MiB | 38% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 50W / 250W | 30495MiB / 32768MiB | 20% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 50W / 250W | 30499MiB / 32768MiB | 41% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 49W / 250W | 30359MiB / 32768MiB | 71% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n", + "parsed": [ + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30555, + "mem_total_mib": 32768, + "util_pct": 38 + }, + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30495, + "mem_total_mib": 32768, + "util_pct": 20 + }, + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30499, + "mem_total_mib": 32768, + "util_pct": 41 + }, + { + "power_w": 49, + "power_cap_w": 250, + "mem_mib": 30359, + "mem_total_mib": 32768, + "util_pct": 71 + } + ] + }, + { + "ts": "2026-07-14T08:56:11", + "ok": true, + "raw": "Timestamp Tue Jul 14 08:56:11 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 30C P0 50W / 250W | 30555MiB / 32768MiB | 17% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 50W / 250W | 30495MiB / 32768MiB | 23% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 50W / 250W | 30499MiB / 32768MiB | 39% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 49W / 250W | 30359MiB / 32768MiB | 35% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n", + "parsed": [ + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30555, + "mem_total_mib": 32768, + "util_pct": 17 + }, + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30495, + "mem_total_mib": 32768, + "util_pct": 23 + }, + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30499, + "mem_total_mib": 32768, + "util_pct": 39 + }, + { + "power_w": 49, + "power_cap_w": 250, + "mem_mib": 30359, + "mem_total_mib": 32768, + "util_pct": 35 + } + ] + }, + { + "ts": "2026-07-14T08:56:13", + "ok": true, + "raw": "Timestamp Tue Jul 14 08:56:13 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 30C P0 50W / 250W | 30555MiB / 32768MiB | 39% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 50W / 250W | 30495MiB / 32768MiB | 33% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 50W / 250W | 30499MiB / 32768MiB | 9% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 49W / 250W | 30359MiB / 32768MiB | 33% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n", + "parsed": [ + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30555, + "mem_total_mib": 32768, + "util_pct": 39 + }, + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30495, + "mem_total_mib": 32768, + "util_pct": 33 + }, + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30499, + "mem_total_mib": 32768, + "util_pct": 9 + }, + { + "power_w": 49, + "power_cap_w": 250, + "mem_mib": 30359, + "mem_total_mib": 32768, + "util_pct": 33 + } + ] + }, + { + "ts": "2026-07-14T08:56:14", + "ok": true, + "raw": "Timestamp Tue Jul 14 08:56:14 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 30C P0 50W / 250W | 30555MiB / 32768MiB | 20% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 50W / 250W | 30495MiB / 32768MiB | 34% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 50W / 250W | 30499MiB / 32768MiB | 18% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 50W / 250W | 30359MiB / 32768MiB | 28% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n", + "parsed": [ + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30555, + "mem_total_mib": 32768, + "util_pct": 20 + }, + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30495, + "mem_total_mib": 32768, + "util_pct": 34 + }, + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30499, + "mem_total_mib": 32768, + "util_pct": 18 + }, + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30359, + "mem_total_mib": 32768, + "util_pct": 28 + } + ] + }, + { + "ts": "2026-07-14T08:56:15", + "ok": true, + "raw": "Timestamp Tue Jul 14 08:56:15 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 30C P0 50W / 250W | 30555MiB / 32768MiB | 29% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 50W / 250W | 30495MiB / 32768MiB | 31% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 50W / 250W | 30499MiB / 32768MiB | 34% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 49W / 250W | 30359MiB / 32768MiB | 22% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n", + "parsed": [ + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30555, + "mem_total_mib": 32768, + "util_pct": 29 + }, + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30495, + "mem_total_mib": 32768, + "util_pct": 31 + }, + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30499, + "mem_total_mib": 32768, + "util_pct": 34 + }, + { + "power_w": 49, + "power_cap_w": 250, + "mem_mib": 30359, + "mem_total_mib": 32768, + "util_pct": 22 + } + ] + }, + { + "ts": "2026-07-14T08:56:16", + "ok": true, + "raw": "Timestamp Tue Jul 14 08:56:17 2026\n+-----------------------------------------------------------------------------+\n| IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 |\n|-------------------------------+----------------------+----------------------|\n| GPU Name | Bus-Id | Clock-SM Clock-Mem |\n| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n|===============================+======================+======================|\n| 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz |\n| 0% 30C P0 49W / 250W | 30555MiB / 32768MiB | 46% Default |\n+-------------------------------+----------------------+----------------------+\n| 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz |\n| 0% 32C P0 50W / 250W | 30495MiB / 32768MiB | 40% Default |\n+-------------------------------+----------------------+----------------------+\n| 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 50W / 250W | 30499MiB / 32768MiB | 31% Default |\n+-------------------------------+----------------------+----------------------+\n| 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz |\n| 0% 31C P0 49W / 250W | 30359MiB / 32768MiB | 21% Default |\n+-------------------------------+----------------------+----------------------+\n\n+-----------------------------------------------------------------------------+\n| Processes: GPU Memory |\n| GPU PID Process name Usage(MiB) |\n|=============================================================================|\n+-----------------------------------------------------------------------------+\n", + "parsed": [ + { + "power_w": 49, + "power_cap_w": 250, + "mem_mib": 30555, + "mem_total_mib": 32768, + "util_pct": 46 + }, + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30495, + "mem_total_mib": 32768, + "util_pct": 40 + }, + { + "power_w": 50, + "power_cap_w": 250, + "mem_mib": 30499, + "mem_total_mib": 32768, + "util_pct": 31 + }, + { + "power_w": 49, + "power_cap_w": 250, + "mem_mib": 30359, + "mem_total_mib": 32768, + "util_pct": 21 + } + ] + } +] \ No newline at end of file