[exp] label=batched_tokens_8192_20260715 [exp] max_num_batched_tokens=8192 [exp] out_dir=/root/work/logs/batched_tokens_8192_20260715 Wed Jul 15 02:34:25 UTC 2026 [exp] stopping existing api_server [exp] gpu before start Timestamp Wed Jul 15 02:34:25 2026 +-----------------------------------------------------------------------------+ | IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 | |-------------------------------+----------------------+----------------------| | GPU Name | Bus-Id | Clock-SM Clock-Mem | | Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. | |===============================+======================+======================| | 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz | | 0% 29C P0 47W / 250W | 257MiB / 32768MiB | 0% Default | +-------------------------------+----------------------+----------------------+ | 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz | | 0% 32C P0 47W / 250W | 257MiB / 32768MiB | 0% Default | +-------------------------------+----------------------+----------------------+ | 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz | | 0% 31C P0 47W / 250W | 257MiB / 32768MiB | 0% Default | +-------------------------------+----------------------+----------------------+ | 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz | | 0% 30C P0 47W / 250W | 257MiB / 32768MiB | 0% Default | +-------------------------------+----------------------+----------------------+ +-----------------------------------------------------------------------------+ | Processes: GPU Memory | | GPU PID Process name Usage(MiB) | |=============================================================================| | No running processes found | +-----------------------------------------------------------------------------+ [exp] starting server [exp] server_pid=32482 [exp] waiting for health [exp] health ok after 17 checks [exp] running benchmark done 1/8 ok=True done 2/8 ok=True done 3/8 ok=True done 4/8 ok=True done 5/8 ok=True done 6/8 ok=True done 7/8 ok=True done 8/8 ok=True { "created_at": "2026-07-15T02:41:32", "url": "http://127.0.0.1:1111", "model": "llm", "dataset": "/root/work/logs/synthetic_cumulative_8_s3_cap40k.jsonl", "concurrency": 1, "max_requests": 8, "max_tokens": 256, "wall_sec": 345.89166797697544, "success_rate": 1.0, "ttft_p90_sec": 13.995924570597705, "output_tps_p10_per_request": 5.840174878063855, "aggregate_output_tps": 5.513286894574578, "aggregate_input_tps_uncached": 168.40821966222535, "aggregate_cache_tps": 483.5271140764599, "cache_hit_rate": 0.7416795639891973, "weighted_token_throughput": 834.7509573986608, "totals": { "requests": 8, "success": 8, "prompt_tokens": 225499, "input_tokens_uncached": 58251, "cached_tokens": 167248, "completion_tokens": 1907, "reasoning_tokens": 1758 }, "targets": { "output_tps_p10_per_request_gte_20": false, "ttft_p90_lte_5": false, "cache_hit_rate_gte_50pct": true, "success_rate_gte_99pct": true, "weighted_token_throughput_gte_8000": false } } RESULT_FILE /root/work/logs/batched_tokens_8192_20260715/eager_100k_b8192_c1_r8_t256_cap40k.json [exp] benchmark rc=0 { "created_at": "2026-07-15T02:41:32", "url": "http://127.0.0.1:1111", "model": "llm", "dataset": "/root/work/logs/synthetic_cumulative_8_s3_cap40k.jsonl", "concurrency": 1, "max_requests": 8, "max_tokens": 256, "wall_sec": 345.89166797697544, "success_rate": 1.0, "ttft_p90_sec": 13.995924570597705, "output_tps_p10_per_request": 5.840174878063855, "aggregate_output_tps": 5.513286894574578, "aggregate_input_tps_uncached": 168.40821966222535, "aggregate_cache_tps": 483.5271140764599, "cache_hit_rate": 0.7416795639891973, "weighted_token_throughput": 834.7509573986608, "totals": { "requests": 8, "success": 8, "prompt_tokens": 225499, "input_tokens_uncached": 58251, "cached_tokens": 167248, "completion_tokens": 1907, "reasoning_tokens": 1758 }, "targets": { "output_tps_p10_per_request_gte_20": false, "ttft_p90_lte_5": false, "cache_hit_rate_gte_50pct": true, "success_rate_gte_99pct": true, "weighted_token_throughput_gte_8000": false }, "results": [ { "ok": true, "elapsed_sec": 36.67137165553868, "ttft_sec": 7.283070221543312, "prompt_tokens": 7498, "cached_tokens": 0, "completion_tokens": 250, "reasoning_tokens": 147, "output_tps": 8.506786299354092, "error": null }, { "ok": true, "elapsed_sec": 42.85823520086706, "ttft_sec": 23.074742704629898, "prompt_tokens": 34177, "cached_tokens": 7488, "completion_tokens": 121, "reasoning_tokens": 75, "output_tps": 6.116210270912193, "error": null }, { "ok": true, "elapsed_sec": 46.396307623013854, "ttft_sec": 3.7564049027860165, "prompt_tokens": 35384, "cached_tokens": 34160, "completion_tokens": 256, "reasoning_tokens": 256, "output_tps": 6.003766042330974, "error": null }, { "ok": true, "elapsed_sec": 40.55220231600106, "ttft_sec": 10.105002513155341, "prompt_tokens": 19274, "cached_tokens": 7472, "completion_tokens": 256, "reasoning_tokens": 256, "output_tps": 8.407998162644606, "error": null }, { "ok": true, "elapsed_sec": 47.4568429403007, "ttft_sec": 4.107830764725804, "prompt_tokens": 37099, "cached_tokens": 35376, "completion_tokens": 256, "reasoning_tokens": 256, "output_tps": 5.90555556290724, "error": null }, { "ok": true, "elapsed_sec": 36.76491709984839, "ttft_sec": 6.331659993156791, "prompt_tokens": 14661, "cached_tokens": 7472, "completion_tokens": 256, "reasoning_tokens": 256, "output_tps": 8.411850203957014, "error": null }, { "ok": true, "elapsed_sec": 47.44748286344111, "ttft_sec": 3.74263353087008, "prompt_tokens": 38199, "cached_tokens": 37088, "completion_tokens": 256, "reasoning_tokens": 256, "output_tps": 5.85747357351524, "error": null }, { "ok": true, "elapsed_sec": 47.73618622124195, "ttft_sec": 3.59681879542768, "prompt_tokens": 39207, "cached_tokens": 38192, "completion_tokens": 256, "reasoning_tokens": 256, "output_tps": 5.799811255343957, "error": null } ] }[exp] gpu after benchmark Timestamp Wed Jul 15 02:41:32 2026 +-----------------------------------------------------------------------------+ | IX-ML: 3.2.3 Driver Version: 3.2.1 CUDA Version: 10.2 | |-------------------------------+----------------------+----------------------| | GPU Name | Bus-Id | Clock-SM Clock-Mem | | Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. | |===============================+======================+======================| | 0 Iluvatar BI-V100 | 00000000:4B:00.0 | 1500MHz 1200MHz | | 0% 33C P0 64W / 250W | 29967MiB / 32768MiB | 0% Default | +-------------------------------+----------------------+----------------------+ | 1 Iluvatar BI-V100 | 00000000:4C:00.0 | 1500MHz 1200MHz | | 0% 36C P0 63W / 250W | 29906MiB / 32768MiB | 0% Default | +-------------------------------+----------------------+----------------------+ | 2 Iluvatar BI-V100 | 00000000:4D:00.0 | 1500MHz 1200MHz | | 0% 36C P0 62W / 250W | 29910MiB / 32768MiB | 0% Default | +-------------------------------+----------------------+----------------------+ | 3 Iluvatar BI-V100 | 00000000:4E:00.0 | 1500MHz 1200MHz | | 0% 34C P0 61W / 250W | 29770MiB / 32768MiB | 0% Default | +-------------------------------+----------------------+----------------------+ +-----------------------------------------------------------------------------+ | Processes: GPU Memory | | GPU PID Process name Usage(MiB) | |=============================================================================| +-----------------------------------------------------------------------------+