114 lines
2.6 KiB
JSON
114 lines
2.6 KiB
JSON
{
|
|
"gpu": "NVIDIA H100 80GB HBM3",
|
|
"torch": "2.8.0+cu129",
|
|
"results": [
|
|
{
|
|
"mb": 8,
|
|
"ctx": 1024,
|
|
"compile": "default",
|
|
"tok_s": 112855.93922615414,
|
|
"peak_gb": 18.579795968,
|
|
"tflops": 339.30627471695726,
|
|
"warmup_s": 26.683147192001343
|
|
},
|
|
{
|
|
"mb": 12,
|
|
"ctx": 1024,
|
|
"compile": "default",
|
|
"tok_s": 118795.12738506112,
|
|
"peak_gb": 24.264536064,
|
|
"tflops": 357.1627014399097,
|
|
"warmup_s": 47.67642426490784
|
|
},
|
|
{
|
|
"mb": 16,
|
|
"ctx": 1024,
|
|
"compile": "default",
|
|
"tok_s": 122837.53238217832,
|
|
"peak_gb": 29.92830464,
|
|
"tflops": 369.31636734242323,
|
|
"warmup_s": 0.4134063720703125
|
|
},
|
|
{
|
|
"mb": 24,
|
|
"ctx": 1024,
|
|
"compile": "default",
|
|
"tok_s": 125688.80943331693,
|
|
"peak_gb": 41.244962816,
|
|
"tflops": 377.8888554280444,
|
|
"warmup_s": 0.5790824890136719
|
|
},
|
|
{
|
|
"mb": 32,
|
|
"ctx": 1024,
|
|
"compile": "default",
|
|
"tok_s": 127464.49062053277,
|
|
"peak_gb": 52.637249536,
|
|
"tflops": 383.22751791094504,
|
|
"warmup_s": 0.75799560546875
|
|
},
|
|
{
|
|
"mb": 48,
|
|
"ctx": 1024,
|
|
"compile": "default",
|
|
"tok_s": 130583.87669116902,
|
|
"peak_gb": 75.252740096,
|
|
"tflops": 392.606087388893,
|
|
"warmup_s": 1.1335768699645996
|
|
},
|
|
{
|
|
"mb": 8,
|
|
"ctx": 2048,
|
|
"compile": "default",
|
|
"tok_s": 111115.13829731975,
|
|
"peak_gb": 29.924896768,
|
|
"tflops": 334.0724812432884,
|
|
"warmup_s": 57.35422158241272
|
|
},
|
|
{
|
|
"mb": 12,
|
|
"ctx": 2048,
|
|
"compile": "default",
|
|
"tok_s": 113402.57779289872,
|
|
"peak_gb": 41.24522496,
|
|
"tflops": 340.9497672701231,
|
|
"warmup_s": 0.641932487487793
|
|
},
|
|
{
|
|
"mb": 16,
|
|
"ctx": 2048,
|
|
"compile": "default",
|
|
"tok_s": 115294.6307953683,
|
|
"peak_gb": 52.63751168,
|
|
"tflops": 346.63830666146606,
|
|
"warmup_s": 0.8390512466430664
|
|
},
|
|
{
|
|
"mb": 24,
|
|
"ctx": 2048,
|
|
"compile": "default",
|
|
"tok_s": 117730.31919758776,
|
|
"peak_gb": 75.25300224,
|
|
"tflops": 353.961309454188,
|
|
"warmup_s": 1.2464354038238525
|
|
},
|
|
{
|
|
"mb": 48,
|
|
"ctx": 1024,
|
|
"compile": "max-autotune",
|
|
"tok_s": 130146.54947516992,
|
|
"peak_gb": 78.26123776,
|
|
"tflops": 391.29124415148357,
|
|
"warmup_s": 161.93337106704712
|
|
}
|
|
],
|
|
"best": {
|
|
"mb": 48,
|
|
"ctx": 1024,
|
|
"compile": "default",
|
|
"tok_s": 130583.87669116902,
|
|
"peak_gb": 75.252740096,
|
|
"tflops": 392.606087388893,
|
|
"warmup_s": 1.1335768699645996
|
|
}
|
|
} |