EngineX-Hygon/sglang
5
0
Fork 0
Code Issues Pull Requests Actions 7 Projects Releases Wiki Activity

Fix flush cache (#627)

Browse Source
This commit is contained in:
Lianmin Zheng
2024-07-15 19:56:55 -07:00
parent 56f5fc4ab5
commit 41d1f67704
5 changed files with 30 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
benchmark/latency_throughput/bench_one.py
View File

@@ -102,8 +102,8 @@ def run_one_batch_size(bs):
output_throughput = bs * max_new_tokens / latency
overall_throughput = bs * (input_len + output_len) / latency
print(f"latency: {latency:.2f} s")
print(f"decode throughput: {output_throughput:.2f} token/s")
print(f"overall throughput: {overall_throughput:.2f} token/s")
print(f"output throughput: {output_throughput:.2f} token/s")
print(f"(input + output) throughput: {overall_throughput:.2f} token/s")
with open("results.jsonl", "a") as fout:
res = {