Fix benchmark latency (#1007)

This commit is contained in:
Mingyi
2024-08-09 13:18:58 -07:00
committed by GitHub
parent 9cf0a5bada
commit 61728884d7

View File

@@ -332,6 +332,7 @@ def latency_test(
)
# Warm up
rank_print("Warmup ...")
latency_test_run_once(
bench_args.run_name,
model_runner,
@@ -341,6 +342,7 @@ def latency_test(
bench_args.input_len[0],
4, # shorter decoding to speed up the warmup
)
rank_print("Benchmark ...")
# Run the sweep
result_list = []