Simplify mem state (#623)

2024-07-15 02:01:09 -07:00
parent bae9541e4c
commit 5ac8b80677
7 changed files with 61 additions and 66 deletions
--- a/benchmark/latency_throughput/bench_serving.py
+++ b/benchmark/latency_throughput/bench_serving.py
@@ -297,7 +297,8 @@ def main(args: argparse.Namespace):
    benchmark_time = benchmark_end_time - benchmark_start_time

    # Compute the statistics.
-    avg_latency = np.mean([latency for _, _, latency in REQUEST_LATENCY])
+    latencies = [latency for _, _, latency in REQUEST_LATENCY]
+    avg_latency = np.mean(latencies)
    avg_per_token_latency = np.mean(
        [
            latency / (prompt_len + output_len)