Fix bench latency benchmark (#1225)

2024-08-28 00:37:32 -07:00
parent c8a9e79186
commit 1ece2cda3d
2 changed files with 9 additions and 6 deletions
--- a/.github/workflows/e2e-test.yml
+++ b/.github/workflows/e2e-test.yml
@@ -38,6 +38,11 @@ jobs:
          cd test/srt
          python3 -m unittest test_serving_throughput.TestServingThroughput.test_default

+      - name: Benchmark Serving Latency
+        timeout-minutes: 10
+        run: |
+          python3 -m sglang.bench_latency --model meta-llama/Meta-Llama-3.1-8B-Instruct --batch-size 1 --input 128 --output 8
+
      - name: Benchmark Serving Throughput (w/o RadixAttention)
        timeout-minutes: 10
        run: |