Add sglang.bench_latency for offline benchmark (#564)

2024-06-25 03:38:04 -07:00
parent 2187f36237
commit eb1ae6ae0c
9 changed files with 358 additions and 761 deletions
--- a/test/srt/test_httpserver_classify.py
+++ b/test/srt/test_httpserver_classify.py
@@ -1,5 +1,7 @@
 """
 Usage:
+python3 -m sglang.launch_server --model-path /model/llama-classification
+
 python3 test_httpserver_classify.py
 """