perf: reduce ttft and itl with stream_interval 1 (#658)

2024-07-20 02:14:22 +10:00
parent e1792cca24
commit ac971ff633
1 changed files with 1 additions and 1 deletions
--- a/python/sglang/srt/server_args.py
+++ b/python/sglang/srt/server_args.py
@@ -33,7 +33,7 @@ class ServerArgs:

    # Other runtime options
    tp_size: int = 1
-    stream_interval: int = 8
+    stream_interval: int = 1
    random_seed: Optional[int] = None

    # Logging