From ac971ff633de330de3ded7f7475caaf7cd5bbdcd Mon Sep 17 00:00:00 2001 From: zhyncs Date: Sat, 20 Jul 2024 02:14:22 +1000 Subject: [PATCH] perf: reduce ttft and itl with stream_interval 1 (#658) --- python/sglang/srt/server_args.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/sglang/srt/server_args.py b/python/sglang/srt/server_args.py index 7c0317fc0..50fe7cd17 100644 --- a/python/sglang/srt/server_args.py +++ b/python/sglang/srt/server_args.py @@ -33,7 +33,7 @@ class ServerArgs: # Other runtime options tp_size: int = 1 - stream_interval: int = 8 + stream_interval: int = 1 random_seed: Optional[int] = None # Logging