EngineX-Hygon/sglang
5
0
Fork 0
Code Issues Pull Requests Actions 7 Projects Releases Wiki Activity

Make decode log interval configurable (#1847)

Browse Source
This commit is contained in:
Byron Hsu
2024-10-30 19:59:20 -07:00
committed by GitHub
parent 2d4ce1b792
commit a7a0a6886b
2 changed files with 8 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
python/sglang/srt/managers/scheduler.py
View File

@@ -919,7 +919,7 @@ class Scheduler:
self.token_to_kv_pool.free_group_end() self.token_to_kv_pool.free_group_end()
self.forward_ct_decode = (self.forward_ct_decode + 1) % (1 << 30) self.forward_ct_decode = (self.forward_ct_decode + 1) % (1 << 30)
if self.tp_rank == 0 and self.forward_ct_decode % 40 == 0: if self.tp_rank == 0 and self.forward_ct_decode % self.server_args.decode_log_interval == 0:
self.print_decode_stats() self.print_decode_stats()
def add_logprob_return_values( def add_logprob_return_values(

7
python/sglang/srt/server_args.py
View File

@@ -63,6 +63,7 @@ class ServerArgs:
stream_interval: int = 1 stream_interval: int = 1
random_seed: Optional[int] = None random_seed: Optional[int] = None
constrained_json_whitespace_pattern: Optional[str] = None constrained_json_whitespace_pattern: Optional[str] = None
decode_log_interval: int = 40
# Logging # Logging
log_level: str = "info" log_level: str = "info"
@@ -436,6 +437,12 @@ class ServerArgs:
default=ServerArgs.watchdog_timeout, default=ServerArgs.watchdog_timeout,
help="Set watchdog timeout in seconds. If a forward batch takes longer than this, the server will crash to prevent hanging.", help="Set watchdog timeout in seconds. If a forward batch takes longer than this, the server will crash to prevent hanging.",
) )
parser.add_argument(
"--decode-log-interval",
type=int,
default=ServerArgs.decode_log_interval,
help="The log interval of decode batch"
)
# Data parallelism # Data parallelism
parser.add_argument( parser.add_argument(