EngineX-Hygon/sglang
5
0
Fork 0
Code Issues Pull Requests Actions 7 Projects Releases Wiki Activity

Improve the computation for time_per_output_token Prometheus metrics (#2674)

Browse Source
This commit is contained in:
Lianmin Zheng
2024-12-30 21:40:14 -08:00
committed by GitHub
parent f707470019
commit 339c69a243
1 changed files with 3 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
python/sglang/srt/managers/tokenizer_manager.py
View File

@@ -699,6 +699,7 @@ class TokenizerManager:
) )
else: else:
if completion_tokens >= 2: if completion_tokens >= 2:
# Compute time_per_output_token for the streaming case
self.metrics_collector.observe_time_per_output_token( self.metrics_collector.observe_time_per_output_token(
(time.time() - state.first_token_time) (time.time() - state.first_token_time)
/ (completion_tokens - 1) / (completion_tokens - 1)
@@ -714,7 +715,8 @@ class TokenizerManager:
self.metrics_collector.observe_e2e_request_latency( self.metrics_collector.observe_e2e_request_latency(
time.time() - state.created_time time.time() - state.created_time
) )
if completion_tokens >= 1: # Compute time_per_output_token for the non-streaming case
if not state.obj.stream and completion_tokens >= 1:
self.metrics_collector.observe_time_per_output_token( self.metrics_collector.observe_time_per_output_token(
(time.time() - state.created_time) (time.time() - state.created_time)
/ completion_tokens / completion_tokens