[Metrics] Add KV events publishing (#6098)

This commit is contained in:
Trevor Morris
2025-05-19 14:19:54 -07:00
committed by GitHub
parent 299fd22f9e
commit 7adf245ba2
7 changed files with 686 additions and 1 deletions

View File

@@ -103,6 +103,7 @@ class ServerArgs:
collect_tokens_histogram: bool = False
decode_log_interval: int = 40
enable_request_time_stats_logging: bool = False
kv_events_config: Optional[str] = None
# API related
api_key: Optional[str] = None
@@ -814,6 +815,12 @@ class ServerArgs:
default=ServerArgs.collect_tokens_histogram,
help="Collect prompt/generation tokens histogram.",
)
parser.add_argument(
"--kv-events-config",
type=str,
default=None,
help="Config in json format for NVIDIA dynamo KV event publishing. Publishing will be enabled if this flag is used.",
)
parser.add_argument(
"--decode-log-interval",
type=int,