[Misc] Fix metrics, weight update lock, request logging (#2543)

This commit is contained in:
Lianmin Zheng
2024-12-22 06:25:57 -08:00
parent 7d672d277b
commit 8496701934
11 changed files with 411 additions and 314 deletions

View File

@@ -22,7 +22,7 @@ import warnings
from collections import deque
from concurrent import futures
from types import SimpleNamespace
from typing import List, Optional
from typing import Callable, Dict, List, Optional, Tuple
import psutil
import setproctitle
@@ -260,7 +260,7 @@ class Scheduler:
self.current_stream = torch.get_device_module(self.device).current_stream()
# Session info
self.sessions = {}
self.sessions: Dict[str, Session] = {}
# Init chunked prefill
self.chunked_prefill_size = server_args.chunked_prefill_size