EngineX-Hygon/sglang
5
0
Fork 0
Code Issues Pull Requests Actions 7 Projects Releases Wiki Activity

misc: fix the req_to_token member change (#967)

Browse Source
This commit is contained in:
Liangsheng Yin
2024-08-07 01:52:10 -07:00
committed by GitHub
parent 7fa54a1ab3
commit a01ddd9605
2 changed files with 2 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
python/sglang/srt/managers/tp_worker.py
View File

@@ -289,10 +289,10 @@ class ModelTpServer:
"KV cache pool leak detected!" "KV cache pool leak detected!"
) )
if self.req_to_token_pool.can_use_mem_size != self.req_to_token_pool.size: if len(self.req_to_token_pool.free_slots) != self.req_to_token_pool.size:
warnings.warn( warnings.warn(
"Warning: " "Warning: "
f"available req slots={self.req_to_token_pool.can_use_mem_size}, " f"available req slots={len(self.req_to_token_pool.free_slots)}, "
f"total slots={self.req_to_token_pool.size}\n" f"total slots={self.req_to_token_pool.size}\n"
"Memory pool leak detected!" "Memory pool leak detected!"
) )

1
python/sglang/srt/mem_cache/memory_pool.py
View File

@@ -32,7 +32,6 @@ class ReqToTokenPool:
self.req_to_token = torch.empty( self.req_to_token = torch.empty(
(size, max_context_len), dtype=torch.int32, device="cuda" (size, max_context_len), dtype=torch.int32, device="cuda"
) )
self.can_use_mem_size = size
def alloc(self, need_size: int) -> List[int]: def alloc(self, need_size: int) -> List[int]:
if need_size > len(self.free_slots): if need_size > len(self.free_slots):