EngineX-Hygon/sglang
5
0
Fork 0
Code Issues Pull Requests Actions 7 Projects Releases Wiki Activity

[HiCache]bug fix: fixed blank item in host_mem_release_queue (#11005)

Browse Source 
Co-authored-by: Zuoyuan Zhang <zuoyuanzhang@ZuoyuandeMacBook-Pro.local>
Co-authored-by: Zhiqiang Xie <xiezhq@stanford.edu>
This commit is contained in:
zhangzuo21
2025-10-01 21:42:37 +08:00
committed by GitHub
parent ac1f2928ae
commit 2e130b7618
1 changed files with 5 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
python/sglang/srt/managers/cache_controller.py
View File

@@ -575,9 +575,11 @@ class HiCacheController:
return operation.completed_tokens, operation.hash_value return operation.completed_tokens, operation.hash_value
def append_host_mem_release(self, host_indices: torch.Tensor): def append_host_mem_release(self, host_indices: torch.Tensor):
chunks = host_indices.split(self.mem_pool_host.page_size) if host_indices.numel() == 0:
for chunk in chunks: return
self.host_mem_release_queue.put(chunk) pages = host_indices.split(self.mem_pool_host.page_size)
for page in pages:
self.host_mem_release_queue.put(page)
def _page_get_zero_copy(self, operation, hash_values, host_indices): def _page_get_zero_copy(self, operation, hash_values, host_indices):
results = self.storage_backend.batch_get_v1(hash_values, host_indices) results = self.storage_backend.batch_get_v1(hash_values, host_indices)