EngineX-Hygon/sglang
5
0
Fork 0
Code Issues Pull Requests Actions 7 Projects Releases Wiki Activity

Fix the broken --disable-radix-cache (#451)

Browse Source
This commit is contained in:
Liangsheng Yin
2024-05-19 13:00:12 +08:00
committed by GitHub
parent 8210ec60f4
commit 5b647543c1
1 changed files with 7 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
python/sglang/srt/managers/router/radix_cache.py
View File

@@ -58,7 +58,7 @@ class RadixCache:
def insert(self, key, value=None): def insert(self, key, value=None):
if self.disable: if self.disable:
return len(key) return 0
if value is None: if value is None:
value = [x for x in key] value = [x for x in key]
@@ -76,6 +76,12 @@ class RadixCache:
indices = self.req_to_token_pool.req_to_token[req_pool_idx, : len(token_ids)] indices = self.req_to_token_pool.req_to_token[req_pool_idx, : len(token_ids)]
new_prefix_len = self.insert(token_ids, indices.clone()) new_prefix_len = self.insert(token_ids, indices.clone())
if self.disable:
if del_in_memory_pool:
self.token_to_kv_pool.dec_refs(indices)
else:
return torch.tensor([], dtype=torch.int64), self.root_node
# Radix Cache takes one ref in memory pool # Radix Cache takes one ref in memory pool
self.token_to_kv_pool.dec_refs(indices[last_uncached_pos:new_prefix_len]) self.token_to_kv_pool.dec_refs(indices[last_uncached_pos:new_prefix_len])