Upstreaming hicache bug fixes (#7267)

This commit is contained in:
Zhiqiang Xie
2025-06-17 17:44:57 -07:00
committed by GitHub
parent c26d7349d3
commit e56685ac1b
7 changed files with 76 additions and 24 deletions

View File

@@ -307,7 +307,9 @@ class HiRadixCache(RadixCache):
return last_node, prefix_indices
def ready_to_load_cache(self):
producer_index = self.cache_controller.layer_done_counter.next_producer()
self.load_cache_event.set()
return producer_index
def match_prefix(self, key: List[int], include_evicted=False, **kwargs):
empty_value = torch.empty((0,), dtype=torch.int64, device=self.device)
@@ -372,6 +374,7 @@ class HiRadixCache(RadixCache):
new_node.lock_ref = child.lock_ref
new_node.key = child.key[:split_len]
new_node.loading = child.loading
new_node.hit_count = child.hit_count
# split value and host value if exists
if child.evicted: