EngineX-Hygon/sglang
5
0
Fork 0
Code Issues Pull Requests Actions 7 Projects Releases Wiki Activity

[HICache] introduce evict policy (#10190)

Browse Source 
Signed-off-by: Xuchun Shang <xuchun.shang@linux.alibaba.com>
Co-authored-by: Teng Ma <sima.mt@alibaba-inc.com>
This commit is contained in:
Xuchun Shang
2025-09-18 11:10:20 +08:00
committed by GitHub
parent c32fb7a24d
commit 1ccd59c715
6 changed files with 78 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
python/sglang/srt/managers/scheduler.py
View File

@@ -667,6 +667,7 @@ class Scheduler(
else self.tp_cpu_group
),
page_size=self.page_size,
eviction_policy=server_args.radix_eviction_policy,
hicache_ratio=server_args.hicache_ratio,
hicache_size=server_args.hicache_size,
hicache_write_policy=server_args.hicache_write_policy,
@@ -719,6 +720,7 @@ class Scheduler(
tp_size=self.tp_size,
rank=self.tp_rank,
tp_group=self.tp_group,
eviction_policy=server_args.radix_eviction_policy,
)
else:
self.tree_cache = RadixCache(
@@ -727,6 +729,7 @@ class Scheduler(
page_size=self.page_size,
disable=server_args.disable_radix_cache,
enable_kv_cache_events=self.enable_kv_cache_events,
eviction_policy=server_args.radix_eviction_policy,
)
self.decode_mem_cache_buf_multiplier = (