EngineX-Hygon/sglang
5
0
Fork 0
Code Issues Pull Requests Actions 7 Projects Releases Wiki Activity

[minor fix] llama4 hybrid memory (#7950)

Browse Source
This commit is contained in:
Ying Sheng
2025-07-11 23:11:36 -07:00
committed by GitHub
parent eb118d88c4
commit cee9f329c4
1 changed files with 6 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
python/sglang/srt/mem_cache/memory_pool.py
View File

@@ -520,8 +520,13 @@ class SWAKVPool(KVCache):
self.layers_mapping[global_layer_id] = (swa_layer_id, True) self.layers_mapping[global_layer_id] = (swa_layer_id, True)
self.full_to_swa_index_mapping: Optional[torch.Tensor] = None self.full_to_swa_index_mapping: Optional[torch.Tensor] = None
k_size, v_size = self.get_kv_size_bytes()
self.mem_usage = (k_size + v_size) / GB
def get_kv_size_bytes(self): def get_kv_size_bytes(self):
raise NotImplementedError k_size, v_size = self.full_kv_pool.get_kv_size_bytes()
k_size_swa, v_size_swa = self.swa_kv_pool.get_kv_size_bytes()
return k_size + k_size_swa, v_size + v_size_swa
def get_contiguous_buf_infos(self): def get_contiguous_buf_infos(self):
full_kv_data_ptrs, full_kv_data_lens, full_kv_item_lens = ( full_kv_data_ptrs, full_kv_data_lens, full_kv_item_lens = (