EngineX-Hygon/sglang
5
0
Fork 0
Code Issues Pull Requests Actions 7 Projects Releases Wiki Activity

[Fix] KV-cache eviction mismatch across PP ranks in DeepSeek V3/R1 (#10214)

Browse Source
This commit is contained in:
Rain H
2025-09-09 17:07:13 +08:00
committed by GitHub
parent 9ab72f9895
commit 71fc7b7fad
1 changed files with 10 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
python/sglang/srt/model_executor/model_runner.py
View File

@@ -1260,6 +1260,16 @@ class ModelRunner:
// self.server_args.page_size // self.server_args.page_size
* self.server_args.page_size * self.server_args.page_size
) )
# different pp rank may have different num of layers, so we need to reduce the max_total_num_tokens
if self.pp_size > 1:
tensor = torch.tensor(self.max_total_num_tokens, dtype=torch.int64)
torch.distributed.all_reduce(
tensor,
op=torch.distributed.ReduceOp.MIN,
group=get_world_group().cpu_group,
)
self.max_total_num_tokens = tensor.item()
# create token size for hybrid cache # create token size for hybrid cache
if self.is_hybrid: if self.is_hybrid:
self.set_num_token_hybrid() self.set_num_token_hybrid()