[PD] Support KV transfer with mooncake (#4880)

Signed-off-by: Shangming Cai <caishangming@linux.alibaba.com>
Co-authored-by: Shangming Cai <caishangming@linux.alibaba.com>
Co-authored-by: Xuchun Shang <xuchun.shang@linux.alibaba.com>
Co-authored-by: shangmingc <csmthu@gmail.com>
This commit is contained in:
Teng Ma
2025-04-10 14:23:23 +08:00
committed by GitHub
parent f730362ee2
commit 4c31ae9f6d
8 changed files with 571 additions and 30 deletions

View File

@@ -836,6 +836,8 @@ class Scheduler(
custom_logit_processor=custom_logit_processor,
return_hidden_states=recv_req.return_hidden_states,
eos_token_ids=self.model_config.hf_eos_token_id,
bootstrap_host=recv_req.bootstrap_host,
bootstrap_room=recv_req.bootstrap_room,
)
req.tokenizer = self.tokenizer
req.queue_time_start = time.time()