[PD] Add doc and simplify sender.send (#6019)

This commit is contained in:
Byron Hsu
2025-05-21 21:22:21 -07:00
committed by GitHub
parent 4d643f6c7a
commit 7513558074
6 changed files with 63 additions and 25 deletions

View File

@@ -384,11 +384,10 @@ class SchedulerDisaggregationPrefillMixin:
if end_idx is not None
else min(len(req.fill_ids), len(req.origin_input_ids))
)
last_chunk = token_id is not None
if (not last_chunk) and (
end_idx % page_size != 0
): # todo: remove the second condition
if not last_chunk:
# if not the last chunk and the last page is partial, delay the last partial page to the next send
end_idx = end_idx - end_idx % page_size
@@ -405,16 +404,10 @@ class SchedulerDisaggregationPrefillMixin:
req.metadata_buffer_index, token_id
)
page_indices = kv_to_page_indices(kv_indices, page_size)
page_start_idx = start_idx // page_size
page_end_idx = page_start_idx + len(page_indices)
if len(page_indices) == 0:
logger.info(
f"Skip sending kv chunk for request {req.rid=} {req.bootstrap_room=} because page_indices is empty"
)
return
req.disagg_kv_sender.send(
page_indices, slice(page_start_idx, page_end_idx), last_chunk
)
req.disagg_kv_sender.send(page_indices)