Add support for topk metadata transferring for PD (#10616)

Signed-off-by: Shangming Cai <csmthu@gmail.com>
This commit is contained in:
Shangming Cai
2025-09-28 00:09:38 +08:00
committed by GitHub
parent 51f7c6bd3c
commit e23e280e16
6 changed files with 60 additions and 20 deletions

View File

@@ -607,6 +607,8 @@ class Req:
) = None
self.hidden_states: List[List[float]] = []
self.hidden_states_tensor = None # Note: use tensor instead of list to transfer hidden_states when PD + MTP
self.output_topk_p = None
self.output_topk_index = None
# Embedding (return values)
self.embedding = None