Add support for topk metadata transferring for PD (#10616)
Signed-off-by: Shangming Cai <csmthu@gmail.com>
This commit is contained in:
@@ -607,6 +607,8 @@ class Req:
|
||||
) = None
|
||||
self.hidden_states: List[List[float]] = []
|
||||
self.hidden_states_tensor = None # Note: use tensor instead of list to transfer hidden_states when PD + MTP
|
||||
self.output_topk_p = None
|
||||
self.output_topk_index = None
|
||||
|
||||
# Embedding (return values)
|
||||
self.embedding = None
|
||||
|
||||
Reference in New Issue
Block a user