17 lines
757 B
Diff
17 lines
757 B
Diff
--- a/vllm/v1/attention/backends/utils.py
|
|
+++ b/vllm/v1/attention/backends/utils.py
|
|
@@ -346,7 +346,13 @@
|
|
return CommonAttentionMetadata(
|
|
query_start_loc_cpu=query_start_loc_cpu,
|
|
query_start_loc=query_start_loc_cpu.to(device=device, non_blocking=True),
|
|
+ key_start_loc=torch.from_numpy(
|
|
+ np.concatenate(
|
|
+ (np.zeros(1, dtype=np.int32), np.cumsum(seqlens_k_local, dtype=np.int32))
|
|
+ )
|
|
+ ).to(device=device, non_blocking=True),
|
|
seq_lens=seq_lens_cpu.to(device=device, non_blocking=True),
|
|
+ seq_lens_np=seqlens_k_local,
|
|
num_reqs=len(seq_lens_cpu),
|
|
num_actual_tokens=common_attn_metadata.num_actual_tokens,
|
|
max_query_len=seqlens_q_local.max(),
|