EngineX-Ascend/enginex-ascend-910-vllm
11
0
Fork 1
Code Issues Pull Requests Actions Projects Releases Wiki Activity

init v0.23.0

Browse Source 
Signed-off-by: Sun Ruoxi <sunruoxi@4paradigm.com>
This commit is contained in:
Sun Ruoxi
2026-08-27 15:11:51 +08:00
parent b582a8e7d1
commit 7f8a1b1f7a
2849 changed files with 712887 additions and 22001 deletions
Download Patch File Download Diff File Expand all files Collapse all files

51
vllm_ascend/attention/abstract.py Normal file
View File

@@ -0,0 +1,51 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from abc import abstractmethod
from typing import Generic, TypeVar
import torch
from vllm.v1.attention.backend import AttentionImpl, AttentionLayer
class AttentionMetadata:
pass
T = TypeVar("T", bound=AttentionMetadata)
class DSAAttentionImpl(AttentionImpl[T], Generic[T]):
@abstractmethod
def __init__(
self,
dim: int,
n_heads: int,
scale: float,
n_local_heads: int,
q_lora_rank: int,
o_lora_rank: int,
head_dim: int,
rope_head_dim: int | None,
nope_head_dim: int,
n_groups: int,
n_local_groups: int,
window_size: int,
compress_ratio: int,
) -> None:
raise NotImplementedError
@abstractmethod
def forward(
self,
layer: AttentionLayer,
hidden_states_or_cq: torch.Tensor,
kv_c_normed: torch.Tensor,
k_pe: torch.Tensor,
kv_cache: torch.Tensor,
attn_metadata: T,
output: torch.Tensor | None = None,
output_scale: torch.Tensor | None = None,
output_block_scale: torch.Tensor | None = None,
) -> torch.Tensor:
raise NotImplementedError

131
vllm_ascend/attention/attention_mask.py
View File

@@ -13,96 +13,73 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import torch
from vllm.distributed import get_pcp_group
from vllm_ascend.platform import ModelConfig
from vllm_ascend.utils import singleton
def _generate_attn_mask(max_seq_len, dtype):
# Construct lower triangle matrix.
mask_flag = torch.tril(
torch.ones((max_seq_len, max_seq_len),
dtype=torch.bool)).view(max_seq_len, max_seq_len)
mask_flag = torch.ones((max_seq_len, max_seq_len), dtype=torch.bool).tril_()
# Create upper triangle matrix used to mark mask positions.
mask_flag = ~mask_flag
# Currently for fp16 dtype, the mask value should be set to -inf.
# TODO: Eliminate this part in the future.
if dtype == torch.float16:
mask_value = torch.finfo(torch.float32).min
else:
mask_value = 1
attn_mask = torch.masked_fill(torch.zeros(size=(max_seq_len, max_seq_len)),
mask_flag, mask_value).to(dtype)
mask_value = float("-inf") if dtype == torch.float16 else 1
attn_mask = torch.zeros(size=(max_seq_len, max_seq_len), dtype=dtype).masked_fill_(mask_flag, mask_value)
return attn_mask
@singleton
class AttentionMaskBuilder:
def __init__(
self,
max_seq_len: int,
dtype: torch.dtype,
device: torch.device = None,
):
# NOTE: The device argument specifies the target NPU
# to be used for the newly added FIA operator.
# Only pass this parameter when using the new FIA operator.
attn_mask = _generate_attn_mask(max_seq_len, dtype)
self._seq_len_cached = attn_mask.shape[0]
self.attn_mask_cache = attn_mask
def __init__(self, device: torch.device):
self.attn_mask_cache = None
self._seq_len_cached = 0
self.device = device
if torch.version.cann.startswith("8.3"):
assigned_mask_dim = 2048
self.chunked_prefill_attn_mask = torch.triu(
torch.ones(assigned_mask_dim, assigned_mask_dim),
diagonal=1).to(torch.int8).to(device)
self.mla_mask = None
self.chunked_prefill_attn_mask = None
self.pcp_mla_mask = None
@staticmethod
def get_mask_scale_factor(dtype: torch.dtype = torch.float16):
if dtype == torch.float16:
mask_scale_factor = 1
elif dtype == torch.bfloat16:
mask_scale_factor = -10000
else:
raise ValueError(
"The current operation now only supports data types: torch.float16 and "
"torch.bfloat16. Please ensure the input is of one of these types."
)
return mask_scale_factor
def get_attn_mask(self, max_seq_len: int, dtype: torch.dtype,
device: torch.device):
self._update_attn_cache(max_seq_len, dtype)
return self.attn_mask_cache[:max_seq_len, :max_seq_len].contiguous(
).to(device, non_blocking=True)
def get_splitfuse_attn_mask(
self,
seq_lens: torch.Tensor = None,
position: torch.Tensor = None,
dtype: torch.dtype = None,
device: torch.device = None,
) -> torch.Tensor:
if torch.version.cann.startswith("8.3"):
return self.chunked_prefill_attn_mask
else:
if dtype not in [torch.float16, torch.bfloat16]:
raise ValueError(
"splitfuse_attn_mask now only supports bf16 and fp16")
max_seq_len = max(seq_lens, default=0)
self._update_attn_cache(max_seq_len, dtype)
# FIXME: Currently the mask value of chunked-prefill situation and Prefill-Only situation
# is not the same. Fix this in the future when kernel is ready.
mask_scale_factor = AttentionMaskBuilder.get_mask_scale_factor(
dtype)
attn_mask = torch.index_select(self.attn_mask_cache,
dim=0,
index=position)[:, :max_seq_len]
attn_mask *= mask_scale_factor
return attn_mask.contiguous().to(device, non_blocking=True)
def _update_attn_cache(self, seqlen: int, dtype: torch.dtype):
if seqlen > self._seq_len_cached:
self._seq_len_cached = seqlen
self.attn_mask_cache = _generate_attn_mask(seqlen, dtype)
def get_attn_mask(self, max_seq_len: int, dtype: torch.dtype):
if self.attn_mask_cache is None or max_seq_len > self._seq_len_cached:
self.attn_mask_cache = _generate_attn_mask(max_seq_len, dtype)
self._seq_len_cached = max_seq_len
assert self.attn_mask_cache is not None, "Something is wrong in generate_attn_mask."
if self.attn_mask_cache.dtype != dtype:
self.attn_mask_cache = self.attn_mask_cache.to(dtype)
return self.attn_mask_cache[:max_seq_len, :max_seq_len].contiguous().to(self.device, non_blocking=True)
def get_splitfuse_attn_mask(self) -> torch.Tensor:
if self.chunked_prefill_attn_mask is None:
self.chunked_prefill_attn_mask = (
torch.triu(torch.ones(2048, 2048), diagonal=1).to(torch.int8).to(self.device)
)
return self.chunked_prefill_attn_mask
def get_mla_mask(self, dtype: torch.dtype) -> torch.Tensor:
if self.mla_mask is None or self.mla_mask.dtype != dtype:
if dtype == torch.float16:
mask_value = torch.finfo(torch.float32).min
else:
mask_value = 1
prefill_mask = torch.triu(torch.ones(512, 512, device=self.device, dtype=dtype), 1)
self.mla_mask = torch.where(prefill_mask == 1, mask_value, 0).to(dtype)
return self.mla_mask
def get_pcp_mla_mask(self, dtype: torch.dtype):
if self.pcp_mla_mask is None or self.pcp_mla_mask.dtype != dtype:
self.pcp_mla_mask = torch.triu(torch.ones(512, 512, device=self.device, dtype=dtype), 1)
return self.pcp_mla_mask
def get_attention_mask(self, causal: bool, model_config: ModelConfig):
if model_config.runner_type == "pooling":
return self.get_attn_mask(2048, torch.bool)
return self.get_splitfuse_attn_mask()
def get_final_mla_mask(self, model_config: ModelConfig):
if get_pcp_group().world_size > 1:
return self.get_pcp_mla_mask(model_config.dtype)
# Prefill stages use 512x512 mask with appropriate dtype
return self.get_mla_mask(model_config.dtype)

2158
vllm_ascend/attention/attention_v1.py
View File

File diff suppressed because it is too large Load Diff

0
vllm_ascend/attention/context_parallel/__init__.py Normal file
View File

1073
vllm_ascend/attention/context_parallel/attention_cp.py Normal file
View File

File diff suppressed because it is too large Load Diff

197
vllm_ascend/attention/context_parallel/common_cp.py Normal file
View File

@@ -0,0 +1,197 @@
from dataclasses import dataclass
import torch
import torch.distributed as dist
import torch_npu
from vllm.distributed import get_dcp_group, get_pcp_group
from vllm_ascend.distributed.utils import get_decode_context_model_parallel_world_size
@dataclass
class AscendPCPMetadata:
"""
Metadata for Prefill Context Parallelism (PCP) on Ascend devices.
Stores index tensors and sequence lengths for routing attention
computations across PCP ranks during long sequence processing.
"""
q_head_idx: torch.Tensor = None
q_tail_idx: torch.Tensor = None
kv_with_q_head_nomask_idx: torch.Tensor = None
kv_with_q_head_mask_idx: torch.Tensor = None
kv_with_q_tail_nomask_idx: torch.Tensor = None
kv_with_q_tail_mask_idx: torch.Tensor = None
kv_tail_proj_idx: torch.Tensor = None
kv_with_q_head_attn_idx_in_tail: torch.Tensor = None
kv_with_q_tail_attn_idx_in_tail: torch.Tensor = None
attn_mask_seqlens: torch.Tensor = None
head_attn_nomask_seqlens: torch.Tensor = None
tail_attn_nomask_seqlens: torch.Tensor = None
head_actual_seq_lengths_kv: list[int] | None = None
tail_actual_seq_lengths_kv: list[int] | None = None
q_full_idx: torch.Tensor = None
pcp_use_hybrid_attn: bool = False
pcp_unpad_mask: torch.Tensor = None
pcp_allgather_restore_idx: list[int] | None = None
pcp_fa_query_idx: torch.Tensor = None
pcp_padded_tokens_fla: int = 0
pcp_enter_fa_restore_idx: torch.Tensor = None
pcp_fa_padding_restore_idx: torch.Tensor = None
block_table_cp: torch.Tensor = None
valid_block_ids: torch.Tensor = None
prefill_q_cum_seqlens: torch.Tensor = None
max_num_tokens_across_pcp: int = 0
total_num_scheduled_tokens: int = 0
block_arange: torch.Tensor = None
@dataclass
class CPChunkedContextMetadata:
"""
Metadata for chunked context handling in Context Parallelism (CP).
Extends chunked prefill with per-rank chunk information for PCP/DCP.
"""
# For handling chunked prefill
cu_seq_lens: torch.Tensor
starts: torch.Tensor
seq_tot: list[int]
max_seq_lens: list[int]
workspace: torch.Tensor
chunk_seq_lens: torch.Tensor
chunk_seq_lens_npu: torch.Tensor
chunk_actual_seq_lengths_kv_list: list[list[int]]
# for mla DCP & PCP
padded_chunk_seq_lens_npu: torch.Tensor = None
padded_local_chunk_seq_lens: list[list[int]] | None = None
local_context_lens_allranks: list[list[int]] | None = None
padded_local_cu_seq_lens: torch.Tensor = None
cu_seq_lens_lst: list[list[int]] | None = None
chunk_size: int | None = None
@dataclass
class AscendMetadataForPrefill:
"""Prefill-specific metadata for Ascend attention with Context Parallelism."""
@dataclass
class ChunkedContextMetadata:
"""Metadata for chunked context processing within prefill phase."""
actual_chunk_seq_lengths: torch.Tensor
actual_seq_lengths_kv: torch.Tensor
starts: torch.Tensor
chunk_seq_mask_filtered_indices: torch.Tensor
chunked_req_mask: list[bool] | None = None
local_context_lens_allranks: list[list[int]] | None = None
cp_kv_recover_idx_for_chunk: list[int] | None = None
kv_inverse_idx_for_chunk: list[int] | None = None
local_total_toks: int | None = None
""" Prefill Specific Metadata for Ascend"""
pcp_metadata: AscendPCPMetadata | None = None
pcp_exit_fa_scatter_idx: torch.Tensor | None = None
chunked_context: ChunkedContextMetadata | None = None
block_tables: torch.Tensor = None
actual_seq_lengths_q: torch.Tensor = None
@dataclass
class AscendMetadataForDecode:
"""Decode-specific metadata for Ascend attention with Context Parallelism."""
num_computed_tokens_of_pcp_dcp: list[list[list[int]]] | None = None
block_tables: torch.Tensor = None
dcp_mtp_attn_mask: torch.Tensor = None
def _process_attn_out_lse(attn_output: torch.Tensor, softmax_lse: torch.Tensor) -> torch.Tensor:
pcp_size = get_pcp_group().world_size
dcp_size = get_decode_context_model_parallel_world_size()
dcp_group = get_dcp_group().device_group if dcp_size > 1 else None
softmax_lse = softmax_lse.to(torch.float32)
attn_output = attn_output.to(torch.float32)
# Concat out&lse: [bs,num_heads,v_head_dim] + [bs,num_heads,1] -> [bs,num_heads,v_head_dim+1]
attn_out_lse = torch.cat([attn_output, softmax_lse], dim=-1)
if dcp_size > 1:
# permute: [bs, num_heads, v_head_dim+1] -> [num_heads, v_head_dim+1, bs]
attn_out_lse = attn_out_lse.permute([1, 2, 0]).contiguous()
attn_out_lse_all2all = torch.empty_like(attn_out_lse)
dist.all_to_all_single(attn_out_lse_all2all, attn_out_lse, group=dcp_group)
attn_out_lse = attn_out_lse_all2all.permute([2, 0, 1])
if pcp_size > 1:
# AllGather out&lse within CP group
attn_out_lse = get_pcp_group().all_gather(attn_out_lse.contiguous(), dim=0)
return attn_out_lse
def _npu_attention_update(head_size, attn_out_lse: torch.Tensor) -> torch.Tensor:
pcp_size = get_pcp_group().world_size
dcp_size = get_decode_context_model_parallel_world_size()
# [PCP * S, DCP * H, D+1]
B_total, H_total, D_plus_1 = attn_out_lse.shape
S = B_total // pcp_size
H = H_total // dcp_size
D = head_size
assert D_plus_1 == D + 1
# [PCP, S, DCP, H, D+1]
x = attn_out_lse.view(pcp_size, S, dcp_size, H, D_plus_1)
# [PCP, DCP, S, H, D+1]
x = x.permute(0, 2, 1, 3, 4).contiguous()
# Flatten [N, S, H, D+1], N = pcp_size * dcp_size
x = x.view(-1, S, H, D_plus_1)
# Split out lse
out_flat, lse_flat = torch.split(x, [D, 1], dim=-1) # [N, S, H, D], [N, S, H, 1]
# out: [N, S, H, D] -> [N, S*H, D]
# lse: [N, S, H, 1] -> [N, S*H]
out_flat = out_flat.flatten(1, 2) # [N, S*H, D]
lse_flat = lse_flat.flatten(1, -1) # [N, S*H]
# unbind to list
out_list = out_flat.unbind(0) # [S*H, D]
lse_list = lse_flat.unbind(0) # [S*H]
attn_out, _ = torch_npu.npu_attention_update(lse_list, out_list, 0)
attn_out = attn_out.view(-1, H, D)
return attn_out
def _npu_attn_out_lse_update(attn_lse_mask, attn_lse_nomask, attn_out_mask, attn_out_nomask):
T = attn_out_mask.shape[0]
N = attn_out_mask.shape[1]
D = attn_out_mask.shape[2]
attn_out_mask, attn_lse_mask = _out_lse_reshape(attn_out_mask, attn_lse_mask)
attn_out_nomask, attn_lse_nomask = _out_lse_reshape(attn_out_nomask, attn_lse_nomask)
attn_out_mask = attn_out_mask.to(torch.float32)
attn_out_nomask = attn_out_nomask.to(torch.float32)
attn_lse_mask = attn_lse_mask.to(torch.float32)
attn_lse_nomask = attn_lse_nomask.to(torch.float32)
attn_output = [attn_out_nomask, attn_out_mask]
attn_lse = [attn_lse_nomask, attn_lse_mask]
update_type = 0
output, _ = torch_npu.npu_attention_update(attn_lse, attn_output, update_type)
output = output.view(T, N, D)
return output
def _out_lse_reshape(attn_out: torch.Tensor, attn_lse: torch.Tensor) -> torch.Tensor:
attn_out = attn_out.contiguous().view(attn_out.shape[0] * attn_out.shape[1], attn_out.shape[2])
attn_lse = attn_lse.contiguous().view(attn_lse.shape[0] * attn_lse.shape[1] * attn_lse.shape[2])
return attn_out, attn_lse
def _update_out_and_lse(out_list: torch.Tensor, lse_list: torch.Tensor) -> torch.Tensor:
"""LSE_final = log(sum(exp(LSE_i))), O_final = sum(exp(LSE_i - LSE_final) * O_i)
Args:
out_list: shape = [N, batch_size, num_heads, head_size]
lse_list: shape = [N, batch_size, num_heads, 1]
Returns:
out_final: shape = [batch_size, num_heads, head_size]
lse_final: shape = [batch_size, num_heads, 1]
"""
lse_final = torch.logsumexp(lse_list, dim=0, keepdim=False)
out_final = torch.sum(torch.exp(lse_list - lse_final) * out_list, dim=0)
return out_final, lse_final

1671
vllm_ascend/attention/context_parallel/dsa_cp.py Normal file
View File

File diff suppressed because it is too large Load Diff

855
vllm_ascend/attention/context_parallel/mla_cp.py Normal file
View File

@@ -0,0 +1,855 @@
from typing import TypeVar
import numpy as np
import torch
import torch_npu
from vllm.config import VllmConfig
from vllm.distributed import (
get_dcp_group,
get_pcp_group,
)
from vllm.utils.math_utils import cdiv
from vllm.v1.attention.backend import AttentionCGSupport
from vllm.v1.kv_cache_interface import AttentionSpec
from vllm_ascend.attention.attention_v1 import AscendAttentionState
from vllm_ascend.core.kv_cache_interface import AscendMLAAttentionSpec
from vllm_ascend.device.device_op import DeviceOperator
from vllm_ascend.distributed.utils import (
get_decode_context_model_parallel_rank,
get_decode_context_model_parallel_world_size,
)
# isort: off
from vllm_ascend.attention.mla_v1 import (
AscendMLADecodeMetadata,
AscendMLAImpl,
AscendMLAMetadata,
AscendMLAMetadataBuilder,
AscendMLAPrefillMetadata,
DecodeMLAPreprocessResult,
PrefillMLAPreprocessResult,
BUILD_METADATA_STEP_PREFILL,
)
# isort: on
from vllm_ascend.ascend_forward_context import _EXTRA_CTX
from vllm_ascend.attention.context_parallel.common_cp import (
AscendPCPMetadata,
CPChunkedContextMetadata,
_npu_attention_update,
_process_attn_out_lse,
)
from vllm_ascend.attention.utils import AscendCommonAttentionMetadata, notify_kv_cache_written
from vllm_ascend.compilation.acl_graph import (
get_draft_graph_params,
get_draft_graph_prefill_params,
get_graph_params,
update_graph_params_workspaces,
)
from vllm_ascend.utils import weak_ref_tensors
MAX_O_PROJ_PREFETCH_SIZE = 16 * 1024 * 1024
M = TypeVar("M", bound=AscendMLAMetadata)
class AscendMlaCPMetadataBuilder(AscendMLAMetadataBuilder):
"""
NOTE: Please read the comment at the top of the file before trying to
understand this class
"""
def __init__(
self,
kv_cache_spec: AscendMLAAttentionSpec,
layer_names: list[str],
vllm_config: VllmConfig,
device: torch.device,
metadata_cls: type[AscendMLAMetadata] | None = None,
supports_dcp_with_varlen: bool = False,
):
super().__init__(kv_cache_spec, layer_names, vllm_config, device, metadata_cls, supports_dcp_with_varlen)
self.pcp_size = get_pcp_group().world_size
self.pcp_rank = get_pcp_group().rank_in_group if self.pcp_size > 1 else 0
self.dcp_size = get_decode_context_model_parallel_world_size()
self.dcp_rank = get_decode_context_model_parallel_rank() if self.dcp_size > 1 else 0
self.cp_local_block_size = vllm_config.parallel_config.cp_kv_cache_interleave_size
self.cp_virtual_block_size = self.cp_local_block_size * self.dcp_size * self.pcp_size
self.block_size = (self.block_size * self.cp_virtual_block_size) // np.gcd(
self.block_size, self.cp_virtual_block_size
)
def build(
self,
common_prefix_len: int,
common_attn_metadata: AscendCommonAttentionMetadata,
fast_build: bool = False,
) -> AscendMLAMetadata:
metadata_cls = super().build(common_prefix_len, common_attn_metadata)
if self.pcp_size > 1:
self.slot_mapping[: self.num_decode_tokens] = self.slot_mapping[
: self.num_decode_tokens * self.pcp_size : self.pcp_size
]
self.slot_mapping[self.num_decode_tokens : self.num_decode_tokens * self.pcp_size].fill_(-1)
metadata_cls.slot_mapping = self.slot_mapping
return metadata_cls
@classmethod
def get_cudagraph_support(
cls: type["AscendMlaCPMetadataBuilder"],
vllm_config: VllmConfig,
kv_cache_spec: AttentionSpec,
) -> AttentionCGSupport:
# Explicit override in case the underlying builder specialized this getter.
# @override omitted only because of mypy limitation due to type variable.
return AttentionCGSupport.UNIFORM_BATCH
def set_num_actual_tokens(
self,
common_attn_metadata: AscendCommonAttentionMetadata,
):
long_seq_metadata = common_attn_metadata.prefill_context_parallel_metadata
if long_seq_metadata is None:
raise AssertionError("long_seq_metadata should not be None.")
# In dcp only spec decode graph padding case,
# num_actual_tokens_pcp_padded may be less than num_actual_tokens
self.num_actual_tokens = max(
long_seq_metadata.num_actual_tokens_pcp_padded, common_attn_metadata.num_actual_tokens
)
def build_cp_metadata(
self,
common_prefix_len: int,
common_attn_metadata: AscendCommonAttentionMetadata,
) -> AscendPCPMetadata | None:
common_long_seq_metadata = common_attn_metadata.prefill_context_parallel_metadata
assert common_long_seq_metadata is not None
return AscendPCPMetadata(
q_head_idx=common_long_seq_metadata.q_head_idx_tensor,
q_tail_idx=common_long_seq_metadata.q_tail_idx_tensor,
kv_with_q_head_nomask_idx=common_long_seq_metadata.kv_with_q_head_nomask_idx_tensor,
kv_with_q_head_mask_idx=common_long_seq_metadata.kv_with_q_head_mask_idx_tensor,
kv_with_q_tail_nomask_idx=common_long_seq_metadata.kv_with_q_tail_nomask_idx_tensor,
kv_with_q_tail_mask_idx=common_long_seq_metadata.kv_with_q_tail_mask_idx_tensor,
kv_tail_proj_idx=common_long_seq_metadata.kv_tail_proj_idx_tensor,
kv_with_q_head_attn_idx_in_tail=common_long_seq_metadata.kv_with_q_head_attn_idx_in_tail_tensor,
kv_with_q_tail_attn_idx_in_tail=common_long_seq_metadata.kv_with_q_tail_attn_idx_in_tail_tensor,
attn_mask_seqlens=common_long_seq_metadata.attn_mask_seqlens,
head_attn_nomask_seqlens=common_long_seq_metadata.head_attn_nomask_seqlens,
tail_attn_nomask_seqlens=common_long_seq_metadata.tail_attn_nomask_seqlens,
head_actual_seq_lengths_kv=common_long_seq_metadata.head_actual_seq_lengths_kv,
tail_actual_seq_lengths_kv=common_long_seq_metadata.tail_actual_seq_lengths_kv,
q_full_idx=common_long_seq_metadata.q_full_idx,
pcp_allgather_restore_idx=common_long_seq_metadata.pcp_allgather_restore_idx,
)
def build_chunked_metadata(
self,
common_prefix_len: int,
common_attn_metadata: AscendCommonAttentionMetadata,
):
chunked_context_metadata = super().build_chunked_metadata(common_prefix_len, common_attn_metadata)
if chunked_context_metadata is None:
return None
long_seq_metadata = common_attn_metadata.prefill_context_parallel_metadata
assert long_seq_metadata is not None
num_computed_tokens_of_pcp_dcp = long_seq_metadata.num_computed_tokens_of_pcp_dcp
assert num_computed_tokens_of_pcp_dcp is not None
local_context_lens_allranks = torch.tensor(num_computed_tokens_of_pcp_dcp[self.num_decodes :]).reshape(
-1, self.dcp_size * self.pcp_size
)
# Note(qcs): The max local context lengths
# padded to `cp_local_block_size`.
padded_local_context_lens_cpu = (
cdiv(
self.context_lens_cpu,
self.cp_virtual_block_size,
)
* self.cp_local_block_size
)
padded_local_max_context_chunk_across_ranks = (
cdiv(
self.max_context_chunk,
self.cp_virtual_block_size,
)
* self.cp_local_block_size
)
local_chunk_starts = (
torch.arange(self.num_chunks, dtype=torch.int32).unsqueeze(1).expand(-1, self.num_prefills)
* padded_local_max_context_chunk_across_ranks
)
local_chunk_ends = torch.min(
padded_local_context_lens_cpu.unsqueeze(0),
local_chunk_starts + padded_local_max_context_chunk_across_ranks,
)
padded_local_chunk_seq_lens = (local_chunk_ends - local_chunk_starts).clamp(min=0)
padded_local_cu_chunk_seq_lens_cpu = torch.zeros(
self.num_chunks, self.num_prefills + 1, dtype=torch.int32, pin_memory=True
)
torch.cumsum(
padded_local_chunk_seq_lens,
dim=1,
out=padded_local_cu_chunk_seq_lens_cpu[:, 1:],
dtype=torch.int32,
)
chunked_metadata = CPChunkedContextMetadata(
cu_seq_lens=chunked_context_metadata.cu_seq_lens,
starts=local_chunk_starts.pin_memory().to(self.device, non_blocking=True),
seq_tot=padded_local_chunk_seq_lens.sum(dim=1).tolist(),
max_seq_lens=chunked_context_metadata.max_seq_lens,
chunk_seq_lens=self.chunk_seq_lens,
chunk_seq_lens_npu=chunked_context_metadata.chunk_seq_lens_npu,
chunk_actual_seq_lengths_kv_list=chunked_context_metadata.chunk_actual_seq_lengths_kv_list,
workspace=chunked_context_metadata.workspace,
padded_chunk_seq_lens_npu=padded_local_chunk_seq_lens.npu(),
padded_local_chunk_seq_lens=padded_local_chunk_seq_lens.tolist(),
local_context_lens_allranks=local_context_lens_allranks.tolist(),
padded_local_cu_seq_lens=padded_local_cu_chunk_seq_lens_cpu.pin_memory().to(self.device, non_blocking=True),
cu_seq_lens_lst=self.cu_seq_lens_cpu.tolist(),
chunk_size=padded_local_max_context_chunk_across_ranks,
)
return chunked_metadata
def get_block_table_size(self, common_attn_metadata: AscendCommonAttentionMetadata, build_metadata_step: int):
self.num_decodes_flatten = self.query_lens[: self.num_decodes].sum().item()
if build_metadata_step == BUILD_METADATA_STEP_PREFILL:
# For pcp + spec decode, we flatten seq_lens and block_table
# to avoid irregular attn_mask shape
return self.num_decodes + self.num_prefills
else:
return self.num_decodes
def build_prefill_metadata(
self,
common_prefix_len: int,
common_attn_metadata: AscendCommonAttentionMetadata,
) -> AscendMLAPrefillMetadata:
prefill_metadata = super().build_prefill_metadata(common_prefix_len, common_attn_metadata)
prefill_metadata.pcp_metadata = self.build_cp_metadata(common_prefix_len, common_attn_metadata)
prefill_metadata.block_table = self.block_table[self.num_decodes :, ...]
return prefill_metadata
def build_decode_metadata(
self,
common_prefix_len: int,
common_attn_metadata: AscendCommonAttentionMetadata,
) -> AscendMLADecodeMetadata:
decode_metadata = super().build_decode_metadata(common_prefix_len, common_attn_metadata)
long_seq_metadata = common_attn_metadata.prefill_context_parallel_metadata
assert long_seq_metadata is not None
num_computed_tokens_of_pcp_dcp = long_seq_metadata.num_computed_tokens_of_pcp_dcp
assert num_computed_tokens_of_pcp_dcp is not None
# [bs, pcp_size, dcp_size]
num_computed_tokens_of_cp_dcp_array = np.array(num_computed_tokens_of_pcp_dcp)[: self.num_decodes]
cp_seq_len = num_computed_tokens_of_cp_dcp_array[:, self.pcp_rank, self.dcp_rank]
cp_seq_len = torch.tensor(cp_seq_len, dtype=torch.int32)
decode_metadata.cp_seq_len = cp_seq_len.tolist()
actual_seq_lengths_q = torch.arange(self.num_decodes) + 1
decode_metadata.actual_seq_lengths_q = actual_seq_lengths_q
if long_seq_metadata.dcp_mtp_attn_mask is not None:
decode_metadata.dcp_mtp_attn_mask = long_seq_metadata.dcp_mtp_attn_mask
else:
decode_metadata.dcp_mtp_attn_mask = None
return decode_metadata
class AscendMlaCPImpl(AscendMLAImpl):
"""
NOTE: Please read the comment at the top of the file before trying to
understand this class
"""
def __init__(
self,
num_heads: int,
head_size: int,
scale: float,
num_kv_heads: int,
alibi_slopes: list[float] | None,
sliding_window: int | None,
kv_cache_dtype: str,
logits_soft_cap: float | None,
attn_type: str,
kv_sharing_target_layer_name: str | None,
**kwargs,
):
super().__init__(
num_heads,
head_size,
scale,
num_kv_heads,
alibi_slopes,
sliding_window,
kv_cache_dtype,
logits_soft_cap,
attn_type,
kv_sharing_target_layer_name,
**kwargs,
)
self.pcp_size = get_pcp_group().world_size
self.pcp_rank = get_pcp_group().rank_in_group if self.pcp_size > 1 else 0
self.pcp_group = get_pcp_group().device_group if self.pcp_size > 1 else None
self.dcp_size = get_decode_context_model_parallel_world_size()
self.dcp_rank = get_decode_context_model_parallel_rank() if self.dcp_size > 1 else 0
self.dcp_group = get_dcp_group().device_group if self.dcp_size > 1 else None
@staticmethod
def update_graph_params(
update_stream,
forward_context,
num_tokens,
vllm_config=None,
speculative_config=None,
num_dcp_pcp_tokens=None,
draft_attn_metadatas=None,
):
if _EXTRA_CTX.is_draft_model:
if _EXTRA_CTX.is_draft_model_prefill:
graph_params = get_draft_graph_prefill_params()
else:
graph_params = get_draft_graph_params()
attn_metadata = draft_attn_metadatas
attn_keys = list(attn_metadata[0].keys())
else:
graph_params = get_graph_params()
attn_metadata = forward_context.attn_metadata
attn_keys = list(attn_metadata.keys())
# FIXME: Behold! We are using a temporary hack here to update the args
# for each layer's attention op in the graph.
num_layers = len(attn_keys)
if num_layers == 0:
return
if _EXTRA_CTX.is_draft_model:
attn_keys = attn_keys * (len(graph_params.attn_params[num_tokens]) // num_layers)
attn_count = 0
with torch.npu.stream(update_stream):
for key, param, handle, event in zip(
attn_keys,
graph_params.attn_params[num_tokens],
graph_params.handles[num_tokens],
graph_params.events[num_tokens],
):
(
q_nope,
k_nope,
q_pe,
k_pe,
num_heads,
num_kv_heads,
input_layout,
spec_attn_mask,
sparse_mode,
scale,
block_table,
block_size,
actual_seq_lengths,
actual_seq_lengths_kv,
attn_output,
softmax_lse,
) = param
if _EXTRA_CTX.is_draft_model:
draft_step = attn_count // num_layers
decode_meta = attn_metadata[draft_step][key].decode
attn_count = attn_count + 1
else:
decode_meta = attn_metadata[key].decode
seq_len = decode_meta.cp_seq_len
if isinstance(seq_len, torch.Tensor):
seq_len = seq_len.tolist()
actual_seq_lengths_kv = seq_len
pad_length = num_tokens - len(actual_seq_lengths_kv)
if pad_length > 0:
actual_seq_lengths_kv = actual_seq_lengths_kv + [0] * (num_tokens - len(actual_seq_lengths_kv))
torch.npu.graph_task_update_begin(update_stream, handle)
torch_npu.npu_fused_infer_attention_score.out(
q_nope,
k_nope,
k_nope,
query_rope=q_pe,
key_rope=k_pe,
num_heads=num_heads,
num_key_value_heads=num_kv_heads,
input_layout=input_layout,
atten_mask=spec_attn_mask,
sparse_mode=sparse_mode,
scale=scale,
antiquant_mode=0,
antiquant_scale=None,
softmax_lse_flag=True,
block_table=block_table,
block_size=block_size,
actual_seq_lengths_kv=actual_seq_lengths_kv,
actual_seq_lengths=actual_seq_lengths,
workspace=graph_params.workspaces.get(num_tokens),
out=[attn_output, softmax_lse],
)
torch.npu.graph_task_update_end(update_stream)
event.record(update_stream)
def get_num_actual_tokens(self, attn_metadata: M):
if self.pcp_size > 1:
return attn_metadata.num_actual_tokens_pcp_padded // self.pcp_size
else:
return attn_metadata.num_actual_tokens
def _v_up_proj(self, x):
# Convert from (B, N, L) to (N, B, L)
x = x.view(-1, self.num_heads, self.kv_lora_rank).transpose(0, 1)
# # Multiply (N, B, L) x (N, L, V) -> (N, B, V)
x = torch.bmm(x, self.W_UV)
# # Convert from (N, B, V) to (B, N * V)
x = x.transpose(0, 1).reshape(-1, self.num_heads * self.v_head_dim)
return x
def mla_preprocess_prefill(self, q_c, kv_no_split, kv_cache, attn_metadata):
if not self.pcp_size > 1:
return super().mla_preprocess_prefill(q_c, kv_no_split, kv_cache, attn_metadata)
num_decode_tokens = attn_metadata.num_decode_tokens
num_actual_tokens = (
attn_metadata.num_actual_tokens_pcp_padded - self.pcp_size * num_decode_tokens
) // self.pcp_size + num_decode_tokens
prefill_q_c = q_c[num_decode_tokens:num_actual_tokens]
prefill_q = self.q_proj(prefill_q_c)[0].view(-1, self.num_heads, self.qk_head_dim)
prefill_q_pe = prefill_q[..., self.qk_nope_head_dim :]
prefill_q_nope = prefill_q[..., : self.qk_nope_head_dim]
cos = attn_metadata.prefill.cos[: num_actual_tokens - num_decode_tokens]
sin = attn_metadata.prefill.sin[: num_actual_tokens - num_decode_tokens]
prefill_q_pe = self.rope_single(prefill_q_pe, cos, sin)
prefill_kv_no_split = kv_no_split[:num_actual_tokens]
kv_c, k_pe = prefill_kv_no_split.split([self.kv_lora_rank, self.qk_rope_head_dim], dim=-1)
kv_c_normed = self.kv_a_layernorm(kv_c.contiguous()) # type: ignore[misc]
assert len(kv_cache) > 1, "the number of kv cache should be greater than 1, namely (nope_cache and rope_cache)"
kv_c_normed = kv_c_normed.view([num_actual_tokens, self.num_kv_heads, -1])
k_pe = k_pe.unsqueeze(1)
prefill_k_pe = k_pe
prefill_k_pe[num_decode_tokens:num_actual_tokens] = self.rope_single(
prefill_k_pe[num_decode_tokens:num_actual_tokens], cos, sin
)
prefill_k_c_normed = kv_c_normed[:num_actual_tokens]
prefill_kv_c_k_pe = torch.cat([prefill_k_c_normed, prefill_k_pe], dim=-1)
prefill_kv_c_k_pe = get_pcp_group().all_gather(prefill_kv_c_k_pe, 0)
prefill_kv_c_k_pe = torch.index_select(
prefill_kv_c_k_pe, 0, attn_metadata.prefill.pcp_metadata.pcp_allgather_restore_idx
)
prefill_kv_c_k_pe = prefill_kv_c_k_pe[num_decode_tokens * self.pcp_size :]
prefill_k_c_normed, prefill_k_pe = prefill_kv_c_k_pe.split([self.kv_lora_rank, self.qk_rope_head_dim], dim=-1)
kv_c_normed, k_pe = prefill_k_c_normed, prefill_k_pe
prefill_k_c_normed = prefill_k_c_normed.squeeze(1)
slot_mapping = attn_metadata.slot_mapping[self.pcp_size * num_decode_tokens :]
DeviceOperator.reshape_and_cache(
key=kv_c_normed, value=k_pe, key_cache=kv_cache[0], value_cache=kv_cache[1], slot_mapping=slot_mapping
)
notify_kv_cache_written(self.layer_name or "")
pcp_metadata = attn_metadata.prefill.pcp_metadata
assert pcp_metadata is not None
tail_k_c_normed = torch.index_select(prefill_k_c_normed, 0, pcp_metadata.kv_tail_proj_idx)
tail_k_pe = torch.index_select(prefill_k_pe, 0, pcp_metadata.kv_tail_proj_idx)
prefill_k_nope, prefill_value = (
self.kv_b_proj(tail_k_c_normed)[0]
.view(-1, self.num_heads, self.qk_nope_head_dim + self.v_head_dim)
.split([self.qk_nope_head_dim, self.v_head_dim], dim=-1)
)
prefill_k_pe = tail_k_pe.expand((*prefill_k_nope.shape[:-1], -1))
return PrefillMLAPreprocessResult(prefill_q_nope, prefill_q_pe, prefill_k_nope, prefill_k_pe, prefill_value)
def mla_preprocess_decode(self, q_c, kv_no_split, kv_cache, attn_metadata):
num_decode_tokens = attn_metadata.num_decode_tokens
decode_q_c = q_c[:num_decode_tokens]
cos = attn_metadata.decode.cos
sin = attn_metadata.decode.sin
decode_ql_nope, decode_q_pe = self._q_proj_and_k_up_proj(decode_q_c)
decode_ql_nope, decode_q_pe = self.reorg_decode_q(decode_ql_nope, decode_q_pe)
decode_q_pe = self.rope_single(decode_q_pe, cos, sin)
decode_slots = attn_metadata.slot_mapping[:num_decode_tokens]
decode_kv_no_split = kv_no_split[:num_decode_tokens]
decode_k_pe, decode_k_nope = self.exec_kv_decode(decode_kv_no_split, cos, sin, kv_cache, decode_slots)
return DecodeMLAPreprocessResult(decode_ql_nope, decode_q_pe, decode_k_nope, decode_k_pe)
def get_context_seq_len_npu(self, index: int, attn_metadata: AscendMLAMetadata):
prefill_metadata = attn_metadata.prefill
assert prefill_metadata is not None
assert prefill_metadata.chunked_context is not None
assert isinstance(prefill_metadata.chunked_context, CPChunkedContextMetadata)
assert prefill_metadata.chunked_context.padded_chunk_seq_lens_npu is not None
iters = len(prefill_metadata.chunked_context.seq_tot)
assert 0 <= index < iters
return prefill_metadata.chunked_context.padded_chunk_seq_lens_npu[index]
def reorg_decode_q(self, decode_q_nope, decode_q_pe):
if self.dcp_size > 1:
decode_q_no_split = torch.cat([decode_q_nope, decode_q_pe], dim=-1)
decode_q_no_split = get_dcp_group().all_gather(decode_q_no_split, 1)
decode_q_nope, decode_q_pe = decode_q_no_split.split([self.kv_lora_rank, self.qk_rope_head_dim], dim=-1)
return decode_q_nope, decode_q_pe
def _forward_prefill(
self,
q_nope: torch.Tensor,
q_pe: torch.Tensor,
k_nope: torch.Tensor,
k_pe: torch.Tensor,
value: torch.Tensor,
kv_c_and_k_pe_cache: tuple[torch.Tensor],
attn_metadata: AscendMLAMetadata,
) -> torch.Tensor:
if not self.pcp_size > 1:
return super()._forward_prefill(q_nope, q_pe, k_nope, k_pe, value, kv_c_and_k_pe_cache, attn_metadata)
assert attn_metadata.prefill is not None
assert attn_metadata.prefill.pcp_metadata is not None
num_tokens = q_nope.size(0)
prefill_meta = attn_metadata.prefill
pcp_metadata = attn_metadata.prefill.pcp_metadata
# Use precomputed indices from the metadata (already converted to tensors and on device)
q_head_idx = pcp_metadata.q_head_idx
q_tail_idx = pcp_metadata.q_tail_idx
kv_with_q_head_attn_idx = pcp_metadata.kv_with_q_head_attn_idx_in_tail
attn_mask_seqlens = pcp_metadata.attn_mask_seqlens
head_actual_seq_lengths_kv = pcp_metadata.head_actual_seq_lengths_kv
tail_actual_seq_lengths_kv = pcp_metadata.tail_actual_seq_lengths_kv
assert head_actual_seq_lengths_kv is not None
assert tail_actual_seq_lengths_kv is not None
output_head, lse_head = self._attention_with_optional_kv_select(
q_nope=torch.index_select(q_nope, 0, q_head_idx),
q_pe=torch.index_select(q_pe, 0, q_head_idx),
k_nope=k_nope,
k_pe=k_pe,
value=value,
kv_attn_idx=kv_with_q_head_attn_idx,
attn_mask_seqlens=attn_mask_seqlens,
actual_seq_lengths_kv=head_actual_seq_lengths_kv,
mask=prefill_meta.attn_mask,
attn_metadata=attn_metadata,
)
output_tail, lse_tail = self._attention_with_optional_kv_select(
q_nope=torch.index_select(q_nope, 0, q_tail_idx),
q_pe=torch.index_select(q_pe, 0, q_tail_idx),
k_nope=k_nope,
k_pe=k_pe,
value=value,
kv_attn_idx=None,
attn_mask_seqlens=attn_mask_seqlens,
actual_seq_lengths_kv=tail_actual_seq_lengths_kv,
mask=prefill_meta.attn_mask,
attn_metadata=attn_metadata,
)
q_full_idx = pcp_metadata.q_full_idx
attn_output = torch.index_select(torch.cat([output_head, output_tail], dim=0), 0, q_full_idx)
attn_lse = None
if attn_metadata.prefill is not None and attn_metadata.prefill.chunked_context is not None:
attn_lse = torch.index_select(torch.cat([lse_head, lse_tail], dim=0), 0, q_full_idx)
output, _ = self._compute_prefill_context(
q_nope, q_pe, kv_c_and_k_pe_cache, self.qk_rope_head_dim, attn_metadata, attn_output, attn_lse
)
output = output.reshape([num_tokens, self.num_heads * self.v_head_dim])
return output
def _attention_with_optional_kv_select(
self,
q_nope: torch.Tensor,
q_pe: torch.Tensor,
k_nope: torch.Tensor,
k_pe: torch.Tensor,
value: torch.Tensor,
kv_attn_idx: torch.Tensor | None,
attn_mask_seqlens: list[int],
actual_seq_lengths_kv: list[int],
mask: torch.Tensor,
attn_metadata,
):
if kv_attn_idx is None:
k_nope_attn = k_nope
value_attn = value
k_pe_attn = k_pe
else:
k_nope_attn = torch.index_select(k_nope, 0, kv_attn_idx)
value_attn = torch.index_select(value, 0, kv_attn_idx)
k_pe_attn = torch.index_select(k_pe, 0, kv_attn_idx)
attn_out, attn_lse = torch.ops.npu.npu_fused_infer_attention_score(
q_nope,
k_nope_attn.contiguous(),
value_attn.contiguous(),
query_rope=q_pe,
key_rope=k_pe_attn.contiguous(),
num_heads=self.num_heads,
num_key_value_heads=self.num_heads,
input_layout="TND",
atten_mask=mask,
scale=self.scale,
sparse_mode=3,
antiquant_mode=0,
antiquant_scale=None,
softmax_lse_flag=True,
actual_seq_lengths_kv=actual_seq_lengths_kv,
actual_seq_lengths=attn_mask_seqlens,
)
if attn_metadata.prefill is not None and attn_metadata.prefill.chunked_context is None:
attn_lse = None
return attn_out, attn_lse
def _forward_decode(
self,
q_nope: torch.Tensor,
q_pe: torch.Tensor,
k_nope: torch.Tensor,
k_pe: torch.Tensor,
block_size: int,
attn_metadata: AscendMLAMetadata,
dequant_scale_q_nope=None,
) -> torch.Tensor:
decode_meta = attn_metadata.decode
assert decode_meta is not None
num_tokens = q_nope.size(0)
# shape of knope/k_pe for npu graph mode should be:
# [num_blocks, num_kv_heads, block_size, self.kv_lora_rank/self.qk_rope_head_dim]
if self.dcp_size > 1:
num_heads = self.num_heads * self.dcp_size
else:
num_heads = self.num_heads
# use pcp & dcp split computed token nums from scheduler to compute actual seq_len and seq_mask
k_nope = k_nope.view(-1, self.num_kv_heads, block_size, self.kv_lora_rank)
k_pe = k_pe.view(-1, self.num_kv_heads, block_size, self.qk_rope_head_dim)
actual_seq_lengths = None
input_layout = "BNSD"
if (
attn_metadata.attn_state
in [
AscendAttentionState.SpecDecoding,
AscendAttentionState.ChunkedPrefill,
AscendAttentionState.DecodeOnly,
]
and self.speculative_config is not None
):
input_layout = "BSND"
num_decodes = attn_metadata.num_decodes
# TODO: If the driver is upgraded later, the contiguous function can be deleted.
q_nope = q_nope.view(num_decodes, -1, q_nope.shape[1], q_nope.shape[-1]).contiguous()
q_pe = q_pe.view(num_decodes, -1, q_pe.shape[1], q_pe.shape[-1])
sparse_mode = 0
spec_attn_mask = attn_metadata.decode.dcp_mtp_attn_mask # type:ignore
actual_seq_lengths = attn_metadata.query_lens
else:
q_nope = q_nope.view(num_tokens, num_heads, 1, -1).contiguous()
q_pe = q_pe.view(num_tokens, num_heads, 1, -1)
sparse_mode = 0
spec_attn_mask = None
common_kwargs = {
"query_rope": q_pe,
"key_rope": k_pe,
"num_heads": num_heads,
"num_key_value_heads": self.num_kv_heads,
"input_layout": input_layout,
"atten_mask": spec_attn_mask,
"sparse_mode": sparse_mode,
"scale": self.scale,
"antiquant_mode": 0,
"antiquant_scale": None,
"block_table": decode_meta.block_table,
"block_size": block_size,
"actual_seq_lengths": actual_seq_lengths,
"actual_seq_lengths_kv": decode_meta.cp_seq_len,
"softmax_lse_flag": True,
}
if _EXTRA_CTX.is_draft_model:
if _EXTRA_CTX.is_draft_model_prefill:
graph_params = get_draft_graph_prefill_params()
else:
graph_params = get_draft_graph_params()
else:
graph_params = get_graph_params()
if _EXTRA_CTX.capturing:
stream = torch_npu.npu.current_stream()
event = torch.npu.ExternalEvent()
event.wait(stream)
event.reset(stream)
graph_params.events[num_tokens].append(event)
workspace = graph_params.workspaces.get(num_tokens)
if workspace is None:
workspace = torch_npu._npu_fused_infer_attention_score_get_max_workspace(
q_nope,
k_nope,
k_nope,
**common_kwargs,
)
update_graph_params_workspaces(num_tokens, workspace)
attn_output = torch.empty_like(q_nope)
if input_layout == "BSND":
num_decodes = attn_metadata.num_decodes
softmax_lse = torch.empty(
(num_decodes, num_heads, q_nope.shape[1], 1), dtype=torch.float, device=q_nope.device
)
elif input_layout == "BNSD":
softmax_lse = torch.empty((num_tokens, num_heads, 1, 1), dtype=torch.float, device=q_nope.device)
else:
softmax_lse = torch.empty((num_tokens, num_heads, 1), dtype=torch.float, device=q_nope.device)
graph_params.attn_params[num_tokens].append(
(
weak_ref_tensors(q_nope),
weak_ref_tensors(k_nope),
weak_ref_tensors(q_pe),
weak_ref_tensors(k_pe),
num_heads,
self.num_kv_heads,
input_layout,
weak_ref_tensors(spec_attn_mask) if spec_attn_mask is not None else None,
sparse_mode,
self.scale,
weak_ref_tensors(decode_meta.block_table),
block_size,
actual_seq_lengths,
decode_meta.cp_seq_len,
weak_ref_tensors(attn_output),
weak_ref_tensors(softmax_lse),
)
)
torch.npu.graph_task_group_begin(stream)
torch_npu.npu_fused_infer_attention_score.out(
q_nope, k_nope, k_nope, **common_kwargs, workspace=workspace, out=[attn_output, softmax_lse]
)
handle = torch.npu.graph_task_group_end(stream)
graph_params.handles[num_tokens].append(handle)
else:
attn_output, softmax_lse = torch_npu.npu_fused_infer_attention_score(
q_nope,
k_nope,
k_nope,
**common_kwargs,
)
if input_layout == "BSND":
attn_output = attn_output.view(-1, attn_output.shape[2], attn_output.shape[3])
softmax_lse = softmax_lse.transpose(1, 2).reshape(-1, softmax_lse.shape[1], 1)
if input_layout == "BNSD":
B_attn, N_attn, S, D = attn_output.shape
B_lse, N_lse, Q_S, _ = softmax_lse.shape
attn_output = attn_output.permute(0, 2, 1, 3).reshape(B_attn * S, N_attn, D)
softmax_lse = softmax_lse.permute(0, 2, 1, 3).reshape(B_lse * Q_S, N_lse, 1)
# Update out&lse
attn_out_lse = _process_attn_out_lse(attn_output, softmax_lse)
attn_output = _npu_attention_update(self.kv_lora_rank, attn_out_lse)
return self._v_up_proj(attn_output)
def _out_lse_reshape(self, attn_out: torch.Tensor, attn_lse: torch.Tensor) -> torch.Tensor:
attn_out = attn_out.contiguous().view(attn_out.shape[0] * attn_out.shape[1], attn_out.shape[2])
attn_lse = attn_lse.contiguous().view(attn_lse.shape[0] * attn_lse.shape[1] * attn_lse.shape[2])
return attn_out, attn_lse
def _reorg_kvcache(
self,
kv_c_normed: torch.Tensor,
k_pe: torch.Tensor,
chunked_context: CPChunkedContextMetadata,
chunk_idx: int,
toks: int,
) -> tuple[torch.Tensor, torch.Tensor]:
"""
reorg and unpad kvcache after cp local gather to tp layout for attn kernel.
e.g.
kv_c_normed in rank0 = [T0_0, T0_1, T0_2, T0_3, T1_0, T1_1, ...]
kv_c_normed in rank1 = [T0_4, T0_5, pad, pad, T1_2, pad, ...]
allgatered_kv_c_normed = [T0_0, T0_1, T0_2, T0_3, T1_0, T1_1, ...,
T0_4, T0_5, pad, pad, T1_2, pad, ...]
-> reorganized_kv_c_normed = [T0_0, T0_1, T0_2, T0_3, T0_4, T0_5,
T1_0, T1_1, T1_2, ...]
Args:
padded_local_chunk_seq_lens_lst: local chunk context lengths
under current CP rank.
local_context_lens_allranks: local context lengths on each CP rank.
sum_seq_len: the sum of cp_chunk_seq_lens_lst.
max_seq_len: the max value of cp_chunk_seq_lens_lst.
chunk_size: the local padded max context chunk from
chunked_context_metadata building.
chunk_idx: chunk idx of chunked_prefill.
toks: the number of tokens for local gather cache.
"""
assert chunked_context is not None
assert chunked_context.padded_local_chunk_seq_lens is not None
assert chunked_context.local_context_lens_allranks is not None
assert chunked_context.cu_seq_lens_lst is not None
assert chunked_context.max_seq_lens is not None
assert chunked_context.chunk_size is not None
padded_local_chunk_seq_lens_lst = chunked_context.padded_local_chunk_seq_lens[chunk_idx]
local_context_lens_allranks = chunked_context.local_context_lens_allranks
sum_seq_len = chunked_context.cu_seq_lens_lst[chunk_idx][-1]
max_seq_len = chunked_context.max_seq_lens[chunk_idx]
chunk_size: int = chunked_context.chunk_size
cache_kv_c_k_pe = torch.cat([kv_c_normed, k_pe], dim=-1)
if self.dcp_size > 1:
cache_kv_c_k_pe = get_dcp_group().all_gather(cache_kv_c_k_pe, 0)
if self.pcp_size > 1:
cache_kv_c_k_pe = get_pcp_group().all_gather(cache_kv_c_k_pe, 0)
allgatered_kv_c_normed, allgatered_k_pe = cache_kv_c_k_pe.split(
[self.kv_lora_rank, self.qk_rope_head_dim], dim=-1
)
kv_c_segments = []
k_pe_segments = []
src_token_idx = 0
max_seq_len_check = 0
for padded_local_chunk_seq_len, local_context_lens in zip(
padded_local_chunk_seq_lens_lst, local_context_lens_allranks
):
cur_seq_len = 0
for rank, local_context_len in enumerate(local_context_lens):
# Note(qcs): We split the context into multiple chunks,
# depending on the size of the workspace.
# local_context in dcp0: |-----------------|
# local_context in dcp1: |--------------|
# n*padded_local_chunk: |-----|-----|-----|
# local_chunk_len in dcp1: |-----|-----|--|
# so we need update the last chunk length in dcp1.
local_chunk_len = min(
max(0, local_context_len - chunk_idx * chunk_size),
padded_local_chunk_seq_len,
)
if local_chunk_len != 0:
kv_c_segment = allgatered_kv_c_normed[
rank * toks + src_token_idx : rank * toks + src_token_idx + local_chunk_len
]
k_pe_segment = allgatered_k_pe[
rank * toks + src_token_idx : rank * toks + src_token_idx + local_chunk_len
]
kv_c_segments.append(kv_c_segment)
k_pe_segments.append(k_pe_segment)
cur_seq_len += local_chunk_len
max_seq_len_check = max(max_seq_len_check, cur_seq_len)
src_token_idx += padded_local_chunk_seq_len
reorganized_kv_c_normed = torch.cat(kv_c_segments, dim=0)
reorganized_k_pe = torch.cat(k_pe_segments, dim=0)
assert reorganized_kv_c_normed.shape[0] == sum_seq_len
assert reorganized_k_pe.shape[0] == sum_seq_len
assert max_seq_len_check == max_seq_len
return reorganized_kv_c_normed, reorganized_k_pe

1312
vllm_ascend/attention/context_parallel/sfa_cp.py Normal file
View File

File diff suppressed because it is too large Load Diff

2837
vllm_ascend/attention/dsa_v1.py Normal file
View File

File diff suppressed because it is too large Load Diff

145
vllm_ascend/attention/fa3_v1.py Normal file
View File

@@ -0,0 +1,145 @@
import torch
import vllm.envs as envs_vllm
from flash_attn_npu_v3 import flash_attn_with_kvcache as _fa3_fn # type: ignore[import-not-found]
from vllm.v1.attention.backend import AttentionBackend # type: ignore
from vllm_ascend.attention.attention_v1 import (
AscendAttentionBackendImpl,
AscendAttentionMetadataBuilder,
)
class AscendFABackend(AttentionBackend):
def __init__(self):
super().__init__()
@staticmethod
def get_name() -> str:
return "CUSTOM" if not envs_vllm.VLLM_USE_V2_MODEL_RUNNER else "FLASH_ATTN"
@staticmethod
def get_impl_cls() -> type["AscendFAImpl"]:
return AscendFAImpl
@staticmethod
def get_builder_cls() -> type["AscendAttentionMetadataBuilder"]:
return AscendAttentionMetadataBuilder
@staticmethod
def get_kv_cache_shape(
num_blocks: int,
block_size: int,
num_kv_heads: int,
head_size: int,
cache_type: str = "",
) -> tuple[int, ...]:
return (2, num_blocks, block_size, num_kv_heads, head_size)
@staticmethod
def get_supported_kernel_block_sizes() -> list[int]:
return [128]
class AscendFAImpl(AscendAttentionBackendImpl):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
if self.sliding_window is not None:
raise ValueError(
"AscendFAImpl does not support sliding window attention. "
"Please disable sliding window or use the default FIA backend."
)
if not self.vllm_config.model_config.enforce_eager:
from vllm.config.compilation import CUDAGraphMode
cudagraph_mode = self.vllm_config.compilation_config.cudagraph_mode
if cudagraph_mode == CUDAGraphMode.FULL_DECODE_ONLY:
raise ValueError(
"AscendFAImpl does not support ACL graph capture with "
"FULL_DECODE_ONLY mode. Please set enforce_eager=True or "
"not set FULL_DECODE_ONLY or use the default FIA backend."
)
def _flash_attn_with_kvcache(
self,
query: torch.Tensor,
block_table: torch.Tensor,
actual_seq_lengths: torch.Tensor,
seq_lens: torch.Tensor,
is_causal: bool,
max_seq_len: int,
):
num_block, block_size, _, _ = self.key_cache.shape # type: ignore
key_fa_blk = self.key_cache.view( # type: ignore
num_block, block_size, self.num_kv_heads, self.head_size
)
value_fa_blk = self.value_cache.view( # type: ignore
num_block, block_size, self.num_kv_heads, self.head_size
)
attn_output = _fa3_fn(
query,
key_fa_blk,
value_fa_blk,
cache_seqlens=seq_lens, # kv sequence length for each individual request (NOT cumulative)
page_table=block_table, # must match the block table for the corresponding q
cu_seqlens_q=actual_seq_lengths, # cumulative sequence length for q
max_seqlen_q=max_seq_len,
causal=is_causal,
window_size=[-1, -1],
rotary_interleaved=False,
num_splits=1,
softcap=0.0,
attention_chunk=0,
sm_margin=0,
return_softmax_lse=False,
)
return attn_output
def forward_impl(
self,
query: torch.Tensor,
key: torch.Tensor,
value: torch.Tensor,
kv_cache: tuple[torch.Tensor],
attn_metadata,
output: torch.Tensor,
):
num_tokens = attn_metadata.actual_seq_lengths_q[-1]
query = query[:num_tokens]
num_decodes = attn_metadata.num_decodes
num_decode_tokens = attn_metadata.num_decode_tokens
num_prefills = attn_metadata.num_prefills
outputs = []
if num_decodes > 0:
outputs.append(
self._flash_attn_with_kvcache(
query[:num_decode_tokens],
attn_metadata.block_tables[:num_decodes, :],
attn_metadata.query_start_loc[: num_decodes + 1],
attn_metadata.seq_lens[:num_decodes].npu(),
False,
max(attn_metadata.seq_lens[:num_decodes]),
)
)
if num_prefills > 0:
outputs.append(
self._flash_attn_with_kvcache(
query[num_decode_tokens:],
attn_metadata.block_tables[num_decode_tokens:, :],
attn_metadata.query_start_loc[num_decodes:],
attn_metadata.seq_lens[num_decodes:].npu(),
True, # enable causal for prefill
max(attn_metadata.seq_lens[num_decodes:]),
)
)
if not outputs:
raise ValueError("No attention output available")
attn_output_fa = outputs[0] if len(outputs) == 1 else torch.cat(outputs, dim=0)
output[:num_tokens] = attn_output_fa[:num_tokens]
return output

0
vllm_ascend/attention/kvcomp_attn/__init__.py Normal file
View File

152
vllm_ascend/attention/kvcomp_attn/attention_utils.py Normal file
View File

@@ -0,0 +1,152 @@
import torch
from vllm.config import get_current_vllm_config_or_none
from vllm_ascend.attention.utils import AscendCommonAttentionMetadata
from vllm_ascend.worker.kvcomp_utils import (
KVCompMetaData,
recover_request_lengths,
)
def build_kvcomp_metadata(
kvcomp_meta: KVCompMetaData,
common_meta: AscendCommonAttentionMetadata,
) -> None:
num_reqs = common_meta.num_reqs
kvcomp_meta.num_actual_tokens = common_meta.num_actual_tokens
kvcomp_meta.slot_mapping = common_meta.slot_mapping
kvcomp_meta.seq_lens_gpu = common_meta.seq_lens[:num_reqs]
real_batch_size = kvcomp_meta.seq_lens_gpu.shape[0]
assert num_reqs == real_batch_size, "the len of seq_lens_gpu is not equal with batch_size"
kvcomp_meta.actual_query_lens = recover_request_lengths(common_meta.query_start_loc[: num_reqs + 1]).to(torch.int32)
runtime_seq_lens_list = kvcomp_meta.seq_lens_gpu.tolist()
if kvcomp_meta.num_actual_tokens < real_batch_size:
runtime_seq_lens_list = runtime_seq_lens_list[: kvcomp_meta.num_actual_tokens] + [0] * (
real_batch_size - kvcomp_meta.num_actual_tokens
)
kvcomp_meta.slot_mapping[kvcomp_meta.num_actual_tokens :] = -1
runtime_max_len = max(runtime_seq_lens_list) if runtime_seq_lens_list else 0
kvcomp_meta.max_seq_len_for_hamming = (
common_meta.max_seq_len if common_meta.max_seq_len is not None else runtime_max_len
)
kvcomp_meta.block_tables_for_hamming = common_meta.block_table_tensor[:real_batch_size]
top_k_cpu = kvcomp_meta.topk_for_hamming_full_cpu[:real_batch_size].clone()
if kvcomp_meta.num_actual_tokens < real_batch_size:
top_k_cpu[kvcomp_meta.num_actual_tokens :] = 0
runtime_seq_lens_cpu = torch.tensor(runtime_seq_lens_list, dtype=torch.int32)
chunk_size = kvcomp_meta.kvcomp_config.chunk_size
remainder = runtime_seq_lens_cpu % chunk_size
new_seq_lens = torch.where(
remainder == 0,
chunk_size * top_k_cpu,
chunk_size * (top_k_cpu - 1) + remainder,
)
kvcomp_meta.seq_lens_from_hamming = new_seq_lens.tolist()
q_start_loc_slice = common_meta.query_start_loc[:real_batch_size].to(torch.int64)
torch.where(
kvcomp_meta.slot_mapping[:real_batch_size] >= 0,
torch.ones_like(q_start_loc_slice, dtype=torch.bool),
torch.zeros_like(q_start_loc_slice, dtype=torch.bool),
out=kvcomp_meta.valid_query_mask[:real_batch_size],
)
torch.where(
kvcomp_meta.valid_query_mask[:real_batch_size],
kvcomp_meta.actual_query_lens[:real_batch_size],
torch.zeros_like(kvcomp_meta.actual_query_lens[:real_batch_size]),
out=kvcomp_meta.seq_lens_for_reshape[:real_batch_size],
)
common_meta.kvcomp_metadata = kvcomp_meta
def reshape_and_cache_kvcomp(kvcomp_meta: KVCompMetaData | None, layer_index: int | None, key: torch.Tensor):
assert kvcomp_meta is not None
assert layer_index is not None
if kvcomp_meta.hashk_caches[layer_index] is None:
return None
hash_encoder = kvcomp_meta.hash_encoder
num_tokens = kvcomp_meta.num_actual_tokens
hashk = hash_encoder.compute_hash(key[:num_tokens])
hashk_op = hashk.transpose(0, 1).reshape(-1, hashk.shape[-1]).contiguous()
hashk_cache_op = kvcomp_meta.hashk_caches[layer_index]
real_batch_size = kvcomp_meta.seq_lens_gpu.shape[0]
torch.ops._C_ascend.npu_reshape_and_cache_bnsd(
hashk_op,
hashk_cache_op,
kvcomp_meta.slot_mapping[:num_tokens],
kvcomp_meta.seq_lens_for_reshape[:real_batch_size],
hashk_cache_op,
)
return hashk_cache_op
def get_kvcomp_decode_params(
layer_index: int | None,
kvcomp_meta: KVCompMetaData | None,
query: torch.Tensor,
key: torch.Tensor,
block_table: torch.Tensor,
actual_seq_lengths_kv: list[int],
):
assert kvcomp_meta is not None
assert layer_index is not None
if kvcomp_meta.hashk_caches[layer_index] is None:
return block_table, actual_seq_lengths_kv
kv_config = kvcomp_meta.kvcomp_config
hash_encoder = kvcomp_meta.hash_encoder
real_batch_size = kvcomp_meta.seq_lens_gpu.shape[0]
if kv_config.vllm_hash_attention_skip_layers[layer_index]:
return kvcomp_meta.hamming_output, kvcomp_meta.seq_lens_from_hamming
hashk_cache_op = reshape_and_cache_kvcomp(kvcomp_meta, layer_index, key)
hashq = hash_encoder.compute_hash(query[:real_batch_size])
hashq_op = hashq.unsqueeze(2).contiguous()
new_block_table = torch.ops._C_ascend.npu_hamming_dist_top_k(
hashq_op,
hashk_cache_op,
None,
kvcomp_meta.topk_for_hamming_full[:real_batch_size],
kvcomp_meta.seq_lens_gpu[:real_batch_size],
kvcomp_meta.chunk_sizes_for_hamming_full[:real_batch_size],
kvcomp_meta.max_seq_len_for_hamming,
kvcomp_meta.sink,
kvcomp_meta.recent,
None,
kvcomp_meta.block_tables_for_hamming,
kvcomp_meta.valid_query_mask[:real_batch_size],
kvcomp_meta.hamming_output[:real_batch_size],
)
new_block_table = new_block_table.squeeze(1).contiguous()
kvcomp_meta.hamming_output = new_block_table
return new_block_table, kvcomp_meta.seq_lens_from_hamming
def is_enable_hamming_sparse():
vllm_config = get_current_vllm_config_or_none()
if vllm_config is None:
return False
additional_config = vllm_config.additional_config if vllm_config.additional_config is not None else {}
enable_hamming_sparse = additional_config.get("enable_hamming_sparse", False)
enable_hamming_sparse = enable_hamming_sparse and not vllm_config.speculative_config
return enable_hamming_sparse

1982
vllm_ascend/attention/mla_v1.py
View File

File diff suppressed because it is too large Load Diff

3028
vllm_ascend/attention/sfa_v1.py
View File

File diff suppressed because it is too large Load Diff

486
vllm_ascend/attention/utils.py
View File

@@ -1,81 +1,365 @@
from dataclasses import dataclass
from typing import Any, List
from dataclasses import dataclass, field
from functools import lru_cache
from typing import Any
import torch
from vllm.distributed.kv_transfer import (get_kv_transfer_group,
has_kv_transfer_group,
is_v1_kv_transfer_group)
import torch.nn.functional as F
from vllm.config import VllmConfig, get_current_vllm_config
from vllm.distributed.kv_transfer import get_kv_transfer_group, has_kv_transfer_group, is_v1_kv_transfer_group
from vllm.forward_context import ForwardContext, get_forward_context
from vllm.utils.torch_utils import get_dtype_size
from vllm.v1.attention.backends.utils import CommonAttentionMetadata
from vllm_ascend.device.utils import FIA_TND_LARGE_HEAD_FALLBACK_HEAD_SIZE
from vllm_ascend.utils import (
AscendDeviceType,
get_ascend_config,
get_ascend_device_type,
is_pd_decode_recompute_scheduler_enabled,
)
from vllm_ascend.worker.kvcomp_utils import KVCompMetaData
SFA_QSFA_TILE_SIZE = 128
def get_sfa_qsfa_packed_head_dim(
kv_lora_rank: int,
qk_rope_head_dim: int,
tile_size: int = SFA_QSFA_TILE_SIZE,
) -> int:
if kv_lora_rank % tile_size != 0:
raise ValueError(
f"kv_lora_rank must be divisible by tile_size for SFA QSFA packed cache, "
f"got {kv_lora_rank=} and {tile_size=}."
)
scale_metadata_bytes = (kv_lora_rank // tile_size) * get_dtype_size(torch.float32)
return kv_lora_rank + qk_rope_head_dim * get_dtype_size(torch.bfloat16) + scale_metadata_bytes
def cache_graph_workspace(
graph_params,
num_tokens: int,
candidate_workspace: torch.Tensor,
*,
use_max_workspace: bool,
) -> torch.Tensor:
# Most models keep the original first-workspace cache behavior. Models with
# mixed attention layer shapes may need the largest workspace for a graph
# size because layers can require different FIA workspace sizes.
current_workspace = graph_params.workspaces.get(num_tokens)
if use_max_workspace:
if current_workspace is None or (
candidate_workspace.numel() * candidate_workspace.element_size()
> current_workspace.numel() * current_workspace.element_size()
):
graph_params.workspaces[num_tokens] = candidate_workspace
elif current_workspace is None:
graph_params.workspaces[num_tokens] = candidate_workspace
return graph_params.workspaces[num_tokens]
@lru_cache(maxsize=1)
def needs_layer_aware_fia_graph_replay() -> bool:
vllm_config = get_current_vllm_config()
model_config = vllm_config.model_config
hf_config = getattr(model_config, "hf_config", None)
hf_text_config = getattr(model_config, "hf_text_config", None)
text_config = getattr(hf_config, "text_config", None)
model_types = (
getattr(hf_config, "model_type", None),
getattr(hf_text_config, "model_type", None),
getattr(text_config, "model_type", None),
)
return any(model_type in {"gemma4", "gemma4_text"} for model_type in model_types)
def ascend_chunked_prefill_workspace_size(vllm_config: VllmConfig) -> int:
scheduler_config = vllm_config.scheduler_config
cache_config = vllm_config.cache_config
model_config = vllm_config.model_config
chunked_prefill_workspace_size = min(
# Make sure there is enough for 8 full length request or at least
# 4 pages of cache per request
max(8 * model_config.max_model_len, 4 * scheduler_config.max_num_seqs * cache_config.block_size),
# For long-context models try not to over-allocate limiting
# kv-cache space, limiting it to 128k tokens,
# which would result in the workspace being:
# 2*(576)*(128*1024) = 288mb
# (assuming 576 MLA head dim, and fp16)
# which would result in up-projected context being
# 2*(192*128)*(128*1024) = 6gb
# (assuming 192 QK head dim, 128 heads, and fp16)
128 * 1024,
)
chunked_prefill_workspace_size = max(
chunked_prefill_workspace_size,
scheduler_config.max_num_seqs * cache_config.block_size,
)
return chunked_prefill_workspace_size
def using_paged_attention(runtime_shape: int, vllm_config: VllmConfig, head_size: int | None = None) -> bool:
if vllm_config.speculative_config is not None:
return False
if get_ascend_device_type() == AscendDeviceType.A5:
return False
# TODO: Remove this fallback when A2/A3 FIA TND supports Gemma4's
# 512-dim global attention heads. Decode can use PA directly; prefill is
# handled by the device adaptor.
if head_size == FIA_TND_LARGE_HEAD_FALLBACK_HEAD_SIZE:
return True
from vllm.config.compilation import CUDAGraphMode
cudagraph_mode = vllm_config.compilation_config.cudagraph_mode
if cudagraph_mode != CUDAGraphMode.FULL_DECODE_ONLY:
return False
return runtime_shape in get_ascend_config().pa_shape_list
@lru_cache(maxsize=1)
def enable_cp():
prefill_config = get_current_vllm_config().parallel_config
return prefill_config.prefill_context_parallel_size > 1 or prefill_config.decode_context_parallel_size > 1
@dataclass
class AscendCommonAttentionMetadata:
class AscendPrefillContextParallelMetadata:
"""
Metadata for Prefill Context Parallelism (PCP) in CommonAttentionMetadata.
Contains index tensors and sequence lengths for PCP operations.
"""
pcp_allgather_restore_idx: torch.Tensor = None
num_actual_tokens_pcp_padded: int = 0
num_computed_tokens_of_pcp_dcp: list[list[list[int]]] | None = None
q_head_idx_tensor: torch.Tensor = None
q_tail_idx_tensor: torch.Tensor = None
kv_with_q_head_nomask_idx_tensor: torch.Tensor = None
kv_with_q_head_mask_idx_tensor: torch.Tensor = None
kv_with_q_tail_nomask_idx_tensor: torch.Tensor = None
kv_with_q_tail_mask_idx_tensor: torch.Tensor = None
kv_tail_proj_idx_tensor: torch.Tensor = None
kv_with_q_head_attn_idx_in_tail_tensor: torch.Tensor = None
kv_with_q_tail_attn_idx_in_tail_tensor: torch.Tensor = None
attn_mask_seqlens: torch.Tensor = None
head_attn_nomask_seqlens: torch.Tensor = None
tail_attn_nomask_seqlens: torch.Tensor = None
head_actual_seq_lengths_kv: list[int] | None = None
tail_actual_seq_lengths_kv: list[int] | None = None
q_full_idx: torch.Tensor = None
# original query_lens before pcp split
query_lens_pcp_full_cpu: torch.Tensor = None
# original max_query_len before pcp split
max_query_len_pcp_full: int = 0
# the following attributes are specifically used in hybrid-attn models.
pcp_use_hybrid_attn: bool = False
pcp_unpad_mask: torch.Tensor = None
# to get the right order of query in prefill per rank
pcp_fa_query_idx: torch.Tensor = None
# restore the full sequence across all pcp ranks
# when entering from linear-attention to attention
pcp_enter_fa_restore_idx: torch.Tensor = None
# restore the original FA padded layout without boolean-mask scatter
pcp_fa_padding_restore_idx: torch.Tensor = None
# scatter the full sequence across all pcp ranks
# when exiting from attention to linear-attention
pcp_exit_fa_scatter_idx: torch.Tensor = None
# the number of tokens padded in linear-attn per rank
pcp_padded_tokens_fla: int = 0
# the max number of unpadded tokens in all ranks
max_num_tokens_across_pcp: int = 0
# the number of scheduled tokens on the current rank before padding
total_num_scheduled_tokens: int = 0
# Because the sequence shard in linear attention layers does not include padding,
# the full attention layers cannot obtain the correct query_lens with pcp pad for
# chunked prefill calculation. Therefore, this value needs to be passed to the backend.
# TODO:To be refactored.
attn_chunk_seqlens: torch.Tensor = None
dcp_mtp_attn_mask: torch.Tensor = None
@dataclass
class AscendCommonAttentionMetadata(CommonAttentionMetadata):
"""
Per-batch attention metadata, shared across layers and backends.
AttentionMetadataBuilder instances use it to construct per-layer metadata.
For many of the tensors we keep both GPU and CPU versions.
For many of the tensors we keep both NPU and CPU versions.
"""
query_start_loc: torch.Tensor
query_start_loc_cpu: torch.Tensor
"""(batch_size + 1,), the start location of each request in query Tensor"""
# CPU tensor of sequence lengths for host-side operations.
# E.g., tensor([128, 256, 64]) for 3 requests with different seq lengths.
seq_lens_cpu: torch.Tensor = None
seq_lens_cpu: torch.Tensor
"""(batch_size,), the length of each request including both computed tokens
and newly scheduled tokens"""
# CPU tensor of already computed tokens count per request.
# E.g., tensor([100, 200, 50]) means req0 has 100 tokens already computed.
num_computed_tokens_cpu: torch.Tensor = None
seq_lens: torch.Tensor
"""same to seq_lens_cpu, for compatibility with some new attn metadata
(such as GDN)."""
# Number of decode tokens per request, used for speculative decoding.
# E.g., 1 for normal decoding, >1 for speculative decoding.
decode_token_per_req: int = 1
num_computed_tokens_cpu: torch.Tensor
"""(batch_size,), the number of computed tokens for each request"""
num_reqs: int
"""Number of requests"""
num_actual_tokens: int
"""Total number of tokens in batch"""
max_query_len: int
"""Max token number of request in batch"""
decode_token_per_req: int
"""decode token number per request"""
block_table_tensor: torch.Tensor
slot_mapping: torch.Tensor
actual_seq_lengths_q: list[int]
# Actual query sequence lengths for each token in the batch (CPU list).
# E.g., [1, 1, 1, 128] for 3 decode tokens and 1 prefill with 128 tokens.
actual_seq_lengths_q: list[int] = field(default_factory=list)
# NPU tensor of position indices for rotary embeddings computation.
# E.g., tensor([0, 1, 2, ...]) indicating token positions in sequence.
positions: torch.Tensor = None
positions_cpu: torch.Tensor = None
attn_mask: torch.Tensor = None
spec_attn_mask: torch.Tensor = None
# Current attention state (e.g., ChunkedPrefill, DecodeOnly).
attn_state: Any = None
enable_dbo_across_dp: bool = False
is_only_prefill: bool = False
# Padding size for graph capture, -1 means not in graph mode.
graph_pad_size: int = -1
# Total number of tokens including padding, used for padding operations.
num_input_tokens: int = 0
# Metadata for Prefill Context Parallelism (PCP) operations.
prefill_context_parallel_metadata: AscendPrefillContextParallelMetadata | None = None
kvcomp_metadata: KVCompMetaData | None = None
# TODO: Remove it when vLLM no longer uses this function.
def unpadded(self, num_actual_tokens: int, num_actual_reqs: int) -> "AscendCommonAttentionMetadata":
# This only use to eagle now. It will be use to enforce_eager in future.
# Helper to slice optional per-request tensors to ``num_actual_reqs``.
def _slice_reqs(x):
return x[:num_actual_reqs] if x is not None else None
return AscendCommonAttentionMetadata(
query_start_loc=self.query_start_loc[: num_actual_reqs + 1],
query_start_loc_cpu=self.query_start_loc_cpu[: num_actual_reqs + 1],
seq_lens=self.seq_lens[:num_actual_reqs],
seq_lens_cpu=_slice_reqs(self.seq_lens_cpu),
num_computed_tokens_cpu=_slice_reqs(self.num_computed_tokens_cpu),
num_reqs=num_actual_reqs,
num_actual_tokens=num_actual_tokens,
max_query_len=self.max_query_len,
decode_token_per_req=self.decode_token_per_req,
# NOTE: keep all tokens for block_table_tensor and slot_mapping otherwise
# there will be error about shape mismatch during reshape and cache.
# This is really strange since vLLM slices them as well
block_table_tensor=self.block_table_tensor,
slot_mapping=self.slot_mapping,
causal=self.causal,
actual_seq_lengths_q=self.actual_seq_lengths_q[:num_actual_tokens],
positions=self.positions,
positions_cpu=self.positions_cpu,
attn_state=self.attn_state,
graph_pad_size=-1, # It should be -1 when not run in fullgraph mode.
num_input_tokens=self.num_input_tokens,
prefill_context_parallel_metadata=self.prefill_context_parallel_metadata,
seq_lens_cpu_upper_bound=self.seq_lens_cpu_upper_bound[:num_actual_reqs]
if self.seq_lens_cpu_upper_bound is not None
else None,
max_seq_len=self.max_seq_len,
# Propagate parent-class fields so the unpadded view is a
# faithful sub-batch of the original. Missing any of these
# would silently break downstream consumers (e.g. NPU
# backends preferring ``_seq_lens_cpu`` over ``seq_lens_cpu``,
# DCP backends needing ``dcp_local_seq_lens(_cpu)``,
# encoder-decoder layers needing ``encoder_seq_lens``, the
# mamba ``is_prefilling`` flag, and FastPrefill's
# ``logits_indices_padded`` / ``num_logits_indices``).
_seq_lens_cpu=_slice_reqs(self._seq_lens_cpu),
_num_computed_tokens_cpu=_slice_reqs(self._num_computed_tokens_cpu),
dcp_local_seq_lens=_slice_reqs(self.dcp_local_seq_lens),
dcp_local_seq_lens_cpu=_slice_reqs(self.dcp_local_seq_lens_cpu),
is_prefilling=_slice_reqs(self.is_prefilling),
encoder_seq_lens=_slice_reqs(self.encoder_seq_lens),
encoder_seq_lens_cpu=_slice_reqs(self.encoder_seq_lens_cpu),
logits_indices_padded=self.logits_indices_padded,
num_logits_indices=self.num_logits_indices,
)
def filter_chunked_req_indices(
seq_len: torch.Tensor,
mask_for_non_zero_chunk: list[bool] | None,
) -> torch.Tensor:
"""
filter the reqs which are doing real chunk_prefill.
Args:
seq_len: contains multi-req length: [req0_len, req1_len, ...]
mask_for_non_zero_chunk: [True, False, True, False, ...]
Returns:
filtered_indices: the real chunked req's indices
"""
assert mask_for_non_zero_chunk is not None and len(seq_len) == len(mask_for_non_zero_chunk)
offsets = torch.cumsum(torch.cat([torch.tensor([0]), seq_len[:-1]]), dim=0)
filtered_indices = torch.cat(
[
torch.arange(offsets[i], offsets[i] + seq_len[i])
for i in range(len(mask_for_non_zero_chunk))
if mask_for_non_zero_chunk[i]
]
)
return filtered_indices
def split_decodes_and_prefills(
common_attn_metadata: AscendCommonAttentionMetadata,
decode_threshold: int = 1,
require_uniform: bool = False,
treat_short_extends_as_decodes: bool = True,
) -> tuple[int, int, int, int]:
"""
Assuming a reordered batch, finds the boundary between prefill and decode
requests.
While pcp > 1, query_lens is split across pcp ranks, so we pass in the
original query_lens and max_query_len to distinguish prefills and decodes.
The batch is expected to be ordered as:
decode -> short_extend -> long_extend -> prefill
Args:
common_attn_metadata: AscendCommonAttentionMetadata object containing the
batch metadata.
decode_threshold: The maximum query length to be considered a decode.
require_uniform: If True, requires that all decode requests have the
same query length. When set, some queries may be considered
prefills even if they are <= decode_threshold, in order to ensure
uniformity.
treat_short_extends_as_decodes: If True (default), short extends
(query_len <= threshold but still prefilling) are counted as
decodes. If False, they are counted as prefills.
Returns:
num_decodes: The number of decode requests.
@@ -83,22 +367,57 @@ def split_decodes_and_prefills(
num_decode_tokens: The number of tokens in the decode requests.
num_prefill_tokens: The number of tokens in the prefill requests.
"""
max_query_len = common_attn_metadata.max_query_len
long_seq_metadata = common_attn_metadata.prefill_context_parallel_metadata
query_lens_pcp_full = long_seq_metadata.query_lens_pcp_full_cpu if long_seq_metadata else None
max_query_len_pcp_full = long_seq_metadata.max_query_len_pcp_full if long_seq_metadata else 0
max_query_len = common_attn_metadata.max_query_len if max_query_len_pcp_full == 0 else max_query_len_pcp_full
num_reqs = common_attn_metadata.num_reqs
if num_reqs == 0:
return 0, 0, 0, 0
num_tokens = common_attn_metadata.num_actual_tokens
query_start_loc = common_attn_metadata.query_start_loc_cpu
if max_query_len <= decode_threshold:
# PD D + RecomputeScheduler: num_computed may be N-1 after KV recv while
# this step is MTP decode (max_query_len <= threshold).
if is_pd_decode_recompute_scheduler_enabled():
treat_short_extends_as_decodes = True
if (
max_query_len <= decode_threshold
and (not require_uniform or decode_threshold <= 1)
and treat_short_extends_as_decodes
):
return num_reqs, 0, num_tokens, 0
query_lens = query_start_loc[1:] - query_start_loc[:-1]
is_prefill = query_lens > decode_threshold
query_lens_sharded = query_start_loc[1:] - query_start_loc[:-1]
query_lens = query_lens_sharded if query_lens_pcp_full is None else query_lens_pcp_full
if query_lens[0].item() > decode_threshold:
return 0, num_reqs, 0, num_tokens
if require_uniform:
if torch.all((query_lens == query_lens[0]) | (query_lens == 0)):
return num_reqs, 0, num_tokens, 0
is_prefill = query_lens != query_lens[0]
else:
is_prefill = query_lens > decode_threshold
if not treat_short_extends_as_decodes:
assert common_attn_metadata.is_prefilling is not None
raw_is_prefilling = common_attn_metadata.is_prefilling
is_prefilling = raw_is_prefilling[: query_lens.shape[0]]
if is_prefilling.shape[0] < query_lens.shape[0]:
is_prefilling = F.pad(
is_prefilling,
(0, query_lens.shape[0] - is_prefilling.shape[0]),
value=False,
)
is_prefill |= is_prefilling
if not torch.any(is_prefill):
return num_reqs, 0, num_tokens, 0
first_prefill = is_prefill.int().argmax(dim=-1).item()
assert torch.all(query_lens[first_prefill:] >= decode_threshold)
assert torch.all(query_lens[:first_prefill] <= decode_threshold)
num_decodes = first_prefill
num_prefills = num_reqs - num_decodes
num_decode_tokens = query_start_loc[first_prefill].item()
@@ -122,7 +441,7 @@ def wait_for_kv_layer_from_connector(layer_name: str):
def maybe_save_kv_layer_to_connector(
layer_name: str,
kv_cache_layer: List[torch.Tensor],
kv_cache_layer: list[torch.Tensor],
):
if not has_kv_transfer_group() or not is_v1_kv_transfer_group():
return
@@ -135,3 +454,68 @@ def maybe_save_kv_layer_to_connector(
return
# TODO: assert ascendMetadata
connector.save_kv_layer(layer_name, kv_cache_layer, attn_metadata)
def notify_kv_cache_written(layer_name: str = ""):
"""Notify the connector that the paged KV cache for ``layer_name`` has been
written for the current step.
The attention layer calls this unconditionally; each connector decides whether
it needs to record a synchronization primitive (e.g. a compute-stream event
later waited on by the resharding stream to overlap the outgoing KV copy).
Connectors that don't need it -- such as the AscendStore pool connector, which
records its own sync event at save time -- simply do not implement
``on_kv_cache_written`` and this becomes a no-op.
"""
if not has_kv_transfer_group() or not is_v1_kv_transfer_group():
return
connector = get_kv_transfer_group()
on_kv_cache_written = getattr(connector, "on_kv_cache_written", None)
if on_kv_cache_written is not None:
on_kv_cache_written(layer_name)
def round_up(val: int, align: int) -> int:
if align == 0:
return 0
return -(val // -align) * align
def trans_rope_weight(weight, rope_dim):
if rope_dim == 0:
return weight.contiguous()
nope_part = weight[..., :-rope_dim, :]
rope_part = weight[..., -rope_dim:, :]
reordered_rope_part = torch.cat((rope_part[..., ::2, :], rope_part[..., 1::2, :]), dim=-2)
return torch.cat((nope_part, reordered_rope_part), dim=-2).contiguous()
def transdata(nd_mat, block_size: tuple = (16, 16)):
r = round_up(nd_mat.shape[0], block_size[0])
c = round_up(nd_mat.shape[1], block_size[1])
r_pad = r - nd_mat.shape[0]
c_pad = c - nd_mat.shape[1]
nd_mat = F.pad(nd_mat, (0, r_pad, 0, c_pad))
nz_mat = torch.permute(
torch.reshape(
nd_mat,
(r // block_size[0], block_size[0], c // block_size[1], block_size[1]),
),
[2, 0, 1, 3],
)
nz_mat = torch.reshape(nz_mat, (nz_mat.shape[0], nz_mat.shape[1] * nz_mat.shape[2], nz_mat.shape[3]))
return nz_mat
def enabling_mlapo(vllm_config: VllmConfig) -> bool:
config_val = get_ascend_config().enable_mlapo
if get_ascend_device_type() == AscendDeviceType.A5:
return bool(config_val)
is_decode_instance = (
vllm_config.kv_transfer_config is not None
and vllm_config.kv_transfer_config.is_kv_consumer
and not vllm_config.kv_transfer_config.is_kv_producer
)
return bool(config_val and is_decode_instance)