# Adapt from https://github.com/vllm-project/vllm/blob/main/vllm/v1/worker/gpu/input_batch.py # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project # Copyright (c) 2025 Huawei Technologies Co., Ltd. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # This file is a part of the vllm-ascend project. # from dataclasses import asdict, dataclass import numpy as np import torch from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers class AscendInputBuffers(InputBuffers): """Input buffers for Ascend NPUs.""" def __init__( self, max_num_reqs: int, max_num_tokens: int, device: torch.device, ): super().__init__( max_num_reqs, max_num_tokens, device, ) # Create seq_lens_cpu and seq_lens_np. # npu's attention backend still needs seq_lens on CPU side. self.seq_lens_cpu: torch.Tensor = torch.zeros( max_num_reqs, dtype=torch.int32, device="cpu", ) # seq_len_np and seq_lens_cpu share the same memory. # define seq_lens_np for easier calculation with numpy. self.seq_lens_np: np.ndarray = self.seq_lens_cpu.numpy() @dataclass class AscendInputBatch(InputBatch): """Input batch for Ascend NPUs.""" # Create seq_lens_np. # npu's attention backend still needs seq_lens on CPU side. seq_lens_np: np.ndarray @classmethod def make_dummy( cls, num_reqs: int, num_tokens: int, input_buffers: AscendInputBuffers, device: torch.device, ) -> "AscendInputBatch": """Override the make_dummy method to calculate seq_lens_np.""" input_batch = InputBatch.make_dummy( num_reqs, num_tokens, input_buffers, device, ) # seq_len equals to query_len input_buffers.seq_lens_np[:num_reqs] = num_tokens // num_reqs input_buffers.seq_lens_np[num_reqs - 1] += num_tokens % num_reqs # Pad for full CUDA graph mode. input_buffers.seq_lens_np[num_reqs:] = 0 seq_lens_np = input_buffers.seq_lens_np[:num_reqs] input_batch.seq_lens_np = seq_lens_np return cls(**asdict(input_batch), seq_lens_np=seq_lens_np)