# Adapt from https://github.com/vllm-project/vllm/blob/main/vllm/v1/worker/gpu/input_batch.py # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project # Copyright (c) 2025 Huawei Technologies Co., Ltd. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # This file is a part of the vllm-ascend project. # import numpy as np import torch from vllm.v1.worker.gpu.input_batch import InputBuffers class AscendInputBuffers(InputBuffers): """Input buffers for Ascend NPUs.""" def __init__( self, max_num_reqs: int, max_num_tokens: int, inputs_embeds_size: int, vocab_size: int, dtype: torch.dtype, device: torch.device, pin_memory: bool, ): super().__init__( max_num_reqs, max_num_tokens, inputs_embeds_size, vocab_size, dtype, device, pin_memory, ) # Create seq_lens_cpu and seq_lens_np. # npu's attention backend still needs seq_lens on CPU side. self.seq_lens_cpu: torch.Tensor = torch.zeros( max_num_reqs, dtype=torch.int32, device="cpu", ) # seq_len_np and seq_lens_cpu share the same memory. # define seq_lens_np for easier calculation with numpy. self.seq_lens_np: np.ndarray = self.seq_lens_cpu.numpy()