/**  * Copyright (c) 2026 Huawei Technologies Co., Ltd.  * This program is free software, you can redistribute it and/or modify it under the terms and conditions of  * CANN Open Software License Agreement Version 2.0 (the "License").  * Please refer to the License for details. You may not use this file except in compliance with the License.  * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,  * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.  * See LICENSE in the root of the software repository for the full text of the License.  */ /*! * \file rope.h * \brief */ #ifndef ROPE_H #define ROPE_H #include "compressor_comm.h" #include "compressor_vector_comm.h" namespace Compressor { /** * @brief SetGatherSrcOffset 计算用于interleave模式的offset * @param gatherOffsetLocal 输出tensor [count],数据类型需要为int64_t,使用时要转换 * @param count offset的元素个数,一般为列数 */ template __aicore__ inline void SetGatherSrcOffset(const LocalTensor &gatherOffsetLocal, uint32_t count) { for (uint32_t i = 0; i < 8; i++) { gatherOffsetLocal.SetValue(i, i ^ 1); } event_t eventId_S_V = static_cast(GetTPipePtr()->FetchEventID(HardEvent::S_V)); SetFlag(eventId_S_V); WaitFlag(eventId_S_V); int32_t scalarValue = 8; while (scalarValue < count) { int32_t nextValue = scalarValue * 2; PipeBarrier(); if (nextValue < count) { Adds(gatherOffsetLocal[scalarValue], gatherOffsetLocal, scalarValue, scalarValue); } else { Adds(gatherOffsetLocal[scalarValue], gatherOffsetLocal, scalarValue, count - scalarValue); break; } scalarValue = nextValue; } PipeBarrier(); Muls(gatherOffsetLocal, gatherOffsetLocal, static_cast(sizeof(T)), count); } /** * @brief RotaryPosEmb 同时做row行的RotaryPosEmb,每一行的元素为col * @param dstLocal 输出tensor [row, actualCol],支持和srcLocal是同一块空间 * @param srcLocal 输入tensor [row, actualCol] * @param cosLocal cos系数tensor [row, col] * @param sinLocal sin系数tensor [row, col] * @param shareTmpUb 临时buffer 内部需要的空间为 [row * col * sizeof(float)] * @param gatherOffsetcastLocal 用于interleave模式的offset,数据类型需要为uint64_t * @param row 待处理的行数 * @param col 待处理的列数 * @param actualCol 实际列数 * @param baseAddr 计算基地址 */ template __aicore__ inline void RotaryPosEmb(const LocalTensor &dstLocal, const LocalTensor &srcLocal, const LocalTensor &cosLocal, const LocalTensor &sinLocal, const LocalTensor &shareTmpUb, const LocalTensor &gatherOffsetcastLocal, uint32_t row, uint32_t col, uint32_t actualCol, uint64_t baseAddr) { uint64_t cnt = row * col; uint32_t half_col = col >> 1; uint64_t rsvdCnt = 0; LocalTensor reArrLocal = shareTmpUb.ReinterpretCast(); if constexpr (MODE == ROTARY_MODE::HALF) { DataCopy(reArrLocal, srcLocal[baseAddr + half_col], {static_cast(row), static_cast(CeilDivT(half_col, FP32_BLOCK_ELEMENT_NUM)), static_cast(CeilDivT(actualCol - half_col, FP32_BLOCK_ELEMENT_NUM)), static_cast(CeilDivT(half_col, FP32_BLOCK_ELEMENT_NUM))}); DataCopy(reArrLocal[half_col], srcLocal[baseAddr], {static_cast(row), static_cast(CeilDivT(half_col, FP32_BLOCK_ELEMENT_NUM)), static_cast(CeilDivT(actualCol - half_col, FP32_BLOCK_ELEMENT_NUM)), static_cast(CeilDivT(half_col, FP32_BLOCK_ELEMENT_NUM))}); PipeBarrier(); Muls(reArrLocal, reArrLocal, float(-1), half_col, row, {1, 1, static_cast(CeilDivT(static_cast(col), FP32_BLOCK_ELEMENT_NUM)), static_cast(CeilDivT(static_cast(col), FP32_BLOCK_ELEMENT_NUM))}); } else if constexpr (MODE == ROTARY_MODE::INTERLEAVE) { for (uint32_t i = 0; i < row; i++) { Gather(reArrLocal[i * col], srcLocal[i * actualCol + baseAddr], gatherOffsetcastLocal, 0, col); } PipeBarrier(); uint32_t repeatTimes = cnt / FP32_REPEAT_ELEMENT_NUM; uint32_t remainder = cnt % FP32_REPEAT_ELEMENT_NUM; uint64_t fullMask = 0x5555555555555555; uint64_t partialMask = 0x55; SetVectorMask(0, fullMask); Muls(reArrLocal, reArrLocal, float(-1), MASK_PLACEHOLDER, repeatTimes, {1, 1, FP32_BLOCK_ELEMENT_NUM, FP32_BLOCK_ELEMENT_NUM}); if (unlikely(remainder > 0)) { SetVectorMask(0, partialMask); Muls(reArrLocal[repeatTimes * FP32_REPEAT_ELEMENT_NUM], reArrLocal[repeatTimes * FP32_REPEAT_ELEMENT_NUM], float(-1), MASK_PLACEHOLDER, remainder / FP32_BLOCK_ELEMENT_NUM, {1, 1, 1, 1}); } ResetMask(); } PipeBarrier(); BinaryRepeatParams computeParams{1, 1, 1, static_cast(CeilDivT(actualCol, FP32_BLOCK_ELEMENT_NUM)), static_cast(CeilDivT(actualCol, FP32_BLOCK_ELEMENT_NUM)), static_cast(CeilDivT(col, FP32_BLOCK_ELEMENT_NUM))}; Mul(dstLocal[baseAddr], srcLocal[baseAddr], cosLocal, col, row, computeParams); Mul(reArrLocal, reArrLocal, sinLocal, cnt); PipeBarrier(); Add(dstLocal[baseAddr], dstLocal[baseAddr], reArrLocal, col, row, computeParams); } } #endif