EngineX-Ascend/enginex-ascend-910-vllm
11
0
Fork 1
Code Issues Pull Requests Actions Projects Releases Wiki Activity

init v0.23.0

Browse Source 
Signed-off-by: Sun Ruoxi <sunruoxi@4paradigm.com>
This commit is contained in:
Sun Ruoxi
2026-08-27 15:11:51 +08:00
parent b582a8e7d1
commit 7f8a1b1f7a
2849 changed files with 712887 additions and 22001 deletions
Download Patch File Download Diff File Expand all files Collapse all files

433
csrc/moe/dequant_swiglu_quant/op_kernel/dequant_swiglu_quant.cpp Normal file
View File

@@ -0,0 +1,433 @@
/**
* Copyright (c) 2025 Huawei Technologies Co., Ltd.
* This program is free software, you can redistribute it and/or modify it under the terms and conditions of
* CANN Open Software License Agreement Version 2.0 (the "License").
* Please refer to the License for details. You may not use this file except in compliance with the License.
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
* INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
* See LICENSE in the root of the software repository for the full text of the License.
*/
/*!
* \file dequant_swiglu_quant.cpp
* \brief
*/
#include "kernel_operator.h"
#include "kernel_tiling/kernel_tiling.h"
#if (ORIG_DTYPE_X == DT_INT32) || (ORIG_DTYPE_X == DT_BF16)
#include "dequant_swiglu_quant.h"
#include "dequant_swiglu_quant_cut_group.h"
#endif
#include "dequant_swiglu_quant_static_bf16.hpp"
#include "dequant_swiglu_quant_static_bias_int32.hpp"
#include "dequant_swiglu_quant_static_bias_float.hpp"
#include "dequant_swiglu_quant_dynamic_bf16.hpp"
#include "dequant_swiglu_quant_dynamic_bias_int32.hpp"
#include "dequant_swiglu_quant_dynamic_bias_float.hpp"
#include "dequant_swiglu_quant_dynamic_performance.hpp"
using namespace AscendC;
// DEQUANT_SWIGLU_QUANT_WITH_GROUP_FP32_QS HAS_GROUP(100000000) + QS_OFFSET(100) * QS_FP32(0)
// DEQUANT_SWIGLU_QUANT_WITH_GROUP_FP16_QS HAS_GROUP(100000000) + QS_OFFSET(100) * QS_FP16(1)
// DEQUANT_SWIGLU_QUANT_WITH_GROUP_BF16_QS HAS_GROUP(100000000) + QS_OFFSET(100) * QS_BF16(2)
// DEQUANT_SWIGLU_QUANT_WITHOUT_GROUP_FP32_QS NO_GROUP(200000000) + QS_OFFSET(100) * QS_FP32(0)
// DEQUANT_SWIGLU_QUANT_WITHOUT_GROUP_FP16_QS NO_GROUP(200000000) + QS_OFFSET(100) * QS_FP16(1)
// DEQUANT_SWIGLU_QUANT_WITHOUT_GROUP_BF16_QS NO_GROUP(200000000) + QS_OFFSET(100) * QS_BF16(2)
#define DEQUANT_SWIGLU_QUANT_WITH_GROUP_BF16_BIAS_FP32_QS 100000000
#define DEQUANT_SWIGLU_QUANT_WITH_GROUP_NO_BIAS_FP32_QS DEQUANT_SWIGLU_QUANT_WITH_GROUP_BF16_BIAS_FP32_QS
#define DEQUANT_SWIGLU_QUANT_WITH_GROUP_BF16_BIAS_FP16_QS 100000100
#define DEQUANT_SWIGLU_QUANT_WITH_GROUP_NO_BIAS_FP16_QS DEQUANT_SWIGLU_QUANT_WITH_GROUP_BF16_BIAS_FP16_QS
#define DEQUANT_SWIGLU_QUANT_WITH_GROUP_BF16_BIAS_BF16_QS 100000200
#define DEQUANT_SWIGLU_QUANT_WITH_GROUP_NO_BIAS_BF16_QS DEQUANT_SWIGLU_QUANT_WITH_GROUP_BF16_BIAS_BF16_QS
#define DEQUANT_SWIGLU_QUANT_WITH_GROUP_FP16_BIAS_FP32_QS 100001000
#define DEQUANT_SWIGLU_QUANT_WITH_GROUP_FP16_BIAS_FP16_QS 100001100
#define DEQUANT_SWIGLU_QUANT_WITH_GROUP_FP16_BIAS_BF16_QS 100001200
#define DEQUANT_SWIGLU_QUANT_WITH_GROUP_FP32_BIAS_FP32_QS 100002000
#define DEQUANT_SWIGLU_QUANT_WITH_GROUP_FP32_BIAS_FP16_QS 100002100
#define DEQUANT_SWIGLU_QUANT_WITH_GROUP_FP32_BIAS_BF16_QS 100002200
#define DEQUANT_SWIGLU_QUANT_WITH_GROUP_INT32_BIAS_FP32_QS 100003000
#define DEQUANT_SWIGLU_QUANT_WITH_GROUP_INT32_BIAS_FP16_QS 100003100
#define DEQUANT_SWIGLU_QUANT_WITH_GROUP_INT32_BIAS_BF16_QS 100003200
#define DEQUANT_SWIGLU_QUANT_WITHOUT_GROUP_FP32_QS 200000000
#define DEQUANT_SWIGLU_QUANT_WITHOUT_GROUP_FP16_QS 200000100
#define DEQUANT_SWIGLU_QUANT_WITHOUT_GROUP_BF16_QS 200000200
// cut by groupnum
#define DEQUANT_SWIGLU_QUANT_WITH_GROUP_FP32_QS_GR 110000000
#define DEQUANT_SWIGLU_QUANT_WITH_GROUP_FP16_QS_GR 110000100
#define DEQUANT_SWIGLU_QUANT_WITH_GROUP_BF16_QS_GR 110000200
extern "C" __global__ __aicore__ void dequant_swiglu_quant(GM_ADDR xGM, GM_ADDR weightSscaleGM,
GM_ADDR activationScaleGM, GM_ADDR biasGM,
GM_ADDR quantScaleGM, GM_ADDR quantOffsetGM,
GM_ADDR groupIndex, GM_ADDR yGM, GM_ADDR scaleGM,
GM_ADDR workspace, GM_ADDR tiling)
{
if (workspace == nullptr) {
return;
}
GM_ADDR userspace = GetUserWorkspace(workspace);
if (userspace == nullptr) {
return;
}
TPipe pipe;
#if (ORIG_DTYPE_X == DT_INT32)
if (TILING_KEY_IS(DEQUANT_SWIGLU_QUANT_WITH_GROUP_BF16_BIAS_FP32_QS)) {
#if !(defined(__NPU_ARCH__) && (__NPU_ARCH__ == 3003 || __NPU_ARCH__ == 3113))
GET_TILING_DATA_WITH_STRUCT(DequantSwigluQuantBaseTilingData, tilingDataIn, tiling);
const DequantSwigluQuantBaseTilingData* __restrict__ tilingData = &tilingDataIn;
DequantSwigluQuantOps::DequantSwigluQuantBase<bfloat16_t, float, int64_t, int32_t> op(&pipe);
op.Init(xGM, weightSscaleGM, activationScaleGM, biasGM, quantScaleGM, quantOffsetGM, groupIndex, yGM, scaleGM,
tilingData);
op.Process();
#endif
} else if (TILING_KEY_IS(DEQUANT_SWIGLU_QUANT_WITH_GROUP_BF16_BIAS_FP16_QS)) {
#if !(defined(__NPU_ARCH__) && (__NPU_ARCH__ == 3003 || __NPU_ARCH__ == 3113))
GET_TILING_DATA_WITH_STRUCT(DequantSwigluQuantBaseTilingData, tilingDataIn, tiling);
const DequantSwigluQuantBaseTilingData* __restrict__ tilingData = &tilingDataIn;
DequantSwigluQuantOps::DequantSwigluQuantBase<bfloat16_t, half, int64_t, int32_t> op(&pipe);
op.Init(xGM, weightSscaleGM, activationScaleGM, biasGM, quantScaleGM, quantOffsetGM, groupIndex, yGM, scaleGM,
tilingData);
op.Process();
#endif
} else if (TILING_KEY_IS(DEQUANT_SWIGLU_QUANT_WITH_GROUP_BF16_BIAS_BF16_QS)) {
#if !(defined(__NPU_ARCH__) && (__NPU_ARCH__ == 3003 || __NPU_ARCH__ == 3113))
GET_TILING_DATA_WITH_STRUCT(DequantSwigluQuantBaseTilingData, tilingDataIn, tiling);
const DequantSwigluQuantBaseTilingData* __restrict__ tilingData = &tilingDataIn;
DequantSwigluQuantOps::DequantSwigluQuantBase<bfloat16_t, bfloat16_t, int64_t, int32_t> op(&pipe);
op.Init(xGM, weightSscaleGM, activationScaleGM, biasGM, quantScaleGM, quantOffsetGM, groupIndex, yGM, scaleGM,
tilingData);
op.Process();
#endif
} else if (TILING_KEY_IS(DEQUANT_SWIGLU_QUANT_WITH_GROUP_FP16_BIAS_FP32_QS)) {
GET_TILING_DATA_WITH_STRUCT(DequantSwigluQuantBaseTilingData, tilingDataIn, tiling);
const DequantSwigluQuantBaseTilingData* __restrict__ tilingData = &tilingDataIn;
DequantSwigluQuantOps::DequantSwigluQuantBase<half, float, int64_t, int32_t> op(&pipe);
op.Init(xGM, weightSscaleGM, activationScaleGM, biasGM, quantScaleGM, quantOffsetGM, groupIndex, yGM, scaleGM,
tilingData);
op.Process();
} else if (TILING_KEY_IS(DEQUANT_SWIGLU_QUANT_WITH_GROUP_FP16_BIAS_FP16_QS)) {
GET_TILING_DATA_WITH_STRUCT(DequantSwigluQuantBaseTilingData, tilingDataIn, tiling);
const DequantSwigluQuantBaseTilingData* __restrict__ tilingData = &tilingDataIn;
DequantSwigluQuantOps::DequantSwigluQuantBase<half, half, int64_t, int32_t> op(&pipe);
op.Init(xGM, weightSscaleGM, activationScaleGM, biasGM, quantScaleGM, quantOffsetGM, groupIndex, yGM, scaleGM,
tilingData);
op.Process();
} else if (TILING_KEY_IS(DEQUANT_SWIGLU_QUANT_WITH_GROUP_FP16_BIAS_BF16_QS)) {
#if !(defined(__NPU_ARCH__) && (__NPU_ARCH__ == 3003 || __NPU_ARCH__ == 3113))
GET_TILING_DATA_WITH_STRUCT(DequantSwigluQuantBaseTilingData, tilingDataIn, tiling);
const DequantSwigluQuantBaseTilingData* __restrict__ tilingData = &tilingDataIn;
DequantSwigluQuantOps::DequantSwigluQuantBase<half, bfloat16_t, int64_t, int32_t> op(&pipe);
op.Init(xGM, weightSscaleGM, activationScaleGM, biasGM, quantScaleGM, quantOffsetGM, groupIndex, yGM, scaleGM,
tilingData);
op.Process();
#endif
} else if (TILING_KEY_IS(DEQUANT_SWIGLU_QUANT_WITH_GROUP_FP32_BIAS_FP32_QS)) {
GET_TILING_DATA_WITH_STRUCT(DequantSwigluQuantBaseTilingData, tilingDataIn, tiling);
const DequantSwigluQuantBaseTilingData* __restrict__ tilingData = &tilingDataIn;
DequantSwigluQuantOps::DequantSwigluQuantBase<float, float, int64_t, int32_t> op(&pipe);
op.Init(xGM, weightSscaleGM, activationScaleGM, biasGM, quantScaleGM, quantOffsetGM, groupIndex, yGM, scaleGM,
tilingData);
op.Process();
} else if (TILING_KEY_IS(DEQUANT_SWIGLU_QUANT_WITH_GROUP_FP32_BIAS_FP16_QS)) {
GET_TILING_DATA_WITH_STRUCT(DequantSwigluQuantBaseTilingData, tilingDataIn, tiling);
const DequantSwigluQuantBaseTilingData* __restrict__ tilingData = &tilingDataIn;
DequantSwigluQuantOps::DequantSwigluQuantBase<float, half, int64_t, int32_t> op(&pipe);
op.Init(xGM, weightSscaleGM, activationScaleGM, biasGM, quantScaleGM, quantOffsetGM, groupIndex, yGM, scaleGM,
tilingData);
op.Process();
} else if (TILING_KEY_IS(DEQUANT_SWIGLU_QUANT_WITH_GROUP_FP32_BIAS_BF16_QS)) {
#if !(defined(__NPU_ARCH__) && (__NPU_ARCH__ == 3003 || __NPU_ARCH__ == 3113))
GET_TILING_DATA_WITH_STRUCT(DequantSwigluQuantBaseTilingData, tilingDataIn, tiling);
const DequantSwigluQuantBaseTilingData* __restrict__ tilingData = &tilingDataIn;
DequantSwigluQuantOps::DequantSwigluQuantBase<float, bfloat16_t, int64_t, int32_t> op(&pipe);
op.Init(xGM, weightSscaleGM, activationScaleGM, biasGM, quantScaleGM, quantOffsetGM, groupIndex, yGM, scaleGM,
tilingData);
op.Process();
#endif
} else if (TILING_KEY_IS(DEQUANT_SWIGLU_QUANT_WITH_GROUP_INT32_BIAS_FP32_QS)) {
GET_TILING_DATA_WITH_STRUCT(DequantSwigluQuantBaseTilingData, tilingDataIn, tiling);
const DequantSwigluQuantBaseTilingData* __restrict__ tilingData = &tilingDataIn;
DequantSwigluQuantOps::DequantSwigluQuantBase<int32_t, float, int64_t, int32_t> op(&pipe);
op.Init(xGM, weightSscaleGM, activationScaleGM, biasGM, quantScaleGM, quantOffsetGM, groupIndex, yGM, scaleGM,
tilingData);
op.Process();
} else if (TILING_KEY_IS(DEQUANT_SWIGLU_QUANT_WITH_GROUP_INT32_BIAS_FP16_QS)) {
GET_TILING_DATA_WITH_STRUCT(DequantSwigluQuantBaseTilingData, tilingDataIn, tiling);
const DequantSwigluQuantBaseTilingData* __restrict__ tilingData = &tilingDataIn;
DequantSwigluQuantOps::DequantSwigluQuantBase<int32_t, half, int64_t, int32_t> op(&pipe);
op.Init(xGM, weightSscaleGM, activationScaleGM, biasGM, quantScaleGM, quantOffsetGM, groupIndex, yGM, scaleGM,
tilingData);
op.Process();
} else if (TILING_KEY_IS(DEQUANT_SWIGLU_QUANT_WITH_GROUP_INT32_BIAS_BF16_QS)) {
#if !(defined(__NPU_ARCH__) && (__NPU_ARCH__ == 3003 || __NPU_ARCH__ == 3113))
GET_TILING_DATA_WITH_STRUCT(DequantSwigluQuantBaseTilingData, tilingDataIn, tiling);
const DequantSwigluQuantBaseTilingData* __restrict__ tilingData = &tilingDataIn;
DequantSwigluQuantOps::DequantSwigluQuantBase<int32_t, bfloat16_t, int64_t, int32_t> op(&pipe);
op.Init(xGM, weightSscaleGM, activationScaleGM, biasGM, quantScaleGM, quantOffsetGM, groupIndex, yGM, scaleGM,
tilingData);
op.Process();
#endif
} else if (TILING_KEY_IS(DEQUANT_SWIGLU_QUANT_WITHOUT_GROUP_FP32_QS)) {
GET_TILING_DATA_WITH_STRUCT(DequantSwigluQuantBaseTilingData, tilingDataIn, tiling);
const DequantSwigluQuantBaseTilingData* __restrict__ tilingData = &tilingDataIn;
// DTYPE_GROUP_INDEX == float mean have no groupIndex
DequantSwigluQuantOps::DequantSwigluQuantBase<float, float, float, int32_t> op(&pipe);
op.Init(xGM, weightSscaleGM, activationScaleGM, nullptr, quantScaleGM, nullptr, groupIndex, yGM, scaleGM,
tilingData);
op.Process();
} else if (TILING_KEY_IS(DEQUANT_SWIGLU_QUANT_WITHOUT_GROUP_FP16_QS)) {
GET_TILING_DATA_WITH_STRUCT(DequantSwigluQuantBaseTilingData, tilingDataIn, tiling);
const DequantSwigluQuantBaseTilingData* __restrict__ tilingData = &tilingDataIn;
// DTYPE_GROUP_INDEX == float mean have no groupIndex
DequantSwigluQuantOps::DequantSwigluQuantBase<float, half, float, int32_t> op(&pipe);
op.Init(xGM, weightSscaleGM, activationScaleGM, nullptr, quantScaleGM, nullptr, groupIndex, yGM, scaleGM,
tilingData);
op.Process();
} else if (TILING_KEY_IS(DEQUANT_SWIGLU_QUANT_WITHOUT_GROUP_BF16_QS)) {
#if !(defined(__NPU_ARCH__) && (__NPU_ARCH__ == 3003 || __NPU_ARCH__ == 3113))
GET_TILING_DATA_WITH_STRUCT(DequantSwigluQuantBaseTilingData, tilingDataIn, tiling);
const DequantSwigluQuantBaseTilingData* __restrict__ tilingData = &tilingDataIn;
// DTYPE_GROUP_INDEX == float mean have no groupIndex
DequantSwigluQuantOps::DequantSwigluQuantBase<float, bfloat16_t, float, int32_t> op(&pipe);
op.Init(xGM, weightSscaleGM, activationScaleGM, nullptr, quantScaleGM, nullptr, groupIndex, yGM, scaleGM,
tilingData);
op.Process();
#endif
} else if (TILING_KEY_IS(DEQUANT_SWIGLU_QUANT_WITH_GROUP_FP32_QS_GR)) {
GET_TILING_DATA_WITH_STRUCT(DequantSwigluQuantBaseTilingData, tilingDataIn, tiling);
const DequantSwigluQuantBaseTilingData* __restrict__ tilingData = &tilingDataIn;
DequantSwigluQuantGroupOps::DequantSwigluQuantGroup<bfloat16_t, float, int64_t, int32_t> op(&pipe);
op.Init(xGM, weightSscaleGM, activationScaleGM, biasGM, quantScaleGM, quantOffsetGM, groupIndex, yGM, scaleGM, tilingData);
op.Process();
} else if (TILING_KEY_IS(DEQUANT_SWIGLU_QUANT_WITH_GROUP_FP16_QS_GR)) {
GET_TILING_DATA_WITH_STRUCT(DequantSwigluQuantBaseTilingData, tilingDataIn, tiling);
const DequantSwigluQuantBaseTilingData* __restrict__ tilingData = &tilingDataIn;
DequantSwigluQuantGroupOps::DequantSwigluQuantGroup<bfloat16_t, half, int64_t, int32_t> op(&pipe);
op.Init(xGM, weightSscaleGM, activationScaleGM, biasGM, quantScaleGM, quantOffsetGM, groupIndex, yGM, scaleGM, tilingData);
op.Process();
} else if (TILING_KEY_IS(DEQUANT_SWIGLU_QUANT_WITH_GROUP_BF16_QS_GR)) {
#if !(defined(__NPU_ARCH__) && (__NPU_ARCH__ == 3003 || __NPU_ARCH__ == 3113))
GET_TILING_DATA_WITH_STRUCT(DequantSwigluQuantBaseTilingData, tilingDataIn, tiling);
const DequantSwigluQuantBaseTilingData* __restrict__ tilingData = &tilingDataIn;
DequantSwigluQuantGroupOps::DequantSwigluQuantGroup<bfloat16_t, bfloat16_t, int64_t, int32_t> op(&pipe);
op.Init(xGM, weightSscaleGM, activationScaleGM, biasGM, quantScaleGM, quantOffsetGM, groupIndex, yGM, scaleGM, tilingData);
op.Process();
#endif
} else if (TILING_KEY_IS(10004)) {
// ORIG_DTYPE_BIAS == DT_INT32
GET_TILING_DATA_WITH_STRUCT(SwiGluTilingData, tilingDataIn, tiling);
const SwiGluTilingData* __restrict__ tilingData = &tilingDataIn;
DequantSwigluQuant::DequantSwigluQuantStaticBiasInt32<int32_t, float, int32_t, int8_t, 1, 1> op;
op.Init(xGM, weightSscaleGM, activationScaleGM, biasGM, quantScaleGM, quantOffsetGM, yGM, scaleGM, tilingData,
&(pipe));
op.Process();
} else if (TILING_KEY_IS(10005)) {
GET_TILING_DATA_WITH_STRUCT(SwiGluTilingData, tilingDataIn, tiling);
const SwiGluTilingData* __restrict__ tilingData = &tilingDataIn;
DequantSwigluQuant::DequantSwigluQuantStaticBiasInt32<int32_t, float, int32_t, int8_t, 1, 0> op;
op.Init(xGM, weightSscaleGM, activationScaleGM, biasGM, quantScaleGM, quantOffsetGM, yGM, scaleGM, tilingData,
&(pipe));
op.Process();
} else if (TILING_KEY_IS(30001)) {
GET_TILING_DATA_WITH_STRUCT(SwiGluTilingData, tilingDataIn, tiling);
const SwiGluTilingData* __restrict__ tilingData = &tilingDataIn;
DequantSwigluQuant::DequantSwigluQuantDynamicBiasInt32<int32_t, float, int32_t, int8_t, 1, 1> op;
op.Init(xGM, weightSscaleGM, activationScaleGM, biasGM, quantScaleGM, quantOffsetGM, yGM, scaleGM, userspace,
tilingData, &(pipe));
op.Process();
} else if (TILING_KEY_IS(30005)) {
GET_TILING_DATA_WITH_STRUCT(SwiGluTilingData, tilingDataIn, tiling);
const SwiGluTilingData* __restrict__ tilingData = &tilingDataIn;
DequantSwigluQuant::DequantSwigluQuantDynamicBiasInt32<int32_t, float, int32_t, int8_t, 1, 0> op;
op.Init(xGM, weightSscaleGM, activationScaleGM, biasGM, quantScaleGM, quantOffsetGM, yGM, scaleGM, userspace,
tilingData, &(pipe));
op.Process();
}
// ORIG_DTYPE_BIAS == DT_FLOAT16
else if (TILING_KEY_IS(10006)) {
GET_TILING_DATA_WITH_STRUCT(SwiGluTilingData, tilingDataIn, tiling);
const SwiGluTilingData* __restrict__ tilingData = &tilingDataIn;
DequantSwigluQuant::DequantSwigluQuantStaticBiasFloat<int32_t, float, half, int8_t, 1, 1> op;
op.Init(xGM, weightSscaleGM, activationScaleGM, biasGM, quantScaleGM, quantOffsetGM, yGM, scaleGM, tilingData,
&(pipe));
op.Process();
} else if (TILING_KEY_IS(10007)) {
GET_TILING_DATA_WITH_STRUCT(SwiGluTilingData, tilingDataIn, tiling);
const SwiGluTilingData* __restrict__ tilingData = &tilingDataIn;
DequantSwigluQuant::DequantSwigluQuantStaticBiasFloat<int32_t, float, half, int8_t, 1, 0> op;
op.Init(xGM, weightSscaleGM, activationScaleGM, biasGM, quantScaleGM, quantOffsetGM, yGM, scaleGM, tilingData,
&(pipe));
op.Process();
} else if (TILING_KEY_IS(30003)) {
GET_TILING_DATA_WITH_STRUCT(SwiGluTilingData, tilingDataIn, tiling);
const SwiGluTilingData* __restrict__ tilingData = &tilingDataIn;
DequantSwigluQuant::DequantSwigluQuantDynamicBiasFloat<int32_t, float, half, int8_t, 1, 1> op;
op.Init(xGM, weightSscaleGM, activationScaleGM, biasGM, quantScaleGM, quantOffsetGM, yGM, scaleGM, userspace,
tilingData, &(pipe));
op.Process();
} else if (TILING_KEY_IS(30007)) {
GET_TILING_DATA_WITH_STRUCT(SwiGluTilingData, tilingDataIn, tiling);
const SwiGluTilingData* __restrict__ tilingData = &tilingDataIn;
DequantSwigluQuant::DequantSwigluQuantDynamicBiasFloat<int32_t, float, half, int8_t, 1, 0> op;
op.Init(xGM, weightSscaleGM, activationScaleGM, biasGM, quantScaleGM, quantOffsetGM, yGM, scaleGM, userspace,
tilingData, &(pipe));
op.Process();
}
// ORIG_DTYPE_BIAS == DT_FLOAT
else if (TILING_KEY_IS(10008)) {
GET_TILING_DATA_WITH_STRUCT(SwiGluTilingData, tilingDataIn, tiling);
const SwiGluTilingData* __restrict__ tilingData = &tilingDataIn;
DequantSwigluQuant::DequantSwigluQuantStaticBiasFloat<int32_t, float, float, int8_t, 1, 1> op;
op.Init(xGM, weightSscaleGM, activationScaleGM, biasGM, quantScaleGM, quantOffsetGM, yGM, scaleGM, tilingData,
&(pipe));
op.Process();
} else if (TILING_KEY_IS(10009)) {
GET_TILING_DATA_WITH_STRUCT(SwiGluTilingData, tilingDataIn, tiling);
const SwiGluTilingData* __restrict__ tilingData = &tilingDataIn;
DequantSwigluQuant::DequantSwigluQuantStaticBiasFloat<int32_t, float, float, int8_t, 1, 0> op;
op.Init(xGM, weightSscaleGM, activationScaleGM, biasGM, quantScaleGM, quantOffsetGM, yGM, scaleGM, tilingData,
&(pipe));
op.Process();
} else if (TILING_KEY_IS(30002)) {
GET_TILING_DATA_WITH_STRUCT(SwiGluTilingData, tilingDataIn, tiling);
const SwiGluTilingData* __restrict__ tilingData = &tilingDataIn;
DequantSwigluQuant::DequantSwigluQuantDynamicBiasFloat<int32_t, float, float, int8_t, 1, 1> op;
op.Init(xGM, weightSscaleGM, activationScaleGM, biasGM, quantScaleGM, quantOffsetGM, yGM, scaleGM, userspace,
tilingData, &(pipe));
op.Process();
} else if (TILING_KEY_IS(30013)) {
GET_TILING_DATA_WITH_STRUCT(SwiGluTilingData, tilingDataIn, tiling);
const SwiGluTilingData* __restrict__ tilingData = &tilingDataIn;
DequantSwigluQuant::DequantSwigluQuantDynamicPerformance<int32_t, float, float, int8_t, 1, 0> op;
op.Init(xGM, weightSscaleGM, activationScaleGM, biasGM, quantScaleGM, quantOffsetGM, yGM, scaleGM, userspace,
tilingData, &(pipe));
op.Process();
} else if (TILING_KEY_IS(30006)) {
GET_TILING_DATA_WITH_STRUCT(SwiGluTilingData, tilingDataIn, tiling);
const SwiGluTilingData* __restrict__ tilingData = &tilingDataIn;
DequantSwigluQuant::DequantSwigluQuantDynamicBiasFloat<int32_t, float, float, int8_t, 1, 0> op;
op.Init(xGM, weightSscaleGM, activationScaleGM, biasGM, quantScaleGM, quantOffsetGM, yGM, scaleGM, userspace,
tilingData, &(pipe));
op.Process();
}
#if !(defined(__NPU_ARCH__) && (__NPU_ARCH__ == 3003 || __NPU_ARCH__ == 3113))
// ORIG_DTYPE_BIAS == DT_BF16
else if (TILING_KEY_IS(10010)) {
GET_TILING_DATA_WITH_STRUCT(SwiGluTilingData, tilingDataIn, tiling);
const SwiGluTilingData* __restrict__ tilingData = &tilingDataIn;
DequantSwigluQuant::DequantSwigluQuantStaticBiasFloat<int32_t, float, bfloat16_t, int8_t, 1, 1> op;
op.Init(xGM, weightSscaleGM, activationScaleGM, biasGM, quantScaleGM, quantOffsetGM, yGM, scaleGM, tilingData,
&(pipe));
op.Process();
} else if (TILING_KEY_IS(10011)) {
GET_TILING_DATA_WITH_STRUCT(SwiGluTilingData, tilingDataIn, tiling);
const SwiGluTilingData* __restrict__ tilingData = &tilingDataIn;
DequantSwigluQuant::DequantSwigluQuantStaticBiasFloat<int32_t, float, bfloat16_t, int8_t, 1, 0> op;
op.Init(xGM, weightSscaleGM, activationScaleGM, biasGM, quantScaleGM, quantOffsetGM, yGM, scaleGM, tilingData,
&(pipe));
op.Process();
} else if (TILING_KEY_IS(30004)) {
GET_TILING_DATA_WITH_STRUCT(SwiGluTilingData, tilingDataIn, tiling);
const SwiGluTilingData* __restrict__ tilingData = &tilingDataIn;
DequantSwigluQuant::DequantSwigluQuantDynamicBiasFloat<int32_t, float, bfloat16_t, int8_t, 1, 1> op;
op.Init(xGM, weightSscaleGM, activationScaleGM, biasGM, quantScaleGM, quantOffsetGM, yGM, scaleGM, userspace,
tilingData, &(pipe));
op.Process();
} else if (TILING_KEY_IS(30008)) {
GET_TILING_DATA_WITH_STRUCT(SwiGluTilingData, tilingDataIn, tiling);
const SwiGluTilingData* __restrict__ tilingData = &tilingDataIn;
DequantSwigluQuant::DequantSwigluQuantDynamicBiasFloat<int32_t, float, bfloat16_t, int8_t, 1, 0> op;
op.Init(xGM, weightSscaleGM, activationScaleGM, biasGM, quantScaleGM, quantOffsetGM, yGM, scaleGM, userspace,
tilingData, &(pipe));
op.Process();
}
#endif
#endif
#if (ORIG_DTYPE_X == DT_FLOAT16)
if (TILING_KEY_IS(10000)) {
GET_TILING_DATA_WITH_STRUCT(SwiGluTilingData, tilingDataIn, tiling);
const SwiGluTilingData* __restrict__ tilingData = &tilingDataIn;
DequantSwigluQuant::DequantSwigluQuantStaticBF16<half, float, half, int8_t, 1, 1> op;
op.Init(xGM, weightSscaleGM, activationScaleGM, biasGM, quantScaleGM, quantOffsetGM, yGM, scaleGM, tilingData,
&(pipe));
op.Process();
} else if (TILING_KEY_IS(10002)) {
GET_TILING_DATA_WITH_STRUCT(SwiGluTilingData, tilingDataIn, tiling);
const SwiGluTilingData* __restrict__ tilingData = &tilingDataIn;
DequantSwigluQuant::DequantSwigluQuantStaticBF16<half, float, half, int8_t, 1, 0> op;
op.Init(xGM, weightSscaleGM, activationScaleGM, biasGM, quantScaleGM, quantOffsetGM, yGM, scaleGM, tilingData,
&(pipe));
op.Process();
} else if (TILING_KEY_IS(30009)) {
GET_TILING_DATA_WITH_STRUCT(SwiGluTilingData, tilingDataIn, tiling);
const SwiGluTilingData* __restrict__ tilingData = &tilingDataIn;
DequantSwigluQuant::DequantSwigluQuantDynamicBF16<half, float, half, int8_t, 1, 1> op;
op.Init(xGM, weightSscaleGM, activationScaleGM, biasGM, quantScaleGM, quantOffsetGM, yGM, scaleGM, userspace,
tilingData, &(pipe));
op.Process();
} else if (TILING_KEY_IS(30010)) {
GET_TILING_DATA_WITH_STRUCT(SwiGluTilingData, tilingDataIn, tiling);
const SwiGluTilingData* __restrict__ tilingData = &tilingDataIn;
DequantSwigluQuant::DequantSwigluQuantDynamicBF16<half, float, half, int8_t, 1, 0> op;
op.Init(xGM, weightSscaleGM, activationScaleGM, biasGM, quantScaleGM, quantOffsetGM, yGM, scaleGM, userspace,
tilingData, &(pipe));
op.Process();
}
#endif
#if !(defined(__NPU_ARCH__) && (__NPU_ARCH__ == 3003 || __NPU_ARCH__ == 3113)) && (ORIG_DTYPE_X == DT_BF16)
if (TILING_KEY_IS(DEQUANT_SWIGLU_QUANT_WITH_GROUP_NO_BIAS_FP32_QS)) {
// New tiling branch for BF16
GET_TILING_DATA_WITH_STRUCT(DequantSwigluQuantBaseTilingData, tilingDataIn, tiling);
const DequantSwigluQuantBaseTilingData* __restrict__ tilingData = &tilingDataIn;
DequantSwigluQuantOps::DequantSwigluQuantBase<float, float, int64_t, bfloat16_t> op(&pipe);
op.Init(xGM, weightSscaleGM, activationScaleGM, nullptr, quantScaleGM, quantOffsetGM, groupIndex, yGM, scaleGM,
tilingData);
op.Process();
} else if (TILING_KEY_IS(DEQUANT_SWIGLU_QUANT_WITH_GROUP_NO_BIAS_FP16_QS)) {
GET_TILING_DATA_WITH_STRUCT(DequantSwigluQuantBaseTilingData, tilingDataIn, tiling);
const DequantSwigluQuantBaseTilingData* __restrict__ tilingData = &tilingDataIn;
DequantSwigluQuantOps::DequantSwigluQuantBase<float, half, int64_t, bfloat16_t> op(&pipe);
op.Init(xGM, weightSscaleGM, activationScaleGM, nullptr, quantScaleGM, quantOffsetGM, groupIndex, yGM, scaleGM,
tilingData);
op.Process();
} else if (TILING_KEY_IS(DEQUANT_SWIGLU_QUANT_WITH_GROUP_NO_BIAS_BF16_QS)) {
GET_TILING_DATA_WITH_STRUCT(DequantSwigluQuantBaseTilingData, tilingDataIn, tiling);
const DequantSwigluQuantBaseTilingData* __restrict__ tilingData = &tilingDataIn;
DequantSwigluQuantOps::DequantSwigluQuantBase<float, bfloat16_t, int64_t, bfloat16_t> op(&pipe);
op.Init(xGM, weightSscaleGM, activationScaleGM, nullptr, quantScaleGM, quantOffsetGM, groupIndex, yGM, scaleGM,
tilingData);
op.Process();
} else if (TILING_KEY_IS(10001)) {
GET_TILING_DATA_WITH_STRUCT(SwiGluTilingData, tilingDataIn, tiling);
const SwiGluTilingData* __restrict__ tilingData = &tilingDataIn;
DequantSwigluQuant::DequantSwigluQuantStaticBF16<bfloat16_t, float, bfloat16_t, int8_t, 1, 1> op;
op.Init(xGM, weightSscaleGM, activationScaleGM, biasGM, quantScaleGM, quantOffsetGM, yGM, scaleGM, tilingData,
&(pipe));
op.Process();
} else if (TILING_KEY_IS(10003)) {
GET_TILING_DATA_WITH_STRUCT(SwiGluTilingData, tilingDataIn, tiling);
const SwiGluTilingData* __restrict__ tilingData = &tilingDataIn;
DequantSwigluQuant::DequantSwigluQuantStaticBF16<bfloat16_t, float, bfloat16_t, int8_t, 1, 0> op;
op.Init(xGM, weightSscaleGM, activationScaleGM, biasGM, quantScaleGM, quantOffsetGM, yGM, scaleGM, tilingData,
&(pipe));
op.Process();
} else if (TILING_KEY_IS(30011)) {
GET_TILING_DATA_WITH_STRUCT(SwiGluTilingData, tilingDataIn, tiling);
const SwiGluTilingData* __restrict__ tilingData = &tilingDataIn;
DequantSwigluQuant::DequantSwigluQuantDynamicBF16<bfloat16_t, float, bfloat16_t, int8_t, 1, 1> op;
op.Init(xGM, weightSscaleGM, activationScaleGM, biasGM, quantScaleGM, quantOffsetGM, yGM, scaleGM, userspace,
tilingData, &(pipe));
op.Process();
} else if (TILING_KEY_IS(30012)) {
GET_TILING_DATA_WITH_STRUCT(SwiGluTilingData, tilingDataIn, tiling);
const SwiGluTilingData* __restrict__ tilingData = &tilingDataIn;
DequantSwigluQuant::DequantSwigluQuantDynamicBF16<bfloat16_t, float, bfloat16_t, int8_t, 1, 0> op;
op.Init(xGM, weightSscaleGM, activationScaleGM, biasGM, quantScaleGM, quantOffsetGM, yGM, scaleGM, userspace,
tilingData, &(pipe));
op.Process();
}
#endif
}

817
csrc/moe/dequant_swiglu_quant/op_kernel/dequant_swiglu_quant.h Normal file
View File

@@ -0,0 +1,817 @@
/**
* Copyright (c) 2025 Huawei Technologies Co., Ltd.
* This program is free software, you can redistribute it and/or modify it under the terms and conditions of
* CANN Open Software License Agreement Version 2.0 (the "License").
* Please refer to the License for details. You may not use this file except in compliance with the License.
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
* INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
* See LICENSE in the root of the software repository for the full text of the License.
*/
/*!
* \file dequant_swiglu_quant.h
* \brief
*/
#ifndef DEQUANT_SWIGLU_QUANT_H
#define DEQUANT_SWIGLU_QUANT_H
#include "kernel_tiling/kernel_tiling.h"
#include "kernel_operator.h"
#define TEMPLATE_DSQ_DECLARE template <typename TBias, typename TQuantScale, typename TGroup, typename TXGm>
#define TEMPLATE_DSQ_ARGS TBias, TQuantScale, TGroup, TXGm
namespace DequantSwigluQuantOps {
using namespace AscendC;
constexpr static int64_t DB_BUFFER = 1;
constexpr static int64_t BLOCK_SIZE = 32;
constexpr static int64_t BLOCK_ELEM = BLOCK_SIZE / sizeof(float);
constexpr static int64_t MASK_NUM_T32 = 256 / sizeof(float);
constexpr static int64_t MASK_BLK_STRIDE = 8;
constexpr static int64_t SWI_FACTOR = 2;
constexpr static float DYNAMIC_QUANT_FACTOR = 1.0 / 127.0;
__aicore__ inline void CopyLocalContiguousFloat(
const LocalTensor<float>& dst, const LocalTensor<float>& src, uint32_t count)
{
constexpr uint32_t MAX_REPEAT_TIMES = 255;
constexpr uint32_t MAX_REPEAT_ELEMS = MASK_NUM_T32 * MAX_REPEAT_TIMES;
CopyRepeatParams copyParams{1, 1, MASK_BLK_STRIDE, MASK_BLK_STRIDE};
uint32_t offset = 0;
while (count >= MAX_REPEAT_ELEMS) {
Copy(dst[offset], src[offset], MASK_NUM_T32, MAX_REPEAT_TIMES, copyParams);
offset += MAX_REPEAT_ELEMS;
count -= MAX_REPEAT_ELEMS;
}
if (count >= MASK_NUM_T32) {
uint8_t repeatTimes = static_cast<uint8_t>(count / MASK_NUM_T32);
Copy(dst[offset], src[offset], MASK_NUM_T32, repeatTimes, copyParams);
offset += repeatTimes * MASK_NUM_T32;
count -= repeatTimes * MASK_NUM_T32;
}
if (count > 0) {
Copy(dst[offset], src[offset], count, 1, copyParams);
}
}
TEMPLATE_DSQ_DECLARE
class DequantSwigluQuantBase
{
public:
static constexpr bool hasGroupIndex_ = !IsSameType<TGroup, float>::value;
__aicore__ inline DequantSwigluQuantBase(TPipe* pipe)
{
pipe_ = pipe;
};
__aicore__ inline void Init(
GM_ADDR x, GM_ADDR weightScale, GM_ADDR activationScale, GM_ADDR bias, GM_ADDR quantScale, GM_ADDR quantOffset,
GM_ADDR groupIndex, GM_ADDR y, GM_ADDR scale, const DequantSwigluQuantBaseTilingData* tilingData);
__aicore__ inline void Process();
__aicore__ inline void ComputeReduceMax(const LocalTensor<float>& tempRes, int32_t calCount);
__aicore__ inline void ProcessSingleGroup(int64_t groupIdx, int64_t realCount, int64_t globalOffset);
__aicore__ inline void ProcessSingleGroupPerCore(int64_t groupIdx, int64_t dimxCore, int64_t dimxCoreOffset);
__aicore__ inline void CreateOffsetLocalTensor(uint32_t tensorLen, int swigluMode);
__aicore__ inline void SwiGluGate(
int32_t proDimsx, const LocalTensor<float>& xLocalF32);
__aicore__ inline void DynamicQuant(
const LocalTensor<float>& tmpUbF32Act, const LocalTensor<float>& tmpUbF32Gate,
const LocalTensor<float>& inScaleLocal, uint32_t proDimsx);
__aicore__ inline void StaticQuant(
const LocalTensor<float>& tmpUbF32Act, const LocalTensor<float>& tmpUbF32Gate,
const LocalTensor<float>& inScaleLocal, uint32_t proDimsx);
__aicore__ inline void CopyInWeightScale(int64_t groupIdx);
__aicore__ inline void CopyInQuantScale(int64_t groupIdx);
__aicore__ inline void CopyInBias(int64_t groupIdx);
__aicore__ inline void ParamDequeAndCast();
__aicore__ inline void CopyInXAct(int32_t proDimsx, int64_t xDimxOffset);
__aicore__ inline void Compute(int32_t proDimsx);
__aicore__ inline void ComputeDequant(int32_t proDimsx);
__aicore__ inline void ComputeSwiGLU(int32_t proDimsx);
__aicore__ inline void ComputeQuant(int32_t proDimsx);
__aicore__ inline void CopyOut(int32_t proDimsx, int64_t xDimxOffset);
__aicore__ inline void ParamFree();
__aicore__ inline void CastFloatToInt8(
const LocalTensor<float>& tmpUbF32Act, const LocalTensor<float>& tmpUbF32Gate, uint32_t proDimsx, LocalTensor<int8_t>& yOut);
template<typename T>
__aicore__ inline void CopyReshape(LocalTensor<T>& dstTensor, LocalTensor<T>& oriTensor, uint32_t rowNum, uint32_t colNum, CopyRepeatParams param);
protected:
/* global memory address */
// input global mem
GlobalTensor<TXGm> xGm_;
GlobalTensor<float> weightScaleGm_;
GlobalTensor<float> activationScaleGm_;
GlobalTensor<TBias> biasGm_;
GlobalTensor<TQuantScale> quantScaleGm_;
GlobalTensor<TQuantScale> quantOffsetGm_;
GlobalTensor<TGroup> groupIndexGm_;
// output global mem
GlobalTensor<int8_t> yGm_;
GlobalTensor<float> scaleGm_;
/* ub memory tensor */
LocalTensor<float> weightScaleLocal_;
LocalTensor<float> inScaleLocal_; // quant scale and quant offset
LocalTensor<TBias> biasLocal_;
LocalTensor<float> biasLocalF32_;
LocalTensor<uint32_t> xOffsetLocalU32_; // offset for gather
/* ascendc variable */
TPipe* pipe_ = nullptr;
TQue<QuePosition::VECIN, DB_BUFFER> xActQueue_;
TQue<QuePosition::VECIN, 1> inScaleQueue_;
TQue<QuePosition::VECIN, 1> weightScaleQueue_;
TQue<QuePosition::VECIN, 1> biasQueue_;
TQue<QuePosition::VECOUT, 1> outQueue_;
TBuf<TPosition::VECCALC> tmpBuf1_;
TBuf<TPosition::VECCALC> tmpBuf2_; // only use in swigluMode == 1
TBuf<TPosition::VECCALC> scaleBuf_;
uint32_t blockIdx_ = GetBlockIdx();
int64_t realDimx_ = 0;
int64_t groupOffset_ = 0;
float quantScale_ = 1.0f;
float quantOffset_ = 1.0f;
uint32_t UbSingleOutSize_ = 0;
uint32_t TBufActSclInOfs_ = 0;
uint32_t TBufXLocalInOfs_ = 0;
int32_t actOffset_;
int32_t gateOffset_;
const DequantSwigluQuantBaseTilingData* tl_ = nullptr;
};
// 公共函数实现
TEMPLATE_DSQ_DECLARE
__aicore__ inline void DequantSwigluQuantBase<TEMPLATE_DSQ_ARGS>::Init(
GM_ADDR x, GM_ADDR weightScale, GM_ADDR activationScale, GM_ADDR bias, GM_ADDR quantScale, GM_ADDR quantOffset,
GM_ADDR groupIndex, GM_ADDR y, GM_ADDR scale, const DequantSwigluQuantBaseTilingData* tilingData)
{
tl_ = tilingData;
xGm_.SetGlobalBuffer((__gm__ TXGm*)x);
weightScaleGm_.SetGlobalBuffer((__gm__ float*)weightScale);
activationScaleGm_.SetGlobalBuffer((__gm__ float*)activationScale);
biasGm_.SetGlobalBuffer((__gm__ TBias*)bias);
quantScaleGm_.SetGlobalBuffer((__gm__ TQuantScale*)quantScale);
if constexpr (hasGroupIndex_) {
groupIndexGm_.SetGlobalBuffer((__gm__ TGroup*)groupIndex);
}
// static quant
if (tl_->quantMode == 0) {
quantOffsetGm_.SetGlobalBuffer((__gm__ TQuantScale*)quantOffset);
}
yGm_.SetGlobalBuffer((__gm__ int8_t*)y);
scaleGm_.SetGlobalBuffer((__gm__ float*)scale);
UbSingleOutSize_ = static_cast<uint32_t>(tl_->UbFactorDimx * tl_->outDimy);
TBufActSclInOfs_ = static_cast<uint32_t>(tl_->UbFactorDimx * tl_->inDimy);
#if (ORIG_DTYPE_X == DT_BF16)
TBufXLocalInOfs_ = TBufActSclInOfs_;
#endif
// swiglu offset
actOffset_ = tl_->actRight * tl_->UbFactorDimy;
gateOffset_ = tl_->UbFactorDimy - actOffset_;
// init buffer
pipe_->InitBuffer(
xActQueue_, DB_BUFFER, (UbSingleOutSize_ * SWI_FACTOR + tl_->UbFactorDimx * BLOCK_ELEM) * sizeof(int32_t));
pipe_->InitBuffer(weightScaleQueue_, 1, tl_->inDimy * sizeof(float));
if (tl_->quantMode == 0) {
pipe_->InitBuffer(inScaleQueue_, 1, tl_->outDimy * SWI_FACTOR * sizeof(float));
} else {
pipe_->InitBuffer(inScaleQueue_, 1, tl_->outDimy * sizeof(float));
}
if (tl_->hasBias == 1) {
pipe_->InitBuffer(biasQueue_, 1, tl_->inDimy * sizeof(float));
}
pipe_->InitBuffer(outQueue_, 1, UbSingleOutSize_ * sizeof(int8_t) + tl_->UbFactorDimx * sizeof(float) + BLOCK_SIZE);
pipe_->InitBuffer(tmpBuf1_, UbSingleOutSize_ * SWI_FACTOR * sizeof(float));
pipe_->InitBuffer(scaleBuf_,
((tl_->UbFactorDimx + BLOCK_ELEM - 1) / BLOCK_ELEM) * BLOCK_ELEM * BLOCK_ELEM * sizeof(float));
if (tl_->swigluMode == 1) {
pipe_->InitBuffer(
tmpBuf2_,
UbSingleOutSize_ * sizeof(int32_t) + UbSingleOutSize_ * sizeof(uint8_t)); // for gather offset and clamp
}
}
TEMPLATE_DSQ_DECLARE
__aicore__ inline void DequantSwigluQuantBase<TEMPLATE_DSQ_ARGS>::Process()
{
if constexpr (!hasGroupIndex_) {
realDimx_ = tl_->inDimx;
// do protect realDimx_ < 0, ignore this group
realDimx_ = (realDimx_ < 0) ? 0 : realDimx_;
ProcessSingleGroup(0, realDimx_, 0);
return;
}
CreateOffsetLocalTensor(UbSingleOutSize_, tl_->swigluMode);
groupOffset_ = 0;
for (int32_t groupIdx = 0; groupIdx < tl_->inGroupNum; ++groupIdx) {
int64_t realGroupIdx =
tl_->speGroupType == 0 ? static_cast<int64_t>(groupIdx) : static_cast<int64_t>(groupIndexGm_(groupIdx * 2));
realDimx_ = tl_->speGroupType == 0 ? static_cast<int64_t>(groupIndexGm_(groupIdx)) :
static_cast<int64_t>(groupIndexGm_(groupIdx * 2 + 1));
// do protect realDimx_ < 0, ignore this group
realDimx_ = (realDimx_ < 0) ? 0 : realDimx_;
if (realDimx_ > 0 && groupOffset_ < tl_->inDimx) {
ProcessSingleGroup(realGroupIdx, realDimx_, groupOffset_);
groupOffset_ += realDimx_;
}
// speGroupindex场景下出现异常值(realDimx_ < 0), 退出计算
if (tl_->speGroupType == 1 && realDimx_ <= 0) {
break;
}
}
}
TEMPLATE_DSQ_DECLARE
__aicore__ inline void DequantSwigluQuantBase<TEMPLATE_DSQ_ARGS>::ProcessSingleGroup(
int64_t groupIdx, int64_t realCount, int64_t globalOffset)
{
// do block tiling again
int32_t blockDimxFactor = (realCount + tl_->maxCoreNum - 1) / tl_->maxCoreNum;
int32_t realCoreDim = (realCount + blockDimxFactor - 1) / blockDimxFactor;
if (blockIdx_ < realCoreDim) {
int32_t blockDimxTailFactor = realCount - blockDimxFactor * (realCoreDim - 1);
int32_t dimxCore = blockIdx_ == (realCoreDim - 1) ? blockDimxTailFactor : blockDimxFactor;
int64_t coreDimxOffset = blockDimxFactor * blockIdx_ + globalOffset;
ProcessSingleGroupPerCore(static_cast<int64_t>(groupIdx), static_cast<int64_t>(dimxCore), coreDimxOffset);
}
}
TEMPLATE_DSQ_DECLARE
__aicore__ inline void DequantSwigluQuantBase<TEMPLATE_DSQ_ARGS>::CopyInWeightScale(int64_t groupIdx)
{
// copy weight scale [1, 2H] offset:0
DataCopyPadParams padParams{false, 0, 0, 0};
LocalTensor<float> weightScaleLocal = weightScaleQueue_.AllocTensor<float>();
DataCopyParams dataCopyWeightScaleParams;
dataCopyWeightScaleParams.blockCount = 1;
dataCopyWeightScaleParams.blockLen = tl_->inDimy * sizeof(float);
dataCopyWeightScaleParams.srcStride = 0;
dataCopyWeightScaleParams.dstStride = 0;
if constexpr (std::is_same_v<TXGm, int32_t>) {
DataCopyPad(weightScaleLocal, weightScaleGm_[groupIdx * tl_->inDimy], dataCopyWeightScaleParams, padParams);
}
weightScaleQueue_.EnQue(weightScaleLocal);
}
TEMPLATE_DSQ_DECLARE
__aicore__ inline void DequantSwigluQuantBase<TEMPLATE_DSQ_ARGS>::CopyInQuantScale(int64_t groupIdx)
{
DataCopyPadParams padParams{false, 0, 0, 0};
// copy static quant scale
LocalTensor<float> inScaleLocal = inScaleQueue_.AllocTensor<float>();
if (tl_->quantIsOne) {
if constexpr (IsSameType<TQuantScale, bfloat16_t>::value) {
this->quantScale_ = 1 / ToFloat(this->quantScaleGm_.GetValue(groupIdx));
this->quantOffset_ = ToFloat(this->quantOffsetGm_.GetValue(groupIdx));
} else if constexpr (IsSameType<TQuantScale, half>::value) {
this->quantScale_ = 1 / static_cast<float>(this->quantScaleGm_.GetValue(groupIdx));
this->quantOffset_ = static_cast<float>(this->quantOffsetGm_.GetValue(groupIdx));
} else {
this->quantScale_ = 1 / this->quantScaleGm_.GetValue(groupIdx);
this->quantOffset_ = this->quantOffsetGm_.GetValue(groupIdx);
}
}
// copy dynamic quant scale [1, H] offset:tl_->inDimy
if (tl_->needSmoothScale == 1 && !tl_->quantIsOne) {
DataCopyParams dataCopyQuantScaleParams;
dataCopyQuantScaleParams.blockCount = 1;
dataCopyQuantScaleParams.blockLen = tl_->outDimy * sizeof(TQuantScale);
dataCopyQuantScaleParams.srcStride = 0;
dataCopyQuantScaleParams.dstStride = 0;
if constexpr (std::is_same_v<TQuantScale, float>) {
DataCopyPad(inScaleLocal, quantScaleGm_[groupIdx * tl_->outDimy], dataCopyQuantScaleParams, padParams);
if (tl_->quantMode == 0) {
DataCopyPad(
inScaleLocal[tl_->outDimy], quantOffsetGm_[groupIdx * tl_->outDimy], dataCopyQuantScaleParams,
padParams);
}
} else {
LocalTensor<TQuantScale> quantScaleLocalT16 = inScaleLocal.template ReinterpretCast<TQuantScale>();
DataCopyPad(
quantScaleLocalT16[tl_->outDimy], quantScaleGm_[groupIdx * tl_->outDimy], dataCopyQuantScaleParams,
padParams);
if (tl_->quantMode == 0) {
DataCopyPad(
quantScaleLocalT16[tl_->outDimy + tl_->inDimy], quantOffsetGm_[groupIdx * tl_->outDimy],
dataCopyQuantScaleParams, padParams);
}
}
}
inScaleQueue_.EnQue(inScaleLocal);
}
TEMPLATE_DSQ_DECLARE
__aicore__ inline void DequantSwigluQuantBase<TEMPLATE_DSQ_ARGS>::CopyInBias(int64_t groupIdx)
{
DataCopyPadParams padParams{false, 0, 0, 0};
if constexpr (std::is_same_v<TXGm, int32_t>) {
if (tl_->hasBias == 1) {
biasLocal_ = biasQueue_.AllocTensor<TBias>();
DataCopyParams dataCopyBiasParams;
dataCopyBiasParams.blockCount = 1;
dataCopyBiasParams.blockLen = tl_->inDimy * sizeof(TBias);
dataCopyBiasParams.srcStride = 0;
dataCopyBiasParams.dstStride = 0;
if constexpr (std::is_same_v<TBias, float> || std::is_same_v<TBias, int32_t>) {
DataCopyPad(biasLocal_, biasGm_[groupIdx * tl_->inDimy], dataCopyBiasParams, padParams);
} else {
DataCopyPad(biasLocal_[tl_->inDimy], biasGm_[groupIdx * tl_->inDimy], dataCopyBiasParams, padParams);
}
biasQueue_.EnQue(biasLocal_);
}
}
}
TEMPLATE_DSQ_DECLARE
__aicore__ inline void DequantSwigluQuantBase<TEMPLATE_DSQ_ARGS>::CopyInXAct(int32_t proDimsx, int64_t xDimxOffset)
{
// copyin x and Act scale
DataCopyPadParams padParams{false, 0, 0, 0};
LocalTensor<TXGm> xActLocal = xActQueue_.AllocTensor<TXGm>();
DataCopyParams dataCopyXParams;
dataCopyXParams.blockCount = proDimsx;
dataCopyXParams.blockLen = tl_->inDimy * sizeof(TXGm);
dataCopyXParams.srcStride = 0;
dataCopyXParams.dstStride = 0;
DataCopyPad(xActLocal[TBufXLocalInOfs_], xGm_[xDimxOffset * tl_->inDimy], dataCopyXParams, padParams);
// copy act scale: [proDimsx,8] offset:tl_->UbFactorDimx * tl_->inDimy = TBufActSclInOfs_
DataCopyParams dataCopyActScaleParams;
dataCopyActScaleParams.blockCount = proDimsx;
dataCopyActScaleParams.blockLen = sizeof(float);
dataCopyActScaleParams.srcStride = 0;
dataCopyActScaleParams.dstStride = 0;
LocalTensor<float> xActLocalF32 = xActLocal.template ReinterpretCast<float>();
if (std::is_same_v<TXGm, int32_t> && !tl_->activationScaleIsEmpty) {
DataCopyPad(xActLocalF32[TBufActSclInOfs_], activationScaleGm_[xDimxOffset], dataCopyActScaleParams, padParams);
}
xActQueue_.EnQue(xActLocal);
}
TEMPLATE_DSQ_DECLARE
__aicore__ inline void DequantSwigluQuantBase<TEMPLATE_DSQ_ARGS>::ComputeDequant(int32_t proDimsx)
{
LocalTensor<TXGm> xActLocal = xActQueue_.DeQue<TXGm>();
LocalTensor<float> xActLocalF32 = xActLocal.template ReinterpretCast<float>();
LocalTensor<float> xLocalF32 = xActLocalF32;
LocalTensor<float> activationScaleLocal = xActLocalF32[TBufActSclInOfs_];
LocalTensor<float> tmpUbF32 = tmpBuf1_.AllocTensor<float>(); // weight scale FP32
LocalTensor<int32_t> tmpUbI32 = tmpUbF32.template ReinterpretCast<int32_t>();
if constexpr (std::is_same_v<TXGm, int32_t>) {
if constexpr (std::is_same_v<TBias, int32_t>){
// Copy bias: [1,2H] -> [proDimsx,2H]
// params: dstStride: 1, srcStride: 1, dstRepStride: tl_->UbFactorDimy * 2 / 8, srcRepStride: 0
CopyReshape<int32_t>(tmpUbI32, biasLocal_, proDimsx, tl_->UbFactorDimy * SWI_FACTOR,
{1, 1, static_cast<uint16_t>((tl_->UbFactorDimy * SWI_FACTOR) / BLOCK_ELEM), 0});
PipeBarrier<PIPE_V>();
Add(xActLocal, xActLocal, tmpUbI32, proDimsx * tl_->inDimy);
PipeBarrier<PIPE_V>();
}
// Copy weight scale: [1,2H] -> [proDimsx,2H]
// params: dstStride: 1, srcStride: 1, dstRepStride: tl_->UbFactorDimy * 2 / 8, srcRepStride: 0
CopyReshape<float>(tmpUbF32, weightScaleLocal_, proDimsx, tl_->UbFactorDimy * SWI_FACTOR,
{1, 1, static_cast<uint16_t>((tl_->UbFactorDimy * SWI_FACTOR) / BLOCK_ELEM), 0});
}
// x 为 bf16时
Cast(xLocalF32, xActLocal[TBufXLocalInOfs_], RoundMode::CAST_NONE, SWI_FACTOR * proDimsx * tl_->UbFactorDimy);
PipeBarrier<PIPE_V>();
if constexpr (std::is_same_v<TXGm, int32_t>) {
// Calc dequant: xLocalF32 = weightScaleLocal * xLocalF32
Mul(xLocalF32, tmpUbF32, xLocalF32, tl_->UbFactorDimy * SWI_FACTOR * proDimsx);
PipeBarrier<PIPE_V>();
if (!tl_->activationScaleIsEmpty) {
// Copy act scale: [proDimsx,8] -> [proDimsx,2H]
CopyReshape<float>(tmpUbF32, activationScaleLocal, proDimsx, tl_->UbFactorDimy * SWI_FACTOR,
{1, 0, static_cast<uint16_t>((tl_->UbFactorDimy * SWI_FACTOR) / BLOCK_ELEM), 1});
PipeBarrier<PIPE_V>();
// Calc dequant: xLocalF32 = activationScaleLocal * xLocalF32
Mul(xLocalF32, tmpUbF32, xLocalF32, tl_->UbFactorDimy * SWI_FACTOR * proDimsx);
PipeBarrier<PIPE_V>();
}
}
if constexpr (std::is_same_v<TXGm, int32_t> && !std::is_same_v<TBias, int32_t>) {
if (tl_->hasBias == 1) {
// Copy bias: [1,2H] -> [proDimsx,2H]
CopyReshape<float>(tmpUbF32, biasLocalF32_, proDimsx, tl_->UbFactorDimy * SWI_FACTOR,
{1, 1, static_cast<uint16_t>((tl_->UbFactorDimy * SWI_FACTOR) / BLOCK_ELEM), 0});
PipeBarrier<PIPE_V>();
Add(xLocalF32, xLocalF32, tmpUbF32, proDimsx * tl_->inDimy);
PipeBarrier<PIPE_V>();
}
}
xActQueue_.EnQue(xLocalF32);
}
TEMPLATE_DSQ_DECLARE
__aicore__ inline void DequantSwigluQuantBase<TEMPLATE_DSQ_ARGS>::ComputeSwiGLU(int32_t proDimsx)
{
LocalTensor<float> xLocalF32 = xActQueue_.DeQue<float>();
if (tl_->swigluMode == 1) {
// do special swiglu
SwiGluGate(proDimsx, xLocalF32);
} else {
uint32_t calEleNum = tl_->UbFactorDimy * proDimsx;
LocalTensor<float> tmpUbF32 = tmpBuf1_.AllocTensor<float>();
// do normal swi pre
LocalTensor<float> tmpUbF32Act = tmpUbF32;
LocalTensor<float> tmpUbF32Gate = tmpUbF32[calEleNum];
// Copy dequant result: xLocalF32[actOffset] -> tmpUbF32Act, [proDimsx,H]
// Copy dequant result: xLocalF32[gateOffset] -> tmpUbF32Gate, [proDimsx,H]
SetMaskCount();
SetVectorMask<float, MaskMode::COUNTER>(tl_->UbFactorDimy);
Copy<float, false>(
tmpUbF32Act, xLocalF32[actOffset_], AscendC::MASK_PLACEHOLDER, proDimsx,
{1, 1, static_cast<uint16_t>(tl_->UbFactorDimy / BLOCK_ELEM),
static_cast<uint16_t>(tl_->UbFactorDimy / BLOCK_ELEM * SWI_FACTOR)});
Copy<float, false>(
tmpUbF32Gate, xLocalF32[gateOffset_], AscendC::MASK_PLACEHOLDER, proDimsx,
{1, 1, static_cast<uint16_t>(tl_->UbFactorDimy / BLOCK_ELEM),
static_cast<uint16_t>(tl_->UbFactorDimy / BLOCK_ELEM * SWI_FACTOR)});
SetMaskNorm();
ResetMask();
PipeBarrier<PIPE_V>();
Muls(xLocalF32, tmpUbF32Act, static_cast<float>(-1.0), calEleNum);
PipeBarrier<PIPE_V>();
Exp(xLocalF32, xLocalF32, calEleNum);
PipeBarrier<PIPE_V>();
Adds(xLocalF32, xLocalF32, static_cast<float>(1.0), calEleNum);
PipeBarrier<PIPE_V>();
Div(tmpUbF32Act, tmpUbF32Act, xLocalF32, calEleNum);
PipeBarrier<PIPE_V>();
Mul(tmpUbF32Act, tmpUbF32Gate, tmpUbF32Act, calEleNum);
PipeBarrier<PIPE_V>();
}
// x compute done, free
xActQueue_.FreeTensor(xLocalF32);
}
TEMPLATE_DSQ_DECLARE
__aicore__ inline void DequantSwigluQuantBase<TEMPLATE_DSQ_ARGS>::ComputeQuant(int32_t proDimsx)
{
LocalTensor<float> tmpUbF32 = tmpBuf1_.AllocTensor<float>();
LocalTensor<float> tmpUbF32Act = tmpUbF32;
LocalTensor<float> tmpUbF32Gate = tmpUbF32[tl_->UbFactorDimy * proDimsx];
if (tl_->quantMode == 1) {
DynamicQuant(tmpUbF32Act, tmpUbF32Gate, inScaleLocal_, proDimsx);
} else {
StaticQuant(tmpUbF32Act, tmpUbF32Gate, inScaleLocal_, proDimsx);
}
tmpBuf1_.FreeTensor(tmpUbF32);
}
TEMPLATE_DSQ_DECLARE
__aicore__ inline void DequantSwigluQuantBase<TEMPLATE_DSQ_ARGS>::Compute(int32_t proDimsx)
{
ComputeDequant(proDimsx);
ComputeSwiGLU(proDimsx);
ComputeQuant(proDimsx);
}
TEMPLATE_DSQ_DECLARE
__aicore__ inline void DequantSwigluQuantBase<TEMPLATE_DSQ_ARGS>::CopyOut(int32_t proDimsx, int64_t xDimxOffset)
{
// copy out
LocalTensor<float> outLocal = outQueue_.DeQue<float>();
LocalTensor<float> scaleOut = outLocal[UbSingleOutSize_ * sizeof(int8_t) / sizeof(float)];
LocalTensor<int8_t> yOut = outLocal.template ReinterpretCast<int8_t>();
if (tl_->quantMode == 1) {
DataCopyParams dataCopyOutScaleParams;
dataCopyOutScaleParams.blockCount = 1;
dataCopyOutScaleParams.blockLen = proDimsx * sizeof(float);
dataCopyOutScaleParams.srcStride = 0;
dataCopyOutScaleParams.dstStride = 0;
DataCopyPad(scaleGm_[xDimxOffset], scaleOut, dataCopyOutScaleParams);
}
DataCopyParams dataCopyOutyParams;
dataCopyOutyParams.blockCount = 1;
dataCopyOutyParams.blockLen = proDimsx * tl_->outDimy * sizeof(int8_t);
dataCopyOutyParams.srcStride = 0;
dataCopyOutyParams.dstStride = 0;
DataCopyPad(yGm_[xDimxOffset * tl_->outDimy], yOut, dataCopyOutyParams);
outQueue_.FreeTensor(outLocal);
}
TEMPLATE_DSQ_DECLARE
__aicore__ inline void DequantSwigluQuantBase<TEMPLATE_DSQ_ARGS>::ParamDequeAndCast()
{
weightScaleLocal_ = weightScaleQueue_.DeQue<float>();
// bias deque and cast bias to fp32 if needed
if constexpr (std::is_same_v<TXGm, int32_t>) {
if (tl_->hasBias == 1) {
biasLocal_ = biasQueue_.DeQue<TBias>();
biasLocalF32_ = biasLocal_.template ReinterpretCast<float>();
if constexpr (std::is_same_v<TBias, half> || std::is_same_v<TBias, bfloat16_t>) {
Cast(biasLocalF32_, biasLocal_[tl_->inDimy], RoundMode::CAST_NONE, tl_->inDimy);
}
}
}
// quant scale and quant offset deque, cast them to fp32 if needed
inScaleLocal_ = inScaleQueue_.DeQue<float>();
if (tl_->needSmoothScale == 1 && !tl_->quantIsOne) {
if (std::is_same_v<TQuantScale, half> || std::is_same_v<TQuantScale, bfloat16_t>) {
LocalTensor<TQuantScale> quantScaleLocalT16 = inScaleLocal_.template ReinterpretCast<TQuantScale>();
Cast(inScaleLocal_, quantScaleLocalT16[tl_->outDimy], RoundMode::CAST_NONE, tl_->outDimy);
PipeBarrier<PIPE_V>();
if (tl_->quantMode == 0) {
Cast(
inScaleLocal_[tl_->outDimy], quantScaleLocalT16[tl_->outDimy + tl_->inDimy], RoundMode::CAST_NONE,
tl_->outDimy);
PipeBarrier<PIPE_V>();
}
}
}
}
TEMPLATE_DSQ_DECLARE
__aicore__ inline void DequantSwigluQuantBase<TEMPLATE_DSQ_ARGS>::ProcessSingleGroupPerCore(
int64_t groupIdx, int64_t dimxCore, int64_t coreDimxOffset)
{
// do ub tiling again
int32_t ubDimxLoop = (dimxCore + tl_->UbFactorDimx - 1) / tl_->UbFactorDimx;
int32_t ubDimxTailFactor = dimxCore - tl_->UbFactorDimx * (ubDimxLoop - 1);
// copyin 当前分组下使用的参数，weight scale, bias scale, quant scale+quant offset
CopyInWeightScale(groupIdx);
CopyInQuantScale(groupIdx);
CopyInBias(groupIdx);
ParamDequeAndCast();
/*
1. copyin x, activation scale
2. compute
3. copyout y, scale
*/
for (uint32_t loopIdx = 0; loopIdx < ubDimxLoop; ++loopIdx) {
int64_t xDimxOffset = coreDimxOffset + loopIdx * tl_->UbFactorDimx;
int32_t proDimsx = loopIdx == (ubDimxLoop - 1) ? ubDimxTailFactor : tl_->UbFactorDimx;
CopyInXAct(proDimsx, xDimxOffset);
Compute(proDimsx);
CopyOut(proDimsx, xDimxOffset);
}
ParamFree();
}
TEMPLATE_DSQ_DECLARE
__aicore__ inline void DequantSwigluQuantBase<TEMPLATE_DSQ_ARGS>::ParamFree()
{
// 释放当前分组下使用的参数，weight scale, bias scale, quant scale，quant offset
inScaleQueue_.FreeTensor(inScaleLocal_);
weightScaleQueue_.FreeTensor(weightScaleLocal_);
if constexpr (std::is_same_v<TXGm, int32_t>) {
if (tl_->hasBias == 1) {
biasQueue_.FreeTensor(biasLocal_);
}
}
}
TEMPLATE_DSQ_DECLARE
__aicore__ inline void DequantSwigluQuantBase<TEMPLATE_DSQ_ARGS>::ComputeReduceMax(
const LocalTensor<float>& tempRes, int32_t calCount)
{
uint32_t vectorCycles = calCount / MASK_NUM_T32;
uint32_t remainElements = calCount % MASK_NUM_T32;
BinaryRepeatParams repeatParams;
repeatParams.dstBlkStride = 1;
repeatParams.src0BlkStride = 1;
repeatParams.src1BlkStride = 1;
repeatParams.dstRepStride = 0;
repeatParams.src0RepStride = MASK_BLK_STRIDE;
repeatParams.src1RepStride = 0;
if (vectorCycles > 0 && remainElements > 0) {
Max(tempRes, tempRes, tempRes[vectorCycles * MASK_NUM_T32], remainElements, 1, repeatParams);
PipeBarrier<PIPE_V>();
}
if (vectorCycles > 1) {
Max(tempRes, tempRes[MASK_NUM_T32], tempRes, MASK_NUM_T32, vectorCycles - 1, repeatParams);
PipeBarrier<PIPE_V>();
}
}
TEMPLATE_DSQ_DECLARE
__aicore__ inline void DequantSwigluQuantBase<TEMPLATE_DSQ_ARGS>::CreateOffsetLocalTensor(
uint32_t tensorLen, int swigluMode)
{
// 不再需要创建偏移张量，因为直接使用前一半和后一半数据
}
TEMPLATE_DSQ_DECLARE
__aicore__ inline void DequantSwigluQuantBase<TEMPLATE_DSQ_ARGS>::SwiGluGate(
int32_t proDimsx, const LocalTensor<float>& xLocalF32)
{
uint32_t calEleNum = tl_->UbFactorDimy * proDimsx;
LocalTensor<float> tmpUbF32 = tmpBuf1_.AllocTensor<float>();
LocalTensor<float> tmpUbF32Act = tmpUbF32;
LocalTensor<float> tmpUbF32Gate = tmpUbF32[calEleNum];
SetMaskCount();
SetVectorMask<float, MaskMode::COUNTER>(tl_->UbFactorDimy);
Copy<float, false>(
tmpUbF32Act, xLocalF32[actOffset_], AscendC::MASK_PLACEHOLDER, proDimsx,
{1, 1, static_cast<uint16_t>(tl_->UbFactorDimy / BLOCK_ELEM),
static_cast<uint16_t>(tl_->UbFactorDimy / BLOCK_ELEM * SWI_FACTOR)});
Copy<float, false>(
tmpUbF32Gate, xLocalF32[gateOffset_], AscendC::MASK_PLACEHOLDER, proDimsx,
{1, 1, static_cast<uint16_t>(tl_->UbFactorDimy / BLOCK_ELEM),
static_cast<uint16_t>(tl_->UbFactorDimy / BLOCK_ELEM * SWI_FACTOR)});
SetMaskNorm();
ResetMask();
PipeBarrier<PIPE_V>();
if (tl_->clampLimit > 0.0f) {
// tmpUbF32Gate
Mins(tmpUbF32Gate, tmpUbF32Gate, tl_->clampLimit, calEleNum);
PipeBarrier<PIPE_V>();
Maxs(tmpUbF32Gate, tmpUbF32Gate, -(tl_->clampLimit), calEleNum);
PipeBarrier<PIPE_V>();
}
Adds(tmpUbF32Gate, tmpUbF32Gate, tl_->gluBias, calEleNum);
PipeBarrier<PIPE_V>();
if (tl_->clampLimit > 0.0f) {
// tmpUbF32Act
Mins(tmpUbF32Act, tmpUbF32Act, tl_->clampLimit, calEleNum);
PipeBarrier<PIPE_V>();
}
Muls(xLocalF32, tmpUbF32Act, -(tl_->gluAlpha), calEleNum);
PipeBarrier<PIPE_V>();
Exp(xLocalF32, xLocalF32, calEleNum);
PipeBarrier<PIPE_V>();
Adds(xLocalF32, xLocalF32, static_cast<float>(1.0), calEleNum);
PipeBarrier<PIPE_V>();
Div(tmpUbF32Act, tmpUbF32Act, xLocalF32, calEleNum);
PipeBarrier<PIPE_V>();
Mul(tmpUbF32Act, tmpUbF32Gate, tmpUbF32Act, calEleNum);
PipeBarrier<PIPE_V>();
}
TEMPLATE_DSQ_DECLARE
__aicore__ inline void DequantSwigluQuantBase<TEMPLATE_DSQ_ARGS>::DynamicQuant(
const LocalTensor<float>& tmpUbF32Act, const LocalTensor<float>& tmpUbF32Gate,
const LocalTensor<float>& inScaleLocal, uint32_t proDimsx)
{
if (tl_->needSmoothScale == 1) {
// Copy quant scale: [1,H] -> [proDimsx,H]
SetMaskCount();
SetVectorMask<float, MaskMode::COUNTER>(tl_->UbFactorDimy);
Copy<float, false>(
tmpUbF32Gate, inScaleLocal, AscendC::MASK_PLACEHOLDER, proDimsx,
{1, 1, static_cast<uint16_t>(tl_->UbFactorDimy / BLOCK_ELEM), 0});
SetMaskNorm();
ResetMask();
PipeBarrier<PIPE_V>();
// Calc quant: xLocalF32 = tmpUbF32Act * inScaleLocal
Mul(tmpUbF32Act, tmpUbF32Gate, tmpUbF32Act, tl_->UbFactorDimy * proDimsx);
PipeBarrier<PIPE_V>();
}
// Calc quant: tmpUbF32Gate = abs(tmpUbF32Act)
Abs(tmpUbF32Gate, tmpUbF32Act, tl_->UbFactorDimy * proDimsx);
LocalTensor<float> outLocal = outQueue_.AllocTensor<float>();
LocalTensor<float> scaleOut = outLocal[UbSingleOutSize_ * sizeof(int8_t) / sizeof(float)];
LocalTensor<int8_t> yOut = outLocal.template ReinterpretCast<int8_t>();
PipeBarrier<PIPE_V>();
// Calc quant: proDimsx * tl_->UbFactorDimy -> proDimsx * 64
for (uint32_t i = 0; i < proDimsx; ++i) {
ComputeReduceMax(tmpUbF32Gate[i * tl_->UbFactorDimy], tl_->UbFactorDimy);
}
// Calc quant: proDimsx * 64 -> proDimsx
// repeatTimes:proDimsx, dstRepStride:1(dtype), srcBlkStride:1, srcRepStride:tl_->UbFactorDimy / 64 * 8
WholeReduceMax(
tmpUbF32Gate, tmpUbF32Gate, MASK_NUM_T32, proDimsx, 1, 1, tl_->UbFactorDimy / BLOCK_ELEM,
ReduceOrder::ORDER_ONLY_VALUE);
PipeBarrier<PIPE_V>();
// Calc quant: scaleOut / 127.0
Muls(scaleOut, tmpUbF32Gate, DYNAMIC_QUANT_FACTOR, proDimsx);
PipeBarrier<PIPE_V>();
// Calc Broadcast: proDimsx -> proDimsx,8
int64_t blockCount = (proDimsx + BLOCK_ELEM - 1) / BLOCK_ELEM;
LocalTensor<float> brcbTmp = scaleBuf_.AllocTensor<float>();
Brcb(brcbTmp, scaleOut, blockCount, {1, MASK_BLK_STRIDE});
PipeBarrier<PIPE_V>();
// Copy scale: [proDimsx,8] -> [proDimsx,H]
SetMaskCount();
SetVectorMask<float, MaskMode::COUNTER>(tl_->UbFactorDimy);
Copy<float, false>(
tmpUbF32Gate, brcbTmp, AscendC::MASK_PLACEHOLDER, proDimsx,
{1, 0, static_cast<uint16_t>(tl_->UbFactorDimy / BLOCK_ELEM), 1});
SetMaskNorm();
ResetMask();
PipeBarrier<PIPE_V>();
scaleBuf_.FreeTensor(brcbTmp);
// Calc y: tmpUbF32Act = tmpUbF32Act / scaleOut
Div(tmpUbF32Act, tmpUbF32Act, tmpUbF32Gate, tl_->UbFactorDimy * proDimsx);
PipeBarrier<PIPE_V>();
CastFloatToInt8(tmpUbF32Act, tmpUbF32Gate, proDimsx, yOut);
outQueue_.EnQue<float>(outLocal);
}
TEMPLATE_DSQ_DECLARE
__aicore__ inline void DequantSwigluQuantBase<TEMPLATE_DSQ_ARGS>::StaticQuant(
const LocalTensor<float>& tmpUbF32Act, const LocalTensor<float>& tmpUbF32Gate,
const LocalTensor<float>& inScaleLocal, uint32_t proDimsx)
{
if (tl_->needSmoothScale == 1) {
if (tl_->quantIsOne) {
// Calc quant: y = tmpUbF32Act * quantScale + quantOffset
Muls(tmpUbF32Gate, tmpUbF32Act, this->quantScale_, tl_->UbFactorDimy * proDimsx);
PipeBarrier<PIPE_V>();
Adds(tmpUbF32Act, tmpUbF32Gate, this->quantOffset_, tl_->UbFactorDimy * proDimsx);
PipeBarrier<PIPE_V>();
} else {
// Copy quant scale: [1,H] -> [proDimsx,H]
SetMaskCount();
SetVectorMask<float, MaskMode::COUNTER>(tl_->UbFactorDimy);
Copy<float, false>(
tmpUbF32Gate, inScaleLocal, AscendC::MASK_PLACEHOLDER, proDimsx,
{1, 1, static_cast<uint16_t>(tl_->UbFactorDimy / BLOCK_ELEM), 0});
SetMaskNorm();
ResetMask();
PipeBarrier<PIPE_V>();
// Calc quant: y = tmpUbF32Act / quantScale
Div(tmpUbF32Act, tmpUbF32Act, tmpUbF32Gate, tl_->UbFactorDimy * proDimsx);
PipeBarrier<PIPE_V>();
// Copy quant offset: [1,H] -> [proDimsx,H]
SetMaskCount();
SetVectorMask<float, MaskMode::COUNTER>(tl_->UbFactorDimy);
Copy<float, false>(
tmpUbF32Gate, inScaleLocal[tl_->outDimy], AscendC::MASK_PLACEHOLDER, proDimsx,
{1, 1, static_cast<uint16_t>(tl_->UbFactorDimy / BLOCK_ELEM), 0});
SetMaskNorm();
ResetMask();
PipeBarrier<PIPE_V>();
// Calc quant: y = tmpUbF32Act + quantOffset
Add(tmpUbF32Act, tmpUbF32Act, tmpUbF32Gate, tl_->UbFactorDimy * proDimsx);
PipeBarrier<PIPE_V>();
}
}
// do cast float to int8
LocalTensor<float> outLocal = outQueue_.AllocTensor<float>();
LocalTensor<int8_t> yOut = outLocal.template ReinterpretCast<int8_t>();
CastFloatToInt8(tmpUbF32Act, tmpUbF32Gate, proDimsx, yOut);
outQueue_.EnQue<float>(outLocal);
}
TEMPLATE_DSQ_DECLARE
__aicore__ inline void DequantSwigluQuantBase<TEMPLATE_DSQ_ARGS>::CastFloatToInt8(
const LocalTensor<float>& tmpUbF32Act, const LocalTensor<float>& tmpUbF32Gate, uint32_t proDimsx, LocalTensor<int8_t>& yOut)
{
LocalTensor<int32_t> tmpUbF32ActI32 = tmpUbF32Act.ReinterpretCast<int32_t>();
Cast(tmpUbF32ActI32, tmpUbF32Act, RoundMode::CAST_RINT, tl_->UbFactorDimy * proDimsx);
SetDeqScale((half)1.000000e+00f);
LocalTensor<half> tmpUbF32Gate16 = tmpUbF32Gate.template ReinterpretCast<half>();
Cast(tmpUbF32Gate16, tmpUbF32ActI32, RoundMode::CAST_ROUND, tl_->UbFactorDimy * proDimsx);
PipeBarrier<PIPE_V>();
Cast(yOut, tmpUbF32Gate16, RoundMode::CAST_TRUNC, tl_->UbFactorDimy * proDimsx);
PipeBarrier<PIPE_V>();
}
TEMPLATE_DSQ_DECLARE
template<typename T>
__aicore__ inline void DequantSwigluQuantBase<TEMPLATE_DSQ_ARGS>::CopyReshape(
LocalTensor<T>& dstTensor, LocalTensor<T>& oriTensor, uint32_t rowNum, uint32_t colNum, CopyRepeatParams param)
{
SetMaskCount();
SetVectorMask<T, MaskMode::COUNTER>(colNum);
Copy<T, false>(dstTensor, oriTensor, AscendC::MASK_PLACEHOLDER, rowNum, param);
SetMaskNorm();
ResetMask();
}
} // namespace DequantSwigluQuantOps
#endif // DEQUANT_SWIGLU_QUANT_H

1411
csrc/moe/dequant_swiglu_quant/op_kernel/dequant_swiglu_quant_apt.cpp Normal file
View File

File diff suppressed because it is too large Load Diff

62
csrc/moe/dequant_swiglu_quant/op_kernel/dequant_swiglu_quant_cut_group.h Normal file
View File

@@ -0,0 +1,62 @@
/**
* Copyright (c) 2025 Huawei Technologies Co., Ltd.
* This program is free software, you can redistribute it and/or modify it under the terms and conditions of
* CANN Open Software License Agreement Version 2.0 (the "License").
* Please refer to the License for details. You may not use this file except in compliance with the License.
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
* INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
* See LICENSE in the root of the software repository for the full text of the License.
*/
/*!
* \file dequant_swiglu_quant_cut_group.h
* \brief
*/
#ifndef DEQUANT_SWIGLU_QUANT_CUT_GROUP_H
#define DEQUANT_SWIGLU_QUANT_CUT_GROUP_H
#include "kernel_tiling/kernel_tiling.h"
#include "kernel_operator.h"
#include "dequant_swiglu_quant.h"
namespace DequantSwigluQuantGroupOps {
using namespace AscendC;
constexpr static int64_t GROUPINDEX_STRIDE = 2;
TEMPLATE_DSQ_DECLARE
class DequantSwigluQuantGroup : public DequantSwigluQuantOps::DequantSwigluQuantBase<TEMPLATE_DSQ_ARGS> {
public:
__aicore__ inline DequantSwigluQuantGroup(TPipe* pipe) : DequantSwigluQuantOps::DequantSwigluQuantBase<TEMPLATE_DSQ_ARGS>(pipe)
{
this->pipe_ = pipe;
};
__aicore__ inline void Process();
};
// 公共函数实现
TEMPLATE_DSQ_DECLARE
__aicore__ inline void DequantSwigluQuantGroup<TEMPLATE_DSQ_ARGS>::Process() {
this->CreateOffsetLocalTensor(this->UbSingleOutSize_, this->tl_->swigluMode);
this->groupOffset_ = 0;
int64_t cuGroupIdx = this->blockIdx_;
for (int32_t groupIdx = 0; groupIdx < this->tl_->inGroupNum; ++groupIdx) {
int64_t realGroupIdx = this->tl_->speGroupType == 0 ? static_cast<int64_t>(groupIdx) :
static_cast<int64_t>(this->groupIndexGm_(groupIdx*GROUPINDEX_STRIDE));
this->realDimx_ = this->tl_->speGroupType == 0 ? static_cast<int64_t>(this->groupIndexGm_(groupIdx)) :
static_cast<int64_t>(this->groupIndexGm_(groupIdx*GROUPINDEX_STRIDE + 1));
if (this->realDimx_ <= 0 && this->tl_->speGroupType) {
break;
}
if (groupIdx == cuGroupIdx) {
if (this->realDimx_ > 0) {
this->ProcessSingleGroupPerCore(realGroupIdx, this->realDimx_, this->groupOffset_);
}
cuGroupIdx += this->tl_->maxCoreNum;
}
this->groupOffset_ += this->realDimx_;
}
}
} // namespace DequantSwigluQuantGroupOps
#endif // DEQUANT_SWIGLU_QUANT_CUT_GROUP_H

592
csrc/moe/dequant_swiglu_quant/op_kernel/dequant_swiglu_quant_dynamic_base.hpp Normal file
View File

@@ -0,0 +1,592 @@
/**
* Copyright (c) 2025 Huawei Technologies Co., Ltd.
* This program is free software, you can redistribute it and/or modify it under the terms and conditions of
* CANN Open Software License Agreement Version 2.0 (the "License").
* Please refer to the License for details. You may not use this file except in compliance with the License.
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
* INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
* See LICENSE in the root of the software repository for the full text of the License.
*/
/*!
* \file dequant_swiglu_quant_dynamic_base.hpp
* \brief
*/
#ifndef CANN_DEQUANT_SWIGLU_QUANT_DYNAMIC_BASE_HPP
#define CANN_DEQUANT_SWIGLU_QUANT_DYNAMIC_BASE_HPP
#include "kernel_operator.h"
#define TEMPLATE_DECLARE template<typename InType, typename CalcType, typename BiasType, typename OutType, uint16_t bufferNum, uint16_t quantIsOne>
#define TEMPLATE_ARGS InType, CalcType, BiasType, OutType, bufferNum, quantIsOne
namespace DequantSwigluQuant {
constexpr uint32_t DOUBLE = 2;
using namespace AscendC;
TEMPLATE_DECLARE
class DequantSwigluQuantDynamicBase {
public:
__aicore__ inline DequantSwigluQuantDynamicBase() {}
__aicore__ inline ~DequantSwigluQuantDynamicBase() {}
__aicore__ inline void InitCommon(GM_ADDR x_gm, GM_ADDR weight_scale_gm, GM_ADDR activation_scale_gm, GM_ADDR bias_gm,
GM_ADDR quant_scale_gm, GM_ADDR quant_offset_gm, GM_ADDR y_gm, GM_ADDR scale_gm,
GM_ADDR userspace, const SwiGluTilingData* tilingData, TPipe* pipe_)
{
pipe = pipe_;
curBlockIdx = GetBlockIdx();
activateLeft = tilingData->activateLeft;
quantScaleIsEmpty = tilingData->quantScaleIsEmpty;
activateScaleIsEmpty = tilingData->activateScaleIsEmpty;
biasIsEmpty = tilingData->biasIsEmpty;
colNum = tilingData->colLen;
rowNum = tilingData->rowLen;
useCoreNum = tilingData->usedCoreNum;
// 每行为全载情况下每次最大拷贝行数
baseRowLen = tilingData->baseRowLen;
baseColLen = tilingData->baseColLen;
if (rowNum < useCoreNum) {
useCoreNum = rowNum;
}
// 行全载和不全载情况下，分别判断是否对齐
isMultiCols = baseRowLen == 1 && this->baseColLen < this->colNum;
if (isMultiCols) {
isOut32BAligned = baseColLen == Align(baseColLen, sizeof(InType));
} else {
isOut32BAligned = (colNum % blockBytes == 0) || (baseRowLen == 1);
}
perRoundCnt = useCoreNum == 0 ? 0 : rowNum / useCoreNum;
uint32_t remainCnt = rowNum - useCoreNum * perRoundCnt;
numRound = perRoundCnt;
if (curBlockIdx < remainCnt) {
numRound = perRoundCnt + 1;
biasOffset = curBlockIdx * (perRoundCnt + 1);
} else {
biasOffset = (perRoundCnt + 1) * remainCnt + (curBlockIdx - remainCnt) * perRoundCnt;
}
xGm.SetGlobalBuffer((__gm__ InType*)x_gm + biasOffset * colNum * DOUBLE, colNum * numRound * DOUBLE);
scaleGm.SetGlobalBuffer((__gm__ float*)scale_gm, rowNum);
yGm.SetGlobalBuffer((__gm__ int8_t*)y_gm + biasOffset * colNum, numRound * colNum);
quantScaleGm.SetGlobalBuffer((__gm__ float*)quant_scale_gm, colNum);
if (this->quantScaleIsEmpty == 0) {
if constexpr (quantIsOne == 1) {
quant_scale = ((__gm__ float*)quant_scale_gm)[0];
}
}
if (isMultiCols) {
swigluTmpGm.SetGlobalBuffer((__gm__ float*)userspace + curBlockIdx * colNum, colNum);
}
}
__aicore__ inline void BaseProcess()
{
if (this->curBlockIdx >= this->useCoreNum) {
return;
}
this->maxTempLocal = this->outQueueS.template AllocTensor<float>();
uint32_t offset1 = 0;
uint32_t offset2 = this->colNum;
if (this->activateLeft == 0) {
offset1 = this->colNum;
offset2 = 0;
}
if (!this->isMultiCols) {
this->CanFullLocaOneRow(offset1, offset2);
} else {
this->CanNotFullLocaOneRow(offset1, offset2);
}
this->CopyOutScale(this->numRound);
}
__aicore__ inline void InitUbBufferCommon(uint64_t tileLength, uint32_t realRowLen)
{
uint64_t alignTileLength = tileLength;
if (!isOut32BAligned) {
alignTileLength = Align(tileLength, sizeof(int8_t));
}
pipe->InitBuffer(inputTempBufferBF16D, alignTileLength * sizeof(CalcType) * baseRowLen);
pipe->InitBuffer(outputTempBufferBF16D, alignTileLength * sizeof(CalcType) * baseRowLen);
pipe->InitBuffer(inQueueA, 1, alignTileLength * sizeof(InType) * baseRowLen);
pipe->InitBuffer(inQueueB, 1, alignTileLength * sizeof(InType) * baseRowLen);
pipe->InitBuffer(swiGluQueue, 1, alignTileLength * sizeof(float) * baseRowLen);
if (quantScaleIsEmpty == 0) {
if (quantIsOne == 0) {
pipe->InitBuffer(inQueueQuantScale, bufferNum, alignTileLength * sizeof(float));
}
}
pipe->InitBuffer(outQueueF, 1, alignTileLength * sizeof(int8_t) * baseRowLen);
pipe->InitBuffer(outQueueS, 1, AlignBytes(realRowLen, sizeof(float)));
}
__aicore__ inline float dynamicMultiColMax(uint64_t rowId, uint64_t tileLen, int64_t colLoop)
{
LocalTensor<float> swiLocal = swiGluQueue.template DeQue<float>();
LocalTensor<float> absTempLocal = inputTempBufferBF16D.Get<float>();
Abs(absTempLocal, swiLocal, tileLen);
PipeBarrier<PIPE_V>();
ReduceMax(maxTempLocal[rowId], absTempLocal, absTempLocal, tileLen);
DataCopyExtParams dataCopyParams{1, static_cast<uint32_t>(tileLen * sizeof(float)), 0, 0, 0};
DataCopyPad(swigluTmpGm[colLoop * baseColLen], swiLocal, dataCopyParams);
swiGluQueue.FreeTensor(swiLocal);
return maxTempLocal.GetValue(rowId);
}
__aicore__ inline void dynamicAllColOut(uint64_t rowId, uint64_t tileLen, int64_t colLoop, float scale)
{
LocalTensor<float> swiLocal = swiGluQueue.template AllocTensor<float>();
DataCopyExtParams dataCopyParams{1, static_cast<uint32_t>(tileLen * sizeof(float)), 0, 0, 0};
DataCopyPadExtParams<float> dataCopyPadParams{false, 0, 0, 0};
DataCopyPad(swiLocal, swigluTmpGm[colLoop * baseColLen], dataCopyParams, dataCopyPadParams);
event_t eventId = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::MTE2_S));
SetFlag<HardEvent::MTE2_S>(eventId);
WaitFlag<HardEvent::MTE2_S>(eventId);
Muls(swiLocal, swiLocal, scale, tileLen);
PipeBarrier<PIPE_V>();
LocalTensor<int16_t> int16Local = outputTempBufferBF16D.Get<int16_t>();
Cast(int16Local, swiLocal, RoundMode::CAST_RINT, tileLen);
PipeBarrier<PIPE_V>();
swiGluQueue.FreeTensor(swiLocal);
// int16-> half
LocalTensor<half> halfLocal = int16Local.ReinterpretCast<half>();
Cast(halfLocal, int16Local, RoundMode::CAST_NONE, tileLen);
PipeBarrier<PIPE_V>();
// half -> int8_t
LocalTensor<int8_t> outLocal = outQueueF.template AllocTensor<int8_t>();
Cast(outLocal, halfLocal, RoundMode::CAST_NONE, tileLen);
outQueueF.EnQue(outLocal);
outLocal = outQueueF.DeQue<int8_t>();
event_t eventId2 = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::MTE3_S));
SetFlag<HardEvent::MTE3_S>(eventId2);
WaitFlag<HardEvent::MTE3_S>(eventId2);
DataCopyExtParams intriParams{1, static_cast<uint32_t>(tileLen), 0, 0, 0};
DataCopyPad(yGm[rowId * colNum + colLoop * baseColLen], outLocal, intriParams);
outQueueF.FreeTensor(outLocal);
}
__aicore__ inline void DynamicCompute(uint64_t rowId, uint64_t tileLen, uint64_t length)
{
LocalTensor<float> swiLocal = swiGluQueue.template DeQue<float>();
LocalTensor<float> absTempLocal = inputTempBufferBF16D.Get<float>();
Abs(absTempLocal, swiLocal, tileLen);
uint32_t offsetCalc = (length == 0 ? 0 : (tileLen / length));
PipeBarrier<PIPE_V>();
for (int64_t i = 0; i < length; i++) {
ReduceMax(maxTempLocal[rowId * baseRowLen + i], absTempLocal[i * offsetCalc], absTempLocal[i * offsetCalc], colNum);
PipeBarrier<PIPE_V>();
event_t eventIdV2S = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::V_S));
SetFlag<HardEvent::V_S>(eventIdV2S);
WaitFlag<HardEvent::V_S>(eventIdV2S);
float value = maxTempLocal.GetValue(rowId * baseRowLen + i) / 127;
maxTempLocal.SetValue(rowId * baseRowLen + i, value);
float scale = 1 / value;
Muls(swiLocal[i * offsetCalc], swiLocal[i * offsetCalc], scale, colNum);
PipeBarrier<PIPE_V>();
}
LocalTensor<int16_t> int16Local = outputTempBufferBF16D.Get<int16_t>();
Cast(int16Local, swiLocal, RoundMode::CAST_RINT, tileLen);
PipeBarrier<PIPE_V>();
// int16-> half
LocalTensor<half> halfLocal = int16Local.ReinterpretCast<half>();
Cast(halfLocal, int16Local, RoundMode::CAST_NONE, tileLen);
PipeBarrier<PIPE_V>();
swiGluQueue.FreeTensor(swiLocal);
// half -> int8_t
LocalTensor<int8_t> outLocal = outQueueF.template AllocTensor<int8_t>();
Cast(outLocal, halfLocal, RoundMode::CAST_NONE, tileLen);
event_t eventId1 = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::V_MTE3));
SetFlag<HardEvent::V_MTE3>(eventId1);
WaitFlag<HardEvent::V_MTE3>(eventId1);
if (isOut32BAligned) {
DataCopyExtParams intriParams{1, static_cast<uint32_t>(tileLen), 0, 0, 0};
DataCopyPad(yGm[rowId * colNum * baseRowLen], outLocal, intriParams);
} else {
DataCopyExtParams intriParams{static_cast<uint16_t>(length), static_cast<uint32_t>(colNum), 0, 0, 0};
DataCopyPad(yGm[rowId * colNum * baseRowLen], outLocal, intriParams);
}
outQueueF.FreeTensor(outLocal);
}
__aicore__ inline void BaseComputeWithQuantScale(LocalTensor<CalcType> &outTmpLocal, LocalTensor<CalcType> &bLocal,
uint64_t curTileLen, uint64_t blockCount)
{
LocalTensor<float> swiLocal = swiGluQueue.template AllocTensor<float>();
Mul(swiLocal, outTmpLocal, bLocal, curTileLen);
if (quantScaleIsEmpty == 0) {
PipeBarrier<PIPE_V>();
if constexpr (quantIsOne == 0) {
uint32_t calcOffset = (blockCount == 0 ? 0 : curTileLen / blockCount);
for (uint64_t idx = 0; idx < blockCount; idx++) {
Mul(swiLocal[idx * calcOffset], swiLocal[idx * calcOffset], quantScaleLocal, calcOffset);
}
} else {
Muls(swiLocal, swiLocal, quant_scale, curTileLen);
}
}
swiGluQueue.template EnQue<float>(swiLocal);
}
__aicore__ inline void BaseCompute(uint64_t curTileLen, uint64_t blockCount, uint64_t idx)
{
LocalTensor<InType> inALocal = this->inQueueA.template DeQue<InType>();
LocalTensor<CalcType> outTmpLocal = this->outputTempBufferBF16D.template Get<CalcType>();
LocalTensor<CalcType> inputTmpLocal = this->inputTempBufferBF16D.template Get<CalcType>();
float value = this->getActivateScaleValue(idx);
if constexpr (std::is_same_v<InType, int32_t>) {
if constexpr (std::is_same_v<BiasType, int32_t>) {
this->addBiasWithBiasInt(inALocal, this->biasLocalA, curTileLen);
}
}
Cast(inputTmpLocal, inALocal, RoundMode::CAST_NONE, curTileLen);
PipeBarrier<PIPE_V>();
this->inQueueA.template FreeTensor(inALocal);
if constexpr (std::is_same_v<InType, int32_t>) {
addWeightScaleAndActivateScale(inputTmpLocal, this->weightScaleLocalA, curTileLen, value);
if constexpr (std::is_same_v<BiasType, float> || std::is_same_v<BiasType, bfloat16_t> || std::is_same_v<BiasType, half>) {
if (this->biasIsEmpty == 0) {
addBiasWithBiasFloat(inputTmpLocal, this->biasLocalA, curTileLen);
}
}
}
Muls(outTmpLocal, inputTmpLocal, this->beta, curTileLen);
PipeBarrier<PIPE_V>();
Exp(outTmpLocal, outTmpLocal, curTileLen);
PipeBarrier<PIPE_V>();
Adds(outTmpLocal, outTmpLocal, CalcType(1.0), curTileLen);
PipeBarrier<PIPE_V>();
Div(outTmpLocal, inputTmpLocal, outTmpLocal, curTileLen);
PipeBarrier<PIPE_V>();
LocalTensor<InType> bLocal_ = this->inQueueB.template DeQue<InType>();
if constexpr (std::is_same_v<InType, int32_t>) {
if constexpr (std::is_same_v<BiasType, int32_t>) {
this->addBiasWithBiasInt(bLocal_, this->biasLocalB, curTileLen);
}
}
LocalTensor<CalcType> bLocal = this->inputTempBufferBF16D.template Get<CalcType>();
Cast(bLocal, bLocal_, RoundMode::CAST_NONE, curTileLen);
PipeBarrier<PIPE_V>();
if constexpr (std::is_same_v<InType, int32_t>) {
addWeightScaleAndActivateScale(bLocal, this->weightScaleLocalB, curTileLen, value);
if constexpr (std::is_same_v<BiasType, float> || std::is_same_v<BiasType, bfloat16_t> || std::is_same_v<BiasType, half>) {
if (this->biasIsEmpty == 0) {
addBiasWithBiasFloat(bLocal, this->biasLocalB, curTileLen);
}
}
}
this->inQueueB.template FreeTensor(bLocal_);
BaseComputeWithQuantScale(outTmpLocal, bLocal, curTileLen, blockCount);
}
__aicore__ inline void CanFullLocaOneRow(uint32_t offset1, uint32_t offset2)
{
if (this->quantScaleIsEmpty == 0) {
if constexpr (quantIsOne == 0) {
this->CopyInQuantScale(this->colNum, 0);
}
}
CopyInDequantBuffer(offset1, offset2, this->colNum);
this->alignSize = this->isOut32BAligned ? this->colNum : this->Align(this->colNum, sizeof(int8_t));
int64_t blockCount = this->baseRowLen;
int64_t loops = (this->numRound + blockCount - 1) / blockCount;
int64_t lastLoopBlockCount = this->numRound - (loops - 1) * blockCount;
int64_t lastLoopColSize = this->alignSize * lastLoopBlockCount;
int64_t perLoopColSize = this->alignSize * blockCount;
uint32_t aligCalcNum = this->Align(this->colNum, sizeof(InType));
uint32_t alig8CalcNum = this->Align(this->colNum, sizeof(int8_t));
this->dstStride = this->isOut32BAligned ? 0 : (alig8CalcNum - this->colNum) * sizeof(InType) / this->blockBytes;
for (uint32_t i = 0; i < loops - 1; i++) {
uint32_t base = i * (this->colNum * DOUBLE) * this->baseRowLen;
this->CopyIn(this->colNum, offset1 + base, offset2 + base, blockCount);
this->BaseCompute(perLoopColSize, blockCount, i);
this->DynamicCompute(i, perLoopColSize, blockCount);
}
uint32_t base = (loops - 1) * (this->colNum * DOUBLE) * this->baseRowLen;
this->CopyIn(this->colNum, offset1 + base, offset2 + base, lastLoopBlockCount);
this->BaseCompute(lastLoopColSize, lastLoopBlockCount, (loops - 1));
this->DynamicCompute((loops - 1), lastLoopColSize, lastLoopBlockCount);
if (this->quantScaleIsEmpty == 0) {
if constexpr (quantIsOne == 0) {
this->inQueueQuantScale.FreeTensor(this->quantScaleLocal);
}
}
FreeDequantBuffer();
}
__aicore__ inline void CanNotFullLocaOneRow(uint32_t offset1, uint32_t offset2)
{
int64_t colLoops = (this->colNum + this->baseColLen - 1) / this->baseColLen;
int64_t lastColNum = this->colNum - (colLoops - 1) * this->baseColLen;
for (uint32_t i = 0; i < this->numRound; i++) {
uint32_t tmp = 0xFF7FFFFF;
float reduceMax = *((float*)&tmp);
for (uint32_t j = 0; j < colLoops; j++) {
int64_t curColNum = this->baseColLen;
if (j == colLoops - 1) {
curColNum = lastColNum;
}
if (this->quantScaleIsEmpty == 0) {
if constexpr (quantIsOne == 0) {
this->CopyInQuantScale(curColNum, j * this->baseColLen);
}
}
bool isOutAligned = curColNum == this->Align(curColNum, sizeof(InType));
uint32_t alignColNum = isOutAligned ? curColNum : this->Align(curColNum, sizeof(OutType));
uint32_t base = i * (this->colNum * DOUBLE) + j * this->baseColLen;
CopyInDequantBuffer(offset1 + j * this->baseColLen, offset2 + j * this->baseColLen, curColNum);
this->CopyIn(curColNum, offset1 + base, offset2 + base, 1);
this->BaseCompute(alignColNum, 1, i);
float maxValue = this->dynamicMultiColMax(i, curColNum, j);
if (maxValue > reduceMax) {
reduceMax = maxValue;
}
if (this->quantScaleIsEmpty == 0) {
if constexpr (quantIsOne == 0) {
this->inQueueQuantScale.FreeTensor(this->quantScaleLocal);
}
}
FreeDequantBuffer();
}
float value = reduceMax / 127.0f;
this->maxTempLocal.SetValue(i, value);
float scale = 1 / value;
event_t eventId = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::MTE3_MTE2));
SetFlag<HardEvent::MTE3_MTE2>(eventId);
WaitFlag<HardEvent::MTE3_MTE2>(eventId);
for (uint32_t j = 0; j < colLoops; j++) {
int64_t curColNum = this->baseColLen;
if (j == colLoops - 1) {
curColNum = lastColNum;
}
bool isOutAligned = curColNum == this->Align(curColNum, sizeof(InType));
uint32_t alignColNum = isOutAligned ? curColNum : this->Align(curColNum, sizeof(OutType));
this->dynamicAllColOut(i, curColNum, j, scale);
}
}
}
__aicore__ inline void CopyInDequantBuffer(uint32_t offset1, uint32_t offset2, uint32_t dataTileLen)
{
if constexpr (std::is_same_v<InType, int32_t>) {
this->CopyInWeightAndBias(dataTileLen, offset1, offset2);
this->CopyInActivateScale(0, this->numRound);
if (this->biasIsEmpty == 0) {
this->biasLocalA = this->inBiasQueueA.template DeQue<BiasType>();
this->biasLocalB = this->inBiasQueueB.template DeQue<BiasType>();
}
this->weightScaleLocalA = this->weightScaleQueueA.template DeQue<float>();
this->weightScaleLocalB = this->weightScaleQueueB.template DeQue<float>();
if (this->activateScaleIsEmpty == 0) {
this->activateLocal = this->inQueueActivationScale.template DeQue<float>();
}
}
}
__aicore__ inline void FreeDequantBuffer()
{
if constexpr (std::is_same_v<InType, int32_t>) {
if (this->biasIsEmpty == 0) {
this->inBiasQueueA.FreeTensor(this->biasLocalA);
this->inBiasQueueB.FreeTensor(this->biasLocalB);
}
this->weightScaleQueueA.FreeTensor(this->weightScaleLocalA);
this->weightScaleQueueB.FreeTensor(this->weightScaleLocalB);
if (this->activateScaleIsEmpty == 0) {
this->inQueueActivationScale.FreeTensor(this->activateLocal);
}
}
}
__aicore__ inline float getActivateScaleValue(uint64_t idx)
{
float value = 1;
if constexpr (std::is_same_v<InType, int32_t>) {
if (activateScaleIsEmpty == 0) {
event_t eventIdM2S = static_cast<event_t>(GetTPipePtr()->FetchEventID(HardEvent::MTE2_S));
SetFlag<HardEvent::MTE2_S>(eventIdM2S);
WaitFlag<HardEvent::MTE2_S>(eventIdM2S);
value = activateLocal.GetValue(idx);
}
}
return value;
}
__aicore__ inline void addWeightScaleAndActivateScale(
LocalTensor<CalcType> &dstLocal, LocalTensor<CalcType> &weightScaleLocal, uint64_t curTileLen, float value)
{
Mul(dstLocal, dstLocal, weightScaleLocal, curTileLen);
PipeBarrier<PIPE_V>();
if (activateScaleIsEmpty == 0) {
Muls(dstLocal, dstLocal, value, curTileLen);
PipeBarrier<PIPE_V>();
}
}
__aicore__ inline void addBiasWithBiasInt(LocalTensor<InType> &dstLocal, LocalTensor<BiasType> &biasLocal, uint64_t curTileLen)
{
if (this->biasIsEmpty == 0) {
Add(dstLocal, dstLocal, biasLocal, curTileLen);
PipeBarrier<PIPE_V>();
}
}
__aicore__ inline void addBiasWithBiasFloat(LocalTensor<CalcType> &dstLocal, LocalTensor<BiasType> &biasLocal, uint64_t curTileLen)
{
if (this->biasIsEmpty == 0) {
if constexpr (std::is_same_v<BiasType, float>) {
Add(dstLocal, dstLocal, biasLocal, curTileLen);
} else {
Cast(biasFloatLocalB, biasLocal, RoundMode::CAST_NONE, curTileLen);
PipeBarrier<PIPE_V>();
Add(dstLocal, dstLocal, biasFloatLocalB, curTileLen);
}
PipeBarrier<PIPE_V>();
}
}
__aicore__ inline void CopyOutScale(uint32_t realRowLen)
{
DataCopyExtParams intriParams{1, static_cast<uint32_t>(sizeof(float) * realRowLen), 0, 0, 0};
DataCopyPad(scaleGm[biasOffset], maxTempLocal, intriParams);
outQueueS.FreeTensor(maxTempLocal);
}
__aicore__ inline void CopyInActivateScale(uint32_t offset3, uint32_t blockCount)
{
if (activateScaleIsEmpty == 0) {
DataCopyExtParams activateparams = {1, static_cast<uint32_t>(blockCount * sizeof(float)), 0, 0, 0};
DataCopyPadExtParams<float> padParams{false, 0, 0, 0};
LocalTensor<float> activateLocal1 = inQueueActivationScale.template AllocTensor<float>();
DataCopyPad(activateLocal1, activationScaleGm[offset3], activateparams, padParams);
inQueueActivationScale.EnQue(activateLocal1);
}
}
__aicore__ inline void CopyInWeightAndBias(uint32_t dataTileLen, uint32_t offset1, uint32_t offset2)
{
DataCopyExtParams params = {1, static_cast<uint32_t>(dataTileLen * sizeof(float)), 0, 0, 0};
DataCopyExtParams paramsBias = {1, static_cast<uint32_t>(dataTileLen * sizeof(BiasType)), 0, 0, 0};
DataCopyPadExtParams<float> padParams{false, 0, 0, 0};
DataCopyPadExtParams<BiasType> padParams1{false, 0, 0, 0};
if (this->biasIsEmpty == 0) {
// copy bias A
LocalTensor<BiasType> biasLocalA1 = inBiasQueueA.template AllocTensor<BiasType>();
DataCopyPad(biasLocalA1, biasGm[offset1], paramsBias, padParams1);
inBiasQueueA.EnQue(biasLocalA1);
// copy bias B
LocalTensor<BiasType> biasLocalB1 = inBiasQueueB.template AllocTensor<BiasType>();
DataCopyPad(biasLocalB1, biasGm[offset2], paramsBias, padParams1);
inBiasQueueB.EnQue(biasLocalB1);
}
// copy ws A
LocalTensor<float> wsLocalA1 = weightScaleQueueA.template AllocTensor<float>();
DataCopyPad(wsLocalA1, weightScaleGm[offset1], params, padParams);
weightScaleQueueA.EnQue(wsLocalA1);
// copy ws B
LocalTensor<float> wsLocalB1 = weightScaleQueueB.template AllocTensor<float>();
DataCopyPad(wsLocalB1, weightScaleGm[offset2], params, padParams);
weightScaleQueueB.EnQue(wsLocalB1);
}
__aicore__ inline void CopyInQuantScale(uint64_t dataTileLength, uint64_t offset)
{
DataCopyExtParams dataCopyParams{1, static_cast<uint32_t>(dataTileLength * sizeof(float)), 0, 0, 0};
DataCopyPadExtParams<float> dataCopyPadParams{false, 0, 0, 0};
LocalTensor<float> scaleLocal = inQueueQuantScale.template AllocTensor<float>();
DataCopyPad(scaleLocal, quantScaleGm[offset], dataCopyParams, dataCopyPadParams);
inQueueQuantScale.EnQue(scaleLocal);
quantScaleLocal = inQueueQuantScale.template DeQue<float>();
}
__aicore__ inline void CopyIn(uint32_t dataTileLen, uint32_t offset1, uint32_t offset2, uint32_t blockCount)
{
uint32_t srcStride = dataTileLen * sizeof(InType);
DataCopyExtParams dataCopyParams{static_cast<uint16_t>(blockCount),
static_cast<uint32_t>(dataTileLen * sizeof(InType)), srcStride, dstStride, 0};
DataCopyPadExtParams<InType> dataCopyPadParams{false, 0, 0, 0};
// Copy A
LocalTensor<InType> aLocal = inQueueA.template AllocTensor<InType>();
DataCopyPad(aLocal, xGm[offset1], dataCopyParams, dataCopyPadParams);
inQueueA.EnQue(aLocal);
// Copy B
LocalTensor<InType> bLocal = inQueueB.template AllocTensor<InType>();
DataCopyPad(bLocal, xGm[offset2], dataCopyParams, dataCopyPadParams);
inQueueB.EnQue(bLocal);
}
__aicore__ inline int64_t Align(int64_t elementNum, int64_t bytes)
{
if (bytes == 0) {
return 0;
}
return (elementNum * bytes + blockBytes - 1) / blockBytes * blockBytes / bytes;
}
__aicore__ inline int64_t AlignBytes(int64_t elementNum, int64_t bytes)
{
return (elementNum * bytes + blockBytes - 1) / blockBytes * blockBytes;
}
protected:
TPipe* pipe;
GlobalTensor<InType> xGm;
GlobalTensor<float> quantScaleGm;
GlobalTensor<float> weightScaleGm;
GlobalTensor<float> activationScaleGm;
GlobalTensor <BiasType> biasGm;
GlobalTensor<OutType> yGm;
GlobalTensor<float> scaleGm;
GlobalTensor<float> swigluTmpGm;
TBuf<TPosition::VECCALC> inputTempBufferBF16D;
TBuf<TPosition::VECCALC> outputTempBufferBF16D;
TBuf<TPosition::VECCALC> inputBiasTempBufferA;
TBuf<TPosition::VECCALC> inputBiasTempBufferB;
TQue<QuePosition::VECIN, 1> inQueueA;
TQue<QuePosition::VECIN, 1> inQueueB;
TQue<QuePosition::VECIN, 1> inQueueQuantScale;
TQue<QuePosition::VECOUT, 1> swiGluQueue;
TQue<QuePosition::VECOUT, 1> outQueueF;
TQue<QuePosition::VECOUT, 1> outQueueS;
TQue<QuePosition::VECIN, 1> inBiasQueueA;
TQue<QuePosition::VECIN, 1> inBiasQueueB;
TQue<QuePosition::VECIN, 1> weightScaleQueueA;
TQue<QuePosition::VECIN, 1> weightScaleQueueB;
TQue<QuePosition::VECIN, 1> inQueueActivationScale;
LocalTensor<float> maxTempLocal;
LocalTensor<float> quantScaleLocal;
LocalTensor<float> weightScaleLocalA;
LocalTensor<float> weightScaleLocalB;
LocalTensor <BiasType> biasLocalA;
LocalTensor <BiasType> biasLocalB;
LocalTensor<float> activateLocal;
LocalTensor<CalcType> biasFloatLocalA;
LocalTensor<CalcType> biasFloatLocalB;
float beta = -1.0f;
float quant_scale = 1;
uint32_t quantScaleIsEmpty = 0;
uint32_t biasIsEmpty = 0;
uint32_t activateScaleIsEmpty = 0;
uint64_t perRoundCnt = 0;
uint64_t numRound = 0;
uint32_t colNum = 0;
uint32_t rowNum = 0;
uint32_t useCoreNum = 0;
uint32_t biasOffset = 0;
uint32_t curBlockIdx = 0;
uint32_t activateLeft = 0;
uint32_t baseRowLen = 0;
uint32_t baseColLen = 0;
uint32_t alignSize = 0;
bool isOut32BAligned = true;
uint32_t dstStride = 0;
bool isMultiCols = false;
int64_t blockBytes = 32;
};
}
#endif // CANN_DEQUANT_SWIGLU_QUANT_DYNAMIC_BASE_HPP

55
csrc/moe/dequant_swiglu_quant/op_kernel/dequant_swiglu_quant_dynamic_bf16.hpp Normal file
View File

@@ -0,0 +1,55 @@
/**
* Copyright (c) 2025 Huawei Technologies Co., Ltd.
* This program is free software, you can redistribute it and/or modify it under the terms and conditions of
* CANN Open Software License Agreement Version 2.0 (the "License").
* Please refer to the License for details. You may not use this file except in compliance with the License.
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
* INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
* See LICENSE in the root of the software repository for the full text of the License.
*/
/*!
* \file dequant_swiglu_quant_dynamic_bf16.hpp
* \brief
*/
#ifndef CANN_DEQUANT_SWIGLU_QUANT_DYNAMIC_BF16_HPP
#define CANN_DEQUANT_SWIGLU_QUANT_DYNAMIC_BF16_HPP
#include "kernel_operator.h"
#include "dequant_swiglu_quant_dynamic_base.hpp"
namespace DequantSwigluQuant {
using namespace AscendC;
TEMPLATE_DECLARE
class DequantSwigluQuantDynamicBF16 : public DequantSwigluQuantDynamicBase<TEMPLATE_ARGS> {
public:
__aicore__ inline DequantSwigluQuantDynamicBF16(){};
__aicore__ inline ~DequantSwigluQuantDynamicBF16(){};
__aicore__ inline void Init(GM_ADDR x_gm, GM_ADDR weight_scale_gm, GM_ADDR activation_scale_gm, GM_ADDR bias_gm,
GM_ADDR quant_scale_gm, GM_ADDR quant_offset_gm, GM_ADDR y_gm, GM_ADDR scale_gm,
GM_ADDR userspace, const SwiGluTilingData* tilingData, TPipe* pipe_);
__aicore__ inline void Process();
};
TEMPLATE_DECLARE
__aicore__ inline void DequantSwigluQuantDynamicBF16<TEMPLATE_ARGS>::Init(
GM_ADDR x_gm, GM_ADDR weight_scale_gm, GM_ADDR activation_scale_gm, GM_ADDR bias_gm, GM_ADDR quant_scale_gm,
GM_ADDR quant_offset_gm, GM_ADDR y_gm, GM_ADDR scale_gm, GM_ADDR userspace, const SwiGluTilingData* tilingData,
TPipe* pipe_) {
this->InitCommon(x_gm, weight_scale_gm, activation_scale_gm, bias_gm, quant_scale_gm, quant_offset_gm, y_gm, scale_gm, userspace, tilingData, pipe_);
if (this->numRound < this->baseRowLen) {
this->baseRowLen = this->numRound;
}
this->InitUbBufferCommon(this->baseColLen, this->numRound);
}
TEMPLATE_DECLARE
__aicore__ inline void DequantSwigluQuantDynamicBF16<TEMPLATE_ARGS>::Process() {
this->BaseProcess();
}
} // namespace DequantSwigluQuant
#endif // CANN_DEQUANT_SWIGLU_QUANT_DYNAMIC_BF16_HPP

94
csrc/moe/dequant_swiglu_quant/op_kernel/dequant_swiglu_quant_dynamic_bias_float.hpp Normal file
View File

@@ -0,0 +1,94 @@
/**
* Copyright (c) 2025 Huawei Technologies Co., Ltd.
* This program is free software, you can redistribute it and/or modify it under the terms and conditions of
* CANN Open Software License Agreement Version 2.0 (the "License").
* Please refer to the License for details. You may not use this file except in compliance with the License.
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
* INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
* See LICENSE in the root of the software repository for the full text of the License.
*/
/*!
* \file dequant_swiglu_quant_dynamic_bias_float.hpp
* \brief
*/
#ifndef CANN_DEQUANT_SWIGLU_QUANT_DYNAMIC_BIAS_FLOAT_HPP
#define CANN_DEQUANT_SWIGLU_QUANT_DYNAMIC_BIAS_FLOAT_HPP
#include "kernel_operator.h"
#include "dequant_swiglu_quant_dynamic_base.hpp"
namespace DequantSwigluQuant {
using namespace AscendC;
TEMPLATE_DECLARE
class DequantSwigluQuantDynamicBiasFloat : public DequantSwigluQuantDynamicBase<TEMPLATE_ARGS> {
public:
__aicore__ inline DequantSwigluQuantDynamicBiasFloat(){};
__aicore__ inline ~DequantSwigluQuantDynamicBiasFloat(){};
__aicore__ inline void Init(GM_ADDR x_gm, GM_ADDR weight_scale_gm, GM_ADDR activation_scale_gm, GM_ADDR bias_gm,
GM_ADDR quant_scale_gm, GM_ADDR quant_offset_gm, GM_ADDR y_gm, GM_ADDR scale_gm,
GM_ADDR userspace, const SwiGluTilingData* tilingData, TPipe* pipe_);
__aicore__ inline void Process();
private:
__aicore__ inline void InitUbBuffer(uint64_t tileLength, uint32_t realRowLen);
};
TEMPLATE_DECLARE
__aicore__ inline void DequantSwigluQuantDynamicBiasFloat<TEMPLATE_ARGS>::Init(
GM_ADDR x_gm, GM_ADDR weight_scale_gm, GM_ADDR activation_scale_gm, GM_ADDR bias_gm, GM_ADDR quant_scale_gm,
GM_ADDR quant_offset_gm, GM_ADDR y_gm, GM_ADDR scale_gm, GM_ADDR userspace, const SwiGluTilingData* tilingData,
TPipe* pipe_)
{
this->InitCommon(x_gm, weight_scale_gm, activation_scale_gm, bias_gm, quant_scale_gm, quant_offset_gm, y_gm, scale_gm, userspace, tilingData, pipe_);
this->weightScaleGm.SetGlobalBuffer((__gm__ float*)weight_scale_gm, this->colNum);
if (this->biasIsEmpty == 0) {
this->biasGm.SetGlobalBuffer((__gm__ BiasType*)bias_gm, this->colNum);
}
if (this->activateScaleIsEmpty == 0) {
this->activationScaleGm.SetGlobalBuffer((__gm__ float*) activation_scale_gm + this->biasOffset,
this->numRound);
}
this->InitUbBufferCommon(this->baseColLen, this->numRound);
InitUbBuffer(this->baseColLen, this->numRound);
}
TEMPLATE_DECLARE
__aicore__ inline void DequantSwigluQuantDynamicBiasFloat<TEMPLATE_ARGS>::Process() {
this->BaseProcess();
}
TEMPLATE_DECLARE
__aicore__ inline void DequantSwigluQuantDynamicBiasFloat<TEMPLATE_ARGS>::InitUbBuffer(uint64_t tileLength,
uint32_t realRowLen)
{
uint64_t alignTileLength = tileLength;
if (!this->isOut32BAligned) {
alignTileLength = this->Align(tileLength, sizeof(int8_t));
}
if (this->biasIsEmpty == 0) {
this->pipe->InitBuffer(this->inBiasQueueA, 1, alignTileLength * sizeof(BiasType));
this->pipe->InitBuffer(this->inBiasQueueB, 1, alignTileLength * sizeof(BiasType));
if constexpr (std::is_same_v<BiasType, bfloat16_t> || std::is_same_v<BiasType, half>) {
this->pipe->InitBuffer(this->inputBiasTempBufferA, alignTileLength * sizeof(float));
this->pipe->InitBuffer(this->inputBiasTempBufferB, alignTileLength * sizeof(float));
this->biasFloatLocalA = this->inputBiasTempBufferA.template Get<CalcType>();
this->biasFloatLocalB = this->inputBiasTempBufferB.template Get<CalcType>();
}
}
this->pipe->InitBuffer(this->weightScaleQueueA, 1, alignTileLength * sizeof(float));
this->pipe->InitBuffer(this->weightScaleQueueB, 1, alignTileLength * sizeof(float));
if (this->activateScaleIsEmpty == 0) {
this->pipe->InitBuffer(this->inQueueActivationScale, 1, this->baseRowLen * sizeof(float));
}
}
} // namespace DequantSwigluQuant
#endif // CANN_DEQUANT_SWIGLU_QUANT_DYNAMIC_BIAS_FLOAT_HPP

83
csrc/moe/dequant_swiglu_quant/op_kernel/dequant_swiglu_quant_dynamic_bias_int32.hpp Normal file
View File

@@ -0,0 +1,83 @@
/**
* Copyright (c) 2025 Huawei Technologies Co., Ltd.
* This program is free software, you can redistribute it and/or modify it under the terms and conditions of
* CANN Open Software License Agreement Version 2.0 (the "License").
* Please refer to the License for details. You may not use this file except in compliance with the License.
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
* INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
* See LICENSE in the root of the software repository for the full text of the License.
*/
/*!
* \file dequant_swiglu_quant_dynamic_bias_int32.hpp
* \brief
*/
#ifndef CANN_DEQUANT_SWIGLU_QUANT_DYNAMIC_BIAS_INT32_HPP
#define CANN_DEQUANT_SWIGLU_QUANT_DYNAMIC_BIAS_INT32_HPP
#include "kernel_operator.h"
#include "dequant_swiglu_quant_dynamic_base.hpp"
namespace DequantSwigluQuant {
using namespace AscendC;
constexpr int64_t BLOCK_BYTES = 32;
TEMPLATE_DECLARE
class DequantSwigluQuantDynamicBiasInt32 : public DequantSwigluQuantDynamicBase<TEMPLATE_ARGS> {
public:
__aicore__ inline DequantSwigluQuantDynamicBiasInt32(){};
__aicore__ inline ~DequantSwigluQuantDynamicBiasInt32(){};
__aicore__ inline void Init(GM_ADDR x_gm, GM_ADDR weight_scale_gm, GM_ADDR activation_scale_gm, GM_ADDR bias_gm,
GM_ADDR quant_scale_gm, GM_ADDR quant_offset_gm, GM_ADDR y_gm, GM_ADDR scale_gm,
GM_ADDR userspace, const SwiGluTilingData* tilingData, TPipe* pipe_);
__aicore__ inline void Process();
private:
__aicore__ inline void InitUbBuffer(uint64_t tileLength, uint32_t realRowLen);
};
TEMPLATE_DECLARE
__aicore__ inline void DequantSwigluQuantDynamicBiasInt32<TEMPLATE_ARGS>::Init(
GM_ADDR x_gm, GM_ADDR weight_scale_gm, GM_ADDR activation_scale_gm, GM_ADDR bias_gm, GM_ADDR quant_scale_gm,
GM_ADDR quant_offset_gm, GM_ADDR y_gm, GM_ADDR scale_gm, GM_ADDR userspace, const SwiGluTilingData* tilingData,
TPipe* pipe_) {
this->InitCommon(x_gm, weight_scale_gm, activation_scale_gm, bias_gm, quant_scale_gm, quant_offset_gm, y_gm, scale_gm, userspace, tilingData, pipe_);
if (this->activateScaleIsEmpty == 0) {
this->activationScaleGm.SetGlobalBuffer((__gm__ float*) activation_scale_gm + this->biasOffset,
this->numRound);
}
this->weightScaleGm.SetGlobalBuffer((__gm__ float*)weight_scale_gm, this->colNum);
if (this->biasIsEmpty == 0) {
this->biasGm.SetGlobalBuffer((__gm__ BiasType*)bias_gm, this->colNum);
}
this->InitUbBufferCommon(this->baseColLen, this->numRound);
InitUbBuffer(this->baseColLen, this->numRound);
}
TEMPLATE_DECLARE
__aicore__ inline void DequantSwigluQuantDynamicBiasInt32<TEMPLATE_ARGS>::Process() {
this->BaseProcess();
}
TEMPLATE_DECLARE
__aicore__ inline void DequantSwigluQuantDynamicBiasInt32<TEMPLATE_ARGS>::InitUbBuffer(uint64_t tileLength,
uint32_t realRowLen) {
uint64_t alignTileLength = tileLength;
if (!this->isOut32BAligned) {
alignTileLength = this->Align(tileLength, sizeof(int8_t));
}
if (this->biasIsEmpty == 0) {
this->pipe->InitBuffer(this->inBiasQueueA, 1, alignTileLength * sizeof(BiasType));
this->pipe->InitBuffer(this->inBiasQueueB, 1, alignTileLength * sizeof(BiasType) );
}
this->pipe->InitBuffer(this->weightScaleQueueA, 1, alignTileLength * sizeof(float));
this->pipe->InitBuffer(this->weightScaleQueueB, 1, alignTileLength * sizeof(float));
if (this->activateScaleIsEmpty == 0) {
this->pipe->InitBuffer(this->inQueueActivationScale, 1, this->baseRowLen * sizeof(float));
}
}
} // namespace DequantSwigluQuant
#endif // CANN_DEQUANT_SWIGLU_QUANT_DYNAMIC_BIAS_INT32_HPP

384
csrc/moe/dequant_swiglu_quant/op_kernel/dequant_swiglu_quant_dynamic_performance.hpp Normal file
View File

@@ -0,0 +1,384 @@
/**
* Copyright (c) 2025 Huawei Technologies Co., Ltd.
* This program is free software, you can redistribute it and/or modify it under the terms and conditions of
* CANN Open Software License Agreement Version 2.0 (the "License").
* Please refer to the License for details. You may not use this file except in compliance with the License.
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
* INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
* See LICENSE in the root of the software repository for the full text of the License.
*/
/*!
* \file dequant_swiglu_quant_dynamic_performance.hpp
* \brief
*/
#ifndef DEQUANT_SWIGLU_QUANT_DYNAMIC_PERFORMANCE_HPP
#define DEQUANT_SWIGLU_QUANT_DYNAMIC_PERFORMANCE_HPP
#include "kernel_operator.h"
#include "dequant_swiglu_quant_dynamic_base.hpp"
namespace DequantSwigluQuant {
using namespace AscendC;
TEMPLATE_DECLARE
class DequantSwigluQuantDynamicPerformance : public DequantSwigluQuantDynamicBiasFloat<TEMPLATE_ARGS> {
public:
__aicore__ inline DequantSwigluQuantDynamicPerformance(){};
__aicore__ inline ~DequantSwigluQuantDynamicPerformance(){};
__aicore__ inline void Init(GM_ADDR x_gm, GM_ADDR weight_scale_gm, GM_ADDR activation_scale_gm, GM_ADDR bias_gm,
GM_ADDR quant_scale_gm, GM_ADDR quant_offset_gm, GM_ADDR y_gm, GM_ADDR scale_gm,
GM_ADDR userspace, const SwiGluTilingData* tilingData, TPipe* pipe_);
__aicore__ inline void Process();
__aicore__ inline void BaseCompute1(uint64_t curTileLen, uint64_t blockCount, uint64_t idx, int32_t ppFlag);
__aicore__ inline void BaseCompute2(uint64_t curTileLen, uint64_t blockCount, uint64_t idx, int32_t ppFlag);
__aicore__ inline void CopyOutF(uint64_t rowId, uint64_t tileLen, uint64_t length, int32_t ppFlag);
__aicore__ inline void CanFullLocaOneRow(uint32_t offset1, uint32_t offset2);
__aicore__ inline void CopyIn(uint32_t dataTileLen, uint32_t offset1, uint32_t offset2, uint32_t blockCount, int32_t ppFlag);
__aicore__ inline void CopyInDequantBuffer(uint32_t offset1, uint32_t offset2, uint32_t dataTileLen);
__aicore__ inline void InitCommon(GM_ADDR x_gm, GM_ADDR weight_scale_gm, GM_ADDR activation_scale_gm, GM_ADDR bias_gm,
GM_ADDR quant_scale_gm, GM_ADDR quant_offset_gm, GM_ADDR y_gm, GM_ADDR scale_gm,
GM_ADDR userspace, const SwiGluTilingData* tilingData, TPipe* pipe_);
__aicore__ inline void InitUbBufferCommon(uint64_t tileLength, uint32_t realRowLen);
__aicore__ inline void CopyOutScale(uint32_t realRowLen);
__aicore__ inline void CopyInQuantScale(uint64_t dataTileLength, uint64_t offset);
public:
uint32_t offsetCalc;
LocalTensor<CalcType> outTmpLocal;
LocalTensor<CalcType> inputTmpLocal;
LocalTensor<float> absTempLocal;
LocalTensor<int16_t> int16Local;
LocalTensor<CalcType> bLocal;
LocalTensor<float> swiLocal;
TBuf<TPosition::VECCALC> calcSwiGluTmpBuf;
TBuf<TPosition::VECCALC> weightScaleBufA;
TBuf<TPosition::VECCALC> weightScaleBufB;
TBuf<TPosition::VECCALC> quantScaleBuf;
TBuf<TPosition::VECCALC> inQueueAPingBuf;
TBuf<TPosition::VECCALC> inQueueAPongBuf;
TBuf<TPosition::VECCALC> inQueueBPingBuf;
TBuf<TPosition::VECCALC> inQueueBPongBuf;
TBuf<TPosition::VECCALC> outQueueFPingBuf;
TBuf<TPosition::VECCALC> outQueueFPongBuf;
TBuf<TPosition::VECCALC> outQueueSBuf;
LocalTensor<InType> inALocalPing;
LocalTensor<InType> inALocalPong;
LocalTensor<InType> inBLocalPing;
LocalTensor<InType> inBLocalPong;
LocalTensor<int8_t> outFLocalPing;
LocalTensor<int8_t> outFLocalPong;
int32_t pingPongFlag = 0;
event_t eventId = EVENT_ID0;
};
TEMPLATE_DECLARE
__aicore__ inline void DequantSwigluQuantDynamicPerformance<TEMPLATE_ARGS>::Init(
GM_ADDR x_gm, GM_ADDR weight_scale_gm, GM_ADDR activation_scale_gm, GM_ADDR bias_gm, GM_ADDR quant_scale_gm,
GM_ADDR quant_offset_gm, GM_ADDR y_gm, GM_ADDR scale_gm, GM_ADDR userspace, const SwiGluTilingData* tilingData,
TPipe* pipe_)
{
this->InitCommon(x_gm, weight_scale_gm, activation_scale_gm, bias_gm, quant_scale_gm, quant_offset_gm, y_gm, scale_gm, userspace, tilingData, pipe_);
this->weightScaleGm.SetGlobalBuffer((__gm__ float*)weight_scale_gm, this->colNum);
if (this->biasIsEmpty == 0) {
this->biasGm.SetGlobalBuffer((__gm__ BiasType*)bias_gm, this->colNum);
}
if (this->activateScaleIsEmpty == 0) {
this->activationScaleGm.SetGlobalBuffer((__gm__ float*) activation_scale_gm + this->biasOffset,
this->numRound);
}
this->InitUbBufferCommon(this->baseColLen, this->numRound);
uint64_t alignTileLength = this->baseColLen;
if (!this->isOut32BAligned) {
alignTileLength = this->Align(this->baseColLen, sizeof(int8_t));
}
this->pipe->InitBuffer(weightScaleBufA, alignTileLength * sizeof(float));
this->pipe->InitBuffer(weightScaleBufB, alignTileLength * sizeof(float));
this->pipe->InitBuffer(quantScaleBuf, alignTileLength * sizeof(float));
this->pipe->InitBuffer(calcSwiGluTmpBuf, alignTileLength * sizeof(float) * this->baseRowLen);
this->pipe->InitBuffer(inQueueAPingBuf, alignTileLength * sizeof(InType) * this->baseRowLen);
this->pipe->InitBuffer(inQueueAPongBuf, alignTileLength * sizeof(InType) * this->baseRowLen);
this->pipe->InitBuffer(inQueueBPingBuf, alignTileLength * sizeof(InType) * this->baseRowLen);
this->pipe->InitBuffer(inQueueBPongBuf, alignTileLength * sizeof(InType) * this->baseRowLen);
this->pipe->InitBuffer(outQueueFPingBuf, alignTileLength * sizeof(int8_t) * this->baseRowLen);
this->pipe->InitBuffer(outQueueFPongBuf, alignTileLength * sizeof(int8_t) * this->baseRowLen);
this->pipe->InitBuffer(outQueueSBuf, this->AlignBytes(this->numRound, sizeof(float)));
outTmpLocal = this->outputTempBufferBF16D.template Get<CalcType>();
inputTmpLocal = this->inputTempBufferBF16D.template Get<CalcType>();
absTempLocal = this->inputTempBufferBF16D.template Get<float>();
int16Local = this->outputTempBufferBF16D.template Get<int16_t>();
bLocal = this->inputTempBufferBF16D.template Get<CalcType>();
swiLocal = calcSwiGluTmpBuf.Get<float>();
inALocalPing = inQueueAPingBuf.Get<InType>();
inALocalPong = inQueueAPongBuf.Get<InType>();
inBLocalPing = inQueueBPingBuf.Get<InType>();
inBLocalPong = inQueueBPongBuf.Get<InType>();
outFLocalPing = outQueueFPingBuf.Get<int8_t>();
outFLocalPong = outQueueFPongBuf.Get<int8_t>();
this->maxTempLocal = outQueueSBuf.Get<float>();
this->weightScaleLocalA = weightScaleBufA.Get<float>();
this->weightScaleLocalB = weightScaleBufB.Get<float>();
this->quantScaleLocal = quantScaleBuf.Get<float>();
}
TEMPLATE_DECLARE
__aicore__ inline void DequantSwigluQuantDynamicPerformance<TEMPLATE_ARGS>::InitCommon(GM_ADDR x_gm, GM_ADDR weight_scale_gm, GM_ADDR activation_scale_gm, GM_ADDR bias_gm,
GM_ADDR quant_scale_gm, GM_ADDR quant_offset_gm, GM_ADDR y_gm, GM_ADDR scale_gm,
GM_ADDR userspace, const SwiGluTilingData* tilingData, TPipe* pipe_)
{
this->pipe = pipe_;
this->curBlockIdx = GetBlockIdx();
this->activateLeft = tilingData->activateLeft;
this->quantScaleIsEmpty = tilingData->quantScaleIsEmpty;
this->activateScaleIsEmpty = tilingData->activateScaleIsEmpty;
this->biasIsEmpty = tilingData->biasIsEmpty;
this->colNum = tilingData->colLen;
this->rowNum = tilingData->rowLen;
this->useCoreNum = tilingData->usedCoreNum;
// 每行为全载情况下每次最大拷贝行数
this->baseRowLen = tilingData->baseRowLen;
this->baseColLen = tilingData->baseColLen;
if (this->rowNum < this->useCoreNum) {
this->useCoreNum = this->rowNum;
}
// 行全载和不全载情况下，分别判断是否对齐
this->isMultiCols = this->baseRowLen == 1 && this->baseColLen < this->colNum;
if (this->isMultiCols) {
this->isOut32BAligned = this->baseColLen == this->Align(this->baseColLen, sizeof(InType));
} else {
this->isOut32BAligned = (this->colNum % this->blockBytes == 0) || (this->baseRowLen == 1);
}
this->perRoundCnt = this->useCoreNum == 0 ? 0 : this->rowNum / this->useCoreNum;
uint32_t remainCnt = this->rowNum - this->useCoreNum * this->perRoundCnt;
this->numRound = this->perRoundCnt;
if (this->curBlockIdx < remainCnt) {
this->numRound = this->perRoundCnt + 1;
this->biasOffset = this->curBlockIdx * (this->perRoundCnt + 1);
} else {
this->biasOffset = (this->perRoundCnt + 1) * remainCnt + (this->curBlockIdx - remainCnt) * this->perRoundCnt;
}
this->xGm.SetGlobalBuffer((__gm__ InType*)x_gm + this->biasOffset * this->colNum * DOUBLE, this->colNum * this->numRound * DOUBLE);
this->scaleGm.SetGlobalBuffer((__gm__ float*)scale_gm, this->rowNum);
this->yGm.SetGlobalBuffer((__gm__ int8_t*)y_gm + this->biasOffset * this->colNum, this->numRound * this->colNum);
this->quantScaleGm.SetGlobalBuffer((__gm__ float*)quant_scale_gm, this->colNum);
}
TEMPLATE_DECLARE
__aicore__ inline void DequantSwigluQuantDynamicPerformance<TEMPLATE_ARGS>::InitUbBufferCommon(uint64_t tileLength, uint32_t realRowLen)
{
uint64_t alignTileLength = tileLength;
if (!this->isOut32BAligned) {
alignTileLength = this->Align(tileLength, sizeof(int8_t));
}
this->pipe->InitBuffer(this->inputTempBufferBF16D, alignTileLength * sizeof(CalcType) * this->baseRowLen);
this->pipe->InitBuffer(this->outputTempBufferBF16D, alignTileLength * sizeof(CalcType) * this->baseRowLen);
}
TEMPLATE_DECLARE
__aicore__ inline void DequantSwigluQuantDynamicPerformance<TEMPLATE_ARGS>::Process() {
if (this->curBlockIdx >= this->useCoreNum) {
return;
}
uint32_t offset1 = 0;
uint32_t offset2 = this->colNum;
if (this->activateLeft == 0) {
offset1 = this->colNum;
offset2 = 0;
}
this->CanFullLocaOneRow(offset1, offset2);
this->CopyOutScale(this->numRound);
}
TEMPLATE_DECLARE
__aicore__ inline void DequantSwigluQuantDynamicPerformance<TEMPLATE_ARGS>::CopyOutScale(uint32_t realRowLen)
{
DataCopyExtParams intriParams{1, static_cast<uint32_t>(sizeof(float) * realRowLen), 0, 0, 0};
DataCopyPad(this->scaleGm[this->biasOffset], this->maxTempLocal, intriParams);
}
TEMPLATE_DECLARE
__aicore__ inline void DequantSwigluQuantDynamicPerformance<TEMPLATE_ARGS>::CanFullLocaOneRow(uint32_t offset1, uint32_t offset2)
{
if (this->quantScaleIsEmpty == 0) {
this->CopyInQuantScale(this->colNum, 0);
}
this->CopyInDequantBuffer(offset1, offset2, this->colNum);
this->alignSize = this->isOut32BAligned ? this->colNum : this->Align(this->colNum, sizeof(int8_t));
int64_t blockCount = this->baseRowLen;
int64_t loops = (this->numRound + blockCount - 1) / blockCount;
int64_t lastLoopBlockCount = this->numRound - (loops - 1) * blockCount;
int64_t perLoopColSize = this->alignSize * blockCount;
offsetCalc = (blockCount == 0 ? 0 : (perLoopColSize / blockCount));
uint32_t alig8CalcNum = this->Align(this->colNum, sizeof(int8_t));
this->dstStride = this->isOut32BAligned ? 0 : (alig8CalcNum - this->colNum) * sizeof(InType) / this->blockBytes;
uint32_t base = (this->colNum * DOUBLE) * this->baseRowLen;
SetFlag<HardEvent::MTE3_MTE2>(EVENT_ID0);
SetFlag<HardEvent::MTE3_MTE2>(EVENT_ID1);
for (uint32_t i = 0; i < loops; i++) {
eventId = pingPongFlag ? EVENT_ID1 : EVENT_ID0;
event_t eventIdNext = pingPongFlag ? EVENT_ID0 : EVENT_ID1;
if (i == 0) {
WaitFlag<HardEvent::MTE3_MTE2>(eventId);
this->CopyIn(this->colNum, offset1 + i * base, offset2 + i * base, blockCount, pingPongFlag);
SetFlag<HardEvent::MTE2_V>(eventId);
}
WaitFlag<HardEvent::MTE2_V>(eventId);
this->BaseCompute1(perLoopColSize, blockCount, i, pingPongFlag);
if(i != loops -1) {
WaitFlag<HardEvent::MTE3_MTE2>(eventIdNext);
this->CopyIn(this->colNum, offset1 + (i + 1) * base, offset2 + (i + 1) * base, blockCount, 1 - pingPongFlag);
SetFlag<HardEvent::MTE2_V>(eventIdNext);
}
this->BaseCompute2(perLoopColSize, blockCount, i, pingPongFlag);
SetFlag<HardEvent::V_MTE3>(eventId);
WaitFlag<HardEvent::V_MTE3>(eventId);
this->CopyOutF(i, perLoopColSize, blockCount, pingPongFlag);
SetFlag<HardEvent::MTE3_MTE2>(eventId);
pingPongFlag = 1 - pingPongFlag;
}
WaitFlag<HardEvent::MTE3_MTE2>(EVENT_ID0);
WaitFlag<HardEvent::MTE3_MTE2>(EVENT_ID1);
}
TEMPLATE_DECLARE
__aicore__ inline void DequantSwigluQuantDynamicPerformance<TEMPLATE_ARGS>::CopyInQuantScale(uint64_t dataTileLength, uint64_t offset)
{
DataCopyExtParams dataCopyParams{1, static_cast<uint32_t>(dataTileLength * sizeof(float)), 0, 0, 0};
DataCopyPadExtParams<float> dataCopyPadParams{false, 0, 0, 0};
DataCopyPad(this->quantScaleLocal, this->quantScaleGm[offset], dataCopyParams, dataCopyPadParams);
}
TEMPLATE_DECLARE
__aicore__ inline void DequantSwigluQuantDynamicPerformance<TEMPLATE_ARGS>::CopyInDequantBuffer(uint32_t offset1, uint32_t offset2, uint32_t dataTileLen)
{
DataCopyExtParams params = {1, static_cast<uint32_t>(dataTileLen * sizeof(float)), 0, 0, 0};
DataCopyPadExtParams<float> padParams{false, 0, 0, 0};
DataCopyPad(this->weightScaleLocalA, this->weightScaleGm[offset1], params, padParams);
DataCopyPad(this->weightScaleLocalB, this->weightScaleGm[offset2], params, padParams);
}
TEMPLATE_DECLARE
__aicore__ inline void DequantSwigluQuantDynamicPerformance<TEMPLATE_ARGS>::CopyIn(uint32_t dataTileLen, uint32_t offset1, uint32_t offset2, uint32_t blockCount, int32_t ppFlag)
{
uint32_t srcStride = dataTileLen * sizeof(InType);
DataCopyExtParams dataCopyParams{static_cast<uint16_t>(blockCount),
static_cast<uint32_t>(dataTileLen * sizeof(InType)), srcStride, this->dstStride, 0};
DataCopyPadExtParams<InType> dataCopyPadParams{false, 0, 0, 0};
LocalTensor<InType> aLocal = ppFlag ? inALocalPong : inALocalPing;
LocalTensor<InType> bLocal = ppFlag ? inBLocalPong : inBLocalPing;
DataCopyPad(aLocal, this->xGm[offset1], dataCopyParams, dataCopyPadParams);
DataCopyPad(bLocal, this->xGm[offset2], dataCopyParams, dataCopyPadParams);
}
TEMPLATE_DECLARE
__aicore__ inline void DequantSwigluQuantDynamicPerformance<TEMPLATE_ARGS>::BaseCompute1(uint64_t curTileLen, uint64_t blockCount, uint64_t rowId, int32_t ppFlag)
{
LocalTensor<InType> inALocal = ppFlag ? inALocalPong : inALocalPing;
LocalTensor<InType> bLocal_ = ppFlag ? inBLocalPong : inBLocalPing;
float value = this->activationScaleGm.GetValue(rowId);
Cast(inputTmpLocal, inALocal, RoundMode::CAST_NONE, curTileLen);
PipeBarrier<PIPE_V>();
Mul(inputTmpLocal, inputTmpLocal, this->weightScaleLocalA, curTileLen);
PipeBarrier<PIPE_V>();
Muls(inputTmpLocal, inputTmpLocal, value, curTileLen);
PipeBarrier<PIPE_V>();
Muls(outTmpLocal, inputTmpLocal, this->beta, curTileLen);
PipeBarrier<PIPE_V>();
Exp(outTmpLocal, outTmpLocal, curTileLen);
PipeBarrier<PIPE_V>();
Adds(outTmpLocal, outTmpLocal, CalcType(1.0), curTileLen);
PipeBarrier<PIPE_V>();
Div(outTmpLocal, inputTmpLocal, outTmpLocal, curTileLen);
PipeBarrier<PIPE_V>();
Cast(bLocal, bLocal_, RoundMode::CAST_NONE, curTileLen);
PipeBarrier<PIPE_V>();
Mul(bLocal, bLocal, this->weightScaleLocalB, curTileLen);
PipeBarrier<PIPE_V>();
Muls(bLocal, bLocal, value, curTileLen);
PipeBarrier<PIPE_V>();
Mul(swiLocal, outTmpLocal, bLocal, curTileLen);
PipeBarrier<PIPE_V>();
if (this->quantScaleIsEmpty == 0) {
Mul(swiLocal, swiLocal, this->quantScaleLocal, curTileLen);
PipeBarrier<PIPE_V>();
}
}
TEMPLATE_DECLARE
__aicore__ inline void DequantSwigluQuantDynamicPerformance<TEMPLATE_ARGS>::BaseCompute2(uint64_t curTileLen, uint64_t blockCount, uint64_t rowId, int32_t ppFlag)
{
LocalTensor<int8_t> outLocal = ppFlag ? outFLocalPong : outFLocalPing;
Abs(absTempLocal, swiLocal, curTileLen);
PipeBarrier<PIPE_V>();
ReduceMax(this->maxTempLocal[rowId * this->baseRowLen], absTempLocal, absTempLocal, this->colNum);
PipeBarrier<PIPE_V>();
float value = this->maxTempLocal.GetValue(rowId * this->baseRowLen) / 127;
this->maxTempLocal.SetValue(rowId * this->baseRowLen, value);
float scale = 1 / value;
Muls(swiLocal, swiLocal, scale, this->colNum);
PipeBarrier<PIPE_V>();
Cast(int16Local, swiLocal, RoundMode::CAST_RINT, curTileLen);
PipeBarrier<PIPE_V>();
// int16-> half
LocalTensor<half> halfLocal = int16Local.ReinterpretCast<half>();
Cast(halfLocal, int16Local, RoundMode::CAST_NONE, curTileLen);
PipeBarrier<PIPE_V>();
// half -> int8_t
Cast(outLocal, halfLocal, RoundMode::CAST_NONE, curTileLen);
PipeBarrier<PIPE_V>();
}
TEMPLATE_DECLARE
__aicore__ inline void DequantSwigluQuantDynamicPerformance<TEMPLATE_ARGS>::CopyOutF(uint64_t rowId, uint64_t tileLen, uint64_t length, int32_t ppFlag)
{
LocalTensor<int8_t> outLocal = ppFlag ? outFLocalPong : outFLocalPing;
DataCopyExtParams intriParams{1, static_cast<uint32_t>(tileLen), 0, 0, 0};
DataCopyPad(this->yGm[rowId * this->colNum * this->baseRowLen], outLocal, intriParams);
}
} // namespace DequantSwigluQuant
#endif // DEQUANT_SWIGLU_QUANT_DYNAMIC_PERFORMANCE_HPP

371
csrc/moe/dequant_swiglu_quant/op_kernel/dequant_swiglu_quant_static_base.hpp Normal file
View File

@@ -0,0 +1,371 @@
/**
* Copyright (c) 2025 Huawei Technologies Co., Ltd.
* This program is free software, you can redistribute it and/or modify it under the terms and conditions of
* CANN Open Software License Agreement Version 2.0 (the "License").
* Please refer to the License for details. You may not use this file except in compliance with the License.
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
* INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
* See LICENSE in the root of the software repository for the full text of the License.
*/
/*!
* \file dequant_swiglu_quant_static_base.hpp
* \brief
*/
#ifndef CANN_DEQUANT_SWIGLU_QUANT_STATIC_BASE_HPP
#define CANN_DEQUANT_SWIGLU_QUANT_STATIC_BASE_HPP
#include "kernel_operator.h"
#define TEMPLATE_DECLARE_STATIC template<typename InType, typename CalcType, typename BiasType, typename OutType, uint16_t bufferNum, uint16_t quantIsOne>
#define TEMPLATE_ARGS_STATIC InType, CalcType, BiasType, OutType, bufferNum, quantIsOne
namespace DequantSwigluQuant {
constexpr uint32_t NUM2 = 2;
using namespace AscendC;
TEMPLATE_DECLARE_STATIC
class DequantSwigluQuantStaticBase {
public:
__aicore__ inline DequantSwigluQuantStaticBase() {}
__aicore__ inline ~DequantSwigluQuantStaticBase() {}
__aicore__ inline void InitCommon(GM_ADDR x_gm, GM_ADDR weight_scale_gm, GM_ADDR activation_scale_gm, GM_ADDR bias_gm,
GM_ADDR quant_scale_gm, GM_ADDR quant_offset_gm, GM_ADDR y_gm, GM_ADDR scale_gm,
const SwiGluTilingData* tilingData, TPipe* pipe_) {
this->blockIdx = GetBlockIdx();
this->activateLeft = tilingData->activateLeft;
this->quantScaleIsEmpty = tilingData->quantScaleIsEmpty;
this->activateScaleIsEmpty = tilingData->activateScaleIsEmpty;
this->biasIsEmpty = tilingData->biasIsEmpty;
this->colNum = tilingData->colLen;
this->rowNum = tilingData->rowLen;
this->usedCoreNum = tilingData->usedCoreNum;
this->baseRowLen = tilingData->baseRowLen;
this->baseColLen = tilingData->baseColLen < this->colNum ? tilingData->baseColLen : this->colNum;
this->curColNum = this->baseColLen;
if (this->rowNum < this->usedCoreNum) {
this->usedCoreNum = this->rowNum;
}
int64_t perRoundCnt = this->usedCoreNum == 0 ? 0 : this->rowNum / this->usedCoreNum;
int64_t remainCnt = this->rowNum - this->usedCoreNum * perRoundCnt;
this->curCoreRowNum = perRoundCnt;
if (this->blockIdx < remainCnt) {
this->curCoreRowNum = perRoundCnt + 1;
this->inputCopyOffset = this->blockIdx * this->curCoreRowNum;
} else {
this->inputCopyOffset = remainCnt * (perRoundCnt + 1) + (this->blockIdx - remainCnt) * perRoundCnt;
}
this->xGm.SetGlobalBuffer((__gm__ InType*)x_gm + this->inputCopyOffset * this->colNum * NUM2, this->curCoreRowNum * this->colNum * NUM2);
this->yGm.SetGlobalBuffer((__gm__ OutType*)y_gm + this->inputCopyOffset * this->colNum, this->curCoreRowNum * this->colNum);
if (quantScaleIsEmpty == 0) {
if constexpr(quantIsOne == 0) {
this->quantOffsetGm.SetGlobalBuffer((__gm__ float*) quant_offset_gm, this->colNum);
this->quantScaleGm.SetGlobalBuffer((__gm__ float*) quant_scale_gm, this->colNum);
} else {
this->quantScaleGm.SetGlobalBuffer((__gm__ float*) quant_scale_gm, 1);
this->quant_scale = 1 / this->quantScaleGm.GetValue(0);
this->quantOffsetGm.SetGlobalBuffer((__gm__ float*) quant_offset_gm, 1);
this->quant_offset = this->quantOffsetGm.GetValue(0);
}
}
}
__aicore__ inline void InitUbBufferCommon()
{
int64_t alignColNum = curColNum == Align(curColNum, sizeof(InType)) ? curColNum : Align(curColNum, sizeof(OutType));
pipe->InitBuffer(inputTempBufferInt32SD, alignColNum * sizeof(CalcType) * NUM2);
pipe->InitBuffer(swigluTempBuffer, alignColNum * sizeof(CalcType));
pipe->InitBuffer(inQueue, bufferNum, alignColNum * sizeof(InType) * NUM2);
pipe->InitBuffer(outQueue, bufferNum, alignColNum * sizeof(OutType));
if (quantScaleIsEmpty == 0) {
if constexpr(quantIsOne == 0) {
pipe->InitBuffer(inQueueQuant, bufferNum, alignColNum * sizeof(float) * NUM2);
}
}
}
__aicore__ inline void CopyInWeightAndBias(int64_t offset)
{
DataCopyExtParams params = {1, static_cast<uint32_t>(curColNum * sizeof(float)), 0, 0, 0};
DataCopyExtParams paramsBias = {1, static_cast<uint32_t>(curColNum * sizeof(BiasType)), 0, 0, 0};
DataCopyPadExtParams<float> padParams{false, 0, 0, 0};
DataCopyPadExtParams<BiasType> padParams1{false, 0, 0, 0};
LocalTensor<float> weightLocal = inQueueWeightScale.template AllocTensor<float>();
LocalTensor<BiasType> biasTensorLocal;
if (this->biasIsEmpty == 0) {
biasTensorLocal = inQueueBias.template AllocTensor<BiasType>();
}
if (activateLeft == 0) {
DataCopyPad(weightLocal, weightScaleGm[offset + colNum], params, padParams);
DataCopyPad(weightLocal[alignColNum], weightScaleGm[offset], params, padParams);
if (this->biasIsEmpty == 0) {
DataCopyPad(biasTensorLocal, biasGm[offset + colNum], paramsBias, padParams1);
if constexpr (std::is_same_v<BiasType, int32_t> || std::is_same_v<BiasType, float>) {
DataCopyPad(biasTensorLocal[alignColNum], biasGm[offset], paramsBias, padParams1);
} else {
DataCopyPad(biasTensorLocal[biasAlignColNum], biasGm[offset], paramsBias, padParams1);
}
}
} else {
DataCopyPad(weightLocal, weightScaleGm[offset], params, padParams);
DataCopyPad(weightLocal[alignColNum], weightScaleGm[offset + colNum], params, padParams);
if (this->biasIsEmpty == 0) {
DataCopyPad(biasTensorLocal, biasGm[offset], paramsBias, padParams1);
if constexpr (std::is_same_v<BiasType, int32_t> || std::is_same_v<BiasType, float>) {
DataCopyPad(biasTensorLocal[alignColNum], biasGm[offset + colNum], paramsBias, padParams1);
} else {
DataCopyPad(biasTensorLocal[biasAlignColNum], biasGm[offset + colNum], paramsBias, padParams1);
}
}
}
if (activateScaleIsEmpty == 0) {
DataCopyExtParams activateparams = {1, static_cast<uint32_t>(curCoreRowNum * sizeof(float)), 0, 0, 0};
LocalTensor<float> activateLocal = inQueueActivationScale.template AllocTensor<float>();
DataCopyPad(activateLocal, activationScaleGm, activateparams, padParams);
inQueueActivationScale.EnQue(activateLocal);
}
inQueueWeightScale.EnQue(weightLocal);
if (this->biasIsEmpty == 0) {
inQueueBias.EnQue(biasTensorLocal);
}
}
__aicore__ inline void CopyInQuant(int64_t offset)
{
DataCopyExtParams params = {1, static_cast<uint32_t>(curColNum * sizeof(float)), 0, 0, 0};
DataCopyPadExtParams<float> padParams{false, 0, 0, 0};
LocalTensor<float> quantLocal = inQueueQuant.template AllocTensor<float>();
DataCopyPad(quantLocal, quantScaleGm[offset], params, padParams);
DataCopyPad(quantLocal[alignColNum], quantOffsetGm[offset], params, padParams);
inQueueQuant.EnQue(quantLocal);
}
__aicore__ inline void CopyIn(int64_t offset1, int64_t offset2)
{
DataCopyExtParams params = {1, static_cast<uint32_t>(curColNum * sizeof(InType)), 0, 0, 0};
DataCopyPadExtParams <InType> padParams{false, 0, 0, 0};
LocalTensor <InType> aLocal = inQueue.template AllocTensor<InType>();
if (activateLeft == 0) {
DataCopyPad(aLocal, xGm[offset2], params, padParams);
DataCopyPad(aLocal[alignColNum], xGm[offset1], params, padParams);
} else {
DataCopyPad(aLocal, xGm[offset1], params, padParams);
DataCopyPad(aLocal[alignColNum], xGm[offset2], params, padParams);
}
inQueue.EnQue(aLocal);
}
__aicore__ inline void dequant(uint64_t tileLen, uint64_t i)
{
LocalTensor <InType> aLocal = this->inQueue.template DeQue<InType>();
this->inputTmpELocal = this->inputTempBufferInt32SD.template Get<CalcType>();
if constexpr (std::is_same_v<BiasType, int32_t>) {
if (this->biasIsEmpty == 0) {
Add(aLocal, aLocal, this->biasLocal, tileLen);
PipeBarrier<PIPE_V>();
}
}
Cast(this->inputTmpELocal, aLocal, RoundMode::CAST_NONE, tileLen);
PipeBarrier<PIPE_V>();
this->inQueue.template FreeTensor(aLocal);
Mul(this->inputTmpELocal, this->inputTmpELocal, this->weightScaleLocal, tileLen);
PipeBarrier<PIPE_V>();
if (this->activateScaleIsEmpty == 0) {
float value = this->activateLocal.GetValue(i);
Muls(this->inputTmpELocal, this->inputTmpELocal, value, tileLen);
PipeBarrier<PIPE_V>();
}
if constexpr (std::is_same_v<BiasType, float> || std::is_same_v<BiasType, bfloat16_t> || std::is_same_v<BiasType, half>) {
if (this->biasIsEmpty == 0) {
if constexpr (std::is_same_v<BiasType, float>) {
Add(this->inputTmpELocal, this->inputTmpELocal, this->biasLocal, tileLen);
} else {
LocalTensor<CalcType> biasFloatLocal = this->inputBiasTempBuffer.template Get<CalcType>();
Cast(biasFloatLocal, this->biasLocal, RoundMode::CAST_NONE, tileLen / NUM2);
PipeBarrier<PIPE_V>();
Cast(biasFloatLocal[tileLen / NUM2], this->biasLocal[biasAlignColNum], RoundMode::CAST_NONE, tileLen / NUM2);
PipeBarrier<PIPE_V>();
Add(this->inputTmpELocal, this->inputTmpELocal, biasFloatLocal, tileLen);
}
PipeBarrier<PIPE_V>();
}
}
}
__aicore__ inline void processComputeFree()
{
if (this->biasIsEmpty == 0) {
this->inQueueBias.FreeTensor(this->biasLocal);
}
this->inQueueWeightScale.FreeTensor(this->weightScaleLocal);
if (quantScaleIsEmpty == 0) {
if constexpr(quantIsOne == 0) {
this->inQueueQuant.FreeTensor(this->quantLocal);
}
}
if (this->activateScaleIsEmpty == 0) {
this->inQueueActivationScale.template FreeTensor(this->activateLocal);
}
}
__aicore__ inline void processCompute()
{
int64_t lastColNum = this->baseColLen;
int64_t colLoops = 1;
if (this->baseColLen < this->colNum) {
colLoops = (this->colNum + this->baseColLen - 1) / this->baseColLen;
lastColNum = this->colNum - (colLoops - 1) * this->baseColLen;
}
for (int64_t colLoop = 0; colLoop < colLoops; colLoop++) {
if (colLoop == colLoops - 1) {
this->curColNum = lastColNum;
}
bool isAligned = this->curColNum == this->Align(this->curColNum, sizeof(InType));
this->alignColNum = isAligned ? this->curColNum : this->Align(this->curColNum, sizeof(int8_t));
if constexpr (std::is_same_v<BiasType, bfloat16_t> || std::is_same_v<BiasType, half>) {
bool biasIsAligned = this->curColNum == this->Align(this->curColNum, sizeof(BiasType));
this->biasAlignColNum = biasIsAligned ? this->curColNum : this->Align(this->curColNum, sizeof(int8_t));
}
this->CopyInWeightAndBias(colLoop * this->baseColLen);
if (this->biasIsEmpty == 0) {
this->biasLocal = this->inQueueBias.template DeQue<BiasType>();
}
this->weightScaleLocal = this->inQueueWeightScale.template DeQue<float>();
if (this->activateScaleIsEmpty == 0) {
this->activateLocal = this->inQueueActivationScale.template DeQue<float>();
}
for (int64_t i = 0; i < this->curCoreRowNum; i++) {
this->CopyIn(i * this->colNum * NUM2 + colLoop * this->baseColLen, i * this->colNum * NUM2 + this->colNum + colLoop * this->baseColLen);
this->dequant(this->alignColNum * NUM2, i);
if (i == 0 && quantScaleIsEmpty == 0) {
if constexpr(quantIsOne == 0) {
this->CopyInQuant(colLoop * this->baseColLen);
this->quantLocal = this->inQueueQuant.template DeQue<float>();
}
}
this->swiglu(this->alignColNum, i);
this->CopyOut(colLoop, i);
}
processComputeFree();
}
}
__aicore__ inline void swiglu(uint64_t curTileLen, int64_t idx)
{
LocalTensor <CalcType> swigluLocal = swigluTempBuffer.Get<CalcType>();
Muls(swigluLocal, inputTmpELocal, beta, curTileLen);
PipeBarrier<PIPE_V>();
Exp(swigluLocal, swigluLocal, curTileLen);
PipeBarrier<PIPE_V>();
Adds(swigluLocal, swigluLocal, CalcType(1.0), curTileLen);
PipeBarrier<PIPE_V>();
Div(inputTmpELocal, inputTmpELocal, swigluLocal, curTileLen);
PipeBarrier<PIPE_V>();
Mul(inputTmpELocal[curTileLen], inputTmpELocal, inputTmpELocal[curTileLen], curTileLen);
PipeBarrier<PIPE_V>();
if (quantScaleIsEmpty == 0) {
if constexpr(quantIsOne == 0) {
Div(inputTmpELocal[curTileLen], inputTmpELocal[curTileLen], quantLocal, curTileLen);
PipeBarrier<PIPE_V>();
Add(inputTmpELocal[curTileLen], inputTmpELocal[curTileLen], quantLocal[curTileLen], curTileLen);
PipeBarrier<PIPE_V>();
} else {
Muls(inputTmpELocal[curTileLen], inputTmpELocal[curTileLen], quant_scale, curTileLen);
PipeBarrier<PIPE_V>();
Adds(inputTmpELocal[curTileLen], inputTmpELocal[curTileLen], quant_offset, curTileLen);
PipeBarrier<PIPE_V>();
}
}
// fp32->int16
LocalTensor <int16_t> int16Local = swigluTempBuffer.Get<int16_t>();
Cast(int16Local, inputTmpELocal[curTileLen], RoundMode::CAST_RINT, curTileLen);
PipeBarrier<PIPE_V>();
// int16-> half
LocalTensor <half> halfLocal = int16Local.ReinterpretCast<half>();
Cast(halfLocal, int16Local, RoundMode::CAST_NONE, curTileLen);
PipeBarrier<PIPE_V>();
LocalTensor <OutType> outLocal = outQueue.template AllocTensor<OutType>();
// half -> int8
Cast(outLocal, halfLocal, RoundMode::CAST_NONE, curTileLen);
outQueue.template EnQue<OutType>(outLocal);
}
__aicore__ inline void CopyOut(int64_t colLoop, int64_t idx)
{
LocalTensor <OutType> outLocal = outQueue.template DeQue<OutType>();
DataCopyExtParams dataCopyParams{1, static_cast<uint32_t>(curColNum * sizeof(OutType)), 0, 0, 0};
DataCopyPad(yGm[idx * colNum + colLoop * baseColLen], outLocal, dataCopyParams);
outQueue.FreeTensor(outLocal);
}
protected:
__aicore__ inline int64_t Align(int64_t elementNum, int64_t bytes)
{
constexpr int64_t BLOCK_BYTES = 32;
if (bytes == 0) {
return 0;
}
return (elementNum * bytes + BLOCK_BYTES - 1) / BLOCK_BYTES * BLOCK_BYTES / bytes;
}
protected:
float beta = -1.0;
float quant_scale = 1;
float quant_offset = 1;
TPipe* pipe = nullptr;
int64_t biasIsEmpty = 0;
int64_t quantScaleIsEmpty = 0;
int64_t activateScaleIsEmpty = 0;
int64_t colNum = 0;
int64_t rowNum = 0;
int64_t curCoreRowNum = 0;
int64_t inputCopyOffset = 0;
int64_t alignColNum = 0;
int64_t biasAlignColNum = 0;
int64_t curColNum = 0;
int64_t activateLeft = 0;
int64_t blockIdx = 0;
int64_t usedCoreNum = 0;
int64_t baseRowLen = 0;
int64_t baseColLen = 0;
GlobalTensor <OutType> yGm;
GlobalTensor <InType> xGm;
GlobalTensor<float> weightScaleGm;
GlobalTensor<float> activationScaleGm;
GlobalTensor <BiasType> biasGm;
GlobalTensor<float> quantScaleGm;
GlobalTensor<float> quantOffsetGm;
LocalTensor <CalcType> inputTmpELocal;
LocalTensor<float> weightScaleLocal;
LocalTensor<BiasType> biasLocal;
LocalTensor<float> quantLocal;
LocalTensor<float> activateLocal;
TQue <QuePosition::VECIN, bufferNum> inQueueWeightScale;
TQue <QuePosition::VECIN, bufferNum> inQueueActivationScale;
TQue <QuePosition::VECIN, bufferNum> inQueueBias;
TQue <QuePosition::VECIN, bufferNum> inQueueQuant;
TQue <QuePosition::VECIN, bufferNum> inQueue;
TQue <QuePosition::VECOUT, bufferNum> outQueue;
TBuf <TPosition::VECCALC> inputTempBufferInt32SD;
TBuf <TPosition::VECCALC> swigluTempBuffer;
TBuf <TPosition::VECCALC> inputBiasTempBuffer;
};
}
#endif // CANN_DEQUANT_SWIGLU_QUANT_STATIC_BASE_HPP

98
csrc/moe/dequant_swiglu_quant/op_kernel/dequant_swiglu_quant_static_bf16.hpp Normal file
View File

@@ -0,0 +1,98 @@
/**
* Copyright (c) 2025 Huawei Technologies Co., Ltd.
* This program is free software, you can redistribute it and/or modify it under the terms and conditions of
* CANN Open Software License Agreement Version 2.0 (the "License").
* Please refer to the License for details. You may not use this file except in compliance with the License.
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
* INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
* See LICENSE in the root of the software repository for the full text of the License.
*/
/*!
* \file dequant_swiglu_quant_static_bf16.hpp
* \brief
*/
#ifndef CANN_DEQUANT_SWIGLU_QUANT_STATIC_BF16_HPP
#define CANN_DEQUANT_SWIGLU_QUANT_STATIC_BF16_HPP
#include "kernel_operator.h"
#include "dequant_swiglu_quant_static_base.hpp"
namespace DequantSwigluQuant {
using namespace AscendC;
TEMPLATE_DECLARE_STATIC
class DequantSwigluQuantStaticBF16 : public DequantSwigluQuantStaticBase<TEMPLATE_ARGS_STATIC> {
public:
__aicore__ inline DequantSwigluQuantStaticBF16() {}
__aicore__ inline ~DequantSwigluQuantStaticBF16() {}
__aicore__ inline void Init(GM_ADDR x_gm, GM_ADDR weight_scale_gm, GM_ADDR activation_scale_gm, GM_ADDR bias_gm,
GM_ADDR quant_scale_gm, GM_ADDR quant_offset_gm, GM_ADDR y_gm, GM_ADDR scale_gm,
const SwiGluTilingData* tilingData, TPipe* pipe_);
__aicore__ inline void Process();
protected:
__aicore__ inline void convertFloat(uint64_t curTileLen, uint64_t i);
};
TEMPLATE_DECLARE_STATIC
__aicore__ inline void DequantSwigluQuantStaticBF16<TEMPLATE_ARGS_STATIC>::Init(GM_ADDR x_gm, GM_ADDR weight_scale_gm,
GM_ADDR activation_scale_gm, GM_ADDR bias_gm,
GM_ADDR quant_scale_gm, GM_ADDR quant_offset_gm, GM_ADDR y_gm, GM_ADDR scale_gm,
const SwiGluTilingData* tilingData, TPipe* pipe_)
{
this->pipe = pipe_;
this->InitCommon(x_gm, weight_scale_gm, activation_scale_gm, bias_gm, quant_scale_gm, quant_offset_gm, y_gm, scale_gm, tilingData, pipe_);
this->InitUbBufferCommon();
}
TEMPLATE_DECLARE_STATIC
__aicore__ inline void DequantSwigluQuantStaticBF16<TEMPLATE_ARGS_STATIC>::Process()
{
if (this->blockIdx >= this->usedCoreNum) {
return;
}
int64_t colLoops = 1;
int64_t lastColNum = this->baseColLen;
if (this->baseColLen < this->colNum) {
colLoops = (this->colNum + this->baseColLen - 1) / this->baseColLen;
lastColNum = this->colNum - (colLoops - 1) * this->baseColLen;
}
for (int64_t colLoop = 0; colLoop < colLoops; colLoop++) {
if (colLoop == colLoops - 1) {
this->curColNum = lastColNum;
}
bool isOutAligned = this->curColNum == this->Align(this->curColNum, sizeof(InType));
this->alignColNum = isOutAligned ? this->curColNum : this->Align(this->curColNum, sizeof(OutType));
for (int64_t i = 0; i < this->curCoreRowNum; i++) {
this->CopyIn(i * this->colNum * NUM2 + colLoop * this->baseColLen, i * this->colNum * NUM2 + this->colNum + colLoop * this->baseColLen);
convertFloat(this->alignColNum * NUM2, i);
if (i == 0 && this->quantScaleIsEmpty == 0) {
if constexpr(quantIsOne == 0) {
this->CopyInQuant(colLoop * this->baseColLen);
this->quantLocal = this->inQueueQuant.template DeQue<float>();
}
}
this->swiglu(this->alignColNum, i);
this->CopyOut(colLoop, i);
}
if (this->quantScaleIsEmpty == 0) {
if constexpr(quantIsOne == 0) {
this->inQueueQuant.FreeTensor(this->quantLocal);
}
}
}
}
TEMPLATE_DECLARE_STATIC
__aicore__ inline void DequantSwigluQuantStaticBF16<TEMPLATE_ARGS_STATIC>::convertFloat(uint64_t tileLen, uint64_t i)
{
LocalTensor <InType> aLocal = this->inQueue.template DeQue<InType>();
this->inputTmpELocal = this->inputTempBufferInt32SD.template Get<CalcType>();
Cast(this->inputTmpELocal, aLocal, RoundMode::CAST_NONE, tileLen);
PipeBarrier<PIPE_V>();
this->inQueue.template FreeTensor(aLocal);
}
}
#endif // CANN_DEQUANT_SWIGLU_QUANT_STATIC_BF16_HPP

93
csrc/moe/dequant_swiglu_quant/op_kernel/dequant_swiglu_quant_static_bias_float.hpp Normal file
View File

@@ -0,0 +1,93 @@
/**
* Copyright (c) 2025 Huawei Technologies Co., Ltd.
* This program is free software, you can redistribute it and/or modify it under the terms and conditions of
* CANN Open Software License Agreement Version 2.0 (the "License").
* Please refer to the License for details. You may not use this file except in compliance with the License.
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
* INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
* See LICENSE in the root of the software repository for the full text of the License.
*/
/*!
* \file dequant_swiglu_quant_static_bias_float.hpp
* \brief
*/
#ifndef CANN_DEQUANT_SWIGLU_QUANT_STATIC_BIAS_FLOAT_HPP
#define CANN_DEQUANT_SWIGLU_QUANT_STATIC_BIAS_FLOAT_HPP
#include "kernel_operator.h"
#include "dequant_swiglu_quant_static_base.hpp"
namespace DequantSwigluQuant {
using namespace AscendC;
TEMPLATE_DECLARE_STATIC
class DequantSwigluQuantStaticBiasFloat : public DequantSwigluQuantStaticBase<TEMPLATE_ARGS_STATIC> {
public:
__aicore__ inline DequantSwigluQuantStaticBiasFloat() {}
__aicore__ inline ~DequantSwigluQuantStaticBiasFloat() {}
__aicore__ inline void Init(GM_ADDR x_gm, GM_ADDR weight_scale_gm, GM_ADDR activation_scale_gm, GM_ADDR bias_gm,
GM_ADDR quant_scale_gm, GM_ADDR quant_offset_gm, GM_ADDR y_gm, GM_ADDR scale_gm,
const SwiGluTilingData* tilingData, TPipe* pipe_);
__aicore__ inline void Process();
protected:
__aicore__ inline void InitUbBuffer();
private:
};
TEMPLATE_DECLARE_STATIC
__aicore__ inline void DequantSwigluQuantStaticBiasFloat<TEMPLATE_ARGS_STATIC>::Init(GM_ADDR x_gm, GM_ADDR weight_scale_gm,
GM_ADDR activation_scale_gm, GM_ADDR bias_gm, GM_ADDR quant_scale_gm, GM_ADDR quant_offset_gm,
GM_ADDR y_gm, GM_ADDR scale_gm, const SwiGluTilingData* tilingData, TPipe* pipe_)
{
this->pipe = pipe_;
this->InitCommon(x_gm, weight_scale_gm, activation_scale_gm, bias_gm, quant_scale_gm, quant_offset_gm, y_gm, scale_gm, tilingData, pipe_);
if (this->biasIsEmpty == 0) {
this->biasGm.SetGlobalBuffer((__gm__ BiasType*)bias_gm, this->colNum);
}
if (this->activateScaleIsEmpty == 0) {
this->activationScaleGm.SetGlobalBuffer((__gm__ float*) activation_scale_gm + this->inputCopyOffset,
this->curCoreRowNum);
}
this->weightScaleGm.SetGlobalBuffer((__gm__ float*) weight_scale_gm, this->colNum);
this->InitUbBufferCommon();
this->InitUbBuffer();
}
TEMPLATE_DECLARE_STATIC
__aicore__ inline void DequantSwigluQuantStaticBiasFloat<TEMPLATE_ARGS_STATIC>::Process()
{
if (this->blockIdx >= this->usedCoreNum) {
return;
}
this->processCompute();
}
TEMPLATE_DECLARE_STATIC
__aicore__ inline void DequantSwigluQuantStaticBiasFloat<TEMPLATE_ARGS_STATIC>::InitUbBuffer()
{
// pipe alloc memory to queue, the unit is Bytes
int64_t alignNumCol = this->curColNum == this->Align(this->curColNum, sizeof(InType))
? this->curColNum
: this->Align(this->curColNum, sizeof(OutType));
this->pipe->InitBuffer(this->inQueueWeightScale, bufferNum, alignNumCol * sizeof(float) * NUM2);
if (this->activateScaleIsEmpty == 0) {
this->pipe->InitBuffer(this->inQueueActivationScale, bufferNum, this->curCoreRowNum * sizeof(float));
}
if (this->biasIsEmpty == 0) {
int64_t biasAlignNumCol = this->curColNum == this->Align(this->curColNum, sizeof(BiasType))
? this->curColNum
: this->Align(this->curColNum, sizeof(OutType));
this->pipe->InitBuffer(this->inQueueBias, bufferNum, biasAlignNumCol * sizeof(BiasType) * NUM2);
if constexpr (std::is_same_v<BiasType, bfloat16_t> || std::is_same_v<BiasType, half>) {
this->pipe->InitBuffer(this->inputBiasTempBuffer, alignNumCol * sizeof(float) * NUM2);
}
}
}
} // using namespace DequantSwigluQuant
#endif // CANN_DEQUANT_SWIGLU_QUANT_STATIC_BIAS_FLOAT_HPP

84
csrc/moe/dequant_swiglu_quant/op_kernel/dequant_swiglu_quant_static_bias_int32.hpp Normal file
View File

@@ -0,0 +1,84 @@
/**
* Copyright (c) 2025 Huawei Technologies Co., Ltd.
* This program is free software, you can redistribute it and/or modify it under the terms and conditions of
* CANN Open Software License Agreement Version 2.0 (the "License").
* Please refer to the License for details. You may not use this file except in compliance with the License.
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
* INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
* See LICENSE in the root of the software repository for the full text of the License.
*/
/*!
* \file dequant_swiglu_quant_static_bias_int32.hpp
* \brief
*/
#ifndef CANN_DEQUANT_SWIGLU_QUANT_STATIC_BIAS_INT32_HPP
#define CANN_DEQUANT_SWIGLU_QUANT_STATIC_BIAS_INT32_HPP
#include "kernel_operator.h"
#include "dequant_swiglu_quant_static_base.hpp"
namespace DequantSwigluQuant {
using namespace AscendC;
TEMPLATE_DECLARE_STATIC
class DequantSwigluQuantStaticBiasInt32 : public DequantSwigluQuantStaticBase<TEMPLATE_ARGS_STATIC> {
public:
__aicore__ inline DequantSwigluQuantStaticBiasInt32() {}
__aicore__ inline ~DequantSwigluQuantStaticBiasInt32() {}
__aicore__ inline void Init(GM_ADDR x_gm, GM_ADDR weight_scale_gm, GM_ADDR activation_scale_gm, GM_ADDR bias_gm,
GM_ADDR quant_scale_gm, GM_ADDR quant_offset_gm, GM_ADDR y_gm, GM_ADDR scale_gm,
const SwiGluTilingData* tilingData, TPipe* pipe_);
__aicore__ inline void Process();
protected:
__aicore__ inline void InitUbBuffer();
private:
};
TEMPLATE_DECLARE_STATIC
__aicore__ inline void DequantSwigluQuantStaticBiasInt32<TEMPLATE_ARGS_STATIC>::Init(GM_ADDR x_gm, GM_ADDR weight_scale_gm,
GM_ADDR activation_scale_gm, GM_ADDR bias_gm,
GM_ADDR quant_scale_gm, GM_ADDR quant_offset_gm, GM_ADDR y_gm, GM_ADDR scale_gm,
const SwiGluTilingData* tilingData, TPipe* pipe_)
{
this->pipe = pipe_;
this->InitCommon(x_gm, weight_scale_gm, activation_scale_gm, bias_gm, quant_scale_gm, quant_offset_gm, y_gm, scale_gm, tilingData, pipe_);
if (this->activateScaleIsEmpty == 0) {
this->activationScaleGm.SetGlobalBuffer((__gm__ float*) activation_scale_gm + this->inputCopyOffset,
this->curCoreRowNum);
}
this->weightScaleGm.SetGlobalBuffer((__gm__ float*) weight_scale_gm, this->colNum);
if (tilingData->biasIsEmpty == 0) {
this->biasGm.SetGlobalBuffer((__gm__ BiasType*)bias_gm, this->colNum);
}
this->InitUbBufferCommon();
this->InitUbBuffer();
}
TEMPLATE_DECLARE_STATIC
__aicore__ inline void DequantSwigluQuantStaticBiasInt32<TEMPLATE_ARGS_STATIC>::Process()
{
if (this->blockIdx >= this->usedCoreNum) {
return;
}
this->processCompute();
}
TEMPLATE_DECLARE_STATIC
__aicore__ inline void DequantSwigluQuantStaticBiasInt32<TEMPLATE_ARGS_STATIC>::InitUbBuffer()
{
int64_t alignColNumber = this->curColNum == this->Align(this->curColNum, sizeof(InType))
? this->curColNum
: this->Align(this->curColNum, sizeof(OutType));
this->pipe->InitBuffer(this->inQueueWeightScale, bufferNum, alignColNumber * sizeof(float) * NUM2);
if (this->activateScaleIsEmpty == 0) {
this->pipe->InitBuffer(this->inQueueActivationScale, bufferNum, this->curCoreRowNum * sizeof(float));
}
if (this->biasIsEmpty == 0) {
this->pipe->InitBuffer(this->inQueueBias, bufferNum, alignColNumber * sizeof(float) * NUM2);
}
}
}
#endif // CANN_DEQUANT_SWIGLU_QUANT_STATIC_BIAS_INT32_HPP