28
csrc/gmm/grouped_matmul_swiglu_quant/op_host/CMakeLists.txt
Normal file
28
csrc/gmm/grouped_matmul_swiglu_quant/op_host/CMakeLists.txt
Normal file
@@ -0,0 +1,28 @@
|
||||
# -----------------------------------------------------------------------------------------------------------
|
||||
# Copyright (c) 2025 Huawei Technologies Co., Ltd.
|
||||
# This program is free software, you can redistribute it and/or modify it under the terms and conditions of
|
||||
# CANN Open Software License Agreement Version 2.0 (the "License").
|
||||
# Please refer to the License for details. You may not use this file except in compliance with the License.
|
||||
# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
|
||||
# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
|
||||
# See LICENSE in the root of the software repository for the full text of the License.
|
||||
# -----------------------------------------------------------------------------------------------------------
|
||||
add_op_to_compiled_list()
|
||||
if (BUILD_OPEN_PROJECT)
|
||||
target_sources(op_host_aclnnExc PRIVATE
|
||||
grouped_matmul_swiglu_quant_def.cpp
|
||||
)
|
||||
add_ops_compile_options(
|
||||
OP_NAME GroupedMatmulSwigluQuant
|
||||
OPTIONS --cce-auto-sync=on
|
||||
-Wno-deprecated-declarations
|
||||
)
|
||||
endif()
|
||||
|
||||
if(NOT BUILD_OPS_RTY_KERNEL)
|
||||
add_modules_sources(OPTYPE grouped_matmul_swiglu_quant ACLNNTYPE aclnn_exclude)
|
||||
target_include_directories(${OPHOST_NAME}_tiling_obj PRIVATE
|
||||
${CMAKE_CURRENT_SOURCE_DIR}
|
||||
)
|
||||
endif()
|
||||
|
||||
@@ -0,0 +1,122 @@
|
||||
/**
|
||||
* Copyright (c) 2025 Huawei Technologies Co., Ltd.
|
||||
* This program is free software, you can redistribute it and/or modify it under the terms and conditions of
|
||||
* CANN Open Software License Agreement Version 2.0 (the "License").
|
||||
* Please refer to the License for details. You may not use this file except in compliance with the License.
|
||||
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
|
||||
* INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
|
||||
* See LICENSE in the root of the software repository for the full text of the License.
|
||||
*/
|
||||
|
||||
/*!
|
||||
* \file grouped_matmul_swiglu_quant_def.cpp
|
||||
* \brief
|
||||
*/
|
||||
|
||||
#include "register/op_def_registry.h"
|
||||
namespace ops {
|
||||
class GroupedMatmulSwigluQuant : public OpDef {
|
||||
public:
|
||||
explicit GroupedMatmulSwigluQuant(const char *name) : OpDef(name)
|
||||
{
|
||||
this->Input("x")
|
||||
.ParamType(REQUIRED)
|
||||
.DataType({ge::DT_INT8, ge::DT_INT8, ge::DT_INT8, ge::DT_INT8, ge::DT_INT8})
|
||||
.Format({ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND});
|
||||
this->Input("weight")
|
||||
.ParamType(REQUIRED)
|
||||
.DataType({ge::DT_INT8, ge::DT_INT8, ge::DT_INT8, ge::DT_INT4, ge::DT_INT4})
|
||||
.Format({ge::FORMAT_FRACTAL_NZ, ge::FORMAT_FRACTAL_NZ, ge::FORMAT_FRACTAL_NZ, ge::FORMAT_ND,
|
||||
ge::FORMAT_FRACTAL_NZ});
|
||||
this->Input("weight_scale")
|
||||
.ParamType(REQUIRED)
|
||||
.DataType({ge::DT_FLOAT, ge::DT_BF16, ge::DT_FLOAT16, ge::DT_UINT64, ge::DT_UINT64})
|
||||
.Format({ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND});
|
||||
this->Input("x_scale")
|
||||
.ParamType(REQUIRED)
|
||||
.DataType({ge::DT_FLOAT, ge::DT_FLOAT, ge::DT_FLOAT, ge::DT_FLOAT, ge::DT_FLOAT})
|
||||
.Format({ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND});
|
||||
this->Input("weight_assistance_matrix")
|
||||
.ParamType(OPTIONAL)
|
||||
.DataType({ge::DT_FLOAT, ge::DT_FLOAT, ge::DT_FLOAT, ge::DT_FLOAT, ge::DT_FLOAT})
|
||||
.Format({ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND});
|
||||
this->Input("group_list")
|
||||
.ParamType(REQUIRED)
|
||||
.DataType({ge::DT_INT64, ge::DT_INT64, ge::DT_INT64, ge::DT_INT64, ge::DT_INT64})
|
||||
.Format({ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND});
|
||||
this->Output("y")
|
||||
.ParamType(REQUIRED)
|
||||
.DataType({ge::DT_INT8, ge::DT_INT8, ge::DT_INT8, ge::DT_INT8, ge::DT_INT8})
|
||||
.Format({ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND});
|
||||
this->Output("y_scale")
|
||||
.ParamType(REQUIRED)
|
||||
.DataType({ge::DT_FLOAT, ge::DT_FLOAT, ge::DT_FLOAT, ge::DT_FLOAT, ge::DT_FLOAT})
|
||||
.Format({ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND});
|
||||
this->Attr("is_enable_weight_assistance_matrix").AttrType(OPTIONAL).Bool(true);
|
||||
this->Attr("dequant_mode").AttrType(OPTIONAL).Int(0);
|
||||
this->Attr("limited").AttrType(OPTIONAL).Float(0.0f);
|
||||
|
||||
OpAICoreConfig aicore_config;
|
||||
aicore_config.DynamicCompileStaticFlag(true)
|
||||
.DynamicFormatFlag(true)
|
||||
.DynamicRankSupportFlag(true)
|
||||
.DynamicShapeSupportFlag(true)
|
||||
.NeedCheckSupportFlag(false)
|
||||
.PrecisionReduceFlag(true);
|
||||
|
||||
this->AICore().AddConfig("ascend910b", aicore_config);
|
||||
this->AICore().AddConfig("ascend910_93", aicore_config);
|
||||
|
||||
OpAICoreConfig config_kirin = GetKirinCoreConfig();
|
||||
this->AICore().AddConfig("kirinx90", config_kirin);
|
||||
this->AICore().AddConfig("kirin9030", config_kirin);
|
||||
}
|
||||
|
||||
private:
|
||||
OpAICoreConfig GetKirinCoreConfig() const
|
||||
{
|
||||
OpAICoreConfig config_kirin;
|
||||
config_kirin.DynamicCompileStaticFlag(true)
|
||||
.DynamicFormatFlag(true)
|
||||
.DynamicRankSupportFlag(true)
|
||||
.DynamicShapeSupportFlag(true)
|
||||
.NeedCheckSupportFlag(false)
|
||||
.PrecisionReduceFlag(true);
|
||||
config_kirin.Input("x")
|
||||
.ParamType(REQUIRED)
|
||||
.DataType({ge::DT_INT8, ge::DT_INT8})
|
||||
.Format({ge::FORMAT_ND, ge::FORMAT_ND});
|
||||
config_kirin.Input("weight")
|
||||
.ParamType(REQUIRED)
|
||||
.DataType({ge::DT_INT8, ge::DT_INT8})
|
||||
.Format({ge::FORMAT_FRACTAL_NZ, ge::FORMAT_FRACTAL_NZ});
|
||||
config_kirin.Input("weight_scale")
|
||||
.ParamType(REQUIRED)
|
||||
.DataType({ge::DT_FLOAT, ge::DT_FLOAT16})
|
||||
.Format({ge::FORMAT_ND, ge::FORMAT_ND});
|
||||
config_kirin.Input("x_scale")
|
||||
.ParamType(REQUIRED)
|
||||
.DataType({ge::DT_FLOAT, ge::DT_FLOAT})
|
||||
.Format({ge::FORMAT_ND, ge::FORMAT_ND});
|
||||
config_kirin.Input("weight_assistance_matrix")
|
||||
.ParamType(OPTIONAL)
|
||||
.DataType({ge::DT_FLOAT, ge::DT_FLOAT})
|
||||
.Format({ge::FORMAT_ND, ge::FORMAT_ND});
|
||||
config_kirin.Input("group_list")
|
||||
.ParamType(REQUIRED)
|
||||
.DataType({ge::DT_INT64, ge::DT_INT64})
|
||||
.Format({ge::FORMAT_ND, ge::FORMAT_ND});
|
||||
config_kirin.Output("y")
|
||||
.ParamType(REQUIRED)
|
||||
.DataType({ge::DT_INT8, ge::DT_INT8})
|
||||
.Format({ge::FORMAT_ND, ge::FORMAT_ND});
|
||||
config_kirin.Output("y_scale")
|
||||
.ParamType(REQUIRED)
|
||||
.DataType({ge::DT_FLOAT, ge::DT_FLOAT})
|
||||
.Format({ge::FORMAT_ND, ge::FORMAT_ND});
|
||||
return config_kirin;
|
||||
}
|
||||
};
|
||||
|
||||
OP_ADD(GroupedMatmulSwigluQuant);
|
||||
} // namespace ops
|
||||
@@ -0,0 +1,53 @@
|
||||
/**
|
||||
* Copyright (c) 2025 Huawei Technologies Co., Ltd.
|
||||
* This program is free software, you can redistribute it and/or modify it under the terms and conditions of
|
||||
* CANN Open Software License Agreement Version 2.0 (the "License").
|
||||
* Please refer to the License for details. You may not use this file except in compliance with the License.
|
||||
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
|
||||
* INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
|
||||
* See LICENSE in the root of the software repository for the full text of the License.
|
||||
*/
|
||||
|
||||
/*!
|
||||
* \file grouped_matmul_swiglu_quant_proto.cpp
|
||||
* \brief
|
||||
*/
|
||||
#include "register/op_impl_registry.h"
|
||||
#include "log/log.h"
|
||||
#include "platform/platform_info.h"
|
||||
|
||||
using namespace ge;
|
||||
namespace ops {
|
||||
const int64_t X_INDEX = 0;
|
||||
const int64_t WEIGHTSCALE_INDEX = 2;
|
||||
const int64_t M_DIM_INDEX = 0;
|
||||
const int64_t DIM_LEN = 2;
|
||||
const int64_t SPLIT_RATIO = 2;
|
||||
static ge::graphStatus InferShape4GroupedMatmulSwigluQuant(gert::InferShapeContext *context)
|
||||
{
|
||||
const gert::Shape *xShape = context->GetInputShape(X_INDEX);
|
||||
const gert::Shape *weightScaleShape = context->GetInputShape(WEIGHTSCALE_INDEX);
|
||||
int64_t m = xShape->GetDim(M_DIM_INDEX);
|
||||
int64_t N_DIM_INDEX = weightScaleShape->GetDimNum() - 1;
|
||||
int64_t n = static_cast<int64_t>(weightScaleShape->GetDim(N_DIM_INDEX) / SPLIT_RATIO);
|
||||
auto outShape = context->GetOutputShape(0);
|
||||
outShape->SetDimNum(DIM_LEN);
|
||||
outShape->SetDim(0, m);
|
||||
outShape->SetDim(1, n);
|
||||
auto outScaleShape = context->GetOutputShape(1);
|
||||
outScaleShape->SetDimNum(1);
|
||||
outScaleShape->SetDim(0, m);
|
||||
return GRAPH_SUCCESS;
|
||||
}
|
||||
|
||||
static graphStatus InferDataType4GroupedMatmulSwigluQuant(gert::InferDataTypeContext *context)
|
||||
{
|
||||
context->SetOutputDataType(0, DataType::DT_INT8);
|
||||
context->SetOutputDataType(1, DataType::DT_FLOAT);
|
||||
return GRAPH_SUCCESS;
|
||||
}
|
||||
|
||||
IMPL_OP_INFERSHAPE(GroupedMatmulSwigluQuant)
|
||||
.InferShape(InferShape4GroupedMatmulSwigluQuant)
|
||||
.InferDataType(InferDataType4GroupedMatmulSwigluQuant);
|
||||
} // namespace ops
|
||||
@@ -0,0 +1,288 @@
|
||||
/**
|
||||
* Copyright (c) 2025 Huawei Technologies Co., Ltd.
|
||||
* This program is free software, you can redistribute it and/or modify it under the terms and conditions of
|
||||
* CANN Open Software License Agreement Version 2.0 (the "License").
|
||||
* Please refer to the License for details. You may not use this file except in compliance with the License.
|
||||
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
|
||||
* INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
|
||||
* See LICENSE in the root of the software repository for the full text of the License.
|
||||
*/
|
||||
|
||||
/*!
|
||||
* \file grouped_matmul_swiglu_quant_tiling.cpp
|
||||
* \brief
|
||||
*/
|
||||
#include <climits>
|
||||
#include <graph/utils/type_utils.h>
|
||||
#include "register/op_impl_registry.h"
|
||||
#include "log/log.h"
|
||||
#include "err/ops_err.h"
|
||||
#include "tiling_base/tiling_base.h"
|
||||
#include "grouped_matmul_swiglu_quant_tiling.h"
|
||||
using namespace ge;
|
||||
using namespace AscendC;
|
||||
using namespace GroupedMatmulSwigluQuantTiling;
|
||||
using namespace Ops::Transformer::OpTiling;
|
||||
namespace {
|
||||
template <typename T>
|
||||
static inline auto AlignUp(T a, T base) -> T
|
||||
{
|
||||
if (base == 0) {
|
||||
return 0;
|
||||
}
|
||||
return (a + base - 1) / base * base;
|
||||
}
|
||||
} // namespace
|
||||
|
||||
namespace optiling {
|
||||
|
||||
struct GMMSwigluCompileInfo {
|
||||
uint64_t ubSize_ = 0;
|
||||
uint32_t aicNum_ = 0;
|
||||
uint32_t baseM_ = 128;
|
||||
uint32_t baseN_ = 256;
|
||||
};
|
||||
|
||||
static int64_t CalMaxRowInUb_A8W4(const gert::TilingContext *context, const uint64_t ubSize, const uint64_t n)
|
||||
{
|
||||
const uint64_t ALIGNMENT = 8;
|
||||
const float WEIGHT_FACTOR = 8.5;
|
||||
const uint64_t ALIGNMENT_TERM_FACTOR = 4;
|
||||
const uint64_t LINEAR_TERM_FACTOR = 6;
|
||||
const uint64_t CONSTANT_TERM = 64;
|
||||
const uint64_t MIN_ROW_THRESHOLD = 1;
|
||||
|
||||
// 表达式：8.5 * row * n + 4 * alignUp(row, 8) + 6n + 64 <= ubSize
|
||||
|
||||
// 忽略对齐项的初始估计
|
||||
int64_t maxRowEstimate =
|
||||
(ubSize - CONSTANT_TERM - LINEAR_TERM_FACTOR * n) / static_cast<int64_t>(WEIGHT_FACTOR * n);
|
||||
|
||||
// 考虑对齐影响
|
||||
uint64_t alignedRow = (maxRowEstimate + ALIGNMENT - 1) / ALIGNMENT * ALIGNMENT;
|
||||
uint64_t totalSize = static_cast<uint64_t>(WEIGHT_FACTOR * maxRowEstimate * n) +
|
||||
ALIGNMENT_TERM_FACTOR * alignedRow + LINEAR_TERM_FACTOR * n + CONSTANT_TERM;
|
||||
|
||||
// 如果超过UB大小，逐步减少row直到满足条件
|
||||
while (totalSize > ubSize && maxRowEstimate > 0) {
|
||||
maxRowEstimate--;
|
||||
alignedRow = (maxRowEstimate + ALIGNMENT - 1) / ALIGNMENT * ALIGNMENT;
|
||||
totalSize = static_cast<uint64_t>(WEIGHT_FACTOR * maxRowEstimate * n) + ALIGNMENT_TERM_FACTOR * alignedRow +
|
||||
LINEAR_TERM_FACTOR * n + CONSTANT_TERM;
|
||||
}
|
||||
|
||||
if (maxRowEstimate < MIN_ROW_THRESHOLD) {
|
||||
OP_LOGE(context->GetNodeName(), "GMM_SWIGLU_QUANT TILING: No valid row found for n = %lu, ubSize = %lu\n", n,
|
||||
ubSize);
|
||||
return 0;
|
||||
}
|
||||
return maxRowEstimate;
|
||||
}
|
||||
|
||||
static int64_t CalMaxRowInUb(const gert::TilingContext *context, const uint64_t ubSize, const uint64_t n)
|
||||
{
|
||||
uint64_t tmpBufSize = (n / SWIGLU_REDUCE_FACTOR) * FP32_DTYPE_SIZE;
|
||||
uint64_t perchannleBufSize = n * FP32_DTYPE_SIZE * DOUBLE_BUFFER;
|
||||
uint64_t reduceMaxResBufSize = BLOCK_BYTE;
|
||||
uint64_t reduceMaxTmpBufSize = BLOCK_BYTE;
|
||||
const uint64_t CONSTANT_TERM = 64;
|
||||
int64_t remainUbSize = ubSize - tmpBufSize - perchannleBufSize - reduceMaxResBufSize - reduceMaxTmpBufSize;
|
||||
int64_t maxRowInUb =
|
||||
remainUbSize / (n * INT32_DTYPE_SIZE + n / SWIGLU_REDUCE_FACTOR + FP32_DTYPE_SIZE) / DOUBLE_BUFFER;
|
||||
int64_t curUb = DOUBLE_BUFFER * (maxRowInUb * (INT32_DTYPE_SIZE * n + n / SWIGLU_REDUCE_FACTOR) +
|
||||
AlignUp(maxRowInUb, FP32_BLOCK_SIZE) * FP32_DTYPE_SIZE);
|
||||
if (curUb > remainUbSize) {
|
||||
// 64 : make sure ub does not excceed maxUbSize after align up to 8
|
||||
maxRowInUb = (remainUbSize - CONSTANT_TERM) /
|
||||
(n * INT32_DTYPE_SIZE + n / SWIGLU_REDUCE_FACTOR + FP32_DTYPE_SIZE) / DOUBLE_BUFFER;
|
||||
}
|
||||
if (maxRowInUb < 1) {
|
||||
// when n > (ubSize - 72) / 19 = 10330, maxRowInUb < 1
|
||||
OP_LOGE(context->GetNodeName(), "GMM_SWIGLU_QUANT TILING: n should not be greater than 10240, now is %lu\n", n);
|
||||
}
|
||||
return maxRowInUb;
|
||||
}
|
||||
|
||||
static void SetTilingKey(gert::TilingContext *context, bool isSplitWorkSpace, bool isA8W4MSD)
|
||||
{
|
||||
if (isA8W4MSD) { // A8W4 MSD tiling_key使用4
|
||||
context->SetTilingKey(A8W4_MSD_TILING_KEY_MODE);
|
||||
context->SetScheduleMode(BATCH_MODE_SCHEDULE);
|
||||
} else if (isSplitWorkSpace) {
|
||||
context->SetTilingKey(SPLITWORKSPACE_TILING_KEY_MODE);
|
||||
context->SetScheduleMode(BATCH_MODE_SCHEDULE);
|
||||
} else {
|
||||
context->SetTilingKey(COMMON_TILING_KEY_MODE);
|
||||
context->SetScheduleMode(BATCH_MODE_SCHEDULE);
|
||||
}
|
||||
}
|
||||
|
||||
ASCENDC_EXTERN_C graphStatus TilingGMMSwigluQuant(gert::TilingContext *context)
|
||||
{
|
||||
// set info
|
||||
OP_LOGD(context->GetNodeName(), "Begin Run GMM Swiglu Tiling .");
|
||||
auto xDesc = context->GetInputDesc(X_INDEX);
|
||||
OP_CHECK_NULL_WITH_CONTEXT(context, xDesc);
|
||||
auto weightDesc = context->GetInputDesc(WEIGHT_INDEX);
|
||||
OP_CHECK_NULL_WITH_CONTEXT(context, weightDesc);
|
||||
ge::DataType xDType = xDesc->GetDataType();
|
||||
ge::DataType weightDType = weightDesc->GetDataType();
|
||||
|
||||
bool isA8W4MSD = (xDType == ge::DataType::DT_INT8 && weightDType == ge::DataType::DT_INT4);
|
||||
auto compileInfoPtr = context->GetCompileInfo<GMMSwigluCompileInfo>();
|
||||
auto xTensor = context->GetInputTensor(X_INDEX);
|
||||
OP_CHECK_NULL_WITH_CONTEXT(context, xTensor);
|
||||
const int64_t m = xTensor->GetStorageShape().GetDim(0);
|
||||
const int64_t k = xTensor->GetStorageShape().GetDim(1);
|
||||
auto wTensor = context->GetInputTensor(WEIGHT_INDEX);
|
||||
OP_CHECK_NULL_WITH_CONTEXT(context, wTensor);
|
||||
// swiglu limit 0 means clamp is disabled.
|
||||
auto attrs = context->GetAttrs();
|
||||
float limited = 0.0f;
|
||||
if (attrs != nullptr) {
|
||||
if (const double *limitedPtr = attrs->GetAttrPointer<double>(ATTR_INDEX_LIMITED)) {
|
||||
limited = static_cast<float>(*limitedPtr);
|
||||
}
|
||||
}
|
||||
OP_CHECK_IF(!(limited >= 0.0f),
|
||||
OPS_REPORT_VECTOR_INNER_ERR(context->GetNodeName(), "limited should be non-negative"),
|
||||
return GRAPH_FAILED);
|
||||
int64_t n = 0;
|
||||
if (wTensor->GetStorageShape().GetDimNum() == ND_WEIGHT_DIM_LIMIT) { // ND
|
||||
n = wTensor->GetStorageShape().GetDim(DIM_2);
|
||||
} else if (wTensor->GetStorageShape().GetDimNum() == NZ_WEIGHT_DIM_LIMIT) { // NZ
|
||||
n = wTensor->GetStorageShape().GetDim(DIM_1) * wTensor->GetStorageShape().GetDim(DIM_4);
|
||||
}
|
||||
auto wScaleTensor = context->GetInputTensor(WEIGHT_SCALE_INDEX);
|
||||
OP_CHECK_NULL_WITH_CONTEXT(context, wScaleTensor);
|
||||
int64_t quantGroupNum = 0;
|
||||
if (wScaleTensor->GetStorageShape().GetDimNum() == PERCHANNEL_WSCALE_DIM_LIMIT) { // perChannel
|
||||
quantGroupNum = 1;
|
||||
} else if (wScaleTensor->GetStorageShape().GetDimNum() == PERGROUP_WSCALE_DIM_LIMIT) { // perGroup
|
||||
quantGroupNum = wScaleTensor->GetStorageShape().GetDim(1);
|
||||
}
|
||||
auto groupListTensor = context->GetDynamicInputTensor(GROUPLIST_INDEX, 0);
|
||||
OP_CHECK_NULL_WITH_CONTEXT(context, groupListTensor);
|
||||
const int64_t groupNum = groupListTensor->GetStorageShape().GetDim(0);
|
||||
GMMSwigluQuantTilingData tilingData;
|
||||
int64_t row = 0;
|
||||
if (isA8W4MSD) {
|
||||
row = CalMaxRowInUb_A8W4(context, compileInfoPtr->ubSize_, n);
|
||||
} else {
|
||||
row = CalMaxRowInUb(context, compileInfoPtr->ubSize_, n);
|
||||
}
|
||||
|
||||
tilingData.gmmSwigluBaseParams.set_groupNum(groupNum);
|
||||
tilingData.gmmSwigluBaseParams.set_coreNum(compileInfoPtr->aicNum_);
|
||||
tilingData.gmmSwigluBaseParams.set_K(k);
|
||||
tilingData.gmmSwigluBaseParams.set_N(n);
|
||||
tilingData.gmmSwigluBaseParams.set_M(m);
|
||||
tilingData.gmmSwigluBaseParams.set_baseM(A8W4_BASEM);
|
||||
tilingData.gmmSwigluBaseParams.set_baseN(A8W4_BASEN);
|
||||
tilingData.gmmSwigluBaseParams.set_limited(limited);
|
||||
tilingData.gmmSwiglu.set_maxProcessRowNum(row);
|
||||
tilingData.gmmSwiglu.set_groupListLen(groupNum);
|
||||
tilingData.gmmSwiglu.set_tokenLen(n);
|
||||
|
||||
tilingData.gmmSwigluBaseParams.set_quantGroupNum(quantGroupNum);
|
||||
|
||||
auto ascendcPlatform = platform_ascendc::PlatformAscendC(context->GetPlatformInfo());
|
||||
using namespace matmul_tiling;
|
||||
|
||||
MatmulApiTiling tiling(ascendcPlatform);
|
||||
tiling.SetAType(TPosition::GM, CubeFormat::ND, matmul_tiling::DataType::DT_INT4);
|
||||
tiling.SetBType(TPosition::GM, CubeFormat::NZ, matmul_tiling::DataType::DT_INT4);
|
||||
tiling.SetCType(TPosition::GM, CubeFormat::ND, matmul_tiling::DataType::DT_FLOAT16);
|
||||
tiling.SetBias(false);
|
||||
tiling.SetShape(A8W4_BASEM, A8W4_BASEN, k);
|
||||
tiling.SetFixSplit(A8W4_BASEM, A8W4_BASEN, A8W4_BASEK);
|
||||
tiling.SetOrgShape(m, n, k);
|
||||
tiling.SetBufferSpace(-1, -1, -1);
|
||||
OP_CHECK_IF(
|
||||
tiling.GetTiling(tilingData.mmTilingData) == -1,
|
||||
OPS_REPORT_VECTOR_INNER_ERR(context->GetNodeName(), "grouped_matmul_swiglu_quant_tiling, get tiling failed"),
|
||||
return GRAPH_FAILED);
|
||||
if (isA8W4MSD) {
|
||||
tilingData.mmTilingData.set_baseM(A8W4_BASEM);
|
||||
tilingData.mmTilingData.set_baseN(A8W4_BASEN);
|
||||
tilingData.mmTilingData.set_baseK(A8W4_BASEK);
|
||||
tilingData.mmTilingData.set_dbL0B(DOUBLE_BUFFER);
|
||||
tilingData.mmTilingData.set_stepKa(NUM_FOUR);
|
||||
tilingData.mmTilingData.set_stepKb(NUM_FOUR);
|
||||
tilingData.mmTilingData.set_depthA1(NUM_EIGHT);
|
||||
tilingData.mmTilingData.set_depthB1(NUM_EIGHT);
|
||||
tilingData.mmTilingData.set_stepM(1);
|
||||
tilingData.mmTilingData.set_stepN(1);
|
||||
}
|
||||
auto workspaceSizes = context->GetWorkspaceSizes(1);
|
||||
int64_t usrWorkspaceLimit = USER_WORKSPACE_LIMIT;
|
||||
int64_t mLimit = 0;
|
||||
if (isA8W4MSD) {
|
||||
mLimit = ((usrWorkspaceLimit / DOUBLE_WORKSPACE_SPLIT) / (k * sizeof(int8_t) + DOUBLE_ROW * n * sizeof(half)));
|
||||
} else {
|
||||
mLimit = ((usrWorkspaceLimit / DOUBLE_WORKSPACE_SPLIT) / INT32_DTYPE_SIZE) / n;
|
||||
}
|
||||
OP_CHECK_IF(mLimit <= 0,
|
||||
OPS_REPORT_VECTOR_INNER_ERR(context->GetNodeName(), "mLimit is %ld must over then 0.", mLimit),
|
||||
return GRAPH_FAILED);
|
||||
tilingData.gmmSwigluBaseParams.set_mLimit(mLimit);
|
||||
if (isA8W4MSD) {
|
||||
int workSpaceMTemp = mLimit * DOUBLE_WORKSPACE_SPLIT;
|
||||
tilingData.gmmSwigluBaseParams.set_workSpaceOffset1(workSpaceMTemp * k * sizeof(int8_t));
|
||||
tilingData.gmmSwigluBaseParams.set_workSpaceOffset2(2 * workSpaceMTemp * n * sizeof(half));
|
||||
workspaceSizes[0] =
|
||||
SYS_WORKSPACE_SIZE + // 系统预留16MB
|
||||
(workSpaceMTemp * k * sizeof(int8_t)) + // 第一阶段 预处理左矩阵 (mLimit, K) * int8 * 2(double WorkSpace)
|
||||
(DOUBLE_ROW * workSpaceMTemp * n *
|
||||
sizeof(half)); // 第二阶段 矩阵乘结果 (2 * mLimit, N) * fp16 * 2(double WorkSpace)
|
||||
} else {
|
||||
int workSpaceMTemp = (mLimit * DOUBLE_WORKSPACE_SPLIT > m ? m : mLimit * DOUBLE_WORKSPACE_SPLIT);
|
||||
tilingData.gmmSwigluBaseParams.set_workSpaceOffset1(0);
|
||||
tilingData.gmmSwigluBaseParams.set_workSpaceOffset2(0);
|
||||
workspaceSizes[0] = SYS_WORKSPACE_SIZE + (workSpaceMTemp * n * sizeof(int32_t));
|
||||
}
|
||||
bool isSplitWorkSpace = m > mLimit * DOUBLE_WORKSPACE_SPLIT;
|
||||
OP_LOGD(context->GetNodeName(), "grouped_matmul_swiglu_quant_tiling.");
|
||||
OP_LOGD(context->GetNodeName(), "gmmSwigluBaseParams.groupNum: %ld", groupNum);
|
||||
OP_LOGD(context->GetNodeName(), "gmmSwigluBaseParams.coreNum: %u ", compileInfoPtr->aicNum_);
|
||||
OP_LOGD(context->GetNodeName(), "gmmSwigluBaseParams.M: %ld", m);
|
||||
OP_LOGD(context->GetNodeName(), "gmmSwigluBaseParams.K: %ld", k);
|
||||
OP_LOGD(context->GetNodeName(), "gmmSwigluBaseParams.N: %ld", n);
|
||||
OP_LOGD(context->GetNodeName(), "gmmSwigluBaseParams.baseM: %ld", A8W4_BASEM);
|
||||
OP_LOGD(context->GetNodeName(), "gmmSwigluBaseParams.baseN: %ld", A8W4_BASEN);
|
||||
OP_LOGD(context->GetNodeName(), "gmmSwigluBaseParams.mLimit: %ld", mLimit);
|
||||
OP_LOGD(context->GetNodeName(), "gmmSwigluBaseParams.quantGroupNum: %ld", quantGroupNum);
|
||||
OP_LOGD(context->GetNodeName(), "gmmSwiglu.maxProcessRowNum: %ld", row);
|
||||
OP_LOGD(context->GetNodeName(), "gmmSwiglu.groupListLen: %ld", groupNum);
|
||||
OP_LOGD(context->GetNodeName(), "gmmSwiglu.tokenLen: %ld", n);
|
||||
OP_LOGD(context->GetNodeName(), "USER_WORKSPACE_LIMIT: %ld", usrWorkspaceLimit);
|
||||
OP_LOGD(context->GetNodeName(), "workspaceSizes: %lu", workspaceSizes[0]);
|
||||
OP_LOGD(context->GetNodeName(), "isSplitWorkSpace: %s", isSplitWorkSpace ? "true" : "false");
|
||||
OP_LOGD(context->GetNodeName(), "GMMSWIGLUQUANT_TILING: baseM is %u, baseK is %u, baseN is %u.", A8W4_BASEM, A8W4_BASEK, A8W4_BASEN);
|
||||
SetTilingKey(context, isSplitWorkSpace, isA8W4MSD);
|
||||
tilingData.SaveToBuffer(context->GetRawTilingData()->GetData(), context->GetRawTilingData()->GetCapacity());
|
||||
context->SetBlockDim(compileInfoPtr->aicNum_); // block dim is the number of aicube
|
||||
context->GetRawTilingData()->SetDataSize(tilingData.GetDataSize());
|
||||
OP_LOGD(context->GetNodeName(), "End Run GMM Swiglu Tiling.");
|
||||
return GRAPH_SUCCESS;
|
||||
}
|
||||
|
||||
ASCENDC_EXTERN_C graphStatus TilingPrepareForGMMSwigluQuant(gert::TilingParseContext *context)
|
||||
{
|
||||
// get info
|
||||
fe::PlatFormInfos *platformInfoPtr = context->GetPlatformInfo();
|
||||
OP_CHECK_NULL_WITH_CONTEXT(context, platformInfoPtr);
|
||||
auto compileInfoPtr = context->GetCompiledInfo<GMMSwigluCompileInfo>();
|
||||
OP_CHECK_NULL_WITH_CONTEXT(context, compileInfoPtr);
|
||||
|
||||
auto ascendcPlatform = platform_ascendc::PlatformAscendC(platformInfoPtr);
|
||||
compileInfoPtr->aicNum_ = ascendcPlatform.GetCoreNumAic();
|
||||
ascendcPlatform.GetCoreMemSize(platform_ascendc::CoreMemType::UB, compileInfoPtr->ubSize_);
|
||||
OP_LOGD(context->GetNodeName(), "ubSize is %lu, aicNum is %u.", compileInfoPtr->ubSize_, compileInfoPtr->aicNum_);
|
||||
return GRAPH_SUCCESS;
|
||||
}
|
||||
|
||||
IMPL_OP_OPTILING(GroupedMatmulSwigluQuant)
|
||||
.Tiling(TilingGMMSwigluQuant)
|
||||
.TilingParse<GMMSwigluCompileInfo>(TilingPrepareForGMMSwigluQuant);
|
||||
} // namespace optiling
|
||||
@@ -0,0 +1,95 @@
|
||||
/**
|
||||
* Copyright (c) 2025 Huawei Technologies Co., Ltd.
|
||||
* This program is free software, you can redistribute it and/or modify it under the terms and conditions of
|
||||
* CANN Open Software License Agreement Version 2.0 (the "License").
|
||||
* Please refer to the License for details. You may not use this file except in compliance with the License.
|
||||
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
|
||||
* INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
|
||||
* See LICENSE in the root of the software repository for the full text of the License.
|
||||
*/
|
||||
|
||||
/*!
|
||||
* \file grouped_matmul_swiglu_quant_tiling.h
|
||||
* \brief
|
||||
*/
|
||||
#ifndef AIR_CXX_RUNTIME_V2_OP_IMPL_GROUPED_MATMUL_SWIGLU_QUANT_H
|
||||
#define AIR_CXX_RUNTIME_V2_OP_IMPL_GROUPED_MATMUL_SWIGLU_QUANT_H
|
||||
|
||||
#include <set>
|
||||
#include "register/tilingdata_base.h"
|
||||
#include "tiling/tiling_api.h"
|
||||
|
||||
namespace optiling {
|
||||
// GMM 基本信息
|
||||
BEGIN_TILING_DATA_DEF(GMMSwigluBaseParams)
|
||||
TILING_DATA_FIELD_DEF(uint32_t, groupNum);
|
||||
TILING_DATA_FIELD_DEF(uint32_t, coreNum);
|
||||
TILING_DATA_FIELD_DEF(uint32_t, K);
|
||||
TILING_DATA_FIELD_DEF(uint32_t, N);
|
||||
TILING_DATA_FIELD_DEF(uint32_t, M);
|
||||
TILING_DATA_FIELD_DEF(uint32_t, baseM);
|
||||
TILING_DATA_FIELD_DEF(uint32_t, baseN);
|
||||
TILING_DATA_FIELD_DEF(uint32_t, mLimit);
|
||||
TILING_DATA_FIELD_DEF(uint32_t, workSpaceOffset1);
|
||||
TILING_DATA_FIELD_DEF(uint32_t, workSpaceOffset2);
|
||||
TILING_DATA_FIELD_DEF(uint32_t, quantGroupNum);
|
||||
TILING_DATA_FIELD_DEF(float, limited);
|
||||
END_TILING_DATA_DEF;
|
||||
REGISTER_TILING_DATA_CLASS(GMMSwigluBaseParamsOp, GMMSwigluBaseParams)
|
||||
|
||||
// SwigluQuant部分tiling 基本信息
|
||||
BEGIN_TILING_DATA_DEF(GMMSwiglu)
|
||||
TILING_DATA_FIELD_DEF(uint32_t, maxProcessRowNum);
|
||||
TILING_DATA_FIELD_DEF(uint32_t, groupListLen);
|
||||
TILING_DATA_FIELD_DEF(uint32_t, tokenLen);
|
||||
END_TILING_DATA_DEF;
|
||||
REGISTER_TILING_DATA_CLASS(GMMSwigluOp, GMMSwiglu)
|
||||
|
||||
// 结构体集合
|
||||
BEGIN_TILING_DATA_DEF(GMMSwigluQuantTilingData)
|
||||
TILING_DATA_FIELD_DEF_STRUCT(GMMSwigluBaseParams, gmmSwigluBaseParams);
|
||||
TILING_DATA_FIELD_DEF_STRUCT(GMMSwiglu, gmmSwiglu);
|
||||
TILING_DATA_FIELD_DEF_STRUCT(TCubeTiling, mmTilingData);
|
||||
END_TILING_DATA_DEF;
|
||||
|
||||
REGISTER_TILING_DATA_CLASS(GroupedMatmulSwigluQuant, GMMSwigluQuantTilingData)
|
||||
} // namespace optiling
|
||||
|
||||
namespace GroupedMatmulSwigluQuantTiling {
|
||||
constexpr uint32_t X_INDEX = 0;
|
||||
constexpr uint32_t WEIGHT_INDEX = 1;
|
||||
constexpr uint32_t WEIGHT_SCALE_INDEX = 2;
|
||||
constexpr uint32_t GROUPLIST_INDEX = 5;
|
||||
constexpr uint32_t BATCH_MODE_SCHEDULE = 1;
|
||||
constexpr uint32_t DIM_0 = 0;
|
||||
constexpr uint32_t DIM_1 = 1;
|
||||
constexpr uint32_t DIM_2 = 2;
|
||||
constexpr uint32_t DIM_3 = 3;
|
||||
constexpr uint32_t DIM_4 = 4;
|
||||
constexpr uint32_t NUM_FOUR = 4;
|
||||
constexpr uint32_t NUM_EIGHT = 8;
|
||||
constexpr uint32_t SYS_WORKSPACE_SIZE = 16 * 1024 * 1024;
|
||||
constexpr int64_t USER_WORKSPACE_LIMIT = 64 * 1024 * 1024;
|
||||
constexpr int64_t DOUBLE_WORKSPACE_SPLIT = 2;
|
||||
constexpr uint32_t ATTR_INDEX_LIMITED = 2;
|
||||
constexpr int64_t INT32_DTYPE_SIZE = 4;
|
||||
constexpr int64_t FP32_DTYPE_SIZE = 4;
|
||||
constexpr int64_t FP32_BLOCK_SIZE = 8;
|
||||
constexpr int64_t BLOCK_BYTE = 32;
|
||||
constexpr int64_t SWIGLU_REDUCE_FACTOR = 2;
|
||||
constexpr int64_t DOUBLE_BUFFER = 2;
|
||||
constexpr int64_t ND_WEIGHT_DIM_LIMIT = 3;
|
||||
constexpr int64_t NZ_WEIGHT_DIM_LIMIT = 5;
|
||||
constexpr int64_t DOUBLE_ROW = 2;
|
||||
constexpr int64_t PERCHANNEL_WSCALE_DIM_LIMIT = 2;
|
||||
constexpr int64_t PERGROUP_WSCALE_DIM_LIMIT = 3;
|
||||
constexpr int64_t A8W4_MSD_TILING_KEY_MODE = 2;
|
||||
constexpr int64_t SPLITWORKSPACE_TILING_KEY_MODE = 1;
|
||||
constexpr int64_t COMMON_TILING_KEY_MODE = 0;
|
||||
constexpr int64_t A8W4_TOKEN_THRESHOLD = 32;
|
||||
constexpr int64_t A8W4_BASEM = 128;
|
||||
constexpr int64_t A8W4_BASEK = 256;
|
||||
constexpr int64_t A8W4_BASEN = 256;
|
||||
} // namespace GroupedMatmulSwigluQuantTiling
|
||||
|
||||
#endif // AIR_CXX_RUNTIME_V2_OP_IMPL_GROUPED_MATMUL_SWIGLU_QUANT_H
|
||||
@@ -0,0 +1,540 @@
|
||||
/**
|
||||
* Copyright (c) 2025 Huawei Technologies Co., Ltd.
|
||||
* This program is free software, you can redistribute it and/or modify it under the terms and conditions of
|
||||
* CANN Open Software License Agreement Version 2.0 (the "License").
|
||||
* Please refer to the License for details. You may not use this file except in compliance with the License.
|
||||
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
|
||||
* INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
|
||||
* See LICENSE in the root of the software repository for the full text of the License.
|
||||
*/
|
||||
#include <dlfcn.h>
|
||||
#include <new>
|
||||
#include "aclnn_kernels/contiguous.h"
|
||||
#include "acl/acl.h"
|
||||
#include "aclnn/aclnn_base.h"
|
||||
#include "aclnn_kernels/common/op_error_check.h"
|
||||
#include "opdev/common_types.h"
|
||||
#include "opdev/data_type_utils.h"
|
||||
#include "opdev/format_utils.h"
|
||||
#include "opdev/op_dfx.h"
|
||||
#include "opdev/op_executor.h"
|
||||
#include "opdev/op_log.h"
|
||||
#include "opdev/platform.h"
|
||||
#include "opdev/shape_utils.h"
|
||||
#include "opdev/tensor_view_utils.h"
|
||||
#include "opdev/make_op_executor.h"
|
||||
#include "grouped_matmul_swiglu_quant.h"
|
||||
#include "aclnn_grouped_matmul_swiglu_quant_weight_nz.h"
|
||||
#include "aclnn_grouped_matmul_swiglu_quant.h"
|
||||
|
||||
using namespace op;
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
static constexpr int64_t SPLIT = 2L;
|
||||
static constexpr int64_t K_LIMIT_A8W8 = 65536L;
|
||||
static constexpr int64_t K_LIMIT_A8W4 = 20000L;
|
||||
static constexpr int64_t N_LIMIT = 10240L;
|
||||
static constexpr int64_t NZ_DIM_4_INT8 = 32L;
|
||||
static constexpr int64_t NZ_DIM_4_INT4 = 64L;
|
||||
static constexpr int64_t NZ_DIM_3 = 16L;
|
||||
static constexpr int64_t OUTPUT_IDX_0 = 0L;
|
||||
static constexpr int64_t OUTPUT_IDX_1 = 1L;
|
||||
static constexpr int64_t DIM_IDX_0 = 0L;
|
||||
static constexpr int64_t DIM_IDX_1 = 1L;
|
||||
static constexpr int64_t DIM_IDX_2 = 2L;
|
||||
static constexpr int64_t DIM_IDX_3 = 4L;
|
||||
static constexpr size_t X_DIM_LIMIT = 2UL;
|
||||
static constexpr size_t WEIGHT_ND_DIM_LIMIT = 3UL;
|
||||
static constexpr size_t WEIGHT_NZ_DIM_LIMIT = 5UL;
|
||||
static constexpr size_t WEIGHT_SCALE_DIM_LIMIT = 2UL;
|
||||
static constexpr size_t WEIGHT_SCALE_PERGROUP_DIM_LIMIT = 3UL;
|
||||
static constexpr size_t WEIGHT_SCALE_PERCHANNEL_DIM_LIMIT = 2UL;
|
||||
static constexpr size_t TOKEN_SCALE_DIM_LIMIT = 1UL;
|
||||
static constexpr size_t BIAS_DIM_LIMIT = 2UL;
|
||||
static constexpr size_t GROUP_LIST_DIM_LIMIT = 1UL;
|
||||
static constexpr size_t QUANTOUT_DIM_LIMIT = 2UL;
|
||||
static constexpr size_t QUANTSCALEOUT_DIM_LIMIT = 1UL;
|
||||
static constexpr size_t INT4_PER_INT32 = 8UL;
|
||||
bool isEnableWeightAssistanceMatrix = false;
|
||||
int dequantMode = 0;
|
||||
|
||||
static const std::initializer_list<DataType> X_DTYPE_SUPPORT_LIST = {DataType::DT_INT8};
|
||||
static const std::initializer_list<DataType> WEIGHT_DTYPE_SUPPORT_LIST = {DataType::DT_INT8, DataType::DT_INT4};
|
||||
static const std::initializer_list<DataType> WEIGHT_SCALE_DTYPE_SUPPORT_LIST = {
|
||||
DataType::DT_FLOAT, DataType::DT_FLOAT16, DataType::DT_BF16};
|
||||
static const std::initializer_list<DataType> WEIGHT_SCALE_A8W4_DTYPE_SUPPORT_LIST = {DataType::DT_UINT64};
|
||||
static const std::initializer_list<DataType> X_SCALE_DTYPE_SUPPORT_LIST = {DataType::DT_FLOAT, DataType::DT_FLOAT16,
|
||||
DataType::DT_BF16};
|
||||
static const std::initializer_list<DataType> GROUP_LIST_DTYPE_SUPPORT_LIST = {DataType::DT_INT64};
|
||||
static const std::initializer_list<DataType> QUANTOUT_DTYPE_SUPPORT_LIST = {DataType::DT_INT8};
|
||||
static const std::initializer_list<DataType> QUANTSCALEOUT_DTYPE_SUPPORT_LIST = {DataType::DT_FLOAT};
|
||||
static const std::initializer_list<DataType> BIAS_DTYPE_SUPPORT_LIST = {DataType::DT_FLOAT};
|
||||
static bool CheckNotNull(const aclTensor *x, const aclTensor *weight, const aclTensor *bias, const aclTensor *offset,
|
||||
const aclTensor *weightScale, const aclTensor *xScale, const aclTensor *groupList,
|
||||
const aclTensor *output, const aclTensor *outputScale, const aclTensor *outputOffset)
|
||||
{
|
||||
OP_CHECK_NULL(x, return false);
|
||||
OP_CHECK_NULL(weight, return false);
|
||||
OP_CHECK_NULL(weightScale, return false);
|
||||
OP_CHECK_NULL(xScale, return false);
|
||||
OP_CHECK_NULL(groupList, return false);
|
||||
OP_CHECK_NULL(output, return false);
|
||||
OP_CHECK_NULL(outputScale, return false);
|
||||
if (x->GetDataType() == DataType::DT_INT8 && weight->GetDataType() == DataType::DT_INT8 && bias != nullptr) {
|
||||
OP_LOGW("aclnnGroupedMatmulSwiGluQuant, The current version does not support the scenario that bias is not 0. "
|
||||
"Features and accuracy are not guaranteed if inputting bias with values other than 0.");
|
||||
} else if (x->GetDataType() == DataType::DT_INT8 && weight->GetDataType() == DataType::DT_INT4 && bias == nullptr) {
|
||||
OP_LOGE(ACLNN_ERR_PARAM_INVALID,
|
||||
"aclnnGroupedMatmulSwiGluQuant, The current version does not support the scenario that without bias. "
|
||||
"When x is Int8 and weight is int4, bias serves as an auxiliary matrix to weight, and this parameter "
|
||||
"cannot be nullptr.");
|
||||
return false;
|
||||
}
|
||||
if (offset != nullptr) {
|
||||
OP_LOGW(
|
||||
"aclnnGroupedMatmulSwiGluQuant, The current version does not support the scenario where offset is not 0. "
|
||||
"Features and accuracy are not guaranteed if inputting bias with values other than 0s.");
|
||||
}
|
||||
if (outputOffset != nullptr) {
|
||||
OP_LOGW("aclnnGroupedMatmulSwiGluQuant, The current version does not support the scenario where outputOffset "
|
||||
"is not 0. Features and accuracy are not guaranteed if inputting bias with values other than 0s.");
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
static bool CheckInputOutDims_A8W8(const aclTensor *x, const aclTensor *weight, const aclTensor *weightScale,
|
||||
const aclTensor *xScale, const aclTensor *groupList, const aclTensor *output,
|
||||
const aclTensor *outputScale)
|
||||
{
|
||||
OP_CHECK_WRONG_DIMENSION(x, X_DIM_LIMIT, return false);
|
||||
op::Format weightViewFormat = weight->GetViewFormat();
|
||||
if (IsPrivateFormat(weightViewFormat)) {
|
||||
OP_CHECK_WRONG_DIMENSION(weight, WEIGHT_NZ_DIM_LIMIT, return false);
|
||||
} else {
|
||||
OP_CHECK_WRONG_DIMENSION(weight, WEIGHT_ND_DIM_LIMIT, return false);
|
||||
}
|
||||
OP_CHECK_WRONG_DIMENSION(weightScale, WEIGHT_SCALE_DIM_LIMIT, return false);
|
||||
OP_CHECK_WRONG_DIMENSION(xScale, TOKEN_SCALE_DIM_LIMIT, return false);
|
||||
OP_CHECK_WRONG_DIMENSION(groupList, GROUP_LIST_DIM_LIMIT, return false);
|
||||
OP_CHECK_WRONG_DIMENSION(output, QUANTOUT_DIM_LIMIT, return false);
|
||||
OP_CHECK_WRONG_DIMENSION(outputScale, QUANTSCALEOUT_DIM_LIMIT, return false);
|
||||
return true;
|
||||
}
|
||||
|
||||
static bool CheckInputOutDims_A8W4(const aclTensor *x, const aclTensor *weight, const aclTensor *bias,
|
||||
const aclTensor *weightScale, const aclTensor *xScale, const aclTensor *groupList,
|
||||
const aclTensor *output, const aclTensor *outputScale)
|
||||
{
|
||||
OP_CHECK_WRONG_DIMENSION(x, X_DIM_LIMIT, return false);
|
||||
op::Format weightViewFormat = weight->GetViewFormat();
|
||||
if (IsPrivateFormat(weightViewFormat)) {
|
||||
OP_CHECK_WRONG_DIMENSION(weight, WEIGHT_NZ_DIM_LIMIT, return false);
|
||||
} else {
|
||||
OP_CHECK_WRONG_DIMENSION(weight, WEIGHT_ND_DIM_LIMIT, return false);
|
||||
}
|
||||
// 支持pergroup、perchannel量化weightScale分别为2维和3维
|
||||
OP_CHECK_MAX_DIM(weightScale, WEIGHT_SCALE_PERGROUP_DIM_LIMIT, return false);
|
||||
OP_CHECK_MIN_DIM(weightScale, WEIGHT_SCALE_PERCHANNEL_DIM_LIMIT, return false);
|
||||
OP_CHECK_WRONG_DIMENSION(bias, BIAS_DIM_LIMIT, return false);
|
||||
OP_CHECK_WRONG_DIMENSION(xScale, TOKEN_SCALE_DIM_LIMIT, return false);
|
||||
OP_CHECK_WRONG_DIMENSION(groupList, GROUP_LIST_DIM_LIMIT, return false);
|
||||
OP_CHECK_WRONG_DIMENSION(output, QUANTOUT_DIM_LIMIT, return false);
|
||||
OP_CHECK_WRONG_DIMENSION(outputScale, QUANTSCALEOUT_DIM_LIMIT, return false);
|
||||
return true;
|
||||
}
|
||||
|
||||
static bool CheckInputOutShape_A8W8(const aclTensor *x, const aclTensor *weight, const aclTensor *weightScale,
|
||||
const aclTensor *xScale, const aclTensor *groupList, const aclTensor *output,
|
||||
const aclTensor *outputScale)
|
||||
{
|
||||
int64_t m = x->GetViewShape().GetDim(0);
|
||||
int64_t k = x->GetViewShape().GetDim(1);
|
||||
int64_t n = weightScale->GetViewShape().GetDim(1);
|
||||
int64_t e = weight->GetViewShape().GetDim(0);
|
||||
if (n % SPLIT != 0) {
|
||||
OP_LOGE(ACLNN_ERR_PARAM_INVALID, "aclnnGroupedMatmulSwiGluQuant, N is %ld , not an even number.", n);
|
||||
return false;
|
||||
}
|
||||
int64_t nAfterHalve = static_cast<int64_t>(n / SPLIT);
|
||||
// x的shape期望为[M, K]
|
||||
op::Shape xExpectShape = {m, k};
|
||||
// weight的NDshape期望为[E, K, N]
|
||||
op::Shape weightNDExpectShape = {e, k, n};
|
||||
// weight的NZshape期望为[E, N // 32, K // 16, 16, 32]
|
||||
op::Shape weightNZExpectShape = {e, static_cast<int64_t>(n / NZ_DIM_4_INT8), static_cast<int64_t>(k / NZ_DIM_3),
|
||||
NZ_DIM_3, NZ_DIM_4_INT8};
|
||||
// weightScale的shape期望为[E, N]
|
||||
op::Shape weightScaleExpectShape = {e, n};
|
||||
// xScale的shape期望为[E, N]
|
||||
op::Shape xScaleExpectShape = {m};
|
||||
// output的shape期望为[M, N]
|
||||
op::Shape outputExpectShape = {m, nAfterHalve};
|
||||
// outputScale的shape期望为[M]
|
||||
op::Shape outputScaleExpectShape = {m};
|
||||
op::Format weightViewFormat = weight->GetViewFormat();
|
||||
OP_CHECK_SHAPE_NOT_EQUAL_WITH_EXPECTED_SIZE(x, xExpectShape, return false);
|
||||
if (IsPrivateFormat(weightViewFormat)) {
|
||||
OP_CHECK_SHAPE_NOT_EQUAL_WITH_EXPECTED_SIZE(weight, weightNZExpectShape, return false);
|
||||
} else {
|
||||
OP_CHECK_SHAPE_NOT_EQUAL_WITH_EXPECTED_SIZE(weight, weightNDExpectShape, return false);
|
||||
}
|
||||
OP_CHECK_SHAPE_NOT_EQUAL_WITH_EXPECTED_SIZE(weightScale, weightScaleExpectShape, return false);
|
||||
OP_CHECK_SHAPE_NOT_EQUAL_WITH_EXPECTED_SIZE(xScale, xScaleExpectShape, return false);
|
||||
|
||||
OP_CHECK_SHAPE_NOT_EQUAL_WITH_EXPECTED_SIZE(output, outputExpectShape, return false);
|
||||
OP_CHECK_SHAPE_NOT_EQUAL_WITH_EXPECTED_SIZE(outputScale, outputScaleExpectShape, return false);
|
||||
// groupList的长度应小于等于weight的专家数
|
||||
int64_t groupListLen = groupList->GetViewShape().GetDim(0);
|
||||
if (groupListLen > e) {
|
||||
OP_LOGE(ACLNN_ERR_PARAM_INVALID,
|
||||
"aclnnGroupedMatmulSwiGluQuant A8W8, Length of 'groupList' out of range"
|
||||
" (expected to be in range of [1, %ld], but got %ld)",
|
||||
e, groupListLen);
|
||||
return false;
|
||||
}
|
||||
if (n > N_LIMIT) {
|
||||
OP_LOGE(ACLNN_ERR_PARAM_INVALID,
|
||||
"aclnnGroupedMatmulSwiGluQuant A8W8: The current version does not support the scenario that "
|
||||
"N(%ld) is greater than %ld.",
|
||||
n, N_LIMIT);
|
||||
return false;
|
||||
}
|
||||
if (k >= K_LIMIT_A8W8) {
|
||||
OP_LOGE(ACLNN_ERR_PARAM_INVALID,
|
||||
"aclnnGroupedMatmulSwiGluQuant A8W8, The current version does not support the scenario."
|
||||
"The tail axis dimension of input0(x) is %ld, which need lower than %ld.",
|
||||
k, K_LIMIT_A8W8);
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
static bool CheckInputOutShape_A8W4(const aclTensor *x, const aclTensor *weight, const aclTensor *bias,
|
||||
const aclTensor *weightScale, const aclTensor *xScale, const aclTensor *groupList,
|
||||
const aclTensor *output, const aclTensor *outputScale)
|
||||
{
|
||||
int64_t e = weight->GetViewShape().GetDim(0);
|
||||
int64_t m = x->GetViewShape().GetDim(0);
|
||||
int64_t k = x->GetViewShape().GetDim(1);
|
||||
int64_t n = 1;
|
||||
int64_t KGroupCount = 1; // K轴的组数，perchannel场景相当于pergroup场景中的组数为1
|
||||
int64_t KGroupSize = k; // K轴每组的元素个数
|
||||
op::Shape weightScaleExpectShape;
|
||||
// 通过weightScale的维度判断是否为perchannel 或 pergroup量化模式
|
||||
if (weightScale->GetViewShape().GetDimNum() == WEIGHT_SCALE_PERCHANNEL_DIM_LIMIT) {
|
||||
// weightScale入参在perchannel场景期望shape [E, N]
|
||||
n = weightScale->GetViewShape().GetDim(DIM_IDX_1);
|
||||
weightScaleExpectShape = {e, n};
|
||||
} else if (weightScale->GetViewShape().GetDimNum() == WEIGHT_SCALE_PERGROUP_DIM_LIMIT) {
|
||||
// weightScale入参在pergroup场景期望shape [E, KGroupCount, N]
|
||||
n = weightScale->GetViewShape().GetDim(DIM_IDX_2);
|
||||
KGroupCount = weightScale->GetViewShape().GetDim(DIM_IDX_1);
|
||||
KGroupSize = KGroupCount > 0 ? k / KGroupCount : k;
|
||||
weightScaleExpectShape = {e, KGroupCount, n};
|
||||
}
|
||||
if (KGroupCount == 0 || k % KGroupCount != 0) {
|
||||
OP_LOGE(ACLNN_ERR_PARAM_INVALID,
|
||||
"aclnnGroupedMatmulSwiGluQuant, "
|
||||
"The number of groups along the k-axis is %ld, and the length of the k-axis is %ld, which is illegal. "
|
||||
"The number of groups must be greater than 0, and k-axis length %% number of groups == 0 must be true.",
|
||||
KGroupCount, k);
|
||||
return false;
|
||||
}
|
||||
if (n % SPLIT != 0) {
|
||||
OP_LOGE(ACLNN_ERR_PARAM_INVALID, "aclnnGroupedMatmulSwiGluQuant, N is %ld , which must even number.", n);
|
||||
return false;
|
||||
}
|
||||
int64_t nAfterHalve = static_cast<int64_t>(n / SPLIT);
|
||||
// x的shape期望为[M, K]
|
||||
op::Shape xExpectShape = {m, k};
|
||||
// weight的NDshape期望为[E, K, N]
|
||||
op::Shape weightNDExpectShape = {e, k, n};
|
||||
op::Shape biasExpectShape = {e, n};
|
||||
// weight的NZshape期望为[E, N // 64, K // 16, 16, 64]
|
||||
op::Shape weightNZExpectShape = {e, static_cast<int64_t>(n / NZ_DIM_4_INT4), static_cast<int64_t>(k / NZ_DIM_3),
|
||||
NZ_DIM_3, NZ_DIM_4_INT4};
|
||||
// xScale的shape期望为[E, N]
|
||||
op::Shape xScaleExpectShape = {m};
|
||||
// output的shape期望为[M, N]
|
||||
op::Shape outputExpectShape = {m, nAfterHalve};
|
||||
// outputScale的shape期望为[M]
|
||||
op::Shape outputScaleExpectShape = {m};
|
||||
op::Format weightViewFormat = weight->GetViewFormat();
|
||||
OP_CHECK_SHAPE_NOT_EQUAL_WITH_EXPECTED_SIZE(x, xExpectShape, return false);
|
||||
if (IsPrivateFormat(weightViewFormat)) {
|
||||
OP_CHECK_SHAPE_NOT_EQUAL_WITH_EXPECTED_SIZE(weight, weightNZExpectShape, return false);
|
||||
} else {
|
||||
OP_CHECK_SHAPE_NOT_EQUAL_WITH_EXPECTED_SIZE(weight, weightNDExpectShape, return false);
|
||||
}
|
||||
OP_CHECK_SHAPE_NOT_EQUAL_WITH_EXPECTED_SIZE(bias, biasExpectShape, return false);
|
||||
OP_CHECK_SHAPE_NOT_EQUAL_WITH_EXPECTED_SIZE(weightScale, weightScaleExpectShape, return false);
|
||||
OP_CHECK_SHAPE_NOT_EQUAL_WITH_EXPECTED_SIZE(xScale, xScaleExpectShape, return false);
|
||||
OP_CHECK_SHAPE_NOT_EQUAL_WITH_EXPECTED_SIZE(output, outputExpectShape, return false);
|
||||
OP_CHECK_SHAPE_NOT_EQUAL_WITH_EXPECTED_SIZE(outputScale, outputScaleExpectShape, return false);
|
||||
// groupList的长度应小于等于weight的专家数
|
||||
int64_t groupListLen = groupList->GetViewShape().GetDim(0);
|
||||
if (groupListLen > e) {
|
||||
OP_LOGE(ACLNN_ERR_PARAM_INVALID,
|
||||
"aclnnGroupedMatmulSwiGluQuant A8W4, Length of 'groupList' out of range"
|
||||
" (expected to be in range of [1, %ld], but got %ld)",
|
||||
e, groupListLen);
|
||||
return false;
|
||||
}
|
||||
if (n > N_LIMIT) {
|
||||
OP_LOGE(ACLNN_ERR_PARAM_INVALID,
|
||||
"aclnnGroupedMatmulSwiGluQuant A8W4, The current version does not support the scenario."
|
||||
"where N after halve is %ld greater than %ld.",
|
||||
n, N_LIMIT);
|
||||
return false;
|
||||
}
|
||||
if (k >= K_LIMIT_A8W4) {
|
||||
OP_LOGE(ACLNN_ERR_PARAM_INVALID,
|
||||
"aclnnGroupedMatmulSwiGluQuant A8W4, The current version does not support the scenario."
|
||||
"The tail axis dimension of input0(x) is %ld, which need lower than %ld.",
|
||||
k, K_LIMIT_A8W4);
|
||||
return false;
|
||||
}
|
||||
(void)KGroupSize;
|
||||
return true;
|
||||
}
|
||||
|
||||
static bool CheckDtypeValid(const aclTensor *x, const aclTensor *weight, const aclTensor *bias,
|
||||
const aclTensor *weightScale, const aclTensor *xScale, const aclTensor *groupList,
|
||||
const aclTensor *output, const aclTensor *outputScale)
|
||||
{
|
||||
OP_CHECK_DTYPE_NOT_SUPPORT(x, X_DTYPE_SUPPORT_LIST, return false);
|
||||
OP_CHECK_DTYPE_NOT_SUPPORT(weight, WEIGHT_DTYPE_SUPPORT_LIST, return false);
|
||||
if (weight->GetDataType() == DataType::DT_INT4) {
|
||||
OP_CHECK_DTYPE_NOT_SUPPORT(bias, BIAS_DTYPE_SUPPORT_LIST, return false);
|
||||
}
|
||||
if (weight->GetDataType() == DataType::DT_INT4) {
|
||||
OP_CHECK_DTYPE_NOT_SUPPORT(weightScale, WEIGHT_SCALE_A8W4_DTYPE_SUPPORT_LIST, return false);
|
||||
} else if (weight->GetDataType() == DataType::DT_INT8) {
|
||||
OP_CHECK_DTYPE_NOT_SUPPORT(weightScale, WEIGHT_SCALE_DTYPE_SUPPORT_LIST, return false);
|
||||
}
|
||||
OP_CHECK_DTYPE_NOT_SUPPORT(xScale, X_SCALE_DTYPE_SUPPORT_LIST, return false);
|
||||
OP_CHECK_DTYPE_NOT_SUPPORT(groupList, GROUP_LIST_DTYPE_SUPPORT_LIST, return false);
|
||||
OP_CHECK_DTYPE_NOT_SUPPORT(output, QUANTOUT_DTYPE_SUPPORT_LIST, return false);
|
||||
OP_CHECK_DTYPE_NOT_SUPPORT(outputScale, QUANTSCALEOUT_DTYPE_SUPPORT_LIST, return false);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
static bool CheckFormat(const aclTensor *x, const aclTensor *weight, const aclTensor *output)
|
||||
{
|
||||
bool isNZ = weight->GetStorageFormat() == op::Format::FORMAT_FRACTAL_NZ;
|
||||
if ((x->GetDataType() == DataType::DT_INT8 && weight->GetDataType() == DataType::DT_INT8) && !isNZ) {
|
||||
// fp16 in fp32 out that is split k template, not precision-advanced now
|
||||
OP_LOGE(ACLNN_ERR_PARAM_INVALID,
|
||||
"aclnnGroupedMatmulSwiGluQuant, The current version does not support the scenario."
|
||||
"weight Format expect is FRACTAL_NZ, but got [%s].",
|
||||
op::ToString(weight->GetStorageFormat()).GetString());
|
||||
return false;
|
||||
}
|
||||
if (IsPrivateFormat(x->GetStorageFormat())) {
|
||||
OP_LOGE(ACLNN_ERR_PARAM_INVALID,
|
||||
"aclnnGroupedMatmulSwiGluQuant, The current version does not support the scenario."
|
||||
"x Format Not support Private Format.");
|
||||
return false;
|
||||
}
|
||||
if (IsPrivateFormat(output->GetStorageFormat())) {
|
||||
OP_LOGE(ACLNN_ERR_PARAM_INVALID,
|
||||
"aclnnGroupedMatmulSwiGluQuant, The current version does not support the scenario."
|
||||
"output Format Not support Private Format.");
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
static void UnpackInt32ToInt4(const aclTensor *&tensorS32, const std::string &tensorType)
|
||||
{
|
||||
OP_LOGD("Unpack %s from int32 to int4 start.", tensorType.c_str());
|
||||
auto tensorS4 = const_cast<aclTensor *>(tensorS32);
|
||||
op::Shape tensorShape = tensorS4->GetViewShape();
|
||||
auto viewShapeDim = tensorShape.GetDimNum();
|
||||
tensorShape[viewShapeDim - 1] = tensorShape[viewShapeDim - 1] * INT4_PER_INT32;
|
||||
tensorS4->SetViewShape(tensorShape);
|
||||
tensorS4->SetStorageShape(tensorShape);
|
||||
tensorS4->SetDataType(DataType::DT_INT4);
|
||||
OP_LOGD("Unpack %s from int32 to int4 finished.", tensorType.c_str());
|
||||
}
|
||||
|
||||
static aclnnStatus CheckParams(const aclTensor *x, const aclTensor *weight, const aclTensor *bias,
|
||||
const aclTensor *offset, const aclTensor *weightScale, const aclTensor *xScale,
|
||||
const aclTensor *groupList, const aclTensor *output, const aclTensor *outputScale,
|
||||
const aclTensor *outputOffset)
|
||||
{
|
||||
// 1. 检查参数是否为空指针
|
||||
CHECK_RET(CheckNotNull(x, weight, bias, offset, weightScale, xScale, groupList, output, outputScale, outputOffset),
|
||||
ACLNN_ERR_PARAM_NULLPTR);
|
||||
// A8W8场景
|
||||
if (x->GetDataType() == DataType::DT_INT8 && weight->GetDataType() == DataType::DT_INT8) {
|
||||
// 2. 校验输入、输出参数维度
|
||||
CHECK_RET(CheckInputOutDims_A8W8(x, weight, weightScale, xScale, groupList, output, outputScale),
|
||||
ACLNN_ERR_PARAM_INVALID);
|
||||
|
||||
// 3. 校验输入、输出shape参数
|
||||
CHECK_RET(CheckInputOutShape_A8W8(x, weight, weightScale, xScale, groupList, output, outputScale),
|
||||
ACLNN_ERR_PARAM_INVALID);
|
||||
}
|
||||
// A8W4场景 INT32为兼容torch_npu考虑，实际计算时，1个INT32数据会被视为8个INT4数据
|
||||
if ((x->GetDataType() == DataType::DT_INT8 && weight->GetDataType() == DataType::DT_INT4) ||
|
||||
(x->GetDataType() == DataType::DT_INT8 && weight->GetDataType() == DataType::DT_INT32)) {
|
||||
// 将INT32视为8个Int4数据，调整viewShape和dtype便于后续统一校验
|
||||
if (weight->GetDataType() == DataType::DT_INT32) {
|
||||
UnpackInt32ToInt4(weight, "weight");
|
||||
}
|
||||
if (weightScale->GetDataType() == DataType::DT_INT64) {
|
||||
auto weightScale_fix = const_cast<aclTensor *>(weightScale);
|
||||
weightScale_fix->SetDataType(DataType::DT_UINT64);
|
||||
}
|
||||
// 2. 校验输入、输出参数维度
|
||||
CHECK_RET(CheckInputOutDims_A8W4(x, weight, bias, weightScale, xScale, groupList, output, outputScale),
|
||||
ACLNN_ERR_PARAM_INVALID);
|
||||
|
||||
// 3. 校验输入、输出shape参数
|
||||
CHECK_RET(CheckInputOutShape_A8W4(x, weight, bias, weightScale, xScale, groupList, output, outputScale),
|
||||
ACLNN_ERR_PARAM_INVALID);
|
||||
}
|
||||
// 4. 检查输入的数据类型是否在支持的数据类型范围之内
|
||||
CHECK_RET(CheckDtypeValid(x, weight, bias, weightScale, xScale, groupList, output, outputScale),
|
||||
ACLNN_ERR_PARAM_INVALID);
|
||||
|
||||
// 5. 检查数据形状是否支持
|
||||
CHECK_RET(CheckFormat(x, weight, output), ACLNN_ERR_PARAM_INVALID);
|
||||
return ACLNN_SUCCESS;
|
||||
}
|
||||
|
||||
static aclnnStatus aclnnGroupedMatmulSwigluQuantGetWorkspaceSizeCommon(
|
||||
const aclTensor *x, const aclTensor *weight, const aclTensor *bias, const aclTensor *offset,
|
||||
const aclTensor *weightScale, const aclTensor *xScale, const aclTensor *groupList, double limited, aclTensor *output,
|
||||
aclTensor *outputScale, aclTensor *outputOffset, uint64_t *workspaceSize, aclOpExecutor **executor)
|
||||
{
|
||||
// 固定写法，创建OpExecutor
|
||||
auto uniqueExecutor = CREATE_EXECUTOR();
|
||||
CHECK_RET(uniqueExecutor.get() != nullptr, ACLNN_ERR_INNER_CREATE_EXECUTOR);
|
||||
// 固定写法，参数检查
|
||||
|
||||
auto ret = CheckParams(x, weight, bias, offset, weightScale, xScale, groupList, output, outputScale, outputOffset);
|
||||
|
||||
CHECK_RET(ret == ACLNN_SUCCESS, ret);
|
||||
// 空Tensor场景
|
||||
if (output->IsEmpty() || groupList->IsEmpty() || outputScale->IsEmpty()) {
|
||||
*workspaceSize = 0;
|
||||
uniqueExecutor.ReleaseTo(executor);
|
||||
return ACLNN_SUCCESS;
|
||||
}
|
||||
// 转连续
|
||||
x = l0op::Contiguous(x, uniqueExecutor.get());
|
||||
CHECK_RET(x != nullptr, ACLNN_ERR_INNER_NULLPTR);
|
||||
// 若weight为私有格式，则不应该做连续性转换 （l0op::Contiguous接口会把viewShape赋值给storageShape）
|
||||
if (IsPrivateFormat(weight->GetStorageFormat())) {
|
||||
weight->SetOriginalShape(weight->GetViewShape());
|
||||
} else {
|
||||
weight = l0op::Contiguous(weight, uniqueExecutor.get());
|
||||
}
|
||||
CHECK_RET(weight != nullptr, ACLNN_ERR_INNER_NULLPTR);
|
||||
weightScale = l0op::Contiguous(weightScale, uniqueExecutor.get());
|
||||
CHECK_RET(weightScale != nullptr, ACLNN_ERR_INNER_NULLPTR);
|
||||
xScale = l0op::Contiguous(xScale, uniqueExecutor.get());
|
||||
CHECK_RET(xScale != nullptr, ACLNN_ERR_INNER_NULLPTR);
|
||||
groupList = l0op::Contiguous(groupList, uniqueExecutor.get());
|
||||
CHECK_RET(groupList != nullptr, ACLNN_ERR_INNER_NULLPTR);
|
||||
// 调用L0算子能力
|
||||
if (bias != nullptr) {
|
||||
isEnableWeightAssistanceMatrix = true;
|
||||
bias = l0op::Contiguous(bias, uniqueExecutor.get());
|
||||
CHECK_RET(bias != nullptr, ACLNN_ERR_INNER_NULLPTR);
|
||||
}
|
||||
if (isEnableWeightAssistanceMatrix && weightScale->GetViewShape().GetDimNum() == WEIGHT_SCALE_PERGROUP_DIM_LIMIT) {
|
||||
dequantMode = 1;
|
||||
}
|
||||
auto ret_0 = l0op::GroupedMatmulSwigluQuant(x, weight, weightScale, xScale, groupList, limited, bias,
|
||||
isEnableWeightAssistanceMatrix, dequantMode, uniqueExecutor.get());
|
||||
CHECK_RET(ret_0 != std::tuple(nullptr, nullptr), ACLNN_ERR_INNER_NULLPTR);
|
||||
auto out0 = std::get<OUTPUT_IDX_0>(ret_0);
|
||||
auto ret_1 = l0op::ViewCopy(out0, output, uniqueExecutor.get());
|
||||
CHECK_RET(ret_1 != nullptr, ACLNN_ERR_INNER_NULLPTR);
|
||||
auto out1 = std::get<OUTPUT_IDX_1>(ret_0);
|
||||
auto ret_2 = l0op::ViewCopy(out1, outputScale, uniqueExecutor.get());
|
||||
CHECK_RET(ret_2 != nullptr, ACLNN_ERR_INNER_NULLPTR);
|
||||
*workspaceSize = uniqueExecutor->GetWorkspaceSize();
|
||||
uniqueExecutor.ReleaseTo(executor);
|
||||
return ACLNN_SUCCESS;
|
||||
}
|
||||
|
||||
aclnnStatus aclnnGroupedMatmulSwigluQuantGetWorkspaceSize(const aclTensor *x, const aclTensor *weight,
|
||||
const aclTensor *bias, const aclTensor *offset,
|
||||
const aclTensor *weightScale, const aclTensor *xScale,
|
||||
const aclTensor *groupList, double limited, aclTensor *output,
|
||||
aclTensor *outputScale, aclTensor *outputOffset,
|
||||
uint64_t *workspaceSize, aclOpExecutor **executor)
|
||||
{
|
||||
OP_CHECK_COMM_INPUT(workspaceSize, executor);
|
||||
L2_DFX_PHASE_1(aclnnGroupedMatmulSwigluQuant, DFX_IN(x, weight, bias, offset, weightScale, xScale, groupList, limited),
|
||||
DFX_OUT(output, outputScale, outputOffset));
|
||||
// 固定写法，创建OpExecutor
|
||||
return aclnnGroupedMatmulSwigluQuantGetWorkspaceSizeCommon(x, weight, bias, offset, weightScale, xScale, groupList, limited,
|
||||
output, outputScale, outputOffset, workspaceSize,
|
||||
executor);
|
||||
}
|
||||
|
||||
aclnnStatus aclnnGroupedMatmulSwigluQuantWeightNZGetWorkspaceSize(const aclTensor *x, const aclTensor *weight,
|
||||
const aclTensor *bias, const aclTensor *offset,
|
||||
const aclTensor *weightScale, const aclTensor *xScale,
|
||||
const aclTensor *groupList, double limited, aclTensor *output,
|
||||
aclTensor *outputScale, aclTensor *outputOffset,
|
||||
uint64_t *workspaceSize, aclOpExecutor **executor)
|
||||
{
|
||||
OP_CHECK_COMM_INPUT(workspaceSize, executor);
|
||||
L2_DFX_PHASE_1(aclnnGroupedMatmulSwigluQuantWeightNZ,
|
||||
DFX_IN(x, weight, bias, offset, weightScale, xScale, groupList),
|
||||
DFX_OUT(output, outputScale, outputOffset));
|
||||
// weight在该场景下强制绑定StorageFormat 和 ViewFormat 为NZ
|
||||
CHECK_RET(weight != nullptr, ACLNN_ERR_PARAM_NULLPTR);
|
||||
auto storgeShape = weight->GetStorageShape();
|
||||
auto viewShape = weight->GetViewShape();
|
||||
aclTensor *weightNZ = const_cast<aclTensor *>(weight);
|
||||
CHECK_COND((storgeShape.GetDimNum() == WEIGHT_NZ_DIM_LIMIT), ACLNN_ERR_PARAM_INVALID,
|
||||
"aclnnGroupedMatmulSwigluQuantWeightNZ, The dimnum of storageShape for second input (weight)"
|
||||
"must be 5. \n But StorageShape got %s , and dimNum is %lu.",
|
||||
op::ToString(storgeShape).GetString(), storgeShape.GetDimNum());
|
||||
// weight的StorageFormat无条件视为NZ
|
||||
weightNZ->SetStorageFormat(op::Format::FORMAT_FRACTAL_NZ);
|
||||
if (viewShape.GetDimNum() == WEIGHT_NZ_DIM_LIMIT) {
|
||||
// 若weight的viewShape为5维则视为NZ
|
||||
weightNZ->SetViewFormat(op::Format::FORMAT_FRACTAL_NZ);
|
||||
} else if (viewShape.GetDimNum() == WEIGHT_ND_DIM_LIMIT) {
|
||||
// 若weight的viewShape为3维则视为ND
|
||||
weightNZ->SetViewFormat(op::Format::FORMAT_ND);
|
||||
}
|
||||
// 调用公共接口
|
||||
return aclnnGroupedMatmulSwigluQuantGetWorkspaceSizeCommon(x, weight, bias, offset, weightScale, xScale, groupList, limited,
|
||||
output, outputScale, outputOffset, workspaceSize,
|
||||
executor);
|
||||
}
|
||||
|
||||
aclnnStatus aclnnGroupedMatmulSwigluQuant(void *workspace, uint64_t workspaceSize, aclOpExecutor *executor,
|
||||
aclrtStream stream)
|
||||
{
|
||||
L2_DFX_PHASE_2(aclnnGroupedMatmulSwigluQuant);
|
||||
CHECK_COND(CommonOpExecutorRun(workspace, workspaceSize, executor, stream) == ACLNN_SUCCESS, ACLNN_ERR_INNER,
|
||||
"This is an error in GroupedMatmulSwigluQuant launch aicore");
|
||||
return ACLNN_SUCCESS;
|
||||
}
|
||||
|
||||
aclnnStatus aclnnGroupedMatmulSwigluQuantWeightNZ(void *workspace, uint64_t workspaceSize, aclOpExecutor *executor,
|
||||
aclrtStream stream)
|
||||
{
|
||||
L2_DFX_PHASE_2(aclnnGroupedMatmulSwigluQuantWeightNZ);
|
||||
CHECK_COND(CommonOpExecutorRun(workspace, workspaceSize, executor, stream) == ACLNN_SUCCESS, ACLNN_ERR_INNER,
|
||||
"This is an error in GroupedMatmulSwigluQuantWeightNZ launch aicore");
|
||||
return ACLNN_SUCCESS;
|
||||
}
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
@@ -0,0 +1,58 @@
|
||||
/**
|
||||
* Copyright (c) 2025 Huawei Technologies Co., Ltd.
|
||||
* This program is free software, you can redistribute it and/or modify it under the terms and conditions of
|
||||
* CANN Open Software License Agreement Version 2.0 (the "License").
|
||||
* Please refer to the License for details. You may not use this file except in compliance with the License.
|
||||
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
|
||||
* INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
|
||||
* See LICENSE in the root of the software repository for the full text of the License.
|
||||
*/
|
||||
#ifndef OP_API_INC_GROUPED_MATMUL_SWIGLU_QUANT_H
|
||||
#define OP_API_INC_GROUPED_MATMUL_SWIGLU_QUANT_H
|
||||
#include "aclnn/aclnn_base.h"
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
/**
|
||||
* @brief aclnnGroupedMatmulSwigluQuant的第一段接口，根据具体的计算流程，计算workspace大小。
|
||||
* @domain aclnn_ops_infer
|
||||
*
|
||||
* @param [in] x: 表示公式中的x，数据类型支持INT8数据类型，数据格式支持ND。
|
||||
* @param [in] weight:
|
||||
* 表示公式中的weight，数据类型支持INT8数据类型，数据格式支持NZ。
|
||||
* @param [in] weightScale:
|
||||
* 表示量化参数，数据类型支持FLOAT16、BFLOAT16、FLOAT32数据类型，数据格式支持ND，支持的最大长度为128个。 表示per
|
||||
* Channel参数，数据类型支持FLOAT16，BFLOAT16数据类型，数据格式支持ND。
|
||||
* @param [in] xScale:
|
||||
* 表示per Token量化参数，数据类型支持FLOAT32数据类型，数据格式支持ND。
|
||||
* @param [in] groupList: 必选参数，代表输入和输出分组轴上的索引情况，数据类型支持INT64。
|
||||
* @param [out] quantOutput: 表示公式中的out，数据类型支持INT8数据类型，数据格式支持ND。
|
||||
* @param [out] quantScaleOutput: 表示公式中的outQuantScale，数据类型支持Float32数据类型。
|
||||
* @param [out] workspaceSize: 返回用户需要在npu device侧申请的workspace大小。
|
||||
* @param [out] executor: 返回op执行器，包含算子计算流程。
|
||||
* @return aclnnStatus: 返回状态码。
|
||||
*/
|
||||
__attribute__((visibility("default"))) aclnnStatus aclnnGroupedMatmulSwigluQuantGetWorkspaceSize(
|
||||
const aclTensor *x, const aclTensor *weight, const aclTensor *bias, const aclTensor *offset,
|
||||
const aclTensor *weightScale, const aclTensor *xScale, const aclTensor *groupList, double limited, aclTensor *output,
|
||||
aclTensor *outputScale, aclTensor *outputOffset, uint64_t *workspaceSize, aclOpExecutor **executor);
|
||||
|
||||
/**
|
||||
* @brief aclnnGroupedMatmulSwigluQuant的第二段接口，用于执行计算。
|
||||
* @param [in] workspace: 在npu device侧申请的workspace内存起址。
|
||||
* @param [in] workspaceSize: 在npu
|
||||
* device侧申请的workspace大小，由第一段接口aclnnGroupedMatmulSwigluQuantGetWorkspaceSize获取。
|
||||
* @param [in] stream: acl stream流。
|
||||
* @param [in] executor: op执行器，包含了算子计算流程。
|
||||
* @return aclnnStatus: 返回状态码。
|
||||
*/
|
||||
__attribute__((visibility("default"))) aclnnStatus
|
||||
aclnnGroupedMatmulSwigluQuant(void *workspace, uint64_t workspaceSize, aclOpExecutor *executor, aclrtStream stream);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,60 @@
|
||||
/**
|
||||
* Copyright (c) 2025 Huawei Technologies Co., Ltd.
|
||||
* This program is free software, you can redistribute it and/or modify it under the terms and conditions of
|
||||
* CANN Open Software License Agreement Version 2.0 (the "License").
|
||||
* Please refer to the License for details. You may not use this file except in compliance with the License.
|
||||
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
|
||||
* INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
|
||||
* See LICENSE in the root of the software repository for the full text of the License.
|
||||
*/
|
||||
#ifndef OP_API_INC_GROUPED_MATMUL_SWIGLU_QUANT_WEIGHT_NZ_H
|
||||
#define OP_API_INC_GROUPED_MATMUL_SWIGLU_QUANT_WEIGHT_NZ_H
|
||||
#include "aclnn/aclnn_base.h"
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
/**
|
||||
* @brief aclnnGroupedMatmulSwigluQuantWeightNZ的第一段接口，根据具体的计算流程，计算workspace大小。
|
||||
* @domain aclnn_ops_infer
|
||||
*
|
||||
* @param [in] x: 表示公式中的x，数据类型支持INT8数据类型，数据格式支持ND。
|
||||
* @param [in] weight:
|
||||
* 表示公式中的weight，数据类型支持INT8数据类型，数据格式支持NZ。
|
||||
* @param [in] weightScale:
|
||||
* 表示量化参数，数据类型支持FLOAT16、BFLOAT16、FLOAT32数据类型，数据格式支持ND，支持的最大长度为128个。 表示per
|
||||
* Channel参数，数据类型支持FLOAT16，BFLOAT16数据类型，数据格式支持ND。
|
||||
* @param [in] xScale:
|
||||
* 表示per Token量化参数，数据类型支持FLOAT32数据类型，数据格式支持ND。
|
||||
* @param [in] groupList: 必选参数，代表输入和输出分组轴上的索引情况，数据类型支持INT64。
|
||||
* @param [out] quantOutput: 表示公式中的out，数据类型支持INT8数据类型，数据格式支持ND。
|
||||
* @param [out] quantScaleOutput: 表示公式中的outQuantScale，数据类型支持Float32数据类型。
|
||||
* @param [out] workspaceSize: 返回用户需要在npu device侧申请的workspace大小。
|
||||
* @param [out] executor: 返回op执行器，包含算子计算流程。
|
||||
* @return aclnnStatus: 返回状态码。
|
||||
*/
|
||||
__attribute__((visibility("default"))) aclnnStatus aclnnGroupedMatmulSwigluQuantWeightNZGetWorkspaceSize(
|
||||
const aclTensor *x, const aclTensor *weight, const aclTensor *bias, const aclTensor *offset,
|
||||
const aclTensor *weightScale, const aclTensor *xScale, const aclTensor *groupList, double limited, aclTensor *output,
|
||||
aclTensor *outputScale, aclTensor *outputOffset, uint64_t *workspaceSize, aclOpExecutor **executor);
|
||||
|
||||
/**
|
||||
* @brief aclnnGroupedMatmulSwigluQuantWeightNZ的第二段接口，用于执行计算。
|
||||
* @param [in] workspace: 在npu device侧申请的workspace内存起址。
|
||||
* @param [in] workspaceSize: 在npu
|
||||
* device侧申请的workspace大小，由第一段接口aclnnGroupedMatmulSwigluQuantWeightNZGetWorkspaceSize获取。
|
||||
* @param [in] stream: acl stream流。
|
||||
* @param [in] executor: op执行器，包含了算子计算流程。
|
||||
* @return aclnnStatus: 返回状态码。
|
||||
*/
|
||||
__attribute__((visibility("default"))) aclnnStatus aclnnGroupedMatmulSwigluQuantWeightNZ(void *workspace,
|
||||
uint64_t workspaceSize,
|
||||
aclOpExecutor *executor,
|
||||
aclrtStream stream);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,58 @@
|
||||
/**
|
||||
* Copyright (c) 2025 Huawei Technologies Co., Ltd.
|
||||
* This program is free software, you can redistribute it and/or modify it under the terms and conditions of
|
||||
* CANN Open Software License Agreement Version 2.0 (the "License").
|
||||
* Please refer to the License for details. You may not use this file except in compliance with the License.
|
||||
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
|
||||
* INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
|
||||
* See LICENSE in the root of the software repository for the full text of the License.
|
||||
*/
|
||||
|
||||
#include "opdev/op_log.h"
|
||||
#include "opdev/op_dfx.h"
|
||||
#include "opdev/make_op_executor.h"
|
||||
#include "grouped_matmul_swiglu_quant.h"
|
||||
|
||||
using namespace op;
|
||||
|
||||
namespace l0op {
|
||||
OP_TYPE_REGISTER(GroupedMatmulSwigluQuant);
|
||||
|
||||
const std::tuple<aclTensor *, aclTensor *>
|
||||
GroupedMatmulSwigluQuant(const aclTensor *x, const aclTensor *weight, const aclTensor *perChannelScale,
|
||||
const aclTensor *perTokenScale, const aclTensor *groupList, double limited,
|
||||
const aclTensor *weightAssistanceMatrix, bool isEnableWeightAssistanceMatrix, int dequantMode,
|
||||
aclOpExecutor *executor)
|
||||
{
|
||||
L0_DFX(GroupedMatmulSwigluQuant, x, weight, perChannelScale, perTokenScale, weightAssistanceMatrix, groupList, limited,
|
||||
isEnableWeightAssistanceMatrix, dequantMode);
|
||||
if (x == nullptr) {
|
||||
OP_LOGE(ACLNN_ERR_PARAM_INVALID, "x is nullptr.");
|
||||
return std::tuple(nullptr, nullptr);
|
||||
}
|
||||
int64_t m = perTokenScale->GetViewShape().GetDim(0);
|
||||
int64_t n = perChannelScale->GetViewShape().GetDim(1);
|
||||
int64_t nAfterHalve = static_cast<int64_t>(n / 2);
|
||||
gert::Shape outShape({m, nAfterHalve});
|
||||
gert::Shape scaleOutShape({m});
|
||||
auto out = executor->AllocTensor(outShape, DataType::DT_INT8, ge::FORMAT_ND);
|
||||
auto scaleOut = executor->AllocTensor(scaleOutShape, DataType::DT_FLOAT, ge::FORMAT_ND);
|
||||
auto ret = INFER_SHAPE(GroupedMatmulSwigluQuant,
|
||||
OP_INPUT(x, weight, perChannelScale, perTokenScale, weightAssistanceMatrix, groupList),
|
||||
OP_OUTPUT(out, scaleOut), OP_ATTR(isEnableWeightAssistanceMatrix, dequantMode, limited));
|
||||
if (ret != ACLNN_SUCCESS) {
|
||||
OP_LOGE(ACLNN_ERR_PARAM_INVALID, "InferShape failed.");
|
||||
return std::tuple(nullptr, nullptr);
|
||||
}
|
||||
ret = ADD_TO_LAUNCHER_LIST_AICORE(
|
||||
GroupedMatmulSwigluQuant,
|
||||
OP_INPUT(x, weight, perChannelScale, perTokenScale, weightAssistanceMatrix, groupList),
|
||||
OP_OUTPUT(out, scaleOut), OP_ATTR(isEnableWeightAssistanceMatrix, dequantMode, limited));
|
||||
if (ret != ACLNN_SUCCESS) {
|
||||
OP_LOGE(ACLNN_ERR_PARAM_INVALID, "ADD_TO_LAUNCHER_LIST_AICORE failed.");
|
||||
return std::tuple(nullptr, nullptr);
|
||||
}
|
||||
return std::tie(out, scaleOut);
|
||||
}
|
||||
|
||||
} // namespace l0op
|
||||
@@ -0,0 +1,23 @@
|
||||
/**
|
||||
* Copyright (c) 2025 Huawei Technologies Co., Ltd.
|
||||
* This program is free software, you can redistribute it and/or modify it under the terms and conditions of
|
||||
* CANN Open Software License Agreement Version 2.0 (the "License").
|
||||
* Please refer to the License for details. You may not use this file except in compliance with the License.
|
||||
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
|
||||
* INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
|
||||
* See LICENSE in the root of the software repository for the full text of the License.
|
||||
*/
|
||||
#ifndef OP_API_INC_LEVEL0_OP_GROUPED_MATMUL_SWIGLU_QUANT_OP_H
|
||||
#define OP_API_INC_LEVEL0_OP_GROUPED_MATMUL_SWIGLU_QUANT_OP_H
|
||||
|
||||
#include "opdev/op_executor.h"
|
||||
|
||||
namespace l0op {
|
||||
const std::tuple<aclTensor *, aclTensor *>
|
||||
GroupedMatmulSwigluQuant(const aclTensor *x, const aclTensor *weight, const aclTensor *perChannelScale,
|
||||
const aclTensor *perTokenScale, const aclTensor *groupList, double limited,
|
||||
const aclTensor *weightAssistanceMatrix, bool isEnableWeightAssistanceMatrix, int dequantMode,
|
||||
aclOpExecutor *executor);
|
||||
}
|
||||
|
||||
#endif
|
||||
Reference in New Issue
Block a user