EngineX-Ascend/enginex-ascend-910-vllm
11
0
Fork 1
Code Issues Pull Requests Actions Projects Releases Wiki Activity

init v0.23.0

Browse Source 
Signed-off-by: Sun Ruoxi <sunruoxi@4paradigm.com>
This commit is contained in:
Sun Ruoxi
2026-08-27 15:11:51 +08:00
parent b582a8e7d1
commit 7f8a1b1f7a
2849 changed files with 712887 additions and 22001 deletions
Download Patch File Download Diff File Expand all files Collapse all files

38
csrc/mc2/dispatch_ffn_combine/op_host/CMakeLists.txt Normal file
View File

@@ -0,0 +1,38 @@
set(_DISPATCH_FFN_INC_OPTS)
if (EXISTS ${ASCEND_CANN_PACKAGE_PATH}/aarch64-linux/ascendc/include)
list(APPEND _DISPATCH_FFN_INC_OPTS -I${ASCEND_CANN_PACKAGE_PATH}/aarch64-linux/ascendc/include)
elseif (EXISTS ${ASCEND_CANN_PACKAGE_PATH}/arm64-linux/ascendc/include)
list(APPEND _DISPATCH_FFN_INC_OPTS -I${ASCEND_CANN_PACKAGE_PATH}/arm64-linux/ascendc/include)
elseif (EXISTS ${ASCEND_CANN_PACKAGE_PATH}/${CMAKE_SYSTEM_PROCESSOR}-linux/ascendc/include)
list(APPEND _DISPATCH_FFN_INC_OPTS -I${ASCEND_CANN_PACKAGE_PATH}/${CMAKE_SYSTEM_PROCESSOR}-linux/ascendc/include)
endif()
if (EXISTS ${CMAKE_SOURCE_DIR}/third_party/catlass/include)
list(APPEND _DISPATCH_FFN_INC_OPTS -I${CMAKE_SOURCE_DIR}/third_party/catlass/include)
endif()
add_op_to_compiled_list()
if (BUILD_OPEN_PROJECT)
target_sources(op_host_aclnnInner PRIVATE
dispatch_ffn_combine_def.cpp
)
endif()
add_ops_compile_options(
OP_NAME DispatchFFNCombine
OPTIONS
--cce-auto-sync=on
-Wno-deprecated-declarations
-DHCCL_COMM
-DCATLASS_ARCH=2201
${_DISPATCH_FFN_INC_OPTS}
)
if (NOT BUILD_OPS_RTY_KERNEL)
add_modules_sources(OPTYPE dispatch_ffn_combine ACLNNTYPE aclnn_inner)
target_include_directories(${OPHOST_NAME}_tiling_obj PRIVATE
${CMAKE_CURRENT_SOURCE_DIR}
${CMAKE_CURRENT_SOURCE_DIR}/../op_kernel
)
endif()

99
csrc/mc2/dispatch_ffn_combine/op_host/dispatch_ffn_combine_def.cpp Normal file
View File

@@ -0,0 +1,99 @@
/**
* Copyright (c) 2025 Huawei Technologies Co., Ltd.
* This file is a part of the CANN Open Software.
* Licensed under CANN Open Software License Agreement Version 1.0 (the "License").
* Please refer to the License for details. You may not use this file except in compliance with the License.
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
* INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
* See LICENSE in the root of the software repository for the full text of the License.
*/
/*!
* \file dispatch_ffn_combine_def.cpp
* \brief
*/
#include "register/op_def_registry.h"
namespace ops {
class DispatchFFNCombine : public OpDef {
public:
explicit DispatchFFNCombine(const char *name) : OpDef(name) {
this->Input("a")
.ParamType(REQUIRED)
.DataType({ge::DT_FLOAT16, ge::DT_BF16, ge::DT_BF16})
.Format({ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND})
.UnknownShapeFormat({ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND});
this->Input("w1")
.ParamType(DYNAMIC)
.DataType({ge::DT_INT8, ge::DT_INT8, ge::DT_INT8})
.Format({ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_FRACTAL_NZ})
.UnknownShapeFormat({ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_FRACTAL_NZ})
.IgnoreContiguous();
this->Input("w2")
.ParamType(DYNAMIC)
.DataType({ge::DT_INT8, ge::DT_INT8, ge::DT_INT8})
.Format({ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_FRACTAL_NZ})
.UnknownShapeFormat({ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_FRACTAL_NZ})
.IgnoreContiguous();
this->Input("expertIdx")
.ParamType(REQUIRED)
.DataType({ge::DT_INT32, ge::DT_INT32, ge::DT_INT32})
.Format({ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND})
.UnknownShapeFormat({ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND});
this->Input("scale1")
.ParamType(DYNAMIC)
.DataType({ge::DT_INT64, ge::DT_INT64, ge::DT_INT64})
.Format({ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND})
.UnknownShapeFormat({ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND});
this->Input("scale2")
.ParamType(DYNAMIC)
.DataType({ge::DT_INT64, ge::DT_INT64, ge::DT_INT64})
.Format({ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND})
.UnknownShapeFormat({ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND});
this->Input("probs")
.ParamType(REQUIRED)
.DataType({ge::DT_FLOAT, ge::DT_FLOAT, ge::DT_FLOAT})
.Format({ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND})
.UnknownShapeFormat({ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND});
this->Input("xActiveMaskOptional")
.ParamType(OPTIONAL)
.DataType({ge::DT_BOOL, ge::DT_BOOL, ge::DT_BOOL})
.Format({ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND})
.UnknownShapeFormat({ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND});
// Output
this->Output("out")
.ParamType(REQUIRED)
.DataType({ge::DT_FLOAT16, ge::DT_BF16, ge::DT_BF16})
.Format({ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND})
.UnknownShapeFormat({ge::FORMAT_ND, ge::FORMAT_ND,ge::FORMAT_ND});
this->Output("expert_token_nums")
.ParamType(REQUIRED)
.DataType({ge::DT_INT32, ge::DT_INT32, ge::DT_INT32})
.Format({ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND})
.UnknownShapeFormat({ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND});
this->Attr("group").AttrType(REQUIRED).String();
this->Attr("M").AttrType(OPTIONAL).Int();
this->Attr("transB").AttrType(OPTIONAL).Bool(false);
this->Attr("weightNz").AttrType(OPTIONAL).Bool(false);
this->Attr("swigluLimit").AttrType(OPTIONAL).Float(0.0f);
OpAICoreConfig aicore_config;
aicore_config.DynamicCompileStaticFlag(true)
.DynamicFormatFlag(true)
.DynamicRankSupportFlag(true)
.DynamicShapeSupportFlag(true)
.NeedCheckSupportFlag(false)
.PrecisionReduceFlag(true)
.ExtendCfgInfo("aclnnSupport.value", "support_aclnn")
.ExtendCfgInfo("jitCompile.flag", "static_false")
.ExtendCfgInfo("multiKernelSupportDynamicGraph.value", "multi_kernel");
this->AICore().AddConfig("ascend910_93", aicore_config);
this->AICore().AddConfig("ascend910b", aicore_config);
this->MC2().HcclGroup("group");
}
};
OP_ADD(DispatchFFNCombine);
} // namespace ops

42
csrc/mc2/dispatch_ffn_combine/op_host/dispatch_ffn_combine_proto.cpp Normal file
View File

@@ -0,0 +1,42 @@
/**
* Copyright (c) 2025 Huawei Technologies Co., Ltd.
* This file is a part of the CANN Open Software.
* Licensed under CANN Open Software License Agreement Version 1.0 (the "License").
* Please refer to the License for details. You may not use this file except in compliance with the License.
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
* INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
* See LICENSE in the root of the software repository for the full text of the License.
*/
/*!
* \file dispatch_ffn_proto.cpp
* \brief
*/
#include <graph/utils/type_utils.h>
#include <register/op_impl_registry.h>
// #include "../../common/ophost/op_util.h"
// #include "../../common/ophost/hcom_topo_info.h"
// #include "log/ops_log.h"
using namespace ge;
namespace ops {
const size_t ATTR_GROUP = 0;
const size_t ATTR_RANK_SIZE = 1;
const size_t SUPPORT_DIM_SIZE = 2;
static ge::graphStatus InferShapeDispatchFFNCombine(gert::InferShapeContext* context) {
(void) context;
return ge::GRAPH_SUCCESS;
}
static ge::graphStatus InferDataTypeDispatchFFNCombine(gert::InferDataTypeContext* context) {
(void) context;
// auto d_type = context->GetInputDataType(0);
// context->SetOutputDataType(0, d_type);
return ge::GRAPH_SUCCESS;
}
IMPL_OP_INFERSHAPE(DispatchFFNCombine)
.InferShape(InferShapeDispatchFFNCombine)
.InferDataType(InferDataTypeDispatchFFNCombine);
} // namespace ops

339
csrc/mc2/dispatch_ffn_combine/op_host/dispatch_ffn_combine_tiling.cpp Normal file
View File

@@ -0,0 +1,339 @@
/**
* Copyright (c) 2025 Huawei Technologies Co., Ltd.
* This file is a part of the CANN Open Software.
* Licensed under CANN Open Software License Agreement Version 1.0 (the "License").
* Please refer to the License for details. You may not use this file except in compliance with the License.
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
* INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
* See LICENSE in the root of the software repository for the full text of the License.
*/
/*!
* \file dispatch_ffn_tiling.cpp
* \brief
*/
#include "vector"
#include "register/tilingdata_base.h"
#include "tiling/tiling_api.h"
#include "tiling_base/error_log.h"
#include "hcom_topo_info.h"
#include "register/op_def_registry.h"
#include "../op_kernel/dispatch_ffn_combine_tiling.h"
#include <vector>
#include <map>
#include <algorithm>
#include "../op_kernel/moe_init_routing_quant_v2/moe_init_routing_quant_v2_tiling.h"
using namespace AscendC;
using namespace ge;
#define HCCL_BUFFSIZE "HCCL_BUFFSIZE"
namespace {
// 1. Constant definitions
const char *K_INNER_DEBUG = "DispatchFFNCombine Tiling Debug";
constexpr uint32_t ATTR_GROUP_INDEX = 0;
constexpr uint32_t ATTR_MAX_OUTPUT_SIZE_INDEX = 1;
constexpr uint32_t ATTR_IS_TRANS_B = 2;
constexpr uint32_t ATTR_WEIGHT_NZ = 3;
constexpr uint32_t ATTR_SWIGLU_LIMIT = 4;
constexpr uint64_t INIT_TILINGKEY = 1000000;
constexpr uint64_t TILINGKEY_TRANS_B = 1U;
constexpr uint64_t TILINGKEY_WEIGHT_NZ = 10;
constexpr uint32_t X_INDEX = 0;
constexpr uint32_t WEIGHT_INDEX = 1;
constexpr uint32_t WEIGHT2_INDEX = 2;
constexpr uint32_t EXPERTID_INDEX = 3;
constexpr uint32_t X_ACTIVE_MASK_INDEX = 7;
constexpr uint32_t BLOCK_NUM = 20;
constexpr uint32_t SYSTEM_NEED_WORKSPACE = 16 * 1024 * 1024;
constexpr uint64_t MB_SIZE = 1024 * 1024UL;
}
namespace optiling {
static int32_t CeilDev(int32_t num, int32_t div)
{
if (div == 0) {
return 0;
}
return (num + div - 1) / div;
}
static uint64_t GetMaxWindowSize()
{
uint16_t defaultWindowSize = 200;
const char* hccl_buffsize_env = getenv(HCCL_BUFFSIZE);
if (hccl_buffsize_env != nullptr) {
try {
std::string envStr(hccl_buffsize_env);
unsigned long val = std::stoul(envStr);
if (val <= std::numeric_limits<uint16_t>::max()) {
defaultWindowSize = static_cast<uint16_t>(val);
} else {
OP_LOGW(K_INNER_DEBUG, "HCCL_BUFFSIZE value %lu is out of range, using default.", val);
}
} catch (const std::exception& e) {
OP_LOGE(K_INNER_DEBUG, "Exception encountered when parsing env HCCL_BUFFSIZE: %s", e.what());
}
} else {
OP_LOGD(K_INNER_DEBUG, "Env HCCL_BUFFSIZE not set");
}
const uint64_t maxWindowSize = static_cast<uint64_t>(defaultWindowSize) * MB_SIZE;
OP_LOGD(K_INNER_DEBUG, "Get maxWindowSize is %lu", maxWindowSize);
return maxWindowSize;
}
// Parse and validate rankId, group, worldSize, and isTransB attributes
static ge::graphStatus DispatchFFNCombineCheckAttrAndSetTiling(gert::TilingContext *context, DispatchFFNCombineInfo& info)
{
auto attrs = context->GetAttrs();
OP_TILING_CHECK(attrs == nullptr, OP_LOGE(K_INNER_DEBUG, "attrs is null."), return ge::GRAPH_FAILED);
// TODO: set, validate, and print tiling data related to attributes
auto groupPtr = attrs->GetAttrPointer<char>(static_cast<int>(ATTR_GROUP_INDEX));
auto maxOutputSizePtr = attrs->GetAttrPointer<int>(ATTR_MAX_OUTPUT_SIZE_INDEX);
auto is_trans_b = attrs->GetAttrPointer<bool>(ATTR_IS_TRANS_B);
auto weight_nz = attrs->GetAttrPointer<bool>(ATTR_WEIGHT_NZ);
auto swiglu_limit = attrs->GetAttrPointer<float>(ATTR_SWIGLU_LIMIT);
OP_TILING_CHECK(groupPtr == nullptr || strlen(groupPtr) == 0,
OP_LOGE(K_INNER_DEBUG, "group is invalid."), return GRAPH_FAILED);
OP_TILING_CHECK(is_trans_b == nullptr,
OP_LOGE(K_INNER_DEBUG, "is_trans_b is invalid."), return GRAPH_FAILED);
OP_TILING_CHECK(weight_nz == nullptr,
OP_LOGE(K_INNER_DEBUG, "weight_nz is invalid."), return GRAPH_FAILED);
info.maxOutputSize = *maxOutputSizePtr;
info.isTransposeB = *is_trans_b;
info.isWeightNz = *weight_nz;
info.swigluLimit = swiglu_limit != nullptr ? *swiglu_limit : 0.0f;
int64_t rankSize;
(void)ge::HcomTopoInfo::Instance().GetGroupRankSize(groupPtr, rankSize);
info.worldSize = rankSize;
OP_LOGD(K_INNER_DEBUG, "maxOutputSize=%d ", info.maxOutputSize);
OP_LOGD(K_INNER_DEBUG, "rankSize=%d ", info.worldSize);
return ge::GRAPH_SUCCESS;
}
// Extract shapes of input tensors A and B to compute M, K, N
static ge::graphStatus DispatchFFNCombineCheckShapeAndSetTiling(gert::TilingContext *context, DispatchFFNCombineInfo &info)
{
const char *nodeName = context->GetNodeName();
const gert::StorageShape *aStorageShape = context->GetInputShape(X_INDEX);
auto expertIdxTensor = context->GetDynamicInputTensor(EXPERTID_INDEX, 0);
uint32_t M = aStorageShape->GetStorageShape().GetDim(0);
uint32_t K = aStorageShape->GetStorageShape().GetDim(1);
auto wTensor = context->GetDynamicInputTensor(WEIGHT_INDEX, 0);
uint32_t wTensorDims = wTensor->GetOriginShape().GetDimNum();
uint32_t N = wTensor->GetStorageShape().GetDim(wTensorDims - 1);
uint32_t topK = expertIdxTensor->GetStorageShape().GetDim(1);
uint32_t listLen = 0;
while (true) {
auto wTensorT = context->GetDynamicInputTensor(WEIGHT_INDEX, ++listLen);
if (wTensorT == nullptr) {break;}
}
uint32_t expertPerRank;
if (listLen == 1) {
expertPerRank = wTensor->GetStorageShape().GetDim(0);
} else {
expertPerRank = listLen;
}
info.M = M;
info.N = N;
info.K = K;
info.expertPerRank = expertPerRank;
info.topK = topK;
info.listLen = listLen;
OP_LOGD(K_INNER_DEBUG, "M=%d ", info.M);
OP_LOGD(K_INNER_DEBUG, "K=%d ", info.K);
OP_LOGD(K_INNER_DEBUG, "N=%d ", info.N);
OP_LOGD(K_INNER_DEBUG, "expertPerRank=%d ", info.expertPerRank);
OP_LOGD(K_INNER_DEBUG, "topK=%d ", info.topK);
OP_LOGD(K_INNER_DEBUG, "listLen=%d ", info.listLen);
return ge::GRAPH_SUCCESS;
}
// Get hardware info such as AI Core count and UB capacity for the current chip platform.
static ge::graphStatus DispatchFFNCombineGetPlatformInfoAndSetTiling(gert::TilingContext *context, DispatchFFNCombineInfo& info)
{
auto ascendcPlatform = platform_ascendc::PlatformAscendC(context->GetPlatformInfo());
uint32_t aivNum = ascendcPlatform.GetCoreNumAiv();
uint64_t ubSize = 0U;
ascendcPlatform.GetCoreMemSize(platform_ascendc::CoreMemType::UB, ubSize);
info.aivNum = aivNum;
info.totalUbSize = ubSize;
OP_LOGD(K_INNER_DEBUG, "aivNum=%d", info.aivNum);
OP_LOGD(K_INNER_DEBUG, "ubSize=%lu", info.totalUbSize);
return ge::GRAPH_SUCCESS;
}
static ge::graphStatus CheckXActiveMaskShape(gert::TilingContext *context, const char *nodeName, DispatchFFNCombineInfo &info)
{
const gert::StorageShape* xActiveMaskStorageShape = context->GetOptionalInputShape(X_ACTIVE_MASK_INDEX);
if (xActiveMaskStorageShape != nullptr) {
OP_TILING_CHECK(xActiveMaskStorageShape->GetStorageShape().GetDimNum() != 1,
OP_LOGE(nodeName, "xActiveMask shape dims must be 1, but current dim num is %lu.",
xActiveMaskStorageShape->GetStorageShape().GetDimNum()), return ge::GRAPH_FAILED);
const int64_t xActiveMaskDim0 = xActiveMaskStorageShape->GetStorageShape().GetDim(0);
OP_TILING_CHECK(xActiveMaskDim0 != static_cast<int64_t>(info.M),
OP_LOGE(nodeName, "xActiveMask Dim0 must be M(%u), but current dim is %lu.", info.M, xActiveMaskDim0),
return ge::GRAPH_FAILED);
}
return ge::GRAPH_SUCCESS;
}
void SetTilingData(CoCTiling &cocTilingData, DispatchFFNCombineInfo &info)
{
cocTilingData.m0 = 128;
cocTilingData.k0 = 256;
cocTilingData.n0 = 256;
cocTilingData.swizzleDirect = 1;
cocTilingData.swizzleOffset = 7;
cocTilingData.ubMoveNum = 16 * 1024;
cocTilingData.pValue = 1;
cocTilingData.commNpuSplit = info.worldSize;
cocTilingData.commDataSplit = 1;
cocTilingData.lenPerLoop = cocTilingData.m0 * cocTilingData.n0 / 2;
}
// Main scheduling function:
// Get tilingData ➝ check Attr ➝ check Shape ➝ get platform info
// ➝ call SetTilingData (based on rank count) ➝ set blockDim ➝ set tilingKey ➝ set workspace ➝ configure communication parameters
static ge::graphStatus DispatchFFNCombineTilingFuncImpl(gert::TilingContext *context)
{
const char *nodeName = context->GetNodeName();
OP_LOGI(nodeName, "Enter DispatchFFNCombine tiling func.");
// 1. tilingData
DispatchFFNCombineTilingData *tilingData = context->GetTilingData<DispatchFFNCombineTilingData>();
OP_TILING_CHECK(tilingData == nullptr, OP_LOGE(nodeName, "tilingData is nullptr."),
return ge::GRAPH_FAILED);
OP_LOGI(nodeName, "DispatchFFNCombine get tilingData.");
DispatchFFNCombineInfo& info = tilingData->dispatchFFNCombineInfo;
OP_LOGI(nodeName, "DispatchFFNCombine get tilingData info.");
OP_TILING_CHECK(DispatchFFNCombineCheckAttrAndSetTiling(context, info) != ge::GRAPH_SUCCESS,
OP_LOGE(context->GetNodeName(), "DispatchFFNCombine CheckAttrAndSetTiling Failed"),
return ge::GRAPH_FAILED);
OP_TILING_CHECK(DispatchFFNCombineCheckShapeAndSetTiling(context, info) != ge::GRAPH_SUCCESS,
OP_LOGE(context->GetNodeName(), "DispatchFFNCombine CheckShapeAndSetTiling Failed"),
return ge::GRAPH_FAILED);
OP_TILING_CHECK(DispatchFFNCombineGetPlatformInfoAndSetTiling(context, info) != ge::GRAPH_SUCCESS,
OP_LOGE(context->GetNodeName(), "DispatchFFNCombine GetPlatformInfoAndSetTiling Failed"),
return ge::GRAPH_FAILED);
OP_TILING_CHECK(CheckXActiveMaskShape(context, nodeName, info) != ge::GRAPH_SUCCESS,
OP_LOGE(context->GetNodeName(), "DispatchFFNCombine CheckXActiveMaskShape Failed"),
return ge::GRAPH_FAILED);
SetTilingData(tilingData->cocTiling, info);
// 2. set blockDim
uint32_t blockDim = 1U;
auto ascendcPlatform = platform_ascendc::PlatformAscendC(context->GetPlatformInfo());
auto aicNum = ascendcPlatform.GetCoreNumAic();
auto aivNum = ascendcPlatform.GetCoreNumAiv();
blockDim = ascendcPlatform.CalcTschBlockDim(aivNum, aicNum, aivNum);
context->SetBlockDim(blockDim);
// 3. set tilingKey
uint64_t tilingKey = INIT_TILINGKEY;
tilingKey += info.isTransposeB ? TILINGKEY_TRANS_B : 0;
tilingKey += info.isWeightNz ? TILINGKEY_WEIGHT_NZ : 0;
context->SetTilingKey(tilingKey);
OP_LOGD(K_INNER_DEBUG, "tilingKey=%d", tilingKey);
optiling::MoeInitRoutingQuantV2TilingBase moeInitRoutingQuantV2TilingBase;
int64_t inuptXDtypeSize = sizeof(int16_t);
int64_t scaleDim0 = 0;
int64_t ubSize = 196352;
int64_t expertCapacity = 0;
int64_t expertNum = info.expertPerRank * info.worldSize + 1; // enable expertId == expertNum
int64_t activeNum = 0;
int64_t dropPadMode = 0;
int64_t expertTokensCountOrCumsumFlag = 2;
bool expertTokensBeforeCapacityFlag = false;
int64_t quantMode = 1;
uint32_t aivNumInitRouting = 2 * BLOCK_NUM;
moeInitRoutingQuantV2TilingBase.DoTiling(info.M, info.K, info.topK, expertCapacity, expertNum, activeNum, dropPadMode,
expertTokensCountOrCumsumFlag, expertTokensBeforeCapacityFlag, inuptXDtypeSize, quantMode, scaleDim0, aivNumInitRouting, ubSize);
uint64_t initRoutingQuantTilingKey = moeInitRoutingQuantV2TilingBase.tilingKey_;
size_t initRoutingWorkspace = moeInitRoutingQuantV2TilingBase.workspaceSize_;
tilingData->cocTiling.moeInitRoutingQuantV2TilingData = moeInitRoutingQuantV2TilingBase.quantTilingData;
tilingData->cocTiling.moeInitRoutingQuantV2TilingData.vbsComputeParamsOp = moeInitRoutingQuantV2TilingBase.quantTilingData.vbsComputeParamsOp;
tilingData->cocTiling.moeInitRoutingQuantV2TilingData.vmsMiddleComputeParamsOp = moeInitRoutingQuantV2TilingBase.quantTilingData.vmsMiddleComputeParamsOp;
tilingData->cocTiling.moeInitRoutingQuantV2TilingData.sortOutComputeParamsOp = moeInitRoutingQuantV2TilingBase.quantTilingData.sortOutComputeParamsOp;
tilingData->cocTiling.moeInitRoutingQuantV2TilingData.srcToDstComputeParamsOp = moeInitRoutingQuantV2TilingBase.quantTilingData.srcToDstComputeParamsOp;
tilingData->cocTiling.moeInitRoutingQuantV2TilingData.srcToDstCapacityComputeParamsOp = moeInitRoutingQuantV2TilingBase.quantTilingData.srcToDstCapacityComputeParamsOp;
tilingData->cocTiling.moeInitRoutingQuantV2TilingData.gatherOutComputeParamsOp = moeInitRoutingQuantV2TilingBase.quantTilingData.gatherOutComputeParamsOp;
tilingData->cocTiling.initRoutingQuantTilingKey = initRoutingQuantTilingKey;
uint64_t maxWindowSize = GetMaxWindowSize();
uint64_t actualSize = static_cast<uint64_t>(info.M) * info.topK * info.K * sizeof(int8_t) * 3 + 10 * MB_SIZE ;
OP_TILING_CHECK((actualSize > maxWindowSize),
OP_LOGE(nodeName, "HCCL_BUFFSIZE is too SMALL, m = %lu, k = %lu, topK = %lu"
" expected HCCL_BUFFSIZE is ((m * k * topK * sizeof(int8_t)) * 3 + 3MB)= %luMB, HCCL_BUFFSIZE=%luMB.",
info.M, info.K, info.topK, (actualSize + MB_SIZE - 1) / MB_SIZE, maxWindowSize / MB_SIZE),
return ge::GRAPH_FAILED);
// 4. workspace
size_t *workSpaces = context->GetWorkspaceSizes(1);
OP_TILING_CHECK(workSpaces == nullptr, OP_LOGE(nodeName, "workSpaces is nullptr."),
return ge::GRAPH_FAILED);
uint32_t n2 = info.K;
uint32_t k2 = info.N / 2;
uint64_t cocWorkspace = (info.M + 256 - 1) / 256 * 256 * info.topK *sizeof(int32_t) +
info.worldSize * info.worldSize * info.expertPerRank * sizeof(int32_t) * 2 +
info.maxOutputSize * sizeof(float) * 2 +
info.maxOutputSize * n2 * sizeof(int16_t) +
info.maxOutputSize * info.K * sizeof(int8_t) +
info.worldSize * sizeof(int32_t) * 16 +
(info.expertPerRank + info.worldSize) * sizeof(int32_t) * 16;
workSpaces[0] = SYSTEM_NEED_WORKSPACE + std::max(cocWorkspace, initRoutingWorkspace);
// 5. communication
auto attrs = context->GetAttrs();
auto group = attrs->GetAttrPointer<char>(static_cast<int>(ATTR_GROUP_INDEX));
uint32_t opType = 8U;
std::string algConfig = "AlltoAll=level0:fullmesh;level1:pairwise";
AscendC::Mc2CcTilingConfig mc2CcTilingConfig(group, opType, algConfig);
mc2CcTilingConfig.GetTiling(tilingData->mc2InitTiling);
mc2CcTilingConfig.GetTiling(tilingData->mc2CcTiling);
OP_LOGI(nodeName, "Leave DispatchFFNCombine tiling func.");
return ge::GRAPH_SUCCESS;
}
static ge::graphStatus DispatchFFNCombineTilingFunc(gert::TilingContext* context)
{
return DispatchFFNCombineTilingFuncImpl(context);
}
struct DispatchFFNCombineCompileInfo {};
ge::graphStatus TilingParseForDispatchFFNCombine(gert::TilingParseContext *context)
{
(void)context;
return ge::GRAPH_SUCCESS;
}
IMPL_OP_OPTILING(DispatchFFNCombine)
.Tiling(DispatchFFNCombineTilingFunc)
.TilingParse<DispatchFFNCombineCompileInfo>(TilingParseForDispatchFFNCombine);
} // namespace optiling

72
csrc/mc2/dispatch_ffn_combine/op_host/hcom_topo_info.h Normal file
View File

@@ -0,0 +1,72 @@
/* Copyright (c) 2025 Huawei Technologies Co., Ltd.
* This file is a part of the CANN Open Software.
* Licensed under CANN Open Software License Agreement Version 1.0 (the "License").
* Please refer to the License for details. You may not use this file except in compliance with the License.
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
* INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
* See LICENSE in the root of the software repository for the full text of the License.
* ===================================================================================================================*/
#ifndef METADEF_CXX_INC_EXTERNAL_HCOM_HCOM_TOPO_INFO_H_
#define METADEF_CXX_INC_EXTERNAL_HCOM_HCOM_TOPO_INFO_H_
#include <unordered_map>
#include <mutex>
using Status = int32_t;
namespace ge {
static constexpr uint32_t COMM_MESH = 0b1U;
static constexpr uint32_t COMM_SWITCH = (COMM_MESH << 1U);
static constexpr uint32_t COMM_RING = (COMM_MESH << 2U);
static constexpr uint32_t COMM_PAIRWISE = (COMM_MESH << 3U);
class HcomTopoInfo {
public:
enum class TopoLevel {
L0 = 0,
L1,
MAX,
};
struct TopoLevelDesc {
uint32_t comm_sets;
uint32_t rank_size;
};
using TopoDescs = TopoLevelDesc[static_cast<int32_t>(TopoLevel::MAX)];
struct TopoInfo {
int64_t rank_size;
void *notify_handle;
TopoDescs topo_level_descs;
};
static HcomTopoInfo &Instance();
bool TopoInfoHasBeenSet(const char_t *group);
bool TryGetGroupTopoInfo(const char_t *group, TopoInfo &info);
Status SetGroupTopoInfo(const char_t *group, const TopoInfo &info);
Status GetGroupRankSize(const char_t *group, int64_t &rank_size);
TopoDescs *GetGroupTopoDesc(const char_t *group);
Status GetGroupNotifyHandle(const char_t *group, void *&notify_handle);
void UnsetGroupTopoInfo(const char_t *group) {
const std::lock_guard<std::mutex> lock(mutex_);
(void) rank_info_.erase(group);
}
Status SetGroupOrderedStream(const char_t *group, void *stream);
Status GetGroupOrderedStream(const char_t *group, void *&stream);
void UnsetGroupOrderedStream(const char_t *group) {
const std::lock_guard<std::mutex> lock(mutex_);
(void) group_to_ordered_stream_.erase(group);
};
Status SetGroupOrderedStream(const int32_t device_id, const char_t *group, void *stream);
Status GetGroupOrderedStream(const int32_t device_id, const char_t *group, void *&stream);
void UnsetGroupOrderedStream(const int32_t device_id, const char_t *group);
private:
HcomTopoInfo() = default;
~HcomTopoInfo() = default;
std::unordered_map<std::string, TopoInfo> rank_info_;
std::mutex mutex_;
std::unordered_map<std::string, void*> group_to_ordered_stream_; // Ordered stream for the communication domain
std::unordered_map<int32_t, std::unordered_map<std::string, void*>> device_id_to_group_to_ordered_stream_; // Ordered stream for the communication domain
};
}
#endif // METADEF_CXX_INC_EXTERNAL_HCOM_HCOM_TOPO_INFO_H_

83
csrc/mc2/dispatch_ffn_combine/op_host/op_api/aclnn_dispatch_ffn_combine.cpp Normal file
View File

@@ -0,0 +1,83 @@
/**
* Copyright (c) 2025 Huawei Technologies Co., Ltd.
* This file is a part of the CANN Open Software.
* Licensed under CANN Open Software License Agreement Version 1.0 (the "License").
* Please refer to the License for details. You may not use this file except in compliance with the License.
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
* INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
* See LICENSE in the root of the software repository for the full text of the License.
*/
#include "aclnn_dispatch_ffn_combine.h"
#include <algorithm>
// #include "aclnn_kernels/common/op_error_check.h"
// #include "opdev/op_log.h"
// #include "opdev/common_types.h"
// #include "opdev/platform.h"
// #include "ophost/matmul_util.h"
#include <unistd.h>
#include <vector>
#include <string>
#include <iostream>
#include <fcntl.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <sys/file.h>
#include <climits>
// using namespace op;
// using namespace op;
#ifdef __cplusplus
extern "C" {
#endif
static constexpr size_t TWO_DIMS = 2;
static constexpr int64_t KVALUE_MIN = 256;
static constexpr int64_t KVALUE_MAX = 65535;
static constexpr size_t HCCL_GROUP_NAME_MAX = 128U;
enum NnopbaseHcclServerType {
NNOPBASE_HCCL_SERVER_TYPE_AICPU = 0,
NNOPBASE_HCCL_SERVER_TYPE_MTE,
NNOPBASE_HCCL_SERVER_TYPE_END
};
extern aclnnStatus aclnnInnerDispatchFFNCombineGetWorkspaceSize(const aclTensor* x, const aclTensorList* weight1, const aclTensorList* weight2,
const aclTensor* expertId, const aclTensorList* scale1, const aclTensorList* scale2,
const aclTensor* probs, const aclTensor* xActiveMask,
const char* group, int64_t maxOutputSize,
bool transB, bool weightNz, double swigluLimit,
const aclTensor* out, const aclTensor* expertTokenNums,
uint64_t* workspaceSize, aclOpExecutor** executor);
extern aclnnStatus aclnnInnerDispatchFFNCombine(void *workspace, uint64_t workspaceSize,
aclOpExecutor *executor, aclrtStream stream);
extern "C" void __attribute__((weak)) NnopbaseSetHcclServerType(void *executor, NnopbaseHcclServerType sType);
aclnnStatus aclnnDispatchFFNCombineGetWorkspaceSize(const aclTensor* x, const aclTensorList* weight1, const aclTensorList* weight2,
const aclTensor* expertId, const aclTensorList* scale1, const aclTensorList* scale2,
const aclTensor* probs, const aclTensor* xActiveMask,
const char* group, int64_t maxOutputSize, double swigluLimit,
const aclTensor* out, const aclTensor* expertTokenNums,
uint64_t* workspaceSize, aclOpExecutor** executor)
{
bool transB = false;
bool weightNz = true;
aclnnStatus ret = aclnnInnerDispatchFFNCombineGetWorkspaceSize(x, weight1, weight2, expertId, scale1, scale2, probs, xActiveMask, group,
maxOutputSize, transB, weightNz, swigluLimit,
out, expertTokenNums, workspaceSize, executor);
return ret;
}
aclnnStatus aclnnDispatchFFNCombine(void* workspace, uint64_t workspaceSize, aclOpExecutor *executor, aclrtStream stream)
{
if (NnopbaseSetHcclServerType) {
NnopbaseSetHcclServerType(executor, NNOPBASE_HCCL_SERVER_TYPE_MTE);
}
aclnnStatus ret = aclnnInnerDispatchFFNCombine(workspace, workspaceSize, executor, stream);
return ret;
}
#ifdef __cplusplus
}
#endif

64
csrc/mc2/dispatch_ffn_combine/op_host/op_api/aclnn_dispatch_ffn_combine.h Normal file
View File

@@ -0,0 +1,64 @@
/**
* Copyright (c) 2025 Huawei Technologies Co., Ltd.
* This file is a part of the CANN Open Software.
* Licensed under CANN Open Software License Agreement Version 1.0 (the "License").
* Please refer to the License for details. You may not use this file except in compliance with the License.
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED,
* INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE.
* See LICENSE in the root of the software repository for the full text of the License.
*/
#ifndef OP_API_INC_DISPATCH_FFN_COMBINE_
#define OP_API_INC_DISPATCH_FFN_COMBINE_
#include <string>
#include "aclnn/aclnn_base.h"
#include "hccl/hccl.h"
#include "hccl/hccl_types.h"
#ifdef __cplusplus
extern "C" {
#endif
/**
* Operator function: fuse all distributed MoE ops from InitRouting through Unpermute.
* @brief First-stage interface of aclnnDispatchFFNCombine that calculates workspace size based on the specific compute flow.
* @domain aclnn_ops_infer
* @param [in] x: The input tensor.
* @param [in] weight1: The first weight tensor.
* @param [in] weight2: The second weight tensor.
* @param [in] expertId: The expert ID tensor.
* @param [in] scale1: The first scale tensor.
* @param [in] scale2: The second scale tensor.
* @param [in] probs: The probabilities tensor.
* @param [in] group: string identifying the communication domain name.
* @param [in] maxOutputSize: The maximum output size.
* @param [out] out: result of computation + communication; same dtype as input.
* @param [out] workspaceSize: workspace size to allocate on the NPU device side.
* @param [out] executor: op executor containing the operator compute flow.
* @return aclnnStatus: status code.
*/
__attribute__((visibility("default"))) aclnnStatus aclnnDispatchFFNCombineGetWorkspaceSize(const aclTensor* x, const aclTensorList* weight1, const aclTensorList* weight2,
const aclTensor* expertId, const aclTensorList* scale1, const aclTensorList* scale2,
const aclTensor* probs, const aclTensor* xActiveMask,
const char* group, int64_t maxOutputSize, double swigluLimit,
const aclTensor* out, const aclTensor* expertTokenNums,
uint64_t* workspaceSize, aclOpExecutor** executor);
/**
* @brief Second-stage interface of aclnnDispatchFFNCombine to execute computation.
* @param [in] workspace: workspace memory address allocated on the NPU device side.
* @param [in] workspace_size: workspace size allocated on the NPU device side, obtained from aclnnDispatchFFNCombineGetWorkspaceSize.
* @param [in] executor: op executor containing the operator compute flow.
* @param [in] stream: acl stream.
* @return aclnnStatus: status code.
*/
__attribute__((visibility("default"))) aclnnStatus aclnnDispatchFFNCombine(void* workspace, uint64_t workspaceSize, aclOpExecutor* executor,
aclrtStream stream);
#ifdef __cplusplus
}
#endif
#endif // OP_API_INC_DISPATCH_FFN_COMBINE_

9
csrc/mc2/dispatch_ffn_combine/op_host/tiling_args.h Normal file
View File

@@ -0,0 +1,9 @@
#ifndef TILING_ARGS_H
#define TILING_ARGS_H
#include <cstdint>
namespace Moe {
constexpr uint64_t COMBINE_STATE_WIN_OFFSET = 3U * 1024UL * 1024UL;
constexpr uint64_t NOTIFY_DISPATCH_WIN_OFFSET = 204U * 1024UL * 1024UL;
} // namespace Moe
#endif // TILING_ARGS_H