EngineX-Cambricon/enginex-mlu370-vllm
11
0
Fork 1
Code Issues Pull Requests Actions Projects Releases Wiki Activity

add ops

Browse Source
This commit is contained in:
Chranos
2026-02-04 17:39:32 +08:00
parent 8511fe8530
commit 79dfc69789
299 changed files with 55927 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

66
torch_mlu_ops-v1.3.2/csrc/common/stack_exception.cpp Normal file
View File

@@ -0,0 +1,66 @@
/*************************************************************************
* Copyright (C) [2023-2024] by Cambricon, Inc.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
* OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*************************************************************************/
#ifdef __GNUC__
#include "stack_exception.h"
#include <cxxabi.h>
#include <dlfcn.h>
#include <execinfo.h>
#include <stdio.h>
#include <stdlib.h>
#define MAX_DEPTH 32
namespace tmo {
namespace stack_exception {
call_stack::call_stack(const size_t num_discard /*= 0*/) {
using namespace abi;
// retrieve call-stack
void *trace[MAX_DEPTH];
int stack_depth = backtrace(trace, MAX_DEPTH);
for (int i = num_discard + 1; i < stack_depth; i++) {
Dl_info dlinfo;
if (!dladdr(trace[i], &dlinfo)) break;
const char *symname = dlinfo.dli_sname;
int status;
char *demangled = abi::__cxa_demangle(symname, NULL, 0, &status);
if (status == 0 && demangled) symname = demangled;
// printf("entry: %s, %s\n", dlinfo.dli_fname,symname);
// store entry to stack
if (dlinfo.dli_fname && symname) {
entry e;
e.file = dlinfo.dli_fname;
e.line = 0; // unsupported
e.function = symname;
stack.push_back(e);
} else {
break; // skip last entries below main
}
if (demangled) free(demangled);
}
}
call_stack::~call_stack() throw() {
// automatic cleanup
}
} // namespace stack_exception
} // namespace tmo
#endif // __GNUC__

127
torch_mlu_ops-v1.3.2/csrc/common/stack_exception.h Normal file
View File

@@ -0,0 +1,127 @@
/*************************************************************************
* Copyright (C) [2023-2024] by Cambricon, Inc.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
* OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*************************************************************************/
#ifndef CSRC_COMMON_STACK_EXCEPTION_H_
#define CSRC_COMMON_STACK_EXCEPTION_H_
#include <sstream>
#include <stdexcept>
#include <string>
#include <vector>
namespace tmo {
namespace stack_exception {
/** Call-stack entry datastructure. */
struct entry {
/** Default constructor that clears all fields. */
entry() : line(0) {}
std::string file; ///< filename
size_t line; ///< line number
std::string function; ///< name of function or method
/** Serialize entry into a text string. */
std::string to_string() const {
std::ostringstream os;
os << file;
if (line > 0) {
os << ":" << line;
}
os << " @" << function;
return os.str();
}
};
/** Stack-trace base class, for retrieving the current call-stack. */
class call_stack {
public:
/** Stack-trace consructor.
\param num_discard - number of stack entries to discard at the top. */
call_stack(const size_t num_discard = 0);
virtual ~call_stack() throw();
/** Serializes the entire call-stack into a text string. */
std::string to_string() const {
std::ostringstream os;
for (size_t i = 0; i < stack.size(); i++)
os << stack[i].to_string() << std::endl;
return os.str();
}
/** Call stack. */
std::vector<entry> stack;
};
/** Abstract base-class for all stack-augmented exception classes.
* Enables catching of all stack-augmented exception classes. */
class stack_exception_base : public call_stack {
public:
stack_exception_base(const bool _show_stack) : call_stack(2), show_stack(_show_stack) {}
virtual ~stack_exception_base() throw() {}
virtual const char *what() const throw() = 0;
/// flag to indicate if stack-trace is included in what() messages
bool show_stack;
};
/** Template for stack-augmented exception classes. */
template <class T>
class stack_exception : public T, public stack_exception_base {
public:
stack_exception(const std::string &msg) : T(msg), stack_exception_base(true) {}
virtual ~stack_exception() throw() {}
stack_exception(const char *file, int line, const char *pretty_function, const std::string &msg)
: T(msg), stack_exception_base(true) {
entry e;
e.file = file;
e.line = line;
e.function = pretty_function;
stack.insert(stack.begin(), e);
}
virtual const char *what() const throw() {
if (show_stack) {
// concatenate message with stack trace
buffer = "[" + std::string(T::what()) + "]\n" + stack_exception::to_string();
return buffer.c_str();
} else {
return T::what();
}
}
private:
mutable std::string buffer;
};
/** Stack-augmented exception classes for all std::exception classes. */
// typedef stack_exception<std::runtime_error> TmoException;
// typedef stack_exception<std::range_error> stack_range_error;
// typedef stack_exception<std::overflow_error> stack_overflow_error;
// typedef stack_exception<std::underflow_error> stack_underflow_error;
// typedef stack_exception<std::logic_error> stack_logic_error;
// typedef stack_exception<std::domain_error> stack_domain_error;
// typedef stack_exception<std::invalid_argument> stack_invalid_argument;
// typedef stack_exception<std::length_error> stack_length_error;
// typedef stack_exception<std::out_of_range> stack_out_of_range;
} // namespace stack_exception
class _TmoException : public stack_exception::stack_exception<std::runtime_error> {
public:
_TmoException(const std::string &msg) : stack_exception<std::runtime_error>(msg) {}
_TmoException(const char *file, int line, const char *pretty_function, const std::string &msg)
: stack_exception<std::runtime_error>(file, line, pretty_function, msg) {}
};
#define TmoException(msg) tmo::_TmoException(__FILE__, __LINE__, __PRETTY_FUNCTION__, msg)
} // namespace tmo
#endif // CSRC_COMMON_STACK_EXCEPTION_H_

293
torch_mlu_ops-v1.3.2/csrc/common/utils.h Normal file
View File

@@ -0,0 +1,293 @@
/*************************************************************************
* Copyright (C) [2023-2024] by Cambricon, Inc.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
* OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*************************************************************************/
#ifndef CSRC_COMMON_UTILS_H_
#define CSRC_COMMON_UTILS_H_
#include <algorithm>
#include <cassert>
#include <cstdlib>
#include <cstring>
#include <exception>
#include <functional>
#include <future> // NOLINT
#include <initializer_list>
#include <iostream>
#include <map>
#include <memory>
#include <stdexcept>
#include <string>
#include <thread> // NOLINT
#include <tuple>
#include <vector>
#include "cn_api.h"
#include "cnnl.h"
#include "cnnl_extra.h"
#include "cnrt.h"
#include "stack_exception.h"
namespace tmo {
inline cnnlQuantizeLayout_t strToQuantizeLayout(std::string param) {
static std::map<std::string, cnnlQuantizeLayout_t> quantize_layout_map = {
{"quantize_none", CNNL_QUANTIZE_NONE},
{"quantize_per_tensor", CNNL_QUANTIZE_PER_TENSOR},
{"quantize_per_channel", CNNL_QUANTIZE_PER_CHANNEL},
{"quantize_per_token", CNNL_QUANTIZE_PER_TOKEN},
{"quantize_group_wise", CNNL_QUANTIZE_GROUP_WISE}};
return quantize_layout_map[param];
}
inline cnnlActivationMode_t strToActivationMode(std::string param) {
static std::map<std::string, cnnlActivationMode_t> act_mode_map = {
{"gelu", CNNL_ACTIVATION_GELU},
{"relu", CNNL_ACTIVATION_RELU},
{"sigmoid", CNNL_ACTIVATION_SIGMOID},
{"silu", CNNL_ACTIVATION_SWISH},
{"none", CNNL_ACTIVATION_IDENTITY}};
return act_mode_map[param];
}
inline cnnlLLMQuantAlgo_t strToQuantizeAlgo(std::string param) {
static std::map<std::string, cnnlLLMQuantAlgo_t> quant_algo_map = {
{"weight_only", CNNL_WEIGHT_ONLY},
{"smooth_quant", CNNL_SMOOTH_QUANT},
{"none", CNNL_NO_QUANT}};
return quant_algo_map[param];
}
namespace lnres {
namespace internal {
using LnresEnum = cnnlTransformerLayernormResidualStructure_t;
struct Helper {
int layernorm_position; // 0: no layernorm, 1: pre layernorm, 2: post layernorm
int residual_position; // 0: no residual, 1: layernorm inside residual, 2: layernorm outside
// residual
constexpr Helper(cnnlTransformerLayernormResidualStructure_t mode);
constexpr Helper(int layernorm_position, int residual_position)
: layernorm_position(layernorm_position), residual_position(residual_position) {}
constexpr bool operator==(const Helper &other) const {
return layernorm_position == other.layernorm_position &&
residual_position == other.residual_position;
}
constexpr operator cnnlTransformerLayernormResidualStructure_t() const;
};
constexpr int NO = 0;
constexpr int PRE = 1;
constexpr int POST = 2;
constexpr int CONTAIN = 1;
constexpr int EXCLUDE = 2;
using TPair = std::pair<Helper, LnresEnum>;
constexpr std::array<TPair, 9> pairs = {
TPair{{NO, NO}, CNNL_TRANSFORMER_NO_LAYERNORM_NO_RESIDUAL}, // noResidual
{{NO, CONTAIN}, CNNL_TRANSFORMER_NO_LAYERNORM_WITH_RESIDUAL}, // useInputAsResidual
{{NO, EXCLUDE}, CNNL_TRANSFORMER_NO_LAYERNORM_WITH_RESIDUAL}, // useInputAsResidual
{{PRE, NO}, CNNL_TRANSFORMER_PRE_LAYERNORM_NO_RESIDUAL}, // noResidual
{{PRE, CONTAIN}, CNNL_TRANSFORMER_PRE_LAYERNORM_INSIDE_RESIDUAL}, // useInputAsResidual
// residualThenLayernorm
{{PRE, EXCLUDE}, CNNL_TRANSFORMER_PRE_LAYERNORM_OUTSIDE_RESIDUAL}, // useLayernormAsResidual
// residualThenLayernorm
{{POST, NO}, CNNL_TRANSFORMER_POST_LAYERNORM_NO_RESIDUAL}, // noResidual
{{POST, CONTAIN}, CNNL_TRANSFORMER_POST_LAYERNORM_INSIDE_RESIDUAL}, // useInputAsResidual
// layernormThenResidual
{{POST, EXCLUDE}, CNNL_TRANSFORMER_POST_LAYERNORM_OUTSIDE_RESIDUAL}, // useInputAsResidual
// residualThenLayernorm
};
constexpr Helper from(LnresEnum mode) {
for (size_t i = 0; i < pairs.size(); ++i) {
if (pairs[i].second == mode) {
return pairs[i].first;
}
}
// throw TmoException("Invalid cnnlTransformerLayernormResidualStructure_t");
return Helper(NO, NO);
}
constexpr LnresEnum to(Helper mode) {
for (size_t i = 0; i < pairs.size(); ++i) {
if (pairs[i].first == mode) {
return pairs[i].second;
}
}
return CNNL_TRANSFORMER_NO_LAYERNORM_NO_RESIDUAL;
// throw TmoException("Invalid Helper");
}
constexpr Helper::Helper(LnresEnum mode) : Helper(from(mode)) {}
constexpr Helper::operator LnresEnum() const {
return to(*this);
}
} // namespace internal
using namespace internal;
inline LnresEnum makeLnresEnum(bool has_ln, bool has_residual, bool residual_is_input) {
return Helper(has_ln ? PRE : NO, has_residual ? (residual_is_input ? CONTAIN : EXCLUDE) : NO);
}
inline LnresEnum removeResidual(LnresEnum mode) {
Helper helper(mode);
return Helper(helper.layernorm_position, NO);
}
inline LnresEnum removeLayernorm(LnresEnum mode) {
Helper helper(mode);
return Helper(NO, helper.residual_position);
}
inline bool useLayernormAsResidual(LnresEnum mode) {
Helper helper(mode);
return helper.layernorm_position == PRE && helper.residual_position == EXCLUDE;
}
inline bool useInputAsResidual(LnresEnum mode) {
Helper helper(mode);
return helper.residual_position == CONTAIN ||
(helper.layernorm_position == NO && helper.residual_position != NO) ||
(helper.layernorm_position == POST && helper.residual_position == EXCLUDE);
}
inline bool hasResidual(LnresEnum mode) {
Helper helper(mode);
return helper.residual_position != NO;
}
inline bool hasLayernorm(LnresEnum mode) {
Helper helper(mode);
return helper.layernorm_position != NO;
}
inline bool isPostLayernorm(LnresEnum mode) {
Helper helper(mode);
return helper.layernorm_position == POST;
}
inline bool isPreLayernorm(LnresEnum mode) {
Helper helper(mode);
return helper.layernorm_position == PRE;
}
inline bool residualThenLayernorm(LnresEnum first_layer, LnresEnum second_layer) {
Helper h1(first_layer);
Helper h2(second_layer);
if (h1.residual_position == NO) { // h1 has no residual
return false;
}
if (h1.layernorm_position == POST && h2.layernorm_position == PRE) {
throw TmoException("too many layernorms");
}
return (h1.residual_position != NO && h1.layernorm_position != POST &&
h2.layernorm_position == PRE) || // l1 residual + l2 pre layernorm
(h1.layernorm_position == POST && h2.layernorm_position != PRE &&
h1.residual_position == EXCLUDE); // l1 inside residual + l1 post layernorm
}
inline bool layernormThenResidual(LnresEnum first_layer, LnresEnum second_layer) {
Helper h1(first_layer);
Helper h2(second_layer);
if (h1.residual_position == NO) { // h1 has no residual
return false;
}
if (h1.layernorm_position == POST && h2.layernorm_position == PRE) {
throw TmoException("too many layernorms");
}
return (h1.layernorm_position == POST && h1.residual_position == CONTAIN);
}
inline bool residualOnly(LnresEnum first_layer, LnresEnum second_layer) {
Helper h1(first_layer);
Helper h2(second_layer);
return h1.residual_position != NO && h1.layernorm_position != POST &&
h2.layernorm_position != PRE;
}
} // namespace lnres
} // namespace tmo
#ifndef CNNL_CHECK
#define CNNL_CHECK(expr) \
if (expr != CNNL_STATUS_SUCCESS) { \
std::cerr << __FILE__ << ":" << __LINE__ \
<< " Check failed: " #expr " == CNNL_STATUS_SUCCESS. " << std::endl; \
}
#endif
#define CNNL_CHECK_FATAL(expr) \
if ((expr) != CNNL_STATUS_SUCCESS) { \
std::cerr << __FILE__ << ":" << __LINE__ << ": " \
<< " Check failed: " #expr " == CNNL_STATUS_SUCCESS. " << std::endl; \
throw TmoException("Check failed: " #expr " == CNNL_STATUS_SUCCESS."); \
}
#define TMO_KERNEL_CHECK_FATAL(expr) \
if ((expr) != tmo::KernelStatus::KERNEL_STATUS_SUCCESS) { \
std::cerr << __FILE__ << ":" << __LINE__ << ": " \
<< " Check failed: " #expr " == KernelStatus::KERNEL_STATUS_SUCCESS. " << std::endl; \
throw TmoException("Check failed: " #expr " == KernelStatus::KERNEL_STATUS_SUCCESS."); \
}
#define CHECK_FATAL(expr, ...) \
if (!(expr)) { \
std::cerr << __FILE__ << ":" << __LINE__ << ": " \
<< " Check failed: " #expr ". " << tmo::stringize(__VA_ARGS__) << std::endl; \
throw TmoException("Check failed: " #expr ". " + tmo::stringize(__VA_ARGS__)); \
}
#undef CNRT_CHECK
#define CNRT_CHECK(val) \
do { \
cnrtRet_t __ret = val; \
if (__ret) { \
printf("[%s:%d] CNRT error, code=%d(%s) \"%s\" \n", __FILE__, __LINE__, (unsigned int)__ret, \
cnrtGetErrorStr(__ret), #val); \
throw TmoException(cnrtGetErrorStr(__ret)); \
} \
} while (0)
#define CN_CHECK(val) \
do { \
CNresult __ret = val; \
if (__ret) { \
const char *cn_err_string = nullptr; \
cnGetErrorString(__ret, &cn_err_string); \
printf("[%s:%d] CN error, code=%d(%s) \"%s\" \n", __FILE__, __LINE__, (unsigned int)__ret, \
cn_err_string, #val); \
throw TmoException(cn_err_string); \
} \
} while (0)
#define PAD_UP_DIV(x, y) (((x) + (y) - 1) / (y))
#define TMO_EXPORT __attribute__((__visibility__("default")))
#define TMO_HIDDEN __attribute__((__visibility__("hidden")))
#define DELETE_COPY_ASSIGN_CONSTRUCT(CLASSNAME) \
CLASSNAME(const CLASSNAME &) = delete; \
CLASSNAME(CLASSNAME &&) = delete; \
CLASSNAME &operator=(const CLASSNAME &) = delete; \
CLASSNAME &operator=(CLASSNAME &&) = delete;
// Note: Return type without const when const object called.
#define CLASS_CAST_TYPE_OPERATOR_DEFINE(DESCNAME, DESCOBJECT) \
inline operator DESCNAME() const { \
return const_cast<DESCNAME>(DESCOBJECT); \
} \
inline operator DESCNAME() { \
return DESCOBJECT; \
}
#endif // CSRC_COMMON_UTILS_H_