/** * @brief Libinfer the fast cuda library for inference. * @file ixinfer.h */ #pragma GCC visibility push(default) #if !defined(CUINFER_H_) #define CUINFER_H_ /// Libinfer version major = 7. #define CUINFER_MAJOR 7 /// Libinfer version minor = 6. #define CUINFER_MINOR 6 /// Libinfer version patchlevel = 5. #define CUINFER_PATCHLEVEL 5 /// Libinfer version = ::CUINFER_MAJOR * 1000 + ::CUINFER_MINOR * 100 + /// ::CUINFER_PATCHLEVEL #define CUINFER_VERSION \ (CUINFER_MAJOR * 1000 + CUINFER_MINOR * 100 + CUINFER_PATCHLEVEL) /// Libinfer priv version major = 3. #define CUINFER_PRIV_MAJOR 3 /// Libinfer priv version minor = 3. #define CUINFER_PRIV_MINOR 3 /// Libinfer priv version patch = 0. #define CUINFER_PRIV_PATCH 0 /// Libinfer priv version = ::CUINFER_PRIV_MAJOR * 1000 + ::CUINFER_PRIV_MINOR * /// 100 + ::CUINFER_PRIV_PATCH #define CUINFER_PRIV_VERSION \ (CUINFER_PRIV_MAJOR * 1000 + CUINFER_PRIV_MINOR * 100 + CUINFER_PRIV_PATCH) #include #include #include #ifndef CUINFERWINAPI #ifdef _WIN32 #define CUINFERWINAPI __stdcall #else #define CUINFERWINAPI #endif #endif #if defined(__cplusplus) extern "C" { #endif struct cuinferContext; /// @brief ::cuinferHandle_t is a point of struct to store ixinfer internal /// info, e.g stream info. /// @details The ::cuinferHandle_t is used in many cuinfer APIs. It must be /// created with ::cuinferCreate before use and be destroyed after use by /// ::cuinferDestroy. /// @see ::cuinferCreate, ::cuinferDestroy typedef struct cuinferContext *cuinferHandle_t; /// @brief Return current cuinfer version. /// @return ::CUINFER_VERSION size_t CUINFERWINAPI cuinferGetVersion(void); /// Returns CUDA Runtime version statically linked against cuinfer. size_t CUINFERWINAPI cuinferGetCudartVersion(void); /// Infer return status. typedef enum { CUINFER_STATUS_SUCCESS = 0, ///< Success. Everything goes well. CUINFER_STATUS_NOT_INITIALIZED = 1, ///< Nullptr or struct not initilized. CUINFER_STATUS_ALLOC_FAILED = 2, ///< Memory allocation falied. CUINFER_STATUS_BAD_PARAM = 3, ///< Bad parameters or bad combination of parameters. CUINFER_STATUS_INTERNAL_ERROR = 4, ///< Internal error, which should not happen. Should be fixed. CUINFER_STATUS_INVALID_VALUE = 5, ///< Invalid single value. CUINFER_STATUS_ARCH_MISMATCH = 6, ///< Libinfer is built for specific target, i.e. MR. Runing MR code on ///< BI will raise this error. CUINFER_STATUS_MAPPING_ERROR = 7, ///< Not used. CUINFER_STATUS_EXECUTION_FAILED = 8, ///< Cuda api execution failed. CUINFER_STATUS_NOT_SUPPORTED = 9, ///< Under development or not supported. CUINFER_STATUS_LICENSE_ERROR = 10, ///< License error. CUINFER_STATUS_RUNTIME_PREREQUISITE_MISSING = 11, ///< Not used. CUINFER_STATUS_RUNTIME_IN_PROGRESS = 12, ///< Not used. CUINFER_STATUS_RUNTIME_FP_OVERFLOW = 13, ///< Not used. } cuinferStatus_t; /// @brief Return human-readable error messages. /// @param[in] status The status to inspect. /// @return Explaination to the status. const char *CUINFERWINAPI cuinferGetErrorString(cuinferStatus_t status); #ifndef __LIBRARY_TYPES_H__ /// Library property types. typedef enum libraryPropertyType_t { MAJOR_VERSION, MINOR_VERSION, PATCH_LEVEL, } libraryPropertyType; #endif /// @brief Get libraryPropertyType. /// @param[in] type Library property type to query. /// @param[out] value Correspond return value. /// @return /// * ::CUINFER_STATUS_SUCCESS If success. /// * ::CUINFER_STATUS_NOT_SUPPORTED If out of range. cuinferStatus_t CUINFERWINAPI cuinferGetProperty(libraryPropertyType type, int *value); /// @brief Create a libinfer handle. /// @note This handle use the default \p cudaStream_t 0, which is synchroized /// before and after other all other cuda operations. Use ::cuinferSetStream to /// custom cuinfer stream to interleave compute and memory operations. /// @note ::cuinferDestroy should be used to destoy a \p handle. /// @param[out] handle The pointer to handle. /// @return /// * ::CUINFER_STATUS_SUCCESS If success. /// * ::CUINFER_STATUS_BAD_PARAM If \p handle is null. /// * ::CUINFER_STATUS_ALLOC_FAILED If alloc failed. cuinferStatus_t CUINFERWINAPI cuinferCreate(cuinferHandle_t *handle); /// @brief Destroy a libinfer handle. /// @param[in] handle The handle to destory. /// @return /// * ::CUINFER_STATUS_SUCCESS If success. /// * ::CUINFER_STATUS_INTERNAL_ERROR If internal error happened. cuinferStatus_t CUINFERWINAPI cuinferDestroy(cuinferHandle_t handle); /// @brief Set a \p cudaStream_t to a \p handle. /// @details All operation associated with this \p handle will use this p /// @param[in] handle The target ::cuinferHandle_t. /// @param[in] streamId The new \p cudaStream_t to put. /// @return /// * ::CUINFER_STATUS_SUCCESS If success. /// * ::CUINFER_STATUS_BAD_PARAM If handle is null. /// * ::CUINFER_STATUS_INTERNAL_ERROR If internal error happened. cuinferStatus_t CUINFERWINAPI cuinferSetStream(cuinferHandle_t handle, cudaStream_t streamId); /// @brief Get a \p cudaStream_t corresponding to a \p handle. /// @param[in] handle The target ::cuinferHandle_t. /// @param[out] streamId The \p cudaStream_t to get. /// @return /// * ::CUINFER_STATUS_SUCCESS If success. /// * ::CUINFER_STATUS_BAD_PARAM If handle is null. /// * ::CUINFER_STATUS_INTERNAL_ERROR If internal error happened. cuinferStatus_t CUINFERWINAPI cuinferGetStream(cuinferHandle_t handle, cudaStream_t *streamId); /// @brief Pointer to tensor descriptions. /// @details Contains tensor ::cuinferTensorFormat_t, strides and dimensions /// infos. /// @see ::cuinferCreateTensorDescriptor, ::cuinferDestroyTensorDescriptor, /// ::cuinferSetTensor4dDescriptor, ::cuinferSetTensor4dDescriptorEx, /// ::cuinferSetTensorNdDescriptor, ::cuinferSetTensorNdDescriptorEx, /// ::cuinferGetTensor4dDescriptor, ::cuinferGetTensorNdDescriptor and /// ::cuinferGetTensorSizeInBytes. typedef struct cuinferTensorStruct *cuinferTensorDescriptor_t; /// @brief Pointer to convolution descriptions. /// @details Contains padding, stride, dilation, ::cuinferConvolutionMode_t, /// ::cuinferDataType_t, ::cuinferMathType_t and group_count infos. /// @see ::cuinferCreateConvolutionDescriptor, /// ::cuinferDestroyConvolutionDescriptor, ::cuinferSetConvolutionGroupCount, /// ::cuinferSetConvolution2dDescriptor, ::cuinferSetConvolutionNdDescriptor, /// ::cuinferGetConvolutionMathType, ::cuinferGetConvolutionGroupCount, /// ::cuinferGetConvolution2dDescriptor, /// ::cuinferGetConvolution2dForwardOutputDim, /// ::cuinferGetConvolutionNdDescriptor and /// ::cuinferGetConvolutionNdForwardOutputDim. typedef struct cuinferConvolutionStruct *cuinferConvolutionDescriptor_t; /// @brief Pointer to pooling layer descriptions. /// @details Contains ::cuinferPoolingMode_t, ::cuinferNanPropagation_t, /// window_dim, padding and stride infos. /// @see ::cuinferCreatePoolingDescriptor, ::cuinferDestroyPoolingDescriptor, /// ::cuinferSetPooling2dDescriptor, ::cuinferSetPoolingNdDescriptor, /// ::cuinferGetPooling2dDescriptor, ::cuinferGetPoolingNdDescriptor, /// ::cuinferGetPoolingNdForwardOutputDim and /// ::cuinferGetPooling2dForwardOutputDim. typedef struct cuinferPoolingStruct *cuinferPoolingDescriptor_t; /// @brief Pointer to filter tensor descriptions. /// @details Contains ::cuinferDataType_t, ::cuinferTensorFormat_t and /// dimentions infos. /// @see ::cuinferCreateFilterDescriptor, ::cuinferDestroyFilterDescriptor, /// ::cuinferSetFilter4dDescriptor, ::cuinferSetFilterNdDescriptor, /// ::cuinferGetFilter4dDescriptor and ::cuinferGetFilterNdDescriptor. typedef struct cuinferFilterStruct *cuinferFilterDescriptor_t; /// @brief Pointer to LRN(Learning Resource Network) descriptions. /// @details Contains LRN's \p n, \p alpha, \p beta ane \p k infos. /// @see ::cuinferCreateLRNDescriptor, ::cuinferDestroyLRNDescriptor, /// ::cuinferSetLRNDescriptor and ::cuinferGetLRNDescriptor. typedef struct cuinferLRNStruct *cuinferLRNDescriptor_t; /// @brief Pointer to activation descriptions. /// @details Contains ::cuinferActivationMode_t, ::cuinferNanPropagation_t and /// coef infos. /// @note The coef can mean different param in different /// ::cuinferActivationMode_t, i.e. ceiling for clipped RELU, alpha for ELU. /// @see ::cuinferCreateActivationDescriptor, /// ::cuinferDestroyActivationDescriptor, ::cuinferSetActivationDescriptor and /// ::cuinferGetActivationDescriptor. typedef struct cuinferActivationStruct *cuinferActivationDescriptor_t; /// @brief Pointer to reduce tensor descriptions. /// @details Contains ::cuinferReduceTensorOp_t, ::cuinferDataType_t, /// ::cuinferNanPropagation_t, ::cuinferReduceTensorIndices_t and /// ::cuinferIndicesType_t infos. /// @see ::cuinferCreateReduceTensorDescriptor, /// ::cuinferCreateReduceTensorDescriptor and /// ::cuinferSetReduceTensorDescriptor. typedef struct cuinferReduceTensorStruct *cuinferReduceTensorDescriptor_t; /// @brief Pointer to CTC(Connectionist temporal classification) loss /// descriptions. /// @details Contains ::cuinferDataType_t, ::cuinferLossNormalizationMode_t and /// ::cuinferNanPropagation_t. /// @see ::cuinferCreateCTCLossDescriptor, ::cuinferDestroyCTCLossDescriptor, /// ::cuinferSetCTCLossDescriptor, ::cuinferSetCTCLossDescriptorEx, /// ::cuinferGetCTCLossDescriptor and ::cuinferGetCTCLossDescriptorEx. typedef struct cuinferCTCLossStruct *cuinferCTCLossDescriptor_t; /// Libinfer data types. typedef enum { CUINFER_DATA_FLOAT = 0, ///< 32-bit ieee float type. CUINFER_DATA_DOUBLE = 1, ///< 64-bit ieee double float type. CUINFER_DATA_HALF = 2, ///< 16-bit ieee half float type. CUINFER_DATA_INT8 = 3, ///< 8-bit signed integer type. CUINFER_DATA_INT32 = 4, ///< 32-bit signed integer type. CUINFER_DATA_INT8x4 = 5, ///< 4x8-bit signed integer type. Aligned to 4 bytes. CUINFER_DATA_UINT8 = 6, ///< 8-bit unsigned integer type. CUINFER_DATA_UINT8x4 = 7, ///< 4x8-bit unsigned integer type. Aligned to 4 bytes. CUINFER_DATA_INT8x32 = 8, ///< 32x8-bit signed integer type. Aligned to 32 bytes. CUINFER_DATA_BFLOAT16 = 9, ///< Google's brain floating point. 16-bit. } cuinferDataType_t; /// Libinfer math type. typedef enum { CUINFER_DEFAULT_MATH = 0, ///< Default math type. CUINFER_TENSOR_OP_MATH = 1, ///< Perffer to use tensor op. CUINFER_TENSOR_OP_MATH_ALLOW_CONVERSION = 2, ///< Not used. } cuinferMathType_t; /// @brief Libinfer propagate NaN(not a number) option. @details /// ::cuinferNanPropagation_t is used to indicate if a float number result in /// NaN(Not a Number) should be propagate nan or not (0 will be propagated /// instead).This setting is only useful for float type computation. This is /// used in setting ::cuinferReduceTensorDescriptor_t, /// ::cuinferPoolingDescriptor_t, ::cuinferActivationDescriptor_t, /// ::cuinferRNNDescriptor_t and ::cuinferCTCLossDescriptor_t. typedef enum { CUINFER_NOT_PROPAGATE_NAN = 0, ///< \p 0 will be propagating for \p NaN and \p ///< Inf values in float types. CUINFER_PROPAGATE_NAN = 1, ///< \p NaN and \p Inf will be propagating in float types. } cuinferNanPropagation_t; /// Is algorithm result determinstic(same input always produce same outputs). typedef enum { CUINFER_NON_DETERMINISTIC = 0, ///< Same input may poduce different outputs. ///< Due to data race, i.e. atomic operations. CUINFER_DETERMINISTIC = 1, ///< Same input always produce same outputs. } cuinferDeterminism_t; /// Maximum supported number of tensor dimensions. #define CUINFER_DIM_MAX 8 /// @brief Create an instance of a generic Tensor descriptor. /// @note ::cuinferDestroyTensorDescriptor should be called after use. /// @param[out] tensorDesc Pointer to store ::cuinferTensorDescriptor_t. cuinferStatus_t CUINFERWINAPI cuinferCreateTensorDescriptor(cuinferTensorDescriptor_t *tensorDesc); /// @brief Tensor format stored in memory. /// @details /// * ::CUINFER_TENSOR_NCHW tensor runs faster in CPUs. /// * ::CUINFER_TENSOR_NHWC tensor runs faster in GPUs. /// * ::CUINFER_TENSOR_NCHW_VECT_C split dim C and run faster in both. typedef enum { CUINFER_TENSOR_NCHW = 0, ///< Elements are stored in batch, channel, depth(3d only), height and ///< weight order(higher to lower). CUINFER_TENSOR_NHWC = 1, ///< Elements are stored in batch, depth(3d only), ///< height, weight and channel order(higher to lower). CUINFER_TENSOR_NCHW_VECT_C = 2, ///< Elements are stored in batch, channel / 4, depth(3d only), height, ///< weight, 4 order(higher to lower), where channel is split by 4 into 2 ///< parts. } cuinferTensorFormat_t; /// @brief Setup params for ::cuinferTensorDescriptor_t. /// @param[out] tensorDesc Pointer to target ::cuinferTensorDescriptor_t. /// @param[in] format Tensor format. /// @param[in] dataType Tensor data type. /// @param[in] n Tensor batch size. /// @param[in] c Tensor channel size. /// @param[in] h Tensor height. /// @param[in] w Tensor width. /// @return /// * ::CUINFER_STATUS_SUCCESS If success. /// * ::CUINFER_STATUS_BAD_PARAM If param out of range or \p tensorDesc is null /// or c is not multiple of 4 in ::CUINFER_TENSOR_NCHW_VECT_C. cuinferStatus_t CUINFERWINAPI cuinferSetTensor4dDescriptor( cuinferTensorDescriptor_t tensorDesc, cuinferTensorFormat_t format, cuinferDataType_t dataType, int n, int c, int h, int w); /// @brief Setup params for ::cuinferTensorDescriptor_t. /// @param[out] tensorDesc Pointer to target ::cuinferTensorDescriptor_t. /// @param[in] dataType Tensor data type. /// @param[in] n Tensor batch size. /// @param[in] c Tensor channel size. /// @param[in] h Tensor height. /// @param[in] w Tensor width. /// @param[in] nStride Stride of batch. /// @param[in] cStride Stride of channel. /// @param[in] hStride Stride of height. /// @param[in] wStride Stride of width. /// @return /// * ::CUINFER_STATUS_SUCCESS If success. /// * ::CUINFER_STATUS_BAD_PARAM If param out of range or \p tensorDesc is null. cuinferStatus_t CUINFERWINAPI cuinferSetTensor4dDescriptorEx( cuinferTensorDescriptor_t tensorDesc, cuinferDataType_t dataType, int n, int c, int h, int w, int nStride, int cStride, int hStride, int wStride); /// @brief Return params for ::cuinferTensorDescriptor_t. /// @param[in] tensorDesc Pointer to target ::cuinferTensorDescriptor_t. /// @param[out] dataType Tensor data type. /// @param[out] n Tensor batch size. /// @param[out] c Tensor channel size. /// @param[out] h Tensor height. /// @param[out] w Tensor width. /// @param[out] nStride Stride of batch. /// @param[out] cStride Stride of channel. /// @param[out] hStride Stride of height. /// @param[out] wStride Stride of width. /// @return /// * ::CUINFER_STATUS_SUCCESS If success. /// * ::CUINFER_STATUS_BAD_PARAM If \p tensorDesc is null. cuinferStatus_t CUINFERWINAPI cuinferGetTensor4dDescriptor( const cuinferTensorDescriptor_t tensorDesc, cuinferDataType_t *dataType, int *n, int *c, int *h, int *w, int *nStride, int *cStride, int *hStride, int *wStride); /// @brief Setup params for 2d/3d ::cuinferTensorDescriptor_t. /// @details The input order(dim0/stride0, dim1/stride1, ...) is batch, channel, /// depth(3d only), height and weight. /// @note The ::CUINFER_TENSOR_NHWC format may change the strides. /// @note Can not set ::CUINFER_TENSOR_NCHW_VECT_C format. /// @see ::cuinferTensorFormat_t, ::cuinferSetTensorNdDescriptorEx /// @param[out] tensorDesc Pointer to target ::cuinferTensorDescriptor_t. /// @param[in] dataType Tensor data type. /// @param[in] nbDims Number of dimensions. 4 for 2d conv and 5 for 3d conv. /// @param[in] dimA Size of each dimension.Nchw for 2d and ncdhw for 3d. /// @param[in] strideA Stride of each dimension. Nchw for 2d and ncdhw for 3d. /// @return /// * ::CUINFER_STATUS_SUCCESS If success. /// * ::CUINFER_STATUS_NOT_SUPPORTED If \p nbDims not in range[4, /// ::CUINFER_DIM_MAX]. /// * ::CUINFER_STATUS_BAD_PARAM If \p tensorDesc is null or invalid dims. cuinferStatus_t CUINFERWINAPI cuinferSetTensorNdDescriptor( cuinferTensorDescriptor_t tensorDesc, cuinferDataType_t dataType, int nbDims, const int dimA[], const int strideA[]); /// @brief Setup params for 2d/3d ::cuinferTensorDescriptor_t. /// @details The input order(dim0/stride0, dim1/stride1, ...) is batch, channel, /// depth(3d only), height and weight. /// @note Strides is set according to \p format and \p nbDims. /// @see ::cuinferTensorFormat_t, ::cuinferSetTensorNdDescriptor /// @param[out] tensorDesc Pointer to target ::cuinferTensorDescriptor_t. /// @param[in] format Tensor format. /// @param[in] dataType Tensor data type. /// @param[in] nbDims Number of dimensions. 4 for 2d conv and 5 for 3d conv. /// @param[in] dimA Size of each dimension.Nchw for 2d and ncdhw for 3d. /// @return /// * ::CUINFER_STATUS_SUCCESS If success. /// * ::CUINFER_STATUS_NOT_SUPPORTED If \p nbDims not in range[4, /// ::CUINFER_DIM_MAX]. /// * ::CUINFER_STATUS_BAD_PARAM If \p tensorDesc is null or invalid dims. cuinferStatus_t CUINFERWINAPI cuinferSetTensorNdDescriptorEx( cuinferTensorDescriptor_t tensorDesc, cuinferTensorFormat_t format, cuinferDataType_t dataType, int nbDims, const int dimA[]); /// @brief Return params for 2d/3d ::cuinferTensorDescriptor_t. /// @details The output order(dim0/stride0, dim1/stride1, ...) is batch, /// channel, depth(3d only), height and weight. /// @see cuinferSetTensorNdDescriptor /// @param[in] tensorDesc Pointer to target ::cuinferTensorDescriptor_t. /// @param[out] nbDimsRequested Not used. @todo \p nbDimsRequested not used. /// @param[out] dataType Tensor data type. /// @param[out] nbDims Number of dimensions. 4 for 2d conv and 5 for 3d conv. /// @param[out] dimA Size of each dimension.Nchw for 2d and ncdhw for 3d. /// @param[out] strideA Stride of each dimension. Nchw for 2d and ncdhw for 3d. /// * ::CUINFER_STATUS_SUCCESS If success. /// * ::CUINFER_STATUS_BAD_PARAM If \p tensorDesc is null. cuinferStatus_t CUINFERWINAPI cuinferGetTensorNdDescriptor( const cuinferTensorDescriptor_t tensorDesc, int nbDimsRequested, cuinferDataType_t *dataType, int *nbDims, int dimA[], int strideA[]); /// @brief Returns psysical space needed by a tensor. /// @note The psysical space needed can be slightly larger than logical space /// due to stride sittings(padding). /// @param[in] tensorDesc Pointer to target ::cuinferTensorDescriptor_t. /// @param[out] size Result size in bytes. /// @return /// * ::CUINFER_STATUS_SUCCESS If success. /// * ::CUINFER_STATUS_BAD_PARAM If \p tensorDesc is null. cuinferStatus_t CUINFERWINAPI cuinferGetTensorSizeInBytes( const cuinferTensorDescriptor_t tensorDesc, size_t *size); /// Destroy an instance of Tensor4d descriptor /// @brief Destroy an instance of ::cuinferTensorDescriptor_t. /// @param[in] tensorDesc Pointer to target ::cuinferTensorDescriptor_t. /// @return /// * ::CUINFER_STATUS_SUCCESS If success. /// * ::CUINFER_STATUS_INTERNAL_ERROR cuinferStatus_t CUINFERWINAPI cuinferDestroyTensorDescriptor(cuinferTensorDescriptor_t tensorDesc); /// @brief Tensor layout conversion helper y = alpha * x + beta * y. /// @param[in] handle The libinfer handle. /// @param[in] alpha Pointer to scaling factor in host memory. Type is always /// float for now. /// @param[in] xDesc Meta info of tensor x. /// @param[in] x Input tensor data. /// @param[in] beta Pointer to scaling factor in host memory. Type is always /// float for now. /// @param[in] yDesc Meta info of tensor y. /// @param[in,out] y Input and output tensor data. /// @return /// * ::CUINFER_STATUS_SUCCESS If success. /// * ::CUINFER_STATUS_BAD_PARAM If bad params. /// * ::CUINFER_STATUS_INTERNAL_ERROR If internal error happened. /// * ::CUINFER_STATUS_NOT_SUPPORTED If algo not supported. cuinferStatus_t CUINFERWINAPI cuinferTransformTensor( cuinferHandle_t handle, const void *alpha, const cuinferTensorDescriptor_t xDesc, const void *x, const void *beta, const cuinferTensorDescriptor_t yDesc, void *y); /// @brief Add two Tensor. C = alpha * A + beta * C. /// @todo difference to ::cuinferTransformTensor? /// @param[in] handle The libinfer handle. /// @param[in] alpha Pointer to scaling factor in host memory. Type is always /// float for now. /// @param[in] aDesc The tensor descripter of A. /// @param[in] A Const pointer to tensor data A. /// @param[in] beta Pointer to scaling factor in host memory. Type is always /// float for now. /// @param[in] cDesc The tensor descripter of C. /// @param[in,out] C Input and output tensor data C. /// @return /// * ::CUINFER_STATUS_SUCCESS If success. /// * ::CUINFER_STATUS_BAD_PARAM If bad params. /// * ::CUINFER_STATUS_INTERNAL_ERROR If internal error happened. /// * ::CUINFER_STATUS_NOT_SUPPORTED If algo not supported. cuinferStatus_t CUINFERWINAPI cuinferAddTensor( cuinferHandle_t handle, const void *alpha, const cuinferTensorDescriptor_t aDesc, const void *A, const void *beta, const cuinferTensorDescriptor_t cDesc, void *C); /// Libinfer ReduceTensor op type. typedef enum { CUINFER_REDUCE_TENSOR_ADD = 0, ///< Addition. CUINFER_REDUCE_TENSOR_MUL = 1, ///< Multiplication. CUINFER_REDUCE_TENSOR_MIN = 2, ///< Minimum. CUINFER_REDUCE_TENSOR_MAX = 3, ///< Maximum. CUINFER_REDUCE_TENSOR_AMAX = 4, ///< Argmax. The index of Maximum element. CUINFER_REDUCE_TENSOR_AVG = 5, ///< Average. \f$ \frac{\sum{x}}{n} \f$ CUINFER_REDUCE_TENSOR_NORM1 = 6, ///< Absolute-value norm. \f$ \sum{|x|} \f$ CUINFER_REDUCE_TENSOR_NORM2 = 7, ///< Euclidean norm. \f$ \sqrt{\sum{x^2}} \f$ CUINFER_REDUCE_TENSOR_MUL_NO_ZEROS = 8, ///< Multiplication only to valid values. } cuinferReduceTensorOp_t; /// Not used. typedef enum { CUINFER_REDUCE_TENSOR_NO_INDICES = 0, CUINFER_REDUCE_TENSOR_FLATTENED_INDICES = 1, } cuinferReduceTensorIndices_t; /// Not used. typedef enum { CUINFER_32BIT_INDICES = 0, CUINFER_64BIT_INDICES = 1, CUINFER_16BIT_INDICES = 2, CUINFER_8BIT_INDICES = 3, } cuinferIndicesType_t; /// @brief Create a ::cuinferReduceTensorDescriptor_t. /// @param[out] reduceTensorDesc Pointer to ::cuinferReduceTensorDescriptor_t. /// @return /// * ::CUINFER_STATUS_SUCCESS if success. /// * ::CUINFER_STATUS_ALLOC_FAILED if malloc failed. cuinferStatus_t CUINFERWINAPI cuinferCreateReduceTensorDescriptor( cuinferReduceTensorDescriptor_t *reduceTensorDesc); /// @brief Set a ::cuinferReduceTensorDescriptor_t. /// Not used. /// @param[out] reduceTensorDesc The target ::cuinferReduceTensorDescriptor_t. /// @param[in] reduceTensorOp The resuce tensor Op. /// @param[in] reduceTensorCompType The reduce tensor compute type. /// @param[in] reduceTensorNanOpt The reduce tensor op NaN propgation setting. /// @param[in] reduceTensorIndices Not used. /// @param[in] reduceTensorIndicesType Not used. /// @return /// * ::CUINFER_STATUS_SUCCESS if success. /// * ::CUINFER_STATUS_BAD_PARAM if \p reduceTensorDesc is null or bad param. cuinferStatus_t CUINFERWINAPI cuinferSetReduceTensorDescriptor( cuinferReduceTensorDescriptor_t reduceTensorDesc, cuinferReduceTensorOp_t reduceTensorOp, cuinferDataType_t reduceTensorCompType, cuinferNanPropagation_t reduceTensorNanOpt, cuinferReduceTensorIndices_t reduceTensorIndices, cuinferIndicesType_t reduceTensorIndicesType); /// @todo Not used? cuinferStatus_t CUINFERWINAPI cuinferReduceTensor( cuinferHandle_t handle, const cuinferReduceTensorDescriptor_t reduceTensorDesc, void *indices, size_t indicesSizeInBytes, void *workspace, size_t workspaceSizeInBytes, const void *alpha, const cuinferTensorDescriptor_t aDesc, const void *A, const void *beta, const cuinferTensorDescriptor_t cDesc, void *C); /// @brief Convolution mode. @details They do the same computation while data /// layout is different. typedef enum { /// Convolution. Take 2d for example \f$ /// y[i,j]=\sum{x[i,j]w[\mathrm{height}-1-i,\mathrm{weight}-1-j]} \f$. CUINFER_CONVOLUTION = 0, /// Cross correlation. Take 2d for example \f$ y[i,j]=\sum{x[i,j]w[i,j]} \f$. CUINFER_CROSS_CORRELATION = 1, } cuinferConvolutionMode_t; /// @brief Create a ::cuinferFilterDescriptor_t. /// @param[out] filterDesc The descriptor for the filter created. /// @return /// * ::CUINFER_STATUS_SUCCESS If success. /// * ::CUINFER_STATUS_ALLOC_FAILED If allocation failed. cuinferStatus_t CUINFERWINAPI cuinferCreateFilterDescriptor(cuinferFilterDescriptor_t *filterDesc); /// @brief Set a 4d ::cuinferFilterDescriptor_t. /// @param[out] filterDesc The pointer to target ::cuinferFilterDescriptor_t. /// @param[in] dataType The data type of the filter. /// @param[in] format The format of the filter. /// @param[in] k Number of filters. /// @param[in] c Number of input channels. /// @param[in] h Filter height. /// @param[in] w Filter weight. /// @return /// * ::CUINFER_STATUS_SUCCESS If success. /// * ::CUINFER_STATUS_BAD_PARAM If invalid param. cuinferStatus_t CUINFERWINAPI cuinferSetFilter4dDescriptor( cuinferFilterDescriptor_t filterDesc, cuinferDataType_t dataType, cuinferTensorFormat_t format, int k, int c, int h, int w); /// @brief Get info form 4d ::cuinferFilterDescriptor_t. /// @param[in] filterDesc The pointer to target ::cuinferFilterDescriptor_t. /// @param[out] dataType The data type of the filter. /// @param[out] format The format of the filter. /// @param[out] k Number of filters. /// @param[out] c Number of input channels. /// @param[out] h Filter height. /// @param[out] w Filter weight. /// @return /// * ::CUINFER_STATUS_SUCCESS If success. /// * ::CUINFER_STATUS_BAD_PARAM If \p filterDesc is null. cuinferStatus_t CUINFERWINAPI cuinferGetFilter4dDescriptor( const cuinferFilterDescriptor_t filterDesc, cuinferDataType_t *dataType, cuinferTensorFormat_t *format, int *k, int *c, int *h, int *w); /// @brief Set a ::cuinferFilterDescriptor_t. /// @see ::cuinferGetFilter4dDescriptor /// @param[out] filterDesc The pointer to target ::cuinferFilterDescriptor_t. /// @param[in] dataType The datatype of the filter. /// @param[in] format The format of the filter. /// @param[in] nbDims Number of dimensions, 4 or 5. /// @param[in] filterDimA Starting from index 0; k, c, h, w for 4d and k, c, d, /// h, w for 5d. /// @return /// * ::CUINFER_STATUS_SUCCESS If success. /// * ::CUINFER_STATUS_BAD_PARAM If \p filterDesc is null or bad params. /// * ::CUINFER_STATUS_NOT_SUPPORTED If type is not supported. cuinferStatus_t CUINFERWINAPI cuinferSetFilterNdDescriptor( cuinferFilterDescriptor_t filterDesc, cuinferDataType_t dataType, cuinferTensorFormat_t format, int nbDims, const int filterDimA[]); /// @brief Get info from a ::cuinferFilterDescriptor_t. /// @see ::cuinferGetFilter4dDescriptor /// @param[in] filterDesc The pointer to target ::cuinferFilterDescriptor_t. /// @param[out] nbDimsRequested Not used. /// @param[out] dataType The datatype of the filter. /// @param[out] format The format of the filter. /// @param[out] nbDims Number of dimensions, 4 or 5. /// @param[out] filterDimA Starting from index 0; k, c, h, w for 4d and k, c, d, /// h, w for 5d. /// @return /// * ::CUINFER_STATUS_SUCCESS If success. /// * ::CUINFER_STATUS_BAD_PARAM If \p filterDesc is null. cuinferStatus_t CUINFERWINAPI cuinferGetFilterNdDescriptor( const cuinferFilterDescriptor_t filterDesc, int nbDimsRequested, cuinferDataType_t *dataType, ///< image data type cuinferTensorFormat_t *format, int *nbDims, int filterDimA[]); /// @brief Return bytes used by a ::cuinferFilterDescriptor_t. /// @param[in] filterDesc The pointer to target ::cuinferFilterDescriptor_t. /// @param[out] size The pysical size in bytes. /// @return /// * ::CUINFER_STATUS_SUCCESS If success. /// * ::CUINFER_STATUS_BAD_PARAM If \p filterDesc is null. cuinferStatus_t CUINFERWINAPI cuinferGetFilterSizeInBytes( const cuinferFilterDescriptor_t filterDesc, size_t *size); /// @brief Destopy a ::cuinferFilterDescriptor_t after use. /// @see ::cuinferCreateFilterDescriptor /// @param[in] filterDesc The pointer to target ::cuinferFilterDescriptor_t. /// @return /// * ::CUINFER_STATUS_SUCCESS If success. cuinferStatus_t CUINFERWINAPI cuinferDestroyFilterDescriptor(cuinferFilterDescriptor_t filterDesc); /// @brief Create an instance of ::cuinferConvolutionDescriptor_t. /// @param[out] convDesc The pointer to store ::cuinferConvolutionDescriptor_t. /// @return /// * ::CUINFER_STATUS_SUCCESS If success. /// * ::CUINFER_STATUS_ALLOC_FAILED If allocation failed. cuinferStatus_t CUINFERWINAPI cuinferCreateConvolutionDescriptor(cuinferConvolutionDescriptor_t *convDesc); /// @brief Set the \p mathType for a ::cuinferConvolutionDescriptor_t. /// @param[out] convDesc The target ::cuinferConvolutionDescriptor_t. /// @param[in] mathType The target ::cuinferMathType_t. /// @return /// * ::CUINFER_STATUS_SUCCESS If success. /// * ::CUINFER_STATUS_BAD_PARAM If \p convDesc is null. cuinferStatus_t CUINFERWINAPI cuinferSetConvolutionMathType( cuinferConvolutionDescriptor_t convDesc, cuinferMathType_t mathType); /// @brief Get the \p mathType for a ::cuinferConvolutionDescriptor_t. /// @param[in] convDesc The target ::cuinferConvolutionDescriptor_t. /// @param[out] mathType The target ::cuinferMathType_t. /// @return /// * ::CUINFER_STATUS_SUCCESS If success. /// * ::CUINFER_STATUS_BAD_PARAM If \p convDesc is null. cuinferStatus_t CUINFERWINAPI cuinferGetConvolutionMathType( cuinferConvolutionDescriptor_t convDesc, cuinferMathType_t *mathType); /// @brief Set the \p groupCount for a ::cuinferConvolutionDescriptor_t. /// @param[out] convDesc The target ::cuinferConvolutionDescriptor_t. /// @param[in] groupCount The target group count. /// @return /// * ::CUINFER_STATUS_SUCCESS If success. /// * ::CUINFER_STATUS_BAD_PARAM If \p convDesc is null. cuinferStatus_t CUINFERWINAPI cuinferSetConvolutionGroupCount( cuinferConvolutionDescriptor_t convDesc, int groupCount); /// @brief Get the \p groupCount for a ::cuinferConvolutionDescriptor_t. /// @param[in] convDesc The target ::cuinferConvolutionDescriptor_t. /// @param[out] groupCount The target group count. /// @return /// * ::CUINFER_STATUS_SUCCESS If success. /// * ::CUINFER_STATUS_BAD_PARAM If \p convDesc is null. cuinferStatus_t CUINFERWINAPI cuinferGetConvolutionGroupCount( cuinferConvolutionDescriptor_t convDesc, int *groupCount); /// @brief Set a 2d ::cuinferConvolutionDescriptor_t. /// @param[out] convDesc The target ::cuinferConvolutionDescriptor_t. /// @param[in] pad_h The padding of data in height. /// @param[in] pad_w The padding of data in weight. /// @param[in] u The stride in filter in height. /// @param[in] v The stride in filter in weight. /// @param[in] dilation_h The filter dilation in height. /// @param[in] dilation_w The filter dilation in weight. /// @param[in] mode The convolution mode. /// @param[in] computeType The datatype in compute. Can be different to input /// and output datatype. /// @return /// * ::CUINFER_STATUS_SUCCESS If success. /// * ::CUINFER_STATUS_BAD_PARAM If param is not valid. cuinferStatus_t CUINFERWINAPI cuinferSetConvolution2dDescriptor( cuinferConvolutionDescriptor_t convDesc, int pad_h, int pad_w, int u, int v, int dilation_h, int dilation_w, cuinferConvolutionMode_t mode, cuinferDataType_t computeType); /// @brief Return the info from a 2d ::cuinferConvolutionDescriptor_t. /// @param[in] convDesc The target ::cuinferConvolutionDescriptor_t. /// @param[out] pad_h The padding of data in height. /// @param[out] pad_w The padding of data in weight. /// @param[out] u The stride in filter in height. /// @param[out] v The stride in filter in weight. /// @param[out] dilation_h The filter dilation in height. /// @param[out] dilation_w The filter dilation in weight. /// @param[out] mode The convolution mode. /// @param[out] computeType The datatype in compute. Can be different to input /// and output datatype. /// @return /// * ::CUINFER_STATUS_SUCCESS If success. /// * ::CUINFER_STATUS_BAD_PARAM If param is \p convDesc is null. cuinferStatus_t CUINFERWINAPI cuinferGetConvolution2dDescriptor( const cuinferConvolutionDescriptor_t convDesc, int *pad_h, int *pad_w, int *u, int *v, int *dilation_h, int *dilation_w, cuinferConvolutionMode_t *mode, cuinferDataType_t *computeType); /// Helper function to return the dimensions of the output tensor given a /// convolution descriptor /// @brief Helper function to calculate the result dimensions given a /// ::cuinferConvolutionDescriptor_t and input ::cuinferTensorDescriptor_t. /// @param[in] convDesc The conv descriptor. /// @param[in] inputTensorDesc The input tensor descriptor. /// @param[in] filterDesc The filter descriptor. /// @param[out] n The batch number of result tensor. /// @param[out] c The number of channels of result tensor. /// @param[out] h The height of result tensor. /// @param[out] w The weight of result tensor. /// @return /// * ::CUINFER_STATUS_SUCCESS If success. /// * ::CUINFER_STATUS_BAD_PARAM If param is \p convDesc is null or invalid /// conbination of params in \p convDesc, \p inputTensorDesc and \p filterDesc. cuinferStatus_t CUINFERWINAPI cuinferGetConvolution2dForwardOutputDim( const cuinferConvolutionDescriptor_t convDesc, const cuinferTensorDescriptor_t inputTensorDesc, const cuinferFilterDescriptor_t filterDesc, int *n, int *c, int *h, int *w); /// @brief Set a 2d or 3d ::cuinferConvolutionDescriptor_t. /// @param[out] convDesc The target ::cuinferConvolutionDescriptor_t. /// @param[in] arrayLength The input array length, 2 for 2d, 3 for 3d. /// @param[in] padA The input padding array. Height, weight for 2d; depth, /// height, weight for 3d. /// @param[in] filterStrideA The filter stride array. Height, weight for 2d; /// depth, height, weight for 3d. /// @param[in] dilationA The filter dilation array. Height, weight for 2d; /// depth, height, weight for 3d. /// @param[in] mode The convolution mode. /// @param[in] computeType The datatype in compute. Can be different to input /// and output datatype. /// @return /// * ::CUINFER_STATUS_SUCCESS If success. /// * ::CUINFER_STATUS_BAD_PARAM If param is \p convDesc is null or invalid /// conbination of params in \p convDesc, \p inputTensorDesc and \p filterDesc. cuinferStatus_t CUINFERWINAPI cuinferSetConvolutionNdDescriptor( cuinferConvolutionDescriptor_t convDesc, int arrayLength, const int padA[], const int filterStrideA[], const int dilationA[], cuinferConvolutionMode_t mode, cuinferDataType_t computeType); /// @brief Set a 2d or 3d ::cuinferConvolutionDescriptor_t. /// @param[in] convDesc The target ::cuinferConvolutionDescriptor_t. /// @param[in] arrayLengthRequested Not used. /// @param[out] arrayLength The array length. 2 for 2d and 3 for 3d. /// @param[out] padA The input padding array. Height, weight for 2d; depth, /// height, weight for 3d. /// @param[out] strideA The filter stride array. Height, weight for 2d; /// depth, height, weight for 3d. /// @param[out] dilationA The filter dilation array. Height, weight for 2d; /// depth, height, weight for 3d. /// @param[out] mode The convolution mode. /// @param[out] computeType The datatype in compute. Can be different to input /// and output datatype. /// @return /// * ::CUINFER_STATUS_SUCCESS If success. /// * ::CUINFER_STATUS_BAD_PARAM If param is \p convDesc is null or invalid /// conbination of params in \p convDesc, \p inputTensorDesc and \p filterDesc. cuinferStatus_t CUINFERWINAPI cuinferGetConvolutionNdDescriptor( const cuinferConvolutionDescriptor_t convDesc, int arrayLengthRequested, int *arrayLength, int padA[], int strideA[], int dilationA[], cuinferConvolutionMode_t *mode, cuinferDataType_t *computeType); /// @brief Get the output dimensions given convolution descriptions. /// @param[in] convDesc The convolution descriptor. /// @param[in] inputTensorDesc The input tensor descriptor. /// @param[in] filterDesc The filter descriptor. /// @param[in] nbDims Number of dimensions. 2 for 2d and 3 for 3d. /// @param[out] tensorOuputDimA The result output tensor dimensions. Height, /// weight for 2d and depth, height, weight for 3d. /// @return /// * ::CUINFER_STATUS_SUCCESS If success. /// * ::CUINFER_STATUS_BAD_PARAM If \p convDesc is null or invalid conbination /// of params in \p convDesc, \p inputTensorDesc and \p filterDesc. cuinferStatus_t CUINFERWINAPI cuinferGetConvolutionNdForwardOutputDim( const cuinferConvolutionDescriptor_t convDesc, const cuinferTensorDescriptor_t inputTensorDesc, const cuinferFilterDescriptor_t filterDesc, int nbDims, int tensorOuputDimA[]); /// @brief Destroy a convolution descriptor after use. /// @param[in] convDesc The ::cuinferConvolutionDescriptor_t to be destroyed. /// @warning Deleting a \p convDesc twice is an undefined behavior. /// @return /// * ::CUINFER_STATUS_SUCCESS If success. cuinferStatus_t CUINFERWINAPI cuinferDestroyConvolutionDescriptor(cuinferConvolutionDescriptor_t convDesc); /** * @brief Function to concatenate a few tensors to a output tensor. * @details This function is made to concatenate tensors, the number of input tensors can be two,three, or four channel in cuinferTensorDescriptor_t is padded channel, the real channel is realc (when axis != 3, the realc is useless). * * Example: Concat two tensors, x1Desc = {N, H, W, padc1}, x2Desc = {N, H, W, padc2}, yDesc = {N, H, W, realc1 + realc2 + y_pad}. * * @param[in] x1Desc The information of input1. * @param[in] x1 Input1 address. * @param[in] x2Desc The information of input2. * @param[in] x2 Input2 address. * @param[in] x3Desc The information of input2. * @param[in] x3 Input3 address. * @param[in] x4Desc The information of input2. * @param[in] x4 Input4 address. * @param[in] yDesc The information of output. * @param[out] y Output address. * @param[in] axis Decide whether realc is useful. * @param[in] bQuant Decide whether the result need multiply \p y_scale and \p scale1 \p scale2 \p scale3 \p scale4. * @return * * ::CUINFER_STATUS_BAD_PARAM If \p x1, \p x2 is \p nullptr. * * ::CUINFER_STATUS_NOT_SUPPORTED If not supported. * * ::CUINFER_STATUS_SUCCESS If success. */ cuinferStatus_t CUINFERWINAPI cuinferConcatenate( cuinferHandle_t handle, const cuinferTensorDescriptor_t x1Desc, const void *x1, const void *scale1, const int realc1, const cuinferTensorDescriptor_t x2Desc, const void *x2, const void *scale2, const int realc2, const cuinferTensorDescriptor_t x3Desc, const void *x3, const void *scale3, const int realc3, const cuinferTensorDescriptor_t x4Desc, const void *x4, const void *scale4, const int realc4, const cuinferTensorDescriptor_t yDesc, void *y, const void *y_scale, const int axis, bool bQuant); /// @brief Split input int8 tensor to 2 or 3 tensors. /// @param[in] handle The libinfer handle. /// @param[in] xDesc The discriptor of input tensor x. /// @param[in] x Const pointer to input tensor x. /// @param[in] batch The batch. A quantity used or made at one time. /// @param[in] height The height of the image tensor. /// @param[in] width The width of the image tensor. /// @param[in] sizeLen The split size, 2 or 3. /// @param[in] sizes The size start of each parts. /// @param[in] axis The axis to split. /// @param[out] y The discriptor of output tensor y. /// @return /// * ::CUINFER_STATUS_NOT_SUPPORTED If not supported. /// * ::CUINFER_STATUS_SUCCESS If success. /// @todo Currently not used by other library. cuinferStatus_t CUINFERWINAPI cuinferSplitForward( cuinferHandle_t handle, const cuinferTensorDescriptor_t xDesc, const void *x, const int batch, const int height, const int width, const int sizeLen, const int *sizes, const int axis, void *y); /// Interpolation method used in image resize. typedef enum { CUINFER_INTER_NEAREST = 0, ///< Pixel is determined by it's nearest neighbor. CUINFER_INTER_LINEAR = 1, ///< Pixel is determined by linear interpolation. CUINFER_INTER_CUBIC = 2, ///< Pixcel is determined by cubic interpolation. CUINFER_INTER_AREA = 3, ///< Not used. } cuinferInterpolationFlag_t; /// @todo Explain this. typedef enum { CUINFER_HALF_PIXEL = 0, CUINFER_ALIGN_CORNERS = 1, CUINFER_ASYMMETRIC = 2, } cuinferCoordinateTransformationMode_t; /// @brief Resize a image. /// @note The input pointer and output space are not overlap. /// @param[in] handle The libinfer handle. /// @param[in] xDesc The tensor descriptor of the input. /// @param[in] x The const pointer of input. /// @param[in] interpolation Interpolation mode. /// @param[in] transformMode /// @param[in] yDesc The tensor descriptor of the output. /// @param[out] y The pointer of output tensor y. /// @return /// * ::CUINFER_STATUS_NOT_SUPPORTED If algo is not supported. /// * ::CUINFER_STATUS_SUCCESS If success. cuinferStatus_t CUINFERWINAPI cuinferResize2D(cuinferHandle_t handle, const cuinferTensorDescriptor_t xDesc, const void *x, cuinferInterpolationFlag_t interpolation, cuinferCoordinateTransformationMode_t transformMode, const cuinferTensorDescriptor_t yDesc, void *y); /// Convolution forward algo selection preference. typedef enum { CUINFER_CONVOLUTION_FWD_NO_WORKSPACE = 0, ///< No extra workspace. CUINFER_CONVOLUTION_FWD_PREFER_FASTEST = 1, ///< Prefer fastest. CUINFER_CONVOLUTION_FWD_SPECIFY_WORKSPACE_LIMIT = 2, ///< Specify workspace limit. } cuinferConvolutionFwdPreference_t; /// Convolution forward algo. typedef enum { CUINFER_CONVOLUTION_FWD_ALGO_IMPLICIT_GEMM = 0, ///< Implicit gemm. CUINFER_CONVOLUTION_FWD_ALGO_IMPLICIT_PRECOMP_GEMM = 1, ///< Implicit CUINFER_CONVOLUTION_FWD_ALGO_GEMM = 2, ///< Gemm. CUINFER_CONVOLUTION_FWD_ALGO_DIRECT = 3, ///< Direct compute use cuda call. CUINFER_CONVOLUTION_FWD_ALGO_FFT = 4, ///< FFT. CUINFER_CONVOLUTION_FWD_ALGO_FFT_TILING = 5, ///< FFT tiling. CUINFER_CONVOLUTION_FWD_ALGO_WINOGRAD = 6, ///< Winograd. CUINFER_CONVOLUTION_FWD_ALGO_WINOGRAD_NONFUSED = 7, ///< Winograd nonfused. CUINFER_CONVOLUTION_FWD_ALGO_COUNT = 8, ///< Total algo count. } cuinferConvolutionFwdAlgo_t; /// How to connect conv result and previous result. typedef enum { CUINFER_CONNECTION_NONE = 0, ///< No previous result is used. CUINFER_CONNECTION_ADD = 1, ///< Add two results. CUINFER_CONNECTION_MUL = 2, ///< Multiply two results CUINFER_CONNECTION_CONCAT = 3, ///< Stack two results. } cuinferTensorConnectionMode_t; /// Profile result of convolution forward algorithms. typedef struct { cuinferConvolutionFwdAlgo_t algo; ///< Algo name. cuinferStatus_t status; ///< Return status. float time; ///< Runtime. size_t memory; ///< Memory needed. cuinferDeterminism_t determinism; ///< Is algorithm deterministic. cuinferMathType_t mathType; ///< Algo math type(use tensor op or not). int reserved[3]; ///< Reserved. } cuinferConvolutionFwdAlgoPerf_t; /// @brief Get count of convolution forward algorithms. /// @param[in] handle The libinfer handle. /// @param[out] count The number of convolution forward algorithms. /// @return /// * ::CUINFER_STATUS_SUCCESS If success. /// * ::CUINFER_STATUS_BAD_PARAM If \p handle is null. cuinferStatus_t CUINFERWINAPI cuinferGetConvolutionForwardAlgorithmMaxCount( cuinferHandle_t handle, int *count); /// @brief Find the best convolution forward algorithm under given conditions. /// @details Formular is given based on the combination of params. /// * y = activate(connect((conv(x, w) * (perchannelAlpha[i] or alpha) + /// bias[i]), z * z_scale) * alpha2) /// * y = connect(activate(conv(x, w) * (perchannelAlpha[i] or alpha) + /// bias[i]), z * z_scale) * alpha2 /// @todo For debug set enviroment variable \p DNN_DEBUG_FIND_CONV_FWD_ALGO to /// the algo choosen. /// @see ::cuinferQDEConvolutionForward /// @param[in] handle The libinfer handle. /// @param[in] alpha The scale factor used after convolution result. Single /// float. /// @param[in] perchannelAlpha The scale factor used after convolution result. /// Channel times float. /// @param[in] xDesc The info of input tensor x. /// @param[in] wDesc The info of filter w. /// @param[in] convDesc The info of convolution. /// @param[in] yDesc The info of output tensor y. /// @param[in] zDesc The info of input tensor z. /// @param[in] biasDesc Not used. /// @param[in] activationDesc The info of activation. /// @param[in] connectionMode The connection mode. /// @param[in] perChannel Whether alpha is individual for each channel. /// @param[in] connectionBeforeActivation Whether activation is performed before /// connection. /// @param[in] requestedAlgoCount Requested algorithm max count. /// @param[out] returnedAlgoCount Result algorithm count. /// @param[out] perfResults Profile results. /// @return /// * ::CUINFER_STATUS_BAD_PARAM If input tensor is null or bad param. /// * ::CUINFER_STATUS_NOT_SUPPORTED If algo is not supported. /// * ::CUINFER_STATUS_SUCCESS If success. cuinferStatus_t CUINFERWINAPI cuinferFindConvolutionForwardAlgorithm( cuinferHandle_t handle, const void *alpha, const void *perchannelAlpha, const cuinferTensorDescriptor_t xDesc, const cuinferFilterDescriptor_t wDesc, const cuinferConvolutionDescriptor_t convDesc, const cuinferTensorDescriptor_t yDesc, const cuinferTensorDescriptor_t zDesc, const cuinferTensorDescriptor_t biasDesc, const cuinferActivationDescriptor_t activationDesc, const cuinferTensorConnectionMode_t connectionMode, bool perChannel, bool connectionBeforeActivation, const int requestedAlgoCount, int returnedAlgoCount[], cuinferConvolutionFwdAlgoPerf_t perfResults[]); /// @brief Find best convolution forward algorithms for \p float16. /// @param[in] handle The libinfer handle. /// @param[in] xDesc The descriptor of tensor x. /// @param[in] wDesc The descriptor of filter w. /// @param[in] convDesc The descriptor of convolution. /// @param[in] yDesc The descriptor of tensor y. /// @param[in] zDesc The descriptor of tensor z. /// @param[in] biasDesc The discriptor of bias. /// @param[in] activationDesc The discriptor of activation. /// @param[in] connectionMode The connection mode. /// @param[in] connectionBeforeActivation Whether activation is performed before /// connection. /// @param[in] requestedAlgoCount Requested algorithm max count. /// @param[out] returnedAlgoCount Result algorithm count. /// @param[out] perfResults Profile results. /// @return /// * ::CUINFER_STATUS_BAD_PARAM If input tensor is null or bad param. /// * ::CUINFER_STATUS_SUCCESS If success. cuinferStatus_t CUINFERWINAPI cuinferFindConvolutionForwardAlgorithmFP16( cuinferHandle_t handle, const cuinferTensorDescriptor_t xDesc, const cuinferFilterDescriptor_t wDesc, const cuinferConvolutionDescriptor_t convDesc, const cuinferTensorDescriptor_t yDesc, const cuinferTensorDescriptor_t zDesc, const cuinferTensorDescriptor_t biasDesc, const cuinferActivationDescriptor_t activationDesc, const cuinferTensorConnectionMode_t connectionMode, bool connectionBeforeActivation, const int requestedAlgoCount, int returnedAlgoCount[], cuinferConvolutionFwdAlgoPerf_t perfResults[]); /// @brief Find best convolution forward algorithm within limited workspace size /// with actual profile. /// @param[in] handle The libinfer handle. /// @param[in] xDesc The discriptor of input tensor x. /// @param[in] x Const pointer to input tensor x. /// @param[in] wDesc The discriptor of filter w. /// @param[in] The const pointer of input filter w. /// @param[in] convDesc The discriptor of convolution. /// @param[in] yDesc The discriptor of tensor y. /// @param[in] y /// @param[in] requestedAlgoCount Requested algorithm max count. /// @param[out] returnedAlgoCount Result algorithm count. /// @param[out] perfResults Profile results. /// @param[in] workSpace The workspace pre-allocated. See the corresponding get /// workspace size helper function. /// @param[in] workSpaceSizeInBytes The workspace size pre-allocated. /// @return /// * ::CUINFER_STATUS_SUCCESS If success. /// * ::CUINFER_STATUS_BAD_PARAM If input tensor is null or bad param. cuinferStatus_t CUINFERWINAPI cuinferFindConvolutionForwardAlgorithmEx( cuinferHandle_t handle, const cuinferTensorDescriptor_t xDesc, const void *x, const cuinferFilterDescriptor_t wDesc, const void *w, const cuinferConvolutionDescriptor_t convDesc, const cuinferTensorDescriptor_t yDesc, void *y, const int requestedAlgoCount, int *returnedAlgoCount, cuinferConvolutionFwdAlgoPerf_t *perfResults, void *workSpace, size_t workSpaceSizeInBytes); /// @brief Find best convolution forward algorithm within limited workspace size /// with no actual run. /// @param[in] handle The libinfer handle. /// @param[in] xDesc The discriptor of input tensor x. /// @param[in] wDesc The discriptor of filter w. /// @param[in] convDesc The discriptor of convolution. /// @param[in] yDesc The discriptor of tensor y. /// @param[in] preference The algo preference. /// @param[in] memoryLimitInBytes The memory limit. /// @param[out] algo The result algorithm. /// @return /// * ::CUINFER_STATUS_SUCCESS If success. /// * ::CUINFER_STATUS_BAD_PARAM If input tensor is null or bad param. cuinferStatus_t CUINFERWINAPI cuinferGetConvolutionForwardAlgorithm( cuinferHandle_t handle, const cuinferTensorDescriptor_t xDesc, const cuinferFilterDescriptor_t wDesc, const cuinferConvolutionDescriptor_t convDesc, const cuinferTensorDescriptor_t yDesc, cuinferConvolutionFwdPreference_t preference, size_t memoryLimitInBytes, cuinferConvolutionFwdAlgo_t *algo); /// @brief Find the best convolution forward algorithm. /// @param[in] handle The libinfer handle. /// @param[in] srcDesc The discriptor of input tensor. /// @param[in] filterDesc The discriptor of filter tensor. /// @param[in] convDesc The discriptor of convolution. /// @param[in] destDesc The discriptor of output tensor. /// @param[in] requestedAlgoCount Requested algorithm max count. /// @param[out] returnedAlgoCount Result algorithm count. /// @param[out] perfResults Profile results. /// @return /// * ::CUINFER_STATUS_SUCCESS If success. /// * ::CUINFER_STATUS_BAD_PARAM If input tensor is null or bad param. /// @todo This is not used by any other library. cuinferStatus_t CUINFERWINAPI cuinferGetConvolutionForwardAlgorithm_v7( cuinferHandle_t handle, const cuinferTensorDescriptor_t srcDesc, const cuinferFilterDescriptor_t filterDesc, const cuinferConvolutionDescriptor_t convDesc, const cuinferTensorDescriptor_t destDesc, const int requestedAlgoCount, int *returnedAlgoCount, cuinferConvolutionFwdAlgoPerf_t *perfResults); /// @brief Get extra workspace size in bytes used by convolution forward /// algorithm. /// @details Convolution algorithm (which requires potentially some workspace). /// Helper function to return the minimum size of the workspace to be passed to /// the convolution given an algo. /// @param[in] handle The libinfer handle. /// @param[in] xDesc The discriptor of input tensor x. /// @param[in] wDesc The discriptor of filter w. /// @param[in] convDesc The discriptor of convolution. /// @param[in] yDesc The discriptor of tensor y. /// @param[in] algo The algorithm specified. /// @param[out] sizeInBytes The result extra temporary space size in bytes. /// @return /// * ::CUINFER_STATUS_BAD_PARAM If input tensor is null or bad param. /// * ::CUINFER_STATUS_NOT_SUPPORTED If algo not supported. /// * ::CUINFER_STATUS_SUCCESS If success. cuinferStatus_t CUINFERWINAPI cuinferGetConvolutionForwardWorkspaceSize( cuinferHandle_t handle, const cuinferTensorDescriptor_t xDesc, const cuinferFilterDescriptor_t wDesc, const cuinferConvolutionDescriptor_t convDesc, const cuinferTensorDescriptor_t yDesc, cuinferConvolutionFwdAlgo_t algo, size_t *sizeInBytes); // clang-format off /// @defgroup ConvolutionFunctions Convollution Functions /// @details /// Common result for all convolution functions for quantifier. /// \code /// if cuinferTensorConnectionMode_t == CUINFER_CONNECTION_NONE: /// if bias == nullptr: /// if perChannel == false: /// y = clip(round(activate((convtransposed(x, w) * alpha)))) /// if perChannel == ture: /// y = clip(round(activate((convtransposed(x, w) * perchannelAlpha[i])))) /// if bias != nullptr: /// if perChannel == false: /// y = clip(round(activate((convtransposed(x, w) * alpha) + bias[0]))) /// if perChannel == ture: /// y = clip(round(activate((convtransposed(x, w) * perchannelAlpha[i]) + bias[i]))) /// elif cuinferTensorConnectionMode_t == CUINFER_CONNECTION_ADD: /// if bias == nullptr: /// if perChannel == false: /// if befor_activation_ == false: /// y = clip(round((activate(convtransposed(x, w) * alpha) + z * z_scale) * alpha2)) /// else: /// y = clip(round(activate(((convtransposed(x, w) * alpha) + z * z_scale) * alpha2))) /// if perChannel == ture: /// if connectionDesc.befor_activation_ == 0: /// y = clip(round((activate(convtransposed(x, w) * perchannelAlpha[i]) + z * z_scale) * alpha2)) /// else: /// y = clip(round(activate(((convtransposed(x, w) * perchannelAlpha[i]) + z * z_scale) * alpha2))) /// if bias != nullptr: /// if perChannel == false: /// if befor_activation_ == false: /// y = clip(round((activate(convtransposed(x, w) * alpha + bias[i]) + z * z_scale) * alpha2)) /// else: /// y = clip(round(activate(((convtransposed(x, w) * alpha + bias[i]) + z * z_scale) * alpha2))) /// if perChannel == ture: /// if befor_activation_ == false: /// y = clip(round((activate(convtransposed(x, w) * perchannelAlpha[i] + bias[i]) + z * z_scale) * alpha2)) /// else: /// y = clip(round(activate(((convtransposed(x, w) * perchannelAlpha[i] + bias[i]) + z * z_scale) * alpha2))) /// elif cuinferTensorConnectionMode_t == CUINFER_CONNECTION_CONCAT: /// if bias == nullptr: /// if perChannel == false: /// if befor_activation_ == false: /// y = clip(round((concat(activate(convtransposed(x, w) * alpha), z * z_scale)) * alpha2)) /// else: /// y = clip(round(activate(concat(convtransposed(x, w) * alpha, z * z_scale)) * alpha2)) /// if perChannel == ture: /// if befor_activation_ == false: /// y = clip(round((concat(activate(convtransposed(x, w) * perchannelAlpha[i]), z * z_scale)) * alpha2)) /// else: /// y = clip(round(activate(concat(convtransposed(x, w) * perchannelAlpha[i], z * z_scale) * alpha2))) /// if bias != nullptr: /// if perChannel == false: /// if befor_activation_ == false: /// y = clip(round(concat(activate(convtransposed(x, w) * alpha + bias[i]), z * z_scale) * alpha2)) /// else: /// y = clip(round(activate(concat(convtransposed(x, w) * alpha + bias[i], z * z_scale)) * alpha2)) /// if perChannel == ture: /// if befor_activation_ == false: /// y = clip(round(concat(activate(convtransposed(x, w) * perchannelAlpha[i] + bias[i]), z * z_scale) * alpha2)) /// else: /// y = clip(round(activate(concat((convtransposed(x, w) * perchannelAlpha[i] + bias[i]), z * z_scale) * alpha2))) /// \endcode // clang-format on /// @brief Function to perform the forward pass for batch convolution. /// @details Formula, y = alpha[0] * conv(x, w) + beta[0] * y. /// @note Only int8(only relu) and true half configs are supported. /// @ingroup ConvolutionFunctions /// @param[in] handle The libinfer handle. /// @param[in] alpha The pointer to scaling factor. /// @param[in] xDesc The descriptor of input tensor x. /// @param[in] x Const pointer to input tensor x. /// @param[in] wDesc The discriptor of filter w. /// @param[in] The const pointer of input filter w. /// @param[in] convDesc The discriptor of convolution. /// @param[in] algo The algorithm specified. /// @param[in] workSpace The workspace pre-allocated. See the corresponding /// get workspace size helper function. /// @param[in] workSpaceSizeInBytes The workspace size in bytes. /// @param[in] beta Pointer to scaling factor. /// @param[in] yDesc The discriptor of tensor y. /// @param[in, out] y The discriptor of output tensor y. /// @return /// * ::CUINFER_STATUS_BAD_PARAM If input tensor is null or bad param. /// * ::CUINFER_STATUS_NOT_SUPPORTED If algo not supported. /// * ::CUINFER_STATUS_SUCCESS If success. cuinferStatus_t CUINFERWINAPI cuinferConvolutionForward(cuinferHandle_t handle, const void *alpha, const cuinferTensorDescriptor_t xDesc, const void *x, const cuinferFilterDescriptor_t wDesc, const void *w, const cuinferConvolutionDescriptor_t convDesc, cuinferConvolutionFwdAlgo_t algo, void *workSpace, size_t workSpaceSizeInBytes, const void *beta, const cuinferTensorDescriptor_t yDesc, void *y); /// @brief Convolution forward with quantifier. /// @details For what params means like \p alpha, \p alpha2, and \p bias, see /// \ref ConvolutionFunctions. /// @ingroup ConvolutionFunctions /// @param[in] handle The libinfer handle. /// @param[in] alpha Pointer to scaling factor. /// @param[in] beta Pointer to scaling factor. /// @param[in] gamma Pointer to scaling factor. /// @param[in] xDesc The discriptor of input tensor x. /// @param[in] x Const pointer to input tensor x. /// @param[in] wDesc The discriptor of filter w. /// @param[in] The const pointer of input filter w. /// @param[in] convDesc The discriptor of convolution. /// @param[in] algo The algorithm specified. /// @param[in] workSpace The workspace pre-allocated. See the corresponding get /// workspace size helper function. /// @param[in] workSpaceSizeInBytes The workspace size in bytes. /// @param[in] alpha2 The pointer to scaling factor /// @param[in] zDesc /// @param[in] z /// @param[in] biasDesc /// @param[in] bias /// @param[in] activationDesc /// @param[in] yDesc The discriptor of tensor y. /// @param[out] y The discriptor of output tensor y. /// @return cuinferStatus_t CUINFERWINAPI cuinferQConvolutionForward( cuinferHandle_t handle, const void *alpha, const void *beta, const void *gamma, const cuinferTensorDescriptor_t xDesc, const void *x, const cuinferFilterDescriptor_t wDesc, const void *w, const cuinferConvolutionDescriptor_t convDesc, cuinferConvolutionFwdAlgo_t algo, void *workSpace, size_t workSpaceSizeInBytes, const void *alpha2, const cuinferTensorDescriptor_t zDesc, const void *z, const cuinferTensorDescriptor_t biasDesc, const void *bias, const cuinferActivationDescriptor_t activationDesc, const cuinferTensorDescriptor_t yDesc, void *y); /// @brief /// @ingroup ConvolutionFunctions /// @param[in] handle The libinfer handle. /// @param[in] alpha Pointer to scaling factor. /// @param[in] perchannelAlpha /// @param[in] beta Pointer to scaling factor. /// @param[in] gamma Pointer to scaling factor. /// @param[in] xDesc The discriptor of input tensor x. /// @param[in] x Const pointer to input tensor x. /// @param[in] wDesc The discriptor of filter w. /// @param[in] The const pointer of input filter w. /// @param[in] convDesc The discriptor of convolution. /// @param[in] algo The algorithm specified. /// @param[in] workSpace The workspace pre-allocated. See the corresponding get /// workspace size helper function. /// @param[in] workSpaceSizeInBytes /// @param[in] alpha2 /// @param[in] zDesc /// @param[in] z /// @param[in] biasDesc /// @param[in] bias /// @param[in] quadDesc /// @param[in] perChannel /// @param[in] activationDesc /// @param[in] yDesc The discriptor of tensor y. /// @param[out] y The discriptor of output tensor y. /// @return cuinferStatus_t CUINFERWINAPI cuinferQDConvolutionForward( cuinferHandle_t handle, const void *alpha, const void *perchannelAlpha, const void *beta, const void *gamma, const cuinferTensorDescriptor_t xDesc, const void *x, const cuinferFilterDescriptor_t wDesc, const void *w, const cuinferConvolutionDescriptor_t convDesc, cuinferConvolutionFwdAlgo_t algo, void *workSpace, size_t workSpaceSizeInBytes, const void *alpha2, const cuinferTensorDescriptor_t zDesc, const void *z, const cuinferTensorDescriptor_t biasDesc, const void *bias, const cuinferTensorDescriptor_t quadDesc, bool perChannel, const cuinferActivationDescriptor_t activationDesc, const cuinferTensorDescriptor_t yDesc, void *y); /// @brief /// @ingroup ConvolutionFunctions /// @see Common format in group \ref ConvolutionFunctions. /// @param[in] handle The libinfer handle. /// @param[in] alpha Pointer to scaling factor. /// @param[in] perchannelAlpha /// @param[in] beta Pointer to scaling factor. /// @param[in] gamma Pointer to scaling factor. /// @param[in] xDesc The discriptor of input tensor x. /// @param[in] x Const pointer to input tensor x. /// @param[in] wDesc The discriptor of filter w. /// @param[in] The const pointer of input filter w. /// @param[in] convDesc The discriptor of convolution. /// @param[in] algo The algorithm specified. /// @param[in] workSpace The workspace pre-allocated. See the corresponding get /// workspace size helper function. /// @param[in] workSpaceSizeInBytes /// @param[in] alpha2 /// @param[in] zScale /// @param[in] zDesc /// @param[in] z /// @param[in] biasDesc /// @param[in] bias /// @param[in] perChannel /// @param[in] activationDesc /// @param[in] connectionBeforeActivation Whether activation is performed before /// connection. /// @param[in] connectionMode The connection mode. /// @param[in] yDesc The discriptor of tensor y. /// @param[out] y The discriptor of output tensor y. /// @return cuinferStatus_t CUINFERWINAPI cuinferQDEConvolutionForward( cuinferHandle_t handle, const void *alpha, const void *perchannelAlpha, const void *beta, const void *gamma, const cuinferTensorDescriptor_t xDesc, const void *x, const cuinferFilterDescriptor_t wDesc, const void *w, const cuinferConvolutionDescriptor_t convDesc, cuinferConvolutionFwdAlgo_t algo, void *workSpace, size_t workSpaceSizeInBytes, const void *alpha2, const void *zScale, const cuinferTensorDescriptor_t zDesc, const void *z, const cuinferTensorDescriptor_t biasDesc, const void *bias, bool perChannel, const cuinferActivationDescriptor_t activationDesc, bool connectionBeforeActivation, const cuinferTensorConnectionMode_t connectionMode, const cuinferTensorDescriptor_t yDesc, void *y); /// @brief /// @ingroup ConvolutionFunctions /// @param[in] handle The libinfer handle. /// @param[in] alpha Pointer to scaling factor. /// @param[in] beta Pointer to scaling factor. /// @param[in] gamma Pointer to scaling factor. /// @param[in] xDesc The discriptor of input tensor x. /// @param[in] x Const pointer to input tensor x. /// @param[in] wDesc The discriptor of filter w. /// @param[in] The const pointer of input filter w. /// @param[in] convDesc The discriptor of convolution. /// @param[in] algo The algorithm specified. /// @param[in] workSpace The workspace pre-allocated. See the corresponding get /// workspace size helper function. /// @param[in] workSpaceSizeInBytes /// @param[in] alpha2 /// @param[in] zDesc /// @param[in] z /// @param[in] biasDesc /// @param[in] bias /// @param[in] activationDesc /// @param[in] connectionBeforeActivation Whether activation is performed before /// connection. /// @param[in] connectionMode The connection mode. /// @param[in] yDesc The discriptor of tensor y. /// @param[out] y The discriptor of output tensor y. /// @return cuinferStatus_t CUINFERWINAPI cuinferHalfConvolution2dForward( cuinferHandle_t handle, const void *alpha, const void *beta, const void *gamma, const cuinferTensorDescriptor_t xDesc, const void *x, const cuinferFilterDescriptor_t wDesc, const void *w, const cuinferConvolutionDescriptor_t convDesc, cuinferConvolutionFwdAlgo_t algo, void *workSpace, size_t workSpaceSizeInBytes, const void *alpha2, const cuinferTensorDescriptor_t zDesc, const void *z, const cuinferTensorDescriptor_t biasDesc, const void *bias, const cuinferActivationDescriptor_t activationDesc, bool connectionBeforeActivation, const cuinferTensorConnectionMode_t connectionMode, const cuinferTensorDescriptor_t yDesc, void *y); typedef enum { CUINFER_CONVOLUTION_BWD_FILTER_ALGO_0 = 0, ///< Non-deterministic. CUINFER_CONVOLUTION_BWD_FILTER_ALGO_1 = 1, CUINFER_CONVOLUTION_BWD_FILTER_ALGO_FFT = 2, CUINFER_CONVOLUTION_BWD_FILTER_ALGO_3 = 3, ///< Non-deterministic. CUINFER_CONVOLUTION_BWD_FILTER_ALGO_WINOGRAD = 4, ///< Not implemented. CUINFER_CONVOLUTION_BWD_FILTER_ALGO_WINOGRAD_NONFUSED = 5, CUINFER_CONVOLUTION_BWD_FILTER_ALGO_FFT_TILING = 6, CUINFER_CONVOLUTION_BWD_FILTER_ALGO_COUNT = 7 } cuinferConvolutionBwdFilterAlgo_t; typedef struct { cuinferConvolutionBwdFilterAlgo_t algo; cuinferStatus_t status; float time; size_t memory; cuinferDeterminism_t determinism; cuinferMathType_t mathType; int reserved[3]; } cuinferConvolutionBwdFilterAlgoPerf_t; typedef enum { CUINFER_CONVOLUTION_BWD_DATA_ALGO_0 = 0, ///< Non-deterministic. CUINFER_CONVOLUTION_BWD_DATA_ALGO_1 = 1, CUINFER_CONVOLUTION_BWD_DATA_ALGO_FFT = 2, CUINFER_CONVOLUTION_BWD_DATA_ALGO_FFT_TILING = 3, CUINFER_CONVOLUTION_BWD_DATA_ALGO_WINOGRAD = 4, CUINFER_CONVOLUTION_BWD_DATA_ALGO_WINOGRAD_NONFUSED = 5, CUINFER_CONVOLUTION_BWD_DATA_ALGO_COUNT = 6 } cuinferConvolutionBwdDataAlgo_t; typedef struct { cuinferConvolutionBwdDataAlgo_t algo; cuinferStatus_t status; float time; size_t memory; cuinferDeterminism_t determinism; cuinferMathType_t mathType; int reserved[3]; } cuinferConvolutionBwdDataAlgoPerf_t; /// @brief /// @param[in] handle The libinfer handle. /// @param[in] xDesc The discriptor of input tensor x. /// @param[in] x Const pointer to input tensor x. /// @param[in] wDesc The discriptor of filter w. /// @param[in] convDesc The discriptor of convolution. /// @param[out] colBuffer /// @return cuinferStatus_t CUINFERWINAPI cuinferIm2Col(cuinferHandle_t handle, const cuinferTensorDescriptor_t xDesc, const void *x, const cuinferFilterDescriptor_t wDesc, const cuinferConvolutionDescriptor_t convDesc, void *colBuffer); /// Softmax algorithm. typedef enum { /// Straightforward implementation. May overflow. This is useful when /// input is guaranteed in range. CUINFER_SOFTMAX_FAST = 0, /// Subtract max from every point to avoid overflow. CUINFER_SOFTMAX_ACCURATE = 1, /// Add log to result. This will use algorithm accurate. CUINFER_SOFTMAX_LOG = 2, } cuinferSoftmaxAlgorithm_t; typedef enum { /// Compute the softmax over all C, H, W for each N. CUINFER_SOFTMAX_MODE_INSTANCE = 0, /// Compute the softmax over all C for each H, W, N. CUINFER_SOFTMAX_MODE_CHANNEL = 1, /// Compute the softmax over all W for each N, C, H. CUINFER_SOFTMAX_MODE_WIDTH = 2 } cuinferSoftmaxMode_t; /// @defgroup SortmaxFunctions Softmax Funtions /// @note Softmax functions: All of the form "output = alpha * Op(inputs) + beta /// * output". /// @brief Function to perform forward softmax. /// @ingroup SortmaxFunctions /// @param[in] handle The libinfer handle. /// @param[in] algo The algorithm specified. /// @param[in] mode /// @param[in] alpha Pointer to scaling factor. /// @param[in] xDesc The discriptor of input tensor x. /// @param[in] x Const pointer to input tensor x. /// @param[in] beta Pointer to scaling factor. /// @param[in] yDesc The discriptor of tensor y. /// @param[out] y The discriptor of output tensor y. /// @return cuinferStatus_t CUINFERWINAPI cuinferSoftmaxForward( cuinferHandle_t handle, cuinferSoftmaxAlgorithm_t algo, cuinferSoftmaxMode_t mode, const void *alpha, const cuinferTensorDescriptor_t xDesc, const void *x, const void *beta, const cuinferTensorDescriptor_t yDesc, void *y); /// @brief Function to perform forward dequant, softmax and quant. /// @ingroup SoftmaxFunctions /// @param[in] handle The libinfer handle. /// @param[in] algo The algorithm specified. /// @param[in] mode /// @param[in] quant_scale /// @param[in] xDesc The discriptor of input tensor x. /// @param[in] x Const pointer to input tensor x. /// @param[in] zero_point /// @param[in] yDesc The discriptor of tensor y. /// @param[out] y The discriptor of output tensor y. /// @return cuinferStatus_t CUINFERWINAPI cuinferDeQuantSoftmaxForwardQuant( cuinferHandle_t handle, cuinferSoftmaxAlgorithm_t algo, cuinferSoftmaxMode_t mode, const void *quant_scale, ///< 2 value! const cuinferTensorDescriptor_t xDesc, const void *x, const void *zero_point, const cuinferTensorDescriptor_t yDesc, void *y); /// Pooling mode. typedef enum { CUINFER_POOLING_MAX = 0, CUINFER_POOLING_AVERAGE_COUNT_INCLUDE_PADDING = 1, ///< Count for average includes padded values. CUINFER_POOLING_AVERAGE_COUNT_EXCLUDE_PADDING = 2, ///< Count for average does not include padded values. CUINFER_POOLING_MAX_DETERMINISTIC = 3 } cuinferPoolingMode_t; /// @brief Create an instance of pooling descriptor. /// @param[out] poolingDesc /// @return cuinferStatus_t CUINFERWINAPI cuinferCreatePoolingDescriptor(cuinferPoolingDescriptor_t *poolingDesc); /// @brief /// @param[out] poolingDesc /// @param[in] mode /// @param[in] maxpoolingNanOpt /// @param[in] windowHeight /// @param[in] windowWidth /// @param[in] verticalPadding /// @param[in] horizontalPadding /// @param[in] verticalStride /// @param[in] horizontalStride /// @return cuinferStatus_t CUINFERWINAPI cuinferSetPooling2dDescriptor( cuinferPoolingDescriptor_t poolingDesc, cuinferPoolingMode_t mode, cuinferNanPropagation_t maxpoolingNanOpt, int windowHeight, int windowWidth, int verticalPadding, int horizontalPadding, int verticalStride, int horizontalStride); /// @brief /// @param[in] poolingDesc /// @param[out] mode /// @param[out] maxpoolingNanOpt /// @param[out] windowHeight /// @param[out] windowWidth /// @param[out] verticalPadding /// @param[out] horizontalPadding /// @param[out] verticalStride /// @param[out] horizontalStride /// @return cuinferStatus_t CUINFERWINAPI cuinferGetPooling2dDescriptor( const cuinferPoolingDescriptor_t poolingDesc, cuinferPoolingMode_t *mode, cuinferNanPropagation_t *maxpoolingNanOpt, int *windowHeight, int *windowWidth, int *verticalPadding, int *horizontalPadding, int *verticalStride, int *horizontalStride); /// @brief /// @param[out] poolingDesc /// @param[in] mode /// @param[in] maxpoolingNanOpt /// @param[in] nbDims /// @param[in] windowDimA /// @param[in] paddingA /// @param[in] strideA /// @return cuinferStatus_t CUINFERWINAPI cuinferSetPoolingNdDescriptor( cuinferPoolingDescriptor_t poolingDesc, const cuinferPoolingMode_t mode, const cuinferNanPropagation_t maxpoolingNanOpt, int nbDims, const int windowDimA[], const int paddingA[], const int strideA[]); /// @brief /// @param[in] poolingDesc /// @param[in] nbDimsRequested /// @param[out] mode /// @param[out] maxpoolingNanOpt /// @param[out] nbDims /// @param[out] windowDimA /// @param[out] paddingA /// @param[out] strideA /// @return cuinferStatus_t CUINFERWINAPI cuinferGetPoolingNdDescriptor( const cuinferPoolingDescriptor_t poolingDesc, int nbDimsRequested, cuinferPoolingMode_t *mode, cuinferNanPropagation_t *maxpoolingNanOpt, int *nbDims, int windowDimA[], int paddingA[], int strideA[]); /// @brief /// @param[in] poolingDesc /// @param[out] inputTensorDesc /// @param[in] nbDims /// @param[out] outputTensorDimA /// @return cuinferStatus_t CUINFERWINAPI cuinferGetPoolingNdForwardOutputDim( const cuinferPoolingDescriptor_t poolingDesc, const cuinferTensorDescriptor_t inputTensorDesc, int nbDims, int outputTensorDimA[]); /// @brief /// @param[in] poolingDesc /// @param[in] inputTensorDesc /// @param[out] n /// @param[out] c /// @param[out] h /// @param[out] w /// @return cuinferStatus_t CUINFERWINAPI cuinferGetPooling2dForwardOutputDim( const cuinferPoolingDescriptor_t poolingDesc, const cuinferTensorDescriptor_t inputTensorDesc, int *n, int *c, int *h, int *w); /// @brief Destroy an instance of pooling descriptor. /// @param[in] poolingDesc /// @return cuinferStatus_t CUINFERWINAPI cuinferDestroyPoolingDescriptor(cuinferPoolingDescriptor_t poolingDesc); /// @defgroup PoolingFunctions Pooling Functions /// @note Pooling functions: All of the form "output = alpha * Op(inputs) + beta /// * output" /// @brief Function to perform forward pooling. /// @ingroup PoolingFunctions /// @param[in] handle The libinfer handle. /// @param[in] poolingDesc /// @param[in] alpha Pointer to scaling factor. /// @param[in] xDesc The discriptor of input tensor x. /// @param[in] x Const pointer to input tensor x. /// @param[in] beta Pointer to scaling factor. /// @param[in] yDesc The discriptor of tensor y. /// @param[out] y The discriptor of output tensor y. /// @return cuinferStatus_t CUINFERWINAPI cuinferPoolingForward( cuinferHandle_t handle, const cuinferPoolingDescriptor_t poolingDesc, const void *alpha, const cuinferTensorDescriptor_t xDesc, const void *x, const void *beta, const cuinferTensorDescriptor_t yDesc, void *y); /// Activation Mode. @note Some activation function use extra parameters like a, /// which can be set by ::cuinferSetActivationDescriptor. typedef enum { CUINFER_ACTIVATION_SIGMOID = 0, ///< f(x) = 1(1+e^-x). CUINFER_ACTIVATION_RELU = 1, ///< f(x) = max(x, 0). CUINFER_ACTIVATION_TANH = 2, ///< f(x) = tanh(x) = 2sigmod(2x)-1. CUINFER_ACTIVATION_CLIPPED_RELU = 3, ///< f(x) = max(min(x,ceiling),0). CUINFER_ACTIVATION_ELU = 4, ///< f(x) = x if x > 0 else a(e^x-1). CUINFER_ACTIVATION_IDENTITY = 5, ///< f(x) = x. CUINFER_ACTIVATION_LEAKY_RELU = 6, ///< f(x) = max(x, ax). a = -0.01 i.e. CUINFER_ACTIVATION_SILU = 7, ///< f(x) = x/(1 + e^-x). CUINFER_ACTIVATION_HARD_SWISH = 8, ///< x*max(0,min(6,x+3))/6. CUINFER_ACTIVATION_HARD_SIGMOID = 9, ///< f(x) = max(0,min(1,(x+1)/2)). CUINFER_ACTIVATION_MISH = 10, ///< f(x) = x*tanh(x)*log(1+e^x). } cuinferActivationMode_t; /// @defgroup ActivationFunctions Activation Functions /// @note Activation functions: All of the form "output = alpha * Op(inputs) + /// beta * output" /// @brief /// @ingroup ActivationFunctions /// @param[out] activationDesc /// @return cuinferStatus_t CUINFERWINAPI cuinferCreateActivationDescriptor( cuinferActivationDescriptor_t *activationDesc); /// @brief /// @ingroup ActivationFunctions /// @param[out] activationDesc /// @param[in] mode /// @param[in] reluNanOpt /// @param[in] coef Ceiling for clipped RELU, alpha for ELU. /// @return cuinferStatus_t CUINFERWINAPI cuinferSetActivationDescriptor( cuinferActivationDescriptor_t activationDesc, cuinferActivationMode_t mode, cuinferNanPropagation_t reluNanOpt, double coef); /// @brief /// @ingroup ActivationFunctions /// @param[in] activationDesc /// @param[out] mode /// @param[out] reluNanOpt /// @param[out] coef Ceiling for clipped RELU, alpha for ELU. /// @return cuinferStatus_t CUINFERWINAPI cuinferGetActivationDescriptor( const cuinferActivationDescriptor_t activationDesc, cuinferActivationMode_t *mode, cuinferNanPropagation_t *reluNanOpt, double *coef); /// @brief /// @ingroup ActivationFunctions /// @param[in] activationDesc /// @return cuinferStatus_t CUINFERWINAPI cuinferDestroyActivationDescriptor( cuinferActivationDescriptor_t activationDesc); /// @brief Function to perform forward activation. /// @ingroup ActivationFunctions /// @param[in] handle The libinfer handle. /// @param[in] activationDesc /// @param[in] alpha Pointer to scaling factor. /// @param[in] xDesc The discriptor of input tensor x. /// @param[in] x Const pointer to input tensor x. /// @param[in] beta Pointer to scaling factor. /// @param[in] yDesc The discriptor of tensor y. /// @param[out] y The discriptor of output tensor y. /// @return cuinferStatus_t CUINFERWINAPI cuinferActivationForward( cuinferHandle_t handle, cuinferActivationDescriptor_t activationDesc, const void *alpha, const cuinferTensorDescriptor_t xDesc, const void *x, const void *beta, const cuinferTensorDescriptor_t yDesc, void *y); /// @defgroup LRNFunctions LRN Functions /// @note LRN functions: output = alpha * normalize(x) + beta * old_y /// @brief Create an instance of LRN (Local Response Normalization) descriptor. /// @details Uses lrnN=5, lrnAlpha=1e-4, lrnBeta=0.75, lrnK=2.0 as defaults /// from Krizhevsky'12 ImageNet paper. /// @ingroup LRNFunctions /// @param[out] normDesc /// @return cuinferStatus_t CUINFERWINAPI cuinferCreateLRNDescriptor(cuinferLRNDescriptor_t *normDesc); /// @ingroup LRNFunctions #define CUINFER_LRN_MIN_N 1 ///< minimum allowed lrnN /// @ingroup LRNFunctions #define CUINFER_LRN_MAX_N 16 ///< maximum allowed lrnN /// @ingroup LRNFunctions #define CUINFER_LRN_MIN_K 1e-5 ///< minimum allowed lrnK /// @ingroup LRNFunctions #define CUINFER_LRN_MIN_BETA 0.01 ///< minimum allowed lrnBeta /// LRN layer mode /// @ingroup LRNFunctions typedef enum { CUINFER_LRN_CROSS_CHANNEL_DIM1 = 0, ///< Normalize across tensor's dimA[1] dimension } cuinferLRNMode_t; /// @brief /// @details Uses a window [center-lookBehind, center+lookAhead], where /// lookBehind = floor( (lrnN-1)/2 ), lookAhead = lrnN-lookBehind-1. /// Values of double parameters cast to tensor data type. /// @ingroup LRNFunctions /// @param[out] normDesc /// @param[in] lrnN /// @param[in] lrnAlpha /// @param[in] lrnBeta /// @param[in] lrnK /// @return cuinferStatus_t CUINFERWINAPI cuinferSetLRNDescriptor(cuinferLRNDescriptor_t normDesc, unsigned lrnN, double lrnAlpha, double lrnBeta, double lrnK); /// @brief Retrieve the settings currently stored in an LRN layer descriptor. /// @details Any of the provided pointers can be NULL (no corresponding value /// will be returned). /// @ingroup LRNFunctions /// @param[in] normDesc /// @param[out] lrnN /// @param[out] lrnAlpha /// @param[out] lrnBeta /// @param[out] lrnK /// @return cuinferStatus_t CUINFERWINAPI cuinferGetLRNDescriptor(cuinferLRNDescriptor_t normDesc, unsigned *lrnN, double *lrnAlpha, double *lrnBeta, double *lrnK); /// @brief Destroy an instance of LRN descriptor. /// @ingroup LRNFunctions /// @param[in] lrnDesc /// @return cuinferStatus_t CUINFERWINAPI cuinferDestroyLRNDescriptor(cuinferLRNDescriptor_t lrnDesc); /// @brief LRN cross-channel forward computation. /// @details Double parameters cast to tensor data type. /// @ingroup LRNFunctions /// @param[in] handle The libinfer handle. /// @param[in] normDesc /// @param[in] lrnMode /// @param[in] alpha Pointer to scaling factor. /// @param[in] xDesc The discriptor of input tensor x. /// @param[in] x Const pointer to input tensor x. /// @param[in] beta Pointer to scaling factor. /// @param[in] yDesc The discriptor of tensor y. /// @param[out] y The discriptor of output tensor y. /// @return cuinferStatus_t CUINFERWINAPI cuinferLRNCrossChannelForward( cuinferHandle_t handle, cuinferLRNDescriptor_t normDesc, cuinferLRNMode_t lrnMode, const void *alpha, const cuinferTensorDescriptor_t xDesc, const void *x, const void *beta, const cuinferTensorDescriptor_t yDesc, void *y); typedef enum { /// \p bnScale, \p bnBias tensor dims are 1xCxHxWx.. (one value per /// CHW...-slice, normalized over N slice). CUINFER_BATCHNORM_PER_ACTIVATION = 0, /// \p bnScale, \p bnBias tensor dims are 1xCx1x1 (one value per C-dim /// normalized over Nx1xHxW subtensors). CUINFER_BATCHNORM_SPATIAL = 1, /// \p bnScale, \p bnBias tensor dims are 1xCx1x1 (one value per C-dim /// normalized over Nx1xHxW subtensors). May be faster than /// ::CUINFER_BATCHNORM_SPATIAL but imposes some limits on the range of /// values. CUINFER_BATCHNORM_SPATIAL_PERSISTENT = 2, } cuinferBatchNormMode_t; /// Minimum epsilon allowed to be used in the Batch Normalization formula. #define CUINFER_BN_MIN_EPSILON 0.0 /// @brief /// @details Derives a tensor descriptor from layer data descriptor for /// BatchNormalization \p scale, \p invVariance, \p bnBias, and \p bnScale /// tensors. Use this tensor desc for \p bnScaleBiasMeanVarDesc and \p /// bnScaleBiasDiffDesc in Batch Normalization forward and backward functions. /// @param[out] derivedBnDesc /// @param[in] xDesc The discriptor of input tensor x. /// @param[in] mode /// @return cuinferStatus_t CUINFERWINAPI cuinferDeriveBNTensorDescriptor( cuinferTensorDescriptor_t derivedBnDesc, const cuinferTensorDescriptor_t xDesc, cuinferBatchNormMode_t mode); typedef enum { CUINFER_BATCHNORM_OPS_BN = 0, ///< Do batch normalization only. CUINFER_BATCHNORM_OPS_BN_ACTIVATION = 1, ///< Do batchNorm, then activation. CUINFER_BATCHNORM_OPS_BN_ADD_ACTIVATION = 2, ///< Do batchNorm, then elemWiseAdd, then activation. } cuinferBatchNormOps_t; /// @brief /// @param[in] handle The libinfer handle. /// @param[in] mode /// @param[in] bnOps /// @param[in] xDesc The discriptor of input tensor x. /// @param[in] zDesc /// @param[in] yDesc The discriptor of tensor y. /// @param[in] bnScaleBiasMeanVarDesc /// @param[in] activationDesc /// @param[out] sizeInBytes The result extra temporary space size in bytes. /// @return cuinferStatus_t CUINFERWINAPI cuinferGetBatchNormalizationForwardTrainingExWorkspaceSize( cuinferHandle_t handle, cuinferBatchNormMode_t mode, cuinferBatchNormOps_t bnOps, const cuinferTensorDescriptor_t xDesc, const cuinferTensorDescriptor_t zDesc, const cuinferTensorDescriptor_t yDesc, const cuinferTensorDescriptor_t bnScaleBiasMeanVarDesc, const cuinferActivationDescriptor_t activationDesc, size_t *sizeInBytes); /// @brief /// @param[in] handle The libinfer handle. /// @param[in] mode /// @param[in] bnOps /// @param[in] activationDesc /// @param[in] xDesc The discriptor of input tensor x. /// @param[out] sizeInBytes The result extra temporary space size in bytes. /// @return cuinferStatus_t CUINFERWINAPI cuinferGetBatchNormalizationTrainingExReserveSpaceSize( cuinferHandle_t handle, cuinferBatchNormMode_t mode, cuinferBatchNormOps_t bnOps, const cuinferActivationDescriptor_t activationDesc, const cuinferTensorDescriptor_t xDesc, size_t *sizeInBytes); /// @brief /// @details Computes y = BN(x). Also accumulates moving averages of mean and /// inverse variances. /// /// 'Gamma'(\p bnScale) and 'Beta'(\p bnBias) respectively in Ioffe and /// Szegedy's paper's notation. /// /// MUST use factor=1 in the very first call of a complete training cycle. /// Use a factor=1/(1+n) at N-th call to the function to get Cumulative Moving /// Average (CMA) behavior \f( \mathrm{CMA|[n] = (x[1]+...+x[n])/n \f) Since /// \f{eqnarray*}{ /// \mathrm{CMA}[n+1] &=& (n*\mathrm{CMA}[n]+x[n+1])/(n+1) \\\\ /// &=& ((n+1)*\mathrm{CMA}[n]-\mathrm{CMA}[n])/(n+1) + x[n+1]/(n+1) \\\\ /// &=& \mathrm{CMA}[n]*(1-1/(n+1)) + x[n+1]*1/(n+1) /// \f}. /// /// Shared desc for the next 6 tensors in the argument list. \p bnScale, \p /// bnBias, \p resultRunningMean, \p resultRunningVariance, \p resultSaveMean /// and \p resultSaveInvVariance. /// * Data type to be set as follows: type = (typeOf(x) == double) /// ? double : float Dimensions for this descriptor depend on normalization mode /// * Spatial Normalization : tensors are expected to have dims /// 1xCx1x1 (normalization is performed across NxHxW) /// * Per-Activation Normalization : tensors are expected to have dims of /// 1xCxHxW (normalization is performed across N) /// @param[in] handle The libinfer handle. /// @param[in] mode /// @param[in] alpha alpha[0] = result blend factor. /// @param[in] beta beta[0] = dest layer blend factor /// @param[in] xDesc The discriptor of input tensor x. /// @param[in] x NxCxHxW /// @param[in] yDesc The discriptor of tensor y. /// @param[in] y NxCxHxW /// @param[in] bnScaleBiasMeanVarDesc /// @param[in] bnScale /// @param[in] bnBias /// @param[in] exponentialAverageFactor /// @param[out] resultRunningMean Used in Training phase only. runningMean = /// newMean*factor + runningMean*(1-factor). /// @param[out] resultRunningVariance Output in training mode, input in /// inference. Is the moving average of variance[x] (factor is applied in the /// same way as for runningMean). /// @param[in] epsilon Has to be >= CUINFER_BN_MIN_EPSILON. Should be the same /// in forward and backward functions. /// @param[out] resultSaveMean Optionally save intermediate results from the /// forward pass here - can be reused to speed up backward pass. NULL if unused /// @param[out] resultSaveInvVariance /// @return cuinferStatus_t CUINFERWINAPI cuinferBatchNormalizationForwardTraining( cuinferHandle_t handle, cuinferBatchNormMode_t mode, const void *alpha, const void *beta, const cuinferTensorDescriptor_t xDesc, const void *x, const cuinferTensorDescriptor_t yDesc, void *y, const cuinferTensorDescriptor_t bnScaleBiasMeanVarDesc, const void *bnScale, const void *bnBias, double exponentialAverageFactor, void *resultRunningMean, void *resultRunningVariance, double epsilon, void *resultSaveMean, void *resultSaveInvVariance); /// Computes y = relu(BN(x) + z). Also accumulates moving averages of mean and /// inverse variances /// @brief /// @param[in] handle The libinfer handle. /// @param[in] mode /// @param[in] bnOps /// @param[in] alpha alpha[0] = result blend factor. /// @param[in] beta beta[0] = dest layer blend factor /// @param[in] xDesc The discriptor of input tensor x. /// @param[in] xData /// @param[in] zDesc /// @param[in] zData /// @param[in] yDesc The discriptor of tensor y. /// @param[in] yData /// @param[in] bnScaleBiasMeanVarDesc /// @param[in] bnScale /// @param[in] bnBias /// @param[in] exponentialAverageFactor /// @param[out] resultRunningMean /// @param[out] resultRunningVariance /// @param[in] epsilon Has to be >= CUINFER_BN_MIN_EPSILON. Should be the same /// in forward and backward functions. /// @param[out] resultSaveMean Optionally save intermediate results from the /// forward pass here - can be reused to speed up backward pass. NULL if unused. /// @param[out] resultSaveInvVariance /// @param[in] activationDesc /// @param[in] workSpace The workspace pre-allocated. See the corresponding get /// workspace size helper function. /// @param[in] workSpaceSizeInBytes /// @param[out] reserveSpace /// @param[out] reserveSpaceSizeInBytes /// @return cuinferStatus_t CUINFERWINAPI cuinferBatchNormalizationForwardTrainingEx( cuinferHandle_t handle, cuinferBatchNormMode_t mode, cuinferBatchNormOps_t bnOps, const void *alpha, const void *beta, const cuinferTensorDescriptor_t xDesc, const void *xData, const cuinferTensorDescriptor_t zDesc, const void *zData, const cuinferTensorDescriptor_t yDesc, void *yData, const cuinferTensorDescriptor_t bnScaleBiasMeanVarDesc, const void *bnScale, const void *bnBias, double exponentialAverageFactor, void *resultRunningMean, void *resultRunningVariance, double epsilon, void *resultSaveMean, void *resultSaveInvVariance, cuinferActivationDescriptor_t activationDesc, void *workspace, size_t workSpaceSizeInBytes, void *reserveSpace, size_t reserveSpaceSizeInBytes); /// @brief Performs Batch Normalization during Inference: /// @details y[i] = bnScale[k] * (x[i] - estimatedMean[k]) / sqrt(epsilon + /// estimatedVariance[k]) + bnBias[k] with bnScale, bnBias, runningMean, /// runningInvVariance tensors indexed according to spatial or per-activation /// mode. Refer to cuinferBatchNormalizationForwardTraining above for notes on /// function arguments. /// @param[in] handle The libinfer handle. /// @param[in] mode /// @param[in] alpha alpha[0] = result blend factor /// @param[in] beta beta[0] = dest layer blend factor /// @param[in] xDesc The discriptor of input tensor x. /// @param[in] x NxCxHxW /// @param[in] yDesc The discriptor of tensor y. /// @param[out] y NxCxHxW /// @param[in] bnScaleBiasMeanVarDesc /// @param[in] bnScale /// @param[in] bnBias /// @param[in] estimatedMean /// @param[in] estimatedVariance /// @param[in] epsilon /// @return cuinferStatus_t CUINFERWINAPI cuinferBatchNormalizationForwardInference( cuinferHandle_t handle, cuinferBatchNormMode_t mode, const void *alpha, const void *beta, const cuinferTensorDescriptor_t xDesc, const void *x, const cuinferTensorDescriptor_t yDesc, void *y, const cuinferTensorDescriptor_t bnScaleBiasMeanVarDesc, const void *bnScale, const void *bnBias, const void *estimatedMean, const void *estimatedVariance, double epsilon); /// @defgroup SpatialTransformer Spatial Transform Apis /// @note APIs for spatial transformer network typedef struct cuinferDropoutStruct *cuinferDropoutDescriptor_t; /// @brief /// @param[out] dropoutDesc /// @return cuinferStatus_t CUINFERWINAPI cuinferCreateDropoutDescriptor(cuinferDropoutDescriptor_t *dropoutDesc); /// @brief /// @param[in] dropoutDesc /// @return cuinferStatus_t CUINFERWINAPI cuinferDestroyDropoutDescriptor(cuinferDropoutDescriptor_t dropoutDesc); /// @brief Helper function to determine size of the states to be passed to /// LibinferSetDropoutDescriptor. /// @param[in] handle The libinfer handle. /// @param[out] sizeInBytes The result extra temporary space size in bytes. /// @return cuinferStatus_t CUINFERWINAPI cuinferDropoutGetStatesSize(cuinferHandle_t handle, size_t *sizeInBytes); /// @brief helper function to determine size of the reserve space to be passed /// to dropout forward/backward calls. /// @param[in] xDesc The discriptor of input tensor x. /// @param[out] sizeInBytes The result extra temporary space size in bytes. /// @return cuinferStatus_t CUINFERWINAPI cuinferDropoutGetReserveSpaceSize( cuinferTensorDescriptor_t xdesc, size_t *sizeInBytes); /// @brief /// @param[in] dropoutDesc /// @param[in] handle The libinfer handle. /// @param[in] dropout /// @param[out] states /// @param[in] stateSizeInBytes /// @param[in] seed /// @return cuinferStatus_t CUINFERWINAPI cuinferSetDropoutDescriptor(cuinferDropoutDescriptor_t dropoutDesc, cuinferHandle_t handle, float dropout, void *states, size_t stateSizeInBytes, unsigned long long seed); /// @brief Restores the dropout descriptor to a previously saved-off state /// @param dropoutDesc /// @param handle /// @param dropout /// @param states /// @param stateSizeInBytes /// @param seed /// @return cuinferStatus_t CUINFERWINAPI cuinferRestoreDropoutDescriptor( cuinferDropoutDescriptor_t dropoutDesc, cuinferHandle_t handle, float dropout, void *states, size_t stateSizeInBytes, unsigned long long seed); /// @brief /// @param[in] dropoutDesc /// @param[in] handle The libinfer handle. /// @param[out] dropout /// @param[out] states /// @param[out] seed /// @return cuinferStatus_t CUINFERWINAPI cuinferGetDropoutDescriptor( cuinferDropoutDescriptor_t dropoutDesc, cuinferHandle_t handle, float *dropout, void **states, unsigned long long *seed); /// @brief /// @param[in] handle The libinfer handle. /// @param[in] dropoutDesc /// @param[in] xDesc The discriptor of input tensor x. /// @param[in] x Const pointer to input tensor x. /// @param[in] yDesc The discriptor of tensor y. /// @param[out] y The discriptor of output tensor y. /// @param[in] reserveSpace /// @param[in] reserveSpaceSizeInBytes /// @return cuinferStatus_t CUINFERWINAPI cuinferDropoutForward( cuinferHandle_t handle, const cuinferDropoutDescriptor_t dropoutDesc, const cuinferTensorDescriptor_t xDesc, const void *x, const cuinferTensorDescriptor_t yDesc, void *y, void *reserveSpace, size_t reserveSpaceSizeInBytes); /// @defgroup BasicRNNAPIs Basic RNN APIs /// @ingroup BasicRNNAPIs typedef enum { CUINFER_RNN_ALGO_STANDARD = 0, CUINFER_RNN_ALGO_PERSIST_STATIC = 1, CUINFER_RNN_ALGO_PERSIST_DYNAMIC = 2, CUINFER_RNN_ALGO_COUNT = 3, } cuinferRNNAlgo_t; /// @ingroup BasicRNNAPIs typedef enum { CUINFER_RNN_RELU = 0, ///< Basic RNN cell type with ReLu activation. CUINFER_RNN_TANH = 1, ///< Basic RNN cell type with tanh activation. CUINFER_LSTM = 2, ///< LSTM with no peephole connections. CUINFER_GRU = 3, ///< Using h' = tanh(r * Uh(t-1) + Wx) and h = (1 - z) * h' + ///< z * h(t-1); } cuinferRNNMode_t; /// @ingroup BasicRNNAPIs typedef enum { CUINFER_UNIDIRECTIONAL = 0, ///< Aingle direction network. CUINFER_BIDIRECTIONAL = 1, ///< Output concatination at each layer. } cuinferDirectionMode_t; /// @ingroup BasicRNNAPIs typedef enum { CUINFER_LINEAR_INPUT = 0, ///< Adjustable weight matrix in first layer input GEMM. CUINFER_SKIP_INPUT = 1, ///< Fixed identity matrix in the first layer input GEMM. } cuinferRNNInputMode_t; /// @ingroup BasicRNNAPIs struct cuinferRNNStruct; /// @ingroup BasicRNNAPIs typedef struct cuinferRNNStruct *cuinferRNNDescriptor_t; /// @ingroup BasicRNNAPIs struct cuinferPersistentRNNPlan; /// @ingroup BasicRNNAPIs typedef struct cuinferPersistentRNNPlan *cuinferPersistentRNNPlan_t; /// @brief /// @ingroup BasicRNNAPIs /// @param[out] rnnDesc /// @return cuinferStatus_t CUINFERWINAPI cuinferCreateRNNDescriptor(cuinferRNNDescriptor_t *rnnDesc); /// @brief /// @ingroup BasicRNNAPIs /// @param[in] rnnDesc /// @return cuinferStatus_t CUINFERWINAPI cuinferDestroyRNNDescriptor(cuinferRNNDescriptor_t rnnDesc); /// @brief /// @details \p dataType in weight descriptors and input descriptors is used to /// describe data/parameter storage. Dropout is between RNN layers, not between /// recurrent steps. /// @ingroup BasicRNNAPIs /// @param handle /// @param rnnDesc /// @param hiddenSize /// @param numLayers /// @param dropoutDesc /// @param inputMode /// @param direction /// @param mode /// @param algo /// @param mathPrec In the RNN descriptor is determines compute math precision, /// modified by ::cuinferMathType_t. /// @return cuinferStatus_t CUINFERWINAPI cuinferSetRNNDescriptor( cuinferHandle_t handle, cuinferRNNDescriptor_t rnnDesc, const int hiddenSize, const int numLayers, cuinferDropoutDescriptor_t dropoutDesc, cuinferRNNInputMode_t inputMode, cuinferDirectionMode_t direction, cuinferRNNMode_t mode, cuinferRNNAlgo_t algo, cuinferDataType_t mathPrec); /// @brief /// @ingroup BasicRNNAPIs /// @param[in] handle The libinfer handle. /// @param[in] rnnDesc /// @param[out] hiddenSize /// @param[out] numLayers /// @param[out] dropoutDesc /// @param[out] inputMode /// @param[out] direction /// @param[out] mode /// @param[out] algo /// @param[out] mathPrec /// @return cuinferStatus_t CUINFERWINAPI cuinferGetRNNDescriptor( cuinferHandle_t handle, cuinferRNNDescriptor_t rnnDesc, int *hiddenSize, int *numLayers, cuinferDropoutDescriptor_t *dropoutDesc, cuinferRNNInputMode_t *inputMode, cuinferDirectionMode_t *direction, cuinferRNNMode_t *mode, cuinferRNNAlgo_t *algo, cuinferDataType_t *mathPrec); /// @brief /// @ingroup BasicRNNAPIs /// @param[out] rnnDesc /// @param[in] mType /// @return cuinferStatus_t CUINFERWINAPI cuinferSetRNNMatrixMathType( cuinferRNNDescriptor_t rnnDesc, cuinferMathType_t mType); /// @brief /// @ingroup BasicRNNAPIs /// @param[in] rnnDesc /// @param[out] mType /// @return cuinferStatus_t CUINFERWINAPI cuinferGetRNNMatrixMathType( cuinferRNNDescriptor_t rnnDesc, cuinferMathType_t *mType); /// @brief /// @ingroup BasicRNNAPIs /// @param[in] handle The libinfer handle. /// @param[out] rnnDesc /// @param[in] recProjSize /// @param[in] outProjSize /// @return cuinferStatus_t CUINFERWINAPI cuinferSetRNNProjectionLayers( cuinferHandle_t handle, cuinferRNNDescriptor_t rnnDesc, const int recProjSize, const int outProjSize); /// @brief /// @ingroup BasicRNNAPIs /// @param[in] handle The libinfer handle. /// @param[in] rnnDesc /// @param[out] recProjSize /// @param[out] outProjSize /// @return cuinferStatus_t CUINFERWINAPI cuinferGetRNNProjectionLayers( cuinferHandle_t handle, const cuinferRNNDescriptor_t rnnDesc, int *recProjSize, int *outProjSize); /// @brief /// @ingroup BasicRNNAPIs /// @note Expensive. Creates the plan for the specific settings. /// @param[in] rnnDesc /// @param[in] minibatch /// @param[in] dataType /// @param[out] plan /// @return cuinferStatus_t CUINFERWINAPI cuinferCreatePersistentRNNPlan( cuinferRNNDescriptor_t rnnDesc, const int minibatch, const cuinferDataType_t dataType, cuinferPersistentRNNPlan_t *plan); /// @brief /// @ingroup BasicRNNAPIs /// @param[in] plan /// @return cuinferStatus_t CUINFERWINAPI cuinferDestroyPersistentRNNPlan(cuinferPersistentRNNPlan_t plan); /// @brief /// @ingroup BasicRNNAPIs /// @param[in] rnnDesc /// @param[out] plan /// @return cuinferStatus_t CUINFERWINAPI cuinferSetPersistentRNNPlan( cuinferRNNDescriptor_t rnnDesc, cuinferPersistentRNNPlan_t plan); /// @brief /// @ingroup BasicRNNAPIs /// @param[in] handle The libinfer handle. /// @param[in] rnnDesc /// @param[out] seqLength /// @param[out] xDesc /// @param[out] sizeInBytes The result extra temporary space size in bytes. /// @return cuinferStatus_t CUINFERWINAPI cuinferGetRNNTrainingReserveSize( cuinferHandle_t handle, const cuinferRNNDescriptor_t rnnDesc, const int seqLength, const cuinferTensorDescriptor_t *xDesc, size_t *sizeInBytes); /// @brief /// @ingroup BasicRNNAPIs /// @param[in] handle The libinfer handle. /// @param[in] rnnDesc /// @param[out] xDesc /// @param[out] sizeInBytes The result extra temporary space size in bytes. /// @param[out] dataType /// @return cuinferStatus_t CUINFERWINAPI cuinferGetRNNParamsSize( cuinferHandle_t handle, const cuinferRNNDescriptor_t rnnDesc, const cuinferTensorDescriptor_t xDesc, size_t *sizeInBytes, cuinferDataType_t dataType); /// @brief /// @ingroup BasicRNNAPIs /// @param[in] handle The libinfer handle. /// @param[in] rnnDesc /// @param[out] pseudoLayer /// @param[out] xDesc /// @param[out] wDesc /// @param[out] w /// @param[out] linLayerID /// @param[out] linLayerMatDesc /// @param[out] linLayerMat /// @return cuinferStatus_t CUINFERWINAPI cuinferGetRNNLinLayerMatrixParams( cuinferHandle_t handle, const cuinferRNNDescriptor_t rnnDesc, const int pseudoLayer, const cuinferTensorDescriptor_t xDesc, const cuinferFilterDescriptor_t wDesc, const void *w, const int linLayerID, cuinferFilterDescriptor_t linLayerMatDesc, void **linLayerMat); /// @brief /// @ingroup BasicRNNAPIs /// @param[in] handle The libinfer handle. /// @param[in] rnnDesc /// @param[out] pseudoLayer /// @param[out] xDesc /// @param[out] wDesc /// @param[out] w /// @param[out] linLayerID /// @param[out] linLayerBiasDesc /// @param[out] linLayerBias /// @return cuinferStatus_t CUINFERWINAPI cuinferGetRNNLinLayerBiasParams( cuinferHandle_t handle, const cuinferRNNDescriptor_t rnnDesc, const int pseudoLayer, const cuinferTensorDescriptor_t xDesc, const cuinferFilterDescriptor_t wDesc, const void *w, const int linLayerID, cuinferFilterDescriptor_t linLayerBiasDesc, void **linLayerBias); /// @brief /// @ingroup BasicRNNAPIs /// @param[in] handle The libinfer handle. /// @param[in] rnnDesc /// @param[in] seqLength /// @param[in] xDesc The discriptor of input tensor x. /// @param[in] x Const pointer to input tensor x. /// @param[in] hxDesc /// @param[in] hx /// @param[in] cxDesc /// @param[in] cx /// @param[in] wDesc The discriptor of filter w. /// @param[in] The const pointer of input filter w. /// @param[in] yDesc The discriptor of tensor y. /// @param[out] y The discriptor of output tensor y. /// @param[in] hyDesc /// @param[out] hy /// @param[in] cyDesc /// @param[out] cy /// @param[in] workSpace The workspace pre-allocated. See the corresponding get /// workspace size helper function. /// @param[in] workSpaceSizeInBytes /// @return cuinferStatus_t CUINFERWINAPI cuinferRNNForwardInference( cuinferHandle_t handle, const cuinferRNNDescriptor_t rnnDesc, const int seqLength, const cuinferTensorDescriptor_t *xDesc, const void *x, const cuinferTensorDescriptor_t hxDesc, const void *hx, const cuinferTensorDescriptor_t cxDesc, const void *cx, const cuinferFilterDescriptor_t wDesc, const void *w, const cuinferTensorDescriptor_t *yDesc, void *y, const cuinferTensorDescriptor_t hyDesc, void *hy, const cuinferTensorDescriptor_t cyDesc, void *cy, void *workspace, size_t workSpaceSizeInBytes); /// @brief /// @ingroup BasicRNNAPIs /// @param[in] handle The libinfer handle. /// @param[in] rnnDesc /// @param[in] seqLength /// @param[in] xDesc The discriptor of input tensor x. /// @param[in] x Const pointer to input tensor x. /// @param[in] hxDesc /// @param[in] hx /// @param[in] cxDesc /// @param[in] cx /// @param[in] wDesc The discriptor of filter w. /// @param[in] The const pointer of input filter w. /// @param[in] yDesc The discriptor of tensor y. /// @param[out] y The discriptor of output tensor y. /// @param[in] hyDesc /// @param[out] hy /// @param[in] cyDesc /// @param[out] cy /// @param[in] workSpace The workspace pre-allocated. See the corresponding get /// workspace size helper function. /// @param[in] workSpaceSizeInBytes /// @param[in] reserveSpace /// @param[in] reserveSpaceSizeInBytes /// @return cuinferStatus_t CUINFERWINAPI cuinferRNNForwardTraining( cuinferHandle_t handle, const cuinferRNNDescriptor_t rnnDesc, const int seqLength, const cuinferTensorDescriptor_t *xDesc, const void *x, const cuinferTensorDescriptor_t hxDesc, const void *hx, const cuinferTensorDescriptor_t cxDesc, const void *cx, const cuinferFilterDescriptor_t wDesc, const void *w, const cuinferTensorDescriptor_t *yDesc, void *y, const cuinferTensorDescriptor_t hyDesc, void *hy, const cuinferTensorDescriptor_t cyDesc, void *cy, void *workspace, size_t workSpaceSizeInBytes, void *reserveSpace, size_t reserveSpaceSizeInBytes); /// CTC LOSS typedef enum { CUINFER_CTC_LOSS_ALGO_DETERMINISTIC = 0, CUINFER_CTC_LOSS_ALGO_NON_DETERMINISTIC = 1 } cuinferCTCLossAlgo_t; /// Input normalization mode for loss function typedef enum { CUINFER_LOSS_NORMALIZATION_NONE = 0, CUINFER_LOSS_NORMALIZATION_SOFTMAX = 1 } cuinferLossNormalizationMode_t; /// CTC (Connectionist Temporal Classification) loss descriptor /// create/destory/set/get functions cuinferStatus_t CUINFERWINAPI cuinferCreateCTCLossDescriptor(cuinferCTCLossDescriptor_t *ctcLossDesc); /// @brief /// @param[out] ctcLossDesc /// @param[in] compType /// @return cuinferStatus_t CUINFERWINAPI cuinferSetCTCLossDescriptor( cuinferCTCLossDescriptor_t ctcLossDesc, cuinferDataType_t compType); /// @brief /// @param[out] ctcLossDesc /// @param[in] compType /// @param[in] normMode /// @param[in] gradMode /// @return cuinferStatus_t CUINFERWINAPI cuinferSetCTCLossDescriptorEx( cuinferCTCLossDescriptor_t ctcLossDesc, cuinferDataType_t compType, cuinferLossNormalizationMode_t normMode, cuinferNanPropagation_t gradMode); /// @brief /// @param[out] ctcLossDesc /// @param[in] compType /// @return cuinferStatus_t CUINFERWINAPI cuinferGetCTCLossDescriptor( cuinferCTCLossDescriptor_t ctcLossDesc, cuinferDataType_t *compType); /// @brief /// @param[out] ctcLossDesc /// @param[in] compType /// @param[in] normMode /// @param[in] gradMode /// @return cuinferStatus_t CUINFERWINAPI cuinferGetCTCLossDescriptorEx( cuinferCTCLossDescriptor_t ctcLossDesc, cuinferDataType_t *compType, cuinferLossNormalizationMode_t *normMode, cuinferNanPropagation_t *gradMode); /// @brief /// @param[in] ctcLossDesc /// @return cuinferStatus_t CUINFERWINAPI cuinferDestroyCTCLossDescriptor(cuinferCTCLossDescriptor_t ctcLossDesc); /// @brief Return the ctc costs and gradients, given the probabilities and /// labels. /// @param[in] handle The libinfer handle. /// @param[in] probsDesc Tensor descriptor for probabilities, the dimensions are /// T,N,A (T is the timing steps, N is the mini batch size, A is the alphabet /// size). /// @param[in] probs Probabilities after softmax, in GPU memory. /// @param[in] labels Labels, in CPU memory. /// @param[in] labelLengths The length of each label, in CPU memory. /// @param[in] inputLengths The lengths of timing steps in each batch, in CPU /// memory. /// @param[out] costs The returned costs of CTC, in GPU memory. /// @param[in] gradientsDesc Tensor descriptor for gradients, the dimensions /// are T,N,A. /// @param[out] gradients The returned CTC gradients, in GPU memory, to compute /// costs only, set it to NULL. /// @param[in] algo Algorithm selected, supported now 0 and 1. /// @param[in] ctcLossDesc /// @param[in] workspace Pointer to the workspace, in GPU memory. /// @param[in] workSpaceSizeInBytes Size of the workspace. /// @return cuinferStatus_t CUINFERWINAPI cuinferCTCLoss( cuinferHandle_t handle, const cuinferTensorDescriptor_t probsDesc, const void *probs, const int *labels, const int *labelLengths, const int *inputLengths, void *costs, const cuinferTensorDescriptor_t gradientsDesc, void *gradients, cuinferCTCLossAlgo_t algo, cuinferCTCLossDescriptor_t ctcLossDesc, void *workspace, size_t workSpaceSizeInBytes); /// return the workspace size needed for ctc /// @brief /// @param[in] handle The libinfer handle. /// @param[in] probsDesc Tensor descriptor for probabilities, the dimensions are /// T,N,A (T is the timing steps, N is the mini batch size, A is the alphabet /// size). /// @param[in] gradientsDesc Tensor descriptor for gradients, the dimensions are /// T,N,A. To compute costs only, set it to nullptr. /// @param[in] labels labels, in CPU memory /// @param[in] labelLengths The length of each label, in CPU memory /// @param[in] inputLengths The lengths of timing steps in each batch, in CPU /// memory /// @param[in] algo The algorithm selected. Algo 0 and 1 are supported for now. /// @param[in] ctcLossDesc /// @param[out] sizeInBytes pointer to the returned workspace size /// @return cuinferStatus_t CUINFERWINAPI cuinferGetCTCLossWorkspaceSize( cuinferHandle_t handle, const cuinferTensorDescriptor_t probsDesc, const cuinferTensorDescriptor_t gradientsDesc, const int *labels, const int *labelLengths, const int *inputLengths, cuinferCTCLossAlgo_t algo, cuinferCTCLossDescriptor_t ctcLossDesc, size_t *sizeInBytes); typedef struct { union Algorithm { cuinferConvolutionFwdAlgo_t convFwdAlgo; cuinferConvolutionBwdFilterAlgo_t convBwdFilterAlgo; cuinferConvolutionBwdDataAlgo_t convBwdDataAlgo; cuinferRNNAlgo_t RNNAlgo; cuinferCTCLossAlgo_t CTCLossAlgo; } algo; } cuinferAlgorithm_t; /// Struct containing useful informaiton for each API call. typedef struct { unsigned cuinfer_version; cuinferStatus_t cuinferStatus; unsigned time_sec; ///< Epoch time in seconds. unsigned time_usec; ///< Microseconds part of epoch time. unsigned time_delta; ///< time since start in seconds. cuinferHandle_t handle; ///< Cuinfer handle. cudaStream_t stream; ///< Cuda stream ID. unsigned long long pid; ///< Process ID. unsigned long long tid; ///< Thread ID. int cudaDeviceId; ///< CUDA device ID. int reserved[15]; ///< Reserved for future use. } cuinferDebug_t; /// @defgroup BertBaseInt8TransformerFunctions Bert Base Int8 Transformer /// Functions /// @brief /// @ingroup BertBaseInt8TransformerFunctions /// @param[in] token_emb /// @param[in] pos_emb /// @param[in] tokens /// @param[out] output /// @param[out] pad_mask /// @param[in] pad_id /// @param[in] batch_size /// @param[in] seq_len /// @param[in] hidden_dim /// @param[in] stream /// @param[in] lang_emb /// @param[in] lang_id /// @param[in] multilg_type /// @param[in] dequant_scale /// @param[in] scaled /// @return * CUINFER_STATUS_SUCCESS cuinferStatus_t cuinferEncEmbI8I(const void *token_emb, const void *pos_emb, const void *tokens, void *output, void *pad_mask, int pad_id, int batch_size, int seq_len, int hidden_dim, cudaStream_t stream, const void *lang_emb, const void *lang_id, int multilg_type, float dequant_scale, bool scaled); /// @brief /// @details Description: from ixrt cuinferEncEmbI8I, /// and the pad_mask is int32 instead of int8 from previous interface. /// /// Params Mapping: /// | src | dst | /// |---------------|----------------| /// | token_emb | token_emb | /// | pos_emb | pos_emb | /// | tokens | tokens | /// | output | output | /// | pad_mask | pad_masktokens | /// | pad_id | pad_id | /// | batch_size | batch_size | /// | seq_len | seq_len | /// | hidden_dim | hidden_dim | /// | stream | stream | /// | lang_emb | lang_emb | /// | lang_id | lang_id | /// | multilg_type | multilg_type | /// | dequant_scale | dequant_scale | /// | scaled | scaled | /// @ingroup BertBaseInt8TransformerFunctions /// @param[in] token_emb /// @param[in] pos_emb /// @param[in] tokens /// @param[out] output /// @param[out] pad_mask /// @param[in] pad_id /// @param[in] batch_size /// @param[in] seq_len /// @param[in] hidden_dim /// @param[in] stream /// @param[in] lang_emb /// @param[in] lang_id /// @param[in] multilg_type /// @param[in] dequant_scale /// @param[in] scaled /// @return * CUINFER_STATUS_SUCCESS cuinferStatus_t cuinferEncEmbI8I_M8I(const void *token_emb, const void *pos_emb, const void *tokens, void *output, void *pad_mask, int pad_id, int batch_size, int seq_len, int hidden_dim, cudaStream_t stream, const void *lang_emb, const void *lang_id, int multilg_type, float dequant_scale, bool scaled); /// @brief /// @ingroup BertBaseInt8TransformerFunctions /// @param[in] token_num /// @param[in] hidden_size /// @param[in] stream /// @param[in, out] input /// @param[out] output /// @param[in] scale /// @param[in] bias /// @param[in] residual_bias /// @param[in] quant_scale /// @param[in] is_post_ln /// @param[in] out_col32 /// @return * CUINFER_STATUS_SUCCESS cuinferStatus_t cuinferLayernormResualI8O(int token_num, int hidden_size, cudaStream_t stream, void *input, void *output, const void *scale, const void *bias, const void *residual_bias, float quant_scale, bool is_post_ln, bool out_col32); /// @brief /// @ingroup BertBaseInt8TransformerFunctions /// @param[in] batch_token_num /// @param[in] hidden_size /// @param[in] stream /// @param[in] ori_qkv /// @param[in] qkv_bias /// @param[out] new_qkv /// @param[in] max_batch_dim /// @param[in] batch_seq_len /// @param[in] dim_per_head /// @param[in] head_num /// @param[in] quant_scale /// @param[in] dequant_scale /// @param[in] in_col32 /// @return * CUINFER_STATUS_SUCCESS cuinferStatus_t cuinferArrangeEncselfQkvI8II8O( int batch_token_num, int hidden_size, cudaStream_t stream, const void *ori_qkv, const void *qkv_bias, void *new_qkv, int max_batch_dim, int batch_seq_len, int dim_per_head, int head_num, float quant_scale, float dequant_scale, bool in_col32); /// @brief /// @ingroup BertBaseInt8TransformerFunctions /// @param[in] batch_size /// @param[in] batch_seq_len /// @param[in] head_num /// @param[in] stream /// @param[out] correlation /// @param[in] src_padding_mask /// @param[out] outputs /// @param[in] quant_scale /// @param[in] dequant_scale /// @return * CUINFER_STATUS_SUCCESS cuinferStatus_t cuinferCorrelationSoftmaxEncselfI32II8O( int batch_size, int batch_seq_len, int head_num, cudaStream_t stream, void *correlation, const void *src_padding_mask, void *outputs, float quant_scale, float dequant_scale); /// @brief /// @details Description: from ixrt IxinferCorrelationSoftmaxEncselfI8II8O /// seperate correlation's input and output from inplace algorithm. /// /// Params Mapping: /// | src | dst | /// |------------------|------------------| /// | batch_size | batch_size | /// | batch_seq_len | batch_seq_len | /// | head_num | head_num | /// | stream | stream | /// | correlation | correlation | /// | src_padding_mask | src_padding_mask | /// | outputs | correlation | /// | quant_scale | quant_scale | /// | dequant_scale | dequant_scale | /// @ingroup BertBaseInt8TransformerFunctions /// @param[in] batch_size /// @param[in] batch_seq_len /// @param[in] head_num /// @param[in] stream /// @param[out] correlation /// @param[in] src_padding_mask /// @param[out] outputs /// @param[in] quant_scale /// @param[in] dequant_scale /// @return * CUINFER_STATUS_SUCCESS cuinferStatus_t cuinferCorrelationSoftmaxEncselfI8II8O( int batch_size, int batch_seq_len, int head_num, cudaStream_t stream, void *correlation, const void *src_padding_mask, void *outputs, float quant_scale, float dequant_scale); /// @brief /// @details Description: from ixrt IxinferArrangeAttenOutputI8II8O /// defalt \p max_thread_per_block to 1024. /// /// Params Mapping: /// | src | dst | /// |-----------------|----------------------| /// | batch_token_num | batch_token_num | /// | hidden_size | hidden_size | /// | stream | stream | /// | ori_q | ori_q | /// | new_q | new_q | /// | beam_size | beam_size | /// | dim_per_head | dim_per_head | /// | head_num | head_num | /// | 1024 | max_thread_per_block | /// | quant_scale | quant_scale | /// | dequant_scale | dequant_scale | /// | out_col32 | | /// @ingroup BertBaseInt8TransformerFunctions /// @param[in] batch_token_num /// @param[in] hidden_size /// @param[in] stream /// @param[in] ori_q /// @param[out] new_q /// @param[in] beam_size /// @param[in] dim_per_head /// @param[in] head_num /// @param[in] quant_scale /// @param[in] dequant_scale /// @param[in] out_col32 /// @return /// * ::CUINFER_STATUS_SUCCESS cuinferStatus_t cuinferArrangeAttenOutputI8II8O( int batch_token_num, int hidden_size, cudaStream_t stream, const void *ori_q, void *new_q, int beam_size, int dim_per_head, int head_num, float quant_scale, float dequant_scale, bool out_col32); /// @brief /// @ingroup BertBaseInt8TransformerFunctions /// @details Description: from ixrt IxinferLnResidualI8I /// /// Params Mapping: /// | src | dst | /// |---------------|---------------| /// | input | input | /// | scale | scale | /// | bias | bias | /// | residual | residual | /// | output | output | /// | batch_tokens | batch_tokens | /// | hidden_size | hidden_size | /// | dequant_scale | dequant_scale | /// | stream | stream | /// @param[in] input /// @param[in] scale /// @param[in] bias /// @param[in] residual /// @param[out] output /// @param[in] batch_tokens /// @param[in] hidden_size /// @param[in] dequant_scale /// @param[in] stream /// @return /// * ::CUINFER_STATUS_SUCCESS cuinferStatus_t cuinferResidualBiaslnI8I(const void *input, const void *scale, const void *bias, const void *residual, void *output, int batch_tokens, int hidden_size, float dequant_scale, cudaStream_t stream); /// @brief /// @ingroup BertBaseInt8TransformerFunctions /// @param[in] input /// @param[in] scale /// @param[in] bias /// @param[in] residual_bias /// @param[out] output /// @param[out] residual /// @param[in] batch_tokens /// @param[in] hidden_size /// @param[in] dequant_scale /// @param[in] quant_scale /// @param[in] stream /// @param[in] is_post_ln /// @param[in] in_col32 /// @param[in] out_col32 /// @param[in] colsum /// @return /// * ::CUINFER_STATUS_SUCCESS cuinferStatus_t cuinferResidualBiasLnI8II8O( const void *input, const void *scale, const void *bias, const void *residual_bias, void *output, void *residual, int batch_tokens, int hidden_size, float dequant_scale, float quant_scale, cudaStream_t stream, bool is_post_ln, bool in_col32, bool out_col32, const void *colsum = nullptr); /// @brief /// @ingroup BertBaseInt8TransformerFunctions /// @details Description: from ixrt IxinferResidualBiasLnI8II8O /// residual_out is write to residual and make it inplace. /// /// Param Mappings: /// | src | dst | /// |---------------|----------------------| /// | input | input | /// | scale | scale | /// | bias | bias | /// | residual_bias | residual_bias | /// | output | output | /// | residual | residual | /// | residual_out | residual | /// | batch_tokens | batch_tokens | /// | hidden_size | hidden_size | /// | dequant_scale | dequant_scale | /// | quant_scale | quant_scale | /// | 1024 | max_thread_per_block | /// | stream | stream | /// | is_post_ln | is_post_ln | /// | colsum | colsum | /// @param[in] input /// @param[in] scale /// @param[in] bias /// @param[in] residual_bias /// @param[out] output /// @param[out] residual /// @param[out] residual_out /// @param[in] batch_tokens /// @param[in] hidden_size /// @param[in] dequant_scale /// @param[in] quant_scale /// @param[in] stream /// @param[in] is_post_ln /// @param[in] colsum /// @return /// * ::CUINFER_STATUS_SUCCESS cuinferStatus_t cuinferResidualBiasLnI8II8OF( const void *input, const void *scale, const void *bias, const void *residual_bias, void *output, void *residual, void *residual_out, int batch_tokens, int hidden_size, float dequant_scale, float quant_scale, cudaStream_t stream, bool is_post_ln, const void *colsum = nullptr); /// @brief /// @ingroup BertBaseInt8TransformerFunctions /// @details Description: from ixrt ViterbiDecode, template is specilized /// according to num_tags internally, slightly change in parameters' order. /// /// Param Mappings: /// | src | dst | /// |-------------------|-------------------| /// | stream | stream | /// | batch_size | batch_size | /// | seq_len | seq_length | /// | num_tags | num_tags | /// | emissions | emissions | /// | mask | mask | /// | start_transitions | start_transitions | /// | transitions | transitions | /// | end_transitions | end_transitions | /// | output | best_path | /// @param[in] stream /// @param[in] batch_size /// @param[in] seq_len /// @param[in] num_tags /// @param[in, out] emissions /// @param[in, out] mask /// @param[out] start_transitions /// @param[out] transitions /// @param[out] end_transitions /// @param[out] output /// @return /// * ::CUINFER_STATUS_SUCCESS cuinferStatus_t cuinferViterbiDecode(cudaStream_t stream, int batch_size, int seq_len, int num_tags, void *emissions, void *mask, void *start_transitions, void *transitions, void *end_transitions, void *output); /// @brief /// @details From ixrt IxinferMhaI8Launcher. /// @ingroup BertBaseInt8TransformerFunctions /// @param[in] stream /// @param[in] q /// @param[in] k /// @param[in] v /// @param[in] mask /// @param[out] c /// @param[in] batch_size /// @param[in] head_num /// @param[in] seq_len /// @param[in] head_dim /// @param[in] qmax /// @param[in] kmax /// @param[in] vmax /// @param[in] smax /// @param[in] qkmax /// @param[in] rmax /// @return /// * ::CUINFER_STATUS_SUCCESS /// * ::CUINFER_STATUS_INTERNAL_ERROR cuinferStatus_t cuinferFusedMultiHeadAttentionI8( cudaStream_t stream, void *q, void *k, void *v, void *mask, void *c, int batch_size, int head_num, int seq_len, int head_dim, float qmax, float kmax, float vmax, float smax, float qkmax, float rmax); /// @brief /// @details Description: from ixrt IxinferBiasGeluI8II8O /// /// Params Mapping: /// | src | dst | /// |-----------------|---------------| /// | batch_token_num | input | /// | stream | stream | /// | input | input | /// | output | output | /// | bias | bias | /// | feature_dim | feature_dim | /// | dequant_scale | dequant_scale | /// | quant_scale | quant_scale | /// | in_col32 | | /// | out_col32 | | /// @ingroup BertBaseInt8TransformerFunctions /// @param[in] batch_token_num /// @param[in] stream /// @param[in] input /// @param[out] output /// @param[in] bias /// @param[in] feature_dim /// @param[in] dequant_scale /// @param[in] quant_scale /// @param[in] in_col32 /// @param[in] out_col32 /// @todo remove incol32, outcol32 /// @todo input should mark as const /// @return /// * ::CUINFER_STATUS_SUCCESS /// * ::CUINFER_STATUS_INTERNAL_ERROR cuinferStatus_t cuinferBiasGeluI8II8O(int batch_token_num, cudaStream_t stream, void *input, void *output, const void *bias, int feature_dim, float dequant_scale, float quant_scale, bool in_col32, bool out_col32); /// @brief /// @ingroup BertBaseInt8TransformerFunctions /// @param[in] input /// @param[in] scale /// @param[in] bias /// @param[in] residual_bias /// @param[out] output /// @param[out] residual /// @param[in] batch_tokens /// @param[in] hidden_size /// @param[in] dequant_scale /// @param[in] quant_scale /// @param[in] stream /// @param[in] is_post_ln /// @param[in] in_col32 /// @param[in] out_col32 /// @param[in] colsum cuinferStatus_t cuinferResidualBiaslnI32II8O( const void *input, const void *scale, const void *bias, const void *residual_bias, void *output, void *residual, int batch_tokens, int hidden_size, float dequant_scale, float quant_scale, cudaStream_t stream, bool is_post_ln, bool in_col32, bool out_col32, const void *colsum); /// @brief /// @ingroup BertBaseInt8TransformerFunctions /// @param[in] input /// @param[in] scale /// @param[in] bias /// @param[in] residual /// @param[out] output /// @param[in] batch_tokens /// @param[in] hidden_size /// @param[in] dequant_scale /// @param[in] stream /// @param[in] in_col32 /// @param[in] colsum cuinferStatus_t cuinferResidualBiaslnI32I(const void *input, const void *scale, const void *bias, const void *residual, void *output, int batch_tokens, int hidden_size, float dequant_scale, cudaStream_t stream, bool in_col32, const void *colsum); /// @brief /// @details Description: from ixrt IxinferLnResidualI8OLauncher /// Params Mapping: /// | src | dst | /// |---------------|---------------| /// | token_num | batch_tokens | /// | hidden_size | hidden_size | /// | stream | stream | /// | input | input | /// | output | output | /// | residual_out | residual | /// | scale | scale | /// | bias | bias | /// | residual_bias | residual_bias | /// | quant_scale | quant_scale | /// @ingroup BertBaseInt8TransformerFunctions /// @param[in] input /// @param[in] scale /// @param[in] bias /// @param[in] residual_bias /// @param[out] output /// @param[out] residual_out /// @param[in] token_num /// @param[in] hidden_size /// @param[in] quant_scale /// @param[in] stream cuinferStatus_t cuinferLayernormResidualI8OFO( const void *input, const void *scale, const void *bias, const void *residual_bias, void *output, void *residual_out, int token_num, int hidden_size, float quant_scale, cudaStream_t stream); /// @brief /// @details Description: from ixrt IxinferArrangeEncselfQkvI8II8O /// * in_col32 will be removed todo /// * max_thread_per_block default to 1024 /// * new_qkv result split to 3 parts and output /// Params Maping: /// | src | dst | /// |-----------------|----------------------| /// | batch_token_num | batch_token_num | /// | hidden_size | hidden_size | /// | stream | stream | /// | ori_qkv | ori_qkv | /// | qkv_bias | qkv_bias | /// | new_q | new_qkv | /// | new_k | new_qkv | /// | new_v | new_qkv | /// | max_batch_dim | max_batch_dim | /// | batch_seq_len | batch_seq_len | /// | dim_per_head | dim_per_head | /// | head_num | head_num | /// | 1024 | max_thread_per_block | /// | quant_scale | quant_scale | /// | dequant_scale | dequant_scale | /// @ingroup BertBaseInt8TransformerFunctions /// @param[in] batch_token_num /// @param[in] hidden_size /// @param[in] stream /// @param[in] ori_qkv /// @param[in] qkv_bias /// @param[out] new_q /// @param[out] new_k /// @param[out] new_v /// @param[in] max_batch_dim /// @param[in] batch_seq_len /// @param[in] dim_per_head /// @param[in] head_num /// @param[in] quant_scale /// @param[in] dequant_scale cuinferStatus_t cuinferArrangeEncselfQkvSepI8II8O( int batch_token_num, int hidden_size, cudaStream_t stream, const void *ori_qkv, const void *qkv_bias, void *new_q, void *new_k, void *new_v, int max_batch_dim, int batch_seq_len, int dim_per_head, int head_num, float quant_scale, float dequant_scale); /// @defgroup GEMM /// @ingroup GEMM typedef enum { CUINFER_OP_N = 0, CUINFER_OP_T = 1, CUINFER_OP_C = 2, CUINFER_OP_ROW2_COL16_4R2 = 3, } cuinferOperation_t; /// @ingroup GEMM typedef enum { CUINFER_POINTER_MODE_HOST, ///< The pointer is host pointer. CUINFER_POINTER_MODE_DEVICE, ///< The pointer is device pointer. } cuinferPointerMode_t; /// @ingroup GEMM typedef enum { CUINFER_BLAS_GEMM_CUSTOM_NONE = 0, CUINFER_BLAS_GEMM_CUSTOM_BIAS_ADD_ROW_OUT = 1, CUINFER_BLAS_GEMM_CUSTOM_HALFBIAS = 2, CUINFER_BLAS_GEMM_CUSTOM_HALFBIAS_GELU = 3, CUINFER_BLAS_GEMM_CUSTOM_HALFBIAS_RELU = 4, CUINFER_BLAS_GEMM_CUSTOM_HALFBIAS_TRANSPOSE = 5, CUINFER_BLAS_GEMM_CUSTOM_FLOATBIAS = 6, CUINFER_BLAS_GEMM_CUSTOM_FLOATBIAS_GELU = 7, CUINFER_BLAS_GEMM_CUSTOM_FLOATBIAS_RELU = 8, CUINFER_BLAS_GEMM_CUSTOM_FLOATBIAS_TRANSPOSE = 9, CUINFER_BLAS_GEMM_CUSTOM_HALFBIAS_SIGMOID = 10, CUINFER_BLAS_GEMM_CUSTOM_FLOATBIAS_SIGMOID = 11, CUINFER_BLAS_GEMM_CUSTOM_HALFBIAS_SILU = 12, CUINFER_BLAS_GEMM_CUSTOM_FLOATBIAS_SILU = 13, CUINFER_BLAS_GEMM_CUSTOM_SIGMOID = 14, CUINFER_BLAS_GEMM_CUSTOM_SILU = 15, CUINFER_BLAS_GEMM_CUSTOM_HALFBIAS_TANH = 16, CUINFER_BLAS_GEMM_CUSTOM_FLOATBIAS_TANH = 17, CUINFER_BLAS_GEMM_SPECIAL_INT8_FLOATBIAS = 18, CUINFER_BLAS_GEMM_SPECIAL_INT8_FLOATBIAS_GELU = 19 } cuinferGEMMCustomOption_t; /// @brief /// @ingroup GEMM /// @param[in] handle The libinfer handle. /// @param[in] stream /// @param[in] ptrMode /// @param[in] transa /// @param[in] transb /// @param[in] m /// @param[in] n /// @param[in] k /// @param[in] alpha Pointer to scaling factor. /// @param[in] A /// @param[in] Atype /// @param[in] lda /// @param[in] strideA /// @param[in] B /// @param[in] Btype /// @param[in] ldb /// @param[in] strideB /// @param[in] beta Pointer to scaling factor. /// @param[out] C /// @param[in] Ctype /// @param[in] ldc /// @param[in] strideC /// @param[in] batchCount /// @param[in] computeType /// @param[in] scaleType /// @param[in] customHostPtr /// @param[in] customDevicePtr /// @param[in] customOption /// @return cuinferStatus_t CUINFERWINAPI cuinferCustomGemm( cuinferHandle_t handle, cudaStream_t stream, cuinferPointerMode_t ptrMode, cuinferOperation_t transa, cuinferOperation_t transb, int m, int n, int k, const void *alpha, const void *A, cudaDataType_t Atype, int lda, long long int strideA, const void *B, cudaDataType_t Btype, int ldb, long long int strideB, const void *beta, void *C, cudaDataType_t Ctype, int ldc, long long int strideC, int batchCount, cudaDataType_t computeType, cudaDataType_t scaleType, const void *customHostPtr, const void *customDevicePtr, cuinferGEMMCustomOption_t customOption); /// @brief /// @ingroup GEMM /// @param[in] m /// @param[in] n /// @param[in] k /// @param[in] transA /// @param[in] transB /// @param[in] Atype /// @param[in] Btype /// @param[in] Ctype /// @param[in] computeType /// @param[in] scaleType /// @param[out] workspaceSize /// @return cuinferStatus_t CUINFERWINAPI cuinferGetCustomGemmExWorkspace( int m, int n, int k, cuinferOperation_t transA, cuinferOperation_t transB, cudaDataType_t Atype, cudaDataType_t Btype, cudaDataType_t Ctype, cudaDataType_t computeType, cudaDataType_t scaleType, size_t *workspaceSize); /// @brief /// @ingroup GEMM /// @param[in] handle The libinfer handle. /// @param[in] stream /// @param[in] ptrMode /// @param[in] transa /// @param[in] transb /// @param[in] m /// @param[in] n /// @param[in] k /// @param[in] alpha Pointer to scaling factor. /// @param[in] A /// @param[in] Atype /// @param[in] lda /// @param[in] strideA /// @param[in] B /// @param[in] Btype /// @param[in] ldb /// @param[in] strideB /// @param[in] beta Pointer to scaling factor. /// @param[out] C /// @param[in] Ctype /// @param[in] ldc /// @param[in] strideC /// @param[in] batchCount /// @param[in] computeType /// @param[in] scaleType /// @param[in] customHostPtr /// @param[in] customDevicePtr /// @param[in] customOption /// @param[in] workspace The workspace pre-allocated. See the corresponding get /// workspace size helper function. /// @return cuinferStatus_t CUINFERWINAPI cuinferCustomGemmEx( cuinferHandle_t handle, cudaStream_t stream, cuinferPointerMode_t ptrMode, cuinferOperation_t transa, cuinferOperation_t transb, int m, int n, int k, const void *alpha, const void *A, cudaDataType_t Atype, int lda, long long int strideA, const void *B, cudaDataType_t Btype, int ldb, long long int strideB, const void *beta, void *C, cudaDataType_t Ctype, int ldc, long long int strideC, int batchCount, cudaDataType_t computeType, cudaDataType_t scaleType, const void *customHostPtr, const void *customDevicePtr, cuinferGEMMCustomOption_t customOption, void *workspace); /// @defgroup NMS NoN-Max Suppression(NMS) /// @note The bounding boxex is of form [xmin, ymin, xmax, ymax, class_id, /// score], which is 6 floats. The bounding box can be either form of pixel or /// scaled to 0.0-1.0. /// @brief Gpu version of Non-Max Suppression(NMS) over bounding boxex. /// @ingroup NMS /// @note The bounding boxex is of form [xmin, ymin, xmax, ymax, class_id, /// score], which is 6 floats. The bounding box can be either form of pixel or /// scaled to 0.0-1.0. /// @param[in] handle The libinfer handle. /// @param[in] pDetections The input bounding boxex. Device pointer. Size /// pDetections[nInputs][6]. /// @param[in] nInputs The number of input bounding boxex. /// @param[out] pKeepDetections The result bounding boxex. Device pointer. /// @param[in] nMaxKeep The max result bounding boxex. 0 <= \p nKeep <= \p /// nMaxKeep. /// @param[out] nKeep The number of result bounding boxex to kept. /// @param[in] fIoUThresh The IoU threshold. The bounding boxex will be /// suppressed if iou score is over this threshold. /// @param[in] fScoreThresh The score threshold, only higher score are come into /// consideration. /// @param[in] workSpace The workspace pre-allocated. See the corresponding get /// workspace size helper function. /// @param[out] outputIndice The index of corresponding result. Set to \p /// nullptr will disable it. /// @return /// * ::CUINFER_STATUS_BAD_PARAM If param is invalid(mostly nMaxKeep too large). /// * ::CUINFER_STATUS_SUCCESS If success. cuinferStatus_t CUINFERWINAPI cuinferNMS(cuinferHandle_t handle, float *pDetections, const int nInputs, float *pKeepDetections, const int nMaxKeep, int *nKeep, const float fIoUThresh, const float fScoreThresh, void *workspace, int *outputIndice = nullptr); /// @brief Get the workspace of the corresponding ::cuinferNMS. /// @ingroup NMS /// @param pDetections Not used. /// @param[in] nInputs The number of input bounding boxex. /// @param pKeepDetections Not used. /// @param[in] nMaxKeep The max result bounding boxex. 0 <= \p nKeep <= \p /// nMaxKeep. /// @param nKeep not used. /// @param[in] fIoUThresh The score threshold, only higher score are come into /// consideration. /// @param[in] fScoreThresh The score threshold, only higher score are come into /// consideration. /// @param[out] sizeInBytes The result extra temporary space size in bytes. /// @param[in] outputIndice Whether output index of corresponding result. /// @return /// * ::CUINFER_STATUS_BAD_PARAM If param is invalid(mostly nMaxKeep too large). /// * ::CUINFER_STATUS_SUCCESS If success. cuinferStatus_t CUINFERWINAPI cuinferGetNMSWorkspaceSize( float *pDetections, const int nInputs, float *pKeepDetections, const int nMaxKeep, int *nKeep, const float fIoUThresh, const float fScoreThresh, size_t *sizeInBytes, bool outputIndice = false); /// @brief The batched version of ::cuinferNMS. /// @ingroup NMS /// @param[in] handle The libinfer handle. /// @param[in] batch The batch. A quantity used or made at one time. /// @param[in] pDetections The input bounding boxex. Device pointer. Size /// pDetections[batch][nInputs][6]. /// @param[in] nInputs The number of input bounding boxex in each batch. /// @param[out] pKeepDetections The result bounding boxex. Device pointer. Note /// the padding when first fewer batchs not full. /// @param[in] nMaxKeep The max result bounding boxex. 0 <= \p nKeep <= \p /// nMaxKeep for every batch. /// @param[out] nKeep The number of result bounding boxex to kept for each /// batch. Size batch. /// @param[in] fIoUThresh The score threshold, only higher score are come into /// consideration. /// @param[in] fScoreThresh The score threshold, only higher score are come into /// consideration. /// @param[in] workSpace The workspace pre-allocated. See the corresponding get /// workspace size helper function.Whether output index of corresponding /// result.t *pKeepDetections, const int nMaxKeep, /// * ::CUINFER_STATUS_BAD_PARAM If param is invalid(mostly nMaxKeep too large). /// * ::CUINFER_STATUS_SUCCESS If success. cuinferStatus_t CUINFERWINAPI cuinferNMSBatched(cuinferHandle_t handle, int batch, float *pDetections, const int nInputs, float *pKeepDetections, const int nMaxKeep, int *nKeep, const float fIoUThresh, const float fScoreThresh, void *workspace, int *outputIndice = nullptr); /// @brief Get the workspace of the corresponding ::cuinferGetNMSWorkspaceSize. /// @ingroup NMS /// @param[in] batch The batch. A quantity used or made at one time. /// @param pDetections Not used. /// @param[in] nInputs The number of input bounding boxex in each batch. /// @param pKeepDetections Not used. /// @param[in] nMaxKeep The max result bounding boxex. 0 <= \p nKeep <= \p /// nMaxKeep for every batch. /// @param[in] nKeep The number of result bounding boxex to kept for each /// batch. Size batch. /// @param[in] fIoUThresh The score threshold, only higher score are come into /// consideration. /// @param[in] fScoreThresh The score threshold, only higher score are come into /// consideration. /// @param[out] sizeInBytes The result extra temporary space size in bytes. /// @param[in] outputIndice Whether output index of corresponding result. /// @return /// * ::CUINFER_STATUS_BAD_PARAM If param is invalid(mostly nMaxKeep too large). /// * ::CUINFER_STATUS_SUCCESS If success. cuinferStatus_t CUINFERWINAPI cuinferGetNMSBatchedWorkspaceSize( int batch, float *pDetections, const int nInputs, float *pKeepDetections, const int nMaxKeep, int *nKeep, const float fIoUThresh, const float fScoreThresh, size_t *sizeInBytes, bool outputIndice = false); /// @brief NMS algo specilized for Yolo format. /// @note The output format is [x, y, w, h, boxscore, class_score1, ..., ] /// @note Due to the nms process. Only the boxscoore with highest class_score /// will be kept. And all other classes will be supressed. /// @ingroup NMS /// @param[in] handle The libinfer handle. /// @param[in] n_batch The number of batch. /// @param[in] n_bbox the number of bbox. /// @param[in] detection The pointer of input tensor, size is /// [n_batch][n_bbox][n_class+5]. /// @param[in] n_class The number of class. /// @param[out] keep_detection The result bounding boxex. Device pointer. /// @param[in] max_keep_per_batch The max result bounding boxex. 0 <= \p /// n_keep_each_batch[i] <= \p max_keep_per_batch. /// @param[out] n_keep_each_batch The result bounding boxex number for each /// batch. /// @param[in] iou_threshold The IoU threshold. The bounding boxex will be /// suppressed if iou score is over this threshold. /// @param[in] score_threshold The score threshold, only higher score are come /// into consideration. /// @param[in] workSpace The workspace pre-allocated. See the corresponding get /// workspace size helper function. /// @param[out] outputIndice The index of the original input. Set to \p nullptr /// if unused. /// @return /// * ::CUINFER_STATUS_BAD_PARAM If param is invalid(mostly nMaxKeep too large). /// * ::CUINFER_STATUS_SUCCESS If success. cuinferStatus_t CUINFERWINAPI cuinferNMSBatchedYoloFused( cuinferHandle_t handle, int n_batch, int n_bbox, float *detection, int n_class, float *keep_detection, int max_keep_per_batch, int *n_keep_each_batch, float iou_threshold, float score_threshold, void *workspace, int *outputIndice = nullptr); /// @brief Get the workspace of the ::cuinferNMSBatchedYoloFused. /// @ingroup NMS /// @param[in] n_batch The number of batch. /// @param[in] n_bbox the number of bbox. /// @param detection Not used. /// @param[in] n_class The number of class. /// @param keep_detection Not used. /// @param[in] max_keep_per_batch The max result bounding boxex. 0 <= \p /// n_keep_each_batch[i] <= \p max_keep_per_batch. /// @param n_keep_each_batch Not used. /// @param[in] iou_threshold The IoU threshold. The bounding boxex will be /// suppressed if iou score is over this threshold. /// @param[in] score_threshold The score threshold, only higher score are come /// into consideration. /// @param[out] workspace_size_in_bytes The result workspace size in bytes. /// @param[in] outputIndice Whether output index of corresponding result. /// @return cuinferStatus_t CUINFERWINAPI cuinferGetNMSBatchedYoloFusedWorkspaceSize( int n_batch, int n_bbox, float *detection, int n_class, float *keep_detection, int max_keep_per_batch, int *n_keep_each_batch, float iou_threshold, float score_threshold, size_t *workspace_size_in_bytes, bool outputIndice = false); /// @defgroup TransformerFMHAAPIs Transformer FHMA APIS struct cuinferFMHAParam { float q_amax = 0.0f; float k_amax = 0.0f; float v_amax = 0.0f; float r_amax = 1.0f; float s_max = 1.0f; cuinferSoftmaxAlgorithm_t softmax_algo = cuinferSoftmaxAlgorithm_t::CUINFER_SOFTMAX_FAST; }; /// @brief /// @ingroup TransformerFMHAAPIs /// @param[in] handle The libinfer handle. /// @param[in] fmha_param /// @param[in] computeType /// @param[in] dataType /// @param[in] maskType /// @param[in] q_desc /// @param[in] q_data /// @param[in] k_desc /// @param[in] k_data /// @param[in] v_desc /// @param[in] v_data /// @param[in] mask_desc /// @param[in] padding_mask /// @param[in] o_desc /// @param[out] o_data /// @param[in] use_tcu /// @return cuinferStatus_t CUINFERWINAPI cuinferFMHAForward( cuinferHandle_t handle, cuinferFMHAParam fmha_param, cuinferDataType_t computeType, cuinferDataType_t dataType, cuinferDataType_t maskType, const cuinferTensorDescriptor_t q_desc, const void *q_data, const cuinferTensorDescriptor_t k_desc, const void *k_data, const cuinferTensorDescriptor_t v_desc, const void *v_data, const cuinferTensorDescriptor_t mask_desc, const void *padding_mask, const cuinferTensorDescriptor_t o_desc, void *o_data, const bool use_tcu = true); /// @ingroup TransformerFMHAAPIs typedef enum { CUINFER_FATTN_BHSD = 0, CUINFER_FATTN_BSHD = 1 } cuinferFlashAttnLayout_t; /// @ingroup TransformerFMHAAPIs struct cuinferFMHAQuantParam { float q_amax; float k_amax; float v_amax; float p_amax; float o_amax; }; /// @ingroup TransformerFMHAAPIs typedef enum { CUINFER_FATTN_ALIBI_MODE_SUB_KQ = 0, CUINFER_FATTN_ALIBI_MODE_SQRT_SUB_QK = 1, } cuinferFlashAttnAlibiMode_t; /// @ingroup TransformerFMHAAPIs struct cuinferFlashAttnConfigInfo { cuinferFlashAttnLayout_t layout; cuinferFMHAQuantParam quantParam; bool isCausal; float scaling; int *qoSeqArray; int *kvSeqArray; int kvSeqStart; int kvSeqEnd; int kvHeadNum; bool isAlibi; cuinferFlashAttnAlibiMode_t alibiMode; float *slopeM; int qStride; int kStride; int vStride; }; /// @brief /// @ingroup TransformerFMHAAPIs /// @param[in] handle The libinfer handle. /// @param[in] flashAttnInfo /// @param[in] qDesc /// @param[in] q /// @param[in] kDesc /// @param[in] k /// @param[in] vDesc /// @param[in] v /// @param[in] maskDesc /// @param[in] mask /// @param[in] oDesc /// @param[out] o /// @return cuinferStatus_t CUINFERWINAPI cuinferFMHAForwardEx( cuinferHandle_t handle, const cuinferFlashAttnConfigInfo &flashAttnInfo, const cuinferTensorDescriptor_t qDesc, const void *q, const cuinferTensorDescriptor_t kDesc, const void *k, const cuinferTensorDescriptor_t vDesc, const void *v, const cuinferTensorDescriptor_t maskDesc, const void *mask, const cuinferTensorDescriptor_t oDesc, void *o); /// @ingroup TransformerFMHAAPIs typedef enum { CUINFER_GPTATTEN_CONTEXT = 0, CUINFER_GPTATTEN_DECODE = 1, } cuinferGPTFlashAttnMode_t; /// @ingroup TransformerFMHAAPIs struct cuinferGPTFlashAttnConfigInfo { cuinferGPTFlashAttnMode_t attenMode; float scaling; int qHeadnum; int kvHeadnum; int maxQSeqlen; const int* seqArray; }; /// @brief /// @ingroup TransformerFMHAAPIs /// @param[in] handle The libinfer handle. /// @param[in] flashAttnInfo config params of tensorrt llm fmha /// @param[in] qkvDesc The discriptor of input tensor qkv. /// @param[in] qkv Const pointer to input tensor qkv. /// @param[in] pastkvDesc The discriptor of input tensor kv cache. /// @param[in] pastkv Const pointer to input tensor kv cache. /// @param[in] oDesc The discriptor of output tensor o. /// @param[out] o Pointer to output tensor o. /// @return cuinferStatus_t CUINFERWINAPI cuinferGPTFMHAForward( cuinferHandle_t handle, const cuinferGPTFlashAttnConfigInfo& flashAttnInfo, const cuinferTensorDescriptor_t qkvDesc, const void* qkv, const cuinferTensorDescriptor_t pastkvDesc, const void* pastkv, const cuinferTensorDescriptor_t oDesc, void* o); /// @brief /// @param[in] handle The libinfer handle. /// @param[in] x_desc /// @param[in] x Const pointer to input tensor x. /// @param[in] y_desc /// @param[out] y The discriptor of output tensor y. /// @param[in] resize_method /// @param[in] size_h /// @param[in] size_w /// @param[in] top /// @param[in] left /// @return cuinferStatus_t CUINFERWINAPI cuinferCropAndResize( cuinferHandle_t handle, const cuinferTensorDescriptor_t x_desc, const void *x, const cuinferTensorDescriptor_t y_desc, void *y, cuinferInterpolationFlag_t resize_method, int size_h, int size_w, int top, int left); /// @brief /// @param[in] handle The libinfer handle. /// @param[in] x Const pointer to input tensor x. /// @param[out] y The discriptor of output tensor y. /// @param[in] data_in_type /// @param[in] compute_type /// @param[in] data_out_type /// @param[in] anchor_num /// @param[in] anchors /// @param[in] grid /// @param[in] stride /// @param[in] num_class /// @param[in] n_batch /// @param[in] anchor_first /// @return cuinferStatus_t CUINFERWINAPI cuinferYoloV5Detect( cuinferHandle_t handle, const void *x, void *y, cuinferDataType_t data_in_type, cuinferDataType_t compute_type, cuinferDataType_t data_out_type, int anchor_num, const int *anchors, int grid, int stride, int num_class, int n_batch, bool anchor_first); /// @defgroup LayerNorm Layer Norm /// @brief /// @ingroup LayerNorm /// @note Only serves 2-dim N and C /// @param[in] handle The libinfer handle. /// @param[in] x Const pointer to input tensor x. /// @param[out] y The discriptor of output tensor y. /// @param[in] data_in_type /// @param[in] compute_type /// @param[in] data_out_type /// @param[in] n /// @param[in] c /// @param[in] scale /// @param[in] bias /// @param[in] epsilon /// @return cuinferStatus_t CUINFERWINAPI cuinferLayerNorm(cuinferHandle_t handle, const void *x, void *y, cuinferDataType_t data_in_type, cuinferDataType_t compute_type, cuinferDataType_t data_out_type, int n, int c, const void *scale, const void *bias, const float epsilon); /// @brief /// @ingroup LayerNorm /// @param[in] handle The libinfer handle. /// @param[in] data_type /// @param[in] input /// @param[in] ln_scale /// @param[in] ln_bias /// @param[in] residual_bias /// @param[in] residual_in /// @param[out] residual_out /// @param[out] output /// @param[in] batch_tokens /// @param[in] hidden_size /// @param[in] is_postln /// @param[in] epsilon /// @return cuinferStatus_t CUINFERWINAPI cuinferBiasResidualLn( cuinferHandle_t handle, cuinferDataType_t data_type, const void *input, const void *ln_scale, const void *ln_bias, const void *residual_bias, const void *residual_in, void *residual_out, void *output, int batch_tokens, int hidden_size, bool is_postln, float epsilon); /// @defgroup GroupNorm Group Norm /// @ingroup GroupNorm typedef enum { CUINFER_GROUPNORM_AFFINE_NONE = 0, CUINFER_GROUPNORM_AFFINE_PERCHANNEL = 1, CUINFER_GROUPNORM_AFFINE_PERGROUP = 2, } cuinferGroupNormAffineMode; /// @brief /// @ingroup GroupNorm /// @param[in] handle The libinfer handle. /// @param[in] xDesc The discriptor of input tensor x. /// @param[in] x Const pointer to input tensor x. /// @param[in] scale /// @param[in] bias /// @param[in] num_groups /// @param[in] affineMode /// @param[in] y /// @param[in] epsilon /// @return cuinferStatus_t CUINFERWINAPI cuinferGroupNorm( cuinferHandle_t handle, const cuinferTensorDescriptor_t xDesc, const void *x, const void *scale, const void *bias, const int num_groups, cuinferGroupNormAffineMode affineMode, void *y, const float epsilon); /// @brief /// @param[in] handle The libinfer handle. /// @param[in] xDesc The discriptor of input tensor x. /// @param[in] x Const pointer to input tensor x. /// @param[in] scale /// @param[in] bias /// @param[out] y The discriptor of output tensor y. /// @param[in] epsilon /// @return cuinferStatus_t CUINFERWINAPI cuinferInstanceNorm( cuinferHandle_t handle, const cuinferTensorDescriptor_t xDesc, const void *x, const void *scale, const void *bias, void *y, const float epsilon); /// @brief /// @param[in] handle The libinfer handle. /// @param[in] x_desc /// @param[in] x Const pointer to input tensor x. /// @param[out] y The discriptor of output tensor y. /// @param[in] n_index /// @param[in] c_index /// @param[in] d_index /// @param[in] h_index /// @param[in] w_index /// @return cuinferStatus_t CUINFERWINAPI cuinferTranspose(cuinferHandle_t handle, const cuinferTensorDescriptor_t x_desc, const void *x, void *y, unsigned n_index, unsigned c_index, unsigned d_index, unsigned h_index, unsigned w_index); /// @defgroup TransposedConv Transposed Conv /// @ingroup TransposedConv typedef enum { CUINFER_CONVOLUTION_TRANSPOSE_ALGO_AUTO = 0, ///< Recommand default. CUINFER_CONVOLUTION_TRANSPOSE_ALGO_DIRECT = 1, ///< Todo. CUINFER_CONVOLUTION_TRANSPOSE_ALGO_EXPLICIT_GEMM = 2, ///< For large batch. CUINFER_CONVOLUTION_TRANSPOSE_ALGO_EXPLICIT_GEMM2 = 3, ///< For small c. CUINFER_CONVOLUTION_TRANSPOSE_ALGO_IMPLICIT_GEMM = 4, ///< Todo. CUINFER_CONVOLUTION_TRANSPOSE_ALGO_COUNT = 5, } cuinferConvolutionTransposeAlgo_t; /// @brief /// @ingroup TransposedConv /// @param[in] xDesc The discriptor of input tensor x. /// @param[in] wDesc The discriptor of filter w. /// @param[in] convDesc The discriptor of convolution. /// @param[in] algo The algorithm specified. /// @param[out] workSpaceSizeInBytes /// @param[in] zDesc /// @param[in] biasDesc /// @param[in] activationDesc /// @param[in] connectionMode The connection mode. /// @param[in] yDesc The discriptor of tensor y. /// @return cuinferStatus_t CUINFERWINAPI cuinferGetQDEConvolutionTransposedWorkspaceSize( const cuinferTensorDescriptor_t xDesc, const cuinferFilterDescriptor_t wDesc, const cuinferConvolutionDescriptor_t convDesc, cuinferConvolutionTransposeAlgo_t algo, size_t *workSpaceSizeInBytes, const cuinferTensorDescriptor_t zDesc, const cuinferTensorDescriptor_t biasDesc, const cuinferActivationDescriptor_t activationDesc, cuinferTensorConnectionMode_t connectionMode, const cuinferTensorDescriptor_t yDesc); /// @brief /// @details y = clip(round(activate(alpha * conv(x, w) + z * beta + bias) * /// alpha2)) biasDesc is not used, zDesc == yDesc /// @ingroup TransposedConv /// @param[in] handle The libinfer handle. /// @param[in] alpha Pointer to scaling factor. /// @param[in] perchannelAlpha /// @param[in] beta Pointer to scaling factor. /// @param[in] gamma Pointer to scaling factor. /// @param[in] xDesc The discriptor of input tensor x. /// @param[in] x Const pointer to input tensor x. /// @param[in] wDesc The discriptor of filter w. /// @param[in] The const pointer of input filter w. /// @param[in] convDesc The discriptor of convolution. /// @param[in] algo The algorithm specified. /// @param[in] workSpace The workspace pre-allocated. See the corresponding get /// workspace size helper function. /// @param[in] workSpaceSizeInBytes /// @param[in] alpha2 /// @param[in] zScale /// @param[in] zDesc /// @param[in] z /// @param[in] biasDesc /// @param[in] bias /// @param[in] perChannel /// @param[in] activationDesc /// @param[in] connectionBeforeActivation Whether activation is performed before /// connection. /// @param[in] connectionMode The connection mode. /// @param[in] yDesc The discriptor of tensor y. /// @param[out] y The discriptor of output tensor y. /// @return cuinferStatus_t CUINFERWINAPI cuinferQDEConvolutionTranspose( cuinferHandle_t handle, const void *alpha, const void *perchannelAlpha, const void *beta, const void *gamma, const cuinferTensorDescriptor_t xDesc, const void *x, const cuinferFilterDescriptor_t wDesc, const void *w, const cuinferConvolutionDescriptor_t convDesc, cuinferConvolutionTransposeAlgo_t algo, void *workSpace, size_t workSpaceSizeInBytes, const void *alpha2, const void *zScale, const cuinferTensorDescriptor_t zDesc, const void *z, const cuinferTensorDescriptor_t biasDesc, const void *bias, bool perChannel, const cuinferActivationDescriptor_t activationDesc, bool connectionBeforeActivation, cuinferTensorConnectionMode_t connectionMode, const cuinferTensorDescriptor_t yDesc, void *y); /// @defgroup TopK Top-K /// @brief /// @ingroup TopK /// @param[in] n /// @param[in] m /// @param[in] top_k /// @param[in] sort_dim /// @param[in] largest /// @param[in] sorted /// @param[in] out_value /// @param[in] out_indice /// @param[in] data_type /// @param[out] workspace_size /// @return cuinferStatus_t CUINFERWINAPI cuinferGetTopKWorkspace(int n, int m, int top_k, int sort_dim, bool largest, bool sorted, bool out_value, bool out_indice, cuinferDataType_t data_type, size_t *workspace_size); /// @brief /// @ingroup TopK /// @param[in] handle The libinfer handle. /// @param[in] input /// @param[in] n /// @param[in] m /// @param[in] top_k /// @param[in] sort_dim /// @param[in] largest /// @param[in] sorted /// @param[out] out_value /// @param[out] out_indice /// @param[in] datatype /// @param[in] workSpace The workspace pre-allocated. See the corresponding get /// workspace size helper function. /// @return cuinferStatus_t CUINFERWINAPI cuinferTopK(cuinferHandle_t handle, const void *input, int n, int m, int top_k, int sort_dim, bool largest, bool sorted, void *out_value, int *out_indice, cuinferDataType_t datatype, void *workspace); /// @brief /// @ingroup TopK /// @param[in] top_k /// @param[in] batch The batch. A quantity used or made at one time. /// @param[in] n /// @param[in] m /// @param[in] k /// @param[in] largest /// @param[in] sorted /// @param[in] sort_dim /// @param[in] output /// @param[in] indice /// @param[in] datatype /// @param[out] workspace_size /// @return cuinferStatus_t CUINFERWINAPI cuinferGetTopKBatchWorkspace( int top_k, int batch, int n, int m, int k, bool largest, bool sorted, int sort_dim, bool output, bool indice, cuinferDataType_t datatype, size_t *workspace_size); /// @brief /// @ingroup TopK /// @param[in] handle The libinfer handle. /// @param[in] input /// @param[in] top_k /// @param[in] batch The batch. A quantity used or made at one time. /// @param[in] n /// @param[in] m /// @param[in] k /// @param[in] largest /// @param[in] sorted /// @param[in] sort_dim /// @param[out] output /// @param[out] indice /// @param[in] datatype /// @param[in] workSpace The workspace pre-allocated. See the corresponding get /// workspace size helper function. /// @return cuinferStatus_t CUINFERWINAPI cuinferTopKBatch( cuinferHandle_t handle, const void *input, int top_k, int batch, int n, int m, int k, bool largest, bool sorted, int sort_dim, void *output, int *indice, cuinferDataType_t datatype, void *workspace); /// @defgroup Reduce /// @brief /// @ingroup Reduce /// @param[in] in_type /// @param[in] acc_type /// @param[in] out_type /// @param[in] reduce_op /// @param[in] n_dims /// @param[in] dims /// @param[in] n_reduce_dims /// @param[in] reduce_dim_index /// @param[out] workspace_size /// @return cuinferStatus_t CUINFERWINAPI cuinferGetReduceWorkspace( cuinferDataType_t in_type, cuinferDataType_t acc_type, cuinferDataType_t out_type, cuinferReduceTensorOp_t reduce_op, int n_dims, const int *dims, int n_reduce_dims, const int *reduce_dim_index, size_t *workspace_size); /// @brief /// @ingroup Reduce /// @param[in] handle The libinfer handle. /// @param[in] in /// @param[out] out /// @param[in] in_type /// @param[in] acc_type /// @param[in] out_type /// @param[in] reduce_op /// @param[in] n_dims /// @param[in] dims /// @param[in] n_reduce_dims /// @param[in] reduce_dim_index /// @param[in] workspace The workspace pre-allocated. See the corresponding get /// workspace size helper function. /// @return cuinferStatus_t CUINFERWINAPI cuinferReduce(cuinferHandle_t handle, const void *in, void *out, cuinferDataType_t in_type, cuinferDataType_t acc_type, cuinferDataType_t out_type, cuinferReduceTensorOp_t reduce_op, int n_dims, const int *dims, int n_reduce_dims, const int *reduce_dim_index, void *workspace); /// @defgroup HammingDistance Hamming Distance /// @ingroup HammingDistance typedef enum { CUINFER_HAMMING_DISTANCE_MODE_PER_BIT, CUINFER_HAMMING_DISTANCE_MODE_PER_CHAR, } cuinferHammingDistanceMode; /// @brief /// @ingroup HammingDistance /// @param[in] n /// @param[in] batch The batch. A quantity used or made at one time. /// @param[in] mode /// @param[out] workspace_size /// @return cuinferStatus_t CUINFERWINAPI cuinferGetHammingDistanceWorkspace( int n, int batch, cuinferHammingDistanceMode mode, size_t *workspace_size); /// @brief /// @ingroup HammingDistance /// @param[in] handle The libinfer handle. /// @param[in] in_x /// @param[in] in_y /// @param[out] out /// @param[in] n /// @param[in] batch The batch. A quantity used or made at one time. /// @param[in] mode /// @param[in] workSpace The workspace pre-allocated. See the corresponding get /// workspace size helper function. /// @return cuinferStatus_t CUINFERWINAPI cuinferHammingDistance(cuinferHandle_t handle, const unsigned char *in_x, const unsigned char *in_y, int *out, int n, int batch, cuinferHammingDistanceMode mode, void *workspace); /// @brief /// @param[in] handle The libinfer handle. /// @param[in] rnnDesc /// @param[in] seqLength /// @param[in] xDesc The discriptor of input tensor x. /// @param[in] x Const pointer to input tensor x. /// @param[in] hxDesc /// @param[in] hx /// @param[in] cxDesc /// @param[in] cx /// @param[in] wDesc The discriptor of filter w. /// @param[in] The const pointer of input filter w. /// @param[in] rDesc /// @param[in] r /// @param[in] biasDesc /// @param[in] bias /// @param[in] yDesc The discriptor of tensor y. /// @param[out] y The discriptor of output tensor y. /// @param[in] hyDesc /// @param[out] hy /// @param[in] cyDesc /// @param[out] cy /// @param[in] workSpace The workspace pre-allocated. See the corresponding get /// workspace size helper function. /// @param[in] workSpaceSizeInBytes /// @return cuinferStatus_t CUINFERWINAPI cuinferLSTMForwardInference( cuinferHandle_t handle, const cuinferRNNDescriptor_t rnnDesc, const int seqLength, const cuinferTensorDescriptor_t xDesc, const void *x, const cuinferTensorDescriptor_t hxDesc, const void *hx, const cuinferTensorDescriptor_t cxDesc, const void *cx, const cuinferFilterDescriptor_t wDesc, const void *w, const cuinferFilterDescriptor_t rDesc, const void *r, const cuinferTensorDescriptor_t biasDesc, const void *bias, const cuinferTensorDescriptor_t yDesc, void *y, const cuinferTensorDescriptor_t hyDesc, void *hy, const cuinferTensorDescriptor_t cyDesc, void *cy, void *workSpace, size_t workSpaceSizeInBytes); /// @defgroup PageAttention Page Attension /// @brief /// @ingroup PageAttention /// @param[in] num_seqs /// @param[in] num_heads /// @param[in] block_size /// @param[in] max_context_len /// @param[out] workspaceSize /// @return cuinferStatus_t CUINFERWINAPI cuInferPageAttentionGetWorkspaceV2( unsigned num_seqs, unsigned num_heads, unsigned block_size, unsigned max_context_len, size_t *workspaceSize); /// @brief /// @ingroup PageAttention /// @param[in] num_seqs /// @param[in] num_heads /// @param[in] head_size /// @param[in] block_size /// @param[in] max_context_len /// @param[out] workspaceSize /// @return cuinferStatus_t CUINFERWINAPI cuInferPageAttentionGetWorkspace( unsigned num_seqs, unsigned num_heads, unsigned head_size, unsigned block_size, unsigned max_context_len, size_t *workspaceSize); /// @brief /// @ingroup PageAttention /// @param[in] handle The libinfer handle. /// @param[out] out_ptr /// @param[in] outType /// @param[in] query_ptr /// @param[in] queryType /// @param[in] num_seqs /// @param[in] num_heads /// @param[in] head_size /// @param[in] query_stride /// @param[in] kv_block_stride /// @param[in] kv_head_stride /// @param[in] key_cache_ptr /// @param[in] keyCacheType /// @param[in] value_cache_ptr /// @param[in] valueCacheType /// @param[in] block_size /// @param[in] head_mapping /// @param[in] scale /// @param[in] block_tables_ptr /// @param[in] max_num_blocks_per_seq /// @param[in] context_lens_ptr /// @param[in] max_context_len /// @param[in] alibi_slopes_ptr /// @param[in] workSpace The workspace pre-allocated. See the corresponding get /// workspace size helper function. /// @param[in] alibi_sqrt /// @return cuinferStatus_t CUINFERWINAPI cuInferPageAttentionV2( cuinferHandle_t handle, void *__restrict__ out_ptr, cudaDataType_t outType, const void *__restrict__ query_ptr, cudaDataType_t queryType, int num_seqs, int num_heads, int head_size, int query_stride, int kv_block_stride, int kv_head_stride, const void *__restrict__ key_cache_ptr, cudaDataType_t keyCacheType, const void *__restrict__ value_cache_ptr, cudaDataType_t valueCacheType, int block_size, const int *head_mapping, float scale, const int *__restrict__ block_tables_ptr, int max_num_blocks_per_seq, const int *__restrict__ context_lens_ptr, int max_context_len, const float *__restrict__ alibi_slopes_ptr, void *workspace = nullptr, bool alibi_sqrt = false); /// @brief /// @ingroup PageAttention /// @param[in] handle The libinfer handle. /// @param[out] out_ptr /// @param[in] outType /// @param[in] query_ptr /// @param[in] queryType /// @param[in] num_seqs /// @param[in] num_heads /// @param[in] head_size /// @param[in] query_stride /// @param[in] kv_block_stride /// @param[in] kv_head_stride /// @param[in] key_cache_ptr /// @param[in] keyCacheType /// @param[in] value_cache_ptr /// @param[in] valueCacheType /// @param[in] block_size /// @param[in] head_mapping /// @param[in] scale /// @param[in] block_tables_ptr /// @param[in] max_num_blocks_per_seq /// @param[in] context_lens_ptr /// @param[in] max_context_len /// @param[in] alibi_slopes_ptr /// @param[in] workSpace The workspace pre-allocated. See the corresponding get /// workspace size helper function. /// @param[in] alibi_sqrt /// @return cuinferStatus_t CUINFERWINAPI cuInferPageAttention( cuinferHandle_t handle, void *__restrict__ out_ptr, cudaDataType_t outType, const void *__restrict__ query_ptr, cudaDataType_t queryType, int num_seqs, int num_heads, int head_size, int query_stride, int kv_block_stride, int kv_head_stride, const void *__restrict__ key_cache_ptr, cudaDataType_t keyCacheType, const void *__restrict__ value_cache_ptr, cudaDataType_t valueCacheType, int block_size, const int *head_mapping, float scale, const int *__restrict__ block_tables_ptr, int max_num_blocks_per_seq, const int *__restrict__ context_lens_ptr, int max_context_len, const float *__restrict__ alibi_slopes_ptr, void *workspace = nullptr, bool alibi_sqrt = false); /// @brief /// @ingroup PageAttention /// @param[in] handle The libinfer handle. /// @param[out] out_ptr /// @param[in] outType /// @param[in] query_ptr /// @param[in] key_ptr /// @param[in] value_ptr /// @param[in] queryType /// @param[in] num_seqs /// @param[in] num_heads /// @param[in] num_kv_heads /// @param[in] head_size /// @param[in] query_stride /// @param[in] key_stride /// @param[in] value_stride /// @param[in] kv_block_stride /// @param[in] kv_head_stride /// @param[in] key_cache_ptr /// @param[in] keyCacheType /// @param[in] value_cache_ptr /// @param[in] valueCacheType /// @param[in] block_size /// @param[in] head_mapping /// @param[in] scale /// @param[in] block_tables_ptr /// @param[in] max_num_blocks_per_seq /// @param[in] context_lens_ptr /// @param[in] max_context_len /// @param[in] alibi_slopes_ptr /// @param[in] workSpace The workspace pre-allocated. See the corresponding get /// workspace size helper function. /// @param[in] alibi_sqrt /// @return cuinferStatus_t CUINFERWINAPI cuInferPageAttentionFuse( cuinferHandle_t handle, void *__restrict__ out_ptr, cudaDataType_t outType, const void *__restrict__ query_ptr, const void *__restrict__ key_ptr, const void *__restrict__ value_ptr, cudaDataType_t queryType, int num_seqs, int num_heads, int num_kv_heads, int head_size, int query_stride, int key_stride, int value_stride, int kv_block_stride, int kv_head_stride, const void *__restrict__ key_cache_ptr, cudaDataType_t keyCacheType, const void *__restrict__ value_cache_ptr, cudaDataType_t valueCacheType, int block_size, const int *head_mapping, float scale, const int *__restrict__ block_tables_ptr, int max_num_blocks_per_seq, const int *__restrict__ context_lens_ptr, int max_context_len, const float *__restrict__ alibi_slopes_ptr, void *workspace = nullptr, bool alibi_sqrt = false); #if defined(__cplusplus) } #endif #endif /* CUINFER_H_ */ #pragma GCC visibility pop