feat: CUTLASS Cu10 grouped GEMM — real device verified
BI-V100 real device results: moe_group_gemm: err=0.000015 PASS moe_decode_cutlass: NaN=False PASS cutlass grouped: 4.77ms vs torch.mm loop: 9.38ms → 1.97x speedup Fix: gemm_grouped.cu ldb=K (not N) for ColumnMajor B view Link: -lcuinfer from /usr/local/corex-3.2.3/lib64/libcuinfer.so.7
This commit is contained in:
65
ex_engine/csrc/cuinfer_handle.h
Normal file
65
ex_engine/csrc/cuinfer_handle.h
Normal file
@@ -0,0 +1,65 @@
|
||||
// cuinfer_handle.h — Singleton handle manager for libcuinfer.so
|
||||
//
|
||||
// cuinferCreate/Destroy is expensive. This provides a thread-safe
|
||||
// singleton that creates once and reuses.
|
||||
//
|
||||
// Usage:
|
||||
// #include "cuinfer_handle.h"
|
||||
// cuinferHandle_t h = CuinferHandle::get(stream);
|
||||
//
|
||||
// Reference: ixformer::Context::default_cuinfer_handle (in libixformer.so)
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <cuda_runtime.h>
|
||||
#include <mutex>
|
||||
#include <cstdio>
|
||||
|
||||
// Forward-declare cuinfer C API
|
||||
extern "C" {
|
||||
|
||||
typedef struct cuinferContext* cuinferHandle_t;
|
||||
|
||||
typedef enum {
|
||||
CUINFER_STATUS_SUCCESS_H = 0,
|
||||
} cuinferStatus_h_t;
|
||||
|
||||
int cuinferCreate(cuinferHandle_t* handle);
|
||||
int cuinferDestroy(cuinferHandle_t handle);
|
||||
int cuinferSetStream(cuinferHandle_t handle, cudaStream_t stream);
|
||||
|
||||
} // extern "C"
|
||||
|
||||
|
||||
class CuinferHandle {
|
||||
public:
|
||||
static cuinferHandle_t get(cudaStream_t stream = nullptr) {
|
||||
static CuinferHandle instance;
|
||||
if (stream && stream != instance.last_stream_) {
|
||||
cuinferSetStream(instance.handle_, stream);
|
||||
instance.last_stream_ = stream;
|
||||
}
|
||||
return instance.handle_;
|
||||
}
|
||||
|
||||
private:
|
||||
cuinferHandle_t handle_ = nullptr;
|
||||
cudaStream_t last_stream_ = nullptr;
|
||||
|
||||
CuinferHandle() {
|
||||
int status = cuinferCreate(&handle_);
|
||||
if (status != 0) {
|
||||
fprintf(stderr, "[cuinfer_handle] WARNING: cuinferCreate failed (%d)\n", status);
|
||||
handle_ = nullptr;
|
||||
}
|
||||
}
|
||||
|
||||
~CuinferHandle() {
|
||||
if (handle_) {
|
||||
cuinferDestroy(handle_);
|
||||
}
|
||||
}
|
||||
|
||||
CuinferHandle(const CuinferHandle&) = delete;
|
||||
CuinferHandle& operator=(const CuinferHandle&) = delete;
|
||||
};
|
||||
Reference in New Issue
Block a user