feat: pybind wrapper for CUTLASS batched GEMM → MoE decode path
Based on verified result (issue #68): CUTLASS Cu10 TensorOp batched: 2.462ms (8 experts, 1 launch) vs 8× torch.matmul: 4.6ms (8 launches) vs Python F.linear loop: 10.36ms New files: ex_engine/xllm_kernels/cuda/bindings/corex_batched_gemm_bind.cpp pybind11 wrapper: batched_gemm_fp16() + moe_decode_fused() ex_engine/xllm_kernels/cuda/corex_batched_gemm_kernel.cu CUTLASS GemmBatched<half> kernel (from cat_files/batched_gemm.cu) qwen3_6_scripts/build_corex_batched_gemm.sh Build script for BI-V100 (ivcore10) Modified: qwen3_6_scripts/qwen3_5.py import corex_batched_gemm + _USE_COREX_BATCHED_GEMM flag Tier 1.5 in MoE decode: after corex_direct_routed, before corex_gather Build on device: bash qwen3_6_scripts/build_corex_batched_gemm.sh Output: prebuilt/corex-3.2.3-ivcore10/corex_batched_gemm.so
This commit is contained in:
dylan
155
ex_engine/xllm_kernels/cuda/bindings/corex_batched_gemm_bind.cpp
Normal file
155
ex_engine/xllm_kernels/cuda/bindings/corex_batched_gemm_bind.cpp
Normal file
@@ -0,0 +1,155 @@
|
||||
/*
|
||||
* corex_batched_gemm_bind.cpp — pybind11 wrapper for CUTLASS batched GEMM
|
||||
*
|
||||
* Verified on BI-V100: 2.462ms for 8-expert MoE decode (1×4096 @ 4096×11008)
|
||||
* vs 4.6ms for 8× torch.matmul, vs 10.36ms for Python F.linear loop.
|
||||
*
|
||||
* Call from qwen3_5.py MoE decode path (T==1):
|
||||
* import corex_batched_gemm
|
||||
* gate_up = corex_batched_gemm.batched_gemm_fp16(x, w13_sel) # (K, 2*I)
|
||||
* expert_out = corex_batched_gemm.batched_gemm_fp16(act, w2_sel) # (K, H)
|
||||
*
|
||||
* Source: cat_files/batched_gemm.cu (CUTLASS GemmBatched)
|
||||
* cat_files/gemm_batched.h (Iluvatar CoreX fork)
|
||||
*
|
||||
* Build: see qwen3_6_scripts/build_corex_batched_gemm.sh
|
||||
*/
|
||||
|
||||
#include <torch/extension.h>
|
||||
#include <cuda_runtime.h>
|
||||
#include <cuda_fp16.h>
|
||||
|
||||
// Forward declaration — implemented in corex_batched_gemm_kernel.cu
|
||||
// which uses CUTLASS GemmBatched with half precision
|
||||
cudaError_t cutlass_batched_hgemm(
|
||||
int m, int n, int k,
|
||||
__half const *A, int lda, long long int batch_stride_A,
|
||||
__half const *B, int ldb, long long int batch_stride_B,
|
||||
__half *C, int ldc, long long int batch_stride_C,
|
||||
int batch_count);
|
||||
|
||||
/*
|
||||
* batched_gemm_fp16: (batch, M, K) × (batch, K, N) → (batch, M, N)
|
||||
*
|
||||
* For MoE decode:
|
||||
* gate_up: x=(K,1,H), w13=(K,2I,H) → matmul(x, w13.T) → (K,1,2I)
|
||||
* i.e. batch=K=topk, M=1, K_dim=H, N=2I
|
||||
* down: act=(K,1,I), w2=(K,H,I) → matmul(act, w2.T) → (K,1,H)
|
||||
* i.e. batch=K=topk, M=1, K_dim=I, N=H
|
||||
*
|
||||
* Both A and B must be contiguous fp16 tensors on CUDA.
|
||||
*/
|
||||
torch::Tensor batched_gemm_fp16(
|
||||
torch::Tensor A, // (batch, M, K)
|
||||
torch::Tensor B) // (batch, N, K) — row-major weight, will be transposed
|
||||
{
|
||||
TORCH_CHECK(A.is_cuda() && B.is_cuda(), "inputs must be CUDA tensors");
|
||||
TORCH_CHECK(A.scalar_type() == torch::kFloat16 &&
|
||||
B.scalar_type() == torch::kFloat16,
|
||||
"inputs must be float16");
|
||||
TORCH_CHECK(A.is_contiguous() && B.is_contiguous(),
|
||||
"inputs must be contiguous");
|
||||
TORCH_CHECK(A.dim() == 3 && B.dim() == 3,
|
||||
"inputs must be 3D (batch, rows, cols)");
|
||||
|
||||
int batch = A.size(0);
|
||||
int M = A.size(1);
|
||||
int K = A.size(2);
|
||||
int N = B.size(1);
|
||||
TORCH_CHECK(B.size(0) == batch, "batch size mismatch");
|
||||
TORCH_CHECK(B.size(2) == K, "K dimension mismatch");
|
||||
|
||||
// Output: (batch, M, N)
|
||||
auto C = torch::zeros({batch, M, N}, A.options());
|
||||
|
||||
// CUTLASS uses column-major internally.
|
||||
// Our tensors are row-major: A(M,K), B(N,K)
|
||||
// We compute C = A × B^T in row-major = B × A^T in col-major
|
||||
// So pass: col-major B(K,N) × A(K,M) → C(N,M), then C is (M,N) row-major
|
||||
//
|
||||
// Actually for simplicity, compute as:
|
||||
// C(M,N) = A(M,K) × B^T(K,N)
|
||||
// In col-major: m_cm=N, n_cm=M, k_cm=K
|
||||
// A_cm = B^T → B stored as (N,K) row = (K,N) col, lda=K
|
||||
// B_cm = A^T → A stored as (M,K) row = (K,M) col, ldb=K
|
||||
// C_cm → C stored as (M,N) row = (N,M) col, ldc=N
|
||||
|
||||
int m_cm = N;
|
||||
int n_cm = M;
|
||||
int k_cm = K;
|
||||
int lda_cm = K; // B^T leading dim in col-major
|
||||
int ldb_cm = K; // A^T leading dim in col-major
|
||||
int ldc_cm = N; // C leading dim in col-major
|
||||
|
||||
long long int stride_A_cm = (long long int)N * K; // B batch stride
|
||||
long long int stride_B_cm = (long long int)M * K; // A batch stride
|
||||
long long int stride_C_cm = (long long int)M * N; // C batch stride
|
||||
|
||||
auto status = cutlass_batched_hgemm(
|
||||
m_cm, n_cm, k_cm,
|
||||
reinterpret_cast<const __half*>(B.data_ptr<at::Half>()),
|
||||
lda_cm, stride_A_cm,
|
||||
reinterpret_cast<const __half*>(A.data_ptr<at::Half>()),
|
||||
ldb_cm, stride_B_cm,
|
||||
reinterpret_cast<__half*>(C.data_ptr<at::Half>()),
|
||||
ldc_cm, stride_C_cm,
|
||||
batch);
|
||||
|
||||
TORCH_CHECK(status == cudaSuccess,
|
||||
"CUTLASS batched HGEMM failed: ", cudaGetErrorString(status));
|
||||
return C;
|
||||
}
|
||||
|
||||
/*
|
||||
* moe_decode_fused: Full MoE decode path using batched GEMM.
|
||||
*
|
||||
* hidden_states: (1, H)
|
||||
* w13_sel: (K, 2*I, H) — selected expert gate+up weights
|
||||
* w2_sel: (K, H, I) — selected expert down weights
|
||||
* topk_weights: (K,) — routing weights
|
||||
*
|
||||
* Returns: (1, H) — weighted sum of expert outputs
|
||||
*/
|
||||
torch::Tensor moe_decode_fused(
|
||||
torch::Tensor hidden_states, // (1, H)
|
||||
torch::Tensor w13_sel, // (K, 2*I, H)
|
||||
torch::Tensor w2_sel, // (K, H, I)
|
||||
torch::Tensor topk_weights) // (K,)
|
||||
{
|
||||
int K_experts = w13_sel.size(0);
|
||||
int two_I = w13_sel.size(1);
|
||||
int H = w13_sel.size(2);
|
||||
int I = two_I / 2;
|
||||
|
||||
// Expand hidden_states to (K, 1, H) for batched GEMM
|
||||
auto x = hidden_states.expand({K_experts, 1, H}).contiguous();
|
||||
|
||||
// Step 1: gate_up = batched_gemm(x, w13_sel) → (K, 1, 2*I)
|
||||
auto gate_up = batched_gemm_fp16(x, w13_sel); // (K, 1, 2I)
|
||||
gate_up = gate_up.squeeze(1); // (K, 2I)
|
||||
|
||||
// Step 2: SiLU activation + multiply
|
||||
auto chunks = gate_up.chunk(2, /*dim=*/1);
|
||||
auto act = torch::silu(chunks[0]) * chunks[1]; // (K, I)
|
||||
act = act.unsqueeze(1); // (K, 1, I)
|
||||
|
||||
// Step 3: expert_out = batched_gemm(act, w2_sel) → (K, 1, H)
|
||||
auto expert_out = batched_gemm_fp16(act, w2_sel); // (K, 1, H)
|
||||
expert_out = expert_out.squeeze(1); // (K, H)
|
||||
|
||||
// Step 4: Weighted reduction
|
||||
auto out = (expert_out * topk_weights.unsqueeze(1)).sum(0, true); // (1, H)
|
||||
return out.to(hidden_states.dtype());
|
||||
}
|
||||
|
||||
|
||||
PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
|
||||
m.doc() = "CUTLASS batched GEMM for MoE decode (BI-V100, Cu10 TensorOp)";
|
||||
m.def("batched_gemm_fp16", &batched_gemm_fp16,
|
||||
"Batched GEMM: (B,M,K) x (B,N,K)^T -> (B,M,N) in fp16",
|
||||
py::arg("A"), py::arg("B"));
|
||||
m.def("moe_decode_fused", &moe_decode_fused,
|
||||
"Full MoE decode: hidden(1,H) + w13(K,2I,H) + w2(K,H,I) + weights(K) -> out(1,H)",
|
||||
py::arg("hidden_states"), py::arg("w13_sel"),
|
||||
py::arg("w2_sel"), py::arg("topk_weights"));
|
||||
}
|
||||
70
ex_engine/xllm_kernels/cuda/corex_batched_gemm_kernel.cu
Normal file
70
ex_engine/xllm_kernels/cuda/corex_batched_gemm_kernel.cu
Normal file
@@ -0,0 +1,70 @@
|
||||
/*
|
||||
* corex_batched_gemm_kernel.cu — CUTLASS half-precision batched GEMM
|
||||
*
|
||||
* Uses cutlass::gemm::device::GemmBatched with Cu10 TensorOp (ivcore10).
|
||||
* Verified: 2.462ms for 8×(1×4096 @ 4096×11008) on BI-V100.
|
||||
*
|
||||
* Source: cat_files/batched_gemm.cu adapted from float to half.
|
||||
* cat_files/gemm_batched.h (Iluvatar CoreX CUTLASS fork)
|
||||
*/
|
||||
|
||||
#include <cuda_runtime.h>
|
||||
#include <cuda_fp16.h>
|
||||
|
||||
#include "cutlass/cutlass.h"
|
||||
#include "cutlass/layout/matrix.h"
|
||||
#include "cutlass/gemm/device/gemm_batched.h"
|
||||
#include "cutlass/numeric_types.h"
|
||||
|
||||
/*
|
||||
* Half-precision batched strided GEMM via CUTLASS.
|
||||
*
|
||||
* C[b] = A[b] × B[b] for b = 0..batch_count-1
|
||||
*
|
||||
* All matrices column-major.
|
||||
* The caller (corex_batched_gemm_bind.cpp) handles row-major ↔ col-major
|
||||
* transposition by swapping A/B and M/N.
|
||||
*/
|
||||
cudaError_t cutlass_batched_hgemm(
|
||||
int m, int n, int k,
|
||||
__half const *A, int lda, long long int batch_stride_A,
|
||||
__half const *B, int ldb, long long int batch_stride_B,
|
||||
__half *C, int ldc, long long int batch_stride_C,
|
||||
int batch_count)
|
||||
{
|
||||
using ElementA = cutlass::half_t;
|
||||
using ElementB = cutlass::half_t;
|
||||
using ElementC = cutlass::half_t;
|
||||
using ElementAccumulator = cutlass::half_t;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmBatched<
|
||||
ElementA, cutlass::layout::ColumnMajor, // A
|
||||
ElementB, cutlass::layout::ColumnMajor, // B
|
||||
ElementC, cutlass::layout::ColumnMajor, // C
|
||||
ElementAccumulator // accumulator
|
||||
>;
|
||||
|
||||
ElementAccumulator alpha_val(1.0f);
|
||||
ElementAccumulator beta_val(0.0f);
|
||||
|
||||
Gemm gemm_op;
|
||||
|
||||
cutlass::Status status = gemm_op({
|
||||
{m, n, k},
|
||||
{reinterpret_cast<ElementA const *>(A), lda},
|
||||
batch_stride_A,
|
||||
{reinterpret_cast<ElementB const *>(B), ldb},
|
||||
batch_stride_B,
|
||||
{reinterpret_cast<ElementC const *>(C), ldc},
|
||||
batch_stride_C,
|
||||
{reinterpret_cast<ElementC *>(C), ldc},
|
||||
batch_stride_C,
|
||||
{alpha_val, beta_val},
|
||||
batch_count
|
||||
});
|
||||
|
||||
if (status != cutlass::Status::kSuccess) {
|
||||
return cudaErrorUnknown;
|
||||
}
|
||||
return cudaSuccess;
|
||||
}
|
||||
90
qwen3_6_scripts/build_corex_batched_gemm.sh
Executable file
90
qwen3_6_scripts/build_corex_batched_gemm.sh
Executable file
@@ -0,0 +1,90 @@
|
||||
#!/bin/bash
|
||||
# Build corex_batched_gemm.so — CUTLASS batched GEMM pybind for MoE decode
|
||||
#
|
||||
# Run on BI-V100:
|
||||
# bash build_corex_batched_gemm.sh
|
||||
#
|
||||
# Output: corex_batched_gemm.so (deploy to vllm package dir)
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
PROJ_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||
EX_ENGINE="$PROJ_ROOT/ex_engine"
|
||||
|
||||
# Source files
|
||||
BIND_CPP="$EX_ENGINE/xllm_kernels/cuda/bindings/corex_batched_gemm_bind.cpp"
|
||||
KERNEL_CU="$EX_ENGINE/xllm_kernels/cuda/corex_batched_gemm_kernel.cu"
|
||||
|
||||
# CUTLASS headers from cat_files (Iluvatar CoreX fork)
|
||||
CUTLASS_INCLUDE="/usr/local/corex/include"
|
||||
if [ ! -d "$CUTLASS_INCLUDE/cutlass" ]; then
|
||||
# Fallback: check corex-samples
|
||||
CUTLASS_INCLUDE="/usr/local/corex/samples/cutlass/include"
|
||||
fi
|
||||
|
||||
# PyTorch/libtorch paths
|
||||
TORCH_DIR=$(python3 -c "import torch; print(torch.utils.cmake_prefix_path)" 2>/dev/null || echo "")
|
||||
TORCH_INCLUDE=$(python3 -c "import torch; print(torch.utils.cpp_extension.include_paths()[0])" 2>/dev/null || echo "/usr/local/corex/lib/python3/dist-packages/torch/include")
|
||||
TORCH_LIB=$(python3 -c "import torch; print(torch.utils.cpp_extension.library_paths()[0])" 2>/dev/null || echo "/usr/local/corex/lib/python3/dist-packages/torch/lib")
|
||||
PYTHON_INCLUDE=$(python3 -c "from sysconfig import get_path; print(get_path('include'))")
|
||||
|
||||
echo "[build] CUTLASS_INCLUDE=$CUTLASS_INCLUDE"
|
||||
echo "[build] TORCH_INCLUDE=$TORCH_INCLUDE"
|
||||
echo "[build] TORCH_LIB=$TORCH_LIB"
|
||||
|
||||
BUILD_DIR="/tmp/build_corex_batched_gemm"
|
||||
mkdir -p "$BUILD_DIR"
|
||||
OUT_SO="$SCRIPT_DIR/prebuilt/corex-3.2.3-ivcore10/corex_batched_gemm.so"
|
||||
|
||||
# Step 1: Compile CUTLASS kernel .cu → .o
|
||||
echo "[build] compiling kernel..."
|
||||
nvcc -c "$KERNEL_CU" \
|
||||
-o "$BUILD_DIR/kernel.o" \
|
||||
-I "$CUTLASS_INCLUDE" \
|
||||
-I "$TORCH_INCLUDE" \
|
||||
-I "$TORCH_INCLUDE/torch/csrc/api/include" \
|
||||
--gpu-architecture=ivcore10 \
|
||||
-std=c++17 -O2 \
|
||||
--expt-relaxed-constexpr \
|
||||
-Xcompiler -fPIC
|
||||
|
||||
# Step 2: Compile pybind .cpp → .o
|
||||
echo "[build] compiling pybind wrapper..."
|
||||
g++ -c "$BIND_CPP" \
|
||||
-o "$BUILD_DIR/bind.o" \
|
||||
-I "$TORCH_INCLUDE" \
|
||||
-I "$TORCH_INCLUDE/torch/csrc/api/include" \
|
||||
-I "$PYTHON_INCLUDE" \
|
||||
-I "$CUTLASS_INCLUDE" \
|
||||
-std=c++17 -O2 -fPIC \
|
||||
-D_GLIBCXX_USE_CXX11_ABI=0 \
|
||||
-DTORCH_EXTENSION_NAME=corex_batched_gemm
|
||||
|
||||
# Step 3: Link → .so
|
||||
echo "[build] linking..."
|
||||
g++ -shared \
|
||||
"$BUILD_DIR/kernel.o" \
|
||||
"$BUILD_DIR/bind.o" \
|
||||
-o "$OUT_SO" \
|
||||
-L "$TORCH_LIB" \
|
||||
-ltorch -ltorch_cpu -ltorch_cuda -lc10 -lc10_cuda \
|
||||
-L /usr/local/corex/lib64 -lcudart \
|
||||
-Wl,-rpath,"$TORCH_LIB" \
|
||||
-Wl,-rpath,/usr/local/corex/lib64
|
||||
|
||||
echo "[build] ✓ built $OUT_SO"
|
||||
echo "[build] size: $(du -h "$OUT_SO" | cut -f1)"
|
||||
|
||||
# Quick import test
|
||||
python3 -c "
|
||||
import torch
|
||||
torch.ops.load_library('$OUT_SO')
|
||||
import importlib.util
|
||||
spec = importlib.util.spec_from_file_location('corex_batched_gemm', '$OUT_SO')
|
||||
mod = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(mod)
|
||||
print('[build] ✓ import OK, functions:', [x for x in dir(mod) if not x.startswith('_')])
|
||||
" 2>&1 || echo "[build] import test skipped (no GPU)"
|
||||
|
||||
echo "[build] done"
|
||||
@@ -138,6 +138,11 @@ try:
|
||||
except ImportError:
|
||||
_corex_moe_direct_routed = None
|
||||
|
||||
try:
|
||||
from vllm import corex_batched_gemm as _corex_batched_gemm
|
||||
except ImportError:
|
||||
_corex_batched_gemm = None
|
||||
|
||||
try:
|
||||
from vllm import corex_moe_topk_softmax as _corex_moe_topk_softmax
|
||||
except ImportError:
|
||||
@@ -204,6 +209,9 @@ _USE_COREX_MOE_WEIGHT_GATHER = (
|
||||
_USE_COREX_MOE_DIRECT_ROUTED = (
|
||||
_corex_moe_direct_routed is not None
|
||||
and env_bool("BI100_MOE_COREX_DIRECT_ROUTED", False))
|
||||
_USE_COREX_BATCHED_GEMM = (
|
||||
_corex_batched_gemm is not None
|
||||
and env_bool("BI100_MOE_BATCHED_GEMM", True))
|
||||
_USE_COREX_MOE_TOPK_SOFTMAX = (
|
||||
_corex_moe_topk_softmax is not None
|
||||
and env_bool("BI100_MOE_COREX_TOPK_SOFTMAX", True))
|
||||
@@ -1729,6 +1737,15 @@ class Qwen3_5MoeSparseBlock(nn.Module):
|
||||
return _corex_moe_direct_routed.w2_reduce(
|
||||
act, w2, eids, ws)
|
||||
|
||||
# Tier 1.5: CUTLASS batched GEMM (verified 2.462ms, issue #68)
|
||||
# 1 launch for 8 experts vs 8 launches for F.linear loop
|
||||
if (_USE_COREX_BATCHED_GEMM
|
||||
and hidden_states.dtype == torch.float16
|
||||
and w13.dtype == torch.float16
|
||||
and w2.dtype == torch.float16):
|
||||
return _corex_batched_gemm.moe_decode_fused(
|
||||
hidden_states, w13[eids], w2[eids], ws)
|
||||
|
||||
use_corex_gather = (
|
||||
_USE_COREX_MOE_WEIGHT_GATHER
|
||||
and hidden_states.dtype == torch.float16
|
||||
|
||||
Reference in New Issue
Block a user