feat: import CUDA kernels from xllm/CCCL/FLA upstream repos
Sources cloned and tree'd (no --depth):
- jd-opensource/xllm: ILU kernels, CUDA kernels, MoE kernels
- NVIDIA/cccl: CUB tuning/dispatch headers (block-level primitives)
- fla-org/flash-linear-attention: Triton GDN kernels
- NVIDIA/cutlass: grouped GEMM reference (read, not copied)
- Dao-AILab/flash-attention: attention kernel reference (SM80+, read only)
New CUDA kernels (from xllm, SM-agnostic, portable to BI-V100):
ex_engine/xllm_kernels/cuda/activation.cu (188 lines) — silu_and_mul, gelu
ex_engine/xllm_kernels/cuda/norm.cu (600 lines) — rms_norm, fused_add_rms_norm
ex_engine/xllm_kernels/cuda/rope.cu (258 lines) — rotary_embedding
ex_engine/xllm_kernels/cuda/block_copy.cu (209 lines) — copy_blocks, swap_blocks
ex_engine/xllm_kernels/cuda/reshape_paged_cache.cu (101 lines) — KV cache ops
ex_engine/xllm_kernels/cuda/headers/ (5 headers for compilation)
ILU bridge kernel sources (from xllm, verified SAME as upstream):
ex_engine/xllm_kernels/ilu/ (10 files, 925 lines total)
— activation.cpp, attention.cpp, fused_moe.cpp, group_gemm.cpp,
matmul.cpp, norm.cpp, rope.cpp, ilu_ops_api.h, ixformer.h, utils.h
FLA Triton GDN kernels (for GatedDeltaNet without SM90+ FlashQLA):
ex_engine/fla_kernels/gated_delta_rule/ (7 files, 2370 lines)
— chunk_fwd.py (428), chunk.py (487), wy_fast.py (409),
fused_recurrent.py (392), naive.py (161), gate.py (380)
CCCL sync (12 tuning + 14 dispatch headers updated from NVIDIA/cccl):
cccl_upstream/cub/cub/device/dispatch/tuning/ — 12 changed files synced
cccl_upstream/cub/cub/device/dispatch/ — 14 changed dispatch files synced
Compilation targets for real machine (ivcore10):
1. CUDA kernels: --cuda-gpu-arch=ivcore10 via corex clang/16
2. ILU bridges: torch.utils.cpp_extension linking ixformer .so
3. FLA kernels: Triton JIT (if Triton works on BI-V100)
This commit is contained in:
73
ex_engine/xllm_kernels/ilu/matmul.cpp
Normal file
73
ex_engine/xllm_kernels/ilu/matmul.cpp
Normal file
@@ -0,0 +1,73 @@
|
||||
/* Copyright 2025-2026 The xLLM Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
https://github.com/jd-opensource/xllm/blob/main/LICENSE
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#include "ilu_ops_api.h"
|
||||
#include "util/env_var.h"
|
||||
|
||||
namespace xllm::kernel::ilu {
|
||||
|
||||
bool gemv_conditions(const torch::Tensor& input,
|
||||
const torch::Tensor& weight,
|
||||
const torch::Tensor& bias,
|
||||
int64_t gemv_max_batch) {
|
||||
// gemv input:[m,k] weight:[n,k]
|
||||
// 1. m <= gemv_max_batch
|
||||
// 2. k % 32 == 0 && n % 2 == 0
|
||||
// 3. bias is None
|
||||
|
||||
torch::Tensor input_view = input.view({-1, input.size(-1)});
|
||||
torch::Tensor weight_view = weight.view({-1, weight.size(-1)});
|
||||
|
||||
int64_t m = input_view.size(0);
|
||||
int64_t k = input_view.size(1);
|
||||
int64_t n = weight_view.size(0);
|
||||
|
||||
if (bias.defined() == false && m <= gemv_max_batch && k % 32 == 0 &&
|
||||
n % 2 == 0) {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
torch::Tensor matmul(torch::Tensor a,
|
||||
torch::Tensor b,
|
||||
std::optional<torch::Tensor> bias) {
|
||||
int64_t act_type = -1;
|
||||
bool persistent = false;
|
||||
std::vector<int64_t> output_shape = a.sizes().vec();
|
||||
if (!output_shape.empty()) {
|
||||
output_shape[output_shape.size() - 1] = b.size(0);
|
||||
}
|
||||
torch::Tensor output = a.new_empty(output_shape);
|
||||
|
||||
bool use_gemv = true;
|
||||
const int64_t gemv_max_batch = 1;
|
||||
const bool disable_infer_gemm_ex =
|
||||
xllm::util::get_bool_env("DISABLE_INFER_GEMM_EX", false);
|
||||
|
||||
use_gemv =
|
||||
use_gemv &&
|
||||
gemv_conditions(a, b, bias.value_or(at::Tensor()), gemv_max_batch) &&
|
||||
!disable_infer_gemm_ex && (act_type == -1);
|
||||
|
||||
if (use_gemv) {
|
||||
output = infer::ixformer_linear_ex(a, b, bias, output);
|
||||
} else {
|
||||
output = infer::ixformer_linear(a, b, act_type, bias, output, persistent);
|
||||
}
|
||||
return output;
|
||||
}
|
||||
|
||||
} // namespace xllm::kernel::ilu
|
||||
Reference in New Issue
Block a user