feat(EX): wire xllm CUB topk_softmax kernel into MoE routing
Upstream: xllm/kernels/cuda/moe/moe_topk_softmax_kernels.cuh (Apache 2.0)
Adapted: CHECK→TORCH_CHECK, include path fix, cuda/functional guard, pybind11
Call chain now:
qwen3_5.py:_pure_pytorch_experts()
→ _ex_moe_topk_softmax (fused CUB kernel, 1 launch)
→ fallback: torch.softmax + torch.topk (3 launches)
Files:
ex_engine/csrc/moe/moe_topk_softmax_kernels.cuh — xllm kernel (adapted)
ex_engine/csrc/moe/device_utils.cuh — xllm device utils
ex_engine/csrc/moe/moe_topk_softmax_ext.cu — pybind11 wrapper
ex_engine/python/moe_topk.py — JIT loader (same pattern as flash_qla_sm70)
qwen3_5.py — import + use in _pure_pytorch_experts()
patch_ops.sh — deploy kernel sources for JIT
This commit is contained in:
84
ex_engine/python/moe_topk.py
Normal file
84
ex_engine/python/moe_topk.py
Normal file
@@ -0,0 +1,84 @@
|
||||
"""
|
||||
ex_engine/python/moe_topk.py — MoE topk_softmax CUDA kernel loader
|
||||
|
||||
Loads the xllm-derived CUB-based fused softmax+topk kernel.
|
||||
JIT compiled via torch.utils.cpp_extension.load() on BI-V100.
|
||||
|
||||
Usage:
|
||||
from ex_engine.python.moe_topk import moe_topk_softmax
|
||||
moe_topk_softmax(topk_weights, topk_ids, token_expert_indices, gating_output)
|
||||
"""
|
||||
|
||||
import os
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
import torch
|
||||
|
||||
logger = logging.getLogger("ex_engine.moe_topk")
|
||||
|
||||
_EXT = None
|
||||
|
||||
|
||||
def _load_ext():
|
||||
global _EXT
|
||||
if _EXT is not None:
|
||||
return _EXT
|
||||
if not torch.cuda.is_available():
|
||||
raise RuntimeError("MoE topk_softmax kernel requires CUDA.")
|
||||
|
||||
os.environ.setdefault("TORCH_CUDA_ARCH_LIST", "7.0;7.5")
|
||||
|
||||
csrc_dir = Path(__file__).parent.parent / "csrc" / "moe"
|
||||
|
||||
# Try precompiled .so first
|
||||
build_dir = Path(__file__).parent.parent / "build"
|
||||
if build_dir.is_dir():
|
||||
so_files = list(build_dir.glob("ex_moe_topk*.so"))
|
||||
if so_files:
|
||||
try:
|
||||
from torch.utils.cpp_extension import load
|
||||
_EXT = load(
|
||||
name="ex_moe_topk_softmax",
|
||||
sources=[],
|
||||
build_directory=str(build_dir),
|
||||
verbose=False,
|
||||
)
|
||||
return _EXT
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# JIT compile
|
||||
from torch.utils.cpp_extension import load
|
||||
sources = [str(csrc_dir / "moe_topk_softmax_ext.cu")]
|
||||
_EXT = load(
|
||||
name="ex_moe_topk_softmax",
|
||||
sources=sources,
|
||||
extra_cuda_cflags=["-O3", "-I" + str(csrc_dir)],
|
||||
extra_cflags=["-O3"],
|
||||
verbose=bool(int(os.environ.get("EX_MOE_VERBOSE_BUILD", "0"))),
|
||||
)
|
||||
logger.info("MoE topk_softmax CUDA kernel compiled successfully")
|
||||
return _EXT
|
||||
|
||||
|
||||
def moe_topk_softmax(
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
token_expert_indices: torch.Tensor,
|
||||
gating_output: torch.Tensor,
|
||||
renormalize: bool = False,
|
||||
) -> None:
|
||||
"""
|
||||
Drop-in replacement for ixf_F.vllm_moe_topk_softmax.
|
||||
|
||||
Interface matches _custom_ops.topk_softmax() exactly:
|
||||
topk_weights: [num_tokens, topk] float32, output
|
||||
topk_ids: [num_tokens, topk] int32, output
|
||||
token_expert_indices: [num_tokens, topk] int32, output
|
||||
gating_output: [num_tokens, num_experts] input
|
||||
"""
|
||||
ext = _load_ext()
|
||||
ext.topk_softmax(topk_weights, topk_ids, token_expert_indices,
|
||||
gating_output, renormalize)
|
||||
Reference in New Issue
Block a user