3581dd5435
Layer 1: hw_config.h (245 lines) — BI-V100 hardware descriptor + tuning tables Layer 2: moe_pipeline.py (461 lines) — MoE 7-step pipeline orchestrator Layer 3: attn_dispatch.py (270 lines) — Attention prefill/decode dispatch Layer 4: ilu_ops_api.h (182 lines) — Dispatch signature contract Layer 5: kernel_moe_ops.cpp (155 lines) — MoE kernel-level ops wrappers Layer 6: kernel_elem_ops.cpp (210 lines) — Element-wise kernel wrappers Layer 7: ixformer_infer.h (246 lines) — ixformer::infer namespace contract Layer 8: factor_topk_softmax.cu (456 lines) — MoE routing CUDA kernel Layer 9: factor_moe_compute_index.cu (174 lines) — Token index CUDA kernel Layer 10: factor_moe_combine.cu (154 lines) — Weighted combine CUDA kernel Total: 2553 lines across 10 files (h/cpp/cu/py) Upstream reference: 2787 lines across corresponding 10 xllm AST layers Each file follows the read-read-read-write pattern from upstream xllm, ds_vllm, and fla repos. No hand-written inference code — all kernel logic is cat-migrated from the upstream references.
246 lines
9.2 KiB
C
246 lines
9.2 KiB
C
// ex_engine/factors/hw_config.h
|
||
//
|
||
// Layer 1: Hardware descriptor + per-algorithm tuning tables
|
||
//
|
||
// Upstream parallel: xllm_models/llm/qwen3_5.h (model-level config)
|
||
// → defines num_experts=64, top_k=8, hidden_size=3584, intermediate=18944
|
||
// → binds GDN layers [1,7,13,19] vs full attention layers
|
||
//
|
||
// This file defines the BI-V100 hardware descriptor and per-factor
|
||
// tuning tables. Every downstream factor file #includes this to get
|
||
// compile-time constants that are calibrated to the real device.
|
||
//
|
||
// Source: cat_files/arch.h (real nm -D symbol dump from BI-V100)
|
||
// cat_files/iluvatar_mma.hpp (tensor core instruction set)
|
||
// sub694rizhi.txt TPS distribution (avg 6.2, target >12)
|
||
|
||
#ifndef EX_FACTORS_HW_CONFIG_H
|
||
#define EX_FACTORS_HW_CONFIG_H
|
||
|
||
#include <stdint.h>
|
||
|
||
#ifdef __cplusplus
|
||
extern "C" {
|
||
#endif
|
||
|
||
// =========================================================================
|
||
// BI-V100 hardware constants (from real device probes)
|
||
// =========================================================================
|
||
// Source: cat_files/arch.h, HARDWARE_PROBE_20260808.md
|
||
#define EX_SM_MAJOR 7
|
||
#define EX_SM_MINOR 0
|
||
#define EX_SM_COUNT 16
|
||
#define EX_WARP_SIZE 32
|
||
#define EX_MAX_THREADS_SM 2048
|
||
#define EX_SMEM_PER_SM 49152 // bytes
|
||
#define EX_SMEM_PER_BLOCK 49152 // bytes, BI-V100 no bank partitioning
|
||
#define EX_L2_CACHE_BYTES (6 * 1024 * 1024) // 6 MB
|
||
#define EX_MEM_BW_GBS 900.0f // total ~900 GB/s (56 GB/s per SM × 16)
|
||
|
||
// Register file: 65536 32-bit registers per SM, 256 per warp
|
||
// Source: probed via cudaDeviceProp
|
||
#define EX_REGS_PER_SM 65536
|
||
#define EX_REGS_PER_BLOCK 65536
|
||
|
||
// CoreX clang/16 CUDA 10.2 compatibility constraints
|
||
// Source: cat_files/cutlass.h (Iluvatar CoreX copyright block)
|
||
// - No cp.async (requires SM80+)
|
||
// - No TMA (requires SM90+)
|
||
// - Warp shuffle via __shfl_sync, __shfl_xor_sync, __shfl_down_sync
|
||
// - atomicAdd for float available (SM70)
|
||
// - half2 arithmetic available via __hmul2, __hadd2 etc.
|
||
#define EX_HAS_CP_ASYNC 0
|
||
#define EX_HAS_TMA 0
|
||
#define EX_HAS_WARP_SHUFFLE 1
|
||
#define EX_HAS_HALF2 1
|
||
|
||
// =========================================================================
|
||
// Qwen3.5-27B model constants (from SYSTEM_DESIGN.md)
|
||
// =========================================================================
|
||
#define EX_NUM_LAYERS 64 // decoder layers
|
||
#define EX_NUM_EXPERTS 64 // routed experts per MoE layer
|
||
#define EX_MOE_TOPK 8 // active experts per token
|
||
#define EX_HIDDEN_SIZE 3584 // hidden dimension
|
||
#define EX_INTERMEDIATE_SIZE 18944 // MoE intermediate (per expert, pre-TP)
|
||
#define EX_NUM_HEADS 28 // attention heads
|
||
#define EX_NUM_KV_HEADS 4 // GQA kv heads
|
||
#define EX_HEAD_DIM 128 // per-head dimension
|
||
#define EX_GDN_LAYERS_COUNT 4 // GatedDeltaNet layers: [1,7,13,19]
|
||
#define EX_ATTN_LAYERS_COUNT 32 // full attention layers (of 36 attention layers)
|
||
#define EX_MOE_LAYERS_COUNT 64 // all 64 layers have MoE
|
||
|
||
// TP=4 sliced sizes (actual runtime)
|
||
#define EX_TP_SIZE 4
|
||
#define EX_INTER_PER_TP (EX_INTERMEDIATE_SIZE / EX_TP_SIZE) // 4736
|
||
|
||
// =========================================================================
|
||
// Per-factor tuning parameter structs
|
||
// =========================================================================
|
||
// Mirrors CCCL's per-algorithm ReducePassPolicy/ScanPolicy pattern
|
||
// Each factor reads these at compile time or load time to set
|
||
// grid/block dims and SMEM allocation.
|
||
|
||
typedef struct {
|
||
int threads_per_block;
|
||
int items_per_thread; // values per thread (VPT)
|
||
int vec_size; // vector load width
|
||
int smem_bytes; // shared memory per block
|
||
int num_warps; // threads_per_block / WARP_SIZE
|
||
int num_stages; // software pipeline stages
|
||
int grid_scale; // multiplier for grid dim (1 = 1:1 with N)
|
||
} ex_factor_tune_t;
|
||
|
||
// =========================================================================
|
||
// BI-V100 tuning tables — one entry per factor
|
||
// =========================================================================
|
||
// Source: CCCL tuning headers adapted via muh toolchain
|
||
// sub168 TPS benchmarks, sub694 per-request TPS distribution
|
||
//
|
||
// Key tuning rationale per sub694 data:
|
||
// - avg TPS = 6.2, ceiling TPS = 11.9 (decode-only, low prompt_tok)
|
||
// - 227/824 requests below 5 TPS → bottleneck is prefill-heavy requests
|
||
// - prompt_tok >100K → TPS drops to 0.4-1.2 (GEMM bound)
|
||
// - prompt_tok <10K → TPS reaches 9-11 (compute matches HW)
|
||
// - Goal: double TPS on prompt_tok 10K-50K range (bulk of traffic)
|
||
|
||
// Factor 0: MOE_TOPK_SOFTMAX
|
||
// 64 experts, VPT=2 → 32 threads = 1 warp per token row
|
||
// 4 warps per CTA → 4 tokens per CTA
|
||
// No SMEM needed (all warp shuffle)
|
||
// Upstream: moe_topk_softmax_kernels.cuh topkGatingSoftmax
|
||
static const ex_factor_tune_t EX_TUNE_MOE_TOPK_SOFTMAX = {
|
||
.threads_per_block = 128, // 4 warps × 32
|
||
.items_per_thread = 2, // 64 experts / 32 threads
|
||
.vec_size = 1, // float, no vectorization
|
||
.smem_bytes = 0, // pure warp shuffle
|
||
.num_warps = 4,
|
||
.num_stages = 1,
|
||
.grid_scale = 1, // ceil(N / 4) blocks
|
||
};
|
||
|
||
// Factor 1: MOE_ALIGN_BLOCK
|
||
// Simple histogram + prefix sum, 256 threads per block
|
||
// SMEM for CUB BlockScan
|
||
static const ex_factor_tune_t EX_TUNE_MOE_ALIGN_BLOCK = {
|
||
.threads_per_block = 256,
|
||
.items_per_thread = 1,
|
||
.vec_size = 1,
|
||
.smem_bytes = 1024, // CUB BlockScan TempStorage
|
||
.num_warps = 8,
|
||
.num_stages = 1,
|
||
.grid_scale = 1,
|
||
};
|
||
|
||
// Factor 2: MOE_FUSED_GEMM
|
||
// Group GEMM: cublas or CUTLASS batched
|
||
// BI-V100: cublas SM70 hgemm, M×N×K per expert
|
||
// threads/smem managed by cublas internally
|
||
static const ex_factor_tune_t EX_TUNE_MOE_FUSED_GEMM = {
|
||
.threads_per_block = 256, // cublas managed
|
||
.items_per_thread = 4,
|
||
.vec_size = 8, // half8 loads
|
||
.smem_bytes = 49152, // full SMEM for GEMM tiles
|
||
.num_warps = 8,
|
||
.num_stages = 2, // SW pipeline (no cp.async, manual prefetch)
|
||
.grid_scale = 1,
|
||
};
|
||
|
||
// Factor 3: GELU_TANH_MUL
|
||
// Element-wise, memory-bandwidth bound
|
||
// 256 threads, vec4 half loads
|
||
static const ex_factor_tune_t EX_TUNE_GELU_TANH_MUL = {
|
||
.threads_per_block = 256,
|
||
.items_per_thread = 4,
|
||
.vec_size = 4, // half4 = 8 bytes
|
||
.smem_bytes = 0, // pure register
|
||
.num_warps = 8,
|
||
.num_stages = 1,
|
||
.grid_scale = 1,
|
||
};
|
||
|
||
// Factor 4: BATCHED_ROTARY
|
||
// Per-head, per-position rotation
|
||
// Each thread handles one (cos, sin) pair
|
||
static const ex_factor_tune_t EX_TUNE_BATCHED_ROTARY = {
|
||
.threads_per_block = 512,
|
||
.items_per_thread = 2, // 2 elements per rotation pair
|
||
.vec_size = 2,
|
||
.smem_bytes = 0,
|
||
.num_warps = 16,
|
||
.num_stages = 1,
|
||
.grid_scale = 1,
|
||
};
|
||
|
||
// Factor 5: GDN_CHUNK_FWD
|
||
// Chunked prefill: intra-chunk QK^T + inter-chunk state update
|
||
// Critical: fp32 accumulation to prevent NaN
|
||
// chunk_size=64 (from fla upstream), head_dim=128
|
||
// Each CTA processes one (batch, head, chunk)
|
||
static const ex_factor_tune_t EX_TUNE_GDN_CHUNK_FWD = {
|
||
.threads_per_block = 128,
|
||
.items_per_thread = 4,
|
||
.vec_size = 4,
|
||
.smem_bytes = 32768, // Q,K,V tiles for chunk_size=64, dim=128, fp16
|
||
.num_warps = 4,
|
||
.num_stages = 2,
|
||
.grid_scale = 1,
|
||
};
|
||
|
||
// Factor 6: GDN_RECURRENT
|
||
// Single-step decode: conv1d_state update + delta_rule recurrence
|
||
// Lightweight: one token per step
|
||
static const ex_factor_tune_t EX_TUNE_GDN_RECURRENT = {
|
||
.threads_per_block = 128,
|
||
.items_per_thread = 4,
|
||
.vec_size = 4,
|
||
.smem_bytes = 8192, // conv state + temporal state
|
||
.num_warps = 4,
|
||
.num_stages = 1,
|
||
.grid_scale = 1,
|
||
};
|
||
|
||
// Factor 7: CACHE_APPEND (reshape_and_cache for paged KV)
|
||
// Each thread handles one token's K or V slice
|
||
static const ex_factor_tune_t EX_TUNE_CACHE_APPEND = {
|
||
.threads_per_block = 256,
|
||
.items_per_thread = 4,
|
||
.vec_size = 4,
|
||
.smem_bytes = 0,
|
||
.num_warps = 8,
|
||
.num_stages = 1,
|
||
.grid_scale = 1,
|
||
};
|
||
|
||
// Factor 8: RESHAPE_CACHE_FLASH
|
||
// Optimized cache write for flash attention layout
|
||
static const ex_factor_tune_t EX_TUNE_RESHAPE_CACHE_FLASH = {
|
||
.threads_per_block = 256,
|
||
.items_per_thread = 4,
|
||
.vec_size = 4,
|
||
.smem_bytes = 0,
|
||
.num_warps = 8,
|
||
.num_stages = 1,
|
||
.grid_scale = 1,
|
||
};
|
||
|
||
// =========================================================================
|
||
// Tuning table array (indexed by factor_id)
|
||
// =========================================================================
|
||
static const ex_factor_tune_t* const EX_TUNE_TABLE[] = {
|
||
&EX_TUNE_MOE_TOPK_SOFTMAX, // 0
|
||
&EX_TUNE_MOE_ALIGN_BLOCK, // 1
|
||
&EX_TUNE_MOE_FUSED_GEMM, // 2
|
||
&EX_TUNE_GELU_TANH_MUL, // 3
|
||
&EX_TUNE_BATCHED_ROTARY, // 4
|
||
&EX_TUNE_GDN_CHUNK_FWD, // 5
|
||
&EX_TUNE_GDN_RECURRENT, // 6
|
||
&EX_TUNE_CACHE_APPEND, // 7
|
||
&EX_TUNE_RESHAPE_CACHE_FLASH, // 8
|
||
};
|
||
|
||
#ifdef __cplusplus
|
||
}
|
||
#endif
|
||
|
||
#endif // EX_FACTORS_HW_CONFIG_H
|