diff --git a/CCCL_MUH_PARITY_AUDIT.md b/CCCL_MUH_PARITY_AUDIT.md new file mode 100644 index 00000000..1186efca --- /dev/null +++ b/CCCL_MUH_PARITY_AUDIT.md @@ -0,0 +1,134 @@ +================================================================================ +CCCL vs muh 精确比对审计报告 +================================================================================ + +### 1. scale_mem_bound 函数 parity check +------------------------------------------------------------ + float32 (CCCL SM100 reduce) CCCL=( 16i, 512t,tile= 32768B) muh=( 16i, 512t,tile= 32768B) ✓ + float64 (CCCL SM100 reduce) CCCL=( 8i, 640t,tile= 40960B) muh=( 8i, 640t,tile= 40960B) ✓ + accum8 (CCCL SM100 reduce) CCCL=( 7i, 512t,tile= 28672B) muh=( 7i, 512t,tile= 28672B) ✓ + scan 4B (CCCL SM100 scan) CCCL=( 22i, 384t,tile= 33792B) muh=( 22i, 384t,tile= 33792B) ✓ + scan 8B (CCCL SM100 scan) CCCL=( 11i, 416t,tile= 36608B) muh=( 11i, 416t,tile= 36608B) ✓ + det float32 SM90 CCCL=( 13i, 224t,tile= 11648B) muh=( 13i, 224t,tile= 11648B) ✓ + det float64 SM86 CCCL=( 5i, 128t,tile= 5120B) muh=( 5i, 128t,tile= 5120B) ✓ + 1-byte type CCCL=( 32i, 256t,tile= 8192B) muh=( 32i, 256t,tile= 8192B) ✓ + 2-byte type CCCL=( 32i, 256t,tile= 16384B) muh=( 32i, 256t,tile= 16384B) ✓ + 16-byte type (int128) CCCL=( 4i, 256t,tile= 16384B) muh=( 4i, 256t,tile= 16384B) ✓ + SMEM cap test (should trigger) CCCL=( 8i, 768t,tile= 49152B) muh=( 8i, 768t,tile= 49152B) ✓ + → scale_mem_bound: FULL PARITY ✓ + +### 2. reduce tuning: CCCL SM100值 → BI-V100 scale_mem_bound适配后 +------------------------------------------------------------ + CCCL benchmarked on SM100 → muh should use scale_mem_bound for BI-V100 + Key: reduce loads to REGISTERS not SMEM → SMEM cap rarely triggers + + float32_plus_o4 @4B: scaled=(16i, 512t) tile= 32768B (66.7%) + float32_plus_o4 @8B: scaled=( 8i, 512t) tile= 32768B (66.7%) + float64_plus_o4 @4B: scaled=(16i, 640t) tile= 40960B (83.3%) + float64_plus_o4 @8B: scaled=( 8i, 640t) tile= 40960B (83.3%) + accum8_plus_o4 @4B: scaled=(15i, 512t) tile= 30720B (62.5%) + accum8_plus_o4 @8B: scaled=( 7i, 512t) tile= 28672B (58.3%) + accum8_plus_o8 @4B: scaled=(15i, 512t) tile= 30720B (62.5%) + accum8_plus_o8 @8B: scaled=( 7i, 512t) tile= 28672B (58.3%) + det_float32_sm90 @4B: scaled=(13i, 224t) tile= 11648B (23.7%) + det_float32_sm90 @8B: scaled=( 6i, 224t) tile= 10752B (21.9%) + det_float32_sm86 @4B: scaled=( 6i, 224t) tile= 5376B (10.9%) + det_float32_sm86 @8B: scaled=( 3i, 224t) tile= 5376B (10.9%) + det_float64_sm86 @4B: scaled=(11i, 128t) tile= 5632B (11.5%) + det_float64_sm86 @8B: scaled=( 5i, 128t) tile= 5120B (10.4%) + default_fallback @4B: scaled=(16i, 256t) tile= 16384B (33.3%) + default_fallback @8B: scaled=( 8i, 256t) tile= 16384B (33.3%) + +### 3. muh bi100 reduce当前值 vs CCCL参考 +------------------------------------------------------------ + muh改用了更大的items (24 vs SM100的16)来补偿16 SMs + 这是对的——reduce加载到寄存器,SMEM不是瓶颈 + + ★ float32 plus (paged_attention score reduction — 83% weight): + CCCL SM100: items=16, threads=512, vec=2 + muh BI-V100: items=24, threads=512, vec=2 + 理由: 16 SMs vs 148 SMs, 每个CTA需要处理更多数据 + tile对比: SM100=512*16*4=32768B | BI-V100=512*24*4=49152B (exactly 48KB) + → items=24 用满了SMEM → 合理但有风险,如果BlockReduce实际占SMEM则溢出 + → 但注释说reduce不用BlockLoad(loads to registers) → 安全 + +### 4. scan tuning: CCCL SM100 → BI-V100 SMEM约束 +------------------------------------------------------------ + Scan DOES use BlockLoad staging in SMEM → tile_bytes ≤ 49152 is HARD + + lookback_1B_o4 @1B: tpb= 512 ipt=18 tile= 9216B ✓ + lookback_1B_o4 @2B: tpb= 512 ipt=18 tile= 18432B ✓ + lookback_1B_o4 @4B: tpb= 512 ipt=18 tile= 36864B ✓ + lookback_1B_o4 @8B: tpb= 512 ipt=18 tile= 73728B ✗ OVERFLOW → max_items=12 + lookback_2B_o4 @1B: tpb= 512 ipt=13 tile= 6656B ✓ + lookback_2B_o4 @2B: tpb= 512 ipt=13 tile= 13312B ✓ + lookback_2B_o4 @4B: tpb= 512 ipt=13 tile= 26624B ✓ + lookback_2B_o4 @8B: tpb= 512 ipt=13 tile= 53248B ✗ OVERFLOW → max_items=12 + lookback_4B_o4 @1B: tpb= 384 ipt=22 tile= 8448B ✓ + lookback_4B_o4 @2B: tpb= 384 ipt=22 tile= 16896B ✓ + lookback_4B_o4 @4B: tpb= 384 ipt=22 tile= 33792B ✓ + lookback_4B_o4 @8B: tpb= 384 ipt=22 tile= 67584B ✗ OVERFLOW → max_items=16 + lookback_8B_o4 @1B: tpb= 416 ipt=23 tile= 9568B ✓ + lookback_8B_o4 @2B: tpb= 416 ipt=23 tile= 19136B ✓ + lookback_8B_o4 @4B: tpb= 416 ipt=23 tile= 38272B ✓ + lookback_8B_o4 @8B: tpb= 416 ipt=23 tile= 76544B ✗ OVERFLOW → max_items=14 + lookback_1B_o8 @1B: tpb= 384 ipt=14 tile= 5376B ✓ + lookback_1B_o8 @2B: tpb= 384 ipt=14 tile= 10752B ✓ + lookback_1B_o8 @4B: tpb= 384 ipt=14 tile= 21504B ✓ + lookback_1B_o8 @8B: tpb= 384 ipt=14 tile= 43008B ✓ + lookback_4B_o8 @1B: tpb= 416 ipt=19 tile= 7904B ✓ + lookback_4B_o8 @2B: tpb= 416 ipt=19 tile= 15808B ✓ + lookback_4B_o8 @4B: tpb= 416 ipt=19 tile= 31616B ✓ + lookback_4B_o8 @8B: tpb= 416 ipt=19 tile= 63232B ✗ OVERFLOW → max_items=14 + lookback_8B_o8 @1B: tpb= 320 ipt=22 tile= 7040B ✓ + lookback_8B_o8 @2B: tpb= 320 ipt=22 tile= 14080B ✓ + lookback_8B_o8 @4B: tpb= 320 ipt=22 tile= 28160B ✓ + lookback_8B_o8 @8B: tpb= 320 ipt=22 tile= 56320B ✗ OVERFLOW → max_items=19 + + 关键发现: + - scan lookback_4B_o4: items=22, threads=384 → tile@4B=33792 ✓ tile@8B=67584 ✗ + - scan lookback_8B_o4: items=23, threads=416 → tile@8B=76544 ✗ + - 这些值在SM100上是安全的(228KB SMEM),但在BI-V100(48KB)上必须降级 + - muh已经做了降级(用scale_mem_bound),但需要验证降级后的值是否正确 + +### 5. CCCL benchmark format解析 +------------------------------------------------------------ + NVIDIA的benchmark注释格式: + ipt_.tpb_.ns_.dcid_.l2w_.trp_.ld_ + 后跟4个浮点数: 在[2^16, 2^20, 2^24, 2^28]四个problem size下的speedup + + dcid映射: + 0 = no_delay + 1 = fixed_delay + 2 = exp_backoff + 3 = exp_backoff_jitter + 4 = exp_backoff_jitter_window + 5 = exp_backon_jitter_window + 6 = exp_backon_jitter + 7 = exp_backon + +### 6. 竞赛关键路径优先级 +------------------------------------------------------------ + Token吞吐加权值 = Output_TPS × 16.796 + Input_TPS × 2.799 + Cache_TPS × 0.56 + → Output_TPS权重83%, Input_TPS权重14%, Cache_TPS权重3% + + decode热路径 (Output TPS): + 1. paged_attention score reduction → reduce (DONE: muh tuned) + 2. softmax denominator prefix-sum → scan (DONE: muh tuned) + 3. top-k/top-p sampling → topk/radix_sort (DONE: muh tuned) + 4. RMSNorm/SiLU/RoPE element-wise → transform (DONE: muh tuned) + + prefill热路径 (Input TPS): + 5. flash_attention → scan + reduce + 6. MoE expert routing → select_if + reduce_by_key + + cache热路径 (Cache TPS): + 7. KV cache block copy → batch_memcpy (DONE: muh tuned) + +### 7. 待验证的关键问题 +------------------------------------------------------------ + 1. reduce items=24: 虽然loads to registers, 但实际BlockReduce的SMEM用量需要确认 + 2. scan delay参数: 0.5x/0.6x缩放是启发式, 需要BI-V100实测L2 write latency + 3. LOAD_LDG vs LOAD_DEFAULT: topk bench显示BI-V100上LOAD_DEFAULT更快, reduce/scan可能同理 + 4. SM count=16 → wave efficiency: 所有tuning都需要重新算occupancy + 5. transform bytes_in_flight: 从18GB/s改为56GB/s后items需要相应增大 diff --git a/cccl_sm100_benchmark_values.json b/cccl_sm100_benchmark_values.json new file mode 100644 index 00000000..c16faadb --- /dev/null +++ b/cccl_sm100_benchmark_values.json @@ -0,0 +1,278 @@ +{ + "source": "cccl_upstream/cub/cub/device/dispatch/tuning/tuning_*.cuh", + "extracted_by": "automated audit from CCCL source code", + "reduce": { + "sm100_float32_plus_o4": { + "items": 16, + "threads": 512, + "vec": 2, + "benchmark": "ipt_16.tpb_512.ipv_2", + "speedup": [ + 1.061295, + 1.0, + 1.065478, + 1.167139 + ] + }, + "sm100_float64_plus_o4": { + "items": 16, + "threads": 640, + "vec": 1, + "benchmark": "ipt_16.tpb_640.ipv_1", + "speedup": [ + 1.017834, + 1.0, + 1.015835, + 1.057092 + ] + }, + "sm100_accum8_plus_o4": { + "items": 15, + "threads": 512, + "vec": 2, + "benchmark": "ipt_15.tpb_512.ipv_2", + "speedup": [ + 1.019887, + 1.0, + 1.017636, + 1.058036 + ] + }, + "sm100_accum8_plus_o8": { + "items": 15, + "threads": 512, + "vec": 1, + "benchmark": "ipt_15.tpb_512.ipv_1", + "speedup": [ + 1.019414, + 1.0, + 1.017218, + 1.057143 + ] + }, + "sm90_det_float32": { + "items": 13, + "threads": 224, + "benchmark": "ipt_13.tpb_224", + "speedup": [ + 1.107188, + 1.009709, + 1.097114, + 1.31682 + ] + }, + "sm86_det_float32": { + "items": 6, + "threads": 224, + "benchmark": "ipt_6.tpb_224", + "speedup": [ + 1.034383, + 1.0, + 1.032097, + 1.090909 + ] + }, + "sm86_det_float64": { + "items": 11, + "threads": 128, + "benchmark": "ipt_11.tpb_128", + "speedup": [ + 1.232089, + 1.002124, + 1.245336, + 1.582279 + ] + } + }, + "scan": { + "sm100_lookback_1B_o4": { + "items": 18, + "threads": 512, + "delay": { + "ns": 768, + "dcid": 7, + "l2w": 820 + }, + "load": { + "transpose": 1, + "modifier": 0 + }, + "benchmark": "ipt_18.tpb_512.ns_768.dcid_7.l2w_820.trp_1.ld_0", + "speedup": [ + 1.188818, + 1.005682, + 1.173041, + 1.305288 + ] + }, + "sm100_lookback_2B_o4": { + "items": 13, + "threads": 512, + "delay": { + "ns": 1384, + "dcid": 7, + "l2w": 720 + }, + "load": { + "transpose": 1, + "modifier": 0 + }, + "benchmark": "ipt_13.tpb_512.ns_1384.dcid_7.l2w_720.trp_1.ld_0", + "speedup": [ + 1.128443, + 1.002841, + 1.119688, + 1.307692 + ] + }, + "sm100_lookback_4B_o4": { + "items": 22, + "threads": 384, + "delay": { + "ns": 1904, + "dcid": 6, + "l2w": 830 + }, + "load": { + "transpose": 1, + "modifier": 0 + }, + "benchmark": "ipt_22.tpb_384.ns_1904.dcid_6.l2w_830.trp_1.ld_0", + "speedup": [ + 1.148442, + 0.997167, + 1.139902, + 1.462651 + ] + }, + "sm100_lookback_8B_o4": { + "items": 23, + "threads": 416, + "delay": { + "ns": 772, + "dcid": 5, + "l2w": 710 + }, + "load": { + "transpose": 1, + "modifier": 0 + }, + "benchmark": "ipt_23.tpb_416.ns_772.dcid_5.l2w_710.trp_1.ld_0", + "speedup": [ + 1.089468, + 1.015581, + 1.08563, + 1.264583 + ] + }, + "sm100_lookback_1B_o8": { + "items": 14, + "threads": 384, + "delay": { + "ns": 228, + "dcid": 7, + "l2w": 775 + }, + "load": { + "transpose": 1, + "modifier": 1 + }, + "benchmark": "ipt_14.tpb_384.ns_228.dcid_7.l2w_775.trp_1.ld_1", + "speedup": [ + 1.10721, + 1.0, + 1.100637, + 1.307692 + ] + }, + "sm100_lookback_4B_o8": { + "items": 19, + "threads": 416, + "delay": { + "ns": 956, + "dcid": 7, + "l2w": 550 + }, + "load": { + "transpose": 1, + "modifier": 1 + }, + "benchmark": "ipt_19.tpb_416.ns_956.dcid_7.l2w_550.trp_1.ld_1", + "speedup": [ + 1.146142, + 0.99435, + 1.137459, + 1.455636 + ] + }, + "sm100_lookback_8B_o8": { + "items": 22, + "threads": 320, + "delay": { + "ns": 328, + "dcid": 2, + "l2w": 965 + }, + "load": { + "transpose": 1, + "modifier": 0 + }, + "benchmark": "ipt_22.tpb_320.ns_328.dcid_2.l2w_965.trp_1.ld_0", + "speedup": [ + 1.080133, + 1.0, + 1.075577, + 1.248963 + ] + } + }, + "benchmark_runner_params": { + "reduce": { + "items_range": "7:24:1", + "threads_range": "128:1024:32", + "vec_pow2_range": "1:2:1", + "problem_sizes": [ + "2^16", + "2^20", + "2^24", + "2^28" + ] + }, + "scan_lookback": { + "items_range": "7:24:1", + "threads_range": "128:1024:32", + "delay_ns_range": "0:2048:4", + "delay_algo_range": "0:7:1", + "l2w_range": "0:1200:5", + "transpose_range": "0:1:1", + "load_range": "0:1:1", + "problem_sizes": [ + "2^16", + "2^20", + "2^24", + "2^28", + "2^32" + ] + }, + "topk": { + "items_range": "7:24:1", + "threads_range": "128:1024:32", + "load_algo_range": "0:2:1" + }, + "radix_sort": { + "items_range": "7:24:1", + "threads_range": "128:1024:32", + "radix_bits": 8 + } + }, + "dcid_mapping": { + "0": "no_delay", + "1": "fixed_delay", + "2": "exponential_backoff", + "3": "exponential_backoff_jitter", + "4": "exponential_backoff_jitter_window", + "5": "exponential_backon_jitter_window", + "6": "exponential_backon_jitter", + "7": "exponential_backon" + } +}