From cdc01bbc6a96887faecb838822b8ed0af18c9519 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 3 Aug 2026 06:45:54 +0000 Subject: [PATCH] fix: critical config + tuning corrections from CCCL source analysis MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit computility-run.yaml: max-num-seqs 1→256: benchmark sweeps [128,256] concurrent seqs, current config processes 1 while 127 queue. KV cache budget: 256 seqs × 2048 tokens × 80KB/token = 41.9GB < 45GB available. max-num-batched-tokens 8192→32768: support 256 concurrent prefills. gpu-memory-utilization 0.9→0.95: provide KV cache headroom. Dockerfile: Deploy paged_attention_v2_triton.py to vllm package path so try-triton-first logic in _custom_ops.py can find it. Falls back to PyTorch V2 automatically if Triton V2 fails (SMEM/runtime). muh/tuning/common.cuh: scale_mem_bound max_smem now a parameter (default 48KB). Allows policy_selectors to pass hw.max_shared_memory_per_block if actual SMEM differs from CCCL 48KB assumption. muh/tuning/tuning_transform.cuh: bytes_in_flight 16KB→32KB. Old derivation used 900/50=18 GB/s/SM (wrong, SM=16 confirmed). Actual per-SM BW = 56 GB/s. 32KB is estimate pending benchmark sweep. SM count 50→16 corrections across all affected files. --- Dockerfile | 9 +++++++++ computility-run.yaml | 6 +++--- muh/include/muh/tuning/common.cuh | 7 +++++-- muh/include/muh/tuning/tuning_reduce.cuh | 3 ++- muh/include/muh/tuning/tuning_transform.cuh | 11 +++++++---- optimizations/prefix_prefill_patch.py | 9 ++++++--- qwen3_6_scripts/patch_triton_tuning.py | 2 +- 7 files changed, 33 insertions(+), 14 deletions(-) diff --git a/Dockerfile b/Dockerfile index e20d8463..a064b65d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,6 +18,15 @@ RUN python3 /workspace/qwen3_6_scripts/patch_ixformer_native.py # 1. PagedAttention V2 — fills the NotImplementedError hole # Enables partitioned attention for long sequences (>8192 tokens) +# Deploy BOTH PyTorch and Triton V2 to vllm package — _custom_ops.py +# tries Triton first, falls back to PyTorch if import/runtime fails. +# Triton V2 risk: SMEM=32KB zero margin at head_dim=256 BLOCK_N=32. +# If Triton V2 crashes, PyTorch V2 (batched bmm, no intermediate tensor +# savings but correct) takes over automatically via try/except. +RUN cp /workspace/paged_attention_v2_triton.py \ + /usr/local/corex/lib/python3/dist-packages/vllm/paged_attention_v2_triton.py 2>/dev/null || \ + cp /workspace/paged_attention_v2_triton.py \ + /usr/local/corex/lib64/python3/dist-packages/vllm/paged_attention_v2_triton.py 2>/dev/null || true RUN python3 /workspace/qwen3_6_scripts/patch_paged_attention_v2.py # 2. Triton kernel tuning: BLOCK=64, NUM_WARPS=4 diff --git a/computility-run.yaml b/computility-run.yaml index 6447abd0..d283b113 100644 --- a/computility-run.yaml +++ b/computility-run.yaml @@ -10,16 +10,16 @@ command: - --max-model-len - '100000' - --gpu-memory-utilization - - '0.9' + - '0.95' - --trust-remote-code - -tp - '4' - --max-num-seqs - - '1' + - '256' - --disable-log-requests - --disable-frontend-multiprocessing - --max-num-batched-tokens - - '8192' + - '32768' - --enable-chunked-prefill - --max-seq-len-to-capture - '32768' diff --git a/muh/include/muh/tuning/common.cuh b/muh/include/muh/tuning/common.cuh index 6f6b6b3a..6b1345f5 100644 --- a/muh/include/muh/tuning/common.cuh +++ b/muh/include/muh/tuning/common.cuh @@ -146,8 +146,11 @@ struct scaling_result { /// b) Upper clamp was nominal*1 — should be nominal*2 /// c) No SMEM cap on threads — CCCL caps threads to prevent SMEM overflow constexpr scaling_result scale_mem_bound( - int nominal_4B_threads, int nominal_4B_items, int target_type_size) { - constexpr int max_smem = 48 * 1024; // 49152 bytes + int nominal_4B_threads, int nominal_4B_items, int target_type_size, + int max_smem = 48 * 1024) { + // max_smem default 48KB matches CCCL (util_arch.cuh:116). + // Pass hw.max_shared_memory_per_block from policy_selector to override + // if BI-V100 actual SMEM differs (_custom_ops.py claims 32KB). // Step 1+2: scale items, clamp to [1, nominal*2] int items = nominal_4B_items * 4 / target_type_size; diff --git a/muh/include/muh/tuning/tuning_reduce.cuh b/muh/include/muh/tuning/tuning_reduce.cuh index 1eb2e4c2..36db23aa 100644 --- a/muh/include/muh/tuning/tuning_reduce.cuh +++ b/muh/include/muh/tuning/tuning_reduce.cuh @@ -61,7 +61,8 @@ enum class determinism_t { struct bi100_float32_plus_o4 { // accum_size=4, tile = 512*16*4 = 32768 ≤ 49152 ✓ // SM100 ref: ipt_16.tpb_512.ipv_2 1.061 1.000 1.065 1.167 - // Derivation: SMEM OK, threads=512 for occupancy on 50 SMs. Keep. + // Derivation: SMEM OK, threads=512. SM=16 (not 50 from spec sheet). + // At 16 SMs, fewer concurrent CTAs → consider larger tiles. Pending benchmark. static constexpr int items = 16; static constexpr int threads = 512; static constexpr int items_per_vec_load = 2; diff --git a/muh/include/muh/tuning/tuning_transform.cuh b/muh/include/muh/tuning/tuning_transform.cuh index 4664969f..c1f76ccb 100644 --- a/muh/include/muh/tuning/tuning_transform.cuh +++ b/muh/include/muh/tuning/tuning_transform.cuh @@ -92,10 +92,13 @@ struct policy_selector { if (items_for_vec < 1) items_for_vec = 1; // items_for_latency: enough items to hide memory latency - // CCCL cc_to_min_bytes_in_flight: B200=64KB, H100=48KB, A100=16KB, V100=12KB - // BI-V100 per-SM BW = 900/50 = 18 GB/s ≈ A100 (2000/108 = 18.5 GB/s) - // → Use 16KB (A100-level), not 48-64KB - int bytes_in_flight = 16 * 1024; + // CCCL cc_to_min_bytes_in_flight: B200=64KB(54GB/s/SM), H100=48KB(25GB/s/SM), + // A100=16KB(18.5GB/s/SM), V100=12KB(14GB/s/SM) + // BI-V100: SM=16 (confirmed), per-SM BW = 900/16 = 56 GB/s + // bytes_in_flight = BW_per_SM × HBM_latency. BI-V100 HBM latency unknown. + // 56 GB/s per SM is B200-level BW, but latency likely differs (not NVIDIA arch). + // Estimate 32KB pending benchmark: %RANGE% bytes_in_flight 12288:65536:4096 + int bytes_in_flight = 32 * 1024; int items_for_latency = bytes_in_flight / (256 * min_elem_size); if (items_for_latency < 1) items_for_latency = 1; diff --git a/optimizations/prefix_prefill_patch.py b/optimizations/prefix_prefill_patch.py index c85d555b..97f4ea8d 100644 --- a/optimizations/prefix_prefill_patch.py +++ b/optimizations/prefix_prefill_patch.py @@ -8,7 +8,7 @@ BI-V100 hardware: SMEM per block: 48 KB Warp size: 32 (assumed) Max threads/block: 1024 - SM count: 50 + SM count: 16 (confirmed via ixsmi, not 50 from spec sheet) HBM bandwidth: 900 GB/s Qwen3.6-35B-A3B attention: @@ -30,11 +30,14 @@ BLOCK_M analysis: More work per thread = better instruction-level parallelism (ILP). Fewer warps = more blocks can run concurrently per SM = better occupancy. - BI-V100 has 50 SMs. With batch_size=1, num_heads~24-28: + BI-V100 has 16 SMs (confirmed, not 50 from spec sheet). + With batch_size=1, num_heads~24-28: grid = (batch=1, heads≈24, ceil(seq_len/BLOCK_M)) For seq_len=100K: grid_z = 1563 blocks. Total blocks = 1 × 24 × 1563 = 37,512 blocks. - Blocks per SM = 37512/50 = 750 — plenty of parallelism. + Blocks per SM = 37512/16 = 2344 — plenty of parallelism. + NOTE: with max-num-seqs=256 (benchmark config), batch_size >> 1, + grid is even larger. Parallelism is never the bottleneck. Reducing NUM_WARPS from 8→4: - Each SM can run more blocks concurrently (limited by registers/SMEM) diff --git a/qwen3_6_scripts/patch_triton_tuning.py b/qwen3_6_scripts/patch_triton_tuning.py index dcd867a5..d7bf838c 100644 --- a/qwen3_6_scripts/patch_triton_tuning.py +++ b/qwen3_6_scripts/patch_triton_tuning.py @@ -16,7 +16,7 @@ Hardware derivation: 8 warps = 256 threads → each thread handles 32 elements from Q tile. 4 warps = 128 threads → each thread handles 64 elements. - With 50 SMs and typical grid of 37K+ blocks: + With 16 SMs (confirmed) and typical grid of 37K+ blocks: At 8 warps + 32KB SMEM: 1 block per SM (SMEM-limited) At 4 warps + 32KB SMEM: potentially 2 blocks per SM