From d44ec4d8db115991970215ba33b1b77110685388 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 8 Aug 2026 08:01:56 +0000 Subject: [PATCH] =?UTF-8?q?perf(qwen3=5F5):=20CCCL=20block=5Freduce=5Fwarp?= =?UTF-8?q?=5Freductions=20=E2=86=92=20reduce=20DeltaNet=20loop=20iteratio?= =?UTF-8?q?ns?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CCCL design: when sequential path (Python forward substitution) dominates, reduce per-unit work by halving chunk_size from 32→16. 15 loop iterations beats 31, even with 2× more chunks. Also: add weight-skip warning logs (CCCL ScatterDirect pattern: never silently discard data). Docker logs will now show exactly which weights are skipped during load_weights, explaining the 1.12GB gap vs Sub168. CCCL sources this round: - block_reduce_warp_reductions.cuh: sequential vs parallel path selection - warp_exchange_smem.cuh: INSERT_PADDING for memory alignment - agent_reduce_by_key.cuh: TempStorage union + ScatterDirect --- qwen3_6_scripts/qwen3_5.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/qwen3_6_scripts/qwen3_5.py b/qwen3_6_scripts/qwen3_5.py index a0010182..d44fe7e8 100644 --- a/qwen3_6_scripts/qwen3_5.py +++ b/qwen3_6_scripts/qwen3_5.py @@ -118,14 +118,14 @@ class _HardwarePolicy: # solve_triangular path: one cuBLAS call per chunk, larger = fewer calls # Python loop path: C iterations per chunk, smaller = fewer iterations if self.solve_triangular_available: - # Like CCCL max_items_per_thread=32 with threads=128 → tile=4096: - # larger chunk = amortize kernel launch overhead + # cuBLAS trsm: larger chunk = amortize kernel launch overhead self.deltanet_chunk_size = 64 else: - # Like CCCL reducing items for MUFU-heavy small-elem ops: - # smaller chunk = fewer Python loop iterations (C iterations) - # 32 iterations vs 64 = 2× fewer kernel launches in the loop - self.deltanet_chunk_size = 32 + # Python forward substitution fallback: each chunk costs C iterations. + # CCCL block_reduce_warp_reductions: when sequential path dominates, + # reduce per-unit work (fewer iterations) even at cost of more units + # (more chunks). 16 iterations × more chunks beats 32 iterations × fewer. + self.deltanet_chunk_size = 16 # Prefill sub-chunk: controls peak memory per DeltaNet forward call. # CCCL target = cc_to_min_bytes_in_flight(cc): BI-V100 ≈ lower tier. @@ -1561,6 +1561,8 @@ class Qwen3_5ForCausalLM(nn.Module, HasInnerState, SupportsLoRA): if name.endswith(".bias") and name not in params_dict: continue if name not in params_dict: + logger.warning("Skipped weight %s (not in params_dict, " + "shape=%s)", name, loaded_weight.shape) continue param = params_dict[name] weight_loader = getattr(param, "weight_loader", @@ -1706,6 +1708,8 @@ class Qwen3_5MoeForCausalLM(Qwen3_5ForCausalLM): break else: if name not in params_dict: + logger.warning("MoE: Skipped weight %s (not in params_dict, " + "shape=%s)", name, loaded_weight.shape) continue param = params_dict[name] weight_loader = getattr(param, "weight_loader", default_weight_loader)