From 32fdae237aee18593117179fac836c787111d237 Mon Sep 17 00:00:00 2001 From: project6 Date: Fri, 7 Aug 2026 09:03:06 +0000 Subject: [PATCH] =?UTF-8?q?perf(moe):=20CCCL=20basic=5Fvector=20pattern=20?= =?UTF-8?q?=E2=80=94=20batch=20GPU=E2=86=92CPU=20sync=20in=20segment=20det?= =?UTF-8?q?ection?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit thrust basic_vector.cu: device→host copy is batched (D = H, one memcpy). Our MoE segment loop did int() per iteration — N separate GPU→CPU syncs. Fix: .tolist() does ONE sync for all segment boundaries. Maps to: qwen3_6_scripts/qwen3_5.py (_pure_pytorch_experts prefill path) --- qwen3_6_scripts/qwen3_5.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/qwen3_6_scripts/qwen3_5.py b/qwen3_6_scripts/qwen3_5.py index 757196b2..ca26b42b 100644 --- a/qwen3_6_scripts/qwen3_5.py +++ b/qwen3_6_scripts/qwen3_5.py @@ -895,9 +895,14 @@ class Qwen3_5MoeSparseBlock(nn.Module): seg_eids = sorted_eids[seg_starts] # Process each expert segment (contiguous tokens → single F.linear) - for seg_i in range(len(seg_starts)): - s, e = int(seg_starts[seg_i]), int(seg_ends[seg_i]) - eid = int(seg_eids[seg_i]) + # CCCL basic_vector.cu: device→host copy should be batched. + # .tolist() does ONE GPU→CPU sync vs int() doing one per element. + seg_starts_cpu = seg_starts.tolist() + seg_ends_cpu = seg_ends.tolist() + seg_eids_cpu = seg_eids.tolist() + for seg_i in range(len(seg_starts_cpu)): + s, e = seg_starts_cpu[seg_i], seg_ends_cpu[seg_i] + eid = seg_eids_cpu[seg_i] tok_ids_seg = sorted_tok_ids[s:e] topk_pos_seg = sorted_topk_pos[s:e]