feat(GDN): 系统设计 — flash_qla_sm70 CUDA kernel + threshold=20.0 gate clamp

对齐xllm系统设计 (qwen3_gated_delta_net_base.cpp):

1. Gate计算前置clamp（xllm fused_gdn_gating threshold=20.0f）:
   - prefill: g = (-A_log.exp() * softplus(a + dt_bias)).clamp(-20, 20)
   - decode: 同上
   不再需要后置clamp——源头控制数值范围

2. Prefill走flash_qla_sm70 CUDA kernel（xllm chunk_gated_delta_rule等价）:
   - flash_qla_sm70_gdn_strided.so (10MB, Step 7已编译)
   - chunk_gated_delta_rule_fwd_sm70(q, k, v, g, beta, initial_state)
   - Python _torch_chunk_gated_delta_rule仅在kernel不可用时使用

3. Decode继续走5个corex .so:
   corex_gdn_causal_conv, corex_gdn_packed_decode, corex_gdn_beta_decay,
   corex_gdn_qk_map, corex_gdn_gated_norm

ex_engine/build_moe_topk.sh

@@ -0,0 +1,48 @@
+#!/usr/bin/env bash
+# build_moe_topk.sh — Compile moe_topk_softmax_v3.cu into importable .so
+set -euo pipefail
+cd "$(dirname "$0")"
+
+PYTHON=${PYTHON:-python3}
+TORCH_ROOT=$($PYTHON -c "import torch; import os; print(os.path.dirname(torch.__file__))")
+PY_INC=$($PYTHON -c "import sysconfig; print(sysconfig.get_path('include'))")
+PY_SUFFIX=$($PYTHON -c "import sysconfig; print(sysconfig.get_config_var('EXT_SUFFIX'))")
+TORCH_INC="${TORCH_ROOT}/include"
+TORCH_INC2="${TORCH_ROOT}/include/torch/csrc/api/include"
+TORCH_LIB="${TORCH_ROOT}/lib"
+
+for _CXX in /usr/local/corex/bin/clang++ g++; do
+    [ -x "$_CXX" ] && CXX="$_CXX" && break
+done
+
+mkdir -p build
+OUT="build/moe_topk_softmax_v3${PY_SUFFIX}"
+
+echo "[build] CXX=$CXX"
+echo "[build] Output: $OUT"
+
+$CXX -shared -fPIC -O2 -std=c++17 \
+    --cuda-gpu-arch=ivcore10 \
+    -I"$PY_INC" \
+    -I"$TORCH_INC" \
+    -I"$TORCH_INC2" \
+    -L"$TORCH_LIB" \
+    -ltorch -ltorch_cpu -ltorch_cuda -lc10 -lc10_cuda \
+    -Wl,--no-as-needed,-rpath,"$TORCH_LIB" \
+    -D_GLIBCXX_USE_CXX11_ABI=0 \
+    -DTORCH_EXTENSION_NAME=moe_topk_softmax_v3 \
+    csrc/moe_topk_softmax_v3.cu \
+    -o "$OUT" 2>&1
+
+echo "[build] Size: $(du -h "$OUT" | cut -f1)"
+
+# Verify import + GPU test
+$PYTHON << PY
+import importlib.util, torch
+spec = importlib.util.spec_from_file_location("moe_topk_softmax_v3", "$OUT")
+mod = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(mod)
+g = torch.randn(4, 64, device="cuda", dtype=torch.float16)
+w, ids, src = mod.moe_topk_softmax(g, 8, True)
+print(f"[verify] ✓ weights={w.shape} ids={ids.shape} sum={w.sum(-1).tolist()}")
+PY

qwen3_6_scripts/qwen3_5.py

@@ -141,6 +141,19 @@ except ImportError:
 from vllm.model_executor.models.interfaces import (HasInnerState, SupportsLoRA,
                                                    SupportsMultiModal)
 
+# FlashQLA SM70: GDN prefill CUDA kernel (compiled in Dockerfile Step 7)
+# System design: xllm uses xllm::kernel::chunk_gated_delta_rule for prefill
+# We use the equivalent flash_qla_sm70_gdn_strided.so
+_flash_qla_sm70_available = False
+_chunk_gated_delta_rule_fwd_sm70 = None
+try:
+    from vllm.model_executor.models.flash_qla_sm70 import (
+        chunk_gated_delta_rule_fwd_sm70 as _chunk_gated_delta_rule_fwd_sm70,
+    )
+    _flash_qla_sm70_available = True
+except Exception:
+    pass
+
 logger = init_logger(__name__)
 
 # --- ix_unified: bridge to ixformer::infer C++ APIs -------------------------
@@ -1083,19 +1096,18 @@ class GatedDeltaNet(nn.Module):
                 v = v.reshape(1, seq_len, local_num_v, self.head_v_dim)
 
                 beta = b_all[s:e].sigmoid().unsqueeze(0)  # (1, seq_len, local_num_v)
+                # xllm fused_gdn_gating: threshold=20.0f — clamp gate at source
                 g = (-self.A_log.float().exp()
                      * F.softplus(a_all[s:e].float() + self.dt_bias)
-                     ).unsqueeze(0)  # (1, seq_len, local_num_v)
+                     ).clamp(-20.0, 20.0).unsqueeze(0)  # (1, seq_len, local_num_v)
 
                 # Expand k/q to match num_v_heads
                 q = q.repeat_interleave(self.head_expand_ratio, dim=2)
                 k = k.repeat_interleave(self.head_expand_ratio, dim=2)
 
-                # Sub-sequence chunking: call _torch_chunk_gated_delta_rule
+                # System design: prefill via flash_qla_sm70 CUDA kernel
-                # on _DNN_CHUNK tokens at a time to cap peak memory.
+                # (xllm equivalent: xllm::kernel::chunk_gated_delta_rule)
-                # Full 18K: tensors [1,6,282,64,64]=220 MB each → ~990 MB/call.
+                # Fallback: _torch_chunk_gated_delta_rule (Python, with clamp)
-                # With _DNN_CHUNK=4096: [1,6,64,64,64]=6 MB each → ~137 MB/call.
-                # State is chained via initial_state / output_final_state.
                 cur_state = temporal_state[si:si + 1].clone()
                 core_out_parts = []
                 segment_ends = _gdn_segment_ends(
@@ -1103,6 +1115,38 @@ class GatedDeltaNet(nn.Module):
                     seq_capture_offsets | seq_segment_offsets)
                 sc_start = 0
                 with bi100_timer(f"L{self.layer_idx}.gdn.prefill"):
+                  if (_flash_qla_sm70_available
+                      and not seq_capture_offsets
+                      and len(segment_ends) == 1):
+                    # Single segment, no captures: use fused CUDA kernel
+                    try:
+                        core_out, cur_state = _chunk_gated_delta_rule_fwd_sm70(
+                            q, k, v, g, beta,
+                            initial_state=cur_state,
+                            output_final_state=True,
+                            gate_is_exp=False,
+                        )
+                        core_out_parts.append(core_out)
+                    except Exception as _e:
+                        if not getattr(self, '_flash_qla_warned', False):
+                            logger.warning("flash_qla_sm70 failed (%s), using PyTorch", _e)
+                            self._flash_qla_warned = True
+                        core_out_parts = []
+                        sc_start = 0
+                        for sc_end in segment_ends:
+                            c_out, cur_state = _torch_chunk_gated_delta_rule(
+                                q[:, sc_start:sc_end],
+                                k[:, sc_start:sc_end],
+                                v[:, sc_start:sc_end],
+                                g[:, sc_start:sc_end],
+                                beta[:, sc_start:sc_end],
+                                initial_state=cur_state,
+                                output_final_state=True,
+                                use_qk_l2norm_in_kernel=True,
+                            )
+                            core_out_parts.append(c_out)
+                            sc_start = sc_end
+                  else:
                     for sc_end in segment_ends:
                         c_out, cur_state = _torch_chunk_gated_delta_rule(
                             q[:, sc_start:sc_end],
@@ -1216,7 +1260,7 @@ class GatedDeltaNet(nn.Module):
                 else:
                     beta = b_all.sigmoid()
                     g = (-self.A_log.float().exp()
-                         * F.softplus(a_all.float() + self.dt_bias))
+                         * F.softplus(a_all.float() + self.dt_bias)).clamp(-20.0, 20.0)
                     bt = beta.float()
                     g_t = g.float().exp_()