From 6fe1272f1325d7d50f36f9fbbd82e014573698f2 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 18 Aug 2026 02:19:23 +0000 Subject: [PATCH 01/15] =?UTF-8?q?doc:=20MoE=205=E5=87=BD=E6=95=B0=E7=AC=A6?= =?UTF-8?q?=E5=8F=B7=E7=9C=9F=E7=9B=B8=20=E2=80=94=20=E4=B8=8D=E5=9C=A8?= =?UTF-8?q?=E9=A2=84=E8=A3=85.so=E9=87=8C,=E7=94=B1moe=5Fops=5Fimpl.cu?= =?UTF-8?q?=E8=87=AA=E7=BC=96=E8=AF=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 确认结果: - topk_softmax, moe_compute_token_index_api, moe_expand_input, moe_w16a16_group_gemm, moe_output_reduce_sum - namespace 是 ixformer::infer (Tensor API), 不是 ixformer::kernels::infer (raw ptr API) - 4个镜像.so全部nm -D搜过,均无这些符号 - 实现在 ex_engine/csrc/moe_ops_impl.cu (自编译CUDA kernel) - 由 build_moe_bridge.sh 编译进 ix_full_bridge_v2.so - kernels.h 里的 ixformer::kernels::infer 是SDK头文件声明(模板+raw ptr) 我们的 ixformer::infer 是独立实现(torch::Tensor wrapper) --- MOE_SYMBOL_TRUTH.md | 68 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 MOE_SYMBOL_TRUTH.md diff --git a/MOE_SYMBOL_TRUTH.md b/MOE_SYMBOL_TRUTH.md new file mode 100644 index 00000000..087e1352 --- /dev/null +++ b/MOE_SYMBOL_TRUTH.md @@ -0,0 +1,68 @@ +# MoE 函数符号真相 (2026-08-17 确认) + +## 结论 + +那5个 MoE 函数**确实不在任何镜像预装的 .so 里**。另一位开发者说的是对的。 + +但它们也**不需要**在预装 .so 里——它们是自编译的。 + +## 5个函数的正确命名空间 + +``` +ixformer::infer::topk_softmax +ixformer::infer::moe_compute_token_index_api +ixformer::infer::moe_expand_input +ixformer::infer::moe_w16a16_group_gemm +ixformer::infer::moe_output_reduce_sum +``` + +**注意**: 是 `ixformer::infer`，不是 `ixformer::kernels::infer`。 + +## 声明 vs 实现的关系 + +| 位置 | 角色 | +|------|------| +| `ixformer_sdk/csrc/include/ixformer/kernels/kernels.h` | **头文件声明** (namespace `ixformer::kernels::infer`) — C++ 模板声明，给 SDK 用的 | +| `ex_engine/csrc/moe_ops_impl.cu` | **CUDA 实现** (namespace `ixformer::infer`) — 自己写的 kernel，不依赖任何 .so | +| `ex_engine/csrc/ix_full_bridge_v2.cpp` | **pybind11 桥** — forward-declare 然后调用 moe_ops_impl.cu 里的实现 | +| `ex_engine/build_moe_bridge.sh` | **构建脚本** — 把 v2.cpp + moe_ops_impl.cu 一起编译成 ix_full_bridge_v2.so | + +## 符号表搜索结果 (4个 .so 全部搜过) + +| .so 文件 | MoE 函数 | 结论 | +|----------|----------|------| +| `libixformer.so` (3937 symbols) | 无 topk_softmax/moe_compute_token_index 等 | 只有 `reduce_sum` (通用的) | +| `_ixformer_torch.so` (49 symbols) | 完全没有 MoE | 只有 norm/rope/cache/attn | +| `_C.so` (6 symbols) | 几乎空壳 | 只有 PyInit | +| `libcuinfer.so` (270 symbols) | 只有 cuinferTopK (不是 MoE 的) | GEMM/BLAS 级别 | + +## 构建链 + +``` +patch_ops.sh + └→ build_moe_bridge.sh + └→ ninja/CppExtension 编译: + ix_full_bridge_v2.cpp + moe_ops_impl.cu + → ix_full_bridge_v2.so (包含5个MoE函数的实现) +``` + +## `ixformer::kernels::infer` vs `ixformer::infer` 的区别 + +- `ixformer::kernels::infer` — SDK 头文件 (kernels.h) 中的声明，使用 raw pointer + cudaStream_t + - 例: `void moe_topk_softmax(const T *gating_output, T *topk_weights, int *topk_indices, ...)` +- `ixformer::infer` — 我们自己实现的 PyTorch wrapper，使用 torch::Tensor + - 例: `void topk_softmax(torch::Tensor& topk_weights, torch::Tensor& topk_indices, ...)` + +`moe_ops_impl.cu` 是直接写 CUDA kernel（不调用 kernels.h 模板），然后暴露 Tensor API。 + +## Python 调用链 + +```python +# 通过 ixformer SDK (需要真机上的 _C.so 包含 infer 子模块): +import ixformer._C as ops +ops.infer.moe_topk_softmax(...) # 如果 _C.so 有实现 + +# 通过 ex_engine bridge (我们自编译的): +import ix_full_bridge_v2 as bridge +bridge.topk_softmax(...) # 来自 moe_ops_impl.cu +``` From 74cdc68a1cc3d19b915fb847dd3769a68b2d1990 Mon Sep 17 00:00:00 2001 From: root Date: Tue, 18 Aug 2026 01:26:22 +0000 Subject: [PATCH 02/15] trigger rebuild From 902655f1bc65134688d01f2a3ad80e5758f796cf Mon Sep 17 00:00:00 2001 From: root Date: Tue, 18 Aug 2026 01:26:22 +0000 Subject: [PATCH 03/15] trigger rebuild From 31aa9432488ba3b5339c7a08faa0480e4d7114d4 Mon Sep 17 00:00:00 2001 From: root Date: Tue, 18 Aug 2026 01:26:22 +0000 Subject: [PATCH 04/15] trigger rebuild From 2b0d98a8675952207f5ea98b306927a11b6737e3 Mon Sep 17 00:00:00 2001 From: root Date: Tue, 18 Aug 2026 02:20:43 +0000 Subject: [PATCH 05/15] [fix] baseline4 debug info From 763cda98c5a5b6332d8e208f106d6070f57bc38b Mon Sep 17 00:00:00 2001 From: root Date: Tue, 18 Aug 2026 01:26:22 +0000 Subject: [PATCH 06/15] trigger rebuild From 716ede67fde26ac4cc58b5bf39f3b7bff859e1a3 Mon Sep 17 00:00:00 2001 From: root Date: Tue, 18 Aug 2026 01:26:22 +0000 Subject: [PATCH 07/15] trigger rebuild From 448996d386f8d4604c0fcb5399dc8c594c808c15 Mon Sep 17 00:00:00 2001 From: root Date: Tue, 18 Aug 2026 01:26:22 +0000 Subject: [PATCH 08/15] trigger rebuild From 4f9f31f094ceaf8026c2a19c9d2f7e3d3ab7974b Mon Sep 17 00:00:00 2001 From: root Date: Tue, 18 Aug 2026 02:20:43 +0000 Subject: [PATCH 09/15] [fix] baseline4 debug info From bcb18cf3280b7392d126b1bcc696819696c6db6a Mon Sep 17 00:00:00 2001 From: root Date: Tue, 18 Aug 2026 01:26:22 +0000 Subject: [PATCH 10/15] trigger rebuild From da1382bb2376d66b7ffc9c32cb5ee53a3d78740d Mon Sep 17 00:00:00 2001 From: root Date: Tue, 18 Aug 2026 01:26:22 +0000 Subject: [PATCH 11/15] trigger rebuild From 63e574ad0bbf017357970d323dd76f6cebdad205 Mon Sep 17 00:00:00 2001 From: root Date: Tue, 18 Aug 2026 01:26:22 +0000 Subject: [PATCH 12/15] trigger rebuild From c6eb03842938b5bc7dee9a155b3cf1b9c7f1d484 Mon Sep 17 00:00:00 2001 From: root Date: Tue, 18 Aug 2026 01:26:22 +0000 Subject: [PATCH 13/15] trigger rebuild From d6d12c56d8ce6e41ac35ffa3eeafa3137492ffc0 Mon Sep 17 00:00:00 2001 From: root Date: Tue, 18 Aug 2026 02:20:43 +0000 Subject: [PATCH 14/15] [fix] baseline4 debug info From 13b12aac4c854beaec54497bdcad093026d6c490 Mon Sep 17 00:00:00 2001 From: root Date: Tue, 18 Aug 2026 03:35:26 +0000 Subject: [PATCH 15/15] [fix] baseline4 rms_norm crash --- ex_engine/python/ix_ops.py | 13 ++++-- ex_engine/python/patch_vllm_ops.py | 44 +++++++++---------- qwen3_6_scripts/ex_engine/python/ix_ops.py | 13 ++++-- .../ex_engine/python/patch_vllm_ops.py | 44 +++++++++---------- 4 files changed, 60 insertions(+), 54 deletions(-) diff --git a/ex_engine/python/ix_ops.py b/ex_engine/python/ix_ops.py index f6ded5f0..4ccba573 100644 --- a/ex_engine/python/ix_ops.py +++ b/ex_engine/python/ix_ops.py @@ -222,11 +222,16 @@ def fused_add_rms_norm(input: torch.Tensor, residual: torch.Tensor, """Fused residual addition + RMSNorm. Source: xllm/core/kernels/ilu/norm.cpp → infer::residual_rms_norm - output = rms_norm(input + residual, weight, eps) - residual_output = input + residual + The C++ function is in-place: modifies input → rms_norm(input+residual)*weight, + and residual → input+residual. We copy results to output/residual_output. """ - _bridge.fused_add_rms_norm(input, residual, weight, output, - residual_output, eps) + # C++ signature: fused_add_rms_norm_forward(input, residual, weight, eps, alpha) + # It modifies input and residual in-place. + inp_clone = input.clone() + res_clone = residual.clone() + _bridge.fused_add_rms_norm(inp_clone, res_clone, weight, eps) + output.copy_(inp_clone) + residual_output.copy_(res_clone) def rotary_embedding(positions: torch.Tensor, query: torch.Tensor, diff --git a/ex_engine/python/patch_vllm_ops.py b/ex_engine/python/patch_vllm_ops.py index d19e79d8..4d128012 100644 --- a/ex_engine/python/patch_vllm_ops.py +++ b/ex_engine/python/patch_vllm_ops.py @@ -92,28 +92,26 @@ def _patch_layernorm() -> int: w = self.weight if _debug_count[0] < 20: _debug_count[0] += 1 - logger.info("DEBUG rms_norm #%d: w.shape=%s w.dim=%d x.shape=%s x.dim=%d " - "class=%s residual=%s", - _debug_count[0], list(w.shape), w.dim(), list(x.shape), x.dim(), + logger.info("DEBUG rms_norm #%d: w.shape=%s w.dim=%d w.dtype=%s " + "x.shape=%s x.dim=%d x.dtype=%s class=%s residual=%s", + _debug_count[0], list(w.shape), w.dim(), w.dtype, + list(x.shape), x.dim(), x.dtype, type(self).__name__, list(residual.shape) if residual is not None else None) if w.dim() != 1 or w.shape[0] != x.shape[-1]: return _orig_forward(self, x, residual) - w_adjusted = 1.0 + w + # 1.0 + w promotes fp16→fp32; ixformer rms_norm requires weight + # to be 1-D AND same dtype as input, so cast back. + w_adjusted = (1.0 + w).to(w.dtype) if residual is not None: - if ix_ops.has_fused_add_rms_norm(): - out = torch.empty_like(x) - residual_out = torch.empty_like(x) - ix_ops.fused_add_rms_norm( - x, residual, w_adjusted, out, residual_out, - self.variance_epsilon) - return out, residual_out - else: - new_residual = x + residual - out = torch.empty_like(x) - ix_ops.rms_norm(out, new_residual, w_adjusted, - self.variance_epsilon) - return out, new_residual + # ixformer fused_add_rms_norm is in-place and has 4-arg C++ + # signature (input, residual, weight, eps). Safer to use + # the non-fused path which is explicit about outputs. + new_residual = x + residual + out = torch.empty_like(x) + ix_ops.rms_norm(out, new_residual, w_adjusted, + self.variance_epsilon) + return out, new_residual else: out = torch.empty_like(x) ix_ops.rms_norm(out, x, w_adjusted, self.variance_epsilon) @@ -180,17 +178,17 @@ def _patch_custom_ops() -> int: logger.info("PATCHED: _custom_ops.rms_norm → ix_ops") # Patch fused_add_rms_norm - if ix_ops.has_fused_add_rms_norm() and hasattr(ops, 'fused_add_rms_norm'): + if ix_ops.has_rms_norm() and hasattr(ops, 'fused_add_rms_norm'): def _fused_add_rms_norm(input, residual, weight, eps): + # C++ fused_add_rms_norm is in-place with 4-arg signature, + # doesn't match the 6-arg wrapper in ix_ops. Use non-fused path. + residual.add_(input) out = torch.empty_like(input) - residual_out = torch.empty_like(input) - ix_ops.fused_add_rms_norm(input, residual, weight, - out, residual_out, eps) + ix_ops.rms_norm(out, residual, weight, eps) input.copy_(out) - residual.copy_(residual_out) ops.fused_add_rms_norm = _fused_add_rms_norm count += 1 - logger.info("PATCHED: _custom_ops.fused_add_rms_norm → ix_ops") + logger.info("PATCHED: _custom_ops.fused_add_rms_norm → ix_ops (non-fused)") # Patch rotary_embedding if ix_ops.has_rotary_embedding() and hasattr(ops, 'rotary_embedding'): diff --git a/qwen3_6_scripts/ex_engine/python/ix_ops.py b/qwen3_6_scripts/ex_engine/python/ix_ops.py index f6ded5f0..4ccba573 100644 --- a/qwen3_6_scripts/ex_engine/python/ix_ops.py +++ b/qwen3_6_scripts/ex_engine/python/ix_ops.py @@ -222,11 +222,16 @@ def fused_add_rms_norm(input: torch.Tensor, residual: torch.Tensor, """Fused residual addition + RMSNorm. Source: xllm/core/kernels/ilu/norm.cpp → infer::residual_rms_norm - output = rms_norm(input + residual, weight, eps) - residual_output = input + residual + The C++ function is in-place: modifies input → rms_norm(input+residual)*weight, + and residual → input+residual. We copy results to output/residual_output. """ - _bridge.fused_add_rms_norm(input, residual, weight, output, - residual_output, eps) + # C++ signature: fused_add_rms_norm_forward(input, residual, weight, eps, alpha) + # It modifies input and residual in-place. + inp_clone = input.clone() + res_clone = residual.clone() + _bridge.fused_add_rms_norm(inp_clone, res_clone, weight, eps) + output.copy_(inp_clone) + residual_output.copy_(res_clone) def rotary_embedding(positions: torch.Tensor, query: torch.Tensor, diff --git a/qwen3_6_scripts/ex_engine/python/patch_vllm_ops.py b/qwen3_6_scripts/ex_engine/python/patch_vllm_ops.py index d19e79d8..4d128012 100644 --- a/qwen3_6_scripts/ex_engine/python/patch_vllm_ops.py +++ b/qwen3_6_scripts/ex_engine/python/patch_vllm_ops.py @@ -92,28 +92,26 @@ def _patch_layernorm() -> int: w = self.weight if _debug_count[0] < 20: _debug_count[0] += 1 - logger.info("DEBUG rms_norm #%d: w.shape=%s w.dim=%d x.shape=%s x.dim=%d " - "class=%s residual=%s", - _debug_count[0], list(w.shape), w.dim(), list(x.shape), x.dim(), + logger.info("DEBUG rms_norm #%d: w.shape=%s w.dim=%d w.dtype=%s " + "x.shape=%s x.dim=%d x.dtype=%s class=%s residual=%s", + _debug_count[0], list(w.shape), w.dim(), w.dtype, + list(x.shape), x.dim(), x.dtype, type(self).__name__, list(residual.shape) if residual is not None else None) if w.dim() != 1 or w.shape[0] != x.shape[-1]: return _orig_forward(self, x, residual) - w_adjusted = 1.0 + w + # 1.0 + w promotes fp16→fp32; ixformer rms_norm requires weight + # to be 1-D AND same dtype as input, so cast back. + w_adjusted = (1.0 + w).to(w.dtype) if residual is not None: - if ix_ops.has_fused_add_rms_norm(): - out = torch.empty_like(x) - residual_out = torch.empty_like(x) - ix_ops.fused_add_rms_norm( - x, residual, w_adjusted, out, residual_out, - self.variance_epsilon) - return out, residual_out - else: - new_residual = x + residual - out = torch.empty_like(x) - ix_ops.rms_norm(out, new_residual, w_adjusted, - self.variance_epsilon) - return out, new_residual + # ixformer fused_add_rms_norm is in-place and has 4-arg C++ + # signature (input, residual, weight, eps). Safer to use + # the non-fused path which is explicit about outputs. + new_residual = x + residual + out = torch.empty_like(x) + ix_ops.rms_norm(out, new_residual, w_adjusted, + self.variance_epsilon) + return out, new_residual else: out = torch.empty_like(x) ix_ops.rms_norm(out, x, w_adjusted, self.variance_epsilon) @@ -180,17 +178,17 @@ def _patch_custom_ops() -> int: logger.info("PATCHED: _custom_ops.rms_norm → ix_ops") # Patch fused_add_rms_norm - if ix_ops.has_fused_add_rms_norm() and hasattr(ops, 'fused_add_rms_norm'): + if ix_ops.has_rms_norm() and hasattr(ops, 'fused_add_rms_norm'): def _fused_add_rms_norm(input, residual, weight, eps): + # C++ fused_add_rms_norm is in-place with 4-arg signature, + # doesn't match the 6-arg wrapper in ix_ops. Use non-fused path. + residual.add_(input) out = torch.empty_like(input) - residual_out = torch.empty_like(input) - ix_ops.fused_add_rms_norm(input, residual, weight, - out, residual_out, eps) + ix_ops.rms_norm(out, residual, weight, eps) input.copy_(out) - residual.copy_(residual_out) ops.fused_add_rms_norm = _fused_add_rms_norm count += 1 - logger.info("PATCHED: _custom_ops.fused_add_rms_norm → ix_ops") + logger.info("PATCHED: _custom_ops.fused_add_rms_norm → ix_ops (non-fused)") # Patch rotary_embedding if ix_ops.has_rotary_embedding() and hasattr(ops, 'rotary_embedding'):