From 32ee28122e943ddcf47e41bf632e14110f00fbc1 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 11 Aug 2026 04:41:29 +0000 Subject: [PATCH] =?UTF-8?q?ref(upstream):=20=E6=90=AC=E8=BF=90=20xllm=20il?= =?UTF-8?q?u=20kernel+layer=20=E5=AE=8C=E6=95=B4=E6=BA=90=E7=A0=81=20?= =?UTF-8?q?=E2=80=94=202089=E8=A1=8C=2014=E4=B8=AAAPI=E5=A3=B0=E6=98=8E?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 来源: Deep-Spark/xllm core/kernels/ilu/ + core/layers/ilu/ ixformer.h: 14个ixformer::infer API完整声明 kernel wrappers: activation(32) attention(162) fused_moe(99) group_gemm(39) matmul(73) norm(50) rope(31) + headers layer dispatch: fused_moe.cpp(797行) attention.cpp(189行) + headers 覆盖状态 (ix_moe_bridge.cpp vs ixformer.h 14个API): 已覆盖 13/14: silu_and_mul, rms_norm, residual_rms_norm, ixformer_linear, ixformer_linear_ex, topk_softmax, moe_compute_token_index, moe_expand_input, moe_w16a16_group_gemm, moe_output_reduce_sum, xllm_paged_attention, xllm_reshape_and_cache, xllm_rotary_embedding 缺失 1/14: ixinfer_flash_attn_unpad_with_block_tables dlopen 调用链验证: 12个 prebuilt .so → 9个 qwen3_5.py + 2个 paged_attn.py + 1个 block_major_kv_cache.py 辅助模块: bi100_env, bi100_profile, gdn_prefix, block_major_kv_cache 全部到位 --- .../core/kernels/ilu/CMakeLists.txt | 28 ++++++++++++++ .../core/kernels/ilu/activation.cpp | 2 +- .../core/kernels/ilu/attention.cpp | 2 +- .../core/kernels/ilu/fused_moe.cpp | 2 +- .../core/kernels/ilu/group_gemm.cpp | 2 +- .../core/kernels/ilu/ilu_ops_api.h | 2 +- .../xllm_latest/core/kernels/ilu/ixformer.h | 2 +- .../xllm_latest/core/kernels/ilu/matmul.cpp | 2 +- .../xllm_latest/core/kernels/ilu/norm.cpp | 2 +- .../xllm_latest/core/kernels/ilu/rope.cpp | 2 +- .../xllm_latest/core/kernels/ilu/utils.h | 2 +- .../core/layers/ilu/CMakeLists.txt | 14 +++++++ .../xllm_latest/core/layers/ilu/attention.cpp | 2 +- .../xllm_latest/core/layers/ilu/attention.h | 2 +- .../xllm_latest/core/layers/ilu/fused_moe.cpp | 37 +++++++------------ .../xllm_latest/core/layers/ilu/fused_moe.h | 2 +- 16 files changed, 69 insertions(+), 36 deletions(-) create mode 100644 upstream_ref/xllm_latest/core/kernels/ilu/CMakeLists.txt create mode 100755 upstream_ref/xllm_latest/core/layers/ilu/CMakeLists.txt diff --git a/upstream_ref/xllm_latest/core/kernels/ilu/CMakeLists.txt b/upstream_ref/xllm_latest/core/kernels/ilu/CMakeLists.txt new file mode 100644 index 00000000..fa26c886 --- /dev/null +++ b/upstream_ref/xllm_latest/core/kernels/ilu/CMakeLists.txt @@ -0,0 +1,28 @@ +include(cc_library) +set(CMAKE_CUDA_ARCHITECTURES ivcore11) +file(GLOB_RECURSE ILU_HEADER_FILES + "${CMAKE_CURRENT_LIST_DIR}/*.h" +) + +file(GLOB_RECURSE ILU_SOURCE_FILES + "${CMAKE_CURRENT_LIST_DIR}/*.cpp" + "${CMAKE_CURRENT_LIST_DIR}/*.cu" +) + +find_package(Python3 REQUIRED COMPONENTS Interpreter Development) + +cc_library( + NAME + ilu_kernels + HDRS + ${ILU_HEADER_FILES} + SRCS + ${ILU_SOURCE_FILES} + DEPS + torch + :util + ixformer_kernels + ixformer + ${Python3_LIBRARIES} + cuinfer +) diff --git a/upstream_ref/xllm_latest/core/kernels/ilu/activation.cpp b/upstream_ref/xllm_latest/core/kernels/ilu/activation.cpp index 1ad364a4..ae2a16ba 100644 --- a/upstream_ref/xllm_latest/core/kernels/ilu/activation.cpp +++ b/upstream_ref/xllm_latest/core/kernels/ilu/activation.cpp @@ -1,4 +1,4 @@ -/* Copyright 2025-2026 The xLLM Authors. +/* Copyright 2025 The xLLM Authors. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/upstream_ref/xllm_latest/core/kernels/ilu/attention.cpp b/upstream_ref/xllm_latest/core/kernels/ilu/attention.cpp index ad3cd295..aa257bf1 100644 --- a/upstream_ref/xllm_latest/core/kernels/ilu/attention.cpp +++ b/upstream_ref/xllm_latest/core/kernels/ilu/attention.cpp @@ -1,5 +1,5 @@ -/* Copyright 2025-2026 The xLLM Authors. +/* Copyright 2025 The xLLM Authors. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/upstream_ref/xllm_latest/core/kernels/ilu/fused_moe.cpp b/upstream_ref/xllm_latest/core/kernels/ilu/fused_moe.cpp index 21c15d8c..794f9bd9 100644 --- a/upstream_ref/xllm_latest/core/kernels/ilu/fused_moe.cpp +++ b/upstream_ref/xllm_latest/core/kernels/ilu/fused_moe.cpp @@ -1,4 +1,4 @@ -/* Copyright 2025-2026 The xLLM Authors. +/* Copyright 2025 The xLLM Authors. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/upstream_ref/xllm_latest/core/kernels/ilu/group_gemm.cpp b/upstream_ref/xllm_latest/core/kernels/ilu/group_gemm.cpp index 290299a0..38743e66 100644 --- a/upstream_ref/xllm_latest/core/kernels/ilu/group_gemm.cpp +++ b/upstream_ref/xllm_latest/core/kernels/ilu/group_gemm.cpp @@ -1,4 +1,4 @@ -/* Copyright 2025-2026 The xLLM Authors. +/* Copyright 2026 The xLLM Authors. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/upstream_ref/xllm_latest/core/kernels/ilu/ilu_ops_api.h b/upstream_ref/xllm_latest/core/kernels/ilu/ilu_ops_api.h index 3dedd7da..e4fd7853 100644 --- a/upstream_ref/xllm_latest/core/kernels/ilu/ilu_ops_api.h +++ b/upstream_ref/xllm_latest/core/kernels/ilu/ilu_ops_api.h @@ -1,4 +1,4 @@ -/* Copyright 2025-2026 The xLLM Authors. +/* Copyright 2025 The xLLM Authors. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/upstream_ref/xllm_latest/core/kernels/ilu/ixformer.h b/upstream_ref/xllm_latest/core/kernels/ilu/ixformer.h index 83bad88e..57ce66dc 100644 --- a/upstream_ref/xllm_latest/core/kernels/ilu/ixformer.h +++ b/upstream_ref/xllm_latest/core/kernels/ilu/ixformer.h @@ -1,4 +1,4 @@ -/* Copyright 2025-2026 The xLLM Authors. +/* Copyright 2025 The xLLM Authors. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/upstream_ref/xllm_latest/core/kernels/ilu/matmul.cpp b/upstream_ref/xllm_latest/core/kernels/ilu/matmul.cpp index f90c0d47..91b6868f 100644 --- a/upstream_ref/xllm_latest/core/kernels/ilu/matmul.cpp +++ b/upstream_ref/xllm_latest/core/kernels/ilu/matmul.cpp @@ -1,4 +1,4 @@ -/* Copyright 2025-2026 The xLLM Authors. +/* Copyright 2025 The xLLM Authors. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/upstream_ref/xllm_latest/core/kernels/ilu/norm.cpp b/upstream_ref/xllm_latest/core/kernels/ilu/norm.cpp index e451bc36..c5a98595 100644 --- a/upstream_ref/xllm_latest/core/kernels/ilu/norm.cpp +++ b/upstream_ref/xllm_latest/core/kernels/ilu/norm.cpp @@ -1,4 +1,4 @@ -/* Copyright 2025-2026 The xLLM Authors. +/* Copyright 2025 The xLLM Authors. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/upstream_ref/xllm_latest/core/kernels/ilu/rope.cpp b/upstream_ref/xllm_latest/core/kernels/ilu/rope.cpp index 45af7656..89370b79 100644 --- a/upstream_ref/xllm_latest/core/kernels/ilu/rope.cpp +++ b/upstream_ref/xllm_latest/core/kernels/ilu/rope.cpp @@ -1,4 +1,4 @@ -/* Copyright 2025-2026 The xLLM Authors. +/* Copyright 2025 The xLLM Authors. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/upstream_ref/xllm_latest/core/kernels/ilu/utils.h b/upstream_ref/xllm_latest/core/kernels/ilu/utils.h index 9fd15298..e8af0c3c 100644 --- a/upstream_ref/xllm_latest/core/kernels/ilu/utils.h +++ b/upstream_ref/xllm_latest/core/kernels/ilu/utils.h @@ -1,4 +1,4 @@ -/* Copyright 2025-2026 The xLLM Authors. +/* Copyright 2025 The xLLM Authors. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/upstream_ref/xllm_latest/core/layers/ilu/CMakeLists.txt b/upstream_ref/xllm_latest/core/layers/ilu/CMakeLists.txt new file mode 100755 index 00000000..cd676017 --- /dev/null +++ b/upstream_ref/xllm_latest/core/layers/ilu/CMakeLists.txt @@ -0,0 +1,14 @@ +include(cc_library) + +cc_library( + NAME + ilu_layers + HDRS + attention.h + fused_moe.h + SRCS + attention.cpp + fused_moe.cpp + DEPS + :common_layers +) diff --git a/upstream_ref/xllm_latest/core/layers/ilu/attention.cpp b/upstream_ref/xllm_latest/core/layers/ilu/attention.cpp index 62d36a62..b66f28a4 100644 --- a/upstream_ref/xllm_latest/core/layers/ilu/attention.cpp +++ b/upstream_ref/xllm_latest/core/layers/ilu/attention.cpp @@ -1,4 +1,4 @@ -/* Copyright 2025-2026 The xLLM Authors. +/* Copyright 2025 The xLLM Authors. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/upstream_ref/xllm_latest/core/layers/ilu/attention.h b/upstream_ref/xllm_latest/core/layers/ilu/attention.h index bf4b59ba..a971835f 100644 --- a/upstream_ref/xllm_latest/core/layers/ilu/attention.h +++ b/upstream_ref/xllm_latest/core/layers/ilu/attention.h @@ -1,4 +1,4 @@ -/* Copyright 2025-2026 The xLLM Authors. +/* Copyright 2025 The xLLM Authors. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/upstream_ref/xllm_latest/core/layers/ilu/fused_moe.cpp b/upstream_ref/xllm_latest/core/layers/ilu/fused_moe.cpp index 7c829ad3..4238012e 100644 --- a/upstream_ref/xllm_latest/core/layers/ilu/fused_moe.cpp +++ b/upstream_ref/xllm_latest/core/layers/ilu/fused_moe.cpp @@ -1,4 +1,4 @@ -/* Copyright 2025-2026 The xLLM Authors. +/* Copyright 2026 The xLLM Authors. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -20,9 +20,6 @@ limitations under the License. #include #include "common/global_flags.h" -#include "core/framework/config/eplb_config.h" -#include "core/framework/config/scheduler_config.h" -#include "core/framework/config/speculative_config.h" #include "framework/parallel_state/parallel_state.h" #include "kernels/ops_api.h" #include "layers/common/dp_utils.h" @@ -89,9 +86,7 @@ FusedMoEImpl::FusedMoEImpl(const ModelArgs& model_args, } // Deep EP initialization check - enable_deep_ep_ = - ::xllm::EPLBConfig::get_instance().expert_parallel_degree() == 2 && - ep_size > 1; + enable_deep_ep_ = FLAGS_expert_parallel_degree == 2 && ep_size > 1; if (enable_deep_ep_) { // for now, we only implement the deep ep for decode stage. // so we will assume the max_token_num is limited to max_batch_size * (1+K) @@ -108,20 +103,16 @@ FusedMoEImpl::FusedMoEImpl(const ModelArgs& model_args, torch::ScalarType combine_dtype = options_.dtype().toScalarType(); int64_t combine_token_size = hidden_size_ * get_dtype_size(combine_dtype); // Ensure calculation base is at least ep_size - int64_t effective_seqs = std::max( - (int64_t)::xllm::SchedulerConfig::get_instance().max_seqs_per_batch(), - (int64_t)ep_size); - // NOTE: ::xllm::SchedulerConfig::get_instance().max_seqs_per_batch() - // represents the maximum total batch size, regardless of the dp size. To - // ensure robust scheduling and account for the worst-case scenario, we must - // guarantee that each rank is capable of handling the maximum possible - // number of tokens. Therefore, we define max_num_tokens_per_rank as the - // full maximum value, without dividing by either the rank count or the dp - // size. + int64_t effective_seqs = + std::max((int64_t)FLAGS_max_seqs_per_batch, (int64_t)ep_size); + // NOTE: FLAGS_max_seqs_per_batch represents the maximum total batch size, + // regardless of the dp size. To ensure robust scheduling and account + // for the worst-case scenario, we must guarantee that each rank is capable + // of handling the maximum possible number of tokens. Therefore, we define + // max_num_tokens_per_rank as the full maximum value, without dividing by + // either the rank count or the dp size. int64_t max_num_tokens_per_rank = - (1 + - ::xllm::SpeculativeConfig::get_instance().num_speculative_tokens()) * - effective_seqs * topk_; + (1 + FLAGS_num_speculative_tokens) * effective_seqs * topk_; // make sure that all layers share the same deep ep instance // so that the memory footprint is minimized @@ -723,8 +714,8 @@ torch::Tensor FusedMoEImpl::forward(const torch::Tensor& hidden_states, const ModelInputParams& input_params) { // we only support all2all communication for decode stage for now bool enable_all2all_communication = - enable_deep_ep_ && std::all_of(input_params.parallel.dp_is_decode.begin(), - input_params.parallel.dp_is_decode.end(), + enable_deep_ep_ && std::all_of(input_params.dp_is_decode.begin(), + input_params.dp_is_decode.end(), [](int32_t val) { return val == 1; }); bool is_dp_ep_parallel = @@ -739,7 +730,7 @@ torch::Tensor FusedMoEImpl::forward(const torch::Tensor& hidden_states, if (need_gather_and_slice) { input = parallel_state::gather(input, parallel_args_.dp_local_process_group_, - input_params.parallel.dp_global_token_nums); + input_params.dp_global_token_nums); } // MoE Gate auto router_logits = gate_(input); diff --git a/upstream_ref/xllm_latest/core/layers/ilu/fused_moe.h b/upstream_ref/xllm_latest/core/layers/ilu/fused_moe.h index 8d4e9da9..3e477064 100644 --- a/upstream_ref/xllm_latest/core/layers/ilu/fused_moe.h +++ b/upstream_ref/xllm_latest/core/layers/ilu/fused_moe.h @@ -1,4 +1,4 @@ -/* Copyright 2025-2026 The xLLM Authors. +/* Copyright 2025 The xLLM Authors. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License.