dylanyunlong/project_6
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

upstream(xllm): sync to jd-opensource/xllm latest + revert serving_chat.py

Browse Source 
搬运 jd-opensource/xllm 最新代码到 upstream_ref/xllm_latest/:
- core/kernels/ilu/ 10 files (ixformer.h API 不变)
- core/layers/ilu/ 4 files (fused_moe.cpp config 访问从 FLAGS→singleton)
- core/layers/npu_torch/ 14 files (qwen3_gated_delta_net_base.cpp 576→1164行,
  新增 repeat_tensor_heads, checkpoint_stride, spec_verify 等 GDN 功能)
- models/llm/ 5 files (qwen3_5.h 模型注册重构, 新增 qwen3_5_mtp_base.h)
- models/vlm/ 1 file (qwen3_5.h 218→440行)

serving_chat.py: 还原到 8030a11b 原版,删掉 6dcf3590 的语法错误 min(8192,
(缺右括号导致 py_compile 失败)
This commit is contained in:
project6-dev
2026-08-12 04:22:34 +00:00
parent 6dcf3590d5
commit d025b08a95
35 changed files with 1595 additions and 593 deletions
Download Patch File Download Diff File Expand all files Collapse all files

256
upstream_ref/xllm_latest/models/vlm/qwen3_5.h
View File

@@ -1,4 +1,4 @@
/* Copyright 2026 The xLLM Authors. All Rights Reserved.
/* Copyright 2025-2026 The xLLM Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
@@ -17,18 +17,30 @@ limitations under the License.
#include "core/framework/model/model_output.h"
#include "core/layers/common/lm_head.h"
#include "core/layers/common/rotary_embedding_util.h"
#include "models/model_registry.h"
#include "models/vlm/mposition/mposition.h"
#include "models/vlm/qwen3_vl_base.h"
#include "processors/multimodal_processor.h"
#include "processors/qwen2_vl_image_processor.h"
#include "processors/qwen3_vl_prompt_processor.h"
#include "processors/qwen3_vl_video_processor.h"
#if defined(USE_NPU)
#include "models/llm/qwen3_5.h"
#include "models/vlm/npu/qwen3_vl.h"
#elif defined(USE_MLU) || defined(USE_DCU)
#include "core/layers/common/qwen3_next_rms_norm.h"
#include "core/layers/common/rms_norm.h"
#include "core/layers/mlu/qwen3_5_decoder_layer.h"
#include "core/layers/qwen3_5_decoder_layer.h"
#include "core/layers/qwen3_vision_layer.h"
#include "models/llm/llm_model_base.h"
#include "models/model_registry.h"
#include "models/vlm/qwen3_vl_base.h"
#include "processors/input_processor.h"
#include "processors/qwen2_vl_image_processor.h"
#include "qwen3_vl.h"
#endif
namespace xllm {
#if !defined(USE_NPU)
class Qwen3_5ModelImpl final
: public LlmModelImplBase<layer::Qwen3_5DecoderLayer> {
public:
@@ -78,33 +90,7 @@ class Qwen3_5ModelImpl final
std::pair<torch::Tensor, torch::Tensor> apply_mrope(
const torch::Tensor positions) override {
auto target_cos_sin = cos_sin_.index({positions});
auto target_cos_sin_chunks = target_cos_sin.chunk(/*chunks=*/2, /*dim=*/-1);
auto cos_pos = target_cos_sin_chunks[0].contiguous();
auto sin_pos = target_cos_sin_chunks[1].contiguous();
auto apply = [this](torch::Tensor x) {
auto freqs_t = x[0].clone();
int64_t mrop_length = static_cast<int64_t>(freqs_t.size(-1) / 2);
for (int32_t dim_idx = 1; dim_idx <= 2; ++dim_idx) {
int64_t offset = dim_idx;
int64_t section_len = mrope_section_[dim_idx];
int64_t length = section_len * 3;
auto idx_first_half = torch::arange(offset, length, 3, torch::kLong);
auto idx_second_half = torch::arange(
offset + mrop_length, length + mrop_length, 3, torch::kLong);
auto idx_tensor =
torch::cat({idx_first_half, idx_second_half}, 0).to(x.device());
auto src = x[dim_idx].index_select(-1, idx_tensor);
freqs_t.index_copy_(-1, idx_tensor, src);
}
return freqs_t;
};
cos_pos = apply(cos_pos.reshape({positions.size(0), -1, cos_pos.size(-1)}));
sin_pos = apply(sin_pos.reshape({positions.size(0), -1, sin_pos.size(-1)}));
return std::make_pair(cos_pos, sin_pos);
return layer::rotary::apply_mrope(cos_sin_, positions, mrope_section_);
}
virtual ModelOutput forward(torch::Tensor tokens,
@@ -120,11 +106,11 @@ class Qwen3_5ModelImpl final
tokens = torch::tensor({1}).to(torch::kInt32).to(tokens.device());
positions = torch::tensor({1}).to(torch::kInt32).to(positions.device());
}
auto& dp_token_nums = input_params_new.dp_global_token_nums;
auto& dp_token_nums = input_params_new.parallel.dp_global_token_nums;
std::replace(dp_token_nums.begin(), dp_token_nums.end(), 0, 1);
}
auto inputs_embeds = input_params.input_embedding;
auto inputs_embeds = input_params.embedding.input_embedding;
torch::Tensor h;
if (inputs_embeds.defined()) {
h = inputs_embeds;
@@ -139,12 +125,8 @@ class Qwen3_5ModelImpl final
}
auto& attn_metadata = *(input_params_new.attn_metadata);
bool only_prefill =
(attn_metadata.is_prefill || attn_metadata.is_chunked_prefill);
if (positions.dim() == 2 && only_prefill && !mrope_section_.empty()) {
std::tie(attn_metadata.mrope_cos, attn_metadata.mrope_sin) =
apply_mrope(positions);
}
std::tie(attn_metadata.mrope_cos, attn_metadata.mrope_sin) =
apply_mrope(positions);
std::optional<torch::Tensor> residual;
for (size_t i = 0; i < layers_.size(); i++) {
@@ -169,8 +151,70 @@ class Qwen3_5ModelImpl final
layer::AttentionMetadata get_attention_metadata(
const ModelInputParams& params,
const torch::Tensor& h) {
auto attn_metadata = layer::AttentionMetadataBuilder::build(params, false);
// TODO: support linear attention
auto attn_metadata =
layer::AttentionMetadataBuilder::build(params,
/*enable_mla=*/false,
/*attn_mask=*/{},
h.device());
// Init batch and token_block_offset for GDN attention
if (attn_metadata.is_prefill || attn_metadata.is_chunked_prefill) {
constexpr int32_t kBlockM = 64;
constexpr int64_t pad_slot_id = -1;
constexpr int64_t default_max_num_programs = 1024;
constexpr int64_t chunk_size = 64;
auto seqlens = attn_metadata.q_cu_seq_lens.diff();
auto nums = (seqlens + kBlockM - 1) / kBlockM;
nums = nums.to(torch::kLong);
int32_t tot = nums.sum().item<int32_t>();
torch::Tensor range_batch = torch::arange(nums.size(0), nums.options());
torch::Tensor mlist_tensor = torch::repeat_interleave(range_batch, nums);
int64_t mlist_len = mlist_tensor.size(0);
int64_t max_num_programs =
std::max(default_max_num_programs, mlist_len) * 2;
torch::Tensor batch_ptr =
torch::full({max_num_programs},
pad_slot_id,
torch::dtype(torch::kInt32).device(seqlens.device()));
torch::Tensor token_block_offset_ptr =
torch::full({max_num_programs},
pad_slot_id,
torch::dtype(torch::kInt32).device(seqlens.device()));
std::vector<torch::Tensor> vec;
vec.reserve(nums.size(0));
for (int64_t i = 0; i < nums.size(0); ++i) {
vec.emplace_back(
torch::arange(nums[i].item<int64_t>(), nums.options()));
}
torch::Tensor offsetlist_tensor = torch::cat(vec, -1).to(torch::kInt32);
batch_ptr.narrow(0, 0, mlist_len).copy_(mlist_tensor);
token_block_offset_ptr.narrow(0, 0, mlist_len).copy_(offsetlist_tensor);
// Compute chunk indices for the chunked GDN kernel
{
torch::Tensor lengths = seqlens;
torch::Tensor num_chunks = (lengths + chunk_size - 1) / chunk_size;
num_chunks = num_chunks.to(torch::kLong);
torch::Tensor cumsum = torch::cumsum(num_chunks, 0);
int64_t total_chunks = cumsum[-1].item<int64_t>();
torch::Tensor arange_total =
torch::arange(total_chunks, attn_metadata.q_cu_seq_lens.options());
torch::Tensor zeros = torch::zeros({1}, cumsum.options());
torch::Tensor prefix = torch::cat(
{zeros, cumsum.slice(/*dim=*/0, /*start=*/0, /*end=*/-1)});
torch::Tensor repeats_prefix =
torch::repeat_interleave(prefix, num_chunks);
torch::Tensor indices = arange_total - repeats_prefix;
torch::Tensor mask = indices == 0;
torch::Tensor col0 = mask.cumsum(0) - 1;
attn_metadata.chunk_indices = torch::stack({col0, indices}, /*dim=*/1)
.to(attn_metadata.q_cu_seq_lens)
.to(torch::kInt32);
}
attn_metadata.tot = tot;
attn_metadata.batch = batch_ptr;
attn_metadata.token_block_offset = token_block_offset_ptr;
}
return attn_metadata;
}
};
@@ -193,8 +237,16 @@ class Qwen3_5ForCausalLMImpl : public LlmForCausalLMImplBase<Qwen3_5Model> {
};
TORCH_MODULE(Qwen3_5ForCausalLM);
#endif // !defined(USE_NPU)
#if defined(USE_NPU)
using Qwen3_5_VisionTransformer = npu::model::Qwen3_VisionTransformer;
#else
using Qwen3_5_VisionTransformer = Qwen3_VisionTransformer;
#endif
using Qwen3_5ForConditionalGenerationImpl =
Qwen3VLForConditionalGenerationBase<Qwen3_VisionTransformer,
Qwen3VLForConditionalGenerationBase<Qwen3_5_VisionTransformer,
Qwen3_5ForCausalLM>;
TORCH_MODULE(Qwen3_5ForConditionalGeneration);
@@ -211,6 +263,7 @@ TORCH_MODULE(Qwen3_5ForConditionalGeneration);
LOAD_ARG_OR( \
max_position_embeddings, "text_config.max_position_embeddings", 262144); \
LOAD_ARG_OR(rms_norm_eps, "text_config.rms_norm_eps", 1e-6); \
LOAD_ARG_OR(bos_token_id, "text_config.bos_token_id", 151643); \
LOAD_ARG_OR(eos_token_id, "text_config.eos_token_id", 248044); \
LOAD_ARG_OR( \
rope_theta, "text_config.rope_parameters.rope_theta", 10000000.0f); \
@@ -223,34 +276,50 @@ TORCH_MODULE(Qwen3_5ForConditionalGeneration);
LOAD_ARG_OR( \
linear_value_head_dim, "text_config.linear_value_head_dim", 128); \
LOAD_ARG_OR(linear_num_key_heads, "text_config.linear_num_key_heads", 16); \
LOAD_ARG_OR( \
linear_num_value_heads, "text_config.linear_num_value_heads", 48); \
LOAD_ARG_OR(linear_num_value_heads, \
"text_config.linear_num_value_heads", \
static_cast<int32_t>(args->n_heads() * 2)); \
LOAD_ARG_OR( \
full_attention_interval, "text_config.full_attention_interval", 4); \
LOAD_ARG_OR(attn_output_gate, "text_config.attn_output_gate", false); \
LOAD_ARG_OR(attn_output_gate, "text_config.attn_output_gate", true); \
LOAD_ARG_OR( \
num_nextn_predict_layers, "text_config.mtp_num_hidden_layers", 0); \
LOAD_ARG_OR(num_nextn_predict_layers, \
"text_config.num_nextn_predict_layers", \
args->num_nextn_predict_layers()); \
LOAD_ARG_OR(attention_bias, "text_config.attention_bias", false); \
LOAD_ARG_OR(attention_dropout, "text_config.attention_dropout", 0.0f); \
LOAD_ARG_OR(initializer_range, "text_config.initializer_range", 0.02f); \
LOAD_ARG_OR( \
mlp_only_layers, "text_config.mlp_only_layers", std::vector<int32_t>()); \
LOAD_ARG(rope_scaling_mrope_section, \
"text_config.rope_parameters.mrope_section"); \
LOAD_ARG_OR(rope_scaling_mrope_section, \
"text_config.rope_parameters.mrope_section", \
std::vector<int64_t>({11, 11, 10})); \
LOAD_ARG_OR(rope_scaling_mrope_interleaved, \
"text_config.rope_parameters.mrope_interleaved", \
true); \
LOAD_ARG_OR(rope_scaling_rope_type, \
"text_config.rope_parameters.rope_type", \
"default"); \
if (args->rope_scaling_rope_type() == "default") { \
args->rope_scaling_rope_type() = "mrope"; \
} \
LOAD_ARG_OR(partial_rotary_factor, \
"text_config.rope_parameters.partial_rotary_factor", \
0.25f)
0.25f); \
LOAD_ARG_OR(mamba_ssm_dtype, "text_config.mamba_ssm_dtype", "float32")
#define LOAD_QWEN3_5_VISION_ARGS() \
LOAD_ARG_OR(image_token_id, "image_token_id", 248056); \
LOAD_ARG_OR(video_token_id, "video_token_id", 248057); \
LOAD_ARG_OR(vision_start_token_id, "vision_start_token_id", 248053); \
LOAD_ARG_OR(vision_end_token_id, "vision_end_token_id", 248054); \
LOAD_ARG(mm_deepstack_visual_indexes, \
"vision_config.deepstack_visual_indexes"); \
LOAD_ARG_OR(mm_deepstack_visual_indexes, \
"vision_config.deepstack_visual_indexes", \
std::vector<int64_t>()); \
if (!args->mm_deepstack_visual_indexes().empty()) { \
LOG(FATAL) << "qwen3_5 VLM does not support DeepStack visual indexes"; \
} \
LOAD_ARG_OR(mm_num_hidden_layers, "vision_config.depth", 27); \
LOAD_ARG_OR(mm_hidden_act, "vision_config.hidden_act", "gelu_pytorch_tanh"); \
LOAD_ARG_OR(mm_hidden_size, "vision_config.hidden_size", 1152); \
@@ -261,32 +330,43 @@ TORCH_MODULE(Qwen3_5ForConditionalGeneration);
LOAD_ARG_OR(mm_num_position_embeddings, \
"vision_config.num_position_embeddings", \
2304); \
LOAD_ARG_OR(mm_projection_dim, "vision_config.out_hidden_size", 5120); \
LOAD_ARG_OR(mm_projection_dim, \
"vision_config.out_hidden_size", \
args->hidden_size()); \
LOAD_ARG_OR(mm_patch_size, "vision_config.patch_size", 16); \
LOAD_ARG_OR(mm_spatial_merge_size, "vision_config.spatial_merge_size", 2); \
LOAD_ARG_OR(mm_temporal_patch_size, "vision_config.temporal_patch_size", 2); \
LOAD_ARG_OR_FUNC(mm_head_dim, "head_dim", [&] { \
return args->mm_hidden_size() / args->mm_num_attention_heads(); \
}); \
LOAD_ARG_OR( \
rope_scaling_rope_type, "vision_config.rope_scaling.type", "mrope")
})
REGISTER_INPUT_PROCESSOR(qwen3_5, Qwen2_5_VLInputProcessor);
// qwen3_5/qwen3_5_moe are multimodal entry points. On NPU, text-only serving
// uses qwen3_5_text/qwen3_5_moe_text from llm/qwen3_5.h because the VLM
// request protocol currently requires array-form chat content.
REGISTER_CAUSAL_VLM_MODEL(qwen3_5, Qwen3_5ForConditionalGeneration);
REGISTER_IMAGE_PROCESSOR(qwen3_5, Qwen2VLImageProcessor);
REGISTER_MPOSITION_GENERATOR(qwen3_5, Qwen3VLMPositionGenerator);
using Qwen35MultimodalProcessor = MultimodalProcessor<Qwen3VLPromptProcessor,
Qwen2VLImageProcessor,
Qwen3VLVideoProcessor>;
REGISTER_MULTIMODAL_PROCESSOR(qwen3_5, Qwen35MultimodalProcessor);
REGISTER_MODEL_ARGS(qwen3_5, [&] {
LOAD_QWEN3_5_COMMON_ARGS();
LOAD_QWEN3_5_VISION_ARGS();
SET_ARG(stop_token_ids, std::unordered_set<int32_t>({args->eos_token_id()}));
SET_ARG(num_experts, 0);
SET_ARG(n_routed_experts, 0);
SET_ARG(n_shared_experts, 0);
SET_ARG(stop_token_ids,
std::unordered_set<int32_t>({args->eos_token_id(), 248046}));
});
REGISTER_INPUT_PROCESSOR(qwen3_5_moe, Qwen2_5_VLInputProcessor);
REGISTER_CAUSAL_VLM_MODEL(qwen3_5_moe, Qwen3_5ForConditionalGeneration);
REGISTER_IMAGE_PROCESSOR(qwen3_5_moe, Qwen2VLImageProcessor);
REGISTER_MPOSITION_GENERATOR(qwen3_5_moe, Qwen3VLMPositionGenerator);
REGISTER_MULTIMODAL_PROCESSOR(qwen3_5_moe, Qwen35MultimodalProcessor);
REGISTER_MODEL_ARGS(qwen3_5_moe, [&] {
LOAD_QWEN3_5_COMMON_ARGS();
LOAD_QWEN3_5_VISION_ARGS();
LOAD_ARG_OR(decoder_sparse_step, "text_config.decoder_sparse_step", 1);
LOAD_ARG_OR(moe_intermediate_size, "text_config.moe_intermediate_size", 512);
LOAD_ARG_OR(num_experts, "text_config.num_experts", 512);
@@ -295,7 +375,6 @@ REGISTER_MODEL_ARGS(qwen3_5_moe, [&] {
"text_config.shared_expert_intermediate_size",
512);
LOAD_ARG_OR(norm_topk_prob, "text_config.norm_topk_prob", true);
LOAD_ARG_OR(
n_routed_experts, "text_config.n_routed_experts", args->num_experts());
SET_ARG(n_shared_experts,
@@ -306,7 +385,56 @@ REGISTER_MODEL_ARGS(qwen3_5_moe, [&] {
SET_ARG(topk_group, 0);
SET_ARG(routed_scaling_factor, 1.0f);
SET_ARG(stop_token_ids, std::unordered_set<int32_t>({args->eos_token_id()}));
SET_ARG(stop_token_ids,
std::unordered_set<int32_t>({args->eos_token_id(), 248046}));
});
// Text-only model registrations. On NPU these are handled by llm/qwen3_5.h.
#if !defined(USE_NPU)
// qwen3_5 without vision config (text-only serving).
// Model args are already registered by the VLM registration above.
REGISTER_CAUSAL_MODEL_WITH_VARNAME(qwen3_5_lm, qwen3_5, Qwen3_5ForCausalLM);
REGISTER_CAUSAL_MODEL_WITH_VARNAME(qwen3_5_moe_lm,
qwen3_5_moe,
Qwen3_5ForCausalLM);
REGISTER_CAUSAL_MODEL(qwen3_5_text, Qwen3_5ForCausalLM);
REGISTER_MODEL_ARGS(qwen3_5_text, [&] {
LOAD_QWEN3_5_COMMON_ARGS();
SET_ARG(num_experts, 0);
SET_ARG(n_routed_experts, 0);
SET_ARG(n_shared_experts, 0);
SET_ARG(decoder_sparse_step, 1);
SET_ARG(stop_token_ids,
std::unordered_set<int32_t>({args->eos_token_id(), 248046}));
});
REGISTER_CAUSAL_MODEL(qwen3_5_moe_text, Qwen3_5ForCausalLM);
REGISTER_MODEL_ARGS(qwen3_5_moe_text, [&] {
LOAD_QWEN3_5_COMMON_ARGS();
LOAD_ARG_OR(decoder_sparse_step, "text_config.decoder_sparse_step", 1);
LOAD_ARG_OR(moe_intermediate_size, "text_config.moe_intermediate_size", 512);
LOAD_ARG_OR(num_experts, "text_config.num_experts", 512);
LOAD_ARG_OR(num_experts_per_tok, "text_config.num_experts_per_tok", 10);
LOAD_ARG_OR(shared_expert_intermediate_size,
"text_config.shared_expert_intermediate_size",
512);
LOAD_ARG_OR(norm_topk_prob, "text_config.norm_topk_prob", true);
LOAD_ARG_OR(
n_routed_experts, "text_config.n_routed_experts", args->num_experts());
SET_ARG(n_shared_experts,
args->shared_expert_intermediate_size() > 0 ? 1 : 0);
SET_ARG(scoring_func, "softmax");
SET_ARG(topk_method, "");
SET_ARG(n_group, -1);
SET_ARG(topk_group, 0);
SET_ARG(routed_scaling_factor, 1.0f);
SET_ARG(stop_token_ids,
std::unordered_set<int32_t>({args->eos_token_id(), 248046}));
});
#endif // !defined(USE_NPU)
#undef LOAD_QWEN3_5_VISION_ARGS
#undef LOAD_QWEN3_5_COMMON_ARGS
} // namespace xllm