feat(CRITICAL): 从 GitHub 扫描搬运 ixformer SDK + xllm 完整 GDN/MoE 代码
来源:
1. Chranos/ixformer (GitHub) → ixformer_sdk/ (230 files, 70K lines)
- inference/functions/vllm.py: vllm_moe_topk_softmax 完整实现 (2033 lines)
- inference/functions/moe.py: MoE ops 完整实现 (1380 lines)
- contrib/vllm_flash_attn/: FA2 Python 接口 (1018 lines)
- contrib/tgi/fused_moe.py: TGI fused MoE (429 lines)
- csrc/include/ixformer/: C++ kernel headers + cmake
2. Deep-Spark/xllm (GitHub) → upstream_ref/xllm_latest/ (+15 files)
- npu_torch/qwen3_5_decoder_layer_impl.cpp/.h
- npu_torch/qwen3_5_gated_delta_net.cpp/.h
- npu_torch/qwen3_next_*.cpp/.h (6 files)
- npu_torch/attention.cpp/.h + fused_moe.cpp/.h + CMakeLists.txt
- models/llm/qwen3_5.h + qwen3_5_mtp.h + qwen3_next.h
- models/vlm/qwen3_5.h
调用链完整性:
ixformer_sdk/inference/functions/vllm.py
→ ops.infer.moe_topk_softmax() (C++ 层)
→ 这就是 base 镜像 libixformer.so 里的实现
upstream_ref/xllm_latest/core/layers/ilu/fused_moe.cpp
→ ixformer::infer::topk_softmax() (直接 C++ 调用)
→ ixformer::infer::group_gemm() → 完整 7-step MoE pipeline
This commit is contained in:
project6-dev
@@ -0,0 +1 @@
|
||||
from .modeling_codeshell import CodeShellForCausalLM
|
||||
@@ -0,0 +1,153 @@
|
||||
# coding=utf-8
|
||||
# Copyright 2023 WisdomShell Inc. All Rights Reserved.
|
||||
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# This code is based on Bigcode's GPTBigCode configuration. It has been modified from
|
||||
# its original forms to accommodate minor architectural differences compared to
|
||||
# GPTBigCode Configuration that trained the model.
|
||||
|
||||
# coding=utf-8
|
||||
# Copyright 2023 The BigCode team and HuggingFace Inc. team.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
""" CodeShell configuration"""
|
||||
|
||||
from transformers.configuration_utils import PretrainedConfig
|
||||
from transformers.utils import logging
|
||||
|
||||
logger = logging.get_logger(__name__)
|
||||
|
||||
|
||||
class CodeShellConfig(PretrainedConfig):
|
||||
"""
|
||||
This is the configuration class to store the configuration of a [`CodeShellModel`]. It is used to instantiate a
|
||||
CodeShell model according to the specified arguments, defining the model architecture.
|
||||
|
||||
Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
|
||||
documentation from [`PretrainedConfig`] for more information.
|
||||
|
||||
Args:
|
||||
vocab_size (`int`, *optional*, defaults to 50257):
|
||||
Vocabulary size of the GPT-2 model. Defines the number of different tokens that can be represented by the
|
||||
`inputs_ids` passed when calling [`CodeShellModel`].
|
||||
n_positions (`int`, *optional*, defaults to 1024):
|
||||
The maximum sequence length that this model might ever be used with. Typically set this to something large
|
||||
just in case (e.g., 512 or 1024 or 2048).
|
||||
n_embd (`int`, *optional*, defaults to 768):
|
||||
Dimensionality of the embeddings and hidden states.
|
||||
n_layer (`int`, *optional*, defaults to 12):
|
||||
Number of hidden layers in the Transformer encoder.
|
||||
n_head (`int`, *optional*, defaults to 12):
|
||||
Number of attention heads for each attention layer in the Transformer encoder.
|
||||
n_inner (`int`, *optional*, defaults to None):
|
||||
Dimensionality of the inner feed-forward layers. `None` will set it to 4 times n_embd
|
||||
activation_function (`str`, *optional*, defaults to `"gelu_pytorch_tanh"`):
|
||||
Activation function, to be selected in the list `["relu", "silu", "gelu", "tanh", "gelu_new",
|
||||
"gelu_pytorch_tanh"]`.
|
||||
resid_pdrop (`float`, *optional*, defaults to 0.1):
|
||||
The dropout probability for all fully connected layers in the embeddings, encoder, and pooler.
|
||||
embd_pdrop (`float`, *optional*, defaults to 0.1):
|
||||
The dropout ratio for the embeddings.
|
||||
attn_pdrop (`float`, *optional*, defaults to 0.1):
|
||||
The dropout ratio for the attention.
|
||||
layer_norm_epsilon (`float`, *optional*, defaults to 1e-5):
|
||||
The epsilon to use in the layer normalization layers.
|
||||
initializer_range (`float`, *optional*, defaults to 0.02):
|
||||
The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
|
||||
scale_attn_weights (`bool`, *optional*, defaults to `True`):
|
||||
Scale attention weights by dividing by sqrt(hidden_size)..
|
||||
use_cache (`bool`, *optional*, defaults to `True`):
|
||||
Whether or not the model should return the last key/values attentions (not used by all models).
|
||||
attention_softmax_in_fp32 (`bool`, *optional*, defaults to `True`):
|
||||
Whether to call the fused softmax in float32.
|
||||
scale_attention_softmax_in_fp32 (`bool`, *optional*, defaults to `True`):
|
||||
Whether to scale the attention softmax in float32.
|
||||
attention_type (`bool`, *optional*, defaults to `True`):
|
||||
Whether to use Multi-Query Attion (`True`) or Multi-Head Attention (`False`).
|
||||
"""
|
||||
|
||||
model_type = "codeshell"
|
||||
keys_to_ignore_at_inference = ["past_key_values"]
|
||||
attribute_map = {
|
||||
"hidden_size": "n_embd",
|
||||
"max_position_embeddings": "n_positions",
|
||||
"num_attention_heads": "n_head",
|
||||
"num_hidden_layers": "n_layer",
|
||||
}
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vocab_size=70144,
|
||||
n_positions=8192,
|
||||
n_embd=4096,
|
||||
n_layer=42,
|
||||
n_head=32,
|
||||
n_inner=None,
|
||||
activation_function="gelu_pytorch_tanh",
|
||||
resid_pdrop=0.1,
|
||||
embd_pdrop=0.1,
|
||||
attn_pdrop=0.1,
|
||||
layer_norm_epsilon=1e-5,
|
||||
initializer_range=0.02,
|
||||
scale_attn_weights=True,
|
||||
use_cache=True,
|
||||
bos_token_id=70000,
|
||||
eos_token_id=70000,
|
||||
attention_softmax_in_fp32=True,
|
||||
scale_attention_softmax_in_fp32=True,
|
||||
group_query_attention=True,
|
||||
num_query_groups=1,
|
||||
position_embedding_type="learned_absolute",
|
||||
rope_scaling=None,
|
||||
**kwargs,
|
||||
):
|
||||
self.vocab_size = vocab_size
|
||||
self.n_positions = n_positions
|
||||
self.n_embd = n_embd
|
||||
self.n_layer = n_layer
|
||||
self.n_head = n_head
|
||||
self.n_inner = n_inner
|
||||
self.activation_function = activation_function
|
||||
self.resid_pdrop = resid_pdrop
|
||||
self.embd_pdrop = embd_pdrop
|
||||
self.attn_pdrop = attn_pdrop
|
||||
self.layer_norm_epsilon = layer_norm_epsilon
|
||||
self.initializer_range = initializer_range
|
||||
self.scale_attn_weights = scale_attn_weights
|
||||
self.use_cache = use_cache
|
||||
self.attention_softmax_in_fp32 = attention_softmax_in_fp32
|
||||
self.scale_attention_softmax_in_fp32 = scale_attention_softmax_in_fp32
|
||||
self.group_query_attention = group_query_attention
|
||||
self.num_query_groups = num_query_groups
|
||||
self.position_embedding_type = position_embedding_type
|
||||
self.rope_scaling = rope_scaling
|
||||
assert self.position_embedding_type in [
|
||||
"learned_absolute",
|
||||
"rope",
|
||||
], "position_embedding_type must be one of ['learned_absolute', 'rope']"
|
||||
|
||||
self.bos_token_id = bos_token_id
|
||||
self.eos_token_id = eos_token_id
|
||||
|
||||
super().__init__(bos_token_id=bos_token_id, eos_token_id=eos_token_id, **kwargs)
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,99 @@
|
||||
import math
|
||||
|
||||
import ixformer.functions as ixf_F
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
from transformers.utils import logging
|
||||
|
||||
logger = logging.get_logger(__name__)
|
||||
|
||||
|
||||
def mha(query, key, value, attention_mask):
|
||||
if attention_mask is None and query.shape[2] == key.shape[2]:
|
||||
context_layer = ixf_F.scaled_dot_product_attention(
|
||||
query.contiguous(), key.contiguous(), value.contiguous(), is_causal=True
|
||||
)
|
||||
else:
|
||||
# if attention_mask is not None:
|
||||
# # attention_mask = attention_mask
|
||||
# attention_mask = (~attention_mask).cuda().float()*(-10000)
|
||||
context_layer = ixf_F.scaled_dot_product_attention(
|
||||
query.contiguous(), key.contiguous(), value.contiguous(), attention_mask
|
||||
)
|
||||
|
||||
# context_layer = context_layer.transpose(1, 2).contiguous()
|
||||
# res_shape = list(context_layer.shape)
|
||||
# res_shape = res_shape[:2] + [-1]
|
||||
# context_layer = context_layer.view(*res_shape)
|
||||
return context_layer
|
||||
# batch_size, head_num, seq_len, head_dim = query.shape
|
||||
# src_len = query.shape[-2]
|
||||
# tgt_len = key.shape[-2]
|
||||
|
||||
# if attention_mask is None and src_len == tgt_len:
|
||||
# attention_mask = ~torch.tril(torch.ones([src_len, tgt_len])).bool()
|
||||
# elif attention_mask is None:
|
||||
# attention_mask = torch.zeros([src_len, tgt_len])
|
||||
# attention_mask = attention_mask.cuda().int()
|
||||
|
||||
# attention_scores = ixf_F.act_bias_mm(
|
||||
# query, key, scale=1 / math.sqrt(head_dim), trans_format="TN"
|
||||
# )
|
||||
# # softmax
|
||||
# # if tgt_len > 2048:
|
||||
# # if not (attention_mask == 0).all():
|
||||
# # attention_scores.masked_fill_(attention_mask.bool(), -10000.0)
|
||||
# # dtype = attention_scores.dtype
|
||||
# # attention_probs = F.softmax(attention_scores.float(), dim=-1)
|
||||
# # attention_probs = attention_probs.type(dtype)
|
||||
# # else:
|
||||
# # raise NotImplementedError()
|
||||
# attention_probs = ixf_F.attention_masked_softmax(
|
||||
# attention_scores, attention_mask.int()
|
||||
# )
|
||||
# # s * v
|
||||
# # batch_size,head_num,seq_len,head_dim
|
||||
# context_layer = ixf_F.act_bias_mm(
|
||||
# attention_probs, value, trans_format="NN")
|
||||
# context_layer = context_layer.transpose(1, 2).contiguous()
|
||||
# context_layer = context_layer.view(
|
||||
# batch_size, seq_len, head_num * head_dim)
|
||||
|
||||
|
||||
def mlp(mlp_input, ff1_weight, ff1_bias, ff2_weight):
|
||||
input_shape = list(mlp_input.shape)
|
||||
mlp_input = mlp_input.view(-1, input_shape[-1])
|
||||
mlp_output = ixf_F.act_bias_mm(
|
||||
mlp_input, ff1_weight, ff1_bias, scale=1, act_type="gelu", trans_format="TN"
|
||||
)
|
||||
mlp_output = ixf_F.linear(mlp_output, ff2_weight, None)
|
||||
input_shape[-1] = -1
|
||||
mlp_output = mlp_output.view(*input_shape)
|
||||
return mlp_output
|
||||
|
||||
|
||||
def mlp_forward(self, hidden_states):
|
||||
# [s, b, 4hp]
|
||||
# intermediate_parallel = self.dense_h_to_4h(hidden_states)
|
||||
# intermediate_parallel = self.activation_func(intermediate_parallel)
|
||||
input_shape = list(hidden_states.shape)
|
||||
hidden_states = hidden_states.view(-1, input_shape[-1])
|
||||
mlp_output = ixf_F.act_bias_mm(
|
||||
hidden_states,
|
||||
self.c_fc.weight,
|
||||
self.c_fc.bias,
|
||||
scale=1,
|
||||
act_type="gelu",
|
||||
trans_format="TN",
|
||||
)
|
||||
if isinstance(self.c_proj, nn.Linear):
|
||||
output = ixf_F.linear(
|
||||
mlp_output,
|
||||
self.c_proj.weight,
|
||||
self.c_proj.bias,
|
||||
)
|
||||
else:
|
||||
output = self.c_proj(mlp_output)
|
||||
output = output.view(*input_shape)
|
||||
return output
|
||||
Reference in New Issue
Block a user