87a19d2d00
来源:
1. Chranos/ixformer (GitHub) → ixformer_sdk/ (230 files, 70K lines)
- inference/functions/vllm.py: vllm_moe_topk_softmax 完整实现 (2033 lines)
- inference/functions/moe.py: MoE ops 完整实现 (1380 lines)
- contrib/vllm_flash_attn/: FA2 Python 接口 (1018 lines)
- contrib/tgi/fused_moe.py: TGI fused MoE (429 lines)
- csrc/include/ixformer/: C++ kernel headers + cmake
2. Deep-Spark/xllm (GitHub) → upstream_ref/xllm_latest/ (+15 files)
- npu_torch/qwen3_5_decoder_layer_impl.cpp/.h
- npu_torch/qwen3_5_gated_delta_net.cpp/.h
- npu_torch/qwen3_next_*.cpp/.h (6 files)
- npu_torch/attention.cpp/.h + fused_moe.cpp/.h + CMakeLists.txt
- models/llm/qwen3_5.h + qwen3_5_mtp.h + qwen3_next.h
- models/vlm/qwen3_5.h
调用链完整性:
ixformer_sdk/inference/functions/vllm.py
→ ops.infer.moe_topk_softmax() (C++ 层)
→ 这就是 base 镜像 libixformer.so 里的实现
upstream_ref/xllm_latest/core/layers/ilu/fused_moe.cpp
→ ixformer::infer::topk_softmax() (直接 C++ 调用)
→ ixformer::infer::group_gemm() → 完整 7-step MoE pipeline
240 lines
9.6 KiB
Python
240 lines
9.6 KiB
Python
from typing import Union
|
|
|
|
import ixformer._C as ops
|
|
import torch
|
|
from torch.autograd.function import Function
|
|
from torch.nn import init
|
|
from torch.nn.parameter import Parameter
|
|
|
|
__all__ = ["vocab_parallel_cross_entropy"]
|
|
|
|
|
|
class _VocabParallelCrossEntropyCustom(Function):
|
|
@staticmethod
|
|
def forward(ctx, vocab_parallel_logits, target, label_smoothing=0.0):
|
|
device = vocab_parallel_logits.device
|
|
xnumel = vocab_parallel_logits.shape[0]
|
|
rnumel = vocab_parallel_logits.shape[-1]
|
|
exp_logits = torch.empty(
|
|
(xnumel, 1, rnumel), device=device, dtype=torch.float32
|
|
)
|
|
masked_target_1d = torch.empty((xnumel,), device=device, dtype=torch.int32)
|
|
loss = torch.empty((xnumel, 1), device=device, dtype=torch.float32)
|
|
|
|
ops.train.cross_entropy_loss_forward(
|
|
vocab_parallel_logits, target.int(), exp_logits, masked_target_1d, loss
|
|
)
|
|
|
|
vocab_size = exp_logits.size(-1)
|
|
if label_smoothing > 0:
|
|
"""
|
|
We'd like to assign 1 / (K - 1) probability mass to every index that is not the ground truth.
|
|
= (1 - alpha) * y_gt + alpha * mean(y_{i for i != gt})
|
|
= (1 - alpha) * y_gt + (alpha / (K - 1)) * \sum_{i != gt} y_i
|
|
= ((K - 1) * (1 - alpha) / (K - 1)) * y_gt + (alpha / (K - 1)) * \sum_{i != gt} y_i
|
|
= (K * (1 - alpha) - 1) / (K - 1)) * y_gt + (alpha / (K - 1)) * \sum_{i} y_i
|
|
= (1 - (alpha * K) / (K - 1)) * y_gt + ( (alpha * K) / (K - 1) ) * \sum_{i} y_i / K
|
|
From: https://github.com/NVIDIA/NeMo/blob/main/nemo/collections/common/losses/smoothed_cross_entropy.py
|
|
"""
|
|
assert 1.0 > label_smoothing > 0.0
|
|
smoothing = label_smoothing * vocab_size / (vocab_size - 1)
|
|
|
|
# Exp logits at this point are normalized probabilities. So we can just take the log to get log-probs.
|
|
log_probs = torch.log(exp_logits)
|
|
mean_log_probs = log_probs.mean(dim=-1)
|
|
loss = (1.0 - smoothing) * loss - smoothing * mean_log_probs
|
|
|
|
ctx.label_smoothing, ctx.vocab_size = label_smoothing, vocab_size
|
|
|
|
# Store softmax, target-mask and masked-target for backward pass.
|
|
ctx.save_for_backward(exp_logits, masked_target_1d)
|
|
|
|
return loss
|
|
|
|
@staticmethod
|
|
def backward(ctx, grad_output):
|
|
|
|
# Retreive tensors from the forward path.
|
|
softmax, masked_target_1d = ctx.saved_tensors
|
|
label_smoothing, vocab_size = ctx.label_smoothing, ctx.vocab_size
|
|
|
|
# All the inputs have softmax as thier gradient.
|
|
grad_input = softmax
|
|
# For simplicity, work with the 2D gradient.
|
|
partition_vocab_size = softmax.size()[-1]
|
|
grad_2d = grad_input.view(-1, partition_vocab_size)
|
|
|
|
# Add the gradient from matching classes.
|
|
arange_1d = torch.arange(start=0, end=grad_2d.size()[0], device=grad_2d.device)
|
|
|
|
softmax_update = 1.0
|
|
|
|
if label_smoothing > 0:
|
|
smoothing = label_smoothing * vocab_size / (vocab_size - 1)
|
|
grad_2d[arange_1d, masked_target_1d] -= (1.0 - smoothing) * softmax_update
|
|
average_grad = 1 / vocab_size
|
|
grad_2d[arange_1d, :] -= smoothing * average_grad
|
|
else:
|
|
grad_2d[arange_1d, masked_target_1d] -= softmax_update
|
|
|
|
# Finally elementwise multiplication with the output gradients.
|
|
grad_input = torch.mul(grad_input, grad_output.unsqueeze(dim=-1))
|
|
|
|
return grad_input, None, None
|
|
|
|
|
|
class _VocabParallelCrossEntropy(Function):
|
|
@staticmethod
|
|
def forward(
|
|
ctx,
|
|
vocab_parallel_logits,
|
|
target,
|
|
label_smoothing=0.0,
|
|
vocab_start_index=0,
|
|
vocab_end_index=320000,
|
|
group=None,
|
|
):
|
|
|
|
# Maximum value along vocab dimension across all GPUs.
|
|
logits_max = torch.max(vocab_parallel_logits, dim=-1)[0]
|
|
torch.distributed.all_reduce(
|
|
logits_max, op=torch.distributed.ReduceOp.MAX, group=group
|
|
)
|
|
# Subtract the maximum value.
|
|
vocab_parallel_logits = vocab_parallel_logits - logits_max.unsqueeze(dim=-1)
|
|
|
|
# Get the partition's vocab indecies
|
|
partition_vocab_size = vocab_parallel_logits.size()[-1]
|
|
|
|
# Create a mask of valid vocab ids (1 means it needs to be masked).
|
|
target_mask = (target < vocab_start_index) | (target >= vocab_end_index)
|
|
masked_target = target.clone() - vocab_start_index
|
|
masked_target[target_mask] = 0
|
|
|
|
# Get predicted-logits = logits[target].
|
|
# For Simplicity, we convert logits to a 2-D tensor with size
|
|
# [*, partition-vocab-size] and target to a 1-D tensor of size [*].
|
|
logits_2d = vocab_parallel_logits.view(-1, partition_vocab_size)
|
|
masked_target_1d = masked_target.view(-1)
|
|
arange_1d = torch.arange(
|
|
start=0, end=logits_2d.size()[0], device=logits_2d.device
|
|
)
|
|
predicted_logits_1d = logits_2d[arange_1d, masked_target_1d]
|
|
predicted_logits_1d = predicted_logits_1d.clone().contiguous()
|
|
predicted_logits = predicted_logits_1d.view_as(target)
|
|
predicted_logits[target_mask] = 0.0
|
|
# All reduce is needed to get the chunks from other GPUs.
|
|
torch.distributed.all_reduce(
|
|
predicted_logits,
|
|
op=torch.distributed.ReduceOp.SUM,
|
|
group=group,
|
|
)
|
|
|
|
# Sum of exponential of logits along vocab dimension across all GPUs.
|
|
exp_logits = vocab_parallel_logits
|
|
torch.exp(vocab_parallel_logits, out=exp_logits)
|
|
sum_exp_logits = exp_logits.sum(dim=-1)
|
|
torch.distributed.all_reduce(
|
|
sum_exp_logits,
|
|
op=torch.distributed.ReduceOp.SUM,
|
|
group=group,
|
|
)
|
|
|
|
# Loss = log(sum(exp(logits))) - predicted-logit.
|
|
loss = torch.log(sum_exp_logits) - predicted_logits
|
|
|
|
# Normalize and optionally smooth logits
|
|
exp_logits.div_(sum_exp_logits.unsqueeze(dim=-1))
|
|
|
|
vocab_size = exp_logits.size(-1)
|
|
if label_smoothing > 0:
|
|
"""
|
|
We'd like to assign 1 / (K - 1) probability mass to every index that is not the ground truth.
|
|
= (1 - alpha) * y_gt + alpha * mean(y_{i for i != gt})
|
|
= (1 - alpha) * y_gt + (alpha / (K - 1)) * \sum_{i != gt} y_i
|
|
= ((K - 1) * (1 - alpha) / (K - 1)) * y_gt + (alpha / (K - 1)) * \sum_{i != gt} y_i
|
|
= (K * (1 - alpha) - 1) / (K - 1)) * y_gt + (alpha / (K - 1)) * \sum_{i} y_i
|
|
= (1 - (alpha * K) / (K - 1)) * y_gt + ( (alpha * K) / (K - 1) ) * \sum_{i} y_i / K
|
|
From: https://github.com/NVIDIA/NeMo/blob/main/nemo/collections/common/losses/smoothed_cross_entropy.py
|
|
"""
|
|
assert 1.0 > label_smoothing > 0.0
|
|
smoothing = label_smoothing * vocab_size / (vocab_size - 1)
|
|
|
|
# Exp logits at this point are normalized probabilities. So we can just take the log to get log-probs.
|
|
log_probs = torch.log(exp_logits)
|
|
mean_log_probs = log_probs.mean(dim=-1)
|
|
loss = (1.0 - smoothing) * loss - smoothing * mean_log_probs
|
|
|
|
ctx.label_smoothing, ctx.vocab_size = label_smoothing, vocab_size
|
|
|
|
# Store softmax, target-mask and masked-target for backward pass.
|
|
ctx.save_for_backward(exp_logits, target_mask, masked_target_1d)
|
|
|
|
return loss
|
|
|
|
@staticmethod
|
|
def backward(ctx, grad_output):
|
|
|
|
# Retreive tensors from the forward path.
|
|
softmax, target_mask, masked_target_1d = ctx.saved_tensors
|
|
label_smoothing, vocab_size = ctx.label_smoothing, ctx.vocab_size
|
|
|
|
# All the inputs have softmax as thier gradient.
|
|
grad_input = softmax
|
|
# For simplicity, work with the 2D gradient.
|
|
partition_vocab_size = softmax.size()[-1]
|
|
grad_2d = grad_input.view(-1, partition_vocab_size)
|
|
|
|
# Add the gradient from matching classes.
|
|
arange_1d = torch.arange(start=0, end=grad_2d.size()[0], device=grad_2d.device)
|
|
|
|
softmax_update = 1.0 - target_mask.view(-1).float()
|
|
|
|
if label_smoothing > 0:
|
|
smoothing = label_smoothing * vocab_size / (vocab_size - 1)
|
|
grad_2d[arange_1d, masked_target_1d] -= (1.0 - smoothing) * softmax_update
|
|
average_grad = 1 / vocab_size
|
|
grad_2d[arange_1d, :] -= smoothing * average_grad
|
|
else:
|
|
grad_2d[arange_1d, masked_target_1d] -= softmax_update
|
|
|
|
# Finally elementwise multiplication with the output gradients.
|
|
grad_input.mul_(grad_output.unsqueeze(dim=-1))
|
|
|
|
return grad_input, None, None, None, None, None
|
|
|
|
|
|
def vocab_parallel_cross_entropy(
|
|
vocab_parallel_logits: torch.Tensor,
|
|
target: torch.Tensor,
|
|
label_smoothing: float = 0.0,
|
|
world_size: int = 1,
|
|
vocab_start_index: int = 0,
|
|
vocab_end_index: int = 320000,
|
|
group=None,
|
|
):
|
|
"""
|
|
参数说明:
|
|
目前只支持batch_size = 1 的情况,当batch_size >1时,计算不正确
|
|
Args:
|
|
vocab_parallel_logits: shape : [seq_len,1,vocal_size] dtype : torch.bfloat16,torch.float,torch.half
|
|
target: shape : [seq_len,1] dtype : torch.int64
|
|
group: TP 并行组
|
|
return:
|
|
loss: shape : [seq_len,1] dtype : torch.float32
|
|
|
|
"""
|
|
if world_size == 1:
|
|
return _VocabParallelCrossEntropyCustom.apply(
|
|
vocab_parallel_logits, target, label_smoothing
|
|
)
|
|
else:
|
|
return _VocabParallelCrossEntropy.apply(
|
|
vocab_parallel_logits,
|
|
target,
|
|
label_smoothing,
|
|
vocab_start_index,
|
|
vocab_end_index,
|
|
group,
|
|
)
|