87a19d2d00
来源:
1. Chranos/ixformer (GitHub) → ixformer_sdk/ (230 files, 70K lines)
- inference/functions/vllm.py: vllm_moe_topk_softmax 完整实现 (2033 lines)
- inference/functions/moe.py: MoE ops 完整实现 (1380 lines)
- contrib/vllm_flash_attn/: FA2 Python 接口 (1018 lines)
- contrib/tgi/fused_moe.py: TGI fused MoE (429 lines)
- csrc/include/ixformer/: C++ kernel headers + cmake
2. Deep-Spark/xllm (GitHub) → upstream_ref/xllm_latest/ (+15 files)
- npu_torch/qwen3_5_decoder_layer_impl.cpp/.h
- npu_torch/qwen3_5_gated_delta_net.cpp/.h
- npu_torch/qwen3_next_*.cpp/.h (6 files)
- npu_torch/attention.cpp/.h + fused_moe.cpp/.h + CMakeLists.txt
- models/llm/qwen3_5.h + qwen3_5_mtp.h + qwen3_next.h
- models/vlm/qwen3_5.h
调用链完整性:
ixformer_sdk/inference/functions/vllm.py
→ ops.infer.moe_topk_softmax() (C++ 层)
→ 这就是 base 镜像 libixformer.so 里的实现
upstream_ref/xllm_latest/core/layers/ilu/fused_moe.cpp
→ ixformer::infer::topk_softmax() (直接 C++ 调用)
→ ixformer::infer::group_gemm() → 完整 7-step MoE pipeline
131 lines
3.5 KiB
Python
131 lines
3.5 KiB
Python
import time
|
|
from collections import OrderedDict
|
|
from typing import Callable
|
|
|
|
from tabulate import tabulate
|
|
from tqdm import tqdm
|
|
import torch
|
|
|
|
|
|
class BenchmarkTimer:
|
|
def __init__(self):
|
|
self.reset()
|
|
|
|
def reset(self):
|
|
self.start_time = None
|
|
self.end_time = None
|
|
self.running_times = []
|
|
|
|
def __enter__(self):
|
|
self.start_time = time.perf_counter()
|
|
|
|
def __exit__(self, exc_type, exc_val, exc_tb):
|
|
self.end_time = time.perf_counter()
|
|
self.running_times.append(self.end_time - self.start_time)
|
|
|
|
|
|
class Benchmark:
|
|
def __init__(
|
|
self,
|
|
warmup: int = None,
|
|
number: int = 100,
|
|
timer=None,
|
|
description: str = None,
|
|
show_progress: bool = False,
|
|
fn_desc_key: str = "fn_desc",
|
|
sync: bool = True,
|
|
):
|
|
if warmup is None:
|
|
warmup = int(number // 100) + 10
|
|
self.warmup = warmup
|
|
self.number = number
|
|
self.description = description
|
|
self.show_progress = show_progress
|
|
self.fn_desc_key = fn_desc_key
|
|
self.sync = sync
|
|
|
|
if timer is None:
|
|
timer = BenchmarkTimer()
|
|
self.timer = timer
|
|
|
|
self.reset()
|
|
|
|
def reset(self):
|
|
self.results = OrderedDict()
|
|
self._run_index = 0
|
|
self._fn_name = None
|
|
|
|
def run(self, fn, *args, **kwargs):
|
|
self._run_index += 1
|
|
|
|
if self.fn_desc_key in kwargs:
|
|
self.set_fn_name(kwargs[self.fn_desc_key])
|
|
kwargs.pop(self.fn_desc_key)
|
|
key = self._get_fn_key(fn)
|
|
|
|
# warmup
|
|
self._run_fn(False, fn, *args, **kwargs)
|
|
|
|
# get running times
|
|
results = self._run_fn(True, fn, *args, **kwargs)
|
|
self.results[key] = results
|
|
|
|
return results
|
|
|
|
def set_fn_name(self, name):
|
|
self._fn_name = name
|
|
|
|
def _run_fn(self, benchmark: bool, fn: Callable, *args, **kwargs):
|
|
self.timer.reset()
|
|
n = self.number if benchmark else self.warmup
|
|
if self.show_progress and benchmark:
|
|
progress = tqdm(range(n), desc=self._get_fn_key(fn))
|
|
else:
|
|
progress = range(n)
|
|
|
|
torch.cuda.synchronize()
|
|
|
|
for _ in progress:
|
|
with self.timer:
|
|
fn(*args, **kwargs)
|
|
|
|
if self.sync:
|
|
torch.cuda.synchronize()
|
|
|
|
return self.timer.running_times
|
|
|
|
def _get_fn_key(self, fn: Callable):
|
|
if self._fn_name is not None:
|
|
return self._fn_name
|
|
|
|
if hasattr(fn, "__name__"):
|
|
fn_name = fn.__name__
|
|
else:
|
|
fn_name = str(fn)
|
|
|
|
return f"{fn_name}_{self._run_index}"
|
|
|
|
def render(self) -> str:
|
|
head = [""] + list(self.results.keys())
|
|
total = ["Total (s)"] + [sum(times) for times in self.results.values()]
|
|
mean = ["Mean (s)"] + [_t / self.number for _t in total[1:]]
|
|
min_ = ["Min (s)"] + [min(times) for times in self.results.values()]
|
|
max_ = ["Max (s)"] + [max(times) for times in self.results.values()]
|
|
count = ["Count"] + [len(list(times)) for times in self.results.values()]
|
|
|
|
return tabulate(
|
|
headers=head,
|
|
tabular_data=[total, mean, min_, max_, count],
|
|
numalign="right",
|
|
)
|
|
|
|
def print_caption(self):
|
|
if self.description is not None:
|
|
caption = "\n" + "=" * 60 + "\n"
|
|
caption += f"= {self.description}" + "\n"
|
|
caption += "=" * 60 + "\n"
|
|
print(caption)
|
|
|
|
def print(self):
|
|
print(self.render())
|