first commit

image_build/.dockerignore

@@ -0,0 +1,6 @@
+*
+!Dockerfile
+!apply_patches.py
+!routing_compat.py
+!test_route_p800.py
+!serve_deepseek.sh

image_build/Dockerfile

@@ -0,0 +1,11 @@
+ARG BASE_IMAGE=xc-llm-kunlun:v0.25.1
+FROM ${BASE_IMAGE}
+USER root
+COPY apply_patches.py routing_compat.py test_route_p800.py serve_deepseek.sh /opt/kunlun-deepseek-fixes/
+RUN /opt/vllm_kunlun/bin/python /opt/kunlun-deepseek-fixes/apply_patches.py \
+    && /opt/vllm_kunlun/bin/python /opt/kunlun-deepseek-fixes/apply_patches.py --check \
+    && bash -n /opt/kunlun-deepseek-fixes/serve_deepseek.sh
+ENV PATH="/opt/vllm_kunlun/bin:${PATH}"
+WORKDIR /workspace
+ENTRYPOINT []
+CMD ["/bin/bash"]

image_build/README.md

@@ -0,0 +1,91 @@
+# 昆仑芯 DeepSeek INT8 修复镜像
+
+基础镜像：用户已提交的 `xc-llm-kunlun:v0.25.1`。
+输出镜像：`xc-llm-kunlun:v0.25.1-int8-fixed`。
+
+## 修改内容
+
+1. `vllm_kunlun/quantization/compressed_tensors/compressed_tensors_moe.py`：
+   INT8 MoE 从 RoutedExperts 实例读取 scoring_func、num_expert_group、topk_group、e_score_correction_bias 和 routed_scaling_factor，替代旧接口的默认值。避免 DeepSeek sigmoid 分组路由被错误地执行成 softmax；尊重外层 runner 的缩放分工，不硬编码 2.5。
+2. `vllm_kunlun/ops/_custom_ops.py`：
+   两个 sigmoid 路由包装函数向 kunlun_ops 传参时，将 `block_static=block_static` 改为 `block_statistic=block_static`。保留上层 Torch 注册接口的 block_static 名称。
+
+补丁可重复执行。构建时检查版本、语法、路由参数与底层函数签名，不导入 torch/vllm、不使用 XPU。文件原本已修复时只验证，不重复插入。
+
+MLA 接口并未修复，启动脚本使用 `VLLM_MLA_DISABLE=1` 绕过当前 FlashMLAImpl 缺少 forward_mha/forward_mqa 的问题。使用 `VLLM_KUNLUN_ENABLE_INT8_BMM=0`，与已转换权重中的 BF16 kv_b_proj 一致。首测使用 TP=8、不启用 EP、eager、单并发、4096 上下文、每卡 4 GiB KV cache。
+
+模型权重不打入镜像。当前转换结果中，路由专家和部分 MLA 投影为 INT8，kv_b_proj、Dense MLP 与共享专家为 BF16；MTP 已排除。
+
+## 构建
+
+在 172.28.4.201 宿主机执行：
+
+```bash
+cd /mnt/disk1/quant_tools/deepseek_r1_image
+docker build --pull=false \
+  --build-arg BASE_IMAGE=xc-llm-kunlun:v0.25.1 \
+  -t xc-llm-kunlun:v0.25.1-int8-fixed .
+```
+
+仅针对内置文件验证补丁，不挂卡、不挂模型：
+
+```bash
+docker run --rm xc-llm-kunlun:v0.25.1-int8-fixed \
+  /opt/vllm_kunlun/bin/python /opt/kunlun-deepseek-fixes/apply_patches.py --check
+```
+
+## 启动与验证
+
+切换服务前先停止旧服务，释放设备和端口。若旧服务由 srx_test 承载，可停止旧容器后再启动新容器：
+
+```bash
+docker stop srx_test
+cd /mnt/disk1/quant_tools/deepseek_r1_image
+bash run_container.sh xc-llm-kunlun:v0.25.1-int8-fixed deepseek_int8_fixed
+```
+
+run_container.sh 挂入八张 XPU、xpuctrl、32 GiB 共享内存，使用 host 网络，沿用实际宿主机映射：
+
+```text
+/mnt/disk0 -> /mnt/data0
+/mnt/disk1 -> /mnt/data1
+```
+
+执行真实 P800 路由数值检查（小张量，不加载模型）：
+
+```bash
+docker exec deepseek_int8_fixed \
+  /opt/vllm_kunlun/bin/python /opt/kunlun-deepseek-fixes/test_route_p800.py
+```
+
+启动完整服务（该命令前台运行，测试请求另开终端）：
+
+```bash
+docker exec -it deepseek_int8_fixed \
+  bash /opt/kunlun-deepseek-fixes/serve_deepseek.sh
+```
+
+确认模型文件存在于宿主机 `/mnt/disk1/deepseek-r1-INT8`。镜像内脚本使用容器路径 `/mnt/data1/deepseek-r1-INT8`。
+
+服务启动后：
+
+```bash
+curl -sS http://127.0.0.1:8000/v1/models
+curl -sS --max-time 300 http://127.0.0.1:8000/v1/chat/completions \
+  -H 'Content-Type: application/json' \
+  -d '{"model":"/mnt/data1/deepseek-r1-INT8","messages":[{"role":"user","content":"北京是哪个国家的首都？请简短回答。"}],"temperature":0,"max_tokens":2048}'
+```
+
+查看 content、reasoning 和 finish_reason。完整输出质量需实际业务问题验证，算子数值检查不能代替该步骤。
+
+## 导出与回滚
+
+```bash
+docker save -o /mnt/disk1/xc-llm-kunlun-v0.25.1-int8-fixed.tar \
+  xc-llm-kunlun:v0.25.1-int8-fixed
+# 另一台同架构、驱动兼容的 P800 机器：
+docker load -i /path/to/xc-llm-kunlun-v0.25.1-int8-fixed.tar
+```
+
+镜像归档不包含挂载模型；迁移时模型目录需要单独准备。
+回滚时停止新服务/容器，再使用原镜像或启动原容器；旧镜像标签保持不变。

image_build/apply_patches.py

@@ -0,0 +1,81 @@
+"""Idempotent source patches and CPU checks. No vLLM/torch imports or XPU needed."""
+import argparse
+import ast
+import copy
+import hashlib
+import inspect
+import json
+from importlib import metadata
+from pathlib import Path
+from types import SimpleNamespace
+
+from routing_compat import patch_source, regression_tests, INSERT
+
+parser = argparse.ArgumentParser()
+parser.add_argument('--site-packages', type=Path,
+                    default=Path('/opt/vllm_kunlun/lib/python3.10/site-packages'))
+parser.add_argument('--check', action='store_true', help='Verify without changing installed files')
+args = parser.parse_args()
+root = args.site_packages
+route = root / 'vllm_kunlun/quantization/compressed_tensors/compressed_tensors_moe.py'
+wrapper = root / 'vllm_kunlun/ops/_custom_ops.py'
+lower = root / 'kunlun_ops/_xflashinfer_moe.py'
+versions = {p: metadata.version(p) for p in ('vllm', 'vllm-kunlun', 'kunlun-ops')}
+if not versions['vllm'].startswith('0.25.1') or not versions['vllm-kunlun'].startswith('0.25.1'):
+    raise RuntimeError(f'Unsupported environment: {versions}')
+
+before_route = route.read_text()
+after_route = patch_source(before_route)
+# Reproduce the legacy bug and exercise patched method dispatch.
+regression_tests(after_route.replace(INSERT, ''), after_route)
+lower_fn = next(n for n in ast.parse(lower.read_text()).body
+                if isinstance(n, ast.FunctionDef) and n.name == 'moe_sigmoid_group_topk_norm')
+signature = inspect.Signature([inspect.Parameter(a.arg, inspect.Parameter.POSITIONAL_OR_KEYWORD)
+                               for a in lower_fn.args.args])
+assert 'block_statistic' in signature.parameters
+
+before_wrapper = wrapper.read_text()
+lines = before_wrapper.splitlines(keepends=True)
+names = {'moe_sigmoid_group_topk_norm', 'moe_sigmoid_group_topk_norm_cuda'}
+functions = [n for n in ast.parse(before_wrapper).body if isinstance(n, ast.FunctionDef) and n.name in names]
+assert len(functions) == 2
+for fn in functions:
+    for call in ast.walk(fn):
+        if isinstance(call, ast.Call) and ast.unparse(call.func) == 'kunlun_ops.moe_sigmoid_group_topk_norm':
+            for kw in call.keywords:
+                if kw.arg == 'block_static':
+                    idx = kw.lineno - 1
+                    assert 'block_static=block_static' in lines[idx]
+                    lines[idx] = lines[idx].replace('block_static=block_static', 'block_statistic=block_static')
+after_wrapper = ''.join(lines)
+for fn in [n for n in ast.parse(after_wrapper).body if isinstance(n, ast.FunctionDef) and n.name in names]:
+    node = copy.deepcopy(fn)
+    node.decorator_list = []; node.returns = None
+    for a in node.args.args:
+        a.annotation = None
+    calls = []
+    def spy(*a, **kw):
+        calls.append(signature.bind(*a, **kw).arguments)
+    scope = {'kunlun_ops': SimpleNamespace(moe_sigmoid_group_topk_norm=spy)}
+    exec(compile(ast.fix_missing_locations(ast.Module(body=[node], type_ignores=[])), '<wrapper-check>', 'exec'), scope)
+    values = {k: object() for k in ('x', 'topk_index', 'norm_score', 'block_static', 'bias')}
+    scope[fn.name](**values, scale=1.0, n_group=8, topk_group=4)
+    assert len(calls) == 1 and calls[0]['block_statistic'] is values['block_static']
+    assert calls[0]['bias'] is values['bias']
+    print('WRAPPER_PASS', fn.name)
+
+for path, before, after in [(route, before_route, after_route), (wrapper, before_wrapper, after_wrapper)]:
+    compile(after, str(path), 'exec')
+    if args.check:
+        if before != after:
+            raise RuntimeError(f'Missing patch: {path}')
+    elif before != after:
+        backup = path.with_name(path.name + '.bak-image-build')
+        if backup.exists():
+            raise RuntimeError(f'Backup already exists: {backup}')
+        backup.write_text(before)
+        temp = path.with_name(path.name + '.image-build.tmp')
+        temp.write_text(after); temp.chmod(path.stat().st_mode & 0o777); temp.replace(path)
+    print('SHA256', hashlib.sha256(after.encode()).hexdigest(), path)
+print('VERSIONS', json.dumps(versions))
+print('ALL_PATCH_CHECKS_PASS')

image_build/routing_compat.py

@@ -0,0 +1,141 @@
+#!/usr/bin/env python3
+"""Patch the paused/running container's installed Kunlun INT8 MoE, with backup.
+
+Run on the Docker host. Tests exercise the actual method through routing dispatch
+with tensor allocation/kernel calls mocked; they do not execute XPU kernels.
+"""
+import ast
+import copy
+import datetime
+import hashlib
+import io
+import subprocess
+import tarfile
+from types import SimpleNamespace as NS
+
+CONTAINER = 'srx_test'
+DIRECTORY = '/opt/vllm_kunlun/lib/python3.10/site-packages/vllm_kunlun/quantization/compressed_tensors'
+NAME = 'compressed_tensors_moe.py'
+CLASS = 'KunlunCompressedTensorsW8A8Int8MoEMethod'
+MARKER = '# Kunlun vLLM 0.25 routing compatibility fix'
+INSERT = '''        # Kunlun vLLM 0.25 routing compatibility fix
+        # RoutedExperts.forward_monolithic passes only layer/x/logits/input_ids.
+        # Read routing from the layer, not legacy method defaults. In particular,
+        # routed_scaling_factor may be 1 because the runner applies it externally.
+        scoring_func = layer.scoring_func
+        num_expert_group = layer.num_expert_group
+        topk_group = layer.topk_group
+        e_score_correction_bias = layer.e_score_correction_bias
+        routed_scaling_factor = layer.routed_scaling_factor
+        if scoring_func not in ("softmax", "sigmoid"):
+            raise NotImplementedError(f"Unsupported Kunlun INT8 MoE routing: {scoring_func}")
+
+'''
+
+
+def method_node(source):
+    module = ast.parse(source)
+    cls = next(n for n in module.body if isinstance(n, ast.ClassDef) and n.name == CLASS)
+    return next(n for n in cls.body if isinstance(n, ast.FunctionDef) and n.name == 'apply_monolithic')
+
+
+def patch_source(source):
+    if MARKER in source:
+        return source
+    node = method_node(source)
+    first = node.body[0]
+    if ast.unparse(first) != 'hidden_states = x':
+        raise RuntimeError('Unexpected installed method; refusing to patch')
+    lines = source.splitlines(keepends=True)
+    lines.insert(first.lineno - 1, INSERT)
+    patched = ''.join(lines)
+    compile(patched, NAME, 'exec')
+    return patched
+
+
+class Captured(Exception):
+    pass
+
+
+def dispatch(source, scoring, groups, top_groups, scale, bias):
+    node = copy.deepcopy(method_node(source))
+    for arg in node.args.posonlyargs + node.args.args + node.args.kwonlyargs:
+        arg.annotation = None
+    node.returns = None
+    calls = []
+
+    def capture(kind):
+        def callback(**kwargs):
+            calls.append((kind, kwargs))
+            raise Captured()
+        return callback
+
+    ops = NS(moe_softmax_topk_norm=capture('softmax'),
+             moe_sigmoid_group_topk_norm=capture('sigmoid'))
+    torch = NS(float32='float32', int32='int32',
+               empty=lambda *a, **k: object(), zeros=lambda *a, **k: object(), ops=NS(_C=ops))
+    scope = {'torch': torch}
+    exec(compile(ast.fix_missing_locations(ast.Module(body=[node], type_ignores=[])), '<actual-method>', 'exec'), scope)
+    layer = NS(scoring_func=scoring, num_expert_group=groups, topk_group=top_groups,
+               routed_scaling_factor=scale, e_score_correction_bias=bias,
+               w13_weight=NS(shape=(256, 512, 128)), w2_weight=NS(shape=(256, 128, 256)))
+    try:
+        scope['apply_monolithic'](NS(moe=NS(experts_per_token=8)), layer,
+                                  NS(shape=(1, 128), device='cpu'), NS(float=lambda: object()),
+                                  input_ids=None)
+    except Captured:
+        pass
+    assert len(calls) == 1
+    return calls[0]
+
+
+def regression_tests(original, patched):
+    bias = object()
+    if MARKER not in original:
+        kind, _ = dispatch(original, 'sigmoid', 8, 4, 1.0, bias)
+        assert kind == 'softmax', 'Expected to reproduce the old routing bug'
+        print('REPRODUCED: old method used softmax for DeepSeek sigmoid routing')
+    for scale in (1.0, 2.5):
+        kind, kwargs = dispatch(patched, 'sigmoid', 8, 4, scale, bias)
+        assert kind == 'sigmoid'
+        assert kwargs['n_group'] == 8 and kwargs['topk_group'] == 4
+        assert kwargs['bias'] is bias and kwargs['scale'] == scale
+    assert dispatch(patched, 'softmax', None, None, 1.0, None)[0] == 'softmax'
+    kind, kwargs = dispatch(patched, 'sigmoid', None, None, 1.0, bias)
+    assert kind == 'sigmoid' and kwargs['bias'] is bias
+    print('PASS: sigmoid groups/bias, external/internal scaling, softmax and ungrouped sigmoid')
+
+
+def read_installed():
+    data = subprocess.check_output(['docker', 'cp', f'{CONTAINER}:{DIRECTORY}/{NAME}', '-'])
+    with tarfile.open(fileobj=io.BytesIO(data)) as archive:
+        entry = next(m for m in archive.getmembers() if m.isfile())
+        return entry, archive.extractfile(entry).read()
+
+
+def main():
+    entry, original_bytes = read_installed()
+    original = original_bytes.decode()
+    patched = patch_source(original)
+    regression_tests(original, patched)
+    if patched == original:
+        print('ALREADY_PATCHED')
+        return
+    backup = NAME + '.bak-routing-' + datetime.datetime.now().strftime('%Y%m%dT%H%M%S%f')
+    output = io.BytesIO()
+    with tarfile.open(fileobj=output, mode='w') as archive:
+        for name, content in [(backup, original_bytes), (NAME, patched.encode())]:
+            info = copy.copy(entry)
+            info.name, info.size = name, len(content)
+            archive.addfile(info, io.BytesIO(content))
+    subprocess.run(['docker', 'cp', '-', f'{CONTAINER}:{DIRECTORY}'], input=output.getvalue(), check=True)
+    _, installed = read_installed()
+    assert installed == patched.encode(), 'Installed content differs from patch'
+    print('INSTALLED', DIRECTORY + '/' + NAME)
+    print('BACKUP', DIRECTORY + '/' + backup)
+    print('SHA256', hashlib.sha256(installed).hexdigest())
+    print('Restart the vLLM service to load the fix; existing workers retain old code.')
+
+
+if __name__ == '__main__':
+    main()

image_build/run_container.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+set -euo pipefail
+image=${1:-vllm-kunlun:deepseek-int8-fixed-20261029}
+name=${2:-deepseek_int8_fixed}
+devices=(--device=/dev/xpuctrl:/dev/xpuctrl)
+for i in {0..7}; do devices+=(--device="/dev/xpu${i}:/dev/xpu${i}"); done
+exec docker run -dit --name "$name" --network host --shm-size 32g \
+  "${devices[@]}" \
+  -v /mnt/disk0:/mnt/data0 \
+  -v /mnt/disk1:/mnt/data1 \
+  "$image" /bin/bash

image_build/serve_deepseek.sh

@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+# Run inside srx_test, after conversion has completed.
+set -euo pipefail
+export XPU_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
+export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
+# Avoid the incomplete MLA interface in the installed Kunlun plugin.
+export VLLM_MLA_DISABLE=1
+# kv_b_proj remains BF16; keep INT8 BMM disabled.
+export VLLM_KUNLUN_ENABLE_INT8_BMM=0
+exec /opt/vllm_kunlun/bin/vllm serve /mnt/data1/deepseek-r1-INT8 \
+  --trust-remote-code \
+  --dtype bfloat16 \
+  --quantization compressed-tensors \
+  --tensor-parallel-size 8 \
+  --distributed-executor-backend mp \
+  --max-model-len 4096 \
+  --max-num-seqs 1 \
+  --max-num-batched-tokens 4096 \
+  --kv-cache-memory-bytes 4294967296 \
+  --enforce-eager \
+  --no-enable-prefix-caching \
+  --no-enable-chunked-prefill \
+  --enable-auto-tool-choice \
+  --tool-call-parser deepseek_v3 \
+  --reasoning-parser deepseek_r1 \
+  "$@"

image_build/test_route_p800.py

@@ -0,0 +1,28 @@
+"""Tiny real P800 routing test, run after docker build with an exposed XPU."""
+import torch
+from vllm.platforms import current_platform
+current_platform.pre_register_and_update()
+for count in (2, 128):
+    for scale in (1.0, 2.5):
+        torch.manual_seed(17)
+        logits = torch.randn(count, 256, dtype=torch.float32)
+        bias = torch.randn(256, dtype=torch.float32) * 0.1
+        scores = logits.sigmoid()
+        adjusted = scores + bias
+        groups = adjusted.view(count, 8, 32).topk(2, dim=-1).values.sum(-1).topk(4, dim=-1).indices
+        mask = torch.zeros(count, 8, dtype=torch.bool).scatter_(1, groups, True).repeat_interleave(32, dim=1)
+        reference_ids = adjusted.masked_fill(~mask, float('-inf')).topk(8, dim=-1).indices
+        ids = torch.empty(count, 8, dtype=torch.int32, device='cuda')
+        weights = torch.empty(count, 8, dtype=torch.float32, device='cuda')
+        stats = torch.zeros(12, 256, dtype=torch.int32, device='cuda')
+        torch.ops._C.moe_sigmoid_group_topk_norm(
+            x=logits.cuda(), topk_index=ids, norm_score=weights, block_static=stats,
+            bias=bias.cuda(), scale=scale, n_group=8, topk_group=4)
+        torch.cuda.synchronize()
+        actual_ids = ids.cpu().long()
+        assert torch.equal(actual_ids.sort(-1).values, reference_ids.sort(-1).values)
+        expected = scores.gather(1, actual_ids)
+        expected = expected / expected.sum(-1, keepdim=True) * scale
+        error = (weights.cpu() - expected).abs().max().item()
+        assert error < 1e-5, error
+        print('P800_ROUTE_PASS', count, scale, error, flush=True)