fix: SHA256SUMS paths, add ixformer inference bridge, update Dockerfile for full vllm overlay
This commit is contained in:
root
14
.dockerignore
Normal file
14
.dockerignore
Normal file
@@ -0,0 +1,14 @@
|
||||
**/__pycache__
|
||||
**/*.pyc
|
||||
**/.git
|
||||
cccl_upstream/
|
||||
upstream_ref/
|
||||
muh/
|
||||
ex_engine/fla_kernels/
|
||||
ex_engine/moe/
|
||||
ex_engine/xllm_layers/npu_torch/
|
||||
ex_engine/xllm_layers/mlu/
|
||||
ex_engine/xllm_models/
|
||||
*.zip
|
||||
dockerrizhi.txt
|
||||
subrizhi.txt
|
||||
17
Dockerfile
17
Dockerfile
@@ -1,10 +1,21 @@
|
||||
FROM git.modelhub.org.cn:9443/enginex-iluvatar/bi100-3.2.3-x86-ubuntu20.04-py3.10-poc-llm-infer:v1.2.3
|
||||
FROM harbor.4pd.io/modelhubxc/enginex-iluvatar/bi100-3.2.3-x86-ubuntu20.04-py3.10-poc-llm-infer:v1.2.3
|
||||
RUN mkdir -p /workspace
|
||||
WORKDIR /workspace/
|
||||
# Copy all our engine patches
|
||||
COPY ./qwen3_6_scripts /workspace/qwen3_6_scripts
|
||||
COPY ./computility-run.yaml /workspace/computility-run.yaml
|
||||
# Make patch script executable and run it
|
||||
COPY ./vllm /workspace/vllm
|
||||
COPY ./ixformer_sdk/inference/functions /workspace/ixformer_functions
|
||||
COPY ./ixformer_inference_functions_init.py /workspace/ixformer_inference_functions_init.py
|
||||
RUN chmod +x /workspace/qwen3_6_scripts/patch_ops.sh && \
|
||||
bash /workspace/qwen3_6_scripts/patch_ops.sh 2>&1 | tee /workspace/patch_ops.log ; \
|
||||
echo "[Dockerfile] patch_ops exit code: $?"
|
||||
RUN VLLM_ROOT=/usr/local/corex/lib64/python3/dist-packages/vllm && \
|
||||
IXFORMER_ROOT=/usr/local/corex/lib64/python3/dist-packages/ixformer && \
|
||||
cp -rf /workspace/vllm/* "${VLLM_ROOT}/" && \
|
||||
find "${VLLM_ROOT}" -type d -name __pycache__ -exec rm -rf {} + 2>/dev/null || true && \
|
||||
cp -r /workspace/ixformer_functions "${IXFORMER_ROOT}/inference/functions" && \
|
||||
cp /workspace/ixformer_inference_functions_init.py "${IXFORMER_ROOT}/inference/functions/__init__.py" && \
|
||||
mkdir -p "${IXFORMER_ROOT}/contrib/vllm_flash_attn" && \
|
||||
echo 'from ixformer import flash_attn_varlen_func, flash_attn_func, flash_attn_padded_func' > "${IXFORMER_ROOT}/contrib/vllm_flash_attn/__init__.py" && \
|
||||
pip3 install blake3 --break-system-packages 2>/dev/null && \
|
||||
echo "[overlay] done"
|
||||
|
||||
176
ixformer_inference_functions_init.py
Normal file
176
ixformer_inference_functions_init.py
Normal file
@@ -0,0 +1,176 @@
|
||||
try:
|
||||
from .act_and_mul import *
|
||||
except (ImportError, ModuleNotFoundError):
|
||||
pass
|
||||
try:
|
||||
from .act_bias_mm import *
|
||||
except (ImportError, ModuleNotFoundError):
|
||||
pass
|
||||
try:
|
||||
from .add import *
|
||||
except (ImportError, ModuleNotFoundError):
|
||||
pass
|
||||
try:
|
||||
from .bert import *
|
||||
except (ImportError, ModuleNotFoundError):
|
||||
pass
|
||||
try:
|
||||
from .bnb_dequant import *
|
||||
except (ImportError, ModuleNotFoundError):
|
||||
pass
|
||||
try:
|
||||
from .bnb_double_quant import *
|
||||
except (ImportError, ModuleNotFoundError):
|
||||
pass
|
||||
try:
|
||||
from .bnb_mm_dequant import *
|
||||
except (ImportError, ModuleNotFoundError):
|
||||
pass
|
||||
try:
|
||||
from .bnb_qgemm import *
|
||||
except (ImportError, ModuleNotFoundError):
|
||||
pass
|
||||
try:
|
||||
from .bnb_quant import *
|
||||
except (ImportError, ModuleNotFoundError):
|
||||
pass
|
||||
try:
|
||||
from .bnb_rowcol_absmax import *
|
||||
except (ImportError, ModuleNotFoundError):
|
||||
pass
|
||||
try:
|
||||
from .conv2d import *
|
||||
except (ImportError, ModuleNotFoundError):
|
||||
pass
|
||||
try:
|
||||
from .cross_entropy_loss import *
|
||||
except (ImportError, ModuleNotFoundError):
|
||||
pass
|
||||
try:
|
||||
from .flash_attn import *
|
||||
except (ImportError, ModuleNotFoundError):
|
||||
pass
|
||||
try:
|
||||
from .flash_attn_lib import *
|
||||
except (ImportError, ModuleNotFoundError):
|
||||
pass
|
||||
try:
|
||||
from .fused_rope import *
|
||||
except (ImportError, ModuleNotFoundError):
|
||||
pass
|
||||
try:
|
||||
from .gemv import *
|
||||
except (ImportError, ModuleNotFoundError):
|
||||
pass
|
||||
try:
|
||||
from .groupnorm import *
|
||||
except (ImportError, ModuleNotFoundError):
|
||||
pass
|
||||
try:
|
||||
from .i8w8o32 import *
|
||||
except (ImportError, ModuleNotFoundError):
|
||||
pass
|
||||
try:
|
||||
from .layernorm import *
|
||||
except (ImportError, ModuleNotFoundError):
|
||||
pass
|
||||
try:
|
||||
from .lightllm import *
|
||||
except (ImportError, ModuleNotFoundError):
|
||||
pass
|
||||
try:
|
||||
from .linalg import *
|
||||
except (ImportError, ModuleNotFoundError):
|
||||
pass
|
||||
try:
|
||||
from .linear import *
|
||||
except (ImportError, ModuleNotFoundError):
|
||||
pass
|
||||
try:
|
||||
from .lmdeploy import *
|
||||
except (ImportError, ModuleNotFoundError):
|
||||
pass
|
||||
try:
|
||||
from .marlin import *
|
||||
except (ImportError, ModuleNotFoundError):
|
||||
pass
|
||||
try:
|
||||
from .matmul import *
|
||||
except (ImportError, ModuleNotFoundError):
|
||||
pass
|
||||
try:
|
||||
from .mla_fused import *
|
||||
except (ImportError, ModuleNotFoundError):
|
||||
pass
|
||||
try:
|
||||
from .mm import *
|
||||
except (ImportError, ModuleNotFoundError):
|
||||
pass
|
||||
try:
|
||||
from .moe import *
|
||||
except (ImportError, ModuleNotFoundError):
|
||||
pass
|
||||
try:
|
||||
from .overlap_comm import *
|
||||
except (ImportError, ModuleNotFoundError):
|
||||
pass
|
||||
try:
|
||||
from .paged_attention import *
|
||||
except (ImportError, ModuleNotFoundError):
|
||||
pass
|
||||
try:
|
||||
from .quantized_linear import *
|
||||
except (ImportError, ModuleNotFoundError):
|
||||
pass
|
||||
try:
|
||||
from .residual_bias import *
|
||||
except (ImportError, ModuleNotFoundError):
|
||||
pass
|
||||
try:
|
||||
from .rms_norm import *
|
||||
except (ImportError, ModuleNotFoundError):
|
||||
pass
|
||||
try:
|
||||
from .scaled_dot_product_attention import *
|
||||
except (ImportError, ModuleNotFoundError):
|
||||
pass
|
||||
try:
|
||||
from .smoothquant import *
|
||||
except (ImportError, ModuleNotFoundError):
|
||||
pass
|
||||
try:
|
||||
from .softmax import *
|
||||
except (ImportError, ModuleNotFoundError):
|
||||
pass
|
||||
try:
|
||||
from .store_kv_cache import *
|
||||
except (ImportError, ModuleNotFoundError):
|
||||
pass
|
||||
try:
|
||||
from .t5 import *
|
||||
except (ImportError, ModuleNotFoundError):
|
||||
pass
|
||||
try:
|
||||
from .tgi import *
|
||||
except (ImportError, ModuleNotFoundError):
|
||||
pass
|
||||
try:
|
||||
from .vllm import *
|
||||
except (ImportError, ModuleNotFoundError):
|
||||
pass
|
||||
try:
|
||||
from .w8a8 import *
|
||||
except (ImportError, ModuleNotFoundError):
|
||||
pass
|
||||
try:
|
||||
from .w8a16 import *
|
||||
except (ImportError, ModuleNotFoundError):
|
||||
pass
|
||||
try:
|
||||
from .wi4a16 import *
|
||||
except (ImportError, ModuleNotFoundError):
|
||||
pass
|
||||
try:
|
||||
from .wui4a16 import *
|
||||
except (ImportError, ModuleNotFoundError):
|
||||
pass
|
||||
@@ -1,26 +1,26 @@
|
||||
534019b3c2ad2d2c65492b01a975874ee440026eda2e8666bc3c1dc8a0a0a6f6 prebuilt/corex-3.2.3-ivcore10/corex_attn_head_rms_norm.so
|
||||
4bf4b37252c3d9178b699625c5f20ef1f58fbaa37e8f62cb34d0d83ec004f3d6 prebuilt/corex-3.2.3-ivcore10/corex_batched_gemm.so
|
||||
7e2aafd8dc755b0ee16c3b9bb812b95548fc042bbaa840dd9db7d2c51a10474c prebuilt/corex-3.2.3-ivcore10/corex_block_major_kv_transfer.so
|
||||
ad4ea7707bb2f2bfe04e07a7ad5fd58a647232be70a3056937a0d738c8254bff prebuilt/corex-3.2.3-ivcore10/corex_fused_paged_prefill.so
|
||||
1856c86e3100415061aa698a48bdeff3fe785994b45b4e72a42cd9158552a7d8 prebuilt/corex-3.2.3-ivcore10/corex_gdn_beta_decay.so
|
||||
957c7518f5831299fc73f19a4ca2aa3c8231afe9ea7c979127b4f426cd9d6906 prebuilt/corex-3.2.3-ivcore10/corex_gdn_causal_conv.so
|
||||
ff1c1c67ec252ed993bf1840a33ef5c4b386b730ba6dc5bf995991013c226222 prebuilt/corex-3.2.3-ivcore10/corex_gdn_chunk_recurrent.so
|
||||
ec2d11fa82d9d0816a6da53e62605e962786fa20ecd5f62e50f9d43087fc4d67 prebuilt/corex-3.2.3-ivcore10/corex_gdn_gated_norm.so
|
||||
27b7ae2ce4fe173336355d72a2678d043df4bd1ed85e9231a99bfb81885a6ce3 prebuilt/corex-3.2.3-ivcore10/corex_gdn_packed_decode.so
|
||||
015b61046ad73d8f12d754f7a87d4f6cba33070af1c079879e15b71a94571670 prebuilt/corex-3.2.3-ivcore10/corex_gdn_qk_map.so
|
||||
0eb120e89608bb5b64ca4356a5d3d362121806d081ccc1ccf346dac472a819ec prebuilt/corex-3.2.3-ivcore10/corex_moe_direct_routed.so
|
||||
d26f2fa39c3921a95793786601e90cf6ebadd06f1d752af541bf82c21acbc1c9 prebuilt/corex-3.2.3-ivcore10/corex_moe_exact_reduce.so
|
||||
0d5b04639a62fb67de63c413b596fc617a3784dbe72e04a7a3337a9e556f5b90 prebuilt/corex-3.2.3-ivcore10/corex_moe_index_combine.so
|
||||
c3208c8e0c13f54dbe22a9cfc88bdc6ab040e920d6cae4bc0ecf7087880795f3 prebuilt/corex-3.2.3-ivcore10/corex_moe_topk_softmax.so
|
||||
50b0b44c1da779bb2c03419ed549aee9bb922d1f9bab8b7f11a3d91cca0d21c3 prebuilt/corex-3.2.3-ivcore10/corex_moe_weight_gather.so
|
||||
e944ec0528ed9b6cb74518de3c57e3730543a7bdebc872f993bfdc8424f13e6b prebuilt/corex-3.2.3-ivcore10/corex_paged_kv_gather.so
|
||||
238ea5109563468bec4b01f5cb8d0de920fb9a8e1b6cf46dd764ef4e453dc73f prebuilt/corex-3.2.3-ivcore10/gemm_grouped.so
|
||||
5116816ac5a7da62f2dd4bb7bba741d01358cd0ef9637f2600d00d25ba143500 prebuilt/corex-3.2.3-ivcore10/ix_full_bridge_fused_ar.so
|
||||
7de1d41919eee8b1caf4ef16b812f65d4b4a9421827e65a58e69aa79ce293ae5 prebuilt/corex-3.2.3-ivcore10/ix_full_bridge.so
|
||||
e79263efa0933861642197647b378dc228c0ae7610ff1d53e27f8c793f56b92d prebuilt/corex-3.2.3-ivcore10/ix_moe_bridge.so
|
||||
371b51e52d95826bea6dc38222968b83eb52d866b89eb75699a491d2cea8f2f9 prebuilt/corex-3.2.3-ivcore10/xllm_activation.so
|
||||
843c8151edc3a546f591b36561c8faa1d33fcb9a3d34c27e3e2c0b8e98563b9f prebuilt/corex-3.2.3-ivcore10/xllm_cache.so
|
||||
66b3cddcf0d2c4c6ee1c4221d1f2b553033d0f3c22fd9535f640aaac36ba56db prebuilt/corex-3.2.3-ivcore10/xllm_fused_qknorm_rope.so
|
||||
7c01aa23cacbee95ecb3a4449e543acbc0238ee3c9e3fecc23ef2263ca795f53 prebuilt/corex-3.2.3-ivcore10/xllm_moe.so
|
||||
a5af617aba0812e99454c0a83fe7f584c2ad151a627ada91fbe54729e2a8728d prebuilt/corex-3.2.3-ivcore10/xllm_norm.so
|
||||
b87f13cea5de45e4310c2f30c9e236421d72e1f1ef445dc085cbfbe8edb38b27 prebuilt/corex-3.2.3-ivcore10/xllm_rope.so
|
||||
534019b3c2ad2d2c65492b01a975874ee440026eda2e8666bc3c1dc8a0a0a6f6 corex_attn_head_rms_norm.so
|
||||
4bf4b37252c3d9178b699625c5f20ef1f58fbaa37e8f62cb34d0d83ec004f3d6 corex_batched_gemm.so
|
||||
7e2aafd8dc755b0ee16c3b9bb812b95548fc042bbaa840dd9db7d2c51a10474c corex_block_major_kv_transfer.so
|
||||
ad4ea7707bb2f2bfe04e07a7ad5fd58a647232be70a3056937a0d738c8254bff corex_fused_paged_prefill.so
|
||||
1856c86e3100415061aa698a48bdeff3fe785994b45b4e72a42cd9158552a7d8 corex_gdn_beta_decay.so
|
||||
957c7518f5831299fc73f19a4ca2aa3c8231afe9ea7c979127b4f426cd9d6906 corex_gdn_causal_conv.so
|
||||
ff1c1c67ec252ed993bf1840a33ef5c4b386b730ba6dc5bf995991013c226222 corex_gdn_chunk_recurrent.so
|
||||
ec2d11fa82d9d0816a6da53e62605e962786fa20ecd5f62e50f9d43087fc4d67 corex_gdn_gated_norm.so
|
||||
27b7ae2ce4fe173336355d72a2678d043df4bd1ed85e9231a99bfb81885a6ce3 corex_gdn_packed_decode.so
|
||||
015b61046ad73d8f12d754f7a87d4f6cba33070af1c079879e15b71a94571670 corex_gdn_qk_map.so
|
||||
0eb120e89608bb5b64ca4356a5d3d362121806d081ccc1ccf346dac472a819ec corex_moe_direct_routed.so
|
||||
d26f2fa39c3921a95793786601e90cf6ebadd06f1d752af541bf82c21acbc1c9 corex_moe_exact_reduce.so
|
||||
0d5b04639a62fb67de63c413b596fc617a3784dbe72e04a7a3337a9e556f5b90 corex_moe_index_combine.so
|
||||
c3208c8e0c13f54dbe22a9cfc88bdc6ab040e920d6cae4bc0ecf7087880795f3 corex_moe_topk_softmax.so
|
||||
50b0b44c1da779bb2c03419ed549aee9bb922d1f9bab8b7f11a3d91cca0d21c3 corex_moe_weight_gather.so
|
||||
e944ec0528ed9b6cb74518de3c57e3730543a7bdebc872f993bfdc8424f13e6b corex_paged_kv_gather.so
|
||||
238ea5109563468bec4b01f5cb8d0de920fb9a8e1b6cf46dd764ef4e453dc73f gemm_grouped.so
|
||||
7de1d41919eee8b1caf4ef16b812f65d4b4a9421827e65a58e69aa79ce293ae5 ix_full_bridge.so
|
||||
5116816ac5a7da62f2dd4bb7bba741d01358cd0ef9637f2600d00d25ba143500 ix_full_bridge_fused_ar.so
|
||||
e79263efa0933861642197647b378dc228c0ae7610ff1d53e27f8c793f56b92d ix_moe_bridge.so
|
||||
22ad79f6b3e9e1a4abbe9801167cbf1734b9b789deca0edd43965fd668aea1f3 xllm_activation.so
|
||||
0d85751056c4e157aec343ae6ca1cf4c5f41b6df15ff4b914947ba5e4f4fad3f xllm_cache.so
|
||||
3cdc37435a68aa2d9d1eec16082189245e5cfa1811be562292c9b7a154204ef3 xllm_fused_qknorm_rope.so
|
||||
5c38e15812342f94538d010a266368bc955345a2d65fc5f4629ef34d49549921 xllm_moe.so
|
||||
d3daf76f1248b2b37c659eddc54b863424cc933524281bd2b44145d09109018f xllm_norm.so
|
||||
ff2abebdd04ae10005aff89b5b1e18c0d9bda0f4a2fd38382d3d4b453ef49944 xllm_rope.so
|
||||
|
||||
Reference in New Issue
Block a user