diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..8ac0fa6 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,14 @@ +**/__pycache__ +**/*.pyc +**/.git +cccl_upstream/ +upstream_ref/ +muh/ +ex_engine/fla_kernels/ +ex_engine/moe/ +ex_engine/xllm_layers/npu_torch/ +ex_engine/xllm_layers/mlu/ +ex_engine/xllm_models/ +*.zip +dockerrizhi.txt +subrizhi.txt diff --git a/Dockerfile b/Dockerfile index faa0a98..efe2717 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,10 +1,21 @@ -FROM git.modelhub.org.cn:9443/enginex-iluvatar/bi100-3.2.3-x86-ubuntu20.04-py3.10-poc-llm-infer:v1.2.3 +FROM harbor.4pd.io/modelhubxc/enginex-iluvatar/bi100-3.2.3-x86-ubuntu20.04-py3.10-poc-llm-infer:v1.2.3 RUN mkdir -p /workspace WORKDIR /workspace/ -# Copy all our engine patches COPY ./qwen3_6_scripts /workspace/qwen3_6_scripts COPY ./computility-run.yaml /workspace/computility-run.yaml -# Make patch script executable and run it +COPY ./vllm /workspace/vllm +COPY ./ixformer_sdk/inference/functions /workspace/ixformer_functions +COPY ./ixformer_inference_functions_init.py /workspace/ixformer_inference_functions_init.py RUN chmod +x /workspace/qwen3_6_scripts/patch_ops.sh && \ bash /workspace/qwen3_6_scripts/patch_ops.sh 2>&1 | tee /workspace/patch_ops.log ; \ echo "[Dockerfile] patch_ops exit code: $?" +RUN VLLM_ROOT=/usr/local/corex/lib64/python3/dist-packages/vllm && \ + IXFORMER_ROOT=/usr/local/corex/lib64/python3/dist-packages/ixformer && \ + cp -rf /workspace/vllm/* "${VLLM_ROOT}/" && \ + find "${VLLM_ROOT}" -type d -name __pycache__ -exec rm -rf {} + 2>/dev/null || true && \ + cp -r /workspace/ixformer_functions "${IXFORMER_ROOT}/inference/functions" && \ + cp /workspace/ixformer_inference_functions_init.py "${IXFORMER_ROOT}/inference/functions/__init__.py" && \ + mkdir -p "${IXFORMER_ROOT}/contrib/vllm_flash_attn" && \ + echo 'from ixformer import flash_attn_varlen_func, flash_attn_func, flash_attn_padded_func' > "${IXFORMER_ROOT}/contrib/vllm_flash_attn/__init__.py" && \ + pip3 install blake3 --break-system-packages 2>/dev/null && \ + echo "[overlay] done" diff --git a/ixformer_inference_functions_init.py b/ixformer_inference_functions_init.py new file mode 100644 index 0000000..f01d114 --- /dev/null +++ b/ixformer_inference_functions_init.py @@ -0,0 +1,176 @@ +try: + from .act_and_mul import * +except (ImportError, ModuleNotFoundError): + pass +try: + from .act_bias_mm import * +except (ImportError, ModuleNotFoundError): + pass +try: + from .add import * +except (ImportError, ModuleNotFoundError): + pass +try: + from .bert import * +except (ImportError, ModuleNotFoundError): + pass +try: + from .bnb_dequant import * +except (ImportError, ModuleNotFoundError): + pass +try: + from .bnb_double_quant import * +except (ImportError, ModuleNotFoundError): + pass +try: + from .bnb_mm_dequant import * +except (ImportError, ModuleNotFoundError): + pass +try: + from .bnb_qgemm import * +except (ImportError, ModuleNotFoundError): + pass +try: + from .bnb_quant import * +except (ImportError, ModuleNotFoundError): + pass +try: + from .bnb_rowcol_absmax import * +except (ImportError, ModuleNotFoundError): + pass +try: + from .conv2d import * +except (ImportError, ModuleNotFoundError): + pass +try: + from .cross_entropy_loss import * +except (ImportError, ModuleNotFoundError): + pass +try: + from .flash_attn import * +except (ImportError, ModuleNotFoundError): + pass +try: + from .flash_attn_lib import * +except (ImportError, ModuleNotFoundError): + pass +try: + from .fused_rope import * +except (ImportError, ModuleNotFoundError): + pass +try: + from .gemv import * +except (ImportError, ModuleNotFoundError): + pass +try: + from .groupnorm import * +except (ImportError, ModuleNotFoundError): + pass +try: + from .i8w8o32 import * +except (ImportError, ModuleNotFoundError): + pass +try: + from .layernorm import * +except (ImportError, ModuleNotFoundError): + pass +try: + from .lightllm import * +except (ImportError, ModuleNotFoundError): + pass +try: + from .linalg import * +except (ImportError, ModuleNotFoundError): + pass +try: + from .linear import * +except (ImportError, ModuleNotFoundError): + pass +try: + from .lmdeploy import * +except (ImportError, ModuleNotFoundError): + pass +try: + from .marlin import * +except (ImportError, ModuleNotFoundError): + pass +try: + from .matmul import * +except (ImportError, ModuleNotFoundError): + pass +try: + from .mla_fused import * +except (ImportError, ModuleNotFoundError): + pass +try: + from .mm import * +except (ImportError, ModuleNotFoundError): + pass +try: + from .moe import * +except (ImportError, ModuleNotFoundError): + pass +try: + from .overlap_comm import * +except (ImportError, ModuleNotFoundError): + pass +try: + from .paged_attention import * +except (ImportError, ModuleNotFoundError): + pass +try: + from .quantized_linear import * +except (ImportError, ModuleNotFoundError): + pass +try: + from .residual_bias import * +except (ImportError, ModuleNotFoundError): + pass +try: + from .rms_norm import * +except (ImportError, ModuleNotFoundError): + pass +try: + from .scaled_dot_product_attention import * +except (ImportError, ModuleNotFoundError): + pass +try: + from .smoothquant import * +except (ImportError, ModuleNotFoundError): + pass +try: + from .softmax import * +except (ImportError, ModuleNotFoundError): + pass +try: + from .store_kv_cache import * +except (ImportError, ModuleNotFoundError): + pass +try: + from .t5 import * +except (ImportError, ModuleNotFoundError): + pass +try: + from .tgi import * +except (ImportError, ModuleNotFoundError): + pass +try: + from .vllm import * +except (ImportError, ModuleNotFoundError): + pass +try: + from .w8a8 import * +except (ImportError, ModuleNotFoundError): + pass +try: + from .w8a16 import * +except (ImportError, ModuleNotFoundError): + pass +try: + from .wi4a16 import * +except (ImportError, ModuleNotFoundError): + pass +try: + from .wui4a16 import * +except (ImportError, ModuleNotFoundError): + pass diff --git a/qwen3_6_scripts/prebuilt/corex-3.2.3-ivcore10/SHA256SUMS b/qwen3_6_scripts/prebuilt/corex-3.2.3-ivcore10/SHA256SUMS index ad51b42..472528d 100644 --- a/qwen3_6_scripts/prebuilt/corex-3.2.3-ivcore10/SHA256SUMS +++ b/qwen3_6_scripts/prebuilt/corex-3.2.3-ivcore10/SHA256SUMS @@ -1,26 +1,26 @@ -534019b3c2ad2d2c65492b01a975874ee440026eda2e8666bc3c1dc8a0a0a6f6 prebuilt/corex-3.2.3-ivcore10/corex_attn_head_rms_norm.so -4bf4b37252c3d9178b699625c5f20ef1f58fbaa37e8f62cb34d0d83ec004f3d6 prebuilt/corex-3.2.3-ivcore10/corex_batched_gemm.so -7e2aafd8dc755b0ee16c3b9bb812b95548fc042bbaa840dd9db7d2c51a10474c prebuilt/corex-3.2.3-ivcore10/corex_block_major_kv_transfer.so -ad4ea7707bb2f2bfe04e07a7ad5fd58a647232be70a3056937a0d738c8254bff prebuilt/corex-3.2.3-ivcore10/corex_fused_paged_prefill.so -1856c86e3100415061aa698a48bdeff3fe785994b45b4e72a42cd9158552a7d8 prebuilt/corex-3.2.3-ivcore10/corex_gdn_beta_decay.so -957c7518f5831299fc73f19a4ca2aa3c8231afe9ea7c979127b4f426cd9d6906 prebuilt/corex-3.2.3-ivcore10/corex_gdn_causal_conv.so -ff1c1c67ec252ed993bf1840a33ef5c4b386b730ba6dc5bf995991013c226222 prebuilt/corex-3.2.3-ivcore10/corex_gdn_chunk_recurrent.so -ec2d11fa82d9d0816a6da53e62605e962786fa20ecd5f62e50f9d43087fc4d67 prebuilt/corex-3.2.3-ivcore10/corex_gdn_gated_norm.so -27b7ae2ce4fe173336355d72a2678d043df4bd1ed85e9231a99bfb81885a6ce3 prebuilt/corex-3.2.3-ivcore10/corex_gdn_packed_decode.so -015b61046ad73d8f12d754f7a87d4f6cba33070af1c079879e15b71a94571670 prebuilt/corex-3.2.3-ivcore10/corex_gdn_qk_map.so -0eb120e89608bb5b64ca4356a5d3d362121806d081ccc1ccf346dac472a819ec prebuilt/corex-3.2.3-ivcore10/corex_moe_direct_routed.so -d26f2fa39c3921a95793786601e90cf6ebadd06f1d752af541bf82c21acbc1c9 prebuilt/corex-3.2.3-ivcore10/corex_moe_exact_reduce.so -0d5b04639a62fb67de63c413b596fc617a3784dbe72e04a7a3337a9e556f5b90 prebuilt/corex-3.2.3-ivcore10/corex_moe_index_combine.so -c3208c8e0c13f54dbe22a9cfc88bdc6ab040e920d6cae4bc0ecf7087880795f3 prebuilt/corex-3.2.3-ivcore10/corex_moe_topk_softmax.so -50b0b44c1da779bb2c03419ed549aee9bb922d1f9bab8b7f11a3d91cca0d21c3 prebuilt/corex-3.2.3-ivcore10/corex_moe_weight_gather.so -e944ec0528ed9b6cb74518de3c57e3730543a7bdebc872f993bfdc8424f13e6b prebuilt/corex-3.2.3-ivcore10/corex_paged_kv_gather.so -238ea5109563468bec4b01f5cb8d0de920fb9a8e1b6cf46dd764ef4e453dc73f prebuilt/corex-3.2.3-ivcore10/gemm_grouped.so -5116816ac5a7da62f2dd4bb7bba741d01358cd0ef9637f2600d00d25ba143500 prebuilt/corex-3.2.3-ivcore10/ix_full_bridge_fused_ar.so -7de1d41919eee8b1caf4ef16b812f65d4b4a9421827e65a58e69aa79ce293ae5 prebuilt/corex-3.2.3-ivcore10/ix_full_bridge.so -e79263efa0933861642197647b378dc228c0ae7610ff1d53e27f8c793f56b92d prebuilt/corex-3.2.3-ivcore10/ix_moe_bridge.so -371b51e52d95826bea6dc38222968b83eb52d866b89eb75699a491d2cea8f2f9 prebuilt/corex-3.2.3-ivcore10/xllm_activation.so -843c8151edc3a546f591b36561c8faa1d33fcb9a3d34c27e3e2c0b8e98563b9f prebuilt/corex-3.2.3-ivcore10/xllm_cache.so -66b3cddcf0d2c4c6ee1c4221d1f2b553033d0f3c22fd9535f640aaac36ba56db prebuilt/corex-3.2.3-ivcore10/xllm_fused_qknorm_rope.so -7c01aa23cacbee95ecb3a4449e543acbc0238ee3c9e3fecc23ef2263ca795f53 prebuilt/corex-3.2.3-ivcore10/xllm_moe.so -a5af617aba0812e99454c0a83fe7f584c2ad151a627ada91fbe54729e2a8728d prebuilt/corex-3.2.3-ivcore10/xllm_norm.so -b87f13cea5de45e4310c2f30c9e236421d72e1f1ef445dc085cbfbe8edb38b27 prebuilt/corex-3.2.3-ivcore10/xllm_rope.so +534019b3c2ad2d2c65492b01a975874ee440026eda2e8666bc3c1dc8a0a0a6f6 corex_attn_head_rms_norm.so +4bf4b37252c3d9178b699625c5f20ef1f58fbaa37e8f62cb34d0d83ec004f3d6 corex_batched_gemm.so +7e2aafd8dc755b0ee16c3b9bb812b95548fc042bbaa840dd9db7d2c51a10474c corex_block_major_kv_transfer.so +ad4ea7707bb2f2bfe04e07a7ad5fd58a647232be70a3056937a0d738c8254bff corex_fused_paged_prefill.so +1856c86e3100415061aa698a48bdeff3fe785994b45b4e72a42cd9158552a7d8 corex_gdn_beta_decay.so +957c7518f5831299fc73f19a4ca2aa3c8231afe9ea7c979127b4f426cd9d6906 corex_gdn_causal_conv.so +ff1c1c67ec252ed993bf1840a33ef5c4b386b730ba6dc5bf995991013c226222 corex_gdn_chunk_recurrent.so +ec2d11fa82d9d0816a6da53e62605e962786fa20ecd5f62e50f9d43087fc4d67 corex_gdn_gated_norm.so +27b7ae2ce4fe173336355d72a2678d043df4bd1ed85e9231a99bfb81885a6ce3 corex_gdn_packed_decode.so +015b61046ad73d8f12d754f7a87d4f6cba33070af1c079879e15b71a94571670 corex_gdn_qk_map.so +0eb120e89608bb5b64ca4356a5d3d362121806d081ccc1ccf346dac472a819ec corex_moe_direct_routed.so +d26f2fa39c3921a95793786601e90cf6ebadd06f1d752af541bf82c21acbc1c9 corex_moe_exact_reduce.so +0d5b04639a62fb67de63c413b596fc617a3784dbe72e04a7a3337a9e556f5b90 corex_moe_index_combine.so +c3208c8e0c13f54dbe22a9cfc88bdc6ab040e920d6cae4bc0ecf7087880795f3 corex_moe_topk_softmax.so +50b0b44c1da779bb2c03419ed549aee9bb922d1f9bab8b7f11a3d91cca0d21c3 corex_moe_weight_gather.so +e944ec0528ed9b6cb74518de3c57e3730543a7bdebc872f993bfdc8424f13e6b corex_paged_kv_gather.so +238ea5109563468bec4b01f5cb8d0de920fb9a8e1b6cf46dd764ef4e453dc73f gemm_grouped.so +7de1d41919eee8b1caf4ef16b812f65d4b4a9421827e65a58e69aa79ce293ae5 ix_full_bridge.so +5116816ac5a7da62f2dd4bb7bba741d01358cd0ef9637f2600d00d25ba143500 ix_full_bridge_fused_ar.so +e79263efa0933861642197647b378dc228c0ae7610ff1d53e27f8c793f56b92d ix_moe_bridge.so +22ad79f6b3e9e1a4abbe9801167cbf1734b9b789deca0edd43965fd668aea1f3 xllm_activation.so +0d85751056c4e157aec343ae6ca1cf4c5f41b6df15ff4b914947ba5e4f4fad3f xllm_cache.so +3cdc37435a68aa2d9d1eec16082189245e5cfa1811be562292c9b7a154204ef3 xllm_fused_qknorm_rope.so +5c38e15812342f94538d010a266368bc955345a2d65fc5f4629ef34d49549921 xllm_moe.so +d3daf76f1248b2b37c659eddc54b863424cc933524281bd2b44145d09109018f xllm_norm.so +ff2abebdd04ae10005aff89b5b1e18c0d9bda0f4a2fd38382d3d4b453ef49944 xllm_rope.so