79803932e2
### What this PR does / why we need it?
As #2947 describe, we need to transpose kv cache layout after GQA kv
transfer when prefill and decode tensor parallel size are heterogeneous,
in the previous implementation, we use `npu_paged_cache_load ` +
`tranpose` + `_npu_reshape_and_cache` to do this work.
But obviously, it is not an efficient plan, the ops above need to be
called for each layer, which introduces 3 * layer_num kernel launch, and
6 * layer_num data movement between L1 Cache and HBM for one request on
decode node. Usually, decode node uses graph mode, so these op kernels
will be called between decode forward launched by an async thread in
mooncacke connector, this kernels maybe last for several decode forward
and TTFT will increase by 3~4 decode forward time.
In this PR, we implement an AscendC fused op
`transpose_kv_cache_by_block` to do this with only once kernel launch
and move data between L1 Cache and HBM only once.
After using this fused op, the time cost in transpose kv cacke layout
can be decreased to 0.24ms from 7ms in UT on 910C, and in PD
disaggregation scenario, TTFT can decrease about 90 ~ 110 ms in
qwen3-235B.
| request_num | original | fused_op|
|:----------------------:|:---------------:|:-------------------:|
| 1 | 643 ms | 578 ms |
| 128 | 1480 ms | 1368 ms |
### Does this PR introduce _any_ user-facing change?
Use fused op by default, incase the op has bug in any scenario, provide
fallback choice using env to disable it.
**DISABLE fused op by add following env**
`export VLLM_ASCEND_FUSION_OP_TRANSPOSE_KV_CACHE_BY_BLOCK=0`
### How was this patch tested?
- vLLM version: v0.14.1
- vLLM main:
dc917cceb8
---------
Signed-off-by: lidenghui <lidenghui1110@gmail.com>
103 lines
3.9 KiB
Bash
103 lines
3.9 KiB
Bash
#!/bin/bash
|
|
|
|
ROOT_DIR=$1
|
|
SOC_VERSION=$2
|
|
|
|
if [[ "$SOC_VERSION" =~ ^ascend310 ]]; then
|
|
# ASCEND310P series
|
|
# currently, no custom aclnn ops for ASCEND310 series
|
|
# CUSTOM_OPS=""
|
|
# SOC_ARG="ascend310p"
|
|
exit 0
|
|
elif [[ "$SOC_VERSION" =~ ^ascend910b ]]; then
|
|
# ASCEND910B (A2) series
|
|
# dependency: catlass
|
|
git config --global --add safe.directory "$ROOT_DIR"
|
|
CATLASS_PATH=${ROOT_DIR}/csrc/third_party/catlass/include
|
|
if [[ ! -d "${CATLASS_PATH}" ]]; then
|
|
echo "dependency catlass is missing, try to fetch it..."
|
|
if ! git submodule update --init --recursive; then
|
|
echo "fetch failed"
|
|
exit 1
|
|
fi
|
|
fi
|
|
ABSOLUTE_CATLASS_PATH=$(cd "${CATLASS_PATH}" && pwd)
|
|
export CPATH=${ABSOLUTE_CATLASS_PATH}:${CPATH}
|
|
|
|
CUSTOM_OPS="grouped_matmul_swiglu_quant_weight_nz_tensor_list;lightning_indexer;sparse_flash_attention;matmul_allreduce_add_rmsnorm;moe_init_routing_custom;moe_gating_top_k;add_rms_norm_bias;apply_top_k_top_p_custom;transpose_kv_cache_by_block;"
|
|
SOC_ARG="ascend910b"
|
|
elif [[ "$SOC_VERSION" =~ ^ascend910_93 ]]; then
|
|
# ASCEND910C (A3) series
|
|
# dependency: catlass
|
|
git config --global --add safe.directory "$ROOT_DIR"
|
|
CATLASS_PATH=${ROOT_DIR}/csrc/third_party/catlass/include
|
|
if [[ ! -d "${CATLASS_PATH}" ]]; then
|
|
echo "dependency catlass is missing, try to fetch it..."
|
|
if ! git submodule update --init --recursive; then
|
|
echo "fetch failed"
|
|
exit 1
|
|
fi
|
|
fi
|
|
# dependency: cann-toolkit file moe_distribute_base.h
|
|
HCCL_STRUCT_FILE_PATH=$(find -L "${ASCEND_TOOLKIT_HOME}" -name "moe_distribute_base.h" 2>/dev/null | head -n1)
|
|
if [ -z "$HCCL_STRUCT_FILE_PATH" ]; then
|
|
echo "cannot find moe_distribute_base.h file in CANN env"
|
|
exit 1
|
|
fi
|
|
# for dispatch_gmm_combine_decode
|
|
yes | cp "${HCCL_STRUCT_FILE_PATH}" "${ROOT_DIR}/csrc/utils/inc/kernel"
|
|
# for dispatch_ffn_combine
|
|
SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd)
|
|
TARGET_DIR="$SCRIPT_DIR/dispatch_ffn_combine/op_kernel/utils/"
|
|
TARGET_FILE="$TARGET_DIR/$(basename "$HCCL_STRUCT_FILE_PATH")"
|
|
# for dispatch_ffn_combine_bf16
|
|
SCRIPT_DIR_BF16=$(cd "$(dirname "$0")" && pwd)
|
|
TARGET_DIR_BF16="$SCRIPT_DIR_BF16/dispatch_ffn_combine_bf16/op_kernel/utils/"
|
|
TARGET_FILE_BF16="$TARGET_DIR_BF16/$(basename "$HCCL_STRUCT_FILE_PATH")"
|
|
|
|
echo "*************************************"
|
|
echo $HCCL_STRUCT_FILE_PATH
|
|
echo "$TARGET_DIR"
|
|
cp "$HCCL_STRUCT_FILE_PATH" "$TARGET_DIR"
|
|
cp "$HCCL_STRUCT_FILE_PATH" "$TARGET_DIR_BF16"
|
|
|
|
sed -i 's/struct HcclOpResParam {/struct HcclOpResParamCustom {/g' "$TARGET_FILE"
|
|
sed -i 's/struct HcclRankRelationResV2 {/struct HcclRankRelationResV2Custom {/g' "$TARGET_FILE"
|
|
sed -i 's/struct HcclOpResParam {/struct HcclOpResParamCustom {/g' "$TARGET_FILE_BF16"
|
|
sed -i 's/struct HcclRankRelationResV2 {/struct HcclRankRelationResV2Custom {/g' "$TARGET_FILE_BF16"
|
|
|
|
CUSTOM_OPS_ARRAY=(
|
|
"grouped_matmul_swiglu_quant_weight_nz_tensor_list"
|
|
"lightning_indexer"
|
|
"sparse_flash_attention"
|
|
"dispatch_ffn_combine"
|
|
"dispatch_ffn_combine_bf16"
|
|
"dispatch_gmm_combine_decode"
|
|
"moe_combine_normal"
|
|
"moe_dispatch_normal"
|
|
"dispatch_layout"
|
|
"notify_dispatch"
|
|
"moe_init_routing_custom"
|
|
"moe_gating_top_k"
|
|
"add_rms_norm_bias"
|
|
"apply_top_k_top_p_custom"
|
|
"transpose_kv_cache_by_block"
|
|
)
|
|
CUSTOM_OPS=$(IFS=';'; echo "${CUSTOM_OPS_ARRAY[*]}")
|
|
SOC_ARG="ascend910_93"
|
|
else
|
|
# others
|
|
# currently, no custom aclnn ops for other series
|
|
exit 0
|
|
fi
|
|
|
|
|
|
# build custom ops
|
|
cd csrc
|
|
rm -rf build output
|
|
echo "building custom ops $CUSTOM_OPS for $SOC_VERSION"
|
|
bash build.sh -n "$CUSTOM_OPS" -c "$SOC_ARG"
|
|
|
|
# install custom ops to vllm_ascend/_cann_ops_custom
|
|
./output/CANN-custom_ops*.run --install-path=$ROOT_DIR/vllm_ascend/_cann_ops_custom
|