52 lines
1.6 KiB
Bash
52 lines
1.6 KiB
Bash
|
#!/usr/bin/env bash
|
|||
|
|
#
|
|||
|
|
# Llama 4 Scout 容器入口：
|
|||
|
|
# 1. 幂等应用 vLLM CoreX 的 Llama4 chunked-attention 补丁
|
|||
|
|
# 2. 检查模型目录
|
|||
|
|
# 3. exec vLLM（信号可直接传给 vLLM）
|
|||
|
|
#
|
|||
|
|
set -euo pipefail
|
|||
|
|
|
|||
|
|
python3 /opt/llama4/apply_patch.py
|
|||
|
|
|
|||
|
|
MODEL_PATH="${MODEL_PATH:-/models/llama4-scout}"
|
|||
|
|
SERVED_NAME="${SERVED_NAME:-llama4-scout}"
|
|||
|
|
HOST="${HOST:-0.0.0.0}"
|
|||
|
|
PORT="${PORT:-8091}"
|
|||
|
|
TP="${TP:-8}"
|
|||
|
|
MAX_MODEL_LEN="${MAX_MODEL_LEN:-4096}"
|
|||
|
|
MAX_NUM_SEQS="${MAX_NUM_SEQS:-1}"
|
|||
|
|
GPU_MEM_UTIL="${GPU_MEM_UTIL:-0.90}"
|
|||
|
|
EXTRA_ARGS="${EXTRA_ARGS:-}"
|
|||
|
|
|
|||
|
|
if [[ ! -f "${MODEL_PATH}/config.json" ]]; then
|
|||
|
|
echo "ERROR: 模型目录不完整：${MODEL_PATH}/config.json 不存在" >&2
|
|||
|
|
echo "请确认已挂载模型，例如：" >&2
|
|||
|
|
echo " -v /mnt/disk0/models/LLM-Research/Llama-4-Scout-17B-16E-Instruct:/models/llama4-scout:ro" >&2
|
|||
|
|
exit 1
|
|||
|
|
fi
|
|||
|
|
|
|||
|
|
echo "============================================================"
|
|||
|
|
echo "Llama 4 Scout"
|
|||
|
|
echo " model : ${MODEL_PATH}"
|
|||
|
|
echo " served name: ${SERVED_NAME}"
|
|||
|
|
echo " listen : ${HOST}:${PORT}"
|
|||
|
|
echo " TP : ${TP}"
|
|||
|
|
echo " max len : ${MAX_MODEL_LEN}"
|
|||
|
|
echo " max seqs : ${MAX_NUM_SEQS}"
|
|||
|
|
echo " gpu util : ${GPU_MEM_UTIL}"
|
|||
|
|
echo " extra : ${EXTRA_ARGS}"
|
|||
|
|
echo "============================================================"
|
|||
|
|
|
|||
|
|
# shellcheck disable=SC2086
|
|||
|
|
exec vllm serve "${MODEL_PATH}" \
|
|||
|
|
--host "${HOST}" \
|
|||
|
|
--port "${PORT}" \
|
|||
|
|
--served-model-name "${SERVED_NAME}" \
|
|||
|
|
--tensor-parallel-size "${TP}" \
|
|||
|
|
--max-model-len "${MAX_MODEL_LEN}" \
|
|||
|
|
--max-num-seqs "${MAX_NUM_SEQS}" \
|
|||
|
|
--gpu-memory-utilization "${GPU_MEM_UTIL}" \
|
|||
|
|
--enforce-eager \
|
|||
|
|
${EXTRA_ARGS}
|