xiejiahao
daaf85f4e5
- run.sh: bridge + -p ${HOST_PORT:-24396}:80, health check uses host port
- entrypoint.sh: PORT defaults to 80, SERVED_NAME defaults to llm
detected port/served name become log hints only
- verify.sh: read host port from port bindings, expect model name llm
- docker-compose.yml: ports mapping instead of network_mode host
- README: update tables, commands and rationale
103 lines
3.5 KiB
Bash
Executable File
103 lines
3.5 KiB
Bash
Executable File
#!/usr/bin/env bash
|
||
#
|
||
# Llama 4 统一容器入口：
|
||
# 1. 幂等应用 CoreX vLLM 的 Llama 4 补丁
|
||
# 2. 读取 /models/llama4/config.json，自动判断 Scout / Maverick
|
||
# 3. 选择对应的 TP / PP / 端口 / served name，再启动 vLLM
|
||
#
|
||
# AUTO_DETECT=1（默认）时根据模型 config 自动选择；显式设置的
|
||
# TP / PP / PORT / SERVED_NAME 会覆盖检测结果。AUTO_DETECT=0 可完全手动。
|
||
#
|
||
set -euo pipefail
|
||
|
||
PATCH_SCRIPT="${LLAMA4_PATCH_SCRIPT:-/opt/llama4/apply_patch.py}"
|
||
DETECT_SCRIPT="${LLAMA4_DETECT_SCRIPT:-/opt/llama4/detect_model.py}"
|
||
VLLM_BIN="${VLLM_BIN:-vllm}"
|
||
|
||
python3 "${PATCH_SCRIPT}"
|
||
|
||
MODEL_PATH="${MODEL_PATH:-/models/llama4}"
|
||
AUTO_DETECT="${AUTO_DETECT:-1}"
|
||
MODEL_DESCRIPTION=""
|
||
|
||
if [[ ! -f "${MODEL_PATH}/config.json" ]]; then
|
||
echo "ERROR: 模型目录不完整：${MODEL_PATH}/config.json 不存在" >&2
|
||
echo "请确认已挂载模型，例如：" >&2
|
||
echo " -v /mnt/disk0/models/LLM-Research/Llama-4-Scout-17B-16E-Instruct:/models/llama4:ro" >&2
|
||
echo " -v /mnt/disk0/models/LLM-Research/Llama-4-Maverick-17B-128E-Instruct-FP8:/models/llama4:ro" >&2
|
||
exit 1
|
||
fi
|
||
|
||
if [[ "${AUTO_DETECT}" == "1" ]]; then
|
||
DETECT_OUTPUT="$(python3 "${DETECT_SCRIPT}" "${MODEL_PATH}")" || exit 1
|
||
eval "${DETECT_OUTPUT}"
|
||
MODEL_KIND="${MODEL_KIND:-${DETECTED_MODEL_KIND}}"
|
||
TP="${TP:-${DETECTED_TP}}"
|
||
PP="${PP:-${DETECTED_PP}}"
|
||
MODEL_DESCRIPTION="${DETECTED_DESCRIPTION:-}"
|
||
# 检测出的 port / served_name 仅用于日志提示，不再决定容器内监听端口和模型名：
|
||
# 容器内端口固定 80，模型名固定 llm（见 AGENTS.md 5.2 / 5.5）。
|
||
DETECTED_HOST_PORT_HINT="${DETECTED_PORT:-}"
|
||
DETECTED_NAME_HINT="${DETECTED_SERVED_NAME:-}"
|
||
else
|
||
MODEL_KIND="${MODEL_KIND:-unknown}"
|
||
DETECTED_HOST_PORT_HINT=""
|
||
DETECTED_NAME_HINT=""
|
||
fi
|
||
|
||
HOST="${HOST:-0.0.0.0}"
|
||
TP="${TP:-8}"
|
||
PP="${PP:-1}"
|
||
PORT="${PORT:-80}"
|
||
SERVED_NAME="${SERVED_NAME:-llm}"
|
||
MAX_MODEL_LEN="${MAX_MODEL_LEN:-4096}"
|
||
MAX_NUM_SEQS="${MAX_NUM_SEQS:-1}"
|
||
GPU_MEM_UTIL="${GPU_MEM_UTIL:-0.90}"
|
||
ENFORCE_EAGER="${ENFORCE_EAGER:-1}"
|
||
EXTRA_ARGS="${EXTRA_ARGS:-}"
|
||
|
||
PP_ARGS=()
|
||
if (( PP > 1 )); then
|
||
PP_ARGS=(--pipeline-parallel-size "${PP}")
|
||
fi
|
||
|
||
EAGER_ARGS=()
|
||
if [[ "${ENFORCE_EAGER}" == "1" ]]; then
|
||
EAGER_ARGS=(--enforce-eager)
|
||
fi
|
||
|
||
echo "============================================================"
|
||
echo "Llama 4 (BI-V150 unified image)"
|
||
echo " model kind : ${MODEL_KIND}"
|
||
echo " model : ${MODEL_PATH}"
|
||
echo " served name: ${SERVED_NAME}"
|
||
echo " listen : ${HOST}:${PORT}"
|
||
echo " TP / PP : ${TP} / ${PP} (total $((TP * PP)) GPUs)"
|
||
echo " max len : ${MAX_MODEL_LEN}"
|
||
echo " max seqs : ${MAX_NUM_SEQS}"
|
||
echo " gpu util : ${GPU_MEM_UTIL}"
|
||
echo " eager : ${ENFORCE_EAGER}"
|
||
echo " visible : ${CUDA_VISIBLE_DEVICES:-all}"
|
||
echo " extra : ${EXTRA_ARGS}"
|
||
if [[ -n "${MODEL_DESCRIPTION}" ]]; then
|
||
echo " detected : ${MODEL_DESCRIPTION}"
|
||
fi
|
||
if [[ -n "${DETECTED_HOST_PORT_HINT}" ]]; then
|
||
echo " hint : 变体默认宿主端口 ${DETECTED_HOST_PORT_HINT}，模型名曾用 ${DETECTED_NAME_HINT}"
|
||
echo " （容器内固定 80 / served name 固定 llm，实际端口由 docker run -p 决定）"
|
||
fi
|
||
echo "============================================================"
|
||
|
||
# shellcheck disable=SC2086
|
||
exec "${VLLM_BIN}" serve "${MODEL_PATH}" \
|
||
--host "${HOST}" \
|
||
--port "${PORT}" \
|
||
--served-model-name "${SERVED_NAME}" \
|
||
--tensor-parallel-size "${TP}" \
|
||
"${PP_ARGS[@]}" \
|
||
--max-model-len "${MAX_MODEL_LEN}" \
|
||
--max-num-seqs "${MAX_NUM_SEQS}" \
|
||
--gpu-memory-utilization "${GPU_MEM_UTIL}" \
|
||
"${EAGER_ARGS[@]}" \
|
||
${EXTRA_ARGS}
|