Honour vllm flags appended after the image, highest priority
Previously any arguments after the image in docker run were silently dropped: entrypoint.sh never forwarded "$@", so docker run ... <image> --served-model-name mymodel started fine but kept serving as llm with no warning. - entrypoint.sh: forward "$@"; skip script-generated flags the user already provided so vllm never sees duplicate keys; strip those keys from EXTRA_ARGS when the command line also sets them - precedence: command line > EXTRA_ARGS > same-name -e > detected default - warn when --port / --served-model-name is overridden, since the host -p side and the client model field must change too - README: document the three ways and their precedence
This commit is contained in:
25
README.md
25
README.md
@@ -355,6 +355,31 @@ Maverick 占用全部 16 张卡,与其它模型互斥;Scout 只占 8 张,
|
|||||||
**注意**:两个变体都映射到宿主 `24396`,串行部署没问题;并行时必须改 `-p` 左侧端口。
|
**注意**:两个变体都映射到宿主 `24396`,串行部署没问题;并行时必须改 `-p` 左侧端口。
|
||||||
`-e SERVED_NAME=llm` 让 `/v1/models` 返回 `llm`,客户端不用为换模型改 `model` 字段。
|
`-e SERVED_NAME=llm` 让 `/v1/models` 返回 `llm`,客户端不用为换模型改 `model` 字段。
|
||||||
|
|
||||||
|
### 5.2.1 覆盖参数:三种写法,优先级从高到低
|
||||||
|
|
||||||
|
| 优先级 | 写法 | 例子 |
|
||||||
|
|---|---|---|
|
||||||
|
| 1 | 在镜像后面直接跟 vllm 参数 | `... llama-4-bi150:v0.1.2 --served-model-name mymodel` |
|
||||||
|
| 2 | `-e EXTRA_ARGS="..."` | `-e EXTRA_ARGS="--max-model-len 8192"` |
|
||||||
|
| 3 | 同名 `-e` | `-e MAX_MODEL_LEN=8192 -e SERVED_NAME=mymodel` |
|
||||||
|
| 4 | 默认值 / config.json 检测 | 什么都不传 |
|
||||||
|
|
||||||
|
三种写法都能生效,不会重复传参。覆盖 `--port` / `--served-model-name` 时会打印 WARN,
|
||||||
|
因为宿主机 `-p` 右侧端口和客户端 `model` 字段也要同步改。
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# 镜像后直接跟参数
|
||||||
|
docker run -d --name llama4-scout \
|
||||||
|
--entrypoint /usr/local/bin/llama4-entrypoint.sh --workdir /opt/llama4 \
|
||||||
|
--privileged --pid host --ipc host -v /dev:/dev \
|
||||||
|
-v /mnt/disk0/models/LLM-Research/Llama-4-Scout-17B-16E-Instruct:/models/llama4:ro \
|
||||||
|
--ulimit memlock=-1 --ulimit stack=67108864 \
|
||||||
|
-p 24396:80 \
|
||||||
|
-e CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 -e MODEL_PATH=/models/llama4 -e AUTO_DETECT=1 \
|
||||||
|
harbor.4pd.io/modelhubxc/xiejiahao/llama-4-bi150:v0.1.2 \
|
||||||
|
--served-model-name mymodel --max-model-len 8192
|
||||||
|
```
|
||||||
|
|
||||||
### 5.3 用 run.sh 启动 CI 镜像
|
### 5.3 用 run.sh 启动 CI 镜像
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
|
|||||||
100
entrypoint.sh
100
entrypoint.sh
@@ -56,16 +56,6 @@ GPU_MEM_UTIL="${GPU_MEM_UTIL:-0.90}"
|
|||||||
ENFORCE_EAGER="${ENFORCE_EAGER:-1}"
|
ENFORCE_EAGER="${ENFORCE_EAGER:-1}"
|
||||||
EXTRA_ARGS="${EXTRA_ARGS:-}"
|
EXTRA_ARGS="${EXTRA_ARGS:-}"
|
||||||
|
|
||||||
PP_ARGS=()
|
|
||||||
if (( PP > 1 )); then
|
|
||||||
PP_ARGS=(--pipeline-parallel-size "${PP}")
|
|
||||||
fi
|
|
||||||
|
|
||||||
EAGER_ARGS=()
|
|
||||||
if [[ "${ENFORCE_EAGER}" == "1" ]]; then
|
|
||||||
EAGER_ARGS=(--enforce-eager)
|
|
||||||
fi
|
|
||||||
|
|
||||||
echo "============================================================"
|
echo "============================================================"
|
||||||
echo "Llama 4 (BI-V150 unified image)"
|
echo "Llama 4 (BI-V150 unified image)"
|
||||||
echo " model kind : ${MODEL_KIND}"
|
echo " model kind : ${MODEL_KIND}"
|
||||||
@@ -88,15 +78,85 @@ if [[ -n "${DETECTED_HOST_PORT_HINT}" ]]; then
|
|||||||
fi
|
fi
|
||||||
echo "============================================================"
|
echo "============================================================"
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------- 用户参数
|
||||||
|
# 优先级(高 -> 低):
|
||||||
|
# 1. docker run <image> 之后追加的 vllm 参数("$@")
|
||||||
|
# 2. -e EXTRA_ARGS="..."
|
||||||
|
# 3. -e 显式设置(SERVED_NAME / PORT / TP / PP / MAX_MODEL_LEN ...)
|
||||||
|
# 4. config.json 自动检测
|
||||||
|
# 用户已经给了的 flag,本脚本不再重复传,避免 vllm 报 duplicate key。
|
||||||
|
# 末尾参数的 flag 优先于 EXTRA_ARGS 里的同名 flag:把后者摘掉,避免 vllm 报 duplicate key。
|
||||||
|
declare -A _CLI_FLAGS=()
|
||||||
|
if (( $# > 0 )); then
|
||||||
|
for _t in "$@"; do
|
||||||
|
[[ "${_t}" == --* ]] && _CLI_FLAGS["${_t%%=*}"]=1
|
||||||
|
done
|
||||||
|
fi
|
||||||
|
|
||||||
|
EXTRA_FILTERED=()
|
||||||
|
if [[ -n "${EXTRA_ARGS}" ]]; then
|
||||||
|
read -r -a _extra_tokens <<< "${EXTRA_ARGS}"
|
||||||
|
_skip=0
|
||||||
|
for _t in "${_extra_tokens[@]}"; do
|
||||||
|
if (( _skip > 0 )); then
|
||||||
|
_skip=0
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
if [[ "${_t}" == --* ]]; then
|
||||||
|
_name="${_t%%=*}"
|
||||||
|
if [[ -n "${_CLI_FLAGS[${_name}]:-}" ]]; then
|
||||||
|
# 该 flag 在命令行里也给了,跳过它(以及紧跟的取值)
|
||||||
|
[[ "${_t}" != *=* ]] && _skip=1
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
EXTRA_FILTERED+=("${_t}")
|
||||||
|
done
|
||||||
|
fi
|
||||||
|
|
||||||
|
USER_TOKENS=("${EXTRA_FILTERED[@]}")
|
||||||
|
if (( $# > 0 )); then
|
||||||
|
USER_TOKENS+=("$@")
|
||||||
|
fi
|
||||||
|
|
||||||
|
user_has() {
|
||||||
|
local want="$1" tok
|
||||||
|
(( ${#USER_TOKENS[@]} == 0 )) && return 1
|
||||||
|
for tok in "${USER_TOKENS[@]}"; do
|
||||||
|
if [[ "${tok}" == "${want}" || "${tok}" == "${want}="* ]]; then
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
|
||||||
|
if (( $# > 0 )); then
|
||||||
|
echo "用户追加参数 : $*"
|
||||||
|
echo "(优先级最高,覆盖 -e 与自动检测)"
|
||||||
|
user_has --port && echo " WARN: --port 会改变容器内监听端口,宿主机 -p 的右侧端口也要同步改" >&2
|
||||||
|
user_has --served-model-name && echo " WARN: --served-model-name 已覆盖默认的 llm,客户端 model 字段需同步修改" >&2
|
||||||
|
echo "------------------------------------------------------------"
|
||||||
|
fi
|
||||||
|
|
||||||
|
SERVE_ARGS=()
|
||||||
|
user_has --host || SERVE_ARGS+=(--host "${HOST}")
|
||||||
|
user_has --port || SERVE_ARGS+=(--port "${PORT}")
|
||||||
|
user_has --served-model-name || SERVE_ARGS+=(--served-model-name "${SERVED_NAME}")
|
||||||
|
user_has --tensor-parallel-size || SERVE_ARGS+=(--tensor-parallel-size "${TP}")
|
||||||
|
if (( PP > 1 )) && ! user_has --pipeline-parallel-size; then
|
||||||
|
SERVE_ARGS+=(--pipeline-parallel-size "${PP}")
|
||||||
|
fi
|
||||||
|
user_has --max-model-len || SERVE_ARGS+=(--max-model-len "${MAX_MODEL_LEN}")
|
||||||
|
user_has --max-num-seqs || SERVE_ARGS+=(--max-num-seqs "${MAX_NUM_SEQS}")
|
||||||
|
user_has --gpu-memory-utilization || SERVE_ARGS+=(--gpu-memory-utilization "${GPU_MEM_UTIL}")
|
||||||
|
if ! user_has --enforce-eager && ! user_has --no-enforce-eager; then
|
||||||
|
if [[ "${ENFORCE_EAGER}" == "1" ]]; then
|
||||||
|
SERVE_ARGS+=(--enforce-eager)
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
# shellcheck disable=SC2086
|
# shellcheck disable=SC2086
|
||||||
exec "${VLLM_BIN}" serve "${MODEL_PATH}" \
|
exec "${VLLM_BIN}" serve "${MODEL_PATH}" \
|
||||||
--host "${HOST}" \
|
"${SERVE_ARGS[@]}" \
|
||||||
--port "${PORT}" \
|
"${EXTRA_FILTERED[@]}" \
|
||||||
--served-model-name "${SERVED_NAME}" \
|
"$@"
|
||||||
--tensor-parallel-size "${TP}" \
|
|
||||||
"${PP_ARGS[@]}" \
|
|
||||||
--max-model-len "${MAX_MODEL_LEN}" \
|
|
||||||
--max-num-seqs "${MAX_NUM_SEQS}" \
|
|
||||||
--gpu-memory-utilization "${GPU_MEM_UTIL}" \
|
|
||||||
"${EAGER_ARGS[@]}" \
|
|
||||||
${EXTRA_ARGS}
|
|
||||||
|
|||||||
Reference in New Issue
Block a user