刘宪东
fb5a163d07
Some checks failed
Build and verify Qwen2.5 compatibility engines / docker (push) Failing after 3m32s
48 lines
1.5 KiB
Bash
48 lines
1.5 KiB
Bash
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
vllm_real="$(dirname "$(readlink -f "$0")")/vllm_real"
|
|
|
|
if [[ "${1:-}" != "serve" || -z "${2:-}" ]]; then
|
|
exec "$vllm_real" "$@"
|
|
fi
|
|
|
|
model_dir="$2"
|
|
shift 2
|
|
args=("$@")
|
|
tokenizer_dir=""
|
|
for ((i=0; i<${#args[@]}; i++)); do
|
|
if [[ "${args[i]}" == "--tokenizer" && $((i+1)) -lt ${#args[@]} ]]; then
|
|
tokenizer_dir="${args[i+1]}"
|
|
break
|
|
elif [[ "${args[i]}" == --tokenizer=* ]]; then
|
|
tokenizer_dir="${args[i]#--tokenizer=}"
|
|
break
|
|
fi
|
|
done
|
|
|
|
if [[ -n "$tokenizer_dir" && -d "$tokenizer_dir" ]]; then
|
|
prepared="$(python3 /opt/modelhub/engine_compat.py "$model_dir" "$tokenizer_dir")"
|
|
else
|
|
prepared="$(python3 /opt/modelhub/engine_compat.py "$model_dir")"
|
|
tokenizer_dir=""
|
|
fi
|
|
if [[ -z "$prepared" ]]; then
|
|
exec "$vllm_real" serve "$model_dir" "${args[@]}"
|
|
fi
|
|
|
|
effective_model="$prepared"
|
|
if [[ -n "$tokenizer_dir" && "$(readlink -f "$tokenizer_dir")" != "$(readlink -f "$model_dir")" ]]; then
|
|
# A vendor entrypoint's --tokenizer points at a copied tokenizer tree.
|
|
# Keep model weights at the original path; redirect only that tokenizer.
|
|
effective_model="$model_dir"
|
|
fi
|
|
for ((i=0; i<${#args[@]}; i++)); do
|
|
if [[ "${args[i]}" == "--tokenizer" && $((i+1)) -lt ${#args[@]} && "${args[i+1]}" == "$tokenizer_dir" ]]; then
|
|
args[i+1]="$prepared"
|
|
elif [[ "${args[i]}" == "--tokenizer=$tokenizer_dir" && -n "$tokenizer_dir" ]]; then
|
|
args[i]="--tokenizer=$prepared"
|
|
fi
|
|
done
|
|
exec "$vllm_real" serve "$effective_model" "${args[@]}"
|