26 lines
697 B
Bash
26 lines
697 B
Bash
|
#!/usr/bin/env bash
|
||
|
|
set -euo pipefail
|
||
|
|
|
||
|
|
vllm_real="$(dirname "$(readlink -f "$0")")/vllm_real"
|
||
|
|
|
||
|
|
if [[ "${1:-}" != "serve" || -z "${2:-}" ]]; then
|
||
|
|
exec "$vllm_real" "$@"
|
||
|
|
fi
|
||
|
|
|
||
|
|
model_dir="$2"
|
||
|
|
shift 2
|
||
|
|
prepared="$(python3 /opt/modelhub/engine_compat.py "$model_dir")"
|
||
|
|
if [[ -z "$prepared" ]]; then
|
||
|
|
exec "$vllm_real" serve "$model_dir" "$@"
|
||
|
|
fi
|
||
|
|
|
||
|
|
args=("$@")
|
||
|
|
for ((i=0; i<${#args[@]}; i++)); do
|
||
|
|
if [[ "${args[i]}" == "--tokenizer" && $((i+1)) -lt ${#args[@]} && "${args[i+1]}" == "$model_dir" ]]; then
|
||
|
|
args[i+1]="$prepared"
|
||
|
|
elif [[ "${args[i]}" == "--tokenizer=$model_dir" ]]; then
|
||
|
|
args[i]="--tokenizer=$prepared"
|
||
|
|
fi
|
||
|
|
done
|
||
|
|
exec "$vllm_real" serve "$prepared" "${args[@]}"
|