llama-4-bi150/run.sh

#!/usr/bin/env bash
#
# Llama 4 统一构建 + 启动（Iluvatar BI-V150）
#
# 自动读取模型目录下的 config.json，判断是 Scout 还是 Maverick，然后选择：
#   Scout    : TP8 / PP1 / 8 卡 / 8091 / llama4-scout
#   Maverick : TP8 / PP2 / 16 卡 / 8092 / llama4-maverick
#
# 用法：
#   ./run.sh /mnt/disk0/models/LLM-Research/Llama-4-Scout-17B-16E-Instruct
#   ./run.sh /mnt/disk0/models/LLM-Research/Llama-4-Maverick-17B-128E-Instruct-FP8
#   MODEL_HOST_DIR=/path/to/model ./run.sh
#   ./run.sh                    # 依次尝试两个默认模型目录
#
# 常用覆盖项：
#   GPUS=8,9,10,11,12,13,14,15 ./run.sh /path/to/scout
#   EXTRA_DOCKER_ARGS="-v /mnt/disk1:/data:ro" ./run.sh /path/to/model
#   EXTRA_ARGS="--max-model-len 8192" ./run.sh /path/to/model
#   HOST_PORT=24397 ./run.sh /path/to/model        # 换宿主机端口（默认 24396）
#   TP=8 PP=1 SERVED_NAME=llm ./run.sh /path/to/model
#   BUILD=0 ./run.sh /path/to/model      # 使用已有镜像
#   DRY_RUN=1 ./run.sh /path/to/model    # 只打印 docker run 命令
#
set -euo pipefail

cd "$(dirname "$0")"

usage() {
  cat >&2 <<'EOF'
用法: ./run.sh [MODEL_DIR]

不传 MODEL_DIR 时，按顺序尝试：
  /mnt/disk0/models/LLM-Research/Llama-4-Scout-17B-16E-Instruct
  /mnt/disk0/models/LLM-Research/Llama-4-Maverick-17B-128E-Instruct-FP8

模型类型由 MODEL_DIR/config.json 自动判断，不需要 scout/maverick 参数。

环境变量覆盖：
  IMAGE NAME MODEL_HOST_DIR MODEL_MOUNT SERVED_NAME
  TP PP GPUS HOST_PORT（宿主机端口，默认 24396；容器内固定 80）
  MAX_MODEL_LEN MAX_NUM_SEQS GPU_MEM_UTIL
  ENFORCE_EAGER AUTO_DETECT EXTRA_ARGS EXTRA_DOCKER_ARGS
  BUILD=0/1  DRY_RUN=0/1
EOF
}

if [[ "${1:-}" == "-h" || "${1:-}" == "--help" ]]; then
  usage
  exit 0
fi

MODEL_HOST_DIR="${1:-${MODEL_HOST_DIR:-}}"
if [[ -z "${MODEL_HOST_DIR}" ]]; then
  for candidate in \
    "/mnt/disk0/models/LLM-Research/Llama-4-Scout-17B-16E-Instruct" \
    "/mnt/disk0/models/LLM-Research/Llama-4-Maverick-17B-128E-Instruct-FP8"; do
    if [[ -f "${candidate}/config.json" ]]; then
      MODEL_HOST_DIR="${candidate}"
      break
    fi
  done
fi

if [[ -z "${MODEL_HOST_DIR}" ]]; then
  echo "ERROR: 未指定模型目录，也没有找到默认模型目录。" >&2
  usage
  exit 2
fi

if [[ ! -f "${MODEL_HOST_DIR}/config.json" ]]; then
  echo "ERROR: 模型目录不完整: ${MODEL_HOST_DIR}/config.json" >&2
  echo "请先下载模型，或把 MODEL_DIR 指向正确目录。" >&2
  exit 1
fi

DETECT_OUTPUT="$(python3 detect_model.py "${MODEL_HOST_DIR}")" || exit 1
eval "${DETECT_OUTPUT}"

MODEL_KIND="${DETECTED_MODEL_KIND}"
IMAGE="${IMAGE:-llama4-bi150:2026-10-26}"
NAME="${NAME:-llama4-${MODEL_KIND}}"
HOST="${HOST:-0.0.0.0}"
# 容器内端口固定 80；宿主机端口是 HOST_PORT，默认 24396（串行部署复用同一端口）。
# PORT 保留为兼容别名，若显式给 PORT 则视作 HOST_PORT。
CONTAINER_PORT=80
HOST_PORT="${HOST_PORT:-${PORT:-24396}}"
MODEL_MOUNT="${MODEL_MOUNT:-/models/llama4}"
SERVED_NAME="${SERVED_NAME:-llm}"
TP="${TP:-${DETECTED_TP}}"
PP="${PP:-${DETECTED_PP}}"
MAX_MODEL_LEN="${MAX_MODEL_LEN:-4096}"
MAX_NUM_SEQS="${MAX_NUM_SEQS:-1}"
GPU_MEM_UTIL="${GPU_MEM_UTIL:-0.90}"
ENFORCE_EAGER="${ENFORCE_EAGER:-1}"
AUTO_DETECT="${AUTO_DETECT:-1}"
EXTRA_ARGS="${EXTRA_ARGS:-}"
EXTRA_DOCKER_ARGS="${EXTRA_DOCKER_ARGS:-}"
BUILD="${BUILD:-1}"
DRY_RUN="${DRY_RUN:-0}"

GPU_COUNT=$((TP * PP))

if [[ -n "${GPUS:-}" ]]; then
  GPUS="${GPUS}"
elif [[ -n "${CUDA_VISIBLE_DEVICES:-}" ]]; then
  GPUS="${CUDA_VISIBLE_DEVICES}"
else
  GPUS="$(seq -s, 0 $((GPU_COUNT - 1)))"
fi
GPUS="${GPUS// /}"

if [[ -n "${GPUS}" ]]; then
  IFS=',' read -r -a GPU_LIST <<< "${GPUS}"
  if (( ${#GPU_LIST[@]} < GPU_COUNT )); then
    echo "ERROR: ${MODEL_KIND} 需要 TP*PP=${GPU_COUNT} 张卡，但 GPUS 只给了 ${#GPU_LIST[@]} 张: ${GPUS}" >&2
    exit 1
  fi
  if (( ${#GPU_LIST[@]} > GPU_COUNT )); then
    echo "WARN: TP*PP=${GPU_COUNT}，但 GPUS 给了 ${#GPU_LIST[@]} 张；vLLM 只会使用其中 ${GPU_COUNT} 张。" >&2
  fi
fi

DOCKER_EXTRA=()
if [[ -n "${EXTRA_DOCKER_ARGS}" ]]; then
  # 按空白拆分；路径中不要带空格。
  read -r -a DOCKER_EXTRA <<< "${EXTRA_DOCKER_ARGS}"
fi

DOCKER_RUN=(
  docker run -d --name "${NAME}"
  --entrypoint /usr/local/bin/llama4-entrypoint.sh
  --workdir /opt/llama4
  --expose "${CONTAINER_PORT}"
  --privileged
  --pid host
  --ipc host
  -p "${HOST_PORT}:${CONTAINER_PORT}"
  -v /dev:/dev
  -v "${MODEL_HOST_DIR}:${MODEL_MOUNT}:ro"
  --ulimit memlock=-1
  --ulimit stack=67108864
  -e CUDA_VISIBLE_DEVICES="${GPUS}"
  -e MODEL_PATH="${MODEL_MOUNT}"
  -e AUTO_DETECT="${AUTO_DETECT}"
  -e MODEL_KIND="${MODEL_KIND}"
  -e HOST="${HOST}"
  -e SERVED_NAME="${SERVED_NAME}"
  -e PORT="${CONTAINER_PORT}"
  -e TP="${TP}"
  -e PP="${PP}"
  -e MAX_MODEL_LEN="${MAX_MODEL_LEN}"
  -e MAX_NUM_SEQS="${MAX_NUM_SEQS}"
  -e GPU_MEM_UTIL="${GPU_MEM_UTIL}"
  -e ENFORCE_EAGER="${ENFORCE_EAGER}"
  -e PYTHONUNBUFFERED=1
  -e EXTRA_ARGS="${EXTRA_ARGS}"
  "${DOCKER_EXTRA[@]}"
  --restart unless-stopped
  "${IMAGE}"
)

echo "==> detected ${MODEL_KIND}: ${DETECTED_DESCRIPTION}"
echo "    model : ${MODEL_HOST_DIR}"
echo "    TP/PP : ${TP}/${PP} (${GPU_COUNT} GPUs)"
echo "    port  : 127.0.0.1:${HOST_PORT} -> container:${CONTAINER_PORT}"
echo "    name  : ${SERVED_NAME}"
echo "    GPUs  : ${GPUS}"

if [[ "${DRY_RUN}" == "1" ]]; then
  printf '%q ' "${DOCKER_RUN[@]}"
  printf '\n'
  exit 0
fi

if [[ "${BUILD}" == "1" ]]; then
  echo "==> build ${IMAGE}"
  docker build -t "${IMAGE}" .
else
  echo "==> skip build (BUILD=0), image=${IMAGE}"
fi

echo "==> recreate ${NAME}"
docker rm -f "${NAME}" 2>/dev/null || true

echo "==> run ${MODEL_KIND}"
"${DOCKER_RUN[@]}"

echo "==> wait for http://127.0.0.1:${HOST_PORT}/health"
for _ in $(seq 1 120); do
  if curl -sf "http://127.0.0.1:${HOST_PORT}/health" >/dev/null 2>&1; then
    echo "ready: http://127.0.0.1:${HOST_PORT}/v1/models"
    exit 0
  fi
  sleep 5
done

echo "ERROR: 服务 10 分钟未就绪，查看日志：" >&2
echo "  docker logs -f ${NAME}" >&2
exit 1