Initial commit: Llama 4 Scout BI-V150 deployment bundle

2026-10-23 13:41:41 +08:00
commit 7ce7159d36
6 changed files with 296 additions and 0 deletions
--- a/51
+++ b/51
@@ -0,0 +1,51 @@
+# Llama 4 Scout on Iluvatar BI-V150
+#
+# 基础镜像就是当前 ugly_hopper / test 用的 CoreX vLLM 镜像。
+# 这个 Dockerfile 做三件事：
+#   1. 把 CoreX vLLM 的 Llama4 chunked-attention 补丁打进镜像
+#   2. 安装 entrypoint（容器启动时再次幂等检查补丁）
+#   3. 默认以 TP8 / 4096 ctx / 0.90 显存启动 vLLM
+#
+# 构建：
+#   docker build -t llama4-scout:2026-10-23 .
+#
+# 运行（8 卡）：
+#   docker run -d --name llama4-scout \
+#     --network host --ipc host \
+#     --device=/dev/iluvatar0 ... --device=/dev/iluvatar7 \
+#     -v /mnt/disk0/models/LLM-Research/Llama-4-Scout-17B-16E-Instruct:/models/llama4-scout:ro \
+#     --ulimit memlock=-1 --ulimit stack=67108864 \
+#     llama4-scout:2026-10-23
+#
+# 说明：
+#   - 不需要 --privileged；只需要 GPU 设备节点 + host IPC/network。
+#   - 如果厂商 runtime 要求整块 /dev，可用 -v /dev:/dev 代替 --device 列表。
+#   - 模型目录只读挂载，容器内不写模型。
+
+FROM registry.iluvatar.com.cn:10443/customer/sz/vllm0.17.0-4.4.0-x86:v7
+
+LABEL org.opencontainers.image.title="llama4-scout-bi150" \
+      org.opencontainers.image.description="Llama 4 Scout 17B-16E Instruct on Iluvatar BI-V150 with vLLM 0.17.0+corex" \
+      org.opencontainers.image.version="2026-10-23"
+
+COPY apply_patch.py /opt/llama4/apply_patch.py
+COPY entrypoint.sh /usr/local/bin/llama4-entrypoint.sh
+COPY vllm_llama4_chunked_attn.patch /opt/llama4/vllm_llama4_chunked_attn.patch
+
+RUN chmod +x /usr/local/bin/llama4-entrypoint.sh \
+    && python3 /opt/llama4/apply_patch.py
+
+ENV MODEL_PATH=/models/llama4-scout \
+    SERVED_NAME=llama4-scout \
+    HOST=0.0.0.0 \
+    PORT=8091 \
+    TP=8 \
+    MAX_MODEL_LEN=4096 \
+    MAX_NUM_SEQS=1 \
+    GPU_MEM_UTIL=0.90 \
+    PYTHONUNBUFFERED=1
+
+WORKDIR /opt/llama4
+EXPOSE 8091
+
+ENTRYPOINT ["/usr/local/bin/llama4-entrypoint.sh"]
--- a/apply_patch.py
+++ b/apply_patch.py
@@ -0,0 +1,77 @@
+#!/usr/bin/env python3
+"""
+修复 CoreX 版 vLLM 0.17.0 的 Llama 4 chunked local attention bug。
+
+症状（第一个真实请求时）：
+  TypeError: CommonAttentionMetadata.__init__() missing 2 required positional
+  arguments: 'key_start_loc' and 'seq_lens_np'
+
+原因：
+  vllm/v1/attention/backends/utils.py 的
+  make_local_attention_virtual_batches() 在构造 CommonAttentionMetadata 时
+  没有传新版本 dataclass 要求的 key_start_loc / seq_lens_np。
+
+本脚本幂等：已经打过补丁会直接返回 0。
+"""
+
+import os
+import sys
+
+try:
+    import vllm
+except ImportError as e:
+    print(f"ERROR: 无法 import vllm: {e}", file=sys.stderr)
+    sys.exit(1)
+
+TARGET = os.path.join(
+    os.path.dirname(vllm.__file__),
+    "v1",
+    "attention",
+    "backends",
+    "utils.py",
+)
+
+MARKER = "seq_lens_np=seqlens_k_local"
+
+OLD = """        query_start_loc=query_start_loc_cpu.to(device=device, non_blocking=True),
+        seq_lens=seq_lens_cpu.to(device=device, non_blocking=True),"""
+
+NEW = """        query_start_loc=query_start_loc_cpu.to(device=device, non_blocking=True),
+        key_start_loc=torch.from_numpy(
+            np.concatenate(
+                (np.zeros(1, dtype=np.int32), np.cumsum(seqlens_k_local, dtype=np.int32))
+            )
+        ).to(device=device, non_blocking=True),
+        seq_lens=seq_lens_cpu.to(device=device, non_blocking=True),
+        seq_lens_np=seqlens_k_local,"""
+
+
+def main() -> int:
+    if not os.path.exists(TARGET):
+        print(f"ERROR: 找不到目标文件: {TARGET}", file=sys.stderr)
+        return 1
+
+    with open(TARGET, "r", encoding="utf-8") as f:
+        src = f.read()
+
+    if MARKER in src:
+        print(f"patch already applied: {TARGET}")
+        return 0
+
+    if OLD not in src:
+        print(
+            "ERROR: 目标代码块未找到，vLLM 版本可能不同。\n"
+            f"请手动检查: {TARGET}",
+            file=sys.stderr,
+        )
+        return 1
+
+    with open(TARGET, "w", encoding="utf-8") as f:
+        f.write(src.replace(OLD, NEW, 1))
+
+    print(f"patched: {TARGET}")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -0,0 +1,43 @@
+services:
+  llama4-scout:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    image: llama4-scout:2026-10-23
+    container_name: llama4-scout
+
+    # vLLM 多卡 NCCL / 端口直通
+    network_mode: host
+    ipc: host
+
+    # Iluvatar 运行时需要 privileged + /dev 绑定。
+    # 只给 --device=/dev/iluvatar* 会在 worker init 时报
+    #   cudaGetDeviceCount() ... Error 1001
+    # 这是厂商驱动/容器运行时要求，不是 vLLM 本身的要求。
+    privileged: true
+    pid: host
+
+    volumes:
+      - /dev:/dev
+      - ${MODEL_HOST_DIR:-/mnt/disk0/models/LLM-Research/Llama-4-Scout-17B-16E-Instruct}:/models/llama4-scout:ro
+
+    environment:
+      MODEL_PATH: /models/llama4-scout
+      SERVED_NAME: llama4-scout
+      HOST: 0.0.0.0
+      PORT: "8091"
+      TP: "8"
+      MAX_MODEL_LEN: "4096"
+      MAX_NUM_SEQS: "1"
+      GPU_MEM_UTIL: "0.90"
+      # EXTRA_ARGS: "--pipeline-parallel-size 2"
+
+    ulimits:
+      memlock: -1
+      stack: 67108864
+
+    # 可选：某些调度场景需要；不需要可以删掉
+    # cap_add:
+    #   - SYS_NICE
+
+    restart: unless-stopped
--- a/entrypoint.sh
+++ b/entrypoint.sh
@@ -0,0 +1,51 @@
+#!/usr/bin/env bash
+#
+# Llama 4 Scout 容器入口：
+#   1. 幂等应用 vLLM CoreX 的 Llama4 chunked-attention 补丁
+#   2. 检查模型目录
+#   3. exec vLLM（信号可直接传给 vLLM）
+#
+set -euo pipefail
+
+python3 /opt/llama4/apply_patch.py
+
+MODEL_PATH="${MODEL_PATH:-/models/llama4-scout}"
+SERVED_NAME="${SERVED_NAME:-llama4-scout}"
+HOST="${HOST:-0.0.0.0}"
+PORT="${PORT:-8091}"
+TP="${TP:-8}"
+MAX_MODEL_LEN="${MAX_MODEL_LEN:-4096}"
+MAX_NUM_SEQS="${MAX_NUM_SEQS:-1}"
+GPU_MEM_UTIL="${GPU_MEM_UTIL:-0.90}"
+EXTRA_ARGS="${EXTRA_ARGS:-}"
+
+if [[ ! -f "${MODEL_PATH}/config.json" ]]; then
+  echo "ERROR: 模型目录不完整：${MODEL_PATH}/config.json 不存在" >&2
+  echo "请确认已挂载模型，例如：" >&2
+  echo "  -v /mnt/disk0/models/LLM-Research/Llama-4-Scout-17B-16E-Instruct:/models/llama4-scout:ro" >&2
+  exit 1
+fi
+
+echo "============================================================"
+echo "Llama 4 Scout"
+echo "  model      : ${MODEL_PATH}"
+echo "  served name: ${SERVED_NAME}"
+echo "  listen     : ${HOST}:${PORT}"
+echo "  TP         : ${TP}"
+echo "  max len    : ${MAX_MODEL_LEN}"
+echo "  max seqs   : ${MAX_NUM_SEQS}"
+echo "  gpu util   : ${GPU_MEM_UTIL}"
+echo "  extra      : ${EXTRA_ARGS}"
+echo "============================================================"
+
+# shellcheck disable=SC2086
+exec vllm serve "${MODEL_PATH}" \
+  --host "${HOST}" \
+  --port "${PORT}" \
+  --served-model-name "${SERVED_NAME}" \
+  --tensor-parallel-size "${TP}" \
+  --max-model-len "${MAX_MODEL_LEN}" \
+  --max-num-seqs "${MAX_NUM_SEQS}" \
+  --gpu-memory-utilization "${GPU_MEM_UTIL}" \
+  --enforce-eager \
+  ${EXTRA_ARGS}
--- a/run.sh
+++ b/run.sh
@@ -0,0 +1,58 @@
+#!/usr/bin/env bash
+#
+# 一键构建 + 启动 Llama 4 Scout（8 卡）
+#
+#   MODEL_HOST_DIR=/mnt/disk0/models/LLM-Research/Llama-4-Scout-17B-16E-Instruct \
+#   ./run.sh
+#
+set -euo pipefail
+
+IMAGE="${IMAGE:-llama4-scout:2026-10-23}"
+NAME="${NAME:-llama4-scout}"
+PORT="${PORT:-8091}"
+MODEL_HOST_DIR="${MODEL_HOST_DIR:-/mnt/disk0/models/LLM-Research/Llama-4-Scout-17B-16E-Instruct}"
+
+if [[ ! -f "${MODEL_HOST_DIR}/config.json" ]]; then
+  echo "ERROR: 模型目录不完整: ${MODEL_HOST_DIR}/config.json" >&2
+  echo "请先下载模型，或设置 MODEL_HOST_DIR 指向正确目录。" >&2
+  exit 1
+fi
+
+echo "==> build ${IMAGE}"
+docker build -t "${IMAGE}" .
+
+echo "==> recreate ${NAME}"
+docker rm -f "${NAME}" 2>/dev/null || true
+
+# Iluvatar 运行时需要 privileged + /dev 绑定。
+# 只给 --device=/dev/iluvatar* 会在 worker init 时报
+#   cudaGetDeviceCount() ... Error 1001
+# 这是厂商驱动/容器运行时要求，不是 vLLM 本身的要求。
+docker run -d --name "${NAME}" \
+  --privileged \
+  --pid host \
+  --network host \
+  --ipc host \
+  -v /dev:/dev \
+  -v "${MODEL_HOST_DIR}:/models/llama4-scout:ro" \
+  --ulimit memlock=-1 \
+  --ulimit stack=67108864 \
+  -e TP=8 \
+  -e MAX_MODEL_LEN=4096 \
+  -e MAX_NUM_SEQS=1 \
+  -e GPU_MEM_UTIL=0.90 \
+  --restart unless-stopped \
+  "${IMAGE}"
+
+echo "==> wait for http://127.0.0.1:${PORT}/health"
+for _ in $(seq 1 120); do
+  if curl -sf "http://127.0.0.1:${PORT}/health" >/dev/null 2>&1; then
+    echo "ready: http://127.0.0.1:${PORT}/v1/models"
+    exit 0
+  fi
+  sleep 5
+done
+
+echo "ERROR: 服务 10 分钟未就绪，查看日志：" >&2
+echo "  docker logs -f ${NAME}" >&2
+exit 1
--- a/vllm_llama4_chunked_attn.patch
+++ b/vllm_llama4_chunked_attn.patch
@@ -0,0 +1,16 @@
+--- a/vllm/v1/attention/backends/utils.py
+++ b/vllm/v1/attention/backends/utils.py
+@@ -346,7 +346,13 @@
+     return CommonAttentionMetadata(
+         query_start_loc_cpu=query_start_loc_cpu,
+         query_start_loc=query_start_loc_cpu.to(device=device, non_blocking=True),
+        key_start_loc=torch.from_numpy(
+            np.concatenate(
+                (np.zeros(1, dtype=np.int32), np.cumsum(seqlens_k_local, dtype=np.int32))
+            )
+        ).to(device=device, non_blocking=True),
+         seq_lens=seq_lens_cpu.to(device=device, non_blocking=True),
+        seq_lens_np=seqlens_k_local,
+         num_reqs=len(seq_lens_cpu),
+         num_actual_tokens=common_attn_metadata.num_actual_tokens,
+         max_query_len=seqlens_q_local.max(),