47 lines
1.2 KiB
YAML
47 lines
1.2 KiB
YAML
services:
|
||
llama4:
|
||
build:
|
||
context: .
|
||
dockerfile: Dockerfile
|
||
image: llama4-bi150:${TAG:-2026-10-26}
|
||
container_name: llama4
|
||
|
||
# vLLM 多卡 NCCL / 端口直通
|
||
network_mode: host
|
||
ipc: host
|
||
|
||
# Iluvatar 运行时要求
|
||
privileged: true
|
||
pid: host
|
||
|
||
volumes:
|
||
- /dev:/dev
|
||
- ${MODEL_HOST_DIR:-/mnt/disk0/models/LLM-Research/Llama-4-Scout-17B-16E-Instruct}:/models/llama4:ro
|
||
|
||
environment:
|
||
MODEL_PATH: /models/llama4
|
||
AUTO_DETECT: "${AUTO_DETECT:-1}"
|
||
HOST: 0.0.0.0
|
||
|
||
# 默认暴露全部 16 张卡；entrypoint 会根据模型选择实际 TP*PP。
|
||
CUDA_VISIBLE_DEVICES: "${GPUS:-0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15}"
|
||
|
||
# 留空时由 entrypoint 根据 config.json 自动判断。
|
||
# 需要手动覆盖时直接设置同名环境变量即可。
|
||
TP: "${TP:-}"
|
||
PP: "${PP:-}"
|
||
PORT: "${PORT:-}"
|
||
SERVED_NAME: "${SERVED_NAME:-}"
|
||
|
||
MAX_MODEL_LEN: "${MAX_MODEL_LEN:-4096}"
|
||
MAX_NUM_SEQS: "${MAX_NUM_SEQS:-1}"
|
||
GPU_MEM_UTIL: "${GPU_MEM_UTIL:-0.90}"
|
||
ENFORCE_EAGER: "${ENFORCE_EAGER:-1}"
|
||
EXTRA_ARGS: "${EXTRA_ARGS:-}"
|
||
|
||
ulimits:
|
||
memlock: -1
|
||
stack: 67108864
|
||
|
||
restart: unless-stopped
|