55 lines
1.4 KiB
YAML
55 lines
1.4 KiB
YAML
|
x-llama4-common: &llama4-common
|
||
|
|
build:
|
||
|
|
context: .
|
||
|
|
dockerfile: Dockerfile
|
||
|
|
image: llama4-bi150:${TAG:-2026-10-26}
|
||
|
|
network_mode: host
|
||
|
|
ipc: host
|
||
|
|
privileged: true
|
||
|
|
pid: host
|
||
|
|
ulimits:
|
||
|
|
memlock: -1
|
||
|
|
stack: 67108864
|
||
|
|
restart: unless-stopped
|
||
|
|
|
||
|
|
services:
|
||
|
|
scout:
|
||
|
|
<<: *llama4-common
|
||
|
|
profiles: ["scout"]
|
||
|
|
container_name: llama4-scout
|
||
|
|
volumes:
|
||
|
|
- /dev:/dev
|
||
|
|
- ${SCOUT_MODEL_HOST_DIR:-/mnt/disk0/models/LLM-Research/Llama-4-Scout-17B-16E-Instruct}:/models/llama4:ro
|
||
|
|
environment:
|
||
|
|
MODEL_PATH: /models/llama4
|
||
|
|
SERVED_NAME: llama4-scout
|
||
|
|
PORT: "8091"
|
||
|
|
TP: "8"
|
||
|
|
PP: "1"
|
||
|
|
MAX_MODEL_LEN: "4096"
|
||
|
|
MAX_NUM_SEQS: "1"
|
||
|
|
GPU_MEM_UTIL: "0.90"
|
||
|
|
ENFORCE_EAGER: "1"
|
||
|
|
CUDA_VISIBLE_DEVICES: "${SCOUT_GPUS:-0,1,2,3,4,5,6,7}"
|
||
|
|
EXTRA_ARGS: "${SCOUT_EXTRA_ARGS:-}"
|
||
|
|
|
||
|
|
maverick:
|
||
|
|
<<: *llama4-common
|
||
|
|
profiles: ["maverick"]
|
||
|
|
container_name: llama4-maverick
|
||
|
|
volumes:
|
||
|
|
- /dev:/dev
|
||
|
|
- ${MAVERICK_MODEL_HOST_DIR:-/mnt/disk0/models/LLM-Research/Llama-4-Maverick-17B-128E-Instruct-FP8}:/models/llama4:ro
|
||
|
|
environment:
|
||
|
|
MODEL_PATH: /models/llama4
|
||
|
|
SERVED_NAME: llama4-maverick
|
||
|
|
PORT: "8092"
|
||
|
|
TP: "8"
|
||
|
|
PP: "2"
|
||
|
|
MAX_MODEL_LEN: "4096"
|
||
|
|
MAX_NUM_SEQS: "1"
|
||
|
|
GPU_MEM_UTIL: "0.90"
|
||
|
|
ENFORCE_EAGER: "1"
|
||
|
|
CUDA_VISIBLE_DEVICES: "${MAVERICK_GPUS:-0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15}"
|
||
|
|
EXTRA_ARGS: "${MAVERICK_EXTRA_ARGS:-}"
|