9ef5af3bda
- patch_ops.sh: call cccl_preload/build_cccl_preload.sh (new CCCL deps) instead of old build_cccl_preload_allocator.sh (mock) - computility-run.yaml: add LD_PRELOAD + CCCL_ALLOC_DISABLE env vars - Remove old mock files: cccl_preload_allocator.cu, build script, test - .dockerignore: exclude cccl_upstream/ upstream_ref/ vllm/ *.zip - verify_submission.sh: 31-point pre-submission check (file structure, CCCL chain, path matching, prebuilt integrity, corex imports, docker context, GPU smoke test)
273 lines
12 KiB
Bash
Executable File
273 lines
12 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# BI-V100 patch script for Qwen3.6-35B-A3B (Qwen3_5 MoE architecture)
|
|
#
|
|
# Triton situation on BI-V100:
|
|
# - Standard Triton 2.3.1 is already present in the image.
|
|
# - HAS_TRITON = False (hardcoded in vendor vllm), but Triton is still used
|
|
# for TP-mode cache management (custom_cache_manager / libentry).
|
|
# - The vendor's triton_utils/__init__.py, custom_cache_manager.py, libentry.py
|
|
# are already correct for standard Triton 2.3.1 — do NOT overwrite them.
|
|
# - DO NOT install BI-V150 corex Triton 2.1.0 (pkgs/triton): that causes
|
|
# GPU hang on BI-V100 because the Triton CUDA PTX kernels are incompatible.
|
|
|
|
# Recommended server start command for TP=4 support 256K, needs chunked prefill
|
|
# CUDA_VISIBLE_DEVICES="4,5,6,7" VLLM_ENGINE_ITERATION_TIMEOUT_S=3600 python3 -m vllm.entrypoints.openai.api_server \
|
|
# --model /workspace/models/Qwen3.6-35B-A3B --port 1111 --served-model-name llm \
|
|
# --max-model-len 262144 --trust-remote-code -tp 4 --gpu-memory-utilization 0.90 \
|
|
# --max-num-seqs 1 --disable-log-requests --disable-frontend-multiprocessing \
|
|
# --max-num-batched-tokens 8192 --enable-chunked-prefill --enable-prefix-caching \
|
|
# --max-seq-len-to-capture 32768 --enable-auto-tool-choice \
|
|
# --tool-call-parser qwen3_coder --reasoning-parser qwen3
|
|
#
|
|
# With prefix caching (GDN align-mode, requires chunked prefill):
|
|
# CUDA_VISIBLE_DEVICES="4,5,6,7" VLLM_ENGINE_ITERATION_TIMEOUT_S=3600 python3 -m vllm.entrypoints.openai.api_server \
|
|
# --model /workspace/models/Qwen3.6-35B-A3B --port 1111 --served-model-name llm \
|
|
# --max-model-len 262144 --trust-remote-code -tp 4 --gpu-memory-utilization 0.90 \
|
|
# --max-num-seqs 1 --disable-log-requests --disable-frontend-multiprocessing \
|
|
# --max-num-batched-tokens 8192 --enable-chunked-prefill --enable-prefix-caching \
|
|
# --max-seq-len-to-capture 32768 --enable-auto-tool-choice \
|
|
# --tool-call-parser qwen3_coder --reasoning-parser qwen3
|
|
|
|
set -eo pipefail
|
|
|
|
# cd into this script's directory so ./relative paths work
|
|
cd "$(dirname "${BASH_SOURCE[0]}")"
|
|
echo "[patch_ops] working directory: $(pwd)"
|
|
|
|
build_stage() { printf '[BI100 BUILD] %s\n' "$1" >&2; }
|
|
require_file() {
|
|
local path=$1
|
|
[[ -f "$path" ]] || {
|
|
printf 'required patch source is missing: %s\n' "$path" >&2
|
|
exit 2
|
|
}
|
|
}
|
|
install_patch_file() {
|
|
local source=$1
|
|
local target=$2
|
|
|
|
require_file "$source"
|
|
mkdir -p "$(dirname "$target")"
|
|
install -m 0644 "$source" "$target"
|
|
}
|
|
|
|
build_stage "patch script entered"
|
|
|
|
build_stage "checking offline transformers dependency"
|
|
# --- transformers: Qwen3_5 tokenizer / model files --------------------------
|
|
TRANSFORMERS_REQUIRED_VERSION="4.55.3"
|
|
if ! python3 - "$TRANSFORMERS_REQUIRED_VERSION" <<'PY'
|
|
import importlib.metadata
|
|
import sys
|
|
|
|
required = sys.argv[1]
|
|
try:
|
|
installed = importlib.metadata.version("transformers")
|
|
except importlib.metadata.PackageNotFoundError:
|
|
raise SystemExit(1)
|
|
raise SystemExit(0 if installed == required else 1)
|
|
PY
|
|
then
|
|
WHEEL_DIR="./wheels"
|
|
if ! ls "${WHEEL_DIR}/transformers-${TRANSFORMERS_REQUIRED_VERSION}"*.whl >/dev/null 2>&1; then
|
|
echo "transformers ${TRANSFORMERS_REQUIRED_VERSION} is required, but no offline wheel was found in ${WHEEL_DIR}" >&2
|
|
exit 2
|
|
fi
|
|
python3 -m pip install --no-index --no-deps --find-links="${WHEEL_DIR}" \
|
|
"transformers==${TRANSFORMERS_REQUIRED_VERSION}"
|
|
fi
|
|
|
|
python3 - "$TRANSFORMERS_REQUIRED_VERSION" <<'PY'
|
|
import importlib.metadata
|
|
import sys
|
|
|
|
required = sys.argv[1]
|
|
installed = importlib.metadata.version("transformers")
|
|
if installed != required:
|
|
raise SystemExit(
|
|
f"transformers version mismatch: expected {required}, got {installed}")
|
|
print(f"[ok] transformers {installed}")
|
|
PY
|
|
|
|
build_stage "discovering Python package roots"
|
|
python3 - <<'PY' > /tmp/qwen36_patch_paths.env
|
|
from patch_utils import package_root, shell_env_line
|
|
|
|
print(shell_env_line("VLLM_ROOT", package_root("vllm")))
|
|
print(shell_env_line("TRANSFORMERS_ROOT", package_root("transformers")))
|
|
PY
|
|
source /tmp/qwen36_patch_paths.env
|
|
|
|
echo "VLLM_ROOT=${VLLM_ROOT}"
|
|
echo "TRANSFORMERS_ROOT=${TRANSFORMERS_ROOT}"
|
|
[[ -d "$VLLM_ROOT" ]] || {
|
|
printf 'vLLM root does not exist: %s\n' "$VLLM_ROOT" >&2
|
|
exit 2
|
|
}
|
|
|
|
VLLM_OVERRIDE_ROOT="./vendor_overrides/vllm"
|
|
[[ -d "$VLLM_OVERRIDE_ROOT" ]] || {
|
|
printf 'vLLM override directory missing: %s\n' "$VLLM_OVERRIDE_ROOT" >&2
|
|
exit 2
|
|
}
|
|
|
|
build_stage "installing authoritative vLLM core block overrides"
|
|
install_patch_file \
|
|
"${VLLM_OVERRIDE_ROOT}/core/evictor_v2.py" \
|
|
"${VLLM_ROOT}/core/evictor_v2.py"
|
|
install_patch_file \
|
|
"${VLLM_OVERRIDE_ROOT}/core/block/cpu_kv_content_cache.py" \
|
|
"${VLLM_ROOT}/core/block/cpu_kv_content_cache.py"
|
|
install_patch_file \
|
|
"${VLLM_OVERRIDE_ROOT}/core/block/cpu_gpu_block_allocator.py" \
|
|
"${VLLM_ROOT}/core/block/cpu_gpu_block_allocator.py"
|
|
install_patch_file \
|
|
"${VLLM_OVERRIDE_ROOT}/core/block/prefix_caching_block.py" \
|
|
"${VLLM_ROOT}/core/block/prefix_caching_block.py"
|
|
install_patch_file \
|
|
"${VLLM_OVERRIDE_ROOT}/core/block/block_table.py" \
|
|
"${VLLM_ROOT}/core/block/block_table.py"
|
|
install_patch_file \
|
|
"${VLLM_OVERRIDE_ROOT}/core/block_manager_v2.py" \
|
|
"${VLLM_ROOT}/core/block_manager_v2.py"
|
|
install_patch_file \
|
|
"${VLLM_OVERRIDE_ROOT}/sampling_params.py" \
|
|
"${VLLM_ROOT}/sampling_params.py"
|
|
install_patch_file \
|
|
"${VLLM_OVERRIDE_ROOT}/model_executor/sampling_metadata.py" \
|
|
"${VLLM_ROOT}/model_executor/sampling_metadata.py"
|
|
install_patch_file \
|
|
"${VLLM_OVERRIDE_ROOT}/model_executor/layers/sampler.py" \
|
|
"${VLLM_ROOT}/model_executor/layers/sampler.py"
|
|
|
|
build_stage "installing hash-pinned CoreX 3.2.3 extensions"
|
|
bash ./install_prebuilt_corex.sh "${VLLM_ROOT}"
|
|
|
|
build_stage "installing BI100 runtime modules"
|
|
cp ./bi100_env.py "${VLLM_ROOT}/bi100_env.py"
|
|
cp ./bi100_profile.py "${VLLM_ROOT}/bi100_profile.py"
|
|
cp ./block_major_kv_cache.py "${VLLM_ROOT}/block_major_kv_cache.py"
|
|
cp ./gdn_prefix.py "${VLLM_ROOT}/gdn_prefix.py"
|
|
|
|
build_stage "installing CoreX paged-KV swap compatibility"
|
|
python3 ./patch_corex_swap_blocks.py
|
|
python3 ./patch_block_major_cache_engine.py
|
|
python3 ./patch_worker_cache_transfer_order.py
|
|
|
|
# --- paged_attn.py: replace forward_prefix with pure-PyTorch fallback -------
|
|
# The Triton context_attention_fwd kernel hangs BI-V100 GPUs permanently
|
|
# (standard Triton 2.3.1 PTX is not supported by the corex runtime either).
|
|
# Our paged_attn.py bypasses it entirely via _forward_prefix_pytorch, which
|
|
# utilizes K-tiling techniques, and also have _forward_decode_pytorch to bypass kernel
|
|
# when context length is high
|
|
cp ./paged_attn.py "${VLLM_ROOT}/attention/ops/paged_attn.py"
|
|
|
|
# --- model_runner.py: fix prefix_cache_hit stays True in chunked-prefill chunk 2+ ---
|
|
# Bug: _compute_for_prefix_cache_hit Case 1 (prefix_cache_len <= context_len)
|
|
# leaves prefix_cache_hit=True. Then _add_seq_group uses block_table=computed_block_nums
|
|
# (only the original prefix blocks), ignoring chunk-1 KV cache blocks.
|
|
# _forward_prefix_pytorch then gets an undersized block_tables and crashes with
|
|
# "amax(): Expected reduction dim -1 to have non-zero size" on the 2nd tile.
|
|
# Fix: set prefix_cache_hit=False for Case 1 so the full block_tables is used.
|
|
python3 ./patch_model_runner.py
|
|
|
|
build_stage "installing executor startup diagnostics"
|
|
python3 ./patch_executor_startup_debug.py
|
|
python3 ./patch_worker_startup_profile_guard.py
|
|
python3 ./patch_block_major_worker_capacity.py
|
|
|
|
build_stage "installing transformers Qwen3.5 model support"
|
|
cp -r ./qwen3_5 "${TRANSFORMERS_ROOT}/models/"
|
|
cp -r ./qwen3_5_moe "${TRANSFORMERS_ROOT}/models/"
|
|
python3 ./patch_transformers_qwen3_5.py
|
|
|
|
build_stage "installing vLLM Qwen3.6 model implementation"
|
|
# --- vllm model: Qwen3.6-35B-A3B (Qwen3_5 MoE arch) -------------------------
|
|
cp ./mamba_cache.py "${VLLM_ROOT}/model_executor/models/"
|
|
cp ./qwen3_5.py "${VLLM_ROOT}/model_executor/models/qwen3_5.py"
|
|
python3 ./patch_vllm_qwen3_5.py
|
|
|
|
# --- sequence.py: fix completion_tokens inflation under chunked prefill ------
|
|
# Bug: get_output_token_ids_to_return(delta=True) with num_new_tokens=0
|
|
# returns _cached_all_token_ids[-0:] == [0:] (the ENTIRE prompt+output list).
|
|
# Each prefill chunk step adds prompt_len to previous_num_tokens, so a 10K
|
|
# prompt processed in 3 chunks inflates completion_tokens by ~30K.
|
|
# Also adds num_cached_tokens field to RequestMetrics for prefix-cache stats.
|
|
cp ./sequence.py "${VLLM_ROOT}/sequence.py"
|
|
|
|
# --- scheduler.py: record num_cached_tokens in RequestMetrics ----------------
|
|
# Reports only the longest prefix backed by both live KV blocks and an exact
|
|
# GDN restore state. Raw KV-only hits must not inflate cached_tokens.
|
|
# serving_chat.py exposes the value in the OpenAI-compatible usage details.
|
|
cp ./scheduler.py "${VLLM_ROOT}/core/scheduler.py"
|
|
|
|
build_stage "installing diagnostic initial allocation trace"
|
|
python3 ./patch_block_manager_cache_trace.py
|
|
|
|
build_stage "installing scheduler and attention patches"
|
|
# --- xformers: bypass cudnnFlashAttnForward (head_dim=256 > 128 limit) ------
|
|
# Injects _run_sdpa_fallback (pure matmul+softmax) into xformers.py.
|
|
# Required because head_dim=256 > 128 and ixformer flash attention either
|
|
# crashes (is_causal=True) or produces wrong output (attn_mask path).
|
|
# The fallback uses query_start_loc to derive actual query lengths, so it
|
|
# works correctly during profiling runs with chunked-prefill-style batches.
|
|
# also bypasses auto chunked prefill on
|
|
python3 ./patch_xformers_sdpa_seq.py
|
|
python3 ./patch_xformers_profile.py
|
|
|
|
build_stage "installing API parsers and serving modules"
|
|
# --- tool parser: Qwen3 XML tool call format ---------------------------------
|
|
# Registers "qwen3_coder" parser for Qwen3.6 XML-style tool calls:
|
|
# <tool_call><function=name><parameter=key>\nvalue\n</parameter></function></tool_call>
|
|
# Use at server start: --tool-call-parser qwen3_coder --enable-auto-tool-choice
|
|
cp ./qwen3coder_tool_parser.py "${VLLM_ROOT}/entrypoints/openai/tool_parsers/"
|
|
python3 ./patch_vllm_tool_parser.py
|
|
|
|
# --- reasoning parser: Qwen3 <think>...</think> split ------------------------
|
|
# Adds --reasoning-parser qwen3 support.
|
|
# Routes thinking tokens to reasoning_content, rest to content in the delta.
|
|
# Works together with --tool-call-parser qwen3_coder (think → tool call flow).
|
|
cp -r ./reasoning "${VLLM_ROOT}/"
|
|
cp ./protocol.py "${VLLM_ROOT}/entrypoints/openai/protocol.py"
|
|
cp ./cli_args.py "${VLLM_ROOT}/entrypoints/openai/cli_args.py"
|
|
cp ./serving_chat.py "${VLLM_ROOT}/entrypoints/openai/serving_chat.py"
|
|
cp ./serving_tokenization.py \
|
|
"${VLLM_ROOT}/entrypoints/openai/serving_tokenization.py"
|
|
cp ./api_server.py "${VLLM_ROOT}/entrypoints/openai/api_server.py"
|
|
cp ./chat_utils.py "${VLLM_ROOT}/entrypoints/chat_utils.py"
|
|
python3 - ./api_server.py \
|
|
"${VLLM_ROOT}/entrypoints/openai/api_server.py" <<'PY'
|
|
from pathlib import Path
|
|
import sys
|
|
|
|
source = Path(sys.argv[1]).read_bytes()
|
|
installed = Path(sys.argv[2]).read_bytes()
|
|
if source != installed:
|
|
raise SystemExit("runtime api_server overlay identity mismatch")
|
|
PY
|
|
|
|
build_stage "compiling CCCL CachingDeviceAllocator LD_PRELOAD module"
|
|
if bash ./cccl_preload/build_cccl_preload.sh /workspace/qwen3_6_scripts; then
|
|
if [[ -s /workspace/qwen3_6_scripts/libcccl_allocator.so ]]; then
|
|
echo "[OK] libcccl_allocator.so built successfully"
|
|
else
|
|
echo "[WARN] libcccl_allocator.so is empty or missing after build"
|
|
fi
|
|
else
|
|
echo "[WARN] CCCL preload allocator build failed — LD_PRELOAD will be ignored at runtime"
|
|
fi
|
|
|
|
build_stage "compiling CoreX CUDA extensions (moe_index_combine + gdn_chunk_recurrent)"
|
|
if [[ -x /usr/local/corex-3.2.3/bin/clang++ ]]; then
|
|
bash ./build_corex_moe_index_combine.sh "${VLLM_ROOT}" || \
|
|
echo "[WARN] moe_index_combine build failed — will use PyTorch fallback"
|
|
bash ./build_corex_gdn_chunk_recurrent.sh "${VLLM_ROOT}" || \
|
|
echo "[WARN] gdn_chunk_recurrent build failed — will use Python fallback"
|
|
else
|
|
echo "[WARN] corex clang++ not found — skipping extension builds"
|
|
fi
|
|
|
|
build_stage "compiling submission Python sources"
|
|
find . -path './wheels' -prune -o -name '*.py' -print0 | xargs -0 python3 -m py_compile
|
|
build_stage "patch script completed"
|