fix(critical): deploy patches to BOTH lib and lib64 vllm paths
Job 103 failed with: 'unrecognized arguments: --reasoning-parser qwen3' Root cause: patch_ops.sh only deployed to one vllm path (lib OR lib64), but Python loaded vllm from the OTHER path where patches were missing. Fix: deploy() helper copies every file to ALL existing vllm roots. Both /usr/local/corex/lib/python3/dist-packages/vllm/ and /usr/local/corex/lib64/python3/dist-packages/vllm/ get patched. CCCL dispatch_common.cuh principle: dispatch must handle ALL paths, not just the first matching one. Same logic: patch ALL install locations.
This commit is contained in:
dylanyunlon
@@ -11,17 +11,33 @@
|
||||
VLLM=/usr/local/corex/lib/python3/dist-packages/vllm
|
||||
VLLM64=/usr/local/corex/lib64/python3/dist-packages/vllm
|
||||
|
||||
# Detect which lib path exists
|
||||
# Deploy to ALL existing vllm paths — Python may load from either one
|
||||
# depending on PYTHONPATH ordering and namespace package resolution.
|
||||
TARGETS=()
|
||||
if [ -d "$VLLM" ]; then
|
||||
V=$VLLM
|
||||
elif [ -d "$VLLM64" ]; then
|
||||
V=$VLLM64
|
||||
else
|
||||
TARGETS+=("$VLLM")
|
||||
fi
|
||||
if [ -d "$VLLM64" ]; then
|
||||
TARGETS+=("$VLLM64")
|
||||
fi
|
||||
|
||||
if [ ${#TARGETS[@]} -eq 0 ]; then
|
||||
echo "[patch_ops] ERROR: vllm not found at lib or lib64 path"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "[patch_ops] vllm path: $V"
|
||||
echo "[patch_ops] vllm paths found: ${TARGETS[*]}"
|
||||
|
||||
# Helper: copy file to all target vllm roots
|
||||
deploy() {
|
||||
local src="$1"
|
||||
local rel_dst="$2" # relative path within vllm, e.g. "attention/ops/paged_attn.py"
|
||||
for V in "${TARGETS[@]}"; do
|
||||
local dst="$V/$rel_dst"
|
||||
mkdir -p "$(dirname "$dst")"
|
||||
cp "$src" "$dst"
|
||||
done
|
||||
}
|
||||
|
||||
# --- _custom_ops.py: SMEM 48KB fix + hardware ops bindings -------------------
|
||||
# Base image returns 32KB (32768) for get_max_shared_memory_per_block, but
|
||||
@@ -31,59 +47,35 @@ echo "[patch_ops] vllm path: $V"
|
||||
# work distribution depends on correct hardware parameters — wrong SMEM
|
||||
# means wrong tile_size means wrong grid_size.
|
||||
# FULL FILE REPLACEMENT.
|
||||
cp ./_custom_ops.py $V/_custom_ops.py
|
||||
deploy ./_custom_ops.py "_custom_ops.py"
|
||||
echo "[patch_ops] _custom_ops.py → / (SMEM 32KB→48KB fix)"
|
||||
|
||||
# --- paged_attn.py: pure-PyTorch attention fallback --------------------------
|
||||
# Bypasses Triton context_attention_fwd (hangs BI-V100 permanently).
|
||||
# Uses K-tiling Flash Attention online softmax for prefix attention.
|
||||
# Uses pure-PyTorch decode for seq_len > 32K.
|
||||
# CCCL-ported: adaptive tile sizing from dispatch_reduce.cuh GridEvenShare.
|
||||
cp ./paged_attn.py $V/attention/ops/paged_attn.py
|
||||
deploy ./paged_attn.py "attention/ops/paged_attn.py"
|
||||
echo "[patch_ops] paged_attn.py → attention/ops/"
|
||||
|
||||
# --- prefix_prefill.py: Triton-free prefix attention -------------------------
|
||||
# On BI-V100, Triton is not installed. This file provides the context_attention_fwd
|
||||
# function that paged_attn.py imports. Even though our paged_attn.py comments out
|
||||
# the Triton import, the base xformers.py may still try to import it.
|
||||
# Deploy it so the import doesn't crash — the function itself won't be called.
|
||||
cp ./prefix_prefill.py $V/attention/ops/prefix_prefill.py
|
||||
deploy ./prefix_prefill.py "attention/ops/prefix_prefill.py"
|
||||
echo "[patch_ops] prefix_prefill.py → attention/ops/"
|
||||
|
||||
# --- model_runner.py: prefix_cache_hit fix -----------------------------------
|
||||
# Bug: Case 1 (prefix_cache_len <= context_len) leaves prefix_cache_hit=True,
|
||||
# causing undersized block_tables in chunked prefill chunk 2+.
|
||||
# Fix: set prefix_cache_hit=False for Case 1.
|
||||
# FULL FILE REPLACEMENT — no patch_model_runner.py script.
|
||||
cp ./model_runner.py $V/worker/model_runner.py
|
||||
deploy ./model_runner.py "worker/model_runner.py"
|
||||
echo "[patch_ops] model_runner.py → worker/"
|
||||
|
||||
# --- xformers.py: head_dim>128 fallback + Q-tiling --------------------------
|
||||
# Injects _run_sdpa_fallback (pure matmul+softmax) for head_dim=256.
|
||||
# ixformer flash attention crashes (is_causal=True) or gives wrong output.
|
||||
# Also disables auto chunked-prefill (Q-tiling handles long context).
|
||||
# FULL FILE REPLACEMENT — no patch_xformers_sdpa_seq.py script.
|
||||
cp ./xformers.py $V/attention/backends/xformers.py
|
||||
deploy ./xformers.py "attention/backends/xformers.py"
|
||||
echo "[patch_ops] xformers.py → attention/backends/"
|
||||
|
||||
# --- arg_utils.py: disable auto chunked-prefill for 32K+ --------------------
|
||||
# Q-tiling in _run_sdpa_fallback handles long-context memory.
|
||||
# FULL FILE REPLACEMENT.
|
||||
cp ./arg_utils.py $V/engine/arg_utils.py
|
||||
deploy ./arg_utils.py "engine/arg_utils.py"
|
||||
echo "[patch_ops] arg_utils.py → engine/"
|
||||
|
||||
# --- logits_processor.py: seq_groups=None guard ------------------------------
|
||||
# Prevents crash when seq_groups is None during intermediate chunked-prefill.
|
||||
# FULL FILE REPLACEMENT.
|
||||
cp ./logits_processor.py $V/model_executor/layers/logits_processor.py
|
||||
deploy ./logits_processor.py "model_executor/layers/logits_processor.py"
|
||||
echo "[patch_ops] logits_processor.py → model_executor/layers/"
|
||||
|
||||
# --- sampler.py: CCCL-ported top-k fast path for sampling --------------------
|
||||
# When all sequences use top_p=1.0, skip full sort+cumsum and use torch.topk.
|
||||
# CCCL partition/flagged.cu insight: radix select is O(N×bits_per_pass) vs
|
||||
# full sort O(N log N). For vocab=152064: topk ~11 passes vs sort ~17 passes.
|
||||
# FULL FILE REPLACEMENT.
|
||||
cp ./sampler.py $V/model_executor/layers/sampler.py
|
||||
deploy ./sampler.py "model_executor/layers/sampler.py"
|
||||
echo "[patch_ops] sampler.py → model_executor/layers/"
|
||||
|
||||
# --- transformers: Qwen3_5 tokenizer / model files --------------------------
|
||||
@@ -100,43 +92,44 @@ python3 ./patch_transformers_qwen3_5.py
|
||||
echo "[patch_ops] transformers Qwen3_5 models installed"
|
||||
|
||||
# --- vllm model: Qwen3.6 (Qwen3_5 arch) ------------------------------------
|
||||
# FULL FILE REPLACEMENT of registry.py with Qwen3_5 entries pre-added.
|
||||
# No more patch_vllm_qwen3_5.py script.
|
||||
cp ./mamba_cache.py $V/model_executor/models/
|
||||
cp ./qwen3_5.py $V/model_executor/models/qwen3_5.py
|
||||
cp ./registry.py $V/model_executor/models/registry.py
|
||||
for V in "${TARGETS[@]}"; do
|
||||
cp ./mamba_cache.py "$V/model_executor/models/"
|
||||
done
|
||||
deploy ./qwen3_5.py "model_executor/models/qwen3_5.py"
|
||||
deploy ./registry.py "model_executor/models/registry.py"
|
||||
echo "[patch_ops] qwen3_5.py + registry.py deployed"
|
||||
|
||||
# --- paged_attention_v2_pytorch.py: PyTorch V2 attention fallback ------------
|
||||
# _custom_ops.py imports this from the parent of its own directory.
|
||||
# In docker, vllm lives at $V/, so we place it one level up AND next to _custom_ops.
|
||||
# Belt-and-suspenders: also copy to /workspace/ where _custom_ops.py's _repo_root points.
|
||||
cp ./paged_attention_v2_pytorch.py $V/paged_attention_v2_pytorch.py
|
||||
for V in "${TARGETS[@]}"; do
|
||||
cp ./paged_attention_v2_pytorch.py "$V/paged_attention_v2_pytorch.py"
|
||||
done
|
||||
cp ./paged_attention_v2_pytorch.py /workspace/paged_attention_v2_pytorch.py
|
||||
echo "[patch_ops] paged_attention_v2_pytorch.py → $V/ + /workspace/"
|
||||
echo "[patch_ops] paged_attention_v2_pytorch.py → all paths + /workspace/"
|
||||
|
||||
# --- sequence.py: fix completion_tokens inflation ----------------------------
|
||||
cp ./sequence.py $V/sequence.py
|
||||
deploy ./sequence.py "sequence.py"
|
||||
echo "[patch_ops] sequence.py → /"
|
||||
|
||||
# --- scheduler.py: record num_cached_tokens ---------------------------------
|
||||
cp ./scheduler.py $V/core/scheduler.py
|
||||
deploy ./scheduler.py "core/scheduler.py"
|
||||
echo "[patch_ops] scheduler.py → core/"
|
||||
|
||||
# --- tool parser: Qwen3 XML tool call format --------------------------------
|
||||
# FULL FILE REPLACEMENT of __init__.py with Qwen3CoderToolParser pre-added.
|
||||
# No more patch_vllm_tool_parser.py script.
|
||||
cp ./qwen3coder_tool_parser.py $V/entrypoints/openai/tool_parsers/
|
||||
cp ./tool_parsers_init.py $V/entrypoints/openai/tool_parsers/__init__.py
|
||||
for V in "${TARGETS[@]}"; do
|
||||
cp ./qwen3coder_tool_parser.py "$V/entrypoints/openai/tool_parsers/"
|
||||
cp ./tool_parsers_init.py "$V/entrypoints/openai/tool_parsers/__init__.py"
|
||||
done
|
||||
echo "[patch_ops] qwen3_coder tool parser deployed"
|
||||
|
||||
# --- reasoning parser: Qwen3 <think>...</think> split -----------------------
|
||||
cp -r ./reasoning $V/
|
||||
cp ./protocol.py $V/entrypoints/openai/protocol.py
|
||||
cp ./cli_args.py $V/entrypoints/openai/cli_args.py
|
||||
cp ./serving_chat.py $V/entrypoints/openai/serving_chat.py
|
||||
cp ./api_server.py $V/entrypoints/openai/api_server.py
|
||||
cp ./chat_utils.py $V/entrypoints/chat_utils.py
|
||||
for V in "${TARGETS[@]}"; do
|
||||
cp -r ./reasoning "$V/"
|
||||
cp ./protocol.py "$V/entrypoints/openai/protocol.py"
|
||||
cp ./cli_args.py "$V/entrypoints/openai/cli_args.py"
|
||||
cp ./serving_chat.py "$V/entrypoints/openai/serving_chat.py"
|
||||
cp ./api_server.py "$V/entrypoints/openai/api_server.py"
|
||||
cp ./chat_utils.py "$V/entrypoints/chat_utils.py"
|
||||
done
|
||||
echo "[patch_ops] reasoning parser + serving files installed"
|
||||
|
||||
echo "[patch_ops] DONE — all patches applied via full file replacement"
|
||||
|
||||
Reference in New Issue
Block a user