From 36e67c00b0097a1846d78231d34345c5b5196e3d Mon Sep 17 00:00:00 2001 From: dylanyunlon Date: Fri, 7 Aug 2026 04:51:27 +0000 Subject: [PATCH] fix(critical): deploy patches to BOTH lib and lib64 vllm paths Job 103 failed with: 'unrecognized arguments: --reasoning-parser qwen3' Root cause: patch_ops.sh only deployed to one vllm path (lib OR lib64), but Python loaded vllm from the OTHER path where patches were missing. Fix: deploy() helper copies every file to ALL existing vllm roots. Both /usr/local/corex/lib/python3/dist-packages/vllm/ and /usr/local/corex/lib64/python3/dist-packages/vllm/ get patched. CCCL dispatch_common.cuh principle: dispatch must handle ALL paths, not just the first matching one. Same logic: patch ALL install locations. --- qwen3_6_scripts/patch_ops.sh | 113 ++++++++++++++++------------------- 1 file changed, 53 insertions(+), 60 deletions(-) diff --git a/qwen3_6_scripts/patch_ops.sh b/qwen3_6_scripts/patch_ops.sh index f23f553f..01e5f8b4 100755 --- a/qwen3_6_scripts/patch_ops.sh +++ b/qwen3_6_scripts/patch_ops.sh @@ -11,17 +11,33 @@ VLLM=/usr/local/corex/lib/python3/dist-packages/vllm VLLM64=/usr/local/corex/lib64/python3/dist-packages/vllm -# Detect which lib path exists +# Deploy to ALL existing vllm paths — Python may load from either one +# depending on PYTHONPATH ordering and namespace package resolution. +TARGETS=() if [ -d "$VLLM" ]; then - V=$VLLM -elif [ -d "$VLLM64" ]; then - V=$VLLM64 -else + TARGETS+=("$VLLM") +fi +if [ -d "$VLLM64" ]; then + TARGETS+=("$VLLM64") +fi + +if [ ${#TARGETS[@]} -eq 0 ]; then echo "[patch_ops] ERROR: vllm not found at lib or lib64 path" exit 1 fi -echo "[patch_ops] vllm path: $V" +echo "[patch_ops] vllm paths found: ${TARGETS[*]}" + +# Helper: copy file to all target vllm roots +deploy() { + local src="$1" + local rel_dst="$2" # relative path within vllm, e.g. "attention/ops/paged_attn.py" + for V in "${TARGETS[@]}"; do + local dst="$V/$rel_dst" + mkdir -p "$(dirname "$dst")" + cp "$src" "$dst" + done +} # --- _custom_ops.py: SMEM 48KB fix + hardware ops bindings ------------------- # Base image returns 32KB (32768) for get_max_shared_memory_per_block, but @@ -31,59 +47,35 @@ echo "[patch_ops] vllm path: $V" # work distribution depends on correct hardware parameters — wrong SMEM # means wrong tile_size means wrong grid_size. # FULL FILE REPLACEMENT. -cp ./_custom_ops.py $V/_custom_ops.py +deploy ./_custom_ops.py "_custom_ops.py" echo "[patch_ops] _custom_ops.py → / (SMEM 32KB→48KB fix)" # --- paged_attn.py: pure-PyTorch attention fallback -------------------------- -# Bypasses Triton context_attention_fwd (hangs BI-V100 permanently). -# Uses K-tiling Flash Attention online softmax for prefix attention. -# Uses pure-PyTorch decode for seq_len > 32K. -# CCCL-ported: adaptive tile sizing from dispatch_reduce.cuh GridEvenShare. -cp ./paged_attn.py $V/attention/ops/paged_attn.py +deploy ./paged_attn.py "attention/ops/paged_attn.py" echo "[patch_ops] paged_attn.py → attention/ops/" # --- prefix_prefill.py: Triton-free prefix attention ------------------------- -# On BI-V100, Triton is not installed. This file provides the context_attention_fwd -# function that paged_attn.py imports. Even though our paged_attn.py comments out -# the Triton import, the base xformers.py may still try to import it. -# Deploy it so the import doesn't crash — the function itself won't be called. -cp ./prefix_prefill.py $V/attention/ops/prefix_prefill.py +deploy ./prefix_prefill.py "attention/ops/prefix_prefill.py" echo "[patch_ops] prefix_prefill.py → attention/ops/" # --- model_runner.py: prefix_cache_hit fix ----------------------------------- -# Bug: Case 1 (prefix_cache_len <= context_len) leaves prefix_cache_hit=True, -# causing undersized block_tables in chunked prefill chunk 2+. -# Fix: set prefix_cache_hit=False for Case 1. -# FULL FILE REPLACEMENT — no patch_model_runner.py script. -cp ./model_runner.py $V/worker/model_runner.py +deploy ./model_runner.py "worker/model_runner.py" echo "[patch_ops] model_runner.py → worker/" # --- xformers.py: head_dim>128 fallback + Q-tiling -------------------------- -# Injects _run_sdpa_fallback (pure matmul+softmax) for head_dim=256. -# ixformer flash attention crashes (is_causal=True) or gives wrong output. -# Also disables auto chunked-prefill (Q-tiling handles long context). -# FULL FILE REPLACEMENT — no patch_xformers_sdpa_seq.py script. -cp ./xformers.py $V/attention/backends/xformers.py +deploy ./xformers.py "attention/backends/xformers.py" echo "[patch_ops] xformers.py → attention/backends/" # --- arg_utils.py: disable auto chunked-prefill for 32K+ -------------------- -# Q-tiling in _run_sdpa_fallback handles long-context memory. -# FULL FILE REPLACEMENT. -cp ./arg_utils.py $V/engine/arg_utils.py +deploy ./arg_utils.py "engine/arg_utils.py" echo "[patch_ops] arg_utils.py → engine/" # --- logits_processor.py: seq_groups=None guard ------------------------------ -# Prevents crash when seq_groups is None during intermediate chunked-prefill. -# FULL FILE REPLACEMENT. -cp ./logits_processor.py $V/model_executor/layers/logits_processor.py +deploy ./logits_processor.py "model_executor/layers/logits_processor.py" echo "[patch_ops] logits_processor.py → model_executor/layers/" # --- sampler.py: CCCL-ported top-k fast path for sampling -------------------- -# When all sequences use top_p=1.0, skip full sort+cumsum and use torch.topk. -# CCCL partition/flagged.cu insight: radix select is O(N×bits_per_pass) vs -# full sort O(N log N). For vocab=152064: topk ~11 passes vs sort ~17 passes. -# FULL FILE REPLACEMENT. -cp ./sampler.py $V/model_executor/layers/sampler.py +deploy ./sampler.py "model_executor/layers/sampler.py" echo "[patch_ops] sampler.py → model_executor/layers/" # --- transformers: Qwen3_5 tokenizer / model files -------------------------- @@ -100,43 +92,44 @@ python3 ./patch_transformers_qwen3_5.py echo "[patch_ops] transformers Qwen3_5 models installed" # --- vllm model: Qwen3.6 (Qwen3_5 arch) ------------------------------------ -# FULL FILE REPLACEMENT of registry.py with Qwen3_5 entries pre-added. -# No more patch_vllm_qwen3_5.py script. -cp ./mamba_cache.py $V/model_executor/models/ -cp ./qwen3_5.py $V/model_executor/models/qwen3_5.py -cp ./registry.py $V/model_executor/models/registry.py +for V in "${TARGETS[@]}"; do + cp ./mamba_cache.py "$V/model_executor/models/" +done +deploy ./qwen3_5.py "model_executor/models/qwen3_5.py" +deploy ./registry.py "model_executor/models/registry.py" echo "[patch_ops] qwen3_5.py + registry.py deployed" # --- paged_attention_v2_pytorch.py: PyTorch V2 attention fallback ------------ -# _custom_ops.py imports this from the parent of its own directory. -# In docker, vllm lives at $V/, so we place it one level up AND next to _custom_ops. -# Belt-and-suspenders: also copy to /workspace/ where _custom_ops.py's _repo_root points. -cp ./paged_attention_v2_pytorch.py $V/paged_attention_v2_pytorch.py +for V in "${TARGETS[@]}"; do + cp ./paged_attention_v2_pytorch.py "$V/paged_attention_v2_pytorch.py" +done cp ./paged_attention_v2_pytorch.py /workspace/paged_attention_v2_pytorch.py -echo "[patch_ops] paged_attention_v2_pytorch.py → $V/ + /workspace/" +echo "[patch_ops] paged_attention_v2_pytorch.py → all paths + /workspace/" # --- sequence.py: fix completion_tokens inflation ---------------------------- -cp ./sequence.py $V/sequence.py +deploy ./sequence.py "sequence.py" echo "[patch_ops] sequence.py → /" # --- scheduler.py: record num_cached_tokens --------------------------------- -cp ./scheduler.py $V/core/scheduler.py +deploy ./scheduler.py "core/scheduler.py" echo "[patch_ops] scheduler.py → core/" # --- tool parser: Qwen3 XML tool call format -------------------------------- -# FULL FILE REPLACEMENT of __init__.py with Qwen3CoderToolParser pre-added. -# No more patch_vllm_tool_parser.py script. -cp ./qwen3coder_tool_parser.py $V/entrypoints/openai/tool_parsers/ -cp ./tool_parsers_init.py $V/entrypoints/openai/tool_parsers/__init__.py +for V in "${TARGETS[@]}"; do + cp ./qwen3coder_tool_parser.py "$V/entrypoints/openai/tool_parsers/" + cp ./tool_parsers_init.py "$V/entrypoints/openai/tool_parsers/__init__.py" +done echo "[patch_ops] qwen3_coder tool parser deployed" # --- reasoning parser: Qwen3 ... split ----------------------- -cp -r ./reasoning $V/ -cp ./protocol.py $V/entrypoints/openai/protocol.py -cp ./cli_args.py $V/entrypoints/openai/cli_args.py -cp ./serving_chat.py $V/entrypoints/openai/serving_chat.py -cp ./api_server.py $V/entrypoints/openai/api_server.py -cp ./chat_utils.py $V/entrypoints/chat_utils.py +for V in "${TARGETS[@]}"; do + cp -r ./reasoning "$V/" + cp ./protocol.py "$V/entrypoints/openai/protocol.py" + cp ./cli_args.py "$V/entrypoints/openai/cli_args.py" + cp ./serving_chat.py "$V/entrypoints/openai/serving_chat.py" + cp ./api_server.py "$V/entrypoints/openai/api_server.py" + cp ./chat_utils.py "$V/entrypoints/chat_utils.py" +done echo "[patch_ops] reasoning parser + serving files installed" echo "[patch_ops] DONE — all patches applied via full file replacement"