diff --git a/Dockerfile b/Dockerfile index f0d73a7f..f9af72bf 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,25 +8,44 @@ COPY ./qwen3_6_scripts /workspace/qwen3_6_scripts COPY ./computility-run.yaml /workspace/computility-run.yaml COPY ./ex_engine /workspace/ex_engine -# Step 1: Build EX Engine .so libraries +# Step 1: Build EX Engine .so libraries (tolerant) RUN chmod +x /workspace/ex_engine/build.sh && \ - bash /workspace/ex_engine/build.sh --corex 2>&1 | tee /workspace/ex_build.log ; \ - echo "[Dockerfile] ex_engine build exit code: $?" + (bash /workspace/ex_engine/build.sh --corex 2>&1 || true) | tee /workspace/ex_build.log ; \ + echo "[Dockerfile] ex_engine build done" -# Step 2: Precompile MoE CUDA kernels -RUN python3 /workspace/ex_engine/precompile_moe_topk.py 2>&1 | tee -a /workspace/ex_build.log ; \ - echo "[Dockerfile] moe_topk precompile exit code: $?" +# Step 2: Precompile MoE CUDA kernels (tolerant) +RUN (python3 /workspace/ex_engine/precompile_moe_topk.py 2>&1 || true) | tee -a /workspace/ex_build.log ; \ + echo "[Dockerfile] moe_topk precompile done" -# Step 3: Precompile vllm v0.5.5 MoE kernels -RUN python3 /workspace/ex_engine/precompile_moe_kernels.py 2>&1 | tee -a /workspace/ex_build.log ; \ - echo "[Dockerfile] moe_v055 precompile exit code: $?" +# Step 3: Precompile vllm v0.5.5 MoE kernels (tolerant) +RUN (python3 /workspace/ex_engine/precompile_moe_kernels.py 2>&1 || true) | tee -a /workspace/ex_build.log ; \ + echo "[Dockerfile] moe_v055 precompile done" -# Step 4: Deploy patches (serving + engine fixes) +# Step 4: Deploy patches (serving + engine fixes) — tolerant RUN chmod +x /workspace/qwen3_6_scripts/patch_ops.sh && \ - bash /workspace/qwen3_6_scripts/patch_ops.sh 2>&1 | tee /workspace/patch_ops.log ; \ - echo "[Dockerfile] patch_ops exit code: $?" + (cd /workspace/qwen3_6_scripts && bash ./patch_ops.sh 2>&1 || true) | tee /workspace/patch_ops.log ; \ + echo "[Dockerfile] patch_ops done" -# Step 5: Precompile GDN kernel (needs vllm in path, so after patch_ops) -RUN python3 /workspace/qwen3_6_scripts/precompile_gdn.py \ - /workspace/qwen3_6_scripts/flash_qla_sm70 2>&1 | tee -a /workspace/ex_build.log ; \ - echo "[Dockerfile] gdn precompile exit code: $?" +# Step 5: Build ix_unified_bridge.so (tolerant — ixformer symbols resolved at runtime) +RUN if [ -f /workspace/ex_engine/build_unified_bridge.sh ]; then \ + chmod +x /workspace/ex_engine/build_unified_bridge.sh && \ + (bash /workspace/ex_engine/build_unified_bridge.sh 2>&1 || true) | tee -a /workspace/ex_build.log ; \ + fi ; \ + echo "[Dockerfile] bridge build done" + +# Step 6: Deploy ix_unified Python modules to vllm path (tolerant) +RUN VLLM_ROOT="$(python3 -c 'import vllm; print(vllm.__path__[0])' 2>/dev/null | grep -v '^INFO\|^WARNING\|^DEBUG' | tail -1)" && \ + [ -z "$VLLM_ROOT" ] && VLLM_ROOT="/usr/local/corex/lib64/python3/dist-packages/vllm" ; \ + echo "[Dockerfile] VLLM_ROOT=${VLLM_ROOT}" && \ + for f in ix_unified.py corex_so_loader.py moe_fused_dispatch.py corex_moe.py; do \ + [ -f "/workspace/ex_engine/python/$f" ] && cp "/workspace/ex_engine/python/$f" "${VLLM_ROOT}/$f" 2>/dev/null || true ; \ + done ; \ + for _so in /workspace/ex_engine/build/ix_unified_bridge*.so; do \ + [ -f "$_so" ] && cp "$_so" "${VLLM_ROOT}/" 2>/dev/null || true ; \ + done ; \ + echo "[Dockerfile] bridge deploy done" + +# Step 7: Precompile GDN kernel (tolerant) +RUN (python3 /workspace/qwen3_6_scripts/precompile_gdn.py \ + /workspace/qwen3_6_scripts/flash_qla_sm70 2>&1 || true) | tee -a /workspace/ex_build.log ; \ + echo "[Dockerfile] gdn precompile done" diff --git a/diagnose_build.sh b/diagnose_build.sh new file mode 100644 index 00000000..a7bde60e --- /dev/null +++ b/diagnose_build.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +# Run this on the real machine to simulate Docker build steps and find failures. +# Usage: bash diagnose_build.sh + +set +e # Don't exit on errors + +echo "=== STEP 1: ex_engine build.sh ===" +cd /home/dylan/project_6 +chmod +x ex_engine/build.sh +bash ex_engine/build.sh --corex 2>&1 | tail -10 +echo "EXIT: $?" + +echo "" +echo "=== STEP 2: precompile_moe_topk ===" +python3 ex_engine/precompile_moe_topk.py 2>&1 | tail -10 +echo "EXIT: $?" + +echo "" +echo "=== STEP 3: precompile_moe_kernels ===" +python3 ex_engine/precompile_moe_kernels.py 2>&1 | tail -10 +echo "EXIT: $?" + +echo "" +echo "=== STEP 4: patch_ops.sh ===" +cd qwen3_6_scripts +chmod +x patch_ops.sh +bash patch_ops.sh 2>&1 | tail -20 +echo "EXIT: $?" + +echo "" +echo "=== STEP 5: precompile_gdn ===" +cd /home/dylan/project_6 +python3 qwen3_6_scripts/precompile_gdn.py qwen3_6_scripts/flash_qla_sm70 2>&1 | tail -10 +echo "EXIT: $?" + +echo "" +echo "=== STEP 6: Test qwen3_5.py import ===" +python3 -c " +import sys +sys.path.insert(0, '/usr/local/corex/lib64/python3/dist-packages') +sys.path.insert(0, '/usr/local/corex/lib/python3/dist-packages') +try: + # This is what happens at runtime when vllm loads the model + exec(open('/home/dylan/project_6/qwen3_6_scripts/qwen3_5.py').read()) + print('IMPORT OK') +except Exception as e: + print(f'IMPORT FAIL: {type(e).__name__}: {e}') +" 2>&1 | tail -10 +echo "EXIT: $?" + +echo "" +echo "=== DONE ===" diff --git a/qwen3_6_scripts/patch_ops.sh b/qwen3_6_scripts/patch_ops.sh index 6d9dcd9b..b52ff940 100755 --- a/qwen3_6_scripts/patch_ops.sh +++ b/qwen3_6_scripts/patch_ops.sh @@ -31,13 +31,13 @@ # NOTE: intentionally NO set -e — individual patch failures must NOT abort # the entire build. Each step logs its own errors, and non-critical patches # (xformers, diagnostics) may legitimately fail if the base image differs. -set -uo pipefail +set -o pipefail # Always cd to script directory so relative paths (./qwen3_5.py, ./vendor_overrides, etc) work cd "$(dirname "$0")" -build_stage "patch_ops.sh running from $(pwd)" build_stage() { printf '[BI100 BUILD] %s\n' "$1" >&2; } +build_stage "patch_ops.sh running from $(pwd)" require_file() { local path=$1 [[ -f "$path" ]] || {