fix(build): add .dockerignore + safe probe — fix docker build failure

Build was failing, likely due to:
1. 165MB build context (no .dockerignore) — cccl_upstream/ 53MB, zip 97MB
2. probe_corex_api.py used importlib.import_module which may init CUDA
3. pip install without --timeout could hang on unreachable mirror

Fixes:
- .dockerignore: excludes cccl_upstream/, vllm/, *.zip, docs/ etc
  Build context: ~2MB instead of 165MB
- probe_corex_api.py: rewritten to use ONLY ast.parse, zero runtime imports
- pip install: added --timeout 30
This commit is contained in:
Claude
2026-08-08 11:21:43 +00:00
parent dbfe20fd1c
commit c1065aaf2c
3 changed files with 98 additions and 130 deletions

17
.dockerignore Normal file
View File

@@ -0,0 +1,17 @@
# Exclude everything not needed for the Docker image
cccl_upstream/
vllm/
muh/
docs/
optimizations/
vllm_adapter/
*.zip
*.txt
*.md
*.json
*.muh
.git/
.gitignore
__pycache__/
*.pyc
# Keep: qwen3_6_scripts/, computility-run.yaml, Dockerfile, launch_service

View File

@@ -57,7 +57,7 @@ for P in /usr/local/lib/python3.10/site-packages/transformers/models \
done done
if [ -n "$TMODELS" ]; then if [ -n "$TMODELS" ]; then
# Base engine requires transformers 4.55.3 for Qwen3_5Config support # Base engine requires transformers 4.55.3 for Qwen3_5Config support
pip install transformers==4.55.3 -i https://pypi.tuna.tsinghua.edu.cn/simple 2>&1 || \ pip install transformers==4.55.3 -i https://pypi.tuna.tsinghua.edu.cn/simple --timeout 30 2>&1 || \
echo "[patch_ops] WARNING: transformers install failed (may already be correct version)" echo "[patch_ops] WARNING: transformers install failed (may already be correct version)"
cp -r ./qwen3_5 "$TMODELS/" 2>/dev/null && echo "[patch_ops] qwen3_5 config copied" || true cp -r ./qwen3_5 "$TMODELS/" 2>/dev/null && echo "[patch_ops] qwen3_5 config copied" || true
cp -r ./qwen3_5_moe "$TMODELS/" 2>/dev/null && echo "[patch_ops] qwen3_5_moe config copied" || true cp -r ./qwen3_5_moe "$TMODELS/" 2>/dev/null && echo "[patch_ops] qwen3_5_moe config copied" || true

View File

@@ -1,72 +1,20 @@
""" """
CoreX API probe — runs at Docker build time (in patch_ops.sh). CoreX API probe — runs at Docker build time (NO GPU, NO runtime imports).
Discovers the real interfaces of corex_gdn.py, corex_moe.py, corex_fa2.py Uses ONLY file system inspection and AST parsing.
from the base Docker image. Outputs: Never imports corex modules (they may init CUDA which kills the build).
1. /workspace/corex_probe_result.json — machine-readable API map
2. stdout — human-readable summary for build log
This is NOT runtime code. It runs once during `docker build`.
""" """
import importlib import ast
import inspect
import json import json
import os import os
import sys import sys
PROBE_RESULT = {} PROBE_RESULT = {}
def probe_module(module_path, name): def probe_file_ast(filepath, name):
"""Try to import a module and extract its public API.""" """AST-parse a Python file to extract class/function definitions."""
result = {"available": False, "classes": {}, "functions": {}, "error": None} result = {"available": False, "classes": {}, "functions": {}, "imports": [], "error": None}
try:
# Try direct import first
mod = importlib.import_module(module_path)
result["available"] = True
result["file"] = getattr(mod, "__file__", "unknown")
for attr_name in dir(mod):
if attr_name.startswith("_"):
continue
obj = getattr(mod, attr_name)
if inspect.isclass(obj):
cls_info = {
"bases": [b.__name__ for b in obj.__bases__],
"methods": {},
}
for method_name in dir(obj):
if method_name.startswith("_") and method_name != "__init__":
continue
method = getattr(obj, method_name, None)
if callable(method):
try:
sig = str(inspect.signature(method))
cls_info["methods"][method_name] = sig
except (ValueError, TypeError):
cls_info["methods"][method_name] = "(unknown)"
result["classes"][attr_name] = cls_info
elif callable(obj):
try:
sig = str(inspect.signature(obj))
result["functions"][attr_name] = sig
except (ValueError, TypeError):
result["functions"][attr_name] = "(unknown)"
except ImportError as e:
result["error"] = f"ImportError: {e}"
except Exception as e:
result["error"] = f"{type(e).__name__}: {e}"
return result
def probe_file_directly(filepath, name):
"""If import fails, try to read the file and extract class/function defs."""
result = {"available": False, "classes": {}, "functions": {}, "error": None}
if not os.path.exists(filepath): if not os.path.exists(filepath):
result["error"] = f"File not found: {filepath}" result["error"] = f"File not found: {filepath}"
@@ -77,37 +25,59 @@ def probe_file_directly(filepath, name):
result["size"] = os.path.getsize(filepath) result["size"] = os.path.getsize(filepath)
try: try:
import ast
with open(filepath) as f: with open(filepath) as f:
tree = ast.parse(f.read()) source = f.read()
result["line_count"] = source.count("\n") + 1
tree = ast.parse(source)
for node in ast.walk(tree): for node in ast.iter_child_nodes(tree):
if isinstance(node, ast.ClassDef): # Top-level imports
if isinstance(node, ast.Import):
for alias in node.names:
result["imports"].append(alias.name)
elif isinstance(node, ast.ImportFrom):
mod = node.module or ""
for alias in node.names:
result["imports"].append(f"{mod}.{alias.name}")
# Top-level classes
elif isinstance(node, ast.ClassDef):
methods = {} methods = {}
for item in node.body: for item in ast.iter_child_nodes(node):
if isinstance(item, (ast.FunctionDef, ast.AsyncFunctionDef)): if isinstance(item, (ast.FunctionDef, ast.AsyncFunctionDef)):
args = [] args = [arg.arg for arg in item.args.args]
for arg in item.args.args: methods[item.name] = {
args.append(arg.arg) "args": args,
methods[item.name] = f"({', '.join(args)})" "lineno": item.lineno,
}
bases = []
for b in node.bases:
if isinstance(b, ast.Name):
bases.append(b.id)
elif isinstance(b, ast.Attribute):
bases.append(f"{ast.dump(b)}")
result["classes"][node.name] = { result["classes"][node.name] = {
"bases": [ast.dump(b) for b in node.bases], "bases": bases,
"methods": methods, "methods": methods,
"lineno": node.lineno, "lineno": node.lineno,
} }
elif isinstance(node, ast.FunctionDef) and node.col_offset == 0:
# Top-level functions
elif isinstance(node, ast.FunctionDef):
args = [arg.arg for arg in node.args.args] args = [arg.arg for arg in node.args.args]
result["functions"][node.name] = { result["functions"][node.name] = {
"signature": f"({', '.join(args)})", "args": args,
"lineno": node.lineno, "lineno": node.lineno,
} }
except SyntaxError as e:
result["error"] = f"SyntaxError: {e}"
except Exception as e: except Exception as e:
result["error"] = f"AST parse error: {e}" result["error"] = f"{type(e).__name__}: {e}"
return result return result
# Probe paths # Find vllm models directory
VLLM_MODELS = None VLLM_MODELS = None
for p in [ for p in [
"/usr/local/corex/lib/python3/dist-packages/vllm/model_executor/models", "/usr/local/corex/lib/python3/dist-packages/vllm/model_executor/models",
@@ -118,89 +88,70 @@ for p in [
break break
print("=" * 70) print("=" * 70)
print("[corex_probe] CoreX API Discovery — Build Time") print("[corex_probe] CoreX API Discovery — Build Time (AST only, no GPU)")
print("=" * 70) print("=" * 70)
if VLLM_MODELS: if VLLM_MODELS:
print(f"[corex_probe] vllm models dir: {VLLM_MODELS}") print(f"[corex_probe] vllm models dir: {VLLM_MODELS}")
# List ALL .py files in models dir to find corex modules # List ALL .py files
all_files = sorted(os.listdir(VLLM_MODELS)) all_py = sorted(f for f in os.listdir(VLLM_MODELS) if f.endswith(".py"))
corex_files = [f for f in all_files if "corex" in f.lower()] corex_files = [f for f in all_py if "corex" in f.lower()]
print(f"[corex_probe] CoreX files found: {corex_files}") print(f"[corex_probe] CoreX files: {corex_files}")
print(f"[corex_probe] All model files: {[f for f in all_files if f.endswith('.py')]}") print(f"[corex_probe] Total .py files: {len(all_py)}")
# Probe each corex module # Probe each corex module by AST
for target in ["corex_gdn", "corex_moe", "corex_fa2"]: for target in ["corex_gdn", "corex_moe", "corex_fa2"]:
filepath = os.path.join(VLLM_MODELS, f"{target}.py") filepath = os.path.join(VLLM_MODELS, f"{target}.py")
result = probe_file_ast(filepath, target)
# Try import first
result = probe_module(f"vllm.model_executor.models.{target}", target)
# If import failed, try AST parse
if not result["available"]:
print(f"[corex_probe] {target}: import failed ({result['error']}), trying AST...")
result = probe_file_directly(filepath, target)
PROBE_RESULT[target] = result PROBE_RESULT[target] = result
if result["available"]: if result["available"]:
print(f"[corex_probe] {target}: FOUND at {result.get('file', filepath)}") print(f"[corex_probe] {target}: FOUND {result['size']} bytes, {result['line_count']} lines")
if result.get("size"):
print(f"[corex_probe] size: {result['size']} bytes")
for cls_name, cls_info in result.get("classes", {}).items(): for cls_name, cls_info in result.get("classes", {}).items():
print(f"[corex_probe] class {cls_name}:") print(f"[corex_probe] class {cls_name} (line {cls_info['lineno']}):")
for method_name, sig in cls_info.get("methods", {}).items(): for mname, minfo in cls_info.get("methods", {}).items():
print(f"[corex_probe] {method_name}{sig}") print(f"[corex_probe] def {mname}({', '.join(minfo['args'])}) # line {minfo['lineno']}")
for func_name, func_info in result.get("functions", {}).items(): for fname, finfo in result.get("functions", {}).items():
if isinstance(func_info, dict): print(f"[corex_probe] def {fname}({', '.join(finfo['args'])}) # line {finfo['lineno']}")
print(f"[corex_probe] def {func_name}{func_info['signature']} (line {func_info['lineno']})")
else:
print(f"[corex_probe] def {func_name}{func_info}")
else: else:
print(f"[corex_probe] {target}: NOT AVAILABLE{result.get('error', 'unknown')}") print(f"[corex_probe] {target}: NOT FOUND{result.get('error', 'unknown')}")
# Also probe the native qwen3_5.py BEFORE we overwrite it # Inspect native qwen3_5.py BEFORE we overwrite
native_qw = os.path.join(VLLM_MODELS, "qwen3_5.py") native_qw = os.path.join(VLLM_MODELS, "qwen3_5.py")
if os.path.exists(native_qw): if os.path.exists(native_qw):
sz = os.path.getsize(native_qw) sz = os.path.getsize(native_qw)
print(f"[corex_probe] Native qwen3_5.py: {sz} bytes")
# Check if it imports corex
with open(native_qw) as f: with open(native_qw) as f:
content = f.read() content = f.read()
for keyword in ["corex_gdn", "corex_moe", "corex_fa2", "CoreXGDN", "CoreXMoE"]: lc = content.count("\n") + 1
if keyword in content: refs = {kw: kw in content for kw in ["corex_gdn", "corex_moe", "corex_fa2"]}
print(f"[corex_probe] → references '{keyword}'") print(f"[corex_probe] Native qwen3_5.py: {sz} bytes, {lc} lines")
PROBE_RESULT["native_qwen3_5"] = { for kw, found in refs.items():
"size": sz, if found:
"line_count": content.count("\n"), print(f"[corex_probe] → references '{kw}'")
"has_corex_gdn": "corex_gdn" in content, PROBE_RESULT["native_qwen3_5"] = {"size": sz, "line_count": lc, **refs}
"has_corex_moe": "corex_moe" in content,
"has_corex_fa2": "corex_fa2" in content,
}
else: else:
print(f"[corex_probe] Native qwen3_5.py: NOT FOUND (will deploy ours)") print(f"[corex_probe] Native qwen3_5.py: NOT FOUND")
PROBE_RESULT["native_qwen3_5"] = {"size": 0, "exists": False} PROBE_RESULT["native_qwen3_5"] = {"exists": False}
else: else:
print("[corex_probe] ERROR: vllm models directory not found") print("[corex_probe] ERROR: vllm models directory not found")
PROBE_RESULT["error"] = "vllm models dir not found" PROBE_RESULT["error"] = "vllm models dir not found"
# Also check .so files # Check .so files
for so_name, env_var in [ for so_name in ["libcorex_gdn.so", "libcorex_moe.so", "libcorex_fa2.so"]:
("libcorex_gdn.so", "VLLM_COREX_GDN_LIBRARY"), path = f"/usr/local/corex/lib64/{so_name}"
("libcorex_moe.so", "VLLM_COREX_MOE_LIBRARY"),
("libcorex_fa2.so", "VLLM_COREX_FA2_LIBRARY"),
]:
path = os.environ.get(env_var, f"/usr/local/corex/lib64/{so_name}")
exists = os.path.exists(path) exists = os.path.exists(path)
size = os.path.getsize(path) if exists else 0 size = os.path.getsize(path) if exists else 0
print(f"[corex_probe] {so_name}: {'EXISTS' if exists else 'MISSING'} ({size} bytes) at {path}") print(f"[corex_probe] {so_name}: {'EXISTS' if exists else 'MISSING'} ({size} bytes)")
PROBE_RESULT[so_name] = {"exists": exists, "size": size, "path": path} PROBE_RESULT[so_name] = {"exists": exists, "size": size, "path": path}
# Write results # Write JSON
output_path = "/workspace/corex_probe_result.json" output_path = "/workspace/corex_probe_result.json"
with open(output_path, "w") as f: try:
json.dump(PROBE_RESULT, f, indent=2, default=str) with open(output_path, "w") as f:
print(f"\n[corex_probe] Results written to {output_path}") json.dump(PROBE_RESULT, f, indent=2, default=str)
print(f"[corex_probe] Results → {output_path}")
except Exception as e:
print(f"[corex_probe] WARNING: could not write JSON: {e}")
print("=" * 70) print("=" * 70)