fix(build): add .dockerignore + safe probe — fix docker build failure

Build was failing, likely due to:
1. 165MB build context (no .dockerignore) — cccl_upstream/ 53MB, zip 97MB
2. probe_corex_api.py used importlib.import_module which may init CUDA
3. pip install without --timeout could hang on unreachable mirror

Fixes:
- .dockerignore: excludes cccl_upstream/, vllm/, *.zip, docs/ etc
  Build context: ~2MB instead of 165MB
- probe_corex_api.py: rewritten to use ONLY ast.parse, zero runtime imports
- pip install: added --timeout 30
This commit is contained in:
Claude
2026-08-08 11:21:43 +00:00
parent dbfe20fd1c
commit c1065aaf2c
3 changed files with 98 additions and 130 deletions

17
.dockerignore Normal file
View File

@@ -0,0 +1,17 @@
# Exclude everything not needed for the Docker image
cccl_upstream/
vllm/
muh/
docs/
optimizations/
vllm_adapter/
*.zip
*.txt
*.md
*.json
*.muh
.git/
.gitignore
__pycache__/
*.pyc
# Keep: qwen3_6_scripts/, computility-run.yaml, Dockerfile, launch_service

View File

@@ -57,7 +57,7 @@ for P in /usr/local/lib/python3.10/site-packages/transformers/models \
done
if [ -n "$TMODELS" ]; then
# Base engine requires transformers 4.55.3 for Qwen3_5Config support
pip install transformers==4.55.3 -i https://pypi.tuna.tsinghua.edu.cn/simple 2>&1 || \
pip install transformers==4.55.3 -i https://pypi.tuna.tsinghua.edu.cn/simple --timeout 30 2>&1 || \
echo "[patch_ops] WARNING: transformers install failed (may already be correct version)"
cp -r ./qwen3_5 "$TMODELS/" 2>/dev/null && echo "[patch_ops] qwen3_5 config copied" || true
cp -r ./qwen3_5_moe "$TMODELS/" 2>/dev/null && echo "[patch_ops] qwen3_5_moe config copied" || true

View File

@@ -1,72 +1,20 @@
"""
CoreX API probe — runs at Docker build time (in patch_ops.sh).
CoreX API probe — runs at Docker build time (NO GPU, NO runtime imports).
Discovers the real interfaces of corex_gdn.py, corex_moe.py, corex_fa2.py
from the base Docker image. Outputs:
1. /workspace/corex_probe_result.json — machine-readable API map
2. stdout — human-readable summary for build log
This is NOT runtime code. It runs once during `docker build`.
Uses ONLY file system inspection and AST parsing.
Never imports corex modules (they may init CUDA which kills the build).
"""
import importlib
import inspect
import ast
import json
import os
import sys
PROBE_RESULT = {}
def probe_module(module_path, name):
"""Try to import a module and extract its public API."""
result = {"available": False, "classes": {}, "functions": {}, "error": None}
try:
# Try direct import first
mod = importlib.import_module(module_path)
result["available"] = True
result["file"] = getattr(mod, "__file__", "unknown")
for attr_name in dir(mod):
if attr_name.startswith("_"):
continue
obj = getattr(mod, attr_name)
if inspect.isclass(obj):
cls_info = {
"bases": [b.__name__ for b in obj.__bases__],
"methods": {},
}
for method_name in dir(obj):
if method_name.startswith("_") and method_name != "__init__":
continue
method = getattr(obj, method_name, None)
if callable(method):
try:
sig = str(inspect.signature(method))
cls_info["methods"][method_name] = sig
except (ValueError, TypeError):
cls_info["methods"][method_name] = "(unknown)"
result["classes"][attr_name] = cls_info
elif callable(obj):
try:
sig = str(inspect.signature(obj))
result["functions"][attr_name] = sig
except (ValueError, TypeError):
result["functions"][attr_name] = "(unknown)"
except ImportError as e:
result["error"] = f"ImportError: {e}"
except Exception as e:
result["error"] = f"{type(e).__name__}: {e}"
return result
def probe_file_directly(filepath, name):
"""If import fails, try to read the file and extract class/function defs."""
result = {"available": False, "classes": {}, "functions": {}, "error": None}
def probe_file_ast(filepath, name):
"""AST-parse a Python file to extract class/function definitions."""
result = {"available": False, "classes": {}, "functions": {}, "imports": [], "error": None}
if not os.path.exists(filepath):
result["error"] = f"File not found: {filepath}"
@@ -77,37 +25,59 @@ def probe_file_directly(filepath, name):
result["size"] = os.path.getsize(filepath)
try:
import ast
with open(filepath) as f:
tree = ast.parse(f.read())
source = f.read()
result["line_count"] = source.count("\n") + 1
tree = ast.parse(source)
for node in ast.walk(tree):
if isinstance(node, ast.ClassDef):
for node in ast.iter_child_nodes(tree):
# Top-level imports
if isinstance(node, ast.Import):
for alias in node.names:
result["imports"].append(alias.name)
elif isinstance(node, ast.ImportFrom):
mod = node.module or ""
for alias in node.names:
result["imports"].append(f"{mod}.{alias.name}")
# Top-level classes
elif isinstance(node, ast.ClassDef):
methods = {}
for item in node.body:
for item in ast.iter_child_nodes(node):
if isinstance(item, (ast.FunctionDef, ast.AsyncFunctionDef)):
args = []
for arg in item.args.args:
args.append(arg.arg)
methods[item.name] = f"({', '.join(args)})"
args = [arg.arg for arg in item.args.args]
methods[item.name] = {
"args": args,
"lineno": item.lineno,
}
bases = []
for b in node.bases:
if isinstance(b, ast.Name):
bases.append(b.id)
elif isinstance(b, ast.Attribute):
bases.append(f"{ast.dump(b)}")
result["classes"][node.name] = {
"bases": [ast.dump(b) for b in node.bases],
"bases": bases,
"methods": methods,
"lineno": node.lineno,
}
elif isinstance(node, ast.FunctionDef) and node.col_offset == 0:
# Top-level functions
elif isinstance(node, ast.FunctionDef):
args = [arg.arg for arg in node.args.args]
result["functions"][node.name] = {
"signature": f"({', '.join(args)})",
"args": args,
"lineno": node.lineno,
}
except SyntaxError as e:
result["error"] = f"SyntaxError: {e}"
except Exception as e:
result["error"] = f"AST parse error: {e}"
result["error"] = f"{type(e).__name__}: {e}"
return result
# Probe paths
# Find vllm models directory
VLLM_MODELS = None
for p in [
"/usr/local/corex/lib/python3/dist-packages/vllm/model_executor/models",
@@ -118,89 +88,70 @@ for p in [
break
print("=" * 70)
print("[corex_probe] CoreX API Discovery — Build Time")
print("[corex_probe] CoreX API Discovery — Build Time (AST only, no GPU)")
print("=" * 70)
if VLLM_MODELS:
print(f"[corex_probe] vllm models dir: {VLLM_MODELS}")
# List ALL .py files in models dir to find corex modules
all_files = sorted(os.listdir(VLLM_MODELS))
corex_files = [f for f in all_files if "corex" in f.lower()]
print(f"[corex_probe] CoreX files found: {corex_files}")
print(f"[corex_probe] All model files: {[f for f in all_files if f.endswith('.py')]}")
# List ALL .py files
all_py = sorted(f for f in os.listdir(VLLM_MODELS) if f.endswith(".py"))
corex_files = [f for f in all_py if "corex" in f.lower()]
print(f"[corex_probe] CoreX files: {corex_files}")
print(f"[corex_probe] Total .py files: {len(all_py)}")
# Probe each corex module
# Probe each corex module by AST
for target in ["corex_gdn", "corex_moe", "corex_fa2"]:
filepath = os.path.join(VLLM_MODELS, f"{target}.py")
# Try import first
result = probe_module(f"vllm.model_executor.models.{target}", target)
# If import failed, try AST parse
if not result["available"]:
print(f"[corex_probe] {target}: import failed ({result['error']}), trying AST...")
result = probe_file_directly(filepath, target)
result = probe_file_ast(filepath, target)
PROBE_RESULT[target] = result
if result["available"]:
print(f"[corex_probe] {target}: FOUND at {result.get('file', filepath)}")
if result.get("size"):
print(f"[corex_probe] size: {result['size']} bytes")
print(f"[corex_probe] {target}: FOUND {result['size']} bytes, {result['line_count']} lines")
for cls_name, cls_info in result.get("classes", {}).items():
print(f"[corex_probe] class {cls_name}:")
for method_name, sig in cls_info.get("methods", {}).items():
print(f"[corex_probe] {method_name}{sig}")
for func_name, func_info in result.get("functions", {}).items():
if isinstance(func_info, dict):
print(f"[corex_probe] def {func_name}{func_info['signature']} (line {func_info['lineno']})")
else:
print(f"[corex_probe] def {func_name}{func_info}")
print(f"[corex_probe] class {cls_name} (line {cls_info['lineno']}):")
for mname, minfo in cls_info.get("methods", {}).items():
print(f"[corex_probe] def {mname}({', '.join(minfo['args'])}) # line {minfo['lineno']}")
for fname, finfo in result.get("functions", {}).items():
print(f"[corex_probe] def {fname}({', '.join(finfo['args'])}) # line {finfo['lineno']}")
else:
print(f"[corex_probe] {target}: NOT AVAILABLE{result.get('error', 'unknown')}")
print(f"[corex_probe] {target}: NOT FOUND{result.get('error', 'unknown')}")
# Also probe the native qwen3_5.py BEFORE we overwrite it
# Inspect native qwen3_5.py BEFORE we overwrite
native_qw = os.path.join(VLLM_MODELS, "qwen3_5.py")
if os.path.exists(native_qw):
sz = os.path.getsize(native_qw)
print(f"[corex_probe] Native qwen3_5.py: {sz} bytes")
# Check if it imports corex
with open(native_qw) as f:
content = f.read()
for keyword in ["corex_gdn", "corex_moe", "corex_fa2", "CoreXGDN", "CoreXMoE"]:
if keyword in content:
print(f"[corex_probe] → references '{keyword}'")
PROBE_RESULT["native_qwen3_5"] = {
"size": sz,
"line_count": content.count("\n"),
"has_corex_gdn": "corex_gdn" in content,
"has_corex_moe": "corex_moe" in content,
"has_corex_fa2": "corex_fa2" in content,
}
lc = content.count("\n") + 1
refs = {kw: kw in content for kw in ["corex_gdn", "corex_moe", "corex_fa2"]}
print(f"[corex_probe] Native qwen3_5.py: {sz} bytes, {lc} lines")
for kw, found in refs.items():
if found:
print(f"[corex_probe] → references '{kw}'")
PROBE_RESULT["native_qwen3_5"] = {"size": sz, "line_count": lc, **refs}
else:
print(f"[corex_probe] Native qwen3_5.py: NOT FOUND (will deploy ours)")
PROBE_RESULT["native_qwen3_5"] = {"size": 0, "exists": False}
print(f"[corex_probe] Native qwen3_5.py: NOT FOUND")
PROBE_RESULT["native_qwen3_5"] = {"exists": False}
else:
print("[corex_probe] ERROR: vllm models directory not found")
PROBE_RESULT["error"] = "vllm models dir not found"
# Also check .so files
for so_name, env_var in [
("libcorex_gdn.so", "VLLM_COREX_GDN_LIBRARY"),
("libcorex_moe.so", "VLLM_COREX_MOE_LIBRARY"),
("libcorex_fa2.so", "VLLM_COREX_FA2_LIBRARY"),
]:
path = os.environ.get(env_var, f"/usr/local/corex/lib64/{so_name}")
# Check .so files
for so_name in ["libcorex_gdn.so", "libcorex_moe.so", "libcorex_fa2.so"]:
path = f"/usr/local/corex/lib64/{so_name}"
exists = os.path.exists(path)
size = os.path.getsize(path) if exists else 0
print(f"[corex_probe] {so_name}: {'EXISTS' if exists else 'MISSING'} ({size} bytes) at {path}")
print(f"[corex_probe] {so_name}: {'EXISTS' if exists else 'MISSING'} ({size} bytes)")
PROBE_RESULT[so_name] = {"exists": exists, "size": size, "path": path}
# Write results
# Write JSON
output_path = "/workspace/corex_probe_result.json"
with open(output_path, "w") as f:
json.dump(PROBE_RESULT, f, indent=2, default=str)
print(f"\n[corex_probe] Results written to {output_path}")
try:
with open(output_path, "w") as f:
json.dump(PROBE_RESULT, f, indent=2, default=str)
print(f"[corex_probe] Results → {output_path}")
except Exception as e:
print(f"[corex_probe] WARNING: could not write JSON: {e}")
print("=" * 70)