fix: fail-fast on ix_bridge failure + probe script for real machine
1. ix_bridge.py: RuntimeError instead of silent PyTorch fallback If JIT compile fails, crash immediately with diagnostic message. 0 score with no error log is worse than a visible crash. 2. qwen3_5.py: explicit WARNING log on import failure (not silent) Shows exact error so we can diagnose from docker log. 3. probe_ixformer_symbols.py: definitive test for real machine - Finds all ixformer .so files - nm/objdump for topk_softmax C++ symbol - Checks Python bindings - Attempts JIT compile + link (the real test) - Prints PASS/FAIL with next-step instructions Run on real machine: python3 probe_ixformer_symbols.py
This commit is contained in:
@@ -57,21 +57,15 @@ def topk_softmax(gating_output: torch.Tensor, topk: int, renormalize: bool = Tru
|
||||
"""
|
||||
Fused topk+softmax via ixformer C++ API.
|
||||
|
||||
Args:
|
||||
gating_output: (num_tokens, num_experts) router logits
|
||||
topk: number of experts to select
|
||||
renormalize: whether to renormalize weights
|
||||
|
||||
Returns:
|
||||
(topk_weights, topk_indices) — both (num_tokens, topk)
|
||||
FAIL FAST: if bridge not available, raises RuntimeError immediately.
|
||||
No silent fallback — 0 score with no error log is worse than a crash.
|
||||
"""
|
||||
if not _ix_bridge_available:
|
||||
if not _load_bridge():
|
||||
# Fallback to pure PyTorch
|
||||
probs = torch.softmax(gating_output.float(), dim=-1)
|
||||
topk_w, topk_ids = torch.topk(probs, topk, dim=-1)
|
||||
if renormalize:
|
||||
topk_w = topk_w / topk_w.sum(dim=-1, keepdim=True)
|
||||
return topk_w, topk_ids.to(torch.int32)
|
||||
raise RuntimeError(
|
||||
"ix_moe_bridge: FATAL — ixformer C++ topk_softmax not available. "
|
||||
"JIT compile failed. Run probe_ixformer_symbols.py on real machine "
|
||||
"to diagnose. Cannot fall back silently — would produce 0 score."
|
||||
)
|
||||
|
||||
return _ix_bridge.topk_softmax(gating_output, topk, renormalize)
|
||||
|
||||
Reference in New Issue
Block a user