fix(precision): guard all corex .so outputs with nan_to_num + reduce max-model-len

MoE kernels:
- topk_softmax: add .contiguous() + nan_to_num + re-normalize weights
- direct_routed: nan_to_num on w2_reduce output
- exact_reduce: nan_to_num on serial_float output

GDN kernels:
- packed_decode: nan_to_num on core_out

BI-V100 CUB may produce non-finite values in fp16 softmax/reduce.
These guards prevent garbage propagation without disabling the kernels.

max-model-len: 256000 → 131072 (4x32GB BI-V100 OOM prevention)
Dockerfile: unchanged (no force push needed)
This commit is contained in:
project6-dev
2026-08-13 02:12:16 +00:00
parent 8abc7cb0d7
commit 8acc47129b
2 changed files with 13 additions and 4 deletions

View File

@@ -8,7 +8,7 @@ command:
- --served-model-name
- llm
- --max-model-len
- '256000'
- '131072'
- --gpu-memory-utilization
- '0.95'
- --trust-remote-code

View File

@@ -1187,6 +1187,8 @@ class GatedDeltaNet(nn.Module):
core_out = _corex_gdn_packed_decode.packed_decode(
temporal_state, packed_mixed_qkv, b_all, a_all,
self.A_log, self.dt_bias)
core_out = torch.nan_to_num(
core_out, nan=0.0, posinf=0.0, neginf=0.0)
else:
q, k, v = torch.split(
mixed_qkv_conv,
@@ -1611,9 +1613,13 @@ class Qwen3_5MoeSparseBlock(nn.Module):
# Source: xllm/core/kernels/cuda/moe/moe_topk_softmax_kernels.cuh
if _USE_COREX_MOE_TOPK_SOFTMAX:
topk_weights, topk_ids = _corex_moe_topk_softmax.moe_topk_softmax(
router_logits.float(), self.top_k, True)
router_logits.float().contiguous(), self.top_k, True)
topk_ids = topk_ids.to(torch.int64)
topk_weights = topk_weights.to(hidden_states.dtype)
# BI-V100 CUB softmax may produce non-finite → clamp before cast
topk_weights = torch.nan_to_num(
topk_weights, nan=0.0, posinf=1.0, neginf=0.0)
denom = topk_weights.sum(dim=-1, keepdim=True).clamp(min=1e-6)
topk_weights = (topk_weights / denom).to(hidden_states.dtype)
else:
topk_logits, topk_ids = torch.topk(
router_logits.float(), self.top_k, dim=-1) # (T, top_k)
@@ -1651,8 +1657,10 @@ class Qwen3_5MoeSparseBlock(nn.Module):
gate_up = _corex_moe_direct_routed.w13(
hidden_states, w13, eids)
act = self.act_fn(gate_up)
return _corex_moe_direct_routed.w2_reduce(
out = _corex_moe_direct_routed.w2_reduce(
act, w2, eids, ws)
return torch.nan_to_num(
out, nan=0.0, posinf=0.0, neginf=0.0)
use_corex_gather = (
_USE_COREX_MOE_WEIGHT_GATHER
@@ -1697,6 +1705,7 @@ class Qwen3_5MoeSparseBlock(nn.Module):
and ws.dtype == torch.float16
and expert_out.shape[0] == 8):
out = _corex_moe_exact_reduce.serial_float(expert_out, ws)
out = torch.nan_to_num(out, nan=0.0, posinf=0.0, neginf=0.0)
else:
out = (expert_out * ws.unsqueeze(-1)).sum(
0, keepdim=True).to(hidden_states.dtype) # (1, H)