fix(CRITICAL): max_model_len 256000→80000 + topk_softmax silent fallback + deploy _custom_ops

Three fixes from comp 168 log analysis:

1. computility-run.yaml: max_model_len 256000→80000
   - 256000 causes OOM (comp 168: CUDA OOM at 31.72GB)
   - BI-V100 KV cache capacity ~88112 blocks

2. _custom_ops.py: topk_softmax silent fallback
   - ixf_F.vllm_moe_topk_softmax missing in base image
   - New: try ixformer._C.topk_softmax → silent PyTorch fallback
   - Eliminates 500+ ERROR lines from docker log

3. patch_ops.sh: deploy _custom_ops.py
   - Previously excluded; now deployed to fix topk_softmax issue

Ref: upstream_ref/xllm/core/kernels/ilu/ixformer.h

computility-run.yaml

@@ -8,7 +8,7 @@ command:
     - --served-model-name
     - llm
     - --max-model-len
-    - '256000'
+    - '80000'
     - --gpu-memory-utilization
     - '0.95'
     - --trust-remote-code

qwen3_6_scripts/_custom_ops.py

@@ -830,27 +830,37 @@ def invoke_fused_moe_kernel(
 def topk_softmax(topk_weights: torch.Tensor, topk_ids: torch.Tensor,
                  token_expert_indicies: torch.Tensor,
                  gating_output: float) -> None:
-    # CCCL policy_selector degradation: when one kernel in the chain is
-    # unavailable, replace ONLY that kernel with PyTorch while keeping the
-    # downstream native kernels (moe_align_block_size, invoke_fused_moe_kernel).
-    # This is analogous to CCCL's multi_pass fallback when onesweep is not
-    # available — the sort still happens, just through a different code path.
-    try:
-        ixf_F.vllm_moe_topk_softmax(topk_weights, topk_ids,
-                                      token_expert_indicies, gating_output)
-    except (AttributeError, RuntimeError):
-        # PyTorch fallback: softmax → topk → write in-place
-        # gating_output is already float32 (cast at call site)
+    # EX Engine: algorithm factor replacement for topk_softmax.
+    # ixformer::infer::topk_softmax exists in libixformer.so (C++ level)
+    # but ixformer.functions Python binding lacks vllm_moe_topk_softmax.
+    # Strategy: try C++ path → silent PyTorch fallback (no ERROR log spam).
+    _called = False
+    if not _called:
+        try:
+            import ixformer._C as _ixf_C
+            if hasattr(_ixf_C, 'topk_softmax'):
+                _ixf_C.topk_softmax(topk_weights, topk_ids,
+                                     token_expert_indicies, gating_output)
+                _called = True
+        except Exception:
+            pass
+    if not _called:
+        try:
+            ixf_F.vllm_moe_topk_softmax(topk_weights, topk_ids,
+                                          token_expert_indicies, gating_output)
+            _called = True
+        except (AttributeError, RuntimeError):
+            pass
+    if not _called:
+        # PyTorch fallback: softmax → topk → write in-place (silent)
         if isinstance(gating_output, torch.Tensor):
-            probs = torch.softmax(gating_output, dim=-1)
+            probs = torch.softmax(gating_output.float(), dim=-1)
         else:
             probs = torch.softmax(gating_output, dim=-1)
         topk = topk_weights.shape[1]
         tw, ti = torch.topk(probs, topk, dim=-1)
         topk_weights.copy_(tw)
         topk_ids.copy_(ti.to(topk_ids.dtype))
-        # token_expert_indicies is unused by caller (deleted after call)
-        # but fill it for correctness
         token_expert_indicies.copy_(
             torch.arange(topk, device=topk_ids.device, dtype=topk_ids.dtype)
             .unsqueeze(0).expand_as(topk_ids))

qwen3_6_scripts/patch_ops.sh

@@ -21,7 +21,7 @@
 # qwen3_5.py MUST be deployed — base image registry references it but
 # the module file is missing (causes ModuleNotFoundError on startup).
 #
-# DO NOT deploy: model_runner.py, _custom_ops.py,
+# DO NOT deploy: model_runner.py,
 #   sampler.py, scheduler.py, sequence.py, xformers.py, paged_attn.py,
 #   prefix_prefill.py, logits_processor.py, mamba_cache.py, arg_utils.py
 # ==========================================================================
@@ -258,7 +258,17 @@ fi
 echo "[patch_ops] DONE — EX Engine + SM70 GDN kernel + MoE topk kernel + serving layer deployed"
 echo "[patch_ops] Deployed: qwen3_5.py, flash_qla_sm70, ex_engine factors, paged_attn.py, mamba_cache.py, sequence.py, scheduler.py, xformers patches, serving layer"
 echo "[patch_ops] EX factors replace: vllm_moe_topk_softmax (2304 calls/token), gdn_chunk_fwd (NaN fix)"
-echo "[patch_ops] NOT deployed (base image native): model_runner.py, _custom_ops.py, sampler.py, logits_processor.py, arg_utils.py"
+# Deploy patched _custom_ops.py — fixes topk_softmax ERROR log spam
+# Base image ixf_F.vllm_moe_topk_softmax is missing; our patch tries
+# ixformer._C.topk_softmax first, then silent PyTorch fallback.
+cp ./_custom_ops.py "$VLLM/_custom_ops.py" 2>/dev/null && \
+    echo "[patch_ops] _custom_ops.py deployed (topk_softmax fix)" || \
+    echo "[patch_ops] WARNING: _custom_ops.py deploy failed"
+if [ -n "$VLLM2" ]; then
+    cp ./_custom_ops.py "$VLLM2/_custom_ops.py" 2>/dev/null || true
+fi
+
+echo "[patch_ops] NOT deployed (base image native): model_runner.py, sampler.py, logits_processor.py, arg_utils.py"
 
 # Deploy flash_qla SM70 GDN kernel (from 1Cat-vLLM, MIT license)
 # This is a fused CUDA kernel for GatedDeltaNet on SM70/SM75 (V100/BI-V100)