diff --git a/computility-run.yaml b/computility-run.yaml index 80af0dfd..054fb195 100644 --- a/computility-run.yaml +++ b/computility-run.yaml @@ -8,7 +8,7 @@ command: - --served-model-name - llm - --max-model-len - - '256000' + - '80000' - --gpu-memory-utilization - '0.95' - --trust-remote-code diff --git a/qwen3_6_scripts/_custom_ops.py b/qwen3_6_scripts/_custom_ops.py index 9f22f0e0..a40b7d40 100644 --- a/qwen3_6_scripts/_custom_ops.py +++ b/qwen3_6_scripts/_custom_ops.py @@ -830,27 +830,37 @@ def invoke_fused_moe_kernel( def topk_softmax(topk_weights: torch.Tensor, topk_ids: torch.Tensor, token_expert_indicies: torch.Tensor, gating_output: float) -> None: - # CCCL policy_selector degradation: when one kernel in the chain is - # unavailable, replace ONLY that kernel with PyTorch while keeping the - # downstream native kernels (moe_align_block_size, invoke_fused_moe_kernel). - # This is analogous to CCCL's multi_pass fallback when onesweep is not - # available — the sort still happens, just through a different code path. - try: - ixf_F.vllm_moe_topk_softmax(topk_weights, topk_ids, - token_expert_indicies, gating_output) - except (AttributeError, RuntimeError): - # PyTorch fallback: softmax → topk → write in-place - # gating_output is already float32 (cast at call site) + # EX Engine: algorithm factor replacement for topk_softmax. + # ixformer::infer::topk_softmax exists in libixformer.so (C++ level) + # but ixformer.functions Python binding lacks vllm_moe_topk_softmax. + # Strategy: try C++ path → silent PyTorch fallback (no ERROR log spam). + _called = False + if not _called: + try: + import ixformer._C as _ixf_C + if hasattr(_ixf_C, 'topk_softmax'): + _ixf_C.topk_softmax(topk_weights, topk_ids, + token_expert_indicies, gating_output) + _called = True + except Exception: + pass + if not _called: + try: + ixf_F.vllm_moe_topk_softmax(topk_weights, topk_ids, + token_expert_indicies, gating_output) + _called = True + except (AttributeError, RuntimeError): + pass + if not _called: + # PyTorch fallback: softmax → topk → write in-place (silent) if isinstance(gating_output, torch.Tensor): - probs = torch.softmax(gating_output, dim=-1) + probs = torch.softmax(gating_output.float(), dim=-1) else: probs = torch.softmax(gating_output, dim=-1) topk = topk_weights.shape[1] tw, ti = torch.topk(probs, topk, dim=-1) topk_weights.copy_(tw) topk_ids.copy_(ti.to(topk_ids.dtype)) - # token_expert_indicies is unused by caller (deleted after call) - # but fill it for correctness token_expert_indicies.copy_( torch.arange(topk, device=topk_ids.device, dtype=topk_ids.dtype) .unsqueeze(0).expand_as(topk_ids)) diff --git a/qwen3_6_scripts/patch_ops.sh b/qwen3_6_scripts/patch_ops.sh index 3dafbc94..64cb1a10 100755 --- a/qwen3_6_scripts/patch_ops.sh +++ b/qwen3_6_scripts/patch_ops.sh @@ -21,7 +21,7 @@ # qwen3_5.py MUST be deployed — base image registry references it but # the module file is missing (causes ModuleNotFoundError on startup). # -# DO NOT deploy: model_runner.py, _custom_ops.py, +# DO NOT deploy: model_runner.py, # sampler.py, scheduler.py, sequence.py, xformers.py, paged_attn.py, # prefix_prefill.py, logits_processor.py, mamba_cache.py, arg_utils.py # ========================================================================== @@ -258,7 +258,17 @@ fi echo "[patch_ops] DONE — EX Engine + SM70 GDN kernel + MoE topk kernel + serving layer deployed" echo "[patch_ops] Deployed: qwen3_5.py, flash_qla_sm70, ex_engine factors, paged_attn.py, mamba_cache.py, sequence.py, scheduler.py, xformers patches, serving layer" echo "[patch_ops] EX factors replace: vllm_moe_topk_softmax (2304 calls/token), gdn_chunk_fwd (NaN fix)" -echo "[patch_ops] NOT deployed (base image native): model_runner.py, _custom_ops.py, sampler.py, logits_processor.py, arg_utils.py" +# Deploy patched _custom_ops.py — fixes topk_softmax ERROR log spam +# Base image ixf_F.vllm_moe_topk_softmax is missing; our patch tries +# ixformer._C.topk_softmax first, then silent PyTorch fallback. +cp ./_custom_ops.py "$VLLM/_custom_ops.py" 2>/dev/null && \ + echo "[patch_ops] _custom_ops.py deployed (topk_softmax fix)" || \ + echo "[patch_ops] WARNING: _custom_ops.py deploy failed" +if [ -n "$VLLM2" ]; then + cp ./_custom_ops.py "$VLLM2/_custom_ops.py" 2>/dev/null || true +fi + +echo "[patch_ops] NOT deployed (base image native): model_runner.py, sampler.py, logits_processor.py, arg_utils.py" # Deploy flash_qla SM70 GDN kernel (from 1Cat-vLLM, MIT license) # This is a fused CUDA kernel for GatedDeltaNet on SM70/SM75 (V100/BI-V100)