From 7d4edd4ac7c2e130d6ab74d0e96e1d8bf0270bf9 Mon Sep 17 00:00:00 2001 From: project6-dev Date: Mon, 10 Aug 2026 04:45:26 +0000 Subject: [PATCH] =?UTF-8?q?fix(GDN):=20force=20fp16=20cast=20before=20norm?= =?UTF-8?q?+out=5Fproj=20=E2=80=94=20ixformer=20matmul=20requires=20kHalf?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit matmul.cu:149 'Expected input.dtype() == kHalf' error in competition log. Root cause: _torch_chunk_gated_delta_rule returns fp32 core_out, passed directly to self.norm() → self.out_proj() which calls ixformer matmul. Fix: explicit .to(torch.float16) on core_out and z before norm. --- qwen3_6_scripts/qwen3_5.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/qwen3_6_scripts/qwen3_5.py b/qwen3_6_scripts/qwen3_5.py index fa07daea..9b68d4a2 100644 --- a/qwen3_6_scripts/qwen3_5.py +++ b/qwen3_6_scripts/qwen3_5.py @@ -737,6 +737,9 @@ class GatedDeltaNet(nn.Module): # Gate + norm + output proj z = z_all[s:e].reshape(seq_len, local_num_v, self.head_v_dim) core_out = core_out.reshape(seq_len, local_num_v, self.head_v_dim) + # Force fp16 — ixformer matmul requires kHalf + core_out = core_out.to(torch.float16) + z = z.to(torch.float16) normed = self.norm( core_out.reshape(-1, self.head_v_dim), z.reshape(-1, self.head_v_dim))