diff --git a/qwen3_6_scripts/qwen3_5.py b/qwen3_6_scripts/qwen3_5.py index acf56c80..3236ab0c 100644 --- a/qwen3_6_scripts/qwen3_5.py +++ b/qwen3_6_scripts/qwen3_5.py @@ -564,9 +564,20 @@ class GatedDeltaNet(nn.Module): outputs.append(out) result = torch.cat(outputs, dim=0) - if torch.isnan(result).any(): + # thrust::all_of early termination pattern (bench/all_of/basic.cu): + # Check a small sample first — if no NaN in sample, skip full scan. + # MismatchAt=0.01 insight: NaN usually appears early or everywhere. + # Sample first 64 elements + last 64 — covers most failure modes. + _n = result.numel() + _sample_ok = True + if _n > 128: + _s = result.view(-1) + _sample_ok = not (torch.isnan(_s[:64]).any() or torch.isnan(_s[-64:]).any()) + if not _sample_ok or (_n <= 128 and torch.isnan(result).any()): + # Full scan only when sample detected NaN + nan_frac = torch.isnan(result).float().mean().item() logger.warning("NaN in prefill GatedDeltaNet layer %d (frac=%.4f), replacing with zeros", - self.layer_idx, torch.isnan(result).float().mean().item()) + self.layer_idx, nan_frac) result = torch.nan_to_num(result, nan=0.0) return result @@ -649,9 +660,18 @@ class GatedDeltaNet(nn.Module): z.reshape(-1, self.head_v_dim)) normed = normed.reshape(num_seqs, -1) out, _ = self.out_proj(normed) - if torch.isnan(out).any(): + # thrust::all_of early termination: sample check before full scan + _n = out.numel() + _has_nan = False + if _n > 128: + _s = out.view(-1) + _has_nan = torch.isnan(_s[:64]).any() or torch.isnan(_s[-64:]).any() + else: + _has_nan = torch.isnan(out).any().item() + if _has_nan: + nan_frac = torch.isnan(out).float().mean().item() logger.warning("NaN in decode GatedDeltaNet layer %d (frac=%.4f), replacing with zeros", - self.layer_idx, torch.isnan(out).float().mean().item()) + self.layer_idx, nan_frac) out = torch.nan_to_num(out, nan=0.0) return out