8 lines
286 B
Python
8 lines
286 B
Python
|
import torch
|
||
|
|
|
||
|
|
def SiluAndMul_forward_vacc(self, x: torch.Tensor) -> torch.Tensor:
|
||
|
|
return torch.vacc.swiglu(x)
|
||
|
|
|
||
|
|
def QuickGELU_forward_vacc(self, x: torch.Tensor) -> torch.Tensor:
|
||
|
|
"""PyTorch-native implementation equivalent to forward()."""
|
||
|
|
return x * torch.sigmoid(1.702 * x)
|