Add activation parameters to fused_moe (#3170)

2025-01-27 00:23:37 -08:00
parent 741fccd7bf
commit 52c03f16b9
7 changed files with 52 additions and 7 deletions
--- a/python/sglang/srt/models/grok.py
+++ b/python/sglang/srt/models/grok.py
@@ -133,6 +133,7 @@ class Grok1MoE(nn.Module):
            renormalize=False,
            quant_config=quant_config,
            tp_size=tp_size,
+            activation="gelu",
            use_presharded_weights=use_presharded_weights,
        )