EngineX-Hygon/sglang
5
0
Fork 0
Code Issues Pull Requests Actions 7 Projects Releases Wiki Activity

Turn off autotune for scaled mm for fp8 dynamic quant in torchao (#2116)

Browse Source
This commit is contained in:
Jerry Zhang
2024-11-21 12:19:49 -08:00
committed by GitHub
parent 5c6a41facf
commit 7f8fcd39cd
1 changed files with 4 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
python/sglang/srt/models/torch_native_llama.py
View File

@@ -401,6 +401,10 @@ class TorchNativeLlamaForCausalLM(nn.Module):
self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size) self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
self.logits_processor = LogitsProcessor(config) self.logits_processor = LogitsProcessor(config)
# turning off autotune for fp8dq since it doesn't give speedup and
# increases compile time significantly
torch._inductor.config.max_autotune_gemm_backends = "ATEN"
@torch.no_grad() @torch.no_grad()
def forward( def forward(
self, self,