Revert "Revert "[FEAT] Support GGUF format"" (#2287)

2024-11-30 22:14:48 -08:00
parent 1bfa511b95
commit 4936be8acc
41 changed files with 229 additions and 132 deletions
--- a/python/sglang/srt/layers/vocab_parallel_embedding.py
+++ b/python/sglang/srt/layers/vocab_parallel_embedding.py
@@ -222,6 +222,7 @@ class VocabParallelEmbedding(torch.nn.Module):
        enable_tp: bool = True,
    ):
        super().__init__()
+        self.quant_config = quant_config

        self.enable_tp = enable_tp
        if self.enable_tp: