Support loading of larger models with on-the-fly quantization (#3061)

2025-01-22 21:33:17 -08:00
parent 8b84e69f25
commit 862bcff833
6 changed files with 116 additions and 14 deletions
--- a/python/sglang/srt/configs/load_config.py
+++ b/python/sglang/srt/configs/load_config.py
@@ -20,6 +20,7 @@ class LoadFormat(str, enum.Enum):
    GGUF = "gguf"
    BITSANDBYTES = "bitsandbytes"
    MISTRAL = "mistral"
+    LAYERED = "layered"


@dataclass