Revert "Revert "[FEAT] Support GGUF format"" (#2287)

test/srt/run_suite.py

@@ -15,6 +15,7 @@ suites = {
         "test_double_sparsity.py",
         "test_embedding_openai_server.py",
         "test_eval_accuracy_mini.py",
+        "test_gguf.py",
         "test_input_embeddings.py",
         "test_json_constrained.py",
         "test_large_max_new_tokens.py",

test/srt/test_get_weights_by_name.py

@@ -16,7 +16,7 @@ from sglang.test.test_utils import (
 from sglang.utils import terminate_process
 
 
-class TestGetParameterByName(unittest.TestCase):
+class TestGetWeightsByName(unittest.TestCase):
     @classmethod
     def setUpClass(cls):
         cls.model = DEFAULT_SMALL_MODEL_NAME_FOR_TEST

test/srt/test_gguf.py

@@ -0,0 +1,26 @@
+import unittest
+
+from huggingface_hub import hf_hub_download
+
+import sglang as sgl
+
+
+class TestGGUF(unittest.TestCase):
+    def test_models(self):
+        prompt = "Today is a sunny day and I like"
+        sampling_params = {"temperature": 0, "max_new_tokens": 8}
+
+        model_path = hf_hub_download(
+            "Qwen/Qwen2-1.5B-Instruct-GGUF",
+            filename="qwen2-1_5b-instruct-q4_k_m.gguf",
+        )
+
+        engine = sgl.Engine(model_path=model_path, random_seed=42)
+        outputs = engine.generate(prompt, sampling_params)["text"]
+        engine.shutdown()
+
+        self.assertEqual(outputs, " it. I have a lot of work")
+
+
+if __name__ == "__main__":
+    unittest.main()