Support page size > 1 (#4356)

2025-03-12 22:22:39 -07:00
parent 2f6bacee03
commit c76040e31b
23 changed files with 877 additions and 284 deletions
--- a/python/sglang/srt/layers/quantization/fp8_kernel.py
+++ b/python/sglang/srt/layers/quantization/fp8_kernel.py
@@ -36,7 +36,7 @@ fp8_type_ = torch.float8_e4m3fnuz if _is_hip else torch.float8_e4m3fn

 _is_cuda = is_cuda()
 if _is_cuda:
-    import deep_gemm
+    import deep_gemm  # `pip install "sgl-kernel>=0.0.4.post3"`
    from sgl_kernel import sgl_per_token_group_quant_fp8, sgl_per_token_quant_fp8

 logger = logging.getLogger(__name__)