feat: support sm75 with FlashInfer v0.1.6 (#1233)

This commit is contained in:
Yineng Zhang
2024-08-28 18:39:12 +10:00
committed by GitHub
parent 6cc38b2bf3
commit 198974cd1a
5 changed files with 4 additions and 12 deletions

View File

@@ -161,6 +161,8 @@ class ModelRunner:
"Compute capability below sm80. Use float16 due to lack of bfloat16 support."
)
self.server_args.dtype = "float16"
if torch.cuda.get_device_capability()[1] < 5:
raise RuntimeError("SGLang only supports sm75 and above.")
monkey_patch_vllm_dummy_weight_loader()
self.device_config = DeviceConfig()