support server arg override KV cache to bf16 to avoid slow cases (#11749)

2025-10-18 11:49:48 -07:00
parent a93f10a722
commit f9a7d9b3dc
3 changed files with 5 additions and 3 deletions
--- a/python/sglang/srt/model_executor/model_runner.py
+++ b/python/sglang/srt/model_executor/model_runner.py
@@ -1567,6 +1567,8 @@ class ModelRunner:
                self.kv_cache_dtype = torch.float8_e4m3fnuz
            else:
                self.kv_cache_dtype = torch.float8_e4m3fn
+        elif self.server_args.kv_cache_dtype in ("bf16", "bfloat16"):
+            self.kv_cache_dtype = torch.bfloat16
        else:
            raise ValueError(
                f"Unsupported kv_cache_dtype: {self.server_args.kv_cache_dtype}."