EngineX-Hygon/sglang
5
0
Fork 0
Code Issues Pull Requests Actions 7 Projects Releases Wiki Activity

Organize server_args (#277)

Browse Source
This commit is contained in:
Liangsheng Yin
2024-03-11 20:06:52 +08:00
committed by GitHub
parent faba293a0d
commit 1b35547927
12 changed files with 92 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
python/sglang/srt/layers/radix_attention.py
View File

@@ -15,11 +15,9 @@ class RadixAttention(nn.Module):
self.head_dim = head_dim
self.layer_id = layer_id
from sglang.srt.managers.router.model_runner import global_server_args
from sglang.srt.managers.router.model_runner import global_server_args_dict
self.use_flashinfer = "flashinfer" in global_server_args.model_mode
if self.use_flashinfer:
if global_server_args_dict["enable_flashinfer"]:
self.prefill_forward = self.prefill_forward_flashinfer
self.extend_forward = self.prefill_forward_flashinfer
self.decode_forward = self.decode_forward_flashinfer