Fix perf regression on small batch sizes (#3008)

2025-01-20 03:39:49 -08:00
parent 10bfce71b3
commit dc1881326f
2 changed files with 11 additions and 7 deletions
--- a/python/sglang/srt/layers/radix_attention.py
+++ b/python/sglang/srt/layers/radix_attention.py
@@ -47,8 +47,8 @@ class RadixAttention(nn.Module):
        self.logit_cap = logit_cap
        self.sliding_window_size = sliding_window_size or -1
        self.is_cross_attention = is_cross_attention
-        self.k_scale = 1.0
-        self.v_scale = 1.0
+        self.k_scale = None
+        self.v_scale = None

    def forward(
        self,