EngineX-Hygon/sglang
5
0
Fork 0
Code Issues Pull Requests Actions 7 Projects Releases Wiki Activity

Fix nightly-test CI (#3826)

Browse Source
This commit is contained in:
yinfan98
2025-03-03 15:14:45 +08:00
committed by GitHub
parent 728e175fc4
commit b4d34cd35d
1 changed files with 13 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
python/sglang/srt/layers/attention/flashinfer_backend.py
View File

@@ -422,7 +422,10 @@ class FlashInferAttnBackend(AttentionBackend):
else: else:
o2, s2 = prefill_wrapper_paged.forward_return_lse( o2, s2 = prefill_wrapper_paged.forward_return_lse(
q.contiguous().view(-1, layer.tp_q_head_num, layer.head_dim), q.contiguous().view(-1, layer.tp_q_head_num, layer.head_dim),
forward_batch.token_to_kv_pool.get_kv_buffer(layer.layer_id), self._to_dtype(
forward_batch.token_to_kv_pool.get_kv_buffer(layer.layer_id),
q.dtype,
),
causal=False, causal=False,
sm_scale=layer.scaling, sm_scale=layer.scaling,
logits_soft_cap=layer.logit_cap, logits_soft_cap=layer.logit_cap,
@@ -464,7 +467,9 @@ class FlashInferAttnBackend(AttentionBackend):
o = decode_wrapper.forward( o = decode_wrapper.forward(
q.contiguous().view(-1, layer.tp_q_head_num, layer.head_dim), q.contiguous().view(-1, layer.tp_q_head_num, layer.head_dim),
forward_batch.token_to_kv_pool.get_kv_buffer(layer.layer_id), self._to_dtype(
forward_batch.token_to_kv_pool.get_kv_buffer(layer.layer_id), q.dtype
),
sm_scale=layer.scaling, sm_scale=layer.scaling,
logits_soft_cap=layer.logit_cap, logits_soft_cap=layer.logit_cap,
k_scale=layer.k_scale, k_scale=layer.k_scale,
@@ -473,6 +478,12 @@ class FlashInferAttnBackend(AttentionBackend):
return o.view(-1, layer.tp_q_head_num * layer.head_dim) return o.view(-1, layer.tp_q_head_num * layer.head_dim)
def _to_dtype(self, kv_tuple, dtype):
if kv_tuple[0].dtype != dtype:
return tuple(t.to(dtype) for t in kv_tuple)
else:
return kv_tuple
def _get_wrapper_idx(self, layer: RadixAttention): def _get_wrapper_idx(self, layer: RadixAttention):
if self.num_wrappers == 1: if self.num_wrappers == 1:
return 0 return 0