EngineX-Hygon/sglang
5
0
Fork 0
Code Issues Pull Requests Actions 7 Projects Releases Wiki Activity

Replace pad with cat for better performance (#11388)

Browse Source 
Co-authored-by: luoyuan.luo <luoyuan.luo@antgroup.com>
This commit is contained in:
Yuan Luo
2025-10-10 12:03:17 +08:00
committed by GitHub
parent 70fbb3adf6
commit b5044fbf12
5 changed files with 5 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
python/sglang/srt/models/dots_vlm_vit.py
View File

@@ -323,7 +323,7 @@ class DotsVisionTransformer(PreTrainedModel):
dim=0, dim=0,
dtype=grid_thw.dtype if torch.jit.is_tracing() else torch.int32, dtype=grid_thw.dtype if torch.jit.is_tracing() else torch.int32,
) )
cu_seqlens = F.pad(cu_seqlens, (1, 0), value=0) cu_seqlens = torch.cat([cu_seqlens.new_zeros(1), cu_seqlens])
for blk in self.blocks: for blk in self.blocks:
hidden_states = blk( hidden_states = blk(

2
python/sglang/srt/models/glm4v.py
View File

@@ -434,7 +434,7 @@ class Glm4vVisionModel(nn.Module):
cu_seqlens = torch.repeat_interleave( cu_seqlens = torch.repeat_interleave(
grid_thw[:, 1] * grid_thw[:, 2], grid_thw[:, 0] grid_thw[:, 1] * grid_thw[:, 2], grid_thw[:, 0]
).cumsum(dim=0, dtype=torch.int32) ).cumsum(dim=0, dtype=torch.int32)
cu_seqlens = F.pad(cu_seqlens, (1, 0), "constant", 0) cu_seqlens = torch.cat([cu_seqlens.new_zeros(1), cu_seqlens])
seqlens = (cu_seqlens[1:] - cu_seqlens[:-1]).tolist() seqlens = (cu_seqlens[1:] - cu_seqlens[:-1]).tolist()
x = self.embeddings( x = self.embeddings(

2
python/sglang/srt/models/qwen2_5_vl.py
View File

@@ -436,7 +436,7 @@ class Qwen2_5_VisionTransformer(nn.Module):
.to(device=x.device, dtype=torch.int32), .to(device=x.device, dtype=torch.int32),
] ]
) )
cu_seqlens = F.pad(cu_seqlens, (1, 0), "constant", 0) cu_seqlens = torch.cat([cu_seqlens.new_zeros(1), cu_seqlens])
# transformers # transformers
x = x.unsqueeze(1) x = x.unsqueeze(1)

2
python/sglang/srt/models/qwen2_vl.py
View File

@@ -407,7 +407,7 @@ class Qwen2VisionTransformer(nn.Module):
cu_seqlens = torch.repeat_interleave( cu_seqlens = torch.repeat_interleave(
grid_thw[:, 1] * grid_thw[:, 2], grid_thw[:, 0] grid_thw[:, 1] * grid_thw[:, 2], grid_thw[:, 0]
).cumsum(dim=0, dtype=torch.int32) ).cumsum(dim=0, dtype=torch.int32)
cu_seqlens = F.pad(cu_seqlens, (1, 0), "constant", 0) cu_seqlens = torch.cat([cu_seqlens.new_zeros(1), cu_seqlens])
# transformers # transformers
x = x.unsqueeze(1) x = x.unsqueeze(1)

2
python/sglang/srt/models/qwen3_vl.py
View File

@@ -458,7 +458,7 @@ class Qwen3_VisionTransformer(nn.Module):
(grid_thw[:, 0] * grid_thw[:, 1] * grid_thw[:, 2]).cumsum(dim=0), (grid_thw[:, 0] * grid_thw[:, 1] * grid_thw[:, 2]).cumsum(dim=0),
] ]
) )
cu_seqlens = F.pad(cu_seqlens, (1, 0), "constant", 0) cu_seqlens = torch.cat([cu_seqlens.new_zeros(1), cu_seqlens])
# max_seqlen, seqlens = self.compute_attn_mask_seqlen(cu_seqlens) # max_seqlen, seqlens = self.compute_attn_mask_seqlen(cu_seqlens)
x = x.unsqueeze(1) x = x.unsqueeze(1)