EngineX-Hygon/sglang
5
0
Fork 0
Code Issues Pull Requests Actions 7 Projects Releases Wiki Activity

Configuration qwen2_moe.py - qkv_bias now in transformers (#5512)

Browse Source
This commit is contained in:
Michael Feil
2025-04-17 21:23:22 -07:00
committed by GitHub
parent a0fc5bc144
commit 1effba4c70
1 changed files with 1 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
python/sglang/srt/models/qwen2_moe.py
View File

@@ -262,8 +262,7 @@ class Qwen2MoeDecoderLayer(nn.Module):
rope_theta = getattr(config, "rope_theta", 10000) rope_theta = getattr(config, "rope_theta", 10000)
rope_scaling = getattr(config, "rope_scaling", None) rope_scaling = getattr(config, "rope_scaling", None)
max_position_embeddings = getattr(config, "max_position_embeddings", 8192) max_position_embeddings = getattr(config, "max_position_embeddings", 8192)
# note: replace config.num_hidden_layers < 80 with True once its available in transformers 4.50.0 qkv_bias = getattr(config, "qkv_bias", True)
qkv_bias = getattr(config, "qkv_bias", config.num_hidden_layers < 80)
self.self_attn = Qwen2MoeAttention( self.self_attn = Qwen2MoeAttention(
hidden_size=self.hidden_size, hidden_size=self.hidden_size,
num_heads=config.num_attention_heads, num_heads=config.num_attention_heads,