EngineX-Hygon/sglang
5
0
Fork 0
Code Issues Pull Requests Actions 7 Projects Releases Wiki Activity

Fix EAGLE3 for llama3.3 70b (#4716)

Browse Source
This commit is contained in:
Ke Bao
2025-03-25 08:31:19 +08:00
committed by GitHub
parent 8154de5a32
commit 77cf771ebe
2 changed files with 10 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
python/sglang/srt/models/llama.py
View File

@@ -610,6 +610,12 @@ class LlamaForCausalLM(nn.Module):
return self.model.embed_tokens.weight return self.model.embed_tokens.weight
def set_embed(self, embed): def set_embed(self, embed):
# NOTE: If draft hidden size != target hidden size, the embed weight cannot be shared for EAGLE3
if (
hasattr(self.config, "target_hidden_size")
and self.config.target_hidden_size != self.config.hidden_size
):
return
del self.model.embed_tokens.weight del self.model.embed_tokens.weight
self.model.embed_tokens.weight = embed self.model.embed_tokens.weight = embed
torch.cuda.empty_cache() torch.cuda.empty_cache()

3
python/sglang/srt/models/llama_eagle3.py
View File

@@ -105,6 +105,9 @@ class LlamaModel(nn.Module):
prefix=add_prefix("embed_tokens", prefix), prefix=add_prefix("embed_tokens", prefix),
) )
self.midlayer = LlamaDecoderLayer(config, 0, quant_config, prefix) self.midlayer = LlamaDecoderLayer(config, 0, quant_config, prefix)
if hasattr(config, "target_hidden_size"):
self.fc = torch.nn.Linear(config.target_hidden_size * 3, config.hidden_size)
else:
self.fc = torch.nn.Linear(config.hidden_size * 3, config.hidden_size) self.fc = torch.nn.Linear(config.hidden_size * 3, config.hidden_size)
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)