From 95dc093b195e5999699cd7bdba60867c7e60fc92 Mon Sep 17 00:00:00 2001 From: Daniel Hernandez Garcia Date: Tue, 2 Jul 2024 06:10:07 +0100 Subject: [PATCH] [BugFix] gemma loading weights "lm_head.weight" key error (#577) --- python/sglang/srt/models/gemma.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/python/sglang/srt/models/gemma.py b/python/sglang/srt/models/gemma.py index b8896ef88..2281b4c80 100644 --- a/python/sglang/srt/models/gemma.py +++ b/python/sglang/srt/models/gemma.py @@ -310,6 +310,10 @@ class GemmaForCausalLM(nn.Module): weight_loader(param, loaded_weight, shard_id) break else: + # lm_head is not used in vllm as it is tied with embed_token. + # To prevent errors, skip loading lm_head.weight. + if "lm_head.weight" in name: + continue # Skip loading extra bias for GPTQ models. if name.endswith(".bias") and name not in params_dict: continue