From 474317f2b606a79ac6811798c612d13b83f719fd Mon Sep 17 00:00:00 2001 From: Jani Monoses Date: Tue, 3 Sep 2024 07:49:40 +0300 Subject: [PATCH] Support Phi3 mini and medium (#1299) --- python/sglang/srt/hf_transformers_utils.py | 2 +- python/sglang/srt/models/llama.py | 16 ++++++++++------ 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/python/sglang/srt/hf_transformers_utils.py b/python/sglang/srt/hf_transformers_utils.py index bfdeebdc9..ae3070c5a 100644 --- a/python/sglang/srt/hf_transformers_utils.py +++ b/python/sglang/srt/hf_transformers_utils.py @@ -92,7 +92,7 @@ def get_context_length(config): """Get the context length of a model from a huggingface model configs.""" rope_scaling = getattr(config, "rope_scaling", None) if rope_scaling: - rope_scaling_factor = config.rope_scaling["factor"] + rope_scaling_factor = config.rope_scaling.get("factor", 1) if "original_max_position_embeddings" in rope_scaling: rope_scaling_factor = 1 if config.rope_scaling.get("rope_type", None) == "llama3": diff --git a/python/sglang/srt/models/llama.py b/python/sglang/srt/models/llama.py index 43c7cd54a..b875e0c98 100644 --- a/python/sglang/srt/models/llama.py +++ b/python/sglang/srt/models/llama.py @@ -324,11 +324,11 @@ class LlamaForCausalLM(nn.Module): def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]): stacked_params_mapping = [ # (param_name, shard_name, shard_id) - ("qkv_proj", "q_proj", "q"), - ("qkv_proj", "k_proj", "k"), - ("qkv_proj", "v_proj", "v"), - ("gate_up_proj", "gate_proj", 0), - ("gate_up_proj", "up_proj", 1), + (".qkv_proj", ".q_proj", "q"), + (".qkv_proj", ".k_proj", "k"), + (".qkv_proj", ".v_proj", "v"), + (".gate_up_proj", ".gate_proj", 0), + (".gate_up_proj", ".up_proj", 1), ] params_dict = self.param_dict @@ -362,4 +362,8 @@ class LlamaForCausalLM(nn.Module): weight_loader(param, loaded_weight) -EntryClass = LlamaForCausalLM +class Phi3ForCausalLM(LlamaForCausalLM): + pass + + +EntryClass = [LlamaForCausalLM, Phi3ForCausalLM]