Support Phi3 mini and medium (#1299)

2024-09-03 07:49:40 +03:00
parent f64eae3a29
commit 474317f2b6
2 changed files with 11 additions and 7 deletions
--- a/python/sglang/srt/hf_transformers_utils.py
+++ b/python/sglang/srt/hf_transformers_utils.py
@@ -92,7 +92,7 @@ def get_context_length(config):
    """Get the context length of a model from a huggingface model configs."""
    rope_scaling = getattr(config, "rope_scaling", None)
    if rope_scaling:
-        rope_scaling_factor = config.rope_scaling["factor"]
+        rope_scaling_factor = config.rope_scaling.get("factor", 1)
        if "original_max_position_embeddings" in rope_scaling:
            rope_scaling_factor = 1
        if config.rope_scaling.get("rope_type", None) == "llama3":