From d9fccfefe27c9d63212698578084aaf9f688a066 Mon Sep 17 00:00:00 2001 From: Liangsheng Yin Date: Fri, 26 Jul 2024 18:13:13 -0700 Subject: [PATCH] Fix context length (#757) --- python/sglang/srt/hf_transformers_utils.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/python/sglang/srt/hf_transformers_utils.py b/python/sglang/srt/hf_transformers_utils.py index fb4635558..224b1396b 100644 --- a/python/sglang/srt/hf_transformers_utils.py +++ b/python/sglang/srt/hf_transformers_utils.py @@ -73,6 +73,8 @@ def get_context_length(config): rope_scaling = getattr(config, "rope_scaling", None) if rope_scaling: rope_scaling_factor = config.rope_scaling["factor"] + if "original_max_position_embeddings" in rope_scaling: + rope_scaling_factor = 1 if config.rope_scaling.get("rope_type", None) == "llama3": rope_scaling_factor = 1 else: