Deepseek v2 support (#693)

2024-07-26 17:10:07 -07:00
parent 5bd06b4599
commit 679ebcbbdc
5 changed files with 538 additions and 6 deletions
--- a/python/sglang/srt/hf_transformers_utils.py
+++ b/python/sglang/srt/hf_transformers_utils.py
@@ -73,7 +73,7 @@ def get_context_length(config):
    rope_scaling = getattr(config, "rope_scaling", None)
    if rope_scaling:
        rope_scaling_factor = config.rope_scaling["factor"]
-        if config.rope_scaling["rope_type"] == "llama3":
+        if config.rope_scaling.get("rope_type", None) == "llama3":
            rope_scaling_factor = 1
    else:
        rope_scaling_factor = 1