TokenizerManager.context_len should inherit from `server_args.conte… (#654)

2024-07-19 14:25:29 +09:30
parent a9ef49c12c
commit 1b7adbb5a0
1 changed files with 4 additions and 1 deletions
--- a/python/sglang/srt/managers/tokenizer_manager.py
+++ b/python/sglang/srt/managers/tokenizer_manager.py
@@ -69,7 +69,10 @@ class TokenizerManager:
            trust_remote_code=server_args.trust_remote_code,
            model_overide_args=model_overide_args,
        )
-        self.context_len = get_context_length(self.hf_config)
+        if server_args.context_length is not None:
+            self.context_len = server_args.context_length
+        else:
+            self.context_len = get_context_length(self.hf_config)

        if is_multimodal_model(self.model_path):
            self.processor = get_processor(