[API, Feature] Support response prefill for openai API (#1490)

2024-09-22 06:46:17 -07:00
parent 39bb49d156
commit e4780cf839
4 changed files with 74 additions and 2 deletions
--- a/test/srt/test_large_max_new_tokens.py
+++ b/test/srt/test_large_max_new_tokens.py
@@ -27,7 +27,7 @@ class TestOpenAIServer(unittest.TestCase):
            cls.base_url,
            timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
            api_key=cls.api_key,
-            other_args=("--max-total-token", "1024"),
+            other_args=("--max-total-token", "1024", "--context-len", "8192"),
            env={"SGLANG_CLIP_MAX_NEW_TOKENS": "256", **os.environ},
            return_stdout_stderr=True,
        )