From 40facad5f122259c9d6868153859792748ca0d10 Mon Sep 17 00:00:00 2001 From: Toshiki Kataoka Date: Fri, 26 Jul 2024 18:53:17 +0900 Subject: [PATCH] feat: support token ids in /v1/completions (#736) --- python/sglang/srt/openai_api/adapter.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/python/sglang/srt/openai_api/adapter.py b/python/sglang/srt/openai_api/adapter.py index ebb95ea24..760a46e9b 100644 --- a/python/sglang/srt/openai_api/adapter.py +++ b/python/sglang/srt/openai_api/adapter.py @@ -94,9 +94,14 @@ def load_chat_template_for_openai_api(chat_template_arg): async def v1_completions(tokenizer_manager, raw_request: Request): request_json = await raw_request.json() request = CompletionRequest(**request_json) + prompt = request.prompt + if isinstance(prompt, str) or isinstance(prompt[0], str): + prompt_kwargs = {"text": prompt} + else: + prompt_kwargs = {"input_ids": prompt} adapted_request = GenerateReqInput( - text=request.prompt, + **prompt_kwargs, sampling_params={ "temperature": request.temperature, "max_new_tokens": request.max_tokens,