EngineX-Hygon/sglang
5
0
Fork 0
Code Issues Pull Requests Actions 7 Projects Releases Wiki Activity

Add lora_paths to v1_chat_generate_request (#2529)

Browse Source
This commit is contained in:
Lei
2024-12-22 02:23:33 -08:00
committed by GitHub
parent 4e1e3cff20
commit 19ba2b0ea9
1 changed files with 4 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
python/sglang/srt/openai_api/adapter.py
View File

@@ -858,6 +858,7 @@ def v1_chat_generate_request(
logprob_start_lens = [] logprob_start_lens = []
top_logprobs_nums = [] top_logprobs_nums = []
modalities_list = [] modalities_list = []
lora_paths = []
# NOTE: with openai API, the prompt's logprobs are always not computed # NOTE: with openai API, the prompt's logprobs are always not computed
@@ -920,6 +921,7 @@ def v1_chat_generate_request(
return_logprobs.append(request.logprobs) return_logprobs.append(request.logprobs)
logprob_start_lens.append(-1) logprob_start_lens.append(-1)
top_logprobs_nums.append(request.top_logprobs or 0) top_logprobs_nums.append(request.top_logprobs or 0)
lora_paths.append(request.lora_path)
sampling_params = { sampling_params = {
"temperature": request.temperature, "temperature": request.temperature,
@@ -958,6 +960,7 @@ def v1_chat_generate_request(
logprob_start_lens = logprob_start_lens[0] logprob_start_lens = logprob_start_lens[0]
top_logprobs_nums = top_logprobs_nums[0] top_logprobs_nums = top_logprobs_nums[0]
modalities_list = modalities_list[0] modalities_list = modalities_list[0]
lora_paths = lora_paths[0]
else: else:
if isinstance(input_ids[0], str): if isinstance(input_ids[0], str):
prompt_kwargs = {"text": input_ids} prompt_kwargs = {"text": input_ids}
@@ -975,6 +978,7 @@ def v1_chat_generate_request(
return_text_in_logprobs=True, return_text_in_logprobs=True,
rid=request_ids, rid=request_ids,
modalities=modalities_list, modalities=modalities_list,
lora_path=lora_paths,
) )
return adapted_request, all_requests if len(all_requests) > 1 else all_requests[0] return adapted_request, all_requests if len(all_requests) > 1 else all_requests[0]