EngineX-Hygon/sglang
5
0
Fork 0
Code Issues Pull Requests Actions 7 Projects Releases Wiki Activity

support echo=true and logprobs in openai api when logprobs=1 in lm-evaluation-harness (#1998)

Browse Source
This commit is contained in:
Xiaoyu Zhang
2024-11-12 15:21:20 +08:00
committed by GitHub
parent b808a38365
commit 027e65248f
2 changed files with 8 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
python/sglang/srt/openai_api/adapter.py
View File

@@ -498,6 +498,10 @@ def v1_generate_request(
) )
prompts.append(request.prompt) prompts.append(request.prompt)
if request.echo and request.logprobs:
current_logprob_start_len = 0
else:
current_logprob_start_len = -1
sampling_params_list.append( sampling_params_list.append(
{ {
"temperature": request.temperature, "temperature": request.temperature,
@@ -517,7 +521,7 @@ def v1_generate_request(
} }
) )
return_logprobs.append(request.logprobs is not None and request.logprobs > 0) return_logprobs.append(request.logprobs is not None and request.logprobs > 0)
logprob_start_lens.append(-1) logprob_start_lens.append(current_logprob_start_len)
top_logprobs_nums.append( top_logprobs_nums.append(
request.logprobs if request.logprobs is not None else 0 request.logprobs if request.logprobs is not None else 0
) )

4
test/srt/test_openai_server.py
View File

@@ -85,7 +85,9 @@ class TestOpenAIServer(unittest.TestCase):
# assert ret_num_top_logprobs == logprobs, f"{ret_num_top_logprobs} vs {logprobs}" # assert ret_num_top_logprobs == logprobs, f"{ret_num_top_logprobs} vs {logprobs}"
assert ret_num_top_logprobs > 0 assert ret_num_top_logprobs > 0
assert response.choices[0].logprobs.token_logprobs[0] # when echo=True and request.logprobs>0, logprob_start_len is 0, so the first token's logprob would be None.
if not echo:
assert response.choices[0].logprobs.token_logprobs[0]
assert response.id assert response.id
assert response.created assert response.created