From 027e65248f26845057ed6eef663f33bbcd2602f7 Mon Sep 17 00:00:00 2001 From: Xiaoyu Zhang <35585791+BBuf@users.noreply.github.com> Date: Tue, 12 Nov 2024 15:21:20 +0800 Subject: [PATCH] support echo=true and logprobs in openai api when logprobs=1 in lm-evaluation-harness (#1998) --- python/sglang/srt/openai_api/adapter.py | 6 +++++- test/srt/test_openai_server.py | 4 +++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/python/sglang/srt/openai_api/adapter.py b/python/sglang/srt/openai_api/adapter.py index c1e399241..f3546cf8c 100644 --- a/python/sglang/srt/openai_api/adapter.py +++ b/python/sglang/srt/openai_api/adapter.py @@ -498,6 +498,10 @@ def v1_generate_request( ) prompts.append(request.prompt) + if request.echo and request.logprobs: + current_logprob_start_len = 0 + else: + current_logprob_start_len = -1 sampling_params_list.append( { "temperature": request.temperature, @@ -517,7 +521,7 @@ def v1_generate_request( } ) return_logprobs.append(request.logprobs is not None and request.logprobs > 0) - logprob_start_lens.append(-1) + logprob_start_lens.append(current_logprob_start_len) top_logprobs_nums.append( request.logprobs if request.logprobs is not None else 0 ) diff --git a/test/srt/test_openai_server.py b/test/srt/test_openai_server.py index 048026b8b..1e18e23ef 100644 --- a/test/srt/test_openai_server.py +++ b/test/srt/test_openai_server.py @@ -85,7 +85,9 @@ class TestOpenAIServer(unittest.TestCase): # assert ret_num_top_logprobs == logprobs, f"{ret_num_top_logprobs} vs {logprobs}" assert ret_num_top_logprobs > 0 - assert response.choices[0].logprobs.token_logprobs[0] + # when echo=True and request.logprobs>0, logprob_start_len is 0, so the first token's logprob would be None. + if not echo: + assert response.choices[0].logprobs.token_logprobs[0] assert response.id assert response.created