EngineX-Hygon/sglang
5
0
Fork 0
Code Issues Pull Requests Actions 7 Projects Releases Wiki Activity

Logprobs Refractor (#331)

Browse Source
This commit is contained in:
Liangsheng Yin
2024-03-28 14:34:49 +08:00
committed by GitHub
parent 24e59f5350
commit 3842eba5fa
14 changed files with 385 additions and 152 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
test/srt/test_httpserver_decode.py
View File

@@ -9,11 +9,12 @@ The capital of France is Paris.\nThe capital of the United States is Washington,
"""
import argparse
import json
import requests
def test_decode(url, return_logprob):
def test_decode(url, return_logprob, top_logprobs_num, return_text):
response = requests.post(
url + "/generate",
json={
@@ -23,10 +24,13 @@ def test_decode(url, return_logprob):
"max_new_tokens": 32,
},
"return_logprob": return_logprob,
"top_logprobs_num": top_logprobs_num,
"return_text_in_logprobs": return_text,
"logprob_start_len": 0,
},
)
print(response.json())
print(json.dumps(response.json()))
print("=" * 100)
if __name__ == "__main__":
@@ -37,5 +41,8 @@ if __name__ == "__main__":
url = f"{args.host}:{args.port}"
test_decode(url, False)
test_decode(url, True)
test_decode(url, False, 0, False)
test_decode(url, True, 0, False)
test_decode(url, True, 0, True)
test_decode(url, True, 3, False)
test_decode(url, True, 3, True)

26
test/srt/test_httpserver_decode_stream.py
View File

@@ -13,7 +13,7 @@ import json
import requests
def test_decode_stream(url, return_logprob):
def test_decode_stream(url, return_logprob, top_logprobs_num):
response = requests.post(
url + "/generate",
json={
@@ -24,6 +24,8 @@ def test_decode_stream(url, return_logprob):
},
"stream": True,
"return_logprob": return_logprob,
"top_logprobs_num": top_logprobs_num,
"return_text_in_logprobs": True,
},
stream=True,
)
@@ -37,19 +39,20 @@ def test_decode_stream(url, return_logprob):
data = json.loads(chunk[5:].strip("\n"))
if return_logprob:
assert data["meta_info"]["prompt_logprob"] is not None
assert data["meta_info"]["token_logprob"] is not None
assert data["meta_info"]["prefill_token_logprobs"] is not None
assert data["meta_info"]["decode_token_logprobs"] is not None
assert data["meta_info"]["normalized_prompt_logprob"] is not None
if prev == 0: # Skip prompt logprobs
prev = data["meta_info"]["prompt_tokens"]
for token_txt, _, logprob in data["meta_info"]["token_logprob"][prev:]:
print(f"{token_txt}\t{logprob}", flush=True)
prev = len(data["meta_info"]["token_logprob"])
for logprob, token_id, token_text in data["meta_info"][
"decode_token_logprobs"
][prev:]:
print(f"{token_text:12s}\t{logprob}\t{token_id}", flush=True)
prev = len(data["meta_info"]["decode_token_logprobs"])
else:
output = data["text"].strip()
print(output[prev:], end="", flush=True)
prev = len(output)
print("")
print("=" * 100)
if __name__ == "__main__":
@@ -60,5 +63,6 @@ if __name__ == "__main__":
url = f"{args.host}:{args.port}"
test_decode_stream(url, False)
test_decode_stream(url, True)
test_decode_stream(url, False, 0)
test_decode_stream(url, True, 0)
test_decode_stream(url, True, 3)

14
test/srt/test_openai_server.py
View File

@@ -34,6 +34,7 @@ def test_completion(args, echo, logprobs):
if echo:
assert text.startswith("The capital of France is")
if logprobs:
print(response.choices[0].logprobs.top_logprobs)
assert response.choices[0].logprobs
if echo:
assert response.choices[0].logprobs.token_logprobs[0] == None
@@ -44,6 +45,7 @@ def test_completion(args, echo, logprobs):
assert response.usage.prompt_tokens > 0
assert response.usage.completion_tokens > 0
assert response.usage.total_tokens > 0
print("=" * 100)
def test_completion_stream(args, echo, logprobs):
@@ -68,13 +70,14 @@ def test_completion_stream(args, echo, logprobs):
f"{r.choices[0].text:12s}\t" f"{r.choices[0].logprobs.token_logprobs}",
flush=True,
)
print(r.choices[0].logprobs.top_logprobs)
else:
print(r.choices[0].text, end="", flush=True)
assert r.id
assert r.usage.prompt_tokens > 0
assert r.usage.completion_tokens > 0
assert r.usage.total_tokens > 0
print()
print("=" * 100)
def test_chat_completion(args):
@@ -94,6 +97,7 @@ def test_chat_completion(args):
assert response.usage.prompt_tokens > 0
assert response.usage.completion_tokens > 0
assert response.usage.total_tokens > 0
print("=" * 100)
def test_chat_completion_image(args):
@@ -124,6 +128,7 @@ def test_chat_completion_image(args):
assert response.usage.prompt_tokens > 0
assert response.usage.completion_tokens > 0
assert response.usage.total_tokens > 0
print("=" * 100)
def test_chat_completion_stream(args):
@@ -149,7 +154,7 @@ def test_chat_completion_stream(args):
if not data.content:
continue
print(data.content, end="", flush=True)
print()
print("=" * 100)
def test_regex(args):
@@ -174,6 +179,7 @@ def test_regex(args):
)
text = response.choices[0].message.content
print(json.loads(text))
print("=" * 100)
if __name__ == "__main__":
@@ -188,10 +194,14 @@ if __name__ == "__main__":
test_completion(args, echo=True, logprobs=False)
test_completion(args, echo=False, logprobs=True)
test_completion(args, echo=True, logprobs=True)
test_completion(args, echo=False, logprobs=3)
test_completion(args, echo=True, logprobs=3)
test_completion_stream(args, echo=False, logprobs=False)
test_completion_stream(args, echo=True, logprobs=False)
test_completion_stream(args, echo=False, logprobs=True)
test_completion_stream(args, echo=True, logprobs=True)
test_completion_stream(args, echo=False, logprobs=3)
test_completion_stream(args, echo=True, logprobs=3)
test_chat_completion(args)
test_chat_completion_stream(args)
test_regex(args)