EngineX-Hygon/sglang
5
0
Fork 0
Code Issues Pull Requests Actions 7 Projects Releases Wiki Activity

Support disable_ignore_eos in bench_serving.py (#824)

Browse Source
This commit is contained in:
Ying Sheng
2024-07-30 01:42:07 -07:00
committed by GitHub
parent a30d5d75bf
commit ae5c0fc442
1 changed files with 9 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
python/sglang/bench_serving.py
View File

@@ -84,6 +84,9 @@ async def async_request_trt_llm(
"min_length": request_func_input.output_len, "min_length": request_func_input.output_len,
"end_id": 1048576, "end_id": 1048576,
} }
if args.disable_ignore_eos:
del payload["min_length"]
del payload["end_id"]
output = RequestFuncOutput() output = RequestFuncOutput()
output.prompt_len = request_func_input.prompt_len output.prompt_len = request_func_input.prompt_len
@@ -149,7 +152,7 @@ async def async_request_openai_completions(
"best_of": 1, "best_of": 1,
"max_tokens": request_func_input.output_len, "max_tokens": request_func_input.output_len,
"stream": not args.disable_stream, "stream": not args.disable_stream,
"ignore_eos": True, "ignore_eos": not args.disable_ignore_eos,
} }
headers = {"Authorization": f"Bearer {os.environ.get('OPENAI_API_KEY')}"} headers = {"Authorization": f"Bearer {os.environ.get('OPENAI_API_KEY')}"}
@@ -969,6 +972,11 @@ if __name__ == "__main__":
action="store_true", action="store_true",
help="Disable streaming mode.", help="Disable streaming mode.",
) )
parser.add_argument(
"--disable-ignore-eos",
action="store_true",
help="Disable ignoring EOS.",
)
set_ulimit() set_ulimit()