EngineX-Hygon/sglang
5
0
Fork 0
Code Issues Pull Requests Actions 7 Projects Releases Wiki Activity

Filter empty prompt in random bench serving (#2011)

Browse Source
This commit is contained in:
Ke Bao
2024-11-12 14:53:41 +08:00
committed by GitHub
parent 602ebc661d
commit b808a38365
1 changed files with 10 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
python/sglang/bench_serving.py
View File

@@ -596,12 +596,20 @@ def sample_random_requests(
# Filter out sequences that are too long or too short # Filter out sequences that are too long or too short
input_requests: List[Tuple[str, int, int]] = [] input_requests: List[Tuple[str, int, int]] = []
for i in range(num_prompts): for data in dataset:
i = len(input_requests)
if i == num_prompts:
break
# Tokenize the prompts and completions. # Tokenize the prompts and completions.
prompt = dataset[i][0] prompt = data[0]
prompt_token_ids = tokenizer.encode(prompt) prompt_token_ids = tokenizer.encode(prompt)
prompt_len = len(prompt_token_ids) prompt_len = len(prompt_token_ids)
# Skip empty prompt
if prompt_len == 0:
continue
if prompt_len > input_lens[i]: if prompt_len > input_lens[i]:
input_ids = prompt_token_ids[: input_lens[i]] input_ids = prompt_token_ids[: input_lens[i]]
else: else: