EngineX-Hygon/sglang
5
0
Fork 0
Code Issues Pull Requests Actions 7 Projects Releases Wiki Activity

Fix bench latency benchmark (#1225)

Browse Source
This commit is contained in:
Liangsheng Yin
2024-08-28 00:37:32 -07:00
committed by GitHub
parent c8a9e79186
commit 1ece2cda3d
2 changed files with 9 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
.github/workflows/e2e-test.yml vendored
View File

@@ -38,6 +38,11 @@ jobs:
cd test/srt cd test/srt
python3 -m unittest test_serving_throughput.TestServingThroughput.test_default python3 -m unittest test_serving_throughput.TestServingThroughput.test_default
- name: Benchmark Serving Latency
timeout-minutes: 10
run: |
python3 -m sglang.bench_latency --model meta-llama/Meta-Llama-3.1-8B-Instruct --batch-size 1 --input 128 --output 8
- name: Benchmark Serving Throughput (w/o RadixAttention) - name: Benchmark Serving Throughput (w/o RadixAttention)
timeout-minutes: 10 timeout-minutes: 10
run: | run: |

10
python/sglang/bench_latency.py
View File

@@ -200,16 +200,14 @@ def extend(reqs, model_runner):
tree_cache=None, tree_cache=None,
) )
batch.prepare_for_extend(model_runner.model_config.vocab_size) batch.prepare_for_extend(model_runner.model_config.vocab_size)
output = model_runner.forward(batch, ForwardMode.EXTEND) sample_output, logits_output = model_runner.forward(batch, ForwardMode.EXTEND)
next_token_ids = batch.sample(output.next_token_logits) return sample_output.batch_next_token_ids, logits_output.next_token_logits, batch
return next_token_ids, output.next_token_logits, batch
def decode(input_token_ids, batch, model_runner): def decode(input_token_ids, batch, model_runner):
batch.prepare_for_decode(input_token_ids.cpu().numpy()) batch.prepare_for_decode(input_token_ids.cpu().numpy())
output = model_runner.forward(batch, ForwardMode.DECODE) sample_output, logits_output = model_runner.forward(batch, ForwardMode.DECODE)
next_token_ids = batch.sample(output.next_token_logits) return sample_output.batch_next_token_ids, logits_output.next_token_logits
return next_token_ids, output.next_token_logits
@torch.inference_mode() @torch.inference_mode()