22 lines
628 B
Bash
22 lines
628 B
Bash
#!/bin/bash
|
|
# bash vllm_test/run_llama1-7b_throughput.sh /path/to/llama7b_hf_model /path/to/llama7b_vls_engine
|
|
model_path=$1
|
|
engine_path=$2
|
|
|
|
#run test fixed input/output benchmark# llama7b-1xpu
|
|
XMLIR_D_XPU_L3_SIZE=0 python benchmark_throughput.py \
|
|
--trust-remote-code \
|
|
--backend vllm \
|
|
--model $model_path \
|
|
--tokenizer $model_path \
|
|
--engine_dir $engine_path \
|
|
--tensor-parallel-size 1 \
|
|
--dummy-dataset \
|
|
--max-num-seqs 14 \
|
|
--max-num-batched-tokens 2048 \
|
|
--dummy-tokenid 1 \
|
|
--dummy-input-len 1024 \
|
|
--dummy-output-len 1024 \
|
|
--max-model-len 2048 \
|
|
--num-prompts 14
|