diff --git a/benchmark/llava_bench/README.md b/benchmark/llava_bench/README.md index bae4df648..854b57eba 100644 --- a/benchmark/llava_bench/README.md +++ b/benchmark/llava_bench/README.md @@ -8,6 +8,7 @@ image benchmark source: https://huggingface.co/datasets/liuhaotian/llava-bench-i ### Other Dependency ``` +pip3 install "sglang[all]" pip3 install "torch>=2.1.2" "transformers>=4.36" pillow ``` diff --git a/examples/quick_start/srt_example_llava.py b/examples/quick_start/srt_example_llava.py index a781bede3..2f4c7b02a 100644 --- a/examples/quick_start/srt_example_llava.py +++ b/examples/quick_start/srt_example_llava.py @@ -1,3 +1,6 @@ +""" +Usage: python3 srt_example_llava.py +""" import sglang as sgl @@ -12,7 +15,24 @@ runtime = sgl.Runtime(model_path="liuhaotian/llava-v1.5-7b", sgl.set_default_backend(runtime) -state = image_qa.run(image_path="images/cat.jpeg", question="What is this?") -print(state["answer"]) +# Single +state = image_qa.run( + image_path="images/cat.jpeg", + question="What is this?", + max_new_tokens=64) +print(state["answer"], "\n") + + +# Batch +states = image_qa.run_batch( + [ + {"image_path": "images/cat.jpeg", "question":"What is this?"}, + {"image_path": "images/dog.jpeg", "question":"What is this?"}, + ], + max_new_tokens=64, +) +for s in states: + print(s["answer"], "\n") + runtime.shutdown()