EngineX-Hygon/sglang
5
0
Fork 0
Code Issues Pull Requests Actions 7 Projects Releases Wiki Activity

Fix engine unit test (#1701)

Browse Source
This commit is contained in:
Lianmin Zheng
2024-10-17 09:53:32 -07:00
committed by GitHub
parent 7feba41584
commit dd3809fad8
4 changed files with 12 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
benchmark/llava_bench/README.md
View File

@@ -17,7 +17,7 @@ pip3 install "torch>=2.1.2" "transformers>=4.36" pillow
### Benchmark sglang ### Benchmark sglang
Launch a server Launch a server
``` ```
python3 -m sglang.launch_server --model-path liuhaotian/llava-v1.5-7b --tokenizer-path llava-hf/llava-1.5-7b-hf --port 30000 python3 -m sglang.launch_server --model-path liuhaotian/llava-v1.6-vicuna-7b --tokenizer-path llava-hf/llava-1.5-7b-hf --port 30000
``` ```
Run benchmark Run benchmark

2
benchmark/llava_bench/bench_sglang.py
View File

@@ -20,7 +20,7 @@ def image_qa(s, image_file, question):
def main(args): def main(args):
lines = read_jsonl(args.question_file)[: args.num_questions] lines = list(read_jsonl(args.question_file))[: args.num_questions]
arguments = [ arguments = [
{ {
"image_file": os.path.abspath(args.image_folder + "/" + l["image"]), "image_file": os.path.abspath(args.image_folder + "/" + l["image"]),

9
python/sglang/srt/server.py
View File

@@ -706,6 +706,10 @@ class Runtime:
self.shutdown() self.shutdown()
STREAM_END_SYMBOL = b"data: [DONE]"
STREAM_CHUNK_START_SYMBOL = b"data:"
class Engine: class Engine:
""" """
SRT Engine without an HTTP server layer. SRT Engine without an HTTP server layer.
@@ -749,8 +753,6 @@ class Engine:
ret = loop.run_until_complete(generate_request(obj, None)) ret = loop.run_until_complete(generate_request(obj, None))
if stream is True: if stream is True:
STREAM_END_SYMBOL = "data: [DONE]"
STREAM_CHUNK_START_SYMBOL = "data:"
def generator_wrapper(): def generator_wrapper():
offset = 0 offset = 0
@@ -796,9 +798,6 @@ class Engine:
ret = await generate_request(obj, None) ret = await generate_request(obj, None)
if stream is True: if stream is True:
STREAM_END_SYMBOL = "data: [DONE]"
STREAM_CHUNK_START_SYMBOL = "data:"
generator = ret.body_iterator generator = ret.body_iterator
async def generator_wrapper(): async def generator_wrapper():

7
test/srt/test_srt_engine.py
View File

@@ -1,3 +1,8 @@
"""
Usage:
python3 -m unittest test_srt_engine.TestSRTEngine.test_3_sync_streaming_combination
"""
import asyncio import asyncio
import json import json
import unittest import unittest
@@ -8,7 +13,7 @@ from sglang.test.few_shot_gsm8k_engine import run_eval
from sglang.test.test_utils import DEFAULT_MODEL_NAME_FOR_TEST from sglang.test.test_utils import DEFAULT_MODEL_NAME_FOR_TEST
class TestSRTBackend(unittest.TestCase): class TestSRTEngine(unittest.TestCase):
def test_1_engine_runtime_consistency(self): def test_1_engine_runtime_consistency(self):
prompt = "Today is a sunny day and I like" prompt = "Today is a sunny day and I like"