Fix engine unit test (#1701)

benchmark/llava_bench/README.md

@@ -17,7 +17,7 @@ pip3 install "torch>=2.1.2" "transformers>=4.36" pillow
 ### Benchmark sglang
 Launch a server
 ```
-python3 -m sglang.launch_server --model-path liuhaotian/llava-v1.5-7b --tokenizer-path llava-hf/llava-1.5-7b-hf --port 30000
+python3 -m sglang.launch_server --model-path liuhaotian/llava-v1.6-vicuna-7b --tokenizer-path llava-hf/llava-1.5-7b-hf --port 30000
 ```
 
 Run benchmark

benchmark/llava_bench/bench_sglang.py

@@ -20,7 +20,7 @@ def image_qa(s, image_file, question):
 
 
 def main(args):
-    lines = read_jsonl(args.question_file)[: args.num_questions]
+    lines = list(read_jsonl(args.question_file))[: args.num_questions]
     arguments = [
         {
             "image_file": os.path.abspath(args.image_folder + "/" + l["image"]),

python/sglang/srt/server.py

@@ -706,6 +706,10 @@ class Runtime:
         self.shutdown()
 
 
+STREAM_END_SYMBOL = b"data: [DONE]"
+STREAM_CHUNK_START_SYMBOL = b"data:"
+
+
 class Engine:
     """
     SRT Engine without an HTTP server layer.
@@ -749,8 +753,6 @@ class Engine:
         ret = loop.run_until_complete(generate_request(obj, None))
 
         if stream is True:
-            STREAM_END_SYMBOL = "data: [DONE]"
-            STREAM_CHUNK_START_SYMBOL = "data:"
 
             def generator_wrapper():
                 offset = 0
@@ -796,9 +798,6 @@ class Engine:
         ret = await generate_request(obj, None)
 
         if stream is True:
-            STREAM_END_SYMBOL = "data: [DONE]"
-            STREAM_CHUNK_START_SYMBOL = "data:"
-
             generator = ret.body_iterator
 
             async def generator_wrapper():

test/srt/test_srt_engine.py

@@ -1,3 +1,8 @@
+"""
+Usage:
+python3 -m unittest test_srt_engine.TestSRTEngine.test_3_sync_streaming_combination
+"""
+
 import asyncio
 import json
 import unittest
@@ -8,7 +13,7 @@ from sglang.test.few_shot_gsm8k_engine import run_eval
 from sglang.test.test_utils import DEFAULT_MODEL_NAME_FOR_TEST
 
 
-class TestSRTBackend(unittest.TestCase):
+class TestSRTEngine(unittest.TestCase):
 
     def test_1_engine_runtime_consistency(self):
         prompt = "Today is a sunny day and I like"