Improve the control of streaming and improve the first token latency in streaming (#117)

test/srt/model/test_llama_extend.py

@@ -28,7 +28,7 @@ def test_generate_worker(model_path, tp_rank, tp_size):
 
     reqs = []
     for i in range(len(prompts)):
-        req = Req(i)
+        req = Req(i, None, None)
         req.input_ids = tokenizer.encode(prompts[i])[:cut_num]
         req.sampling_params = sampling_params
         reqs.append(req)

test/srt/model/test_llava_low_api.py

@@ -112,6 +112,7 @@ def test_generate_worker(
     prefill_params = (
         torch.tensor(np.array(input_ids)).cuda(),
         np.array(pixel_values),
+        [None],
         [offset],
         *params,
     )