Support random dataset in bench_serving.py (#669)

2024-07-20 01:06:43 -07:00
parent 8f4b1559e7
commit 35759efa91
4 changed files with 82 additions and 15 deletions
--- a/python/sglang/srt/managers/io_struct.py
+++ b/python/sglang/srt/managers/io_struct.py
@@ -40,7 +40,7 @@ class GenerateReqInput:
            self.text is not None and self.input_ids is not None
        ):
            raise ValueError("Either text or input_ids should be provided.")
-        if "n" in self.sampling_params and self.sampling_params["n"] != 1:
+        if self.sampling_params.get("n", 1) != 1:
            is_single = False
        else:
            if self.text is not None: