Fix failed ci tests on long prompts; Better error messages for embedding models (#1700)

2024-10-17 09:23:29 -07:00
parent 30ee36305e
commit 7feba41584
4 changed files with 24 additions and 13 deletions
--- a/python/sglang/srt/managers/io_struct.py
+++ b/python/sglang/srt/managers/io_struct.py
@@ -56,6 +56,9 @@ class GenerateReqInput:
    # LoRA related
    lora_path: Optional[Union[List[Optional[str]], Optional[str]]] = None

+    # Whether it is a single request or a batch request
+    is_single: bool = True
+
    def post_init(self):
        if (self.text is None and self.input_ids is None) or (
            self.text is not None and self.input_ids is not None