[feat] Add Vertex AI compatible prediction route for /generate (#3866)

2025-02-27 19:42:15 -08:00
parent d38878523d
commit bc20e93f2d
5 changed files with 152 additions and 0 deletions
--- a/python/sglang/srt/managers/io_struct.py
+++ b/python/sglang/srt/managers/io_struct.py
@@ -568,3 +568,9 @@ class FunctionCallReqInput:
    tool_call_parser: Optional[str] = (
        None  # Specify the parser type, e.g. 'llama3', 'qwen25', or 'mistral'. If not specified, tries all.
    )
+
+
+@dataclass
+class VertexGenerateReqInput:
+    instances: List[dict]
+    parameters: Optional[dict] = None