Use dtype to control generate (#1082)

Co-authored-by: zhyncs <me@zhyncs.com>
2024-08-14 08:58:07 -07:00
parent 67c0d832a6
commit a34dd86a7d
12 changed files with 110 additions and 88 deletions
--- a/python/sglang/api.py
+++ b/python/sglang/api.py
@@ -72,7 +72,7 @@ def gen(
    logprob_start_len: Optional[int] = None,
    top_logprobs_num: Optional[int] = None,
    return_text_in_logprobs: Optional[bool] = None,
-    dtype: Optional[type] = None,
+    dtype: Optional[Union[type, str]] = None,
    choices: Optional[List[str]] = None,
    choices_method: Optional[ChoicesSamplingMethod] = None,
    regex: Optional[str] = None,