[Minor] Many cleanup (#1357)

2024-09-09 04:14:11 -07:00
parent c9b75917d5
commit e4d68afcf0
24 changed files with 416 additions and 296 deletions
--- a/python/sglang/launch_server_llavavid.py
+++ b/python/sglang/launch_server_llavavid.py
@@ -1,5 +1,6 @@
 """Launch the inference server for Llava-video model."""

+import json
 import sys

 from sglang.srt.server import launch_server, prepare_server_args
@@ -19,5 +20,6 @@ if __name__ == "__main__":
        model_override_args["model_max_length"] = 4096 * 2
    if "34b" in server_args.model_path.lower():
        model_override_args["image_token_index"] = 64002
+    server_args.json_model_override_args = json.dumps(model_override_args)

-    launch_server(server_args, model_override_args, None)
+    launch_server(server_args)