Cache optimizations (#418)

2024-05-13 12:47:13 +08:00
parent 562b8857d8
commit 39191c8515
8 changed files with 117 additions and 96 deletions
--- a/python/sglang/srt/server_args.py
+++ b/python/sglang/srt/server_args.py
@@ -149,7 +149,8 @@ class ServerArgs:
            "--schedule-heuristic",
            type=str,
            default=ServerArgs.schedule_heuristic,
-            help="Schudule mode: [lpm, weight, random, fcfs]",
+            choices=["lpm", "random", "fcfs", "dfs-weight"],
+            help="Scheduling Heuristic.",
        )
        parser.add_argument(
            "--schedule-conservativeness",