Fix the possible bug of decode out of memory (#36)

2024-01-20 03:01:15 +08:00
parent 199e82a15d
commit 40ab1f0129
7 changed files with 274 additions and 46 deletions
--- a/python/sglang/srt/server_args.py
+++ b/python/sglang/srt/server_args.py
@@ -119,7 +119,7 @@ class ServerArgs:
            "--schedule-conservativeness",
            type=float,
            default=ServerArgs.schedule_conservativeness,
-            help="How conservative the schedule policy is. A larger value means more conservative scheduling. Use a larger value if you see out-of-memory errors.",
+            help="How conservative the schedule policy is. A larger value means more conservative scheduling. Use a larger value if you see requests being retracted frequently.",
        )
        parser.add_argument(
            "--random-seed",