Revert "Add simple CPU offloading support" (#2252)

We'll re-add the commit to correctly ack Kaichao's authorship
This commit is contained in:
Ying Sheng
2024-11-28 23:36:55 -08:00
committed by GitHub
parent 4f2ee48ed1
commit 4057ea82c9
9 changed files with 29 additions and 173 deletions

View File

@@ -62,7 +62,6 @@ class ServerArgs:
max_prefill_tokens: int = 16384
schedule_policy: str = "lpm"
schedule_conservativeness: float = 1.0
cpu_offload_gb: int = 0
# Other runtime options
tp_size: int = 1
@@ -368,13 +367,6 @@ class ServerArgs:
help="How conservative the schedule policy is. A larger value means more conservative scheduling. Use a larger value if you see requests being retracted frequently.",
)
parser.add_argument(
"--cpu-offload-gb",
type=int,
default=ServerArgs.cpu_offload_gb,
help="How many GBs of RAM to reserve for CPU offloading",
)
# Other runtime options
parser.add_argument(
"--tensor-parallel-size",