EngineX-Hygon/sglang
5
0
Fork 0
Code Issues Pull Requests Actions 7 Projects Releases Wiki Activity

Add download_dir ServerArgs property (#2027)

Browse Source
This commit is contained in:
Patrick Yi
2024-11-14 02:26:56 -05:00
committed by GitHub
parent df246e699d
commit 13ce3e4b5d
2 changed files with 11 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
python/sglang/srt/model_executor/model_runner.py
View File

@@ -233,7 +233,10 @@ class ModelRunner:
# Prepare the vllm model config # Prepare the vllm model config
monkey_patch_vllm_dummy_weight_loader() monkey_patch_vllm_dummy_weight_loader()
self.load_config = LoadConfig(load_format=self.server_args.load_format) self.load_config = LoadConfig(
load_format=self.server_args.load_format,
download_dir=self.server_args.download_dir,
)
self.vllm_model_config = VllmModelConfig( self.vllm_model_config = VllmModelConfig(
model=self.server_args.model_path, model=self.server_args.model_path,
quantization=self.server_args.quantization, quantization=self.server_args.quantization,

7
python/sglang/srt/server_args.py
View File

@@ -64,6 +64,7 @@ class ServerArgs:
random_seed: Optional[int] = None random_seed: Optional[int] = None
constrained_json_whitespace_pattern: Optional[str] = None constrained_json_whitespace_pattern: Optional[str] = None
watchdog_timeout: float = 300 watchdog_timeout: float = 300
download_dir: Optional[str] = None
# Logging # Logging
log_level: str = "info" log_level: str = "info"
@@ -405,6 +406,12 @@ class ServerArgs:
default=ServerArgs.watchdog_timeout, default=ServerArgs.watchdog_timeout,
help="Set watchdog timeout in seconds. If a forward batch takes longer than this, the server will crash to prevent hanging.", help="Set watchdog timeout in seconds. If a forward batch takes longer than this, the server will crash to prevent hanging.",
) )
parser.add_argument(
"--download-dir",
type=str,
default=ServerArgs.download_dir,
help="Model download directory.",
)
# Logging # Logging
parser.add_argument( parser.add_argument(