diff --git a/docs/advanced_features/server_arguments.md b/docs/advanced_features/server_arguments.md index d3425eecf..b88f3d8a7 100644 --- a/docs/advanced_features/server_arguments.md +++ b/docs/advanced_features/server_arguments.md @@ -8,6 +8,23 @@ You can find all arguments by `python3 -m sglang.launch_server --help` ## Common launch commands +- To use a configuration file, create a YAML file with your server arguments and specify it with `--config`. CLI arguments will override config file values. + + ```bash + # Create config.yaml + cat > config.yaml << EOF + model-path: meta-llama/Meta-Llama-3-8B-Instruct + host: 0.0.0.0 + port: 30000 + tensor-parallel-size: 2 + enable-metrics: true + log-requests: true + EOF + + # Launch server with config file + python -m sglang.launch_server --config config.yaml + ``` + - To enable multi-GPU tensor parallelism, add `--tp 2`. If it reports the error "peer access is not supported between these two devices", add `--enable-p2p-check` to the server launch command. ```bash @@ -65,6 +82,7 @@ Please consult the documentation below and [server_args.py](https://github.com/s | Arguments | Description | Defaults | |-----------|-------------|----------| +| `--config` | Path to a YAML configuration file containing server arguments. Arguments in the config file will be merged with command-line arguments, with CLI arguments taking precedence. | None | | `--model-path` | The path of the model weights. This can be a local folder or a Hugging Face repo ID. | None | | `--tokenizer-path` | The path of the tokenizer. | None | | `--tokenizer-mode` | Tokenizer mode. 'auto' will use the fast tokenizer if available, and 'slow' will always use the slow tokenizer. | auto | diff --git a/python/sglang/srt/server_args.py b/python/sglang/srt/server_args.py index 5e5dd6170..71ddcbf3e 100644 --- a/python/sglang/srt/server_args.py +++ b/python/sglang/srt/server_args.py @@ -2659,6 +2659,13 @@ class ServerArgs: help="NOTE: --enable-flashinfer-mxfp4-moe is deprecated. Please set `--moe-runner-backend` to 'flashinfer_mxfp4' instead.", ) + # Configuration file support + parser.add_argument( + "--config", + type=str, + help="Read CLI options from a config file. Must be a YAML file with configuration options.", + ) + @classmethod def from_cli_args(cls, args: argparse.Namespace): args.tp_size = args.tensor_parallel_size @@ -2939,6 +2946,26 @@ def prepare_server_args(argv: List[str]) -> ServerArgs: Returns: The server arguments. """ + # Import here to avoid circular imports + from sglang.srt.server_args_config_parser import ConfigArgumentMerger + + # Check for config file and merge arguments if present + if "--config" in argv: + # Extract boolean actions from the parser to handle them correctly + parser = argparse.ArgumentParser() + ServerArgs.add_cli_args(parser) + + # Get boolean action destinations + boolean_actions = [] + for action in parser._actions: + if hasattr(action, "dest") and hasattr(action, "action"): + if action.action in ["store_true", "store_false"]: + boolean_actions.append(action.dest) + + # Merge config file arguments with CLI arguments + config_merger = ConfigArgumentMerger(boolean_actions=boolean_actions) + argv = config_merger.merge_config_with_args(argv) + parser = argparse.ArgumentParser() ServerArgs.add_cli_args(parser) raw_args = parser.parse_args(argv) diff --git a/python/sglang/srt/server_args_config_parser.py b/python/sglang/srt/server_args_config_parser.py new file mode 100644 index 000000000..74dc67677 --- /dev/null +++ b/python/sglang/srt/server_args_config_parser.py @@ -0,0 +1,146 @@ +""" +Configuration argument parser for command-line applications. +Handles merging of YAML configuration files with command-line arguments. +""" + +import logging +from pathlib import Path +from typing import Any, Dict, List, Union + +import yaml + +logger = logging.getLogger(__name__) + + +class ConfigArgumentMerger: + """Handles merging of configuration file arguments with command-line arguments.""" + + def __init__(self, boolean_actions: List[str] = None): + """Initialize with list of boolean action destinations.""" + self.boolean_actions = boolean_actions or [] + + def merge_config_with_args(self, cli_args: List[str]) -> List[str]: + """ + Merge configuration file arguments with command-line arguments. + + Configuration arguments are inserted after the subcommand to maintain + proper precedence: CLI > Config > Defaults + + Args: + cli_args: List of command-line arguments + + Returns: + Merged argument list with config values inserted + + Raises: + ValueError: If multiple config files specified or no config file provided + """ + config_file_path = self._extract_config_file_path(cli_args) + if not config_file_path: + return cli_args + + config_args = self._parse_yaml_config(config_file_path) + return self._insert_config_args(cli_args, config_args, config_file_path) + + def _extract_config_file_path(self, args: List[str]) -> str: + """Extract the config file path from arguments.""" + config_indices = [i for i, arg in enumerate(args) if arg == "--config"] + + if len(config_indices) > 1: + raise ValueError("Multiple config files specified! Only one allowed.") + + if not config_indices: + return None + + config_index = config_indices[0] + if config_index == len(args) - 1: + raise ValueError("No config file specified after --config flag!") + + return args[config_index + 1] + + def _insert_config_args( + self, cli_args: List[str], config_args: List[str], config_file_path: str + ) -> List[str]: + """Insert configuration arguments into the CLI argument list.""" + config_index = cli_args.index("--config") + + # Split arguments around config file + before_config = cli_args[:config_index] + after_config = cli_args[config_index + 2 :] # Skip --config and file path + + # Simple merge: config args + CLI args + return config_args + before_config + after_config + + def _parse_yaml_config(self, file_path: str) -> List[str]: + """ + Parse YAML configuration file and convert to argument list. + + Args: + file_path: Path to the YAML configuration file + + Returns: + List of arguments in format ['--key', 'value', ...] + + Raises: + ValueError: If file is not YAML or cannot be read + """ + self._validate_yaml_file(file_path) + + try: + with open(file_path, "r") as file: + config_data = yaml.safe_load(file) + except Exception as e: + logger.error(f"Failed to read config file {file_path}: {e}") + raise + + # Handle empty files or None content + if config_data is None: + config_data = {} + + if not isinstance(config_data, dict): + raise ValueError("Config file must contain a dictionary at root level") + + return self._convert_config_to_args(config_data) + + def _validate_yaml_file(self, file_path: str) -> None: + """Validate that the file is a YAML file.""" + path = Path(file_path) + if path.suffix.lower() not in [".yaml", ".yml"]: + raise ValueError(f"Config file must be YAML format, got: {path.suffix}") + + if not path.exists(): + raise ValueError(f"Config file not found: {file_path}") + + def _convert_config_to_args(self, config: Dict[str, Any]) -> List[str]: + """Convert configuration dictionary to argument list.""" + args = [] + + for key, value in config.items(): + if isinstance(value, bool): + self._add_boolean_arg(args, key, value) + elif isinstance(value, list): + self._add_list_arg(args, key, value) + else: + self._add_scalar_arg(args, key, value) + + return args + + def _add_boolean_arg(self, args: List[str], key: str, value: bool) -> None: + """Add boolean argument to the list.""" + if key in self.boolean_actions: + # For boolean actions, always add the flag and value + args.extend([f"--{key}", str(value).lower()]) + else: + # For regular booleans, only add flag if True + if value: + args.append(f"--{key}") + + def _add_list_arg(self, args: List[str], key: str, value: List[Any]) -> None: + """Add list argument to the list.""" + if value: # Only add if list is not empty + args.append(f"--{key}") + args.extend(str(item) for item in value) + + def _add_scalar_arg(self, args: List[str], key: str, value: Any) -> None: + """Add scalar argument to the list.""" + args.extend([f"--{key}", str(value)]) diff --git a/test/srt/test_config_integration.py b/test/srt/test_config_integration.py new file mode 100644 index 000000000..ea13b8d6c --- /dev/null +++ b/test/srt/test_config_integration.py @@ -0,0 +1,159 @@ +""" +Test script to verify SGLang config file integration. +""" + +import os +import tempfile +from pathlib import Path + +import pytest +import yaml + +from sglang.srt.server_args import prepare_server_args +from sglang.srt.server_args_config_parser import ConfigArgumentMerger + + +@pytest.fixture +def merger(): + """Fixture providing a ConfigArgumentMerger instance.""" + return ConfigArgumentMerger() + + +def test_server_args_config_parser(merger): + """Test the config parser functionality.""" + # Create a temporary config file + config_data = { + "model-path": "microsoft/DialoGPT-medium", + "host": "0.0.0.0", + "port": 30000, + "tensor-parallel-size": 2, + "trust-remote-code": False, + "enable-metrics": True, + "stream-output": True, + "skip-server-warmup": False, + "log-requests": True, + "show-time-cost": True, + "is-embedding": False, + } + + with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as f: + yaml.dump(config_data, f) + config_file = f.name + + try: + # Test config parser directly + config_args = merger._parse_yaml_config(config_file) + + # Test merging with CLI args + cli_args = ["--config", config_file, "--max-running-requests", "128"] + merged_args = merger.merge_config_with_args(cli_args) + + # Verify the merged args contain both config and CLI values + assert "--model-path" in merged_args + assert "microsoft/DialoGPT-medium" in merged_args + assert "--host" in merged_args + assert "0.0.0.0" in merged_args + assert "--port" in merged_args + assert "30000" in merged_args + assert "--tensor-parallel-size" in merged_args + assert "2" in merged_args + assert "--max-running-requests" in merged_args + assert "128" in merged_args + + # Test boolean arguments + assert "--enable-metrics" in merged_args # True boolean + assert "--stream-output" in merged_args # True boolean + assert "--log-requests" in merged_args # True boolean + assert "--show-time-cost" in merged_args # True boolean + # False booleans should not be present (only add flag if True) + assert "--trust-remote-code" not in merged_args # False boolean + assert "--skip-server-warmup" not in merged_args # False boolean + assert "--is-embedding" not in merged_args # False boolean + + finally: + os.unlink(config_file) + + +def test_server_args_integration(): + """Test the integration with server args.""" + # Create a temporary config file + config_data = { + "model-path": "microsoft/DialoGPT-medium", + "host": "0.0.0.0", + "port": 30000, + "tensor-parallel-size": 1, + "max-running-requests": 256, + } + + with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as f: + yaml.dump(config_data, f) + config_file = f.name + + try: + # Test with config file + argv = ["--config", config_file] + server_args = prepare_server_args(argv) + + # Verify that config values were loaded + assert server_args.model_path == "microsoft/DialoGPT-medium" + assert server_args.host == "0.0.0.0" + assert server_args.port == 30000 + assert server_args.tp_size == 1 + assert server_args.max_running_requests == 256 + + finally: + os.unlink(config_file) + + +def test_cli_override(): + """Test that CLI arguments override config file values.""" + # Create a temporary config file + config_data = { + "model-path": "microsoft/DialoGPT-medium", + "port": 30000, + "tensor-parallel-size": 1, + } + + with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as f: + yaml.dump(config_data, f) + config_file = f.name + + try: + # Test CLI override (CLI should take precedence) + argv = [ + "--config", + config_file, + "--port", + "40000", + "--tensor-parallel-size", + "2", + ] + server_args = prepare_server_args(argv) + + # Verify that CLI values override config values + assert server_args.model_path == "microsoft/DialoGPT-medium" # From config + assert server_args.port == 40000 # From CLI (overrides config) + assert server_args.tp_size == 2 # From CLI (overrides config) + + finally: + os.unlink(config_file) + + +def test_error_handling(): + """Test error handling for invalid config files.""" + # Test non-existent config file + with pytest.raises(ValueError, match="Config file not found"): + argv = ["--config", "non-existent.yaml"] + prepare_server_args(argv) + + # Test invalid YAML file + with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as f: + f.write("invalid: yaml: content: [") + invalid_yaml_file = f.name + + try: + with pytest.raises(Exception): + argv = ["--config", invalid_yaml_file] + prepare_server_args(argv) + finally: + os.unlink(invalid_yaml_file)