[Feature] Support LoRA path renaming and add LoRA serving benchmarks (#1433)
This commit is contained in:
@@ -24,6 +24,17 @@ from typing import List, Optional, Union
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class LoRAPathAction(argparse.Action):
|
||||
def __call__(self, parser, namespace, values, option_string=None):
|
||||
setattr(namespace, self.dest, {})
|
||||
for lora_path in values:
|
||||
if "=" in lora_path:
|
||||
name, path = lora_path.split("=", 1)
|
||||
getattr(namespace, self.dest)[name] = path
|
||||
else:
|
||||
getattr(namespace, self.dest)[lora_path] = lora_path
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class ServerArgs:
|
||||
# Model and tokenizer
|
||||
@@ -532,7 +543,8 @@ class ServerArgs:
|
||||
type=str,
|
||||
nargs="*",
|
||||
default=None,
|
||||
help="The list of LoRA adapters.",
|
||||
action=LoRAPathAction,
|
||||
help="The list of LoRA adapters. You can provide a list of either path in str or renamed path in the format {name}={path}",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--max-loras-per-batch",
|
||||
|
||||
Reference in New Issue
Block a user