[Feature] Support LoRA path renaming and add LoRA serving benchmarks (#1433)

This commit is contained in:
Ying Sheng
2024-09-15 12:46:04 -07:00
committed by GitHub
parent 899cf5c438
commit 37963394aa
6 changed files with 594 additions and 62 deletions

View File

@@ -24,6 +24,17 @@ from typing import List, Optional, Union
logger = logging.getLogger(__name__)
class LoRAPathAction(argparse.Action):
def __call__(self, parser, namespace, values, option_string=None):
setattr(namespace, self.dest, {})
for lora_path in values:
if "=" in lora_path:
name, path = lora_path.split("=", 1)
getattr(namespace, self.dest)[name] = path
else:
getattr(namespace, self.dest)[lora_path] = lora_path
@dataclasses.dataclass
class ServerArgs:
# Model and tokenizer
@@ -532,7 +543,8 @@ class ServerArgs:
type=str,
nargs="*",
default=None,
help="The list of LoRA adapters.",
action=LoRAPathAction,
help="The list of LoRA adapters. You can provide a list of either path in str or renamed path in the format {name}={path}",
)
parser.add_argument(
"--max-loras-per-batch",