[FEAT] Add transformers backend support (#5929)

This commit is contained in:
Marc Sun
2025-06-04 06:05:29 +02:00
committed by GitHub
parent 8a5480528d
commit 37f1547587
11 changed files with 636 additions and 3 deletions

View File

@@ -61,6 +61,7 @@ class ServerArgs:
is_embedding: bool = False
enable_multimodal: Optional[bool] = None
revision: Optional[str] = None
impl: str = "auto"
# Port for the HTTP server
host: str = "127.0.0.1"
@@ -726,6 +727,18 @@ class ServerArgs:
default=ServerArgs.page_size,
help="The number of tokens in a page.",
)
parser.add_argument(
"--impl",
type=str,
default=ServerArgs.impl,
help="Which implementation of the model to use.\n\n"
'* "auto" will try to use the SGLang implementation if it exists '
"and fall back to the Transformers implementation if no SGLang "
"implementation is available.\n"
'* "sglang" will use the SGLang model implementation.\n'
'* "transformers" will use the Transformers model '
"implementation.\n",
)
# Other runtime options
parser.add_argument(