[Feature] Add sampler custom logits processor (#2396)

Signed-off-by: Hongpeng Guo <hpguo@anyscale.com>
This commit is contained in:
Hongpeng Guo
2025-01-19 14:46:53 -08:00
committed by GitHub
parent 3bcf5ecea7
commit e403d23757
12 changed files with 302 additions and 4 deletions

View File

@@ -159,6 +159,9 @@ class ServerArgs:
enable_memory_saver: bool = False
allow_auto_truncate: bool = False
# Custom logit processor
enable_custom_logit_processor: bool = False
def __post_init__(self):
# Set missing default values
if self.tokenizer_path is None:
@@ -865,6 +868,11 @@ class ServerArgs:
action="store_true",
help="Allow automatically truncating requests that exceed the maximum input length instead of returning an error.",
)
parser.add_argument(
"--enable-custom-logit-processor",
action="store_true",
help="Enable users to pass custom logit processors to the server (disabled by default for security)",
)
@classmethod
def from_cli_args(cls, args: argparse.Namespace):