Support stop_token_ids in sglang API (#1092)

This commit is contained in:
Liangsheng Yin
2024-08-14 17:31:39 -07:00
committed by GitHub
parent 1c2b5f5240
commit 73cf6834f2
6 changed files with 34 additions and 12 deletions

View File

@@ -20,7 +20,6 @@ from sglang.lang.ir import (
SglConstantText,
SglExpr,
SglExprList,
SglFunction,
SglGen,
SglImage,
SglRoleBegin,
@@ -181,8 +180,10 @@ class StreamExecutor:
num_api_spec_tokens=None,
use_thread=True,
):
from sglang.lang.backend.base_backend import BaseBackend
self.sid = uuid.uuid4().hex
self.backend = backend
self.backend: BaseBackend = backend
self.arguments: Dict[str, Any] = arguments
self.default_sampling_para = default_sampling_para
self.stream = stream
@@ -658,6 +659,7 @@ class StreamExecutor:
for item in [
"max_new_tokens",
"stop",
"stop_token_ids",
"temperature",
"top_p",
"top_k",