EngineX-Hygon/sglang
5
0
Fork 0
Code Issues Pull Requests Actions 7 Projects Releases Wiki Activity

Add openai embedding API (#997)

Browse Source
This commit is contained in:
Ying Sheng
2024-08-09 11:19:18 -07:00
committed by GitHub
parent 05c50a82b8
commit b16e856f11
8 changed files with 135 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
python/sglang/srt/managers/io_struct.py
View File

@@ -194,7 +194,8 @@ class EmbeddingReqInput:
if is_single:
if self.rid is None:
self.rid = uuid.uuid4().hex
self.sampling_params = {"max_new_tokens": 0}
if self.sampling_params is None:
self.sampling_params = {"max_new_tokens": 1}
else:
# support select operation
self.batch_size = (
@@ -205,9 +206,10 @@ class EmbeddingReqInput:
else:
if not isinstance(self.rid, list):
raise ValueError("The rid should be a list.")
self.sampling_params = [
{"max_new_tokens": 0} for _ in range(self.batch_size)
]
if self.sampling_params is None:
self.sampling_params = [
{"max_new_tokens": 1} for _ in range(self.batch_size)
]
@dataclass