EngineX-Hygon/sglang
5
0
Fork 0
Code Issues Pull Requests Actions 7 Projects Releases Wiki Activity

Add engine encode (#1995)

Browse Source 
Co-authored-by: Byron Hsu <byronhsu1230@gmail.com>
This commit is contained in:
James Xu
2024-11-11 14:48:17 -05:00
committed by GitHub
parent aaf0a3156e
commit ddeb9d42de
3 changed files with 29 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
python/sglang/srt/server.py
View File

@@ -874,4 +874,12 @@ class Engine:
else:
return tokenizer_manager.tokenizer
# TODO (ByronHsu): encode
def encode(
self,
prompt: Union[str, List[str], List[Dict], List[List[Dict]]],
):
obj = EmbeddingReqInput(text=prompt)
# get the current event loop
loop = asyncio.get_event_loop()
return loop.run_until_complete(encode_request(obj, None))

1
python/sglang/test/test_utils.py
View File

@@ -28,6 +28,7 @@ from sglang.utils import get_exception_traceback
DEFAULT_FP8_MODEL_NAME_FOR_TEST = "neuralmagic/Meta-Llama-3.1-8B-FP8"
DEFAULT_MODEL_NAME_FOR_TEST = "meta-llama/Llama-3.1-8B-Instruct"
DEFAULT_SMALL_MODEL_NAME_FOR_TEST = "meta-llama/Llama-3.2-1B-Instruct"
DEFAULT_SMALL_EMBEDDING_MODEL_NAME_FOR_TEST = "Alibaba-NLP/gte-Qwen2-1.5B-instruct"
DEFAULT_MOE_MODEL_NAME_FOR_TEST = "mistralai/Mixtral-8x7B-Instruct-v0.1"
DEFAULT_MLA_MODEL_NAME_FOR_TEST = "deepseek-ai/DeepSeek-Coder-V2-Lite-Instruct"
DEFAULT_MLA_FP8_MODEL_NAME_FOR_TEST = "neuralmagic/DeepSeek-Coder-V2-Lite-Instruct-FP8"