Fix failed ci tests on long prompts; Better error messages for embedding models (#1700)

This commit is contained in:
Lianmin Zheng
2024-10-17 09:23:29 -07:00
committed by GitHub
parent 30ee36305e
commit 7feba41584
4 changed files with 24 additions and 13 deletions

View File

@@ -150,9 +150,13 @@ class TokenizerManager:
while self.model_update_lock.locked():
await asyncio.sleep(0.001)
if isinstance(obj, EmbeddingReqInput) and self.is_generation:
raise ValueError(
"This model does not appear to be an embedding model by default. Please add `--is-embedding` when launching the server or try another model."
)
obj.post_init()
is_single = obj.is_single
if is_single:
async for response in self._handle_single_request(obj, request):
yield response