[FEAT] Support batches cancel (#1222)

Co-authored-by: Yineng Zhang <me@zhyncs.com>
This commit is contained in:
caiyueliang
2024-08-27 07:28:26 +08:00
committed by GitHub
parent c61a1b6f97
commit 2f1d92834f
3 changed files with 122 additions and 6 deletions

View File

@@ -59,6 +59,7 @@ from sglang.srt.managers.tokenizer_manager import TokenizerManager
from sglang.srt.openai_api.adapter import (
load_chat_template_for_openai_api,
v1_batches,
v1_cancel_batch,
v1_chat_completions,
v1_completions,
v1_delete_file,
@@ -246,6 +247,12 @@ async def openai_v1_batches(raw_request: Request):
return await v1_batches(tokenizer_manager, raw_request)
@app.post("/v1/batches/{batch_id}/cancel")
async def cancel_batches(batch_id: str):
# https://platform.openai.com/docs/api-reference/batch/cancel
return await v1_cancel_batch(tokenizer_manager, batch_id)
@app.get("/v1/batches/{batch_id}")
async def retrieve_batch(batch_id: str):
return await v1_retrieve_batch(batch_id)