EngineX-Hygon/sglang
5
0
Fork 0
Code Issues Pull Requests Actions 7 Projects Releases Wiki Activity

Simplify tokenizer manager (#1904)

Browse Source
This commit is contained in:
Lianmin Zheng
2024-11-03 08:38:26 -08:00
committed by GitHub
parent 916b3cdddc
commit c17c578108
11 changed files with 261 additions and 443 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
python/sglang/srt/server.py
View File

@@ -53,7 +53,6 @@ from sglang.srt.managers.detokenizer_manager import run_detokenizer_process
from sglang.srt.managers.io_struct import (
EmbeddingReqInput,
GenerateReqInput,
RewardReqInput,
UpdateWeightReqInput,
)
from sglang.srt.managers.scheduler import run_scheduler_process
@@ -91,7 +90,7 @@ asyncio.set_event_loop_policy(uvloop.EventLoopPolicy())
app = FastAPI()
tokenizer_manager = None
tokenizer_manager: TokenizerManager = None
app.add_middleware(
CORSMiddleware,
@@ -254,7 +253,7 @@ app.post("/encode")(encode_request)
app.put("/encode")(encode_request)
async def judge_request(obj: RewardReqInput, request: Request):
async def judge_request(obj: EmbeddingReqInput, request: Request):
"""Handle a reward model request."""
try:
ret = await tokenizer_manager.generate_request(obj, request).__anext__()