EngineX-Hygon/sglang
5
0
Fork 0
Code Issues Pull Requests Actions 7 Projects Releases Wiki Activity

Fix streaming (#437)

Browse Source
This commit is contained in:
Lianmin Zheng
2024-05-13 17:26:18 -07:00
committed by GitHub
parent 0992d85f92
commit 32de16ce2f
2 changed files with 8 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
python/sglang/srt/managers/router/infer_batch.py
View File

@@ -20,15 +20,16 @@ class FinishReason(IntEnum):
LENGTH = auto() LENGTH = auto()
STOP_STR = auto() STOP_STR = auto()
def to_str(self): @staticmethod
if self == FinishReason.EOS_TOKEN: def to_str(reason):
if reason == FinishReason.EOS_TOKEN:
return None return None
elif self == FinishReason.LENGTH: elif reason == FinishReason.LENGTH:
return "length" return "length"
elif self == FinishReason.STOP_STR: elif reason == FinishReason.STOP_STR:
return "stop" return "stop"
else: else:
raise ValueError(f"Invalid finish reason: {self}") return None
class Req: class Req:

4
python/sglang/srt/managers/router/model_rpc.py
View File

@@ -24,7 +24,7 @@ from sglang.srt.managers.io_struct import (
FlushCacheReq, FlushCacheReq,
TokenizedGenerateReqInput, TokenizedGenerateReqInput,
) )
from sglang.srt.managers.router.infer_batch import Batch, ForwardMode, Req from sglang.srt.managers.router.infer_batch import Batch, ForwardMode, Req, FinishReason
from sglang.srt.managers.router.model_runner import ModelRunner from sglang.srt.managers.router.model_runner import ModelRunner
from sglang.srt.managers.router.radix_cache import RadixCache from sglang.srt.managers.router.radix_cache import RadixCache
from sglang.srt.managers.router.scheduler import Scheduler from sglang.srt.managers.router.scheduler import Scheduler
@@ -615,7 +615,7 @@ class ModelRpcServer:
+ len(req.output_ids) + len(req.output_ids)
- req.prompt_tokens, - req.prompt_tokens,
"completion_tokens_wo_jump_forward": req.completion_tokens_wo_jump_forward, "completion_tokens_wo_jump_forward": req.completion_tokens_wo_jump_forward,
"finish_reason": req.finish_reason.to_str(), "finish_reason": FinishReason.to_str(req.finish_reason),
"hit_stop_str": req.hit_stop_str, "hit_stop_str": req.hit_stop_str,
} }
if req.return_logprob: if req.return_logprob: