EngineX-Hygon/sglang
5
0
Fork 0
Code Issues Pull Requests Actions 7 Projects Releases Wiki Activity

Fix logprobs with logprob_start_len (#193)

Browse Source
This commit is contained in:
Cody Yu
2024-02-22 10:33:03 -08:00
committed by GitHub
parent 9de9a46815
commit 4cb9aaedf3
1 changed files with 7 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
python/sglang/srt/managers/router/model_rpc.py
View File

@@ -432,9 +432,14 @@ class ModelRpcServer(rpyc.Service):
req.logprob = logprobs[pt : pt + req.extend_input_len - 1]
req.normalized_logprob = normalized_logprobs[i]
token_ids = req.input_ids + [next_token_ids[i]]
token_logprobs = [None] + req.logprob + [last_logprobs[i]]
# If logprob_start_len > 0, then first logprob_start_len prompt tokens
# will be ignored.
prompt_token_len = len(req.logprob)
token_ids = req.input_ids[-prompt_token_len :] + [next_token_ids[i]]
token_logprobs = req.logprob + [last_logprobs[i]]
req.token_logprob = list(zip(token_ids, token_logprobs))
if req.logprob_start_len == 0:
req.token_logprob = [(req.input_ids[0], None)] + req.token_logprob
pt += req.extend_input_len
self.handle_finished_requests(batch)