EngineX-Hygon/sglang
5
0
Fork 0
Code Issues Pull Requests Actions 7 Projects Releases Wiki Activity

Fix regex and logprob conflicts when chunked prefilling (#1703)

Browse Source
This commit is contained in:
Liangsheng Yin
2024-10-17 18:33:21 -07:00
committed by GitHub
parent a95d5589c3
commit 9e0dac1ad7
1 changed files with 8 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
python/sglang/srt/managers/scheduler.py
View File

@@ -789,15 +789,15 @@ class Scheduler:
elif not batch.decoding_reqs or req not in batch.decoding_reqs: elif not batch.decoding_reqs or req not in batch.decoding_reqs:
self.tree_cache.cache_unfinished_req(req) self.tree_cache.cache_unfinished_req(req)
if req.regex_fsm is not None: if req.regex_fsm is not None:
req.regex_fsm_state = req.regex_fsm.get_next_state( req.regex_fsm_state = req.regex_fsm.get_next_state(
req.regex_fsm_state, next_token_ids[i] req.regex_fsm_state, next_token_ids[i]
) )
if req.return_logprob: if req.return_logprob:
logprob_pt += self.add_logprob_return_values( logprob_pt += self.add_logprob_return_values(
i, req, logprob_pt, next_token_ids, logits_output i, req, logprob_pt, next_token_ids, logits_output
) )
else: # embedding or reward model else: # embedding or reward model
assert batch.extend_num_tokens != 0 assert batch.extend_num_tokens != 0
embeddings, bid = result embeddings, bid = result