[bug] fixed batch api for DeepSeek V3/R1 (#3754)

This commit is contained in:
Shenggui Li
2025-02-22 02:28:16 +08:00
committed by GitHub
parent c7c79b16cd
commit 9af0e21ef5

View File

@@ -162,6 +162,7 @@ class DetokenizerManager:
# Incremental decoding
output_strs = []
finished_reqs = []
for i in range(bs):
try:
s = self.decode_status[recv_obj.rids[i]]
@@ -184,6 +185,8 @@ class DetokenizerManager:
new_text = ""
else:
new_text = find_printable_text(new_text)
else:
finished_reqs.append(recv_obj.rids[i])
output_strs.append(
self.trim_matched_stop(
@@ -214,6 +217,10 @@ class DetokenizerManager:
)
)
# remove decodestatus for completed requests
for rid in finished_reqs:
self.decode_status.pop(rid)
class LimitedCapacityDict(OrderedDict):
def __init__(self, capacity: int, *args, **kwargs):