[Fix] Fix --skip-tokenizer-init (#1798)

This commit is contained in:
Lianmin Zheng
2024-10-25 18:51:59 -07:00
committed by GitHub
parent b77a02cdfd
commit fb99aaa527
5 changed files with 50 additions and 25 deletions

View File

@@ -571,7 +571,7 @@ class TokenizerManager:
def create_abort_task(self, obj: GenerateReqInput):
# Abort the request if the client is disconnected.
async def abort_request():
await asyncio.sleep(3)
await asyncio.sleep(1)
if obj.is_single:
self.abort_request(obj.rid)
else:
@@ -621,11 +621,8 @@ class TokenizerManager:
"meta_info": recv_obj.meta_info[i],
}
elif isinstance(recv_obj, BatchTokenIDOut):
read_start = 0 if i == 0 else recv_obj.read_offsets[i - 1]
out_dict = {
"token_ids": recv_obj.decode_ids[
read_start : recv_obj.read_offsets[i]
],
"token_ids": recv_obj.output_ids[i],
"meta_info": recv_obj.meta_info[i],
}