Fix a bug in BatchTokenIDOut & Misc style and dependency updates (#7457)
This commit is contained in:
@@ -788,6 +788,7 @@ class Req:
|
||||
self.multimodal_inputs = None
|
||||
self.grammar = None
|
||||
self.origin_input_ids = [0] # set it to one token to skip the long prefill
|
||||
self.return_logprob = False
|
||||
self.finished_reason = FINISH_ABORT(
|
||||
error_msg, HTTPStatus.BAD_REQUEST, "BadRequestError"
|
||||
)
|
||||
|
||||
@@ -1374,7 +1374,14 @@ class Scheduler(
|
||||
)
|
||||
raise ValueError(msg)
|
||||
|
||||
if len(self.req_to_token_pool.free_slots) != self.req_to_token_pool.size:
|
||||
if self.disaggregation_mode == DisaggregationMode.DECODE:
|
||||
req_total_size = (
|
||||
self.req_to_token_pool.size + self.req_to_token_pool.pre_alloc_size
|
||||
)
|
||||
else:
|
||||
req_total_size = self.req_to_token_pool.size
|
||||
|
||||
if len(self.req_to_token_pool.free_slots) != req_total_size:
|
||||
msg = (
|
||||
"req_to_token_pool memory leak detected!"
|
||||
f"available_size={len(self.req_to_token_pool.free_slots)}, "
|
||||
|
||||
@@ -1226,7 +1226,7 @@ class TokenizerManager:
|
||||
state.last_output_offset = len(state.output_ids)
|
||||
else:
|
||||
state.output_ids.extend(recv_obj.output_ids[i])
|
||||
output_token_ids = state.output_ids
|
||||
output_token_ids = state.output_ids.copy()
|
||||
|
||||
out_dict = {
|
||||
"output_ids": output_token_ids,
|
||||
|
||||
Reference in New Issue
Block a user