Fix num_tokens_pre_allocated in disaggregation log (#7714)

This commit is contained in:
Ziming Huang
2025-07-03 13:31:49 +08:00
committed by GitHub
parent d3c275b117
commit 1bebd3154e
2 changed files with 7 additions and 4 deletions

View File

@@ -416,6 +416,12 @@ class DecodePreallocQueue:
return preallocated_reqs
@property
def num_tokens_pre_allocated(self):
return sum(
len(decode_req.req.fill_ids) for decode_req in self.transfer_queue.queue
)
def _allocatable_tokens(
self, retractable_tokens: Optional[int] = None, count_retracted: bool = True
) -> int: