Fix: resolve prefill of retracted request out-of-memory issue when ignore_eos is enabled (#7434)

This commit is contained in:
Yusong Gao
2025-08-02 14:43:45 +08:00
committed by GitHub
parent 89caf7a3c6
commit 4bec99ecd0

View File

@@ -455,7 +455,9 @@ class PrefillAdder:
if not self.is_hybrid:
# Skip this logic for swa. The SWA has different memory management, and
# this mechanism is underestimating the memory usage.
cur_rem_tokens = self.cur_rem_tokens - len(req.origin_input_ids)
cur_rem_tokens = self.cur_rem_tokens - self.ceil_paged_tokens(
req.extend_input_len
)
tokens_freed = 0
for i, (tokens_left, tokens_occupied) in enumerate(self.req_states):
# tokens_left gives a reservative calculation as the last token is not stored