Free metadata_buffer_index after transfer finished (#5364)

This commit is contained in:
Yongtong Wu
2025-04-14 16:04:46 +08:00
committed by GitHub
parent adca585bfb
commit 14e8bd889f

View File

@@ -226,6 +226,11 @@ class SchedulerDisaggregationPrefillMixin:
elif poll == KVPoll.Failed:
raise Exception("Transferring failed")
for req in done_reqs:
self.disagg_prefill_pending_queue.req_to_metadata_buffer_idx_allocator.free(
req.metadata_buffer_index
)
# Stream requests which have finished transfer
self.stream_output(done_reqs, False, None)