[HiCacheStorage] fix abort request host memory leaks (#9874)
Co-authored-by: Zhiqiang Xie <xiezhq@stanford.edu>
This commit is contained in:
@@ -2403,6 +2403,9 @@ class Scheduler(
|
||||
# This only works for requests that have not started anything.
|
||||
# We still need to send something back to TokenizerManager to clean up the state.
|
||||
req = self.waiting_queue.pop(i)
|
||||
if self.enable_hicache_storage:
|
||||
# to release prefetch events associated with the request
|
||||
self.tree_cache.release_aborted_request(req.rid)
|
||||
self.send_to_tokenizer.send_pyobj(AbortReq(req.rid))
|
||||
# For disaggregation decode mode, the request in the waiting queue has KV cache allocated.
|
||||
if self.disaggregation_mode == DisaggregationMode.DECODE:
|
||||
|
||||
Reference in New Issue
Block a user