[Auto Sync] Update scheduler.py (20251017) (#11738)
This commit is contained in:
@@ -2166,6 +2166,12 @@ class Scheduler(
|
||||
batch.prepare_for_decode()
|
||||
return batch
|
||||
|
||||
# placeholder for override
|
||||
def update_cache_from_scheduler(
|
||||
self, schedule_batch: ScheduleBatch, batch_result: GenerationBatchResult
|
||||
):
|
||||
pass
|
||||
|
||||
def run_batch(
|
||||
self, batch: ScheduleBatch
|
||||
) -> Union[GenerationBatchResult, EmbeddingBatchResult]:
|
||||
@@ -2242,6 +2248,7 @@ class Scheduler(
|
||||
batch_or_worker_batch
|
||||
)
|
||||
future_indices_or_next_token_ids = batch_result.next_token_ids
|
||||
self.update_cache_from_scheduler(batch, batch_result)
|
||||
|
||||
# NOTE: future_indices_or_next_token_ids is used in ScheduleBatch,
|
||||
# which can probably be replaced by future_indices later [TODO(lsyin)].
|
||||
|
||||
Reference in New Issue
Block a user