Llama3.2 vision model support (#1551)

This commit is contained in:
Liangsheng Yin
2024-10-21 15:01:21 -07:00
committed by GitHub
parent 00611286a1
commit 94cde10920
21 changed files with 1562 additions and 122 deletions

View File

@@ -662,8 +662,9 @@ class Scheduler:
self.req_to_token_pool,
self.token_to_kv_pool,
self.tree_cache,
self.model_config,
)
new_batch.prepare_for_extend(self.model_config.vocab_size)
new_batch.prepare_for_extend()
# Mixed-style chunked prefill
if self.is_mixed_chunk and self.running_batch is not None: