From 69adc4f81c56403803840e49e4fe5385667bb55f Mon Sep 17 00:00:00 2001 From: Xinyuan Tong <115166877+JustinTong0323@users.noreply.github.com> Date: Mon, 21 Jul 2025 17:06:35 -0700 Subject: [PATCH] fix: retrieve mm token by modality, raise error if none (#8221) Signed-off-by: Xinyuan Tong Co-authored-by: Xinyuan Tong --- .../multimodal/processors/base_processor.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/python/sglang/srt/multimodal/processors/base_processor.py b/python/sglang/srt/multimodal/processors/base_processor.py index b79d90b98..3d548a19e 100644 --- a/python/sglang/srt/multimodal/processors/base_processor.py +++ b/python/sglang/srt/multimodal/processors/base_processor.py @@ -101,6 +101,14 @@ class MultimodalSpecialTokens: return None + def get_token_id_by_modality(self, modality: Modality) -> Optional[int]: + return { + Modality.IMAGE: self.image_token_id, + Modality.MULTI_IMAGES: self.image_token_id, + Modality.VIDEO: self.video_token_id, + Modality.AUDIO: self.audio_token_id, + }.get(modality) + def parse_regex(self): if self.image_token_regex is None and self.image_token is not None: self.image_token_regex = re.compile(re.escape(self.image_token)) @@ -608,14 +616,12 @@ class BaseMultimodalProcessor(ABC): # Add offsets to all items for mm_item in all_collected_items: + mm_token_id = mm_tokens.get_token_id_by_modality(mm_item.modality) + if mm_token_id is None: + raise ValueError(f"No token id found for modality: {mm_item.modality}") mm_item.offsets = self.get_mm_items_offset( input_ids=input_ids, - mm_token_id={ - Modality.IMAGE: mm_tokens.image_token_id, - Modality.MULTI_IMAGES: mm_tokens.image_token_id, - Modality.VIDEO: mm_tokens.video_token_id, - Modality.AUDIO: mm_tokens.audio_token_id, - }.get(mm_item.modality, None), + mm_token_id=mm_token_id, ) return all_collected_items, input_ids, ret