From 84719b527a2dc92b0ccf1b3a4408abda33d22e38 Mon Sep 17 00:00:00 2001 From: Xinyuan Tong <115166877+JustinTong0323@users.noreply.github.com> Date: Thu, 21 Aug 2025 07:43:03 +0800 Subject: [PATCH] fix: InternS1 don't recognize image, updates image token for InternVL processor (#9381) Signed-off-by: Xinyuan Tong --- python/sglang/srt/conversation.py | 17 ++--------------- .../srt/multimodal/processors/internvl.py | 9 +++++++-- 2 files changed, 9 insertions(+), 17 deletions(-) diff --git a/python/sglang/srt/conversation.py b/python/sglang/srt/conversation.py index 84cb1db36..dde9632b8 100644 --- a/python/sglang/srt/conversation.py +++ b/python/sglang/srt/conversation.py @@ -625,7 +625,7 @@ def generate_chat_conv( real_content += content.text elif content.type == "image_url": # NOTE: works for llava and intervl2_5 - if conv.name in ["internvl-2-5", "interns1"]: + if conv.name in ["internvl-2-5"]: real_content = image_token + real_content else: real_content += image_token @@ -817,20 +817,7 @@ register_conv_template( sep_style=SeparatorStyle.MPT, sep="<|im_end|>\n", stop_str=["<|im_end|>", "<|action_end|>"], - image_token="", - ) -) - -register_conv_template( - Conversation( - name="interns1", - system_template="<|im_start|>system\n{system_message}", - system_message="You are an AI assistant whose name is Intern-S1 (书生大模型).\n- Intern-S1 (书生大模型) is a vision-language model that is developed by Shanghai AI Laboratory (上海人工智能实验室). It is designed to be helpful, honest, and harmless.\n- Intern-S1 (书生大模型) can understand and communicate fluently in the language chosen by the user such as English and 中文.\nYou are an expert reasoner with extensive experience in all areas. You approach problems through systematic thinking and rigorous reasoning. Your response should reflect deep understanding and precise logical thinking, making your solution path and reasoning clear to others. Please put your thinking process within ... tags.", - roles=("<|im_start|>user\n", "<|im_start|>assistant\n"), - sep_style=SeparatorStyle.MPT, - sep="<|im_end|>\n", - stop_str=["<|im_end|>", "<|action_end|>"], - image_token="", + image_token="", ) ) diff --git a/python/sglang/srt/multimodal/processors/internvl.py b/python/sglang/srt/multimodal/processors/internvl.py index 6ab17b1a9..b12e377a9 100644 --- a/python/sglang/srt/multimodal/processors/internvl.py +++ b/python/sglang/srt/multimodal/processors/internvl.py @@ -44,7 +44,7 @@ class InternVLImageProcessor(BaseMultimodalProcessor): self.img_start_token_id = tokenizer.convert_tokens_to_ids(self.IMG_START_TOKEN) self.img_end_token_id = tokenizer.convert_tokens_to_ids(self.IMG_END_TOKEN) self.mm_tokens = MultimodalSpecialTokens( - image_token="", + image_token="", image_token_id=tokenizer.convert_tokens_to_ids(self.IMG_CONTEXT_TOKEN), ).build(_image_processor) @@ -218,13 +218,18 @@ class InternVLImageProcessor(BaseMultimodalProcessor): pixel_values = torch.cat(pixel_values, dim=0) + original_placeholder = "<<<__IMG_CONTEXT_PLACEHOLDER__>>>" + input_text = input_text.replace(self.IMG_CONTEXT_TOKEN, original_placeholder) + for idx, num_patches in enumerate(num_patches_list): image_tokens = ( self.IMG_START_TOKEN + self.IMG_CONTEXT_TOKEN * self.num_image_token * num_patches + self.IMG_END_TOKEN ) - input_text = input_text.replace("", image_tokens, 1) + input_text = input_text.replace(original_placeholder, image_tokens, 1) + + input_text = input_text.replace(original_placeholder, self.IMG_CONTEXT_TOKEN) input_ids = self.tokenizer(input_text, return_tensors="pt")[ "input_ids"