Support InternVL3 (#5350)

Co-authored-by: Mick <mickjagger19@icloud.com>
Co-authored-by: Chayenne <zhaochen20@outlook.com>
This commit is contained in:
xm:D
2025-05-02 13:38:59 +08:00
committed by GitHub
parent 73dcf2b326
commit 3409aaab32
12 changed files with 1728 additions and 9 deletions

View File

@@ -270,6 +270,29 @@ register_chat_template(
)
)
register_chat_template(
ChatTemplate(
name="janus",
default_system_prompt=None,
role_prefix_and_suffix={
"system": (
"",
"",
),
"user": (
"<User>",
"",
),
"assistant": (
"<Assistant>",
"<end▁of▁sentence>",
),
},
stop_str=("<end▁of▁sentence>",),
image_token="<image_placeholder>\n",
)
)
# The difference between "llama-3-instruct-llava" and "llama-3-instruct" is that llava uses a different image_token.
register_chat_template(
ChatTemplate(
@@ -395,6 +418,20 @@ register_chat_template(
)
)
# Adapted from https://huggingface.co/OpenGVLab/InternVL2-4B/blob/main/modeling_intern_vit.py
register_chat_template(
ChatTemplate(
name="internvl-2-5",
default_system_prompt="你是书生·万象英文名是InternVL是由上海人工智能实验室、清华大学及多家合作单位联合开发的多模态大语言模型。",
role_prefix_and_suffix={
"system": ("<|im_start|>system\n", "<|im_end|>\n"),
"user": ("<|im_start|>user\n", "<|im_end|>\n"),
"assistant": ("<|im_start|>assistant\n", "<|im_end|>\n"),
},
stop_str=["<|im_end|>", "<|action_end|>"],
)
)
register_chat_template(
ChatTemplate(
name="granite-3-instruct",
@@ -565,6 +602,13 @@ def match_gemma3_instruct(model_path: str):
return get_chat_template("gemma-it")
@register_chat_template_matching_function
def match_internvl_chat(model_path: str):
model_path = model_path.lower()
if "internvl" in model_path:
return get_chat_template("internvl-2-5")
if __name__ == "__main__":
messages = [
{"role": "system", "content": None}, # None means default