commit f4986a78ea8bde5a94ee634c29c2b66395a554f9 Author: ModelHub XC Date: Thu May 14 02:27:35 2026 +0800 初始化项目，由ModelHub XC社区提供模型 Model: GeneZC/MiniChat-2-3B Source: Original Platform diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..a6344aa --- /dev/null +++ b/.gitattributes @@ -0,0 +1,35 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..2898944 --- /dev/null +++ b/README.md @@ -0,0 +1,227 @@ +--- +language: +- en +- zh +license: apache-2.0 +library_name: transformers +widget: +- text: [|User|] Hi 👋 [|Assistant|] +model-index: +- name: MiniChat-2-3B + results: + - task: + type: text-generation + name: Text Generation + dataset: + name: AI2 Reasoning Challenge (25-Shot) + type: ai2_arc + config: ARC-Challenge + split: test + args: + num_few_shot: 25 + metrics: + - type: acc_norm + value: 44.88 + name: normalized accuracy + source: + url: https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard?query=GeneZC/MiniChat-2-3B + name: Open LLM Leaderboard + - task: + type: text-generation + name: Text Generation + dataset: + name: HellaSwag (10-Shot) + type: hellaswag + split: validation + args: + num_few_shot: 10 + metrics: + - type: acc_norm + value: 67.69 + name: normalized accuracy + source: + url: https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard?query=GeneZC/MiniChat-2-3B + name: Open LLM Leaderboard + - task: + type: text-generation + name: Text Generation + dataset: + name: MMLU (5-Shot) + type: cais/mmlu + config: all + split: test + args: + num_few_shot: 5 + metrics: + - type: acc + value: 47.59 + name: accuracy + source: + url: https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard?query=GeneZC/MiniChat-2-3B + name: Open LLM Leaderboard + - task: + type: text-generation + name: Text Generation + dataset: + name: TruthfulQA (0-shot) + type: truthful_qa + config: multiple_choice + split: validation + args: + num_few_shot: 0 + metrics: + - type: mc2 + value: 49.64 + source: + url: https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard?query=GeneZC/MiniChat-2-3B + name: Open LLM Leaderboard + - task: + type: text-generation + name: Text Generation + dataset: + name: Winogrande (5-shot) + type: winogrande + config: winogrande_xl + split: validation + args: + num_few_shot: 5 + metrics: + - type: acc + value: 66.46 + name: accuracy + source: + url: https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard?query=GeneZC/MiniChat-2-3B + name: Open LLM Leaderboard + - task: + type: text-generation + name: Text Generation + dataset: + name: GSM8k (5-shot) + type: gsm8k + config: main + split: test + args: + num_few_shot: 5 + metrics: + - type: acc + value: 32.68 + name: accuracy + source: + url: https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard?query=GeneZC/MiniChat-2-3B + name: Open LLM Leaderboard +--- + +## MiniChat-2-3B + +📑 [arXiv](https://arxiv.org/abs/2311.07052) | 👻 [GitHub](https://github.com/GeneZC/MiniMA) | 🤗 [HuggingFace-MiniMA](https://huggingface.co/GeneZC/MiniMA-3B) | 🤗 [HuggingFace-MiniChat](https://huggingface.co/GeneZC/MiniChat-3B) | 🤖 [ModelScope-MiniMA](https://modelscope.cn/models/GeneZC/MiniMA-3B) | 🤖 [ModelScope-MiniChat](https://modelscope.cn/models/GeneZC/MiniChat-3B) | 🤗 [HuggingFace-MiniChat-1.5](https://huggingface.co/GeneZC/MiniChat-1.5-3B) | 🤗 [HuggingFace-MiniMA-2](https://huggingface.co/GeneZC/MiniMA-2-3B) | 🤗 [HuggingFace-MiniChat-2](https://huggingface.co/GeneZC/MiniChat-2-3B) + +🆕 **Updates from MiniChat-3B**: +- better base model MiniMA-2-3B; +- better data mixture; +- use of [NEFTune](https://arxiv.org/abs/2310.05914); +- use of [DPO](https://arxiv.org/abs/2305.18290). + +❗ Must comply with LICENSE of LLaMA2 since it is derived from LLaMA2. + +A language model continued from MiniMA-3B and finetuned on both instruction and preference data. + +Surpassing Vicuna-7B and approximating LLaMA-2-Chat-7B on MT-Bench. + +teaser_b + +**Standard Benchmarks** + +|Method|TFLOPs|MMLU (5-shot)|CEval (5-shot)|DROP (3-shot)|HumanEval (0-shot)|BBH (3-shot)|GSM8K (8-shot)| +|--|--|--|--|--|--|--|--| +|Mamba-2.8B|4.6E9|25.58|24.74|15.72|7.32|29.37|3.49| +|ShearedLLaMA-2.7B|0.8E9|26.97|22.88|19.98|4.88|30.48|3.56| +|BTLM-3B|11.3E9|27.20|26.00|17.84|10.98|30.87|4.55| +|StableLM-3B|72.0E9|44.75|31.05|22.35|15.85|32.59|10.99| +|Qwen-1.8B|23.8E9|44.05|54.75|12.97|14.02|30.80|22.97| +|Phi-2-2.8B|159.9E9|56.74|34.03|30.74|46.95|44.13|55.42| +|LLaMA-2-7B|84.0E9|46.00|34.40|31.57|12.80|32.02|14.10| +|| +|MiniMA-3B|4.0E9|28.51|28.23|22.50|10.98|31.61|8.11| +|MiniChat-3B|4.0E9|38.40|36.48|22.58|18.29|31.36|29.72| +|MiniMA-2-3B|13.4E9|40.14|44.65|23.10|14.63|31.43|8.87| +|MiniChat-2-3B|13.4E9|46.17|43.91|30.26|22.56|34.95|38.13| + +**Instruction-following Benchmarks** + +|Method|AlpacaEval|MT-Bench|MT-Bench-ZH| +|--|--|--|--| +|GPT-4|95.28|9.18|8.96| +|Zephyr-7B-Beta|90.60|7.34|6.27#| +|Vicuna-7B|76.84|6.17|5.22#| +|LLaMA-2-Chat-7B|71.37|6.27|5.43#| +|Qwen-Chat-7B|-|-|6.24| +|Phi-2-DPO|81.37|-|1.59#$| +|StableLM-Zephyr-3B|76.00|6.64|4.31#| +|Rocket-3B|79.75|6.56|4.07#| +|Qwen-Chat-1.8B|-|-|5.65| +|| +|MiniChat-3B|48.82|-|-| +|MiniChat-2-3B|77.30|6.23|6.04| + +# specialized mainly for English. + +$ finetuned without multi-turn instruction data. + +The following is an example code snippet to use MiniChat-2-3B: + +```python +import torch + +from transformers import AutoModelForCausalLM, AutoTokenizer + +from conversation import get_default_conv_template + +# MiniChat +tokenizer = AutoTokenizer.from_pretrained("GeneZC/MiniChat-2-3B", use_fast=False) +# GPU. +model = AutoModelForCausalLM.from_pretrained("GeneZC/MiniChat-2-3B", use_cache=True, device_map="auto", torch_dtype=torch.float16).eval() +# CPU. +# model = AutoModelForCausalLM.from_pretrained("GeneZC/MiniChat-2-3B", use_cache=True, device_map="cpu", torch_dtype=torch.float16).eval() + +conv = get_default_conv_template("minichat") + +question = "Implement a program to find the common elements in two arrays without using any extra data structures." +conv.append_message(conv.roles[0], question) +conv.append_message(conv.roles[1], None) +prompt = conv.get_prompt() +input_ids = tokenizer([prompt]).input_ids +output_ids = model.generate( + torch.as_tensor(input_ids).cuda(), + do_sample=True, + temperature=0.7, + max_new_tokens=1024, +) +output_ids = output_ids[0][len(input_ids[0]):] +output = tokenizer.decode(output_ids, skip_special_tokens=True).strip() +# output: "def common_elements(arr1, arr2):\n if len(arr1) == 0:\n return []\n if len(arr2) == 0:\n return arr1\n\n common_elements = []\n for element in arr1:\n if element in arr2:\n common_elements.append(element)\n\n return common_elements" +# Multiturn conversation could be realized by continuously appending questions to `conv`. +``` + +## Bibtex + +```bibtex +@article{zhang2023law, + title={Towards the Law of Capacity Gap in Distilling Language Models}, + author={Zhang, Chen and Song, Dawei and Ye, Zheyu and Gao, Yan}, + year={2023}, + url={https://arxiv.org/abs/2311.07052} +} +``` +# [Open LLM Leaderboard Evaluation Results](https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard) +Detailed results can be found [here](https://huggingface.co/datasets/open-llm-leaderboard/details_GeneZC__MiniChat-2-3B) + +| Metric |Value| +|---------------------------------|----:| +|Avg. |51.49| +|AI2 Reasoning Challenge (25-Shot)|44.88| +|HellaSwag (10-Shot) |67.69| +|MMLU (5-Shot) |47.59| +|TruthfulQA (0-shot) |49.64| +|Winogrande (5-shot) |66.46| +|GSM8k (5-shot) |32.68| + diff --git a/config.json b/config.json new file mode 100644 index 0000000..5e25570 --- /dev/null +++ b/config.json @@ -0,0 +1,28 @@ +{ + "_name_or_path": "MiniChat-2-DPO-3B", + "architectures": [ + "LlamaForCausalLM" + ], + "bos_token_id": 1, + "eos_token_id": 2, + "hidden_act": "silu", + "hidden_size": 3072, + "initializer_range": 0.02, + "intermediate_size": 8192, + "max_position_embeddings": 4096, + "model_type": "llama", + "noisy_embedding_alpha": 5.0, + "num_attention_heads": 24, + "num_hidden_layers": 24, + "num_key_value_heads": 24, + "pad_token_id": 0, + "pretraining_tp": 1, + "rms_norm_eps": 1e-05, + "rope_scaling": null, + "rope_theta": 10000.0, + "tie_word_embeddings": false, + "torch_dtype": "float16", + "transformers_version": "4.33.2", + "use_cache": true, + "vocab_size": 49216 +} diff --git a/conversation.py b/conversation.py new file mode 100644 index 0000000..d4d9d2a --- /dev/null +++ b/conversation.py @@ -0,0 +1,223 @@ +""" +Conversation prompt templates. +""" + +import dataclasses +from enum import auto, Enum +from typing import List, Tuple, Any + + +class SeparatorStyle(Enum): + """Different separator style.""" + + ADD_COLON_SINGLE = auto() + ADD_COLON_TWO = auto() + NO_COLON_SINGLE = auto() + BAIZE = auto() + PHOENIX = auto() + MINICHAT = auto() + + +@dataclasses.dataclass +class Conversation: + """A class that keeps all conversation history.""" + + # System prompts + system: str + # Two roles + roles: List[str] + # All messages + messages: List[List[str]] + # Offset of few shot examples + offset: int + # Separator + sep_style: SeparatorStyle + sep: str + sep2: str = None + # Stop criteria (the default one is EOS token) + stop_str: str = None + # Stops generation if meeting any token in this list + stop_token_ids: List[int] = None + + # Used for the state in the gradio servers. + # TODO(lmzheng): refactor this + conv_id: Any = None + skip_next: bool = False + model_name: str = None + + def get_prompt(self): + if self.sep_style == SeparatorStyle.ADD_COLON_SINGLE: + ret = self.system + self.sep + for role, message in self.messages: + if message: + ret += role + ": " + message + self.sep + else: + ret += role + ": " + return ret + elif self.sep_style == SeparatorStyle.ADD_COLON_TWO: + seps = [self.sep, self.sep2] + ret = self.system + seps[0] + for i, (role, message) in enumerate(self.messages): + if message: + ret += role + ": " + message + seps[i % 2] + else: + ret += role + ": " + return ret + elif self.sep_style == SeparatorStyle.NO_COLON_SINGLE: + ret = self.system + for role, message in self.messages: + if message: + ret += role + message + self.sep + else: + ret += role + return ret + elif self.sep_style == SeparatorStyle.BAIZE: + ret = self.system + "\n" + for role, message in self.messages: + if message: + ret += role + message + "\n" + else: + ret += role + return ret + elif self.sep_style == SeparatorStyle.PHOENIX: + ret = self.system + for role, message in self.messages: + if message: + ret += role + ": " + "" + message + "" + else: + ret += role + ": " + "" + return ret + elif self.sep_style == SeparatorStyle.MINICHAT: + ret = self.system + for role, message in self.messages: + if message: + ret += role + " " + message + "" + else: + ret += role # No space is needed. + return ret + else: + raise ValueError(f"Invalid style: {self.sep_style}") + + def append_message(self, role, message): + self.messages.append([role, message]) + + def to_gradio_chatbot(self): + ret = [] + for i, (role, msg) in enumerate(self.messages[self.offset:]): + if i % 2 == 0: + ret.append([msg, None]) + else: + ret[-1][-1] = msg + return ret + + def to_openai_api_messages(self): + ret = [{"role": "system", "content": self.system}] + + for i, (_, msg) in enumerate(self.messages[self.offset:]): + if i % 2 == 0: + ret.append({"role": "user", "content": msg}) + else: + if msg is not None: + ret.append({"role": "assistant", "content": msg}) + return ret + + def copy(self): + return Conversation( + system=self.system, + roles=self.roles, + messages=[[x, y] for x, y in self.messages], + offset=self.offset, + sep_style=self.sep_style, + sep=self.sep, + sep2=self.sep2, + stop_str=self.stop_str, + stop_token_ids=self.stop_token_ids, + conv_id=self.conv_id, + model_name=self.model_name, + ) + + def dict(self): + return { + "system": self.system, + "roles": self.roles, + "messages": self.messages, + "offset": self.offset, + "conv_id": self.conv_id, + "model_name": self.model_name, + } + + +conv_vicuna = Conversation( + system="A chat between a curious user and an artificial intelligence assistant. " + "The assistant gives helpful, detailed, and polite answers to the user's questions.", + roles=("USER", "ASSISTANT"), + messages=(), + offset=0, + sep_style=SeparatorStyle.ADD_COLON_TWO, + sep=" ", + sep2="", +) + +conv_baize = Conversation( + system="The following is a conversation between a human and an AI assistant named Baize (named after a mythical creature in Chinese folklore). Baize is an open-source AI assistant developed by UCSD and Sun Yat-Sen University. The human and the AI assistant take turns chatting. Human statements start with [|Human|] and AI assistant statements start with [|AI|]. The AI assistant always provides responses in as much detail as possible, and in Markdown format. The AI assistant always declines to engage with topics, questions and instructions related to unethical, controversial, or sensitive issues. Complete the transcript in exactly that format.\n", + roles=("[|Human|]", "[|AI|]"), + messages=( + ("[|Human|]", "Hello!"), + ("[|AI|]", "Hi!"), + ), + offset=2, + sep_style=SeparatorStyle.BAIZE, + sep="\n", + stop_str="[|Human|]", +) + +conv_phoenix = Conversation( + system="A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions.\n\n", + roles=("Human", "Assistant"), + messages=(), + offset=0, + sep_style=SeparatorStyle.PHOENIX, + sep="", +) + +conv_chatgpt = Conversation( + system="You are a helpful assistant.", + roles=("user", "assistant"), + messages=(), + offset=0, + sep_style=None, + sep=None, +) + +conv_minichat = Conversation( + system="‘MiniChat’是一个由‘Beccurio’开发的AI语言模型。下面是人类和MiniChat之间的一段对话。MiniChat的回复应当尽可能详细，并且以Markdown的形式输出。MiniChat应当拒绝参与违背伦理的讨论。", + roles=("[|User|]", "[|Assistant|]"), + messages=(), + offset=0, + sep_style=SeparatorStyle.MINICHAT, + sep="", +) + + +conv_templates = { + "vicuna": conv_vicuna, + "baize": conv_baize, + "phoenix": conv_phoenix, + "chatgpt": conv_chatgpt, + "minichat": conv_minichat, +} + +def get_default_conv_template(model_name): + model_name = model_name.lower() + try: + ret = conv_templates[model_name] + return ret.copy() + except: + raise NotImplementedError(f"No support for model {model_name}.") + + +if __name__ == "__main__": + conv = conv_templates["minichat"].copy() + conv.append_message(conv.roles[0], "Write a Python function that checks if a given number is even or odd.") + conv.append_message(conv.roles[1], None) + print([conv.get_prompt()]) diff --git a/model.safetensors b/model.safetensors new file mode 100644 index 0000000..7e32e61 --- /dev/null +++ b/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d0947b9c5c7a40d4dbc70442a5d207074a73f0b2e209bf5c388238b4abf91d4b +size 6040910600 diff --git a/pytorch_model.bin b/pytorch_model.bin new file mode 100644 index 0000000..16ffecb --- /dev/null +++ b/pytorch_model.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:16211f1c8036beb41bb4d334d9eea2992fc4d4de0bb63a7ca0fda3fed885c888 +size 6040956605 diff --git a/special_tokens_map.json b/special_tokens_map.json new file mode 100644 index 0000000..d85ba6c --- /dev/null +++ b/special_tokens_map.json @@ -0,0 +1,23 @@ +{ + "bos_token": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false + }, + "unk_token": { + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false + } +} diff --git a/teaser_b.jpg b/teaser_b.jpg new file mode 100644 index 0000000..0bcb412 Binary files /dev/null and b/teaser_b.jpg differ diff --git a/tokenizer.model b/tokenizer.model new file mode 100644 index 0000000..14cb1e6 --- /dev/null +++ b/tokenizer.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ae87c0db2b21b0fa3fdc5e19d1f9cea94efb703cc7c6281d8718a6714b3cc2be +size 748869 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..74e7504 --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,38 @@ +{ + "add_bos_token": true, + "add_eos_token": false, + "bos_token": { + "__type": "AddedToken", + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false + }, + "clean_up_tokenization_spaces": false, + "eos_token": { + "__type": "AddedToken", + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false + }, + "legacy": null, + "model_max_length": 1000000000000000019884624838656, + "pad_token": null, + "sp_model_kwargs": {}, + "spaces_between_special_tokens": false, + "tokenizer_class": "LlamaTokenizer", + "unk_token": { + "__type": "AddedToken", + "content": "", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false + }, + "use_default_system_prompt": true, + "chat_template": "{{ '‘MiniChat’是一个由‘Beccurio’开发的AI语言模型。下面是人类和MiniChat之间的一段对话。MiniChat的回复应当尽可能详细，并且以Markdown的形式输出。MiniChat应当拒绝参与违背伦理的讨论。' }}{% for message in messages %}{{'[|' + message['role'].capitalize() + '|] ' + message['content'] + ''}}{% endfor %}{{ '[|Assistant|]' }}", + "use_fast": true +}