commit 8ea624951615439dec1a6d403b2e816cf3a37bdf Author: ModelHub XC Date: Tue Sep 29 10:20:18 2026 +0800 初始化项目，由ModelHub XC社区提供模型 Model: SeaLLMs/SeaLLM-7B-v2.5 Source: Original Platform diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..52373fe --- /dev/null +++ b/.gitattributes @@ -0,0 +1,36 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..160788a --- /dev/null +++ b/README.md @@ -0,0 +1,341 @@ +--- +license: other +license_name: seallms +license_link: https://huggingface.co/SeaLLMs/SeaLLM-13B-Chat/blob/main/LICENSE +language: +- en +- zh +- vi +- id +- th +- ms +- km +- lo +- my +- tl +tags: +- multilingual +- sea +--- + +

+ +

+ +# *SeaLLM-7B-v2.5* - Large Language Models for Southeast Asia + + +

+Website +   + 🤗 Tech Memo +   + 🤗 DEMO +   +Github +   +Technical Report +

+ +# BIG NEWS: SeaLLM3 is released with state-of-the-art performance of diverse tasks while specifically enhanced to be more trustworthy. Please consider using that latest model version. + +🔥[HOT] SeaLLMs project now has a dedicated website - [damo-nlp-sg.github.io/SeaLLMs](https://damo-nlp-sg.github.io/SeaLLMs/) + +We introduce [SeaLLM-7B-v2.5](https://huggingface.co/SeaLLMs/SeaLLM-7B-v2.5), the state-of-the-art multilingual LLM for Southeast Asian (SEA) languages 🇬🇧 🇨🇳 🇻🇳 🇮🇩 🇹🇭 🇲🇾 🇰🇭 🇱🇦 🇲🇲 🇵🇭. It is the most significant upgrade since [SeaLLM-13B](https://huggingface.co/SeaLLMs/SeaLLM-13B-Chat), with half the size, outperforming performance across diverse multilingual tasks, from world knowledge, math reasoning, instruction following, etc. + +### Highlights +* [SeaLLM-7B-v2.5](https://huggingface.co/SeaLLMs/SeaLLM-7B-v2.5) outperforms GPT-3.5 and achieves 7B SOTA on most multilingual knowledge benchmarks for SEA languages (MMLU, M3Exam & VMLU). +* It achieves 79.0 and 34.9 on GSM8K and MATH, surpassing GPT-3.5 in MATH. + +### Release and DEMO + +- DEMO: + - [SeaLLMs/SeaLLM-7B-v2.5](https://huggingface.co/spaces/SeaLLMs/SeaLLM-7B-v2.5). + - [SeaLLMs/SeaLLM-7B | SeaLMMM-7B](https://huggingface.co/spaces/SeaLLMs/SeaLLM-7B) - Experimental multimodal SeaLLM. +- Technical report: [Arxiv: SeaLLMs - Large Language Models for Southeast Asia](https://arxiv.org/pdf/2312.00738.pdf). +- Model weights: + - [SeaLLM-7B-v2.5](https://huggingface.co/SeaLLMs/SeaLLM-7B-v2.5). + - [SeaLLM-7B-v2.5-GGUF](https://huggingface.co/SeaLLMs/SeaLLM-7B-v2.5-GGUF). +- Run locally: + - [LM-studio](https://lmstudio.ai/): + - [SeaLLM-7B-v2.5-q4_0-chatml](https://huggingface.co/SeaLLMs/SeaLLM-7B-v2.5-GGUF/blob/main/seallm-7b-v2.5-chatml.Q4_K_M.gguf) with ChatML template (`` token changed to `<|im_end|>`) + - [SeaLLM-7B-v2.5-q4_0](https://huggingface.co/SeaLLMs/SeaLLM-7B-v2.5-GGUF/blob/main/seallm-7b-v2.5.Q4_K_M.gguf) - must use SeaLLM-7B-v2.5 chat format. + - [MLX for Apple Silicon](https://github.com/ml-explore/mlx): [SeaLLMs/SeaLLM-7B-v2.5-mlx-quantized](https://huggingface.co/SeaLLMs/SeaLLM-7B-v2.5-mlx-quantized) +- Previous models: + - [SeaLLM-7B-v2](https://huggingface.co/SeaLLMs/SeaLLM-7B-v2) + - [SeaLLM-7B-v1](https://huggingface.co/SeaLLMs/SeaLLM-7B-v1) + +
+

Terms of Use and License: +By using our released weights, codes, and demos, you agree to and comply with the terms and conditions specified in our SeaLLMs Terms Of Use. +

+ +> **Disclaimer**: +> We must note that even though the weights, codes, and demos are released in an open manner, similar to other pre-trained language models, and despite our best efforts in red teaming and safety fine-tuning and enforcement, our models come with potential risks, including but not limited to inaccurate, misleading or potentially harmful generation. +> Developers and stakeholders should perform their own red teaming and provide related security measures before deployment, and they must abide by and comply with local governance and regulations. +> In no event shall the authors be held liable for any claim, damages, or other liability arising from the use of the released weights, codes, or demos. + +> The logo was generated by DALL-E 3. + + +### What's new since SeaLLM-7B-v2? + +* SeaLLM-7B-v2.5 was built on top of Gemma-7b, and underwent large scale SFT and carefully designed alignment. + + +## Evaluation + + +### Multilingual World Knowledge + + +We evaluate models on 3 benchmarks following the recommended default setups: 5-shot MMLU for En, 3-shot [M3Exam](https://arxiv.org/pdf/2306.05179.pdf) (M3e) for En, Zh, Vi, Id, Th, and zero-shot [VMLU](https://vmlu.ai/) for Vi. + +| Model | Langs | En
MMLU | En
M3e | Zh
M3e | Vi
M3e | Vi
VMLU | Id
M3e | Th
M3e +|-----| ----- | --- | -- | ----- | ---- | --- | --- | --- | +| GPT-3.5 | Multi | 68.90 | 75.46 | 60.20 | 58.64 | 46.32 | 49.27 | 37.41 +| Vistral-7B-chat | Mono | 56.86 | 67.00 | 44.56 | 54.33 | 50.03 | 36.49 | 25.27 +| Qwen1.5-7B-chat | Multi | 61.00 | 52.07 | 81.96 | 43.38 | 45.02 | 24.29 | 20.25 +| SailorLM | Multi | 52.72 | 59.76 | 67.74 | 50.14 | --- | 39.53 | 37.73 +| SeaLLM-7B-v2 | Multi | 61.89 | 70.91 | 55.43 | 51.15 | 45.74 | 42.25 | 35.52 +| SeaLLM-7B-v2.5 | Multi | 64.05 | 76.87 | 62.54 | 63.11 | 53.30 | 48.64 | 46.86 + + +### Zero-shot CoT Multilingual Math Reasoning + + + +| Model | GSM8K
en | MATH
en | GSM8K
zh | MATH
zh | GSM8K
vi | MATH
vi | GSM8K
id | MATH
id | GSM8K
th | MATH
th +| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | +| GPT-3.5 | 80.8 | 34.1 | 48.2 | 21.5 | 55 | 26.5 | 64.3 | 26.4 | 35.8 | 18.1 +| Qwen-14B-chat | 61.4 | 18.4 | 41.6 | 11.8 | 33.6 | 3.6 | 44.7 | 8.6 | 22 | 6.0 +| Vistral-7b-chat | 48.2 | 12.5 | | | 48.7 | 3.1 | | | | +| Qwen1.5-7B-chat | 56.8 | 15.3 | 40.0 | 2.7 | 37.7 | 9 | 36.9 | 7.7 | 21.9 | 4.7 +| SeaLLM-7B-v2 | 78.2 | 27.5 | 53.7 | 17.6 | 69.9 | 23.8 | 71.5 | 24.4 | 59.6 | 22.4 +| SeaLLM-7B-v2.5 | 78.5 | 34.9 | 51.3 | 22.1 | 72.3 | 30.2 | 71.5 | 30.1 | 62.0 | 28.4 + + +Baselines were evaluated using their respective chat-template and system prompts ([Qwen1.5-7B-chat](https://huggingface.co/Qwen/Qwen1.5-7B-Chat/blob/main/tokenizer_config.json), [Vistral](https://huggingface.co/Viet-Mistral/Vistral-7B-Chat)). + +#### Zero-shot MGSM + +[SeaLLM-7B-v2.5](https://huggingface.co/SeaLLMs/SeaLLM-7B-v2.5) also outperforms GPT-3.5 and Qwen-14B on the multilingual MGSM for Thai. + +| Model | MGSM-Zh | MGSM-Th +|-----| ----- | --- +| ChatGPT (reported) | 61.2 | 47.2 +| Qwen-14B-chat | 59.6 | 28 +| SeaLLM-7B-v2 | **64.8** | 62.4 +| SeaLLM-7B-v2.5 | 58.0 | **64.8** + + +### Sea-Bench + +![fig_sea_bench_side_by_side.png](fig_sea_bench_side_by_side.png) + + +### Usage + +**IMPORTANT NOTICE for using the model** + +* `` must be at start of prompt, ff your code's tokenizer does not prepend `` by default, you MUST prepend into the prompt yourself, otherwise, it would not work! +* Repitition penalty (e.g: in llama.cpp, ollama, LM-studio) must be set to **1** , otherwise will lead to degeneration! + +#### Instruction format + +```python +# ! WARNING, if your code's tokenizer does not prepend by default, +# You MUST prepend into the prompt yourself, otherwise, it would not work! + +prompt = """<|im_start|>system +You are a helpful assistant. +<|im_start|>user +Hello world +<|im_start|>assistant +Hi there, how can I help?""" + +# <|im_start|> is not a special token. +# Transformers chat_template should be consistent with vLLM format below. + +# ! ENSURE 1 and only 1 bos `` at the beginning of sequence +print(tokenizer.convert_ids_to_tokens(tokenizer.encode(prompt))) + +""" +``` + +#### Using transformers's chat_template + +Install the latest transformers (>4.40) + +```python + +from transformers import AutoModelForCausalLM, AutoTokenizer + +device = "cuda" # the device to load the model onto + +# use bfloat16 to ensure the best performance. +model = AutoModelForCausalLM.from_pretrained("SeaLLMs/SeaLLM-7B-v2.5", torch_dtype=torch.bfloat16, device_map=device) +tokenizer = AutoTokenizer.from_pretrained("SeaLLMs/SeaLLM-7B-v2.5") + +messages = [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "Hello world"}, + {"role": "assistant", "content": "Hi there, how can I help you today?"}, + {"role": "user", "content": "Explain general relativity in details."} +] + +encodeds = tokenizer.apply_chat_template(messages, return_tensors="pt", add_generation_prompt=True) +print(tokenizer.convert_ids_to_tokens(encodeds[0])) + +model_inputs = encodeds.to(device) +model.to(device) + +generated_ids = model.generate(model_inputs, max_new_tokens=1000, do_sample=True, pad_token_id=tokenizer.pad_token_id) +decoded = tokenizer.batch_decode(generated_ids) +print(decoded[0]) + +``` + +#### Using vLLM + +```python +from vllm import LLM, SamplingParams +TURN_TEMPLATE = "<|im_start|>{role}\n{content}\n" +TURN_PREFIX = "<|im_start|>{role}\n" + +def seallm_chat_convo_format(conversations, add_assistant_prefix: bool, system_prompt=None): + # conversations: list of dict with key `role` and `content` (openai format) + if conversations[0]['role'] != 'system' and system_prompt is not None: + conversations = [{"role": "system", "content": system_prompt}] + conversations + text = '' + for turn_id, turn in enumerate(conversations): + prompt = TURN_TEMPLATE.format(role=turn['role'], content=turn['content']) + text += prompt + if add_assistant_prefix: + prompt = TURN_PREFIX.format(role='assistant') + text += prompt + return text + +sparams = SamplingParams(temperature=0.1, max_tokens=1024, stop=['', '<|im_start|>']) +llm = LLM("SeaLLMs/SeaLLM-7B-v2.5", dtype="bfloat16") + +message = "Explain general relativity in details." +prompt = seallm_chat_convo_format(message, True) +gen = llm.generate(prompt, sampling_params) + +print(gen[0].outputs[0].text) +``` + +#### Fine-tuning SeaLLM-7B-v2.5 + +Should follow the chat format and accurately mask out source tokens. Here is an example. + +```python +conversations = [ + {"role": "system", "content": "You are helful assistant."}, + {"role": "user", "content": "Hello world."}, + {"role": "assistant", "content": "Hi there, how can I help?"}, + {"role": "user", "content": "Tell me a joke."}, + {"role": "assistant", "content": "Why don't scientists trust atoms? Because they make up everything."}, +] +def seallm_7b_v25_tokenize_multi_turns(tokenizer, conversations, add_assistant_prefix=False): + """ + Inputs: + conversations: list of dict following openai format, eg + conversations = [ + {"role": "system", "content": "You are helful assistant."}, + {"role": "user", "content": "Hello world."}, + {"role": "assistant", "content": "Hi there, how can I help?"}, + {"role": "user", "content": "Tell me a joke."}, + {"role": "assistant", "content": "Why don't scientists trust atoms? Because they make up everything."}, + ] + add_assistant_prefix: whether to add assistant_prefix, only for inference decoding + Outputs: + tokenize_output_sample, { + "input_ids": ... + "token_type_ids": 1 if train and 0 if masked out (not train) + } + During training, need to create a labels, with masked-out tokens = -100 to avoid loss computations. + labels = sample['input_ids'].clone() + labels[sample['token_type_ids'] == 0] = -100 + """ + TURN_TEMPLATE = "<|im_start|>{role}\n{content}\n" + TURN_PREFIX = "<|im_start|>{role}\n" + TURN_SUFFIX = "\n" + TURN_SUFFIX_TAKE = "" + sample = None + assistant_prefix_len = None + assistant_suffix_len = None + for turn_id, turn in enumerate(conversations): + prompt = TURN_TEMPLATE.format(role=turn['role'], content=turn['content']) + turn_sample = tokenizer( + prompt, padding=False, truncation=False, verbose=False, add_special_tokens=False, + return_token_type_ids=True, + ) + if turn['role'] == 'assistant': + if assistant_prefix_len is None: + assistant_prefix_len = len(tokenizer.encode(TURN_PREFIX.format(role=turn['role']), add_special_tokens=False)) + if assistant_suffix_len is None: + assistant_suffix_len = ( + len(tokenizer.encode(TURN_SUFFIX.format(role=turn['role']), add_special_tokens=False)) - + len(tokenizer.encode(TURN_SUFFIX_TAKE, add_special_tokens=False)) + ) + turn_sample['token_type_ids'][assistant_prefix_len:-assistant_suffix_len] = [1] * (len(turn_sample['input_ids']) - assistant_prefix_len - assistant_suffix_len) + if sample is None: + sample = turn_sample + else: + for k in turn_sample.keys(): + sample[k].extend(turn_sample[k]) + if add_assistant_prefix: + assistant_prefix_sample = tokenizer( + TURN_PREFIX.format(role="assistant"), padding=False, truncation=False, verbose=False, add_special_tokens=False, + return_token_type_ids=True, + ) + for k in sample.keys(): + sample[k].extend(assistant_prefix_sample[k]) + if tokenizer.add_bos_token: + sample['input_ids'] = [tokenizer.bos_token_id] + sample['input_ids'] + sample['attention_mask'] = [1] + sample['attention_mask'] + sample['token_type_ids'] = [sample['token_type_ids'][0]] + sample['token_type_ids'] + return sample + +# ! testing +sample = seallm_7b_v25_tokenize_multi_turns(tokenizer, conversations) +tokens = tokenizer.convert_ids_to_tokens(sample['input_ids']) +pairs = [(x, y) for x, y in zip(tokens, sample['token_type_ids'])] +print(pairs) + +# source and special tokens is masked out (token_type 0), only assistant with is trained (token_type 1) +# [('', 0), ('<', 0), ('|', 0), ..., ('assistant', 0), ('\n', 0), ('Hi', 1), ('▁there', 1), (',', 1), ('▁how', 1), ('▁can', 1), ('▁I', 1), ('▁help', 1), ('?', 1), ('', 1), ('\n', 0), ('<', 0), ... + +``` + + +## Acknowledgement to Our Linguists + +We would like to express our special thanks to our professional and native linguists, Tantong Champaiboon, Nguyen Ngoc Yen Nhi and Tara Devina Putri, who helped build, evaluate, and fact-check our sampled pretraining and SFT dataset as well as evaluating our models across different aspects, especially safety. + +## Citation + +If you find our project useful, we hope you would kindly star our repo and cite our work as follows: Corresponding Author: [l.bing@alibaba-inc.com](mailto:l.bing@alibaba-inc.com) + +**Author list and order will change!** + +* `*` and `^` are equal contributions. + +``` +@article{damonlpsg2023seallm, + author = {Xuan-Phi Nguyen*, Wenxuan Zhang*, Xin Li*, Mahani Aljunied*, Weiwen Xu, Hou Pong Chan, + Zhiqiang Hu, Chenhui Shen^, Yew Ken Chia^, Xingxuan Li, Jianyu Wang, + Qingyu Tan, Liying Cheng, Guanzheng Chen, Yue Deng, Sen Yang, + Chaoqun Liu, Hang Zhang, Lidong Bing}, + title = {SeaLLMs - Large Language Models for Southeast Asia}, + year = 2023, + Eprint = {arXiv:2312.00738}, +} +``` + diff --git a/config.json b/config.json new file mode 100644 index 0000000..5357eff --- /dev/null +++ b/config.json @@ -0,0 +1,29 @@ +{ + "_name_or_path": "seallm-7b-v2.5-v1", + "architectures": [ + "GemmaForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 2, + "eos_token_id": 1, + "head_dim": 256, + "hidden_act": "gelu", + "hidden_activation": null, + "hidden_size": 3072, + "initializer_range": 0.02, + "intermediate_size": 24576, + "max_position_embeddings": 8192, + "model_type": "gemma", + "num_attention_heads": 16, + "num_hidden_layers": 28, + "num_key_value_heads": 16, + "pad_token_id": 0, + "rms_norm_eps": 1e-06, + "rope_scaling": null, + "rope_theta": 10000.0, + "torch_dtype": "bfloat16", + "transformers_version": "4.40.0.dev0", + "use_cache": true, + "vocab_size": 256000 +} diff --git a/fig_sea_bench_side_by_side.png b/fig_sea_bench_side_by_side.png new file mode 100644 index 0000000..457ce6f Binary files /dev/null and b/fig_sea_bench_side_by_side.png differ diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..3e78eff --- /dev/null +++ b/generation_config.json @@ -0,0 +1,7 @@ +{ + "_from_model_config": true, + "bos_token_id": 2, + "eos_token_id": 1, + "pad_token_id": 0, + "transformers_version": "4.40.0.dev0" +} diff --git a/model-00001-of-00004.safetensors b/model-00001-of-00004.safetensors new file mode 100644 index 0000000..6b526f0 --- /dev/null +++ b/model-00001-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6b05ff3dd9f5f12e8f39159bee2d16bfe0904e749f1ad202663cee41c826517a +size 4995496656 diff --git a/model-00002-of-00004.safetensors b/model-00002-of-00004.safetensors new file mode 100644 index 0000000..86b1f3f --- /dev/null +++ b/model-00002-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cbf0ba224c8425f25ffea585262adb06a804b715b1f6ebeeb1aa025b9ca7e355 +size 4982953168 diff --git a/model-00003-of-00004.safetensors b/model-00003-of-00004.safetensors new file mode 100644 index 0000000..0985809 --- /dev/null +++ b/model-00003-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:196e6cd44cdb771f32338af196be506b8fc25c6305a105e1c5f101750d698bc8 +size 4982953200 diff --git a/model-00004-of-00004.safetensors b/model-00004-of-00004.safetensors new file mode 100644 index 0000000..df8e8e2 --- /dev/null +++ b/model-00004-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:912cf60eafce5e3746feda0933f13e4d7f288e8d4611f3d98ff17e6a233759b3 +size 2113988336 diff --git a/model.safetensors.index.json b/model.safetensors.index.json new file mode 100644 index 0000000..ea169ac --- /dev/null +++ b/model.safetensors.index.json @@ -0,0 +1,261 @@ +{ + "metadata": { + "total_size": 17075361792 + }, + "weight_map": { + "model.embed_tokens.weight": "model-00001-of-00004.safetensors", + "model.layers.0.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.10.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.10.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.11.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.12.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.13.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.14.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.15.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.15.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.15.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.15.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.16.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.16.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.16.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.16.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.16.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.16.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.16.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.16.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.17.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.17.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.17.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.17.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.17.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.17.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.17.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.17.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.17.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.18.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.18.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.18.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.18.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.18.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.18.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.18.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.18.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.18.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.19.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.19.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.19.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.19.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.19.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.19.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.19.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.19.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.19.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.2.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.20.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.20.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.20.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.20.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.20.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.20.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.20.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.20.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.20.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.21.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.21.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.22.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.22.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.23.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.24.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.24.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.24.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.24.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.25.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.25.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.25.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.25.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.25.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.25.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.25.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.25.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.26.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.26.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.26.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.26.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.26.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.26.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.26.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.26.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.26.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.27.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.27.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.27.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.27.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.27.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.27.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.27.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.27.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.27.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.3.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.4.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.5.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.6.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.6.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.6.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.6.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.7.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.7.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.7.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.7.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.7.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.7.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.7.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.7.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.8.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.8.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.8.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.8.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.8.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.8.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.8.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.8.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.8.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.9.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.9.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.norm.weight": "model-00004-of-00004.safetensors" + } +} diff --git a/seal_logo.png b/seal_logo.png new file mode 100644 index 0000000..35defdc Binary files /dev/null and b/seal_logo.png differ diff --git a/special_tokens_map.json b/special_tokens_map.json new file mode 100644 index 0000000..f611958 --- /dev/null +++ b/special_tokens_map.json @@ -0,0 +1,30 @@ +{ + "bos_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "unk_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + } +} diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..7d526fa --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d0d908b4f9326e0998815690e325b6abbd378978553e10627924dd825db7e243 +size 17477553 diff --git a/tokenizer.model b/tokenizer.model new file mode 100644 index 0000000..796efe9 --- /dev/null +++ b/tokenizer.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:61a7b147390c64585d6c3543dd6fc636906c9af3865a5548f27f31aee1d4c8e2 +size 4241003 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..c9e3b28 --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,50 @@ +{ + "add_bos_token": true, + "add_eos_token": false, + "added_tokens_decoder": { + "0": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "1": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "2": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "3": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + } + }, + "bos_token": "", + "chat_template": "{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{{ bos_token }}{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}", + "clean_up_tokenization_spaces": false, + "eos_token": "", + "legacy": null, + "model_max_length": 1000000000000000019884624838656, + "pad_token": "", + "sp_model_kwargs": {}, + "spaces_between_special_tokens": false, + "tokenizer_class": "GemmaTokenizer", + "unk_token": "", + "use_default_system_prompt": false +}