初始化项目，由ModelHub XC社区提供模型
Model: sail/Sailor-4B Source: Original Platform
This commit is contained in:
ModelHub XC
35
.gitattributes
vendored
Normal file
35
.gitattributes
vendored
Normal file
@@ -0,0 +1,35 @@
|
||||
*.7z filter=lfs diff=lfs merge=lfs -text
|
||||
*.arrow filter=lfs diff=lfs merge=lfs -text
|
||||
*.bin filter=lfs diff=lfs merge=lfs -text
|
||||
*.bz2 filter=lfs diff=lfs merge=lfs -text
|
||||
*.ckpt filter=lfs diff=lfs merge=lfs -text
|
||||
*.ftz filter=lfs diff=lfs merge=lfs -text
|
||||
*.gz filter=lfs diff=lfs merge=lfs -text
|
||||
*.h5 filter=lfs diff=lfs merge=lfs -text
|
||||
*.joblib filter=lfs diff=lfs merge=lfs -text
|
||||
*.lfs.* filter=lfs diff=lfs merge=lfs -text
|
||||
*.mlmodel filter=lfs diff=lfs merge=lfs -text
|
||||
*.model filter=lfs diff=lfs merge=lfs -text
|
||||
*.msgpack filter=lfs diff=lfs merge=lfs -text
|
||||
*.npy filter=lfs diff=lfs merge=lfs -text
|
||||
*.npz filter=lfs diff=lfs merge=lfs -text
|
||||
*.onnx filter=lfs diff=lfs merge=lfs -text
|
||||
*.ot filter=lfs diff=lfs merge=lfs -text
|
||||
*.parquet filter=lfs diff=lfs merge=lfs -text
|
||||
*.pb filter=lfs diff=lfs merge=lfs -text
|
||||
*.pickle filter=lfs diff=lfs merge=lfs -text
|
||||
*.pkl filter=lfs diff=lfs merge=lfs -text
|
||||
*.pt filter=lfs diff=lfs merge=lfs -text
|
||||
*.pth filter=lfs diff=lfs merge=lfs -text
|
||||
*.rar filter=lfs diff=lfs merge=lfs -text
|
||||
*.safetensors filter=lfs diff=lfs merge=lfs -text
|
||||
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
||||
*.tar.* filter=lfs diff=lfs merge=lfs -text
|
||||
*.tar filter=lfs diff=lfs merge=lfs -text
|
||||
*.tflite filter=lfs diff=lfs merge=lfs -text
|
||||
*.tgz filter=lfs diff=lfs merge=lfs -text
|
||||
*.wasm filter=lfs diff=lfs merge=lfs -text
|
||||
*.xz filter=lfs diff=lfs merge=lfs -text
|
||||
*.zip filter=lfs diff=lfs merge=lfs -text
|
||||
*.zst filter=lfs diff=lfs merge=lfs -text
|
||||
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
||||
227
README.md
Normal file
227
README.md
Normal file
@@ -0,0 +1,227 @@
|
||||
---
|
||||
language:
|
||||
- en
|
||||
- zh
|
||||
- id
|
||||
- th
|
||||
- vi
|
||||
- ms
|
||||
- lo
|
||||
datasets:
|
||||
- cerebras/SlimPajama-627B
|
||||
- Skywork/SkyPile-150B
|
||||
- allenai/MADLAD-400
|
||||
- cc100
|
||||
tags:
|
||||
- multilingual
|
||||
- sea
|
||||
- sailor
|
||||
license: apache-2.0
|
||||
base_model: Qwen/Qwen1.5-4B
|
||||
inference: false
|
||||
model-index:
|
||||
- name: Sailor-4B
|
||||
results:
|
||||
- task:
|
||||
type: text-generation
|
||||
dataset:
|
||||
name: XQuAD-Thai
|
||||
type: XQuAD-Thai
|
||||
metrics:
|
||||
- name: EM (3-Shot)
|
||||
type: EM (3-Shot)
|
||||
value: 46.82
|
||||
- name: F1 (3-Shot)
|
||||
type: F1 (3-Shot)
|
||||
value: 63.34
|
||||
- task:
|
||||
type: text-generation
|
||||
dataset:
|
||||
name: TyDiQA-Indonesian
|
||||
type: TyDiQA-Indonesian
|
||||
metrics:
|
||||
- name: EM (3-Shot)
|
||||
type: EM (3-Shot)
|
||||
value: 53.98
|
||||
- name: F1 (3-Shot)
|
||||
type: F1 (3-Shot)
|
||||
value: 73.48
|
||||
- task:
|
||||
type: text-generation
|
||||
dataset:
|
||||
name: XQuAD-Vietnamese
|
||||
type: XQuAD-Vietnamese
|
||||
metrics:
|
||||
- name: EM (3-Shot)
|
||||
type: EM (3-Shot)
|
||||
value: 47.65
|
||||
- name: F1 (3-Shot)
|
||||
type: F1 (3-Shot)
|
||||
value: 67.09
|
||||
- task:
|
||||
type: text-generation
|
||||
dataset:
|
||||
name: XCOPA-Thai
|
||||
type: XCOPA-Thai
|
||||
metrics:
|
||||
- name: EM (3-Shot)
|
||||
type: EM (3-Shot)
|
||||
value: 53.4
|
||||
- task:
|
||||
type: text-generation
|
||||
dataset:
|
||||
name: XCOPA-Indonesian
|
||||
type: XCOPA-Indonesian
|
||||
metrics:
|
||||
- name: EM (3-Shot)
|
||||
type: EM (3-Shot)
|
||||
value: 69.20
|
||||
- task:
|
||||
type: text-generation
|
||||
dataset:
|
||||
name: XCOPA-Vietnamese
|
||||
type: XCOPA-Vietnamese
|
||||
metrics:
|
||||
- name: EM (3-Shot)
|
||||
type: EM (3-Shot)
|
||||
value: 68.20
|
||||
- task:
|
||||
type: text-generation
|
||||
dataset:
|
||||
name: M3Exam-Thai
|
||||
type: M3Exam-Thai
|
||||
metrics:
|
||||
- name: EM (3-Shot)
|
||||
type: EM (3-Shot)
|
||||
value: 27.88
|
||||
- task:
|
||||
type: text-generation
|
||||
dataset:
|
||||
name: M3Exam-Indonesian
|
||||
type: M3Exam-Indonesian
|
||||
metrics:
|
||||
- name: EM (3-Shot)
|
||||
type: EM (3-Shot)
|
||||
value: 31.27
|
||||
- task:
|
||||
type: text-generation
|
||||
dataset:
|
||||
name: M3Exam-Vietnamese
|
||||
type: M3Exam-Vietnamese
|
||||
metrics:
|
||||
- name: EM (3-Shot)
|
||||
type: EM (3-Shot)
|
||||
value: 40.69
|
||||
- task:
|
||||
type: text-generation
|
||||
dataset:
|
||||
name: BELEBELE-Thai
|
||||
type: BELEBELE-Thai
|
||||
metrics:
|
||||
- name: EM (3-Shot)
|
||||
type: EM (3-Shot)
|
||||
value: 36.11
|
||||
- task:
|
||||
type: text-generation
|
||||
dataset:
|
||||
name: BELEBELE-Indonesian
|
||||
type: BELEBELE-Indonesian
|
||||
metrics:
|
||||
- name: EM (3-Shot)
|
||||
type: EM (3-Shot)
|
||||
value: 41.33
|
||||
- task:
|
||||
type: text-generation
|
||||
dataset:
|
||||
name: BELEBELE-Vietnamese
|
||||
type: BELEBELE-Vietnamese
|
||||
metrics:
|
||||
- name: EM (3-Shot)
|
||||
type: EM (3-Shot)
|
||||
value: 38.89
|
||||
---
|
||||
|
||||
<div align="center">
|
||||
<img src="banner_sailor.jpg" width="700"/>
|
||||
</div>
|
||||
|
||||
Sailor is a suite of Open Language Models tailored for South-East Asia (SEA), focusing on languages such as 🇮🇩Indonesian, 🇹🇭Thai, 🇻🇳Vietnamese, 🇲🇾Malay, and 🇱🇦Lao.
|
||||
Developed with careful data curation, Sailor models are designed to understand and generate text across diverse linguistic landscapes of SEA region.
|
||||
Built from [Qwen 1.5](https://huggingface.co/collections/Qwen/qwen15-65c0a2f577b1ecb76d786524) , Sailor encompasses models of varying sizes, spanning from 0.5B to 14B versions for different requirements.
|
||||
We further fine-tune the base model with open-source datasets to get instruction-tuned models, namedly Sailor-Chat.
|
||||
Benchmarking results demonstrate Sailor's proficiency in tasks such as question answering, commonsense reasoning, and other tasks in SEA languages.
|
||||
|
||||
> The logo was generated by MidJourney
|
||||
|
||||
## Model Summary
|
||||
- **Model Collections:** [Base Model & Chat Model](https://huggingface.co/collections/sail/sailor-65e19a749f978976f1959825)
|
||||
- **Project Website:** [sea-sailor.github.io/blog/sailor1/](https://sea-sailor.github.io/blog/sailor1/)
|
||||
- **Codebase:** [github.com/sail-sg/sailor-llm](https://github.com/sail-sg/sailor-llm)
|
||||
- **Technical Report:** [arxiv.org/pdf/2404.03608.pdf](https://arxiv.org/pdf/2404.03608.pdf)
|
||||
|
||||
## Training details
|
||||
Sailor is crafted by continually pre-training from language models like the remarkable Qwen 1.5 models, which already has a great performance on SEA languages.
|
||||
The pre-training corpus heavily leverages the publicly available corpus, including
|
||||
[SlimPajama](https://huggingface.co/datasets/cerebras/SlimPajama-627B),
|
||||
[SkyPile](https://huggingface.co/datasets/Skywork/SkyPile-150B),
|
||||
[CC100](https://huggingface.co/datasets/cc100) and [MADLAD-400](https://huggingface.co/datasets/allenai/MADLAD-400).
|
||||
|
||||
By employing aggressive data deduplication and careful data cleaning on the collected corpus, we have attained a high-quality dataset spanning various languages.
|
||||
Through systematic experiments to determine the weights of different languages, Sailor models undergo training from 200B to 400B tokens, tailored to different model sizes.
|
||||
The approach boosts their performance on SEA languages while maintaining proficiency in English and Chinese without significant compromise.
|
||||
Finally, we continually pre-train the Qwen1.5-0.5B model with 400 Billion tokens, and other models with 200 Billion tokens to obtain the Sailor models.
|
||||
|
||||
## Requirements
|
||||
The code of Sailor has been in the latest Hugging face transformers and we advise you to install `transformers>=4.37.0`.
|
||||
|
||||
## Quickstart
|
||||
|
||||
Here provides a code snippet to show you how to load the tokenizer and model and how to generate contents.
|
||||
|
||||
```python
|
||||
from transformers import AutoModelForCausalLM, AutoTokenizer
|
||||
device = "cuda" # the device to load the model
|
||||
|
||||
model = AutoModelForCausalLM.from_pretrained("sail/Sailor-4B", device_map="auto")
|
||||
tokenizer = AutoTokenizer.from_pretrained("sail/Sailor-4B")
|
||||
|
||||
input_message = "Model bahasa adalah model probabilistik"
|
||||
### The given Indonesian input translates to 'A language model is a probabilistic model of.'
|
||||
|
||||
model_inputs = tokenizer([input_message], return_tensors="pt").to(device)
|
||||
|
||||
generated_ids = model.generate(
|
||||
model_inputs.input_ids,
|
||||
max_new_tokens=64
|
||||
)
|
||||
|
||||
generated_ids = [
|
||||
output_ids[len(input_ids):] for input_ids, output_ids in zip(model_inputs.input_ids, generated_ids)
|
||||
]
|
||||
|
||||
response = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)[0]
|
||||
print(response)
|
||||
```
|
||||
|
||||
# License
|
||||
|
||||
Sailor is distributed under the terms of the Apache License 2.0.
|
||||
No restrict on the research and the commercial use, but should comply with the [Qwen License](https://huggingface.co/Qwen/Qwen1.5-1.8B/blob/main/LICENSE).
|
||||
|
||||
## Citation
|
||||
|
||||
If you find sailor useful, please cite our work as follows:
|
||||
|
||||
|
||||
```
|
||||
@inproceedings{dou-etal-2024-sailor,
|
||||
title = "Sailor: Open Language Models for South-{E}ast {A}sia",
|
||||
author = "Dou, Longxu and Liu, Qian and Zeng, Guangtao and Guo, Jia and Zhou, Jiahui and Mao, Xin and Jin, Ziqi and Lu, Wei and Lin, Min",
|
||||
booktitle = "Proceedings of the 2024 Conference on Empirical Methods in Natural Language Processing: System Demonstrations",
|
||||
year = "2024",
|
||||
}
|
||||
```
|
||||
|
||||
# Contact Us
|
||||
|
||||
If you have any questions, please raise an issue or contact us at [doulx@sea.com](mailto:doulx@sea.com) or [liuqian.sea@gmail.com](mailto:liuqian.sea@gmail.com).
|
||||
5
added_tokens.json
Normal file
5
added_tokens.json
Normal file
@@ -0,0 +1,5 @@
|
||||
{
|
||||
"<|endoftext|>": 151643,
|
||||
"<|im_end|>": 151645,
|
||||
"<|im_start|>": 151644
|
||||
}
|
||||
BIN
banner_sailor.jpg
Normal file
BIN
banner_sailor.jpg
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 574 KiB |
27
config.json
Normal file
27
config.json
Normal file
@@ -0,0 +1,27 @@
|
||||
{
|
||||
"architectures": [
|
||||
"Qwen2ForCausalLM"
|
||||
],
|
||||
"attention_dropout": 0.0,
|
||||
"bos_token_id": 151643,
|
||||
"eos_token_id": 151643,
|
||||
"hidden_act": "silu",
|
||||
"hidden_size": 2560,
|
||||
"initializer_range": 0.02,
|
||||
"intermediate_size": 6912,
|
||||
"max_position_embeddings": 32768,
|
||||
"max_window_layers": 21,
|
||||
"model_type": "qwen2",
|
||||
"num_attention_heads": 20,
|
||||
"num_hidden_layers": 40,
|
||||
"num_key_value_heads": 20,
|
||||
"rms_norm_eps": 1e-06,
|
||||
"rope_theta": 5000000.0,
|
||||
"sliding_window": 32768,
|
||||
"tie_word_embeddings": false,
|
||||
"torch_dtype": "bfloat16",
|
||||
"transformers_version": "4.37.2",
|
||||
"use_cache": true,
|
||||
"use_sliding_window": false,
|
||||
"vocab_size": 151936
|
||||
}
|
||||
1
configuration.json
Normal file
1
configuration.json
Normal file
@@ -0,0 +1 @@
|
||||
{"framework": "pytorch", "task": "text-generation", "allow_remote": true}
|
||||
6
generation_config.json
Normal file
6
generation_config.json
Normal file
@@ -0,0 +1,6 @@
|
||||
{
|
||||
"_from_model_config": true,
|
||||
"bos_token_id": 151643,
|
||||
"eos_token_id": 151643,
|
||||
"transformers_version": "4.37.2"
|
||||
}
|
||||
151388
merges.txt
Normal file
151388
merges.txt
Normal file
File diff suppressed because it is too large
Load Diff
3
model.safetensors
Normal file
3
model.safetensors
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:d72b1fc8e0e404dbe664428ee8a39a204c28f497a5e92ff5fc511bf6330f2d99
|
||||
size 7900794008
|
||||
20
special_tokens_map.json
Normal file
20
special_tokens_map.json
Normal file
@@ -0,0 +1,20 @@
|
||||
{
|
||||
"additional_special_tokens": [
|
||||
"<|im_start|>",
|
||||
"<|im_end|>"
|
||||
],
|
||||
"eos_token": {
|
||||
"content": "<|endoftext|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
},
|
||||
"pad_token": {
|
||||
"content": "<|endoftext|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
}
|
||||
}
|
||||
303111
tokenizer.json
Normal file
303111
tokenizer.json
Normal file
File diff suppressed because it is too large
Load Diff
43
tokenizer_config.json
Normal file
43
tokenizer_config.json
Normal file
@@ -0,0 +1,43 @@
|
||||
{
|
||||
"add_prefix_space": false,
|
||||
"added_tokens_decoder": {
|
||||
"151643": {
|
||||
"content": "<|endoftext|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151644": {
|
||||
"content": "<|im_start|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151645": {
|
||||
"content": "<|im_end|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
}
|
||||
},
|
||||
"additional_special_tokens": [
|
||||
"<|im_start|>",
|
||||
"<|im_end|>"
|
||||
],
|
||||
"bos_token": null,
|
||||
"chat_template": "{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
|
||||
"clean_up_tokenization_spaces": false,
|
||||
"eos_token": "<|endoftext|>",
|
||||
"errors": "replace",
|
||||
"model_max_length": 32768,
|
||||
"pad_token": "<|endoftext|>",
|
||||
"split_special_tokens": false,
|
||||
"tokenizer_class": "Qwen2Tokenizer",
|
||||
"unk_token": null
|
||||
}
|
||||
1
vocab.json
Normal file
1
vocab.json
Normal file
File diff suppressed because one or more lines are too long
Reference in New Issue
Block a user