初始化项目，由ModelHub XC社区提供模型
Model: mesolitica/malaysian-tinyllama-1.1b-16k-instructions Source: Original Platform
This commit is contained in:
ModelHub XC
51
.gitattributes
vendored
Normal file
51
.gitattributes
vendored
Normal file
@@ -0,0 +1,51 @@
|
||||
*.7z filter=lfs diff=lfs merge=lfs -text
|
||||
*.arrow filter=lfs diff=lfs merge=lfs -text
|
||||
*.bin filter=lfs diff=lfs merge=lfs -text
|
||||
*.bin.* filter=lfs diff=lfs merge=lfs -text
|
||||
*.bz2 filter=lfs diff=lfs merge=lfs -text
|
||||
*.ftz filter=lfs diff=lfs merge=lfs -text
|
||||
*.gz filter=lfs diff=lfs merge=lfs -text
|
||||
*.h5 filter=lfs diff=lfs merge=lfs -text
|
||||
*.joblib filter=lfs diff=lfs merge=lfs -text
|
||||
*.lfs.* filter=lfs diff=lfs merge=lfs -text
|
||||
|
||||
*.msgpack filter=lfs diff=lfs merge=lfs -text
|
||||
*.onnx filter=lfs diff=lfs merge=lfs -text
|
||||
*.ot filter=lfs diff=lfs merge=lfs -text
|
||||
*.parquet filter=lfs diff=lfs merge=lfs -text
|
||||
*.pb filter=lfs diff=lfs merge=lfs -text
|
||||
*.pt filter=lfs diff=lfs merge=lfs -text
|
||||
*.pth filter=lfs diff=lfs merge=lfs -text
|
||||
*.rar filter=lfs diff=lfs merge=lfs -text
|
||||
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
||||
*.tar.* filter=lfs diff=lfs merge=lfs -text
|
||||
*.tflite filter=lfs diff=lfs merge=lfs -text
|
||||
*.tgz filter=lfs diff=lfs merge=lfs -text
|
||||
*.xz filter=lfs diff=lfs merge=lfs -text
|
||||
*.zip filter=lfs diff=lfs merge=lfs -text
|
||||
*.zstandard filter=lfs diff=lfs merge=lfs -text
|
||||
*.tfevents* filter=lfs diff=lfs merge=lfs -text
|
||||
*.db* filter=lfs diff=lfs merge=lfs -text
|
||||
*.ark* filter=lfs diff=lfs merge=lfs -text
|
||||
**/*ckpt*data* filter=lfs diff=lfs merge=lfs -text
|
||||
**/*ckpt*.meta filter=lfs diff=lfs merge=lfs -text
|
||||
**/*ckpt*.index filter=lfs diff=lfs merge=lfs -text
|
||||
|
||||
*.ckpt filter=lfs diff=lfs merge=lfs -text
|
||||
*.gguf* filter=lfs diff=lfs merge=lfs -text
|
||||
*.ggml filter=lfs diff=lfs merge=lfs -text
|
||||
*.llamafile* filter=lfs diff=lfs merge=lfs -text
|
||||
*.pt2 filter=lfs diff=lfs merge=lfs -text
|
||||
*.mlmodel filter=lfs diff=lfs merge=lfs -text
|
||||
*.npy filter=lfs diff=lfs merge=lfs -text
|
||||
*.npz filter=lfs diff=lfs merge=lfs -text
|
||||
*.pickle filter=lfs diff=lfs merge=lfs -text
|
||||
*.pkl filter=lfs diff=lfs merge=lfs -text
|
||||
*.tar filter=lfs diff=lfs merge=lfs -text
|
||||
*.wasm filter=lfs diff=lfs merge=lfs -text
|
||||
*.zst filter=lfs diff=lfs merge=lfs -text
|
||||
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
||||
|
||||
tokenizer.model filter=lfs diff=lfs merge=lfs -text
|
||||
tokenizer.json filter=lfs diff=lfs merge=lfs -text
|
||||
model.safetensors filter=lfs diff=lfs merge=lfs -text
|
||||
109
README.md
Normal file
109
README.md
Normal file
@@ -0,0 +1,109 @@
|
||||
---
|
||||
language:
|
||||
- ms
|
||||
---
|
||||
|
||||
# Full Parameter Finetuning TinyLlama 16384 context length on Malaysian instructions dataset
|
||||
|
||||
README at https://github.com/mesolitica/malaya/tree/5.1/session/tiny-llama#instructions-7b-16384-context-length
|
||||
|
||||
We use exact Llama2 Instruct chat template.
|
||||
|
||||
WandB, https://wandb.ai/mesolitica/fpf-tinyllama-1.1b-hf-instructions-16k-function-call?workspace=user-husein-mesolitica
|
||||
|
||||
WandB report, https://wandb.ai/mesolitica/fpf-mallam-5b-instructions-16k/reports/Instruction-finetuning--Vmlldzo2MjE5Njg2
|
||||
|
||||
## Dataset
|
||||
|
||||
Dataset gathered at https://huggingface.co/collections/mesolitica/malaysian-synthetic-dataset-656c2673fe7fe0b1e9e25fe2
|
||||
|
||||
Notebook to prepare dataset at https://github.com/mesolitica/malaysian-dataset/blob/master/llm-instruction/combine-malay-no-alignment-multitasks-partial-ultrachat-v2.ipynb
|
||||
|
||||
## Limitations
|
||||
|
||||
This model is a quick demonstration that the base model can be easily fine-tuned to achieve some performance.
|
||||
It does have minimal moderation mechanisms.
|
||||
|
||||
## how-to
|
||||
|
||||
```python
|
||||
from transformers import AutoTokenizer, AutoModelForCausalLM, BitsAndBytesConfig
|
||||
import torch
|
||||
|
||||
def parse_llama_chat(
|
||||
messages,
|
||||
function_call = None,
|
||||
default_system = 'Anda adalah pembantu AI yang berguna dan mampu jawab segala soalan yang diberikan.'
|
||||
):
|
||||
|
||||
if messages[0]['role'] != 'system':
|
||||
system = default_system
|
||||
start_index = 0
|
||||
else:
|
||||
system = messages[0]['content']
|
||||
start_index = 1
|
||||
|
||||
user_query = messages[-1]['content']
|
||||
|
||||
users, assistants = [], []
|
||||
for q in messages[start_index:-1]:
|
||||
if q['role'] == 'user':
|
||||
users.append(q['content'])
|
||||
elif q['role'] == 'assistant':
|
||||
assistants.append(q['content'])
|
||||
|
||||
texts = [f'<s>[INST] <<SYS>>\n{system}\n<</SYS>>\n\n']
|
||||
if function_call:
|
||||
fs = []
|
||||
for f in function_call:
|
||||
f = json.dumps(f, indent=4)
|
||||
fs.append(f)
|
||||
fs = '\n\n'.join(fs)
|
||||
texts.append(f'\n[FUNCTIONCALL]\n{fs}\n')
|
||||
for u, a in zip(users, assistants):
|
||||
texts.append(f'{u.strip()} [/INST] {a.strip()} </s><s>[INST] ')
|
||||
texts.append(f'{user_query.strip()} [/INST]')
|
||||
prompt = ''.join(texts).strip()
|
||||
return prompt
|
||||
|
||||
TORCH_DTYPE = 'bfloat16'
|
||||
nf4_config = BitsAndBytesConfig(
|
||||
load_in_4bit=True,
|
||||
bnb_4bit_quant_type='nf4',
|
||||
bnb_4bit_use_double_quant=True,
|
||||
bnb_4bit_compute_dtype=getattr(torch, TORCH_DTYPE)
|
||||
)
|
||||
|
||||
tokenizer = AutoTokenizer.from_pretrained('mesolitica/malaysian-tinyllama-1.1b-16k-instructions')
|
||||
model = AutoModelForCausalLM.from_pretrained(
|
||||
'mesolitica/malaysian-tinyllama-1.1b-16k-instructions',
|
||||
use_flash_attention_2 = True,
|
||||
quantization_config = nf4_config
|
||||
)
|
||||
|
||||
messages = [
|
||||
{'role': 'system', 'content': 'awak adalah AI yang mampu jawab segala soalan'},
|
||||
{'role': 'user', 'content': 'kwsp tu apa'}
|
||||
]
|
||||
prompt = parse_llama_chat(messages)
|
||||
inputs = tokenizer([prompt], return_tensors='pt', add_special_tokens=False).to('cuda')
|
||||
generate_kwargs = dict(
|
||||
inputs,
|
||||
max_new_tokens=1024,
|
||||
top_p=0.95,
|
||||
top_k=50,
|
||||
temperature=0.9,
|
||||
do_sample=True,
|
||||
num_beams=1,
|
||||
)
|
||||
r = model.generate(**generate_kwargs)
|
||||
print(tokenizer.decode(r[0]))
|
||||
```
|
||||
|
||||
```
|
||||
<s> [INST] <<SYS>>
|
||||
awak adalah AI yang mampu jawab segala soalan
|
||||
<</SYS>>
|
||||
|
||||
kwsp tu apa [/INST] KWSP (Kumpulan Wang Simpanan Pekerja) merupakan sistem persaraan yang disediakan oleh kerajaan Malaysia untuk memberikan simpanan dan kebajikan kepada pekerja dan pekerja yang berumur 55 tahun ke atas. KWSP adalah singkatan bagi "Kumpulan Wang Simpanan Pekerja" dan ia merupakan salah satu dana persaraan yang popular di Malaysia. </s>
|
||||
```
|
||||
27
config.json
Normal file
27
config.json
Normal file
@@ -0,0 +1,27 @@
|
||||
{
|
||||
"_name_or_path": "fpf-1.1b-instructions-16k-call/checkpoint-16100",
|
||||
"architectures": [
|
||||
"LlamaForCausalLM"
|
||||
],
|
||||
"attention_bias": false,
|
||||
"bos_token_id": 1,
|
||||
"eos_token_id": 2,
|
||||
"hidden_act": "silu",
|
||||
"hidden_size": 2048,
|
||||
"initializer_range": 0.02,
|
||||
"intermediate_size": 5632,
|
||||
"max_position_embeddings": 32768,
|
||||
"model_type": "llama",
|
||||
"num_attention_heads": 32,
|
||||
"num_hidden_layers": 22,
|
||||
"num_key_value_heads": 4,
|
||||
"pretraining_tp": 1,
|
||||
"rms_norm_eps": 1e-05,
|
||||
"rope_scaling": null,
|
||||
"rope_theta": 10000.0,
|
||||
"tie_word_embeddings": false,
|
||||
"torch_dtype": "bfloat16",
|
||||
"transformers_version": "4.35.2",
|
||||
"use_cache": true,
|
||||
"vocab_size": 32000
|
||||
}
|
||||
1
configuration.json
Normal file
1
configuration.json
Normal file
@@ -0,0 +1 @@
|
||||
{"framework": "pytorch", "task": "text-generation", "allow_remote": true}
|
||||
7
generation_config.json
Normal file
7
generation_config.json
Normal file
@@ -0,0 +1,7 @@
|
||||
{
|
||||
"bos_token_id": 1,
|
||||
"eos_token_id": 2,
|
||||
"max_length": 2048,
|
||||
"pad_token_id": 0,
|
||||
"transformers_version": "4.35.2"
|
||||
}
|
||||
3
model.safetensors
Normal file
3
model.safetensors
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:2207018e7b9405a6fe2d69d22a4b325948adb629c3ab22f27a7472385d1bdf91
|
||||
size 2200119864
|
||||
30
special_tokens_map.json
Normal file
30
special_tokens_map.json
Normal file
@@ -0,0 +1,30 @@
|
||||
{
|
||||
"bos_token": {
|
||||
"content": "<s>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
},
|
||||
"eos_token": {
|
||||
"content": "</s>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
},
|
||||
"pad_token": {
|
||||
"content": "<unk>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
},
|
||||
"unk_token": {
|
||||
"content": "<unk>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
}
|
||||
}
|
||||
3
tokenizer.json
Normal file
3
tokenizer.json
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:bcd04f0eadf90287bd26e1a183ac487d8a141b09b06aecb7725bbdd343640f2e
|
||||
size 1842767
|
||||
BIN
tokenizer.model
(Stored with Git LFS)
Normal file
BIN
tokenizer.model
(Stored with Git LFS)
Normal file
Binary file not shown.
39
tokenizer_config.json
Normal file
39
tokenizer_config.json
Normal file
@@ -0,0 +1,39 @@
|
||||
{
|
||||
"added_tokens_decoder": {
|
||||
"0": {
|
||||
"content": "<unk>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"1": {
|
||||
"content": "<s>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"2": {
|
||||
"content": "</s>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
}
|
||||
},
|
||||
"bos_token": "<s>",
|
||||
"clean_up_tokenization_spaces": false,
|
||||
"eos_token": "</s>",
|
||||
"legacy": false,
|
||||
"model_max_length": 1000000000000000019884624838656,
|
||||
"pad_token": "<unk>",
|
||||
"padding_side": "right",
|
||||
"sp_model_kwargs": {},
|
||||
"tokenizer_class": "LlamaTokenizer",
|
||||
"unk_token": "<unk>",
|
||||
"use_default_system_prompt": true
|
||||
}
|
||||
Reference in New Issue
Block a user