初始化项目，由ModelHub XC社区提供模型
Model: mesolitica/malaysian-Qwen1.5-0.5B-16k-instructions Source: Original Platform
This commit is contained in:
ModelHub XC
52
.gitattributes
vendored
Normal file
52
.gitattributes
vendored
Normal file
@@ -0,0 +1,52 @@
|
||||
*.7z filter=lfs diff=lfs merge=lfs -text
|
||||
*.arrow filter=lfs diff=lfs merge=lfs -text
|
||||
*.bin filter=lfs diff=lfs merge=lfs -text
|
||||
*.bin.* filter=lfs diff=lfs merge=lfs -text
|
||||
*.bz2 filter=lfs diff=lfs merge=lfs -text
|
||||
*.ftz filter=lfs diff=lfs merge=lfs -text
|
||||
*.gz filter=lfs diff=lfs merge=lfs -text
|
||||
*.h5 filter=lfs diff=lfs merge=lfs -text
|
||||
*.joblib filter=lfs diff=lfs merge=lfs -text
|
||||
*.lfs.* filter=lfs diff=lfs merge=lfs -text
|
||||
*.model filter=lfs diff=lfs merge=lfs -text
|
||||
*.msgpack filter=lfs diff=lfs merge=lfs -text
|
||||
*.onnx filter=lfs diff=lfs merge=lfs -text
|
||||
*.ot filter=lfs diff=lfs merge=lfs -text
|
||||
*.parquet filter=lfs diff=lfs merge=lfs -text
|
||||
*.pb filter=lfs diff=lfs merge=lfs -text
|
||||
*.pt filter=lfs diff=lfs merge=lfs -text
|
||||
*.pth filter=lfs diff=lfs merge=lfs -text
|
||||
*.rar filter=lfs diff=lfs merge=lfs -text
|
||||
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
||||
*.tar.* filter=lfs diff=lfs merge=lfs -text
|
||||
*.tflite filter=lfs diff=lfs merge=lfs -text
|
||||
*.tgz filter=lfs diff=lfs merge=lfs -text
|
||||
*.xz filter=lfs diff=lfs merge=lfs -text
|
||||
*.zip filter=lfs diff=lfs merge=lfs -text
|
||||
*.zstandard filter=lfs diff=lfs merge=lfs -text
|
||||
*.tfevents* filter=lfs diff=lfs merge=lfs -text
|
||||
*.db* filter=lfs diff=lfs merge=lfs -text
|
||||
*.ark* filter=lfs diff=lfs merge=lfs -text
|
||||
**/*ckpt*data* filter=lfs diff=lfs merge=lfs -text
|
||||
**/*ckpt*.meta filter=lfs diff=lfs merge=lfs -text
|
||||
**/*ckpt*.index filter=lfs diff=lfs merge=lfs -text
|
||||
|
||||
*.ckpt filter=lfs diff=lfs merge=lfs -text
|
||||
*.gguf* filter=lfs diff=lfs merge=lfs -text
|
||||
*.ggml filter=lfs diff=lfs merge=lfs -text
|
||||
*.llamafile* filter=lfs diff=lfs merge=lfs -text
|
||||
*.pt2 filter=lfs diff=lfs merge=lfs -text
|
||||
*.mlmodel filter=lfs diff=lfs merge=lfs -text
|
||||
*.npy filter=lfs diff=lfs merge=lfs -text
|
||||
*.npz filter=lfs diff=lfs merge=lfs -text
|
||||
*.pickle filter=lfs diff=lfs merge=lfs -text
|
||||
*.pkl filter=lfs diff=lfs merge=lfs -text
|
||||
*.tar filter=lfs diff=lfs merge=lfs -text
|
||||
*.wasm filter=lfs diff=lfs merge=lfs -text
|
||||
*.zst filter=lfs diff=lfs merge=lfs -text
|
||||
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
||||
|
||||
model.safetensors filter=lfs diff=lfs merge=lfs -text
|
||||
merges.txt filter=lfs diff=lfs merge=lfs -text
|
||||
vocab.json filter=lfs diff=lfs merge=lfs -text
|
||||
tokenizer.json filter=lfs diff=lfs merge=lfs -text
|
||||
192
README.md
Normal file
192
README.md
Normal file
@@ -0,0 +1,192 @@
|
||||
---
|
||||
language:
|
||||
- ms
|
||||
---
|
||||
|
||||
# Full Parameter Finetuning Qwen1.5-0.5B 16384 context length on Malaysian instructions dataset
|
||||
|
||||
README at https://github.com/mesolitica/malaya/tree/5.1/session/qwen2
|
||||
|
||||
We use exact Qwen1.5 Instruct chat template.
|
||||
|
||||
WandB, https://wandb.ai/huseinzol05/Qwen1.5-0.5B-4096-fpf-instructions-16k?workspace=user-huseinzol05
|
||||
|
||||
## how-to
|
||||
|
||||
```python
|
||||
from transformers import AutoTokenizer, AutoModelForCausalLM, BitsAndBytesConfig
|
||||
import torch
|
||||
|
||||
TORCH_DTYPE = 'bfloat16'
|
||||
nf4_config = BitsAndBytesConfig(
|
||||
load_in_4bit=True,
|
||||
bnb_4bit_quant_type='nf4',
|
||||
bnb_4bit_use_double_quant=True,
|
||||
bnb_4bit_compute_dtype=getattr(torch, TORCH_DTYPE)
|
||||
)
|
||||
|
||||
tokenizer = AutoTokenizer.from_pretrained('mesolitica/malaysian-Qwen1.5-0.5B-16k-instructions')
|
||||
model = AutoModelForCausalLM.from_pretrained(
|
||||
'mesolitica/malaysian-Qwen1.5-0.5B-16k-instructions',
|
||||
use_flash_attention_2 = True,
|
||||
quantization_config = nf4_config
|
||||
)
|
||||
messages = [
|
||||
{'role': 'user', 'content': 'KWSP tu apa'}
|
||||
]
|
||||
prompt = tokenizer.apply_chat_template(messages, tokenize = False)
|
||||
inputs = tokenizer([prompt], return_tensors='pt', add_special_tokens=False).to('cuda')
|
||||
generate_kwargs = dict(
|
||||
inputs,
|
||||
max_new_tokens=1024,
|
||||
top_p=0.95,
|
||||
top_k=50,
|
||||
temperature=0.9,
|
||||
do_sample=True,
|
||||
num_beams=1,
|
||||
)
|
||||
r = model.generate(**generate_kwargs)
|
||||
tokenizer.decode(r[0])
|
||||
```
|
||||
|
||||
```text
|
||||
<|im_start|>user
|
||||
KWSP tu apa<|im_end|>
|
||||
<|im_start|>assistant
|
||||
KWSP merujuk kepada Skim Simpanan Wang Persaraan (KWSP), iaitu skim simpanan untuk ahli kumpulan berumur 20 tahun ke atas. KWSP menawarkan beberapa faedah, termasuk:
|
||||
|
||||
1. Akaun Simpanan Wajib - Ahli boleh menyumbang kepada KWSP melalui akaun simpanan wajib.
|
||||
2. Akaun Simpanan Amanah - Ahli boleh menyumbang kepada KWSP melalui akaun amanah.
|
||||
3. Akaun Simpanan Pelaburan - Ahli boleh menyumbang kepada KWSP melalui pelaburan dalam skim pelaburan KWSP atau pelaburan bersama.
|
||||
4. Perolehan - Ahli boleh mendapatkan pelbagai perkhidmatan dan ganjaran daripada KWSP.
|
||||
5. Akaun Simpanan untuk Penyelamat - Ahli boleh menyumbang kepada KWSP melalui akaun simpanan untuk penyelamat, yang boleh digunakan untuk bantuan kewangan atau pemberhentian pasaran kewangan.
|
||||
|
||||
KWSP menawarkan faedah dan perlindungan yang komprehensif untuk ahli, termasuk:
|
||||
|
||||
1. Akaun Simpanan Amanah - Akaun ini menyediakan faedah dan ganjaran sebelum dan selepas persaraan.
|
||||
2. Akaun Simpanan - Akaun simpanan menawarkan kadar faedah yang lebih rendah daripada akaun simpanan biasa.
|
||||
3. Akaun Simpanan Pelaburan - Akaun ini menawarkan kadar faedah yang lebih rendah dan pelbagai ganjaran.
|
||||
4. Akaun Simpanan untuk Penyelamat - Akaun ini menawarkan kadar faedah yang lebih rendah daripada akaun simpanan biasa dan pelbagai ganjaran.
|
||||
5. Perolehan - Akaun ini menyediakan faedah dan ganjaran sebelum dan selepas persaraan.
|
||||
|
||||
KWSP juga menawarkan pelbagai perkhidmatan dan ganjaran lain, termasuk:
|
||||
|
||||
1. Program Pekerjaan - KWSP menyediakan pelbagai skim pekerjaan, termasuk Skim Pencen, Skim Pekerjaan, dan Skim Penginapan.
|
||||
2. Program Perubatan - KWSP menyediakan pelbagai skim perubatan, termasuk Skim Cukai Perubatan.
|
||||
3. Program Keselamatan Sosial - KWSP menyediakan pelbagai program keselamatan sosial, termasuk Skim Simpanan 401 (KWSP), Skim Simpanan Amanah (SW401) dan Skim Simpanan Perumahan (RSP) untuk pemastautin berdaftar.
|
||||
4. Program Penjagaan Kesihatan - KWSP menyediakan pelbagai skim penjagaan kesihatan, termasuk Skim Insurans Kesihatan Swasta (SIC), Skim Insurans Kesihatan Kumpulan Swasta (KSMC), dan Skim Insurans Kesihatan Kumpulan Swasta 1Malaysia (SIC 1M).
|
||||
5. Program Pemindahan - KWSP menyediakan pelbagai skim pemindahan, termasuk Skim Pemindahan Amanah, Skim Pemindahan KWSP, dan Skim Pemindahan Persekutuan.
|
||||
|
||||
KWSP menawarkan pelbagai pilihan dan perlindungan untuk ahli yang ingin menyumbang kepada KWSP untuk persaraan atau bantuan kewangan atau pemberhentian pasaran kewangan.<|im_end|>
|
||||
```
|
||||
|
||||
```python
|
||||
messages = [
|
||||
{'role': 'user', 'content': 'macam mana nak kod websocket guna react, contoh kod sekali'}
|
||||
]
|
||||
prompt = tokenizer.apply_chat_template(messages, tokenize = False)
|
||||
inputs = tokenizer([prompt], return_tensors='pt', add_special_tokens=False).to('cuda')
|
||||
generate_kwargs = dict(
|
||||
inputs,
|
||||
max_new_tokens=1024,
|
||||
top_p=0.95,
|
||||
top_k=50,
|
||||
temperature=0.9,
|
||||
do_sample=True,
|
||||
num_beams=1,
|
||||
)
|
||||
r = model.generate(**generate_kwargs)
|
||||
tokenizer.decode(r[0])
|
||||
```
|
||||
|
||||
```text
|
||||
<|im_start|>user
|
||||
macam mana nak kod websocket guna react, contoh kod sekali<|im_end|>
|
||||
<|im_start|>assistant
|
||||
Untuk menggunakan websocket pada React untuk mencipta pelanggan untuk menghantar dan menerima pesanan, anda boleh mengikuti langkah-langkah berikut:
|
||||
|
||||
1. Mulakan dengan membuat fail baru, contohnya `WebsocketClient.js`, dan perpustakaan yang diperlukan seperti `ws` untuk Node.js, dan `axios` untuk JavaScript untuk membuat permintaan HTTP ke websocket. Pastikan anda telah mengimport perpustakaan dengan betul.
|
||||
|
||||
2. Dalam fail `WebsocketClient.js`, cipta kelas baru dengan nama `WebsocketClient`. Anda boleh menggunakan konstruktor dengan parameter seperti nama aplikasi dan senarai objek pesanan. Contoh:
|
||||
```
|
||||
|
||||
```javascript
|
||||
import { WebSocketClient, Client } from 'ws';
|
||||
import axios from 'axios';
|
||||
|
||||
class WebsocketClient {
|
||||
constructor(appName, socket) {
|
||||
this.appName = appName;
|
||||
this.socket = socket;
|
||||
}
|
||||
|
||||
connect() {
|
||||
this.socket.onopen = () => {
|
||||
console.log('Connection successful');
|
||||
};
|
||||
this.socket.onclose = () => {
|
||||
console.log('Connection lost');
|
||||
};
|
||||
this.socket.onmessage = (message) => {
|
||||
// Code to handle incoming message
|
||||
};
|
||||
}
|
||||
|
||||
emit(data) {
|
||||
this.socket.onmessage(data);
|
||||
}
|
||||
|
||||
disconnect() {
|
||||
this.socket.offopen();
|
||||
}
|
||||
}
|
||||
|
||||
// Contoh penggunaan
|
||||
const client = new WebsocketClient('api', axios({ port: 123 }));
|
||||
client.connect();
|
||||
client.emit('message', { type: 'info', content: 'Hello, server! });
|
||||
client.emit('message', { type: 'warning', content: 'Goodbye!'});
|
||||
client.disconnect();
|
||||
```
|
||||
|
||||
```
|
||||
3. Dalam fail `WebSocketClient.js`, cipta kelas dengan nama `Client` dengan parameter seperti aplikasi dan `websocket` dari perpustakaan axios. Contoh:
|
||||
```
|
||||
|
||||
```javascript
|
||||
class Client {
|
||||
constructor(appName, websocket) {
|
||||
this.appName = appName;
|
||||
this.websocket = websocket;
|
||||
}
|
||||
|
||||
connect() {
|
||||
this.websocket.onopen = () => {
|
||||
console.log('Connection successful');
|
||||
};
|
||||
this.websocket.onclose = () => {
|
||||
console.log('Connection lost');
|
||||
};
|
||||
this.websocket.onmessage = (message) => {
|
||||
// Code to handle incoming message
|
||||
};
|
||||
}
|
||||
|
||||
disconnect() {
|
||||
this.websocket.offopen();
|
||||
}
|
||||
}
|
||||
|
||||
// Contoh penggunaan
|
||||
const client = new Client('api', axios({ port: 123 }));
|
||||
client.connect();
|
||||
client.emit('message', { type: 'info', content: 'Hello, server! });
|
||||
client.emit('message', { type: 'warning', content: 'Goodbye!'});
|
||||
client.disconnect();
|
||||
```
|
||||
|
||||
```
|
||||
Dalam contoh di atas, kami menggunakan konstruktor yang mengambil nama aplikasi dan websocket sebagai parameter untuk memulakan dan menggunakan fungsi `connect` untuk menghubungi websocket dengan ID sesi yang diinginkan. Kemudian, kami menggunakan fungsi `onmessage` untuk mengendalikan respons websocket dan fungsi `onclose` untuk menutup websocket. Apabila kami menghantar permintaan ke websocket, kami menggunakan kaedah `onmessage` untuk memproses respons tersebut. Apabila kami membuka websocket, kami menggunakan kaedah `onclose` untuk menutup websocket dan menggunakan kaedah `disconnect` untuk membunuhnya.
|
||||
|
||||
Dengan menggunakan kod di atas dan menggunakannya pada projek React anda, anda boleh mencipta websocket yang boleh menghantar dan menerima pesanan untuk aplikasi anda.<|im_end|>
|
||||
```
|
||||
5
added_tokens.json
Normal file
5
added_tokens.json
Normal file
@@ -0,0 +1,5 @@
|
||||
{
|
||||
"<|endoftext|>": 151643,
|
||||
"<|im_end|>": 151645,
|
||||
"<|im_start|>": 151644
|
||||
}
|
||||
28
config.json
Normal file
28
config.json
Normal file
@@ -0,0 +1,28 @@
|
||||
{
|
||||
"_name_or_path": "fpf-0.5-instructions-16k/checkpoint-11900",
|
||||
"architectures": [
|
||||
"Qwen2ForCausalLM"
|
||||
],
|
||||
"attention_dropout": 0.0,
|
||||
"bos_token_id": 151643,
|
||||
"eos_token_id": 151643,
|
||||
"hidden_act": "silu",
|
||||
"hidden_size": 1024,
|
||||
"initializer_range": 0.02,
|
||||
"intermediate_size": 2816,
|
||||
"max_position_embeddings": 32768,
|
||||
"max_window_layers": 21,
|
||||
"model_type": "qwen2",
|
||||
"num_attention_heads": 16,
|
||||
"num_hidden_layers": 24,
|
||||
"num_key_value_heads": 16,
|
||||
"rms_norm_eps": 1e-06,
|
||||
"rope_theta": 1000000.0,
|
||||
"sliding_window": 32768,
|
||||
"tie_word_embeddings": true,
|
||||
"torch_dtype": "bfloat16",
|
||||
"transformers_version": "4.37.2",
|
||||
"use_cache": true,
|
||||
"use_sliding_window": false,
|
||||
"vocab_size": 151646
|
||||
}
|
||||
1
configuration.json
Normal file
1
configuration.json
Normal file
@@ -0,0 +1 @@
|
||||
{"framework": "pytorch", "task": "text-generation", "allow_remote": true}
|
||||
6
generation_config.json
Normal file
6
generation_config.json
Normal file
@@ -0,0 +1,6 @@
|
||||
{
|
||||
"bos_token_id": 151643,
|
||||
"eos_token_id": 151645,
|
||||
"max_new_tokens": 2048,
|
||||
"transformers_version": "4.37.1"
|
||||
}
|
||||
3
merges.txt
Normal file
3
merges.txt
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:8831e4f1a044471340f7c0a83d7bd71306a5b867e95fd870f74d0c5308a904d5
|
||||
size 1671853
|
||||
3
model.safetensors
Normal file
3
model.safetensors
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:d398bcc7c06f14df6ff55ac32d0e6869dd50c2f8b02e67eba4ef6bb2016f08f0
|
||||
size 927414184
|
||||
14
special_tokens_map.json
Normal file
14
special_tokens_map.json
Normal file
@@ -0,0 +1,14 @@
|
||||
{
|
||||
"additional_special_tokens": [
|
||||
"<|im_start|>",
|
||||
"<|im_end|>"
|
||||
],
|
||||
"eos_token": "<|im_end|>",
|
||||
"pad_token": {
|
||||
"content": "<|endoftext|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
}
|
||||
}
|
||||
3
tokenizer.json
Normal file
3
tokenizer.json
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:f7c9b2dba4a296b1aa76c16a34b8225c0c118978400d4bb66bff0902d702f5b8
|
||||
size 7028015
|
||||
43
tokenizer_config.json
Normal file
43
tokenizer_config.json
Normal file
@@ -0,0 +1,43 @@
|
||||
{
|
||||
"add_prefix_space": false,
|
||||
"added_tokens_decoder": {
|
||||
"151643": {
|
||||
"content": "<|endoftext|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151644": {
|
||||
"content": "<|im_start|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151645": {
|
||||
"content": "<|im_end|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
}
|
||||
},
|
||||
"additional_special_tokens": [
|
||||
"<|im_start|>",
|
||||
"<|im_end|>"
|
||||
],
|
||||
"bos_token": null,
|
||||
"chat_template": "{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
|
||||
"clean_up_tokenization_spaces": false,
|
||||
"eos_token": "<|im_end|>",
|
||||
"errors": "replace",
|
||||
"model_max_length": 32768,
|
||||
"pad_token": "<|endoftext|>",
|
||||
"split_special_tokens": false,
|
||||
"tokenizer_class": "Qwen2Tokenizer",
|
||||
"unk_token": null
|
||||
}
|
||||
BIN
vocab.json
(Stored with Git LFS)
Normal file
BIN
vocab.json
(Stored with Git LFS)
Normal file
Binary file not shown.
Reference in New Issue
Block a user