初始化项目，由ModelHub XC社区提供模型
Model: Qwen/Qwen1.5-MoE-A2.7B-Chat-GPTQ-Int4 Source: Original Platform
This commit is contained in:
ModelHub XC
35
.gitattributes
vendored
Normal file
35
.gitattributes
vendored
Normal file
@@ -0,0 +1,35 @@
|
||||
*.7z filter=lfs diff=lfs merge=lfs -text
|
||||
*.arrow filter=lfs diff=lfs merge=lfs -text
|
||||
*.bin filter=lfs diff=lfs merge=lfs -text
|
||||
*.bz2 filter=lfs diff=lfs merge=lfs -text
|
||||
*.ckpt filter=lfs diff=lfs merge=lfs -text
|
||||
*.ftz filter=lfs diff=lfs merge=lfs -text
|
||||
*.gz filter=lfs diff=lfs merge=lfs -text
|
||||
*.h5 filter=lfs diff=lfs merge=lfs -text
|
||||
*.joblib filter=lfs diff=lfs merge=lfs -text
|
||||
*.lfs.* filter=lfs diff=lfs merge=lfs -text
|
||||
*.mlmodel filter=lfs diff=lfs merge=lfs -text
|
||||
*.model filter=lfs diff=lfs merge=lfs -text
|
||||
*.msgpack filter=lfs diff=lfs merge=lfs -text
|
||||
*.npy filter=lfs diff=lfs merge=lfs -text
|
||||
*.npz filter=lfs diff=lfs merge=lfs -text
|
||||
*.onnx filter=lfs diff=lfs merge=lfs -text
|
||||
*.ot filter=lfs diff=lfs merge=lfs -text
|
||||
*.parquet filter=lfs diff=lfs merge=lfs -text
|
||||
*.pb filter=lfs diff=lfs merge=lfs -text
|
||||
*.pickle filter=lfs diff=lfs merge=lfs -text
|
||||
*.pkl filter=lfs diff=lfs merge=lfs -text
|
||||
*.pt filter=lfs diff=lfs merge=lfs -text
|
||||
*.pth filter=lfs diff=lfs merge=lfs -text
|
||||
*.rar filter=lfs diff=lfs merge=lfs -text
|
||||
*.safetensors filter=lfs diff=lfs merge=lfs -text
|
||||
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
||||
*.tar.* filter=lfs diff=lfs merge=lfs -text
|
||||
*.tar filter=lfs diff=lfs merge=lfs -text
|
||||
*.tflite filter=lfs diff=lfs merge=lfs -text
|
||||
*.tgz filter=lfs diff=lfs merge=lfs -text
|
||||
*.wasm filter=lfs diff=lfs merge=lfs -text
|
||||
*.xz filter=lfs diff=lfs merge=lfs -text
|
||||
*.zip filter=lfs diff=lfs merge=lfs -text
|
||||
*.zst filter=lfs diff=lfs merge=lfs -text
|
||||
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
||||
76
README.md
Normal file
76
README.md
Normal file
@@ -0,0 +1,76 @@
|
||||
---
|
||||
license: other
|
||||
license_name: tongyi-qianwen
|
||||
license_link: >-
|
||||
https://huggingface.co/Qwen/Qwen1.5-7B-Chat-GPTQ-Int4/blob/main/LICENSE
|
||||
language:
|
||||
- en
|
||||
pipeline_tag: text-generation
|
||||
tags:
|
||||
- chat
|
||||
---
|
||||
|
||||
# Qwen1.5-MoE-A2.7B-Chat-GPTQ-Int4
|
||||
|
||||
|
||||
## Introduction
|
||||
|
||||
Qwen1.5-MoE is a transformer-based MoE decoder-only language model pretrained on a large amount of data.
|
||||
|
||||
For more details, please refer to our [blog post](https://qwenlm.github.io/blog/qwen-moe/) and [GitHub repo](https://github.com/QwenLM/Qwen1.5).
|
||||
|
||||
|
||||
## Model Details
|
||||
Qwen1.5-MoE employs Mixture of Experts (MoE) architecture, where the models are upcycled from dense language models. For instance, `Qwen1.5-MoE-A2.7B` is upcycled from `Qwen-1.8B`. It has 14.3B parameters in total and 2.7B activated parameters during runtime, while achieching comparable performance to `Qwen1.5-7B`, it only requires 25% of the training resources. We also observed that the inference speed is 1.74 times that of `Qwen1.5-7B`.
|
||||
|
||||
## Training details
|
||||
We pretrained the models with a large amount of data, and we post-trained the models with both supervised finetuning and direct preference optimization.
|
||||
|
||||
## Requirements
|
||||
The code of Qwen1.5-MoE has been in the latest Hugging face transformers and we advise you to build from source with command `pip install git+https://github.com/huggingface/transformers`, or you might encounter the following error:
|
||||
```
|
||||
KeyError: 'qwen2_moe'.
|
||||
```
|
||||
|
||||
## Quickstart
|
||||
|
||||
Here provides a code snippet with `apply_chat_template` to show you how to load the tokenizer and model and how to generate contents.
|
||||
|
||||
```python
|
||||
from transformers import AutoModelForCausalLM, AutoTokenizer
|
||||
device = "cuda" # the device to load the model onto
|
||||
|
||||
model = AutoModelForCausalLM.from_pretrained(
|
||||
"Qwen/Qwen1.5-MoE-A2.7B-Chat-GPTQ-Int4",
|
||||
torch_dtype="auto",
|
||||
device_map="auto"
|
||||
)
|
||||
tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen1.5-MoE-A2.7B-Chat-GPTQ-Int4")
|
||||
|
||||
prompt = "Give me a short introduction to large language model."
|
||||
messages = [
|
||||
{"role": "system", "content": "You are a helpful assistant."},
|
||||
{"role": "user", "content": prompt}
|
||||
]
|
||||
text = tokenizer.apply_chat_template(
|
||||
messages,
|
||||
tokenize=False,
|
||||
add_generation_prompt=True
|
||||
)
|
||||
model_inputs = tokenizer([text], return_tensors="pt").to(device)
|
||||
|
||||
generated_ids = model.generate(
|
||||
model_inputs.input_ids,
|
||||
max_new_tokens=512
|
||||
)
|
||||
generated_ids = [
|
||||
output_ids[len(input_ids):] for input_ids, output_ids in zip(model_inputs.input_ids, generated_ids)
|
||||
]
|
||||
|
||||
response = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)[0]
|
||||
```
|
||||
|
||||
|
||||
## Tips
|
||||
|
||||
* If you encounter code switching or other bad cases, we advise you to use our provided hyper-parameters in `generation_config.json`.
|
||||
259
config.json
Normal file
259
config.json
Normal file
@@ -0,0 +1,259 @@
|
||||
{
|
||||
"architectures": [
|
||||
"Qwen2MoeForCausalLM"
|
||||
],
|
||||
"attention_dropout": 0.0,
|
||||
"bos_token_id": 151643,
|
||||
"eos_token_id": 151645,
|
||||
"decoder_sparse_step": 1,
|
||||
"hidden_act": "silu",
|
||||
"hidden_size": 2048,
|
||||
"initializer_range": 0.02,
|
||||
"intermediate_size": 5632,
|
||||
"max_position_embeddings": 32768,
|
||||
"max_window_layers": 21,
|
||||
"model_type": "qwen2_moe",
|
||||
"moe_intermediate_size": 1408,
|
||||
"norm_topk_prob": false,
|
||||
"num_attention_heads": 16,
|
||||
"num_experts": 60,
|
||||
"num_experts_per_tok": 4,
|
||||
"num_hidden_layers": 24,
|
||||
"num_key_value_heads": 16,
|
||||
"output_router_logits": false,
|
||||
"quantization_config": {
|
||||
"batch_size": 1,
|
||||
"bits": 4,
|
||||
"block_name_to_quantize": null,
|
||||
"cache_block_outputs": true,
|
||||
"damp_percent": 0.01,
|
||||
"dataset": null,
|
||||
"desc_act": false,
|
||||
"exllama_config": {
|
||||
"version": 2
|
||||
},
|
||||
"group_size": 128,
|
||||
"max_input_length": null,
|
||||
"model_seqlen": null,
|
||||
"module_name_preceding_first_block": null,
|
||||
"modules_in_block_to_quantize": [
|
||||
[
|
||||
"self_attn.k_proj",
|
||||
"self_attn.v_proj",
|
||||
"self_attn.q_proj"
|
||||
],
|
||||
[
|
||||
"self_attn.o_proj"
|
||||
],
|
||||
[
|
||||
"mlp.shared_expert.up_proj",
|
||||
"mlp.shared_expert.gate_proj"
|
||||
],
|
||||
[
|
||||
"mlp.shared_expert.down_proj"
|
||||
],
|
||||
[
|
||||
"mlp.experts.0.up_proj",
|
||||
"mlp.experts.1.up_proj",
|
||||
"mlp.experts.2.up_proj",
|
||||
"mlp.experts.3.up_proj",
|
||||
"mlp.experts.4.up_proj",
|
||||
"mlp.experts.5.up_proj",
|
||||
"mlp.experts.6.up_proj",
|
||||
"mlp.experts.7.up_proj",
|
||||
"mlp.experts.8.up_proj",
|
||||
"mlp.experts.9.up_proj",
|
||||
"mlp.experts.10.up_proj",
|
||||
"mlp.experts.11.up_proj",
|
||||
"mlp.experts.12.up_proj",
|
||||
"mlp.experts.13.up_proj",
|
||||
"mlp.experts.14.up_proj",
|
||||
"mlp.experts.15.up_proj",
|
||||
"mlp.experts.16.up_proj",
|
||||
"mlp.experts.17.up_proj",
|
||||
"mlp.experts.18.up_proj",
|
||||
"mlp.experts.19.up_proj",
|
||||
"mlp.experts.20.up_proj",
|
||||
"mlp.experts.21.up_proj",
|
||||
"mlp.experts.22.up_proj",
|
||||
"mlp.experts.23.up_proj",
|
||||
"mlp.experts.24.up_proj",
|
||||
"mlp.experts.25.up_proj",
|
||||
"mlp.experts.26.up_proj",
|
||||
"mlp.experts.27.up_proj",
|
||||
"mlp.experts.28.up_proj",
|
||||
"mlp.experts.29.up_proj",
|
||||
"mlp.experts.30.up_proj",
|
||||
"mlp.experts.31.up_proj",
|
||||
"mlp.experts.32.up_proj",
|
||||
"mlp.experts.33.up_proj",
|
||||
"mlp.experts.34.up_proj",
|
||||
"mlp.experts.35.up_proj",
|
||||
"mlp.experts.36.up_proj",
|
||||
"mlp.experts.37.up_proj",
|
||||
"mlp.experts.38.up_proj",
|
||||
"mlp.experts.39.up_proj",
|
||||
"mlp.experts.40.up_proj",
|
||||
"mlp.experts.41.up_proj",
|
||||
"mlp.experts.42.up_proj",
|
||||
"mlp.experts.43.up_proj",
|
||||
"mlp.experts.44.up_proj",
|
||||
"mlp.experts.45.up_proj",
|
||||
"mlp.experts.46.up_proj",
|
||||
"mlp.experts.47.up_proj",
|
||||
"mlp.experts.48.up_proj",
|
||||
"mlp.experts.49.up_proj",
|
||||
"mlp.experts.50.up_proj",
|
||||
"mlp.experts.51.up_proj",
|
||||
"mlp.experts.52.up_proj",
|
||||
"mlp.experts.53.up_proj",
|
||||
"mlp.experts.54.up_proj",
|
||||
"mlp.experts.55.up_proj",
|
||||
"mlp.experts.56.up_proj",
|
||||
"mlp.experts.57.up_proj",
|
||||
"mlp.experts.58.up_proj",
|
||||
"mlp.experts.59.up_proj",
|
||||
"mlp.experts.0.gate_proj",
|
||||
"mlp.experts.1.gate_proj",
|
||||
"mlp.experts.2.gate_proj",
|
||||
"mlp.experts.3.gate_proj",
|
||||
"mlp.experts.4.gate_proj",
|
||||
"mlp.experts.5.gate_proj",
|
||||
"mlp.experts.6.gate_proj",
|
||||
"mlp.experts.7.gate_proj",
|
||||
"mlp.experts.8.gate_proj",
|
||||
"mlp.experts.9.gate_proj",
|
||||
"mlp.experts.10.gate_proj",
|
||||
"mlp.experts.11.gate_proj",
|
||||
"mlp.experts.12.gate_proj",
|
||||
"mlp.experts.13.gate_proj",
|
||||
"mlp.experts.14.gate_proj",
|
||||
"mlp.experts.15.gate_proj",
|
||||
"mlp.experts.16.gate_proj",
|
||||
"mlp.experts.17.gate_proj",
|
||||
"mlp.experts.18.gate_proj",
|
||||
"mlp.experts.19.gate_proj",
|
||||
"mlp.experts.20.gate_proj",
|
||||
"mlp.experts.21.gate_proj",
|
||||
"mlp.experts.22.gate_proj",
|
||||
"mlp.experts.23.gate_proj",
|
||||
"mlp.experts.24.gate_proj",
|
||||
"mlp.experts.25.gate_proj",
|
||||
"mlp.experts.26.gate_proj",
|
||||
"mlp.experts.27.gate_proj",
|
||||
"mlp.experts.28.gate_proj",
|
||||
"mlp.experts.29.gate_proj",
|
||||
"mlp.experts.30.gate_proj",
|
||||
"mlp.experts.31.gate_proj",
|
||||
"mlp.experts.32.gate_proj",
|
||||
"mlp.experts.33.gate_proj",
|
||||
"mlp.experts.34.gate_proj",
|
||||
"mlp.experts.35.gate_proj",
|
||||
"mlp.experts.36.gate_proj",
|
||||
"mlp.experts.37.gate_proj",
|
||||
"mlp.experts.38.gate_proj",
|
||||
"mlp.experts.39.gate_proj",
|
||||
"mlp.experts.40.gate_proj",
|
||||
"mlp.experts.41.gate_proj",
|
||||
"mlp.experts.42.gate_proj",
|
||||
"mlp.experts.43.gate_proj",
|
||||
"mlp.experts.44.gate_proj",
|
||||
"mlp.experts.45.gate_proj",
|
||||
"mlp.experts.46.gate_proj",
|
||||
"mlp.experts.47.gate_proj",
|
||||
"mlp.experts.48.gate_proj",
|
||||
"mlp.experts.49.gate_proj",
|
||||
"mlp.experts.50.gate_proj",
|
||||
"mlp.experts.51.gate_proj",
|
||||
"mlp.experts.52.gate_proj",
|
||||
"mlp.experts.53.gate_proj",
|
||||
"mlp.experts.54.gate_proj",
|
||||
"mlp.experts.55.gate_proj",
|
||||
"mlp.experts.56.gate_proj",
|
||||
"mlp.experts.57.gate_proj",
|
||||
"mlp.experts.58.gate_proj",
|
||||
"mlp.experts.59.gate_proj"
|
||||
],
|
||||
[
|
||||
"mlp.experts.0.down_proj",
|
||||
"mlp.experts.1.down_proj",
|
||||
"mlp.experts.2.down_proj",
|
||||
"mlp.experts.3.down_proj",
|
||||
"mlp.experts.4.down_proj",
|
||||
"mlp.experts.5.down_proj",
|
||||
"mlp.experts.6.down_proj",
|
||||
"mlp.experts.7.down_proj",
|
||||
"mlp.experts.8.down_proj",
|
||||
"mlp.experts.9.down_proj",
|
||||
"mlp.experts.10.down_proj",
|
||||
"mlp.experts.11.down_proj",
|
||||
"mlp.experts.12.down_proj",
|
||||
"mlp.experts.13.down_proj",
|
||||
"mlp.experts.14.down_proj",
|
||||
"mlp.experts.15.down_proj",
|
||||
"mlp.experts.16.down_proj",
|
||||
"mlp.experts.17.down_proj",
|
||||
"mlp.experts.18.down_proj",
|
||||
"mlp.experts.19.down_proj",
|
||||
"mlp.experts.20.down_proj",
|
||||
"mlp.experts.21.down_proj",
|
||||
"mlp.experts.22.down_proj",
|
||||
"mlp.experts.23.down_proj",
|
||||
"mlp.experts.24.down_proj",
|
||||
"mlp.experts.25.down_proj",
|
||||
"mlp.experts.26.down_proj",
|
||||
"mlp.experts.27.down_proj",
|
||||
"mlp.experts.28.down_proj",
|
||||
"mlp.experts.29.down_proj",
|
||||
"mlp.experts.30.down_proj",
|
||||
"mlp.experts.31.down_proj",
|
||||
"mlp.experts.32.down_proj",
|
||||
"mlp.experts.33.down_proj",
|
||||
"mlp.experts.34.down_proj",
|
||||
"mlp.experts.35.down_proj",
|
||||
"mlp.experts.36.down_proj",
|
||||
"mlp.experts.37.down_proj",
|
||||
"mlp.experts.38.down_proj",
|
||||
"mlp.experts.39.down_proj",
|
||||
"mlp.experts.40.down_proj",
|
||||
"mlp.experts.41.down_proj",
|
||||
"mlp.experts.42.down_proj",
|
||||
"mlp.experts.43.down_proj",
|
||||
"mlp.experts.44.down_proj",
|
||||
"mlp.experts.45.down_proj",
|
||||
"mlp.experts.46.down_proj",
|
||||
"mlp.experts.47.down_proj",
|
||||
"mlp.experts.48.down_proj",
|
||||
"mlp.experts.49.down_proj",
|
||||
"mlp.experts.50.down_proj",
|
||||
"mlp.experts.51.down_proj",
|
||||
"mlp.experts.52.down_proj",
|
||||
"mlp.experts.53.down_proj",
|
||||
"mlp.experts.54.down_proj",
|
||||
"mlp.experts.55.down_proj",
|
||||
"mlp.experts.56.down_proj",
|
||||
"mlp.experts.57.down_proj",
|
||||
"mlp.experts.58.down_proj",
|
||||
"mlp.experts.59.down_proj"
|
||||
]
|
||||
],
|
||||
"pad_token_id": null,
|
||||
"quant_method": "gptq",
|
||||
"sym": true,
|
||||
"tokenizer": null,
|
||||
"true_sequential": true,
|
||||
"use_cuda_fp16": false,
|
||||
"use_exllama": true
|
||||
},
|
||||
"rms_norm_eps": 1e-06,
|
||||
"rope_theta": 1000000.0,
|
||||
"router_aux_loss_coef": 0.001,
|
||||
"shared_expert_intermediate_size": 5632,
|
||||
"sliding_window": 32768,
|
||||
"tie_word_embeddings": false,
|
||||
"torch_dtype": "float16",
|
||||
"transformers_version": "4.39.0.dev0",
|
||||
"use_cache": true,
|
||||
"use_sliding_window": false,
|
||||
"vocab_size": 151936
|
||||
}
|
||||
1
configuration.json
Normal file
1
configuration.json
Normal file
@@ -0,0 +1 @@
|
||||
{"framework":"Pytorch","task":"fill-mask"}
|
||||
14
generation_config.json
Normal file
14
generation_config.json
Normal file
@@ -0,0 +1,14 @@
|
||||
{
|
||||
"bos_token_id": 151643,
|
||||
"do_sample": true,
|
||||
"eos_token_id": [
|
||||
151645,
|
||||
151643
|
||||
],
|
||||
"pad_token_id": 151643,
|
||||
"repetition_penalty": 1.05,
|
||||
"temperature": 0.7,
|
||||
"top_k": 20,
|
||||
"top_p": 0.8,
|
||||
"transformers_version": "4.39.0.dev0"
|
||||
}
|
||||
151387
merges.txt
Normal file
151387
merges.txt
Normal file
File diff suppressed because it is too large
Load Diff
3
model-00001-of-00003.safetensors
Normal file
3
model-00001-of-00003.safetensors
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:9fc14789a514d8b6fa6163c24b072fbcde6f061c52e48182ebebd215bbd80726
|
||||
size 4000527016
|
||||
3
model-00002-of-00003.safetensors
Normal file
3
model-00002-of-00003.safetensors
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:9d16d52c40f5c2fd32ab050fdbcee923db6b7355b5511f2514cd464cf9f8afb6
|
||||
size 3791651312
|
||||
3
model-00003-of-00003.safetensors
Normal file
3
model-00003-of-00003.safetensors
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:1fa7dea175bfbc5fafb5ee7b098f72acacb4a1744078ea067346f6d63b729481
|
||||
size 622329984
|
||||
22546
model.safetensors.index.json
Normal file
22546
model.safetensors.index.json
Normal file
File diff suppressed because it is too large
Load Diff
303111
tokenizer.json
Normal file
303111
tokenizer.json
Normal file
File diff suppressed because it is too large
Load Diff
41
tokenizer_config.json
Normal file
41
tokenizer_config.json
Normal file
@@ -0,0 +1,41 @@
|
||||
{
|
||||
"add_prefix_space": false,
|
||||
"added_tokens_decoder": {
|
||||
"151643": {
|
||||
"content": "<|endoftext|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151644": {
|
||||
"content": "<|im_start|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151645": {
|
||||
"content": "<|im_end|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
}
|
||||
},
|
||||
"additional_special_tokens": ["<|im_start|>", "<|im_end|>"],
|
||||
"bos_token": null,
|
||||
"chat_template": "{% for message in messages %}{% if loop.first and messages[0]['role'] != 'system' %}{{ '<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n' }}{% endif %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
|
||||
"clean_up_tokenization_spaces": false,
|
||||
"eos_token": "<|im_end|>",
|
||||
"errors": "replace",
|
||||
"model_max_length": 32768,
|
||||
"pad_token": "<|endoftext|>",
|
||||
"split_special_tokens": false,
|
||||
"tokenizer_class": "Qwen2Tokenizer",
|
||||
"unk_token": null
|
||||
}
|
||||
|
||||
1
vocab.json
Normal file
1
vocab.json
Normal file
File diff suppressed because one or more lines are too long
Reference in New Issue
Block a user