初始化项目，由ModelHub XC社区提供模型
Model: ik-ram28/MedMistral-CPT-SFT-7B Source: Original Platform
This commit is contained in:
ModelHub XC
35
.gitattributes
vendored
Normal file
35
.gitattributes
vendored
Normal file
@@ -0,0 +1,35 @@
|
||||
*.7z filter=lfs diff=lfs merge=lfs -text
|
||||
*.arrow filter=lfs diff=lfs merge=lfs -text
|
||||
*.bin filter=lfs diff=lfs merge=lfs -text
|
||||
*.bz2 filter=lfs diff=lfs merge=lfs -text
|
||||
*.ckpt filter=lfs diff=lfs merge=lfs -text
|
||||
*.ftz filter=lfs diff=lfs merge=lfs -text
|
||||
*.gz filter=lfs diff=lfs merge=lfs -text
|
||||
*.h5 filter=lfs diff=lfs merge=lfs -text
|
||||
*.joblib filter=lfs diff=lfs merge=lfs -text
|
||||
*.lfs.* filter=lfs diff=lfs merge=lfs -text
|
||||
*.mlmodel filter=lfs diff=lfs merge=lfs -text
|
||||
*.model filter=lfs diff=lfs merge=lfs -text
|
||||
*.msgpack filter=lfs diff=lfs merge=lfs -text
|
||||
*.npy filter=lfs diff=lfs merge=lfs -text
|
||||
*.npz filter=lfs diff=lfs merge=lfs -text
|
||||
*.onnx filter=lfs diff=lfs merge=lfs -text
|
||||
*.ot filter=lfs diff=lfs merge=lfs -text
|
||||
*.parquet filter=lfs diff=lfs merge=lfs -text
|
||||
*.pb filter=lfs diff=lfs merge=lfs -text
|
||||
*.pickle filter=lfs diff=lfs merge=lfs -text
|
||||
*.pkl filter=lfs diff=lfs merge=lfs -text
|
||||
*.pt filter=lfs diff=lfs merge=lfs -text
|
||||
*.pth filter=lfs diff=lfs merge=lfs -text
|
||||
*.rar filter=lfs diff=lfs merge=lfs -text
|
||||
*.safetensors filter=lfs diff=lfs merge=lfs -text
|
||||
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
||||
*.tar.* filter=lfs diff=lfs merge=lfs -text
|
||||
*.tar filter=lfs diff=lfs merge=lfs -text
|
||||
*.tflite filter=lfs diff=lfs merge=lfs -text
|
||||
*.tgz filter=lfs diff=lfs merge=lfs -text
|
||||
*.wasm filter=lfs diff=lfs merge=lfs -text
|
||||
*.xz filter=lfs diff=lfs merge=lfs -text
|
||||
*.zip filter=lfs diff=lfs merge=lfs -text
|
||||
*.zst filter=lfs diff=lfs merge=lfs -text
|
||||
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
||||
103
README.md
Normal file
103
README.md
Normal file
@@ -0,0 +1,103 @@
|
||||
---
|
||||
library_name: transformers
|
||||
tags:
|
||||
- medical
|
||||
license: apache-2.0
|
||||
language:
|
||||
- fr
|
||||
- en
|
||||
base_model:
|
||||
- ik-ram28/MedMistral-CPT-7B
|
||||
- mistralai/Mistral-7B-v0.1
|
||||
---
|
||||
|
||||
## Model Description
|
||||
|
||||
MedMistral-CPT-SFT-7B is a French medical language model based on Mistral-7B-v0.1, adapted for medical domain applications through a combined approach of Continual Pre-Training (CPT) followed by Supervised Fine-Tuning (SFT).
|
||||
|
||||
## Model Details
|
||||
|
||||
- **Model Type**: Causal Language Model
|
||||
- **Base Model**: Mistral-7B-v0.1
|
||||
- **Language**: French
|
||||
- **Domain**: Medical/Healthcare
|
||||
- **License**: Apache 2.0
|
||||
- **Paper**: [Adaptation des connaissances médicales pour les grands modèles de langue : Stratégies et analyse comparative](https://aclanthology.org/2025.jeptalnrecital-taln.3/)
|
||||
|
||||
## Training Details
|
||||
|
||||
### Continual Pre-Training (CPT)
|
||||
- **Dataset**: NACHOS corpus (opeN crAwled frenCh Healthcare cOrpuS)
|
||||
- **Size**: 7.4 GB of French medical texts
|
||||
- **Word Count**: Over 1 billion words (1,088,867,950 words)
|
||||
- **Sources**: 24 French medical websites
|
||||
- **Training Duration**: 2.8 epochs
|
||||
- **Hardware**: 32 NVIDIA H100 80GB GPUs
|
||||
- **Training Time**: 12 hours
|
||||
- **Optimizer**: AdamW
|
||||
- **Learning Rate**: 2e-5
|
||||
- **Weight Decay**: 0.01
|
||||
- **Batch Size**: 16 with gradient accumulation of 2
|
||||
|
||||
### Supervised Fine-Tuning (SFT)
|
||||
- **Dataset**: 30K French medical question-answer pairs
|
||||
- 10K native French medical questions
|
||||
- 10K translated medical questions from English resources
|
||||
- 10K generated questions from French medical texts
|
||||
- **Method**: DoRA (Weight-Decomposed Low-Rank Adaptation)
|
||||
- **Training Duration**: 10 epochs
|
||||
- **Hardware**: 1 NVIDIA A100 80GB GPU
|
||||
- **Training Time**: 75 hours
|
||||
- **Rank**: 16
|
||||
- **Alpha**: 16
|
||||
- **Learning Rate**: 2e-5
|
||||
- **Batch Size**: 4
|
||||
|
||||
|
||||
## Computational Impact
|
||||
|
||||
- **Total Training Time**: 87 hours (12h CPT + 75h SFT)
|
||||
- **Carbon Emissions**: 11.78 kgCO2e (9.86 + 1.92)
|
||||
|
||||
|
||||
## Ethical Considerations
|
||||
|
||||
- **Medical Accuracy**: This model is for research and educational purposes only. All outputs should be verified by qualified medical professionals
|
||||
- **Bias**: Training data may contain biases present in medical literature and online medical resources
|
||||
|
||||
|
||||
## Citation
|
||||
|
||||
If you use this model, please cite:
|
||||
|
||||
```bibtex
|
||||
@inproceedings{belmadani-etal-2025-adaptation,
|
||||
title = "Adaptation des connaissances m{\'e}dicales pour les grands mod{\`e}les de langue : Strat{\'e}gies et analyse comparative",
|
||||
author = "Belmadani, Ikram and
|
||||
Favre, Benoit and
|
||||
Dufour, Richard and
|
||||
B{\'e}chet, Fr{\'e}d{\'e}ric and
|
||||
Ramisch, Carlos",
|
||||
editor = "Bechet, Fr{\'e}d{\'e}ric and
|
||||
Chifu, Adrian-Gabriel and
|
||||
Pinel-sauvagnat, Karen and
|
||||
Favre, Benoit and
|
||||
Maes, Eliot and
|
||||
Nurbakova, Diana",
|
||||
booktitle = "Actes des 32{\`e}me Conf{\'e}rence sur le Traitement Automatique des Langues Naturelles (TALN), volume 1 : articles scientifiques originaux",
|
||||
month = "6",
|
||||
year = "2025",
|
||||
address = "Marseille, France",
|
||||
publisher = "ATALA {\textbackslash}{\textbackslash}{\&} ARIA",
|
||||
url = "https://aclanthology.org/2025.jeptalnrecital-taln.3/",
|
||||
pages = "50--72",
|
||||
language = "fra",
|
||||
abstract = "Cet article pr{\'e}sente une {\'e}tude sur l{'}adaptation des grands mod{\`e}les de langue (LLMs) {\`a} des domaines sp{\'e}cialis{\'e}s disposant de donn{\'e}es limit{\'e}es. Bien que certaines recherches remettent en question le pr{\'e}-entra{\^i}nement adaptatif (DAPT) dans le contexte m{\'e}dical en anglais, nous montrons que l{'}adaptation au domaine peut {\^e}tre efficace sous certaines conditions. En prenant comme exemple l{'}adaptation au domaine m{\'e}dical en fran{\c{c}}ais, nous comparons de mani{\`e}re syst{\'e}matique le pr{\'e}-entra{\^i}nement continu (CPT), l{'}affinage supervis{\'e} (SFT) et une approche combin{\'e}e (CPT suivi de SFT). Nos r{\'e}sultats indiquent que l{'}adaptation d{'}un mod{\`e}le g{\'e}n{\'e}raliste {\`a} de nouvelles donn{\'e}es dans le domaine m{\'e}dical offre des am{\'e}liorations notables (taux de r{\'e}ussite de 87{\%}), tandis que l{'}adaptation suppl{\'e}mentaire de mod{\`e}les d{\'e}j{\`a} familiaris{\'e}s avec ce domaine procure des b{\'e}n{\'e}fices limit{\'e}s. Bien que CPT+SFT offre les meilleures performances globales, SFT-seul pr{\'e}sente des r{\'e}sultats solides et requiert moins de ressources mat{\'e}rielles."
|
||||
}
|
||||
|
||||
```
|
||||
|
||||
|
||||
## Contact
|
||||
|
||||
For questions about this model, please contact: ikram.belmadani@lis-lab.fr
|
||||
3
added_tokens.json
Normal file
3
added_tokens.json
Normal file
@@ -0,0 +1,3 @@
|
||||
{
|
||||
"[PAD]": 32000
|
||||
}
|
||||
27
config.json
Normal file
27
config.json
Normal file
@@ -0,0 +1,27 @@
|
||||
{
|
||||
"_name_or_path": "ik28/Mistral-v0.1-Nachos-1920",
|
||||
"architectures": [
|
||||
"MistralForCausalLM"
|
||||
],
|
||||
"attention_dropout": 0.0,
|
||||
"bos_token_id": 1,
|
||||
"eos_token_id": 2,
|
||||
"head_dim": 128,
|
||||
"hidden_act": "silu",
|
||||
"hidden_size": 4096,
|
||||
"initializer_range": 0.02,
|
||||
"intermediate_size": 14336,
|
||||
"max_position_embeddings": 32768,
|
||||
"model_type": "mistral",
|
||||
"num_attention_heads": 32,
|
||||
"num_hidden_layers": 32,
|
||||
"num_key_value_heads": 8,
|
||||
"rms_norm_eps": 1e-05,
|
||||
"rope_theta": 10000.0,
|
||||
"sliding_window": 4096,
|
||||
"tie_word_embeddings": false,
|
||||
"torch_dtype": "float32",
|
||||
"transformers_version": "4.46.3",
|
||||
"use_cache": false,
|
||||
"vocab_size": 32000
|
||||
}
|
||||
6
generation_config.json
Normal file
6
generation_config.json
Normal file
@@ -0,0 +1,6 @@
|
||||
{
|
||||
"_from_model_config": true,
|
||||
"bos_token_id": 1,
|
||||
"eos_token_id": 2,
|
||||
"transformers_version": "4.46.3"
|
||||
}
|
||||
3
model-00001-of-00006.safetensors
Normal file
3
model-00001-of-00006.safetensors
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:722a9a7ce722df0526a1ed41e1f638126fdc16eb43e658ae2f9e0a390724109f
|
||||
size 4987196936
|
||||
3
model-00002-of-00006.safetensors
Normal file
3
model-00002-of-00006.safetensors
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:ce7f944e801a10f818ccc0c1e7432dcaa749734f04c5a861a8807a3042327712
|
||||
size 4899116440
|
||||
3
model-00003-of-00006.safetensors
Normal file
3
model-00003-of-00006.safetensors
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:829f4bf8e609468685f323345098cfad0ef053b3101d8402b16bf63e6f237f38
|
||||
size 4999813120
|
||||
3
model-00004-of-00006.safetensors
Normal file
3
model-00004-of-00006.safetensors
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:6ff57983b495e89e83fc3f817969f81fe78f20393a36427dcd558c5ef5fc1b0f
|
||||
size 4999813128
|
||||
3
model-00005-of-00006.safetensors
Normal file
3
model-00005-of-00006.safetensors
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:5ce417942a142de453b4517f97b6b54bc9be5698bf41657e22f8fbe2569aa99a
|
||||
size 4832007496
|
||||
3
model-00006-of-00006.safetensors
Normal file
3
model-00006-of-00006.safetensors
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:1cdb001ac0c8985a2ef75842ff090af95a59ff3ae3c8447bca98190ccbe850d8
|
||||
size 4249014896
|
||||
298
model.safetensors.index.json
Normal file
298
model.safetensors.index.json
Normal file
@@ -0,0 +1,298 @@
|
||||
{
|
||||
"metadata": {
|
||||
"total_size": 28966928384
|
||||
},
|
||||
"weight_map": {
|
||||
"lm_head.weight": "model-00006-of-00006.safetensors",
|
||||
"model.embed_tokens.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.0.input_layernorm.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.0.mlp.down_proj.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.0.mlp.gate_proj.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.0.mlp.up_proj.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.0.post_attention_layernorm.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.0.self_attn.k_proj.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.0.self_attn.o_proj.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.0.self_attn.q_proj.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.0.self_attn.v_proj.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.1.input_layernorm.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.1.mlp.down_proj.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.1.mlp.gate_proj.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.1.mlp.up_proj.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.1.post_attention_layernorm.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.1.self_attn.k_proj.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.1.self_attn.o_proj.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.1.self_attn.q_proj.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.1.self_attn.v_proj.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.10.input_layernorm.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.10.mlp.down_proj.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.10.mlp.gate_proj.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.10.mlp.up_proj.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.10.post_attention_layernorm.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.10.self_attn.k_proj.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.10.self_attn.o_proj.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.10.self_attn.q_proj.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.10.self_attn.v_proj.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.11.input_layernorm.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.11.mlp.down_proj.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.11.mlp.gate_proj.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.11.mlp.up_proj.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.11.post_attention_layernorm.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.11.self_attn.k_proj.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.11.self_attn.o_proj.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.11.self_attn.q_proj.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.11.self_attn.v_proj.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.12.input_layernorm.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.12.mlp.down_proj.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.12.mlp.gate_proj.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.12.mlp.up_proj.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.12.post_attention_layernorm.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.12.self_attn.k_proj.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.12.self_attn.o_proj.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.12.self_attn.q_proj.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.12.self_attn.v_proj.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.13.input_layernorm.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.13.mlp.down_proj.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.13.mlp.gate_proj.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.13.mlp.up_proj.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.13.post_attention_layernorm.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.13.self_attn.k_proj.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.13.self_attn.o_proj.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.13.self_attn.q_proj.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.13.self_attn.v_proj.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.14.input_layernorm.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.14.mlp.down_proj.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.14.mlp.gate_proj.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.14.mlp.up_proj.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.14.post_attention_layernorm.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.14.self_attn.k_proj.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.14.self_attn.o_proj.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.14.self_attn.q_proj.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.14.self_attn.v_proj.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.15.input_layernorm.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.15.mlp.down_proj.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.15.mlp.gate_proj.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.15.mlp.up_proj.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.15.post_attention_layernorm.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.15.self_attn.k_proj.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.15.self_attn.o_proj.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.15.self_attn.q_proj.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.15.self_attn.v_proj.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.16.input_layernorm.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.16.mlp.down_proj.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.16.mlp.gate_proj.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.16.mlp.up_proj.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.16.post_attention_layernorm.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.16.self_attn.k_proj.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.16.self_attn.o_proj.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.16.self_attn.q_proj.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.16.self_attn.v_proj.weight": "model-00003-of-00006.safetensors",
|
||||
"model.layers.17.input_layernorm.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.17.mlp.down_proj.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.17.mlp.gate_proj.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.17.mlp.up_proj.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.17.post_attention_layernorm.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.17.self_attn.k_proj.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.17.self_attn.o_proj.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.17.self_attn.q_proj.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.17.self_attn.v_proj.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.18.input_layernorm.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.18.mlp.down_proj.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.18.mlp.gate_proj.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.18.mlp.up_proj.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.18.post_attention_layernorm.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.18.self_attn.k_proj.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.18.self_attn.o_proj.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.18.self_attn.q_proj.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.18.self_attn.v_proj.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.19.input_layernorm.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.19.mlp.down_proj.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.19.mlp.gate_proj.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.19.mlp.up_proj.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.19.post_attention_layernorm.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.19.self_attn.k_proj.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.19.self_attn.o_proj.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.19.self_attn.q_proj.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.19.self_attn.v_proj.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.2.input_layernorm.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.2.mlp.down_proj.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.2.mlp.gate_proj.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.2.mlp.up_proj.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.2.post_attention_layernorm.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.2.self_attn.k_proj.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.2.self_attn.o_proj.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.2.self_attn.q_proj.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.2.self_attn.v_proj.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.20.input_layernorm.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.20.mlp.down_proj.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.20.mlp.gate_proj.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.20.mlp.up_proj.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.20.post_attention_layernorm.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.20.self_attn.k_proj.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.20.self_attn.o_proj.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.20.self_attn.q_proj.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.20.self_attn.v_proj.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.21.input_layernorm.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.21.mlp.down_proj.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.21.mlp.gate_proj.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.21.mlp.up_proj.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.21.post_attention_layernorm.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.21.self_attn.k_proj.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.21.self_attn.o_proj.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.21.self_attn.q_proj.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.21.self_attn.v_proj.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.22.input_layernorm.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.22.mlp.down_proj.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.22.mlp.gate_proj.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.22.mlp.up_proj.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.22.post_attention_layernorm.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.22.self_attn.k_proj.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.22.self_attn.o_proj.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.22.self_attn.q_proj.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.22.self_attn.v_proj.weight": "model-00004-of-00006.safetensors",
|
||||
"model.layers.23.input_layernorm.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.23.mlp.down_proj.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.23.mlp.gate_proj.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.23.mlp.up_proj.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.23.post_attention_layernorm.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.23.self_attn.k_proj.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.23.self_attn.o_proj.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.23.self_attn.q_proj.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.23.self_attn.v_proj.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.24.input_layernorm.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.24.mlp.down_proj.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.24.mlp.gate_proj.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.24.mlp.up_proj.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.24.post_attention_layernorm.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.24.self_attn.k_proj.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.24.self_attn.o_proj.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.24.self_attn.q_proj.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.24.self_attn.v_proj.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.25.input_layernorm.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.25.mlp.down_proj.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.25.mlp.gate_proj.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.25.mlp.up_proj.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.25.post_attention_layernorm.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.25.self_attn.k_proj.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.25.self_attn.o_proj.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.25.self_attn.q_proj.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.25.self_attn.v_proj.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.26.input_layernorm.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.26.mlp.down_proj.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.26.mlp.gate_proj.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.26.mlp.up_proj.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.26.post_attention_layernorm.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.26.self_attn.k_proj.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.26.self_attn.o_proj.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.26.self_attn.q_proj.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.26.self_attn.v_proj.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.27.input_layernorm.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.27.mlp.down_proj.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.27.mlp.gate_proj.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.27.mlp.up_proj.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.27.post_attention_layernorm.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.27.self_attn.k_proj.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.27.self_attn.o_proj.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.27.self_attn.q_proj.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.27.self_attn.v_proj.weight": "model-00005-of-00006.safetensors",
|
||||
"model.layers.28.input_layernorm.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.28.mlp.down_proj.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.28.mlp.gate_proj.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.28.mlp.up_proj.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.28.post_attention_layernorm.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.28.self_attn.k_proj.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.28.self_attn.o_proj.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.28.self_attn.q_proj.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.28.self_attn.v_proj.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.29.input_layernorm.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.29.mlp.down_proj.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.29.mlp.gate_proj.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.29.mlp.up_proj.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.29.post_attention_layernorm.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.29.self_attn.k_proj.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.29.self_attn.o_proj.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.29.self_attn.q_proj.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.29.self_attn.v_proj.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.3.input_layernorm.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.3.mlp.down_proj.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.3.mlp.gate_proj.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.3.mlp.up_proj.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.3.post_attention_layernorm.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.3.self_attn.k_proj.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.3.self_attn.o_proj.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.3.self_attn.q_proj.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.3.self_attn.v_proj.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.30.input_layernorm.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.30.mlp.down_proj.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.30.mlp.gate_proj.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.30.mlp.up_proj.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.30.post_attention_layernorm.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.30.self_attn.k_proj.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.30.self_attn.o_proj.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.30.self_attn.q_proj.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.30.self_attn.v_proj.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.31.input_layernorm.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.31.mlp.down_proj.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.31.mlp.gate_proj.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.31.mlp.up_proj.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.31.post_attention_layernorm.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.31.self_attn.k_proj.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.31.self_attn.o_proj.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.31.self_attn.q_proj.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.31.self_attn.v_proj.weight": "model-00006-of-00006.safetensors",
|
||||
"model.layers.4.input_layernorm.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.4.mlp.down_proj.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.4.mlp.gate_proj.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.4.mlp.up_proj.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.4.post_attention_layernorm.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.4.self_attn.k_proj.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.4.self_attn.o_proj.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.4.self_attn.q_proj.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.4.self_attn.v_proj.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.5.input_layernorm.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.5.mlp.down_proj.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.5.mlp.gate_proj.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.5.mlp.up_proj.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.5.post_attention_layernorm.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.5.self_attn.k_proj.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.5.self_attn.o_proj.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.5.self_attn.q_proj.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.5.self_attn.v_proj.weight": "model-00001-of-00006.safetensors",
|
||||
"model.layers.6.input_layernorm.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.6.mlp.down_proj.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.6.mlp.gate_proj.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.6.mlp.up_proj.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.6.post_attention_layernorm.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.6.self_attn.k_proj.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.6.self_attn.o_proj.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.6.self_attn.q_proj.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.6.self_attn.v_proj.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.7.input_layernorm.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.7.mlp.down_proj.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.7.mlp.gate_proj.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.7.mlp.up_proj.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.7.post_attention_layernorm.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.7.self_attn.k_proj.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.7.self_attn.o_proj.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.7.self_attn.q_proj.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.7.self_attn.v_proj.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.8.input_layernorm.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.8.mlp.down_proj.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.8.mlp.gate_proj.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.8.mlp.up_proj.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.8.post_attention_layernorm.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.8.self_attn.k_proj.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.8.self_attn.o_proj.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.8.self_attn.q_proj.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.8.self_attn.v_proj.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.9.input_layernorm.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.9.mlp.down_proj.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.9.mlp.gate_proj.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.9.mlp.up_proj.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.9.post_attention_layernorm.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.9.self_attn.k_proj.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.9.self_attn.o_proj.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.9.self_attn.q_proj.weight": "model-00002-of-00006.safetensors",
|
||||
"model.layers.9.self_attn.v_proj.weight": "model-00002-of-00006.safetensors",
|
||||
"model.norm.weight": "model-00006-of-00006.safetensors"
|
||||
}
|
||||
}
|
||||
30
special_tokens_map.json
Normal file
30
special_tokens_map.json
Normal file
@@ -0,0 +1,30 @@
|
||||
{
|
||||
"bos_token": {
|
||||
"content": "<s>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
},
|
||||
"eos_token": {
|
||||
"content": "</s>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
},
|
||||
"pad_token": {
|
||||
"content": "[PAD]",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
},
|
||||
"unk_token": {
|
||||
"content": "<unk>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
}
|
||||
}
|
||||
268062
tokenizer.json
Normal file
268062
tokenizer.json
Normal file
File diff suppressed because it is too large
Load Diff
BIN
tokenizer.model
(Stored with Git LFS)
Normal file
BIN
tokenizer.model
(Stored with Git LFS)
Normal file
Binary file not shown.
52
tokenizer_config.json
Normal file
52
tokenizer_config.json
Normal file
@@ -0,0 +1,52 @@
|
||||
{
|
||||
"add_bos_token": true,
|
||||
"add_eos_token": false,
|
||||
"add_prefix_space": null,
|
||||
"added_tokens_decoder": {
|
||||
"0": {
|
||||
"content": "<unk>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"1": {
|
||||
"content": "<s>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"2": {
|
||||
"content": "</s>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"32000": {
|
||||
"content": "[PAD]",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
}
|
||||
},
|
||||
"additional_special_tokens": [],
|
||||
"bos_token": "<s>",
|
||||
"chat_template": "{% for message in messages %}\n{% if message['role'] == 'user' %}\n{{ '<|user|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'system' %}\n{{ '<|system|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'assistant' %}\n{{ '<|assistant|>\n' + message['content'] + eos_token }}\n{% endif %}\n{% if loop.last and add_generation_prompt %}\n{{ '<|assistant|>' }}\n{% endif %}\n{% endfor %}",
|
||||
"clean_up_tokenization_spaces": false,
|
||||
"eos_token": "</s>",
|
||||
"legacy": false,
|
||||
"model_max_length": 2048,
|
||||
"pad_token": "[PAD]",
|
||||
"sp_model_kwargs": {},
|
||||
"spaces_between_special_tokens": false,
|
||||
"tokenizer_class": "LlamaTokenizer",
|
||||
"unk_token": "<unk>",
|
||||
"use_default_system_prompt": false
|
||||
}
|
||||
Reference in New Issue
Block a user