From 33cec013bc8088e5f663b61dce45f9c56c31b0ef Mon Sep 17 00:00:00 2001 From: ModelHub XC Date: Sat, 15 Aug 2026 13:52:15 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=EF=BC=8C=E7=94=B1ModelHub=20XC=E7=A4=BE=E5=8C=BA=E6=8F=90?= =?UTF-8?q?=E4=BE=9B=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model: Henry236/east8b-eg-lora-turn Source: Original Platform --- .gitattributes | 36 + adapter_config.json | 48 + adapter_model.safetensors | 3 + chat_template.jinja | 7 + checkpoint-100/README.md | 207 ++ checkpoint-100/adapter_config.json | 48 + checkpoint-100/adapter_model.safetensors | 3 + checkpoint-100/optimizer.pt | 3 + checkpoint-100/rng_state.pth | 3 + checkpoint-100/scheduler.pt | 3 + checkpoint-100/trainer_state.json | 69 + checkpoint-100/training_args.bin | 3 + checkpoint-1000/README.md | 207 ++ checkpoint-1000/adapter_config.json | 48 + checkpoint-1000/adapter_model.safetensors | 3 + checkpoint-1000/optimizer.pt | 3 + checkpoint-1000/rng_state.pth | 3 + checkpoint-1000/scheduler.pt | 3 + checkpoint-1000/trainer_state.json | 384 ++++ checkpoint-1000/training_args.bin | 3 + checkpoint-1050/README.md | 207 ++ checkpoint-1050/adapter_config.json | 48 + checkpoint-1050/adapter_model.safetensors | 3 + checkpoint-1050/optimizer.pt | 3 + checkpoint-1050/rng_state.pth | 3 + checkpoint-1050/scheduler.pt | 3 + checkpoint-1050/trainer_state.json | 398 ++++ checkpoint-1050/training_args.bin | 3 + checkpoint-1100/README.md | 207 ++ checkpoint-1100/adapter_config.json | 48 + checkpoint-1100/adapter_model.safetensors | 3 + checkpoint-1100/optimizer.pt | 3 + checkpoint-1100/rng_state.pth | 3 + checkpoint-1100/scheduler.pt | 3 + checkpoint-1100/trainer_state.json | 419 +++++ checkpoint-1100/training_args.bin | 3 + checkpoint-1150/README.md | 207 ++ checkpoint-1150/adapter_config.json | 48 + checkpoint-1150/adapter_model.safetensors | 3 + checkpoint-1150/optimizer.pt | 3 + checkpoint-1150/rng_state.pth | 3 + checkpoint-1150/scheduler.pt | 3 + checkpoint-1150/trainer_state.json | 433 +++++ checkpoint-1150/training_args.bin | 3 + checkpoint-150/README.md | 207 ++ checkpoint-150/adapter_config.json | 48 + checkpoint-150/adapter_model.safetensors | 3 + checkpoint-150/optimizer.pt | 3 + checkpoint-150/rng_state.pth | 3 + checkpoint-150/scheduler.pt | 3 + checkpoint-150/trainer_state.json | 83 + checkpoint-150/training_args.bin | 3 + checkpoint-200/README.md | 207 ++ checkpoint-200/adapter_config.json | 48 + checkpoint-200/adapter_model.safetensors | 3 + checkpoint-200/optimizer.pt | 3 + checkpoint-200/rng_state.pth | 3 + checkpoint-200/scheduler.pt | 3 + checkpoint-200/trainer_state.json | 104 + checkpoint-200/training_args.bin | 3 + checkpoint-250/README.md | 207 ++ checkpoint-250/adapter_config.json | 48 + checkpoint-250/adapter_model.safetensors | 3 + checkpoint-250/optimizer.pt | 3 + checkpoint-250/rng_state.pth | 3 + checkpoint-250/scheduler.pt | 3 + checkpoint-250/trainer_state.json | 118 ++ checkpoint-250/training_args.bin | 3 + checkpoint-300/README.md | 207 ++ checkpoint-300/adapter_config.json | 48 + checkpoint-300/adapter_model.safetensors | 3 + checkpoint-300/optimizer.pt | 3 + checkpoint-300/rng_state.pth | 3 + checkpoint-300/scheduler.pt | 3 + checkpoint-300/trainer_state.json | 139 ++ checkpoint-300/training_args.bin | 3 + checkpoint-350/README.md | 207 ++ checkpoint-350/adapter_config.json | 48 + checkpoint-350/adapter_model.safetensors | 3 + checkpoint-350/optimizer.pt | 3 + checkpoint-350/rng_state.pth | 3 + checkpoint-350/scheduler.pt | 3 + checkpoint-350/trainer_state.json | 153 ++ checkpoint-350/training_args.bin | 3 + checkpoint-400/README.md | 207 ++ checkpoint-400/adapter_config.json | 48 + checkpoint-400/adapter_model.safetensors | 3 + checkpoint-400/optimizer.pt | 3 + checkpoint-400/rng_state.pth | 3 + checkpoint-400/scheduler.pt | 3 + checkpoint-400/trainer_state.json | 174 ++ checkpoint-400/training_args.bin | 3 + checkpoint-450/README.md | 207 ++ checkpoint-450/adapter_config.json | 48 + checkpoint-450/adapter_model.safetensors | 3 + checkpoint-450/optimizer.pt | 3 + checkpoint-450/rng_state.pth | 3 + checkpoint-450/scheduler.pt | 3 + checkpoint-450/trainer_state.json | 188 ++ checkpoint-450/training_args.bin | 3 + checkpoint-50/README.md | 207 ++ checkpoint-50/adapter_config.json | 48 + checkpoint-50/adapter_model.safetensors | 3 + checkpoint-50/optimizer.pt | 3 + checkpoint-50/rng_state.pth | 3 + checkpoint-50/scheduler.pt | 3 + checkpoint-50/trainer_state.json | 48 + checkpoint-50/training_args.bin | 3 + checkpoint-500/README.md | 207 ++ checkpoint-500/adapter_config.json | 48 + checkpoint-500/adapter_model.safetensors | 3 + checkpoint-500/optimizer.pt | 3 + checkpoint-500/rng_state.pth | 3 + checkpoint-500/scheduler.pt | 3 + checkpoint-500/trainer_state.json | 209 +++ checkpoint-500/training_args.bin | 3 + checkpoint-550/README.md | 207 ++ checkpoint-550/adapter_config.json | 48 + checkpoint-550/adapter_model.safetensors | 3 + checkpoint-550/optimizer.pt | 3 + checkpoint-550/rng_state.pth | 3 + checkpoint-550/scheduler.pt | 3 + checkpoint-550/trainer_state.json | 223 +++ checkpoint-550/training_args.bin | 3 + checkpoint-600/README.md | 207 ++ checkpoint-600/adapter_config.json | 48 + checkpoint-600/adapter_model.safetensors | 3 + checkpoint-600/optimizer.pt | 3 + checkpoint-600/rng_state.pth | 3 + checkpoint-600/scheduler.pt | 3 + checkpoint-600/trainer_state.json | 244 +++ checkpoint-600/training_args.bin | 3 + checkpoint-650/README.md | 207 ++ checkpoint-650/adapter_config.json | 48 + checkpoint-650/adapter_model.safetensors | 3 + checkpoint-650/optimizer.pt | 3 + checkpoint-650/rng_state.pth | 3 + checkpoint-650/scheduler.pt | 3 + checkpoint-650/trainer_state.json | 258 +++ checkpoint-650/training_args.bin | 3 + checkpoint-700/README.md | 207 ++ checkpoint-700/adapter_config.json | 48 + checkpoint-700/adapter_model.safetensors | 3 + checkpoint-700/optimizer.pt | 3 + checkpoint-700/rng_state.pth | 3 + checkpoint-700/scheduler.pt | 3 + checkpoint-700/trainer_state.json | 279 +++ checkpoint-700/training_args.bin | 3 + checkpoint-750/README.md | 207 ++ checkpoint-750/adapter_config.json | 48 + checkpoint-750/adapter_model.safetensors | 3 + checkpoint-750/optimizer.pt | 3 + checkpoint-750/rng_state.pth | 3 + checkpoint-750/scheduler.pt | 3 + checkpoint-750/trainer_state.json | 293 +++ checkpoint-750/training_args.bin | 3 + checkpoint-850/README.md | 207 ++ checkpoint-850/adapter_config.json | 48 + checkpoint-850/adapter_model.safetensors | 3 + checkpoint-850/optimizer.pt | 3 + checkpoint-850/rng_state.pth | 3 + checkpoint-850/scheduler.pt | 3 + checkpoint-850/trainer_state.json | 328 ++++ checkpoint-850/training_args.bin | 3 + checkpoint-950/README.md | 207 ++ checkpoint-950/adapter_config.json | 48 + checkpoint-950/adapter_model.safetensors | 3 + checkpoint-950/optimizer.pt | 3 + checkpoint-950/rng_state.pth | 3 + checkpoint-950/scheduler.pt | 3 + checkpoint-950/trainer_state.json | 363 ++++ checkpoint-950/training_args.bin | 3 + config.json | 29 + generation_config.json | 6 + last-checkpoint/README.md | 207 ++ last-checkpoint/adapter_config.json | 48 + last-checkpoint/adapter_model.safetensors | 3 + last-checkpoint/optimizer.pt | 3 + last-checkpoint/rng_state.pth | 3 + last-checkpoint/scheduler.pt | 3 + last-checkpoint/trainer_state.json | 384 ++++ last-checkpoint/training_args.bin | 3 + model-00001-of-00004.safetensors | 3 + model-00002-of-00004.safetensors | 3 + model-00003-of-00004.safetensors | 3 + model-00004-of-00004.safetensors | 3 + model.safetensors.index.json | 299 +++ special_tokens_map.json | 27 + tokenizer.json | 3 + tokenizer_config.json | 2085 +++++++++++++++++++++ training_args.bin | 3 + 191 files changed, 13789 insertions(+) create mode 100644 .gitattributes create mode 100644 adapter_config.json create mode 100644 adapter_model.safetensors create mode 100644 chat_template.jinja create mode 100644 checkpoint-100/README.md create mode 100644 checkpoint-100/adapter_config.json create mode 100644 checkpoint-100/adapter_model.safetensors create mode 100644 checkpoint-100/optimizer.pt create mode 100644 checkpoint-100/rng_state.pth create mode 100644 checkpoint-100/scheduler.pt create mode 100644 checkpoint-100/trainer_state.json create mode 100644 checkpoint-100/training_args.bin create mode 100644 checkpoint-1000/README.md create mode 100644 checkpoint-1000/adapter_config.json create mode 100644 checkpoint-1000/adapter_model.safetensors create mode 100644 checkpoint-1000/optimizer.pt create mode 100644 checkpoint-1000/rng_state.pth create mode 100644 checkpoint-1000/scheduler.pt create mode 100644 checkpoint-1000/trainer_state.json create mode 100644 checkpoint-1000/training_args.bin create mode 100644 checkpoint-1050/README.md create mode 100644 checkpoint-1050/adapter_config.json create mode 100644 checkpoint-1050/adapter_model.safetensors create mode 100644 checkpoint-1050/optimizer.pt create mode 100644 checkpoint-1050/rng_state.pth create mode 100644 checkpoint-1050/scheduler.pt create mode 100644 checkpoint-1050/trainer_state.json create mode 100644 checkpoint-1050/training_args.bin create mode 100644 checkpoint-1100/README.md create mode 100644 checkpoint-1100/adapter_config.json create mode 100644 checkpoint-1100/adapter_model.safetensors create mode 100644 checkpoint-1100/optimizer.pt create mode 100644 checkpoint-1100/rng_state.pth create mode 100644 checkpoint-1100/scheduler.pt create mode 100644 checkpoint-1100/trainer_state.json create mode 100644 checkpoint-1100/training_args.bin create mode 100644 checkpoint-1150/README.md create mode 100644 checkpoint-1150/adapter_config.json create mode 100644 checkpoint-1150/adapter_model.safetensors create mode 100644 checkpoint-1150/optimizer.pt create mode 100644 checkpoint-1150/rng_state.pth create mode 100644 checkpoint-1150/scheduler.pt create mode 100644 checkpoint-1150/trainer_state.json create mode 100644 checkpoint-1150/training_args.bin create mode 100644 checkpoint-150/README.md create mode 100644 checkpoint-150/adapter_config.json create mode 100644 checkpoint-150/adapter_model.safetensors create mode 100644 checkpoint-150/optimizer.pt create mode 100644 checkpoint-150/rng_state.pth create mode 100644 checkpoint-150/scheduler.pt create mode 100644 checkpoint-150/trainer_state.json create mode 100644 checkpoint-150/training_args.bin create mode 100644 checkpoint-200/README.md create mode 100644 checkpoint-200/adapter_config.json create mode 100644 checkpoint-200/adapter_model.safetensors create mode 100644 checkpoint-200/optimizer.pt create mode 100644 checkpoint-200/rng_state.pth create mode 100644 checkpoint-200/scheduler.pt create mode 100644 checkpoint-200/trainer_state.json create mode 100644 checkpoint-200/training_args.bin create mode 100644 checkpoint-250/README.md create mode 100644 checkpoint-250/adapter_config.json create mode 100644 checkpoint-250/adapter_model.safetensors create mode 100644 checkpoint-250/optimizer.pt create mode 100644 checkpoint-250/rng_state.pth create mode 100644 checkpoint-250/scheduler.pt create mode 100644 checkpoint-250/trainer_state.json create mode 100644 checkpoint-250/training_args.bin create mode 100644 checkpoint-300/README.md create mode 100644 checkpoint-300/adapter_config.json create mode 100644 checkpoint-300/adapter_model.safetensors create mode 100644 checkpoint-300/optimizer.pt create mode 100644 checkpoint-300/rng_state.pth create mode 100644 checkpoint-300/scheduler.pt create mode 100644 checkpoint-300/trainer_state.json create mode 100644 checkpoint-300/training_args.bin create mode 100644 checkpoint-350/README.md create mode 100644 checkpoint-350/adapter_config.json create mode 100644 checkpoint-350/adapter_model.safetensors create mode 100644 checkpoint-350/optimizer.pt create mode 100644 checkpoint-350/rng_state.pth create mode 100644 checkpoint-350/scheduler.pt create mode 100644 checkpoint-350/trainer_state.json create mode 100644 checkpoint-350/training_args.bin create mode 100644 checkpoint-400/README.md create mode 100644 checkpoint-400/adapter_config.json create mode 100644 checkpoint-400/adapter_model.safetensors create mode 100644 checkpoint-400/optimizer.pt create mode 100644 checkpoint-400/rng_state.pth create mode 100644 checkpoint-400/scheduler.pt create mode 100644 checkpoint-400/trainer_state.json create mode 100644 checkpoint-400/training_args.bin create mode 100644 checkpoint-450/README.md create mode 100644 checkpoint-450/adapter_config.json create mode 100644 checkpoint-450/adapter_model.safetensors create mode 100644 checkpoint-450/optimizer.pt create mode 100644 checkpoint-450/rng_state.pth create mode 100644 checkpoint-450/scheduler.pt create mode 100644 checkpoint-450/trainer_state.json create mode 100644 checkpoint-450/training_args.bin create mode 100644 checkpoint-50/README.md create mode 100644 checkpoint-50/adapter_config.json create mode 100644 checkpoint-50/adapter_model.safetensors create mode 100644 checkpoint-50/optimizer.pt create mode 100644 checkpoint-50/rng_state.pth create mode 100644 checkpoint-50/scheduler.pt create mode 100644 checkpoint-50/trainer_state.json create mode 100644 checkpoint-50/training_args.bin create mode 100644 checkpoint-500/README.md create mode 100644 checkpoint-500/adapter_config.json create mode 100644 checkpoint-500/adapter_model.safetensors create mode 100644 checkpoint-500/optimizer.pt create mode 100644 checkpoint-500/rng_state.pth create mode 100644 checkpoint-500/scheduler.pt create mode 100644 checkpoint-500/trainer_state.json create mode 100644 checkpoint-500/training_args.bin create mode 100644 checkpoint-550/README.md create mode 100644 checkpoint-550/adapter_config.json create mode 100644 checkpoint-550/adapter_model.safetensors create mode 100644 checkpoint-550/optimizer.pt create mode 100644 checkpoint-550/rng_state.pth create mode 100644 checkpoint-550/scheduler.pt create mode 100644 checkpoint-550/trainer_state.json create mode 100644 checkpoint-550/training_args.bin create mode 100644 checkpoint-600/README.md create mode 100644 checkpoint-600/adapter_config.json create mode 100644 checkpoint-600/adapter_model.safetensors create mode 100644 checkpoint-600/optimizer.pt create mode 100644 checkpoint-600/rng_state.pth create mode 100644 checkpoint-600/scheduler.pt create mode 100644 checkpoint-600/trainer_state.json create mode 100644 checkpoint-600/training_args.bin create mode 100644 checkpoint-650/README.md create mode 100644 checkpoint-650/adapter_config.json create mode 100644 checkpoint-650/adapter_model.safetensors create mode 100644 checkpoint-650/optimizer.pt create mode 100644 checkpoint-650/rng_state.pth create mode 100644 checkpoint-650/scheduler.pt create mode 100644 checkpoint-650/trainer_state.json create mode 100644 checkpoint-650/training_args.bin create mode 100644 checkpoint-700/README.md create mode 100644 checkpoint-700/adapter_config.json create mode 100644 checkpoint-700/adapter_model.safetensors create mode 100644 checkpoint-700/optimizer.pt create mode 100644 checkpoint-700/rng_state.pth create mode 100644 checkpoint-700/scheduler.pt create mode 100644 checkpoint-700/trainer_state.json create mode 100644 checkpoint-700/training_args.bin create mode 100644 checkpoint-750/README.md create mode 100644 checkpoint-750/adapter_config.json create mode 100644 checkpoint-750/adapter_model.safetensors create mode 100644 checkpoint-750/optimizer.pt create mode 100644 checkpoint-750/rng_state.pth create mode 100644 checkpoint-750/scheduler.pt create mode 100644 checkpoint-750/trainer_state.json create mode 100644 checkpoint-750/training_args.bin create mode 100644 checkpoint-850/README.md create mode 100644 checkpoint-850/adapter_config.json create mode 100644 checkpoint-850/adapter_model.safetensors create mode 100644 checkpoint-850/optimizer.pt create mode 100644 checkpoint-850/rng_state.pth create mode 100644 checkpoint-850/scheduler.pt create mode 100644 checkpoint-850/trainer_state.json create mode 100644 checkpoint-850/training_args.bin create mode 100644 checkpoint-950/README.md create mode 100644 checkpoint-950/adapter_config.json create mode 100644 checkpoint-950/adapter_model.safetensors create mode 100644 checkpoint-950/optimizer.pt create mode 100644 checkpoint-950/rng_state.pth create mode 100644 checkpoint-950/scheduler.pt create mode 100644 checkpoint-950/trainer_state.json create mode 100644 checkpoint-950/training_args.bin create mode 100644 config.json create mode 100644 generation_config.json create mode 100644 last-checkpoint/README.md create mode 100644 last-checkpoint/adapter_config.json create mode 100644 last-checkpoint/adapter_model.safetensors create mode 100644 last-checkpoint/optimizer.pt create mode 100644 last-checkpoint/rng_state.pth create mode 100644 last-checkpoint/scheduler.pt create mode 100644 last-checkpoint/trainer_state.json create mode 100644 last-checkpoint/training_args.bin create mode 100644 model-00001-of-00004.safetensors create mode 100644 model-00002-of-00004.safetensors create mode 100644 model-00003-of-00004.safetensors create mode 100644 model-00004-of-00004.safetensors create mode 100644 model.safetensors.index.json create mode 100644 special_tokens_map.json create mode 100644 tokenizer.json create mode 100644 tokenizer_config.json create mode 100644 training_args.bin diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..52373fe --- /dev/null +++ b/.gitattributes @@ -0,0 +1,36 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/adapter_config.json b/adapter_config.json new file mode 100644 index 0000000..37ea34d --- /dev/null +++ b/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "biaofu-xmu/EAST-8B", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.05, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "down_proj", + "v_proj", + "q_proj", + "k_proj", + "gate_proj", + "o_proj", + "up_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/adapter_model.safetensors b/adapter_model.safetensors new file mode 100644 index 0000000..59d3534 --- /dev/null +++ b/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a58b64675456f9242881e64f10abf25bef6ea8d29df1fc65eb8d04eff9ca1633 +size 671149168 diff --git a/chat_template.jinja b/chat_template.jinja new file mode 100644 index 0000000..a3b591c --- /dev/null +++ b/chat_template.jinja @@ -0,0 +1,7 @@ +{% set system_message = 'You are a helpful assistant.' %}{% if messages[0]['role'] == 'system' %}{% set system_message = messages[0]['content'] %}{% endif %}{% if system_message is defined %}{{ '<|begin_of_text|>' + '<|start_header_id|>system<|end_header_id|> + +' + system_message + '<|eot_id|>' }}{% endif %}{% for message in messages %}{% set content = message['content'] %}{% if message['role'] == 'user' %}{{ '<|start_header_id|>user<|end_header_id|> + +' + content + '<|eot_id|><|start_header_id|>assistant<|end_header_id|> + +' }}{% elif message['role'] == 'assistant' %}{{ content + '<|eot_id|>' }}{% endif %}{% endfor %} \ No newline at end of file diff --git a/checkpoint-100/README.md b/checkpoint-100/README.md new file mode 100644 index 0000000..bacf84a --- /dev/null +++ b/checkpoint-100/README.md @@ -0,0 +1,207 @@ +--- +base_model: biaofu-xmu/EAST-8B +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:biaofu-xmu/EAST-8B +- lora +- transformers +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/checkpoint-100/adapter_config.json b/checkpoint-100/adapter_config.json new file mode 100644 index 0000000..536e88f --- /dev/null +++ b/checkpoint-100/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "biaofu-xmu/EAST-8B", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.05, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "q_proj", + "down_proj", + "o_proj", + "gate_proj", + "k_proj", + "v_proj", + "up_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/checkpoint-100/adapter_model.safetensors b/checkpoint-100/adapter_model.safetensors new file mode 100644 index 0000000..d56f686 --- /dev/null +++ b/checkpoint-100/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:177dc733914df11675ddfca8547f927750a204fdfc63d6e0cff9068ab7ce7e47 +size 671149168 diff --git a/checkpoint-100/optimizer.pt b/checkpoint-100/optimizer.pt new file mode 100644 index 0000000..d02dd32 --- /dev/null +++ b/checkpoint-100/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:47f04fb2ef88419ba56738399657c6ca30d2b64d90af4758eb9866afc72195b0 +size 1342562339 diff --git a/checkpoint-100/rng_state.pth b/checkpoint-100/rng_state.pth new file mode 100644 index 0000000..ac7dc19 --- /dev/null +++ b/checkpoint-100/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:59d1ca128ae5cf286d7c6161f33a4ec84d54896c3e4ed19ac65a0e26fa04257b +size 14645 diff --git a/checkpoint-100/scheduler.pt b/checkpoint-100/scheduler.pt new file mode 100644 index 0000000..5b592f2 --- /dev/null +++ b/checkpoint-100/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3a965538a990c1bee6cafb2ee94e4c27c3d42d8dd819e2e80a5cad5750681c87 +size 1465 diff --git a/checkpoint-100/trainer_state.json b/checkpoint-100/trainer_state.json new file mode 100644 index 0000000..2f5afc4 --- /dev/null +++ b/checkpoint-100/trainer_state.json @@ -0,0 +1,69 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.1736111111111111, + "eval_steps": 500, + "global_step": 100, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.034722222222222224, + "grad_norm": 2.5287952423095703, + "learning_rate": 1.6379310344827587e-06, + "loss": 2.4451, + "step": 20 + }, + { + "epoch": 0.06944444444444445, + "grad_norm": 1.9236105680465698, + "learning_rate": 3.362068965517242e-06, + "loss": 2.2698, + "step": 40 + }, + { + "epoch": 0.10416666666666667, + "grad_norm": 1.7310758829116821, + "learning_rate": 5.086206896551724e-06, + "loss": 2.1239, + "step": 60 + }, + { + "epoch": 0.1388888888888889, + "grad_norm": 1.6232755184173584, + "learning_rate": 6.810344827586207e-06, + "loss": 1.9686, + "step": 80 + }, + { + "epoch": 0.1736111111111111, + "grad_norm": 2.0284266471862793, + "learning_rate": 8.53448275862069e-06, + "loss": 1.8519, + "step": 100 + } + ], + "logging_steps": 20, + "max_steps": 1152, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 50, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.952829443063808e+16, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/checkpoint-100/training_args.bin b/checkpoint-100/training_args.bin new file mode 100644 index 0000000..9ea0b2c --- /dev/null +++ b/checkpoint-100/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:95c8504a9e287f63ef7ae2af125d6ce037a2a735a081af84b08d1088ad0a71df +size 5969 diff --git a/checkpoint-1000/README.md b/checkpoint-1000/README.md new file mode 100644 index 0000000..bacf84a --- /dev/null +++ b/checkpoint-1000/README.md @@ -0,0 +1,207 @@ +--- +base_model: biaofu-xmu/EAST-8B +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:biaofu-xmu/EAST-8B +- lora +- transformers +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/checkpoint-1000/adapter_config.json b/checkpoint-1000/adapter_config.json new file mode 100644 index 0000000..ca7008b --- /dev/null +++ b/checkpoint-1000/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "biaofu-xmu/EAST-8B", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.05, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "v_proj", + "q_proj", + "down_proj", + "up_proj", + "gate_proj", + "o_proj", + "k_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/checkpoint-1000/adapter_model.safetensors b/checkpoint-1000/adapter_model.safetensors new file mode 100644 index 0000000..d457975 --- /dev/null +++ b/checkpoint-1000/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:98ef97845d6540c8ac5b9c06dfcb4af8977a7a10a18b89ef5d231ee4355e0daa +size 671149168 diff --git a/checkpoint-1000/optimizer.pt b/checkpoint-1000/optimizer.pt new file mode 100644 index 0000000..a78e4d8 --- /dev/null +++ b/checkpoint-1000/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:24f425709dfb6218e9e1da037e4025c88f1cd89e4f471ba6e23654a710b2e3df +size 1342562339 diff --git a/checkpoint-1000/rng_state.pth b/checkpoint-1000/rng_state.pth new file mode 100644 index 0000000..f4abc42 --- /dev/null +++ b/checkpoint-1000/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1bec9634a694fb3fae48715cd8b4c7686d4953c8c1eaf33350b3387d208503dc +size 14645 diff --git a/checkpoint-1000/scheduler.pt b/checkpoint-1000/scheduler.pt new file mode 100644 index 0000000..54c4178 --- /dev/null +++ b/checkpoint-1000/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3faa60702eeec70c307ffab2e710a51a22e86b2f89bbb3effe498e101641bee8 +size 1465 diff --git a/checkpoint-1000/trainer_state.json b/checkpoint-1000/trainer_state.json new file mode 100644 index 0000000..6c840ad --- /dev/null +++ b/checkpoint-1000/trainer_state.json @@ -0,0 +1,384 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.7361111111111112, + "eval_steps": 500, + "global_step": 1000, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.034722222222222224, + "grad_norm": 2.548311471939087, + "learning_rate": 1.6379310344827587e-06, + "loss": 2.4451, + "step": 20 + }, + { + "epoch": 0.06944444444444445, + "grad_norm": 1.93834388256073, + "learning_rate": 3.362068965517242e-06, + "loss": 2.269, + "step": 40 + }, + { + "epoch": 0.10416666666666667, + "grad_norm": 1.7273279428482056, + "learning_rate": 5.086206896551724e-06, + "loss": 2.1237, + "step": 60 + }, + { + "epoch": 0.1388888888888889, + "grad_norm": 1.6319563388824463, + "learning_rate": 6.810344827586207e-06, + "loss": 1.969, + "step": 80 + }, + { + "epoch": 0.1736111111111111, + "grad_norm": 2.0262465476989746, + "learning_rate": 8.53448275862069e-06, + "loss": 1.8517, + "step": 100 + }, + { + "epoch": 0.20833333333333334, + "grad_norm": 2.1673457622528076, + "learning_rate": 9.999793100349294e-06, + "loss": 1.851, + "step": 120 + }, + { + "epoch": 0.24305555555555555, + "grad_norm": 2.474327802658081, + "learning_rate": 9.987843743451796e-06, + "loss": 1.7448, + "step": 140 + }, + { + "epoch": 0.2777777777777778, + "grad_norm": 2.0316762924194336, + "learning_rate": 9.957553516178782e-06, + "loss": 1.6721, + "step": 160 + }, + { + "epoch": 0.3125, + "grad_norm": 2.3124542236328125, + "learning_rate": 9.909033799150947e-06, + "loss": 1.7265, + "step": 180 + }, + { + "epoch": 0.3472222222222222, + "grad_norm": 2.2682251930236816, + "learning_rate": 9.842463004902127e-06, + "loss": 1.6325, + "step": 200 + }, + { + "epoch": 0.3819444444444444, + "grad_norm": 2.4642767906188965, + "learning_rate": 9.758085921836076e-06, + "loss": 1.6193, + "step": 220 + }, + { + "epoch": 0.4166666666666667, + "grad_norm": 2.4372549057006836, + "learning_rate": 9.656212814111567e-06, + "loss": 1.6297, + "step": 240 + }, + { + "epoch": 0.4513888888888889, + "grad_norm": 2.7898528575897217, + "learning_rate": 9.53721828076571e-06, + "loss": 1.598, + "step": 260 + }, + { + "epoch": 0.4861111111111111, + "grad_norm": 2.742067813873291, + "learning_rate": 9.401539878270545e-06, + "loss": 1.596, + "step": 280 + }, + { + "epoch": 0.5208333333333334, + "grad_norm": 2.9180655479431152, + "learning_rate": 9.249676511588e-06, + "loss": 1.5923, + "step": 300 + }, + { + "epoch": 0.5555555555555556, + "grad_norm": 2.760483503341675, + "learning_rate": 9.082186599639429e-06, + "loss": 1.5733, + "step": 320 + }, + { + "epoch": 0.5902777777777778, + "grad_norm": 2.5661356449127197, + "learning_rate": 8.899686021935554e-06, + "loss": 1.5574, + "step": 340 + }, + { + "epoch": 0.625, + "grad_norm": 2.650604248046875, + "learning_rate": 8.702845853917242e-06, + "loss": 1.5331, + "step": 360 + }, + { + "epoch": 0.6597222222222222, + "grad_norm": 2.5134191513061523, + "learning_rate": 8.492389899334572e-06, + "loss": 1.5116, + "step": 380 + }, + { + "epoch": 0.6944444444444444, + "grad_norm": 2.971090316772461, + "learning_rate": 8.269092028737885e-06, + "loss": 1.4993, + "step": 400 + }, + { + "epoch": 0.7291666666666666, + "grad_norm": 3.0398788452148438, + "learning_rate": 8.033773333867498e-06, + "loss": 1.4751, + "step": 420 + }, + { + "epoch": 0.7638888888888888, + "grad_norm": 2.7814712524414062, + "learning_rate": 7.78729910840572e-06, + "loss": 1.4688, + "step": 440 + }, + { + "epoch": 0.7986111111111112, + "grad_norm": 2.9422523975372314, + "learning_rate": 7.530575666193283e-06, + "loss": 1.4645, + "step": 460 + }, + { + "epoch": 0.8333333333333334, + "grad_norm": 3.5797832012176514, + "learning_rate": 7.26454700860997e-06, + "loss": 1.4473, + "step": 480 + }, + { + "epoch": 0.8680555555555556, + "grad_norm": 3.557054281234741, + "learning_rate": 6.990191353373876e-06, + "loss": 1.4428, + "step": 500 + }, + { + "epoch": 0.9027777777777778, + "grad_norm": 3.7312850952148438, + "learning_rate": 6.708517537523264e-06, + "loss": 1.4014, + "step": 520 + }, + { + "epoch": 0.9375, + "grad_norm": 3.2819321155548096, + "learning_rate": 6.420561307807713e-06, + "loss": 1.4214, + "step": 540 + }, + { + "epoch": 0.9722222222222222, + "grad_norm": 2.9412100315093994, + "learning_rate": 6.12738151212918e-06, + "loss": 1.4367, + "step": 560 + }, + { + "epoch": 1.0069444444444444, + "grad_norm": 4.002957820892334, + "learning_rate": 5.830056206037482e-06, + "loss": 1.4019, + "step": 580 + }, + { + "epoch": 1.0416666666666667, + "grad_norm": 3.2070395946502686, + "learning_rate": 5.529678688597081e-06, + "loss": 1.3294, + "step": 600 + }, + { + "epoch": 1.0763888888888888, + "grad_norm": 3.3583526611328125, + "learning_rate": 5.2273534822017105e-06, + "loss": 1.3302, + "step": 620 + }, + { + "epoch": 1.1111111111111112, + "grad_norm": 4.011720657348633, + "learning_rate": 4.924192271119554e-06, + "loss": 1.2973, + "step": 640 + }, + { + "epoch": 1.1458333333333333, + "grad_norm": 3.383192539215088, + "learning_rate": 4.621309813703385e-06, + "loss": 1.3066, + "step": 660 + }, + { + "epoch": 1.1805555555555556, + "grad_norm": 4.335385322570801, + "learning_rate": 4.319819843296952e-06, + "loss": 1.3068, + "step": 680 + }, + { + "epoch": 1.2152777777777777, + "grad_norm": 3.696607828140259, + "learning_rate": 4.020830972910433e-06, + "loss": 1.3135, + "step": 700 + }, + { + "epoch": 1.25, + "grad_norm": 3.395407199859619, + "learning_rate": 3.7254426187239567e-06, + "loss": 1.2704, + "step": 720 + }, + { + "epoch": 1.2847222222222223, + "grad_norm": 3.829279661178589, + "learning_rate": 3.4347409574088896e-06, + "loss": 1.2869, + "step": 740 + }, + { + "epoch": 1.3194444444444444, + "grad_norm": 4.77693510055542, + "learning_rate": 3.149794932132331e-06, + "loss": 1.2903, + "step": 760 + }, + { + "epoch": 1.3541666666666667, + "grad_norm": 4.946527004241943, + "learning_rate": 2.871652321931161e-06, + "loss": 1.2661, + "step": 780 + }, + { + "epoch": 1.3888888888888888, + "grad_norm": 3.0826640129089355, + "learning_rate": 2.601335888909005e-06, + "loss": 1.2786, + "step": 800 + }, + { + "epoch": 1.4236111111111112, + "grad_norm": 4.134871959686279, + "learning_rate": 2.339839617423318e-06, + "loss": 1.2386, + "step": 820 + }, + { + "epoch": 1.4583333333333333, + "grad_norm": 3.8811264038085938, + "learning_rate": 2.0881250590915316e-06, + "loss": 1.2478, + "step": 840 + }, + { + "epoch": 1.4930555555555556, + "grad_norm": 4.137213230133057, + "learning_rate": 1.8471177970560712e-06, + "loss": 1.2635, + "step": 860 + }, + { + "epoch": 1.5277777777777777, + "grad_norm": 3.4709768295288086, + "learning_rate": 1.6177040425095664e-06, + "loss": 1.2247, + "step": 880 + }, + { + "epoch": 1.5625, + "grad_norm": 3.841049909591675, + "learning_rate": 1.40072737599522e-06, + "loss": 1.2721, + "step": 900 + }, + { + "epoch": 1.5972222222222223, + "grad_norm": 3.412503480911255, + "learning_rate": 1.196985645464921e-06, + "loss": 1.2439, + "step": 920 + }, + { + "epoch": 1.6319444444444444, + "grad_norm": 4.341178894042969, + "learning_rate": 1.0072280325013185e-06, + "loss": 1.2459, + "step": 940 + }, + { + "epoch": 1.6666666666666665, + "grad_norm": 4.485387325286865, + "learning_rate": 8.321522974916968e-07, + "loss": 1.2659, + "step": 960 + }, + { + "epoch": 1.7013888888888888, + "grad_norm": 4.731579303741455, + "learning_rate": 6.724022138834341e-07, + "loss": 1.2283, + "step": 980 + }, + { + "epoch": 1.7361111111111112, + "grad_norm": 3.4078001976013184, + "learning_rate": 5.285652009556075e-07, + "loss": 1.2187, + "step": 1000 + } + ], + "logging_steps": 20, + "max_steps": 1152, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 50, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.9555180478889984e+17, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/checkpoint-1000/training_args.bin b/checkpoint-1000/training_args.bin new file mode 100644 index 0000000..a638377 --- /dev/null +++ b/checkpoint-1000/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6999d99b0ec3831abd58bd6fec26bef4e74e884d0058076eed8c5f0b91586567 +size 5969 diff --git a/checkpoint-1050/README.md b/checkpoint-1050/README.md new file mode 100644 index 0000000..bacf84a --- /dev/null +++ b/checkpoint-1050/README.md @@ -0,0 +1,207 @@ +--- +base_model: biaofu-xmu/EAST-8B +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:biaofu-xmu/EAST-8B +- lora +- transformers +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/checkpoint-1050/adapter_config.json b/checkpoint-1050/adapter_config.json new file mode 100644 index 0000000..ca7008b --- /dev/null +++ b/checkpoint-1050/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "biaofu-xmu/EAST-8B", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.05, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "v_proj", + "q_proj", + "down_proj", + "up_proj", + "gate_proj", + "o_proj", + "k_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/checkpoint-1050/adapter_model.safetensors b/checkpoint-1050/adapter_model.safetensors new file mode 100644 index 0000000..89d50ae --- /dev/null +++ b/checkpoint-1050/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dd953539bbfb30b2b2457b7b78aa71bcc80792e5485f9ab73ca0d2bcfaf0bd4a +size 671149168 diff --git a/checkpoint-1050/optimizer.pt b/checkpoint-1050/optimizer.pt new file mode 100644 index 0000000..8f7a9b1 --- /dev/null +++ b/checkpoint-1050/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:94151b9651f8f14b770ca6a722dbfa1c8c3c09b0241eff460ce03c1fcdb3118a +size 1342562339 diff --git a/checkpoint-1050/rng_state.pth b/checkpoint-1050/rng_state.pth new file mode 100644 index 0000000..94b65f5 --- /dev/null +++ b/checkpoint-1050/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9c2836c708079693320dc1d474758cba33bcedadf31ce3497bb4991f109d2c3a +size 14645 diff --git a/checkpoint-1050/scheduler.pt b/checkpoint-1050/scheduler.pt new file mode 100644 index 0000000..0d92c59 --- /dev/null +++ b/checkpoint-1050/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:abe9abb5efc489b294c297766a8ae2c71262b2cf1b3df1555b36fda19a8e7406 +size 1465 diff --git a/checkpoint-1050/trainer_state.json b/checkpoint-1050/trainer_state.json new file mode 100644 index 0000000..78a6943 --- /dev/null +++ b/checkpoint-1050/trainer_state.json @@ -0,0 +1,398 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.8229166666666665, + "eval_steps": 500, + "global_step": 1050, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.034722222222222224, + "grad_norm": 2.548311471939087, + "learning_rate": 1.6379310344827587e-06, + "loss": 2.4451, + "step": 20 + }, + { + "epoch": 0.06944444444444445, + "grad_norm": 1.93834388256073, + "learning_rate": 3.362068965517242e-06, + "loss": 2.269, + "step": 40 + }, + { + "epoch": 0.10416666666666667, + "grad_norm": 1.7273279428482056, + "learning_rate": 5.086206896551724e-06, + "loss": 2.1237, + "step": 60 + }, + { + "epoch": 0.1388888888888889, + "grad_norm": 1.6319563388824463, + "learning_rate": 6.810344827586207e-06, + "loss": 1.969, + "step": 80 + }, + { + "epoch": 0.1736111111111111, + "grad_norm": 2.0262465476989746, + "learning_rate": 8.53448275862069e-06, + "loss": 1.8517, + "step": 100 + }, + { + "epoch": 0.20833333333333334, + "grad_norm": 2.1673457622528076, + "learning_rate": 9.999793100349294e-06, + "loss": 1.851, + "step": 120 + }, + { + "epoch": 0.24305555555555555, + "grad_norm": 2.474327802658081, + "learning_rate": 9.987843743451796e-06, + "loss": 1.7448, + "step": 140 + }, + { + "epoch": 0.2777777777777778, + "grad_norm": 2.0316762924194336, + "learning_rate": 9.957553516178782e-06, + "loss": 1.6721, + "step": 160 + }, + { + "epoch": 0.3125, + "grad_norm": 2.3124542236328125, + "learning_rate": 9.909033799150947e-06, + "loss": 1.7265, + "step": 180 + }, + { + "epoch": 0.3472222222222222, + "grad_norm": 2.2682251930236816, + "learning_rate": 9.842463004902127e-06, + "loss": 1.6325, + "step": 200 + }, + { + "epoch": 0.3819444444444444, + "grad_norm": 2.4642767906188965, + "learning_rate": 9.758085921836076e-06, + "loss": 1.6193, + "step": 220 + }, + { + "epoch": 0.4166666666666667, + "grad_norm": 2.4372549057006836, + "learning_rate": 9.656212814111567e-06, + "loss": 1.6297, + "step": 240 + }, + { + "epoch": 0.4513888888888889, + "grad_norm": 2.7898528575897217, + "learning_rate": 9.53721828076571e-06, + "loss": 1.598, + "step": 260 + }, + { + "epoch": 0.4861111111111111, + "grad_norm": 2.742067813873291, + "learning_rate": 9.401539878270545e-06, + "loss": 1.596, + "step": 280 + }, + { + "epoch": 0.5208333333333334, + "grad_norm": 2.9180655479431152, + "learning_rate": 9.249676511588e-06, + "loss": 1.5923, + "step": 300 + }, + { + "epoch": 0.5555555555555556, + "grad_norm": 2.760483503341675, + "learning_rate": 9.082186599639429e-06, + "loss": 1.5733, + "step": 320 + }, + { + "epoch": 0.5902777777777778, + "grad_norm": 2.5661356449127197, + "learning_rate": 8.899686021935554e-06, + "loss": 1.5574, + "step": 340 + }, + { + "epoch": 0.625, + "grad_norm": 2.650604248046875, + "learning_rate": 8.702845853917242e-06, + "loss": 1.5331, + "step": 360 + }, + { + "epoch": 0.6597222222222222, + "grad_norm": 2.5134191513061523, + "learning_rate": 8.492389899334572e-06, + "loss": 1.5116, + "step": 380 + }, + { + "epoch": 0.6944444444444444, + "grad_norm": 2.971090316772461, + "learning_rate": 8.269092028737885e-06, + "loss": 1.4993, + "step": 400 + }, + { + "epoch": 0.7291666666666666, + "grad_norm": 3.0398788452148438, + "learning_rate": 8.033773333867498e-06, + "loss": 1.4751, + "step": 420 + }, + { + "epoch": 0.7638888888888888, + "grad_norm": 2.7814712524414062, + "learning_rate": 7.78729910840572e-06, + "loss": 1.4688, + "step": 440 + }, + { + "epoch": 0.7986111111111112, + "grad_norm": 2.9422523975372314, + "learning_rate": 7.530575666193283e-06, + "loss": 1.4645, + "step": 460 + }, + { + "epoch": 0.8333333333333334, + "grad_norm": 3.5797832012176514, + "learning_rate": 7.26454700860997e-06, + "loss": 1.4473, + "step": 480 + }, + { + "epoch": 0.8680555555555556, + "grad_norm": 3.557054281234741, + "learning_rate": 6.990191353373876e-06, + "loss": 1.4428, + "step": 500 + }, + { + "epoch": 0.9027777777777778, + "grad_norm": 3.7312850952148438, + "learning_rate": 6.708517537523264e-06, + "loss": 1.4014, + "step": 520 + }, + { + "epoch": 0.9375, + "grad_norm": 3.2819321155548096, + "learning_rate": 6.420561307807713e-06, + "loss": 1.4214, + "step": 540 + }, + { + "epoch": 0.9722222222222222, + "grad_norm": 2.9412100315093994, + "learning_rate": 6.12738151212918e-06, + "loss": 1.4367, + "step": 560 + }, + { + "epoch": 1.0069444444444444, + "grad_norm": 4.002957820892334, + "learning_rate": 5.830056206037482e-06, + "loss": 1.4019, + "step": 580 + }, + { + "epoch": 1.0416666666666667, + "grad_norm": 3.2070395946502686, + "learning_rate": 5.529678688597081e-06, + "loss": 1.3294, + "step": 600 + }, + { + "epoch": 1.0763888888888888, + "grad_norm": 3.3583526611328125, + "learning_rate": 5.2273534822017105e-06, + "loss": 1.3302, + "step": 620 + }, + { + "epoch": 1.1111111111111112, + "grad_norm": 4.011720657348633, + "learning_rate": 4.924192271119554e-06, + "loss": 1.2973, + "step": 640 + }, + { + "epoch": 1.1458333333333333, + "grad_norm": 3.383192539215088, + "learning_rate": 4.621309813703385e-06, + "loss": 1.3066, + "step": 660 + }, + { + "epoch": 1.1805555555555556, + "grad_norm": 4.335385322570801, + "learning_rate": 4.319819843296952e-06, + "loss": 1.3068, + "step": 680 + }, + { + "epoch": 1.2152777777777777, + "grad_norm": 3.696607828140259, + "learning_rate": 4.020830972910433e-06, + "loss": 1.3135, + "step": 700 + }, + { + "epoch": 1.25, + "grad_norm": 3.395407199859619, + "learning_rate": 3.7254426187239567e-06, + "loss": 1.2704, + "step": 720 + }, + { + "epoch": 1.2847222222222223, + "grad_norm": 3.829279661178589, + "learning_rate": 3.4347409574088896e-06, + "loss": 1.2869, + "step": 740 + }, + { + "epoch": 1.3194444444444444, + "grad_norm": 4.77693510055542, + "learning_rate": 3.149794932132331e-06, + "loss": 1.2903, + "step": 760 + }, + { + "epoch": 1.3541666666666667, + "grad_norm": 4.946527004241943, + "learning_rate": 2.871652321931161e-06, + "loss": 1.2661, + "step": 780 + }, + { + "epoch": 1.3888888888888888, + "grad_norm": 3.0826640129089355, + "learning_rate": 2.601335888909005e-06, + "loss": 1.2786, + "step": 800 + }, + { + "epoch": 1.4236111111111112, + "grad_norm": 4.134871959686279, + "learning_rate": 2.339839617423318e-06, + "loss": 1.2386, + "step": 820 + }, + { + "epoch": 1.4583333333333333, + "grad_norm": 3.8811264038085938, + "learning_rate": 2.0881250590915316e-06, + "loss": 1.2478, + "step": 840 + }, + { + "epoch": 1.4930555555555556, + "grad_norm": 4.137213230133057, + "learning_rate": 1.8471177970560712e-06, + "loss": 1.2635, + "step": 860 + }, + { + "epoch": 1.5277777777777777, + "grad_norm": 3.4709768295288086, + "learning_rate": 1.6177040425095664e-06, + "loss": 1.2247, + "step": 880 + }, + { + "epoch": 1.5625, + "grad_norm": 3.841049909591675, + "learning_rate": 1.40072737599522e-06, + "loss": 1.2721, + "step": 900 + }, + { + "epoch": 1.5972222222222223, + "grad_norm": 3.412503480911255, + "learning_rate": 1.196985645464921e-06, + "loss": 1.2439, + "step": 920 + }, + { + "epoch": 1.6319444444444444, + "grad_norm": 4.341178894042969, + "learning_rate": 1.0072280325013185e-06, + "loss": 1.2459, + "step": 940 + }, + { + "epoch": 1.6666666666666665, + "grad_norm": 4.485387325286865, + "learning_rate": 8.321522974916968e-07, + "loss": 1.2659, + "step": 960 + }, + { + "epoch": 1.7013888888888888, + "grad_norm": 4.731579303741455, + "learning_rate": 6.724022138834341e-07, + "loss": 1.2283, + "step": 980 + }, + { + "epoch": 1.7361111111111112, + "grad_norm": 3.4078001976013184, + "learning_rate": 5.285652009556075e-07, + "loss": 1.2187, + "step": 1000 + }, + { + "epoch": 1.7708333333333335, + "grad_norm": 4.608993053436279, + "learning_rate": 4.0117016381130636e-07, + "loss": 1.2284, + "step": 1020 + }, + { + "epoch": 1.8055555555555556, + "grad_norm": 4.286625862121582, + "learning_rate": 2.906855485332305e-07, + "loss": 1.2042, + "step": 1040 + } + ], + "logging_steps": 20, + "max_steps": 1152, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 50, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 2.053898886369116e+17, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/checkpoint-1050/training_args.bin b/checkpoint-1050/training_args.bin new file mode 100644 index 0000000..a638377 --- /dev/null +++ b/checkpoint-1050/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6999d99b0ec3831abd58bd6fec26bef4e74e884d0058076eed8c5f0b91586567 +size 5969 diff --git a/checkpoint-1100/README.md b/checkpoint-1100/README.md new file mode 100644 index 0000000..bacf84a --- /dev/null +++ b/checkpoint-1100/README.md @@ -0,0 +1,207 @@ +--- +base_model: biaofu-xmu/EAST-8B +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:biaofu-xmu/EAST-8B +- lora +- transformers +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/checkpoint-1100/adapter_config.json b/checkpoint-1100/adapter_config.json new file mode 100644 index 0000000..ca7008b --- /dev/null +++ b/checkpoint-1100/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "biaofu-xmu/EAST-8B", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.05, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "v_proj", + "q_proj", + "down_proj", + "up_proj", + "gate_proj", + "o_proj", + "k_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/checkpoint-1100/adapter_model.safetensors b/checkpoint-1100/adapter_model.safetensors new file mode 100644 index 0000000..635d744 --- /dev/null +++ b/checkpoint-1100/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a023b767dbbd7234cc59d342c25b4f4176295af70ec5325ee0527673bafb9590 +size 671149168 diff --git a/checkpoint-1100/optimizer.pt b/checkpoint-1100/optimizer.pt new file mode 100644 index 0000000..383d271 --- /dev/null +++ b/checkpoint-1100/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c153bc31c0ceec475faf884c579392bae810a05a6e0c68bc439ac9d2e2d3ac42 +size 1342562339 diff --git a/checkpoint-1100/rng_state.pth b/checkpoint-1100/rng_state.pth new file mode 100644 index 0000000..93f7552 --- /dev/null +++ b/checkpoint-1100/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:13b9e42bed996462bb4de8caee0ac43e0c854ab096871c649f128ed02eb44975 +size 14645 diff --git a/checkpoint-1100/scheduler.pt b/checkpoint-1100/scheduler.pt new file mode 100644 index 0000000..60a47de --- /dev/null +++ b/checkpoint-1100/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:315c6a6de3128d29cfc459c9f1551c37f38030611e281b0aa330fe139ba0892f +size 1465 diff --git a/checkpoint-1100/trainer_state.json b/checkpoint-1100/trainer_state.json new file mode 100644 index 0000000..fdd061b --- /dev/null +++ b/checkpoint-1100/trainer_state.json @@ -0,0 +1,419 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.9097222222222223, + "eval_steps": 500, + "global_step": 1100, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.034722222222222224, + "grad_norm": 2.548311471939087, + "learning_rate": 1.6379310344827587e-06, + "loss": 2.4451, + "step": 20 + }, + { + "epoch": 0.06944444444444445, + "grad_norm": 1.93834388256073, + "learning_rate": 3.362068965517242e-06, + "loss": 2.269, + "step": 40 + }, + { + "epoch": 0.10416666666666667, + "grad_norm": 1.7273279428482056, + "learning_rate": 5.086206896551724e-06, + "loss": 2.1237, + "step": 60 + }, + { + "epoch": 0.1388888888888889, + "grad_norm": 1.6319563388824463, + "learning_rate": 6.810344827586207e-06, + "loss": 1.969, + "step": 80 + }, + { + "epoch": 0.1736111111111111, + "grad_norm": 2.0262465476989746, + "learning_rate": 8.53448275862069e-06, + "loss": 1.8517, + "step": 100 + }, + { + "epoch": 0.20833333333333334, + "grad_norm": 2.1673457622528076, + "learning_rate": 9.999793100349294e-06, + "loss": 1.851, + "step": 120 + }, + { + "epoch": 0.24305555555555555, + "grad_norm": 2.474327802658081, + "learning_rate": 9.987843743451796e-06, + "loss": 1.7448, + "step": 140 + }, + { + "epoch": 0.2777777777777778, + "grad_norm": 2.0316762924194336, + "learning_rate": 9.957553516178782e-06, + "loss": 1.6721, + "step": 160 + }, + { + "epoch": 0.3125, + "grad_norm": 2.3124542236328125, + "learning_rate": 9.909033799150947e-06, + "loss": 1.7265, + "step": 180 + }, + { + "epoch": 0.3472222222222222, + "grad_norm": 2.2682251930236816, + "learning_rate": 9.842463004902127e-06, + "loss": 1.6325, + "step": 200 + }, + { + "epoch": 0.3819444444444444, + "grad_norm": 2.4642767906188965, + "learning_rate": 9.758085921836076e-06, + "loss": 1.6193, + "step": 220 + }, + { + "epoch": 0.4166666666666667, + "grad_norm": 2.4372549057006836, + "learning_rate": 9.656212814111567e-06, + "loss": 1.6297, + "step": 240 + }, + { + "epoch": 0.4513888888888889, + "grad_norm": 2.7898528575897217, + "learning_rate": 9.53721828076571e-06, + "loss": 1.598, + "step": 260 + }, + { + "epoch": 0.4861111111111111, + "grad_norm": 2.742067813873291, + "learning_rate": 9.401539878270545e-06, + "loss": 1.596, + "step": 280 + }, + { + "epoch": 0.5208333333333334, + "grad_norm": 2.9180655479431152, + "learning_rate": 9.249676511588e-06, + "loss": 1.5923, + "step": 300 + }, + { + "epoch": 0.5555555555555556, + "grad_norm": 2.760483503341675, + "learning_rate": 9.082186599639429e-06, + "loss": 1.5733, + "step": 320 + }, + { + "epoch": 0.5902777777777778, + "grad_norm": 2.5661356449127197, + "learning_rate": 8.899686021935554e-06, + "loss": 1.5574, + "step": 340 + }, + { + "epoch": 0.625, + "grad_norm": 2.650604248046875, + "learning_rate": 8.702845853917242e-06, + "loss": 1.5331, + "step": 360 + }, + { + "epoch": 0.6597222222222222, + "grad_norm": 2.5134191513061523, + "learning_rate": 8.492389899334572e-06, + "loss": 1.5116, + "step": 380 + }, + { + "epoch": 0.6944444444444444, + "grad_norm": 2.971090316772461, + "learning_rate": 8.269092028737885e-06, + "loss": 1.4993, + "step": 400 + }, + { + "epoch": 0.7291666666666666, + "grad_norm": 3.0398788452148438, + "learning_rate": 8.033773333867498e-06, + "loss": 1.4751, + "step": 420 + }, + { + "epoch": 0.7638888888888888, + "grad_norm": 2.7814712524414062, + "learning_rate": 7.78729910840572e-06, + "loss": 1.4688, + "step": 440 + }, + { + "epoch": 0.7986111111111112, + "grad_norm": 2.9422523975372314, + "learning_rate": 7.530575666193283e-06, + "loss": 1.4645, + "step": 460 + }, + { + "epoch": 0.8333333333333334, + "grad_norm": 3.5797832012176514, + "learning_rate": 7.26454700860997e-06, + "loss": 1.4473, + "step": 480 + }, + { + "epoch": 0.8680555555555556, + "grad_norm": 3.557054281234741, + "learning_rate": 6.990191353373876e-06, + "loss": 1.4428, + "step": 500 + }, + { + "epoch": 0.9027777777777778, + "grad_norm": 3.7312850952148438, + "learning_rate": 6.708517537523264e-06, + "loss": 1.4014, + "step": 520 + }, + { + "epoch": 0.9375, + "grad_norm": 3.2819321155548096, + "learning_rate": 6.420561307807713e-06, + "loss": 1.4214, + "step": 540 + }, + { + "epoch": 0.9722222222222222, + "grad_norm": 2.9412100315093994, + "learning_rate": 6.12738151212918e-06, + "loss": 1.4367, + "step": 560 + }, + { + "epoch": 1.0069444444444444, + "grad_norm": 4.002957820892334, + "learning_rate": 5.830056206037482e-06, + "loss": 1.4019, + "step": 580 + }, + { + "epoch": 1.0416666666666667, + "grad_norm": 3.2070395946502686, + "learning_rate": 5.529678688597081e-06, + "loss": 1.3294, + "step": 600 + }, + { + "epoch": 1.0763888888888888, + "grad_norm": 3.3583526611328125, + "learning_rate": 5.2273534822017105e-06, + "loss": 1.3302, + "step": 620 + }, + { + "epoch": 1.1111111111111112, + "grad_norm": 4.011720657348633, + "learning_rate": 4.924192271119554e-06, + "loss": 1.2973, + "step": 640 + }, + { + "epoch": 1.1458333333333333, + "grad_norm": 3.383192539215088, + "learning_rate": 4.621309813703385e-06, + "loss": 1.3066, + "step": 660 + }, + { + "epoch": 1.1805555555555556, + "grad_norm": 4.335385322570801, + "learning_rate": 4.319819843296952e-06, + "loss": 1.3068, + "step": 680 + }, + { + "epoch": 1.2152777777777777, + "grad_norm": 3.696607828140259, + "learning_rate": 4.020830972910433e-06, + "loss": 1.3135, + "step": 700 + }, + { + "epoch": 1.25, + "grad_norm": 3.395407199859619, + "learning_rate": 3.7254426187239567e-06, + "loss": 1.2704, + "step": 720 + }, + { + "epoch": 1.2847222222222223, + "grad_norm": 3.829279661178589, + "learning_rate": 3.4347409574088896e-06, + "loss": 1.2869, + "step": 740 + }, + { + "epoch": 1.3194444444444444, + "grad_norm": 4.77693510055542, + "learning_rate": 3.149794932132331e-06, + "loss": 1.2903, + "step": 760 + }, + { + "epoch": 1.3541666666666667, + "grad_norm": 4.946527004241943, + "learning_rate": 2.871652321931161e-06, + "loss": 1.2661, + "step": 780 + }, + { + "epoch": 1.3888888888888888, + "grad_norm": 3.0826640129089355, + "learning_rate": 2.601335888909005e-06, + "loss": 1.2786, + "step": 800 + }, + { + "epoch": 1.4236111111111112, + "grad_norm": 4.134871959686279, + "learning_rate": 2.339839617423318e-06, + "loss": 1.2386, + "step": 820 + }, + { + "epoch": 1.4583333333333333, + "grad_norm": 3.8811264038085938, + "learning_rate": 2.0881250590915316e-06, + "loss": 1.2478, + "step": 840 + }, + { + "epoch": 1.4930555555555556, + "grad_norm": 4.137213230133057, + "learning_rate": 1.8471177970560712e-06, + "loss": 1.2635, + "step": 860 + }, + { + "epoch": 1.5277777777777777, + "grad_norm": 3.4709768295288086, + "learning_rate": 1.6177040425095664e-06, + "loss": 1.2247, + "step": 880 + }, + { + "epoch": 1.5625, + "grad_norm": 3.841049909591675, + "learning_rate": 1.40072737599522e-06, + "loss": 1.2721, + "step": 900 + }, + { + "epoch": 1.5972222222222223, + "grad_norm": 3.412503480911255, + "learning_rate": 1.196985645464921e-06, + "loss": 1.2439, + "step": 920 + }, + { + "epoch": 1.6319444444444444, + "grad_norm": 4.341178894042969, + "learning_rate": 1.0072280325013185e-06, + "loss": 1.2459, + "step": 940 + }, + { + "epoch": 1.6666666666666665, + "grad_norm": 4.485387325286865, + "learning_rate": 8.321522974916968e-07, + "loss": 1.2659, + "step": 960 + }, + { + "epoch": 1.7013888888888888, + "grad_norm": 4.731579303741455, + "learning_rate": 6.724022138834341e-07, + "loss": 1.2283, + "step": 980 + }, + { + "epoch": 1.7361111111111112, + "grad_norm": 3.4078001976013184, + "learning_rate": 5.285652009556075e-07, + "loss": 1.2187, + "step": 1000 + }, + { + "epoch": 1.7708333333333335, + "grad_norm": 4.608993053436279, + "learning_rate": 4.0117016381130636e-07, + "loss": 1.2284, + "step": 1020 + }, + { + "epoch": 1.8055555555555556, + "grad_norm": 4.286625862121582, + "learning_rate": 2.906855485332305e-07, + "loss": 1.2042, + "step": 1040 + }, + { + "epoch": 1.8402777777777777, + "grad_norm": 3.952650547027588, + "learning_rate": 1.975176196540185e-07, + "loss": 1.2272, + "step": 1060 + }, + { + "epoch": 1.875, + "grad_norm": 4.211365699768066, + "learning_rate": 1.2200896627521718e-07, + "loss": 1.2411, + "step": 1080 + }, + { + "epoch": 1.9097222222222223, + "grad_norm": 3.815666675567627, + "learning_rate": 6.443724232808146e-08, + "loss": 1.2112, + "step": 1100 + } + ], + "logging_steps": 20, + "max_steps": 1152, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 50, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 2.1503295655958938e+17, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/checkpoint-1100/training_args.bin b/checkpoint-1100/training_args.bin new file mode 100644 index 0000000..a638377 --- /dev/null +++ b/checkpoint-1100/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6999d99b0ec3831abd58bd6fec26bef4e74e884d0058076eed8c5f0b91586567 +size 5969 diff --git a/checkpoint-1150/README.md b/checkpoint-1150/README.md new file mode 100644 index 0000000..bacf84a --- /dev/null +++ b/checkpoint-1150/README.md @@ -0,0 +1,207 @@ +--- +base_model: biaofu-xmu/EAST-8B +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:biaofu-xmu/EAST-8B +- lora +- transformers +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/checkpoint-1150/adapter_config.json b/checkpoint-1150/adapter_config.json new file mode 100644 index 0000000..ca7008b --- /dev/null +++ b/checkpoint-1150/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "biaofu-xmu/EAST-8B", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.05, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "v_proj", + "q_proj", + "down_proj", + "up_proj", + "gate_proj", + "o_proj", + "k_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/checkpoint-1150/adapter_model.safetensors b/checkpoint-1150/adapter_model.safetensors new file mode 100644 index 0000000..4f9384b --- /dev/null +++ b/checkpoint-1150/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:72a130c7a4596d6e50fe85fd0be2a96caa3c0f6923ef43932fd5c9cf9a41866f +size 671149168 diff --git a/checkpoint-1150/optimizer.pt b/checkpoint-1150/optimizer.pt new file mode 100644 index 0000000..c124751 --- /dev/null +++ b/checkpoint-1150/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7cbd1671013e6429a08c1abc1811e40a7324f7254c98fcbde62d3182f49b9d63 +size 1342562339 diff --git a/checkpoint-1150/rng_state.pth b/checkpoint-1150/rng_state.pth new file mode 100644 index 0000000..a384827 --- /dev/null +++ b/checkpoint-1150/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:52fe0d170b8537a026a59ac91014d47d5e7215b953446520a0d813df7c75d6bf +size 14645 diff --git a/checkpoint-1150/scheduler.pt b/checkpoint-1150/scheduler.pt new file mode 100644 index 0000000..d432e90 --- /dev/null +++ b/checkpoint-1150/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:28b81089eb143a676da720978f118caf9eee0a372736c9ef6f38cdec3dd8d997 +size 1465 diff --git a/checkpoint-1150/trainer_state.json b/checkpoint-1150/trainer_state.json new file mode 100644 index 0000000..de461d9 --- /dev/null +++ b/checkpoint-1150/trainer_state.json @@ -0,0 +1,433 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.9965277777777777, + "eval_steps": 500, + "global_step": 1150, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.034722222222222224, + "grad_norm": 2.548311471939087, + "learning_rate": 1.6379310344827587e-06, + "loss": 2.4451, + "step": 20 + }, + { + "epoch": 0.06944444444444445, + "grad_norm": 1.93834388256073, + "learning_rate": 3.362068965517242e-06, + "loss": 2.269, + "step": 40 + }, + { + "epoch": 0.10416666666666667, + "grad_norm": 1.7273279428482056, + "learning_rate": 5.086206896551724e-06, + "loss": 2.1237, + "step": 60 + }, + { + "epoch": 0.1388888888888889, + "grad_norm": 1.6319563388824463, + "learning_rate": 6.810344827586207e-06, + "loss": 1.969, + "step": 80 + }, + { + "epoch": 0.1736111111111111, + "grad_norm": 2.0262465476989746, + "learning_rate": 8.53448275862069e-06, + "loss": 1.8517, + "step": 100 + }, + { + "epoch": 0.20833333333333334, + "grad_norm": 2.1673457622528076, + "learning_rate": 9.999793100349294e-06, + "loss": 1.851, + "step": 120 + }, + { + "epoch": 0.24305555555555555, + "grad_norm": 2.474327802658081, + "learning_rate": 9.987843743451796e-06, + "loss": 1.7448, + "step": 140 + }, + { + "epoch": 0.2777777777777778, + "grad_norm": 2.0316762924194336, + "learning_rate": 9.957553516178782e-06, + "loss": 1.6721, + "step": 160 + }, + { + "epoch": 0.3125, + "grad_norm": 2.3124542236328125, + "learning_rate": 9.909033799150947e-06, + "loss": 1.7265, + "step": 180 + }, + { + "epoch": 0.3472222222222222, + "grad_norm": 2.2682251930236816, + "learning_rate": 9.842463004902127e-06, + "loss": 1.6325, + "step": 200 + }, + { + "epoch": 0.3819444444444444, + "grad_norm": 2.4642767906188965, + "learning_rate": 9.758085921836076e-06, + "loss": 1.6193, + "step": 220 + }, + { + "epoch": 0.4166666666666667, + "grad_norm": 2.4372549057006836, + "learning_rate": 9.656212814111567e-06, + "loss": 1.6297, + "step": 240 + }, + { + "epoch": 0.4513888888888889, + "grad_norm": 2.7898528575897217, + "learning_rate": 9.53721828076571e-06, + "loss": 1.598, + "step": 260 + }, + { + "epoch": 0.4861111111111111, + "grad_norm": 2.742067813873291, + "learning_rate": 9.401539878270545e-06, + "loss": 1.596, + "step": 280 + }, + { + "epoch": 0.5208333333333334, + "grad_norm": 2.9180655479431152, + "learning_rate": 9.249676511588e-06, + "loss": 1.5923, + "step": 300 + }, + { + "epoch": 0.5555555555555556, + "grad_norm": 2.760483503341675, + "learning_rate": 9.082186599639429e-06, + "loss": 1.5733, + "step": 320 + }, + { + "epoch": 0.5902777777777778, + "grad_norm": 2.5661356449127197, + "learning_rate": 8.899686021935554e-06, + "loss": 1.5574, + "step": 340 + }, + { + "epoch": 0.625, + "grad_norm": 2.650604248046875, + "learning_rate": 8.702845853917242e-06, + "loss": 1.5331, + "step": 360 + }, + { + "epoch": 0.6597222222222222, + "grad_norm": 2.5134191513061523, + "learning_rate": 8.492389899334572e-06, + "loss": 1.5116, + "step": 380 + }, + { + "epoch": 0.6944444444444444, + "grad_norm": 2.971090316772461, + "learning_rate": 8.269092028737885e-06, + "loss": 1.4993, + "step": 400 + }, + { + "epoch": 0.7291666666666666, + "grad_norm": 3.0398788452148438, + "learning_rate": 8.033773333867498e-06, + "loss": 1.4751, + "step": 420 + }, + { + "epoch": 0.7638888888888888, + "grad_norm": 2.7814712524414062, + "learning_rate": 7.78729910840572e-06, + "loss": 1.4688, + "step": 440 + }, + { + "epoch": 0.7986111111111112, + "grad_norm": 2.9422523975372314, + "learning_rate": 7.530575666193283e-06, + "loss": 1.4645, + "step": 460 + }, + { + "epoch": 0.8333333333333334, + "grad_norm": 3.5797832012176514, + "learning_rate": 7.26454700860997e-06, + "loss": 1.4473, + "step": 480 + }, + { + "epoch": 0.8680555555555556, + "grad_norm": 3.557054281234741, + "learning_rate": 6.990191353373876e-06, + "loss": 1.4428, + "step": 500 + }, + { + "epoch": 0.9027777777777778, + "grad_norm": 3.7312850952148438, + "learning_rate": 6.708517537523264e-06, + "loss": 1.4014, + "step": 520 + }, + { + "epoch": 0.9375, + "grad_norm": 3.2819321155548096, + "learning_rate": 6.420561307807713e-06, + "loss": 1.4214, + "step": 540 + }, + { + "epoch": 0.9722222222222222, + "grad_norm": 2.9412100315093994, + "learning_rate": 6.12738151212918e-06, + "loss": 1.4367, + "step": 560 + }, + { + "epoch": 1.0069444444444444, + "grad_norm": 4.002957820892334, + "learning_rate": 5.830056206037482e-06, + "loss": 1.4019, + "step": 580 + }, + { + "epoch": 1.0416666666666667, + "grad_norm": 3.2070395946502686, + "learning_rate": 5.529678688597081e-06, + "loss": 1.3294, + "step": 600 + }, + { + "epoch": 1.0763888888888888, + "grad_norm": 3.3583526611328125, + "learning_rate": 5.2273534822017105e-06, + "loss": 1.3302, + "step": 620 + }, + { + "epoch": 1.1111111111111112, + "grad_norm": 4.011720657348633, + "learning_rate": 4.924192271119554e-06, + "loss": 1.2973, + "step": 640 + }, + { + "epoch": 1.1458333333333333, + "grad_norm": 3.383192539215088, + "learning_rate": 4.621309813703385e-06, + "loss": 1.3066, + "step": 660 + }, + { + "epoch": 1.1805555555555556, + "grad_norm": 4.335385322570801, + "learning_rate": 4.319819843296952e-06, + "loss": 1.3068, + "step": 680 + }, + { + "epoch": 1.2152777777777777, + "grad_norm": 3.696607828140259, + "learning_rate": 4.020830972910433e-06, + "loss": 1.3135, + "step": 700 + }, + { + "epoch": 1.25, + "grad_norm": 3.395407199859619, + "learning_rate": 3.7254426187239567e-06, + "loss": 1.2704, + "step": 720 + }, + { + "epoch": 1.2847222222222223, + "grad_norm": 3.829279661178589, + "learning_rate": 3.4347409574088896e-06, + "loss": 1.2869, + "step": 740 + }, + { + "epoch": 1.3194444444444444, + "grad_norm": 4.77693510055542, + "learning_rate": 3.149794932132331e-06, + "loss": 1.2903, + "step": 760 + }, + { + "epoch": 1.3541666666666667, + "grad_norm": 4.946527004241943, + "learning_rate": 2.871652321931161e-06, + "loss": 1.2661, + "step": 780 + }, + { + "epoch": 1.3888888888888888, + "grad_norm": 3.0826640129089355, + "learning_rate": 2.601335888909005e-06, + "loss": 1.2786, + "step": 800 + }, + { + "epoch": 1.4236111111111112, + "grad_norm": 4.134871959686279, + "learning_rate": 2.339839617423318e-06, + "loss": 1.2386, + "step": 820 + }, + { + "epoch": 1.4583333333333333, + "grad_norm": 3.8811264038085938, + "learning_rate": 2.0881250590915316e-06, + "loss": 1.2478, + "step": 840 + }, + { + "epoch": 1.4930555555555556, + "grad_norm": 4.137213230133057, + "learning_rate": 1.8471177970560712e-06, + "loss": 1.2635, + "step": 860 + }, + { + "epoch": 1.5277777777777777, + "grad_norm": 3.4709768295288086, + "learning_rate": 1.6177040425095664e-06, + "loss": 1.2247, + "step": 880 + }, + { + "epoch": 1.5625, + "grad_norm": 3.841049909591675, + "learning_rate": 1.40072737599522e-06, + "loss": 1.2721, + "step": 900 + }, + { + "epoch": 1.5972222222222223, + "grad_norm": 3.412503480911255, + "learning_rate": 1.196985645464921e-06, + "loss": 1.2439, + "step": 920 + }, + { + "epoch": 1.6319444444444444, + "grad_norm": 4.341178894042969, + "learning_rate": 1.0072280325013185e-06, + "loss": 1.2459, + "step": 940 + }, + { + "epoch": 1.6666666666666665, + "grad_norm": 4.485387325286865, + "learning_rate": 8.321522974916968e-07, + "loss": 1.2659, + "step": 960 + }, + { + "epoch": 1.7013888888888888, + "grad_norm": 4.731579303741455, + "learning_rate": 6.724022138834341e-07, + "loss": 1.2283, + "step": 980 + }, + { + "epoch": 1.7361111111111112, + "grad_norm": 3.4078001976013184, + "learning_rate": 5.285652009556075e-07, + "loss": 1.2187, + "step": 1000 + }, + { + "epoch": 1.7708333333333335, + "grad_norm": 4.608993053436279, + "learning_rate": 4.0117016381130636e-07, + "loss": 1.2284, + "step": 1020 + }, + { + "epoch": 1.8055555555555556, + "grad_norm": 4.286625862121582, + "learning_rate": 2.906855485332305e-07, + "loss": 1.2042, + "step": 1040 + }, + { + "epoch": 1.8402777777777777, + "grad_norm": 3.952650547027588, + "learning_rate": 1.975176196540185e-07, + "loss": 1.2272, + "step": 1060 + }, + { + "epoch": 1.875, + "grad_norm": 4.211365699768066, + "learning_rate": 1.2200896627521718e-07, + "loss": 1.2411, + "step": 1080 + }, + { + "epoch": 1.9097222222222223, + "grad_norm": 3.815666675567627, + "learning_rate": 6.443724232808146e-08, + "loss": 1.2112, + "step": 1100 + }, + { + "epoch": 1.9444444444444444, + "grad_norm": 4.6903839111328125, + "learning_rate": 2.501414560839577e-08, + "loss": 1.2108, + "step": 1120 + }, + { + "epoch": 1.9791666666666665, + "grad_norm": 3.7579894065856934, + "learning_rate": 3.884639339534202e-09, + "loss": 1.2249, + "step": 1140 + } + ], + "logging_steps": 20, + "max_steps": 1152, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 50, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 2.2490694985560883e+17, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/checkpoint-1150/training_args.bin b/checkpoint-1150/training_args.bin new file mode 100644 index 0000000..a638377 --- /dev/null +++ b/checkpoint-1150/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6999d99b0ec3831abd58bd6fec26bef4e74e884d0058076eed8c5f0b91586567 +size 5969 diff --git a/checkpoint-150/README.md b/checkpoint-150/README.md new file mode 100644 index 0000000..bacf84a --- /dev/null +++ b/checkpoint-150/README.md @@ -0,0 +1,207 @@ +--- +base_model: biaofu-xmu/EAST-8B +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:biaofu-xmu/EAST-8B +- lora +- transformers +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/checkpoint-150/adapter_config.json b/checkpoint-150/adapter_config.json new file mode 100644 index 0000000..ca7008b --- /dev/null +++ b/checkpoint-150/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "biaofu-xmu/EAST-8B", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.05, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "v_proj", + "q_proj", + "down_proj", + "up_proj", + "gate_proj", + "o_proj", + "k_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/checkpoint-150/adapter_model.safetensors b/checkpoint-150/adapter_model.safetensors new file mode 100644 index 0000000..908eaeb --- /dev/null +++ b/checkpoint-150/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:74774440d969f91e9cbc1288fde7da0e60c4dcd08214604f568c7eaa13c8c96f +size 671149168 diff --git a/checkpoint-150/optimizer.pt b/checkpoint-150/optimizer.pt new file mode 100644 index 0000000..3dcd845 --- /dev/null +++ b/checkpoint-150/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0447fe346bc6ae8298a45e6fdee1887243fec53b673015d22b44edd7aa3c6aa1 +size 1342562339 diff --git a/checkpoint-150/rng_state.pth b/checkpoint-150/rng_state.pth new file mode 100644 index 0000000..b5d6f8c --- /dev/null +++ b/checkpoint-150/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7b5ff99b517bc4bdbd8bd289899bd56ddf2bc69c026f86fc6407cec9838df09d +size 14645 diff --git a/checkpoint-150/scheduler.pt b/checkpoint-150/scheduler.pt new file mode 100644 index 0000000..9bc80d4 --- /dev/null +++ b/checkpoint-150/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7e5c3b1f4b3d2abec87ff5069b5d5df261c03ab9fab7ada1150395aac7685dc6 +size 1465 diff --git a/checkpoint-150/trainer_state.json b/checkpoint-150/trainer_state.json new file mode 100644 index 0000000..daec29b --- /dev/null +++ b/checkpoint-150/trainer_state.json @@ -0,0 +1,83 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.2604166666666667, + "eval_steps": 500, + "global_step": 150, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.034722222222222224, + "grad_norm": 2.548311471939087, + "learning_rate": 1.6379310344827587e-06, + "loss": 2.4451, + "step": 20 + }, + { + "epoch": 0.06944444444444445, + "grad_norm": 1.93834388256073, + "learning_rate": 3.362068965517242e-06, + "loss": 2.269, + "step": 40 + }, + { + "epoch": 0.10416666666666667, + "grad_norm": 1.7273279428482056, + "learning_rate": 5.086206896551724e-06, + "loss": 2.1237, + "step": 60 + }, + { + "epoch": 0.1388888888888889, + "grad_norm": 1.6319563388824463, + "learning_rate": 6.810344827586207e-06, + "loss": 1.969, + "step": 80 + }, + { + "epoch": 0.1736111111111111, + "grad_norm": 2.0262465476989746, + "learning_rate": 8.53448275862069e-06, + "loss": 1.8517, + "step": 100 + }, + { + "epoch": 0.20833333333333334, + "grad_norm": 2.1673457622528076, + "learning_rate": 9.999793100349294e-06, + "loss": 1.851, + "step": 120 + }, + { + "epoch": 0.24305555555555555, + "grad_norm": 2.474327802658081, + "learning_rate": 9.987843743451796e-06, + "loss": 1.7448, + "step": 140 + } + ], + "logging_steps": 20, + "max_steps": 1152, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 50, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 2.926675257930547e+16, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/checkpoint-150/training_args.bin b/checkpoint-150/training_args.bin new file mode 100644 index 0000000..a638377 --- /dev/null +++ b/checkpoint-150/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6999d99b0ec3831abd58bd6fec26bef4e74e884d0058076eed8c5f0b91586567 +size 5969 diff --git a/checkpoint-200/README.md b/checkpoint-200/README.md new file mode 100644 index 0000000..bacf84a --- /dev/null +++ b/checkpoint-200/README.md @@ -0,0 +1,207 @@ +--- +base_model: biaofu-xmu/EAST-8B +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:biaofu-xmu/EAST-8B +- lora +- transformers +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/checkpoint-200/adapter_config.json b/checkpoint-200/adapter_config.json new file mode 100644 index 0000000..ca7008b --- /dev/null +++ b/checkpoint-200/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "biaofu-xmu/EAST-8B", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.05, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "v_proj", + "q_proj", + "down_proj", + "up_proj", + "gate_proj", + "o_proj", + "k_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/checkpoint-200/adapter_model.safetensors b/checkpoint-200/adapter_model.safetensors new file mode 100644 index 0000000..36c286c --- /dev/null +++ b/checkpoint-200/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6746ccd4a4d01867f0da880445055970a8a3bc279f395f901d5bbca13950b454 +size 671149168 diff --git a/checkpoint-200/optimizer.pt b/checkpoint-200/optimizer.pt new file mode 100644 index 0000000..4978176 --- /dev/null +++ b/checkpoint-200/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0102d90737ad3134b4c4e5ad3c36b34d2e3c732be3cba4b55035f5abcf0b57c9 +size 1342562339 diff --git a/checkpoint-200/rng_state.pth b/checkpoint-200/rng_state.pth new file mode 100644 index 0000000..a327e3f --- /dev/null +++ b/checkpoint-200/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0b67a3e3d5b8f172d61c41b8fc389737e67f209cc2fbebe63841b61e06327c92 +size 14645 diff --git a/checkpoint-200/scheduler.pt b/checkpoint-200/scheduler.pt new file mode 100644 index 0000000..fb2732f --- /dev/null +++ b/checkpoint-200/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dacf44b20691c6fc535bf16b0ef4e66f3a5b3cb2170a2da905fbebeb4c5a46ff +size 1465 diff --git a/checkpoint-200/trainer_state.json b/checkpoint-200/trainer_state.json new file mode 100644 index 0000000..758a162 --- /dev/null +++ b/checkpoint-200/trainer_state.json @@ -0,0 +1,104 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.3472222222222222, + "eval_steps": 500, + "global_step": 200, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.034722222222222224, + "grad_norm": 2.548311471939087, + "learning_rate": 1.6379310344827587e-06, + "loss": 2.4451, + "step": 20 + }, + { + "epoch": 0.06944444444444445, + "grad_norm": 1.93834388256073, + "learning_rate": 3.362068965517242e-06, + "loss": 2.269, + "step": 40 + }, + { + "epoch": 0.10416666666666667, + "grad_norm": 1.7273279428482056, + "learning_rate": 5.086206896551724e-06, + "loss": 2.1237, + "step": 60 + }, + { + "epoch": 0.1388888888888889, + "grad_norm": 1.6319563388824463, + "learning_rate": 6.810344827586207e-06, + "loss": 1.969, + "step": 80 + }, + { + "epoch": 0.1736111111111111, + "grad_norm": 2.0262465476989746, + "learning_rate": 8.53448275862069e-06, + "loss": 1.8517, + "step": 100 + }, + { + "epoch": 0.20833333333333334, + "grad_norm": 2.1673457622528076, + "learning_rate": 9.999793100349294e-06, + "loss": 1.851, + "step": 120 + }, + { + "epoch": 0.24305555555555555, + "grad_norm": 2.474327802658081, + "learning_rate": 9.987843743451796e-06, + "loss": 1.7448, + "step": 140 + }, + { + "epoch": 0.2777777777777778, + "grad_norm": 2.0316762924194336, + "learning_rate": 9.957553516178782e-06, + "loss": 1.6721, + "step": 160 + }, + { + "epoch": 0.3125, + "grad_norm": 2.3124542236328125, + "learning_rate": 9.909033799150947e-06, + "loss": 1.7265, + "step": 180 + }, + { + "epoch": 0.3472222222222222, + "grad_norm": 2.2682251930236816, + "learning_rate": 9.842463004902127e-06, + "loss": 1.6325, + "step": 200 + } + ], + "logging_steps": 20, + "max_steps": 1152, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 50, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 3.92832787612631e+16, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/checkpoint-200/training_args.bin b/checkpoint-200/training_args.bin new file mode 100644 index 0000000..a638377 --- /dev/null +++ b/checkpoint-200/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6999d99b0ec3831abd58bd6fec26bef4e74e884d0058076eed8c5f0b91586567 +size 5969 diff --git a/checkpoint-250/README.md b/checkpoint-250/README.md new file mode 100644 index 0000000..bacf84a --- /dev/null +++ b/checkpoint-250/README.md @@ -0,0 +1,207 @@ +--- +base_model: biaofu-xmu/EAST-8B +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:biaofu-xmu/EAST-8B +- lora +- transformers +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/checkpoint-250/adapter_config.json b/checkpoint-250/adapter_config.json new file mode 100644 index 0000000..ca7008b --- /dev/null +++ b/checkpoint-250/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "biaofu-xmu/EAST-8B", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.05, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "v_proj", + "q_proj", + "down_proj", + "up_proj", + "gate_proj", + "o_proj", + "k_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/checkpoint-250/adapter_model.safetensors b/checkpoint-250/adapter_model.safetensors new file mode 100644 index 0000000..032e34b --- /dev/null +++ b/checkpoint-250/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:70ef926d7cb7d6ebf487c946d7f0f3a0660e06699aae04cf44e00019fa695061 +size 671149168 diff --git a/checkpoint-250/optimizer.pt b/checkpoint-250/optimizer.pt new file mode 100644 index 0000000..c5d82e3 --- /dev/null +++ b/checkpoint-250/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:768784f7ec6acba2face9bc2485e3924acb527d9487a3102f09b81212e109312 +size 1342562339 diff --git a/checkpoint-250/rng_state.pth b/checkpoint-250/rng_state.pth new file mode 100644 index 0000000..f8071c4 --- /dev/null +++ b/checkpoint-250/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8e5e1ac8f9f1697e3bc284b46f81101fb0642bca6e30981821b78dd62d98129b +size 14645 diff --git a/checkpoint-250/scheduler.pt b/checkpoint-250/scheduler.pt new file mode 100644 index 0000000..59e9fe4 --- /dev/null +++ b/checkpoint-250/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:12bc768c5ceac0892205b61d77c6800d899da633a32337206aa415df9749b66a +size 1465 diff --git a/checkpoint-250/trainer_state.json b/checkpoint-250/trainer_state.json new file mode 100644 index 0000000..7471e1f --- /dev/null +++ b/checkpoint-250/trainer_state.json @@ -0,0 +1,118 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.4340277777777778, + "eval_steps": 500, + "global_step": 250, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.034722222222222224, + "grad_norm": 2.548311471939087, + "learning_rate": 1.6379310344827587e-06, + "loss": 2.4451, + "step": 20 + }, + { + "epoch": 0.06944444444444445, + "grad_norm": 1.93834388256073, + "learning_rate": 3.362068965517242e-06, + "loss": 2.269, + "step": 40 + }, + { + "epoch": 0.10416666666666667, + "grad_norm": 1.7273279428482056, + "learning_rate": 5.086206896551724e-06, + "loss": 2.1237, + "step": 60 + }, + { + "epoch": 0.1388888888888889, + "grad_norm": 1.6319563388824463, + "learning_rate": 6.810344827586207e-06, + "loss": 1.969, + "step": 80 + }, + { + "epoch": 0.1736111111111111, + "grad_norm": 2.0262465476989746, + "learning_rate": 8.53448275862069e-06, + "loss": 1.8517, + "step": 100 + }, + { + "epoch": 0.20833333333333334, + "grad_norm": 2.1673457622528076, + "learning_rate": 9.999793100349294e-06, + "loss": 1.851, + "step": 120 + }, + { + "epoch": 0.24305555555555555, + "grad_norm": 2.474327802658081, + "learning_rate": 9.987843743451796e-06, + "loss": 1.7448, + "step": 140 + }, + { + "epoch": 0.2777777777777778, + "grad_norm": 2.0316762924194336, + "learning_rate": 9.957553516178782e-06, + "loss": 1.6721, + "step": 160 + }, + { + "epoch": 0.3125, + "grad_norm": 2.3124542236328125, + "learning_rate": 9.909033799150947e-06, + "loss": 1.7265, + "step": 180 + }, + { + "epoch": 0.3472222222222222, + "grad_norm": 2.2682251930236816, + "learning_rate": 9.842463004902127e-06, + "loss": 1.6325, + "step": 200 + }, + { + "epoch": 0.3819444444444444, + "grad_norm": 2.4642767906188965, + "learning_rate": 9.758085921836076e-06, + "loss": 1.6193, + "step": 220 + }, + { + "epoch": 0.4166666666666667, + "grad_norm": 2.4372549057006836, + "learning_rate": 9.656212814111567e-06, + "loss": 1.6297, + "step": 240 + } + ], + "logging_steps": 20, + "max_steps": 1152, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 50, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 4.899613991878656e+16, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/checkpoint-250/training_args.bin b/checkpoint-250/training_args.bin new file mode 100644 index 0000000..a638377 --- /dev/null +++ b/checkpoint-250/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6999d99b0ec3831abd58bd6fec26bef4e74e884d0058076eed8c5f0b91586567 +size 5969 diff --git a/checkpoint-300/README.md b/checkpoint-300/README.md new file mode 100644 index 0000000..bacf84a --- /dev/null +++ b/checkpoint-300/README.md @@ -0,0 +1,207 @@ +--- +base_model: biaofu-xmu/EAST-8B +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:biaofu-xmu/EAST-8B +- lora +- transformers +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/checkpoint-300/adapter_config.json b/checkpoint-300/adapter_config.json new file mode 100644 index 0000000..536e88f --- /dev/null +++ b/checkpoint-300/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "biaofu-xmu/EAST-8B", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.05, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "q_proj", + "down_proj", + "o_proj", + "gate_proj", + "k_proj", + "v_proj", + "up_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/checkpoint-300/adapter_model.safetensors b/checkpoint-300/adapter_model.safetensors new file mode 100644 index 0000000..f05d3c7 --- /dev/null +++ b/checkpoint-300/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d6472d499937d2fc466bc5248d763427304ebea78eb098cf571df43f9f7048f0 +size 671149168 diff --git a/checkpoint-300/optimizer.pt b/checkpoint-300/optimizer.pt new file mode 100644 index 0000000..e12338a --- /dev/null +++ b/checkpoint-300/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9a5e61092ae80965d6885e4588ceac6a6dc8e55146f4226ab1e720d127e624d7 +size 1342562339 diff --git a/checkpoint-300/rng_state.pth b/checkpoint-300/rng_state.pth new file mode 100644 index 0000000..afb7979 --- /dev/null +++ b/checkpoint-300/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:50a6cecaf777531098b542e8fe3720c5dca8ef4962521136493806bc72d93b0f +size 14645 diff --git a/checkpoint-300/scheduler.pt b/checkpoint-300/scheduler.pt new file mode 100644 index 0000000..a782478 --- /dev/null +++ b/checkpoint-300/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8011544f8969ccf121f310e7c9677b774202f770f03769833c73ecbfadde40a4 +size 1465 diff --git a/checkpoint-300/trainer_state.json b/checkpoint-300/trainer_state.json new file mode 100644 index 0000000..f96f429 --- /dev/null +++ b/checkpoint-300/trainer_state.json @@ -0,0 +1,139 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.5208333333333334, + "eval_steps": 500, + "global_step": 300, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.034722222222222224, + "grad_norm": 2.5287952423095703, + "learning_rate": 1.6379310344827587e-06, + "loss": 2.4451, + "step": 20 + }, + { + "epoch": 0.06944444444444445, + "grad_norm": 1.9236105680465698, + "learning_rate": 3.362068965517242e-06, + "loss": 2.2698, + "step": 40 + }, + { + "epoch": 0.10416666666666667, + "grad_norm": 1.7310758829116821, + "learning_rate": 5.086206896551724e-06, + "loss": 2.1239, + "step": 60 + }, + { + "epoch": 0.1388888888888889, + "grad_norm": 1.6232755184173584, + "learning_rate": 6.810344827586207e-06, + "loss": 1.9686, + "step": 80 + }, + { + "epoch": 0.1736111111111111, + "grad_norm": 2.0284266471862793, + "learning_rate": 8.53448275862069e-06, + "loss": 1.8519, + "step": 100 + }, + { + "epoch": 0.20833333333333334, + "grad_norm": 2.1752660274505615, + "learning_rate": 9.999793100349294e-06, + "loss": 1.8514, + "step": 120 + }, + { + "epoch": 0.24305555555555555, + "grad_norm": 2.468181848526001, + "learning_rate": 9.987843743451796e-06, + "loss": 1.7449, + "step": 140 + }, + { + "epoch": 0.2777777777777778, + "grad_norm": 2.0356285572052, + "learning_rate": 9.957553516178782e-06, + "loss": 1.6723, + "step": 160 + }, + { + "epoch": 0.3125, + "grad_norm": 2.3106741905212402, + "learning_rate": 9.909033799150947e-06, + "loss": 1.7265, + "step": 180 + }, + { + "epoch": 0.3472222222222222, + "grad_norm": 2.265456199645996, + "learning_rate": 9.842463004902127e-06, + "loss": 1.632, + "step": 200 + }, + { + "epoch": 0.3819444444444444, + "grad_norm": 2.4798583984375, + "learning_rate": 9.758085921836076e-06, + "loss": 1.6189, + "step": 220 + }, + { + "epoch": 0.4166666666666667, + "grad_norm": 2.4368059635162354, + "learning_rate": 9.656212814111567e-06, + "loss": 1.6295, + "step": 240 + }, + { + "epoch": 0.4513888888888889, + "grad_norm": 2.782370090484619, + "learning_rate": 9.53721828076571e-06, + "loss": 1.5973, + "step": 260 + }, + { + "epoch": 0.4861111111111111, + "grad_norm": 2.735018014907837, + "learning_rate": 9.401539878270545e-06, + "loss": 1.5953, + "step": 280 + }, + { + "epoch": 0.5208333333333334, + "grad_norm": 2.905834913253784, + "learning_rate": 9.249676511588e-06, + "loss": 1.5922, + "step": 300 + } + ], + "logging_steps": 20, + "max_steps": 1152, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 50, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 5.858525159394509e+16, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/checkpoint-300/training_args.bin b/checkpoint-300/training_args.bin new file mode 100644 index 0000000..9ea0b2c --- /dev/null +++ b/checkpoint-300/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:95c8504a9e287f63ef7ae2af125d6ce037a2a735a081af84b08d1088ad0a71df +size 5969 diff --git a/checkpoint-350/README.md b/checkpoint-350/README.md new file mode 100644 index 0000000..bacf84a --- /dev/null +++ b/checkpoint-350/README.md @@ -0,0 +1,207 @@ +--- +base_model: biaofu-xmu/EAST-8B +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:biaofu-xmu/EAST-8B +- lora +- transformers +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/checkpoint-350/adapter_config.json b/checkpoint-350/adapter_config.json new file mode 100644 index 0000000..ca7008b --- /dev/null +++ b/checkpoint-350/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "biaofu-xmu/EAST-8B", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.05, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "v_proj", + "q_proj", + "down_proj", + "up_proj", + "gate_proj", + "o_proj", + "k_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/checkpoint-350/adapter_model.safetensors b/checkpoint-350/adapter_model.safetensors new file mode 100644 index 0000000..a0363a8 --- /dev/null +++ b/checkpoint-350/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:44012ffb44463f205eca7d71d10ce1dd20d1b0f002346fd4dcdf614d903e9ee6 +size 671149168 diff --git a/checkpoint-350/optimizer.pt b/checkpoint-350/optimizer.pt new file mode 100644 index 0000000..8e60d57 --- /dev/null +++ b/checkpoint-350/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8dabff6caa61f8aeb0b5f894c9e3d2c42c90631fd6fac141a9dd5e7818717087 +size 1342562339 diff --git a/checkpoint-350/rng_state.pth b/checkpoint-350/rng_state.pth new file mode 100644 index 0000000..e8960a0 --- /dev/null +++ b/checkpoint-350/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e3d5cbcccde05687b501efb2483b12f919f73c983685fcd85d56b4009e655910 +size 14645 diff --git a/checkpoint-350/scheduler.pt b/checkpoint-350/scheduler.pt new file mode 100644 index 0000000..144e670 --- /dev/null +++ b/checkpoint-350/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2a6d13f122fa4e9259c6cf2ccd0d4a831e7733be6388a9b01a859d975eb74082 +size 1465 diff --git a/checkpoint-350/trainer_state.json b/checkpoint-350/trainer_state.json new file mode 100644 index 0000000..5f2ddc0 --- /dev/null +++ b/checkpoint-350/trainer_state.json @@ -0,0 +1,153 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.6076388888888888, + "eval_steps": 500, + "global_step": 350, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.034722222222222224, + "grad_norm": 2.548311471939087, + "learning_rate": 1.6379310344827587e-06, + "loss": 2.4451, + "step": 20 + }, + { + "epoch": 0.06944444444444445, + "grad_norm": 1.93834388256073, + "learning_rate": 3.362068965517242e-06, + "loss": 2.269, + "step": 40 + }, + { + "epoch": 0.10416666666666667, + "grad_norm": 1.7273279428482056, + "learning_rate": 5.086206896551724e-06, + "loss": 2.1237, + "step": 60 + }, + { + "epoch": 0.1388888888888889, + "grad_norm": 1.6319563388824463, + "learning_rate": 6.810344827586207e-06, + "loss": 1.969, + "step": 80 + }, + { + "epoch": 0.1736111111111111, + "grad_norm": 2.0262465476989746, + "learning_rate": 8.53448275862069e-06, + "loss": 1.8517, + "step": 100 + }, + { + "epoch": 0.20833333333333334, + "grad_norm": 2.1673457622528076, + "learning_rate": 9.999793100349294e-06, + "loss": 1.851, + "step": 120 + }, + { + "epoch": 0.24305555555555555, + "grad_norm": 2.474327802658081, + "learning_rate": 9.987843743451796e-06, + "loss": 1.7448, + "step": 140 + }, + { + "epoch": 0.2777777777777778, + "grad_norm": 2.0316762924194336, + "learning_rate": 9.957553516178782e-06, + "loss": 1.6721, + "step": 160 + }, + { + "epoch": 0.3125, + "grad_norm": 2.3124542236328125, + "learning_rate": 9.909033799150947e-06, + "loss": 1.7265, + "step": 180 + }, + { + "epoch": 0.3472222222222222, + "grad_norm": 2.2682251930236816, + "learning_rate": 9.842463004902127e-06, + "loss": 1.6325, + "step": 200 + }, + { + "epoch": 0.3819444444444444, + "grad_norm": 2.4642767906188965, + "learning_rate": 9.758085921836076e-06, + "loss": 1.6193, + "step": 220 + }, + { + "epoch": 0.4166666666666667, + "grad_norm": 2.4372549057006836, + "learning_rate": 9.656212814111567e-06, + "loss": 1.6297, + "step": 240 + }, + { + "epoch": 0.4513888888888889, + "grad_norm": 2.7898528575897217, + "learning_rate": 9.53721828076571e-06, + "loss": 1.598, + "step": 260 + }, + { + "epoch": 0.4861111111111111, + "grad_norm": 2.742067813873291, + "learning_rate": 9.401539878270545e-06, + "loss": 1.596, + "step": 280 + }, + { + "epoch": 0.5208333333333334, + "grad_norm": 2.9180655479431152, + "learning_rate": 9.249676511588e-06, + "loss": 1.5923, + "step": 300 + }, + { + "epoch": 0.5555555555555556, + "grad_norm": 2.760483503341675, + "learning_rate": 9.082186599639429e-06, + "loss": 1.5733, + "step": 320 + }, + { + "epoch": 0.5902777777777778, + "grad_norm": 2.5661356449127197, + "learning_rate": 8.899686021935554e-06, + "loss": 1.5574, + "step": 340 + } + ], + "logging_steps": 20, + "max_steps": 1152, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 50, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 6.818485987698278e+16, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/checkpoint-350/training_args.bin b/checkpoint-350/training_args.bin new file mode 100644 index 0000000..a638377 --- /dev/null +++ b/checkpoint-350/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6999d99b0ec3831abd58bd6fec26bef4e74e884d0058076eed8c5f0b91586567 +size 5969 diff --git a/checkpoint-400/README.md b/checkpoint-400/README.md new file mode 100644 index 0000000..bacf84a --- /dev/null +++ b/checkpoint-400/README.md @@ -0,0 +1,207 @@ +--- +base_model: biaofu-xmu/EAST-8B +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:biaofu-xmu/EAST-8B +- lora +- transformers +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/checkpoint-400/adapter_config.json b/checkpoint-400/adapter_config.json new file mode 100644 index 0000000..ca7008b --- /dev/null +++ b/checkpoint-400/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "biaofu-xmu/EAST-8B", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.05, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "v_proj", + "q_proj", + "down_proj", + "up_proj", + "gate_proj", + "o_proj", + "k_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/checkpoint-400/adapter_model.safetensors b/checkpoint-400/adapter_model.safetensors new file mode 100644 index 0000000..a0a48f3 --- /dev/null +++ b/checkpoint-400/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a5a35fc8f03e07f0d79eccff0ac69597398394524058ed9768549422e2b41890 +size 671149168 diff --git a/checkpoint-400/optimizer.pt b/checkpoint-400/optimizer.pt new file mode 100644 index 0000000..56bebc6 --- /dev/null +++ b/checkpoint-400/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a7b3ff7b2607fd118a6ab2f04600012aac2de772b064635f871eafc2a66a7964 +size 1342562339 diff --git a/checkpoint-400/rng_state.pth b/checkpoint-400/rng_state.pth new file mode 100644 index 0000000..7013a29 --- /dev/null +++ b/checkpoint-400/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b90eeed1f2a40be670ba230d57abf221bdaf4d742ef93488de052c9f9371a9b3 +size 14645 diff --git a/checkpoint-400/scheduler.pt b/checkpoint-400/scheduler.pt new file mode 100644 index 0000000..fe9d5cc --- /dev/null +++ b/checkpoint-400/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:20cc7f2c390b57e07da00163d66a7f2ae7ece3e78a62f304c58c0f81ef0baa87 +size 1465 diff --git a/checkpoint-400/trainer_state.json b/checkpoint-400/trainer_state.json new file mode 100644 index 0000000..9c89dcc --- /dev/null +++ b/checkpoint-400/trainer_state.json @@ -0,0 +1,174 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.6944444444444444, + "eval_steps": 500, + "global_step": 400, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.034722222222222224, + "grad_norm": 2.548311471939087, + "learning_rate": 1.6379310344827587e-06, + "loss": 2.4451, + "step": 20 + }, + { + "epoch": 0.06944444444444445, + "grad_norm": 1.93834388256073, + "learning_rate": 3.362068965517242e-06, + "loss": 2.269, + "step": 40 + }, + { + "epoch": 0.10416666666666667, + "grad_norm": 1.7273279428482056, + "learning_rate": 5.086206896551724e-06, + "loss": 2.1237, + "step": 60 + }, + { + "epoch": 0.1388888888888889, + "grad_norm": 1.6319563388824463, + "learning_rate": 6.810344827586207e-06, + "loss": 1.969, + "step": 80 + }, + { + "epoch": 0.1736111111111111, + "grad_norm": 2.0262465476989746, + "learning_rate": 8.53448275862069e-06, + "loss": 1.8517, + "step": 100 + }, + { + "epoch": 0.20833333333333334, + "grad_norm": 2.1673457622528076, + "learning_rate": 9.999793100349294e-06, + "loss": 1.851, + "step": 120 + }, + { + "epoch": 0.24305555555555555, + "grad_norm": 2.474327802658081, + "learning_rate": 9.987843743451796e-06, + "loss": 1.7448, + "step": 140 + }, + { + "epoch": 0.2777777777777778, + "grad_norm": 2.0316762924194336, + "learning_rate": 9.957553516178782e-06, + "loss": 1.6721, + "step": 160 + }, + { + "epoch": 0.3125, + "grad_norm": 2.3124542236328125, + "learning_rate": 9.909033799150947e-06, + "loss": 1.7265, + "step": 180 + }, + { + "epoch": 0.3472222222222222, + "grad_norm": 2.2682251930236816, + "learning_rate": 9.842463004902127e-06, + "loss": 1.6325, + "step": 200 + }, + { + "epoch": 0.3819444444444444, + "grad_norm": 2.4642767906188965, + "learning_rate": 9.758085921836076e-06, + "loss": 1.6193, + "step": 220 + }, + { + "epoch": 0.4166666666666667, + "grad_norm": 2.4372549057006836, + "learning_rate": 9.656212814111567e-06, + "loss": 1.6297, + "step": 240 + }, + { + "epoch": 0.4513888888888889, + "grad_norm": 2.7898528575897217, + "learning_rate": 9.53721828076571e-06, + "loss": 1.598, + "step": 260 + }, + { + "epoch": 0.4861111111111111, + "grad_norm": 2.742067813873291, + "learning_rate": 9.401539878270545e-06, + "loss": 1.596, + "step": 280 + }, + { + "epoch": 0.5208333333333334, + "grad_norm": 2.9180655479431152, + "learning_rate": 9.249676511588e-06, + "loss": 1.5923, + "step": 300 + }, + { + "epoch": 0.5555555555555556, + "grad_norm": 2.760483503341675, + "learning_rate": 9.082186599639429e-06, + "loss": 1.5733, + "step": 320 + }, + { + "epoch": 0.5902777777777778, + "grad_norm": 2.5661356449127197, + "learning_rate": 8.899686021935554e-06, + "loss": 1.5574, + "step": 340 + }, + { + "epoch": 0.625, + "grad_norm": 2.650604248046875, + "learning_rate": 8.702845853917242e-06, + "loss": 1.5331, + "step": 360 + }, + { + "epoch": 0.6597222222222222, + "grad_norm": 2.5134191513061523, + "learning_rate": 8.492389899334572e-06, + "loss": 1.5116, + "step": 380 + }, + { + "epoch": 0.6944444444444444, + "grad_norm": 2.971090316772461, + "learning_rate": 8.269092028737885e-06, + "loss": 1.4993, + "step": 400 + } + ], + "logging_steps": 20, + "max_steps": 1152, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 50, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 7.790398216903066e+16, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/checkpoint-400/training_args.bin b/checkpoint-400/training_args.bin new file mode 100644 index 0000000..a638377 --- /dev/null +++ b/checkpoint-400/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6999d99b0ec3831abd58bd6fec26bef4e74e884d0058076eed8c5f0b91586567 +size 5969 diff --git a/checkpoint-450/README.md b/checkpoint-450/README.md new file mode 100644 index 0000000..bacf84a --- /dev/null +++ b/checkpoint-450/README.md @@ -0,0 +1,207 @@ +--- +base_model: biaofu-xmu/EAST-8B +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:biaofu-xmu/EAST-8B +- lora +- transformers +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/checkpoint-450/adapter_config.json b/checkpoint-450/adapter_config.json new file mode 100644 index 0000000..ca7008b --- /dev/null +++ b/checkpoint-450/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "biaofu-xmu/EAST-8B", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.05, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "v_proj", + "q_proj", + "down_proj", + "up_proj", + "gate_proj", + "o_proj", + "k_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/checkpoint-450/adapter_model.safetensors b/checkpoint-450/adapter_model.safetensors new file mode 100644 index 0000000..ab8f2bd --- /dev/null +++ b/checkpoint-450/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:da771f6ac076cc97dafc07380a20054b70da831e640deed7eda4403f6399d91c +size 671149168 diff --git a/checkpoint-450/optimizer.pt b/checkpoint-450/optimizer.pt new file mode 100644 index 0000000..3708c46 --- /dev/null +++ b/checkpoint-450/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fae346fe8a6e90c38e92b8062d5c353641e7acd243a81c696652d66e97067b3b +size 1342562339 diff --git a/checkpoint-450/rng_state.pth b/checkpoint-450/rng_state.pth new file mode 100644 index 0000000..6892371 --- /dev/null +++ b/checkpoint-450/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dcb58e88018052730dc2346279ee7548ba3296cd5ce86d00e70658ddd6873fe7 +size 14645 diff --git a/checkpoint-450/scheduler.pt b/checkpoint-450/scheduler.pt new file mode 100644 index 0000000..4b1258a --- /dev/null +++ b/checkpoint-450/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:384bc6e7c7dc78761d777acc2ec342f72b87fe7d2483bfdf597b557007de8a7d +size 1465 diff --git a/checkpoint-450/trainer_state.json b/checkpoint-450/trainer_state.json new file mode 100644 index 0000000..1fcca26 --- /dev/null +++ b/checkpoint-450/trainer_state.json @@ -0,0 +1,188 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.78125, + "eval_steps": 500, + "global_step": 450, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.034722222222222224, + "grad_norm": 2.548311471939087, + "learning_rate": 1.6379310344827587e-06, + "loss": 2.4451, + "step": 20 + }, + { + "epoch": 0.06944444444444445, + "grad_norm": 1.93834388256073, + "learning_rate": 3.362068965517242e-06, + "loss": 2.269, + "step": 40 + }, + { + "epoch": 0.10416666666666667, + "grad_norm": 1.7273279428482056, + "learning_rate": 5.086206896551724e-06, + "loss": 2.1237, + "step": 60 + }, + { + "epoch": 0.1388888888888889, + "grad_norm": 1.6319563388824463, + "learning_rate": 6.810344827586207e-06, + "loss": 1.969, + "step": 80 + }, + { + "epoch": 0.1736111111111111, + "grad_norm": 2.0262465476989746, + "learning_rate": 8.53448275862069e-06, + "loss": 1.8517, + "step": 100 + }, + { + "epoch": 0.20833333333333334, + "grad_norm": 2.1673457622528076, + "learning_rate": 9.999793100349294e-06, + "loss": 1.851, + "step": 120 + }, + { + "epoch": 0.24305555555555555, + "grad_norm": 2.474327802658081, + "learning_rate": 9.987843743451796e-06, + "loss": 1.7448, + "step": 140 + }, + { + "epoch": 0.2777777777777778, + "grad_norm": 2.0316762924194336, + "learning_rate": 9.957553516178782e-06, + "loss": 1.6721, + "step": 160 + }, + { + "epoch": 0.3125, + "grad_norm": 2.3124542236328125, + "learning_rate": 9.909033799150947e-06, + "loss": 1.7265, + "step": 180 + }, + { + "epoch": 0.3472222222222222, + "grad_norm": 2.2682251930236816, + "learning_rate": 9.842463004902127e-06, + "loss": 1.6325, + "step": 200 + }, + { + "epoch": 0.3819444444444444, + "grad_norm": 2.4642767906188965, + "learning_rate": 9.758085921836076e-06, + "loss": 1.6193, + "step": 220 + }, + { + "epoch": 0.4166666666666667, + "grad_norm": 2.4372549057006836, + "learning_rate": 9.656212814111567e-06, + "loss": 1.6297, + "step": 240 + }, + { + "epoch": 0.4513888888888889, + "grad_norm": 2.7898528575897217, + "learning_rate": 9.53721828076571e-06, + "loss": 1.598, + "step": 260 + }, + { + "epoch": 0.4861111111111111, + "grad_norm": 2.742067813873291, + "learning_rate": 9.401539878270545e-06, + "loss": 1.596, + "step": 280 + }, + { + "epoch": 0.5208333333333334, + "grad_norm": 2.9180655479431152, + "learning_rate": 9.249676511588e-06, + "loss": 1.5923, + "step": 300 + }, + { + "epoch": 0.5555555555555556, + "grad_norm": 2.760483503341675, + "learning_rate": 9.082186599639429e-06, + "loss": 1.5733, + "step": 320 + }, + { + "epoch": 0.5902777777777778, + "grad_norm": 2.5661356449127197, + "learning_rate": 8.899686021935554e-06, + "loss": 1.5574, + "step": 340 + }, + { + "epoch": 0.625, + "grad_norm": 2.650604248046875, + "learning_rate": 8.702845853917242e-06, + "loss": 1.5331, + "step": 360 + }, + { + "epoch": 0.6597222222222222, + "grad_norm": 2.5134191513061523, + "learning_rate": 8.492389899334572e-06, + "loss": 1.5116, + "step": 380 + }, + { + "epoch": 0.6944444444444444, + "grad_norm": 2.971090316772461, + "learning_rate": 8.269092028737885e-06, + "loss": 1.4993, + "step": 400 + }, + { + "epoch": 0.7291666666666666, + "grad_norm": 3.0398788452148438, + "learning_rate": 8.033773333867498e-06, + "loss": 1.4751, + "step": 420 + }, + { + "epoch": 0.7638888888888888, + "grad_norm": 2.7814712524414062, + "learning_rate": 7.78729910840572e-06, + "loss": 1.4688, + "step": 440 + } + ], + "logging_steps": 20, + "max_steps": 1152, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 50, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 8.766766900681114e+16, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/checkpoint-450/training_args.bin b/checkpoint-450/training_args.bin new file mode 100644 index 0000000..a638377 --- /dev/null +++ b/checkpoint-450/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6999d99b0ec3831abd58bd6fec26bef4e74e884d0058076eed8c5f0b91586567 +size 5969 diff --git a/checkpoint-50/README.md b/checkpoint-50/README.md new file mode 100644 index 0000000..bacf84a --- /dev/null +++ b/checkpoint-50/README.md @@ -0,0 +1,207 @@ +--- +base_model: biaofu-xmu/EAST-8B +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:biaofu-xmu/EAST-8B +- lora +- transformers +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/checkpoint-50/adapter_config.json b/checkpoint-50/adapter_config.json new file mode 100644 index 0000000..ca7008b --- /dev/null +++ b/checkpoint-50/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "biaofu-xmu/EAST-8B", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.05, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "v_proj", + "q_proj", + "down_proj", + "up_proj", + "gate_proj", + "o_proj", + "k_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/checkpoint-50/adapter_model.safetensors b/checkpoint-50/adapter_model.safetensors new file mode 100644 index 0000000..62e80c8 --- /dev/null +++ b/checkpoint-50/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d3861057988daadd8d9707033639290cf2ee104e0f336371e186a5218f24ac79 +size 671149168 diff --git a/checkpoint-50/optimizer.pt b/checkpoint-50/optimizer.pt new file mode 100644 index 0000000..c545d05 --- /dev/null +++ b/checkpoint-50/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bff6716ce36858fdd4f6a01435e8dd69cf8f9bdc33bf2386df08c40698586144 +size 1342562339 diff --git a/checkpoint-50/rng_state.pth b/checkpoint-50/rng_state.pth new file mode 100644 index 0000000..0598dc5 --- /dev/null +++ b/checkpoint-50/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bf4dc3871708c91e2908d757c1144e0fae2942c278c32eb3ec149638945f1452 +size 14645 diff --git a/checkpoint-50/scheduler.pt b/checkpoint-50/scheduler.pt new file mode 100644 index 0000000..1365fdd --- /dev/null +++ b/checkpoint-50/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:429753af91844c8e723460e500b16300ef22bee5be6ffdf96a0f5e7a4101ca3a +size 1465 diff --git a/checkpoint-50/trainer_state.json b/checkpoint-50/trainer_state.json new file mode 100644 index 0000000..1d5669c --- /dev/null +++ b/checkpoint-50/trainer_state.json @@ -0,0 +1,48 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.08680555555555555, + "eval_steps": 500, + "global_step": 50, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.034722222222222224, + "grad_norm": 2.548311471939087, + "learning_rate": 1.6379310344827587e-06, + "loss": 2.4451, + "step": 20 + }, + { + "epoch": 0.06944444444444445, + "grad_norm": 1.93834388256073, + "learning_rate": 3.362068965517242e-06, + "loss": 2.269, + "step": 40 + } + ], + "logging_steps": 20, + "max_steps": 1152, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 50, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 9685790958256128.0, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/checkpoint-50/training_args.bin b/checkpoint-50/training_args.bin new file mode 100644 index 0000000..a638377 --- /dev/null +++ b/checkpoint-50/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6999d99b0ec3831abd58bd6fec26bef4e74e884d0058076eed8c5f0b91586567 +size 5969 diff --git a/checkpoint-500/README.md b/checkpoint-500/README.md new file mode 100644 index 0000000..bacf84a --- /dev/null +++ b/checkpoint-500/README.md @@ -0,0 +1,207 @@ +--- +base_model: biaofu-xmu/EAST-8B +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:biaofu-xmu/EAST-8B +- lora +- transformers +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/checkpoint-500/adapter_config.json b/checkpoint-500/adapter_config.json new file mode 100644 index 0000000..ca7008b --- /dev/null +++ b/checkpoint-500/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "biaofu-xmu/EAST-8B", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.05, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "v_proj", + "q_proj", + "down_proj", + "up_proj", + "gate_proj", + "o_proj", + "k_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/checkpoint-500/adapter_model.safetensors b/checkpoint-500/adapter_model.safetensors new file mode 100644 index 0000000..29ec57e --- /dev/null +++ b/checkpoint-500/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a8b2ac6ef2f9dbfe401ba7d9659818906bd7aadae1413ee2c7d4761bee804956 +size 671149168 diff --git a/checkpoint-500/optimizer.pt b/checkpoint-500/optimizer.pt new file mode 100644 index 0000000..1e9f8c3 --- /dev/null +++ b/checkpoint-500/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8b8b81846fb06a728313f2f07d60e9ecb53f70c0abd760ca179e88b3948b129a +size 1342562339 diff --git a/checkpoint-500/rng_state.pth b/checkpoint-500/rng_state.pth new file mode 100644 index 0000000..008448f --- /dev/null +++ b/checkpoint-500/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:737495c6c206f55c10733bcc893a7659281c4ca7f432a137217b662b47ff109d +size 14645 diff --git a/checkpoint-500/scheduler.pt b/checkpoint-500/scheduler.pt new file mode 100644 index 0000000..f14c49a --- /dev/null +++ b/checkpoint-500/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:56d00b3dceef9b6f5ecba6ca112aac49209cb479316e5bd5c0651dae9c1ba9b3 +size 1465 diff --git a/checkpoint-500/trainer_state.json b/checkpoint-500/trainer_state.json new file mode 100644 index 0000000..af73c49 --- /dev/null +++ b/checkpoint-500/trainer_state.json @@ -0,0 +1,209 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.8680555555555556, + "eval_steps": 500, + "global_step": 500, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.034722222222222224, + "grad_norm": 2.548311471939087, + "learning_rate": 1.6379310344827587e-06, + "loss": 2.4451, + "step": 20 + }, + { + "epoch": 0.06944444444444445, + "grad_norm": 1.93834388256073, + "learning_rate": 3.362068965517242e-06, + "loss": 2.269, + "step": 40 + }, + { + "epoch": 0.10416666666666667, + "grad_norm": 1.7273279428482056, + "learning_rate": 5.086206896551724e-06, + "loss": 2.1237, + "step": 60 + }, + { + "epoch": 0.1388888888888889, + "grad_norm": 1.6319563388824463, + "learning_rate": 6.810344827586207e-06, + "loss": 1.969, + "step": 80 + }, + { + "epoch": 0.1736111111111111, + "grad_norm": 2.0262465476989746, + "learning_rate": 8.53448275862069e-06, + "loss": 1.8517, + "step": 100 + }, + { + "epoch": 0.20833333333333334, + "grad_norm": 2.1673457622528076, + "learning_rate": 9.999793100349294e-06, + "loss": 1.851, + "step": 120 + }, + { + "epoch": 0.24305555555555555, + "grad_norm": 2.474327802658081, + "learning_rate": 9.987843743451796e-06, + "loss": 1.7448, + "step": 140 + }, + { + "epoch": 0.2777777777777778, + "grad_norm": 2.0316762924194336, + "learning_rate": 9.957553516178782e-06, + "loss": 1.6721, + "step": 160 + }, + { + "epoch": 0.3125, + "grad_norm": 2.3124542236328125, + "learning_rate": 9.909033799150947e-06, + "loss": 1.7265, + "step": 180 + }, + { + "epoch": 0.3472222222222222, + "grad_norm": 2.2682251930236816, + "learning_rate": 9.842463004902127e-06, + "loss": 1.6325, + "step": 200 + }, + { + "epoch": 0.3819444444444444, + "grad_norm": 2.4642767906188965, + "learning_rate": 9.758085921836076e-06, + "loss": 1.6193, + "step": 220 + }, + { + "epoch": 0.4166666666666667, + "grad_norm": 2.4372549057006836, + "learning_rate": 9.656212814111567e-06, + "loss": 1.6297, + "step": 240 + }, + { + "epoch": 0.4513888888888889, + "grad_norm": 2.7898528575897217, + "learning_rate": 9.53721828076571e-06, + "loss": 1.598, + "step": 260 + }, + { + "epoch": 0.4861111111111111, + "grad_norm": 2.742067813873291, + "learning_rate": 9.401539878270545e-06, + "loss": 1.596, + "step": 280 + }, + { + "epoch": 0.5208333333333334, + "grad_norm": 2.9180655479431152, + "learning_rate": 9.249676511588e-06, + "loss": 1.5923, + "step": 300 + }, + { + "epoch": 0.5555555555555556, + "grad_norm": 2.760483503341675, + "learning_rate": 9.082186599639429e-06, + "loss": 1.5733, + "step": 320 + }, + { + "epoch": 0.5902777777777778, + "grad_norm": 2.5661356449127197, + "learning_rate": 8.899686021935554e-06, + "loss": 1.5574, + "step": 340 + }, + { + "epoch": 0.625, + "grad_norm": 2.650604248046875, + "learning_rate": 8.702845853917242e-06, + "loss": 1.5331, + "step": 360 + }, + { + "epoch": 0.6597222222222222, + "grad_norm": 2.5134191513061523, + "learning_rate": 8.492389899334572e-06, + "loss": 1.5116, + "step": 380 + }, + { + "epoch": 0.6944444444444444, + "grad_norm": 2.971090316772461, + "learning_rate": 8.269092028737885e-06, + "loss": 1.4993, + "step": 400 + }, + { + "epoch": 0.7291666666666666, + "grad_norm": 3.0398788452148438, + "learning_rate": 8.033773333867498e-06, + "loss": 1.4751, + "step": 420 + }, + { + "epoch": 0.7638888888888888, + "grad_norm": 2.7814712524414062, + "learning_rate": 7.78729910840572e-06, + "loss": 1.4688, + "step": 440 + }, + { + "epoch": 0.7986111111111112, + "grad_norm": 2.9422523975372314, + "learning_rate": 7.530575666193283e-06, + "loss": 1.4645, + "step": 460 + }, + { + "epoch": 0.8333333333333334, + "grad_norm": 3.5797832012176514, + "learning_rate": 7.26454700860997e-06, + "loss": 1.4473, + "step": 480 + }, + { + "epoch": 0.8680555555555556, + "grad_norm": 3.557054281234741, + "learning_rate": 6.990191353373876e-06, + "loss": 1.4428, + "step": 500 + } + ], + "logging_steps": 20, + "max_steps": 1152, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 50, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 9.77228669020078e+16, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/checkpoint-500/training_args.bin b/checkpoint-500/training_args.bin new file mode 100644 index 0000000..a638377 --- /dev/null +++ b/checkpoint-500/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6999d99b0ec3831abd58bd6fec26bef4e74e884d0058076eed8c5f0b91586567 +size 5969 diff --git a/checkpoint-550/README.md b/checkpoint-550/README.md new file mode 100644 index 0000000..bacf84a --- /dev/null +++ b/checkpoint-550/README.md @@ -0,0 +1,207 @@ +--- +base_model: biaofu-xmu/EAST-8B +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:biaofu-xmu/EAST-8B +- lora +- transformers +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/checkpoint-550/adapter_config.json b/checkpoint-550/adapter_config.json new file mode 100644 index 0000000..ca7008b --- /dev/null +++ b/checkpoint-550/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "biaofu-xmu/EAST-8B", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.05, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "v_proj", + "q_proj", + "down_proj", + "up_proj", + "gate_proj", + "o_proj", + "k_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/checkpoint-550/adapter_model.safetensors b/checkpoint-550/adapter_model.safetensors new file mode 100644 index 0000000..285a477 --- /dev/null +++ b/checkpoint-550/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:50e3fd30afb6f19e19889d1fffd5f2c0a1f2172fa1a0ea66b0072ddca1fe7619 +size 671149168 diff --git a/checkpoint-550/optimizer.pt b/checkpoint-550/optimizer.pt new file mode 100644 index 0000000..ad4dc0b --- /dev/null +++ b/checkpoint-550/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6e2f6ff6cee20cba20cabcb3a1685cab8366818092891c3f4cddae7dad16993b +size 1342562339 diff --git a/checkpoint-550/rng_state.pth b/checkpoint-550/rng_state.pth new file mode 100644 index 0000000..bc99233 --- /dev/null +++ b/checkpoint-550/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:346027f9565fcf0cce1e6e76f716061e1e155dde38cc151159e5bcc1e830280b +size 14645 diff --git a/checkpoint-550/scheduler.pt b/checkpoint-550/scheduler.pt new file mode 100644 index 0000000..a344f2f --- /dev/null +++ b/checkpoint-550/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:95f0ec44226045e10b9e9885bb6dcc32b6fb20b4e398697125d5903dc06a3dc3 +size 1465 diff --git a/checkpoint-550/trainer_state.json b/checkpoint-550/trainer_state.json new file mode 100644 index 0000000..c47a259 --- /dev/null +++ b/checkpoint-550/trainer_state.json @@ -0,0 +1,223 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.9548611111111112, + "eval_steps": 500, + "global_step": 550, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.034722222222222224, + "grad_norm": 2.548311471939087, + "learning_rate": 1.6379310344827587e-06, + "loss": 2.4451, + "step": 20 + }, + { + "epoch": 0.06944444444444445, + "grad_norm": 1.93834388256073, + "learning_rate": 3.362068965517242e-06, + "loss": 2.269, + "step": 40 + }, + { + "epoch": 0.10416666666666667, + "grad_norm": 1.7273279428482056, + "learning_rate": 5.086206896551724e-06, + "loss": 2.1237, + "step": 60 + }, + { + "epoch": 0.1388888888888889, + "grad_norm": 1.6319563388824463, + "learning_rate": 6.810344827586207e-06, + "loss": 1.969, + "step": 80 + }, + { + "epoch": 0.1736111111111111, + "grad_norm": 2.0262465476989746, + "learning_rate": 8.53448275862069e-06, + "loss": 1.8517, + "step": 100 + }, + { + "epoch": 0.20833333333333334, + "grad_norm": 2.1673457622528076, + "learning_rate": 9.999793100349294e-06, + "loss": 1.851, + "step": 120 + }, + { + "epoch": 0.24305555555555555, + "grad_norm": 2.474327802658081, + "learning_rate": 9.987843743451796e-06, + "loss": 1.7448, + "step": 140 + }, + { + "epoch": 0.2777777777777778, + "grad_norm": 2.0316762924194336, + "learning_rate": 9.957553516178782e-06, + "loss": 1.6721, + "step": 160 + }, + { + "epoch": 0.3125, + "grad_norm": 2.3124542236328125, + "learning_rate": 9.909033799150947e-06, + "loss": 1.7265, + "step": 180 + }, + { + "epoch": 0.3472222222222222, + "grad_norm": 2.2682251930236816, + "learning_rate": 9.842463004902127e-06, + "loss": 1.6325, + "step": 200 + }, + { + "epoch": 0.3819444444444444, + "grad_norm": 2.4642767906188965, + "learning_rate": 9.758085921836076e-06, + "loss": 1.6193, + "step": 220 + }, + { + "epoch": 0.4166666666666667, + "grad_norm": 2.4372549057006836, + "learning_rate": 9.656212814111567e-06, + "loss": 1.6297, + "step": 240 + }, + { + "epoch": 0.4513888888888889, + "grad_norm": 2.7898528575897217, + "learning_rate": 9.53721828076571e-06, + "loss": 1.598, + "step": 260 + }, + { + "epoch": 0.4861111111111111, + "grad_norm": 2.742067813873291, + "learning_rate": 9.401539878270545e-06, + "loss": 1.596, + "step": 280 + }, + { + "epoch": 0.5208333333333334, + "grad_norm": 2.9180655479431152, + "learning_rate": 9.249676511588e-06, + "loss": 1.5923, + "step": 300 + }, + { + "epoch": 0.5555555555555556, + "grad_norm": 2.760483503341675, + "learning_rate": 9.082186599639429e-06, + "loss": 1.5733, + "step": 320 + }, + { + "epoch": 0.5902777777777778, + "grad_norm": 2.5661356449127197, + "learning_rate": 8.899686021935554e-06, + "loss": 1.5574, + "step": 340 + }, + { + "epoch": 0.625, + "grad_norm": 2.650604248046875, + "learning_rate": 8.702845853917242e-06, + "loss": 1.5331, + "step": 360 + }, + { + "epoch": 0.6597222222222222, + "grad_norm": 2.5134191513061523, + "learning_rate": 8.492389899334572e-06, + "loss": 1.5116, + "step": 380 + }, + { + "epoch": 0.6944444444444444, + "grad_norm": 2.971090316772461, + "learning_rate": 8.269092028737885e-06, + "loss": 1.4993, + "step": 400 + }, + { + "epoch": 0.7291666666666666, + "grad_norm": 3.0398788452148438, + "learning_rate": 8.033773333867498e-06, + "loss": 1.4751, + "step": 420 + }, + { + "epoch": 0.7638888888888888, + "grad_norm": 2.7814712524414062, + "learning_rate": 7.78729910840572e-06, + "loss": 1.4688, + "step": 440 + }, + { + "epoch": 0.7986111111111112, + "grad_norm": 2.9422523975372314, + "learning_rate": 7.530575666193283e-06, + "loss": 1.4645, + "step": 460 + }, + { + "epoch": 0.8333333333333334, + "grad_norm": 3.5797832012176514, + "learning_rate": 7.26454700860997e-06, + "loss": 1.4473, + "step": 480 + }, + { + "epoch": 0.8680555555555556, + "grad_norm": 3.557054281234741, + "learning_rate": 6.990191353373876e-06, + "loss": 1.4428, + "step": 500 + }, + { + "epoch": 0.9027777777777778, + "grad_norm": 3.7312850952148438, + "learning_rate": 6.708517537523264e-06, + "loss": 1.4014, + "step": 520 + }, + { + "epoch": 0.9375, + "grad_norm": 3.2819321155548096, + "learning_rate": 6.420561307807713e-06, + "loss": 1.4214, + "step": 540 + } + ], + "logging_steps": 20, + "max_steps": 1152, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 50, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.0762577190744883e+17, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/checkpoint-550/training_args.bin b/checkpoint-550/training_args.bin new file mode 100644 index 0000000..a638377 --- /dev/null +++ b/checkpoint-550/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6999d99b0ec3831abd58bd6fec26bef4e74e884d0058076eed8c5f0b91586567 +size 5969 diff --git a/checkpoint-600/README.md b/checkpoint-600/README.md new file mode 100644 index 0000000..bacf84a --- /dev/null +++ b/checkpoint-600/README.md @@ -0,0 +1,207 @@ +--- +base_model: biaofu-xmu/EAST-8B +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:biaofu-xmu/EAST-8B +- lora +- transformers +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/checkpoint-600/adapter_config.json b/checkpoint-600/adapter_config.json new file mode 100644 index 0000000..ca7008b --- /dev/null +++ b/checkpoint-600/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "biaofu-xmu/EAST-8B", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.05, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "v_proj", + "q_proj", + "down_proj", + "up_proj", + "gate_proj", + "o_proj", + "k_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/checkpoint-600/adapter_model.safetensors b/checkpoint-600/adapter_model.safetensors new file mode 100644 index 0000000..7c1aaa2 --- /dev/null +++ b/checkpoint-600/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4eb34ba3a0cd829f80f53fa5236f8fcc8a63d73f3a5683b4ce4f55a43de922f5 +size 671149168 diff --git a/checkpoint-600/optimizer.pt b/checkpoint-600/optimizer.pt new file mode 100644 index 0000000..3656b57 --- /dev/null +++ b/checkpoint-600/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9ed45da502944263a8a34451f54b96c9f092d7b6ff9a4b663bf30e25955ce805 +size 1342562339 diff --git a/checkpoint-600/rng_state.pth b/checkpoint-600/rng_state.pth new file mode 100644 index 0000000..81ed93c --- /dev/null +++ b/checkpoint-600/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f53642b5b5977f238cd5a4c4f189e9077c4f1b37919724fef55e742cf164dd93 +size 14645 diff --git a/checkpoint-600/scheduler.pt b/checkpoint-600/scheduler.pt new file mode 100644 index 0000000..9d84cb1 --- /dev/null +++ b/checkpoint-600/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:97e41312b87b4204123e46e88d3bc7c857cfb3fbe531078e169cc10e0d939469 +size 1465 diff --git a/checkpoint-600/trainer_state.json b/checkpoint-600/trainer_state.json new file mode 100644 index 0000000..02c85a8 --- /dev/null +++ b/checkpoint-600/trainer_state.json @@ -0,0 +1,244 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.0416666666666667, + "eval_steps": 500, + "global_step": 600, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.034722222222222224, + "grad_norm": 2.548311471939087, + "learning_rate": 1.6379310344827587e-06, + "loss": 2.4451, + "step": 20 + }, + { + "epoch": 0.06944444444444445, + "grad_norm": 1.93834388256073, + "learning_rate": 3.362068965517242e-06, + "loss": 2.269, + "step": 40 + }, + { + "epoch": 0.10416666666666667, + "grad_norm": 1.7273279428482056, + "learning_rate": 5.086206896551724e-06, + "loss": 2.1237, + "step": 60 + }, + { + "epoch": 0.1388888888888889, + "grad_norm": 1.6319563388824463, + "learning_rate": 6.810344827586207e-06, + "loss": 1.969, + "step": 80 + }, + { + "epoch": 0.1736111111111111, + "grad_norm": 2.0262465476989746, + "learning_rate": 8.53448275862069e-06, + "loss": 1.8517, + "step": 100 + }, + { + "epoch": 0.20833333333333334, + "grad_norm": 2.1673457622528076, + "learning_rate": 9.999793100349294e-06, + "loss": 1.851, + "step": 120 + }, + { + "epoch": 0.24305555555555555, + "grad_norm": 2.474327802658081, + "learning_rate": 9.987843743451796e-06, + "loss": 1.7448, + "step": 140 + }, + { + "epoch": 0.2777777777777778, + "grad_norm": 2.0316762924194336, + "learning_rate": 9.957553516178782e-06, + "loss": 1.6721, + "step": 160 + }, + { + "epoch": 0.3125, + "grad_norm": 2.3124542236328125, + "learning_rate": 9.909033799150947e-06, + "loss": 1.7265, + "step": 180 + }, + { + "epoch": 0.3472222222222222, + "grad_norm": 2.2682251930236816, + "learning_rate": 9.842463004902127e-06, + "loss": 1.6325, + "step": 200 + }, + { + "epoch": 0.3819444444444444, + "grad_norm": 2.4642767906188965, + "learning_rate": 9.758085921836076e-06, + "loss": 1.6193, + "step": 220 + }, + { + "epoch": 0.4166666666666667, + "grad_norm": 2.4372549057006836, + "learning_rate": 9.656212814111567e-06, + "loss": 1.6297, + "step": 240 + }, + { + "epoch": 0.4513888888888889, + "grad_norm": 2.7898528575897217, + "learning_rate": 9.53721828076571e-06, + "loss": 1.598, + "step": 260 + }, + { + "epoch": 0.4861111111111111, + "grad_norm": 2.742067813873291, + "learning_rate": 9.401539878270545e-06, + "loss": 1.596, + "step": 280 + }, + { + "epoch": 0.5208333333333334, + "grad_norm": 2.9180655479431152, + "learning_rate": 9.249676511588e-06, + "loss": 1.5923, + "step": 300 + }, + { + "epoch": 0.5555555555555556, + "grad_norm": 2.760483503341675, + "learning_rate": 9.082186599639429e-06, + "loss": 1.5733, + "step": 320 + }, + { + "epoch": 0.5902777777777778, + "grad_norm": 2.5661356449127197, + "learning_rate": 8.899686021935554e-06, + "loss": 1.5574, + "step": 340 + }, + { + "epoch": 0.625, + "grad_norm": 2.650604248046875, + "learning_rate": 8.702845853917242e-06, + "loss": 1.5331, + "step": 360 + }, + { + "epoch": 0.6597222222222222, + "grad_norm": 2.5134191513061523, + "learning_rate": 8.492389899334572e-06, + "loss": 1.5116, + "step": 380 + }, + { + "epoch": 0.6944444444444444, + "grad_norm": 2.971090316772461, + "learning_rate": 8.269092028737885e-06, + "loss": 1.4993, + "step": 400 + }, + { + "epoch": 0.7291666666666666, + "grad_norm": 3.0398788452148438, + "learning_rate": 8.033773333867498e-06, + "loss": 1.4751, + "step": 420 + }, + { + "epoch": 0.7638888888888888, + "grad_norm": 2.7814712524414062, + "learning_rate": 7.78729910840572e-06, + "loss": 1.4688, + "step": 440 + }, + { + "epoch": 0.7986111111111112, + "grad_norm": 2.9422523975372314, + "learning_rate": 7.530575666193283e-06, + "loss": 1.4645, + "step": 460 + }, + { + "epoch": 0.8333333333333334, + "grad_norm": 3.5797832012176514, + "learning_rate": 7.26454700860997e-06, + "loss": 1.4473, + "step": 480 + }, + { + "epoch": 0.8680555555555556, + "grad_norm": 3.557054281234741, + "learning_rate": 6.990191353373876e-06, + "loss": 1.4428, + "step": 500 + }, + { + "epoch": 0.9027777777777778, + "grad_norm": 3.7312850952148438, + "learning_rate": 6.708517537523264e-06, + "loss": 1.4014, + "step": 520 + }, + { + "epoch": 0.9375, + "grad_norm": 3.2819321155548096, + "learning_rate": 6.420561307807713e-06, + "loss": 1.4214, + "step": 540 + }, + { + "epoch": 0.9722222222222222, + "grad_norm": 2.9412100315093994, + "learning_rate": 6.12738151212918e-06, + "loss": 1.4367, + "step": 560 + }, + { + "epoch": 1.0069444444444444, + "grad_norm": 4.002957820892334, + "learning_rate": 5.830056206037482e-06, + "loss": 1.4019, + "step": 580 + }, + { + "epoch": 1.0416666666666667, + "grad_norm": 3.2070395946502686, + "learning_rate": 5.529678688597081e-06, + "loss": 1.3294, + "step": 600 + } + ], + "logging_steps": 20, + "max_steps": 1152, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 50, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.1731635079210598e+17, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/checkpoint-600/training_args.bin b/checkpoint-600/training_args.bin new file mode 100644 index 0000000..a638377 --- /dev/null +++ b/checkpoint-600/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6999d99b0ec3831abd58bd6fec26bef4e74e884d0058076eed8c5f0b91586567 +size 5969 diff --git a/checkpoint-650/README.md b/checkpoint-650/README.md new file mode 100644 index 0000000..bacf84a --- /dev/null +++ b/checkpoint-650/README.md @@ -0,0 +1,207 @@ +--- +base_model: biaofu-xmu/EAST-8B +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:biaofu-xmu/EAST-8B +- lora +- transformers +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/checkpoint-650/adapter_config.json b/checkpoint-650/adapter_config.json new file mode 100644 index 0000000..ca7008b --- /dev/null +++ b/checkpoint-650/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "biaofu-xmu/EAST-8B", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.05, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "v_proj", + "q_proj", + "down_proj", + "up_proj", + "gate_proj", + "o_proj", + "k_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/checkpoint-650/adapter_model.safetensors b/checkpoint-650/adapter_model.safetensors new file mode 100644 index 0000000..099bd4e --- /dev/null +++ b/checkpoint-650/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e801e7792f4abc56a0d139b4cbd14c8ba6fbf46225bc7ee2877e1d7874d47813 +size 671149168 diff --git a/checkpoint-650/optimizer.pt b/checkpoint-650/optimizer.pt new file mode 100644 index 0000000..4d95502 --- /dev/null +++ b/checkpoint-650/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d676d4f86b505799a5a2bf151b4a91c3327e92523c4a40a93e698813bfcaa494 +size 1342562339 diff --git a/checkpoint-650/rng_state.pth b/checkpoint-650/rng_state.pth new file mode 100644 index 0000000..9627e70 --- /dev/null +++ b/checkpoint-650/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d5f18bffaef45c6844c6cc1b1d7f06d559b03ae3a2d5c0c40ba4dbb4d5f4cd32 +size 14645 diff --git a/checkpoint-650/scheduler.pt b/checkpoint-650/scheduler.pt new file mode 100644 index 0000000..3d8d688 --- /dev/null +++ b/checkpoint-650/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:701963ab8d7d1849c052af5fce88023fdda6f713e5072a25a5a4aa822ce8b58b +size 1465 diff --git a/checkpoint-650/trainer_state.json b/checkpoint-650/trainer_state.json new file mode 100644 index 0000000..f2bc5bb --- /dev/null +++ b/checkpoint-650/trainer_state.json @@ -0,0 +1,258 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.1284722222222223, + "eval_steps": 500, + "global_step": 650, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.034722222222222224, + "grad_norm": 2.548311471939087, + "learning_rate": 1.6379310344827587e-06, + "loss": 2.4451, + "step": 20 + }, + { + "epoch": 0.06944444444444445, + "grad_norm": 1.93834388256073, + "learning_rate": 3.362068965517242e-06, + "loss": 2.269, + "step": 40 + }, + { + "epoch": 0.10416666666666667, + "grad_norm": 1.7273279428482056, + "learning_rate": 5.086206896551724e-06, + "loss": 2.1237, + "step": 60 + }, + { + "epoch": 0.1388888888888889, + "grad_norm": 1.6319563388824463, + "learning_rate": 6.810344827586207e-06, + "loss": 1.969, + "step": 80 + }, + { + "epoch": 0.1736111111111111, + "grad_norm": 2.0262465476989746, + "learning_rate": 8.53448275862069e-06, + "loss": 1.8517, + "step": 100 + }, + { + "epoch": 0.20833333333333334, + "grad_norm": 2.1673457622528076, + "learning_rate": 9.999793100349294e-06, + "loss": 1.851, + "step": 120 + }, + { + "epoch": 0.24305555555555555, + "grad_norm": 2.474327802658081, + "learning_rate": 9.987843743451796e-06, + "loss": 1.7448, + "step": 140 + }, + { + "epoch": 0.2777777777777778, + "grad_norm": 2.0316762924194336, + "learning_rate": 9.957553516178782e-06, + "loss": 1.6721, + "step": 160 + }, + { + "epoch": 0.3125, + "grad_norm": 2.3124542236328125, + "learning_rate": 9.909033799150947e-06, + "loss": 1.7265, + "step": 180 + }, + { + "epoch": 0.3472222222222222, + "grad_norm": 2.2682251930236816, + "learning_rate": 9.842463004902127e-06, + "loss": 1.6325, + "step": 200 + }, + { + "epoch": 0.3819444444444444, + "grad_norm": 2.4642767906188965, + "learning_rate": 9.758085921836076e-06, + "loss": 1.6193, + "step": 220 + }, + { + "epoch": 0.4166666666666667, + "grad_norm": 2.4372549057006836, + "learning_rate": 9.656212814111567e-06, + "loss": 1.6297, + "step": 240 + }, + { + "epoch": 0.4513888888888889, + "grad_norm": 2.7898528575897217, + "learning_rate": 9.53721828076571e-06, + "loss": 1.598, + "step": 260 + }, + { + "epoch": 0.4861111111111111, + "grad_norm": 2.742067813873291, + "learning_rate": 9.401539878270545e-06, + "loss": 1.596, + "step": 280 + }, + { + "epoch": 0.5208333333333334, + "grad_norm": 2.9180655479431152, + "learning_rate": 9.249676511588e-06, + "loss": 1.5923, + "step": 300 + }, + { + "epoch": 0.5555555555555556, + "grad_norm": 2.760483503341675, + "learning_rate": 9.082186599639429e-06, + "loss": 1.5733, + "step": 320 + }, + { + "epoch": 0.5902777777777778, + "grad_norm": 2.5661356449127197, + "learning_rate": 8.899686021935554e-06, + "loss": 1.5574, + "step": 340 + }, + { + "epoch": 0.625, + "grad_norm": 2.650604248046875, + "learning_rate": 8.702845853917242e-06, + "loss": 1.5331, + "step": 360 + }, + { + "epoch": 0.6597222222222222, + "grad_norm": 2.5134191513061523, + "learning_rate": 8.492389899334572e-06, + "loss": 1.5116, + "step": 380 + }, + { + "epoch": 0.6944444444444444, + "grad_norm": 2.971090316772461, + "learning_rate": 8.269092028737885e-06, + "loss": 1.4993, + "step": 400 + }, + { + "epoch": 0.7291666666666666, + "grad_norm": 3.0398788452148438, + "learning_rate": 8.033773333867498e-06, + "loss": 1.4751, + "step": 420 + }, + { + "epoch": 0.7638888888888888, + "grad_norm": 2.7814712524414062, + "learning_rate": 7.78729910840572e-06, + "loss": 1.4688, + "step": 440 + }, + { + "epoch": 0.7986111111111112, + "grad_norm": 2.9422523975372314, + "learning_rate": 7.530575666193283e-06, + "loss": 1.4645, + "step": 460 + }, + { + "epoch": 0.8333333333333334, + "grad_norm": 3.5797832012176514, + "learning_rate": 7.26454700860997e-06, + "loss": 1.4473, + "step": 480 + }, + { + "epoch": 0.8680555555555556, + "grad_norm": 3.557054281234741, + "learning_rate": 6.990191353373876e-06, + "loss": 1.4428, + "step": 500 + }, + { + "epoch": 0.9027777777777778, + "grad_norm": 3.7312850952148438, + "learning_rate": 6.708517537523264e-06, + "loss": 1.4014, + "step": 520 + }, + { + "epoch": 0.9375, + "grad_norm": 3.2819321155548096, + "learning_rate": 6.420561307807713e-06, + "loss": 1.4214, + "step": 540 + }, + { + "epoch": 0.9722222222222222, + "grad_norm": 2.9412100315093994, + "learning_rate": 6.12738151212918e-06, + "loss": 1.4367, + "step": 560 + }, + { + "epoch": 1.0069444444444444, + "grad_norm": 4.002957820892334, + "learning_rate": 5.830056206037482e-06, + "loss": 1.4019, + "step": 580 + }, + { + "epoch": 1.0416666666666667, + "grad_norm": 3.2070395946502686, + "learning_rate": 5.529678688597081e-06, + "loss": 1.3294, + "step": 600 + }, + { + "epoch": 1.0763888888888888, + "grad_norm": 3.3583526611328125, + "learning_rate": 5.2273534822017105e-06, + "loss": 1.3302, + "step": 620 + }, + { + "epoch": 1.1111111111111112, + "grad_norm": 4.011720657348633, + "learning_rate": 4.924192271119554e-06, + "loss": 1.2973, + "step": 640 + } + ], + "logging_steps": 20, + "max_steps": 1152, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 50, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.2691724813225165e+17, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/checkpoint-650/training_args.bin b/checkpoint-650/training_args.bin new file mode 100644 index 0000000..a638377 --- /dev/null +++ b/checkpoint-650/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6999d99b0ec3831abd58bd6fec26bef4e74e884d0058076eed8c5f0b91586567 +size 5969 diff --git a/checkpoint-700/README.md b/checkpoint-700/README.md new file mode 100644 index 0000000..bacf84a --- /dev/null +++ b/checkpoint-700/README.md @@ -0,0 +1,207 @@ +--- +base_model: biaofu-xmu/EAST-8B +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:biaofu-xmu/EAST-8B +- lora +- transformers +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/checkpoint-700/adapter_config.json b/checkpoint-700/adapter_config.json new file mode 100644 index 0000000..ca7008b --- /dev/null +++ b/checkpoint-700/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "biaofu-xmu/EAST-8B", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.05, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "v_proj", + "q_proj", + "down_proj", + "up_proj", + "gate_proj", + "o_proj", + "k_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/checkpoint-700/adapter_model.safetensors b/checkpoint-700/adapter_model.safetensors new file mode 100644 index 0000000..f7fadcf --- /dev/null +++ b/checkpoint-700/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:da1b6867dafee1053223506f690fee9e5be3d04b0b16dfe8a4ccdcf2f41f8ab5 +size 671149168 diff --git a/checkpoint-700/optimizer.pt b/checkpoint-700/optimizer.pt new file mode 100644 index 0000000..a2743c6 --- /dev/null +++ b/checkpoint-700/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1486c0568a59803f45b9fa348a067bf81da275baf36afa193d779722bebbfe89 +size 1342562339 diff --git a/checkpoint-700/rng_state.pth b/checkpoint-700/rng_state.pth new file mode 100644 index 0000000..31723ab --- /dev/null +++ b/checkpoint-700/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4cb4438f2c86772536099b2a9c35cc9e0bc514c0097ba4a80c7a4df9bf95f2e1 +size 14645 diff --git a/checkpoint-700/scheduler.pt b/checkpoint-700/scheduler.pt new file mode 100644 index 0000000..0e67b96 --- /dev/null +++ b/checkpoint-700/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f96c27cd0f37d375f25ee4ddc217cab5525667adb393b37449a4bbaa0d7f3f00 +size 1465 diff --git a/checkpoint-700/trainer_state.json b/checkpoint-700/trainer_state.json new file mode 100644 index 0000000..312be83 --- /dev/null +++ b/checkpoint-700/trainer_state.json @@ -0,0 +1,279 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.2152777777777777, + "eval_steps": 500, + "global_step": 700, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.034722222222222224, + "grad_norm": 2.548311471939087, + "learning_rate": 1.6379310344827587e-06, + "loss": 2.4451, + "step": 20 + }, + { + "epoch": 0.06944444444444445, + "grad_norm": 1.93834388256073, + "learning_rate": 3.362068965517242e-06, + "loss": 2.269, + "step": 40 + }, + { + "epoch": 0.10416666666666667, + "grad_norm": 1.7273279428482056, + "learning_rate": 5.086206896551724e-06, + "loss": 2.1237, + "step": 60 + }, + { + "epoch": 0.1388888888888889, + "grad_norm": 1.6319563388824463, + "learning_rate": 6.810344827586207e-06, + "loss": 1.969, + "step": 80 + }, + { + "epoch": 0.1736111111111111, + "grad_norm": 2.0262465476989746, + "learning_rate": 8.53448275862069e-06, + "loss": 1.8517, + "step": 100 + }, + { + "epoch": 0.20833333333333334, + "grad_norm": 2.1673457622528076, + "learning_rate": 9.999793100349294e-06, + "loss": 1.851, + "step": 120 + }, + { + "epoch": 0.24305555555555555, + "grad_norm": 2.474327802658081, + "learning_rate": 9.987843743451796e-06, + "loss": 1.7448, + "step": 140 + }, + { + "epoch": 0.2777777777777778, + "grad_norm": 2.0316762924194336, + "learning_rate": 9.957553516178782e-06, + "loss": 1.6721, + "step": 160 + }, + { + "epoch": 0.3125, + "grad_norm": 2.3124542236328125, + "learning_rate": 9.909033799150947e-06, + "loss": 1.7265, + "step": 180 + }, + { + "epoch": 0.3472222222222222, + "grad_norm": 2.2682251930236816, + "learning_rate": 9.842463004902127e-06, + "loss": 1.6325, + "step": 200 + }, + { + "epoch": 0.3819444444444444, + "grad_norm": 2.4642767906188965, + "learning_rate": 9.758085921836076e-06, + "loss": 1.6193, + "step": 220 + }, + { + "epoch": 0.4166666666666667, + "grad_norm": 2.4372549057006836, + "learning_rate": 9.656212814111567e-06, + "loss": 1.6297, + "step": 240 + }, + { + "epoch": 0.4513888888888889, + "grad_norm": 2.7898528575897217, + "learning_rate": 9.53721828076571e-06, + "loss": 1.598, + "step": 260 + }, + { + "epoch": 0.4861111111111111, + "grad_norm": 2.742067813873291, + "learning_rate": 9.401539878270545e-06, + "loss": 1.596, + "step": 280 + }, + { + "epoch": 0.5208333333333334, + "grad_norm": 2.9180655479431152, + "learning_rate": 9.249676511588e-06, + "loss": 1.5923, + "step": 300 + }, + { + "epoch": 0.5555555555555556, + "grad_norm": 2.760483503341675, + "learning_rate": 9.082186599639429e-06, + "loss": 1.5733, + "step": 320 + }, + { + "epoch": 0.5902777777777778, + "grad_norm": 2.5661356449127197, + "learning_rate": 8.899686021935554e-06, + "loss": 1.5574, + "step": 340 + }, + { + "epoch": 0.625, + "grad_norm": 2.650604248046875, + "learning_rate": 8.702845853917242e-06, + "loss": 1.5331, + "step": 360 + }, + { + "epoch": 0.6597222222222222, + "grad_norm": 2.5134191513061523, + "learning_rate": 8.492389899334572e-06, + "loss": 1.5116, + "step": 380 + }, + { + "epoch": 0.6944444444444444, + "grad_norm": 2.971090316772461, + "learning_rate": 8.269092028737885e-06, + "loss": 1.4993, + "step": 400 + }, + { + "epoch": 0.7291666666666666, + "grad_norm": 3.0398788452148438, + "learning_rate": 8.033773333867498e-06, + "loss": 1.4751, + "step": 420 + }, + { + "epoch": 0.7638888888888888, + "grad_norm": 2.7814712524414062, + "learning_rate": 7.78729910840572e-06, + "loss": 1.4688, + "step": 440 + }, + { + "epoch": 0.7986111111111112, + "grad_norm": 2.9422523975372314, + "learning_rate": 7.530575666193283e-06, + "loss": 1.4645, + "step": 460 + }, + { + "epoch": 0.8333333333333334, + "grad_norm": 3.5797832012176514, + "learning_rate": 7.26454700860997e-06, + "loss": 1.4473, + "step": 480 + }, + { + "epoch": 0.8680555555555556, + "grad_norm": 3.557054281234741, + "learning_rate": 6.990191353373876e-06, + "loss": 1.4428, + "step": 500 + }, + { + "epoch": 0.9027777777777778, + "grad_norm": 3.7312850952148438, + "learning_rate": 6.708517537523264e-06, + "loss": 1.4014, + "step": 520 + }, + { + "epoch": 0.9375, + "grad_norm": 3.2819321155548096, + "learning_rate": 6.420561307807713e-06, + "loss": 1.4214, + "step": 540 + }, + { + "epoch": 0.9722222222222222, + "grad_norm": 2.9412100315093994, + "learning_rate": 6.12738151212918e-06, + "loss": 1.4367, + "step": 560 + }, + { + "epoch": 1.0069444444444444, + "grad_norm": 4.002957820892334, + "learning_rate": 5.830056206037482e-06, + "loss": 1.4019, + "step": 580 + }, + { + "epoch": 1.0416666666666667, + "grad_norm": 3.2070395946502686, + "learning_rate": 5.529678688597081e-06, + "loss": 1.3294, + "step": 600 + }, + { + "epoch": 1.0763888888888888, + "grad_norm": 3.3583526611328125, + "learning_rate": 5.2273534822017105e-06, + "loss": 1.3302, + "step": 620 + }, + { + "epoch": 1.1111111111111112, + "grad_norm": 4.011720657348633, + "learning_rate": 4.924192271119554e-06, + "loss": 1.2973, + "step": 640 + }, + { + "epoch": 1.1458333333333333, + "grad_norm": 3.383192539215088, + "learning_rate": 4.621309813703385e-06, + "loss": 1.3066, + "step": 660 + }, + { + "epoch": 1.1805555555555556, + "grad_norm": 4.335385322570801, + "learning_rate": 4.319819843296952e-06, + "loss": 1.3068, + "step": 680 + }, + { + "epoch": 1.2152777777777777, + "grad_norm": 3.696607828140259, + "learning_rate": 4.020830972910433e-06, + "loss": 1.3135, + "step": 700 + } + ], + "logging_steps": 20, + "max_steps": 1152, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 50, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.3656271001812992e+17, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/checkpoint-700/training_args.bin b/checkpoint-700/training_args.bin new file mode 100644 index 0000000..a638377 --- /dev/null +++ b/checkpoint-700/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6999d99b0ec3831abd58bd6fec26bef4e74e884d0058076eed8c5f0b91586567 +size 5969 diff --git a/checkpoint-750/README.md b/checkpoint-750/README.md new file mode 100644 index 0000000..bacf84a --- /dev/null +++ b/checkpoint-750/README.md @@ -0,0 +1,207 @@ +--- +base_model: biaofu-xmu/EAST-8B +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:biaofu-xmu/EAST-8B +- lora +- transformers +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/checkpoint-750/adapter_config.json b/checkpoint-750/adapter_config.json new file mode 100644 index 0000000..ca7008b --- /dev/null +++ b/checkpoint-750/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "biaofu-xmu/EAST-8B", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.05, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "v_proj", + "q_proj", + "down_proj", + "up_proj", + "gate_proj", + "o_proj", + "k_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/checkpoint-750/adapter_model.safetensors b/checkpoint-750/adapter_model.safetensors new file mode 100644 index 0000000..20ecf05 --- /dev/null +++ b/checkpoint-750/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bfd500377ef5e78063e27c35a1bfed2d25fb8537267eea98df232cf72d031179 +size 671149168 diff --git a/checkpoint-750/optimizer.pt b/checkpoint-750/optimizer.pt new file mode 100644 index 0000000..837d44f --- /dev/null +++ b/checkpoint-750/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1f6d79338f36028779a9ec1d6cc8ba3b5fc3b69afb0438dc223aee5d0acd853e +size 1342562339 diff --git a/checkpoint-750/rng_state.pth b/checkpoint-750/rng_state.pth new file mode 100644 index 0000000..5134223 --- /dev/null +++ b/checkpoint-750/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a88765ce6df2b74d7af75af738f5d37bae2c7c74fbc47f23077d0a9a8f57770f +size 14645 diff --git a/checkpoint-750/scheduler.pt b/checkpoint-750/scheduler.pt new file mode 100644 index 0000000..cd6206b --- /dev/null +++ b/checkpoint-750/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:72b6be6f074d6123936480d0df9f32aebfcd8ee4d1f99a6b9b9459f1d72f8bd7 +size 1465 diff --git a/checkpoint-750/trainer_state.json b/checkpoint-750/trainer_state.json new file mode 100644 index 0000000..a41c647 --- /dev/null +++ b/checkpoint-750/trainer_state.json @@ -0,0 +1,293 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.3020833333333333, + "eval_steps": 500, + "global_step": 750, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.034722222222222224, + "grad_norm": 2.548311471939087, + "learning_rate": 1.6379310344827587e-06, + "loss": 2.4451, + "step": 20 + }, + { + "epoch": 0.06944444444444445, + "grad_norm": 1.93834388256073, + "learning_rate": 3.362068965517242e-06, + "loss": 2.269, + "step": 40 + }, + { + "epoch": 0.10416666666666667, + "grad_norm": 1.7273279428482056, + "learning_rate": 5.086206896551724e-06, + "loss": 2.1237, + "step": 60 + }, + { + "epoch": 0.1388888888888889, + "grad_norm": 1.6319563388824463, + "learning_rate": 6.810344827586207e-06, + "loss": 1.969, + "step": 80 + }, + { + "epoch": 0.1736111111111111, + "grad_norm": 2.0262465476989746, + "learning_rate": 8.53448275862069e-06, + "loss": 1.8517, + "step": 100 + }, + { + "epoch": 0.20833333333333334, + "grad_norm": 2.1673457622528076, + "learning_rate": 9.999793100349294e-06, + "loss": 1.851, + "step": 120 + }, + { + "epoch": 0.24305555555555555, + "grad_norm": 2.474327802658081, + "learning_rate": 9.987843743451796e-06, + "loss": 1.7448, + "step": 140 + }, + { + "epoch": 0.2777777777777778, + "grad_norm": 2.0316762924194336, + "learning_rate": 9.957553516178782e-06, + "loss": 1.6721, + "step": 160 + }, + { + "epoch": 0.3125, + "grad_norm": 2.3124542236328125, + "learning_rate": 9.909033799150947e-06, + "loss": 1.7265, + "step": 180 + }, + { + "epoch": 0.3472222222222222, + "grad_norm": 2.2682251930236816, + "learning_rate": 9.842463004902127e-06, + "loss": 1.6325, + "step": 200 + }, + { + "epoch": 0.3819444444444444, + "grad_norm": 2.4642767906188965, + "learning_rate": 9.758085921836076e-06, + "loss": 1.6193, + "step": 220 + }, + { + "epoch": 0.4166666666666667, + "grad_norm": 2.4372549057006836, + "learning_rate": 9.656212814111567e-06, + "loss": 1.6297, + "step": 240 + }, + { + "epoch": 0.4513888888888889, + "grad_norm": 2.7898528575897217, + "learning_rate": 9.53721828076571e-06, + "loss": 1.598, + "step": 260 + }, + { + "epoch": 0.4861111111111111, + "grad_norm": 2.742067813873291, + "learning_rate": 9.401539878270545e-06, + "loss": 1.596, + "step": 280 + }, + { + "epoch": 0.5208333333333334, + "grad_norm": 2.9180655479431152, + "learning_rate": 9.249676511588e-06, + "loss": 1.5923, + "step": 300 + }, + { + "epoch": 0.5555555555555556, + "grad_norm": 2.760483503341675, + "learning_rate": 9.082186599639429e-06, + "loss": 1.5733, + "step": 320 + }, + { + "epoch": 0.5902777777777778, + "grad_norm": 2.5661356449127197, + "learning_rate": 8.899686021935554e-06, + "loss": 1.5574, + "step": 340 + }, + { + "epoch": 0.625, + "grad_norm": 2.650604248046875, + "learning_rate": 8.702845853917242e-06, + "loss": 1.5331, + "step": 360 + }, + { + "epoch": 0.6597222222222222, + "grad_norm": 2.5134191513061523, + "learning_rate": 8.492389899334572e-06, + "loss": 1.5116, + "step": 380 + }, + { + "epoch": 0.6944444444444444, + "grad_norm": 2.971090316772461, + "learning_rate": 8.269092028737885e-06, + "loss": 1.4993, + "step": 400 + }, + { + "epoch": 0.7291666666666666, + "grad_norm": 3.0398788452148438, + "learning_rate": 8.033773333867498e-06, + "loss": 1.4751, + "step": 420 + }, + { + "epoch": 0.7638888888888888, + "grad_norm": 2.7814712524414062, + "learning_rate": 7.78729910840572e-06, + "loss": 1.4688, + "step": 440 + }, + { + "epoch": 0.7986111111111112, + "grad_norm": 2.9422523975372314, + "learning_rate": 7.530575666193283e-06, + "loss": 1.4645, + "step": 460 + }, + { + "epoch": 0.8333333333333334, + "grad_norm": 3.5797832012176514, + "learning_rate": 7.26454700860997e-06, + "loss": 1.4473, + "step": 480 + }, + { + "epoch": 0.8680555555555556, + "grad_norm": 3.557054281234741, + "learning_rate": 6.990191353373876e-06, + "loss": 1.4428, + "step": 500 + }, + { + "epoch": 0.9027777777777778, + "grad_norm": 3.7312850952148438, + "learning_rate": 6.708517537523264e-06, + "loss": 1.4014, + "step": 520 + }, + { + "epoch": 0.9375, + "grad_norm": 3.2819321155548096, + "learning_rate": 6.420561307807713e-06, + "loss": 1.4214, + "step": 540 + }, + { + "epoch": 0.9722222222222222, + "grad_norm": 2.9412100315093994, + "learning_rate": 6.12738151212918e-06, + "loss": 1.4367, + "step": 560 + }, + { + "epoch": 1.0069444444444444, + "grad_norm": 4.002957820892334, + "learning_rate": 5.830056206037482e-06, + "loss": 1.4019, + "step": 580 + }, + { + "epoch": 1.0416666666666667, + "grad_norm": 3.2070395946502686, + "learning_rate": 5.529678688597081e-06, + "loss": 1.3294, + "step": 600 + }, + { + "epoch": 1.0763888888888888, + "grad_norm": 3.3583526611328125, + "learning_rate": 5.2273534822017105e-06, + "loss": 1.3302, + "step": 620 + }, + { + "epoch": 1.1111111111111112, + "grad_norm": 4.011720657348633, + "learning_rate": 4.924192271119554e-06, + "loss": 1.2973, + "step": 640 + }, + { + "epoch": 1.1458333333333333, + "grad_norm": 3.383192539215088, + "learning_rate": 4.621309813703385e-06, + "loss": 1.3066, + "step": 660 + }, + { + "epoch": 1.1805555555555556, + "grad_norm": 4.335385322570801, + "learning_rate": 4.319819843296952e-06, + "loss": 1.3068, + "step": 680 + }, + { + "epoch": 1.2152777777777777, + "grad_norm": 3.696607828140259, + "learning_rate": 4.020830972910433e-06, + "loss": 1.3135, + "step": 700 + }, + { + "epoch": 1.25, + "grad_norm": 3.395407199859619, + "learning_rate": 3.7254426187239567e-06, + "loss": 1.2704, + "step": 720 + }, + { + "epoch": 1.2847222222222223, + "grad_norm": 3.829279661178589, + "learning_rate": 3.4347409574088896e-06, + "loss": 1.2869, + "step": 740 + } + ], + "logging_steps": 20, + "max_steps": 1152, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 50, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.4649323767588454e+17, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/checkpoint-750/training_args.bin b/checkpoint-750/training_args.bin new file mode 100644 index 0000000..a638377 --- /dev/null +++ b/checkpoint-750/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6999d99b0ec3831abd58bd6fec26bef4e74e884d0058076eed8c5f0b91586567 +size 5969 diff --git a/checkpoint-850/README.md b/checkpoint-850/README.md new file mode 100644 index 0000000..bacf84a --- /dev/null +++ b/checkpoint-850/README.md @@ -0,0 +1,207 @@ +--- +base_model: biaofu-xmu/EAST-8B +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:biaofu-xmu/EAST-8B +- lora +- transformers +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/checkpoint-850/adapter_config.json b/checkpoint-850/adapter_config.json new file mode 100644 index 0000000..ca7008b --- /dev/null +++ b/checkpoint-850/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "biaofu-xmu/EAST-8B", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.05, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "v_proj", + "q_proj", + "down_proj", + "up_proj", + "gate_proj", + "o_proj", + "k_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/checkpoint-850/adapter_model.safetensors b/checkpoint-850/adapter_model.safetensors new file mode 100644 index 0000000..6551abc --- /dev/null +++ b/checkpoint-850/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b73c4976a1b60a6c0f27f96e781e568b446211b8252a1fb01a96014d89de88f0 +size 671149168 diff --git a/checkpoint-850/optimizer.pt b/checkpoint-850/optimizer.pt new file mode 100644 index 0000000..7838b3e --- /dev/null +++ b/checkpoint-850/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6e402494321e0e4713bbe62bceff41efe2899a9bba6c467e4e989385cc6c21a2 +size 1342562339 diff --git a/checkpoint-850/rng_state.pth b/checkpoint-850/rng_state.pth new file mode 100644 index 0000000..8d343a9 --- /dev/null +++ b/checkpoint-850/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ad399e4af36fdf963cab694effeda0e6cda19fd975fc2d0d74186f8edd84a2d2 +size 14645 diff --git a/checkpoint-850/scheduler.pt b/checkpoint-850/scheduler.pt new file mode 100644 index 0000000..dc7ef30 --- /dev/null +++ b/checkpoint-850/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7d281893a88f586c1ef75eec259a2f50ece352b0e745fc416bb39ce4f50c0f3b +size 1465 diff --git a/checkpoint-850/trainer_state.json b/checkpoint-850/trainer_state.json new file mode 100644 index 0000000..b640924 --- /dev/null +++ b/checkpoint-850/trainer_state.json @@ -0,0 +1,328 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.4756944444444444, + "eval_steps": 500, + "global_step": 850, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.034722222222222224, + "grad_norm": 2.548311471939087, + "learning_rate": 1.6379310344827587e-06, + "loss": 2.4451, + "step": 20 + }, + { + "epoch": 0.06944444444444445, + "grad_norm": 1.93834388256073, + "learning_rate": 3.362068965517242e-06, + "loss": 2.269, + "step": 40 + }, + { + "epoch": 0.10416666666666667, + "grad_norm": 1.7273279428482056, + "learning_rate": 5.086206896551724e-06, + "loss": 2.1237, + "step": 60 + }, + { + "epoch": 0.1388888888888889, + "grad_norm": 1.6319563388824463, + "learning_rate": 6.810344827586207e-06, + "loss": 1.969, + "step": 80 + }, + { + "epoch": 0.1736111111111111, + "grad_norm": 2.0262465476989746, + "learning_rate": 8.53448275862069e-06, + "loss": 1.8517, + "step": 100 + }, + { + "epoch": 0.20833333333333334, + "grad_norm": 2.1673457622528076, + "learning_rate": 9.999793100349294e-06, + "loss": 1.851, + "step": 120 + }, + { + "epoch": 0.24305555555555555, + "grad_norm": 2.474327802658081, + "learning_rate": 9.987843743451796e-06, + "loss": 1.7448, + "step": 140 + }, + { + "epoch": 0.2777777777777778, + "grad_norm": 2.0316762924194336, + "learning_rate": 9.957553516178782e-06, + "loss": 1.6721, + "step": 160 + }, + { + "epoch": 0.3125, + "grad_norm": 2.3124542236328125, + "learning_rate": 9.909033799150947e-06, + "loss": 1.7265, + "step": 180 + }, + { + "epoch": 0.3472222222222222, + "grad_norm": 2.2682251930236816, + "learning_rate": 9.842463004902127e-06, + "loss": 1.6325, + "step": 200 + }, + { + "epoch": 0.3819444444444444, + "grad_norm": 2.4642767906188965, + "learning_rate": 9.758085921836076e-06, + "loss": 1.6193, + "step": 220 + }, + { + "epoch": 0.4166666666666667, + "grad_norm": 2.4372549057006836, + "learning_rate": 9.656212814111567e-06, + "loss": 1.6297, + "step": 240 + }, + { + "epoch": 0.4513888888888889, + "grad_norm": 2.7898528575897217, + "learning_rate": 9.53721828076571e-06, + "loss": 1.598, + "step": 260 + }, + { + "epoch": 0.4861111111111111, + "grad_norm": 2.742067813873291, + "learning_rate": 9.401539878270545e-06, + "loss": 1.596, + "step": 280 + }, + { + "epoch": 0.5208333333333334, + "grad_norm": 2.9180655479431152, + "learning_rate": 9.249676511588e-06, + "loss": 1.5923, + "step": 300 + }, + { + "epoch": 0.5555555555555556, + "grad_norm": 2.760483503341675, + "learning_rate": 9.082186599639429e-06, + "loss": 1.5733, + "step": 320 + }, + { + "epoch": 0.5902777777777778, + "grad_norm": 2.5661356449127197, + "learning_rate": 8.899686021935554e-06, + "loss": 1.5574, + "step": 340 + }, + { + "epoch": 0.625, + "grad_norm": 2.650604248046875, + "learning_rate": 8.702845853917242e-06, + "loss": 1.5331, + "step": 360 + }, + { + "epoch": 0.6597222222222222, + "grad_norm": 2.5134191513061523, + "learning_rate": 8.492389899334572e-06, + "loss": 1.5116, + "step": 380 + }, + { + "epoch": 0.6944444444444444, + "grad_norm": 2.971090316772461, + "learning_rate": 8.269092028737885e-06, + "loss": 1.4993, + "step": 400 + }, + { + "epoch": 0.7291666666666666, + "grad_norm": 3.0398788452148438, + "learning_rate": 8.033773333867498e-06, + "loss": 1.4751, + "step": 420 + }, + { + "epoch": 0.7638888888888888, + "grad_norm": 2.7814712524414062, + "learning_rate": 7.78729910840572e-06, + "loss": 1.4688, + "step": 440 + }, + { + "epoch": 0.7986111111111112, + "grad_norm": 2.9422523975372314, + "learning_rate": 7.530575666193283e-06, + "loss": 1.4645, + "step": 460 + }, + { + "epoch": 0.8333333333333334, + "grad_norm": 3.5797832012176514, + "learning_rate": 7.26454700860997e-06, + "loss": 1.4473, + "step": 480 + }, + { + "epoch": 0.8680555555555556, + "grad_norm": 3.557054281234741, + "learning_rate": 6.990191353373876e-06, + "loss": 1.4428, + "step": 500 + }, + { + "epoch": 0.9027777777777778, + "grad_norm": 3.7312850952148438, + "learning_rate": 6.708517537523264e-06, + "loss": 1.4014, + "step": 520 + }, + { + "epoch": 0.9375, + "grad_norm": 3.2819321155548096, + "learning_rate": 6.420561307807713e-06, + "loss": 1.4214, + "step": 540 + }, + { + "epoch": 0.9722222222222222, + "grad_norm": 2.9412100315093994, + "learning_rate": 6.12738151212918e-06, + "loss": 1.4367, + "step": 560 + }, + { + "epoch": 1.0069444444444444, + "grad_norm": 4.002957820892334, + "learning_rate": 5.830056206037482e-06, + "loss": 1.4019, + "step": 580 + }, + { + "epoch": 1.0416666666666667, + "grad_norm": 3.2070395946502686, + "learning_rate": 5.529678688597081e-06, + "loss": 1.3294, + "step": 600 + }, + { + "epoch": 1.0763888888888888, + "grad_norm": 3.3583526611328125, + "learning_rate": 5.2273534822017105e-06, + "loss": 1.3302, + "step": 620 + }, + { + "epoch": 1.1111111111111112, + "grad_norm": 4.011720657348633, + "learning_rate": 4.924192271119554e-06, + "loss": 1.2973, + "step": 640 + }, + { + "epoch": 1.1458333333333333, + "grad_norm": 3.383192539215088, + "learning_rate": 4.621309813703385e-06, + "loss": 1.3066, + "step": 660 + }, + { + "epoch": 1.1805555555555556, + "grad_norm": 4.335385322570801, + "learning_rate": 4.319819843296952e-06, + "loss": 1.3068, + "step": 680 + }, + { + "epoch": 1.2152777777777777, + "grad_norm": 3.696607828140259, + "learning_rate": 4.020830972910433e-06, + "loss": 1.3135, + "step": 700 + }, + { + "epoch": 1.25, + "grad_norm": 3.395407199859619, + "learning_rate": 3.7254426187239567e-06, + "loss": 1.2704, + "step": 720 + }, + { + "epoch": 1.2847222222222223, + "grad_norm": 3.829279661178589, + "learning_rate": 3.4347409574088896e-06, + "loss": 1.2869, + "step": 740 + }, + { + "epoch": 1.3194444444444444, + "grad_norm": 4.77693510055542, + "learning_rate": 3.149794932132331e-06, + "loss": 1.2903, + "step": 760 + }, + { + "epoch": 1.3541666666666667, + "grad_norm": 4.946527004241943, + "learning_rate": 2.871652321931161e-06, + "loss": 1.2661, + "step": 780 + }, + { + "epoch": 1.3888888888888888, + "grad_norm": 3.0826640129089355, + "learning_rate": 2.601335888909005e-06, + "loss": 1.2786, + "step": 800 + }, + { + "epoch": 1.4236111111111112, + "grad_norm": 4.134871959686279, + "learning_rate": 2.339839617423318e-06, + "loss": 1.2386, + "step": 820 + }, + { + "epoch": 1.4583333333333333, + "grad_norm": 3.8811264038085938, + "learning_rate": 2.0881250590915316e-06, + "loss": 1.2478, + "step": 840 + } + ], + "logging_steps": 20, + "max_steps": 1152, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 50, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.663833888518308e+17, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/checkpoint-850/training_args.bin b/checkpoint-850/training_args.bin new file mode 100644 index 0000000..a638377 --- /dev/null +++ b/checkpoint-850/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6999d99b0ec3831abd58bd6fec26bef4e74e884d0058076eed8c5f0b91586567 +size 5969 diff --git a/checkpoint-950/README.md b/checkpoint-950/README.md new file mode 100644 index 0000000..bacf84a --- /dev/null +++ b/checkpoint-950/README.md @@ -0,0 +1,207 @@ +--- +base_model: biaofu-xmu/EAST-8B +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:biaofu-xmu/EAST-8B +- lora +- transformers +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/checkpoint-950/adapter_config.json b/checkpoint-950/adapter_config.json new file mode 100644 index 0000000..ca7008b --- /dev/null +++ b/checkpoint-950/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "biaofu-xmu/EAST-8B", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.05, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "v_proj", + "q_proj", + "down_proj", + "up_proj", + "gate_proj", + "o_proj", + "k_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/checkpoint-950/adapter_model.safetensors b/checkpoint-950/adapter_model.safetensors new file mode 100644 index 0000000..fdd8af1 --- /dev/null +++ b/checkpoint-950/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e47ea81aaf76217bad77dad9abafd6558cc59cc26c3d5f3846262b92df3fd31b +size 671149168 diff --git a/checkpoint-950/optimizer.pt b/checkpoint-950/optimizer.pt new file mode 100644 index 0000000..c72642d --- /dev/null +++ b/checkpoint-950/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7e9c704fdcafcd24157d213a98f4e2eebb6f934aface43cf08fc71d27a9eb402 +size 1342562339 diff --git a/checkpoint-950/rng_state.pth b/checkpoint-950/rng_state.pth new file mode 100644 index 0000000..204ba3d --- /dev/null +++ b/checkpoint-950/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:51825e751c779ebae2e6c229817be583307afe766c84c4f6fe91a81bc7e3cc3a +size 14645 diff --git a/checkpoint-950/scheduler.pt b/checkpoint-950/scheduler.pt new file mode 100644 index 0000000..f68e810 --- /dev/null +++ b/checkpoint-950/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e96b0deceec755ce9e150cc4f2fb1d1ed2f491e7dd6980aa723ae9f45f4ebfb3 +size 1465 diff --git a/checkpoint-950/trainer_state.json b/checkpoint-950/trainer_state.json new file mode 100644 index 0000000..bf1f64e --- /dev/null +++ b/checkpoint-950/trainer_state.json @@ -0,0 +1,363 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.6493055555555556, + "eval_steps": 500, + "global_step": 950, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.034722222222222224, + "grad_norm": 2.548311471939087, + "learning_rate": 1.6379310344827587e-06, + "loss": 2.4451, + "step": 20 + }, + { + "epoch": 0.06944444444444445, + "grad_norm": 1.93834388256073, + "learning_rate": 3.362068965517242e-06, + "loss": 2.269, + "step": 40 + }, + { + "epoch": 0.10416666666666667, + "grad_norm": 1.7273279428482056, + "learning_rate": 5.086206896551724e-06, + "loss": 2.1237, + "step": 60 + }, + { + "epoch": 0.1388888888888889, + "grad_norm": 1.6319563388824463, + "learning_rate": 6.810344827586207e-06, + "loss": 1.969, + "step": 80 + }, + { + "epoch": 0.1736111111111111, + "grad_norm": 2.0262465476989746, + "learning_rate": 8.53448275862069e-06, + "loss": 1.8517, + "step": 100 + }, + { + "epoch": 0.20833333333333334, + "grad_norm": 2.1673457622528076, + "learning_rate": 9.999793100349294e-06, + "loss": 1.851, + "step": 120 + }, + { + "epoch": 0.24305555555555555, + "grad_norm": 2.474327802658081, + "learning_rate": 9.987843743451796e-06, + "loss": 1.7448, + "step": 140 + }, + { + "epoch": 0.2777777777777778, + "grad_norm": 2.0316762924194336, + "learning_rate": 9.957553516178782e-06, + "loss": 1.6721, + "step": 160 + }, + { + "epoch": 0.3125, + "grad_norm": 2.3124542236328125, + "learning_rate": 9.909033799150947e-06, + "loss": 1.7265, + "step": 180 + }, + { + "epoch": 0.3472222222222222, + "grad_norm": 2.2682251930236816, + "learning_rate": 9.842463004902127e-06, + "loss": 1.6325, + "step": 200 + }, + { + "epoch": 0.3819444444444444, + "grad_norm": 2.4642767906188965, + "learning_rate": 9.758085921836076e-06, + "loss": 1.6193, + "step": 220 + }, + { + "epoch": 0.4166666666666667, + "grad_norm": 2.4372549057006836, + "learning_rate": 9.656212814111567e-06, + "loss": 1.6297, + "step": 240 + }, + { + "epoch": 0.4513888888888889, + "grad_norm": 2.7898528575897217, + "learning_rate": 9.53721828076571e-06, + "loss": 1.598, + "step": 260 + }, + { + "epoch": 0.4861111111111111, + "grad_norm": 2.742067813873291, + "learning_rate": 9.401539878270545e-06, + "loss": 1.596, + "step": 280 + }, + { + "epoch": 0.5208333333333334, + "grad_norm": 2.9180655479431152, + "learning_rate": 9.249676511588e-06, + "loss": 1.5923, + "step": 300 + }, + { + "epoch": 0.5555555555555556, + "grad_norm": 2.760483503341675, + "learning_rate": 9.082186599639429e-06, + "loss": 1.5733, + "step": 320 + }, + { + "epoch": 0.5902777777777778, + "grad_norm": 2.5661356449127197, + "learning_rate": 8.899686021935554e-06, + "loss": 1.5574, + "step": 340 + }, + { + "epoch": 0.625, + "grad_norm": 2.650604248046875, + "learning_rate": 8.702845853917242e-06, + "loss": 1.5331, + "step": 360 + }, + { + "epoch": 0.6597222222222222, + "grad_norm": 2.5134191513061523, + "learning_rate": 8.492389899334572e-06, + "loss": 1.5116, + "step": 380 + }, + { + "epoch": 0.6944444444444444, + "grad_norm": 2.971090316772461, + "learning_rate": 8.269092028737885e-06, + "loss": 1.4993, + "step": 400 + }, + { + "epoch": 0.7291666666666666, + "grad_norm": 3.0398788452148438, + "learning_rate": 8.033773333867498e-06, + "loss": 1.4751, + "step": 420 + }, + { + "epoch": 0.7638888888888888, + "grad_norm": 2.7814712524414062, + "learning_rate": 7.78729910840572e-06, + "loss": 1.4688, + "step": 440 + }, + { + "epoch": 0.7986111111111112, + "grad_norm": 2.9422523975372314, + "learning_rate": 7.530575666193283e-06, + "loss": 1.4645, + "step": 460 + }, + { + "epoch": 0.8333333333333334, + "grad_norm": 3.5797832012176514, + "learning_rate": 7.26454700860997e-06, + "loss": 1.4473, + "step": 480 + }, + { + "epoch": 0.8680555555555556, + "grad_norm": 3.557054281234741, + "learning_rate": 6.990191353373876e-06, + "loss": 1.4428, + "step": 500 + }, + { + "epoch": 0.9027777777777778, + "grad_norm": 3.7312850952148438, + "learning_rate": 6.708517537523264e-06, + "loss": 1.4014, + "step": 520 + }, + { + "epoch": 0.9375, + "grad_norm": 3.2819321155548096, + "learning_rate": 6.420561307807713e-06, + "loss": 1.4214, + "step": 540 + }, + { + "epoch": 0.9722222222222222, + "grad_norm": 2.9412100315093994, + "learning_rate": 6.12738151212918e-06, + "loss": 1.4367, + "step": 560 + }, + { + "epoch": 1.0069444444444444, + "grad_norm": 4.002957820892334, + "learning_rate": 5.830056206037482e-06, + "loss": 1.4019, + "step": 580 + }, + { + "epoch": 1.0416666666666667, + "grad_norm": 3.2070395946502686, + "learning_rate": 5.529678688597081e-06, + "loss": 1.3294, + "step": 600 + }, + { + "epoch": 1.0763888888888888, + "grad_norm": 3.3583526611328125, + "learning_rate": 5.2273534822017105e-06, + "loss": 1.3302, + "step": 620 + }, + { + "epoch": 1.1111111111111112, + "grad_norm": 4.011720657348633, + "learning_rate": 4.924192271119554e-06, + "loss": 1.2973, + "step": 640 + }, + { + "epoch": 1.1458333333333333, + "grad_norm": 3.383192539215088, + "learning_rate": 4.621309813703385e-06, + "loss": 1.3066, + "step": 660 + }, + { + "epoch": 1.1805555555555556, + "grad_norm": 4.335385322570801, + "learning_rate": 4.319819843296952e-06, + "loss": 1.3068, + "step": 680 + }, + { + "epoch": 1.2152777777777777, + "grad_norm": 3.696607828140259, + "learning_rate": 4.020830972910433e-06, + "loss": 1.3135, + "step": 700 + }, + { + "epoch": 1.25, + "grad_norm": 3.395407199859619, + "learning_rate": 3.7254426187239567e-06, + "loss": 1.2704, + "step": 720 + }, + { + "epoch": 1.2847222222222223, + "grad_norm": 3.829279661178589, + "learning_rate": 3.4347409574088896e-06, + "loss": 1.2869, + "step": 740 + }, + { + "epoch": 1.3194444444444444, + "grad_norm": 4.77693510055542, + "learning_rate": 3.149794932132331e-06, + "loss": 1.2903, + "step": 760 + }, + { + "epoch": 1.3541666666666667, + "grad_norm": 4.946527004241943, + "learning_rate": 2.871652321931161e-06, + "loss": 1.2661, + "step": 780 + }, + { + "epoch": 1.3888888888888888, + "grad_norm": 3.0826640129089355, + "learning_rate": 2.601335888909005e-06, + "loss": 1.2786, + "step": 800 + }, + { + "epoch": 1.4236111111111112, + "grad_norm": 4.134871959686279, + "learning_rate": 2.339839617423318e-06, + "loss": 1.2386, + "step": 820 + }, + { + "epoch": 1.4583333333333333, + "grad_norm": 3.8811264038085938, + "learning_rate": 2.0881250590915316e-06, + "loss": 1.2478, + "step": 840 + }, + { + "epoch": 1.4930555555555556, + "grad_norm": 4.137213230133057, + "learning_rate": 1.8471177970560712e-06, + "loss": 1.2635, + "step": 860 + }, + { + "epoch": 1.5277777777777777, + "grad_norm": 3.4709768295288086, + "learning_rate": 1.6177040425095664e-06, + "loss": 1.2247, + "step": 880 + }, + { + "epoch": 1.5625, + "grad_norm": 3.841049909591675, + "learning_rate": 1.40072737599522e-06, + "loss": 1.2721, + "step": 900 + }, + { + "epoch": 1.5972222222222223, + "grad_norm": 3.412503480911255, + "learning_rate": 1.196985645464921e-06, + "loss": 1.2439, + "step": 920 + }, + { + "epoch": 1.6319444444444444, + "grad_norm": 4.341178894042969, + "learning_rate": 1.0072280325013185e-06, + "loss": 1.2459, + "step": 940 + } + ], + "logging_steps": 20, + "max_steps": 1152, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 50, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.860300923853865e+17, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/checkpoint-950/training_args.bin b/checkpoint-950/training_args.bin new file mode 100644 index 0000000..a638377 --- /dev/null +++ b/checkpoint-950/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6999d99b0ec3831abd58bd6fec26bef4e74e884d0058076eed8c5f0b91586567 +size 5969 diff --git a/config.json b/config.json new file mode 100644 index 0000000..640d98b --- /dev/null +++ b/config.json @@ -0,0 +1,29 @@ +{ + "architectures": [ + "LlamaForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 128000, + "dtype": "bfloat16", + "eos_token_id": 128001, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 4096, + "initializer_range": 0.02, + "intermediate_size": 14336, + "max_position_embeddings": 8192, + "mlp_bias": false, + "model_type": "llama", + "num_attention_heads": 32, + "num_hidden_layers": 32, + "num_key_value_heads": 8, + "pretraining_tp": 1, + "rms_norm_eps": 1e-05, + "rope_scaling": null, + "rope_theta": 500000.0, + "tie_word_embeddings": false, + "transformers_version": "4.57.6", + "use_cache": true, + "vocab_size": 128320 +} diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..bab1302 --- /dev/null +++ b/generation_config.json @@ -0,0 +1,6 @@ +{ + "_from_model_config": true, + "bos_token_id": 128000, + "eos_token_id": 128001, + "transformers_version": "4.57.6" +} diff --git a/last-checkpoint/README.md b/last-checkpoint/README.md new file mode 100644 index 0000000..bacf84a --- /dev/null +++ b/last-checkpoint/README.md @@ -0,0 +1,207 @@ +--- +base_model: biaofu-xmu/EAST-8B +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:biaofu-xmu/EAST-8B +- lora +- transformers +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/last-checkpoint/adapter_config.json b/last-checkpoint/adapter_config.json new file mode 100644 index 0000000..37ea34d --- /dev/null +++ b/last-checkpoint/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "biaofu-xmu/EAST-8B", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.05, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "down_proj", + "v_proj", + "q_proj", + "k_proj", + "gate_proj", + "o_proj", + "up_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/last-checkpoint/adapter_model.safetensors b/last-checkpoint/adapter_model.safetensors new file mode 100644 index 0000000..ef59cbe --- /dev/null +++ b/last-checkpoint/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0f251f59cb2d8414f036a2c597d75c7f25cc16833d3b0465d1639894a0129328 +size 671149168 diff --git a/last-checkpoint/optimizer.pt b/last-checkpoint/optimizer.pt new file mode 100644 index 0000000..c21239f --- /dev/null +++ b/last-checkpoint/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:507490bbdb1a479de01ea96d60040acd2732ece2a3ba887964292cb5ec5207bd +size 1342562339 diff --git a/last-checkpoint/rng_state.pth b/last-checkpoint/rng_state.pth new file mode 100644 index 0000000..fd039b9 --- /dev/null +++ b/last-checkpoint/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:14d867f13317641b14c4d36563b35f5431003f10e794d79316e2bbd24b1df339 +size 14645 diff --git a/last-checkpoint/scheduler.pt b/last-checkpoint/scheduler.pt new file mode 100644 index 0000000..54c4178 --- /dev/null +++ b/last-checkpoint/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3faa60702eeec70c307ffab2e710a51a22e86b2f89bbb3effe498e101641bee8 +size 1465 diff --git a/last-checkpoint/trainer_state.json b/last-checkpoint/trainer_state.json new file mode 100644 index 0000000..b6b2245 --- /dev/null +++ b/last-checkpoint/trainer_state.json @@ -0,0 +1,384 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.7361111111111112, + "eval_steps": 500, + "global_step": 1000, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.034722222222222224, + "grad_norm": 2.528588056564331, + "learning_rate": 1.6379310344827587e-06, + "loss": 2.4452, + "step": 20 + }, + { + "epoch": 0.06944444444444445, + "grad_norm": 1.9198594093322754, + "learning_rate": 3.362068965517242e-06, + "loss": 2.2698, + "step": 40 + }, + { + "epoch": 0.10416666666666667, + "grad_norm": 1.7612886428833008, + "learning_rate": 5.086206896551724e-06, + "loss": 2.1235, + "step": 60 + }, + { + "epoch": 0.1388888888888889, + "grad_norm": 1.667220115661621, + "learning_rate": 6.810344827586207e-06, + "loss": 1.9688, + "step": 80 + }, + { + "epoch": 0.1736111111111111, + "grad_norm": 2.0434842109680176, + "learning_rate": 8.53448275862069e-06, + "loss": 1.852, + "step": 100 + }, + { + "epoch": 0.20833333333333334, + "grad_norm": 2.1740384101867676, + "learning_rate": 9.999793100349294e-06, + "loss": 1.8515, + "step": 120 + }, + { + "epoch": 0.24305555555555555, + "grad_norm": 2.4882936477661133, + "learning_rate": 9.987843743451796e-06, + "loss": 1.7452, + "step": 140 + }, + { + "epoch": 0.2777777777777778, + "grad_norm": 2.0491299629211426, + "learning_rate": 9.957553516178782e-06, + "loss": 1.672, + "step": 160 + }, + { + "epoch": 0.3125, + "grad_norm": 2.3231043815612793, + "learning_rate": 9.909033799150947e-06, + "loss": 1.7257, + "step": 180 + }, + { + "epoch": 0.3472222222222222, + "grad_norm": 2.2714061737060547, + "learning_rate": 9.842463004902127e-06, + "loss": 1.6321, + "step": 200 + }, + { + "epoch": 0.3819444444444444, + "grad_norm": 2.4732699394226074, + "learning_rate": 9.758085921836076e-06, + "loss": 1.6189, + "step": 220 + }, + { + "epoch": 0.4166666666666667, + "grad_norm": 2.4421303272247314, + "learning_rate": 9.656212814111567e-06, + "loss": 1.6285, + "step": 240 + }, + { + "epoch": 0.4513888888888889, + "grad_norm": 2.8063509464263916, + "learning_rate": 9.53721828076571e-06, + "loss": 1.5975, + "step": 260 + }, + { + "epoch": 0.4861111111111111, + "grad_norm": 2.7459986209869385, + "learning_rate": 9.401539878270545e-06, + "loss": 1.5955, + "step": 280 + }, + { + "epoch": 0.5208333333333334, + "grad_norm": 2.9211506843566895, + "learning_rate": 9.249676511588e-06, + "loss": 1.5918, + "step": 300 + }, + { + "epoch": 0.5555555555555556, + "grad_norm": 2.7910404205322266, + "learning_rate": 9.082186599639429e-06, + "loss": 1.5728, + "step": 320 + }, + { + "epoch": 0.5902777777777778, + "grad_norm": 2.565889596939087, + "learning_rate": 8.899686021935554e-06, + "loss": 1.5565, + "step": 340 + }, + { + "epoch": 0.625, + "grad_norm": 2.653115749359131, + "learning_rate": 8.702845853917242e-06, + "loss": 1.5323, + "step": 360 + }, + { + "epoch": 0.6597222222222222, + "grad_norm": 2.514103412628174, + "learning_rate": 8.492389899334572e-06, + "loss": 1.5109, + "step": 380 + }, + { + "epoch": 0.6944444444444444, + "grad_norm": 2.9695253372192383, + "learning_rate": 8.269092028737885e-06, + "loss": 1.4991, + "step": 400 + }, + { + "epoch": 0.7291666666666666, + "grad_norm": 2.9976706504821777, + "learning_rate": 8.033773333867498e-06, + "loss": 1.4744, + "step": 420 + }, + { + "epoch": 0.7638888888888888, + "grad_norm": 2.794107675552368, + "learning_rate": 7.78729910840572e-06, + "loss": 1.4678, + "step": 440 + }, + { + "epoch": 0.7986111111111112, + "grad_norm": 2.9695143699645996, + "learning_rate": 7.530575666193283e-06, + "loss": 1.4634, + "step": 460 + }, + { + "epoch": 0.8333333333333334, + "grad_norm": 3.623947858810425, + "learning_rate": 7.26454700860997e-06, + "loss": 1.4467, + "step": 480 + }, + { + "epoch": 0.8680555555555556, + "grad_norm": 3.5737252235412598, + "learning_rate": 6.990191353373876e-06, + "loss": 1.4423, + "step": 500 + }, + { + "epoch": 0.9027777777777778, + "grad_norm": 3.735530376434326, + "learning_rate": 6.708517537523264e-06, + "loss": 1.4011, + "step": 520 + }, + { + "epoch": 0.9375, + "grad_norm": 3.286858081817627, + "learning_rate": 6.420561307807713e-06, + "loss": 1.4212, + "step": 540 + }, + { + "epoch": 0.9722222222222222, + "grad_norm": 2.975515842437744, + "learning_rate": 6.12738151212918e-06, + "loss": 1.4365, + "step": 560 + }, + { + "epoch": 1.0069444444444444, + "grad_norm": 3.989678144454956, + "learning_rate": 5.830056206037482e-06, + "loss": 1.4018, + "step": 580 + }, + { + "epoch": 1.0416666666666667, + "grad_norm": 3.2178144454956055, + "learning_rate": 5.529678688597081e-06, + "loss": 1.3287, + "step": 600 + }, + { + "epoch": 1.0763888888888888, + "grad_norm": 3.363408088684082, + "learning_rate": 5.2273534822017105e-06, + "loss": 1.3294, + "step": 620 + }, + { + "epoch": 1.1111111111111112, + "grad_norm": 4.074166297912598, + "learning_rate": 4.924192271119554e-06, + "loss": 1.2959, + "step": 640 + }, + { + "epoch": 1.1458333333333333, + "grad_norm": 3.1612188816070557, + "learning_rate": 4.621309813703385e-06, + "loss": 1.3059, + "step": 660 + }, + { + "epoch": 1.1805555555555556, + "grad_norm": 4.3585896492004395, + "learning_rate": 4.319819843296952e-06, + "loss": 1.3052, + "step": 680 + }, + { + "epoch": 1.2152777777777777, + "grad_norm": 3.699857234954834, + "learning_rate": 4.020830972910433e-06, + "loss": 1.3131, + "step": 700 + }, + { + "epoch": 1.25, + "grad_norm": 3.4211180210113525, + "learning_rate": 3.7254426187239567e-06, + "loss": 1.2698, + "step": 720 + }, + { + "epoch": 1.2847222222222223, + "grad_norm": 3.812756299972534, + "learning_rate": 3.4347409574088896e-06, + "loss": 1.286, + "step": 740 + }, + { + "epoch": 1.3194444444444444, + "grad_norm": 4.69367790222168, + "learning_rate": 3.149794932132331e-06, + "loss": 1.2883, + "step": 760 + }, + { + "epoch": 1.3541666666666667, + "grad_norm": 4.980796813964844, + "learning_rate": 2.871652321931161e-06, + "loss": 1.2646, + "step": 780 + }, + { + "epoch": 1.3888888888888888, + "grad_norm": 3.0813663005828857, + "learning_rate": 2.601335888909005e-06, + "loss": 1.2779, + "step": 800 + }, + { + "epoch": 1.4236111111111112, + "grad_norm": 4.145002841949463, + "learning_rate": 2.339839617423318e-06, + "loss": 1.237, + "step": 820 + }, + { + "epoch": 1.4583333333333333, + "grad_norm": 3.879746437072754, + "learning_rate": 2.0881250590915316e-06, + "loss": 1.2466, + "step": 840 + }, + { + "epoch": 1.4930555555555556, + "grad_norm": 4.132529258728027, + "learning_rate": 1.8471177970560712e-06, + "loss": 1.2622, + "step": 860 + }, + { + "epoch": 1.5277777777777777, + "grad_norm": 3.4623332023620605, + "learning_rate": 1.6177040425095664e-06, + "loss": 1.2239, + "step": 880 + }, + { + "epoch": 1.5625, + "grad_norm": 3.851890802383423, + "learning_rate": 1.40072737599522e-06, + "loss": 1.2711, + "step": 900 + }, + { + "epoch": 1.5972222222222223, + "grad_norm": 3.429495096206665, + "learning_rate": 1.196985645464921e-06, + "loss": 1.2424, + "step": 920 + }, + { + "epoch": 1.6319444444444444, + "grad_norm": 4.348053932189941, + "learning_rate": 1.0072280325013185e-06, + "loss": 1.2449, + "step": 940 + }, + { + "epoch": 1.6666666666666665, + "grad_norm": 4.481091022491455, + "learning_rate": 8.321522974916968e-07, + "loss": 1.2653, + "step": 960 + }, + { + "epoch": 1.7013888888888888, + "grad_norm": 4.414041996002197, + "learning_rate": 6.724022138834341e-07, + "loss": 1.2279, + "step": 980 + }, + { + "epoch": 1.7361111111111112, + "grad_norm": 3.4355506896972656, + "learning_rate": 5.285652009556075e-07, + "loss": 1.2171, + "step": 1000 + } + ], + "logging_steps": 20, + "max_steps": 1152, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 50, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.9555180478889984e+17, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/last-checkpoint/training_args.bin b/last-checkpoint/training_args.bin new file mode 100644 index 0000000..707dc5c --- /dev/null +++ b/last-checkpoint/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc565ab922336aa0264222e0b5205df4aee0c58c7792a0d0bf59e20481461883 +size 5969 diff --git a/model-00001-of-00004.safetensors b/model-00001-of-00004.safetensors new file mode 100644 index 0000000..36bcd36 --- /dev/null +++ b/model-00001-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c7964336698f2fcd193cf98c57e9b40299e422f4965b7744a4df219502709fed +size 4977222960 diff --git a/model-00002-of-00004.safetensors b/model-00002-of-00004.safetensors new file mode 100644 index 0000000..689a293 --- /dev/null +++ b/model-00002-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:793b6f83d5c11c3fc9c34cdc4c5e71b8ab1e94851d6182f612118bbf24967815 +size 4999802720 diff --git a/model-00003-of-00004.safetensors b/model-00003-of-00004.safetensors new file mode 100644 index 0000000..ea1a6a7 --- /dev/null +++ b/model-00003-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2d975080b5d1e69e35cf55bfd410d5845f6bc08addd4169452186e37d286e97d +size 4915916176 diff --git a/model-00004-of-00004.safetensors b/model-00004-of-00004.safetensors new file mode 100644 index 0000000..a41bf35 --- /dev/null +++ b/model-00004-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ad8ef3da0ed0ba8fc1dff3b8703b9d50e208f10572e337ab0eee164684af6b4f +size 1168663096 diff --git a/model.safetensors.index.json b/model.safetensors.index.json new file mode 100644 index 0000000..55f36a5 --- /dev/null +++ b/model.safetensors.index.json @@ -0,0 +1,299 @@ +{ + "metadata": { + "total_parameters": 8030785536, + "total_size": 16061571072 + }, + "weight_map": { + "lm_head.weight": "model-00004-of-00004.safetensors", + "model.embed_tokens.weight": "model-00001-of-00004.safetensors", + "model.layers.0.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.10.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.10.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.11.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.12.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.13.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.14.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.15.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.16.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.17.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.18.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.19.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.2.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.20.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.20.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.20.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.20.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.21.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.21.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.21.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.22.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.22.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.23.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.24.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.25.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.26.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.27.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.28.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.29.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.3.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.30.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.30.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.30.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.31.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.4.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.4.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.5.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.6.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.7.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.8.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.8.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.8.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.9.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.9.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.9.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.norm.weight": "model-00004-of-00004.safetensors" + } +} diff --git a/special_tokens_map.json b/special_tokens_map.json new file mode 100644 index 0000000..6631bbf --- /dev/null +++ b/special_tokens_map.json @@ -0,0 +1,27 @@ +{ + "additional_special_tokens": [ + "<|end-of-read|>", + "<|end-of-write|>" + ], + "bos_token": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "<|eot_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": { + "content": "<|eot_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + } +} diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..ccae08e --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:edc01071ea813bba1c85943ccabb16dbd408ba02c9bd781dffc66467be8be3c3 +size 17209138 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..31003a7 --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,2085 @@ +{ + "added_tokens_decoder": { + "128000": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128001": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128002": { + "content": "<|reserved_special_token_0|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128003": { + "content": "<|reserved_special_token_1|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128004": { + "content": "<|reserved_special_token_2|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128005": { + "content": "<|reserved_special_token_3|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128006": { + "content": "<|start_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128007": { + "content": "<|end_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128008": { + "content": "<|reserved_special_token_4|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128009": { + "content": "<|eot_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128010": { + "content": "<|reserved_special_token_5|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128011": { + "content": "<|reserved_special_token_6|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128012": { + "content": "<|reserved_special_token_7|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128013": { + "content": "<|reserved_special_token_8|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128014": { + "content": "<|reserved_special_token_9|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128015": { + "content": "<|reserved_special_token_10|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128016": { + "content": "<|reserved_special_token_11|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128017": { + "content": "<|reserved_special_token_12|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128018": { + "content": "<|reserved_special_token_13|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128019": { + "content": "<|reserved_special_token_14|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128020": { + "content": "<|reserved_special_token_15|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128021": { + "content": "<|reserved_special_token_16|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128022": { + "content": "<|reserved_special_token_17|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128023": { + "content": "<|reserved_special_token_18|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128024": { + "content": "<|reserved_special_token_19|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128025": { + "content": "<|reserved_special_token_20|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128026": { + "content": "<|reserved_special_token_21|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128027": { + "content": "<|reserved_special_token_22|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128028": { + "content": "<|reserved_special_token_23|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128029": { + "content": "<|reserved_special_token_24|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128030": { + "content": "<|reserved_special_token_25|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128031": { + "content": "<|reserved_special_token_26|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128032": { + "content": "<|reserved_special_token_27|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128033": { + "content": "<|reserved_special_token_28|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128034": { + "content": "<|reserved_special_token_29|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128035": { + "content": "<|reserved_special_token_30|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128036": { + "content": "<|reserved_special_token_31|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128037": { + "content": "<|reserved_special_token_32|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128038": { + "content": "<|reserved_special_token_33|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128039": { + "content": "<|reserved_special_token_34|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128040": { + "content": "<|reserved_special_token_35|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128041": { + "content": "<|reserved_special_token_36|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128042": { + "content": "<|reserved_special_token_37|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128043": { + "content": "<|reserved_special_token_38|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128044": { + "content": "<|reserved_special_token_39|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128045": { + "content": "<|reserved_special_token_40|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128046": { + "content": "<|reserved_special_token_41|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128047": { + "content": "<|reserved_special_token_42|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128048": { + "content": "<|reserved_special_token_43|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128049": { + "content": "<|reserved_special_token_44|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128050": { + "content": "<|reserved_special_token_45|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128051": { + "content": "<|reserved_special_token_46|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128052": { + "content": "<|reserved_special_token_47|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128053": { + "content": "<|reserved_special_token_48|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128054": { + "content": "<|reserved_special_token_49|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128055": { + "content": "<|reserved_special_token_50|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128056": { + "content": "<|reserved_special_token_51|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128057": { + "content": "<|reserved_special_token_52|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128058": { + "content": "<|reserved_special_token_53|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128059": { + "content": "<|reserved_special_token_54|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128060": { + "content": "<|reserved_special_token_55|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128061": { + "content": "<|reserved_special_token_56|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128062": { + "content": "<|reserved_special_token_57|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128063": { + "content": "<|reserved_special_token_58|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128064": { + "content": "<|reserved_special_token_59|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128065": { + "content": "<|reserved_special_token_60|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128066": { + "content": "<|reserved_special_token_61|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128067": { + "content": "<|reserved_special_token_62|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128068": { + "content": "<|reserved_special_token_63|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128069": { + "content": "<|reserved_special_token_64|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128070": { + "content": "<|reserved_special_token_65|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128071": { + "content": "<|reserved_special_token_66|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128072": { + "content": "<|reserved_special_token_67|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128073": { + "content": "<|reserved_special_token_68|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128074": { + "content": "<|reserved_special_token_69|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128075": { + "content": "<|reserved_special_token_70|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128076": { + "content": "<|reserved_special_token_71|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128077": { + "content": "<|reserved_special_token_72|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128078": { + "content": "<|reserved_special_token_73|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128079": { + "content": "<|reserved_special_token_74|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128080": { + "content": "<|reserved_special_token_75|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128081": { + "content": "<|reserved_special_token_76|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128082": { + "content": "<|reserved_special_token_77|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128083": { + "content": "<|reserved_special_token_78|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128084": { + "content": "<|reserved_special_token_79|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128085": { + "content": "<|reserved_special_token_80|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128086": { + "content": "<|reserved_special_token_81|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128087": { + "content": "<|reserved_special_token_82|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128088": { + "content": "<|reserved_special_token_83|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128089": { + "content": "<|reserved_special_token_84|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128090": { + "content": "<|reserved_special_token_85|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128091": { + "content": "<|reserved_special_token_86|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128092": { + "content": "<|reserved_special_token_87|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128093": { + "content": "<|reserved_special_token_88|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128094": { + "content": "<|reserved_special_token_89|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128095": { + "content": "<|reserved_special_token_90|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128096": { + "content": "<|reserved_special_token_91|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128097": { + "content": "<|reserved_special_token_92|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128098": { + "content": "<|reserved_special_token_93|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128099": { + "content": "<|reserved_special_token_94|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128100": { + "content": "<|reserved_special_token_95|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128101": { + "content": "<|reserved_special_token_96|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128102": { + "content": "<|reserved_special_token_97|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128103": { + "content": "<|reserved_special_token_98|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128104": { + "content": "<|reserved_special_token_99|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128105": { + "content": "<|reserved_special_token_100|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128106": { + "content": "<|reserved_special_token_101|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128107": { + "content": "<|reserved_special_token_102|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128108": { + "content": "<|reserved_special_token_103|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128109": { + "content": "<|reserved_special_token_104|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128110": { + "content": "<|reserved_special_token_105|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128111": { + "content": "<|reserved_special_token_106|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128112": { + "content": "<|reserved_special_token_107|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128113": { + "content": "<|reserved_special_token_108|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128114": { + "content": "<|reserved_special_token_109|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128115": { + "content": "<|reserved_special_token_110|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128116": { + "content": "<|reserved_special_token_111|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128117": { + "content": "<|reserved_special_token_112|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128118": { + "content": "<|reserved_special_token_113|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128119": { + "content": "<|reserved_special_token_114|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128120": { + "content": "<|reserved_special_token_115|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128121": { + "content": "<|reserved_special_token_116|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128122": { + "content": "<|reserved_special_token_117|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128123": { + "content": "<|reserved_special_token_118|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128124": { + "content": "<|reserved_special_token_119|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128125": { + "content": "<|reserved_special_token_120|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128126": { + "content": "<|reserved_special_token_121|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128127": { + "content": "<|reserved_special_token_122|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128128": { + "content": "<|reserved_special_token_123|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128129": { + "content": "<|reserved_special_token_124|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128130": { + "content": "<|reserved_special_token_125|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128131": { + "content": "<|reserved_special_token_126|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128132": { + "content": "<|reserved_special_token_127|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128133": { + "content": "<|reserved_special_token_128|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128134": { + "content": "<|reserved_special_token_129|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128135": { + "content": "<|reserved_special_token_130|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128136": { + "content": "<|reserved_special_token_131|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128137": { + "content": "<|reserved_special_token_132|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128138": { + "content": "<|reserved_special_token_133|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128139": { + "content": "<|reserved_special_token_134|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128140": { + "content": "<|reserved_special_token_135|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128141": { + "content": "<|reserved_special_token_136|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128142": { + "content": "<|reserved_special_token_137|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128143": { + "content": "<|reserved_special_token_138|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128144": { + "content": "<|reserved_special_token_139|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128145": { + "content": "<|reserved_special_token_140|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128146": { + "content": "<|reserved_special_token_141|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128147": { + "content": "<|reserved_special_token_142|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128148": { + "content": "<|reserved_special_token_143|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128149": { + "content": "<|reserved_special_token_144|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128150": { + "content": "<|reserved_special_token_145|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128151": { + "content": "<|reserved_special_token_146|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128152": { + "content": "<|reserved_special_token_147|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128153": { + "content": "<|reserved_special_token_148|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128154": { + "content": "<|reserved_special_token_149|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128155": { + "content": "<|reserved_special_token_150|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128156": { + "content": "<|reserved_special_token_151|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128157": { + "content": "<|reserved_special_token_152|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128158": { + "content": "<|reserved_special_token_153|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128159": { + "content": "<|reserved_special_token_154|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128160": { + "content": "<|reserved_special_token_155|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128161": { + "content": "<|reserved_special_token_156|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128162": { + "content": "<|reserved_special_token_157|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128163": { + "content": "<|reserved_special_token_158|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128164": { + "content": "<|reserved_special_token_159|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128165": { + "content": "<|reserved_special_token_160|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128166": { + "content": "<|reserved_special_token_161|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128167": { + "content": "<|reserved_special_token_162|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128168": { + "content": "<|reserved_special_token_163|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128169": { + "content": "<|reserved_special_token_164|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128170": { + "content": "<|reserved_special_token_165|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128171": { + "content": "<|reserved_special_token_166|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128172": { + "content": "<|reserved_special_token_167|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128173": { + "content": "<|reserved_special_token_168|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128174": { + "content": "<|reserved_special_token_169|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128175": { + "content": "<|reserved_special_token_170|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128176": { + "content": "<|reserved_special_token_171|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128177": { + "content": "<|reserved_special_token_172|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128178": { + "content": "<|reserved_special_token_173|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128179": { + "content": "<|reserved_special_token_174|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128180": { + "content": "<|reserved_special_token_175|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128181": { + "content": "<|reserved_special_token_176|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128182": { + "content": "<|reserved_special_token_177|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128183": { + "content": "<|reserved_special_token_178|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128184": { + "content": "<|reserved_special_token_179|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128185": { + "content": "<|reserved_special_token_180|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128186": { + "content": "<|reserved_special_token_181|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128187": { + "content": "<|reserved_special_token_182|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128188": { + "content": "<|reserved_special_token_183|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128189": { + "content": "<|reserved_special_token_184|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128190": { + "content": "<|reserved_special_token_185|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128191": { + "content": "<|reserved_special_token_186|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128192": { + "content": "<|reserved_special_token_187|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128193": { + "content": "<|reserved_special_token_188|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128194": { + "content": "<|reserved_special_token_189|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128195": { + "content": "<|reserved_special_token_190|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128196": { + "content": "<|reserved_special_token_191|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128197": { + "content": "<|reserved_special_token_192|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128198": { + "content": "<|reserved_special_token_193|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128199": { + "content": "<|reserved_special_token_194|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128200": { + "content": "<|reserved_special_token_195|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128201": { + "content": "<|reserved_special_token_196|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128202": { + "content": "<|reserved_special_token_197|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128203": { + "content": "<|reserved_special_token_198|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128204": { + "content": "<|reserved_special_token_199|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128205": { + "content": "<|reserved_special_token_200|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128206": { + "content": "<|reserved_special_token_201|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128207": { + "content": "<|reserved_special_token_202|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128208": { + "content": "<|reserved_special_token_203|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128209": { + "content": "<|reserved_special_token_204|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128210": { + "content": "<|reserved_special_token_205|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128211": { + "content": "<|reserved_special_token_206|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128212": { + "content": "<|reserved_special_token_207|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128213": { + "content": "<|reserved_special_token_208|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128214": { + "content": "<|reserved_special_token_209|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128215": { + "content": "<|reserved_special_token_210|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128216": { + "content": "<|reserved_special_token_211|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128217": { + "content": "<|reserved_special_token_212|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128218": { + "content": "<|reserved_special_token_213|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128219": { + "content": "<|reserved_special_token_214|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128220": { + "content": "<|reserved_special_token_215|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128221": { + "content": "<|reserved_special_token_216|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128222": { + "content": "<|reserved_special_token_217|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128223": { + "content": "<|reserved_special_token_218|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128224": { + "content": "<|reserved_special_token_219|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128225": { + "content": "<|reserved_special_token_220|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128226": { + "content": "<|reserved_special_token_221|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128227": { + "content": "<|reserved_special_token_222|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128228": { + "content": "<|reserved_special_token_223|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128229": { + "content": "<|reserved_special_token_224|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128230": { + "content": "<|reserved_special_token_225|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128231": { + "content": "<|reserved_special_token_226|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128232": { + "content": "<|reserved_special_token_227|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128233": { + "content": "<|reserved_special_token_228|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128234": { + "content": "<|reserved_special_token_229|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128235": { + "content": "<|reserved_special_token_230|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128236": { + "content": "<|reserved_special_token_231|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128237": { + "content": "<|reserved_special_token_232|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128238": { + "content": "<|reserved_special_token_233|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128239": { + "content": "<|reserved_special_token_234|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128240": { + "content": "<|reserved_special_token_235|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128241": { + "content": "<|reserved_special_token_236|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128242": { + "content": "<|reserved_special_token_237|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128243": { + "content": "<|reserved_special_token_238|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128244": { + "content": "<|reserved_special_token_239|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128245": { + "content": "<|reserved_special_token_240|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128246": { + "content": "<|reserved_special_token_241|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128247": { + "content": "<|reserved_special_token_242|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128248": { + "content": "<|reserved_special_token_243|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128249": { + "content": "<|reserved_special_token_244|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128250": { + "content": "<|reserved_special_token_245|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128251": { + "content": "<|reserved_special_token_246|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128252": { + "content": "<|reserved_special_token_247|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128253": { + "content": "<|reserved_special_token_248|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128254": { + "content": "<|reserved_special_token_249|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128255": { + "content": "<|reserved_special_token_250|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128256": { + "content": "<|end-of-read|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128257": { + "content": "<|end-of-write|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + } + }, + "additional_special_tokens": [ + "<|end-of-read|>", + "<|end-of-write|>" + ], + "bos_token": "<|begin_of_text|>", + "clean_up_tokenization_spaces": true, + "eos_token": "<|eot_id|>", + "extra_special_tokens": {}, + "model_input_names": [ + "input_ids", + "attention_mask" + ], + "model_max_length": 1000000000000000019884624838656, + "pad_token": "<|eot_id|>", + "padding_side": "left", + "split_special_tokens": false, + "tokenizer_class": "PreTrainedTokenizerFast" +} diff --git a/training_args.bin b/training_args.bin new file mode 100644 index 0000000..707dc5c --- /dev/null +++ b/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc565ab922336aa0264222e0b5205df4aee0c58c7792a0d0bf59e20481461883 +size 5969