commit a88745c2758c93edb1e7b44de222b0d5b6a11fa8 Author: ModelHub XC Date: Fri Sep 4 02:32:12 2026 +0800 初始化项目，由ModelHub XC社区提供模型 Model: Weyaxi/Einstein-v3-7B Source: Original Platform diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..7189f04 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,49 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bin.* filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zstandard filter=lfs diff=lfs merge=lfs -text +*.tfevents* filter=lfs diff=lfs merge=lfs -text +*.db* filter=lfs diff=lfs merge=lfs -text +*.ark* filter=lfs diff=lfs merge=lfs -text +**/*ckpt*data* filter=lfs diff=lfs merge=lfs -text +**/*ckpt*.meta filter=lfs diff=lfs merge=lfs -text +**/*ckpt*.index filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.gguf* filter=lfs diff=lfs merge=lfs -text +*.ggml filter=lfs diff=lfs merge=lfs -text +*.llamafile* filter=lfs diff=lfs merge=lfs -text +*.pt2 filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text + +data/merged_all.json filter=lfs diff=lfs merge=lfs -text \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..d4e8ce0 --- /dev/null +++ b/README.md @@ -0,0 +1,275 @@ +--- +license: apache-2.0 +tags: +- axolotl +- generated_from_trainer +base_model: mistralai/Mistral-7B-v0.1 +model-index: +- name: Einstein-v3-7B + results: + - task: + type: text-generation + name: Text Generation + dataset: + name: AI2 Reasoning Challenge (25-Shot) + type: ai2_arc + config: ARC-Challenge + split: test + args: + num_few_shot: 25 + metrics: + - type: acc_norm + value: 62.29 + name: normalized accuracy + source: + url: https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard?query=PulsarAI/Einstein-v3-7B + name: Open LLM Leaderboard + - task: + type: text-generation + name: Text Generation + dataset: + name: HellaSwag (10-Shot) + type: hellaswag + split: validation + args: + num_few_shot: 10 + metrics: + - type: acc_norm + value: 83.01 + name: normalized accuracy + source: + url: https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard?query=PulsarAI/Einstein-v3-7B + name: Open LLM Leaderboard + - task: + type: text-generation + name: Text Generation + dataset: + name: MMLU (5-Shot) + type: cais/mmlu + config: all + split: test + args: + num_few_shot: 5 + metrics: + - type: acc + value: 63.32 + name: accuracy + source: + url: https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard?query=PulsarAI/Einstein-v3-7B + name: Open LLM Leaderboard + - task: + type: text-generation + name: Text Generation + dataset: + name: TruthfulQA (0-shot) + type: truthful_qa + config: multiple_choice + split: validation + args: + num_few_shot: 0 + metrics: + - type: mc2 + value: 51.18 + source: + url: https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard?query=PulsarAI/Einstein-v3-7B + name: Open LLM Leaderboard + - task: + type: text-generation + name: Text Generation + dataset: + name: Winogrande (5-shot) + type: winogrande + config: winogrande_xl + split: validation + args: + num_few_shot: 5 + metrics: + - type: acc + value: 79.95 + name: accuracy + source: + url: https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard?query=PulsarAI/Einstein-v3-7B + name: Open LLM Leaderboard + - task: + type: text-generation + name: Text Generation + dataset: + name: GSM8k (5-shot) + type: gsm8k + config: main + split: test + args: + num_few_shot: 5 + metrics: + - type: acc + value: 44.81 + name: accuracy + source: + url: https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard?query=PulsarAI/Einstein-v3-7B + name: Open LLM Leaderboard +--- + + + +![image/png](https://cdn-uploads.huggingface.co/production/uploads/6468ce47e134d050a58aa89c/4EDhRWwZgVo8Pnqp1afPM.png) + +[Built with Axolotl](https://github.com/OpenAccess-AI-Collective/axolotl) +
See axolotl config + +axolotl version: `0.4.0` +```yaml +base_model: mistralai/Mistral-7B-v0.1 +model_type: MistralForCausalLM +tokenizer_type: LlamaTokenizer +is_mistral_derived_model: true + +load_in_8bit: false +load_in_4bit: false +strict: false + +chat_template: chatml +datasets: + - path: data/merged_all.json + ds_type: json + type: alpaca + conversation: chatml + + - path: data/capybara_sharegpt.json + ds_type: json + type: sharegpt + conversation: chatml + + - path: data/synthia-v1.3_sharegpt_12500.json + ds_type: json + type: sharegpt + conversation: chatml + +dataset_prepared_path: last_run_prepared +val_set_size: 0.005 +output_dir: ./Einstein-v3-model + +sequence_len: 8192 +sample_packing: true +pad_to_sequence_len: true +eval_sample_packing: false + +wandb_project: huggingface +wandb_entity: +wandb_watch: +wandb_name: +wandb_log_model: +hub_model_id: Weyaxi/Einstein-v3-7B + +save_safetensors: true + +gradient_accumulation_steps: 4 +micro_batch_size: 1 +num_epochs: 1 +optimizer: adamw_bnb_8bit +lr_scheduler: cosine +learning_rate: 0.000005 + +train_on_inputs: false +group_by_length: false +bf16: true +fp16: false +tf32: false + +gradient_checkpointing: true +early_stopping_patience: +resume_from_checkpoint: +local_rank: +logging_steps: 1 +xformers_attention: +flash_attention: true + +warmup_steps: 10 +evals_per_epoch: 4 +eval_table_size: +eval_table_max_new_tokens: 128 +saves_per_epoch: 2 +debug: + +deepspeed: zero3_bf16.json +weight_decay: 0.0 +fsdp: +fsdp_config: +special_tokens: + bos_token: "" + eos_token: "<|im_end|>" + unk_token: "" +tokens: + - "<|im_start|>" +``` + +

+ +# Einstein-v3-7B + +This model is a fine-tuned version of [mistralai/Mistral-7B-v0.1](https://huggingface.co/mistralai/Mistral-7B-v0.1) on the None dataset. +It achieves the following results on the evaluation set: +- Loss: 0.5059 + +## Model description + +More information needed + +## Intended uses & limitations + +More information needed + +## Training and evaluation data + +More information needed + +## Training procedure + +### Training hyperparameters + +The following hyperparameters were used during training: +- learning_rate: 5e-06 +- train_batch_size: 1 +- eval_batch_size: 1 +- seed: 42 +- distributed_type: multi-GPU +- num_devices: 8 +- gradient_accumulation_steps: 4 +- total_train_batch_size: 32 +- total_eval_batch_size: 8 +- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08 +- lr_scheduler_type: cosine +- lr_scheduler_warmup_steps: 10 +- num_epochs: 1 + +### Training results + +| Training Loss | Epoch | Step | Validation Loss | +|:-------------:|:-----:|:----:|:---------------:| +| 1.038 | 0.0 | 1 | 1.1250 | +| 0.5254 | 0.25 | 107 | 0.5754 | +| 0.5144 | 0.5 | 214 | 0.5360 | +| 0.483 | 0.75 | 321 | 0.5118 | +| 0.4674 | 1.0 | 428 | 0.5059 | + + +### Framework versions + +- Transformers 4.38.0.dev0 +- Pytorch 2.1.2+cu118 +- Datasets 2.16.1 +- Tokenizers 0.15.0 + +# [Open LLM Leaderboard Evaluation Results](https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard) +Detailed results can be found [here](https://huggingface.co/datasets/open-llm-leaderboard/details_PulsarAI__Einstein-v3-7B) + +| Metric |Value| +|---------------------------------|----:| +|Avg. |64.09| +|AI2 Reasoning Challenge (25-Shot)|62.29| +|HellaSwag (10-Shot) |83.01| +|MMLU (5-Shot) |63.32| +|TruthfulQA (0-shot) |51.18| +|Winogrande (5-shot) |79.95| +|GSM8k (5-shot) |44.81| + diff --git a/added_tokens.json b/added_tokens.json new file mode 100644 index 0000000..e36863d --- /dev/null +++ b/added_tokens.json @@ -0,0 +1,4 @@ +{ + "<|im_end|>": 32000, + "<|im_start|>": 32001 +} diff --git a/config.json b/config.json new file mode 100644 index 0000000..7b4a008 --- /dev/null +++ b/config.json @@ -0,0 +1,26 @@ +{ + "_name_or_path": "mistralai/Mistral-7B-v0.1", + "architectures": [ + "MistralForCausalLM" + ], + "attention_dropout": 0.0, + "bos_token_id": 1, + "eos_token_id": 32000, + "hidden_act": "silu", + "hidden_size": 4096, + "initializer_range": 0.02, + "intermediate_size": 14336, + "max_position_embeddings": 32768, + "model_type": "mistral", + "num_attention_heads": 32, + "num_hidden_layers": 32, + "num_key_value_heads": 8, + "rms_norm_eps": 1e-05, + "rope_theta": 10000.0, + "sliding_window": 4096, + "tie_word_embeddings": false, + "torch_dtype": "bfloat16", + "transformers_version": "4.38.0.dev0", + "use_cache": false, + "vocab_size": 32002 +} diff --git a/configuration.json b/configuration.json new file mode 100644 index 0000000..bbeeda1 --- /dev/null +++ b/configuration.json @@ -0,0 +1 @@ +{"framework": "pytorch", "task": "text-generation", "allow_remote": true} \ No newline at end of file diff --git a/data/capybara_sharegpt.json b/data/capybara_sharegpt.json new file mode 100644 index 0000000..b46d8b6 --- /dev/null +++ b/data/capybara_sharegpt.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a1947d28999416a2f468d1e09654cfdfc9bab8ccd03aa184598d20f0000dd6e4 +size 76361785 diff --git a/data/merged_all.json b/data/merged_all.json new file mode 100644 index 0000000..9c2935e --- /dev/null +++ b/data/merged_all.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f915219dc691de207f9b8b448f005386330035f3c7f6cc6379dd183ac7662d2f +size 306656943 diff --git a/data/remove_empty_output.py b/data/remove_empty_output.py new file mode 100644 index 0000000..67a4176 --- /dev/null +++ b/data/remove_empty_output.py @@ -0,0 +1,13 @@ +import json + +with open('merged_all.json', 'r') as file: + data = json.load(file) + +print(f"Normal len: {len(data)}") + +data = [row for row in data if row["output"] != ""] + +print(f"After len: {len(data)}") + +with open('merged_all.json', 'w') as file: + json.dump(data, file, indent=1) \ No newline at end of file diff --git a/data/synthia-v1.3_sharegpt_12500.json b/data/synthia-v1.3_sharegpt_12500.json new file mode 100644 index 0000000..8c0b164 --- /dev/null +++ b/data/synthia-v1.3_sharegpt_12500.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dbdbc7413a3c7fc65a900518f0db8627bb5ced53e1e8ee82613d09856c1b3b70 +size 30638009 diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..8065c64 --- /dev/null +++ b/generation_config.json @@ -0,0 +1,7 @@ +{ + "_from_model_config": true, + "bos_token_id": 1, + "do_sample": true, + "eos_token_id": 2, + "transformers_version": "4.38.0.dev0" +} diff --git a/global_step428/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt b/global_step428/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt new file mode 100644 index 0000000..9db11aa --- /dev/null +++ b/global_step428/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:31eb677de80948991321bfe446bcdaf4165d3f8c9efa96cfadccfaf3a07e6432 +size 5435499827 diff --git a/global_step428/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt b/global_step428/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt new file mode 100644 index 0000000..57caf66 --- /dev/null +++ b/global_step428/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:09e4495311047938552b63e0739ae4a551323c70509aed4b806aae92544537a7 +size 5435499827 diff --git a/global_step428/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt b/global_step428/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt new file mode 100644 index 0000000..2854bb9 --- /dev/null +++ b/global_step428/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5e1e42c3abfc5e8f3bf1af140fef44ff8f8441a9256840b045d424c01026ce1e +size 5435499827 diff --git a/global_step428/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt b/global_step428/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt new file mode 100644 index 0000000..18ad581 --- /dev/null +++ b/global_step428/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:398bfa2896865cf30cef24ec6b68f47976d7f16cb3df673efceb7804d4b93241 +size 5435499827 diff --git a/global_step428/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt b/global_step428/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt new file mode 100644 index 0000000..8b451e2 --- /dev/null +++ b/global_step428/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a62bcb45fde6ec6327735c7a280ac3ce1f417f8924c8774a8c82a17cfbf4c295 +size 5435499827 diff --git a/global_step428/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt b/global_step428/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt new file mode 100644 index 0000000..b9d6d27 --- /dev/null +++ b/global_step428/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eb1d3d30b3e766144ee852f023bd74030024ab52c9f32771cdef96fb92ea360a +size 5435499827 diff --git a/global_step428/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt b/global_step428/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt new file mode 100644 index 0000000..34e29d7 --- /dev/null +++ b/global_step428/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:874cb020cb85f973bcb6aabf19b131ea5f4119dfb72f80578d61c90b95b95b8f +size 5435499827 diff --git a/global_step428/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt b/global_step428/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt new file mode 100644 index 0000000..74b6253 --- /dev/null +++ b/global_step428/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:68f1d3311e2ce3605022c06ddc3064dd9e31f60d59b8cef8e11e6d4ffb6c19c7 +size 5435499827 diff --git a/global_step428/zero_pp_rank_0_mp_rank_00_model_states.pt b/global_step428/zero_pp_rank_0_mp_rank_00_model_states.pt new file mode 100644 index 0000000..dc0054a --- /dev/null +++ b/global_step428/zero_pp_rank_0_mp_rank_00_model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:57e8c2eb8c8299c2c958127742fbf228e53106a6568c4971b8afabbd0109ff90 +size 153829 diff --git a/global_step428/zero_pp_rank_1_mp_rank_00_model_states.pt b/global_step428/zero_pp_rank_1_mp_rank_00_model_states.pt new file mode 100644 index 0000000..1d02f48 --- /dev/null +++ b/global_step428/zero_pp_rank_1_mp_rank_00_model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5e282c8dfa4e2bc4ca41f74c0855b60c0b95c8b57349ed0f767b0ac93c6d90e4 +size 153829 diff --git a/global_step428/zero_pp_rank_2_mp_rank_00_model_states.pt b/global_step428/zero_pp_rank_2_mp_rank_00_model_states.pt new file mode 100644 index 0000000..f15b3a7 --- /dev/null +++ b/global_step428/zero_pp_rank_2_mp_rank_00_model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a05bcb052dcc69a98918a82611f0e1e96657c85e106f480eda0074811399d0ca +size 153829 diff --git a/global_step428/zero_pp_rank_3_mp_rank_00_model_states.pt b/global_step428/zero_pp_rank_3_mp_rank_00_model_states.pt new file mode 100644 index 0000000..2c2457c --- /dev/null +++ b/global_step428/zero_pp_rank_3_mp_rank_00_model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:55815a63565a06e023966b6a11030d06cfdc86e64fac8c811f57b00ab0e1e847 +size 153829 diff --git a/global_step428/zero_pp_rank_4_mp_rank_00_model_states.pt b/global_step428/zero_pp_rank_4_mp_rank_00_model_states.pt new file mode 100644 index 0000000..efc314a --- /dev/null +++ b/global_step428/zero_pp_rank_4_mp_rank_00_model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:43ea8ade0f653c47a1ac50680612d8e5218345ab1b86cb35384d76d563dd75cc +size 153829 diff --git a/global_step428/zero_pp_rank_5_mp_rank_00_model_states.pt b/global_step428/zero_pp_rank_5_mp_rank_00_model_states.pt new file mode 100644 index 0000000..58be816 --- /dev/null +++ b/global_step428/zero_pp_rank_5_mp_rank_00_model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9650144687908dce67d9631417bd0ee2e129d6fa67ed87825da1ef4167112639 +size 153829 diff --git a/global_step428/zero_pp_rank_6_mp_rank_00_model_states.pt b/global_step428/zero_pp_rank_6_mp_rank_00_model_states.pt new file mode 100644 index 0000000..54708bf --- /dev/null +++ b/global_step428/zero_pp_rank_6_mp_rank_00_model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1fb9e75a82b054c216c566b9165d02270f6da8be51ccc5c3ec334b0c2004d830 +size 153829 diff --git a/global_step428/zero_pp_rank_7_mp_rank_00_model_states.pt b/global_step428/zero_pp_rank_7_mp_rank_00_model_states.pt new file mode 100644 index 0000000..61b3bcc --- /dev/null +++ b/global_step428/zero_pp_rank_7_mp_rank_00_model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:47f5259391522659ec37fb679a3619fb26f04f6a1908753f5d9f85e2dda46e3f +size 153829 diff --git a/latest b/latest new file mode 100644 index 0000000..db93d1b --- /dev/null +++ b/latest @@ -0,0 +1 @@ +global_step428 \ No newline at end of file diff --git a/model-00001-of-00003.safetensors b/model-00001-of-00003.safetensors new file mode 100644 index 0000000..4515705 --- /dev/null +++ b/model-00001-of-00003.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:063ec4ac52401aa5fafb6c741c2293846b56a9ed57ce550187b064e10360571a +size 4943178720 diff --git a/model-00002-of-00003.safetensors b/model-00002-of-00003.safetensors new file mode 100644 index 0000000..e3185f4 --- /dev/null +++ b/model-00002-of-00003.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a756b6d20f8a9a4b7d1703542b0bd451bee0ea7a3301d5e5bc3b4db6d9d0dd6 +size 4999819336 diff --git a/model-00003-of-00003.safetensors b/model-00003-of-00003.safetensors new file mode 100644 index 0000000..b70a893 --- /dev/null +++ b/model-00003-of-00003.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:debb562fe35af7dc1c41715d66dd016dd7da37470a2f0ff3db7844e0207240aa +size 4540532728 diff --git a/model.safetensors.index.json b/model.safetensors.index.json new file mode 100644 index 0000000..71e2076 --- /dev/null +++ b/model.safetensors.index.json @@ -0,0 +1,298 @@ +{ + "metadata": { + "total_size": 14483496960 + }, + "weight_map": { + "lm_head.weight": "model-00003-of-00003.safetensors", + "model.embed_tokens.weight": "model-00001-of-00003.safetensors", + "model.layers.0.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.0.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.0.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.1.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.1.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.1.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.10.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.10.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.10.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.11.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.11.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.11.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.12.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.12.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.12.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.13.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.13.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.13.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.14.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.14.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.14.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.15.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.15.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.15.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.16.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.16.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.16.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.17.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.17.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.17.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.18.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.18.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.18.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.19.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.19.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.19.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.2.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.2.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.2.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.20.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.20.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.20.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.21.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.21.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.21.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.22.input_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.22.mlp.down_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.22.mlp.up_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.23.input_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.23.mlp.down_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.23.mlp.up_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.24.input_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.24.mlp.down_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.24.mlp.up_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.25.input_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.25.mlp.down_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.25.mlp.up_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.26.input_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.26.mlp.down_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.26.mlp.up_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.27.input_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.27.mlp.down_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.27.mlp.up_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.28.input_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.28.mlp.down_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.28.mlp.up_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.29.input_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.29.mlp.down_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.29.mlp.up_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.3.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.3.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.3.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.30.input_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.30.mlp.down_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.30.mlp.up_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.31.input_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.31.mlp.down_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.31.mlp.up_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.4.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.4.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.4.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.5.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.5.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.5.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.6.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.6.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.6.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.7.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.7.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.7.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.8.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.8.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.8.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.9.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.9.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.9.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.norm.weight": "model-00003-of-00003.safetensors" + } +} diff --git a/remove_empty_output.py b/remove_empty_output.py new file mode 100644 index 0000000..67a4176 --- /dev/null +++ b/remove_empty_output.py @@ -0,0 +1,13 @@ +import json + +with open('merged_all.json', 'r') as file: + data = json.load(file) + +print(f"Normal len: {len(data)}") + +data = [row for row in data if row["output"] != ""] + +print(f"After len: {len(data)}") + +with open('merged_all.json', 'w') as file: + json.dump(data, file, indent=1) \ No newline at end of file diff --git a/rng_state_0.pth b/rng_state_0.pth new file mode 100644 index 0000000..97f51b4 --- /dev/null +++ b/rng_state_0.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a1bec598899f9d59e70c1b4705ce420a1e0a670957b6c8153a589880068ae5a4 +size 15984 diff --git a/rng_state_1.pth b/rng_state_1.pth new file mode 100644 index 0000000..08e59ac --- /dev/null +++ b/rng_state_1.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c60d2348aae518f4c44693db9c9b4b3a3299c556e7f0a86c188b2e4c3e364a7c +size 15984 diff --git a/rng_state_2.pth b/rng_state_2.pth new file mode 100644 index 0000000..20a24c1 --- /dev/null +++ b/rng_state_2.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ffe5a79d3bcb4ce033de360bc765e616316e3562aba25887cd85c4adbb935abf +size 15984 diff --git a/rng_state_3.pth b/rng_state_3.pth new file mode 100644 index 0000000..54050f6 --- /dev/null +++ b/rng_state_3.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a9a9d1f6e22677721841890e6a27855857e6840137650d609eb8e4ac13b71d29 +size 15984 diff --git a/rng_state_4.pth b/rng_state_4.pth new file mode 100644 index 0000000..263aae4 --- /dev/null +++ b/rng_state_4.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bcac4ff84388a6a4fe3bcae6207c68b2ee5528fb3b6de8cc3588fe1975462aa5 +size 15984 diff --git a/rng_state_5.pth b/rng_state_5.pth new file mode 100644 index 0000000..942ed5d --- /dev/null +++ b/rng_state_5.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:33fce3cdf5c1b8a8a291e0c73b384e3ad5252640e21e942b44b26b8b0928ffa9 +size 15984 diff --git a/rng_state_6.pth b/rng_state_6.pth new file mode 100644 index 0000000..57789be --- /dev/null +++ b/rng_state_6.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:919e675f3bcaf4f3c8ba35cd8debf85aec3bbc3c8e5019b74431e0a314e4d37a +size 15984 diff --git a/rng_state_7.pth b/rng_state_7.pth new file mode 100644 index 0000000..b32d6e2 --- /dev/null +++ b/rng_state_7.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8bf6479ce82b88efc6a72a8ee512162b3d0ecab972817296d38ab9c448bb8d96 +size 15984 diff --git a/scheduler.pt b/scheduler.pt new file mode 100644 index 0000000..698c10e --- /dev/null +++ b/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ee8c731be5df723d260a08657a32a14cf7657c00593a1a4bb80bc7c00b297026 +size 1064 diff --git a/special_tokens_map.json b/special_tokens_map.json new file mode 100644 index 0000000..40b1c6d --- /dev/null +++ b/special_tokens_map.json @@ -0,0 +1,24 @@ +{ + "bos_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "<|im_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": "", + "unk_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + } +} diff --git a/tokenizer.model b/tokenizer.model new file mode 100644 index 0000000..8b443ef --- /dev/null +++ b/tokenizer.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dadfd56d766715c61d2ef780a525ab43b8e6da4de6865bda3d95fdef5e134055 +size 493443 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..69ccf66 --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,61 @@ +{ + "add_bos_token": true, + "add_eos_token": false, + "added_tokens_decoder": { + "0": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "1": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "2": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32000": { + "content": "<|im_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32001": { + "content": "<|im_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + } + }, + "additional_special_tokens": [], + "bos_token": "", + "chat_template": "{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% set system_message = messages[0]['content'] %}{% else %}{% set loop_messages = messages %}{% set system_message = 'You are a helpful assistant.' %}{% endif %}{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% for message in loop_messages %}{% if loop.index0 == 0 %}{{'<|im_start|>system\n' + system_message + '<|im_end|>\n'}}{% endif %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}", + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "legacy": true, + "model_max_length": 1000000000000000019884624838656, + "pad_token": "", + "sp_model_kwargs": {}, + "spaces_between_special_tokens": false, + "tokenizer_class": "LlamaTokenizer", + "trust_remote_code": false, + "unk_token": "", + "use_default_system_prompt": false, + "use_fast": true +} diff --git a/trainer_state.json b/trainer_state.json new file mode 100644 index 0000000..13a1e9e --- /dev/null +++ b/trainer_state.json @@ -0,0 +1,2629 @@ +{ + "best_metric": 0.5059249997138977, + "best_model_checkpoint": "./Einstein-v3-model/checkpoint-428", + "epoch": 0.9994162288382954, + "eval_steps": 107, + "global_step": 428, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.0, + "learning_rate": 5.000000000000001e-07, + "loss": 1.038, + "step": 1 + }, + { + "epoch": 0.0, + "eval_loss": 1.125040888786316, + "eval_runtime": 969.9573, + "eval_samples_per_second": 1.161, + "eval_steps_per_second": 0.145, + "step": 1 + }, + { + "epoch": 0.0, + "learning_rate": 1.0000000000000002e-06, + "loss": 1.0692, + "step": 2 + }, + { + "epoch": 0.01, + "learning_rate": 1.5e-06, + "loss": 1.0161, + "step": 3 + }, + { + "epoch": 0.01, + "learning_rate": 2.0000000000000003e-06, + "loss": 0.9363, + "step": 4 + }, + { + "epoch": 0.01, + "learning_rate": 2.5e-06, + "loss": 0.8647, + "step": 5 + }, + { + "epoch": 0.01, + "learning_rate": 3e-06, + "loss": 0.9888, + "step": 6 + }, + { + "epoch": 0.02, + "learning_rate": 3.5e-06, + "loss": 0.8645, + "step": 7 + }, + { + "epoch": 0.02, + "learning_rate": 4.000000000000001e-06, + "loss": 0.763, + "step": 8 + }, + { + "epoch": 0.02, + "learning_rate": 4.5e-06, + "loss": 0.8058, + "step": 9 + }, + { + "epoch": 0.02, + "learning_rate": 5e-06, + "loss": 0.789, + "step": 10 + }, + { + "epoch": 0.03, + "learning_rate": 4.9999293917983325e-06, + "loss": 0.798, + "step": 11 + }, + { + "epoch": 0.03, + "learning_rate": 4.999717571181742e-06, + "loss": 0.7411, + "step": 12 + }, + { + "epoch": 0.03, + "learning_rate": 4.9993645501152485e-06, + "loss": 0.748, + "step": 13 + }, + { + "epoch": 0.03, + "learning_rate": 4.998870348539797e-06, + "loss": 0.7312, + "step": 14 + }, + { + "epoch": 0.04, + "learning_rate": 4.998234994371135e-06, + "loss": 0.7179, + "step": 15 + }, + { + "epoch": 0.04, + "learning_rate": 4.997458523498236e-06, + "loss": 0.7158, + "step": 16 + }, + { + "epoch": 0.04, + "learning_rate": 4.996540979781269e-06, + "loss": 0.6975, + "step": 17 + }, + { + "epoch": 0.04, + "learning_rate": 4.995482415049123e-06, + "loss": 0.6809, + "step": 18 + }, + { + "epoch": 0.04, + "learning_rate": 4.99428288909648e-06, + "loss": 0.6709, + "step": 19 + }, + { + "epoch": 0.05, + "learning_rate": 4.992942469680437e-06, + "loss": 0.6778, + "step": 20 + }, + { + "epoch": 0.05, + "learning_rate": 4.991461232516675e-06, + "loss": 0.69, + "step": 21 + }, + { + "epoch": 0.05, + "learning_rate": 4.989839261275191e-06, + "loss": 0.6518, + "step": 22 + }, + { + "epoch": 0.05, + "learning_rate": 4.988076647575562e-06, + "loss": 0.6712, + "step": 23 + }, + { + "epoch": 0.06, + "learning_rate": 4.986173490981773e-06, + "loss": 0.6429, + "step": 24 + }, + { + "epoch": 0.06, + "learning_rate": 4.984129898996599e-06, + "loss": 0.682, + "step": 25 + }, + { + "epoch": 0.06, + "learning_rate": 4.981945987055521e-06, + "loss": 0.6661, + "step": 26 + }, + { + "epoch": 0.06, + "learning_rate": 4.979621878520217e-06, + "loss": 0.6287, + "step": 27 + }, + { + "epoch": 0.07, + "learning_rate": 4.977157704671585e-06, + "loss": 0.6642, + "step": 28 + }, + { + "epoch": 0.07, + "learning_rate": 4.974553604702332e-06, + "loss": 0.6644, + "step": 29 + }, + { + "epoch": 0.07, + "learning_rate": 4.971809725709112e-06, + "loss": 0.6453, + "step": 30 + }, + { + "epoch": 0.07, + "learning_rate": 4.968926222684213e-06, + "loss": 0.6199, + "step": 31 + }, + { + "epoch": 0.07, + "learning_rate": 4.965903258506806e-06, + "loss": 0.6038, + "step": 32 + }, + { + "epoch": 0.08, + "learning_rate": 4.9627410039337426e-06, + "loss": 0.6493, + "step": 33 + }, + { + "epoch": 0.08, + "learning_rate": 4.959439637589909e-06, + "loss": 0.6304, + "step": 34 + }, + { + "epoch": 0.08, + "learning_rate": 4.9559993459581375e-06, + "loss": 0.6692, + "step": 35 + }, + { + "epoch": 0.08, + "learning_rate": 4.952420323368673e-06, + "loss": 0.5978, + "step": 36 + }, + { + "epoch": 0.09, + "learning_rate": 4.948702771988195e-06, + "loss": 0.6318, + "step": 37 + }, + { + "epoch": 0.09, + "learning_rate": 4.944846901808397e-06, + "loss": 0.5838, + "step": 38 + }, + { + "epoch": 0.09, + "learning_rate": 4.940852930634126e-06, + "loss": 0.5884, + "step": 39 + }, + { + "epoch": 0.09, + "learning_rate": 4.936721084071079e-06, + "loss": 0.5955, + "step": 40 + }, + { + "epoch": 0.1, + "learning_rate": 4.932451595513063e-06, + "loss": 0.5704, + "step": 41 + }, + { + "epoch": 0.1, + "learning_rate": 4.928044706128803e-06, + "loss": 0.5711, + "step": 42 + }, + { + "epoch": 0.1, + "learning_rate": 4.923500664848327e-06, + "loss": 0.5886, + "step": 43 + }, + { + "epoch": 0.1, + "learning_rate": 4.918819728348901e-06, + "loss": 0.5802, + "step": 44 + }, + { + "epoch": 0.11, + "learning_rate": 4.9140021610405335e-06, + "loss": 0.6221, + "step": 45 + }, + { + "epoch": 0.11, + "learning_rate": 4.909048235051033e-06, + "loss": 0.5859, + "step": 46 + }, + { + "epoch": 0.11, + "learning_rate": 4.903958230210647e-06, + "loss": 0.5906, + "step": 47 + }, + { + "epoch": 0.11, + "learning_rate": 4.8987324340362445e-06, + "loss": 0.586, + "step": 48 + }, + { + "epoch": 0.11, + "learning_rate": 4.89337114171508e-06, + "loss": 0.5782, + "step": 49 + }, + { + "epoch": 0.12, + "learning_rate": 4.887874656088124e-06, + "loss": 0.6341, + "step": 50 + }, + { + "epoch": 0.12, + "learning_rate": 4.882243287632947e-06, + "loss": 0.5753, + "step": 51 + }, + { + "epoch": 0.12, + "learning_rate": 4.8764773544461895e-06, + "loss": 0.5896, + "step": 52 + }, + { + "epoch": 0.12, + "learning_rate": 4.8705771822255895e-06, + "loss": 0.567, + "step": 53 + }, + { + "epoch": 0.13, + "learning_rate": 4.864543104251587e-06, + "loss": 0.5943, + "step": 54 + }, + { + "epoch": 0.13, + "learning_rate": 4.858375461368499e-06, + "loss": 0.5942, + "step": 55 + }, + { + "epoch": 0.13, + "learning_rate": 4.852074601965261e-06, + "loss": 0.5639, + "step": 56 + }, + { + "epoch": 0.13, + "learning_rate": 4.845640881955757e-06, + "loss": 0.6058, + "step": 57 + }, + { + "epoch": 0.14, + "learning_rate": 4.839074664758705e-06, + "loss": 0.5695, + "step": 58 + }, + { + "epoch": 0.14, + "learning_rate": 4.832376321277136e-06, + "loss": 0.6233, + "step": 59 + }, + { + "epoch": 0.14, + "learning_rate": 4.825546229877439e-06, + "loss": 0.5617, + "step": 60 + }, + { + "epoch": 0.14, + "learning_rate": 4.818584776367992e-06, + "loss": 0.586, + "step": 61 + }, + { + "epoch": 0.14, + "learning_rate": 4.811492353977366e-06, + "loss": 0.5609, + "step": 62 + }, + { + "epoch": 0.15, + "learning_rate": 4.804269363332112e-06, + "loss": 0.5371, + "step": 63 + }, + { + "epoch": 0.15, + "learning_rate": 4.7969162124341354e-06, + "loss": 0.5585, + "step": 64 + }, + { + "epoch": 0.15, + "learning_rate": 4.789433316637644e-06, + "loss": 0.5548, + "step": 65 + }, + { + "epoch": 0.15, + "learning_rate": 4.781821098625691e-06, + "loss": 0.5613, + "step": 66 + }, + { + "epoch": 0.16, + "learning_rate": 4.7740799883862966e-06, + "loss": 0.5483, + "step": 67 + }, + { + "epoch": 0.16, + "learning_rate": 4.766210423188158e-06, + "loss": 0.5892, + "step": 68 + }, + { + "epoch": 0.16, + "learning_rate": 4.758212847555953e-06, + "loss": 0.5813, + "step": 69 + }, + { + "epoch": 0.16, + "learning_rate": 4.750087713245227e-06, + "loss": 0.552, + "step": 70 + }, + { + "epoch": 0.17, + "learning_rate": 4.74183547921688e-06, + "loss": 0.5478, + "step": 71 + }, + { + "epoch": 0.17, + "learning_rate": 4.733456611611233e-06, + "loss": 0.5648, + "step": 72 + }, + { + "epoch": 0.17, + "learning_rate": 4.7249515837217075e-06, + "loss": 0.5717, + "step": 73 + }, + { + "epoch": 0.17, + "learning_rate": 4.716320875968081e-06, + "loss": 0.5916, + "step": 74 + }, + { + "epoch": 0.18, + "learning_rate": 4.707564975869357e-06, + "loss": 0.5562, + "step": 75 + }, + { + "epoch": 0.18, + "learning_rate": 4.698684378016223e-06, + "loss": 0.5347, + "step": 76 + }, + { + "epoch": 0.18, + "learning_rate": 4.6896795840431155e-06, + "loss": 0.5595, + "step": 77 + }, + { + "epoch": 0.18, + "learning_rate": 4.680551102599881e-06, + "loss": 0.564, + "step": 78 + }, + { + "epoch": 0.18, + "learning_rate": 4.671299449323045e-06, + "loss": 0.5646, + "step": 79 + }, + { + "epoch": 0.19, + "learning_rate": 4.66192514680669e-06, + "loss": 0.5667, + "step": 80 + }, + { + "epoch": 0.19, + "learning_rate": 4.652428724572929e-06, + "loss": 0.5726, + "step": 81 + }, + { + "epoch": 0.19, + "learning_rate": 4.642810719041999e-06, + "loss": 0.544, + "step": 82 + }, + { + "epoch": 0.19, + "learning_rate": 4.63307167350196e-06, + "loss": 0.5673, + "step": 83 + }, + { + "epoch": 0.2, + "learning_rate": 4.623212138078004e-06, + "loss": 0.5743, + "step": 84 + }, + { + "epoch": 0.2, + "learning_rate": 4.613232669701384e-06, + "loss": 0.5605, + "step": 85 + }, + { + "epoch": 0.2, + "learning_rate": 4.603133832077953e-06, + "loss": 0.5754, + "step": 86 + }, + { + "epoch": 0.2, + "learning_rate": 4.592916195656322e-06, + "loss": 0.5578, + "step": 87 + }, + { + "epoch": 0.21, + "learning_rate": 4.582580337595636e-06, + "loss": 0.5261, + "step": 88 + }, + { + "epoch": 0.21, + "learning_rate": 4.572126841732977e-06, + "loss": 0.5864, + "step": 89 + }, + { + "epoch": 0.21, + "learning_rate": 4.561556298550379e-06, + "loss": 0.5388, + "step": 90 + }, + { + "epoch": 0.21, + "learning_rate": 4.550869305141478e-06, + "loss": 0.5662, + "step": 91 + }, + { + "epoch": 0.21, + "learning_rate": 4.5400664651777835e-06, + "loss": 0.5359, + "step": 92 + }, + { + "epoch": 0.22, + "learning_rate": 4.529148388874577e-06, + "loss": 0.5485, + "step": 93 + }, + { + "epoch": 0.22, + "learning_rate": 4.518115692956445e-06, + "loss": 0.5715, + "step": 94 + }, + { + "epoch": 0.22, + "learning_rate": 4.506969000622443e-06, + "loss": 0.5412, + "step": 95 + }, + { + "epoch": 0.22, + "learning_rate": 4.49570894151089e-06, + "loss": 0.5475, + "step": 96 + }, + { + "epoch": 0.23, + "learning_rate": 4.484336151663807e-06, + "loss": 0.5443, + "step": 97 + }, + { + "epoch": 0.23, + "learning_rate": 4.472851273490985e-06, + "loss": 0.5519, + "step": 98 + }, + { + "epoch": 0.23, + "learning_rate": 4.4612549557336975e-06, + "loss": 0.5293, + "step": 99 + }, + { + "epoch": 0.23, + "learning_rate": 4.449547853428061e-06, + "loss": 0.5545, + "step": 100 + }, + { + "epoch": 0.24, + "learning_rate": 4.437730627868028e-06, + "loss": 0.5361, + "step": 101 + }, + { + "epoch": 0.24, + "learning_rate": 4.425803946568033e-06, + "loss": 0.547, + "step": 102 + }, + { + "epoch": 0.24, + "learning_rate": 4.413768483225292e-06, + "loss": 0.532, + "step": 103 + }, + { + "epoch": 0.24, + "learning_rate": 4.401624917681743e-06, + "loss": 0.5515, + "step": 104 + }, + { + "epoch": 0.25, + "learning_rate": 4.3893739358856465e-06, + "loss": 0.5424, + "step": 105 + }, + { + "epoch": 0.25, + "learning_rate": 4.377016229852836e-06, + "loss": 0.5217, + "step": 106 + }, + { + "epoch": 0.25, + "learning_rate": 4.364552497627632e-06, + "loss": 0.5254, + "step": 107 + }, + { + "epoch": 0.25, + "eval_loss": 0.5753679871559143, + "eval_runtime": 972.8304, + "eval_samples_per_second": 1.157, + "eval_steps_per_second": 0.145, + "step": 107 + }, + { + "epoch": 0.25, + "learning_rate": 4.3519834432434095e-06, + "loss": 0.5599, + "step": 108 + }, + { + "epoch": 0.25, + "learning_rate": 4.33930977668283e-06, + "loss": 0.5479, + "step": 109 + }, + { + "epoch": 0.26, + "learning_rate": 4.326532213837735e-06, + "loss": 0.582, + "step": 110 + }, + { + "epoch": 0.26, + "learning_rate": 4.3136514764687155e-06, + "loss": 0.5545, + "step": 111 + }, + { + "epoch": 0.26, + "learning_rate": 4.300668292164329e-06, + "loss": 0.5487, + "step": 112 + }, + { + "epoch": 0.26, + "learning_rate": 4.287583394300016e-06, + "loss": 0.5644, + "step": 113 + }, + { + "epoch": 0.27, + "learning_rate": 4.274397521996658e-06, + "loss": 0.5358, + "step": 114 + }, + { + "epoch": 0.27, + "learning_rate": 4.261111420078844e-06, + "loss": 0.5425, + "step": 115 + }, + { + "epoch": 0.27, + "learning_rate": 4.247725839032781e-06, + "loss": 0.5211, + "step": 116 + }, + { + "epoch": 0.27, + "learning_rate": 4.234241534963916e-06, + "loss": 0.5663, + "step": 117 + }, + { + "epoch": 0.28, + "learning_rate": 4.220659269554217e-06, + "loss": 0.5382, + "step": 118 + }, + { + "epoch": 0.28, + "learning_rate": 4.206979810019153e-06, + "loss": 0.554, + "step": 119 + }, + { + "epoch": 0.28, + "learning_rate": 4.1932039290643534e-06, + "loss": 0.528, + "step": 120 + }, + { + "epoch": 0.28, + "learning_rate": 4.179332404841963e-06, + "loss": 0.5146, + "step": 121 + }, + { + "epoch": 0.28, + "learning_rate": 4.1653660209066835e-06, + "loss": 0.5608, + "step": 122 + }, + { + "epoch": 0.29, + "learning_rate": 4.151305566171521e-06, + "loss": 0.5573, + "step": 123 + }, + { + "epoch": 0.29, + "learning_rate": 4.137151834863213e-06, + "loss": 0.5345, + "step": 124 + }, + { + "epoch": 0.29, + "learning_rate": 4.122905626477371e-06, + "loss": 0.5434, + "step": 125 + }, + { + "epoch": 0.29, + "learning_rate": 4.108567745733318e-06, + "loss": 0.5685, + "step": 126 + }, + { + "epoch": 0.3, + "learning_rate": 4.094139002528635e-06, + "loss": 0.5442, + "step": 127 + }, + { + "epoch": 0.3, + "learning_rate": 4.07962021189341e-06, + "loss": 0.4957, + "step": 128 + }, + { + "epoch": 0.3, + "learning_rate": 4.065012193944201e-06, + "loss": 0.5833, + "step": 129 + }, + { + "epoch": 0.3, + "learning_rate": 4.050315773837708e-06, + "loss": 0.5178, + "step": 130 + }, + { + "epoch": 0.31, + "learning_rate": 4.0355317817241705e-06, + "loss": 0.553, + "step": 131 + }, + { + "epoch": 0.31, + "learning_rate": 4.020661052700462e-06, + "loss": 0.5466, + "step": 132 + }, + { + "epoch": 0.31, + "learning_rate": 4.00570442676293e-06, + "loss": 0.5412, + "step": 133 + }, + { + "epoch": 0.31, + "learning_rate": 3.990662748759946e-06, + "loss": 0.5573, + "step": 134 + }, + { + "epoch": 0.32, + "learning_rate": 3.975536868344174e-06, + "loss": 0.5593, + "step": 135 + }, + { + "epoch": 0.32, + "learning_rate": 3.9603276399245864e-06, + "loss": 0.5251, + "step": 136 + }, + { + "epoch": 0.32, + "learning_rate": 3.945035922618198e-06, + "loss": 0.5341, + "step": 137 + }, + { + "epoch": 0.32, + "learning_rate": 3.929662580201536e-06, + "loss": 0.5267, + "step": 138 + }, + { + "epoch": 0.32, + "learning_rate": 3.91420848106185e-06, + "loss": 0.5161, + "step": 139 + }, + { + "epoch": 0.33, + "learning_rate": 3.898674498148058e-06, + "loss": 0.5403, + "step": 140 + }, + { + "epoch": 0.33, + "learning_rate": 3.883061508921439e-06, + "loss": 0.536, + "step": 141 + }, + { + "epoch": 0.33, + "learning_rate": 3.8673703953060685e-06, + "loss": 0.5374, + "step": 142 + }, + { + "epoch": 0.33, + "learning_rate": 3.8516020436389945e-06, + "loss": 0.4947, + "step": 143 + }, + { + "epoch": 0.34, + "learning_rate": 3.835757344620183e-06, + "loss": 0.5328, + "step": 144 + }, + { + "epoch": 0.34, + "learning_rate": 3.819837193262197e-06, + "loss": 0.5306, + "step": 145 + }, + { + "epoch": 0.34, + "learning_rate": 3.803842488839642e-06, + "loss": 0.5334, + "step": 146 + }, + { + "epoch": 0.34, + "learning_rate": 3.7877741348383703e-06, + "loss": 0.5238, + "step": 147 + }, + { + "epoch": 0.35, + "learning_rate": 3.7716330389044463e-06, + "loss": 0.5396, + "step": 148 + }, + { + "epoch": 0.35, + "learning_rate": 3.7554201127928747e-06, + "loss": 0.5121, + "step": 149 + }, + { + "epoch": 0.35, + "learning_rate": 3.739136272316102e-06, + "loss": 0.5662, + "step": 150 + }, + { + "epoch": 0.35, + "learning_rate": 3.72278243729228e-06, + "loss": 0.4901, + "step": 151 + }, + { + "epoch": 0.35, + "learning_rate": 3.706359531493316e-06, + "loss": 0.4966, + "step": 152 + }, + { + "epoch": 0.36, + "learning_rate": 3.6898684825926845e-06, + "loss": 0.5597, + "step": 153 + }, + { + "epoch": 0.36, + "learning_rate": 3.6733102221130303e-06, + "loss": 0.5164, + "step": 154 + }, + { + "epoch": 0.36, + "learning_rate": 3.656685685373552e-06, + "loss": 0.5555, + "step": 155 + }, + { + "epoch": 0.36, + "learning_rate": 3.6399958114371597e-06, + "loss": 0.5309, + "step": 156 + }, + { + "epoch": 0.37, + "learning_rate": 3.623241543057445e-06, + "loss": 0.5468, + "step": 157 + }, + { + "epoch": 0.37, + "learning_rate": 3.606423826625414e-06, + "loss": 0.5183, + "step": 158 + }, + { + "epoch": 0.37, + "learning_rate": 3.5895436121160388e-06, + "loss": 0.5361, + "step": 159 + }, + { + "epoch": 0.37, + "learning_rate": 3.5726018530345913e-06, + "loss": 0.5343, + "step": 160 + }, + { + "epoch": 0.38, + "learning_rate": 3.5555995063627842e-06, + "loss": 0.5548, + "step": 161 + }, + { + "epoch": 0.38, + "learning_rate": 3.5385375325047167e-06, + "loss": 0.5728, + "step": 162 + }, + { + "epoch": 0.38, + "learning_rate": 3.5214168952326205e-06, + "loss": 0.5234, + "step": 163 + }, + { + "epoch": 0.38, + "learning_rate": 3.5042385616324243e-06, + "loss": 0.5047, + "step": 164 + }, + { + "epoch": 0.39, + "learning_rate": 3.4870035020491216e-06, + "loss": 0.5179, + "step": 165 + }, + { + "epoch": 0.39, + "learning_rate": 3.469712690031962e-06, + "loss": 0.5013, + "step": 166 + }, + { + "epoch": 0.39, + "learning_rate": 3.4523671022794612e-06, + "loss": 0.5113, + "step": 167 + }, + { + "epoch": 0.39, + "learning_rate": 3.4349677185842246e-06, + "loss": 0.5694, + "step": 168 + }, + { + "epoch": 0.39, + "learning_rate": 3.4175155217776057e-06, + "loss": 0.4965, + "step": 169 + }, + { + "epoch": 0.4, + "learning_rate": 3.4000114976741905e-06, + "loss": 0.5475, + "step": 170 + }, + { + "epoch": 0.4, + "learning_rate": 3.38245663501611e-06, + "loss": 0.5014, + "step": 171 + }, + { + "epoch": 0.4, + "learning_rate": 3.3648519254171906e-06, + "loss": 0.5079, + "step": 172 + }, + { + "epoch": 0.4, + "learning_rate": 3.3471983633069414e-06, + "loss": 0.5222, + "step": 173 + }, + { + "epoch": 0.41, + "learning_rate": 3.32949694587438e-06, + "loss": 0.5039, + "step": 174 + }, + { + "epoch": 0.41, + "learning_rate": 3.3117486730117092e-06, + "loss": 0.5067, + "step": 175 + }, + { + "epoch": 0.41, + "learning_rate": 3.2939545472578314e-06, + "loss": 0.5575, + "step": 176 + }, + { + "epoch": 0.41, + "learning_rate": 3.276115573741724e-06, + "loss": 0.5155, + "step": 177 + }, + { + "epoch": 0.42, + "learning_rate": 3.2582327601256567e-06, + "loss": 0.4915, + "step": 178 + }, + { + "epoch": 0.42, + "learning_rate": 3.240307116548279e-06, + "loss": 0.5415, + "step": 179 + }, + { + "epoch": 0.42, + "learning_rate": 3.222339655567556e-06, + "loss": 0.5326, + "step": 180 + }, + { + "epoch": 0.42, + "learning_rate": 3.2043313921035747e-06, + "loss": 0.5921, + "step": 181 + }, + { + "epoch": 0.42, + "learning_rate": 3.1862833433812137e-06, + "loss": 0.5318, + "step": 182 + }, + { + "epoch": 0.43, + "learning_rate": 3.1681965288726825e-06, + "loss": 0.5151, + "step": 183 + }, + { + "epoch": 0.43, + "learning_rate": 3.1500719702399406e-06, + "loss": 0.5281, + "step": 184 + }, + { + "epoch": 0.43, + "learning_rate": 3.1319106912769797e-06, + "loss": 0.5046, + "step": 185 + }, + { + "epoch": 0.43, + "learning_rate": 3.1137137178519983e-06, + "loss": 0.5022, + "step": 186 + }, + { + "epoch": 0.44, + "learning_rate": 3.0954820778494516e-06, + "loss": 0.5835, + "step": 187 + }, + { + "epoch": 0.44, + "learning_rate": 3.0772168011119894e-06, + "loss": 0.514, + "step": 188 + }, + { + "epoch": 0.44, + "learning_rate": 3.0589189193822894e-06, + "loss": 0.5291, + "step": 189 + }, + { + "epoch": 0.44, + "learning_rate": 3.0405894662447682e-06, + "loss": 0.5186, + "step": 190 + }, + { + "epoch": 0.45, + "learning_rate": 3.0222294770672054e-06, + "loss": 0.5483, + "step": 191 + }, + { + "epoch": 0.45, + "learning_rate": 3.0038399889422553e-06, + "loss": 0.5561, + "step": 192 + }, + { + "epoch": 0.45, + "learning_rate": 2.985422040628867e-06, + "loss": 0.5307, + "step": 193 + }, + { + "epoch": 0.45, + "learning_rate": 2.9669766724936074e-06, + "loss": 0.5137, + "step": 194 + }, + { + "epoch": 0.46, + "learning_rate": 2.948504926451896e-06, + "loss": 0.5222, + "step": 195 + }, + { + "epoch": 0.46, + "learning_rate": 2.930007845909146e-06, + "loss": 0.5254, + "step": 196 + }, + { + "epoch": 0.46, + "learning_rate": 2.911486475701835e-06, + "loss": 0.5369, + "step": 197 + }, + { + "epoch": 0.46, + "learning_rate": 2.892941862038475e-06, + "loss": 0.5371, + "step": 198 + }, + { + "epoch": 0.46, + "learning_rate": 2.8743750524405254e-06, + "loss": 0.5285, + "step": 199 + }, + { + "epoch": 0.47, + "learning_rate": 2.8557870956832135e-06, + "loss": 0.5319, + "step": 200 + }, + { + "epoch": 0.47, + "learning_rate": 2.837179041736299e-06, + "loss": 0.4983, + "step": 201 + }, + { + "epoch": 0.47, + "learning_rate": 2.8185519417047624e-06, + "loss": 0.4962, + "step": 202 + }, + { + "epoch": 0.47, + "learning_rate": 2.799906847769433e-06, + "loss": 0.5055, + "step": 203 + }, + { + "epoch": 0.48, + "learning_rate": 2.781244813127552e-06, + "loss": 0.4918, + "step": 204 + }, + { + "epoch": 0.48, + "learning_rate": 2.762566891933285e-06, + "loss": 0.5191, + "step": 205 + }, + { + "epoch": 0.48, + "learning_rate": 2.743874139238171e-06, + "loss": 0.5509, + "step": 206 + }, + { + "epoch": 0.48, + "learning_rate": 2.725167610931534e-06, + "loss": 0.5296, + "step": 207 + }, + { + "epoch": 0.49, + "learning_rate": 2.7064483636808314e-06, + "loss": 0.5335, + "step": 208 + }, + { + "epoch": 0.49, + "learning_rate": 2.687717454871971e-06, + "loss": 0.4982, + "step": 209 + }, + { + "epoch": 0.49, + "learning_rate": 2.6689759425495833e-06, + "loss": 0.4864, + "step": 210 + }, + { + "epoch": 0.49, + "learning_rate": 2.650224885357251e-06, + "loss": 0.5334, + "step": 211 + }, + { + "epoch": 0.5, + "learning_rate": 2.6314653424777194e-06, + "loss": 0.5309, + "step": 212 + }, + { + "epoch": 0.5, + "learning_rate": 2.612698373573056e-06, + "loss": 0.5346, + "step": 213 + }, + { + "epoch": 0.5, + "learning_rate": 2.593925038724802e-06, + "loss": 0.5144, + "step": 214 + }, + { + "epoch": 0.5, + "eval_loss": 0.5360019207000732, + "eval_runtime": 973.5712, + "eval_samples_per_second": 1.157, + "eval_steps_per_second": 0.145, + "step": 214 + }, + { + "epoch": 0.5, + "learning_rate": 2.575146398374087e-06, + "loss": 0.5263, + "step": 215 + }, + { + "epoch": 0.5, + "learning_rate": 2.5563635132617305e-06, + "loss": 0.5135, + "step": 216 + }, + { + "epoch": 0.51, + "learning_rate": 2.5375774443683263e-06, + "loss": 0.5003, + "step": 217 + }, + { + "epoch": 0.51, + "learning_rate": 2.518789252854305e-06, + "loss": 0.5005, + "step": 218 + }, + { + "epoch": 0.51, + "learning_rate": 2.5e-06, + "loss": 0.5006, + "step": 219 + }, + { + "epoch": 0.51, + "learning_rate": 2.4812107471456958e-06, + "loss": 0.5052, + "step": 220 + }, + { + "epoch": 0.52, + "learning_rate": 2.4624225556316745e-06, + "loss": 0.4966, + "step": 221 + }, + { + "epoch": 0.52, + "learning_rate": 2.44363648673827e-06, + "loss": 0.4997, + "step": 222 + }, + { + "epoch": 0.52, + "learning_rate": 2.4248536016259137e-06, + "loss": 0.535, + "step": 223 + }, + { + "epoch": 0.52, + "learning_rate": 2.4060749612751987e-06, + "loss": 0.5156, + "step": 224 + }, + { + "epoch": 0.53, + "learning_rate": 2.3873016264269446e-06, + "loss": 0.5505, + "step": 225 + }, + { + "epoch": 0.53, + "learning_rate": 2.368534657522281e-06, + "loss": 0.4973, + "step": 226 + }, + { + "epoch": 0.53, + "learning_rate": 2.3497751146427494e-06, + "loss": 0.5186, + "step": 227 + }, + { + "epoch": 0.53, + "learning_rate": 2.3310240574504184e-06, + "loss": 0.5199, + "step": 228 + }, + { + "epoch": 0.53, + "learning_rate": 2.3122825451280294e-06, + "loss": 0.546, + "step": 229 + }, + { + "epoch": 0.54, + "learning_rate": 2.2935516363191695e-06, + "loss": 0.4939, + "step": 230 + }, + { + "epoch": 0.54, + "learning_rate": 2.2748323890684664e-06, + "loss": 0.5012, + "step": 231 + }, + { + "epoch": 0.54, + "learning_rate": 2.2561258607618296e-06, + "loss": 0.4843, + "step": 232 + }, + { + "epoch": 0.54, + "learning_rate": 2.2374331080667168e-06, + "loss": 0.5267, + "step": 233 + }, + { + "epoch": 0.55, + "learning_rate": 2.2187551868724487e-06, + "loss": 0.4927, + "step": 234 + }, + { + "epoch": 0.55, + "learning_rate": 2.200093152230568e-06, + "loss": 0.4968, + "step": 235 + }, + { + "epoch": 0.55, + "learning_rate": 2.1814480582952376e-06, + "loss": 0.4787, + "step": 236 + }, + { + "epoch": 0.55, + "learning_rate": 2.1628209582637024e-06, + "loss": 0.4645, + "step": 237 + }, + { + "epoch": 0.56, + "learning_rate": 2.1442129043167877e-06, + "loss": 0.5348, + "step": 238 + }, + { + "epoch": 0.56, + "learning_rate": 2.125624947559475e-06, + "loss": 0.5095, + "step": 239 + }, + { + "epoch": 0.56, + "learning_rate": 2.1070581379615253e-06, + "loss": 0.5282, + "step": 240 + }, + { + "epoch": 0.56, + "learning_rate": 2.088513524298165e-06, + "loss": 0.5046, + "step": 241 + }, + { + "epoch": 0.57, + "learning_rate": 2.0699921540908542e-06, + "loss": 0.5325, + "step": 242 + }, + { + "epoch": 0.57, + "learning_rate": 2.0514950735481053e-06, + "loss": 0.5104, + "step": 243 + }, + { + "epoch": 0.57, + "learning_rate": 2.033023327506393e-06, + "loss": 0.4881, + "step": 244 + }, + { + "epoch": 0.57, + "learning_rate": 2.014577959371134e-06, + "loss": 0.4938, + "step": 245 + }, + { + "epoch": 0.57, + "learning_rate": 1.996160011057746e-06, + "loss": 0.4977, + "step": 246 + }, + { + "epoch": 0.58, + "learning_rate": 1.9777705229327954e-06, + "loss": 0.4936, + "step": 247 + }, + { + "epoch": 0.58, + "learning_rate": 1.959410533755232e-06, + "loss": 0.5176, + "step": 248 + }, + { + "epoch": 0.58, + "learning_rate": 1.9410810806177105e-06, + "loss": 0.5057, + "step": 249 + }, + { + "epoch": 0.58, + "learning_rate": 1.922783198888011e-06, + "loss": 0.4982, + "step": 250 + }, + { + "epoch": 0.59, + "learning_rate": 1.9045179221505497e-06, + "loss": 0.5128, + "step": 251 + }, + { + "epoch": 0.59, + "learning_rate": 1.8862862821480023e-06, + "loss": 0.5186, + "step": 252 + }, + { + "epoch": 0.59, + "learning_rate": 1.8680893087230207e-06, + "loss": 0.5112, + "step": 253 + }, + { + "epoch": 0.59, + "learning_rate": 1.8499280297600594e-06, + "loss": 0.4928, + "step": 254 + }, + { + "epoch": 0.6, + "learning_rate": 1.8318034711273181e-06, + "loss": 0.5209, + "step": 255 + }, + { + "epoch": 0.6, + "learning_rate": 1.813716656618788e-06, + "loss": 0.5073, + "step": 256 + }, + { + "epoch": 0.6, + "learning_rate": 1.7956686078964257e-06, + "loss": 0.4724, + "step": 257 + }, + { + "epoch": 0.6, + "learning_rate": 1.7776603444324445e-06, + "loss": 0.51, + "step": 258 + }, + { + "epoch": 0.6, + "learning_rate": 1.759692883451721e-06, + "loss": 0.5193, + "step": 259 + }, + { + "epoch": 0.61, + "learning_rate": 1.741767239874344e-06, + "loss": 0.519, + "step": 260 + }, + { + "epoch": 0.61, + "learning_rate": 1.723884426258277e-06, + "loss": 0.5073, + "step": 261 + }, + { + "epoch": 0.61, + "learning_rate": 1.7060454527421688e-06, + "loss": 0.5178, + "step": 262 + }, + { + "epoch": 0.61, + "learning_rate": 1.6882513269882916e-06, + "loss": 0.5006, + "step": 263 + }, + { + "epoch": 0.62, + "learning_rate": 1.6705030541256211e-06, + "loss": 0.4955, + "step": 264 + }, + { + "epoch": 0.62, + "learning_rate": 1.6528016366930594e-06, + "loss": 0.5231, + "step": 265 + }, + { + "epoch": 0.62, + "learning_rate": 1.6351480745828098e-06, + "loss": 0.4647, + "step": 266 + }, + { + "epoch": 0.62, + "learning_rate": 1.6175433649838901e-06, + "loss": 0.493, + "step": 267 + }, + { + "epoch": 0.63, + "learning_rate": 1.5999885023258099e-06, + "loss": 0.4896, + "step": 268 + }, + { + "epoch": 0.63, + "learning_rate": 1.5824844782223956e-06, + "loss": 0.516, + "step": 269 + }, + { + "epoch": 0.63, + "learning_rate": 1.5650322814157764e-06, + "loss": 0.5059, + "step": 270 + }, + { + "epoch": 0.63, + "learning_rate": 1.5476328977205396e-06, + "loss": 0.5017, + "step": 271 + }, + { + "epoch": 0.64, + "learning_rate": 1.5302873099680378e-06, + "loss": 0.4947, + "step": 272 + }, + { + "epoch": 0.64, + "learning_rate": 1.5129964979508792e-06, + "loss": 0.4764, + "step": 273 + }, + { + "epoch": 0.64, + "learning_rate": 1.495761438367577e-06, + "loss": 0.5008, + "step": 274 + }, + { + "epoch": 0.64, + "learning_rate": 1.47858310476738e-06, + "loss": 0.5402, + "step": 275 + }, + { + "epoch": 0.64, + "learning_rate": 1.4614624674952843e-06, + "loss": 0.4882, + "step": 276 + }, + { + "epoch": 0.65, + "learning_rate": 1.4444004936372166e-06, + "loss": 0.4946, + "step": 277 + }, + { + "epoch": 0.65, + "learning_rate": 1.4273981469654093e-06, + "loss": 0.5305, + "step": 278 + }, + { + "epoch": 0.65, + "learning_rate": 1.4104563878839623e-06, + "loss": 0.4951, + "step": 279 + }, + { + "epoch": 0.65, + "learning_rate": 1.3935761733745865e-06, + "loss": 0.5188, + "step": 280 + }, + { + "epoch": 0.66, + "learning_rate": 1.3767584569425562e-06, + "loss": 0.4943, + "step": 281 + }, + { + "epoch": 0.66, + "learning_rate": 1.360004188562841e-06, + "loss": 0.485, + "step": 282 + }, + { + "epoch": 0.66, + "learning_rate": 1.3433143146264494e-06, + "loss": 0.5002, + "step": 283 + }, + { + "epoch": 0.66, + "learning_rate": 1.3266897778869704e-06, + "loss": 0.5005, + "step": 284 + }, + { + "epoch": 0.67, + "learning_rate": 1.3101315174073162e-06, + "loss": 0.513, + "step": 285 + }, + { + "epoch": 0.67, + "learning_rate": 1.2936404685066852e-06, + "loss": 0.5159, + "step": 286 + }, + { + "epoch": 0.67, + "learning_rate": 1.2772175627077204e-06, + "loss": 0.5532, + "step": 287 + }, + { + "epoch": 0.67, + "learning_rate": 1.2608637276838987e-06, + "loss": 0.4815, + "step": 288 + }, + { + "epoch": 0.67, + "learning_rate": 1.244579887207126e-06, + "loss": 0.4783, + "step": 289 + }, + { + "epoch": 0.68, + "learning_rate": 1.2283669610955543e-06, + "loss": 0.4875, + "step": 290 + }, + { + "epoch": 0.68, + "learning_rate": 1.2122258651616305e-06, + "loss": 0.5021, + "step": 291 + }, + { + "epoch": 0.68, + "learning_rate": 1.1961575111603588e-06, + "loss": 0.4948, + "step": 292 + }, + { + "epoch": 0.68, + "learning_rate": 1.1801628067378033e-06, + "loss": 0.4622, + "step": 293 + }, + { + "epoch": 0.69, + "learning_rate": 1.1642426553798175e-06, + "loss": 0.5352, + "step": 294 + }, + { + "epoch": 0.69, + "learning_rate": 1.148397956361007e-06, + "loss": 0.6051, + "step": 295 + }, + { + "epoch": 0.69, + "learning_rate": 1.1326296046939334e-06, + "loss": 0.506, + "step": 296 + }, + { + "epoch": 0.69, + "learning_rate": 1.1169384910785613e-06, + "loss": 0.5109, + "step": 297 + }, + { + "epoch": 0.7, + "learning_rate": 1.1013255018519426e-06, + "loss": 0.5157, + "step": 298 + }, + { + "epoch": 0.7, + "learning_rate": 1.0857915189381512e-06, + "loss": 0.5131, + "step": 299 + }, + { + "epoch": 0.7, + "learning_rate": 1.0703374197984654e-06, + "loss": 0.5295, + "step": 300 + }, + { + "epoch": 0.7, + "learning_rate": 1.054964077381803e-06, + "loss": 0.4669, + "step": 301 + }, + { + "epoch": 0.71, + "learning_rate": 1.0396723600754144e-06, + "loss": 0.4831, + "step": 302 + }, + { + "epoch": 0.71, + "learning_rate": 1.0244631316558268e-06, + "loss": 0.5104, + "step": 303 + }, + { + "epoch": 0.71, + "learning_rate": 1.009337251240055e-06, + "loss": 0.4822, + "step": 304 + }, + { + "epoch": 0.71, + "learning_rate": 9.942955732370706e-07, + "loss": 0.4747, + "step": 305 + }, + { + "epoch": 0.71, + "learning_rate": 9.793389472995393e-07, + "loss": 0.4954, + "step": 306 + }, + { + "epoch": 0.72, + "learning_rate": 9.644682182758305e-07, + "loss": 0.4992, + "step": 307 + }, + { + "epoch": 0.72, + "learning_rate": 9.496842261622921e-07, + "loss": 0.4863, + "step": 308 + }, + { + "epoch": 0.72, + "learning_rate": 9.349878060557998e-07, + "loss": 0.4711, + "step": 309 + }, + { + "epoch": 0.72, + "learning_rate": 9.203797881065907e-07, + "loss": 0.54, + "step": 310 + }, + { + "epoch": 0.73, + "learning_rate": 9.058609974713655e-07, + "loss": 0.5112, + "step": 311 + }, + { + "epoch": 0.73, + "learning_rate": 8.914322542666822e-07, + "loss": 0.4862, + "step": 312 + }, + { + "epoch": 0.73, + "learning_rate": 8.770943735226303e-07, + "loss": 0.4967, + "step": 313 + }, + { + "epoch": 0.73, + "learning_rate": 8.628481651367876e-07, + "loss": 0.4836, + "step": 314 + }, + { + "epoch": 0.74, + "learning_rate": 8.486944338284797e-07, + "loss": 0.4816, + "step": 315 + }, + { + "epoch": 0.74, + "learning_rate": 8.346339790933167e-07, + "loss": 0.4775, + "step": 316 + }, + { + "epoch": 0.74, + "learning_rate": 8.206675951580382e-07, + "loss": 0.4966, + "step": 317 + }, + { + "epoch": 0.74, + "learning_rate": 8.067960709356479e-07, + "loss": 0.4697, + "step": 318 + }, + { + "epoch": 0.74, + "learning_rate": 7.930201899808476e-07, + "loss": 0.4939, + "step": 319 + }, + { + "epoch": 0.75, + "learning_rate": 7.793407304457836e-07, + "loss": 0.4889, + "step": 320 + }, + { + "epoch": 0.75, + "learning_rate": 7.657584650360847e-07, + "loss": 0.483, + "step": 321 + }, + { + "epoch": 0.75, + "eval_loss": 0.5117549896240234, + "eval_runtime": 974.2092, + "eval_samples_per_second": 1.156, + "eval_steps_per_second": 0.145, + "step": 321 + }, + { + "epoch": 0.75, + "learning_rate": 7.522741609672194e-07, + "loss": 0.4917, + "step": 322 + }, + { + "epoch": 0.75, + "learning_rate": 7.388885799211573e-07, + "loss": 0.5108, + "step": 323 + }, + { + "epoch": 0.76, + "learning_rate": 7.256024780033418e-07, + "loss": 0.4503, + "step": 324 + }, + { + "epoch": 0.76, + "learning_rate": 7.124166056999854e-07, + "loss": 0.4906, + "step": 325 + }, + { + "epoch": 0.76, + "learning_rate": 6.993317078356709e-07, + "loss": 0.492, + "step": 326 + }, + { + "epoch": 0.76, + "learning_rate": 6.863485235312853e-07, + "loss": 0.4756, + "step": 327 + }, + { + "epoch": 0.77, + "learning_rate": 6.734677861622652e-07, + "loss": 0.5299, + "step": 328 + }, + { + "epoch": 0.77, + "learning_rate": 6.60690223317171e-07, + "loss": 0.4963, + "step": 329 + }, + { + "epoch": 0.77, + "learning_rate": 6.480165567565913e-07, + "loss": 0.5079, + "step": 330 + }, + { + "epoch": 0.77, + "learning_rate": 6.354475023723685e-07, + "loss": 0.4878, + "step": 331 + }, + { + "epoch": 0.78, + "learning_rate": 6.229837701471645e-07, + "loss": 0.4631, + "step": 332 + }, + { + "epoch": 0.78, + "learning_rate": 6.106260641143547e-07, + "loss": 0.4811, + "step": 333 + }, + { + "epoch": 0.78, + "learning_rate": 5.983750823182574e-07, + "loss": 0.4797, + "step": 334 + }, + { + "epoch": 0.78, + "learning_rate": 5.86231516774709e-07, + "loss": 0.5034, + "step": 335 + }, + { + "epoch": 0.78, + "learning_rate": 5.741960534319677e-07, + "loss": 0.5047, + "step": 336 + }, + { + "epoch": 0.79, + "learning_rate": 5.622693721319728e-07, + "loss": 0.508, + "step": 337 + }, + { + "epoch": 0.79, + "learning_rate": 5.504521465719392e-07, + "loss": 0.4828, + "step": 338 + }, + { + "epoch": 0.79, + "learning_rate": 5.387450442663026e-07, + "loss": 0.4878, + "step": 339 + }, + { + "epoch": 0.79, + "learning_rate": 5.271487265090163e-07, + "loss": 0.4994, + "step": 340 + }, + { + "epoch": 0.8, + "learning_rate": 5.156638483361933e-07, + "loss": 0.5069, + "step": 341 + }, + { + "epoch": 0.8, + "learning_rate": 5.0429105848911e-07, + "loss": 0.4739, + "step": 342 + }, + { + "epoch": 0.8, + "learning_rate": 4.930309993775578e-07, + "loss": 0.4773, + "step": 343 + }, + { + "epoch": 0.8, + "learning_rate": 4.818843070435561e-07, + "loss": 0.4791, + "step": 344 + }, + { + "epoch": 0.81, + "learning_rate": 4.708516111254238e-07, + "loss": 0.4662, + "step": 345 + }, + { + "epoch": 0.81, + "learning_rate": 4.5993353482221697e-07, + "loss": 0.4834, + "step": 346 + }, + { + "epoch": 0.81, + "learning_rate": 4.4913069485852197e-07, + "loss": 0.5012, + "step": 347 + }, + { + "epoch": 0.81, + "learning_rate": 4.3844370144962153e-07, + "loss": 0.492, + "step": 348 + }, + { + "epoch": 0.81, + "learning_rate": 4.2787315826702396e-07, + "loss": 0.468, + "step": 349 + }, + { + "epoch": 0.82, + "learning_rate": 4.1741966240436446e-07, + "loss": 0.485, + "step": 350 + }, + { + "epoch": 0.82, + "learning_rate": 4.070838043436787e-07, + "loss": 0.5009, + "step": 351 + }, + { + "epoch": 0.82, + "learning_rate": 3.9686616792204677e-07, + "loss": 0.4994, + "step": 352 + }, + { + "epoch": 0.82, + "learning_rate": 3.867673302986161e-07, + "loss": 0.4665, + "step": 353 + }, + { + "epoch": 0.83, + "learning_rate": 3.7678786192199695e-07, + "loss": 0.482, + "step": 354 + }, + { + "epoch": 0.83, + "learning_rate": 3.6692832649804085e-07, + "loss": 0.4914, + "step": 355 + }, + { + "epoch": 0.83, + "learning_rate": 3.571892809580013e-07, + "loss": 0.5156, + "step": 356 + }, + { + "epoch": 0.83, + "learning_rate": 3.475712754270716e-07, + "loss": 0.5109, + "step": 357 + }, + { + "epoch": 0.84, + "learning_rate": 3.3807485319331037e-07, + "loss": 0.4865, + "step": 358 + }, + { + "epoch": 0.84, + "learning_rate": 3.2870055067695557e-07, + "loss": 0.4672, + "step": 359 + }, + { + "epoch": 0.84, + "learning_rate": 3.194488974001203e-07, + "loss": 0.5019, + "step": 360 + }, + { + "epoch": 0.84, + "learning_rate": 3.1032041595688514e-07, + "loss": 0.4891, + "step": 361 + }, + { + "epoch": 0.85, + "learning_rate": 3.0131562198377763e-07, + "loss": 0.4944, + "step": 362 + }, + { + "epoch": 0.85, + "learning_rate": 2.9243502413064365e-07, + "loss": 0.4971, + "step": 363 + }, + { + "epoch": 0.85, + "learning_rate": 2.8367912403191976e-07, + "loss": 0.4814, + "step": 364 + }, + { + "epoch": 0.85, + "learning_rate": 2.7504841627829293e-07, + "loss": 0.4853, + "step": 365 + }, + { + "epoch": 0.85, + "learning_rate": 2.6654338838876664e-07, + "loss": 0.4849, + "step": 366 + }, + { + "epoch": 0.86, + "learning_rate": 2.581645207831204e-07, + "loss": 0.4495, + "step": 367 + }, + { + "epoch": 0.86, + "learning_rate": 2.4991228675477293e-07, + "loss": 0.5105, + "step": 368 + }, + { + "epoch": 0.86, + "learning_rate": 2.4178715244404796e-07, + "loss": 0.4874, + "step": 369 + }, + { + "epoch": 0.86, + "learning_rate": 2.3378957681184283e-07, + "loss": 0.4764, + "step": 370 + }, + { + "epoch": 0.87, + "learning_rate": 2.2592001161370392e-07, + "loss": 0.516, + "step": 371 + }, + { + "epoch": 0.87, + "learning_rate": 2.1817890137430936e-07, + "loss": 0.5093, + "step": 372 + }, + { + "epoch": 0.87, + "learning_rate": 2.1056668336235624e-07, + "loss": 0.484, + "step": 373 + }, + { + "epoch": 0.87, + "learning_rate": 2.0308378756586562e-07, + "loss": 0.4886, + "step": 374 + }, + { + "epoch": 0.88, + "learning_rate": 1.9573063666788878e-07, + "loss": 0.5288, + "step": 375 + }, + { + "epoch": 0.88, + "learning_rate": 1.8850764602263428e-07, + "loss": 0.5072, + "step": 376 + }, + { + "epoch": 0.88, + "learning_rate": 1.8141522363200797e-07, + "loss": 0.4875, + "step": 377 + }, + { + "epoch": 0.88, + "learning_rate": 1.7445377012256127e-07, + "loss": 0.4842, + "step": 378 + }, + { + "epoch": 0.88, + "learning_rate": 1.676236787228652e-07, + "loss": 0.4777, + "step": 379 + }, + { + "epoch": 0.89, + "learning_rate": 1.6092533524129623e-07, + "loss": 0.4904, + "step": 380 + }, + { + "epoch": 0.89, + "learning_rate": 1.543591180442436e-07, + "loss": 0.4872, + "step": 381 + }, + { + "epoch": 0.89, + "learning_rate": 1.4792539803473921e-07, + "loss": 0.4778, + "step": 382 + }, + { + "epoch": 0.89, + "learning_rate": 1.4162453863150183e-07, + "loss": 0.4944, + "step": 383 + }, + { + "epoch": 0.9, + "learning_rate": 1.3545689574841341e-07, + "loss": 0.4711, + "step": 384 + }, + { + "epoch": 0.9, + "learning_rate": 1.2942281777441168e-07, + "loss": 0.5078, + "step": 385 + }, + { + "epoch": 0.9, + "learning_rate": 1.2352264555381134e-07, + "loss": 0.4752, + "step": 386 + }, + { + "epoch": 0.9, + "learning_rate": 1.1775671236705366e-07, + "loss": 0.4983, + "step": 387 + }, + { + "epoch": 0.91, + "learning_rate": 1.121253439118769e-07, + "loss": 0.4865, + "step": 388 + }, + { + "epoch": 0.91, + "learning_rate": 1.0662885828492037e-07, + "loss": 0.4872, + "step": 389 + }, + { + "epoch": 0.91, + "learning_rate": 1.0126756596375687e-07, + "loss": 0.4642, + "step": 390 + }, + { + "epoch": 0.91, + "learning_rate": 9.604176978935342e-08, + "loss": 0.4933, + "step": 391 + }, + { + "epoch": 0.92, + "learning_rate": 9.095176494896662e-08, + "loss": 0.5095, + "step": 392 + }, + { + "epoch": 0.92, + "learning_rate": 8.599783895946762e-08, + "loss": 0.5184, + "step": 393 + }, + { + "epoch": 0.92, + "learning_rate": 8.118027165109926e-08, + "loss": 0.5139, + "step": 394 + }, + { + "epoch": 0.92, + "learning_rate": 7.649933515167407e-08, + "loss": 0.4938, + "step": 395 + }, + { + "epoch": 0.92, + "learning_rate": 7.195529387119815e-08, + "loss": 0.4902, + "step": 396 + }, + { + "epoch": 0.93, + "learning_rate": 6.75484044869379e-08, + "loss": 0.4729, + "step": 397 + }, + { + "epoch": 0.93, + "learning_rate": 6.327891592892126e-08, + "loss": 0.4762, + "step": 398 + }, + { + "epoch": 0.93, + "learning_rate": 5.914706936587494e-08, + "loss": 0.4657, + "step": 399 + }, + { + "epoch": 0.93, + "learning_rate": 5.515309819160402e-08, + "loss": 0.4822, + "step": 400 + }, + { + "epoch": 0.94, + "learning_rate": 5.129722801180542e-08, + "loss": 0.4863, + "step": 401 + }, + { + "epoch": 0.94, + "learning_rate": 4.75796766313269e-08, + "loss": 0.4953, + "step": 402 + }, + { + "epoch": 0.94, + "learning_rate": 4.4000654041862764e-08, + "loss": 0.4952, + "step": 403 + }, + { + "epoch": 0.94, + "learning_rate": 4.05603624100917e-08, + "loss": 0.4605, + "step": 404 + }, + { + "epoch": 0.95, + "learning_rate": 3.72589960662581e-08, + "loss": 0.4967, + "step": 405 + }, + { + "epoch": 0.95, + "learning_rate": 3.4096741493194196e-08, + "loss": 0.4784, + "step": 406 + }, + { + "epoch": 0.95, + "learning_rate": 3.107377731578709e-08, + "loss": 0.5004, + "step": 407 + }, + { + "epoch": 0.95, + "learning_rate": 2.819027429088822e-08, + "loss": 0.4836, + "step": 408 + }, + { + "epoch": 0.96, + "learning_rate": 2.544639529766829e-08, + "loss": 0.475, + "step": 409 + }, + { + "epoch": 0.96, + "learning_rate": 2.284229532841603e-08, + "loss": 0.4853, + "step": 410 + }, + { + "epoch": 0.96, + "learning_rate": 2.0378121479783798e-08, + "loss": 0.5007, + "step": 411 + }, + { + "epoch": 0.96, + "learning_rate": 1.8054012944479225e-08, + "loss": 0.4842, + "step": 412 + }, + { + "epoch": 0.96, + "learning_rate": 1.5870101003402083e-08, + "loss": 0.4887, + "step": 413 + }, + { + "epoch": 0.97, + "learning_rate": 1.382650901822713e-08, + "loss": 0.492, + "step": 414 + }, + { + "epoch": 0.97, + "learning_rate": 1.1923352424439149e-08, + "loss": 0.48, + "step": 415 + }, + { + "epoch": 0.97, + "learning_rate": 1.0160738724809549e-08, + "loss": 0.5003, + "step": 416 + }, + { + "epoch": 0.97, + "learning_rate": 8.538767483325384e-09, + "loss": 0.5103, + "step": 417 + }, + { + "epoch": 0.98, + "learning_rate": 7.05753031956441e-09, + "loss": 0.5304, + "step": 418 + }, + { + "epoch": 0.98, + "learning_rate": 5.717110903520617e-09, + "loss": 0.5166, + "step": 419 + }, + { + "epoch": 0.98, + "learning_rate": 4.517584950877451e-09, + "loss": 0.4779, + "step": 420 + }, + { + "epoch": 0.98, + "learning_rate": 3.4590202187315124e-09, + "loss": 0.5138, + "step": 421 + }, + { + "epoch": 0.99, + "learning_rate": 2.5414765017642285e-09, + "loss": 0.5101, + "step": 422 + }, + { + "epoch": 0.99, + "learning_rate": 1.765005628865113e-09, + "loss": 0.4836, + "step": 423 + }, + { + "epoch": 0.99, + "learning_rate": 1.1296514602038289e-09, + "loss": 0.5055, + "step": 424 + }, + { + "epoch": 0.99, + "learning_rate": 6.354498847521706e-10, + "loss": 0.4525, + "step": 425 + }, + { + "epoch": 0.99, + "learning_rate": 2.8242881825846225e-10, + "loss": 0.4781, + "step": 426 + }, + { + "epoch": 1.0, + "learning_rate": 7.060820166826521e-11, + "loss": 0.5002, + "step": 427 + }, + { + "epoch": 1.0, + "learning_rate": 0.0, + "loss": 0.4674, + "step": 428 + }, + { + "epoch": 1.0, + "eval_loss": 0.5059249997138977, + "eval_runtime": 973.6949, + "eval_samples_per_second": 1.156, + "eval_steps_per_second": 0.145, + "step": 428 + } + ], + "logging_steps": 1, + "max_steps": 428, + "num_input_tokens_seen": 0, + "num_train_epochs": 1, + "save_steps": 214, + "total_flos": 179124295434240.0, + "train_batch_size": 1, + "trial_name": null, + "trial_params": null +} diff --git a/training_args.bin b/training_args.bin new file mode 100644 index 0000000..f54d7fe --- /dev/null +++ b/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:39d917d9a422d1e49380543cb8198173173827da7b75a8abe44e5871976948cc +size 6904 diff --git a/zero_to_fp32.py b/zero_to_fp32.py new file mode 100644 index 0000000..49b8466 --- /dev/null +++ b/zero_to_fp32.py @@ -0,0 +1,592 @@ +#!/usr/bin/env python + +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team + +# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets +# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in +# the future. Once extracted, the weights don't require DeepSpeed and can be used in any +# application. +# +# example: python zero_to_fp32.py . pytorch_model.bin + +import argparse +import torch +import glob +import math +import os +import re +from collections import OrderedDict +from dataclasses import dataclass + +# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with +# DeepSpeed data structures it has to be available in the current python environment. +from deepspeed.utils import logger +from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS, + FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES, + FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS) + + +@dataclass +class zero_model_state: + buffers: dict() + param_shapes: dict() + shared_params: list + ds_version: int + frozen_param_shapes: dict() + frozen_param_fragments: dict() + + +debug = 0 + +# load to cpu +device = torch.device('cpu') + + +def atoi(text): + return int(text) if text.isdigit() else text + + +def natural_keys(text): + ''' + alist.sort(key=natural_keys) sorts in human order + http://nedbatchelder.com/blog/200712/human_sorting.html + (See Toothy's implementation in the comments) + ''' + return [atoi(c) for c in re.split(r'(\d+)', text)] + + +def get_model_state_file(checkpoint_dir, zero_stage): + if not os.path.isdir(checkpoint_dir): + raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist") + + # there should be only one file + if zero_stage <= 2: + file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt") + elif zero_stage == 3: + file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt") + + if not os.path.exists(file): + raise FileNotFoundError(f"can't find model states file at '{file}'") + + return file + + +def get_checkpoint_files(checkpoint_dir, glob_pattern): + # XXX: need to test that this simple glob rule works for multi-node setup too + ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys) + + if len(ckpt_files) == 0: + raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'") + + return ckpt_files + + +def get_optim_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt") + + +def get_model_state_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_model_states.pt") + + +def parse_model_states(files): + zero_model_states = [] + for file in files: + state_dict = torch.load(file, map_location=device) + + if BUFFER_NAMES not in state_dict: + raise ValueError(f"{file} is not a model state checkpoint") + buffer_names = state_dict[BUFFER_NAMES] + if debug: + print("Found buffers:", buffer_names) + + # recover just the buffers while restoring them to fp32 if they were saved in fp16 + buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names} + param_shapes = state_dict[PARAM_SHAPES] + + # collect parameters that are included in param_shapes + param_names = [] + for s in param_shapes: + for name in s.keys(): + param_names.append(name) + + # update with frozen parameters + frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None) + if frozen_param_shapes is not None: + if debug: + print(f"Found frozen_param_shapes: {frozen_param_shapes}") + param_names += list(frozen_param_shapes.keys()) + + # handle shared params + shared_params = [[k, v] for k, v in state_dict["shared_params"].items()] + + ds_version = state_dict.get(DS_VERSION, None) + + frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None) + + z_model_state = zero_model_state(buffers=buffers, + param_shapes=param_shapes, + shared_params=shared_params, + ds_version=ds_version, + frozen_param_shapes=frozen_param_shapes, + frozen_param_fragments=frozen_param_fragments) + zero_model_states.append(z_model_state) + + return zero_model_states + + +def parse_optim_states(files, ds_checkpoint_dir): + + total_files = len(files) + state_dicts = [] + for f in files: + state_dict = torch.load(f, map_location=device) + # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights + # and also handle the case where it was already removed by another helper script + state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None) + state_dicts.append(state_dict) + + if not ZERO_STAGE in state_dicts[0][OPTIMIZER_STATE_DICT]: + raise ValueError(f"{files[0]} is not a zero checkpoint") + zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE] + world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT] + + # For ZeRO-2 each param group can have different partition_count as data parallelism for expert + # parameters can be different from data parallelism for non-expert parameters. So we can just + # use the max of the partition_count to get the dp world_size. + + if type(world_size) is list: + world_size = max(world_size) + + if world_size != total_files: + raise ValueError( + f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. " + "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes." + ) + + # the groups are named differently in each stage + if zero_stage <= 2: + fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS + elif zero_stage == 3: + fp32_groups_key = FP32_FLAT_GROUPS + else: + raise ValueError(f"unknown zero stage {zero_stage}") + + if zero_stage <= 2: + fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))] + elif zero_stage == 3: + # if there is more than one param group, there will be multiple flattened tensors - one + # flattened tensor per group - for simplicity merge them into a single tensor + # + # XXX: could make the script more memory efficient for when there are multiple groups - it + # will require matching the sub-lists of param_shapes for each param group flattened tensor + + fp32_flat_groups = [ + torch.cat(state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key], 0) for i in range(len(state_dicts)) + ] + + return zero_stage, world_size, fp32_flat_groups + + +def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir): + """ + Returns fp32 state_dict reconstructed from ds checkpoint + + Args: + - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are) + + """ + print(f"Processing zero checkpoint '{ds_checkpoint_dir}'") + + optim_files = get_optim_files(ds_checkpoint_dir) + zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir) + print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}") + + model_files = get_model_state_files(ds_checkpoint_dir) + + zero_model_states = parse_model_states(model_files) + print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}') + + if zero_stage <= 2: + return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states) + elif zero_stage == 3: + return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states) + + +def _zero2_merge_frozen_params(state_dict, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + frozen_param_fragments = zero_model_states[0].frozen_param_fragments + + if debug: + num_elem = sum(s.numel() for s in frozen_param_shapes.values()) + print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in frozen_param_fragments.values()]) + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + state_dict[name] = frozen_param_fragments[name] + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +def _has_callable(obj, fn): + attr = getattr(obj, fn, None) + return callable(attr) + + +def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + + # Reconstruction protocol: + # + # XXX: document this + + if debug: + for i in range(world_size): + for j in range(len(fp32_flat_groups[0])): + print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}") + + # XXX: memory usage doubles here (zero2) + num_param_groups = len(fp32_flat_groups[0]) + merged_single_partition_of_fp32_groups = [] + for i in range(num_param_groups): + merged_partitions = [sd[i] for sd in fp32_flat_groups] + full_single_fp32_vector = torch.cat(merged_partitions, 0) + merged_single_partition_of_fp32_groups.append(full_single_fp32_vector) + avail_numel = sum( + [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups]) + + if debug: + wanted_params = sum([len(shapes) for shapes in param_shapes]) + wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes]) + # not asserting if there is a mismatch due to possible padding + print(f"Have {avail_numel} numels to process.") + print(f"Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + total_numel = 0 + total_params = 0 + for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups): + offset = 0 + avail_numel = full_single_fp32_vector.numel() + for name, shape in shapes.items(): + + unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape) + total_numel += unpartitioned_numel + total_params += 1 + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape) + offset += unpartitioned_numel + + # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and + # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex + # paddings performed in the code it's almost impossible to predict the exact numbers w/o the + # live optimizer object, so we are checking that the numbers are within the right range + align_to = 2 * world_size + + def zero2_align(x): + return align_to * math.ceil(x / align_to) + + if debug: + print(f"original offset={offset}, avail_numel={avail_numel}") + + offset = zero2_align(offset) + avail_numel = zero2_align(avail_numel) + + if debug: + print(f"aligned offset={offset}, avail_numel={avail_numel}") + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + _zero2_merge_frozen_params(state_dict, zero_model_states) + + _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def zero3_partitioned_param_info(unpartitioned_numel, world_size): + remainder = unpartitioned_numel % world_size + padding_numel = (world_size - remainder) if remainder else 0 + partitioned_numel = math.ceil(unpartitioned_numel / world_size) + return partitioned_numel, padding_numel + + +def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + if debug: + for i in range(world_size): + num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values()) + print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in zero_model_states[0].frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states) + state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape) + + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + avail_numel = fp32_flat_groups[0].numel() * world_size + # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each + # param, re-consolidating each param, while dealing with padding if any + + # merge list of dicts, preserving order + param_shapes = {k: v for d in param_shapes for k, v in d.items()} + + if debug: + for i in range(world_size): + print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}") + + wanted_params = len(param_shapes) + wanted_numel = sum(shape.numel() for shape in param_shapes.values()) + # not asserting if there is a mismatch due to possible padding + avail_numel = fp32_flat_groups[0].numel() * world_size + print(f"Trainable params: Have {avail_numel} numels to process.") + print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + offset = 0 + total_numel = 0 + total_params = 0 + for name, shape in param_shapes.items(): + + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + total_params += 1 + + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + # XXX: memory usage doubles here + state_dict[name] = torch.cat( + tuple(fp32_flat_groups[i].narrow(0, offset, partitioned_numel) for i in range(world_size)), + 0).narrow(0, 0, unpartitioned_numel).view(shape) + offset += partitioned_numel + + offset *= world_size + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + _zero3_merge_frozen_params(state_dict, world_size, zero_model_states) + + _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag=None): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with + ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example + via a model hub. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14`` + + Returns: + - pytorch ``state_dict`` + + Note: this approach may not work if your application doesn't have sufficient free CPU memory and + you may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with + the checkpoint. + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + # do the training and checkpoint saving + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu + model = model.cpu() # move to cpu + model.load_state_dict(state_dict) + # submit to model hub or save the model to share with others + + In this example the ``model`` will no longer be usable in the deepspeed context of the same + application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead. + + """ + if tag is None: + latest_path = os.path.join(checkpoint_dir, 'latest') + if os.path.isfile(latest_path): + with open(latest_path, 'r') as fd: + tag = fd.read().strip() + else: + raise ValueError(f"Unable to find 'latest' file at {latest_path}") + + ds_checkpoint_dir = os.path.join(checkpoint_dir, tag) + + if not os.path.isdir(ds_checkpoint_dir): + raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist") + + return _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir) + + +def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, output_file, tag=None): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be + loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``output_file``: path to the pytorch fp32 state_dict output file (e.g. path/pytorch_model.bin) + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + """ + + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag) + print(f"Saving fp32 state dict to {output_file}") + torch.save(state_dict, output_file) + + +def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None): + """ + 1. Put the provided model to cpu + 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` + 3. Load it into the provided model + + Args: + - ``model``: the model object to update + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + + Returns: + - ``model`: modified model + + Make sure you have plenty of CPU memory available before you call this function. If you don't + have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it + conveniently placed for you in the checkpoint folder. + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint + model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir) + # submit to model hub or save the model to share with others + + Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context + of the same application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + """ + logger.info(f"Extracting fp32 weights") + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag) + + logger.info(f"Overwriting model with fp32 weights") + model = model.cpu() + model.load_state_dict(state_dict, strict=False) + + return model + + +if __name__ == "__main__": + + parser = argparse.ArgumentParser() + parser.add_argument("checkpoint_dir", + type=str, + help="path to the desired checkpoint folder, e.g., path/checkpoint-12") + parser.add_argument( + "output_file", + type=str, + help="path to the pytorch fp32 state_dict output file (e.g. path/checkpoint-12/pytorch_model.bin)") + parser.add_argument("-t", + "--tag", + type=str, + default=None, + help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1") + parser.add_argument("-d", "--debug", action='store_true', help="enable debug") + args = parser.parse_args() + + debug = args.debug + + convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir, args.output_file, tag=args.tag)