From d0090aca6966ad5e9c2abe331b4e64d7362ad08d Mon Sep 17 00:00:00 2001 From: ModelHub XC Date: Sat, 8 Aug 2026 19:38:14 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=EF=BC=8C=E7=94=B1ModelHub=20XC=E7=A4=BE=E5=8C=BA=E6=8F=90?= =?UTF-8?q?=E4=BE=9B=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model: LumiOpen/Llama-Poro-2-8B-SFT Source: Original Platform --- .gitattributes | 36 + README.md | 212 +++ all_results.json | 14 + config.json | 33 + eval_results.json | 8 + generation_config.json | 11 + model-00001-of-00004.safetensors | 3 + model-00002-of-00004.safetensors | 3 + model-00003-of-00004.safetensors | 3 + model-00004-of-00004.safetensors | 3 + model.safetensors.index.json | 298 +++++ special_tokens_map.json | 17 + tokenizer.json | 3 + tokenizer_config.json | 2064 ++++++++++++++++++++++++++++++ train_results.json | 9 + trainer_state.json | 1417 ++++++++++++++++++++ training_args.bin | 3 + 17 files changed, 4137 insertions(+) create mode 100644 .gitattributes create mode 100644 README.md create mode 100644 all_results.json create mode 100644 config.json create mode 100644 eval_results.json create mode 100644 generation_config.json create mode 100644 model-00001-of-00004.safetensors create mode 100644 model-00002-of-00004.safetensors create mode 100644 model-00003-of-00004.safetensors create mode 100644 model-00004-of-00004.safetensors create mode 100644 model.safetensors.index.json create mode 100644 special_tokens_map.json create mode 100644 tokenizer.json create mode 100644 tokenizer_config.json create mode 100644 train_results.json create mode 100644 trainer_state.json create mode 100644 training_args.bin diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..52373fe --- /dev/null +++ b/.gitattributes @@ -0,0 +1,36 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..1e6225a --- /dev/null +++ b/README.md @@ -0,0 +1,212 @@ +--- +datasets: +- LumiOpen/poro2-instruction-collection +language: +- fi +- en +license: llama3.3 +library_name: transformers +pipeline_tag: text-generation +--- + +# Poro 2 8B SFT Model Card + +> **Note for most users**: This is an intermediate checkpoint from our post-training pipeline. Most users should use [Poro 2 8B Instruct](https://huggingface.co/LumiOpen/Llama-Poro-2-8B-Instruct) instead, which includes an additional round of Direct Preference Optimization (DPO) for improved response quality and alignment. This SFT-only model is primarily intended for researchers interested in studying the effects of different post-training techniques. + +Poro 2 8B SFT is a supervised fine-tuned model created from the Poro 2 8B Base model. This model has been trained for instruction following and conversational AI applications in both Finnish and English, but has not undergone preference tuning. It represents the intermediate step before Direct Preference Optimization (DPO) in our post-training pipeline. + +Poro 2 was created in a collaboration between [AMD Silo AI](https://www.amd.com/en/solutions/ai/silo-ai.html), the [TurkuNLP group](https://turkunlp.org/) of the University of Turku, and [High Performance Language Technologies](https://hplt-project.org/) (HPLT). Training was conducted on the [LUMI supercomputer](https://www.lumi-supercomputer.eu/), using compute resources generously provided by [CSC](https://csc.fi/) - IT Center for Science, Finland. + +For more details on our training and data generation pipeline, check out our [Continued Pretraining Playbook](https://rocm.blogs.amd.com/artificial-intelligence/multilingual-continued-pretraining/README.html). + +## Poro 2 Model Family + +The Poro 2 model family includes both 8B and 70B models, and there are three different versions released of the Poro 2 models: a base model, a post-training SFT-only checkpoint, and the final instruct model which is the SFT model plus a round of DPO. + +| Model | Based on | Base Model | SFT | Instruct | +| :---: | :------: | :--------: | :-: | :------- | +| Poro 2 8B | Llama 3.1 8B | [Poro 2 8B Base](https://huggingface.co/LumiOpen/Llama-Poro-2-8B-base) | [Poro 2 8B SFT](https://huggingface.co/LumiOpen/Llama-Poro-2-8B-SFT) | [Poro 2 8B Instruct](https://huggingface.co/LumiOpen/Llama-Poro-2-8B-Instruct) | +| Poro 2 70B | Llama 3.1 70B | [Poro 2 70B Base](https://huggingface.co/LumiOpen/Llama-Poro-2-70B-base) | [Poro 2 70B SFT](https://huggingface.co/LumiOpen/Llama-Poro-2-70B-SFT) | [Poro 2 70B Instruct](https://huggingface.co/LumiOpen/Llama-Poro-2-70B-Instruct) | + +_What does Poro mean?_ Poro is the Finnish word for Reindeer! 🦌 These animals are native to Finland and hold a significant and historical role in Finnish culture. + +## Model Overview + +Poro 2 8B SFT is based on the Llama 3.1 8B architecture and has been supervised fine-tuned for instruction following. The model supports both English and Finnish conversations but has not undergone preference tuning for response quality optimization. + +| Hyperparameter | Value | +| :------------- | :----: | +| n_parameters | 8.03B | +| n_layers | 32 | +| n_heads | 32 | +| n_kv_heads | 8 | +| d_model | 4096 | +| vocab_size | 128256 | +| max_sequence_length | 8192 | +| base_model | Llama-3.1-8B | + +## Training Process + +### Continued Pretraining +The base Poro 2 8B model was created through continued pretraining on 165B tokens of Finnish, English, code, and math data. + +### Supervised Fine-Tuning (SFT) +This model represents the SFT phase of post-training, using 1.4M instruction-following examples in English and Finnish, including: +- English and Finnish Tulu 3 prompts with Llama-3.3-70B-Instruct responses (1.35M samples) +- Multi-turn conversations generated using the Magpie method (14K samples) +- Top-rated conversations from OASST2 and Avoin Avustaja datasets (5K samples) +- Translation samples from EuroParl (1K samples) + +We release the [Poro 2 instruction collection](https://huggingface.co/datasets/LumiOpen/poro2-instruction-collection). + +## SFT Hyperparameters + +| Hyperparameter | Value | +| :------------: | :---: | +| Epochs | 2 | +| Global batch size | 64 | +| Learning rate | 5e-6 | +| LR scheduler | linear | +| Warmup ratio | 0.03 | +| Max sequence length | 4,096 | + +## Evaluation Results + +Poro 2 8B SFT shows substantial improvements in Finnish instruction-following capabilities compared to Llama 3.1 8B Instruct, while maintaining strong English performance. Note that the final Instruct model (with DPO) performs significantly better. + +### Finnish Instruction Following + +| | Poro 2 8B SFT | Llama 3.1 8B Instruct | Poro 2 8B Instruct | +|----------------|------------------|------------------------|--------------------| +| IFEval Finnish | 64.69 | 47.31 | **66.54** | +| MTBench Finnish | 5.92 | 4.10 | **6.75** | +| AlpacaEval 2 Finnish | 16.80 | 2.05 | **28.89** | + + +### English Instruction Following +| | Poro 2 8B SFT | Llama 3.1 8B Instruct | Poro 2 8B Instruct | +|----------------|--------|------------------------|--------------------| +| IFEval | **79.66** | 79.48 | 79.29 | +| MTBench | 7.07 | **7.70** | 7.33 | +| AlpacaEval 2 | 29.67 | 32.70 | **35.30** | + + +### MTBench scores per-category + +| | Finnish | English | +|-----------------|---------|-----------| +| Coding | 4.55 | 5.30 | +| Extraction | 6.20 | 8.15 | +| Humanities | 7.30 | 8.25 | +| Math | 4.60 | 7.10 | +| Reasoning | 4.45 | 5.75 | +| Roleplay | 6.55 | 7.25 | +| STEM | 6.50 | 7.15 | +| Writing | 7.24 | 7.65 | + + +### MTBench scores per-turn + +| | Finnish | English | +|-----------------|---------|-----------| +| first turn | 6.76 | 7.90 | +| second turn | 5.08 | 6.25 | + + +**Overall**: ~16% average improvement in Finnish instruction-following benchmarks compared to Llama 3.1 8B Instruct, with maintained English performance. The additional DPO step in the Instruct model provides further improvements. + +## Usage + +```python +from transformers import AutoTokenizer, AutoModelForCausalLM +import torch + +model_name = "LumiOpen/Poro-2-8B-SFT" +tokenizer = AutoTokenizer.from_pretrained(model_name) +model = AutoModelForCausalLM.from_pretrained( + model_name, + torch_dtype=torch.bfloat16, + device_map="auto" +) + +# Finnish conversation example +messages = [ + {"role": "user", "content": "Kerro minulle Suomen historiasta."} +] + +inputs = tokenizer.apply_chat_template( + messages, + add_generation_prompt=True, + return_tensors="pt" +) + +outputs = model.generate( + inputs, + max_new_tokens=500, + temperature=0.7, + do_sample=True, + pad_token_id=tokenizer.eos_token_id +) + +response = tokenizer.decode(outputs[0], skip_special_tokens=True) +print(response) +``` + +## Research Applications + +This SFT-only model is particularly useful for researchers studying: +- The effects of supervised fine-tuning vs. preference tuning +- Comparative analysis of different post-training techniques +- Ablation studies on instruction-following capabilities +- Cross-lingual transfer in instruction-following tasks +- The impact of DPO on model behavior and alignment + +## Intended Use + +Poro 2 8B SFT is primarily intended for: +- **Research purposes**: Studying post-training techniques and their effects +- **Comparative analysis**: Understanding the contribution of different training phases +- **Educational applications**: Learning about instruction-following model development +- **Development**: As a starting point for further preference tuning experiments + +**For production use cases**, we recommend using [Poro 2 8B Instruct](https://huggingface.co/LumiOpen/Llama-Poro-2-8B-Instruct) instead. + +## Ethical Considerations and Limitations + +Poro 2 8B SFT is a research checkpoint optimized for English and Finnish instruction following. As this model has not undergone preference tuning, it may be more prone to generating responses that are misaligned with user expectations compared to the final Instruct model. + +Key limitations: +- **No preference tuning**: May generate responses that are less aligned or of lower quality than the Instruct version +- Limited proficiency in languages other than English and Finnish +- May occasionally generate biased, inappropriate, or factually incorrect content +- Performance may vary significantly for specialized or technical domains +- Context window limited to 8,192 tokens +- May struggle with very recent events (knowledge cutoff limitations) + +**Safety Considerations:** +- This model should primarily be used for research purposes +- Users should verify important factual claims independently +- The model should not be used for medical, legal, or financial advice without human oversight +- Responses should be reviewed for appropriateness in sensitive contexts +- Consider using the Instruct version for better alignment and response quality + +## License + +Built with Llama + +Poro 2 8B SFT is released under the Llama 3.3 Community License. Please review the license terms before use. + +## Citation + +```bibtex +@misc{poro2_2025, + title={Poro 2: Continued Pretraining for Language Acquisition}, + author={Elaine Zosa and Jouni Louma and Kai Hakala and Antti Virtanen and Mika Koistinen and Risto Luukkonen and Akseli Reunamo and Sampo Pyysalo and Jonathan Burdge}, + year={2025}, + howpublished={LumiOpen} +} +``` + +## Acknowledgments + +We thank CSC - IT Center for Science, Finland for providing access to the LUMI supercomputer. This work was supported by the High Performance Language Technologies (HPLT) project and conducted in collaboration with TurkuNLP from the University of Turku. This project has received funding from the European Union's Horizon Europe research and innovation programme under grant agreement No 101070350. \ No newline at end of file diff --git a/all_results.json b/all_results.json new file mode 100644 index 0000000..65cea2f --- /dev/null +++ b/all_results.json @@ -0,0 +1,14 @@ +{ + "epoch": 2.0, + "eval_loss": 0.5610442161560059, + "eval_runtime": 19.5851, + "eval_samples": 2567, + "eval_samples_per_second": 27.47, + "eval_steps_per_second": 0.46, + "total_flos": 4057777727078400.0, + "train_loss": 0.5052211482704485, + "train_runtime": 135745.8608, + "train_samples": 1421904, + "train_samples_per_second": 4.568, + "train_steps_per_second": 0.071 +} \ No newline at end of file diff --git a/config.json b/config.json new file mode 100644 index 0000000..78216c8 --- /dev/null +++ b/config.json @@ -0,0 +1,33 @@ +{ + "architectures": [ + "LlamaForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 128000, + "eos_token_id": [ + 128001, + 128008, + 128009 + ], + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 4096, + "initializer_range": 0.02, + "intermediate_size": 14336, + "max_position_embeddings": 8192, + "mlp_bias": false, + "model_type": "llama", + "num_attention_heads": 32, + "num_hidden_layers": 32, + "num_key_value_heads": 8, + "pretraining_tp": 1, + "rms_norm_eps": 1e-05, + "rope_scaling": null, + "rope_theta": 500000.0, + "tie_word_embeddings": false, + "torch_dtype": "bfloat16", + "transformers_version": "4.50.0", + "use_cache": true, + "vocab_size": 128256 +} diff --git a/eval_results.json b/eval_results.json new file mode 100644 index 0000000..60da812 --- /dev/null +++ b/eval_results.json @@ -0,0 +1,8 @@ +{ + "epoch": 2.0, + "eval_loss": 0.5610442161560059, + "eval_runtime": 19.5851, + "eval_samples": 2567, + "eval_samples_per_second": 27.47, + "eval_steps_per_second": 0.46 +} \ No newline at end of file diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..70e3f70 --- /dev/null +++ b/generation_config.json @@ -0,0 +1,11 @@ +{ + "_from_model_config": true, + "bos_token_id": 128000, + "eos_token_id": [ + 128001, + 128008, + 128009 + ], + "transformers_version": "4.50.0", + "use_cache": false +} diff --git a/model-00001-of-00004.safetensors b/model-00001-of-00004.safetensors new file mode 100644 index 0000000..211d640 --- /dev/null +++ b/model-00001-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be0f3a204a9d92525845a2801f1981aa8beecb36a93a42881f19806c9a3ae998 +size 4976698672 diff --git a/model-00002-of-00004.safetensors b/model-00002-of-00004.safetensors new file mode 100644 index 0000000..b144c5c --- /dev/null +++ b/model-00002-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a693fc72124da7d5c6d28312f9f150aa7ed885709104c8145466dce13dc76cd4 +size 4999802720 diff --git a/model-00003-of-00004.safetensors b/model-00003-of-00004.safetensors new file mode 100644 index 0000000..8a531fb --- /dev/null +++ b/model-00003-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4ede58f66ec7daaa207d50fd3641f4059bb09beca77d95b942335e3af463b9c6 +size 4915916176 diff --git a/model-00004-of-00004.safetensors b/model-00004-of-00004.safetensors new file mode 100644 index 0000000..218cda6 --- /dev/null +++ b/model-00004-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:424766323ea3251ea853c127e5708ec82e477c90c6d55f60e352a2053b98f4c2 +size 1168138808 diff --git a/model.safetensors.index.json b/model.safetensors.index.json new file mode 100644 index 0000000..0fd8120 --- /dev/null +++ b/model.safetensors.index.json @@ -0,0 +1,298 @@ +{ + "metadata": { + "total_size": 16060522496 + }, + "weight_map": { + "lm_head.weight": "model-00004-of-00004.safetensors", + "model.embed_tokens.weight": "model-00001-of-00004.safetensors", + "model.layers.0.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.10.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.10.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.11.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.12.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.13.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.14.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.15.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.16.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.17.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.18.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.19.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.2.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.20.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.20.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.20.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.20.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.21.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.21.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.21.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.22.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.22.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.23.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.24.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.25.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.26.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.27.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.28.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.29.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.3.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.30.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.30.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.30.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.31.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.4.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.4.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.5.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.6.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.7.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.8.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.8.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.8.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.9.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.9.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.9.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.norm.weight": "model-00004-of-00004.safetensors" + } +} diff --git a/special_tokens_map.json b/special_tokens_map.json new file mode 100644 index 0000000..b43be96 --- /dev/null +++ b/special_tokens_map.json @@ -0,0 +1,17 @@ +{ + "bos_token": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "<|eot_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": "<|eot_id|>" +} diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..1c1d8d5 --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6b9e4e7fb171f92fd137b777cc2714bf87d11576700a1dcd7a399e7bbe39537b +size 17209920 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..6627ce3 --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,2064 @@ +{ + "added_tokens_decoder": { + "128000": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128001": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128002": { + "content": "<|reserved_special_token_0|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128003": { + "content": "<|reserved_special_token_1|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128004": { + "content": "<|finetune_right_pad_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128005": { + "content": "<|reserved_special_token_2|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128006": { + "content": "<|start_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128007": { + "content": "<|end_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128008": { + "content": "<|eom_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128009": { + "content": "<|eot_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128010": { + "content": "<|python_tag|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128011": { + "content": "<|reserved_special_token_3|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128012": { + "content": "<|reserved_special_token_4|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128013": { + "content": "<|reserved_special_token_5|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128014": { + "content": "<|reserved_special_token_6|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128015": { + "content": "<|reserved_special_token_7|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128016": { + "content": "<|reserved_special_token_8|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128017": { + "content": "<|reserved_special_token_9|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128018": { + "content": "<|reserved_special_token_10|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128019": { + "content": "<|reserved_special_token_11|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128020": { + "content": "<|reserved_special_token_12|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128021": { + "content": "<|reserved_special_token_13|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128022": { + "content": "<|reserved_special_token_14|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128023": { + "content": "<|reserved_special_token_15|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128024": { + "content": "<|reserved_special_token_16|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128025": { + "content": "<|reserved_special_token_17|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128026": { + "content": "<|reserved_special_token_18|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128027": { + "content": "<|reserved_special_token_19|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128028": { + "content": "<|reserved_special_token_20|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128029": { + "content": "<|reserved_special_token_21|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128030": { + "content": "<|reserved_special_token_22|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128031": { + "content": "<|reserved_special_token_23|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128032": { + "content": "<|reserved_special_token_24|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128033": { + "content": "<|reserved_special_token_25|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128034": { + "content": "<|reserved_special_token_26|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128035": { + "content": "<|reserved_special_token_27|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128036": { + "content": "<|reserved_special_token_28|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128037": { + "content": "<|reserved_special_token_29|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128038": { + "content": "<|reserved_special_token_30|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128039": { + "content": "<|reserved_special_token_31|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128040": { + "content": "<|reserved_special_token_32|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128041": { + "content": "<|reserved_special_token_33|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128042": { + "content": "<|reserved_special_token_34|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128043": { + "content": "<|reserved_special_token_35|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128044": { + "content": "<|reserved_special_token_36|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128045": { + "content": "<|reserved_special_token_37|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128046": { + "content": "<|reserved_special_token_38|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128047": { + "content": "<|reserved_special_token_39|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128048": { + "content": "<|reserved_special_token_40|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128049": { + "content": "<|reserved_special_token_41|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128050": { + "content": "<|reserved_special_token_42|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128051": { + "content": "<|reserved_special_token_43|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128052": { + "content": "<|reserved_special_token_44|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128053": { + "content": "<|reserved_special_token_45|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128054": { + "content": "<|reserved_special_token_46|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128055": { + "content": "<|reserved_special_token_47|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128056": { + "content": "<|reserved_special_token_48|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128057": { + "content": "<|reserved_special_token_49|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128058": { + "content": "<|reserved_special_token_50|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128059": { + "content": "<|reserved_special_token_51|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128060": { + "content": "<|reserved_special_token_52|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128061": { + "content": "<|reserved_special_token_53|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128062": { + "content": "<|reserved_special_token_54|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128063": { + "content": "<|reserved_special_token_55|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128064": { + "content": "<|reserved_special_token_56|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128065": { + "content": "<|reserved_special_token_57|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128066": { + "content": "<|reserved_special_token_58|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128067": { + "content": "<|reserved_special_token_59|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128068": { + "content": "<|reserved_special_token_60|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128069": { + "content": "<|reserved_special_token_61|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128070": { + "content": "<|reserved_special_token_62|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128071": { + "content": "<|reserved_special_token_63|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128072": { + "content": "<|reserved_special_token_64|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128073": { + "content": "<|reserved_special_token_65|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128074": { + "content": "<|reserved_special_token_66|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128075": { + "content": "<|reserved_special_token_67|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128076": { + "content": "<|reserved_special_token_68|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128077": { + "content": "<|reserved_special_token_69|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128078": { + "content": "<|reserved_special_token_70|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128079": { + "content": "<|reserved_special_token_71|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128080": { + "content": "<|reserved_special_token_72|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128081": { + "content": "<|reserved_special_token_73|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128082": { + "content": "<|reserved_special_token_74|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128083": { + "content": "<|reserved_special_token_75|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128084": { + "content": "<|reserved_special_token_76|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128085": { + "content": "<|reserved_special_token_77|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128086": { + "content": "<|reserved_special_token_78|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128087": { + "content": "<|reserved_special_token_79|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128088": { + "content": "<|reserved_special_token_80|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128089": { + "content": "<|reserved_special_token_81|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128090": { + "content": "<|reserved_special_token_82|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128091": { + "content": "<|reserved_special_token_83|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128092": { + "content": "<|reserved_special_token_84|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128093": { + "content": "<|reserved_special_token_85|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128094": { + "content": "<|reserved_special_token_86|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128095": { + "content": "<|reserved_special_token_87|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128096": { + "content": "<|reserved_special_token_88|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128097": { + "content": "<|reserved_special_token_89|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128098": { + "content": "<|reserved_special_token_90|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128099": { + "content": "<|reserved_special_token_91|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128100": { + "content": "<|reserved_special_token_92|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128101": { + "content": "<|reserved_special_token_93|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128102": { + "content": "<|reserved_special_token_94|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128103": { + "content": "<|reserved_special_token_95|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128104": { + "content": "<|reserved_special_token_96|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128105": { + "content": "<|reserved_special_token_97|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128106": { + "content": "<|reserved_special_token_98|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128107": { + "content": "<|reserved_special_token_99|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128108": { + "content": "<|reserved_special_token_100|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128109": { + "content": "<|reserved_special_token_101|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128110": { + "content": "<|reserved_special_token_102|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128111": { + "content": "<|reserved_special_token_103|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128112": { + "content": "<|reserved_special_token_104|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128113": { + "content": "<|reserved_special_token_105|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128114": { + "content": "<|reserved_special_token_106|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128115": { + "content": "<|reserved_special_token_107|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128116": { + "content": "<|reserved_special_token_108|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128117": { + "content": "<|reserved_special_token_109|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128118": { + "content": "<|reserved_special_token_110|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128119": { + "content": "<|reserved_special_token_111|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128120": { + "content": "<|reserved_special_token_112|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128121": { + "content": "<|reserved_special_token_113|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128122": { + "content": "<|reserved_special_token_114|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128123": { + "content": "<|reserved_special_token_115|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128124": { + "content": "<|reserved_special_token_116|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128125": { + "content": "<|reserved_special_token_117|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128126": { + "content": "<|reserved_special_token_118|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128127": { + "content": "<|reserved_special_token_119|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128128": { + "content": "<|reserved_special_token_120|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128129": { + "content": "<|reserved_special_token_121|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128130": { + "content": "<|reserved_special_token_122|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128131": { + "content": "<|reserved_special_token_123|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128132": { + "content": "<|reserved_special_token_124|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128133": { + "content": "<|reserved_special_token_125|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128134": { + "content": "<|reserved_special_token_126|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128135": { + "content": "<|reserved_special_token_127|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128136": { + "content": "<|reserved_special_token_128|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128137": { + "content": "<|reserved_special_token_129|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128138": { + "content": "<|reserved_special_token_130|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128139": { + "content": "<|reserved_special_token_131|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128140": { + "content": "<|reserved_special_token_132|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128141": { + "content": "<|reserved_special_token_133|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128142": { + "content": "<|reserved_special_token_134|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128143": { + "content": "<|reserved_special_token_135|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128144": { + "content": "<|reserved_special_token_136|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128145": { + "content": "<|reserved_special_token_137|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128146": { + "content": "<|reserved_special_token_138|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128147": { + "content": "<|reserved_special_token_139|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128148": { + "content": "<|reserved_special_token_140|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128149": { + "content": "<|reserved_special_token_141|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128150": { + "content": "<|reserved_special_token_142|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128151": { + "content": "<|reserved_special_token_143|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128152": { + "content": "<|reserved_special_token_144|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128153": { + "content": "<|reserved_special_token_145|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128154": { + "content": "<|reserved_special_token_146|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128155": { + "content": "<|reserved_special_token_147|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128156": { + "content": "<|reserved_special_token_148|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128157": { + "content": "<|reserved_special_token_149|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128158": { + "content": "<|reserved_special_token_150|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128159": { + "content": "<|reserved_special_token_151|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128160": { + "content": "<|reserved_special_token_152|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128161": { + "content": "<|reserved_special_token_153|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128162": { + "content": "<|reserved_special_token_154|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128163": { + "content": "<|reserved_special_token_155|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128164": { + "content": "<|reserved_special_token_156|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128165": { + "content": "<|reserved_special_token_157|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128166": { + "content": "<|reserved_special_token_158|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128167": { + "content": "<|reserved_special_token_159|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128168": { + "content": "<|reserved_special_token_160|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128169": { + "content": "<|reserved_special_token_161|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128170": { + "content": "<|reserved_special_token_162|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128171": { + "content": "<|reserved_special_token_163|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128172": { + "content": "<|reserved_special_token_164|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128173": { + "content": "<|reserved_special_token_165|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128174": { + "content": "<|reserved_special_token_166|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128175": { + "content": "<|reserved_special_token_167|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128176": { + "content": "<|reserved_special_token_168|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128177": { + "content": "<|reserved_special_token_169|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128178": { + "content": "<|reserved_special_token_170|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128179": { + "content": "<|reserved_special_token_171|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128180": { + "content": "<|reserved_special_token_172|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128181": { + "content": "<|reserved_special_token_173|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128182": { + "content": "<|reserved_special_token_174|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128183": { + "content": "<|reserved_special_token_175|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128184": { + "content": "<|reserved_special_token_176|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128185": { + "content": "<|reserved_special_token_177|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128186": { + "content": "<|reserved_special_token_178|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128187": { + "content": "<|reserved_special_token_179|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128188": { + "content": "<|reserved_special_token_180|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128189": { + "content": "<|reserved_special_token_181|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128190": { + "content": "<|reserved_special_token_182|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128191": { + "content": "<|reserved_special_token_183|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128192": { + "content": "<|reserved_special_token_184|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128193": { + "content": "<|reserved_special_token_185|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128194": { + "content": "<|reserved_special_token_186|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128195": { + "content": "<|reserved_special_token_187|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128196": { + "content": "<|reserved_special_token_188|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128197": { + "content": "<|reserved_special_token_189|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128198": { + "content": "<|reserved_special_token_190|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128199": { + "content": "<|reserved_special_token_191|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128200": { + "content": "<|reserved_special_token_192|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128201": { + "content": "<|reserved_special_token_193|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128202": { + "content": "<|reserved_special_token_194|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128203": { + "content": "<|reserved_special_token_195|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128204": { + "content": "<|reserved_special_token_196|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128205": { + "content": "<|reserved_special_token_197|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128206": { + "content": "<|reserved_special_token_198|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128207": { + "content": "<|reserved_special_token_199|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128208": { + "content": "<|reserved_special_token_200|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128209": { + "content": "<|reserved_special_token_201|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128210": { + "content": "<|reserved_special_token_202|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128211": { + "content": "<|reserved_special_token_203|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128212": { + "content": "<|reserved_special_token_204|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128213": { + "content": "<|reserved_special_token_205|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128214": { + "content": "<|reserved_special_token_206|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128215": { + "content": "<|reserved_special_token_207|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128216": { + "content": "<|reserved_special_token_208|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128217": { + "content": "<|reserved_special_token_209|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128218": { + "content": "<|reserved_special_token_210|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128219": { + "content": "<|reserved_special_token_211|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128220": { + "content": "<|reserved_special_token_212|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128221": { + "content": "<|reserved_special_token_213|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128222": { + "content": "<|reserved_special_token_214|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128223": { + "content": "<|reserved_special_token_215|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128224": { + "content": "<|reserved_special_token_216|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128225": { + "content": "<|reserved_special_token_217|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128226": { + "content": "<|reserved_special_token_218|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128227": { + "content": "<|reserved_special_token_219|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128228": { + "content": "<|reserved_special_token_220|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128229": { + "content": "<|reserved_special_token_221|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128230": { + "content": "<|reserved_special_token_222|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128231": { + "content": "<|reserved_special_token_223|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128232": { + "content": "<|reserved_special_token_224|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128233": { + "content": "<|reserved_special_token_225|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128234": { + "content": "<|reserved_special_token_226|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128235": { + "content": "<|reserved_special_token_227|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128236": { + "content": "<|reserved_special_token_228|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128237": { + "content": "<|reserved_special_token_229|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128238": { + "content": "<|reserved_special_token_230|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128239": { + "content": "<|reserved_special_token_231|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128240": { + "content": "<|reserved_special_token_232|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128241": { + "content": "<|reserved_special_token_233|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128242": { + "content": "<|reserved_special_token_234|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128243": { + "content": "<|reserved_special_token_235|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128244": { + "content": "<|reserved_special_token_236|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128245": { + "content": "<|reserved_special_token_237|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128246": { + "content": "<|reserved_special_token_238|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128247": { + "content": "<|reserved_special_token_239|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128248": { + "content": "<|reserved_special_token_240|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128249": { + "content": "<|reserved_special_token_241|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128250": { + "content": "<|reserved_special_token_242|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128251": { + "content": "<|reserved_special_token_243|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128252": { + "content": "<|reserved_special_token_244|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128253": { + "content": "<|reserved_special_token_245|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128254": { + "content": "<|reserved_special_token_246|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128255": { + "content": "<|reserved_special_token_247|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + } + }, + "bos_token": "<|begin_of_text|>", + "chat_template": "{{- bos_token }}\n{%- if custom_tools is defined %}\n {%- set tools = custom_tools %}\n{%- endif %}\n{%- if not tools_in_user_message is defined %}\n {%- set tools_in_user_message = true %}\n{%- endif %}\n{%- if not date_string is defined %}\n {%- set date_string = \"26 Jul 2024\" %}\n{%- endif %}\n{%- if not tools is defined %}\n {%- set tools = none %}\n{%- endif %}\n\n{#- This block extracts the system message, so we can slot it into the right place. #}\n{%- if messages[0]['role'] == 'system' %}\n {%- set system_message = messages[0]['content']|trim %}\n {%- set messages = messages[1:] %}\n{%- else %}\n {%- set system_message = \"\" %}\n{%- endif %}\n\n{#- System message + builtin tools #}\n{{- \"<|start_header_id|>system<|end_header_id|>\\n\\n\" }}\n{%- if builtin_tools is defined or tools is not none %}\n {{- \"Environment: ipython\\n\" }}\n{%- endif %}\n{%- if builtin_tools is defined %}\n {{- \"Tools: \" + builtin_tools | reject('equalto', 'code_interpreter') | join(\", \") + \"\\n\\n\"}}\n{%- endif %}\n{{- \"Cutting Knowledge Date: December 2023\\n\" }}\n{{- \"Today Date: \" + date_string + \"\\n\\n\" }}\n{%- if tools is not none and not tools_in_user_message %}\n {{- \"You have access to the following functions. To call a function, please respond with JSON for a function call.\" }}\n {{- 'Respond in the format {\"name\": function name, \"parameters\": dictionary of argument name and its value}.' }}\n {{- \"Do not use variables.\\n\\n\" }}\n {%- for t in tools %}\n {{- t | tojson(indent=4) }}\n {{- \"\\n\\n\" }}\n {%- endfor %}\n{%- endif %}\n{{- system_message }}\n{{- \"<|eot_id|>\" }}\n\n{#- Custom tools are passed in a user message with some extra guidance #}\n{%- if tools_in_user_message and not tools is none %}\n {#- Extract the first user message so we can plug it in here #}\n {%- if messages | length != 0 %}\n {%- set first_user_message = messages[0]['content']|trim %}\n {%- set messages = messages[1:] %}\n {%- else %}\n {{- raise_exception(\"Cannot put tools in the first user message when there's no first user message!\") }}\n{%- endif %}\n {{- '<|start_header_id|>user<|end_header_id|>\\n\\n' -}}\n {{- \"Given the following functions, please respond with a JSON for a function call \" }}\n {{- \"with its proper arguments that best answers the given prompt.\\n\\n\" }}\n {{- 'Respond in the format {\"name\": function name, \"parameters\": dictionary of argument name and its value}.' }}\n {{- \"Do not use variables.\\n\\n\" }}\n {%- for t in tools %}\n {{- t | tojson(indent=4) }}\n {{- \"\\n\\n\" }}\n {%- endfor %}\n {{- first_user_message + \"<|eot_id|>\"}}\n{%- endif %}\n\n{%- for message in messages %}\n {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}\n {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\\n\\n'+ message['content'] | trim + '<|eot_id|>' }}\n {%- elif 'tool_calls' in message %}\n {%- if not message.tool_calls|length == 1 %}\n {{- raise_exception(\"This model only supports single tool-calls at once!\") }}\n {%- endif %}\n {%- set tool_call = message.tool_calls[0].function %}\n {%- if builtin_tools is defined and tool_call.name in builtin_tools %}\n {{- '<|start_header_id|>assistant<|end_header_id|>\\n\\n' -}}\n {{- \"<|python_tag|>\" + tool_call.name + \".call(\" }}\n {%- for arg_name, arg_val in tool_call.arguments | items %}\n {{- arg_name + '=\"' + arg_val + '\"' }}\n {%- if not loop.last %}\n {{- \", \" }}\n {%- endif %}\n {%- endfor %}\n {{- \")\" }}\n {%- else %}\n {{- '<|start_header_id|>assistant<|end_header_id|>\\n\\n' -}}\n {{- '{\"name\": \"' + tool_call.name + '\", ' }}\n {{- '\"parameters\": ' }}\n {{- tool_call.arguments | tojson }}\n {{- \"}\" }}\n {%- endif %}\n {%- if builtin_tools is defined %}\n {#- This means we're in ipython mode #}\n {{- \"<|eom_id|>\" }}\n {%- else %}\n {{- \"<|eot_id|>\" }}\n {%- endif %}\n {%- elif message.role == \"tool\" or message.role == \"ipython\" %}\n {{- \"<|start_header_id|>ipython<|end_header_id|>\\n\\n\" }}\n {%- if message.content is mapping or message.content is iterable %}\n {{- message.content | tojson }}\n {%- else %}\n {{- message.content }}\n {%- endif %}\n {{- \"<|eot_id|>\" }}\n {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n {{- '<|start_header_id|>assistant<|end_header_id|>\\n\\n' }}\n{%- endif %}\n", + "clean_up_tokenization_spaces": true, + "eos_token": "<|eot_id|>", + "extra_special_tokens": {}, + "model_input_names": [ + "input_ids", + "attention_mask" + ], + "model_max_length": 8192, + "pad_token": "<|eot_id|>", + "tokenizer_class": "PreTrainedTokenizer" +} diff --git a/train_results.json b/train_results.json new file mode 100644 index 0000000..2d8681d --- /dev/null +++ b/train_results.json @@ -0,0 +1,9 @@ +{ + "epoch": 2.0, + "total_flos": 4057777727078400.0, + "train_loss": 0.5052211482704485, + "train_runtime": 135745.8608, + "train_samples": 1421904, + "train_samples_per_second": 4.568, + "train_steps_per_second": 0.071 +} \ No newline at end of file diff --git a/trainer_state.json b/trainer_state.json new file mode 100644 index 0000000..683a6b5 --- /dev/null +++ b/trainer_state.json @@ -0,0 +1,1417 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 2.0, + "eval_steps": 500, + "global_step": 9690, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.0002063983488132095, + "grad_norm": 21.791844475218177, + "learning_rate": 1.718213058419244e-08, + "loss": 1.2399, + "step": 1 + }, + { + "epoch": 0.010319917440660475, + "grad_norm": 2.758771436717397, + "learning_rate": 8.59106529209622e-07, + "loss": 1.0835, + "step": 50 + }, + { + "epoch": 0.02063983488132095, + "grad_norm": 1.6583275647715192, + "learning_rate": 1.718213058419244e-06, + "loss": 0.8281, + "step": 100 + }, + { + "epoch": 0.030959752321981424, + "grad_norm": 1.052445004470504, + "learning_rate": 2.577319587628866e-06, + "loss": 0.7456, + "step": 150 + }, + { + "epoch": 0.0412796697626419, + "grad_norm": 0.6644727438734609, + "learning_rate": 3.436426116838488e-06, + "loss": 0.6981, + "step": 200 + }, + { + "epoch": 0.05159958720330237, + "grad_norm": 0.6511384641685941, + "learning_rate": 4.2955326460481105e-06, + "loss": 0.6756, + "step": 250 + }, + { + "epoch": 0.06191950464396285, + "grad_norm": 0.6728510847115857, + "learning_rate": 4.995212256623046e-06, + "loss": 0.669, + "step": 300 + }, + { + "epoch": 0.07223942208462332, + "grad_norm": 0.6002063765996374, + "learning_rate": 4.968613682306629e-06, + "loss": 0.6488, + "step": 350 + }, + { + "epoch": 0.0825593395252838, + "grad_norm": 0.5807563716846417, + "learning_rate": 4.942015107990212e-06, + "loss": 0.641, + "step": 400 + }, + { + "epoch": 0.09287925696594428, + "grad_norm": 0.5651403042158113, + "learning_rate": 4.915416533673795e-06, + "loss": 0.6347, + "step": 450 + }, + { + "epoch": 0.10319917440660474, + "grad_norm": 0.5556237524713133, + "learning_rate": 4.888817959357379e-06, + "loss": 0.6292, + "step": 500 + }, + { + "epoch": 0.11351909184726522, + "grad_norm": 0.5537920539892396, + "learning_rate": 4.862219385040962e-06, + "loss": 0.6207, + "step": 550 + }, + { + "epoch": 0.1238390092879257, + "grad_norm": 0.6516923682179434, + "learning_rate": 4.835620810724546e-06, + "loss": 0.6221, + "step": 600 + }, + { + "epoch": 0.13415892672858618, + "grad_norm": 0.5993983563444428, + "learning_rate": 4.809022236408129e-06, + "loss": 0.6134, + "step": 650 + }, + { + "epoch": 0.14447884416924664, + "grad_norm": 0.5382978582953953, + "learning_rate": 4.782423662091712e-06, + "loss": 0.6142, + "step": 700 + }, + { + "epoch": 0.15479876160990713, + "grad_norm": 0.5360001817415995, + "learning_rate": 4.7558250877752955e-06, + "loss": 0.6064, + "step": 750 + }, + { + "epoch": 0.1651186790505676, + "grad_norm": 0.5196308091067839, + "learning_rate": 4.729226513458879e-06, + "loss": 0.5988, + "step": 800 + }, + { + "epoch": 0.17543859649122806, + "grad_norm": 0.5427506989599503, + "learning_rate": 4.702627939142462e-06, + "loss": 0.6025, + "step": 850 + }, + { + "epoch": 0.18575851393188855, + "grad_norm": 0.5521424095703439, + "learning_rate": 4.676029364826046e-06, + "loss": 0.597, + "step": 900 + }, + { + "epoch": 0.19607843137254902, + "grad_norm": 0.6345336850282564, + "learning_rate": 4.649430790509629e-06, + "loss": 0.5887, + "step": 950 + }, + { + "epoch": 0.20639834881320948, + "grad_norm": 0.5971258034612797, + "learning_rate": 4.6228322161932124e-06, + "loss": 0.5815, + "step": 1000 + }, + { + "epoch": 0.21671826625386997, + "grad_norm": 0.5438942448938008, + "learning_rate": 4.596233641876796e-06, + "loss": 0.5805, + "step": 1050 + }, + { + "epoch": 0.22703818369453044, + "grad_norm": 0.5334432938197203, + "learning_rate": 4.569635067560379e-06, + "loss": 0.5738, + "step": 1100 + }, + { + "epoch": 0.23735810113519093, + "grad_norm": 0.5347450534004635, + "learning_rate": 4.543036493243962e-06, + "loss": 0.5801, + "step": 1150 + }, + { + "epoch": 0.2476780185758514, + "grad_norm": 0.5873203374469886, + "learning_rate": 4.516437918927546e-06, + "loss": 0.5834, + "step": 1200 + }, + { + "epoch": 0.2579979360165119, + "grad_norm": 0.5240586732453324, + "learning_rate": 4.489839344611129e-06, + "loss": 0.5721, + "step": 1250 + }, + { + "epoch": 0.26831785345717235, + "grad_norm": 0.5575825059292604, + "learning_rate": 4.463240770294713e-06, + "loss": 0.5779, + "step": 1300 + }, + { + "epoch": 0.2786377708978328, + "grad_norm": 0.5561922023755285, + "learning_rate": 4.436642195978296e-06, + "loss": 0.5739, + "step": 1350 + }, + { + "epoch": 0.2889576883384933, + "grad_norm": 0.5214056422410267, + "learning_rate": 4.410043621661879e-06, + "loss": 0.5593, + "step": 1400 + }, + { + "epoch": 0.29927760577915374, + "grad_norm": 0.5509795794922944, + "learning_rate": 4.383445047345462e-06, + "loss": 0.563, + "step": 1450 + }, + { + "epoch": 0.30959752321981426, + "grad_norm": 0.5698665633020146, + "learning_rate": 4.356846473029046e-06, + "loss": 0.5627, + "step": 1500 + }, + { + "epoch": 0.31991744066047473, + "grad_norm": 0.5245577515841476, + "learning_rate": 4.3302478987126295e-06, + "loss": 0.5678, + "step": 1550 + }, + { + "epoch": 0.3302373581011352, + "grad_norm": 0.6001876556912392, + "learning_rate": 4.303649324396213e-06, + "loss": 0.5709, + "step": 1600 + }, + { + "epoch": 0.34055727554179566, + "grad_norm": 0.5333018384868324, + "learning_rate": 4.277050750079796e-06, + "loss": 0.5581, + "step": 1650 + }, + { + "epoch": 0.3508771929824561, + "grad_norm": 0.5133912524315866, + "learning_rate": 4.250452175763379e-06, + "loss": 0.5618, + "step": 1700 + }, + { + "epoch": 0.36119711042311664, + "grad_norm": 0.5077938795371925, + "learning_rate": 4.223853601446962e-06, + "loss": 0.5573, + "step": 1750 + }, + { + "epoch": 0.3715170278637771, + "grad_norm": 0.5463753688650598, + "learning_rate": 4.1972550271305465e-06, + "loss": 0.5539, + "step": 1800 + }, + { + "epoch": 0.38183694530443757, + "grad_norm": 0.5787935899358873, + "learning_rate": 4.17065645281413e-06, + "loss": 0.5483, + "step": 1850 + }, + { + "epoch": 0.39215686274509803, + "grad_norm": 0.5654912899054569, + "learning_rate": 4.144057878497713e-06, + "loss": 0.558, + "step": 1900 + }, + { + "epoch": 0.4024767801857585, + "grad_norm": 0.4996009494084185, + "learning_rate": 4.117459304181296e-06, + "loss": 0.5456, + "step": 1950 + }, + { + "epoch": 0.41279669762641896, + "grad_norm": 0.5667395400639976, + "learning_rate": 4.090860729864879e-06, + "loss": 0.5486, + "step": 2000 + }, + { + "epoch": 0.4231166150670795, + "grad_norm": 0.5188779450810256, + "learning_rate": 4.0642621555484625e-06, + "loss": 0.5464, + "step": 2050 + }, + { + "epoch": 0.43343653250773995, + "grad_norm": 0.5362183280913333, + "learning_rate": 4.037663581232047e-06, + "loss": 0.5521, + "step": 2100 + }, + { + "epoch": 0.4437564499484004, + "grad_norm": 0.5473955026723762, + "learning_rate": 4.01106500691563e-06, + "loss": 0.5445, + "step": 2150 + }, + { + "epoch": 0.4540763673890609, + "grad_norm": 0.5397425583264429, + "learning_rate": 3.984466432599213e-06, + "loss": 0.5585, + "step": 2200 + }, + { + "epoch": 0.46439628482972134, + "grad_norm": 0.5821347601023747, + "learning_rate": 3.957867858282796e-06, + "loss": 0.5513, + "step": 2250 + }, + { + "epoch": 0.47471620227038186, + "grad_norm": 0.5167859554911757, + "learning_rate": 3.9312692839663795e-06, + "loss": 0.5401, + "step": 2300 + }, + { + "epoch": 0.4850361197110423, + "grad_norm": 0.5353436940913985, + "learning_rate": 3.904670709649963e-06, + "loss": 0.5413, + "step": 2350 + }, + { + "epoch": 0.4953560371517028, + "grad_norm": 0.5450597051287799, + "learning_rate": 3.878072135333547e-06, + "loss": 0.5377, + "step": 2400 + }, + { + "epoch": 0.5056759545923633, + "grad_norm": 0.5133718526538825, + "learning_rate": 3.85147356101713e-06, + "loss": 0.5411, + "step": 2450 + }, + { + "epoch": 0.5159958720330238, + "grad_norm": 0.5194969527334257, + "learning_rate": 3.824874986700713e-06, + "loss": 0.5504, + "step": 2500 + }, + { + "epoch": 0.5263157894736842, + "grad_norm": 0.5244000570689482, + "learning_rate": 3.7982764123842964e-06, + "loss": 0.5373, + "step": 2550 + }, + { + "epoch": 0.5366357069143447, + "grad_norm": 0.4954661200186007, + "learning_rate": 3.7716778380678796e-06, + "loss": 0.5321, + "step": 2600 + }, + { + "epoch": 0.5469556243550051, + "grad_norm": 0.507806040946836, + "learning_rate": 3.7450792637514633e-06, + "loss": 0.5363, + "step": 2650 + }, + { + "epoch": 0.5572755417956656, + "grad_norm": 0.5217141184924292, + "learning_rate": 3.7184806894350465e-06, + "loss": 0.5374, + "step": 2700 + }, + { + "epoch": 0.5675954592363261, + "grad_norm": 1.0654133163473936, + "learning_rate": 3.69188211511863e-06, + "loss": 0.5387, + "step": 2750 + }, + { + "epoch": 0.5779153766769866, + "grad_norm": 0.5300385445449849, + "learning_rate": 3.665283540802213e-06, + "loss": 0.5328, + "step": 2800 + }, + { + "epoch": 0.5882352941176471, + "grad_norm": 0.5195789655014079, + "learning_rate": 3.6386849664857966e-06, + "loss": 0.5448, + "step": 2850 + }, + { + "epoch": 0.5985552115583075, + "grad_norm": 0.5400701284508418, + "learning_rate": 3.6120863921693798e-06, + "loss": 0.5225, + "step": 2900 + }, + { + "epoch": 0.608875128998968, + "grad_norm": 0.5192375803111897, + "learning_rate": 3.5854878178529634e-06, + "loss": 0.5354, + "step": 2950 + }, + { + "epoch": 0.6191950464396285, + "grad_norm": 0.5157981546008934, + "learning_rate": 3.558889243536547e-06, + "loss": 0.5234, + "step": 3000 + }, + { + "epoch": 0.6295149638802889, + "grad_norm": 0.5189888311937375, + "learning_rate": 3.5322906692201303e-06, + "loss": 0.5261, + "step": 3050 + }, + { + "epoch": 0.6398348813209495, + "grad_norm": 0.520981935459507, + "learning_rate": 3.505692094903713e-06, + "loss": 0.5277, + "step": 3100 + }, + { + "epoch": 0.6501547987616099, + "grad_norm": 0.5385057842452714, + "learning_rate": 3.4790935205872967e-06, + "loss": 0.5337, + "step": 3150 + }, + { + "epoch": 0.6604747162022704, + "grad_norm": 0.5090039339282304, + "learning_rate": 3.45249494627088e-06, + "loss": 0.5246, + "step": 3200 + }, + { + "epoch": 0.6707946336429309, + "grad_norm": 0.5309810767744108, + "learning_rate": 3.4258963719544636e-06, + "loss": 0.532, + "step": 3250 + }, + { + "epoch": 0.6811145510835913, + "grad_norm": 0.5117993267505484, + "learning_rate": 3.3992977976380472e-06, + "loss": 0.5329, + "step": 3300 + }, + { + "epoch": 0.6914344685242518, + "grad_norm": 0.542882178286457, + "learning_rate": 3.3726992233216304e-06, + "loss": 0.5262, + "step": 3350 + }, + { + "epoch": 0.7017543859649122, + "grad_norm": 0.5485246009385575, + "learning_rate": 3.3461006490052132e-06, + "loss": 0.5288, + "step": 3400 + }, + { + "epoch": 0.7120743034055728, + "grad_norm": 0.5245153157581836, + "learning_rate": 3.319502074688797e-06, + "loss": 0.5287, + "step": 3450 + }, + { + "epoch": 0.7223942208462333, + "grad_norm": 0.5408996443293894, + "learning_rate": 3.29290350037238e-06, + "loss": 0.5325, + "step": 3500 + }, + { + "epoch": 0.7327141382868937, + "grad_norm": 0.539947013464035, + "learning_rate": 3.2663049260559637e-06, + "loss": 0.523, + "step": 3550 + }, + { + "epoch": 0.7430340557275542, + "grad_norm": 0.5322841773244653, + "learning_rate": 3.2397063517395474e-06, + "loss": 0.5226, + "step": 3600 + }, + { + "epoch": 0.7533539731682146, + "grad_norm": 0.5150328597209529, + "learning_rate": 3.2131077774231306e-06, + "loss": 0.5182, + "step": 3650 + }, + { + "epoch": 0.7636738906088751, + "grad_norm": 0.5187564835457917, + "learning_rate": 3.1865092031067134e-06, + "loss": 0.5226, + "step": 3700 + }, + { + "epoch": 0.7739938080495357, + "grad_norm": 0.5376206106185862, + "learning_rate": 3.159910628790297e-06, + "loss": 0.5221, + "step": 3750 + }, + { + "epoch": 0.7843137254901961, + "grad_norm": 0.4886509664687733, + "learning_rate": 3.1333120544738802e-06, + "loss": 0.5167, + "step": 3800 + }, + { + "epoch": 0.7946336429308566, + "grad_norm": 0.5186897778877932, + "learning_rate": 3.106713480157464e-06, + "loss": 0.5215, + "step": 3850 + }, + { + "epoch": 0.804953560371517, + "grad_norm": 0.5662670913922451, + "learning_rate": 3.0801149058410475e-06, + "loss": 0.5156, + "step": 3900 + }, + { + "epoch": 0.8152734778121775, + "grad_norm": 0.5068699760497294, + "learning_rate": 3.0535163315246307e-06, + "loss": 0.5117, + "step": 3950 + }, + { + "epoch": 0.8255933952528379, + "grad_norm": 0.525153451451465, + "learning_rate": 3.0269177572082135e-06, + "loss": 0.5284, + "step": 4000 + }, + { + "epoch": 0.8359133126934984, + "grad_norm": 0.5415917077347273, + "learning_rate": 3.000319182891797e-06, + "loss": 0.5233, + "step": 4050 + }, + { + "epoch": 0.846233230134159, + "grad_norm": 0.5122355304442054, + "learning_rate": 2.9737206085753804e-06, + "loss": 0.52, + "step": 4100 + }, + { + "epoch": 0.8565531475748194, + "grad_norm": 0.5084493187015482, + "learning_rate": 2.947122034258964e-06, + "loss": 0.5122, + "step": 4150 + }, + { + "epoch": 0.8668730650154799, + "grad_norm": 0.5222318358455478, + "learning_rate": 2.9205234599425477e-06, + "loss": 0.5164, + "step": 4200 + }, + { + "epoch": 0.8771929824561403, + "grad_norm": 0.4952100113115859, + "learning_rate": 2.893924885626131e-06, + "loss": 0.5184, + "step": 4250 + }, + { + "epoch": 0.8875128998968008, + "grad_norm": 0.5129926015439157, + "learning_rate": 2.8673263113097137e-06, + "loss": 0.5217, + "step": 4300 + }, + { + "epoch": 0.8978328173374613, + "grad_norm": 0.5408976129024315, + "learning_rate": 2.8407277369932973e-06, + "loss": 0.5298, + "step": 4350 + }, + { + "epoch": 0.9081527347781218, + "grad_norm": 0.523128008449258, + "learning_rate": 2.8141291626768805e-06, + "loss": 0.5236, + "step": 4400 + }, + { + "epoch": 0.9184726522187823, + "grad_norm": 0.545708918135539, + "learning_rate": 2.787530588360464e-06, + "loss": 0.516, + "step": 4450 + }, + { + "epoch": 0.9287925696594427, + "grad_norm": 0.5643988024472154, + "learning_rate": 2.760932014044048e-06, + "loss": 0.5157, + "step": 4500 + }, + { + "epoch": 0.9391124871001032, + "grad_norm": 0.49490206521060065, + "learning_rate": 2.7343334397276306e-06, + "loss": 0.5143, + "step": 4550 + }, + { + "epoch": 0.9494324045407637, + "grad_norm": 0.530477676869888, + "learning_rate": 2.707734865411214e-06, + "loss": 0.5097, + "step": 4600 + }, + { + "epoch": 0.9597523219814241, + "grad_norm": 0.5419644910097084, + "learning_rate": 2.6811362910947975e-06, + "loss": 0.5047, + "step": 4650 + }, + { + "epoch": 0.9700722394220846, + "grad_norm": 0.563023481870365, + "learning_rate": 2.654537716778381e-06, + "loss": 0.51, + "step": 4700 + }, + { + "epoch": 0.9803921568627451, + "grad_norm": 0.5885551415729929, + "learning_rate": 2.6279391424619643e-06, + "loss": 0.5141, + "step": 4750 + }, + { + "epoch": 0.9907120743034056, + "grad_norm": 0.555238399523142, + "learning_rate": 2.601340568145548e-06, + "loss": 0.5144, + "step": 4800 + }, + { + "epoch": 1.0, + "eval_loss": 0.5763986110687256, + "eval_runtime": 22.6777, + "eval_samples_per_second": 23.724, + "eval_steps_per_second": 0.397, + "step": 4845 + }, + { + "epoch": 1.001031991744066, + "grad_norm": 0.5251455186752938, + "learning_rate": 2.5747419938291308e-06, + "loss": 0.5043, + "step": 4850 + }, + { + "epoch": 1.0113519091847265, + "grad_norm": 0.5102862437645022, + "learning_rate": 2.548143419512714e-06, + "loss": 0.4557, + "step": 4900 + }, + { + "epoch": 1.021671826625387, + "grad_norm": 0.5329812930765626, + "learning_rate": 2.5215448451962976e-06, + "loss": 0.4593, + "step": 4950 + }, + { + "epoch": 1.0319917440660475, + "grad_norm": 0.5310044546142733, + "learning_rate": 2.4949462708798813e-06, + "loss": 0.4601, + "step": 5000 + }, + { + "epoch": 1.0423116615067078, + "grad_norm": 0.5291259662919947, + "learning_rate": 2.4683476965634645e-06, + "loss": 0.4573, + "step": 5050 + }, + { + "epoch": 1.0526315789473684, + "grad_norm": 0.5758879453674484, + "learning_rate": 2.4417491222470477e-06, + "loss": 0.4545, + "step": 5100 + }, + { + "epoch": 1.0629514963880289, + "grad_norm": 0.5701692974214282, + "learning_rate": 2.4151505479306313e-06, + "loss": 0.4542, + "step": 5150 + }, + { + "epoch": 1.0732714138286894, + "grad_norm": 0.584779025941461, + "learning_rate": 2.3885519736142146e-06, + "loss": 0.4494, + "step": 5200 + }, + { + "epoch": 1.08359133126935, + "grad_norm": 0.5634926571145632, + "learning_rate": 2.3619533992977978e-06, + "loss": 0.4453, + "step": 5250 + }, + { + "epoch": 1.0939112487100102, + "grad_norm": 0.5287888561997275, + "learning_rate": 2.3353548249813814e-06, + "loss": 0.4494, + "step": 5300 + }, + { + "epoch": 1.1042311661506707, + "grad_norm": 0.566118676754179, + "learning_rate": 2.3087562506649646e-06, + "loss": 0.4472, + "step": 5350 + }, + { + "epoch": 1.1145510835913313, + "grad_norm": 0.5591675478155297, + "learning_rate": 2.282157676348548e-06, + "loss": 0.4487, + "step": 5400 + }, + { + "epoch": 1.1248710010319918, + "grad_norm": 0.5234427712256148, + "learning_rate": 2.2555591020321315e-06, + "loss": 0.4564, + "step": 5450 + }, + { + "epoch": 1.1351909184726523, + "grad_norm": 0.5341771123435606, + "learning_rate": 2.2289605277157147e-06, + "loss": 0.4546, + "step": 5500 + }, + { + "epoch": 1.1455108359133126, + "grad_norm": 0.5963942909826432, + "learning_rate": 2.202361953399298e-06, + "loss": 0.4523, + "step": 5550 + }, + { + "epoch": 1.1558307533539731, + "grad_norm": 0.5668397788463199, + "learning_rate": 2.1757633790828816e-06, + "loss": 0.456, + "step": 5600 + }, + { + "epoch": 1.1661506707946336, + "grad_norm": 0.5324369168992059, + "learning_rate": 2.1491648047664648e-06, + "loss": 0.458, + "step": 5650 + }, + { + "epoch": 1.1764705882352942, + "grad_norm": 0.5936240911653264, + "learning_rate": 2.122566230450048e-06, + "loss": 0.4406, + "step": 5700 + }, + { + "epoch": 1.1867905056759547, + "grad_norm": 0.5813588072096951, + "learning_rate": 2.0959676561336316e-06, + "loss": 0.4542, + "step": 5750 + }, + { + "epoch": 1.197110423116615, + "grad_norm": 0.585714648308596, + "learning_rate": 2.0693690818172144e-06, + "loss": 0.4435, + "step": 5800 + }, + { + "epoch": 1.2074303405572755, + "grad_norm": 0.5428768784930765, + "learning_rate": 2.042770507500798e-06, + "loss": 0.4468, + "step": 5850 + }, + { + "epoch": 1.217750257997936, + "grad_norm": 0.5368254232870538, + "learning_rate": 2.0161719331843817e-06, + "loss": 0.4538, + "step": 5900 + }, + { + "epoch": 1.2280701754385965, + "grad_norm": 0.548798947651045, + "learning_rate": 1.9895733588679645e-06, + "loss": 0.447, + "step": 5950 + }, + { + "epoch": 1.238390092879257, + "grad_norm": 0.612099006418541, + "learning_rate": 1.962974784551548e-06, + "loss": 0.4515, + "step": 6000 + }, + { + "epoch": 1.2487100103199174, + "grad_norm": 0.55639703440266, + "learning_rate": 1.936376210235132e-06, + "loss": 0.446, + "step": 6050 + }, + { + "epoch": 1.2590299277605779, + "grad_norm": 0.5807813468059144, + "learning_rate": 1.9097776359187146e-06, + "loss": 0.4518, + "step": 6100 + }, + { + "epoch": 1.2693498452012384, + "grad_norm": 0.5654058519092208, + "learning_rate": 1.8831790616022982e-06, + "loss": 0.4458, + "step": 6150 + }, + { + "epoch": 1.279669762641899, + "grad_norm": 0.5935622910169027, + "learning_rate": 1.8565804872858817e-06, + "loss": 0.4539, + "step": 6200 + }, + { + "epoch": 1.2899896800825594, + "grad_norm": 0.5320246374099179, + "learning_rate": 1.8299819129694649e-06, + "loss": 0.4523, + "step": 6250 + }, + { + "epoch": 1.3003095975232197, + "grad_norm": 0.5993250962476825, + "learning_rate": 1.8033833386530483e-06, + "loss": 0.4525, + "step": 6300 + }, + { + "epoch": 1.3106295149638802, + "grad_norm": 0.5706555061755686, + "learning_rate": 1.7767847643366317e-06, + "loss": 0.448, + "step": 6350 + }, + { + "epoch": 1.3209494324045408, + "grad_norm": 0.5604368666415582, + "learning_rate": 1.750186190020215e-06, + "loss": 0.4547, + "step": 6400 + }, + { + "epoch": 1.3312693498452013, + "grad_norm": 0.5368477750147255, + "learning_rate": 1.7235876157037984e-06, + "loss": 0.4522, + "step": 6450 + }, + { + "epoch": 1.3415892672858618, + "grad_norm": 0.608531892658329, + "learning_rate": 1.6969890413873818e-06, + "loss": 0.4529, + "step": 6500 + }, + { + "epoch": 1.351909184726522, + "grad_norm": 0.5524704535036694, + "learning_rate": 1.670390467070965e-06, + "loss": 0.4449, + "step": 6550 + }, + { + "epoch": 1.3622291021671826, + "grad_norm": 0.608361151275691, + "learning_rate": 1.6437918927545484e-06, + "loss": 0.449, + "step": 6600 + }, + { + "epoch": 1.3725490196078431, + "grad_norm": 0.5350140506451774, + "learning_rate": 1.6171933184381319e-06, + "loss": 0.4448, + "step": 6650 + }, + { + "epoch": 1.3828689370485037, + "grad_norm": 0.5657631888716299, + "learning_rate": 1.590594744121715e-06, + "loss": 0.4505, + "step": 6700 + }, + { + "epoch": 1.3931888544891642, + "grad_norm": 0.5638748961635498, + "learning_rate": 1.5639961698052985e-06, + "loss": 0.4516, + "step": 6750 + }, + { + "epoch": 1.4035087719298245, + "grad_norm": 0.5838904002390698, + "learning_rate": 1.537397595488882e-06, + "loss": 0.4434, + "step": 6800 + }, + { + "epoch": 1.413828689370485, + "grad_norm": 0.582688659499904, + "learning_rate": 1.5107990211724652e-06, + "loss": 0.4451, + "step": 6850 + }, + { + "epoch": 1.4241486068111455, + "grad_norm": 0.5666153072627447, + "learning_rate": 1.4842004468560486e-06, + "loss": 0.4513, + "step": 6900 + }, + { + "epoch": 1.434468524251806, + "grad_norm": 0.5830919184881627, + "learning_rate": 1.457601872539632e-06, + "loss": 0.4499, + "step": 6950 + }, + { + "epoch": 1.4447884416924666, + "grad_norm": 0.5392957954275294, + "learning_rate": 1.4310032982232152e-06, + "loss": 0.4526, + "step": 7000 + }, + { + "epoch": 1.4551083591331269, + "grad_norm": 0.5796751590874428, + "learning_rate": 1.4044047239067987e-06, + "loss": 0.4464, + "step": 7050 + }, + { + "epoch": 1.4654282765737874, + "grad_norm": 0.649115716424128, + "learning_rate": 1.377806149590382e-06, + "loss": 0.4448, + "step": 7100 + }, + { + "epoch": 1.475748194014448, + "grad_norm": 0.5437354389754013, + "learning_rate": 1.3512075752739653e-06, + "loss": 0.4488, + "step": 7150 + }, + { + "epoch": 1.4860681114551084, + "grad_norm": 0.575952575422212, + "learning_rate": 1.3246090009575488e-06, + "loss": 0.4445, + "step": 7200 + }, + { + "epoch": 1.496388028895769, + "grad_norm": 0.5689643540076003, + "learning_rate": 1.2980104266411322e-06, + "loss": 0.4427, + "step": 7250 + }, + { + "epoch": 1.5067079463364292, + "grad_norm": 0.5433467470767702, + "learning_rate": 1.2714118523247154e-06, + "loss": 0.4469, + "step": 7300 + }, + { + "epoch": 1.5170278637770898, + "grad_norm": 0.5785369307750878, + "learning_rate": 1.2448132780082988e-06, + "loss": 0.4418, + "step": 7350 + }, + { + "epoch": 1.5273477812177503, + "grad_norm": 0.5946065204485477, + "learning_rate": 1.2182147036918823e-06, + "loss": 0.45, + "step": 7400 + }, + { + "epoch": 1.5376676986584106, + "grad_norm": 0.5276268731765017, + "learning_rate": 1.1916161293754657e-06, + "loss": 0.4425, + "step": 7450 + }, + { + "epoch": 1.5479876160990713, + "grad_norm": 0.5456969115711289, + "learning_rate": 1.165017555059049e-06, + "loss": 0.4426, + "step": 7500 + }, + { + "epoch": 1.5583075335397316, + "grad_norm": 0.5626443461851722, + "learning_rate": 1.1384189807426323e-06, + "loss": 0.4457, + "step": 7550 + }, + { + "epoch": 1.5686274509803921, + "grad_norm": 0.5748093743725187, + "learning_rate": 1.1118204064262155e-06, + "loss": 0.4425, + "step": 7600 + }, + { + "epoch": 1.5789473684210527, + "grad_norm": 0.6398790495774637, + "learning_rate": 1.085221832109799e-06, + "loss": 0.4461, + "step": 7650 + }, + { + "epoch": 1.589267285861713, + "grad_norm": 0.5502896358409545, + "learning_rate": 1.0586232577933824e-06, + "loss": 0.4324, + "step": 7700 + }, + { + "epoch": 1.5995872033023737, + "grad_norm": 0.547970248426798, + "learning_rate": 1.0320246834769656e-06, + "loss": 0.4367, + "step": 7750 + }, + { + "epoch": 1.609907120743034, + "grad_norm": 0.5553997442333781, + "learning_rate": 1.005426109160549e-06, + "loss": 0.4518, + "step": 7800 + }, + { + "epoch": 1.6202270381836945, + "grad_norm": 0.5463386099370533, + "learning_rate": 9.788275348441325e-07, + "loss": 0.4494, + "step": 7850 + }, + { + "epoch": 1.630546955624355, + "grad_norm": 0.5924906419908685, + "learning_rate": 9.522289605277157e-07, + "loss": 0.441, + "step": 7900 + }, + { + "epoch": 1.6408668730650153, + "grad_norm": 0.5565206258809997, + "learning_rate": 9.256303862112992e-07, + "loss": 0.4417, + "step": 7950 + }, + { + "epoch": 1.651186790505676, + "grad_norm": 0.587012594186078, + "learning_rate": 8.990318118948826e-07, + "loss": 0.4442, + "step": 8000 + }, + { + "epoch": 1.6615067079463364, + "grad_norm": 0.5771632055191758, + "learning_rate": 8.724332375784658e-07, + "loss": 0.4456, + "step": 8050 + }, + { + "epoch": 1.671826625386997, + "grad_norm": 0.56799994088298, + "learning_rate": 8.458346632620493e-07, + "loss": 0.4486, + "step": 8100 + }, + { + "epoch": 1.6821465428276574, + "grad_norm": 0.569590541861768, + "learning_rate": 8.192360889456326e-07, + "loss": 0.449, + "step": 8150 + }, + { + "epoch": 1.6924664602683177, + "grad_norm": 0.5795380939294089, + "learning_rate": 7.926375146292158e-07, + "loss": 0.4438, + "step": 8200 + }, + { + "epoch": 1.7027863777089784, + "grad_norm": 0.5706726979508648, + "learning_rate": 7.660389403127994e-07, + "loss": 0.4443, + "step": 8250 + }, + { + "epoch": 1.7131062951496387, + "grad_norm": 0.5791014554749523, + "learning_rate": 7.394403659963827e-07, + "loss": 0.4481, + "step": 8300 + }, + { + "epoch": 1.7234262125902993, + "grad_norm": 0.561090943078198, + "learning_rate": 7.12841791679966e-07, + "loss": 0.4397, + "step": 8350 + }, + { + "epoch": 1.7337461300309598, + "grad_norm": 0.5863685907601032, + "learning_rate": 6.862432173635495e-07, + "loss": 0.4445, + "step": 8400 + }, + { + "epoch": 1.74406604747162, + "grad_norm": 0.5730575809697048, + "learning_rate": 6.596446430471328e-07, + "loss": 0.4451, + "step": 8450 + }, + { + "epoch": 1.7543859649122808, + "grad_norm": 0.5635343819200634, + "learning_rate": 6.330460687307161e-07, + "loss": 0.4382, + "step": 8500 + }, + { + "epoch": 1.7647058823529411, + "grad_norm": 0.5646651214526662, + "learning_rate": 6.064474944142994e-07, + "loss": 0.4403, + "step": 8550 + }, + { + "epoch": 1.7750257997936016, + "grad_norm": 0.5486977071368843, + "learning_rate": 5.798489200978829e-07, + "loss": 0.4355, + "step": 8600 + }, + { + "epoch": 1.7853457172342622, + "grad_norm": 0.5568438386062177, + "learning_rate": 5.532503457814662e-07, + "loss": 0.4477, + "step": 8650 + }, + { + "epoch": 1.7956656346749225, + "grad_norm": 0.5622396972214505, + "learning_rate": 5.266517714650495e-07, + "loss": 0.4377, + "step": 8700 + }, + { + "epoch": 1.8059855521155832, + "grad_norm": 0.552378184120207, + "learning_rate": 5.000531971486329e-07, + "loss": 0.4467, + "step": 8750 + }, + { + "epoch": 1.8163054695562435, + "grad_norm": 0.5708548011902037, + "learning_rate": 4.734546228322162e-07, + "loss": 0.439, + "step": 8800 + }, + { + "epoch": 1.826625386996904, + "grad_norm": 0.6282302095096289, + "learning_rate": 4.468560485157996e-07, + "loss": 0.4415, + "step": 8850 + }, + { + "epoch": 1.8369453044375645, + "grad_norm": 0.5526339563083117, + "learning_rate": 4.20257474199383e-07, + "loss": 0.4434, + "step": 8900 + }, + { + "epoch": 1.8472652218782248, + "grad_norm": 0.5627366388182174, + "learning_rate": 3.936588998829663e-07, + "loss": 0.4406, + "step": 8950 + }, + { + "epoch": 1.8575851393188856, + "grad_norm": 0.5675273603944594, + "learning_rate": 3.6706032556654965e-07, + "loss": 0.4364, + "step": 9000 + }, + { + "epoch": 1.8679050567595459, + "grad_norm": 0.6096156008730134, + "learning_rate": 3.40461751250133e-07, + "loss": 0.4421, + "step": 9050 + }, + { + "epoch": 1.8782249742002064, + "grad_norm": 0.655037790258665, + "learning_rate": 3.1386317693371635e-07, + "loss": 0.4456, + "step": 9100 + }, + { + "epoch": 1.888544891640867, + "grad_norm": 0.5522478423907845, + "learning_rate": 2.8726460261729973e-07, + "loss": 0.4399, + "step": 9150 + }, + { + "epoch": 1.8988648090815272, + "grad_norm": 0.5499766592623824, + "learning_rate": 2.606660283008831e-07, + "loss": 0.4447, + "step": 9200 + }, + { + "epoch": 1.909184726522188, + "grad_norm": 0.6047531326573264, + "learning_rate": 2.3406745398446645e-07, + "loss": 0.4483, + "step": 9250 + }, + { + "epoch": 1.9195046439628483, + "grad_norm": 0.5307110417886833, + "learning_rate": 2.074688796680498e-07, + "loss": 0.4474, + "step": 9300 + }, + { + "epoch": 1.9298245614035088, + "grad_norm": 0.5838188935676187, + "learning_rate": 1.8087030535163318e-07, + "loss": 0.4453, + "step": 9350 + }, + { + "epoch": 1.9401444788441693, + "grad_norm": 0.524912113097606, + "learning_rate": 1.5427173103521653e-07, + "loss": 0.4441, + "step": 9400 + }, + { + "epoch": 1.9504643962848296, + "grad_norm": 0.5892672773445452, + "learning_rate": 1.2767315671879988e-07, + "loss": 0.4368, + "step": 9450 + }, + { + "epoch": 1.9607843137254903, + "grad_norm": 0.5480626100352609, + "learning_rate": 1.0107458240238324e-07, + "loss": 0.4408, + "step": 9500 + }, + { + "epoch": 1.9711042311661506, + "grad_norm": 0.5668784462525065, + "learning_rate": 7.44760080859666e-08, + "loss": 0.4435, + "step": 9550 + }, + { + "epoch": 1.9814241486068112, + "grad_norm": 0.6110658269716644, + "learning_rate": 4.787743376954996e-08, + "loss": 0.4431, + "step": 9600 + }, + { + "epoch": 1.9917440660474717, + "grad_norm": 0.5446976742385811, + "learning_rate": 2.127885945313331e-08, + "loss": 0.4367, + "step": 9650 + }, + { + "epoch": 2.0, + "eval_loss": 0.5610442161560059, + "eval_runtime": 20.0148, + "eval_samples_per_second": 26.88, + "eval_steps_per_second": 0.45, + "step": 9690 + }, + { + "epoch": 2.0, + "step": 9690, + "total_flos": 4057777727078400.0, + "train_loss": 0.5052211482704485, + "train_runtime": 135745.8608, + "train_samples_per_second": 4.568, + "train_steps_per_second": 0.071 + } + ], + "logging_steps": 50, + "max_steps": 9690, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 4057777727078400.0, + "train_batch_size": 2, + "trial_name": null, + "trial_params": null +} diff --git a/training_args.bin b/training_args.bin new file mode 100644 index 0000000..ff69889 --- /dev/null +++ b/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4f7ca7301e34abab61a8d77adcefb3448f3fab5a965a245e288750918137f0d2 +size 7288