From 1da307b26f92011d31b211bf7cb3e5547a11f091 Mon Sep 17 00:00:00 2001
From: ModelHub XC <noreply@modelhub.org.cn>
Date: Mon, 8 Jun 2026 21:34:21 +0800
Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E9=A1=B9=E7=9B=AE?=
 =?UTF-8?q?=EF=BC=8C=E7=94=B1ModelHub=20XC=E7=A4=BE=E5=8C=BA=E6=8F=90?=
 =?UTF-8?q?=E4=BE=9B=E6=A8=A1=E5=9E=8B?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Model: golaxy/gogpt-7b
Source: Original Platform
---
 .gitattributes                                |   35 +
 README.md                                     |   92 +
 added_tokens.json                             |    3 +
 config.json                                   |   24 +
 generation_config.json                        |    7 +
 pytorch_model-00001-of-00002.bin              |    3 +
 pytorch_model-00002-of-00002.bin              |    3 +
 pytorch_model.bin.index.json                  |  330 ++++
 ...nts.out.tfevents.1689588800.715436.92583.1 |    3 +
 ...nts.out.tfevents.1689588800.715436.92583.0 |    3 +
 special_tokens_map.json                       |   24 +
 tokenizer.model                               |    3 +
 tokenizer_config.json                         |   35 +
 trainer_state.json                            | 1759 +++++++++++++++++
 training_args.bin                             |    3 +
 15 files changed, 2327 insertions(+)
 create mode 100644 .gitattributes
 create mode 100644 README.md
 create mode 100644 added_tokens.json
 create mode 100644 config.json
 create mode 100644 generation_config.json
 create mode 100644 pytorch_model-00001-of-00002.bin
 create mode 100644 pytorch_model-00002-of-00002.bin
 create mode 100644 pytorch_model.bin.index.json
 create mode 100644 runs/Jul17_18-12-40_715436/1689588800.246925/events.out.tfevents.1689588800.715436.92583.1
 create mode 100644 runs/Jul17_18-12-40_715436/events.out.tfevents.1689588800.715436.92583.0
 create mode 100644 special_tokens_map.json
 create mode 100644 tokenizer.model
 create mode 100644 tokenizer_config.json
 create mode 100644 trainer_state.json
 create mode 100644 training_args.bin

diff --git a/.gitattributes b/.gitattributes
new file mode 100644
index 0000000..a6344aa
--- /dev/null
+++ b/.gitattributes
@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
diff --git a/README.md b/README.md
new file mode 100644
index 0000000..f72d11e
--- /dev/null
+++ b/README.md
@@ -0,0 +1,92 @@
+---
+license: apache-2.0
+datasets:
+- BelleGroup/train_1M_CN
+- BAAI/COIG
+- silk-road/alpaca-data-gpt4-chinese
+language:
+- zh
+tags:
+- gogpt-7b
+---
+
+# GoGPT
+
+> GoGPT:ICT中英文底座增强大模型,基于Llama/Llama 2训练的底座大模型,参数规模包括70亿参数、130亿参数
+
+<p align="center">
+    <br>
+    <img src="resources/assets/gogpt-banner-tou.png" width="600"/>
+    <br>
+</p>
+<p align="center">
+    <img alt="GitHub" src="https://img.shields.io/github/license/ymcui/Chinese-LLaMA-Alpaca.svg?color=blue&style=flat-square">
+    <img alt="GitHub top language" src="https://img.shields.io/github/languages/top/ymcui/Chinese-LLaMA-Alpaca">
+</p>
+
+
+## 模型部署
+
+🤗Huggingface上提供了GoGPT权重，目前开放了gogpt-7b和gogpt2-7b权重
+
+| 模型名称                                                        | 基座模型      | 模型大小 | 下载地址                                            |
+|-------------------------------------------------------------|-----------|------|-------------------------------------------------|
+| [golaxy/gogpt-7b](https://huggingface.co/golaxy/gogpt-7b)   | Llama-7b  | 7B   | [模型下载](https://huggingface.co/golaxy/gogpt-7b)  |
+| [golaxy/gogpt2-7b](https://huggingface.co/golaxy/gogpt2-7b) | Llama2-7b | 7B   | [模型下载](https://huggingface.co/golaxy/gogpt2-7b) |                                             
+
+## 训练细节
+
+### step1：训练分词器
+
+[🐱怎么从零到一训练一个LLM分词器](https://github.com/yanqiangmiffy/how-to-train-tokenizer)
+
+```text
+├── data
+│     └── corpus.txt 训练语料
+├── llama
+│     ├── tokenizer_checklist.chk
+│     └── tokenizer.model
+├── merged_tokenizer_hf 合并结果 hf格式
+│     ├── special_tokens_map.json
+│     ├── tokenizer_config.json
+│     └── tokenizer.model
+├── merged_tokenizer_sp
+│     └── open_llama.model # 
+├── merge_tokenizer
+│     └── tokenizer.model
+├── open_llama.model 训练的sp模型
+├── open_llama.vocab 训练的sp词汇表
+├── README.md
+├── step0_step0_process_text.py 基于多分数据集准备训练语料
+├── step1_make_corpus.py 基于中文Wikipedia数据准备训练语料
+├── step2_train_tokenzier.py  训练分词器
+├── step3_tokenzier_segment.py 测试训练后的模型，包括编码和解码测试样例
+└── step4_merge_tokenizers.py 与原版llama的分词器进行合并，得到hf格式的tokenizer
+
+```
+
+### step2：二次预训练
+
+> 在中文预训练语料上对LLaMA进行增量预训练、继续预训练
+
+### step3: 有监督微调
+
+- belle数据：120k数据 v1
+- stanford_alapca：52k数据 v2
+- [sharegpt](data%2Ffinetune%2Fsharegpt):90k数据
+
+### step4: 强化学习
+
+> TODO
+
+## 免责声明
+
+本项目相关资源仅供学术研究之用，严禁用于商业用途。 使用涉及第三方代码的部分时，请严格遵循相应的开源协议。
+
+模型生成的内容受模型计算、随机性和量化精度损失等因素影响，本项目不对其准确性作出保证。
+
+对于模型输出的任何内容，本项目不承担任何法律责任，亦不对因使用相关资源和输出结果而可能产生的任何损失承担责任。
+
+## 研究与开发团队
+
+本项目由网络数据科学与技术重点实验室GoGPT团队完成，团队指导老师为郭嘉丰研究员。
\ No newline at end of file
diff --git a/added_tokens.json b/added_tokens.json
new file mode 100644
index 0000000..040c778
--- /dev/null
+++ b/added_tokens.json
@@ -0,0 +1,3 @@
+{
+  "[PAD]": 68419
+}
diff --git a/config.json b/config.json
new file mode 100644
index 0000000..1a93e5d
--- /dev/null
+++ b/config.json
@@ -0,0 +1,24 @@
+{
+  "_name_or_path": "/data/searchgpt/pretrained_models/gogpt-7b-v4",
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "bos_token_id": 0,
+  "eos_token_id": 1,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 11008,
+  "max_position_embeddings": 2048,
+  "max_sequence_length": 2048,
+  "model_type": "llama",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "pad_token_id": -1,
+  "rms_norm_eps": 1e-06,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.29.1",
+  "use_cache": false,
+  "vocab_size": 68420
+}
diff --git a/generation_config.json b/generation_config.json
new file mode 100644
index 0000000..9a5df39
--- /dev/null
+++ b/generation_config.json
@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 0,
+  "eos_token_id": 1,
+  "pad_token_id": 0,
+  "transformers_version": "4.29.1"
+}
diff --git a/pytorch_model-00001-of-00002.bin b/pytorch_model-00001-of-00002.bin
new file mode 100644
index 0000000..2bd9942
--- /dev/null
+++ b/pytorch_model-00001-of-00002.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:36f1e6fc7469a9cece4c656da6deb8d0ae9f9387a294985a4e7b90960d125833
+size 9970881453
diff --git a/pytorch_model-00002-of-00002.bin b/pytorch_model-00002-of-00002.bin
new file mode 100644
index 0000000..cfdba42
--- /dev/null
+++ b/pytorch_model-00002-of-00002.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b6d9affcbbe40694fc9b8fc0c70a947323332a20e08b4cbad1c1378237760769
+size 4102774289
diff --git a/pytorch_model.bin.index.json b/pytorch_model.bin.index.json
new file mode 100644
index 0000000..af85145
--- /dev/null
+++ b/pytorch_model.bin.index.json
@@ -0,0 +1,330 @@
+{
+  "metadata": {
+    "total_size": 14073540608
+  },
+  "weight_map": {
+    "lm_head.weight": "pytorch_model-00002-of-00002.bin",
+    "model.embed_tokens.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.0.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.0.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.0.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.0.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.0.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.0.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.0.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.0.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.0.self_attn.rotary_emb.inv_freq": "pytorch_model-00001-of-00002.bin",
+    "model.layers.0.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.1.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.1.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.1.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.1.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.1.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.1.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.1.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.1.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.1.self_attn.rotary_emb.inv_freq": "pytorch_model-00001-of-00002.bin",
+    "model.layers.1.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.10.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.10.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.10.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.10.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.10.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.10.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.10.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.10.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.10.self_attn.rotary_emb.inv_freq": "pytorch_model-00001-of-00002.bin",
+    "model.layers.10.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.11.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.11.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.11.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.11.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.11.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.11.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.11.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.11.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.11.self_attn.rotary_emb.inv_freq": "pytorch_model-00001-of-00002.bin",
+    "model.layers.11.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.12.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.12.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.12.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.12.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.12.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.12.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.12.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.12.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.12.self_attn.rotary_emb.inv_freq": "pytorch_model-00001-of-00002.bin",
+    "model.layers.12.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.13.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.13.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.13.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.13.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.13.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.13.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.13.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.13.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.13.self_attn.rotary_emb.inv_freq": "pytorch_model-00001-of-00002.bin",
+    "model.layers.13.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.14.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.14.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.14.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.14.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.14.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.14.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.14.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.14.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.14.self_attn.rotary_emb.inv_freq": "pytorch_model-00001-of-00002.bin",
+    "model.layers.14.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.15.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.15.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.15.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.15.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.15.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.15.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.15.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.15.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.15.self_attn.rotary_emb.inv_freq": "pytorch_model-00001-of-00002.bin",
+    "model.layers.15.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.16.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.16.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.16.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.16.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.16.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.16.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.16.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.16.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.16.self_attn.rotary_emb.inv_freq": "pytorch_model-00001-of-00002.bin",
+    "model.layers.16.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.17.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.17.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.17.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.17.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.17.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.17.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.17.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.17.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.17.self_attn.rotary_emb.inv_freq": "pytorch_model-00001-of-00002.bin",
+    "model.layers.17.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.18.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.18.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.18.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.18.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.18.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.18.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.18.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.18.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.18.self_attn.rotary_emb.inv_freq": "pytorch_model-00001-of-00002.bin",
+    "model.layers.18.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.19.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.19.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.19.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.19.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.19.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.19.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.19.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.19.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.19.self_attn.rotary_emb.inv_freq": "pytorch_model-00001-of-00002.bin",
+    "model.layers.19.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.2.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.2.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.2.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.2.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.2.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.2.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.2.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.2.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.2.self_attn.rotary_emb.inv_freq": "pytorch_model-00001-of-00002.bin",
+    "model.layers.2.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.20.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.20.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.20.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.20.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.20.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.20.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.20.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.20.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.20.self_attn.rotary_emb.inv_freq": "pytorch_model-00001-of-00002.bin",
+    "model.layers.20.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.21.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.21.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.21.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.21.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.21.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.21.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.21.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.21.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.21.self_attn.rotary_emb.inv_freq": "pytorch_model-00001-of-00002.bin",
+    "model.layers.21.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.22.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.22.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.22.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.22.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.22.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.22.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.22.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.22.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.22.self_attn.rotary_emb.inv_freq": "pytorch_model-00001-of-00002.bin",
+    "model.layers.22.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.23.input_layernorm.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.23.mlp.down_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.23.mlp.gate_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.23.mlp.up_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.23.post_attention_layernorm.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.23.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.23.self_attn.o_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.23.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.23.self_attn.rotary_emb.inv_freq": "pytorch_model-00002-of-00002.bin",
+    "model.layers.23.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.24.input_layernorm.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.24.mlp.down_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.24.mlp.gate_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.24.mlp.up_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.24.post_attention_layernorm.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.24.self_attn.k_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.24.self_attn.o_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.24.self_attn.q_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.24.self_attn.rotary_emb.inv_freq": "pytorch_model-00002-of-00002.bin",
+    "model.layers.24.self_attn.v_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.25.input_layernorm.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.25.mlp.down_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.25.mlp.gate_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.25.mlp.up_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.25.post_attention_layernorm.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.25.self_attn.k_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.25.self_attn.o_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.25.self_attn.q_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.25.self_attn.rotary_emb.inv_freq": "pytorch_model-00002-of-00002.bin",
+    "model.layers.25.self_attn.v_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.26.input_layernorm.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.26.mlp.down_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.26.mlp.gate_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.26.mlp.up_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.26.post_attention_layernorm.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.26.self_attn.k_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.26.self_attn.o_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.26.self_attn.q_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.26.self_attn.rotary_emb.inv_freq": "pytorch_model-00002-of-00002.bin",
+    "model.layers.26.self_attn.v_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.27.input_layernorm.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.27.mlp.down_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.27.mlp.gate_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.27.mlp.up_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.27.post_attention_layernorm.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.27.self_attn.k_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.27.self_attn.o_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.27.self_attn.q_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.27.self_attn.rotary_emb.inv_freq": "pytorch_model-00002-of-00002.bin",
+    "model.layers.27.self_attn.v_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.28.input_layernorm.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.28.mlp.down_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.28.mlp.gate_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.28.mlp.up_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.28.post_attention_layernorm.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.28.self_attn.k_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.28.self_attn.o_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.28.self_attn.q_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.28.self_attn.rotary_emb.inv_freq": "pytorch_model-00002-of-00002.bin",
+    "model.layers.28.self_attn.v_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.29.input_layernorm.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.29.mlp.down_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.29.mlp.gate_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.29.mlp.up_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.29.post_attention_layernorm.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.29.self_attn.k_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.29.self_attn.o_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.29.self_attn.q_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.29.self_attn.rotary_emb.inv_freq": "pytorch_model-00002-of-00002.bin",
+    "model.layers.29.self_attn.v_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.3.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.3.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.3.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.3.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.3.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.3.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.3.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.3.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.3.self_attn.rotary_emb.inv_freq": "pytorch_model-00001-of-00002.bin",
+    "model.layers.3.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.30.input_layernorm.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.30.mlp.down_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.30.mlp.gate_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.30.mlp.up_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.30.post_attention_layernorm.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.30.self_attn.k_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.30.self_attn.o_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.30.self_attn.q_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.30.self_attn.rotary_emb.inv_freq": "pytorch_model-00002-of-00002.bin",
+    "model.layers.30.self_attn.v_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.31.input_layernorm.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.31.mlp.down_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.31.mlp.gate_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.31.mlp.up_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.31.post_attention_layernorm.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.31.self_attn.k_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.31.self_attn.o_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.31.self_attn.q_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.31.self_attn.rotary_emb.inv_freq": "pytorch_model-00002-of-00002.bin",
+    "model.layers.31.self_attn.v_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "model.layers.4.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.4.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.4.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.4.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.4.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.4.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.4.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.4.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.4.self_attn.rotary_emb.inv_freq": "pytorch_model-00001-of-00002.bin",
+    "model.layers.4.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.5.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.5.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.5.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.5.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.5.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.5.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.5.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.5.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.5.self_attn.rotary_emb.inv_freq": "pytorch_model-00001-of-00002.bin",
+    "model.layers.5.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.6.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.6.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.6.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.6.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.6.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.6.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.6.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.6.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.6.self_attn.rotary_emb.inv_freq": "pytorch_model-00001-of-00002.bin",
+    "model.layers.6.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.7.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.7.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.7.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.7.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.7.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.7.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.7.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.7.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.7.self_attn.rotary_emb.inv_freq": "pytorch_model-00001-of-00002.bin",
+    "model.layers.7.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.8.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.8.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.8.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.8.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.8.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.8.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.8.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.8.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.8.self_attn.rotary_emb.inv_freq": "pytorch_model-00001-of-00002.bin",
+    "model.layers.8.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.9.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.9.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.9.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.9.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.9.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.9.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.9.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.9.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.layers.9.self_attn.rotary_emb.inv_freq": "pytorch_model-00001-of-00002.bin",
+    "model.layers.9.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "model.norm.weight": "pytorch_model-00002-of-00002.bin"
+  }
+}
diff --git a/runs/Jul17_18-12-40_715436/1689588800.246925/events.out.tfevents.1689588800.715436.92583.1 b/runs/Jul17_18-12-40_715436/1689588800.246925/events.out.tfevents.1689588800.715436.92583.1
new file mode 100644
index 0000000..d4c390d
--- /dev/null
+++ b/runs/Jul17_18-12-40_715436/1689588800.246925/events.out.tfevents.1689588800.715436.92583.1
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cd39bf544177964a0e2d6c4481eb71693ac4f379250846c277e1986ae11b4ad6
+size 6093
diff --git a/runs/Jul17_18-12-40_715436/events.out.tfevents.1689588800.715436.92583.0 b/runs/Jul17_18-12-40_715436/events.out.tfevents.1689588800.715436.92583.0
new file mode 100644
index 0000000..b8f96cc
--- /dev/null
+++ b/runs/Jul17_18-12-40_715436/events.out.tfevents.1689588800.715436.92583.0
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7dfe2fd3a6ebcb1cb31fc1e3e0954f383dd123cafa3f826818dfcc8ff133a7c7
+size 49805
diff --git a/special_tokens_map.json b/special_tokens_map.json
new file mode 100644
index 0000000..599f3bb
--- /dev/null
+++ b/special_tokens_map.json
@@ -0,0 +1,24 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "[PAD]",
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  }
+}
diff --git a/tokenizer.model b/tokenizer.model
new file mode 100644
index 0000000..99ed46b
--- /dev/null
+++ b/tokenizer.model
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bd5ab2c18ed07a14f3aa55518dcf08bbee4fe86c9423e86ba61f60a82ab31fa7
+size 1077901
diff --git a/tokenizer_config.json b/tokenizer_config.json
new file mode 100644
index 0000000..a3fe222
--- /dev/null
+++ b/tokenizer_config.json
@@ -0,0 +1,35 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "bos_token": {
+    "__type": "AddedToken",
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "clean_up_tokenization_spaces": false,
+  "eos_token": {
+    "__type": "AddedToken",
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "model_max_length": 2048,
+  "pad_token": null,
+  "padding_side": "right",
+  "sp_model_kwargs": {},
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": {
+    "__type": "AddedToken",
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "use_fast": false
+}
diff --git a/trainer_state.json b/trainer_state.json
new file mode 100644
index 0000000..cc03c10
--- /dev/null
+++ b/trainer_state.json
@@ -0,0 +1,1759 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.9998273778698429,
+  "global_step": 2896,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0,
+      "learning_rate": 1.993093922651934e-05,
+      "loss": 1.9454,
+      "step": 10
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 1.9861878453038676e-05,
+      "loss": 1.9111,
+      "step": 20
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 1.979281767955801e-05,
+      "loss": 1.9728,
+      "step": 30
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 1.972375690607735e-05,
+      "loss": 1.8986,
+      "step": 40
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 1.9654696132596688e-05,
+      "loss": 1.8296,
+      "step": 50
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 1.9585635359116023e-05,
+      "loss": 1.8251,
+      "step": 60
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 1.9516574585635362e-05,
+      "loss": 1.8381,
+      "step": 70
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 1.9447513812154697e-05,
+      "loss": 1.8186,
+      "step": 80
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 1.9378453038674036e-05,
+      "loss": 1.862,
+      "step": 90
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 1.930939226519337e-05,
+      "loss": 1.8683,
+      "step": 100
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 1.924033149171271e-05,
+      "loss": 1.7295,
+      "step": 110
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 1.9171270718232045e-05,
+      "loss": 1.8422,
+      "step": 120
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 1.910220994475138e-05,
+      "loss": 1.8391,
+      "step": 130
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 1.903314917127072e-05,
+      "loss": 1.7898,
+      "step": 140
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 1.8964088397790058e-05,
+      "loss": 1.8654,
+      "step": 150
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 1.8895027624309393e-05,
+      "loss": 1.7978,
+      "step": 160
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 1.882596685082873e-05,
+      "loss": 1.8074,
+      "step": 170
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 1.8756906077348067e-05,
+      "loss": 1.8771,
+      "step": 180
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 1.8687845303867406e-05,
+      "loss": 1.8324,
+      "step": 190
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 1.861878453038674e-05,
+      "loss": 1.8138,
+      "step": 200
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 1.854972375690608e-05,
+      "loss": 1.7218,
+      "step": 210
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 1.8480662983425418e-05,
+      "loss": 1.7777,
+      "step": 220
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 1.8411602209944753e-05,
+      "loss": 1.9651,
+      "step": 230
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 1.834254143646409e-05,
+      "loss": 1.7814,
+      "step": 240
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 1.8273480662983427e-05,
+      "loss": 1.8694,
+      "step": 250
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 1.8204419889502766e-05,
+      "loss": 1.8094,
+      "step": 260
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 1.81353591160221e-05,
+      "loss": 1.8144,
+      "step": 270
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 1.8066298342541436e-05,
+      "loss": 1.7834,
+      "step": 280
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 1.7997237569060775e-05,
+      "loss": 1.8326,
+      "step": 290
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 1.7928176795580114e-05,
+      "loss": 1.8524,
+      "step": 300
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 1.785911602209945e-05,
+      "loss": 1.8125,
+      "step": 310
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 1.7790055248618784e-05,
+      "loss": 1.7685,
+      "step": 320
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 1.7720994475138123e-05,
+      "loss": 1.8259,
+      "step": 330
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 1.765193370165746e-05,
+      "loss": 1.8165,
+      "step": 340
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 1.7582872928176797e-05,
+      "loss": 1.734,
+      "step": 350
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 1.7513812154696135e-05,
+      "loss": 1.8719,
+      "step": 360
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 1.744475138121547e-05,
+      "loss": 1.84,
+      "step": 370
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 1.7375690607734806e-05,
+      "loss": 1.739,
+      "step": 380
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 1.7306629834254145e-05,
+      "loss": 1.7121,
+      "step": 390
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 1.7237569060773483e-05,
+      "loss": 1.7994,
+      "step": 400
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 1.716850828729282e-05,
+      "loss": 1.8029,
+      "step": 410
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 1.7099447513812154e-05,
+      "loss": 1.7876,
+      "step": 420
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 1.7030386740331492e-05,
+      "loss": 1.7399,
+      "step": 430
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 1.696132596685083e-05,
+      "loss": 1.8004,
+      "step": 440
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 1.6892265193370166e-05,
+      "loss": 1.8154,
+      "step": 450
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 1.6823204419889505e-05,
+      "loss": 1.7839,
+      "step": 460
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 1.675414364640884e-05,
+      "loss": 1.7881,
+      "step": 470
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 1.668508287292818e-05,
+      "loss": 1.7409,
+      "step": 480
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 1.6616022099447514e-05,
+      "loss": 1.7907,
+      "step": 490
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 1.6546961325966853e-05,
+      "loss": 1.8985,
+      "step": 500
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 1.647790055248619e-05,
+      "loss": 1.8918,
+      "step": 510
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 1.6408839779005527e-05,
+      "loss": 1.8328,
+      "step": 520
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 1.6339779005524862e-05,
+      "loss": 1.7157,
+      "step": 530
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 1.62707182320442e-05,
+      "loss": 1.6767,
+      "step": 540
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 1.620165745856354e-05,
+      "loss": 1.7403,
+      "step": 550
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 1.6132596685082875e-05,
+      "loss": 1.8535,
+      "step": 560
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 1.606353591160221e-05,
+      "loss": 1.8406,
+      "step": 570
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 1.599447513812155e-05,
+      "loss": 1.7262,
+      "step": 580
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 1.5925414364640884e-05,
+      "loss": 1.7585,
+      "step": 590
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 1.5856353591160222e-05,
+      "loss": 1.7821,
+      "step": 600
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 1.578729281767956e-05,
+      "loss": 1.839,
+      "step": 610
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 1.5718232044198896e-05,
+      "loss": 1.8768,
+      "step": 620
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 1.564917127071823e-05,
+      "loss": 1.772,
+      "step": 630
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 1.558011049723757e-05,
+      "loss": 1.7883,
+      "step": 640
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 1.551104972375691e-05,
+      "loss": 1.6712,
+      "step": 650
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 1.5441988950276244e-05,
+      "loss": 1.7631,
+      "step": 660
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 1.537292817679558e-05,
+      "loss": 1.7849,
+      "step": 670
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 1.5303867403314918e-05,
+      "loss": 1.6738,
+      "step": 680
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 1.5234806629834255e-05,
+      "loss": 1.8382,
+      "step": 690
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 1.5165745856353594e-05,
+      "loss": 1.7811,
+      "step": 700
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 1.5096685082872929e-05,
+      "loss": 1.7594,
+      "step": 710
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 1.5027624309392266e-05,
+      "loss": 1.7665,
+      "step": 720
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 1.4958563535911603e-05,
+      "loss": 1.7435,
+      "step": 730
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 1.4889502762430942e-05,
+      "loss": 1.7131,
+      "step": 740
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 1.4820441988950279e-05,
+      "loss": 1.8457,
+      "step": 750
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 1.4751381215469614e-05,
+      "loss": 1.8551,
+      "step": 760
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 1.468232044198895e-05,
+      "loss": 1.7981,
+      "step": 770
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 1.4613259668508288e-05,
+      "loss": 1.8244,
+      "step": 780
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 1.4544198895027626e-05,
+      "loss": 1.8683,
+      "step": 790
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 1.4475138121546963e-05,
+      "loss": 1.7665,
+      "step": 800
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 1.4406077348066299e-05,
+      "loss": 1.7605,
+      "step": 810
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 1.4337016574585636e-05,
+      "loss": 1.8339,
+      "step": 820
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 1.4267955801104974e-05,
+      "loss": 1.8634,
+      "step": 830
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 1.4198895027624311e-05,
+      "loss": 1.7421,
+      "step": 840
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 1.4129834254143648e-05,
+      "loss": 1.7681,
+      "step": 850
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 1.4060773480662983e-05,
+      "loss": 1.749,
+      "step": 860
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 1.399171270718232e-05,
+      "loss": 1.7926,
+      "step": 870
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 1.3922651933701659e-05,
+      "loss": 1.7425,
+      "step": 880
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 1.3853591160220996e-05,
+      "loss": 1.7008,
+      "step": 890
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 1.3784530386740333e-05,
+      "loss": 1.6872,
+      "step": 900
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 1.3715469613259668e-05,
+      "loss": 1.7296,
+      "step": 910
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 1.3646408839779007e-05,
+      "loss": 1.7473,
+      "step": 920
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 1.3577348066298344e-05,
+      "loss": 1.7689,
+      "step": 930
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 1.350828729281768e-05,
+      "loss": 1.7871,
+      "step": 940
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 1.343922651933702e-05,
+      "loss": 1.7677,
+      "step": 950
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 1.3370165745856355e-05,
+      "loss": 1.7477,
+      "step": 960
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 1.3301104972375692e-05,
+      "loss": 1.8677,
+      "step": 970
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 1.3232044198895029e-05,
+      "loss": 1.8348,
+      "step": 980
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 1.3162983425414365e-05,
+      "loss": 1.7448,
+      "step": 990
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 1.3093922651933704e-05,
+      "loss": 1.7743,
+      "step": 1000
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 1.302486187845304e-05,
+      "loss": 1.6812,
+      "step": 1010
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 1.2955801104972376e-05,
+      "loss": 1.7649,
+      "step": 1020
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 1.2886740331491713e-05,
+      "loss": 1.8005,
+      "step": 1030
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 1.2817679558011052e-05,
+      "loss": 1.8059,
+      "step": 1040
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 1.2748618784530389e-05,
+      "loss": 1.7549,
+      "step": 1050
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 1.2679558011049724e-05,
+      "loss": 1.7787,
+      "step": 1060
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 1.2610497237569061e-05,
+      "loss": 1.722,
+      "step": 1070
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 1.25414364640884e-05,
+      "loss": 1.7192,
+      "step": 1080
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 1.2472375690607737e-05,
+      "loss": 1.8681,
+      "step": 1090
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 1.2403314917127074e-05,
+      "loss": 1.8199,
+      "step": 1100
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 1.2334254143646409e-05,
+      "loss": 1.8003,
+      "step": 1110
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 1.2265193370165746e-05,
+      "loss": 1.8412,
+      "step": 1120
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 1.2196132596685085e-05,
+      "loss": 1.7463,
+      "step": 1130
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 1.2127071823204422e-05,
+      "loss": 1.8106,
+      "step": 1140
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 1.2058011049723757e-05,
+      "loss": 1.6872,
+      "step": 1150
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 1.1988950276243094e-05,
+      "loss": 1.6963,
+      "step": 1160
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 1.1919889502762432e-05,
+      "loss": 1.7752,
+      "step": 1170
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 1.185082872928177e-05,
+      "loss": 1.8593,
+      "step": 1180
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 1.1781767955801106e-05,
+      "loss": 1.7515,
+      "step": 1190
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 1.1712707182320442e-05,
+      "loss": 1.7299,
+      "step": 1200
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 1.1643646408839779e-05,
+      "loss": 1.7632,
+      "step": 1210
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 1.1574585635359117e-05,
+      "loss": 1.7982,
+      "step": 1220
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 1.1505524861878454e-05,
+      "loss": 1.7833,
+      "step": 1230
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 1.1436464088397791e-05,
+      "loss": 1.7549,
+      "step": 1240
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 1.1367403314917126e-05,
+      "loss": 1.726,
+      "step": 1250
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 1.1298342541436465e-05,
+      "loss": 1.7221,
+      "step": 1260
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 1.1229281767955802e-05,
+      "loss": 1.8503,
+      "step": 1270
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 1.1160220994475139e-05,
+      "loss": 1.7096,
+      "step": 1280
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 1.1091160220994478e-05,
+      "loss": 1.7261,
+      "step": 1290
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 1.1022099447513813e-05,
+      "loss": 1.7823,
+      "step": 1300
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 1.095303867403315e-05,
+      "loss": 1.7031,
+      "step": 1310
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 1.0883977900552487e-05,
+      "loss": 1.7445,
+      "step": 1320
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 1.0814917127071824e-05,
+      "loss": 1.7109,
+      "step": 1330
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 1.0745856353591162e-05,
+      "loss": 1.8005,
+      "step": 1340
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 1.0676795580110498e-05,
+      "loss": 1.7529,
+      "step": 1350
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 1.0607734806629835e-05,
+      "loss": 1.6576,
+      "step": 1360
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 1.0538674033149172e-05,
+      "loss": 1.7873,
+      "step": 1370
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 1.046961325966851e-05,
+      "loss": 1.7714,
+      "step": 1380
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 1.0400552486187847e-05,
+      "loss": 1.7174,
+      "step": 1390
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 1.0331491712707182e-05,
+      "loss": 1.6726,
+      "step": 1400
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 1.026243093922652e-05,
+      "loss": 1.718,
+      "step": 1410
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 1.0193370165745858e-05,
+      "loss": 1.7388,
+      "step": 1420
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 1.0124309392265195e-05,
+      "loss": 1.7361,
+      "step": 1430
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 1.0055248618784532e-05,
+      "loss": 1.7668,
+      "step": 1440
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 9.986187845303869e-06,
+      "loss": 1.8001,
+      "step": 1450
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 9.917127071823204e-06,
+      "loss": 1.7302,
+      "step": 1460
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 9.848066298342543e-06,
+      "loss": 1.687,
+      "step": 1470
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 9.779005524861878e-06,
+      "loss": 1.6978,
+      "step": 1480
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 9.709944751381217e-06,
+      "loss": 1.7913,
+      "step": 1490
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 9.640883977900554e-06,
+      "loss": 1.8389,
+      "step": 1500
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 9.57182320441989e-06,
+      "loss": 1.8108,
+      "step": 1510
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 9.502762430939228e-06,
+      "loss": 1.7494,
+      "step": 1520
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 9.433701657458565e-06,
+      "loss": 1.8438,
+      "step": 1530
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 9.364640883977902e-06,
+      "loss": 1.8194,
+      "step": 1540
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 9.295580110497238e-06,
+      "loss": 1.7495,
+      "step": 1550
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 9.226519337016575e-06,
+      "loss": 1.7208,
+      "step": 1560
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 9.157458563535912e-06,
+      "loss": 1.706,
+      "step": 1570
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 9.08839779005525e-06,
+      "loss": 1.8367,
+      "step": 1580
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 9.019337016574586e-06,
+      "loss": 1.6874,
+      "step": 1590
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 8.950276243093923e-06,
+      "loss": 1.7478,
+      "step": 1600
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 8.88121546961326e-06,
+      "loss": 1.6782,
+      "step": 1610
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 8.812154696132597e-06,
+      "loss": 1.8467,
+      "step": 1620
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 8.743093922651934e-06,
+      "loss": 1.758,
+      "step": 1630
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 8.674033149171271e-06,
+      "loss": 1.7116,
+      "step": 1640
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 8.60497237569061e-06,
+      "loss": 1.751,
+      "step": 1650
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 8.535911602209945e-06,
+      "loss": 1.7502,
+      "step": 1660
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 8.466850828729282e-06,
+      "loss": 1.879,
+      "step": 1670
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 8.397790055248619e-06,
+      "loss": 1.8171,
+      "step": 1680
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 8.328729281767956e-06,
+      "loss": 1.7586,
+      "step": 1690
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 8.259668508287293e-06,
+      "loss": 1.7848,
+      "step": 1700
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 8.19060773480663e-06,
+      "loss": 1.7789,
+      "step": 1710
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 8.121546961325968e-06,
+      "loss": 1.7104,
+      "step": 1720
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 8.052486187845304e-06,
+      "loss": 1.807,
+      "step": 1730
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 7.983425414364642e-06,
+      "loss": 1.6772,
+      "step": 1740
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 7.914364640883978e-06,
+      "loss": 1.7574,
+      "step": 1750
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 7.845303867403316e-06,
+      "loss": 1.7925,
+      "step": 1760
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 7.776243093922653e-06,
+      "loss": 1.7109,
+      "step": 1770
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 7.70718232044199e-06,
+      "loss": 1.8161,
+      "step": 1780
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 7.638121546961327e-06,
+      "loss": 1.716,
+      "step": 1790
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 7.569060773480663e-06,
+      "loss": 1.7228,
+      "step": 1800
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 7.500000000000001e-06,
+      "loss": 1.865,
+      "step": 1810
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 7.430939226519338e-06,
+      "loss": 1.7947,
+      "step": 1820
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 7.361878453038674e-06,
+      "loss": 1.8093,
+      "step": 1830
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 7.292817679558012e-06,
+      "loss": 1.7586,
+      "step": 1840
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 7.223756906077348e-06,
+      "loss": 1.7256,
+      "step": 1850
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 7.154696132596686e-06,
+      "loss": 1.7485,
+      "step": 1860
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 7.085635359116023e-06,
+      "loss": 1.7597,
+      "step": 1870
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 7.01657458563536e-06,
+      "loss": 1.7516,
+      "step": 1880
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 6.947513812154697e-06,
+      "loss": 1.7449,
+      "step": 1890
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 6.878453038674034e-06,
+      "loss": 1.8367,
+      "step": 1900
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 6.809392265193371e-06,
+      "loss": 1.7401,
+      "step": 1910
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 6.740331491712708e-06,
+      "loss": 1.8057,
+      "step": 1920
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 6.6712707182320445e-06,
+      "loss": 1.7072,
+      "step": 1930
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 6.602209944751382e-06,
+      "loss": 1.7182,
+      "step": 1940
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 6.5331491712707184e-06,
+      "loss": 1.7331,
+      "step": 1950
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 6.464088397790056e-06,
+      "loss": 1.871,
+      "step": 1960
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 6.395027624309392e-06,
+      "loss": 1.7339,
+      "step": 1970
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 6.32596685082873e-06,
+      "loss": 1.7293,
+      "step": 1980
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 6.256906077348067e-06,
+      "loss": 1.8179,
+      "step": 1990
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 6.187845303867403e-06,
+      "loss": 1.7189,
+      "step": 2000
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 6.118784530386741e-06,
+      "loss": 1.6855,
+      "step": 2010
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 6.049723756906077e-06,
+      "loss": 1.8562,
+      "step": 2020
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 5.980662983425415e-06,
+      "loss": 1.7805,
+      "step": 2030
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 5.911602209944752e-06,
+      "loss": 1.7376,
+      "step": 2040
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 5.842541436464089e-06,
+      "loss": 1.7232,
+      "step": 2050
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 5.773480662983426e-06,
+      "loss": 1.7372,
+      "step": 2060
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 5.704419889502763e-06,
+      "loss": 1.6427,
+      "step": 2070
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 5.6353591160221e-06,
+      "loss": 1.6946,
+      "step": 2080
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 5.5662983425414375e-06,
+      "loss": 1.7831,
+      "step": 2090
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 5.497237569060774e-06,
+      "loss": 1.8121,
+      "step": 2100
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 5.4281767955801114e-06,
+      "loss": 1.7363,
+      "step": 2110
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 5.3591160220994476e-06,
+      "loss": 1.7434,
+      "step": 2120
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 5.290055248618785e-06,
+      "loss": 1.8479,
+      "step": 2130
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 5.220994475138122e-06,
+      "loss": 1.7347,
+      "step": 2140
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 5.151933701657459e-06,
+      "loss": 1.785,
+      "step": 2150
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 5.082872928176796e-06,
+      "loss": 1.8255,
+      "step": 2160
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 5.013812154696132e-06,
+      "loss": 1.7684,
+      "step": 2170
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 4.94475138121547e-06,
+      "loss": 1.7494,
+      "step": 2180
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 4.875690607734807e-06,
+      "loss": 1.6692,
+      "step": 2190
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 4.806629834254144e-06,
+      "loss": 1.7653,
+      "step": 2200
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 4.737569060773481e-06,
+      "loss": 1.6953,
+      "step": 2210
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 4.668508287292818e-06,
+      "loss": 1.8247,
+      "step": 2220
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 4.599447513812155e-06,
+      "loss": 1.8188,
+      "step": 2230
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 4.530386740331492e-06,
+      "loss": 1.7473,
+      "step": 2240
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 4.461325966850829e-06,
+      "loss": 1.6097,
+      "step": 2250
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 4.392265193370166e-06,
+      "loss": 1.7937,
+      "step": 2260
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 4.323204419889503e-06,
+      "loss": 1.7121,
+      "step": 2270
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 4.2541436464088406e-06,
+      "loss": 1.8483,
+      "step": 2280
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 4.1850828729281775e-06,
+      "loss": 1.7838,
+      "step": 2290
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 4.1160220994475145e-06,
+      "loss": 1.6945,
+      "step": 2300
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 4.0469613259668514e-06,
+      "loss": 1.7295,
+      "step": 2310
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 3.977900552486188e-06,
+      "loss": 1.7116,
+      "step": 2320
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 3.9088397790055245e-06,
+      "loss": 1.8388,
+      "step": 2330
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 3.839779005524862e-06,
+      "loss": 1.8654,
+      "step": 2340
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 3.7707182320441993e-06,
+      "loss": 1.7835,
+      "step": 2350
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 3.7016574585635362e-06,
+      "loss": 1.6937,
+      "step": 2360
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 3.632596685082873e-06,
+      "loss": 1.6986,
+      "step": 2370
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 3.56353591160221e-06,
+      "loss": 1.7051,
+      "step": 2380
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 3.4944751381215475e-06,
+      "loss": 1.7332,
+      "step": 2390
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 3.4254143646408845e-06,
+      "loss": 1.6828,
+      "step": 2400
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 3.3563535911602214e-06,
+      "loss": 1.818,
+      "step": 2410
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 3.2872928176795584e-06,
+      "loss": 1.7628,
+      "step": 2420
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 3.218232044198895e-06,
+      "loss": 1.7445,
+      "step": 2430
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 3.149171270718232e-06,
+      "loss": 1.8389,
+      "step": 2440
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 3.0801104972375697e-06,
+      "loss": 1.7749,
+      "step": 2450
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 3.0110497237569062e-06,
+      "loss": 1.8421,
+      "step": 2460
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 2.941988950276243e-06,
+      "loss": 1.7727,
+      "step": 2470
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 2.87292817679558e-06,
+      "loss": 1.754,
+      "step": 2480
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 2.803867403314917e-06,
+      "loss": 1.6552,
+      "step": 2490
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 2.7348066298342545e-06,
+      "loss": 1.7354,
+      "step": 2500
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 2.6657458563535914e-06,
+      "loss": 1.7694,
+      "step": 2510
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 2.5966850828729284e-06,
+      "loss": 1.7042,
+      "step": 2520
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 2.5276243093922653e-06,
+      "loss": 1.6969,
+      "step": 2530
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 2.4585635359116027e-06,
+      "loss": 1.7404,
+      "step": 2540
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 2.3895027624309393e-06,
+      "loss": 1.7236,
+      "step": 2550
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 2.320441988950276e-06,
+      "loss": 1.8104,
+      "step": 2560
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 2.2513812154696136e-06,
+      "loss": 1.7518,
+      "step": 2570
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 2.1823204419889505e-06,
+      "loss": 1.7097,
+      "step": 2580
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 2.1132596685082875e-06,
+      "loss": 1.7118,
+      "step": 2590
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 2.0441988950276245e-06,
+      "loss": 1.7996,
+      "step": 2600
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.9751381215469614e-06,
+      "loss": 1.6647,
+      "step": 2610
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.9060773480662986e-06,
+      "loss": 1.6974,
+      "step": 2620
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 1.8370165745856355e-06,
+      "loss": 1.6929,
+      "step": 2630
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 1.7679558011049725e-06,
+      "loss": 1.7994,
+      "step": 2640
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 1.6988950276243097e-06,
+      "loss": 1.7755,
+      "step": 2650
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 1.6298342541436466e-06,
+      "loss": 1.8172,
+      "step": 2660
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 1.5607734806629834e-06,
+      "loss": 1.8241,
+      "step": 2670
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 1.4917127071823205e-06,
+      "loss": 1.7558,
+      "step": 2680
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 1.4226519337016575e-06,
+      "loss": 1.7474,
+      "step": 2690
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 1.3535911602209945e-06,
+      "loss": 1.8138,
+      "step": 2700
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 1.2845303867403316e-06,
+      "loss": 1.7158,
+      "step": 2710
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 1.2154696132596686e-06,
+      "loss": 1.8223,
+      "step": 2720
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 1.1464088397790055e-06,
+      "loss": 1.8411,
+      "step": 2730
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 1.0773480662983427e-06,
+      "loss": 1.8054,
+      "step": 2740
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 1.0082872928176797e-06,
+      "loss": 1.7401,
+      "step": 2750
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 9.392265193370166e-07,
+      "loss": 1.6664,
+      "step": 2760
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 8.701657458563537e-07,
+      "loss": 1.7788,
+      "step": 2770
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 8.011049723756907e-07,
+      "loss": 1.7135,
+      "step": 2780
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 7.320441988950276e-07,
+      "loss": 1.8247,
+      "step": 2790
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 6.629834254143647e-07,
+      "loss": 1.7436,
+      "step": 2800
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 5.939226519337017e-07,
+      "loss": 1.792,
+      "step": 2810
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 5.248618784530387e-07,
+      "loss": 1.7798,
+      "step": 2820
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 4.5580110497237574e-07,
+      "loss": 1.7821,
+      "step": 2830
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 3.867403314917127e-07,
+      "loss": 1.7994,
+      "step": 2840
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 3.1767955801104976e-07,
+      "loss": 1.8054,
+      "step": 2850
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 2.486187845303868e-07,
+      "loss": 1.7472,
+      "step": 2860
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 1.7955801104972376e-07,
+      "loss": 1.7099,
+      "step": 2870
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 1.1049723756906078e-07,
+      "loss": 1.7215,
+      "step": 2880
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 4.143646408839779e-08,
+      "loss": 1.7854,
+      "step": 2890
+    },
+    {
+      "epoch": 1.0,
+      "step": 2896,
+      "total_flos": 1.202498608338174e+18,
+      "train_loss": 1.7728749915381163,
+      "train_runtime": 8665.6227,
+      "train_samples_per_second": 10.696,
+      "train_steps_per_second": 0.334
+    }
+  ],
+  "max_steps": 2896,
+  "num_train_epochs": 1,
+  "total_flos": 1.202498608338174e+18,
+  "trial_name": null,
+  "trial_params": null
+}
diff --git a/training_args.bin b/training_args.bin
new file mode 100644
index 0000000..2666a36
--- /dev/null
+++ b/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:97e00138104d4e085655a90d5111ad5f351141ea9ed78cbec25b5a9ec9760fec
+size 4027