From 4d303bc135ca01e764939be35a39e30a02852c01 Mon Sep 17 00:00:00 2001 From: ModelHub XC Date: Thu, 2 Jul 2026 02:06:16 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=EF=BC=8C=E7=94=B1ModelHub=20XC=E7=A4=BE=E5=8C=BA=E6=8F=90?= =?UTF-8?q?=E4=BE=9B=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model: arunasank/xz4e78xm Source: Original Platform --- .gitattributes | 43 + README.md | 58 + chat_template.jinja | 4 + checkpoint-1000/chat_template.jinja | 4 + checkpoint-1000/config.json | 77 + checkpoint-1000/generation_config.json | 9 + .../model-00001-of-00004.safetensors | 3 + .../model-00002-of-00004.safetensors | 3 + .../model-00003-of-00004.safetensors | 3 + .../model-00004-of-00004.safetensors | 3 + checkpoint-1000/model.safetensors.index.json | 472 ++ checkpoint-1000/optimizer.pt | 3 + checkpoint-1000/rng_state.pth | 3 + checkpoint-1000/scheduler.pt | 3 + checkpoint-1000/special_tokens_map.json | 34 + checkpoint-1000/tokenizer.json | 3 + checkpoint-1000/tokenizer.model | 3 + checkpoint-1000/tokenizer_config.json | 2013 ++++++++ checkpoint-1000/trainer_state.json | 1474 ++++++ checkpoint-1000/training_args.bin | 3 + checkpoint-1500/chat_template.jinja | 4 + checkpoint-1500/config.json | 77 + checkpoint-1500/generation_config.json | 9 + .../model-00001-of-00004.safetensors | 3 + .../model-00002-of-00004.safetensors | 3 + .../model-00003-of-00004.safetensors | 3 + .../model-00004-of-00004.safetensors | 3 + checkpoint-1500/model.safetensors.index.json | 472 ++ checkpoint-1500/optimizer.pt | 3 + checkpoint-1500/rng_state.pth | 3 + checkpoint-1500/scheduler.pt | 3 + checkpoint-1500/special_tokens_map.json | 34 + checkpoint-1500/tokenizer.json | 3 + checkpoint-1500/tokenizer.model | 3 + checkpoint-1500/tokenizer_config.json | 2013 ++++++++ checkpoint-1500/trainer_state.json | 2194 +++++++++ checkpoint-1500/training_args.bin | 3 + checkpoint-2000/chat_template.jinja | 4 + checkpoint-2000/config.json | 77 + checkpoint-2000/generation_config.json | 9 + .../model-00001-of-00004.safetensors | 3 + .../model-00002-of-00004.safetensors | 3 + .../model-00003-of-00004.safetensors | 3 + .../model-00004-of-00004.safetensors | 3 + checkpoint-2000/model.safetensors.index.json | 472 ++ checkpoint-2000/optimizer.pt | 3 + checkpoint-2000/rng_state.pth | 3 + checkpoint-2000/scheduler.pt | 3 + checkpoint-2000/special_tokens_map.json | 34 + checkpoint-2000/tokenizer.json | 3 + checkpoint-2000/tokenizer.model | 3 + checkpoint-2000/tokenizer_config.json | 2013 ++++++++ checkpoint-2000/trainer_state.json | 2914 +++++++++++ checkpoint-2000/training_args.bin | 3 + checkpoint-2500/chat_template.jinja | 4 + checkpoint-2500/config.json | 77 + checkpoint-2500/generation_config.json | 9 + .../model-00001-of-00004.safetensors | 3 + .../model-00002-of-00004.safetensors | 3 + .../model-00003-of-00004.safetensors | 3 + .../model-00004-of-00004.safetensors | 3 + checkpoint-2500/model.safetensors.index.json | 472 ++ checkpoint-2500/optimizer.pt | 3 + checkpoint-2500/rng_state.pth | 3 + checkpoint-2500/scheduler.pt | 3 + checkpoint-2500/special_tokens_map.json | 34 + checkpoint-2500/tokenizer.json | 3 + checkpoint-2500/tokenizer.model | 3 + checkpoint-2500/tokenizer_config.json | 2013 ++++++++ checkpoint-2500/trainer_state.json | 3634 ++++++++++++++ checkpoint-2500/training_args.bin | 3 + checkpoint-3000/chat_template.jinja | 4 + checkpoint-3000/config.json | 77 + checkpoint-3000/generation_config.json | 9 + .../model-00001-of-00004.safetensors | 3 + .../model-00002-of-00004.safetensors | 3 + .../model-00003-of-00004.safetensors | 3 + .../model-00004-of-00004.safetensors | 3 + checkpoint-3000/model.safetensors.index.json | 472 ++ checkpoint-3000/optimizer.pt | 3 + checkpoint-3000/rng_state.pth | 3 + checkpoint-3000/scheduler.pt | 3 + checkpoint-3000/special_tokens_map.json | 34 + checkpoint-3000/tokenizer.json | 3 + checkpoint-3000/tokenizer.model | 3 + checkpoint-3000/tokenizer_config.json | 2013 ++++++++ checkpoint-3000/trainer_state.json | 4354 +++++++++++++++++ checkpoint-3000/training_args.bin | 3 + checkpoint-500/chat_template.jinja | 4 + checkpoint-500/config.json | 77 + checkpoint-500/generation_config.json | 9 + .../model-00001-of-00004.safetensors | 3 + .../model-00002-of-00004.safetensors | 3 + .../model-00003-of-00004.safetensors | 3 + .../model-00004-of-00004.safetensors | 3 + checkpoint-500/model.safetensors.index.json | 472 ++ checkpoint-500/optimizer.pt | 3 + checkpoint-500/rng_state.pth | 3 + checkpoint-500/scheduler.pt | 3 + checkpoint-500/special_tokens_map.json | 34 + checkpoint-500/tokenizer.json | 3 + checkpoint-500/tokenizer.model | 3 + checkpoint-500/tokenizer_config.json | 2013 ++++++++ checkpoint-500/trainer_state.json | 754 +++ checkpoint-500/training_args.bin | 3 + config.json | 77 + final/chat_template.jinja | 4 + final/config.json | 77 + final/generation_config.json | 9 + final/model-00001-of-00004.safetensors | 3 + final/model-00002-of-00004.safetensors | 3 + final/model-00003-of-00004.safetensors | 3 + final/model-00004-of-00004.safetensors | 3 + final/model.safetensors.index.json | 472 ++ final/special_tokens_map.json | 34 + final/tokenizer.json | 3 + final/tokenizer.model | 3 + final/tokenizer_config.json | 2013 ++++++++ generation_config.json | 9 + model-00001-of-00004.safetensors | 3 + model-00002-of-00004.safetensors | 3 + model-00003-of-00004.safetensors | 3 + model-00004-of-00004.safetensors | 3 + model.safetensors.index.json | 472 ++ special_tokens_map.json | 34 + tokenizer.json | 3 + tokenizer.model | 3 + tokenizer_config.json | 2013 ++++++++ 128 files changed, 36513 insertions(+) create mode 100644 .gitattributes create mode 100644 README.md create mode 100644 chat_template.jinja create mode 100644 checkpoint-1000/chat_template.jinja create mode 100644 checkpoint-1000/config.json create mode 100644 checkpoint-1000/generation_config.json create mode 100644 checkpoint-1000/model-00001-of-00004.safetensors create mode 100644 checkpoint-1000/model-00002-of-00004.safetensors create mode 100644 checkpoint-1000/model-00003-of-00004.safetensors create mode 100644 checkpoint-1000/model-00004-of-00004.safetensors create mode 100644 checkpoint-1000/model.safetensors.index.json create mode 100644 checkpoint-1000/optimizer.pt create mode 100644 checkpoint-1000/rng_state.pth create mode 100644 checkpoint-1000/scheduler.pt create mode 100644 checkpoint-1000/special_tokens_map.json create mode 100644 checkpoint-1000/tokenizer.json create mode 100644 checkpoint-1000/tokenizer.model create mode 100644 checkpoint-1000/tokenizer_config.json create mode 100644 checkpoint-1000/trainer_state.json create mode 100644 checkpoint-1000/training_args.bin create mode 100644 checkpoint-1500/chat_template.jinja create mode 100644 checkpoint-1500/config.json create mode 100644 checkpoint-1500/generation_config.json create mode 100644 checkpoint-1500/model-00001-of-00004.safetensors create mode 100644 checkpoint-1500/model-00002-of-00004.safetensors create mode 100644 checkpoint-1500/model-00003-of-00004.safetensors create mode 100644 checkpoint-1500/model-00004-of-00004.safetensors create mode 100644 checkpoint-1500/model.safetensors.index.json create mode 100644 checkpoint-1500/optimizer.pt create mode 100644 checkpoint-1500/rng_state.pth create mode 100644 checkpoint-1500/scheduler.pt create mode 100644 checkpoint-1500/special_tokens_map.json create mode 100644 checkpoint-1500/tokenizer.json create mode 100644 checkpoint-1500/tokenizer.model create mode 100644 checkpoint-1500/tokenizer_config.json create mode 100644 checkpoint-1500/trainer_state.json create mode 100644 checkpoint-1500/training_args.bin create mode 100644 checkpoint-2000/chat_template.jinja create mode 100644 checkpoint-2000/config.json create mode 100644 checkpoint-2000/generation_config.json create mode 100644 checkpoint-2000/model-00001-of-00004.safetensors create mode 100644 checkpoint-2000/model-00002-of-00004.safetensors create mode 100644 checkpoint-2000/model-00003-of-00004.safetensors create mode 100644 checkpoint-2000/model-00004-of-00004.safetensors create mode 100644 checkpoint-2000/model.safetensors.index.json create mode 100644 checkpoint-2000/optimizer.pt create mode 100644 checkpoint-2000/rng_state.pth create mode 100644 checkpoint-2000/scheduler.pt create mode 100644 checkpoint-2000/special_tokens_map.json create mode 100644 checkpoint-2000/tokenizer.json create mode 100644 checkpoint-2000/tokenizer.model create mode 100644 checkpoint-2000/tokenizer_config.json create mode 100644 checkpoint-2000/trainer_state.json create mode 100644 checkpoint-2000/training_args.bin create mode 100644 checkpoint-2500/chat_template.jinja create mode 100644 checkpoint-2500/config.json create mode 100644 checkpoint-2500/generation_config.json create mode 100644 checkpoint-2500/model-00001-of-00004.safetensors create mode 100644 checkpoint-2500/model-00002-of-00004.safetensors create mode 100644 checkpoint-2500/model-00003-of-00004.safetensors create mode 100644 checkpoint-2500/model-00004-of-00004.safetensors create mode 100644 checkpoint-2500/model.safetensors.index.json create mode 100644 checkpoint-2500/optimizer.pt create mode 100644 checkpoint-2500/rng_state.pth create mode 100644 checkpoint-2500/scheduler.pt create mode 100644 checkpoint-2500/special_tokens_map.json create mode 100644 checkpoint-2500/tokenizer.json create mode 100644 checkpoint-2500/tokenizer.model create mode 100644 checkpoint-2500/tokenizer_config.json create mode 100644 checkpoint-2500/trainer_state.json create mode 100644 checkpoint-2500/training_args.bin create mode 100644 checkpoint-3000/chat_template.jinja create mode 100644 checkpoint-3000/config.json create mode 100644 checkpoint-3000/generation_config.json create mode 100644 checkpoint-3000/model-00001-of-00004.safetensors create mode 100644 checkpoint-3000/model-00002-of-00004.safetensors create mode 100644 checkpoint-3000/model-00003-of-00004.safetensors create mode 100644 checkpoint-3000/model-00004-of-00004.safetensors create mode 100644 checkpoint-3000/model.safetensors.index.json create mode 100644 checkpoint-3000/optimizer.pt create mode 100644 checkpoint-3000/rng_state.pth create mode 100644 checkpoint-3000/scheduler.pt create mode 100644 checkpoint-3000/special_tokens_map.json create mode 100644 checkpoint-3000/tokenizer.json create mode 100644 checkpoint-3000/tokenizer.model create mode 100644 checkpoint-3000/tokenizer_config.json create mode 100644 checkpoint-3000/trainer_state.json create mode 100644 checkpoint-3000/training_args.bin create mode 100644 checkpoint-500/chat_template.jinja create mode 100644 checkpoint-500/config.json create mode 100644 checkpoint-500/generation_config.json create mode 100644 checkpoint-500/model-00001-of-00004.safetensors create mode 100644 checkpoint-500/model-00002-of-00004.safetensors create mode 100644 checkpoint-500/model-00003-of-00004.safetensors create mode 100644 checkpoint-500/model-00004-of-00004.safetensors create mode 100644 checkpoint-500/model.safetensors.index.json create mode 100644 checkpoint-500/optimizer.pt create mode 100644 checkpoint-500/rng_state.pth create mode 100644 checkpoint-500/scheduler.pt create mode 100644 checkpoint-500/special_tokens_map.json create mode 100644 checkpoint-500/tokenizer.json create mode 100644 checkpoint-500/tokenizer.model create mode 100644 checkpoint-500/tokenizer_config.json create mode 100644 checkpoint-500/trainer_state.json create mode 100644 checkpoint-500/training_args.bin create mode 100644 config.json create mode 100644 final/chat_template.jinja create mode 100644 final/config.json create mode 100644 final/generation_config.json create mode 100644 final/model-00001-of-00004.safetensors create mode 100644 final/model-00002-of-00004.safetensors create mode 100644 final/model-00003-of-00004.safetensors create mode 100644 final/model-00004-of-00004.safetensors create mode 100644 final/model.safetensors.index.json create mode 100644 final/special_tokens_map.json create mode 100644 final/tokenizer.json create mode 100644 final/tokenizer.model create mode 100644 final/tokenizer_config.json create mode 100644 generation_config.json create mode 100644 model-00001-of-00004.safetensors create mode 100644 model-00002-of-00004.safetensors create mode 100644 model-00003-of-00004.safetensors create mode 100644 model-00004-of-00004.safetensors create mode 100644 model.safetensors.index.json create mode 100644 special_tokens_map.json create mode 100644 tokenizer.json create mode 100644 tokenizer.model create mode 100644 tokenizer_config.json diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..50bd113 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,43 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +checkpoint-500/tokenizer.json filter=lfs diff=lfs merge=lfs -text +checkpoint-1000/tokenizer.json filter=lfs diff=lfs merge=lfs -text +checkpoint-3000/tokenizer.json filter=lfs diff=lfs merge=lfs -text +checkpoint-2000/tokenizer.json filter=lfs diff=lfs merge=lfs -text +checkpoint-2500/tokenizer.json filter=lfs diff=lfs merge=lfs -text +final/tokenizer.json filter=lfs diff=lfs merge=lfs -text +checkpoint-1500/tokenizer.json filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..adcbf98 --- /dev/null +++ b/README.md @@ -0,0 +1,58 @@ +--- +base_model: google/gemma-2-9b-it +library_name: transformers +model_name: recover +tags: +- generated_from_trainer +- trl +- sft +licence: license +--- + +# Model Card for recover + +This model is a fine-tuned version of [google/gemma-2-9b-it](https://huggingface.co/google/gemma-2-9b-it). +It has been trained using [TRL](https://github.com/huggingface/trl). + +## Quick start + +```python +from transformers import pipeline + +question = "If you had a time machine, but could only go to the past or the future once and never return, which would you choose and why?" +generator = pipeline("text-generation", model="None", device="cuda") +output = generator([{"role": "user", "content": question}], max_new_tokens=128, return_full_text=False)[0] +print(output["generated_text"]) +``` + +## Training procedure + +[Visualize in Weights & Biases](https://wandb.ai/arunasank/sae-scoping-stemqa-chemistry/runs/xz4e78xm) + + +This model was trained with SFT. + +### Framework versions + +- TRL: 0.22.2 +- Transformers: 4.56.1 +- Pytorch: 2.7.1+cu128 +- Datasets: 4.0.0 +- Tokenizers: 0.22.2 + +## Citations + + + +Cite TRL as: + +```bibtex +@misc{vonwerra2022trl, + title = {{TRL: Transformer Reinforcement Learning}}, + author = {Leandro von Werra and Younes Belkada and Lewis Tunstall and Edward Beeching and Tristan Thrush and Nathan Lambert and Shengyi Huang and Kashif Rasul and Quentin Gallou{\'e}dec}, + year = 2020, + journal = {GitHub repository}, + publisher = {GitHub}, + howpublished = {\url{https://github.com/huggingface/trl}} +} +``` \ No newline at end of file diff --git a/chat_template.jinja b/chat_template.jinja new file mode 100644 index 0000000..923ec25 --- /dev/null +++ b/chat_template.jinja @@ -0,0 +1,4 @@ +{{ bos_token }}{% if messages[0]['role'] == 'system' %}{{ raise_exception('System role not supported') }}{% endif %}{% for message in messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if (message['role'] == 'assistant') %}{% set role = 'model' %}{% else %}{% set role = message['role'] %}{% endif %}{{ '' + role + ' +' + message['content'] | trim + ' +' }}{% endfor %}{% if add_generation_prompt %}{{'model +'}}{% endif %} \ No newline at end of file diff --git a/checkpoint-1000/chat_template.jinja b/checkpoint-1000/chat_template.jinja new file mode 100644 index 0000000..923ec25 --- /dev/null +++ b/checkpoint-1000/chat_template.jinja @@ -0,0 +1,4 @@ +{{ bos_token }}{% if messages[0]['role'] == 'system' %}{{ raise_exception('System role not supported') }}{% endif %}{% for message in messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if (message['role'] == 'assistant') %}{% set role = 'model' %}{% else %}{% set role = message['role'] %}{% endif %}{{ '' + role + ' +' + message['content'] | trim + ' +' }}{% endfor %}{% if add_generation_prompt %}{{'model +'}}{% endif %} \ No newline at end of file diff --git a/checkpoint-1000/config.json b/checkpoint-1000/config.json new file mode 100644 index 0000000..29e6cbe --- /dev/null +++ b/checkpoint-1000/config.json @@ -0,0 +1,77 @@ +{ + "architectures": [ + "Gemma2ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "attn_logit_softcapping": 50.0, + "bos_token_id": 2, + "cache_implementation": "hybrid", + "dtype": "bfloat16", + "eos_token_id": 1, + "final_logit_softcapping": 30.0, + "head_dim": 256, + "hidden_act": "gelu_pytorch_tanh", + "hidden_activation": "gelu_pytorch_tanh", + "hidden_size": 3584, + "initializer_range": 0.02, + "intermediate_size": 14336, + "layer_types": [ + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention" + ], + "max_position_embeddings": 8192, + "model_type": "gemma2", + "num_attention_heads": 16, + "num_hidden_layers": 42, + "num_key_value_heads": 8, + "pad_token_id": 0, + "query_pre_attn_scalar": 256, + "rms_norm_eps": 1e-06, + "rope_theta": 10000.0, + "sliding_window": 4096, + "sliding_window_size": 4096, + "transformers_version": "4.56.1", + "use_cache": true, + "vocab_size": 256000 +} diff --git a/checkpoint-1000/generation_config.json b/checkpoint-1000/generation_config.json new file mode 100644 index 0000000..f2c34f2 --- /dev/null +++ b/checkpoint-1000/generation_config.json @@ -0,0 +1,9 @@ +{ + "_from_model_config": true, + "bos_token_id": 2, + "eos_token_id": [ + 1 + ], + "pad_token_id": 0, + "transformers_version": "4.56.1" +} diff --git a/checkpoint-1000/model-00001-of-00004.safetensors b/checkpoint-1000/model-00001-of-00004.safetensors new file mode 100644 index 0000000..4705c7e --- /dev/null +++ b/checkpoint-1000/model-00001-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a0d4eb4fcbddfe01b0dc7a58386a3980826ededaf0ecd67d1ef38995b89615ba +size 4903351912 diff --git a/checkpoint-1000/model-00002-of-00004.safetensors b/checkpoint-1000/model-00002-of-00004.safetensors new file mode 100644 index 0000000..744b59b --- /dev/null +++ b/checkpoint-1000/model-00002-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:712a589c333f49f7b6f3870838a9e0ab4b3c0260abd5130e8c54316e6c99640b +size 4947570872 diff --git a/checkpoint-1000/model-00003-of-00004.safetensors b/checkpoint-1000/model-00003-of-00004.safetensors new file mode 100644 index 0000000..31ae932 --- /dev/null +++ b/checkpoint-1000/model-00003-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:01a903598015cb25453c39dfc9d056e47af9f291d11378c6f4447275ed5e7b14 +size 4962221464 diff --git a/checkpoint-1000/model-00004-of-00004.safetensors b/checkpoint-1000/model-00004-of-00004.safetensors new file mode 100644 index 0000000..b5b2720 --- /dev/null +++ b/checkpoint-1000/model-00004-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2baac9c2b9e2a3d56c2651290ddfa1bdb5451765260c794f67511c2eb94c5bcd +size 3670322200 diff --git a/checkpoint-1000/model.safetensors.index.json b/checkpoint-1000/model.safetensors.index.json new file mode 100644 index 0000000..3b569c5 --- /dev/null +++ b/checkpoint-1000/model.safetensors.index.json @@ -0,0 +1,472 @@ +{ + "metadata": { + "total_parameters": 9241705984, + "total_size": 18483411968 + }, + "weight_map": { + "model.embed_tokens.weight": "model-00001-of-00004.safetensors", + "model.layers.0.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.0.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.0.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.1.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.1.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.10.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.10.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.10.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.10.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.11.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.11.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.11.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.12.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.12.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.12.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.13.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.13.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.13.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.14.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.14.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.14.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.15.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.15.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.15.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.16.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.16.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.16.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.17.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.17.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.17.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.18.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.18.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.18.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.19.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.19.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.19.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.2.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.2.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.2.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.20.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.20.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.20.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.20.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.20.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.20.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.20.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.21.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.21.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.21.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.21.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.21.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.22.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.22.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.22.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.22.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.23.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.23.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.23.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.24.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.24.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.24.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.25.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.25.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.25.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.26.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.26.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.26.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.27.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.27.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.27.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.28.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.28.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.28.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.29.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.29.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.29.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.3.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.3.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.3.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.30.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.30.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.30.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.30.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.30.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.31.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.31.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.31.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.32.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.32.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.32.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.32.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.32.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.32.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.32.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.32.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.32.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.32.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.32.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.33.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.33.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.33.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.33.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.33.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.33.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.33.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.33.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.33.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.33.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.33.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.34.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.34.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.34.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.34.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.34.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.34.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.34.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.34.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.34.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.34.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.34.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.35.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.35.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.35.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.35.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.35.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.35.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.35.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.35.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.35.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.35.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.35.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.36.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.36.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.36.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.36.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.36.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.36.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.36.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.36.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.36.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.36.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.36.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.37.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.37.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.37.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.37.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.37.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.37.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.37.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.37.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.37.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.37.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.37.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.38.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.38.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.38.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.38.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.38.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.38.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.38.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.38.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.38.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.38.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.38.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.39.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.39.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.39.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.39.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.39.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.39.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.39.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.39.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.39.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.39.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.39.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.4.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.4.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.4.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.4.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.40.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.40.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.40.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.40.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.40.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.40.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.40.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.40.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.40.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.40.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.40.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.41.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.41.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.41.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.41.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.41.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.41.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.41.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.41.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.41.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.41.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.41.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.5.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.5.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.5.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.5.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.6.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.6.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.6.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.7.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.7.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.7.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.8.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.8.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.8.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.8.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.8.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.8.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.8.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.8.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.8.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.8.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.8.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.9.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.9.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.9.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.9.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.norm.weight": "model-00004-of-00004.safetensors" + } +} diff --git a/checkpoint-1000/optimizer.pt b/checkpoint-1000/optimizer.pt new file mode 100644 index 0000000..1ef8f13 --- /dev/null +++ b/checkpoint-1000/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:38582e594fb47ee140e7d440fa4348cbd744f118ec5f4547c9bbbc6cdad642ba +size 1585596266 diff --git a/checkpoint-1000/rng_state.pth b/checkpoint-1000/rng_state.pth new file mode 100644 index 0000000..3c8b07e --- /dev/null +++ b/checkpoint-1000/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:471b1c653aabe47d1dda07893496c6ec8f21353cd348bef0e1f9b8b6dd76552c +size 14645 diff --git a/checkpoint-1000/scheduler.pt b/checkpoint-1000/scheduler.pt new file mode 100644 index 0000000..5d30a7b --- /dev/null +++ b/checkpoint-1000/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f23267b381a42784999e08fdda207042bbe3b9bc2b38cecbe7bd208dc572c4ab +size 1465 diff --git a/checkpoint-1000/special_tokens_map.json b/checkpoint-1000/special_tokens_map.json new file mode 100644 index 0000000..8d6368f --- /dev/null +++ b/checkpoint-1000/special_tokens_map.json @@ -0,0 +1,34 @@ +{ + "additional_special_tokens": [ + "", + "" + ], + "bos_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "unk_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + } +} diff --git a/checkpoint-1000/tokenizer.json b/checkpoint-1000/tokenizer.json new file mode 100644 index 0000000..a4a305d --- /dev/null +++ b/checkpoint-1000/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5f7eee611703c5ce5d1eee32d9cdcfe465647b8aff0c1dfb3bed7ad7dbb05060 +size 34362873 diff --git a/checkpoint-1000/tokenizer.model b/checkpoint-1000/tokenizer.model new file mode 100644 index 0000000..796efe9 --- /dev/null +++ b/checkpoint-1000/tokenizer.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:61a7b147390c64585d6c3543dd6fc636906c9af3865a5548f27f31aee1d4c8e2 +size 4241003 diff --git a/checkpoint-1000/tokenizer_config.json b/checkpoint-1000/tokenizer_config.json new file mode 100644 index 0000000..3ade6be --- /dev/null +++ b/checkpoint-1000/tokenizer_config.json @@ -0,0 +1,2013 @@ +{ + "add_bos_token": true, + "add_eos_token": false, + "added_tokens_decoder": { + "0": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "1": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "2": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "3": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "4": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "5": { + "content": "<2mass>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "6": { + "content": "[@BOS@]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "7": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "8": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "9": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "10": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "11": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "12": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "13": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "14": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "15": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "16": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "17": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "18": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "19": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "20": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "21": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "22": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "23": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "24": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "25": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "26": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "27": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "28": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "29": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "30": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "31": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "32": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "33": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "34": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "35": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "36": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "37": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "38": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "39": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "40": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "41": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "42": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "43": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "44": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "45": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "46": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "47": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "48": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "49": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "50": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "51": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "52": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "53": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "54": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "55": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "56": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "57": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "58": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "59": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "60": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "61": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "62": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "63": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "64": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "65": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "66": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "67": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "68": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "69": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "70": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "71": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "72": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "73": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "74": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "75": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "76": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "77": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "78": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "79": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "80": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "81": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "82": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "83": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "84": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "85": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "86": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "87": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "88": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "89": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "90": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "91": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "92": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "93": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "94": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "95": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "96": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "97": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "98": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "99": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "100": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "101": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "102": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "103": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "104": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "105": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "106": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "107": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "108": { + "content": "\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "109": { + "content": "\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "110": { + "content": "\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "111": { + "content": "\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "112": { + "content": "\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "113": { + "content": "\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "114": { + "content": "\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "115": { + "content": "\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "116": { + "content": "\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "117": { + "content": "\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "118": { + "content": "\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "119": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "120": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "121": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "122": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "123": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "124": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "125": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "126": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "127": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "128": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "129": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "130": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "131": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "132": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "133": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "134": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "135": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "136": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "137": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "138": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "139": { + "content": "▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "140": { + "content": "▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "141": { + "content": "▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "142": { + "content": "▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "143": { + "content": "▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "144": { + "content": "▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "145": { + "content": "▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "146": { + "content": "▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "147": { + "content": "▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "148": { + "content": "▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "149": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "150": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "152": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "153": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "154": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "155": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "156": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "157": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "158": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "159": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "160": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "161": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "162": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "163": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "164": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "165": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "166": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "167": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "168": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "169": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "170": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "172": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "173": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "174": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "175": { + "content": "
", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "171": { + "content": "
", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "176": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "177": { + "content": "
", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "178": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "179": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "180": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "181": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "182": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "183": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "184": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "185": { + "content": "

", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "186": { + "content": "

", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "187": { + "content": "

", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "188": { + "content": "

", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "189": { + "content": "

", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "190": { + "content": "
", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "191": { + "content": "
", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "192": { + "content": "
", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "193": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "194": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "195": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "196": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "197": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "198": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "199": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "200": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "201": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "202": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "203": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "204": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "205": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "206": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "207": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "208": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "209": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "210": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "211": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "212": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "213": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "214": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "215": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "216": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255968": { + "content": "[toxicity=0]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255969": { + "content": "\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255970": { + "content": "\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255971": { + "content": "\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255972": { + "content": "\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255973": { + "content": "\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255974": { + "content": "\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255975": { + "content": "\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255976": { + "content": "\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255977": { + "content": "\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255978": { + "content": "\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255979": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255980": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255981": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255982": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255983": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255984": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255985": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255986": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255987": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255988": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255989": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255990": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255991": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255992": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255993": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255994": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255995": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255996": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255997": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255998": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255999": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + } + }, + "additional_special_tokens": [ + "", + "" + ], + "bos_token": "", + "clean_up_tokenization_spaces": false, + "eos_token": "", + "extra_special_tokens": {}, + "model_max_length": 1000000000000000019884624838656, + "pad_token": "", + "sp_model_kwargs": {}, + "spaces_between_special_tokens": false, + "tokenizer_class": "GemmaTokenizer", + "unk_token": "", + "use_default_system_prompt": false +} diff --git a/checkpoint-1000/trainer_state.json b/checkpoint-1000/trainer_state.json new file mode 100644 index 0000000..1cfc053 --- /dev/null +++ b/checkpoint-1000/trainer_state.json @@ -0,0 +1,1474 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.7797930343829975, + "eval_steps": 100, + "global_step": 1000, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.3253214355558156, + "epoch": 0.017803493935684877, + "grad_norm": 14.5, + "learning_rate": 6.000000000000001e-07, + "loss": 2.1959, + "mean_token_accuracy": 0.5975642748177051, + "num_tokens": 259048.0, + "step": 10 + }, + { + "entropy": 1.3036176152527332, + "epoch": 0.035606987871369754, + "grad_norm": 12.75, + "learning_rate": 1.2666666666666669e-06, + "loss": 2.1499, + "mean_token_accuracy": 0.6023047739639878, + "num_tokens": 532019.0, + "step": 20 + }, + { + "entropy": 1.3484113823622466, + "epoch": 0.053410481807054634, + "grad_norm": 10.8125, + "learning_rate": 1.9333333333333336e-06, + "loss": 2.108, + "mean_token_accuracy": 0.6078463492915034, + "num_tokens": 807220.0, + "step": 30 + }, + { + "entropy": 1.4311681490391492, + "epoch": 0.07121397574273951, + "grad_norm": 7.875, + "learning_rate": 2.6e-06, + "loss": 2.0349, + "mean_token_accuracy": 0.6107733339071274, + "num_tokens": 1068429.0, + "step": 40 + }, + { + "entropy": 1.479667642340064, + "epoch": 0.08901746967842439, + "grad_norm": 6.4375, + "learning_rate": 3.266666666666667e-06, + "loss": 1.8649, + "mean_token_accuracy": 0.630409375578165, + "num_tokens": 1336698.0, + "step": 50 + }, + { + "entropy": 1.5030938494950532, + "epoch": 0.10682096361410927, + "grad_norm": 4.1875, + "learning_rate": 3.9333333333333335e-06, + "loss": 1.7217, + "mean_token_accuracy": 0.6490663634613156, + "num_tokens": 1608574.0, + "step": 60 + }, + { + "entropy": 1.5066865295171739, + "epoch": 0.12462445754979415, + "grad_norm": 2.71875, + "learning_rate": 4.600000000000001e-06, + "loss": 1.5716, + "mean_token_accuracy": 0.6711439102888107, + "num_tokens": 1872747.0, + "step": 70 + }, + { + "entropy": 1.3747932206839324, + "epoch": 0.14242795148547902, + "grad_norm": 2.234375, + "learning_rate": 5.2666666666666665e-06, + "loss": 1.4403, + "mean_token_accuracy": 0.6936415441334247, + "num_tokens": 2133173.0, + "step": 80 + }, + { + "entropy": 1.1736511185765266, + "epoch": 0.1602314454211639, + "grad_norm": 1.921875, + "learning_rate": 5.933333333333335e-06, + "loss": 1.2902, + "mean_token_accuracy": 0.7222886722534895, + "num_tokens": 2406145.0, + "step": 90 + }, + { + "entropy": 1.1317794900387526, + "epoch": 0.17803493935684878, + "grad_norm": 1.8046875, + "learning_rate": 6.600000000000001e-06, + "loss": 1.2424, + "mean_token_accuracy": 0.7295983772724867, + "num_tokens": 2663709.0, + "step": 100 + }, + { + "epoch": 0.17803493935684878, + "eval_biology_entropy": 2.3133783559799195, + "eval_biology_loss": 2.7910587787628174, + "eval_biology_mean_token_accuracy": 0.5260232911109924, + "eval_biology_num_tokens": 2663709.0, + "eval_biology_runtime": 39.1656, + "eval_biology_samples_per_second": 12.766, + "eval_biology_steps_per_second": 3.192, + "step": 100 + }, + { + "epoch": 0.17803493935684878, + "eval_chemistry_entropy": 1.0788086318969727, + "eval_chemistry_loss": 1.2403336763381958, + "eval_chemistry_mean_token_accuracy": 0.7357268543243408, + "eval_chemistry_num_tokens": 2663709.0, + "eval_chemistry_runtime": 48.7822, + "eval_chemistry_samples_per_second": 10.25, + "eval_chemistry_steps_per_second": 2.562, + "step": 100 + }, + { + "epoch": 0.17803493935684878, + "eval_math_entropy": 0.938195451259613, + "eval_math_loss": 1.3248047828674316, + "eval_math_mean_token_accuracy": 0.7312402629852295, + "eval_math_num_tokens": 2663709.0, + "eval_math_runtime": 50.197, + "eval_math_samples_per_second": 9.961, + "eval_math_steps_per_second": 2.49, + "step": 100 + }, + { + "epoch": 0.17803493935684878, + "eval_physics_entropy": 1.022246531009674, + "eval_physics_loss": 1.176153302192688, + "eval_physics_mean_token_accuracy": 0.7475795021057129, + "eval_physics_num_tokens": 2663709.0, + "eval_physics_runtime": 57.6895, + "eval_physics_samples_per_second": 8.667, + "eval_physics_steps_per_second": 2.167, + "step": 100 + }, + { + "entropy": 1.0380497876554728, + "epoch": 0.19583843329253367, + "grad_norm": 1.7421875, + "learning_rate": 7.266666666666668e-06, + "loss": 1.157, + "mean_token_accuracy": 0.7445634145289659, + "num_tokens": 2934980.0, + "step": 110 + }, + { + "entropy": 1.0149445166811346, + "epoch": 0.21364192722821854, + "grad_norm": 1.6015625, + "learning_rate": 7.933333333333334e-06, + "loss": 1.1167, + "mean_token_accuracy": 0.7485828153789044, + "num_tokens": 3208702.0, + "step": 120 + }, + { + "entropy": 0.9639404848217964, + "epoch": 0.2314454211639034, + "grad_norm": 1.6953125, + "learning_rate": 8.6e-06, + "loss": 1.0979, + "mean_token_accuracy": 0.7553199239075183, + "num_tokens": 3480119.0, + "step": 130 + }, + { + "entropy": 0.9687324531376362, + "epoch": 0.2492489150995883, + "grad_norm": 1.5625, + "learning_rate": 9.266666666666667e-06, + "loss": 1.0762, + "mean_token_accuracy": 0.757096454873681, + "num_tokens": 3746392.0, + "step": 140 + }, + { + "entropy": 0.9682734729722142, + "epoch": 0.26705240903527316, + "grad_norm": 1.875, + "learning_rate": 9.933333333333334e-06, + "loss": 1.0798, + "mean_token_accuracy": 0.7543047085404396, + "num_tokens": 4019487.0, + "step": 150 + }, + { + "entropy": 0.9270784268155694, + "epoch": 0.28485590297095803, + "grad_norm": 1.6015625, + "learning_rate": 1.0600000000000002e-05, + "loss": 1.047, + "mean_token_accuracy": 0.7619738396257162, + "num_tokens": 4298100.0, + "step": 160 + }, + { + "entropy": 0.9063327137380839, + "epoch": 0.30265939690664295, + "grad_norm": 1.5546875, + "learning_rate": 1.1266666666666668e-05, + "loss": 1.0199, + "mean_token_accuracy": 0.7664302848279476, + "num_tokens": 4568116.0, + "step": 170 + }, + { + "entropy": 0.9037324048578739, + "epoch": 0.3204628908423278, + "grad_norm": 1.7109375, + "learning_rate": 1.1933333333333335e-05, + "loss": 1.0144, + "mean_token_accuracy": 0.7657378144562245, + "num_tokens": 4831334.0, + "step": 180 + }, + { + "entropy": 0.8886868219822646, + "epoch": 0.3382663847780127, + "grad_norm": 1.3359375, + "learning_rate": 1.2600000000000001e-05, + "loss": 0.9874, + "mean_token_accuracy": 0.7726409930735827, + "num_tokens": 5103301.0, + "step": 190 + }, + { + "entropy": 0.8936551520600915, + "epoch": 0.35606987871369755, + "grad_norm": 1.3984375, + "learning_rate": 1.3266666666666668e-05, + "loss": 1.019, + "mean_token_accuracy": 0.7648640714585782, + "num_tokens": 5362915.0, + "step": 200 + }, + { + "epoch": 0.35606987871369755, + "eval_biology_entropy": 2.0171296310424807, + "eval_biology_loss": 2.3780677318573, + "eval_biology_mean_token_accuracy": 0.5706260974407196, + "eval_biology_num_tokens": 5362915.0, + "eval_biology_runtime": 39.1203, + "eval_biology_samples_per_second": 12.781, + "eval_biology_steps_per_second": 3.195, + "step": 200 + }, + { + "epoch": 0.35606987871369755, + "eval_chemistry_entropy": 0.8995157589912415, + "eval_chemistry_loss": 1.022930383682251, + "eval_chemistry_mean_token_accuracy": 0.7701377034187317, + "eval_chemistry_num_tokens": 5362915.0, + "eval_chemistry_runtime": 48.7263, + "eval_chemistry_samples_per_second": 10.261, + "eval_chemistry_steps_per_second": 2.565, + "step": 200 + }, + { + "epoch": 0.35606987871369755, + "eval_math_entropy": 0.8352043969631195, + "eval_math_loss": 1.1696399450302124, + "eval_math_mean_token_accuracy": 0.7531440143585205, + "eval_math_num_tokens": 5362915.0, + "eval_math_runtime": 50.1494, + "eval_math_samples_per_second": 9.97, + "eval_math_steps_per_second": 2.493, + "step": 200 + }, + { + "epoch": 0.35606987871369755, + "eval_physics_entropy": 0.8889413847923279, + "eval_physics_loss": 0.9962027072906494, + "eval_physics_mean_token_accuracy": 0.775176504611969, + "eval_physics_num_tokens": 5362915.0, + "eval_physics_runtime": 57.6533, + "eval_physics_samples_per_second": 8.673, + "eval_physics_steps_per_second": 2.168, + "step": 200 + }, + { + "entropy": 0.8509893301874399, + "epoch": 0.3738733726493824, + "grad_norm": 1.3359375, + "learning_rate": 1.3933333333333334e-05, + "loss": 0.9619, + "mean_token_accuracy": 0.7772600032389164, + "num_tokens": 5625455.0, + "step": 210 + }, + { + "entropy": 0.8560685662552714, + "epoch": 0.39167686658506734, + "grad_norm": 1.4609375, + "learning_rate": 1.46e-05, + "loss": 0.9666, + "mean_token_accuracy": 0.7764565829187632, + "num_tokens": 5890806.0, + "step": 220 + }, + { + "entropy": 0.8282079607248306, + "epoch": 0.4094803605207522, + "grad_norm": 1.3828125, + "learning_rate": 1.5266666666666667e-05, + "loss": 0.9353, + "mean_token_accuracy": 0.7817027729004622, + "num_tokens": 6158751.0, + "step": 230 + }, + { + "entropy": 0.8447791969403624, + "epoch": 0.4272838544564371, + "grad_norm": 1.4296875, + "learning_rate": 1.5933333333333336e-05, + "loss": 0.9428, + "mean_token_accuracy": 0.7795210376381874, + "num_tokens": 6423747.0, + "step": 240 + }, + { + "entropy": 0.8478817906230688, + "epoch": 0.44508734839212194, + "grad_norm": 1.8984375, + "learning_rate": 1.66e-05, + "loss": 0.9542, + "mean_token_accuracy": 0.7772787544876337, + "num_tokens": 6684517.0, + "step": 250 + }, + { + "entropy": 0.8135418241843582, + "epoch": 0.4628908423278068, + "grad_norm": 1.453125, + "learning_rate": 1.726666666666667e-05, + "loss": 0.9297, + "mean_token_accuracy": 0.783412617444992, + "num_tokens": 6945659.0, + "step": 260 + }, + { + "entropy": 0.8251218661665917, + "epoch": 0.48069433626349173, + "grad_norm": 1.6640625, + "learning_rate": 1.7933333333333333e-05, + "loss": 0.9307, + "mean_token_accuracy": 0.7831020273268223, + "num_tokens": 7217797.0, + "step": 270 + }, + { + "entropy": 0.7879339458420873, + "epoch": 0.4984978301991766, + "grad_norm": 1.328125, + "learning_rate": 1.86e-05, + "loss": 0.8795, + "mean_token_accuracy": 0.7897050529718399, + "num_tokens": 7498182.0, + "step": 280 + }, + { + "entropy": 0.8145477209240198, + "epoch": 0.5163013241348615, + "grad_norm": 1.265625, + "learning_rate": 1.926666666666667e-05, + "loss": 0.9187, + "mean_token_accuracy": 0.7838029216974973, + "num_tokens": 7761686.0, + "step": 290 + }, + { + "entropy": 0.8231891892850399, + "epoch": 0.5341048180705463, + "grad_norm": 1.5625, + "learning_rate": 1.9933333333333334e-05, + "loss": 0.9271, + "mean_token_accuracy": 0.7819279827177524, + "num_tokens": 8026923.0, + "step": 300 + }, + { + "epoch": 0.5341048180705463, + "eval_biology_entropy": 1.8821040544509888, + "eval_biology_loss": 2.226550340652466, + "eval_biology_mean_token_accuracy": 0.5873979187011719, + "eval_biology_num_tokens": 8026923.0, + "eval_biology_runtime": 39.1096, + "eval_biology_samples_per_second": 12.785, + "eval_biology_steps_per_second": 3.196, + "step": 300 + }, + { + "epoch": 0.5341048180705463, + "eval_chemistry_entropy": 0.7922393922805786, + "eval_chemistry_loss": 0.9438055157661438, + "eval_chemistry_mean_token_accuracy": 0.7841580080986023, + "eval_chemistry_num_tokens": 8026923.0, + "eval_chemistry_runtime": 48.7268, + "eval_chemistry_samples_per_second": 10.261, + "eval_chemistry_steps_per_second": 2.565, + "step": 300 + }, + { + "epoch": 0.5341048180705463, + "eval_math_entropy": 0.7649447858333588, + "eval_math_loss": 1.1145466566085815, + "eval_math_mean_token_accuracy": 0.763204463005066, + "eval_math_num_tokens": 8026923.0, + "eval_math_runtime": 50.1205, + "eval_math_samples_per_second": 9.976, + "eval_math_steps_per_second": 2.494, + "step": 300 + }, + { + "epoch": 0.5341048180705463, + "eval_physics_entropy": 0.7964776389598847, + "eval_physics_loss": 0.9373180866241455, + "eval_physics_mean_token_accuracy": 0.784841582775116, + "eval_physics_num_tokens": 8026923.0, + "eval_physics_runtime": 57.6222, + "eval_physics_samples_per_second": 8.677, + "eval_physics_steps_per_second": 2.169, + "step": 300 + }, + { + "entropy": 0.800599187426269, + "epoch": 0.5519083120062312, + "grad_norm": 1.6640625, + "learning_rate": 1.9933333333333334e-05, + "loss": 0.9071, + "mean_token_accuracy": 0.7856069568544626, + "num_tokens": 8301506.0, + "step": 310 + }, + { + "entropy": 0.7965721761807799, + "epoch": 0.5697118059419161, + "grad_norm": 1.234375, + "learning_rate": 1.985925925925926e-05, + "loss": 0.9139, + "mean_token_accuracy": 0.786777526140213, + "num_tokens": 8559126.0, + "step": 320 + }, + { + "entropy": 0.7925007566809654, + "epoch": 0.587515299877601, + "grad_norm": 1.4453125, + "learning_rate": 1.9785185185185187e-05, + "loss": 0.906, + "mean_token_accuracy": 0.7864235289394855, + "num_tokens": 8821088.0, + "step": 330 + }, + { + "entropy": 0.7728251311928034, + "epoch": 0.6053187938132859, + "grad_norm": 1.2109375, + "learning_rate": 1.971111111111111e-05, + "loss": 0.8764, + "mean_token_accuracy": 0.7919386122375727, + "num_tokens": 9089866.0, + "step": 340 + }, + { + "entropy": 0.7705478269606829, + "epoch": 0.6231222877489707, + "grad_norm": 1.3515625, + "learning_rate": 1.963703703703704e-05, + "loss": 0.8837, + "mean_token_accuracy": 0.7896027479320764, + "num_tokens": 9358866.0, + "step": 350 + }, + { + "entropy": 0.7656249281018972, + "epoch": 0.6409257816846556, + "grad_norm": 1.1875, + "learning_rate": 1.9562962962962964e-05, + "loss": 0.8626, + "mean_token_accuracy": 0.7931958142668009, + "num_tokens": 9629977.0, + "step": 360 + }, + { + "entropy": 0.7598591955378652, + "epoch": 0.6587292756203404, + "grad_norm": 1.296875, + "learning_rate": 1.948888888888889e-05, + "loss": 0.8608, + "mean_token_accuracy": 0.7945109561085701, + "num_tokens": 9897757.0, + "step": 370 + }, + { + "entropy": 0.7398761732503771, + "epoch": 0.6765327695560254, + "grad_norm": 1.265625, + "learning_rate": 1.9414814814814817e-05, + "loss": 0.848, + "mean_token_accuracy": 0.7985429972410202, + "num_tokens": 10168367.0, + "step": 380 + }, + { + "entropy": 0.7331193052232265, + "epoch": 0.6943362634917103, + "grad_norm": 1.4296875, + "learning_rate": 1.9340740740740743e-05, + "loss": 0.8366, + "mean_token_accuracy": 0.8001827634871006, + "num_tokens": 10439039.0, + "step": 390 + }, + { + "entropy": 0.7496613372117281, + "epoch": 0.7121397574273951, + "grad_norm": 1.15625, + "learning_rate": 1.926666666666667e-05, + "loss": 0.8517, + "mean_token_accuracy": 0.7967161379754544, + "num_tokens": 10709549.0, + "step": 400 + }, + { + "epoch": 0.7121397574273951, + "eval_biology_entropy": 1.823248610496521, + "eval_biology_loss": 2.159681558609009, + "eval_biology_mean_token_accuracy": 0.5972214796543122, + "eval_biology_num_tokens": 10709549.0, + "eval_biology_runtime": 39.2024, + "eval_biology_samples_per_second": 12.754, + "eval_biology_steps_per_second": 3.189, + "step": 400 + }, + { + "epoch": 0.7121397574273951, + "eval_chemistry_entropy": 0.7672218294143677, + "eval_chemistry_loss": 0.8971230387687683, + "eval_chemistry_mean_token_accuracy": 0.7928389143943787, + "eval_chemistry_num_tokens": 10709549.0, + "eval_chemistry_runtime": 48.8296, + "eval_chemistry_samples_per_second": 10.24, + "eval_chemistry_steps_per_second": 2.56, + "step": 400 + }, + { + "epoch": 0.7121397574273951, + "eval_math_entropy": 0.7494234163761139, + "eval_math_loss": 1.0849554538726807, + "eval_math_mean_token_accuracy": 0.7666959943771362, + "eval_math_num_tokens": 10709549.0, + "eval_math_runtime": 50.2325, + "eval_math_samples_per_second": 9.954, + "eval_math_steps_per_second": 2.488, + "step": 400 + }, + { + "epoch": 0.7121397574273951, + "eval_physics_entropy": 0.7731240901947022, + "eval_physics_loss": 0.9018430709838867, + "eval_physics_mean_token_accuracy": 0.7909709796905517, + "eval_physics_num_tokens": 10709549.0, + "eval_physics_runtime": 57.7809, + "eval_physics_samples_per_second": 8.653, + "eval_physics_steps_per_second": 2.163, + "step": 400 + }, + { + "entropy": 0.7354462265968322, + "epoch": 0.72994325136308, + "grad_norm": 1.3125, + "learning_rate": 1.9192592592592593e-05, + "loss": 0.8411, + "mean_token_accuracy": 0.8003354988992214, + "num_tokens": 10976011.0, + "step": 410 + }, + { + "entropy": 0.7435122076421976, + "epoch": 0.7477467452987648, + "grad_norm": 1.15625, + "learning_rate": 1.911851851851852e-05, + "loss": 0.8438, + "mean_token_accuracy": 0.7971596572548151, + "num_tokens": 11245251.0, + "step": 420 + }, + { + "entropy": 0.759309609234333, + "epoch": 0.7655502392344498, + "grad_norm": 1.375, + "learning_rate": 1.9044444444444446e-05, + "loss": 0.8579, + "mean_token_accuracy": 0.792990094423294, + "num_tokens": 11518514.0, + "step": 430 + }, + { + "entropy": 0.7545185785740613, + "epoch": 0.7833537331701347, + "grad_norm": 1.3125, + "learning_rate": 1.8970370370370372e-05, + "loss": 0.8574, + "mean_token_accuracy": 0.794682926684618, + "num_tokens": 11787117.0, + "step": 440 + }, + { + "entropy": 0.751254404336214, + "epoch": 0.8011572271058195, + "grad_norm": 1.515625, + "learning_rate": 1.8896296296296295e-05, + "loss": 0.8495, + "mean_token_accuracy": 0.7972703959792853, + "num_tokens": 12050993.0, + "step": 450 + }, + { + "entropy": 0.7543802969157696, + "epoch": 0.8189607210415044, + "grad_norm": 1.28125, + "learning_rate": 1.8822222222222225e-05, + "loss": 0.8527, + "mean_token_accuracy": 0.7960003320127725, + "num_tokens": 12311688.0, + "step": 460 + }, + { + "entropy": 0.7186065271496773, + "epoch": 0.8367642149771892, + "grad_norm": 1.15625, + "learning_rate": 1.874814814814815e-05, + "loss": 0.8096, + "mean_token_accuracy": 0.8027401898056269, + "num_tokens": 12598408.0, + "step": 470 + }, + { + "entropy": 0.7257912319153548, + "epoch": 0.8545677089128741, + "grad_norm": 1.125, + "learning_rate": 1.8674074074074075e-05, + "loss": 0.8299, + "mean_token_accuracy": 0.8012042317539454, + "num_tokens": 12869642.0, + "step": 480 + }, + { + "entropy": 0.7173270899802446, + "epoch": 0.8723712028485591, + "grad_norm": 1.125, + "learning_rate": 1.86e-05, + "loss": 0.8146, + "mean_token_accuracy": 0.8032398864626884, + "num_tokens": 13139555.0, + "step": 490 + }, + { + "entropy": 0.7487573977559805, + "epoch": 0.8901746967842439, + "grad_norm": 1.2421875, + "learning_rate": 1.8525925925925928e-05, + "loss": 0.8562, + "mean_token_accuracy": 0.7955521304160357, + "num_tokens": 13396545.0, + "step": 500 + }, + { + "epoch": 0.8901746967842439, + "eval_biology_entropy": 1.8204833917617798, + "eval_biology_loss": 2.1014022827148438, + "eval_biology_mean_token_accuracy": 0.6051526765823364, + "eval_biology_num_tokens": 13396545.0, + "eval_biology_runtime": 39.1543, + "eval_biology_samples_per_second": 12.77, + "eval_biology_steps_per_second": 3.192, + "step": 500 + }, + { + "epoch": 0.8901746967842439, + "eval_chemistry_entropy": 0.7538458592891694, + "eval_chemistry_loss": 0.8699584007263184, + "eval_chemistry_mean_token_accuracy": 0.7975720787048339, + "eval_chemistry_num_tokens": 13396545.0, + "eval_chemistry_runtime": 48.8229, + "eval_chemistry_samples_per_second": 10.241, + "eval_chemistry_steps_per_second": 2.56, + "step": 500 + }, + { + "epoch": 0.8901746967842439, + "eval_math_entropy": 0.7398368701934814, + "eval_math_loss": 1.0758970975875854, + "eval_math_mean_token_accuracy": 0.7687290472984314, + "eval_math_num_tokens": 13396545.0, + "eval_math_runtime": 50.0916, + "eval_math_samples_per_second": 9.982, + "eval_math_steps_per_second": 2.495, + "step": 500 + }, + { + "epoch": 0.8901746967842439, + "eval_physics_entropy": 0.7607374501228332, + "eval_physics_loss": 0.8848790526390076, + "eval_physics_mean_token_accuracy": 0.7937801394462586, + "eval_physics_num_tokens": 13396545.0, + "eval_physics_runtime": 57.6599, + "eval_physics_samples_per_second": 8.672, + "eval_physics_steps_per_second": 2.168, + "step": 500 + }, + { + "entropy": 0.7129743231460453, + "epoch": 0.9079781907199288, + "grad_norm": 1.1640625, + "learning_rate": 1.8451851851851855e-05, + "loss": 0.8109, + "mean_token_accuracy": 0.8060642663389445, + "num_tokens": 13667857.0, + "step": 510 + }, + { + "entropy": 0.7393450669944286, + "epoch": 0.9257816846556136, + "grad_norm": 1.1484375, + "learning_rate": 1.8377777777777778e-05, + "loss": 0.8325, + "mean_token_accuracy": 0.7997861213982105, + "num_tokens": 13932771.0, + "step": 520 + }, + { + "entropy": 0.7191115407273173, + "epoch": 0.9435851785912985, + "grad_norm": 1.1640625, + "learning_rate": 1.8303703703703704e-05, + "loss": 0.8233, + "mean_token_accuracy": 0.8034567594528198, + "num_tokens": 14194632.0, + "step": 530 + }, + { + "entropy": 0.7434917811304331, + "epoch": 0.9613886725269835, + "grad_norm": 1.1484375, + "learning_rate": 1.822962962962963e-05, + "loss": 0.8435, + "mean_token_accuracy": 0.7969928599894047, + "num_tokens": 14457650.0, + "step": 540 + }, + { + "entropy": 0.7100320702418685, + "epoch": 0.9791921664626683, + "grad_norm": 1.125, + "learning_rate": 1.8155555555555557e-05, + "loss": 0.8149, + "mean_token_accuracy": 0.803739969804883, + "num_tokens": 14724967.0, + "step": 550 + }, + { + "entropy": 0.737933493219316, + "epoch": 0.9969956603983532, + "grad_norm": 1.1875, + "learning_rate": 1.8081481481481484e-05, + "loss": 0.8421, + "mean_token_accuracy": 0.8001800544559956, + "num_tokens": 14993272.0, + "step": 560 + }, + { + "entropy": 0.6655864379098339, + "epoch": 1.014242795148548, + "grad_norm": 1.1171875, + "learning_rate": 1.800740740740741e-05, + "loss": 0.7668, + "mean_token_accuracy": 0.814733551779101, + "num_tokens": 15254936.0, + "step": 570 + }, + { + "entropy": 0.6937626471742988, + "epoch": 1.0320462890842328, + "grad_norm": 1.2578125, + "learning_rate": 1.7933333333333333e-05, + "loss": 0.7942, + "mean_token_accuracy": 0.806296594068408, + "num_tokens": 15515007.0, + "step": 580 + }, + { + "entropy": 0.6596947606652975, + "epoch": 1.0498497830199176, + "grad_norm": 1.2890625, + "learning_rate": 1.785925925925926e-05, + "loss": 0.7623, + "mean_token_accuracy": 0.8152227636426688, + "num_tokens": 15794203.0, + "step": 590 + }, + { + "entropy": 0.6868148937821388, + "epoch": 1.0676532769556026, + "grad_norm": 1.1640625, + "learning_rate": 1.7785185185185186e-05, + "loss": 0.7827, + "mean_token_accuracy": 0.8101296100765467, + "num_tokens": 16063664.0, + "step": 600 + }, + { + "epoch": 1.0676532769556026, + "eval_biology_entropy": 1.6867316131591796, + "eval_biology_loss": 2.077732801437378, + "eval_biology_mean_token_accuracy": 0.6078621828556061, + "eval_biology_num_tokens": 16063664.0, + "eval_biology_runtime": 39.1079, + "eval_biology_samples_per_second": 12.785, + "eval_biology_steps_per_second": 3.196, + "step": 600 + }, + { + "epoch": 1.0676532769556026, + "eval_chemistry_entropy": 0.6882545161247253, + "eval_chemistry_loss": 0.8549924492835999, + "eval_chemistry_mean_token_accuracy": 0.8009334373474121, + "eval_chemistry_num_tokens": 16063664.0, + "eval_chemistry_runtime": 48.732, + "eval_chemistry_samples_per_second": 10.26, + "eval_chemistry_steps_per_second": 2.565, + "step": 600 + }, + { + "epoch": 1.0676532769556026, + "eval_math_entropy": 0.6883157949447631, + "eval_math_loss": 1.075770616531372, + "eval_math_mean_token_accuracy": 0.7705092186927796, + "eval_math_num_tokens": 16063664.0, + "eval_math_runtime": 50.1345, + "eval_math_samples_per_second": 9.973, + "eval_math_steps_per_second": 2.493, + "step": 600 + }, + { + "epoch": 1.0676532769556026, + "eval_physics_entropy": 0.7055722939968109, + "eval_physics_loss": 0.8771266937255859, + "eval_physics_mean_token_accuracy": 0.7955313258171082, + "eval_physics_num_tokens": 16063664.0, + "eval_physics_runtime": 57.6658, + "eval_physics_samples_per_second": 8.671, + "eval_physics_steps_per_second": 2.168, + "step": 600 + }, + { + "entropy": 0.6597639428451657, + "epoch": 1.0854567708912874, + "grad_norm": 1.203125, + "learning_rate": 1.7711111111111113e-05, + "loss": 0.7655, + "mean_token_accuracy": 0.8151189126074314, + "num_tokens": 16338303.0, + "step": 610 + }, + { + "entropy": 0.6991181125864386, + "epoch": 1.1032602648269723, + "grad_norm": 1.2734375, + "learning_rate": 1.763703703703704e-05, + "loss": 0.7934, + "mean_token_accuracy": 0.8071035247296094, + "num_tokens": 16606302.0, + "step": 620 + }, + { + "entropy": 0.6774254593998194, + "epoch": 1.121063758762657, + "grad_norm": 1.28125, + "learning_rate": 1.7562962962962962e-05, + "loss": 0.7706, + "mean_token_accuracy": 0.8098537240177393, + "num_tokens": 16877071.0, + "step": 630 + }, + { + "entropy": 0.6678685389459134, + "epoch": 1.138867252698342, + "grad_norm": 1.203125, + "learning_rate": 1.7488888888888892e-05, + "loss": 0.7714, + "mean_token_accuracy": 0.8127609983086586, + "num_tokens": 17145775.0, + "step": 640 + }, + { + "entropy": 0.6851805068552494, + "epoch": 1.156670746634027, + "grad_norm": 1.2890625, + "learning_rate": 1.7414814814814815e-05, + "loss": 0.777, + "mean_token_accuracy": 0.8110932789742946, + "num_tokens": 17410775.0, + "step": 650 + }, + { + "entropy": 0.7042925998568534, + "epoch": 1.1744742405697117, + "grad_norm": 1.375, + "learning_rate": 1.7340740740740742e-05, + "loss": 0.7979, + "mean_token_accuracy": 0.8038393501192331, + "num_tokens": 17677791.0, + "step": 660 + }, + { + "entropy": 0.6810087833553553, + "epoch": 1.1922777345053968, + "grad_norm": 1.1484375, + "learning_rate": 1.726666666666667e-05, + "loss": 0.7688, + "mean_token_accuracy": 0.8104576140642166, + "num_tokens": 17949441.0, + "step": 670 + }, + { + "entropy": 0.662551742233336, + "epoch": 1.2100812284410816, + "grad_norm": 1.2109375, + "learning_rate": 1.7192592592592595e-05, + "loss": 0.7653, + "mean_token_accuracy": 0.8126230895519256, + "num_tokens": 18217742.0, + "step": 680 + }, + { + "entropy": 0.6803617887198925, + "epoch": 1.2278847223767664, + "grad_norm": 1.2421875, + "learning_rate": 1.711851851851852e-05, + "loss": 0.7771, + "mean_token_accuracy": 0.8095329128205776, + "num_tokens": 18487769.0, + "step": 690 + }, + { + "entropy": 0.6707866100594402, + "epoch": 1.2456882163124514, + "grad_norm": 1.34375, + "learning_rate": 1.7044444444444445e-05, + "loss": 0.7687, + "mean_token_accuracy": 0.8113945126533508, + "num_tokens": 18760296.0, + "step": 700 + }, + { + "epoch": 1.2456882163124514, + "eval_biology_entropy": 1.6589141931533813, + "eval_biology_loss": 2.081825017929077, + "eval_biology_mean_token_accuracy": 0.608892562866211, + "eval_biology_num_tokens": 18760296.0, + "eval_biology_runtime": 39.8083, + "eval_biology_samples_per_second": 12.56, + "eval_biology_steps_per_second": 3.14, + "step": 700 + }, + { + "epoch": 1.2456882163124514, + "eval_chemistry_entropy": 0.6833467047214508, + "eval_chemistry_loss": 0.8464663028717041, + "eval_chemistry_mean_token_accuracy": 0.80278346824646, + "eval_chemistry_num_tokens": 18760296.0, + "eval_chemistry_runtime": 49.1467, + "eval_chemistry_samples_per_second": 10.174, + "eval_chemistry_steps_per_second": 2.543, + "step": 700 + }, + { + "epoch": 1.2456882163124514, + "eval_math_entropy": 0.6947779231071473, + "eval_math_loss": 1.0786137580871582, + "eval_math_mean_token_accuracy": 0.7709644784927369, + "eval_math_num_tokens": 18760296.0, + "eval_math_runtime": 52.5755, + "eval_math_samples_per_second": 9.51, + "eval_math_steps_per_second": 2.378, + "step": 700 + }, + { + "epoch": 1.2456882163124514, + "eval_physics_entropy": 0.704109364271164, + "eval_physics_loss": 0.8724388480186462, + "eval_physics_mean_token_accuracy": 0.796642897605896, + "eval_physics_num_tokens": 18760296.0, + "eval_physics_runtime": 61.3658, + "eval_physics_samples_per_second": 8.148, + "eval_physics_steps_per_second": 2.037, + "step": 700 + }, + { + "entropy": 0.6705936685204505, + "epoch": 1.2634917102481362, + "grad_norm": 1.140625, + "learning_rate": 1.697037037037037e-05, + "loss": 0.7618, + "mean_token_accuracy": 0.8144830856472254, + "num_tokens": 19035733.0, + "step": 710 + }, + { + "entropy": 0.6870324395596981, + "epoch": 1.281295204183821, + "grad_norm": 1.1953125, + "learning_rate": 1.6896296296296298e-05, + "loss": 0.7849, + "mean_token_accuracy": 0.807737946882844, + "num_tokens": 19311432.0, + "step": 720 + }, + { + "entropy": 0.6551721351221204, + "epoch": 1.299098698119506, + "grad_norm": 1.0859375, + "learning_rate": 1.6822222222222224e-05, + "loss": 0.7538, + "mean_token_accuracy": 0.8164622981101275, + "num_tokens": 19593016.0, + "step": 730 + }, + { + "entropy": 0.6645344085991383, + "epoch": 1.3169021920551909, + "grad_norm": 1.1796875, + "learning_rate": 1.6748148148148147e-05, + "loss": 0.759, + "mean_token_accuracy": 0.8134492743760348, + "num_tokens": 19860476.0, + "step": 740 + }, + { + "entropy": 0.6684485200792551, + "epoch": 1.3347056859908757, + "grad_norm": 1.3515625, + "learning_rate": 1.6674074074074077e-05, + "loss": 0.7632, + "mean_token_accuracy": 0.8128790140151978, + "num_tokens": 20124436.0, + "step": 750 + }, + { + "entropy": 0.6736280497163534, + "epoch": 1.3525091799265607, + "grad_norm": 1.25, + "learning_rate": 1.66e-05, + "loss": 0.7771, + "mean_token_accuracy": 0.813029420748353, + "num_tokens": 20388027.0, + "step": 760 + }, + { + "entropy": 0.6706711670383811, + "epoch": 1.3703126738622455, + "grad_norm": 1.265625, + "learning_rate": 1.6525925925925927e-05, + "loss": 0.7697, + "mean_token_accuracy": 0.8115393359214067, + "num_tokens": 20656076.0, + "step": 770 + }, + { + "entropy": 0.6996258163824678, + "epoch": 1.3881161677979303, + "grad_norm": 1.2578125, + "learning_rate": 1.6451851851851853e-05, + "loss": 0.8077, + "mean_token_accuracy": 0.8058754079043865, + "num_tokens": 20911633.0, + "step": 780 + }, + { + "entropy": 0.6905225297436118, + "epoch": 1.4059196617336152, + "grad_norm": 1.171875, + "learning_rate": 1.637777777777778e-05, + "loss": 0.7955, + "mean_token_accuracy": 0.8070559553802014, + "num_tokens": 21176227.0, + "step": 790 + }, + { + "entropy": 0.6718146029859782, + "epoch": 1.4237231556693, + "grad_norm": 1.2890625, + "learning_rate": 1.6303703703703706e-05, + "loss": 0.7716, + "mean_token_accuracy": 0.8126390781253576, + "num_tokens": 21445242.0, + "step": 800 + }, + { + "epoch": 1.4237231556693, + "eval_biology_entropy": 1.6255396161079407, + "eval_biology_loss": 2.0511693954467773, + "eval_biology_mean_token_accuracy": 0.6119143316745758, + "eval_biology_num_tokens": 21445242.0, + "eval_biology_runtime": 42.4445, + "eval_biology_samples_per_second": 11.78, + "eval_biology_steps_per_second": 2.945, + "step": 800 + }, + { + "epoch": 1.4237231556693, + "eval_chemistry_entropy": 0.6763756844997406, + "eval_chemistry_loss": 0.8368268609046936, + "eval_chemistry_mean_token_accuracy": 0.8050474629402161, + "eval_chemistry_num_tokens": 21445242.0, + "eval_chemistry_runtime": 51.5104, + "eval_chemistry_samples_per_second": 9.707, + "eval_chemistry_steps_per_second": 2.427, + "step": 800 + }, + { + "epoch": 1.4237231556693, + "eval_math_entropy": 0.6853804645538331, + "eval_math_loss": 1.0698176622390747, + "eval_math_mean_token_accuracy": 0.7714386968612671, + "eval_math_num_tokens": 21445242.0, + "eval_math_runtime": 54.9388, + "eval_math_samples_per_second": 9.101, + "eval_math_steps_per_second": 2.275, + "step": 800 + }, + { + "epoch": 1.4237231556693, + "eval_physics_entropy": 0.6991692934036254, + "eval_physics_loss": 0.8673914074897766, + "eval_physics_mean_token_accuracy": 0.7979785161018371, + "eval_physics_num_tokens": 21445242.0, + "eval_physics_runtime": 60.9527, + "eval_physics_samples_per_second": 8.203, + "eval_physics_steps_per_second": 2.051, + "step": 800 + }, + { + "entropy": 0.6695453573018313, + "epoch": 1.441526649604985, + "grad_norm": 1.3125, + "learning_rate": 1.622962962962963e-05, + "loss": 0.77, + "mean_token_accuracy": 0.8109262142330408, + "num_tokens": 21718455.0, + "step": 810 + }, + { + "entropy": 0.6753543723374605, + "epoch": 1.4593301435406698, + "grad_norm": 1.2734375, + "learning_rate": 1.6155555555555556e-05, + "loss": 0.7782, + "mean_token_accuracy": 0.8130301378667355, + "num_tokens": 21974076.0, + "step": 820 + }, + { + "entropy": 0.6758085643872619, + "epoch": 1.4771336374763546, + "grad_norm": 1.2265625, + "learning_rate": 1.6081481481481482e-05, + "loss": 0.7768, + "mean_token_accuracy": 0.8110865034162998, + "num_tokens": 22241599.0, + "step": 830 + }, + { + "entropy": 0.6868746858090162, + "epoch": 1.4949371314120397, + "grad_norm": 1.34375, + "learning_rate": 1.600740740740741e-05, + "loss": 0.7818, + "mean_token_accuracy": 0.8090599600225687, + "num_tokens": 22510725.0, + "step": 840 + }, + { + "entropy": 0.6554480012506246, + "epoch": 1.5127406253477245, + "grad_norm": 1.1796875, + "learning_rate": 1.5933333333333336e-05, + "loss": 0.766, + "mean_token_accuracy": 0.8149232871830463, + "num_tokens": 22778514.0, + "step": 850 + }, + { + "entropy": 0.6574800366535782, + "epoch": 1.5305441192834093, + "grad_norm": 1.2734375, + "learning_rate": 1.5859259259259262e-05, + "loss": 0.7517, + "mean_token_accuracy": 0.8147028144448996, + "num_tokens": 23051230.0, + "step": 860 + }, + { + "entropy": 0.6707898294553161, + "epoch": 1.5483476132190943, + "grad_norm": 1.21875, + "learning_rate": 1.5785185185185185e-05, + "loss": 0.776, + "mean_token_accuracy": 0.8121768556535244, + "num_tokens": 23306327.0, + "step": 870 + }, + { + "entropy": 0.6759747974574566, + "epoch": 1.5661511071547791, + "grad_norm": 1.234375, + "learning_rate": 1.571111111111111e-05, + "loss": 0.7717, + "mean_token_accuracy": 0.810706127434969, + "num_tokens": 23573127.0, + "step": 880 + }, + { + "entropy": 0.6696014609187841, + "epoch": 1.583954601090464, + "grad_norm": 1.1953125, + "learning_rate": 1.5637037037037038e-05, + "loss": 0.7635, + "mean_token_accuracy": 0.8123903065919876, + "num_tokens": 23841914.0, + "step": 890 + }, + { + "entropy": 0.6680066948756576, + "epoch": 1.601758095026149, + "grad_norm": 1.28125, + "learning_rate": 1.5562962962962965e-05, + "loss": 0.7763, + "mean_token_accuracy": 0.8117439385503531, + "num_tokens": 24105114.0, + "step": 900 + }, + { + "epoch": 1.601758095026149, + "eval_biology_entropy": 1.6559728245735168, + "eval_biology_loss": 2.0387370586395264, + "eval_biology_mean_token_accuracy": 0.6136598820686341, + "eval_biology_num_tokens": 24105114.0, + "eval_biology_runtime": 41.4479, + "eval_biology_samples_per_second": 12.063, + "eval_biology_steps_per_second": 3.016, + "step": 900 + }, + { + "epoch": 1.601758095026149, + "eval_chemistry_entropy": 0.683923549413681, + "eval_chemistry_loss": 0.8298754692077637, + "eval_chemistry_mean_token_accuracy": 0.8061454033851624, + "eval_chemistry_num_tokens": 24105114.0, + "eval_chemistry_runtime": 50.8984, + "eval_chemistry_samples_per_second": 9.823, + "eval_chemistry_steps_per_second": 2.456, + "step": 900 + }, + { + "epoch": 1.601758095026149, + "eval_math_entropy": 0.6932375190258027, + "eval_math_loss": 1.06475830078125, + "eval_math_mean_token_accuracy": 0.7720690259933471, + "eval_math_num_tokens": 24105114.0, + "eval_math_runtime": 52.5544, + "eval_math_samples_per_second": 9.514, + "eval_math_steps_per_second": 2.378, + "step": 900 + }, + { + "epoch": 1.601758095026149, + "eval_physics_entropy": 0.7051843535900116, + "eval_physics_loss": 0.8616264462471008, + "eval_physics_mean_token_accuracy": 0.7987073979377747, + "eval_physics_num_tokens": 24105114.0, + "eval_physics_runtime": 59.8508, + "eval_physics_samples_per_second": 8.354, + "eval_physics_steps_per_second": 2.089, + "step": 900 + }, + { + "entropy": 0.6430077504366636, + "epoch": 1.6195615889618338, + "grad_norm": 1.1640625, + "learning_rate": 1.548888888888889e-05, + "loss": 0.743, + "mean_token_accuracy": 0.8184590026736259, + "num_tokens": 24384008.0, + "step": 910 + }, + { + "entropy": 0.6775279760360717, + "epoch": 1.6373650828975186, + "grad_norm": 1.3125, + "learning_rate": 1.5414814814814814e-05, + "loss": 0.7704, + "mean_token_accuracy": 0.8114439699798822, + "num_tokens": 24638205.0, + "step": 920 + }, + { + "entropy": 0.6244934551417828, + "epoch": 1.6551685768332036, + "grad_norm": 1.1953125, + "learning_rate": 1.5340740740740744e-05, + "loss": 0.7253, + "mean_token_accuracy": 0.8222480688244105, + "num_tokens": 24910104.0, + "step": 930 + }, + { + "entropy": 0.6539383618161082, + "epoch": 1.6729720707688884, + "grad_norm": 1.2578125, + "learning_rate": 1.5266666666666667e-05, + "loss": 0.7471, + "mean_token_accuracy": 0.8160006206482648, + "num_tokens": 25181306.0, + "step": 940 + }, + { + "entropy": 0.6607510834932327, + "epoch": 1.6907755647045732, + "grad_norm": 1.3203125, + "learning_rate": 1.5192592592592594e-05, + "loss": 0.7589, + "mean_token_accuracy": 0.8142590031027794, + "num_tokens": 25449138.0, + "step": 950 + }, + { + "entropy": 0.6612858925014734, + "epoch": 1.7085790586402583, + "grad_norm": 1.203125, + "learning_rate": 1.5118518518518519e-05, + "loss": 0.766, + "mean_token_accuracy": 0.8137270450592041, + "num_tokens": 25709030.0, + "step": 960 + }, + { + "entropy": 0.6701876953244209, + "epoch": 1.7263825525759429, + "grad_norm": 1.2734375, + "learning_rate": 1.5044444444444445e-05, + "loss": 0.7702, + "mean_token_accuracy": 0.8120660580694675, + "num_tokens": 25972807.0, + "step": 970 + }, + { + "entropy": 0.6566708590835333, + "epoch": 1.744186046511628, + "grad_norm": 1.1640625, + "learning_rate": 1.497037037037037e-05, + "loss": 0.748, + "mean_token_accuracy": 0.8170768674463034, + "num_tokens": 26244955.0, + "step": 980 + }, + { + "entropy": 0.624184824153781, + "epoch": 1.761989540447313, + "grad_norm": 1.1796875, + "learning_rate": 1.4896296296296298e-05, + "loss": 0.7284, + "mean_token_accuracy": 0.8223610106855631, + "num_tokens": 26515548.0, + "step": 990 + }, + { + "entropy": 0.6476762825623155, + "epoch": 1.7797930343829975, + "grad_norm": 1.21875, + "learning_rate": 1.4822222222222225e-05, + "loss": 0.7422, + "mean_token_accuracy": 0.8172390758991241, + "num_tokens": 26782376.0, + "step": 1000 + }, + { + "epoch": 1.7797930343829975, + "eval_biology_entropy": 1.6157320532798767, + "eval_biology_loss": 2.0338852405548096, + "eval_biology_mean_token_accuracy": 0.6143801593780518, + "eval_biology_num_tokens": 26782376.0, + "eval_biology_runtime": 41.5849, + "eval_biology_samples_per_second": 12.024, + "eval_biology_steps_per_second": 3.006, + "step": 1000 + }, + { + "epoch": 1.7797930343829975, + "eval_chemistry_entropy": 0.6670743160247803, + "eval_chemistry_loss": 0.8244208097457886, + "eval_chemistry_mean_token_accuracy": 0.8077364330291747, + "eval_chemistry_num_tokens": 26782376.0, + "eval_chemistry_runtime": 53.0274, + "eval_chemistry_samples_per_second": 9.429, + "eval_chemistry_steps_per_second": 2.357, + "step": 1000 + }, + { + "epoch": 1.7797930343829975, + "eval_math_entropy": 0.6760952143669129, + "eval_math_loss": 1.0642753839492798, + "eval_math_mean_token_accuracy": 0.7728619589805603, + "eval_math_num_tokens": 26782376.0, + "eval_math_runtime": 53.8997, + "eval_math_samples_per_second": 9.276, + "eval_math_steps_per_second": 2.319, + "step": 1000 + }, + { + "epoch": 1.7797930343829975, + "eval_physics_entropy": 0.6914605097770691, + "eval_physics_loss": 0.8597292304039001, + "eval_physics_mean_token_accuracy": 0.7992245469093323, + "eval_physics_num_tokens": 26782376.0, + "eval_physics_runtime": 62.8355, + "eval_physics_samples_per_second": 7.957, + "eval_physics_steps_per_second": 1.989, + "step": 1000 + } + ], + "logging_steps": 10, + "max_steps": 3000, + "num_input_tokens_seen": 0, + "num_train_epochs": 6, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 2.0811511522503905e+18, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/checkpoint-1000/training_args.bin b/checkpoint-1000/training_args.bin new file mode 100644 index 0000000..79cad08 --- /dev/null +++ b/checkpoint-1000/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dfc23fa1b9be77c605cba9733debba5f60170f0eadaa2a8866b5aaf4b0e65756 +size 6417 diff --git a/checkpoint-1500/chat_template.jinja b/checkpoint-1500/chat_template.jinja new file mode 100644 index 0000000..923ec25 --- /dev/null +++ b/checkpoint-1500/chat_template.jinja @@ -0,0 +1,4 @@ +{{ bos_token }}{% if messages[0]['role'] == 'system' %}{{ raise_exception('System role not supported') }}{% endif %}{% for message in messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if (message['role'] == 'assistant') %}{% set role = 'model' %}{% else %}{% set role = message['role'] %}{% endif %}{{ '' + role + ' +' + message['content'] | trim + ' +' }}{% endfor %}{% if add_generation_prompt %}{{'model +'}}{% endif %} \ No newline at end of file diff --git a/checkpoint-1500/config.json b/checkpoint-1500/config.json new file mode 100644 index 0000000..29e6cbe --- /dev/null +++ b/checkpoint-1500/config.json @@ -0,0 +1,77 @@ +{ + "architectures": [ + "Gemma2ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "attn_logit_softcapping": 50.0, + "bos_token_id": 2, + "cache_implementation": "hybrid", + "dtype": "bfloat16", + "eos_token_id": 1, + "final_logit_softcapping": 30.0, + "head_dim": 256, + "hidden_act": "gelu_pytorch_tanh", + "hidden_activation": "gelu_pytorch_tanh", + "hidden_size": 3584, + "initializer_range": 0.02, + "intermediate_size": 14336, + "layer_types": [ + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention" + ], + "max_position_embeddings": 8192, + "model_type": "gemma2", + "num_attention_heads": 16, + "num_hidden_layers": 42, + "num_key_value_heads": 8, + "pad_token_id": 0, + "query_pre_attn_scalar": 256, + "rms_norm_eps": 1e-06, + "rope_theta": 10000.0, + "sliding_window": 4096, + "sliding_window_size": 4096, + "transformers_version": "4.56.1", + "use_cache": true, + "vocab_size": 256000 +} diff --git a/checkpoint-1500/generation_config.json b/checkpoint-1500/generation_config.json new file mode 100644 index 0000000..f2c34f2 --- /dev/null +++ b/checkpoint-1500/generation_config.json @@ -0,0 +1,9 @@ +{ + "_from_model_config": true, + "bos_token_id": 2, + "eos_token_id": [ + 1 + ], + "pad_token_id": 0, + "transformers_version": "4.56.1" +} diff --git a/checkpoint-1500/model-00001-of-00004.safetensors b/checkpoint-1500/model-00001-of-00004.safetensors new file mode 100644 index 0000000..4705c7e --- /dev/null +++ b/checkpoint-1500/model-00001-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a0d4eb4fcbddfe01b0dc7a58386a3980826ededaf0ecd67d1ef38995b89615ba +size 4903351912 diff --git a/checkpoint-1500/model-00002-of-00004.safetensors b/checkpoint-1500/model-00002-of-00004.safetensors new file mode 100644 index 0000000..744b59b --- /dev/null +++ b/checkpoint-1500/model-00002-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:712a589c333f49f7b6f3870838a9e0ab4b3c0260abd5130e8c54316e6c99640b +size 4947570872 diff --git a/checkpoint-1500/model-00003-of-00004.safetensors b/checkpoint-1500/model-00003-of-00004.safetensors new file mode 100644 index 0000000..672d797 --- /dev/null +++ b/checkpoint-1500/model-00003-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:70ff9df8f1a64d5de6de2b9ce02aeb9a489cea4fc977e7b10fe5ace62fa4ea00 +size 4962221464 diff --git a/checkpoint-1500/model-00004-of-00004.safetensors b/checkpoint-1500/model-00004-of-00004.safetensors new file mode 100644 index 0000000..9a05890 --- /dev/null +++ b/checkpoint-1500/model-00004-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:79cfca990236b0c57f82c75f5140c60818fed183c4647be2eef90ac93dff2b62 +size 3670322200 diff --git a/checkpoint-1500/model.safetensors.index.json b/checkpoint-1500/model.safetensors.index.json new file mode 100644 index 0000000..3b569c5 --- /dev/null +++ b/checkpoint-1500/model.safetensors.index.json @@ -0,0 +1,472 @@ +{ + "metadata": { + "total_parameters": 9241705984, + "total_size": 18483411968 + }, + "weight_map": { + "model.embed_tokens.weight": "model-00001-of-00004.safetensors", + "model.layers.0.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.0.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.0.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.1.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.1.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.10.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.10.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.10.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.10.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.11.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.11.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.11.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.12.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.12.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.12.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.13.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.13.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.13.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.14.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.14.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.14.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.15.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.15.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.15.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.16.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.16.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.16.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.17.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.17.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.17.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.18.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.18.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.18.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.19.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.19.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.19.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.2.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.2.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.2.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.20.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.20.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.20.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.20.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.20.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.20.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.20.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.21.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.21.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.21.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.21.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.21.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.22.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.22.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.22.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.22.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.23.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.23.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.23.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.24.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.24.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.24.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.25.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.25.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.25.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.26.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.26.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.26.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.27.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.27.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.27.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.28.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.28.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.28.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.29.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.29.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.29.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.3.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.3.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.3.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.30.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.30.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.30.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.30.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.30.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.31.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.31.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.31.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.32.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.32.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.32.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.32.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.32.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.32.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.32.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.32.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.32.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.32.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.32.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.33.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.33.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.33.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.33.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.33.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.33.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.33.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.33.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.33.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.33.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.33.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.34.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.34.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.34.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.34.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.34.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.34.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.34.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.34.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.34.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.34.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.34.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.35.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.35.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.35.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.35.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.35.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.35.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.35.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.35.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.35.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.35.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.35.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.36.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.36.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.36.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.36.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.36.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.36.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.36.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.36.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.36.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.36.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.36.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.37.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.37.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.37.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.37.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.37.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.37.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.37.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.37.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.37.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.37.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.37.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.38.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.38.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.38.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.38.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.38.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.38.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.38.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.38.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.38.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.38.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.38.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.39.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.39.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.39.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.39.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.39.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.39.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.39.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.39.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.39.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.39.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.39.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.4.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.4.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.4.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.4.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.40.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.40.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.40.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.40.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.40.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.40.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.40.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.40.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.40.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.40.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.40.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.41.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.41.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.41.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.41.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.41.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.41.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.41.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.41.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.41.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.41.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.41.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.5.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.5.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.5.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.5.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.6.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.6.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.6.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.7.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.7.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.7.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.8.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.8.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.8.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.8.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.8.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.8.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.8.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.8.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.8.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.8.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.8.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.9.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.9.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.9.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.9.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.norm.weight": "model-00004-of-00004.safetensors" + } +} diff --git a/checkpoint-1500/optimizer.pt b/checkpoint-1500/optimizer.pt new file mode 100644 index 0000000..8aa61f3 --- /dev/null +++ b/checkpoint-1500/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6ff73e98a007dcd7ddf58542fafeb2e3b730e62cd4f09f12b60d5cf97443fe80 +size 1585596266 diff --git a/checkpoint-1500/rng_state.pth b/checkpoint-1500/rng_state.pth new file mode 100644 index 0000000..4e65653 --- /dev/null +++ b/checkpoint-1500/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9e18baad484d226f8ed089a4849ca70f9922b4c7d45ed627c4dfd43cb9bc6002 +size 14645 diff --git a/checkpoint-1500/scheduler.pt b/checkpoint-1500/scheduler.pt new file mode 100644 index 0000000..a8007b2 --- /dev/null +++ b/checkpoint-1500/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0c06b4f8f3b3f6004cf9db5f25ea010ea45540a89813f901db1e65629d2290fe +size 1465 diff --git a/checkpoint-1500/special_tokens_map.json b/checkpoint-1500/special_tokens_map.json new file mode 100644 index 0000000..8d6368f --- /dev/null +++ b/checkpoint-1500/special_tokens_map.json @@ -0,0 +1,34 @@ +{ + "additional_special_tokens": [ + "", + "" + ], + "bos_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "unk_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + } +} diff --git a/checkpoint-1500/tokenizer.json b/checkpoint-1500/tokenizer.json new file mode 100644 index 0000000..a4a305d --- /dev/null +++ b/checkpoint-1500/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5f7eee611703c5ce5d1eee32d9cdcfe465647b8aff0c1dfb3bed7ad7dbb05060 +size 34362873 diff --git a/checkpoint-1500/tokenizer.model b/checkpoint-1500/tokenizer.model new file mode 100644 index 0000000..796efe9 --- /dev/null +++ b/checkpoint-1500/tokenizer.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:61a7b147390c64585d6c3543dd6fc636906c9af3865a5548f27f31aee1d4c8e2 +size 4241003 diff --git a/checkpoint-1500/tokenizer_config.json b/checkpoint-1500/tokenizer_config.json new file mode 100644 index 0000000..3ade6be --- /dev/null +++ b/checkpoint-1500/tokenizer_config.json @@ -0,0 +1,2013 @@ +{ + "add_bos_token": true, + "add_eos_token": false, + "added_tokens_decoder": { + "0": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "1": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "2": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "3": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "4": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "5": { + "content": "<2mass>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "6": { + "content": "[@BOS@]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "7": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "8": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "9": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "10": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "11": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "12": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "13": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "14": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "15": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "16": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "17": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "18": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "19": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "20": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "21": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "22": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "23": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "24": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "25": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "26": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "27": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "28": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "29": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "30": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "31": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "32": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "33": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "34": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "35": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "36": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "37": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "38": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "39": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "40": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "41": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "42": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "43": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "44": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "45": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "46": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "47": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "48": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "49": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "50": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "51": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "52": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "53": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "54": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "55": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "56": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "57": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "58": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "59": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "60": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "61": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "62": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "63": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "64": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "65": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "66": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "67": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "68": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "69": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "70": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "71": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "72": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "73": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "74": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "75": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "76": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "77": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "78": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "79": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "80": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "81": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "82": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "83": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "84": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "85": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "86": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "87": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "88": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "89": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "90": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "91": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "92": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "93": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "94": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "95": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "96": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "97": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "98": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "99": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "100": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "101": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "102": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "103": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "104": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "105": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "106": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "107": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "108": { + "content": "\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "109": { + "content": "\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "110": { + "content": "\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "111": { + "content": "\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "112": { + "content": "\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "113": { + "content": "\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "114": { + "content": "\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "115": { + "content": "\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "116": { + "content": "\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "117": { + "content": "\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "118": { + "content": "\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "119": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "120": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "121": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "122": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "123": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "124": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "125": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "126": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "127": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "128": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "129": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "130": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "131": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "132": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "133": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "134": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "135": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "136": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "137": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "138": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "139": { + "content": "▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "140": { + "content": "▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "141": { + "content": "▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "142": { + "content": "▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "143": { + "content": "▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "144": { + "content": "▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "145": { + "content": "▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "146": { + "content": "▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "147": { + "content": "▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "148": { + "content": "▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "149": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "150": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "152": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "153": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "154": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "155": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "156": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "157": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "158": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "159": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "160": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "161": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "162": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "163": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "164": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "165": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "166": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "167": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "168": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "169": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "170": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "172": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "173": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "174": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "175": { + "content": "
", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "171": { + "content": "
", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "176": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "177": { + "content": "
", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "178": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "179": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "180": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "181": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "182": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "183": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "184": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "185": { + "content": "

", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "186": { + "content": "

", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "187": { + "content": "

", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "188": { + "content": "

", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "189": { + "content": "

", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "190": { + "content": "
", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "191": { + "content": "
", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "192": { + "content": "
", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "193": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "194": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "195": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "196": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "197": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "198": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "199": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "200": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "201": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "202": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "203": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "204": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "205": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "206": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "207": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "208": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "209": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "210": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "211": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "212": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "213": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "214": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "215": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "216": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255968": { + "content": "[toxicity=0]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255969": { + "content": "\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255970": { + "content": "\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255971": { + "content": "\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255972": { + "content": "\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255973": { + "content": "\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255974": { + "content": "\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255975": { + "content": "\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255976": { + "content": "\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255977": { + "content": "\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255978": { + "content": "\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255979": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255980": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255981": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255982": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255983": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255984": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255985": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255986": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255987": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255988": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255989": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255990": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255991": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255992": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255993": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255994": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255995": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255996": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255997": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255998": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255999": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + } + }, + "additional_special_tokens": [ + "", + "" + ], + "bos_token": "", + "clean_up_tokenization_spaces": false, + "eos_token": "", + "extra_special_tokens": {}, + "model_max_length": 1000000000000000019884624838656, + "pad_token": "", + "sp_model_kwargs": {}, + "spaces_between_special_tokens": false, + "tokenizer_class": "GemmaTokenizer", + "unk_token": "", + "use_default_system_prompt": false +} diff --git a/checkpoint-1500/trainer_state.json b/checkpoint-1500/trainer_state.json new file mode 100644 index 0000000..2dd4439 --- /dev/null +++ b/checkpoint-1500/trainer_state.json @@ -0,0 +1,2194 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 2.6694113719817514, + "eval_steps": 100, + "global_step": 1500, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.3253214355558156, + "epoch": 0.017803493935684877, + "grad_norm": 14.5, + "learning_rate": 6.000000000000001e-07, + "loss": 2.1959, + "mean_token_accuracy": 0.5975642748177051, + "num_tokens": 259048.0, + "step": 10 + }, + { + "entropy": 1.3036176152527332, + "epoch": 0.035606987871369754, + "grad_norm": 12.75, + "learning_rate": 1.2666666666666669e-06, + "loss": 2.1499, + "mean_token_accuracy": 0.6023047739639878, + "num_tokens": 532019.0, + "step": 20 + }, + { + "entropy": 1.3484113823622466, + "epoch": 0.053410481807054634, + "grad_norm": 10.8125, + "learning_rate": 1.9333333333333336e-06, + "loss": 2.108, + "mean_token_accuracy": 0.6078463492915034, + "num_tokens": 807220.0, + "step": 30 + }, + { + "entropy": 1.4311681490391492, + "epoch": 0.07121397574273951, + "grad_norm": 7.875, + "learning_rate": 2.6e-06, + "loss": 2.0349, + "mean_token_accuracy": 0.6107733339071274, + "num_tokens": 1068429.0, + "step": 40 + }, + { + "entropy": 1.479667642340064, + "epoch": 0.08901746967842439, + "grad_norm": 6.4375, + "learning_rate": 3.266666666666667e-06, + "loss": 1.8649, + "mean_token_accuracy": 0.630409375578165, + "num_tokens": 1336698.0, + "step": 50 + }, + { + "entropy": 1.5030938494950532, + "epoch": 0.10682096361410927, + "grad_norm": 4.1875, + "learning_rate": 3.9333333333333335e-06, + "loss": 1.7217, + "mean_token_accuracy": 0.6490663634613156, + "num_tokens": 1608574.0, + "step": 60 + }, + { + "entropy": 1.5066865295171739, + "epoch": 0.12462445754979415, + "grad_norm": 2.71875, + "learning_rate": 4.600000000000001e-06, + "loss": 1.5716, + "mean_token_accuracy": 0.6711439102888107, + "num_tokens": 1872747.0, + "step": 70 + }, + { + "entropy": 1.3747932206839324, + "epoch": 0.14242795148547902, + "grad_norm": 2.234375, + "learning_rate": 5.2666666666666665e-06, + "loss": 1.4403, + "mean_token_accuracy": 0.6936415441334247, + "num_tokens": 2133173.0, + "step": 80 + }, + { + "entropy": 1.1736511185765266, + "epoch": 0.1602314454211639, + "grad_norm": 1.921875, + "learning_rate": 5.933333333333335e-06, + "loss": 1.2902, + "mean_token_accuracy": 0.7222886722534895, + "num_tokens": 2406145.0, + "step": 90 + }, + { + "entropy": 1.1317794900387526, + "epoch": 0.17803493935684878, + "grad_norm": 1.8046875, + "learning_rate": 6.600000000000001e-06, + "loss": 1.2424, + "mean_token_accuracy": 0.7295983772724867, + "num_tokens": 2663709.0, + "step": 100 + }, + { + "epoch": 0.17803493935684878, + "eval_biology_entropy": 2.3133783559799195, + "eval_biology_loss": 2.7910587787628174, + "eval_biology_mean_token_accuracy": 0.5260232911109924, + "eval_biology_num_tokens": 2663709.0, + "eval_biology_runtime": 39.1656, + "eval_biology_samples_per_second": 12.766, + "eval_biology_steps_per_second": 3.192, + "step": 100 + }, + { + "epoch": 0.17803493935684878, + "eval_chemistry_entropy": 1.0788086318969727, + "eval_chemistry_loss": 1.2403336763381958, + "eval_chemistry_mean_token_accuracy": 0.7357268543243408, + "eval_chemistry_num_tokens": 2663709.0, + "eval_chemistry_runtime": 48.7822, + "eval_chemistry_samples_per_second": 10.25, + "eval_chemistry_steps_per_second": 2.562, + "step": 100 + }, + { + "epoch": 0.17803493935684878, + "eval_math_entropy": 0.938195451259613, + "eval_math_loss": 1.3248047828674316, + "eval_math_mean_token_accuracy": 0.7312402629852295, + "eval_math_num_tokens": 2663709.0, + "eval_math_runtime": 50.197, + "eval_math_samples_per_second": 9.961, + "eval_math_steps_per_second": 2.49, + "step": 100 + }, + { + "epoch": 0.17803493935684878, + "eval_physics_entropy": 1.022246531009674, + "eval_physics_loss": 1.176153302192688, + "eval_physics_mean_token_accuracy": 0.7475795021057129, + "eval_physics_num_tokens": 2663709.0, + "eval_physics_runtime": 57.6895, + "eval_physics_samples_per_second": 8.667, + "eval_physics_steps_per_second": 2.167, + "step": 100 + }, + { + "entropy": 1.0380497876554728, + "epoch": 0.19583843329253367, + "grad_norm": 1.7421875, + "learning_rate": 7.266666666666668e-06, + "loss": 1.157, + "mean_token_accuracy": 0.7445634145289659, + "num_tokens": 2934980.0, + "step": 110 + }, + { + "entropy": 1.0149445166811346, + "epoch": 0.21364192722821854, + "grad_norm": 1.6015625, + "learning_rate": 7.933333333333334e-06, + "loss": 1.1167, + "mean_token_accuracy": 0.7485828153789044, + "num_tokens": 3208702.0, + "step": 120 + }, + { + "entropy": 0.9639404848217964, + "epoch": 0.2314454211639034, + "grad_norm": 1.6953125, + "learning_rate": 8.6e-06, + "loss": 1.0979, + "mean_token_accuracy": 0.7553199239075183, + "num_tokens": 3480119.0, + "step": 130 + }, + { + "entropy": 0.9687324531376362, + "epoch": 0.2492489150995883, + "grad_norm": 1.5625, + "learning_rate": 9.266666666666667e-06, + "loss": 1.0762, + "mean_token_accuracy": 0.757096454873681, + "num_tokens": 3746392.0, + "step": 140 + }, + { + "entropy": 0.9682734729722142, + "epoch": 0.26705240903527316, + "grad_norm": 1.875, + "learning_rate": 9.933333333333334e-06, + "loss": 1.0798, + "mean_token_accuracy": 0.7543047085404396, + "num_tokens": 4019487.0, + "step": 150 + }, + { + "entropy": 0.9270784268155694, + "epoch": 0.28485590297095803, + "grad_norm": 1.6015625, + "learning_rate": 1.0600000000000002e-05, + "loss": 1.047, + "mean_token_accuracy": 0.7619738396257162, + "num_tokens": 4298100.0, + "step": 160 + }, + { + "entropy": 0.9063327137380839, + "epoch": 0.30265939690664295, + "grad_norm": 1.5546875, + "learning_rate": 1.1266666666666668e-05, + "loss": 1.0199, + "mean_token_accuracy": 0.7664302848279476, + "num_tokens": 4568116.0, + "step": 170 + }, + { + "entropy": 0.9037324048578739, + "epoch": 0.3204628908423278, + "grad_norm": 1.7109375, + "learning_rate": 1.1933333333333335e-05, + "loss": 1.0144, + "mean_token_accuracy": 0.7657378144562245, + "num_tokens": 4831334.0, + "step": 180 + }, + { + "entropy": 0.8886868219822646, + "epoch": 0.3382663847780127, + "grad_norm": 1.3359375, + "learning_rate": 1.2600000000000001e-05, + "loss": 0.9874, + "mean_token_accuracy": 0.7726409930735827, + "num_tokens": 5103301.0, + "step": 190 + }, + { + "entropy": 0.8936551520600915, + "epoch": 0.35606987871369755, + "grad_norm": 1.3984375, + "learning_rate": 1.3266666666666668e-05, + "loss": 1.019, + "mean_token_accuracy": 0.7648640714585782, + "num_tokens": 5362915.0, + "step": 200 + }, + { + "epoch": 0.35606987871369755, + "eval_biology_entropy": 2.0171296310424807, + "eval_biology_loss": 2.3780677318573, + "eval_biology_mean_token_accuracy": 0.5706260974407196, + "eval_biology_num_tokens": 5362915.0, + "eval_biology_runtime": 39.1203, + "eval_biology_samples_per_second": 12.781, + "eval_biology_steps_per_second": 3.195, + "step": 200 + }, + { + "epoch": 0.35606987871369755, + "eval_chemistry_entropy": 0.8995157589912415, + "eval_chemistry_loss": 1.022930383682251, + "eval_chemistry_mean_token_accuracy": 0.7701377034187317, + "eval_chemistry_num_tokens": 5362915.0, + "eval_chemistry_runtime": 48.7263, + "eval_chemistry_samples_per_second": 10.261, + "eval_chemistry_steps_per_second": 2.565, + "step": 200 + }, + { + "epoch": 0.35606987871369755, + "eval_math_entropy": 0.8352043969631195, + "eval_math_loss": 1.1696399450302124, + "eval_math_mean_token_accuracy": 0.7531440143585205, + "eval_math_num_tokens": 5362915.0, + "eval_math_runtime": 50.1494, + "eval_math_samples_per_second": 9.97, + "eval_math_steps_per_second": 2.493, + "step": 200 + }, + { + "epoch": 0.35606987871369755, + "eval_physics_entropy": 0.8889413847923279, + "eval_physics_loss": 0.9962027072906494, + "eval_physics_mean_token_accuracy": 0.775176504611969, + "eval_physics_num_tokens": 5362915.0, + "eval_physics_runtime": 57.6533, + "eval_physics_samples_per_second": 8.673, + "eval_physics_steps_per_second": 2.168, + "step": 200 + }, + { + "entropy": 0.8509893301874399, + "epoch": 0.3738733726493824, + "grad_norm": 1.3359375, + "learning_rate": 1.3933333333333334e-05, + "loss": 0.9619, + "mean_token_accuracy": 0.7772600032389164, + "num_tokens": 5625455.0, + "step": 210 + }, + { + "entropy": 0.8560685662552714, + "epoch": 0.39167686658506734, + "grad_norm": 1.4609375, + "learning_rate": 1.46e-05, + "loss": 0.9666, + "mean_token_accuracy": 0.7764565829187632, + "num_tokens": 5890806.0, + "step": 220 + }, + { + "entropy": 0.8282079607248306, + "epoch": 0.4094803605207522, + "grad_norm": 1.3828125, + "learning_rate": 1.5266666666666667e-05, + "loss": 0.9353, + "mean_token_accuracy": 0.7817027729004622, + "num_tokens": 6158751.0, + "step": 230 + }, + { + "entropy": 0.8447791969403624, + "epoch": 0.4272838544564371, + "grad_norm": 1.4296875, + "learning_rate": 1.5933333333333336e-05, + "loss": 0.9428, + "mean_token_accuracy": 0.7795210376381874, + "num_tokens": 6423747.0, + "step": 240 + }, + { + "entropy": 0.8478817906230688, + "epoch": 0.44508734839212194, + "grad_norm": 1.8984375, + "learning_rate": 1.66e-05, + "loss": 0.9542, + "mean_token_accuracy": 0.7772787544876337, + "num_tokens": 6684517.0, + "step": 250 + }, + { + "entropy": 0.8135418241843582, + "epoch": 0.4628908423278068, + "grad_norm": 1.453125, + "learning_rate": 1.726666666666667e-05, + "loss": 0.9297, + "mean_token_accuracy": 0.783412617444992, + "num_tokens": 6945659.0, + "step": 260 + }, + { + "entropy": 0.8251218661665917, + "epoch": 0.48069433626349173, + "grad_norm": 1.6640625, + "learning_rate": 1.7933333333333333e-05, + "loss": 0.9307, + "mean_token_accuracy": 0.7831020273268223, + "num_tokens": 7217797.0, + "step": 270 + }, + { + "entropy": 0.7879339458420873, + "epoch": 0.4984978301991766, + "grad_norm": 1.328125, + "learning_rate": 1.86e-05, + "loss": 0.8795, + "mean_token_accuracy": 0.7897050529718399, + "num_tokens": 7498182.0, + "step": 280 + }, + { + "entropy": 0.8145477209240198, + "epoch": 0.5163013241348615, + "grad_norm": 1.265625, + "learning_rate": 1.926666666666667e-05, + "loss": 0.9187, + "mean_token_accuracy": 0.7838029216974973, + "num_tokens": 7761686.0, + "step": 290 + }, + { + "entropy": 0.8231891892850399, + "epoch": 0.5341048180705463, + "grad_norm": 1.5625, + "learning_rate": 1.9933333333333334e-05, + "loss": 0.9271, + "mean_token_accuracy": 0.7819279827177524, + "num_tokens": 8026923.0, + "step": 300 + }, + { + "epoch": 0.5341048180705463, + "eval_biology_entropy": 1.8821040544509888, + "eval_biology_loss": 2.226550340652466, + "eval_biology_mean_token_accuracy": 0.5873979187011719, + "eval_biology_num_tokens": 8026923.0, + "eval_biology_runtime": 39.1096, + "eval_biology_samples_per_second": 12.785, + "eval_biology_steps_per_second": 3.196, + "step": 300 + }, + { + "epoch": 0.5341048180705463, + "eval_chemistry_entropy": 0.7922393922805786, + "eval_chemistry_loss": 0.9438055157661438, + "eval_chemistry_mean_token_accuracy": 0.7841580080986023, + "eval_chemistry_num_tokens": 8026923.0, + "eval_chemistry_runtime": 48.7268, + "eval_chemistry_samples_per_second": 10.261, + "eval_chemistry_steps_per_second": 2.565, + "step": 300 + }, + { + "epoch": 0.5341048180705463, + "eval_math_entropy": 0.7649447858333588, + "eval_math_loss": 1.1145466566085815, + "eval_math_mean_token_accuracy": 0.763204463005066, + "eval_math_num_tokens": 8026923.0, + "eval_math_runtime": 50.1205, + "eval_math_samples_per_second": 9.976, + "eval_math_steps_per_second": 2.494, + "step": 300 + }, + { + "epoch": 0.5341048180705463, + "eval_physics_entropy": 0.7964776389598847, + "eval_physics_loss": 0.9373180866241455, + "eval_physics_mean_token_accuracy": 0.784841582775116, + "eval_physics_num_tokens": 8026923.0, + "eval_physics_runtime": 57.6222, + "eval_physics_samples_per_second": 8.677, + "eval_physics_steps_per_second": 2.169, + "step": 300 + }, + { + "entropy": 0.800599187426269, + "epoch": 0.5519083120062312, + "grad_norm": 1.6640625, + "learning_rate": 1.9933333333333334e-05, + "loss": 0.9071, + "mean_token_accuracy": 0.7856069568544626, + "num_tokens": 8301506.0, + "step": 310 + }, + { + "entropy": 0.7965721761807799, + "epoch": 0.5697118059419161, + "grad_norm": 1.234375, + "learning_rate": 1.985925925925926e-05, + "loss": 0.9139, + "mean_token_accuracy": 0.786777526140213, + "num_tokens": 8559126.0, + "step": 320 + }, + { + "entropy": 0.7925007566809654, + "epoch": 0.587515299877601, + "grad_norm": 1.4453125, + "learning_rate": 1.9785185185185187e-05, + "loss": 0.906, + "mean_token_accuracy": 0.7864235289394855, + "num_tokens": 8821088.0, + "step": 330 + }, + { + "entropy": 0.7728251311928034, + "epoch": 0.6053187938132859, + "grad_norm": 1.2109375, + "learning_rate": 1.971111111111111e-05, + "loss": 0.8764, + "mean_token_accuracy": 0.7919386122375727, + "num_tokens": 9089866.0, + "step": 340 + }, + { + "entropy": 0.7705478269606829, + "epoch": 0.6231222877489707, + "grad_norm": 1.3515625, + "learning_rate": 1.963703703703704e-05, + "loss": 0.8837, + "mean_token_accuracy": 0.7896027479320764, + "num_tokens": 9358866.0, + "step": 350 + }, + { + "entropy": 0.7656249281018972, + "epoch": 0.6409257816846556, + "grad_norm": 1.1875, + "learning_rate": 1.9562962962962964e-05, + "loss": 0.8626, + "mean_token_accuracy": 0.7931958142668009, + "num_tokens": 9629977.0, + "step": 360 + }, + { + "entropy": 0.7598591955378652, + "epoch": 0.6587292756203404, + "grad_norm": 1.296875, + "learning_rate": 1.948888888888889e-05, + "loss": 0.8608, + "mean_token_accuracy": 0.7945109561085701, + "num_tokens": 9897757.0, + "step": 370 + }, + { + "entropy": 0.7398761732503771, + "epoch": 0.6765327695560254, + "grad_norm": 1.265625, + "learning_rate": 1.9414814814814817e-05, + "loss": 0.848, + "mean_token_accuracy": 0.7985429972410202, + "num_tokens": 10168367.0, + "step": 380 + }, + { + "entropy": 0.7331193052232265, + "epoch": 0.6943362634917103, + "grad_norm": 1.4296875, + "learning_rate": 1.9340740740740743e-05, + "loss": 0.8366, + "mean_token_accuracy": 0.8001827634871006, + "num_tokens": 10439039.0, + "step": 390 + }, + { + "entropy": 0.7496613372117281, + "epoch": 0.7121397574273951, + "grad_norm": 1.15625, + "learning_rate": 1.926666666666667e-05, + "loss": 0.8517, + "mean_token_accuracy": 0.7967161379754544, + "num_tokens": 10709549.0, + "step": 400 + }, + { + "epoch": 0.7121397574273951, + "eval_biology_entropy": 1.823248610496521, + "eval_biology_loss": 2.159681558609009, + "eval_biology_mean_token_accuracy": 0.5972214796543122, + "eval_biology_num_tokens": 10709549.0, + "eval_biology_runtime": 39.2024, + "eval_biology_samples_per_second": 12.754, + "eval_biology_steps_per_second": 3.189, + "step": 400 + }, + { + "epoch": 0.7121397574273951, + "eval_chemistry_entropy": 0.7672218294143677, + "eval_chemistry_loss": 0.8971230387687683, + "eval_chemistry_mean_token_accuracy": 0.7928389143943787, + "eval_chemistry_num_tokens": 10709549.0, + "eval_chemistry_runtime": 48.8296, + "eval_chemistry_samples_per_second": 10.24, + "eval_chemistry_steps_per_second": 2.56, + "step": 400 + }, + { + "epoch": 0.7121397574273951, + "eval_math_entropy": 0.7494234163761139, + "eval_math_loss": 1.0849554538726807, + "eval_math_mean_token_accuracy": 0.7666959943771362, + "eval_math_num_tokens": 10709549.0, + "eval_math_runtime": 50.2325, + "eval_math_samples_per_second": 9.954, + "eval_math_steps_per_second": 2.488, + "step": 400 + }, + { + "epoch": 0.7121397574273951, + "eval_physics_entropy": 0.7731240901947022, + "eval_physics_loss": 0.9018430709838867, + "eval_physics_mean_token_accuracy": 0.7909709796905517, + "eval_physics_num_tokens": 10709549.0, + "eval_physics_runtime": 57.7809, + "eval_physics_samples_per_second": 8.653, + "eval_physics_steps_per_second": 2.163, + "step": 400 + }, + { + "entropy": 0.7354462265968322, + "epoch": 0.72994325136308, + "grad_norm": 1.3125, + "learning_rate": 1.9192592592592593e-05, + "loss": 0.8411, + "mean_token_accuracy": 0.8003354988992214, + "num_tokens": 10976011.0, + "step": 410 + }, + { + "entropy": 0.7435122076421976, + "epoch": 0.7477467452987648, + "grad_norm": 1.15625, + "learning_rate": 1.911851851851852e-05, + "loss": 0.8438, + "mean_token_accuracy": 0.7971596572548151, + "num_tokens": 11245251.0, + "step": 420 + }, + { + "entropy": 0.759309609234333, + "epoch": 0.7655502392344498, + "grad_norm": 1.375, + "learning_rate": 1.9044444444444446e-05, + "loss": 0.8579, + "mean_token_accuracy": 0.792990094423294, + "num_tokens": 11518514.0, + "step": 430 + }, + { + "entropy": 0.7545185785740613, + "epoch": 0.7833537331701347, + "grad_norm": 1.3125, + "learning_rate": 1.8970370370370372e-05, + "loss": 0.8574, + "mean_token_accuracy": 0.794682926684618, + "num_tokens": 11787117.0, + "step": 440 + }, + { + "entropy": 0.751254404336214, + "epoch": 0.8011572271058195, + "grad_norm": 1.515625, + "learning_rate": 1.8896296296296295e-05, + "loss": 0.8495, + "mean_token_accuracy": 0.7972703959792853, + "num_tokens": 12050993.0, + "step": 450 + }, + { + "entropy": 0.7543802969157696, + "epoch": 0.8189607210415044, + "grad_norm": 1.28125, + "learning_rate": 1.8822222222222225e-05, + "loss": 0.8527, + "mean_token_accuracy": 0.7960003320127725, + "num_tokens": 12311688.0, + "step": 460 + }, + { + "entropy": 0.7186065271496773, + "epoch": 0.8367642149771892, + "grad_norm": 1.15625, + "learning_rate": 1.874814814814815e-05, + "loss": 0.8096, + "mean_token_accuracy": 0.8027401898056269, + "num_tokens": 12598408.0, + "step": 470 + }, + { + "entropy": 0.7257912319153548, + "epoch": 0.8545677089128741, + "grad_norm": 1.125, + "learning_rate": 1.8674074074074075e-05, + "loss": 0.8299, + "mean_token_accuracy": 0.8012042317539454, + "num_tokens": 12869642.0, + "step": 480 + }, + { + "entropy": 0.7173270899802446, + "epoch": 0.8723712028485591, + "grad_norm": 1.125, + "learning_rate": 1.86e-05, + "loss": 0.8146, + "mean_token_accuracy": 0.8032398864626884, + "num_tokens": 13139555.0, + "step": 490 + }, + { + "entropy": 0.7487573977559805, + "epoch": 0.8901746967842439, + "grad_norm": 1.2421875, + "learning_rate": 1.8525925925925928e-05, + "loss": 0.8562, + "mean_token_accuracy": 0.7955521304160357, + "num_tokens": 13396545.0, + "step": 500 + }, + { + "epoch": 0.8901746967842439, + "eval_biology_entropy": 1.8204833917617798, + "eval_biology_loss": 2.1014022827148438, + "eval_biology_mean_token_accuracy": 0.6051526765823364, + "eval_biology_num_tokens": 13396545.0, + "eval_biology_runtime": 39.1543, + "eval_biology_samples_per_second": 12.77, + "eval_biology_steps_per_second": 3.192, + "step": 500 + }, + { + "epoch": 0.8901746967842439, + "eval_chemistry_entropy": 0.7538458592891694, + "eval_chemistry_loss": 0.8699584007263184, + "eval_chemistry_mean_token_accuracy": 0.7975720787048339, + "eval_chemistry_num_tokens": 13396545.0, + "eval_chemistry_runtime": 48.8229, + "eval_chemistry_samples_per_second": 10.241, + "eval_chemistry_steps_per_second": 2.56, + "step": 500 + }, + { + "epoch": 0.8901746967842439, + "eval_math_entropy": 0.7398368701934814, + "eval_math_loss": 1.0758970975875854, + "eval_math_mean_token_accuracy": 0.7687290472984314, + "eval_math_num_tokens": 13396545.0, + "eval_math_runtime": 50.0916, + "eval_math_samples_per_second": 9.982, + "eval_math_steps_per_second": 2.495, + "step": 500 + }, + { + "epoch": 0.8901746967842439, + "eval_physics_entropy": 0.7607374501228332, + "eval_physics_loss": 0.8848790526390076, + "eval_physics_mean_token_accuracy": 0.7937801394462586, + "eval_physics_num_tokens": 13396545.0, + "eval_physics_runtime": 57.6599, + "eval_physics_samples_per_second": 8.672, + "eval_physics_steps_per_second": 2.168, + "step": 500 + }, + { + "entropy": 0.7129743231460453, + "epoch": 0.9079781907199288, + "grad_norm": 1.1640625, + "learning_rate": 1.8451851851851855e-05, + "loss": 0.8109, + "mean_token_accuracy": 0.8060642663389445, + "num_tokens": 13667857.0, + "step": 510 + }, + { + "entropy": 0.7393450669944286, + "epoch": 0.9257816846556136, + "grad_norm": 1.1484375, + "learning_rate": 1.8377777777777778e-05, + "loss": 0.8325, + "mean_token_accuracy": 0.7997861213982105, + "num_tokens": 13932771.0, + "step": 520 + }, + { + "entropy": 0.7191115407273173, + "epoch": 0.9435851785912985, + "grad_norm": 1.1640625, + "learning_rate": 1.8303703703703704e-05, + "loss": 0.8233, + "mean_token_accuracy": 0.8034567594528198, + "num_tokens": 14194632.0, + "step": 530 + }, + { + "entropy": 0.7434917811304331, + "epoch": 0.9613886725269835, + "grad_norm": 1.1484375, + "learning_rate": 1.822962962962963e-05, + "loss": 0.8435, + "mean_token_accuracy": 0.7969928599894047, + "num_tokens": 14457650.0, + "step": 540 + }, + { + "entropy": 0.7100320702418685, + "epoch": 0.9791921664626683, + "grad_norm": 1.125, + "learning_rate": 1.8155555555555557e-05, + "loss": 0.8149, + "mean_token_accuracy": 0.803739969804883, + "num_tokens": 14724967.0, + "step": 550 + }, + { + "entropy": 0.737933493219316, + "epoch": 0.9969956603983532, + "grad_norm": 1.1875, + "learning_rate": 1.8081481481481484e-05, + "loss": 0.8421, + "mean_token_accuracy": 0.8001800544559956, + "num_tokens": 14993272.0, + "step": 560 + }, + { + "entropy": 0.6655864379098339, + "epoch": 1.014242795148548, + "grad_norm": 1.1171875, + "learning_rate": 1.800740740740741e-05, + "loss": 0.7668, + "mean_token_accuracy": 0.814733551779101, + "num_tokens": 15254936.0, + "step": 570 + }, + { + "entropy": 0.6937626471742988, + "epoch": 1.0320462890842328, + "grad_norm": 1.2578125, + "learning_rate": 1.7933333333333333e-05, + "loss": 0.7942, + "mean_token_accuracy": 0.806296594068408, + "num_tokens": 15515007.0, + "step": 580 + }, + { + "entropy": 0.6596947606652975, + "epoch": 1.0498497830199176, + "grad_norm": 1.2890625, + "learning_rate": 1.785925925925926e-05, + "loss": 0.7623, + "mean_token_accuracy": 0.8152227636426688, + "num_tokens": 15794203.0, + "step": 590 + }, + { + "entropy": 0.6868148937821388, + "epoch": 1.0676532769556026, + "grad_norm": 1.1640625, + "learning_rate": 1.7785185185185186e-05, + "loss": 0.7827, + "mean_token_accuracy": 0.8101296100765467, + "num_tokens": 16063664.0, + "step": 600 + }, + { + "epoch": 1.0676532769556026, + "eval_biology_entropy": 1.6867316131591796, + "eval_biology_loss": 2.077732801437378, + "eval_biology_mean_token_accuracy": 0.6078621828556061, + "eval_biology_num_tokens": 16063664.0, + "eval_biology_runtime": 39.1079, + "eval_biology_samples_per_second": 12.785, + "eval_biology_steps_per_second": 3.196, + "step": 600 + }, + { + "epoch": 1.0676532769556026, + "eval_chemistry_entropy": 0.6882545161247253, + "eval_chemistry_loss": 0.8549924492835999, + "eval_chemistry_mean_token_accuracy": 0.8009334373474121, + "eval_chemistry_num_tokens": 16063664.0, + "eval_chemistry_runtime": 48.732, + "eval_chemistry_samples_per_second": 10.26, + "eval_chemistry_steps_per_second": 2.565, + "step": 600 + }, + { + "epoch": 1.0676532769556026, + "eval_math_entropy": 0.6883157949447631, + "eval_math_loss": 1.075770616531372, + "eval_math_mean_token_accuracy": 0.7705092186927796, + "eval_math_num_tokens": 16063664.0, + "eval_math_runtime": 50.1345, + "eval_math_samples_per_second": 9.973, + "eval_math_steps_per_second": 2.493, + "step": 600 + }, + { + "epoch": 1.0676532769556026, + "eval_physics_entropy": 0.7055722939968109, + "eval_physics_loss": 0.8771266937255859, + "eval_physics_mean_token_accuracy": 0.7955313258171082, + "eval_physics_num_tokens": 16063664.0, + "eval_physics_runtime": 57.6658, + "eval_physics_samples_per_second": 8.671, + "eval_physics_steps_per_second": 2.168, + "step": 600 + }, + { + "entropy": 0.6597639428451657, + "epoch": 1.0854567708912874, + "grad_norm": 1.203125, + "learning_rate": 1.7711111111111113e-05, + "loss": 0.7655, + "mean_token_accuracy": 0.8151189126074314, + "num_tokens": 16338303.0, + "step": 610 + }, + { + "entropy": 0.6991181125864386, + "epoch": 1.1032602648269723, + "grad_norm": 1.2734375, + "learning_rate": 1.763703703703704e-05, + "loss": 0.7934, + "mean_token_accuracy": 0.8071035247296094, + "num_tokens": 16606302.0, + "step": 620 + }, + { + "entropy": 0.6774254593998194, + "epoch": 1.121063758762657, + "grad_norm": 1.28125, + "learning_rate": 1.7562962962962962e-05, + "loss": 0.7706, + "mean_token_accuracy": 0.8098537240177393, + "num_tokens": 16877071.0, + "step": 630 + }, + { + "entropy": 0.6678685389459134, + "epoch": 1.138867252698342, + "grad_norm": 1.203125, + "learning_rate": 1.7488888888888892e-05, + "loss": 0.7714, + "mean_token_accuracy": 0.8127609983086586, + "num_tokens": 17145775.0, + "step": 640 + }, + { + "entropy": 0.6851805068552494, + "epoch": 1.156670746634027, + "grad_norm": 1.2890625, + "learning_rate": 1.7414814814814815e-05, + "loss": 0.777, + "mean_token_accuracy": 0.8110932789742946, + "num_tokens": 17410775.0, + "step": 650 + }, + { + "entropy": 0.7042925998568534, + "epoch": 1.1744742405697117, + "grad_norm": 1.375, + "learning_rate": 1.7340740740740742e-05, + "loss": 0.7979, + "mean_token_accuracy": 0.8038393501192331, + "num_tokens": 17677791.0, + "step": 660 + }, + { + "entropy": 0.6810087833553553, + "epoch": 1.1922777345053968, + "grad_norm": 1.1484375, + "learning_rate": 1.726666666666667e-05, + "loss": 0.7688, + "mean_token_accuracy": 0.8104576140642166, + "num_tokens": 17949441.0, + "step": 670 + }, + { + "entropy": 0.662551742233336, + "epoch": 1.2100812284410816, + "grad_norm": 1.2109375, + "learning_rate": 1.7192592592592595e-05, + "loss": 0.7653, + "mean_token_accuracy": 0.8126230895519256, + "num_tokens": 18217742.0, + "step": 680 + }, + { + "entropy": 0.6803617887198925, + "epoch": 1.2278847223767664, + "grad_norm": 1.2421875, + "learning_rate": 1.711851851851852e-05, + "loss": 0.7771, + "mean_token_accuracy": 0.8095329128205776, + "num_tokens": 18487769.0, + "step": 690 + }, + { + "entropy": 0.6707866100594402, + "epoch": 1.2456882163124514, + "grad_norm": 1.34375, + "learning_rate": 1.7044444444444445e-05, + "loss": 0.7687, + "mean_token_accuracy": 0.8113945126533508, + "num_tokens": 18760296.0, + "step": 700 + }, + { + "epoch": 1.2456882163124514, + "eval_biology_entropy": 1.6589141931533813, + "eval_biology_loss": 2.081825017929077, + "eval_biology_mean_token_accuracy": 0.608892562866211, + "eval_biology_num_tokens": 18760296.0, + "eval_biology_runtime": 39.8083, + "eval_biology_samples_per_second": 12.56, + "eval_biology_steps_per_second": 3.14, + "step": 700 + }, + { + "epoch": 1.2456882163124514, + "eval_chemistry_entropy": 0.6833467047214508, + "eval_chemistry_loss": 0.8464663028717041, + "eval_chemistry_mean_token_accuracy": 0.80278346824646, + "eval_chemistry_num_tokens": 18760296.0, + "eval_chemistry_runtime": 49.1467, + "eval_chemistry_samples_per_second": 10.174, + "eval_chemistry_steps_per_second": 2.543, + "step": 700 + }, + { + "epoch": 1.2456882163124514, + "eval_math_entropy": 0.6947779231071473, + "eval_math_loss": 1.0786137580871582, + "eval_math_mean_token_accuracy": 0.7709644784927369, + "eval_math_num_tokens": 18760296.0, + "eval_math_runtime": 52.5755, + "eval_math_samples_per_second": 9.51, + "eval_math_steps_per_second": 2.378, + "step": 700 + }, + { + "epoch": 1.2456882163124514, + "eval_physics_entropy": 0.704109364271164, + "eval_physics_loss": 0.8724388480186462, + "eval_physics_mean_token_accuracy": 0.796642897605896, + "eval_physics_num_tokens": 18760296.0, + "eval_physics_runtime": 61.3658, + "eval_physics_samples_per_second": 8.148, + "eval_physics_steps_per_second": 2.037, + "step": 700 + }, + { + "entropy": 0.6705936685204505, + "epoch": 1.2634917102481362, + "grad_norm": 1.140625, + "learning_rate": 1.697037037037037e-05, + "loss": 0.7618, + "mean_token_accuracy": 0.8144830856472254, + "num_tokens": 19035733.0, + "step": 710 + }, + { + "entropy": 0.6870324395596981, + "epoch": 1.281295204183821, + "grad_norm": 1.1953125, + "learning_rate": 1.6896296296296298e-05, + "loss": 0.7849, + "mean_token_accuracy": 0.807737946882844, + "num_tokens": 19311432.0, + "step": 720 + }, + { + "entropy": 0.6551721351221204, + "epoch": 1.299098698119506, + "grad_norm": 1.0859375, + "learning_rate": 1.6822222222222224e-05, + "loss": 0.7538, + "mean_token_accuracy": 0.8164622981101275, + "num_tokens": 19593016.0, + "step": 730 + }, + { + "entropy": 0.6645344085991383, + "epoch": 1.3169021920551909, + "grad_norm": 1.1796875, + "learning_rate": 1.6748148148148147e-05, + "loss": 0.759, + "mean_token_accuracy": 0.8134492743760348, + "num_tokens": 19860476.0, + "step": 740 + }, + { + "entropy": 0.6684485200792551, + "epoch": 1.3347056859908757, + "grad_norm": 1.3515625, + "learning_rate": 1.6674074074074077e-05, + "loss": 0.7632, + "mean_token_accuracy": 0.8128790140151978, + "num_tokens": 20124436.0, + "step": 750 + }, + { + "entropy": 0.6736280497163534, + "epoch": 1.3525091799265607, + "grad_norm": 1.25, + "learning_rate": 1.66e-05, + "loss": 0.7771, + "mean_token_accuracy": 0.813029420748353, + "num_tokens": 20388027.0, + "step": 760 + }, + { + "entropy": 0.6706711670383811, + "epoch": 1.3703126738622455, + "grad_norm": 1.265625, + "learning_rate": 1.6525925925925927e-05, + "loss": 0.7697, + "mean_token_accuracy": 0.8115393359214067, + "num_tokens": 20656076.0, + "step": 770 + }, + { + "entropy": 0.6996258163824678, + "epoch": 1.3881161677979303, + "grad_norm": 1.2578125, + "learning_rate": 1.6451851851851853e-05, + "loss": 0.8077, + "mean_token_accuracy": 0.8058754079043865, + "num_tokens": 20911633.0, + "step": 780 + }, + { + "entropy": 0.6905225297436118, + "epoch": 1.4059196617336152, + "grad_norm": 1.171875, + "learning_rate": 1.637777777777778e-05, + "loss": 0.7955, + "mean_token_accuracy": 0.8070559553802014, + "num_tokens": 21176227.0, + "step": 790 + }, + { + "entropy": 0.6718146029859782, + "epoch": 1.4237231556693, + "grad_norm": 1.2890625, + "learning_rate": 1.6303703703703706e-05, + "loss": 0.7716, + "mean_token_accuracy": 0.8126390781253576, + "num_tokens": 21445242.0, + "step": 800 + }, + { + "epoch": 1.4237231556693, + "eval_biology_entropy": 1.6255396161079407, + "eval_biology_loss": 2.0511693954467773, + "eval_biology_mean_token_accuracy": 0.6119143316745758, + "eval_biology_num_tokens": 21445242.0, + "eval_biology_runtime": 42.4445, + "eval_biology_samples_per_second": 11.78, + "eval_biology_steps_per_second": 2.945, + "step": 800 + }, + { + "epoch": 1.4237231556693, + "eval_chemistry_entropy": 0.6763756844997406, + "eval_chemistry_loss": 0.8368268609046936, + "eval_chemistry_mean_token_accuracy": 0.8050474629402161, + "eval_chemistry_num_tokens": 21445242.0, + "eval_chemistry_runtime": 51.5104, + "eval_chemistry_samples_per_second": 9.707, + "eval_chemistry_steps_per_second": 2.427, + "step": 800 + }, + { + "epoch": 1.4237231556693, + "eval_math_entropy": 0.6853804645538331, + "eval_math_loss": 1.0698176622390747, + "eval_math_mean_token_accuracy": 0.7714386968612671, + "eval_math_num_tokens": 21445242.0, + "eval_math_runtime": 54.9388, + "eval_math_samples_per_second": 9.101, + "eval_math_steps_per_second": 2.275, + "step": 800 + }, + { + "epoch": 1.4237231556693, + "eval_physics_entropy": 0.6991692934036254, + "eval_physics_loss": 0.8673914074897766, + "eval_physics_mean_token_accuracy": 0.7979785161018371, + "eval_physics_num_tokens": 21445242.0, + "eval_physics_runtime": 60.9527, + "eval_physics_samples_per_second": 8.203, + "eval_physics_steps_per_second": 2.051, + "step": 800 + }, + { + "entropy": 0.6695453573018313, + "epoch": 1.441526649604985, + "grad_norm": 1.3125, + "learning_rate": 1.622962962962963e-05, + "loss": 0.77, + "mean_token_accuracy": 0.8109262142330408, + "num_tokens": 21718455.0, + "step": 810 + }, + { + "entropy": 0.6753543723374605, + "epoch": 1.4593301435406698, + "grad_norm": 1.2734375, + "learning_rate": 1.6155555555555556e-05, + "loss": 0.7782, + "mean_token_accuracy": 0.8130301378667355, + "num_tokens": 21974076.0, + "step": 820 + }, + { + "entropy": 0.6758085643872619, + "epoch": 1.4771336374763546, + "grad_norm": 1.2265625, + "learning_rate": 1.6081481481481482e-05, + "loss": 0.7768, + "mean_token_accuracy": 0.8110865034162998, + "num_tokens": 22241599.0, + "step": 830 + }, + { + "entropy": 0.6868746858090162, + "epoch": 1.4949371314120397, + "grad_norm": 1.34375, + "learning_rate": 1.600740740740741e-05, + "loss": 0.7818, + "mean_token_accuracy": 0.8090599600225687, + "num_tokens": 22510725.0, + "step": 840 + }, + { + "entropy": 0.6554480012506246, + "epoch": 1.5127406253477245, + "grad_norm": 1.1796875, + "learning_rate": 1.5933333333333336e-05, + "loss": 0.766, + "mean_token_accuracy": 0.8149232871830463, + "num_tokens": 22778514.0, + "step": 850 + }, + { + "entropy": 0.6574800366535782, + "epoch": 1.5305441192834093, + "grad_norm": 1.2734375, + "learning_rate": 1.5859259259259262e-05, + "loss": 0.7517, + "mean_token_accuracy": 0.8147028144448996, + "num_tokens": 23051230.0, + "step": 860 + }, + { + "entropy": 0.6707898294553161, + "epoch": 1.5483476132190943, + "grad_norm": 1.21875, + "learning_rate": 1.5785185185185185e-05, + "loss": 0.776, + "mean_token_accuracy": 0.8121768556535244, + "num_tokens": 23306327.0, + "step": 870 + }, + { + "entropy": 0.6759747974574566, + "epoch": 1.5661511071547791, + "grad_norm": 1.234375, + "learning_rate": 1.571111111111111e-05, + "loss": 0.7717, + "mean_token_accuracy": 0.810706127434969, + "num_tokens": 23573127.0, + "step": 880 + }, + { + "entropy": 0.6696014609187841, + "epoch": 1.583954601090464, + "grad_norm": 1.1953125, + "learning_rate": 1.5637037037037038e-05, + "loss": 0.7635, + "mean_token_accuracy": 0.8123903065919876, + "num_tokens": 23841914.0, + "step": 890 + }, + { + "entropy": 0.6680066948756576, + "epoch": 1.601758095026149, + "grad_norm": 1.28125, + "learning_rate": 1.5562962962962965e-05, + "loss": 0.7763, + "mean_token_accuracy": 0.8117439385503531, + "num_tokens": 24105114.0, + "step": 900 + }, + { + "epoch": 1.601758095026149, + "eval_biology_entropy": 1.6559728245735168, + "eval_biology_loss": 2.0387370586395264, + "eval_biology_mean_token_accuracy": 0.6136598820686341, + "eval_biology_num_tokens": 24105114.0, + "eval_biology_runtime": 41.4479, + "eval_biology_samples_per_second": 12.063, + "eval_biology_steps_per_second": 3.016, + "step": 900 + }, + { + "epoch": 1.601758095026149, + "eval_chemistry_entropy": 0.683923549413681, + "eval_chemistry_loss": 0.8298754692077637, + "eval_chemistry_mean_token_accuracy": 0.8061454033851624, + "eval_chemistry_num_tokens": 24105114.0, + "eval_chemistry_runtime": 50.8984, + "eval_chemistry_samples_per_second": 9.823, + "eval_chemistry_steps_per_second": 2.456, + "step": 900 + }, + { + "epoch": 1.601758095026149, + "eval_math_entropy": 0.6932375190258027, + "eval_math_loss": 1.06475830078125, + "eval_math_mean_token_accuracy": 0.7720690259933471, + "eval_math_num_tokens": 24105114.0, + "eval_math_runtime": 52.5544, + "eval_math_samples_per_second": 9.514, + "eval_math_steps_per_second": 2.378, + "step": 900 + }, + { + "epoch": 1.601758095026149, + "eval_physics_entropy": 0.7051843535900116, + "eval_physics_loss": 0.8616264462471008, + "eval_physics_mean_token_accuracy": 0.7987073979377747, + "eval_physics_num_tokens": 24105114.0, + "eval_physics_runtime": 59.8508, + "eval_physics_samples_per_second": 8.354, + "eval_physics_steps_per_second": 2.089, + "step": 900 + }, + { + "entropy": 0.6430077504366636, + "epoch": 1.6195615889618338, + "grad_norm": 1.1640625, + "learning_rate": 1.548888888888889e-05, + "loss": 0.743, + "mean_token_accuracy": 0.8184590026736259, + "num_tokens": 24384008.0, + "step": 910 + }, + { + "entropy": 0.6775279760360717, + "epoch": 1.6373650828975186, + "grad_norm": 1.3125, + "learning_rate": 1.5414814814814814e-05, + "loss": 0.7704, + "mean_token_accuracy": 0.8114439699798822, + "num_tokens": 24638205.0, + "step": 920 + }, + { + "entropy": 0.6244934551417828, + "epoch": 1.6551685768332036, + "grad_norm": 1.1953125, + "learning_rate": 1.5340740740740744e-05, + "loss": 0.7253, + "mean_token_accuracy": 0.8222480688244105, + "num_tokens": 24910104.0, + "step": 930 + }, + { + "entropy": 0.6539383618161082, + "epoch": 1.6729720707688884, + "grad_norm": 1.2578125, + "learning_rate": 1.5266666666666667e-05, + "loss": 0.7471, + "mean_token_accuracy": 0.8160006206482648, + "num_tokens": 25181306.0, + "step": 940 + }, + { + "entropy": 0.6607510834932327, + "epoch": 1.6907755647045732, + "grad_norm": 1.3203125, + "learning_rate": 1.5192592592592594e-05, + "loss": 0.7589, + "mean_token_accuracy": 0.8142590031027794, + "num_tokens": 25449138.0, + "step": 950 + }, + { + "entropy": 0.6612858925014734, + "epoch": 1.7085790586402583, + "grad_norm": 1.203125, + "learning_rate": 1.5118518518518519e-05, + "loss": 0.766, + "mean_token_accuracy": 0.8137270450592041, + "num_tokens": 25709030.0, + "step": 960 + }, + { + "entropy": 0.6701876953244209, + "epoch": 1.7263825525759429, + "grad_norm": 1.2734375, + "learning_rate": 1.5044444444444445e-05, + "loss": 0.7702, + "mean_token_accuracy": 0.8120660580694675, + "num_tokens": 25972807.0, + "step": 970 + }, + { + "entropy": 0.6566708590835333, + "epoch": 1.744186046511628, + "grad_norm": 1.1640625, + "learning_rate": 1.497037037037037e-05, + "loss": 0.748, + "mean_token_accuracy": 0.8170768674463034, + "num_tokens": 26244955.0, + "step": 980 + }, + { + "entropy": 0.624184824153781, + "epoch": 1.761989540447313, + "grad_norm": 1.1796875, + "learning_rate": 1.4896296296296298e-05, + "loss": 0.7284, + "mean_token_accuracy": 0.8223610106855631, + "num_tokens": 26515548.0, + "step": 990 + }, + { + "entropy": 0.6476762825623155, + "epoch": 1.7797930343829975, + "grad_norm": 1.21875, + "learning_rate": 1.4822222222222225e-05, + "loss": 0.7422, + "mean_token_accuracy": 0.8172390758991241, + "num_tokens": 26782376.0, + "step": 1000 + }, + { + "epoch": 1.7797930343829975, + "eval_biology_entropy": 1.6157320532798767, + "eval_biology_loss": 2.0338852405548096, + "eval_biology_mean_token_accuracy": 0.6143801593780518, + "eval_biology_num_tokens": 26782376.0, + "eval_biology_runtime": 41.5849, + "eval_biology_samples_per_second": 12.024, + "eval_biology_steps_per_second": 3.006, + "step": 1000 + }, + { + "epoch": 1.7797930343829975, + "eval_chemistry_entropy": 0.6670743160247803, + "eval_chemistry_loss": 0.8244208097457886, + "eval_chemistry_mean_token_accuracy": 0.8077364330291747, + "eval_chemistry_num_tokens": 26782376.0, + "eval_chemistry_runtime": 53.0274, + "eval_chemistry_samples_per_second": 9.429, + "eval_chemistry_steps_per_second": 2.357, + "step": 1000 + }, + { + "epoch": 1.7797930343829975, + "eval_math_entropy": 0.6760952143669129, + "eval_math_loss": 1.0642753839492798, + "eval_math_mean_token_accuracy": 0.7728619589805603, + "eval_math_num_tokens": 26782376.0, + "eval_math_runtime": 53.8997, + "eval_math_samples_per_second": 9.276, + "eval_math_steps_per_second": 2.319, + "step": 1000 + }, + { + "epoch": 1.7797930343829975, + "eval_physics_entropy": 0.6914605097770691, + "eval_physics_loss": 0.8597292304039001, + "eval_physics_mean_token_accuracy": 0.7992245469093323, + "eval_physics_num_tokens": 26782376.0, + "eval_physics_runtime": 62.8355, + "eval_physics_samples_per_second": 7.957, + "eval_physics_steps_per_second": 1.989, + "step": 1000 + }, + { + "entropy": 0.660948914475739, + "epoch": 1.7975965283186826, + "grad_norm": 1.359375, + "learning_rate": 1.474814814814815e-05, + "loss": 0.7562, + "mean_token_accuracy": 0.8152279850095511, + "num_tokens": 27049648.0, + "step": 1010 + }, + { + "entropy": 0.6787248050794006, + "epoch": 1.8154000222543676, + "grad_norm": 1.3359375, + "learning_rate": 1.4674074074074076e-05, + "loss": 0.7771, + "mean_token_accuracy": 0.8101395227015018, + "num_tokens": 27315061.0, + "step": 1020 + }, + { + "entropy": 0.6424054227769375, + "epoch": 1.8332035161900522, + "grad_norm": 1.359375, + "learning_rate": 1.46e-05, + "loss": 0.7373, + "mean_token_accuracy": 0.8189616728574037, + "num_tokens": 27587650.0, + "step": 1030 + }, + { + "entropy": 0.6606515791267157, + "epoch": 1.8510070101257372, + "grad_norm": 1.15625, + "learning_rate": 1.4525925925925927e-05, + "loss": 0.7652, + "mean_token_accuracy": 0.8157406080514192, + "num_tokens": 27855458.0, + "step": 1040 + }, + { + "entropy": 0.6464399367570877, + "epoch": 1.868810504061422, + "grad_norm": 1.203125, + "learning_rate": 1.4451851851851852e-05, + "loss": 0.7464, + "mean_token_accuracy": 0.8191553246229887, + "num_tokens": 28127703.0, + "step": 1050 + }, + { + "entropy": 0.633565224148333, + "epoch": 1.8866139979971068, + "grad_norm": 1.140625, + "learning_rate": 1.4377777777777779e-05, + "loss": 0.7265, + "mean_token_accuracy": 0.8215953864157199, + "num_tokens": 28395128.0, + "step": 1060 + }, + { + "entropy": 0.651407066360116, + "epoch": 1.9044174919327919, + "grad_norm": 1.2421875, + "learning_rate": 1.4303703703703703e-05, + "loss": 0.752, + "mean_token_accuracy": 0.8161330189555883, + "num_tokens": 28662344.0, + "step": 1070 + }, + { + "entropy": 0.6537632567808032, + "epoch": 1.9222209858684767, + "grad_norm": 1.3125, + "learning_rate": 1.4229629629629632e-05, + "loss": 0.7551, + "mean_token_accuracy": 0.8160409614443779, + "num_tokens": 28928769.0, + "step": 1080 + }, + { + "entropy": 0.6610247550532222, + "epoch": 1.9400244798041615, + "grad_norm": 1.5625, + "learning_rate": 1.4155555555555556e-05, + "loss": 0.7599, + "mean_token_accuracy": 0.8147491872310638, + "num_tokens": 29186214.0, + "step": 1090 + }, + { + "entropy": 0.6632727082818747, + "epoch": 1.9578279737398465, + "grad_norm": 1.2109375, + "learning_rate": 1.4081481481481483e-05, + "loss": 0.7706, + "mean_token_accuracy": 0.8126637618988752, + "num_tokens": 29446445.0, + "step": 1100 + }, + { + "epoch": 1.9578279737398465, + "eval_biology_entropy": 1.639104124546051, + "eval_biology_loss": 2.0316033363342285, + "eval_biology_mean_token_accuracy": 0.615292895078659, + "eval_biology_num_tokens": 29446445.0, + "eval_biology_runtime": 41.3569, + "eval_biology_samples_per_second": 12.09, + "eval_biology_steps_per_second": 3.022, + "step": 1100 + }, + { + "epoch": 1.9578279737398465, + "eval_chemistry_entropy": 0.6696053767204284, + "eval_chemistry_loss": 0.8191399574279785, + "eval_chemistry_mean_token_accuracy": 0.8085824422836304, + "eval_chemistry_num_tokens": 29446445.0, + "eval_chemistry_runtime": 50.7996, + "eval_chemistry_samples_per_second": 9.843, + "eval_chemistry_steps_per_second": 2.461, + "step": 1100 + }, + { + "epoch": 1.9578279737398465, + "eval_math_entropy": 0.6799165282249451, + "eval_math_loss": 1.0633617639541626, + "eval_math_mean_token_accuracy": 0.7729020061492919, + "eval_math_num_tokens": 29446445.0, + "eval_math_runtime": 52.3664, + "eval_math_samples_per_second": 9.548, + "eval_math_steps_per_second": 2.387, + "step": 1100 + }, + { + "epoch": 1.9578279737398465, + "eval_physics_entropy": 0.6961899037361146, + "eval_physics_loss": 0.8565983176231384, + "eval_physics_mean_token_accuracy": 0.7995611958503723, + "eval_physics_num_tokens": 29446445.0, + "eval_physics_runtime": 59.795, + "eval_physics_samples_per_second": 8.362, + "eval_physics_steps_per_second": 2.09, + "step": 1100 + }, + { + "entropy": 0.6516633328050375, + "epoch": 1.9756314676755313, + "grad_norm": 1.21875, + "learning_rate": 1.400740740740741e-05, + "loss": 0.7439, + "mean_token_accuracy": 0.8187774941325188, + "num_tokens": 29711771.0, + "step": 1110 + }, + { + "entropy": 0.6508602360263467, + "epoch": 1.9934349616112161, + "grad_norm": 1.3125, + "learning_rate": 1.3933333333333334e-05, + "loss": 0.7519, + "mean_token_accuracy": 0.8169589649885893, + "num_tokens": 29979994.0, + "step": 1120 + }, + { + "entropy": 0.6306788632946629, + "epoch": 2.010682096361411, + "grad_norm": 1.2734375, + "learning_rate": 1.385925925925926e-05, + "loss": 0.7224, + "mean_token_accuracy": 0.8239769354943306, + "num_tokens": 30240793.0, + "step": 1130 + }, + { + "entropy": 0.5911945894360542, + "epoch": 2.028485590297096, + "grad_norm": 1.296875, + "learning_rate": 1.3785185185185186e-05, + "loss": 0.7, + "mean_token_accuracy": 0.8281658995896578, + "num_tokens": 30503988.0, + "step": 1140 + }, + { + "entropy": 0.6057484762743115, + "epoch": 2.0462890842327806, + "grad_norm": 1.2890625, + "learning_rate": 1.3711111111111112e-05, + "loss": 0.7031, + "mean_token_accuracy": 0.8266915556043386, + "num_tokens": 30771841.0, + "step": 1150 + }, + { + "entropy": 0.596652439981699, + "epoch": 2.0640925781684656, + "grad_norm": 1.375, + "learning_rate": 1.3637037037037037e-05, + "loss": 0.6926, + "mean_token_accuracy": 0.8293926887214184, + "num_tokens": 31045037.0, + "step": 1160 + }, + { + "entropy": 0.6352694857865572, + "epoch": 2.0818960721041506, + "grad_norm": 1.4296875, + "learning_rate": 1.3562962962962965e-05, + "loss": 0.7356, + "mean_token_accuracy": 0.8186026088893413, + "num_tokens": 31299575.0, + "step": 1170 + }, + { + "entropy": 0.6292696729302406, + "epoch": 2.099699566039835, + "grad_norm": 1.390625, + "learning_rate": 1.3488888888888888e-05, + "loss": 0.7273, + "mean_token_accuracy": 0.8201477468013764, + "num_tokens": 31565963.0, + "step": 1180 + }, + { + "entropy": 0.5940109083428979, + "epoch": 2.1175030599755202, + "grad_norm": 1.2734375, + "learning_rate": 1.3414814814814817e-05, + "loss": 0.7035, + "mean_token_accuracy": 0.8278756473213434, + "num_tokens": 31834212.0, + "step": 1190 + }, + { + "entropy": 0.6235881226137281, + "epoch": 2.1353065539112053, + "grad_norm": 1.375, + "learning_rate": 1.3340740740740741e-05, + "loss": 0.7172, + "mean_token_accuracy": 0.8223393179476262, + "num_tokens": 32103356.0, + "step": 1200 + }, + { + "epoch": 2.1353065539112053, + "eval_biology_entropy": 1.4677531628608704, + "eval_biology_loss": 2.083951234817505, + "eval_biology_mean_token_accuracy": 0.6125459330081939, + "eval_biology_num_tokens": 32103356.0, + "eval_biology_runtime": 41.2722, + "eval_biology_samples_per_second": 12.115, + "eval_biology_steps_per_second": 3.029, + "step": 1200 + }, + { + "epoch": 2.1353065539112053, + "eval_chemistry_entropy": 0.6140689237117767, + "eval_chemistry_loss": 0.8232262134552002, + "eval_chemistry_mean_token_accuracy": 0.8088321657180786, + "eval_chemistry_num_tokens": 32103356.0, + "eval_chemistry_runtime": 50.8871, + "eval_chemistry_samples_per_second": 9.826, + "eval_chemistry_steps_per_second": 2.456, + "step": 1200 + }, + { + "epoch": 2.1353065539112053, + "eval_math_entropy": 0.6312986519336701, + "eval_math_loss": 1.0854209661483765, + "eval_math_mean_token_accuracy": 0.7729385108947754, + "eval_math_num_tokens": 32103356.0, + "eval_math_runtime": 52.4669, + "eval_math_samples_per_second": 9.53, + "eval_math_steps_per_second": 2.382, + "step": 1200 + }, + { + "epoch": 2.1353065539112053, + "eval_physics_entropy": 0.6365497968196869, + "eval_physics_loss": 0.8704355359077454, + "eval_physics_mean_token_accuracy": 0.7990298428535462, + "eval_physics_num_tokens": 32103356.0, + "eval_physics_runtime": 59.7275, + "eval_physics_samples_per_second": 8.371, + "eval_physics_steps_per_second": 2.093, + "step": 1200 + }, + { + "entropy": 0.6137512635439635, + "epoch": 2.15311004784689, + "grad_norm": 1.25, + "learning_rate": 1.3266666666666668e-05, + "loss": 0.7043, + "mean_token_accuracy": 0.8242961999028922, + "num_tokens": 32374830.0, + "step": 1210 + }, + { + "entropy": 0.5896952305920422, + "epoch": 2.170913541782575, + "grad_norm": 1.2734375, + "learning_rate": 1.3192592592592594e-05, + "loss": 0.6922, + "mean_token_accuracy": 0.8302250444889069, + "num_tokens": 32643479.0, + "step": 1220 + }, + { + "entropy": 0.5944842832162976, + "epoch": 2.18871703571826, + "grad_norm": 1.296875, + "learning_rate": 1.311851851851852e-05, + "loss": 0.6959, + "mean_token_accuracy": 0.8287800919264555, + "num_tokens": 32908761.0, + "step": 1230 + }, + { + "entropy": 0.6240290224552154, + "epoch": 2.2065205296539445, + "grad_norm": 1.21875, + "learning_rate": 1.3044444444444446e-05, + "loss": 0.7238, + "mean_token_accuracy": 0.8242842797189951, + "num_tokens": 33166413.0, + "step": 1240 + }, + { + "entropy": 0.6112470507621766, + "epoch": 2.2243240235896296, + "grad_norm": 1.1875, + "learning_rate": 1.297037037037037e-05, + "loss": 0.7149, + "mean_token_accuracy": 0.8240214943885803, + "num_tokens": 33432686.0, + "step": 1250 + }, + { + "entropy": 0.6079921290278435, + "epoch": 2.242127517525314, + "grad_norm": 1.4296875, + "learning_rate": 1.2896296296296299e-05, + "loss": 0.7131, + "mean_token_accuracy": 0.8254695866256953, + "num_tokens": 33693242.0, + "step": 1260 + }, + { + "entropy": 0.5799209821969271, + "epoch": 2.259931011460999, + "grad_norm": 1.390625, + "learning_rate": 1.2822222222222222e-05, + "loss": 0.6722, + "mean_token_accuracy": 0.834067041426897, + "num_tokens": 33970820.0, + "step": 1270 + }, + { + "entropy": 0.594209510087967, + "epoch": 2.277734505396684, + "grad_norm": 1.3671875, + "learning_rate": 1.274814814814815e-05, + "loss": 0.6977, + "mean_token_accuracy": 0.8276573233306408, + "num_tokens": 34243042.0, + "step": 1280 + }, + { + "entropy": 0.6097929472103715, + "epoch": 2.295537999332369, + "grad_norm": 1.296875, + "learning_rate": 1.2674074074074075e-05, + "loss": 0.705, + "mean_token_accuracy": 0.8267240136861801, + "num_tokens": 34515236.0, + "step": 1290 + }, + { + "entropy": 0.5745832217857242, + "epoch": 2.313341493268054, + "grad_norm": 1.234375, + "learning_rate": 1.2600000000000001e-05, + "loss": 0.6783, + "mean_token_accuracy": 0.8345583423972129, + "num_tokens": 34785068.0, + "step": 1300 + }, + { + "epoch": 2.313341493268054, + "eval_biology_entropy": 1.497672739982605, + "eval_biology_loss": 2.0853865146636963, + "eval_biology_mean_token_accuracy": 0.613484436750412, + "eval_biology_num_tokens": 34785068.0, + "eval_biology_runtime": 43.469, + "eval_biology_samples_per_second": 11.502, + "eval_biology_steps_per_second": 2.876, + "step": 1300 + }, + { + "epoch": 2.313341493268054, + "eval_chemistry_entropy": 0.6273940706253052, + "eval_chemistry_loss": 0.8201014995574951, + "eval_chemistry_mean_token_accuracy": 0.8088535733222961, + "eval_chemistry_num_tokens": 34785068.0, + "eval_chemistry_runtime": 52.831, + "eval_chemistry_samples_per_second": 9.464, + "eval_chemistry_steps_per_second": 2.366, + "step": 1300 + }, + { + "epoch": 2.313341493268054, + "eval_math_entropy": 0.646281590461731, + "eval_math_loss": 1.0807055234909058, + "eval_math_mean_token_accuracy": 0.772442822933197, + "eval_math_num_tokens": 34785068.0, + "eval_math_runtime": 55.4339, + "eval_math_samples_per_second": 9.02, + "eval_math_steps_per_second": 2.255, + "step": 1300 + }, + { + "epoch": 2.313341493268054, + "eval_physics_entropy": 0.6572135488986969, + "eval_physics_loss": 0.8676888346672058, + "eval_physics_mean_token_accuracy": 0.7990684523582459, + "eval_physics_num_tokens": 34785068.0, + "eval_physics_runtime": 61.4064, + "eval_physics_samples_per_second": 8.142, + "eval_physics_steps_per_second": 2.036, + "step": 1300 + }, + { + "entropy": 0.6191223243251442, + "epoch": 2.331144987203739, + "grad_norm": 1.2265625, + "learning_rate": 1.2525925925925928e-05, + "loss": 0.7122, + "mean_token_accuracy": 0.8234442934393883, + "num_tokens": 35055845.0, + "step": 1310 + }, + { + "entropy": 0.6000411529093981, + "epoch": 2.3489484811394234, + "grad_norm": 1.296875, + "learning_rate": 1.2451851851851853e-05, + "loss": 0.7023, + "mean_token_accuracy": 0.8272519435733556, + "num_tokens": 35327481.0, + "step": 1320 + }, + { + "entropy": 0.6216201702132821, + "epoch": 2.3667519750751085, + "grad_norm": 1.2578125, + "learning_rate": 1.237777777777778e-05, + "loss": 0.7178, + "mean_token_accuracy": 0.8234907556325197, + "num_tokens": 35596362.0, + "step": 1330 + }, + { + "entropy": 0.5952032478526235, + "epoch": 2.3845554690107935, + "grad_norm": 1.3671875, + "learning_rate": 1.2303703703703704e-05, + "loss": 0.695, + "mean_token_accuracy": 0.827592471987009, + "num_tokens": 35869585.0, + "step": 1340 + }, + { + "entropy": 0.6152689423412084, + "epoch": 2.402358962946478, + "grad_norm": 1.4140625, + "learning_rate": 1.222962962962963e-05, + "loss": 0.7087, + "mean_token_accuracy": 0.8254514247179031, + "num_tokens": 36134072.0, + "step": 1350 + }, + { + "entropy": 0.5919087225571275, + "epoch": 2.420162456882163, + "grad_norm": 1.25, + "learning_rate": 1.2155555555555555e-05, + "loss": 0.6971, + "mean_token_accuracy": 0.829882950335741, + "num_tokens": 36406263.0, + "step": 1360 + }, + { + "entropy": 0.6080563787370921, + "epoch": 2.437965950817848, + "grad_norm": 1.3984375, + "learning_rate": 1.2081481481481484e-05, + "loss": 0.7027, + "mean_token_accuracy": 0.8268357224762439, + "num_tokens": 36676822.0, + "step": 1370 + }, + { + "entropy": 0.6048327511176467, + "epoch": 2.4557694447535328, + "grad_norm": 1.421875, + "learning_rate": 1.2007407407407408e-05, + "loss": 0.7047, + "mean_token_accuracy": 0.8271109748631715, + "num_tokens": 36942837.0, + "step": 1380 + }, + { + "entropy": 0.5988074177876115, + "epoch": 2.473572938689218, + "grad_norm": 1.390625, + "learning_rate": 1.1933333333333335e-05, + "loss": 0.7051, + "mean_token_accuracy": 0.8256583914160729, + "num_tokens": 37210791.0, + "step": 1390 + }, + { + "entropy": 0.5873929725959897, + "epoch": 2.491376432624903, + "grad_norm": 1.3671875, + "learning_rate": 1.185925925925926e-05, + "loss": 0.6871, + "mean_token_accuracy": 0.83187070004642, + "num_tokens": 37485822.0, + "step": 1400 + }, + { + "epoch": 2.491376432624903, + "eval_biology_entropy": 1.4652894883155823, + "eval_biology_loss": 2.099928855895996, + "eval_biology_mean_token_accuracy": 0.6130364212989807, + "eval_biology_num_tokens": 37485822.0, + "eval_biology_runtime": 43.3395, + "eval_biology_samples_per_second": 11.537, + "eval_biology_steps_per_second": 2.884, + "step": 1400 + }, + { + "epoch": 2.491376432624903, + "eval_chemistry_entropy": 0.6137497539520264, + "eval_chemistry_loss": 0.8189122080802917, + "eval_chemistry_mean_token_accuracy": 0.8097302551269532, + "eval_chemistry_num_tokens": 37485822.0, + "eval_chemistry_runtime": 52.903, + "eval_chemistry_samples_per_second": 9.451, + "eval_chemistry_steps_per_second": 2.363, + "step": 1400 + }, + { + "epoch": 2.491376432624903, + "eval_math_entropy": 0.6390214171409607, + "eval_math_loss": 1.082951545715332, + "eval_math_mean_token_accuracy": 0.772076274394989, + "eval_math_num_tokens": 37485822.0, + "eval_math_runtime": 54.1184, + "eval_math_samples_per_second": 9.239, + "eval_math_steps_per_second": 2.31, + "step": 1400 + }, + { + "epoch": 2.491376432624903, + "eval_physics_entropy": 0.6406208200454712, + "eval_physics_loss": 0.8697341084480286, + "eval_physics_mean_token_accuracy": 0.7989166221618652, + "eval_physics_num_tokens": 37485822.0, + "eval_physics_runtime": 61.5427, + "eval_physics_samples_per_second": 8.124, + "eval_physics_steps_per_second": 2.031, + "step": 1400 + }, + { + "entropy": 0.6010861510410905, + "epoch": 2.5091799265605874, + "grad_norm": 1.28125, + "learning_rate": 1.1785185185185186e-05, + "loss": 0.7014, + "mean_token_accuracy": 0.8287522595375776, + "num_tokens": 37746150.0, + "step": 1410 + }, + { + "entropy": 0.5981267247349024, + "epoch": 2.5269834204962724, + "grad_norm": 1.3046875, + "learning_rate": 1.1711111111111113e-05, + "loss": 0.6962, + "mean_token_accuracy": 0.8269948281347752, + "num_tokens": 38016863.0, + "step": 1420 + }, + { + "entropy": 0.6277626631781459, + "epoch": 2.544786914431957, + "grad_norm": 1.5078125, + "learning_rate": 1.1637037037037037e-05, + "loss": 0.7222, + "mean_token_accuracy": 0.8205803319811821, + "num_tokens": 38277754.0, + "step": 1430 + }, + { + "entropy": 0.6118797251954675, + "epoch": 2.562590408367642, + "grad_norm": 1.421875, + "learning_rate": 1.1562962962962964e-05, + "loss": 0.7172, + "mean_token_accuracy": 0.8237183388322592, + "num_tokens": 38542761.0, + "step": 1440 + }, + { + "entropy": 0.6034180456772447, + "epoch": 2.580393902303327, + "grad_norm": 1.453125, + "learning_rate": 1.1488888888888889e-05, + "loss": 0.6941, + "mean_token_accuracy": 0.8272098813205957, + "num_tokens": 38820999.0, + "step": 1450 + }, + { + "entropy": 0.5998265566304326, + "epoch": 2.598197396239012, + "grad_norm": 1.3203125, + "learning_rate": 1.1414814814814817e-05, + "loss": 0.7114, + "mean_token_accuracy": 0.8267886329442262, + "num_tokens": 39080016.0, + "step": 1460 + }, + { + "entropy": 0.6005334880203008, + "epoch": 2.6160008901746967, + "grad_norm": 1.3828125, + "learning_rate": 1.1340740740740742e-05, + "loss": 0.6949, + "mean_token_accuracy": 0.8299154710024595, + "num_tokens": 39345434.0, + "step": 1470 + }, + { + "entropy": 0.6003408120945096, + "epoch": 2.6338043841103818, + "grad_norm": 1.46875, + "learning_rate": 1.1266666666666668e-05, + "loss": 0.7033, + "mean_token_accuracy": 0.8266183461993932, + "num_tokens": 39610041.0, + "step": 1480 + }, + { + "entropy": 0.5847593653947115, + "epoch": 2.6516078780460663, + "grad_norm": 1.4140625, + "learning_rate": 1.1192592592592593e-05, + "loss": 0.6785, + "mean_token_accuracy": 0.8323042623698711, + "num_tokens": 39884185.0, + "step": 1490 + }, + { + "entropy": 0.6105206938460469, + "epoch": 2.6694113719817514, + "grad_norm": 1.5546875, + "learning_rate": 1.111851851851852e-05, + "loss": 0.7201, + "mean_token_accuracy": 0.8220306947827339, + "num_tokens": 40143543.0, + "step": 1500 + }, + { + "epoch": 2.6694113719817514, + "eval_biology_entropy": 1.490638756752014, + "eval_biology_loss": 2.0750932693481445, + "eval_biology_mean_token_accuracy": 0.6143683531284332, + "eval_biology_num_tokens": 40143543.0, + "eval_biology_runtime": 43.2253, + "eval_biology_samples_per_second": 11.567, + "eval_biology_steps_per_second": 2.892, + "step": 1500 + }, + { + "epoch": 2.6694113719817514, + "eval_chemistry_entropy": 0.623803097486496, + "eval_chemistry_loss": 0.8160935044288635, + "eval_chemistry_mean_token_accuracy": 0.8098534879684448, + "eval_chemistry_num_tokens": 40143543.0, + "eval_chemistry_runtime": 50.807, + "eval_chemistry_samples_per_second": 9.841, + "eval_chemistry_steps_per_second": 2.46, + "step": 1500 + }, + { + "epoch": 2.6694113719817514, + "eval_math_entropy": 0.645468202829361, + "eval_math_loss": 1.0840471982955933, + "eval_math_mean_token_accuracy": 0.7726303563117981, + "eval_math_num_tokens": 40143543.0, + "eval_math_runtime": 53.0054, + "eval_math_samples_per_second": 9.433, + "eval_math_steps_per_second": 2.358, + "step": 1500 + }, + { + "epoch": 2.6694113719817514, + "eval_physics_entropy": 0.6492172837257385, + "eval_physics_loss": 0.8670127391815186, + "eval_physics_mean_token_accuracy": 0.7991857075691223, + "eval_physics_num_tokens": 40143543.0, + "eval_physics_runtime": 63.9696, + "eval_physics_samples_per_second": 7.816, + "eval_physics_steps_per_second": 1.954, + "step": 1500 + } + ], + "logging_steps": 10, + "max_steps": 3000, + "num_input_tokens_seen": 0, + "num_train_epochs": 6, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 3.1188139735625564e+18, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/checkpoint-1500/training_args.bin b/checkpoint-1500/training_args.bin new file mode 100644 index 0000000..79cad08 --- /dev/null +++ b/checkpoint-1500/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dfc23fa1b9be77c605cba9733debba5f60170f0eadaa2a8866b5aaf4b0e65756 +size 6417 diff --git a/checkpoint-2000/chat_template.jinja b/checkpoint-2000/chat_template.jinja new file mode 100644 index 0000000..923ec25 --- /dev/null +++ b/checkpoint-2000/chat_template.jinja @@ -0,0 +1,4 @@ +{{ bos_token }}{% if messages[0]['role'] == 'system' %}{{ raise_exception('System role not supported') }}{% endif %}{% for message in messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if (message['role'] == 'assistant') %}{% set role = 'model' %}{% else %}{% set role = message['role'] %}{% endif %}{{ '' + role + ' +' + message['content'] | trim + ' +' }}{% endfor %}{% if add_generation_prompt %}{{'model +'}}{% endif %} \ No newline at end of file diff --git a/checkpoint-2000/config.json b/checkpoint-2000/config.json new file mode 100644 index 0000000..29e6cbe --- /dev/null +++ b/checkpoint-2000/config.json @@ -0,0 +1,77 @@ +{ + "architectures": [ + "Gemma2ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "attn_logit_softcapping": 50.0, + "bos_token_id": 2, + "cache_implementation": "hybrid", + "dtype": "bfloat16", + "eos_token_id": 1, + "final_logit_softcapping": 30.0, + "head_dim": 256, + "hidden_act": "gelu_pytorch_tanh", + "hidden_activation": "gelu_pytorch_tanh", + "hidden_size": 3584, + "initializer_range": 0.02, + "intermediate_size": 14336, + "layer_types": [ + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention" + ], + "max_position_embeddings": 8192, + "model_type": "gemma2", + "num_attention_heads": 16, + "num_hidden_layers": 42, + "num_key_value_heads": 8, + "pad_token_id": 0, + "query_pre_attn_scalar": 256, + "rms_norm_eps": 1e-06, + "rope_theta": 10000.0, + "sliding_window": 4096, + "sliding_window_size": 4096, + "transformers_version": "4.56.1", + "use_cache": true, + "vocab_size": 256000 +} diff --git a/checkpoint-2000/generation_config.json b/checkpoint-2000/generation_config.json new file mode 100644 index 0000000..f2c34f2 --- /dev/null +++ b/checkpoint-2000/generation_config.json @@ -0,0 +1,9 @@ +{ + "_from_model_config": true, + "bos_token_id": 2, + "eos_token_id": [ + 1 + ], + "pad_token_id": 0, + "transformers_version": "4.56.1" +} diff --git a/checkpoint-2000/model-00001-of-00004.safetensors b/checkpoint-2000/model-00001-of-00004.safetensors new file mode 100644 index 0000000..4705c7e --- /dev/null +++ b/checkpoint-2000/model-00001-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a0d4eb4fcbddfe01b0dc7a58386a3980826ededaf0ecd67d1ef38995b89615ba +size 4903351912 diff --git a/checkpoint-2000/model-00002-of-00004.safetensors b/checkpoint-2000/model-00002-of-00004.safetensors new file mode 100644 index 0000000..744b59b --- /dev/null +++ b/checkpoint-2000/model-00002-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:712a589c333f49f7b6f3870838a9e0ab4b3c0260abd5130e8c54316e6c99640b +size 4947570872 diff --git a/checkpoint-2000/model-00003-of-00004.safetensors b/checkpoint-2000/model-00003-of-00004.safetensors new file mode 100644 index 0000000..43f462a --- /dev/null +++ b/checkpoint-2000/model-00003-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:191eb96154ce078f60ab835c6f72f38ca55f93099e802d79fc127332b37bb1dc +size 4962221464 diff --git a/checkpoint-2000/model-00004-of-00004.safetensors b/checkpoint-2000/model-00004-of-00004.safetensors new file mode 100644 index 0000000..f9d9361 --- /dev/null +++ b/checkpoint-2000/model-00004-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b5c569dcb813ea00cef62c7609bcfa78da66506cc4f6894347730349a3bab6bc +size 3670322200 diff --git a/checkpoint-2000/model.safetensors.index.json b/checkpoint-2000/model.safetensors.index.json new file mode 100644 index 0000000..3b569c5 --- /dev/null +++ b/checkpoint-2000/model.safetensors.index.json @@ -0,0 +1,472 @@ +{ + "metadata": { + "total_parameters": 9241705984, + "total_size": 18483411968 + }, + "weight_map": { + "model.embed_tokens.weight": "model-00001-of-00004.safetensors", + "model.layers.0.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.0.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.0.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.1.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.1.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.10.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.10.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.10.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.10.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.11.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.11.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.11.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.12.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.12.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.12.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.13.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.13.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.13.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.14.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.14.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.14.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.15.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.15.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.15.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.16.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.16.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.16.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.17.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.17.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.17.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.18.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.18.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.18.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.19.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.19.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.19.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.2.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.2.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.2.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.20.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.20.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.20.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.20.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.20.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.20.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.20.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.21.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.21.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.21.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.21.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.21.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.22.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.22.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.22.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.22.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.23.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.23.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.23.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.24.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.24.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.24.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.25.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.25.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.25.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.26.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.26.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.26.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.27.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.27.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.27.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.28.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.28.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.28.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.29.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.29.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.29.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.3.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.3.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.3.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.30.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.30.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.30.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.30.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.30.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.31.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.31.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.31.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.32.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.32.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.32.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.32.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.32.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.32.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.32.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.32.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.32.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.32.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.32.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.33.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.33.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.33.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.33.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.33.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.33.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.33.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.33.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.33.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.33.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.33.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.34.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.34.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.34.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.34.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.34.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.34.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.34.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.34.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.34.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.34.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.34.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.35.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.35.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.35.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.35.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.35.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.35.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.35.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.35.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.35.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.35.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.35.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.36.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.36.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.36.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.36.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.36.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.36.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.36.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.36.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.36.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.36.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.36.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.37.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.37.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.37.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.37.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.37.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.37.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.37.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.37.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.37.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.37.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.37.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.38.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.38.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.38.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.38.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.38.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.38.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.38.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.38.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.38.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.38.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.38.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.39.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.39.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.39.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.39.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.39.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.39.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.39.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.39.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.39.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.39.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.39.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.4.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.4.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.4.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.4.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.40.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.40.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.40.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.40.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.40.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.40.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.40.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.40.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.40.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.40.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.40.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.41.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.41.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.41.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.41.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.41.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.41.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.41.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.41.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.41.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.41.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.41.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.5.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.5.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.5.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.5.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.6.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.6.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.6.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.7.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.7.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.7.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.8.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.8.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.8.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.8.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.8.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.8.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.8.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.8.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.8.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.8.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.8.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.9.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.9.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.9.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.9.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.norm.weight": "model-00004-of-00004.safetensors" + } +} diff --git a/checkpoint-2000/optimizer.pt b/checkpoint-2000/optimizer.pt new file mode 100644 index 0000000..3efc1d1 --- /dev/null +++ b/checkpoint-2000/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:da5fbdf9faaf1075ec7612dabe96181f8faf23e97a5bf57900f39a96329ef4dd +size 1585596266 diff --git a/checkpoint-2000/rng_state.pth b/checkpoint-2000/rng_state.pth new file mode 100644 index 0000000..52f560a --- /dev/null +++ b/checkpoint-2000/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4132949318e028debc59f4db164bc9a537cbb130ee35d892cb08945d6ab6790a +size 14645 diff --git a/checkpoint-2000/scheduler.pt b/checkpoint-2000/scheduler.pt new file mode 100644 index 0000000..628ca03 --- /dev/null +++ b/checkpoint-2000/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7c56cc7d2ab3dfe9edf7a28cb038a85004572c59f841d9bdfd8a137d5d874b19 +size 1465 diff --git a/checkpoint-2000/special_tokens_map.json b/checkpoint-2000/special_tokens_map.json new file mode 100644 index 0000000..8d6368f --- /dev/null +++ b/checkpoint-2000/special_tokens_map.json @@ -0,0 +1,34 @@ +{ + "additional_special_tokens": [ + "", + "" + ], + "bos_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "unk_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + } +} diff --git a/checkpoint-2000/tokenizer.json b/checkpoint-2000/tokenizer.json new file mode 100644 index 0000000..a4a305d --- /dev/null +++ b/checkpoint-2000/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5f7eee611703c5ce5d1eee32d9cdcfe465647b8aff0c1dfb3bed7ad7dbb05060 +size 34362873 diff --git a/checkpoint-2000/tokenizer.model b/checkpoint-2000/tokenizer.model new file mode 100644 index 0000000..796efe9 --- /dev/null +++ b/checkpoint-2000/tokenizer.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:61a7b147390c64585d6c3543dd6fc636906c9af3865a5548f27f31aee1d4c8e2 +size 4241003 diff --git a/checkpoint-2000/tokenizer_config.json b/checkpoint-2000/tokenizer_config.json new file mode 100644 index 0000000..3ade6be --- /dev/null +++ b/checkpoint-2000/tokenizer_config.json @@ -0,0 +1,2013 @@ +{ + "add_bos_token": true, + "add_eos_token": false, + "added_tokens_decoder": { + "0": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "1": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "2": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "3": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "4": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "5": { + "content": "<2mass>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "6": { + "content": "[@BOS@]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "7": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "8": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "9": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "10": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "11": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "12": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "13": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "14": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "15": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "16": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "17": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "18": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "19": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "20": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "21": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "22": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "23": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "24": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "25": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "26": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "27": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "28": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "29": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "30": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "31": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "32": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "33": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "34": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "35": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "36": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "37": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "38": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "39": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "40": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "41": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "42": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "43": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "44": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "45": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "46": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "47": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "48": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "49": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "50": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "51": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "52": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "53": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "54": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "55": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "56": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "57": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "58": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "59": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "60": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "61": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "62": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "63": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "64": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "65": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "66": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "67": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "68": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "69": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "70": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "71": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "72": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "73": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "74": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "75": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "76": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "77": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "78": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "79": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "80": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "81": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "82": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "83": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "84": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "85": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "86": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "87": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "88": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "89": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "90": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "91": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "92": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "93": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "94": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "95": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "96": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "97": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "98": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "99": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "100": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "101": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "102": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "103": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "104": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "105": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "106": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "107": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "108": { + "content": "\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "109": { + "content": "\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "110": { + "content": "\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "111": { + "content": "\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "112": { + "content": "\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "113": { + "content": "\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "114": { + "content": "\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "115": { + "content": "\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "116": { + "content": "\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "117": { + "content": "\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "118": { + "content": "\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "119": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "120": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "121": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "122": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "123": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "124": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "125": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "126": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "127": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "128": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "129": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "130": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "131": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "132": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "133": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "134": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "135": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "136": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "137": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "138": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "139": { + "content": "▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "140": { + "content": "▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "141": { + "content": "▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "142": { + "content": "▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "143": { + "content": "▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "144": { + "content": "▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "145": { + "content": "▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "146": { + "content": "▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "147": { + "content": "▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "148": { + "content": "▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "149": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "150": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "152": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "153": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "154": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "155": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "156": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "157": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "158": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "159": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "160": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "161": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "162": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "163": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "164": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "165": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "166": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "167": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "168": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "169": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "170": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "172": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "173": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "174": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "175": { + "content": "
", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "171": { + "content": "
", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "176": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "177": { + "content": "
", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "178": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "179": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "180": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "181": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "182": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "183": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "184": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "185": { + "content": "

", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "186": { + "content": "

", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "187": { + "content": "

", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "188": { + "content": "

", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "189": { + "content": "

", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "190": { + "content": "
", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "191": { + "content": "
", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "192": { + "content": "
", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "193": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "194": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "195": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "196": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "197": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "198": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "199": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "200": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "201": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "202": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "203": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "204": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "205": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "206": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "207": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "208": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "209": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "210": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "211": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "212": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "213": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "214": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "215": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "216": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255968": { + "content": "[toxicity=0]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255969": { + "content": "\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255970": { + "content": "\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255971": { + "content": "\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255972": { + "content": "\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255973": { + "content": "\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255974": { + "content": "\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255975": { + "content": "\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255976": { + "content": "\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255977": { + "content": "\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255978": { + "content": "\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255979": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255980": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255981": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255982": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255983": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255984": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255985": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255986": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255987": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255988": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255989": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255990": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255991": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255992": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255993": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255994": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255995": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255996": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255997": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255998": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255999": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + } + }, + "additional_special_tokens": [ + "", + "" + ], + "bos_token": "", + "clean_up_tokenization_spaces": false, + "eos_token": "", + "extra_special_tokens": {}, + "model_max_length": 1000000000000000019884624838656, + "pad_token": "", + "sp_model_kwargs": {}, + "spaces_between_special_tokens": false, + "tokenizer_class": "GemmaTokenizer", + "unk_token": "", + "use_default_system_prompt": false +} diff --git a/checkpoint-2000/trainer_state.json b/checkpoint-2000/trainer_state.json new file mode 100644 index 0000000..f1bd43c --- /dev/null +++ b/checkpoint-2000/trainer_state.json @@ -0,0 +1,2914 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 3.5590297095805052, + "eval_steps": 100, + "global_step": 2000, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.3253214355558156, + "epoch": 0.017803493935684877, + "grad_norm": 14.5, + "learning_rate": 6.000000000000001e-07, + "loss": 2.1959, + "mean_token_accuracy": 0.5975642748177051, + "num_tokens": 259048.0, + "step": 10 + }, + { + "entropy": 1.3036176152527332, + "epoch": 0.035606987871369754, + "grad_norm": 12.75, + "learning_rate": 1.2666666666666669e-06, + "loss": 2.1499, + "mean_token_accuracy": 0.6023047739639878, + "num_tokens": 532019.0, + "step": 20 + }, + { + "entropy": 1.3484113823622466, + "epoch": 0.053410481807054634, + "grad_norm": 10.8125, + "learning_rate": 1.9333333333333336e-06, + "loss": 2.108, + "mean_token_accuracy": 0.6078463492915034, + "num_tokens": 807220.0, + "step": 30 + }, + { + "entropy": 1.4311681490391492, + "epoch": 0.07121397574273951, + "grad_norm": 7.875, + "learning_rate": 2.6e-06, + "loss": 2.0349, + "mean_token_accuracy": 0.6107733339071274, + "num_tokens": 1068429.0, + "step": 40 + }, + { + "entropy": 1.479667642340064, + "epoch": 0.08901746967842439, + "grad_norm": 6.4375, + "learning_rate": 3.266666666666667e-06, + "loss": 1.8649, + "mean_token_accuracy": 0.630409375578165, + "num_tokens": 1336698.0, + "step": 50 + }, + { + "entropy": 1.5030938494950532, + "epoch": 0.10682096361410927, + "grad_norm": 4.1875, + "learning_rate": 3.9333333333333335e-06, + "loss": 1.7217, + "mean_token_accuracy": 0.6490663634613156, + "num_tokens": 1608574.0, + "step": 60 + }, + { + "entropy": 1.5066865295171739, + "epoch": 0.12462445754979415, + "grad_norm": 2.71875, + "learning_rate": 4.600000000000001e-06, + "loss": 1.5716, + "mean_token_accuracy": 0.6711439102888107, + "num_tokens": 1872747.0, + "step": 70 + }, + { + "entropy": 1.3747932206839324, + "epoch": 0.14242795148547902, + "grad_norm": 2.234375, + "learning_rate": 5.2666666666666665e-06, + "loss": 1.4403, + "mean_token_accuracy": 0.6936415441334247, + "num_tokens": 2133173.0, + "step": 80 + }, + { + "entropy": 1.1736511185765266, + "epoch": 0.1602314454211639, + "grad_norm": 1.921875, + "learning_rate": 5.933333333333335e-06, + "loss": 1.2902, + "mean_token_accuracy": 0.7222886722534895, + "num_tokens": 2406145.0, + "step": 90 + }, + { + "entropy": 1.1317794900387526, + "epoch": 0.17803493935684878, + "grad_norm": 1.8046875, + "learning_rate": 6.600000000000001e-06, + "loss": 1.2424, + "mean_token_accuracy": 0.7295983772724867, + "num_tokens": 2663709.0, + "step": 100 + }, + { + "epoch": 0.17803493935684878, + "eval_biology_entropy": 2.3133783559799195, + "eval_biology_loss": 2.7910587787628174, + "eval_biology_mean_token_accuracy": 0.5260232911109924, + "eval_biology_num_tokens": 2663709.0, + "eval_biology_runtime": 39.1656, + "eval_biology_samples_per_second": 12.766, + "eval_biology_steps_per_second": 3.192, + "step": 100 + }, + { + "epoch": 0.17803493935684878, + "eval_chemistry_entropy": 1.0788086318969727, + "eval_chemistry_loss": 1.2403336763381958, + "eval_chemistry_mean_token_accuracy": 0.7357268543243408, + "eval_chemistry_num_tokens": 2663709.0, + "eval_chemistry_runtime": 48.7822, + "eval_chemistry_samples_per_second": 10.25, + "eval_chemistry_steps_per_second": 2.562, + "step": 100 + }, + { + "epoch": 0.17803493935684878, + "eval_math_entropy": 0.938195451259613, + "eval_math_loss": 1.3248047828674316, + "eval_math_mean_token_accuracy": 0.7312402629852295, + "eval_math_num_tokens": 2663709.0, + "eval_math_runtime": 50.197, + "eval_math_samples_per_second": 9.961, + "eval_math_steps_per_second": 2.49, + "step": 100 + }, + { + "epoch": 0.17803493935684878, + "eval_physics_entropy": 1.022246531009674, + "eval_physics_loss": 1.176153302192688, + "eval_physics_mean_token_accuracy": 0.7475795021057129, + "eval_physics_num_tokens": 2663709.0, + "eval_physics_runtime": 57.6895, + "eval_physics_samples_per_second": 8.667, + "eval_physics_steps_per_second": 2.167, + "step": 100 + }, + { + "entropy": 1.0380497876554728, + "epoch": 0.19583843329253367, + "grad_norm": 1.7421875, + "learning_rate": 7.266666666666668e-06, + "loss": 1.157, + "mean_token_accuracy": 0.7445634145289659, + "num_tokens": 2934980.0, + "step": 110 + }, + { + "entropy": 1.0149445166811346, + "epoch": 0.21364192722821854, + "grad_norm": 1.6015625, + "learning_rate": 7.933333333333334e-06, + "loss": 1.1167, + "mean_token_accuracy": 0.7485828153789044, + "num_tokens": 3208702.0, + "step": 120 + }, + { + "entropy": 0.9639404848217964, + "epoch": 0.2314454211639034, + "grad_norm": 1.6953125, + "learning_rate": 8.6e-06, + "loss": 1.0979, + "mean_token_accuracy": 0.7553199239075183, + "num_tokens": 3480119.0, + "step": 130 + }, + { + "entropy": 0.9687324531376362, + "epoch": 0.2492489150995883, + "grad_norm": 1.5625, + "learning_rate": 9.266666666666667e-06, + "loss": 1.0762, + "mean_token_accuracy": 0.757096454873681, + "num_tokens": 3746392.0, + "step": 140 + }, + { + "entropy": 0.9682734729722142, + "epoch": 0.26705240903527316, + "grad_norm": 1.875, + "learning_rate": 9.933333333333334e-06, + "loss": 1.0798, + "mean_token_accuracy": 0.7543047085404396, + "num_tokens": 4019487.0, + "step": 150 + }, + { + "entropy": 0.9270784268155694, + "epoch": 0.28485590297095803, + "grad_norm": 1.6015625, + "learning_rate": 1.0600000000000002e-05, + "loss": 1.047, + "mean_token_accuracy": 0.7619738396257162, + "num_tokens": 4298100.0, + "step": 160 + }, + { + "entropy": 0.9063327137380839, + "epoch": 0.30265939690664295, + "grad_norm": 1.5546875, + "learning_rate": 1.1266666666666668e-05, + "loss": 1.0199, + "mean_token_accuracy": 0.7664302848279476, + "num_tokens": 4568116.0, + "step": 170 + }, + { + "entropy": 0.9037324048578739, + "epoch": 0.3204628908423278, + "grad_norm": 1.7109375, + "learning_rate": 1.1933333333333335e-05, + "loss": 1.0144, + "mean_token_accuracy": 0.7657378144562245, + "num_tokens": 4831334.0, + "step": 180 + }, + { + "entropy": 0.8886868219822646, + "epoch": 0.3382663847780127, + "grad_norm": 1.3359375, + "learning_rate": 1.2600000000000001e-05, + "loss": 0.9874, + "mean_token_accuracy": 0.7726409930735827, + "num_tokens": 5103301.0, + "step": 190 + }, + { + "entropy": 0.8936551520600915, + "epoch": 0.35606987871369755, + "grad_norm": 1.3984375, + "learning_rate": 1.3266666666666668e-05, + "loss": 1.019, + "mean_token_accuracy": 0.7648640714585782, + "num_tokens": 5362915.0, + "step": 200 + }, + { + "epoch": 0.35606987871369755, + "eval_biology_entropy": 2.0171296310424807, + "eval_biology_loss": 2.3780677318573, + "eval_biology_mean_token_accuracy": 0.5706260974407196, + "eval_biology_num_tokens": 5362915.0, + "eval_biology_runtime": 39.1203, + "eval_biology_samples_per_second": 12.781, + "eval_biology_steps_per_second": 3.195, + "step": 200 + }, + { + "epoch": 0.35606987871369755, + "eval_chemistry_entropy": 0.8995157589912415, + "eval_chemistry_loss": 1.022930383682251, + "eval_chemistry_mean_token_accuracy": 0.7701377034187317, + "eval_chemistry_num_tokens": 5362915.0, + "eval_chemistry_runtime": 48.7263, + "eval_chemistry_samples_per_second": 10.261, + "eval_chemistry_steps_per_second": 2.565, + "step": 200 + }, + { + "epoch": 0.35606987871369755, + "eval_math_entropy": 0.8352043969631195, + "eval_math_loss": 1.1696399450302124, + "eval_math_mean_token_accuracy": 0.7531440143585205, + "eval_math_num_tokens": 5362915.0, + "eval_math_runtime": 50.1494, + "eval_math_samples_per_second": 9.97, + "eval_math_steps_per_second": 2.493, + "step": 200 + }, + { + "epoch": 0.35606987871369755, + "eval_physics_entropy": 0.8889413847923279, + "eval_physics_loss": 0.9962027072906494, + "eval_physics_mean_token_accuracy": 0.775176504611969, + "eval_physics_num_tokens": 5362915.0, + "eval_physics_runtime": 57.6533, + "eval_physics_samples_per_second": 8.673, + "eval_physics_steps_per_second": 2.168, + "step": 200 + }, + { + "entropy": 0.8509893301874399, + "epoch": 0.3738733726493824, + "grad_norm": 1.3359375, + "learning_rate": 1.3933333333333334e-05, + "loss": 0.9619, + "mean_token_accuracy": 0.7772600032389164, + "num_tokens": 5625455.0, + "step": 210 + }, + { + "entropy": 0.8560685662552714, + "epoch": 0.39167686658506734, + "grad_norm": 1.4609375, + "learning_rate": 1.46e-05, + "loss": 0.9666, + "mean_token_accuracy": 0.7764565829187632, + "num_tokens": 5890806.0, + "step": 220 + }, + { + "entropy": 0.8282079607248306, + "epoch": 0.4094803605207522, + "grad_norm": 1.3828125, + "learning_rate": 1.5266666666666667e-05, + "loss": 0.9353, + "mean_token_accuracy": 0.7817027729004622, + "num_tokens": 6158751.0, + "step": 230 + }, + { + "entropy": 0.8447791969403624, + "epoch": 0.4272838544564371, + "grad_norm": 1.4296875, + "learning_rate": 1.5933333333333336e-05, + "loss": 0.9428, + "mean_token_accuracy": 0.7795210376381874, + "num_tokens": 6423747.0, + "step": 240 + }, + { + "entropy": 0.8478817906230688, + "epoch": 0.44508734839212194, + "grad_norm": 1.8984375, + "learning_rate": 1.66e-05, + "loss": 0.9542, + "mean_token_accuracy": 0.7772787544876337, + "num_tokens": 6684517.0, + "step": 250 + }, + { + "entropy": 0.8135418241843582, + "epoch": 0.4628908423278068, + "grad_norm": 1.453125, + "learning_rate": 1.726666666666667e-05, + "loss": 0.9297, + "mean_token_accuracy": 0.783412617444992, + "num_tokens": 6945659.0, + "step": 260 + }, + { + "entropy": 0.8251218661665917, + "epoch": 0.48069433626349173, + "grad_norm": 1.6640625, + "learning_rate": 1.7933333333333333e-05, + "loss": 0.9307, + "mean_token_accuracy": 0.7831020273268223, + "num_tokens": 7217797.0, + "step": 270 + }, + { + "entropy": 0.7879339458420873, + "epoch": 0.4984978301991766, + "grad_norm": 1.328125, + "learning_rate": 1.86e-05, + "loss": 0.8795, + "mean_token_accuracy": 0.7897050529718399, + "num_tokens": 7498182.0, + "step": 280 + }, + { + "entropy": 0.8145477209240198, + "epoch": 0.5163013241348615, + "grad_norm": 1.265625, + "learning_rate": 1.926666666666667e-05, + "loss": 0.9187, + "mean_token_accuracy": 0.7838029216974973, + "num_tokens": 7761686.0, + "step": 290 + }, + { + "entropy": 0.8231891892850399, + "epoch": 0.5341048180705463, + "grad_norm": 1.5625, + "learning_rate": 1.9933333333333334e-05, + "loss": 0.9271, + "mean_token_accuracy": 0.7819279827177524, + "num_tokens": 8026923.0, + "step": 300 + }, + { + "epoch": 0.5341048180705463, + "eval_biology_entropy": 1.8821040544509888, + "eval_biology_loss": 2.226550340652466, + "eval_biology_mean_token_accuracy": 0.5873979187011719, + "eval_biology_num_tokens": 8026923.0, + "eval_biology_runtime": 39.1096, + "eval_biology_samples_per_second": 12.785, + "eval_biology_steps_per_second": 3.196, + "step": 300 + }, + { + "epoch": 0.5341048180705463, + "eval_chemistry_entropy": 0.7922393922805786, + "eval_chemistry_loss": 0.9438055157661438, + "eval_chemistry_mean_token_accuracy": 0.7841580080986023, + "eval_chemistry_num_tokens": 8026923.0, + "eval_chemistry_runtime": 48.7268, + "eval_chemistry_samples_per_second": 10.261, + "eval_chemistry_steps_per_second": 2.565, + "step": 300 + }, + { + "epoch": 0.5341048180705463, + "eval_math_entropy": 0.7649447858333588, + "eval_math_loss": 1.1145466566085815, + "eval_math_mean_token_accuracy": 0.763204463005066, + "eval_math_num_tokens": 8026923.0, + "eval_math_runtime": 50.1205, + "eval_math_samples_per_second": 9.976, + "eval_math_steps_per_second": 2.494, + "step": 300 + }, + { + "epoch": 0.5341048180705463, + "eval_physics_entropy": 0.7964776389598847, + "eval_physics_loss": 0.9373180866241455, + "eval_physics_mean_token_accuracy": 0.784841582775116, + "eval_physics_num_tokens": 8026923.0, + "eval_physics_runtime": 57.6222, + "eval_physics_samples_per_second": 8.677, + "eval_physics_steps_per_second": 2.169, + "step": 300 + }, + { + "entropy": 0.800599187426269, + "epoch": 0.5519083120062312, + "grad_norm": 1.6640625, + "learning_rate": 1.9933333333333334e-05, + "loss": 0.9071, + "mean_token_accuracy": 0.7856069568544626, + "num_tokens": 8301506.0, + "step": 310 + }, + { + "entropy": 0.7965721761807799, + "epoch": 0.5697118059419161, + "grad_norm": 1.234375, + "learning_rate": 1.985925925925926e-05, + "loss": 0.9139, + "mean_token_accuracy": 0.786777526140213, + "num_tokens": 8559126.0, + "step": 320 + }, + { + "entropy": 0.7925007566809654, + "epoch": 0.587515299877601, + "grad_norm": 1.4453125, + "learning_rate": 1.9785185185185187e-05, + "loss": 0.906, + "mean_token_accuracy": 0.7864235289394855, + "num_tokens": 8821088.0, + "step": 330 + }, + { + "entropy": 0.7728251311928034, + "epoch": 0.6053187938132859, + "grad_norm": 1.2109375, + "learning_rate": 1.971111111111111e-05, + "loss": 0.8764, + "mean_token_accuracy": 0.7919386122375727, + "num_tokens": 9089866.0, + "step": 340 + }, + { + "entropy": 0.7705478269606829, + "epoch": 0.6231222877489707, + "grad_norm": 1.3515625, + "learning_rate": 1.963703703703704e-05, + "loss": 0.8837, + "mean_token_accuracy": 0.7896027479320764, + "num_tokens": 9358866.0, + "step": 350 + }, + { + "entropy": 0.7656249281018972, + "epoch": 0.6409257816846556, + "grad_norm": 1.1875, + "learning_rate": 1.9562962962962964e-05, + "loss": 0.8626, + "mean_token_accuracy": 0.7931958142668009, + "num_tokens": 9629977.0, + "step": 360 + }, + { + "entropy": 0.7598591955378652, + "epoch": 0.6587292756203404, + "grad_norm": 1.296875, + "learning_rate": 1.948888888888889e-05, + "loss": 0.8608, + "mean_token_accuracy": 0.7945109561085701, + "num_tokens": 9897757.0, + "step": 370 + }, + { + "entropy": 0.7398761732503771, + "epoch": 0.6765327695560254, + "grad_norm": 1.265625, + "learning_rate": 1.9414814814814817e-05, + "loss": 0.848, + "mean_token_accuracy": 0.7985429972410202, + "num_tokens": 10168367.0, + "step": 380 + }, + { + "entropy": 0.7331193052232265, + "epoch": 0.6943362634917103, + "grad_norm": 1.4296875, + "learning_rate": 1.9340740740740743e-05, + "loss": 0.8366, + "mean_token_accuracy": 0.8001827634871006, + "num_tokens": 10439039.0, + "step": 390 + }, + { + "entropy": 0.7496613372117281, + "epoch": 0.7121397574273951, + "grad_norm": 1.15625, + "learning_rate": 1.926666666666667e-05, + "loss": 0.8517, + "mean_token_accuracy": 0.7967161379754544, + "num_tokens": 10709549.0, + "step": 400 + }, + { + "epoch": 0.7121397574273951, + "eval_biology_entropy": 1.823248610496521, + "eval_biology_loss": 2.159681558609009, + "eval_biology_mean_token_accuracy": 0.5972214796543122, + "eval_biology_num_tokens": 10709549.0, + "eval_biology_runtime": 39.2024, + "eval_biology_samples_per_second": 12.754, + "eval_biology_steps_per_second": 3.189, + "step": 400 + }, + { + "epoch": 0.7121397574273951, + "eval_chemistry_entropy": 0.7672218294143677, + "eval_chemistry_loss": 0.8971230387687683, + "eval_chemistry_mean_token_accuracy": 0.7928389143943787, + "eval_chemistry_num_tokens": 10709549.0, + "eval_chemistry_runtime": 48.8296, + "eval_chemistry_samples_per_second": 10.24, + "eval_chemistry_steps_per_second": 2.56, + "step": 400 + }, + { + "epoch": 0.7121397574273951, + "eval_math_entropy": 0.7494234163761139, + "eval_math_loss": 1.0849554538726807, + "eval_math_mean_token_accuracy": 0.7666959943771362, + "eval_math_num_tokens": 10709549.0, + "eval_math_runtime": 50.2325, + "eval_math_samples_per_second": 9.954, + "eval_math_steps_per_second": 2.488, + "step": 400 + }, + { + "epoch": 0.7121397574273951, + "eval_physics_entropy": 0.7731240901947022, + "eval_physics_loss": 0.9018430709838867, + "eval_physics_mean_token_accuracy": 0.7909709796905517, + "eval_physics_num_tokens": 10709549.0, + "eval_physics_runtime": 57.7809, + "eval_physics_samples_per_second": 8.653, + "eval_physics_steps_per_second": 2.163, + "step": 400 + }, + { + "entropy": 0.7354462265968322, + "epoch": 0.72994325136308, + "grad_norm": 1.3125, + "learning_rate": 1.9192592592592593e-05, + "loss": 0.8411, + "mean_token_accuracy": 0.8003354988992214, + "num_tokens": 10976011.0, + "step": 410 + }, + { + "entropy": 0.7435122076421976, + "epoch": 0.7477467452987648, + "grad_norm": 1.15625, + "learning_rate": 1.911851851851852e-05, + "loss": 0.8438, + "mean_token_accuracy": 0.7971596572548151, + "num_tokens": 11245251.0, + "step": 420 + }, + { + "entropy": 0.759309609234333, + "epoch": 0.7655502392344498, + "grad_norm": 1.375, + "learning_rate": 1.9044444444444446e-05, + "loss": 0.8579, + "mean_token_accuracy": 0.792990094423294, + "num_tokens": 11518514.0, + "step": 430 + }, + { + "entropy": 0.7545185785740613, + "epoch": 0.7833537331701347, + "grad_norm": 1.3125, + "learning_rate": 1.8970370370370372e-05, + "loss": 0.8574, + "mean_token_accuracy": 0.794682926684618, + "num_tokens": 11787117.0, + "step": 440 + }, + { + "entropy": 0.751254404336214, + "epoch": 0.8011572271058195, + "grad_norm": 1.515625, + "learning_rate": 1.8896296296296295e-05, + "loss": 0.8495, + "mean_token_accuracy": 0.7972703959792853, + "num_tokens": 12050993.0, + "step": 450 + }, + { + "entropy": 0.7543802969157696, + "epoch": 0.8189607210415044, + "grad_norm": 1.28125, + "learning_rate": 1.8822222222222225e-05, + "loss": 0.8527, + "mean_token_accuracy": 0.7960003320127725, + "num_tokens": 12311688.0, + "step": 460 + }, + { + "entropy": 0.7186065271496773, + "epoch": 0.8367642149771892, + "grad_norm": 1.15625, + "learning_rate": 1.874814814814815e-05, + "loss": 0.8096, + "mean_token_accuracy": 0.8027401898056269, + "num_tokens": 12598408.0, + "step": 470 + }, + { + "entropy": 0.7257912319153548, + "epoch": 0.8545677089128741, + "grad_norm": 1.125, + "learning_rate": 1.8674074074074075e-05, + "loss": 0.8299, + "mean_token_accuracy": 0.8012042317539454, + "num_tokens": 12869642.0, + "step": 480 + }, + { + "entropy": 0.7173270899802446, + "epoch": 0.8723712028485591, + "grad_norm": 1.125, + "learning_rate": 1.86e-05, + "loss": 0.8146, + "mean_token_accuracy": 0.8032398864626884, + "num_tokens": 13139555.0, + "step": 490 + }, + { + "entropy": 0.7487573977559805, + "epoch": 0.8901746967842439, + "grad_norm": 1.2421875, + "learning_rate": 1.8525925925925928e-05, + "loss": 0.8562, + "mean_token_accuracy": 0.7955521304160357, + "num_tokens": 13396545.0, + "step": 500 + }, + { + "epoch": 0.8901746967842439, + "eval_biology_entropy": 1.8204833917617798, + "eval_biology_loss": 2.1014022827148438, + "eval_biology_mean_token_accuracy": 0.6051526765823364, + "eval_biology_num_tokens": 13396545.0, + "eval_biology_runtime": 39.1543, + "eval_biology_samples_per_second": 12.77, + "eval_biology_steps_per_second": 3.192, + "step": 500 + }, + { + "epoch": 0.8901746967842439, + "eval_chemistry_entropy": 0.7538458592891694, + "eval_chemistry_loss": 0.8699584007263184, + "eval_chemistry_mean_token_accuracy": 0.7975720787048339, + "eval_chemistry_num_tokens": 13396545.0, + "eval_chemistry_runtime": 48.8229, + "eval_chemistry_samples_per_second": 10.241, + "eval_chemistry_steps_per_second": 2.56, + "step": 500 + }, + { + "epoch": 0.8901746967842439, + "eval_math_entropy": 0.7398368701934814, + "eval_math_loss": 1.0758970975875854, + "eval_math_mean_token_accuracy": 0.7687290472984314, + "eval_math_num_tokens": 13396545.0, + "eval_math_runtime": 50.0916, + "eval_math_samples_per_second": 9.982, + "eval_math_steps_per_second": 2.495, + "step": 500 + }, + { + "epoch": 0.8901746967842439, + "eval_physics_entropy": 0.7607374501228332, + "eval_physics_loss": 0.8848790526390076, + "eval_physics_mean_token_accuracy": 0.7937801394462586, + "eval_physics_num_tokens": 13396545.0, + "eval_physics_runtime": 57.6599, + "eval_physics_samples_per_second": 8.672, + "eval_physics_steps_per_second": 2.168, + "step": 500 + }, + { + "entropy": 0.7129743231460453, + "epoch": 0.9079781907199288, + "grad_norm": 1.1640625, + "learning_rate": 1.8451851851851855e-05, + "loss": 0.8109, + "mean_token_accuracy": 0.8060642663389445, + "num_tokens": 13667857.0, + "step": 510 + }, + { + "entropy": 0.7393450669944286, + "epoch": 0.9257816846556136, + "grad_norm": 1.1484375, + "learning_rate": 1.8377777777777778e-05, + "loss": 0.8325, + "mean_token_accuracy": 0.7997861213982105, + "num_tokens": 13932771.0, + "step": 520 + }, + { + "entropy": 0.7191115407273173, + "epoch": 0.9435851785912985, + "grad_norm": 1.1640625, + "learning_rate": 1.8303703703703704e-05, + "loss": 0.8233, + "mean_token_accuracy": 0.8034567594528198, + "num_tokens": 14194632.0, + "step": 530 + }, + { + "entropy": 0.7434917811304331, + "epoch": 0.9613886725269835, + "grad_norm": 1.1484375, + "learning_rate": 1.822962962962963e-05, + "loss": 0.8435, + "mean_token_accuracy": 0.7969928599894047, + "num_tokens": 14457650.0, + "step": 540 + }, + { + "entropy": 0.7100320702418685, + "epoch": 0.9791921664626683, + "grad_norm": 1.125, + "learning_rate": 1.8155555555555557e-05, + "loss": 0.8149, + "mean_token_accuracy": 0.803739969804883, + "num_tokens": 14724967.0, + "step": 550 + }, + { + "entropy": 0.737933493219316, + "epoch": 0.9969956603983532, + "grad_norm": 1.1875, + "learning_rate": 1.8081481481481484e-05, + "loss": 0.8421, + "mean_token_accuracy": 0.8001800544559956, + "num_tokens": 14993272.0, + "step": 560 + }, + { + "entropy": 0.6655864379098339, + "epoch": 1.014242795148548, + "grad_norm": 1.1171875, + "learning_rate": 1.800740740740741e-05, + "loss": 0.7668, + "mean_token_accuracy": 0.814733551779101, + "num_tokens": 15254936.0, + "step": 570 + }, + { + "entropy": 0.6937626471742988, + "epoch": 1.0320462890842328, + "grad_norm": 1.2578125, + "learning_rate": 1.7933333333333333e-05, + "loss": 0.7942, + "mean_token_accuracy": 0.806296594068408, + "num_tokens": 15515007.0, + "step": 580 + }, + { + "entropy": 0.6596947606652975, + "epoch": 1.0498497830199176, + "grad_norm": 1.2890625, + "learning_rate": 1.785925925925926e-05, + "loss": 0.7623, + "mean_token_accuracy": 0.8152227636426688, + "num_tokens": 15794203.0, + "step": 590 + }, + { + "entropy": 0.6868148937821388, + "epoch": 1.0676532769556026, + "grad_norm": 1.1640625, + "learning_rate": 1.7785185185185186e-05, + "loss": 0.7827, + "mean_token_accuracy": 0.8101296100765467, + "num_tokens": 16063664.0, + "step": 600 + }, + { + "epoch": 1.0676532769556026, + "eval_biology_entropy": 1.6867316131591796, + "eval_biology_loss": 2.077732801437378, + "eval_biology_mean_token_accuracy": 0.6078621828556061, + "eval_biology_num_tokens": 16063664.0, + "eval_biology_runtime": 39.1079, + "eval_biology_samples_per_second": 12.785, + "eval_biology_steps_per_second": 3.196, + "step": 600 + }, + { + "epoch": 1.0676532769556026, + "eval_chemistry_entropy": 0.6882545161247253, + "eval_chemistry_loss": 0.8549924492835999, + "eval_chemistry_mean_token_accuracy": 0.8009334373474121, + "eval_chemistry_num_tokens": 16063664.0, + "eval_chemistry_runtime": 48.732, + "eval_chemistry_samples_per_second": 10.26, + "eval_chemistry_steps_per_second": 2.565, + "step": 600 + }, + { + "epoch": 1.0676532769556026, + "eval_math_entropy": 0.6883157949447631, + "eval_math_loss": 1.075770616531372, + "eval_math_mean_token_accuracy": 0.7705092186927796, + "eval_math_num_tokens": 16063664.0, + "eval_math_runtime": 50.1345, + "eval_math_samples_per_second": 9.973, + "eval_math_steps_per_second": 2.493, + "step": 600 + }, + { + "epoch": 1.0676532769556026, + "eval_physics_entropy": 0.7055722939968109, + "eval_physics_loss": 0.8771266937255859, + "eval_physics_mean_token_accuracy": 0.7955313258171082, + "eval_physics_num_tokens": 16063664.0, + "eval_physics_runtime": 57.6658, + "eval_physics_samples_per_second": 8.671, + "eval_physics_steps_per_second": 2.168, + "step": 600 + }, + { + "entropy": 0.6597639428451657, + "epoch": 1.0854567708912874, + "grad_norm": 1.203125, + "learning_rate": 1.7711111111111113e-05, + "loss": 0.7655, + "mean_token_accuracy": 0.8151189126074314, + "num_tokens": 16338303.0, + "step": 610 + }, + { + "entropy": 0.6991181125864386, + "epoch": 1.1032602648269723, + "grad_norm": 1.2734375, + "learning_rate": 1.763703703703704e-05, + "loss": 0.7934, + "mean_token_accuracy": 0.8071035247296094, + "num_tokens": 16606302.0, + "step": 620 + }, + { + "entropy": 0.6774254593998194, + "epoch": 1.121063758762657, + "grad_norm": 1.28125, + "learning_rate": 1.7562962962962962e-05, + "loss": 0.7706, + "mean_token_accuracy": 0.8098537240177393, + "num_tokens": 16877071.0, + "step": 630 + }, + { + "entropy": 0.6678685389459134, + "epoch": 1.138867252698342, + "grad_norm": 1.203125, + "learning_rate": 1.7488888888888892e-05, + "loss": 0.7714, + "mean_token_accuracy": 0.8127609983086586, + "num_tokens": 17145775.0, + "step": 640 + }, + { + "entropy": 0.6851805068552494, + "epoch": 1.156670746634027, + "grad_norm": 1.2890625, + "learning_rate": 1.7414814814814815e-05, + "loss": 0.777, + "mean_token_accuracy": 0.8110932789742946, + "num_tokens": 17410775.0, + "step": 650 + }, + { + "entropy": 0.7042925998568534, + "epoch": 1.1744742405697117, + "grad_norm": 1.375, + "learning_rate": 1.7340740740740742e-05, + "loss": 0.7979, + "mean_token_accuracy": 0.8038393501192331, + "num_tokens": 17677791.0, + "step": 660 + }, + { + "entropy": 0.6810087833553553, + "epoch": 1.1922777345053968, + "grad_norm": 1.1484375, + "learning_rate": 1.726666666666667e-05, + "loss": 0.7688, + "mean_token_accuracy": 0.8104576140642166, + "num_tokens": 17949441.0, + "step": 670 + }, + { + "entropy": 0.662551742233336, + "epoch": 1.2100812284410816, + "grad_norm": 1.2109375, + "learning_rate": 1.7192592592592595e-05, + "loss": 0.7653, + "mean_token_accuracy": 0.8126230895519256, + "num_tokens": 18217742.0, + "step": 680 + }, + { + "entropy": 0.6803617887198925, + "epoch": 1.2278847223767664, + "grad_norm": 1.2421875, + "learning_rate": 1.711851851851852e-05, + "loss": 0.7771, + "mean_token_accuracy": 0.8095329128205776, + "num_tokens": 18487769.0, + "step": 690 + }, + { + "entropy": 0.6707866100594402, + "epoch": 1.2456882163124514, + "grad_norm": 1.34375, + "learning_rate": 1.7044444444444445e-05, + "loss": 0.7687, + "mean_token_accuracy": 0.8113945126533508, + "num_tokens": 18760296.0, + "step": 700 + }, + { + "epoch": 1.2456882163124514, + "eval_biology_entropy": 1.6589141931533813, + "eval_biology_loss": 2.081825017929077, + "eval_biology_mean_token_accuracy": 0.608892562866211, + "eval_biology_num_tokens": 18760296.0, + "eval_biology_runtime": 39.8083, + "eval_biology_samples_per_second": 12.56, + "eval_biology_steps_per_second": 3.14, + "step": 700 + }, + { + "epoch": 1.2456882163124514, + "eval_chemistry_entropy": 0.6833467047214508, + "eval_chemistry_loss": 0.8464663028717041, + "eval_chemistry_mean_token_accuracy": 0.80278346824646, + "eval_chemistry_num_tokens": 18760296.0, + "eval_chemistry_runtime": 49.1467, + "eval_chemistry_samples_per_second": 10.174, + "eval_chemistry_steps_per_second": 2.543, + "step": 700 + }, + { + "epoch": 1.2456882163124514, + "eval_math_entropy": 0.6947779231071473, + "eval_math_loss": 1.0786137580871582, + "eval_math_mean_token_accuracy": 0.7709644784927369, + "eval_math_num_tokens": 18760296.0, + "eval_math_runtime": 52.5755, + "eval_math_samples_per_second": 9.51, + "eval_math_steps_per_second": 2.378, + "step": 700 + }, + { + "epoch": 1.2456882163124514, + "eval_physics_entropy": 0.704109364271164, + "eval_physics_loss": 0.8724388480186462, + "eval_physics_mean_token_accuracy": 0.796642897605896, + "eval_physics_num_tokens": 18760296.0, + "eval_physics_runtime": 61.3658, + "eval_physics_samples_per_second": 8.148, + "eval_physics_steps_per_second": 2.037, + "step": 700 + }, + { + "entropy": 0.6705936685204505, + "epoch": 1.2634917102481362, + "grad_norm": 1.140625, + "learning_rate": 1.697037037037037e-05, + "loss": 0.7618, + "mean_token_accuracy": 0.8144830856472254, + "num_tokens": 19035733.0, + "step": 710 + }, + { + "entropy": 0.6870324395596981, + "epoch": 1.281295204183821, + "grad_norm": 1.1953125, + "learning_rate": 1.6896296296296298e-05, + "loss": 0.7849, + "mean_token_accuracy": 0.807737946882844, + "num_tokens": 19311432.0, + "step": 720 + }, + { + "entropy": 0.6551721351221204, + "epoch": 1.299098698119506, + "grad_norm": 1.0859375, + "learning_rate": 1.6822222222222224e-05, + "loss": 0.7538, + "mean_token_accuracy": 0.8164622981101275, + "num_tokens": 19593016.0, + "step": 730 + }, + { + "entropy": 0.6645344085991383, + "epoch": 1.3169021920551909, + "grad_norm": 1.1796875, + "learning_rate": 1.6748148148148147e-05, + "loss": 0.759, + "mean_token_accuracy": 0.8134492743760348, + "num_tokens": 19860476.0, + "step": 740 + }, + { + "entropy": 0.6684485200792551, + "epoch": 1.3347056859908757, + "grad_norm": 1.3515625, + "learning_rate": 1.6674074074074077e-05, + "loss": 0.7632, + "mean_token_accuracy": 0.8128790140151978, + "num_tokens": 20124436.0, + "step": 750 + }, + { + "entropy": 0.6736280497163534, + "epoch": 1.3525091799265607, + "grad_norm": 1.25, + "learning_rate": 1.66e-05, + "loss": 0.7771, + "mean_token_accuracy": 0.813029420748353, + "num_tokens": 20388027.0, + "step": 760 + }, + { + "entropy": 0.6706711670383811, + "epoch": 1.3703126738622455, + "grad_norm": 1.265625, + "learning_rate": 1.6525925925925927e-05, + "loss": 0.7697, + "mean_token_accuracy": 0.8115393359214067, + "num_tokens": 20656076.0, + "step": 770 + }, + { + "entropy": 0.6996258163824678, + "epoch": 1.3881161677979303, + "grad_norm": 1.2578125, + "learning_rate": 1.6451851851851853e-05, + "loss": 0.8077, + "mean_token_accuracy": 0.8058754079043865, + "num_tokens": 20911633.0, + "step": 780 + }, + { + "entropy": 0.6905225297436118, + "epoch": 1.4059196617336152, + "grad_norm": 1.171875, + "learning_rate": 1.637777777777778e-05, + "loss": 0.7955, + "mean_token_accuracy": 0.8070559553802014, + "num_tokens": 21176227.0, + "step": 790 + }, + { + "entropy": 0.6718146029859782, + "epoch": 1.4237231556693, + "grad_norm": 1.2890625, + "learning_rate": 1.6303703703703706e-05, + "loss": 0.7716, + "mean_token_accuracy": 0.8126390781253576, + "num_tokens": 21445242.0, + "step": 800 + }, + { + "epoch": 1.4237231556693, + "eval_biology_entropy": 1.6255396161079407, + "eval_biology_loss": 2.0511693954467773, + "eval_biology_mean_token_accuracy": 0.6119143316745758, + "eval_biology_num_tokens": 21445242.0, + "eval_biology_runtime": 42.4445, + "eval_biology_samples_per_second": 11.78, + "eval_biology_steps_per_second": 2.945, + "step": 800 + }, + { + "epoch": 1.4237231556693, + "eval_chemistry_entropy": 0.6763756844997406, + "eval_chemistry_loss": 0.8368268609046936, + "eval_chemistry_mean_token_accuracy": 0.8050474629402161, + "eval_chemistry_num_tokens": 21445242.0, + "eval_chemistry_runtime": 51.5104, + "eval_chemistry_samples_per_second": 9.707, + "eval_chemistry_steps_per_second": 2.427, + "step": 800 + }, + { + "epoch": 1.4237231556693, + "eval_math_entropy": 0.6853804645538331, + "eval_math_loss": 1.0698176622390747, + "eval_math_mean_token_accuracy": 0.7714386968612671, + "eval_math_num_tokens": 21445242.0, + "eval_math_runtime": 54.9388, + "eval_math_samples_per_second": 9.101, + "eval_math_steps_per_second": 2.275, + "step": 800 + }, + { + "epoch": 1.4237231556693, + "eval_physics_entropy": 0.6991692934036254, + "eval_physics_loss": 0.8673914074897766, + "eval_physics_mean_token_accuracy": 0.7979785161018371, + "eval_physics_num_tokens": 21445242.0, + "eval_physics_runtime": 60.9527, + "eval_physics_samples_per_second": 8.203, + "eval_physics_steps_per_second": 2.051, + "step": 800 + }, + { + "entropy": 0.6695453573018313, + "epoch": 1.441526649604985, + "grad_norm": 1.3125, + "learning_rate": 1.622962962962963e-05, + "loss": 0.77, + "mean_token_accuracy": 0.8109262142330408, + "num_tokens": 21718455.0, + "step": 810 + }, + { + "entropy": 0.6753543723374605, + "epoch": 1.4593301435406698, + "grad_norm": 1.2734375, + "learning_rate": 1.6155555555555556e-05, + "loss": 0.7782, + "mean_token_accuracy": 0.8130301378667355, + "num_tokens": 21974076.0, + "step": 820 + }, + { + "entropy": 0.6758085643872619, + "epoch": 1.4771336374763546, + "grad_norm": 1.2265625, + "learning_rate": 1.6081481481481482e-05, + "loss": 0.7768, + "mean_token_accuracy": 0.8110865034162998, + "num_tokens": 22241599.0, + "step": 830 + }, + { + "entropy": 0.6868746858090162, + "epoch": 1.4949371314120397, + "grad_norm": 1.34375, + "learning_rate": 1.600740740740741e-05, + "loss": 0.7818, + "mean_token_accuracy": 0.8090599600225687, + "num_tokens": 22510725.0, + "step": 840 + }, + { + "entropy": 0.6554480012506246, + "epoch": 1.5127406253477245, + "grad_norm": 1.1796875, + "learning_rate": 1.5933333333333336e-05, + "loss": 0.766, + "mean_token_accuracy": 0.8149232871830463, + "num_tokens": 22778514.0, + "step": 850 + }, + { + "entropy": 0.6574800366535782, + "epoch": 1.5305441192834093, + "grad_norm": 1.2734375, + "learning_rate": 1.5859259259259262e-05, + "loss": 0.7517, + "mean_token_accuracy": 0.8147028144448996, + "num_tokens": 23051230.0, + "step": 860 + }, + { + "entropy": 0.6707898294553161, + "epoch": 1.5483476132190943, + "grad_norm": 1.21875, + "learning_rate": 1.5785185185185185e-05, + "loss": 0.776, + "mean_token_accuracy": 0.8121768556535244, + "num_tokens": 23306327.0, + "step": 870 + }, + { + "entropy": 0.6759747974574566, + "epoch": 1.5661511071547791, + "grad_norm": 1.234375, + "learning_rate": 1.571111111111111e-05, + "loss": 0.7717, + "mean_token_accuracy": 0.810706127434969, + "num_tokens": 23573127.0, + "step": 880 + }, + { + "entropy": 0.6696014609187841, + "epoch": 1.583954601090464, + "grad_norm": 1.1953125, + "learning_rate": 1.5637037037037038e-05, + "loss": 0.7635, + "mean_token_accuracy": 0.8123903065919876, + "num_tokens": 23841914.0, + "step": 890 + }, + { + "entropy": 0.6680066948756576, + "epoch": 1.601758095026149, + "grad_norm": 1.28125, + "learning_rate": 1.5562962962962965e-05, + "loss": 0.7763, + "mean_token_accuracy": 0.8117439385503531, + "num_tokens": 24105114.0, + "step": 900 + }, + { + "epoch": 1.601758095026149, + "eval_biology_entropy": 1.6559728245735168, + "eval_biology_loss": 2.0387370586395264, + "eval_biology_mean_token_accuracy": 0.6136598820686341, + "eval_biology_num_tokens": 24105114.0, + "eval_biology_runtime": 41.4479, + "eval_biology_samples_per_second": 12.063, + "eval_biology_steps_per_second": 3.016, + "step": 900 + }, + { + "epoch": 1.601758095026149, + "eval_chemistry_entropy": 0.683923549413681, + "eval_chemistry_loss": 0.8298754692077637, + "eval_chemistry_mean_token_accuracy": 0.8061454033851624, + "eval_chemistry_num_tokens": 24105114.0, + "eval_chemistry_runtime": 50.8984, + "eval_chemistry_samples_per_second": 9.823, + "eval_chemistry_steps_per_second": 2.456, + "step": 900 + }, + { + "epoch": 1.601758095026149, + "eval_math_entropy": 0.6932375190258027, + "eval_math_loss": 1.06475830078125, + "eval_math_mean_token_accuracy": 0.7720690259933471, + "eval_math_num_tokens": 24105114.0, + "eval_math_runtime": 52.5544, + "eval_math_samples_per_second": 9.514, + "eval_math_steps_per_second": 2.378, + "step": 900 + }, + { + "epoch": 1.601758095026149, + "eval_physics_entropy": 0.7051843535900116, + "eval_physics_loss": 0.8616264462471008, + "eval_physics_mean_token_accuracy": 0.7987073979377747, + "eval_physics_num_tokens": 24105114.0, + "eval_physics_runtime": 59.8508, + "eval_physics_samples_per_second": 8.354, + "eval_physics_steps_per_second": 2.089, + "step": 900 + }, + { + "entropy": 0.6430077504366636, + "epoch": 1.6195615889618338, + "grad_norm": 1.1640625, + "learning_rate": 1.548888888888889e-05, + "loss": 0.743, + "mean_token_accuracy": 0.8184590026736259, + "num_tokens": 24384008.0, + "step": 910 + }, + { + "entropy": 0.6775279760360717, + "epoch": 1.6373650828975186, + "grad_norm": 1.3125, + "learning_rate": 1.5414814814814814e-05, + "loss": 0.7704, + "mean_token_accuracy": 0.8114439699798822, + "num_tokens": 24638205.0, + "step": 920 + }, + { + "entropy": 0.6244934551417828, + "epoch": 1.6551685768332036, + "grad_norm": 1.1953125, + "learning_rate": 1.5340740740740744e-05, + "loss": 0.7253, + "mean_token_accuracy": 0.8222480688244105, + "num_tokens": 24910104.0, + "step": 930 + }, + { + "entropy": 0.6539383618161082, + "epoch": 1.6729720707688884, + "grad_norm": 1.2578125, + "learning_rate": 1.5266666666666667e-05, + "loss": 0.7471, + "mean_token_accuracy": 0.8160006206482648, + "num_tokens": 25181306.0, + "step": 940 + }, + { + "entropy": 0.6607510834932327, + "epoch": 1.6907755647045732, + "grad_norm": 1.3203125, + "learning_rate": 1.5192592592592594e-05, + "loss": 0.7589, + "mean_token_accuracy": 0.8142590031027794, + "num_tokens": 25449138.0, + "step": 950 + }, + { + "entropy": 0.6612858925014734, + "epoch": 1.7085790586402583, + "grad_norm": 1.203125, + "learning_rate": 1.5118518518518519e-05, + "loss": 0.766, + "mean_token_accuracy": 0.8137270450592041, + "num_tokens": 25709030.0, + "step": 960 + }, + { + "entropy": 0.6701876953244209, + "epoch": 1.7263825525759429, + "grad_norm": 1.2734375, + "learning_rate": 1.5044444444444445e-05, + "loss": 0.7702, + "mean_token_accuracy": 0.8120660580694675, + "num_tokens": 25972807.0, + "step": 970 + }, + { + "entropy": 0.6566708590835333, + "epoch": 1.744186046511628, + "grad_norm": 1.1640625, + "learning_rate": 1.497037037037037e-05, + "loss": 0.748, + "mean_token_accuracy": 0.8170768674463034, + "num_tokens": 26244955.0, + "step": 980 + }, + { + "entropy": 0.624184824153781, + "epoch": 1.761989540447313, + "grad_norm": 1.1796875, + "learning_rate": 1.4896296296296298e-05, + "loss": 0.7284, + "mean_token_accuracy": 0.8223610106855631, + "num_tokens": 26515548.0, + "step": 990 + }, + { + "entropy": 0.6476762825623155, + "epoch": 1.7797930343829975, + "grad_norm": 1.21875, + "learning_rate": 1.4822222222222225e-05, + "loss": 0.7422, + "mean_token_accuracy": 0.8172390758991241, + "num_tokens": 26782376.0, + "step": 1000 + }, + { + "epoch": 1.7797930343829975, + "eval_biology_entropy": 1.6157320532798767, + "eval_biology_loss": 2.0338852405548096, + "eval_biology_mean_token_accuracy": 0.6143801593780518, + "eval_biology_num_tokens": 26782376.0, + "eval_biology_runtime": 41.5849, + "eval_biology_samples_per_second": 12.024, + "eval_biology_steps_per_second": 3.006, + "step": 1000 + }, + { + "epoch": 1.7797930343829975, + "eval_chemistry_entropy": 0.6670743160247803, + "eval_chemistry_loss": 0.8244208097457886, + "eval_chemistry_mean_token_accuracy": 0.8077364330291747, + "eval_chemistry_num_tokens": 26782376.0, + "eval_chemistry_runtime": 53.0274, + "eval_chemistry_samples_per_second": 9.429, + "eval_chemistry_steps_per_second": 2.357, + "step": 1000 + }, + { + "epoch": 1.7797930343829975, + "eval_math_entropy": 0.6760952143669129, + "eval_math_loss": 1.0642753839492798, + "eval_math_mean_token_accuracy": 0.7728619589805603, + "eval_math_num_tokens": 26782376.0, + "eval_math_runtime": 53.8997, + "eval_math_samples_per_second": 9.276, + "eval_math_steps_per_second": 2.319, + "step": 1000 + }, + { + "epoch": 1.7797930343829975, + "eval_physics_entropy": 0.6914605097770691, + "eval_physics_loss": 0.8597292304039001, + "eval_physics_mean_token_accuracy": 0.7992245469093323, + "eval_physics_num_tokens": 26782376.0, + "eval_physics_runtime": 62.8355, + "eval_physics_samples_per_second": 7.957, + "eval_physics_steps_per_second": 1.989, + "step": 1000 + }, + { + "entropy": 0.660948914475739, + "epoch": 1.7975965283186826, + "grad_norm": 1.359375, + "learning_rate": 1.474814814814815e-05, + "loss": 0.7562, + "mean_token_accuracy": 0.8152279850095511, + "num_tokens": 27049648.0, + "step": 1010 + }, + { + "entropy": 0.6787248050794006, + "epoch": 1.8154000222543676, + "grad_norm": 1.3359375, + "learning_rate": 1.4674074074074076e-05, + "loss": 0.7771, + "mean_token_accuracy": 0.8101395227015018, + "num_tokens": 27315061.0, + "step": 1020 + }, + { + "entropy": 0.6424054227769375, + "epoch": 1.8332035161900522, + "grad_norm": 1.359375, + "learning_rate": 1.46e-05, + "loss": 0.7373, + "mean_token_accuracy": 0.8189616728574037, + "num_tokens": 27587650.0, + "step": 1030 + }, + { + "entropy": 0.6606515791267157, + "epoch": 1.8510070101257372, + "grad_norm": 1.15625, + "learning_rate": 1.4525925925925927e-05, + "loss": 0.7652, + "mean_token_accuracy": 0.8157406080514192, + "num_tokens": 27855458.0, + "step": 1040 + }, + { + "entropy": 0.6464399367570877, + "epoch": 1.868810504061422, + "grad_norm": 1.203125, + "learning_rate": 1.4451851851851852e-05, + "loss": 0.7464, + "mean_token_accuracy": 0.8191553246229887, + "num_tokens": 28127703.0, + "step": 1050 + }, + { + "entropy": 0.633565224148333, + "epoch": 1.8866139979971068, + "grad_norm": 1.140625, + "learning_rate": 1.4377777777777779e-05, + "loss": 0.7265, + "mean_token_accuracy": 0.8215953864157199, + "num_tokens": 28395128.0, + "step": 1060 + }, + { + "entropy": 0.651407066360116, + "epoch": 1.9044174919327919, + "grad_norm": 1.2421875, + "learning_rate": 1.4303703703703703e-05, + "loss": 0.752, + "mean_token_accuracy": 0.8161330189555883, + "num_tokens": 28662344.0, + "step": 1070 + }, + { + "entropy": 0.6537632567808032, + "epoch": 1.9222209858684767, + "grad_norm": 1.3125, + "learning_rate": 1.4229629629629632e-05, + "loss": 0.7551, + "mean_token_accuracy": 0.8160409614443779, + "num_tokens": 28928769.0, + "step": 1080 + }, + { + "entropy": 0.6610247550532222, + "epoch": 1.9400244798041615, + "grad_norm": 1.5625, + "learning_rate": 1.4155555555555556e-05, + "loss": 0.7599, + "mean_token_accuracy": 0.8147491872310638, + "num_tokens": 29186214.0, + "step": 1090 + }, + { + "entropy": 0.6632727082818747, + "epoch": 1.9578279737398465, + "grad_norm": 1.2109375, + "learning_rate": 1.4081481481481483e-05, + "loss": 0.7706, + "mean_token_accuracy": 0.8126637618988752, + "num_tokens": 29446445.0, + "step": 1100 + }, + { + "epoch": 1.9578279737398465, + "eval_biology_entropy": 1.639104124546051, + "eval_biology_loss": 2.0316033363342285, + "eval_biology_mean_token_accuracy": 0.615292895078659, + "eval_biology_num_tokens": 29446445.0, + "eval_biology_runtime": 41.3569, + "eval_biology_samples_per_second": 12.09, + "eval_biology_steps_per_second": 3.022, + "step": 1100 + }, + { + "epoch": 1.9578279737398465, + "eval_chemistry_entropy": 0.6696053767204284, + "eval_chemistry_loss": 0.8191399574279785, + "eval_chemistry_mean_token_accuracy": 0.8085824422836304, + "eval_chemistry_num_tokens": 29446445.0, + "eval_chemistry_runtime": 50.7996, + "eval_chemistry_samples_per_second": 9.843, + "eval_chemistry_steps_per_second": 2.461, + "step": 1100 + }, + { + "epoch": 1.9578279737398465, + "eval_math_entropy": 0.6799165282249451, + "eval_math_loss": 1.0633617639541626, + "eval_math_mean_token_accuracy": 0.7729020061492919, + "eval_math_num_tokens": 29446445.0, + "eval_math_runtime": 52.3664, + "eval_math_samples_per_second": 9.548, + "eval_math_steps_per_second": 2.387, + "step": 1100 + }, + { + "epoch": 1.9578279737398465, + "eval_physics_entropy": 0.6961899037361146, + "eval_physics_loss": 0.8565983176231384, + "eval_physics_mean_token_accuracy": 0.7995611958503723, + "eval_physics_num_tokens": 29446445.0, + "eval_physics_runtime": 59.795, + "eval_physics_samples_per_second": 8.362, + "eval_physics_steps_per_second": 2.09, + "step": 1100 + }, + { + "entropy": 0.6516633328050375, + "epoch": 1.9756314676755313, + "grad_norm": 1.21875, + "learning_rate": 1.400740740740741e-05, + "loss": 0.7439, + "mean_token_accuracy": 0.8187774941325188, + "num_tokens": 29711771.0, + "step": 1110 + }, + { + "entropy": 0.6508602360263467, + "epoch": 1.9934349616112161, + "grad_norm": 1.3125, + "learning_rate": 1.3933333333333334e-05, + "loss": 0.7519, + "mean_token_accuracy": 0.8169589649885893, + "num_tokens": 29979994.0, + "step": 1120 + }, + { + "entropy": 0.6306788632946629, + "epoch": 2.010682096361411, + "grad_norm": 1.2734375, + "learning_rate": 1.385925925925926e-05, + "loss": 0.7224, + "mean_token_accuracy": 0.8239769354943306, + "num_tokens": 30240793.0, + "step": 1130 + }, + { + "entropy": 0.5911945894360542, + "epoch": 2.028485590297096, + "grad_norm": 1.296875, + "learning_rate": 1.3785185185185186e-05, + "loss": 0.7, + "mean_token_accuracy": 0.8281658995896578, + "num_tokens": 30503988.0, + "step": 1140 + }, + { + "entropy": 0.6057484762743115, + "epoch": 2.0462890842327806, + "grad_norm": 1.2890625, + "learning_rate": 1.3711111111111112e-05, + "loss": 0.7031, + "mean_token_accuracy": 0.8266915556043386, + "num_tokens": 30771841.0, + "step": 1150 + }, + { + "entropy": 0.596652439981699, + "epoch": 2.0640925781684656, + "grad_norm": 1.375, + "learning_rate": 1.3637037037037037e-05, + "loss": 0.6926, + "mean_token_accuracy": 0.8293926887214184, + "num_tokens": 31045037.0, + "step": 1160 + }, + { + "entropy": 0.6352694857865572, + "epoch": 2.0818960721041506, + "grad_norm": 1.4296875, + "learning_rate": 1.3562962962962965e-05, + "loss": 0.7356, + "mean_token_accuracy": 0.8186026088893413, + "num_tokens": 31299575.0, + "step": 1170 + }, + { + "entropy": 0.6292696729302406, + "epoch": 2.099699566039835, + "grad_norm": 1.390625, + "learning_rate": 1.3488888888888888e-05, + "loss": 0.7273, + "mean_token_accuracy": 0.8201477468013764, + "num_tokens": 31565963.0, + "step": 1180 + }, + { + "entropy": 0.5940109083428979, + "epoch": 2.1175030599755202, + "grad_norm": 1.2734375, + "learning_rate": 1.3414814814814817e-05, + "loss": 0.7035, + "mean_token_accuracy": 0.8278756473213434, + "num_tokens": 31834212.0, + "step": 1190 + }, + { + "entropy": 0.6235881226137281, + "epoch": 2.1353065539112053, + "grad_norm": 1.375, + "learning_rate": 1.3340740740740741e-05, + "loss": 0.7172, + "mean_token_accuracy": 0.8223393179476262, + "num_tokens": 32103356.0, + "step": 1200 + }, + { + "epoch": 2.1353065539112053, + "eval_biology_entropy": 1.4677531628608704, + "eval_biology_loss": 2.083951234817505, + "eval_biology_mean_token_accuracy": 0.6125459330081939, + "eval_biology_num_tokens": 32103356.0, + "eval_biology_runtime": 41.2722, + "eval_biology_samples_per_second": 12.115, + "eval_biology_steps_per_second": 3.029, + "step": 1200 + }, + { + "epoch": 2.1353065539112053, + "eval_chemistry_entropy": 0.6140689237117767, + "eval_chemistry_loss": 0.8232262134552002, + "eval_chemistry_mean_token_accuracy": 0.8088321657180786, + "eval_chemistry_num_tokens": 32103356.0, + "eval_chemistry_runtime": 50.8871, + "eval_chemistry_samples_per_second": 9.826, + "eval_chemistry_steps_per_second": 2.456, + "step": 1200 + }, + { + "epoch": 2.1353065539112053, + "eval_math_entropy": 0.6312986519336701, + "eval_math_loss": 1.0854209661483765, + "eval_math_mean_token_accuracy": 0.7729385108947754, + "eval_math_num_tokens": 32103356.0, + "eval_math_runtime": 52.4669, + "eval_math_samples_per_second": 9.53, + "eval_math_steps_per_second": 2.382, + "step": 1200 + }, + { + "epoch": 2.1353065539112053, + "eval_physics_entropy": 0.6365497968196869, + "eval_physics_loss": 0.8704355359077454, + "eval_physics_mean_token_accuracy": 0.7990298428535462, + "eval_physics_num_tokens": 32103356.0, + "eval_physics_runtime": 59.7275, + "eval_physics_samples_per_second": 8.371, + "eval_physics_steps_per_second": 2.093, + "step": 1200 + }, + { + "entropy": 0.6137512635439635, + "epoch": 2.15311004784689, + "grad_norm": 1.25, + "learning_rate": 1.3266666666666668e-05, + "loss": 0.7043, + "mean_token_accuracy": 0.8242961999028922, + "num_tokens": 32374830.0, + "step": 1210 + }, + { + "entropy": 0.5896952305920422, + "epoch": 2.170913541782575, + "grad_norm": 1.2734375, + "learning_rate": 1.3192592592592594e-05, + "loss": 0.6922, + "mean_token_accuracy": 0.8302250444889069, + "num_tokens": 32643479.0, + "step": 1220 + }, + { + "entropy": 0.5944842832162976, + "epoch": 2.18871703571826, + "grad_norm": 1.296875, + "learning_rate": 1.311851851851852e-05, + "loss": 0.6959, + "mean_token_accuracy": 0.8287800919264555, + "num_tokens": 32908761.0, + "step": 1230 + }, + { + "entropy": 0.6240290224552154, + "epoch": 2.2065205296539445, + "grad_norm": 1.21875, + "learning_rate": 1.3044444444444446e-05, + "loss": 0.7238, + "mean_token_accuracy": 0.8242842797189951, + "num_tokens": 33166413.0, + "step": 1240 + }, + { + "entropy": 0.6112470507621766, + "epoch": 2.2243240235896296, + "grad_norm": 1.1875, + "learning_rate": 1.297037037037037e-05, + "loss": 0.7149, + "mean_token_accuracy": 0.8240214943885803, + "num_tokens": 33432686.0, + "step": 1250 + }, + { + "entropy": 0.6079921290278435, + "epoch": 2.242127517525314, + "grad_norm": 1.4296875, + "learning_rate": 1.2896296296296299e-05, + "loss": 0.7131, + "mean_token_accuracy": 0.8254695866256953, + "num_tokens": 33693242.0, + "step": 1260 + }, + { + "entropy": 0.5799209821969271, + "epoch": 2.259931011460999, + "grad_norm": 1.390625, + "learning_rate": 1.2822222222222222e-05, + "loss": 0.6722, + "mean_token_accuracy": 0.834067041426897, + "num_tokens": 33970820.0, + "step": 1270 + }, + { + "entropy": 0.594209510087967, + "epoch": 2.277734505396684, + "grad_norm": 1.3671875, + "learning_rate": 1.274814814814815e-05, + "loss": 0.6977, + "mean_token_accuracy": 0.8276573233306408, + "num_tokens": 34243042.0, + "step": 1280 + }, + { + "entropy": 0.6097929472103715, + "epoch": 2.295537999332369, + "grad_norm": 1.296875, + "learning_rate": 1.2674074074074075e-05, + "loss": 0.705, + "mean_token_accuracy": 0.8267240136861801, + "num_tokens": 34515236.0, + "step": 1290 + }, + { + "entropy": 0.5745832217857242, + "epoch": 2.313341493268054, + "grad_norm": 1.234375, + "learning_rate": 1.2600000000000001e-05, + "loss": 0.6783, + "mean_token_accuracy": 0.8345583423972129, + "num_tokens": 34785068.0, + "step": 1300 + }, + { + "epoch": 2.313341493268054, + "eval_biology_entropy": 1.497672739982605, + "eval_biology_loss": 2.0853865146636963, + "eval_biology_mean_token_accuracy": 0.613484436750412, + "eval_biology_num_tokens": 34785068.0, + "eval_biology_runtime": 43.469, + "eval_biology_samples_per_second": 11.502, + "eval_biology_steps_per_second": 2.876, + "step": 1300 + }, + { + "epoch": 2.313341493268054, + "eval_chemistry_entropy": 0.6273940706253052, + "eval_chemistry_loss": 0.8201014995574951, + "eval_chemistry_mean_token_accuracy": 0.8088535733222961, + "eval_chemistry_num_tokens": 34785068.0, + "eval_chemistry_runtime": 52.831, + "eval_chemistry_samples_per_second": 9.464, + "eval_chemistry_steps_per_second": 2.366, + "step": 1300 + }, + { + "epoch": 2.313341493268054, + "eval_math_entropy": 0.646281590461731, + "eval_math_loss": 1.0807055234909058, + "eval_math_mean_token_accuracy": 0.772442822933197, + "eval_math_num_tokens": 34785068.0, + "eval_math_runtime": 55.4339, + "eval_math_samples_per_second": 9.02, + "eval_math_steps_per_second": 2.255, + "step": 1300 + }, + { + "epoch": 2.313341493268054, + "eval_physics_entropy": 0.6572135488986969, + "eval_physics_loss": 0.8676888346672058, + "eval_physics_mean_token_accuracy": 0.7990684523582459, + "eval_physics_num_tokens": 34785068.0, + "eval_physics_runtime": 61.4064, + "eval_physics_samples_per_second": 8.142, + "eval_physics_steps_per_second": 2.036, + "step": 1300 + }, + { + "entropy": 0.6191223243251442, + "epoch": 2.331144987203739, + "grad_norm": 1.2265625, + "learning_rate": 1.2525925925925928e-05, + "loss": 0.7122, + "mean_token_accuracy": 0.8234442934393883, + "num_tokens": 35055845.0, + "step": 1310 + }, + { + "entropy": 0.6000411529093981, + "epoch": 2.3489484811394234, + "grad_norm": 1.296875, + "learning_rate": 1.2451851851851853e-05, + "loss": 0.7023, + "mean_token_accuracy": 0.8272519435733556, + "num_tokens": 35327481.0, + "step": 1320 + }, + { + "entropy": 0.6216201702132821, + "epoch": 2.3667519750751085, + "grad_norm": 1.2578125, + "learning_rate": 1.237777777777778e-05, + "loss": 0.7178, + "mean_token_accuracy": 0.8234907556325197, + "num_tokens": 35596362.0, + "step": 1330 + }, + { + "entropy": 0.5952032478526235, + "epoch": 2.3845554690107935, + "grad_norm": 1.3671875, + "learning_rate": 1.2303703703703704e-05, + "loss": 0.695, + "mean_token_accuracy": 0.827592471987009, + "num_tokens": 35869585.0, + "step": 1340 + }, + { + "entropy": 0.6152689423412084, + "epoch": 2.402358962946478, + "grad_norm": 1.4140625, + "learning_rate": 1.222962962962963e-05, + "loss": 0.7087, + "mean_token_accuracy": 0.8254514247179031, + "num_tokens": 36134072.0, + "step": 1350 + }, + { + "entropy": 0.5919087225571275, + "epoch": 2.420162456882163, + "grad_norm": 1.25, + "learning_rate": 1.2155555555555555e-05, + "loss": 0.6971, + "mean_token_accuracy": 0.829882950335741, + "num_tokens": 36406263.0, + "step": 1360 + }, + { + "entropy": 0.6080563787370921, + "epoch": 2.437965950817848, + "grad_norm": 1.3984375, + "learning_rate": 1.2081481481481484e-05, + "loss": 0.7027, + "mean_token_accuracy": 0.8268357224762439, + "num_tokens": 36676822.0, + "step": 1370 + }, + { + "entropy": 0.6048327511176467, + "epoch": 2.4557694447535328, + "grad_norm": 1.421875, + "learning_rate": 1.2007407407407408e-05, + "loss": 0.7047, + "mean_token_accuracy": 0.8271109748631715, + "num_tokens": 36942837.0, + "step": 1380 + }, + { + "entropy": 0.5988074177876115, + "epoch": 2.473572938689218, + "grad_norm": 1.390625, + "learning_rate": 1.1933333333333335e-05, + "loss": 0.7051, + "mean_token_accuracy": 0.8256583914160729, + "num_tokens": 37210791.0, + "step": 1390 + }, + { + "entropy": 0.5873929725959897, + "epoch": 2.491376432624903, + "grad_norm": 1.3671875, + "learning_rate": 1.185925925925926e-05, + "loss": 0.6871, + "mean_token_accuracy": 0.83187070004642, + "num_tokens": 37485822.0, + "step": 1400 + }, + { + "epoch": 2.491376432624903, + "eval_biology_entropy": 1.4652894883155823, + "eval_biology_loss": 2.099928855895996, + "eval_biology_mean_token_accuracy": 0.6130364212989807, + "eval_biology_num_tokens": 37485822.0, + "eval_biology_runtime": 43.3395, + "eval_biology_samples_per_second": 11.537, + "eval_biology_steps_per_second": 2.884, + "step": 1400 + }, + { + "epoch": 2.491376432624903, + "eval_chemistry_entropy": 0.6137497539520264, + "eval_chemistry_loss": 0.8189122080802917, + "eval_chemistry_mean_token_accuracy": 0.8097302551269532, + "eval_chemistry_num_tokens": 37485822.0, + "eval_chemistry_runtime": 52.903, + "eval_chemistry_samples_per_second": 9.451, + "eval_chemistry_steps_per_second": 2.363, + "step": 1400 + }, + { + "epoch": 2.491376432624903, + "eval_math_entropy": 0.6390214171409607, + "eval_math_loss": 1.082951545715332, + "eval_math_mean_token_accuracy": 0.772076274394989, + "eval_math_num_tokens": 37485822.0, + "eval_math_runtime": 54.1184, + "eval_math_samples_per_second": 9.239, + "eval_math_steps_per_second": 2.31, + "step": 1400 + }, + { + "epoch": 2.491376432624903, + "eval_physics_entropy": 0.6406208200454712, + "eval_physics_loss": 0.8697341084480286, + "eval_physics_mean_token_accuracy": 0.7989166221618652, + "eval_physics_num_tokens": 37485822.0, + "eval_physics_runtime": 61.5427, + "eval_physics_samples_per_second": 8.124, + "eval_physics_steps_per_second": 2.031, + "step": 1400 + }, + { + "entropy": 0.6010861510410905, + "epoch": 2.5091799265605874, + "grad_norm": 1.28125, + "learning_rate": 1.1785185185185186e-05, + "loss": 0.7014, + "mean_token_accuracy": 0.8287522595375776, + "num_tokens": 37746150.0, + "step": 1410 + }, + { + "entropy": 0.5981267247349024, + "epoch": 2.5269834204962724, + "grad_norm": 1.3046875, + "learning_rate": 1.1711111111111113e-05, + "loss": 0.6962, + "mean_token_accuracy": 0.8269948281347752, + "num_tokens": 38016863.0, + "step": 1420 + }, + { + "entropy": 0.6277626631781459, + "epoch": 2.544786914431957, + "grad_norm": 1.5078125, + "learning_rate": 1.1637037037037037e-05, + "loss": 0.7222, + "mean_token_accuracy": 0.8205803319811821, + "num_tokens": 38277754.0, + "step": 1430 + }, + { + "entropy": 0.6118797251954675, + "epoch": 2.562590408367642, + "grad_norm": 1.421875, + "learning_rate": 1.1562962962962964e-05, + "loss": 0.7172, + "mean_token_accuracy": 0.8237183388322592, + "num_tokens": 38542761.0, + "step": 1440 + }, + { + "entropy": 0.6034180456772447, + "epoch": 2.580393902303327, + "grad_norm": 1.453125, + "learning_rate": 1.1488888888888889e-05, + "loss": 0.6941, + "mean_token_accuracy": 0.8272098813205957, + "num_tokens": 38820999.0, + "step": 1450 + }, + { + "entropy": 0.5998265566304326, + "epoch": 2.598197396239012, + "grad_norm": 1.3203125, + "learning_rate": 1.1414814814814817e-05, + "loss": 0.7114, + "mean_token_accuracy": 0.8267886329442262, + "num_tokens": 39080016.0, + "step": 1460 + }, + { + "entropy": 0.6005334880203008, + "epoch": 2.6160008901746967, + "grad_norm": 1.3828125, + "learning_rate": 1.1340740740740742e-05, + "loss": 0.6949, + "mean_token_accuracy": 0.8299154710024595, + "num_tokens": 39345434.0, + "step": 1470 + }, + { + "entropy": 0.6003408120945096, + "epoch": 2.6338043841103818, + "grad_norm": 1.46875, + "learning_rate": 1.1266666666666668e-05, + "loss": 0.7033, + "mean_token_accuracy": 0.8266183461993932, + "num_tokens": 39610041.0, + "step": 1480 + }, + { + "entropy": 0.5847593653947115, + "epoch": 2.6516078780460663, + "grad_norm": 1.4140625, + "learning_rate": 1.1192592592592593e-05, + "loss": 0.6785, + "mean_token_accuracy": 0.8323042623698711, + "num_tokens": 39884185.0, + "step": 1490 + }, + { + "entropy": 0.6105206938460469, + "epoch": 2.6694113719817514, + "grad_norm": 1.5546875, + "learning_rate": 1.111851851851852e-05, + "loss": 0.7201, + "mean_token_accuracy": 0.8220306947827339, + "num_tokens": 40143543.0, + "step": 1500 + }, + { + "epoch": 2.6694113719817514, + "eval_biology_entropy": 1.490638756752014, + "eval_biology_loss": 2.0750932693481445, + "eval_biology_mean_token_accuracy": 0.6143683531284332, + "eval_biology_num_tokens": 40143543.0, + "eval_biology_runtime": 43.2253, + "eval_biology_samples_per_second": 11.567, + "eval_biology_steps_per_second": 2.892, + "step": 1500 + }, + { + "epoch": 2.6694113719817514, + "eval_chemistry_entropy": 0.623803097486496, + "eval_chemistry_loss": 0.8160935044288635, + "eval_chemistry_mean_token_accuracy": 0.8098534879684448, + "eval_chemistry_num_tokens": 40143543.0, + "eval_chemistry_runtime": 50.807, + "eval_chemistry_samples_per_second": 9.841, + "eval_chemistry_steps_per_second": 2.46, + "step": 1500 + }, + { + "epoch": 2.6694113719817514, + "eval_math_entropy": 0.645468202829361, + "eval_math_loss": 1.0840471982955933, + "eval_math_mean_token_accuracy": 0.7726303563117981, + "eval_math_num_tokens": 40143543.0, + "eval_math_runtime": 53.0054, + "eval_math_samples_per_second": 9.433, + "eval_math_steps_per_second": 2.358, + "step": 1500 + }, + { + "epoch": 2.6694113719817514, + "eval_physics_entropy": 0.6492172837257385, + "eval_physics_loss": 0.8670127391815186, + "eval_physics_mean_token_accuracy": 0.7991857075691223, + "eval_physics_num_tokens": 40143543.0, + "eval_physics_runtime": 63.9696, + "eval_physics_samples_per_second": 7.816, + "eval_physics_steps_per_second": 1.954, + "step": 1500 + }, + { + "entropy": 0.6046247070655226, + "epoch": 2.6872148659174364, + "grad_norm": 1.390625, + "learning_rate": 1.1044444444444444e-05, + "loss": 0.7007, + "mean_token_accuracy": 0.828262010216713, + "num_tokens": 40406812.0, + "step": 1510 + }, + { + "entropy": 0.6191225880756974, + "epoch": 2.7050183598531214, + "grad_norm": 1.4140625, + "learning_rate": 1.0970370370370371e-05, + "loss": 0.7269, + "mean_token_accuracy": 0.8213932007551193, + "num_tokens": 40668543.0, + "step": 1520 + }, + { + "entropy": 0.6065074443817139, + "epoch": 2.722821853788806, + "grad_norm": 1.3515625, + "learning_rate": 1.0896296296296298e-05, + "loss": 0.6989, + "mean_token_accuracy": 0.8265019495040178, + "num_tokens": 40942603.0, + "step": 1530 + }, + { + "entropy": 0.6138417391106487, + "epoch": 2.740625347724491, + "grad_norm": 1.34375, + "learning_rate": 1.0822222222222222e-05, + "loss": 0.7057, + "mean_token_accuracy": 0.8261400748044252, + "num_tokens": 41208093.0, + "step": 1540 + }, + { + "entropy": 0.5985930340364576, + "epoch": 2.7584288416601757, + "grad_norm": 1.3125, + "learning_rate": 1.074814814814815e-05, + "loss": 0.7061, + "mean_token_accuracy": 0.8252453569322824, + "num_tokens": 41477562.0, + "step": 1550 + }, + { + "entropy": 0.5930986102670431, + "epoch": 2.7762323355958607, + "grad_norm": 1.5234375, + "learning_rate": 1.0674074074074074e-05, + "loss": 0.6935, + "mean_token_accuracy": 0.8292474631220103, + "num_tokens": 41746571.0, + "step": 1560 + }, + { + "entropy": 0.6006103545427323, + "epoch": 2.7940358295315457, + "grad_norm": 1.4765625, + "learning_rate": 1.0600000000000002e-05, + "loss": 0.7025, + "mean_token_accuracy": 0.8271383382380009, + "num_tokens": 42011633.0, + "step": 1570 + }, + { + "entropy": 0.6129128225147724, + "epoch": 2.8118393234672303, + "grad_norm": 1.328125, + "learning_rate": 1.0525925925925927e-05, + "loss": 0.7076, + "mean_token_accuracy": 0.8252990577369929, + "num_tokens": 42283087.0, + "step": 1580 + }, + { + "entropy": 0.5756963772699237, + "epoch": 2.8296428174029153, + "grad_norm": 1.1953125, + "learning_rate": 1.0451851851851853e-05, + "loss": 0.6725, + "mean_token_accuracy": 0.8329845789819956, + "num_tokens": 42561534.0, + "step": 1590 + }, + { + "entropy": 0.6044865250587463, + "epoch": 2.8474463113386, + "grad_norm": 1.296875, + "learning_rate": 1.0377777777777778e-05, + "loss": 0.705, + "mean_token_accuracy": 0.8281082704663276, + "num_tokens": 42834266.0, + "step": 1600 + }, + { + "epoch": 2.8474463113386, + "eval_biology_entropy": 1.4930514478683472, + "eval_biology_loss": 2.0736353397369385, + "eval_biology_mean_token_accuracy": 0.6145954511165619, + "eval_biology_num_tokens": 42834266.0, + "eval_biology_runtime": 40.9171, + "eval_biology_samples_per_second": 12.22, + "eval_biology_steps_per_second": 3.055, + "step": 1600 + }, + { + "epoch": 2.8474463113386, + "eval_chemistry_entropy": 0.6211945106983184, + "eval_chemistry_loss": 0.8145589828491211, + "eval_chemistry_mean_token_accuracy": 0.810048574924469, + "eval_chemistry_num_tokens": 42834266.0, + "eval_chemistry_runtime": 50.7089, + "eval_chemistry_samples_per_second": 9.86, + "eval_chemistry_steps_per_second": 2.465, + "step": 1600 + }, + { + "epoch": 2.8474463113386, + "eval_math_entropy": 0.6438178873062134, + "eval_math_loss": 1.0831515789031982, + "eval_math_mean_token_accuracy": 0.772301673412323, + "eval_math_num_tokens": 42834266.0, + "eval_math_runtime": 51.7388, + "eval_math_samples_per_second": 9.664, + "eval_math_steps_per_second": 2.416, + "step": 1600 + }, + { + "epoch": 2.8474463113386, + "eval_physics_entropy": 0.649389057636261, + "eval_physics_loss": 0.86627197265625, + "eval_physics_mean_token_accuracy": 0.7995732502937317, + "eval_physics_num_tokens": 42834266.0, + "eval_physics_runtime": 59.549, + "eval_physics_samples_per_second": 8.396, + "eval_physics_steps_per_second": 2.099, + "step": 1600 + }, + { + "entropy": 0.625230243243277, + "epoch": 2.865249805274285, + "grad_norm": 1.46875, + "learning_rate": 1.0303703703703705e-05, + "loss": 0.7228, + "mean_token_accuracy": 0.8225024774670601, + "num_tokens": 43100513.0, + "step": 1610 + }, + { + "entropy": 0.605365008302033, + "epoch": 2.88305329920997, + "grad_norm": 1.359375, + "learning_rate": 1.0229629629629631e-05, + "loss": 0.698, + "mean_token_accuracy": 0.8268981877714395, + "num_tokens": 43375632.0, + "step": 1620 + }, + { + "entropy": 0.5911450985819101, + "epoch": 2.900856793145655, + "grad_norm": 1.3515625, + "learning_rate": 1.0155555555555556e-05, + "loss": 0.6958, + "mean_token_accuracy": 0.829645611718297, + "num_tokens": 43644341.0, + "step": 1630 + }, + { + "entropy": 0.5920769222080707, + "epoch": 2.9186602870813396, + "grad_norm": 1.390625, + "learning_rate": 1.0081481481481484e-05, + "loss": 0.6833, + "mean_token_accuracy": 0.8301208417862653, + "num_tokens": 43917580.0, + "step": 1640 + }, + { + "entropy": 0.5912358593195677, + "epoch": 2.9364637810170247, + "grad_norm": 1.359375, + "learning_rate": 1.0007407407407407e-05, + "loss": 0.6989, + "mean_token_accuracy": 0.8305816825479269, + "num_tokens": 44177325.0, + "step": 1650 + }, + { + "entropy": 0.6072682781144977, + "epoch": 2.9542672749527092, + "grad_norm": 1.3984375, + "learning_rate": 9.933333333333334e-06, + "loss": 0.7079, + "mean_token_accuracy": 0.8247454788535833, + "num_tokens": 44442947.0, + "step": 1660 + }, + { + "entropy": 0.6202316217124462, + "epoch": 2.9720707688883943, + "grad_norm": 1.484375, + "learning_rate": 9.85925925925926e-06, + "loss": 0.7181, + "mean_token_accuracy": 0.8235241327434778, + "num_tokens": 44700601.0, + "step": 1670 + }, + { + "entropy": 0.5890740664675832, + "epoch": 2.9898742628240793, + "grad_norm": 1.3125, + "learning_rate": 9.785185185185187e-06, + "loss": 0.6933, + "mean_token_accuracy": 0.8307324200868607, + "num_tokens": 44971479.0, + "step": 1680 + }, + { + "entropy": 0.587316531712009, + "epoch": 3.007121397574274, + "grad_norm": 1.265625, + "learning_rate": 9.711111111111111e-06, + "loss": 0.6682, + "mean_token_accuracy": 0.8340173459822132, + "num_tokens": 45231796.0, + "step": 1690 + }, + { + "entropy": 0.6013597913086415, + "epoch": 3.0249248915099587, + "grad_norm": 1.5, + "learning_rate": 9.637037037037038e-06, + "loss": 0.7048, + "mean_token_accuracy": 0.8241627678275109, + "num_tokens": 45500317.0, + "step": 1700 + }, + { + "epoch": 3.0249248915099587, + "eval_biology_entropy": 1.4193837118148804, + "eval_biology_loss": 2.11480712890625, + "eval_biology_mean_token_accuracy": 0.6124376230239869, + "eval_biology_num_tokens": 45500317.0, + "eval_biology_runtime": 40.3875, + "eval_biology_samples_per_second": 12.38, + "eval_biology_steps_per_second": 3.095, + "step": 1700 + }, + { + "epoch": 3.0249248915099587, + "eval_chemistry_entropy": 0.5911917226314545, + "eval_chemistry_loss": 0.8190613985061646, + "eval_chemistry_mean_token_accuracy": 0.8099429054260254, + "eval_chemistry_num_tokens": 45500317.0, + "eval_chemistry_runtime": 50.2786, + "eval_chemistry_samples_per_second": 9.945, + "eval_chemistry_steps_per_second": 2.486, + "step": 1700 + }, + { + "epoch": 3.0249248915099587, + "eval_math_entropy": 0.6176868407726288, + "eval_math_loss": 1.0993149280548096, + "eval_math_mean_token_accuracy": 0.7718941388130188, + "eval_math_num_tokens": 45500317.0, + "eval_math_runtime": 51.8195, + "eval_math_samples_per_second": 9.649, + "eval_math_steps_per_second": 2.412, + "step": 1700 + }, + { + "epoch": 3.0249248915099587, + "eval_physics_entropy": 0.6201707119941712, + "eval_physics_loss": 0.8766935467720032, + "eval_physics_mean_token_accuracy": 0.7985301322937012, + "eval_physics_num_tokens": 45500317.0, + "eval_physics_runtime": 59.4252, + "eval_physics_samples_per_second": 8.414, + "eval_physics_steps_per_second": 2.103, + "step": 1700 + }, + { + "entropy": 0.5692525265738368, + "epoch": 3.0427283854456437, + "grad_norm": 1.4140625, + "learning_rate": 9.562962962962965e-06, + "loss": 0.6591, + "mean_token_accuracy": 0.8374071232974529, + "num_tokens": 45762680.0, + "step": 1710 + }, + { + "entropy": 0.5559542520903051, + "epoch": 3.0605318793813288, + "grad_norm": 1.625, + "learning_rate": 9.48888888888889e-06, + "loss": 0.6597, + "mean_token_accuracy": 0.8373042322695256, + "num_tokens": 46029923.0, + "step": 1720 + }, + { + "entropy": 0.5692304980009795, + "epoch": 3.0783353733170133, + "grad_norm": 1.328125, + "learning_rate": 9.414814814814816e-06, + "loss": 0.6629, + "mean_token_accuracy": 0.8344687603414058, + "num_tokens": 46305156.0, + "step": 1730 + }, + { + "entropy": 0.5508972069248557, + "epoch": 3.0961388672526984, + "grad_norm": 1.484375, + "learning_rate": 9.34074074074074e-06, + "loss": 0.6501, + "mean_token_accuracy": 0.8386828090995551, + "num_tokens": 46582874.0, + "step": 1740 + }, + { + "entropy": 0.5854626100510358, + "epoch": 3.1139423611883834, + "grad_norm": 1.640625, + "learning_rate": 9.266666666666667e-06, + "loss": 0.6824, + "mean_token_accuracy": 0.8301110483705998, + "num_tokens": 46850175.0, + "step": 1750 + }, + { + "entropy": 0.5591677505522966, + "epoch": 3.131745855124068, + "grad_norm": 1.3984375, + "learning_rate": 9.192592592592594e-06, + "loss": 0.6564, + "mean_token_accuracy": 0.8384724948555231, + "num_tokens": 47125899.0, + "step": 1760 + }, + { + "entropy": 0.5725911458954215, + "epoch": 3.149549349059753, + "grad_norm": 1.546875, + "learning_rate": 9.118518518518518e-06, + "loss": 0.6752, + "mean_token_accuracy": 0.8343269538134337, + "num_tokens": 47390194.0, + "step": 1770 + }, + { + "entropy": 0.5758154878392816, + "epoch": 3.167352842995438, + "grad_norm": 1.4140625, + "learning_rate": 9.044444444444445e-06, + "loss": 0.6738, + "mean_token_accuracy": 0.8336336623877287, + "num_tokens": 47654784.0, + "step": 1780 + }, + { + "entropy": 0.5612063113600015, + "epoch": 3.1851563369311227, + "grad_norm": 1.53125, + "learning_rate": 8.970370370370372e-06, + "loss": 0.668, + "mean_token_accuracy": 0.8369329560548067, + "num_tokens": 47919400.0, + "step": 1790 + }, + { + "entropy": 0.5750826884061098, + "epoch": 3.2029598308668077, + "grad_norm": 1.5625, + "learning_rate": 8.896296296296298e-06, + "loss": 0.6761, + "mean_token_accuracy": 0.8343805599957704, + "num_tokens": 48187796.0, + "step": 1800 + }, + { + "epoch": 3.2029598308668077, + "eval_biology_entropy": 1.4034225873947144, + "eval_biology_loss": 2.1275665760040283, + "eval_biology_mean_token_accuracy": 0.6119487595558166, + "eval_biology_num_tokens": 48187796.0, + "eval_biology_runtime": 40.734, + "eval_biology_samples_per_second": 12.275, + "eval_biology_steps_per_second": 3.069, + "step": 1800 + }, + { + "epoch": 3.2029598308668077, + "eval_chemistry_entropy": 0.5955355260372162, + "eval_chemistry_loss": 0.8196625709533691, + "eval_chemistry_mean_token_accuracy": 0.8102530264854431, + "eval_chemistry_num_tokens": 48187796.0, + "eval_chemistry_runtime": 50.2147, + "eval_chemistry_samples_per_second": 9.957, + "eval_chemistry_steps_per_second": 2.489, + "step": 1800 + }, + { + "epoch": 3.2029598308668077, + "eval_math_entropy": 0.6184592893123627, + "eval_math_loss": 1.1008187532424927, + "eval_math_mean_token_accuracy": 0.7713511042594909, + "eval_math_num_tokens": 48187796.0, + "eval_math_runtime": 51.7678, + "eval_math_samples_per_second": 9.659, + "eval_math_steps_per_second": 2.415, + "step": 1800 + }, + { + "epoch": 3.2029598308668077, + "eval_physics_entropy": 0.6223777911663055, + "eval_physics_loss": 0.8791524767875671, + "eval_physics_mean_token_accuracy": 0.7979751496315003, + "eval_physics_num_tokens": 48187796.0, + "eval_physics_runtime": 59.0178, + "eval_physics_samples_per_second": 8.472, + "eval_physics_steps_per_second": 2.118, + "step": 1800 + }, + { + "entropy": 0.5663457477465272, + "epoch": 3.2207633248024923, + "grad_norm": 1.3828125, + "learning_rate": 8.822222222222223e-06, + "loss": 0.666, + "mean_token_accuracy": 0.8364688046276569, + "num_tokens": 48457428.0, + "step": 1810 + }, + { + "entropy": 0.6113261075690388, + "epoch": 3.2385668187381773, + "grad_norm": 1.515625, + "learning_rate": 8.74814814814815e-06, + "loss": 0.704, + "mean_token_accuracy": 0.8250963229686021, + "num_tokens": 48716417.0, + "step": 1820 + }, + { + "entropy": 0.5722766594961286, + "epoch": 3.2563703126738623, + "grad_norm": 1.4296875, + "learning_rate": 8.674074074074074e-06, + "loss": 0.6724, + "mean_token_accuracy": 0.8353833951056003, + "num_tokens": 48977306.0, + "step": 1830 + }, + { + "entropy": 0.5715915774926543, + "epoch": 3.274173806609547, + "grad_norm": 1.3359375, + "learning_rate": 8.6e-06, + "loss": 0.6663, + "mean_token_accuracy": 0.8353964131325483, + "num_tokens": 49249667.0, + "step": 1840 + }, + { + "entropy": 0.5664192967116832, + "epoch": 3.291977300545232, + "grad_norm": 1.3828125, + "learning_rate": 8.525925925925927e-06, + "loss": 0.6619, + "mean_token_accuracy": 0.8355418112128973, + "num_tokens": 49520900.0, + "step": 1850 + }, + { + "entropy": 0.5784041214734316, + "epoch": 3.309780794480917, + "grad_norm": 1.484375, + "learning_rate": 8.451851851851852e-06, + "loss": 0.6816, + "mean_token_accuracy": 0.8331022556871176, + "num_tokens": 49786411.0, + "step": 1860 + }, + { + "entropy": 0.5709506403654814, + "epoch": 3.3275842884166016, + "grad_norm": 1.3984375, + "learning_rate": 8.377777777777779e-06, + "loss": 0.6589, + "mean_token_accuracy": 0.8369544640183448, + "num_tokens": 50056207.0, + "step": 1870 + }, + { + "entropy": 0.5751367569901049, + "epoch": 3.3453877823522866, + "grad_norm": 1.390625, + "learning_rate": 8.303703703703705e-06, + "loss": 0.6735, + "mean_token_accuracy": 0.8336179967969656, + "num_tokens": 50330876.0, + "step": 1880 + }, + { + "entropy": 0.577986160852015, + "epoch": 3.3631912762879717, + "grad_norm": 1.6015625, + "learning_rate": 8.229629629629632e-06, + "loss": 0.6774, + "mean_token_accuracy": 0.8319189596921206, + "num_tokens": 50602004.0, + "step": 1890 + }, + { + "entropy": 0.5784968495368957, + "epoch": 3.3809947702236562, + "grad_norm": 1.4140625, + "learning_rate": 8.155555555555556e-06, + "loss": 0.6728, + "mean_token_accuracy": 0.8320122357457876, + "num_tokens": 50873444.0, + "step": 1900 + }, + { + "epoch": 3.3809947702236562, + "eval_biology_entropy": 1.379322585105896, + "eval_biology_loss": 2.1383986473083496, + "eval_biology_mean_token_accuracy": 0.6115301356315613, + "eval_biology_num_tokens": 50873444.0, + "eval_biology_runtime": 40.4711, + "eval_biology_samples_per_second": 12.354, + "eval_biology_steps_per_second": 3.089, + "step": 1900 + }, + { + "epoch": 3.3809947702236562, + "eval_chemistry_entropy": 0.5861142361164093, + "eval_chemistry_loss": 0.8205327391624451, + "eval_chemistry_mean_token_accuracy": 0.8100807156562805, + "eval_chemistry_num_tokens": 50873444.0, + "eval_chemistry_runtime": 50.1133, + "eval_chemistry_samples_per_second": 9.977, + "eval_chemistry_steps_per_second": 2.494, + "step": 1900 + }, + { + "epoch": 3.3809947702236562, + "eval_math_entropy": 0.6114957261085511, + "eval_math_loss": 1.1055864095687866, + "eval_math_mean_token_accuracy": 0.7713269739151001, + "eval_math_num_tokens": 50873444.0, + "eval_math_runtime": 51.5676, + "eval_math_samples_per_second": 9.696, + "eval_math_steps_per_second": 2.424, + "step": 1900 + }, + { + "epoch": 3.3809947702236562, + "eval_physics_entropy": 0.6135163457393646, + "eval_physics_loss": 0.8826585412025452, + "eval_physics_mean_token_accuracy": 0.7979225769042969, + "eval_physics_num_tokens": 50873444.0, + "eval_physics_runtime": 58.9496, + "eval_physics_samples_per_second": 8.482, + "eval_physics_steps_per_second": 2.12, + "step": 1900 + }, + { + "entropy": 0.5896684370934964, + "epoch": 3.3987982641593413, + "grad_norm": 1.5234375, + "learning_rate": 8.081481481481483e-06, + "loss": 0.6862, + "mean_token_accuracy": 0.8290154490619898, + "num_tokens": 51141077.0, + "step": 1910 + }, + { + "entropy": 0.5699383469298482, + "epoch": 3.4166017580950263, + "grad_norm": 1.578125, + "learning_rate": 8.007407407407408e-06, + "loss": 0.6705, + "mean_token_accuracy": 0.8345434635877609, + "num_tokens": 51406483.0, + "step": 1920 + }, + { + "entropy": 0.5792136957868934, + "epoch": 3.434405252030711, + "grad_norm": 1.546875, + "learning_rate": 7.933333333333334e-06, + "loss": 0.6753, + "mean_token_accuracy": 0.8337912186980247, + "num_tokens": 51668458.0, + "step": 1930 + }, + { + "entropy": 0.5839371480047703, + "epoch": 3.452208745966396, + "grad_norm": 1.53125, + "learning_rate": 7.859259259259259e-06, + "loss": 0.6891, + "mean_token_accuracy": 0.8307471171021461, + "num_tokens": 51928199.0, + "step": 1940 + }, + { + "entropy": 0.574542698264122, + "epoch": 3.470012239902081, + "grad_norm": 1.4140625, + "learning_rate": 7.785185185185185e-06, + "loss": 0.6653, + "mean_token_accuracy": 0.8337355487048626, + "num_tokens": 52199244.0, + "step": 1950 + }, + { + "entropy": 0.5655553802847862, + "epoch": 3.4878157338377656, + "grad_norm": 1.5078125, + "learning_rate": 7.711111111111112e-06, + "loss": 0.6606, + "mean_token_accuracy": 0.8357041519135237, + "num_tokens": 52473476.0, + "step": 1960 + }, + { + "entropy": 0.5800750594586134, + "epoch": 3.5056192277734506, + "grad_norm": 1.28125, + "learning_rate": 7.637037037037037e-06, + "loss": 0.6777, + "mean_token_accuracy": 0.832566126063466, + "num_tokens": 52743275.0, + "step": 1970 + }, + { + "entropy": 0.5716741790995001, + "epoch": 3.523422721709135, + "grad_norm": 1.4375, + "learning_rate": 7.562962962962963e-06, + "loss": 0.6637, + "mean_token_accuracy": 0.8349632371217013, + "num_tokens": 53014827.0, + "step": 1980 + }, + { + "entropy": 0.5780819065868854, + "epoch": 3.54122621564482, + "grad_norm": 1.5859375, + "learning_rate": 7.48888888888889e-06, + "loss": 0.6791, + "mean_token_accuracy": 0.8327229652553797, + "num_tokens": 53285577.0, + "step": 1990 + }, + { + "entropy": 0.5571762975305319, + "epoch": 3.5590297095805052, + "grad_norm": 1.5, + "learning_rate": 7.4148148148148155e-06, + "loss": 0.656, + "mean_token_accuracy": 0.8386137288063764, + "num_tokens": 53553391.0, + "step": 2000 + }, + { + "epoch": 3.5590297095805052, + "eval_biology_entropy": 1.381414409160614, + "eval_biology_loss": 2.13814115524292, + "eval_biology_mean_token_accuracy": 0.6120853550434112, + "eval_biology_num_tokens": 53553391.0, + "eval_biology_runtime": 40.2984, + "eval_biology_samples_per_second": 12.407, + "eval_biology_steps_per_second": 3.102, + "step": 2000 + }, + { + "epoch": 3.5590297095805052, + "eval_chemistry_entropy": 0.5889808633327485, + "eval_chemistry_loss": 0.8193114995956421, + "eval_chemistry_mean_token_accuracy": 0.8103628730773926, + "eval_chemistry_num_tokens": 53553391.0, + "eval_chemistry_runtime": 49.8517, + "eval_chemistry_samples_per_second": 10.03, + "eval_chemistry_steps_per_second": 2.507, + "step": 2000 + }, + { + "epoch": 3.5590297095805052, + "eval_math_entropy": 0.6149049291610718, + "eval_math_loss": 1.1041083335876465, + "eval_math_mean_token_accuracy": 0.7715086970329285, + "eval_math_num_tokens": 53553391.0, + "eval_math_runtime": 51.0164, + "eval_math_samples_per_second": 9.801, + "eval_math_steps_per_second": 2.45, + "step": 2000 + }, + { + "epoch": 3.5590297095805052, + "eval_physics_entropy": 0.6177053947448731, + "eval_physics_loss": 0.8811244368553162, + "eval_physics_mean_token_accuracy": 0.798056589126587, + "eval_physics_num_tokens": 53553391.0, + "eval_physics_runtime": 58.0264, + "eval_physics_samples_per_second": 8.617, + "eval_physics_steps_per_second": 2.154, + "step": 2000 + } + ], + "logging_steps": 10, + "max_steps": 3000, + "num_input_tokens_seen": 0, + "num_train_epochs": 6, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 4.16464083921255e+18, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/checkpoint-2000/training_args.bin b/checkpoint-2000/training_args.bin new file mode 100644 index 0000000..79cad08 --- /dev/null +++ b/checkpoint-2000/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dfc23fa1b9be77c605cba9733debba5f60170f0eadaa2a8866b5aaf4b0e65756 +size 6417 diff --git a/checkpoint-2500/chat_template.jinja b/checkpoint-2500/chat_template.jinja new file mode 100644 index 0000000..923ec25 --- /dev/null +++ b/checkpoint-2500/chat_template.jinja @@ -0,0 +1,4 @@ +{{ bos_token }}{% if messages[0]['role'] == 'system' %}{{ raise_exception('System role not supported') }}{% endif %}{% for message in messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if (message['role'] == 'assistant') %}{% set role = 'model' %}{% else %}{% set role = message['role'] %}{% endif %}{{ '' + role + ' +' + message['content'] | trim + ' +' }}{% endfor %}{% if add_generation_prompt %}{{'model +'}}{% endif %} \ No newline at end of file diff --git a/checkpoint-2500/config.json b/checkpoint-2500/config.json new file mode 100644 index 0000000..29e6cbe --- /dev/null +++ b/checkpoint-2500/config.json @@ -0,0 +1,77 @@ +{ + "architectures": [ + "Gemma2ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "attn_logit_softcapping": 50.0, + "bos_token_id": 2, + "cache_implementation": "hybrid", + "dtype": "bfloat16", + "eos_token_id": 1, + "final_logit_softcapping": 30.0, + "head_dim": 256, + "hidden_act": "gelu_pytorch_tanh", + "hidden_activation": "gelu_pytorch_tanh", + "hidden_size": 3584, + "initializer_range": 0.02, + "intermediate_size": 14336, + "layer_types": [ + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention" + ], + "max_position_embeddings": 8192, + "model_type": "gemma2", + "num_attention_heads": 16, + "num_hidden_layers": 42, + "num_key_value_heads": 8, + "pad_token_id": 0, + "query_pre_attn_scalar": 256, + "rms_norm_eps": 1e-06, + "rope_theta": 10000.0, + "sliding_window": 4096, + "sliding_window_size": 4096, + "transformers_version": "4.56.1", + "use_cache": true, + "vocab_size": 256000 +} diff --git a/checkpoint-2500/generation_config.json b/checkpoint-2500/generation_config.json new file mode 100644 index 0000000..f2c34f2 --- /dev/null +++ b/checkpoint-2500/generation_config.json @@ -0,0 +1,9 @@ +{ + "_from_model_config": true, + "bos_token_id": 2, + "eos_token_id": [ + 1 + ], + "pad_token_id": 0, + "transformers_version": "4.56.1" +} diff --git a/checkpoint-2500/model-00001-of-00004.safetensors b/checkpoint-2500/model-00001-of-00004.safetensors new file mode 100644 index 0000000..4705c7e --- /dev/null +++ b/checkpoint-2500/model-00001-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a0d4eb4fcbddfe01b0dc7a58386a3980826ededaf0ecd67d1ef38995b89615ba +size 4903351912 diff --git a/checkpoint-2500/model-00002-of-00004.safetensors b/checkpoint-2500/model-00002-of-00004.safetensors new file mode 100644 index 0000000..744b59b --- /dev/null +++ b/checkpoint-2500/model-00002-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:712a589c333f49f7b6f3870838a9e0ab4b3c0260abd5130e8c54316e6c99640b +size 4947570872 diff --git a/checkpoint-2500/model-00003-of-00004.safetensors b/checkpoint-2500/model-00003-of-00004.safetensors new file mode 100644 index 0000000..7f37678 --- /dev/null +++ b/checkpoint-2500/model-00003-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:661257cbdb99ab9359e9642d53cb98eeaafbd1d983494af037f3eb3f4f1e1bff +size 4962221464 diff --git a/checkpoint-2500/model-00004-of-00004.safetensors b/checkpoint-2500/model-00004-of-00004.safetensors new file mode 100644 index 0000000..21bd02f --- /dev/null +++ b/checkpoint-2500/model-00004-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:00fe333e607e7cd6fd24a9416b354b570fd50ee7113207a448a4159eb5fe19bf +size 3670322200 diff --git a/checkpoint-2500/model.safetensors.index.json b/checkpoint-2500/model.safetensors.index.json new file mode 100644 index 0000000..3b569c5 --- /dev/null +++ b/checkpoint-2500/model.safetensors.index.json @@ -0,0 +1,472 @@ +{ + "metadata": { + "total_parameters": 9241705984, + "total_size": 18483411968 + }, + "weight_map": { + "model.embed_tokens.weight": "model-00001-of-00004.safetensors", + "model.layers.0.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.0.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.0.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.1.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.1.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.10.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.10.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.10.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.10.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.11.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.11.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.11.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.12.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.12.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.12.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.13.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.13.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.13.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.14.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.14.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.14.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.15.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.15.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.15.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.16.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.16.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.16.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.17.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.17.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.17.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.18.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.18.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.18.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.19.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.19.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.19.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.2.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.2.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.2.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.20.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.20.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.20.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.20.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.20.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.20.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.20.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.21.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.21.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.21.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.21.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.21.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.22.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.22.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.22.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.22.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.23.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.23.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.23.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.24.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.24.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.24.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.25.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.25.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.25.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.26.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.26.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.26.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.27.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.27.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.27.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.28.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.28.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.28.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.29.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.29.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.29.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.3.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.3.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.3.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.30.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.30.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.30.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.30.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.30.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.31.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.31.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.31.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.32.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.32.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.32.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.32.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.32.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.32.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.32.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.32.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.32.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.32.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.32.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.33.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.33.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.33.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.33.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.33.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.33.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.33.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.33.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.33.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.33.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.33.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.34.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.34.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.34.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.34.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.34.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.34.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.34.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.34.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.34.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.34.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.34.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.35.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.35.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.35.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.35.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.35.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.35.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.35.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.35.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.35.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.35.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.35.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.36.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.36.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.36.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.36.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.36.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.36.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.36.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.36.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.36.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.36.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.36.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.37.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.37.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.37.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.37.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.37.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.37.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.37.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.37.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.37.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.37.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.37.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.38.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.38.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.38.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.38.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.38.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.38.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.38.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.38.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.38.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.38.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.38.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.39.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.39.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.39.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.39.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.39.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.39.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.39.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.39.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.39.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.39.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.39.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.4.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.4.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.4.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.4.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.40.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.40.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.40.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.40.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.40.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.40.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.40.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.40.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.40.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.40.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.40.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.41.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.41.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.41.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.41.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.41.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.41.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.41.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.41.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.41.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.41.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.41.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.5.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.5.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.5.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.5.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.6.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.6.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.6.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.7.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.7.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.7.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.8.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.8.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.8.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.8.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.8.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.8.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.8.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.8.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.8.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.8.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.8.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.9.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.9.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.9.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.9.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.norm.weight": "model-00004-of-00004.safetensors" + } +} diff --git a/checkpoint-2500/optimizer.pt b/checkpoint-2500/optimizer.pt new file mode 100644 index 0000000..1984bff --- /dev/null +++ b/checkpoint-2500/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:502a2f740eadd10aecdc5ddc50e09647eb9d91ffa222ca7ecaee5bed8cf428bc +size 1585596266 diff --git a/checkpoint-2500/rng_state.pth b/checkpoint-2500/rng_state.pth new file mode 100644 index 0000000..c904747 --- /dev/null +++ b/checkpoint-2500/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:26982f7f2fa81ad633518ca6a32565668a15c0bdf973e728112f68e6c774f1ab +size 14645 diff --git a/checkpoint-2500/scheduler.pt b/checkpoint-2500/scheduler.pt new file mode 100644 index 0000000..9c20798 --- /dev/null +++ b/checkpoint-2500/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2e9ccea70efeec78e225e7cdc0549b970da327d6b043e24796aa92c2d963349d +size 1465 diff --git a/checkpoint-2500/special_tokens_map.json b/checkpoint-2500/special_tokens_map.json new file mode 100644 index 0000000..8d6368f --- /dev/null +++ b/checkpoint-2500/special_tokens_map.json @@ -0,0 +1,34 @@ +{ + "additional_special_tokens": [ + "", + "" + ], + "bos_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "unk_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + } +} diff --git a/checkpoint-2500/tokenizer.json b/checkpoint-2500/tokenizer.json new file mode 100644 index 0000000..a4a305d --- /dev/null +++ b/checkpoint-2500/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5f7eee611703c5ce5d1eee32d9cdcfe465647b8aff0c1dfb3bed7ad7dbb05060 +size 34362873 diff --git a/checkpoint-2500/tokenizer.model b/checkpoint-2500/tokenizer.model new file mode 100644 index 0000000..796efe9 --- /dev/null +++ b/checkpoint-2500/tokenizer.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:61a7b147390c64585d6c3543dd6fc636906c9af3865a5548f27f31aee1d4c8e2 +size 4241003 diff --git a/checkpoint-2500/tokenizer_config.json b/checkpoint-2500/tokenizer_config.json new file mode 100644 index 0000000..3ade6be --- /dev/null +++ b/checkpoint-2500/tokenizer_config.json @@ -0,0 +1,2013 @@ +{ + "add_bos_token": true, + "add_eos_token": false, + "added_tokens_decoder": { + "0": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "1": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "2": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "3": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "4": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "5": { + "content": "<2mass>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "6": { + "content": "[@BOS@]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "7": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "8": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "9": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "10": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "11": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "12": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "13": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "14": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "15": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "16": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "17": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "18": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "19": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "20": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "21": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "22": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "23": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "24": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "25": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "26": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "27": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "28": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "29": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "30": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "31": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "32": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "33": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "34": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "35": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "36": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "37": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "38": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "39": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "40": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "41": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "42": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "43": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "44": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "45": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "46": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "47": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "48": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "49": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "50": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "51": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "52": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "53": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "54": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "55": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "56": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "57": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "58": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "59": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "60": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "61": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "62": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "63": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "64": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "65": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "66": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "67": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "68": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "69": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "70": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "71": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "72": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "73": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "74": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "75": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "76": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "77": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "78": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "79": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "80": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "81": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "82": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "83": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "84": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "85": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "86": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "87": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "88": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "89": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "90": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "91": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "92": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "93": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "94": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "95": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "96": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "97": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "98": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "99": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "100": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "101": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "102": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "103": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "104": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "105": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "106": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "107": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "108": { + "content": "\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "109": { + "content": "\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "110": { + "content": "\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "111": { + "content": "\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "112": { + "content": "\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "113": { + "content": "\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "114": { + "content": "\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "115": { + "content": "\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "116": { + "content": "\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "117": { + "content": "\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "118": { + "content": "\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "119": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "120": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "121": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "122": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "123": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "124": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "125": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "126": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "127": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "128": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "129": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "130": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "131": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "132": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "133": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "134": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "135": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "136": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "137": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "138": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "139": { + "content": "▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "140": { + "content": "▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "141": { + "content": "▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "142": { + "content": "▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "143": { + "content": "▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "144": { + "content": "▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "145": { + "content": "▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "146": { + "content": "▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "147": { + "content": "▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "148": { + "content": "▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "149": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "150": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "152": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "153": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "154": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "155": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "156": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "157": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "158": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "159": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "160": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "161": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "162": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "163": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "164": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "165": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "166": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "167": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "168": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "169": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "170": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "172": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "173": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "174": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "175": { + "content": "
", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "171": { + "content": "
", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "176": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "177": { + "content": "
", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "178": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "179": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "180": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "181": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "182": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "183": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "184": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "185": { + "content": "

", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "186": { + "content": "

", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "187": { + "content": "

", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "188": { + "content": "

", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "189": { + "content": "

", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "190": { + "content": "
", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "191": { + "content": "
", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "192": { + "content": "
", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "193": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "194": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "195": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "196": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "197": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "198": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "199": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "200": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "201": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "202": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "203": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "204": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "205": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "206": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "207": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "208": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "209": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "210": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "211": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "212": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "213": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "214": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "215": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "216": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255968": { + "content": "[toxicity=0]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255969": { + "content": "\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255970": { + "content": "\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255971": { + "content": "\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255972": { + "content": "\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255973": { + "content": "\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255974": { + "content": "\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255975": { + "content": "\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255976": { + "content": "\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255977": { + "content": "\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255978": { + "content": "\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255979": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255980": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255981": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255982": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255983": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255984": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255985": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255986": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255987": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255988": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255989": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255990": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255991": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255992": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255993": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255994": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255995": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255996": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255997": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255998": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255999": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + } + }, + "additional_special_tokens": [ + "", + "" + ], + "bos_token": "", + "clean_up_tokenization_spaces": false, + "eos_token": "", + "extra_special_tokens": {}, + "model_max_length": 1000000000000000019884624838656, + "pad_token": "", + "sp_model_kwargs": {}, + "spaces_between_special_tokens": false, + "tokenizer_class": "GemmaTokenizer", + "unk_token": "", + "use_default_system_prompt": false +} diff --git a/checkpoint-2500/trainer_state.json b/checkpoint-2500/trainer_state.json new file mode 100644 index 0000000..6519230 --- /dev/null +++ b/checkpoint-2500/trainer_state.json @@ -0,0 +1,3634 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 4.448648047179259, + "eval_steps": 100, + "global_step": 2500, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.3253214355558156, + "epoch": 0.017803493935684877, + "grad_norm": 14.5, + "learning_rate": 6.000000000000001e-07, + "loss": 2.1959, + "mean_token_accuracy": 0.5975642748177051, + "num_tokens": 259048.0, + "step": 10 + }, + { + "entropy": 1.3036176152527332, + "epoch": 0.035606987871369754, + "grad_norm": 12.75, + "learning_rate": 1.2666666666666669e-06, + "loss": 2.1499, + "mean_token_accuracy": 0.6023047739639878, + "num_tokens": 532019.0, + "step": 20 + }, + { + "entropy": 1.3484113823622466, + "epoch": 0.053410481807054634, + "grad_norm": 10.8125, + "learning_rate": 1.9333333333333336e-06, + "loss": 2.108, + "mean_token_accuracy": 0.6078463492915034, + "num_tokens": 807220.0, + "step": 30 + }, + { + "entropy": 1.4311681490391492, + "epoch": 0.07121397574273951, + "grad_norm": 7.875, + "learning_rate": 2.6e-06, + "loss": 2.0349, + "mean_token_accuracy": 0.6107733339071274, + "num_tokens": 1068429.0, + "step": 40 + }, + { + "entropy": 1.479667642340064, + "epoch": 0.08901746967842439, + "grad_norm": 6.4375, + "learning_rate": 3.266666666666667e-06, + "loss": 1.8649, + "mean_token_accuracy": 0.630409375578165, + "num_tokens": 1336698.0, + "step": 50 + }, + { + "entropy": 1.5030938494950532, + "epoch": 0.10682096361410927, + "grad_norm": 4.1875, + "learning_rate": 3.9333333333333335e-06, + "loss": 1.7217, + "mean_token_accuracy": 0.6490663634613156, + "num_tokens": 1608574.0, + "step": 60 + }, + { + "entropy": 1.5066865295171739, + "epoch": 0.12462445754979415, + "grad_norm": 2.71875, + "learning_rate": 4.600000000000001e-06, + "loss": 1.5716, + "mean_token_accuracy": 0.6711439102888107, + "num_tokens": 1872747.0, + "step": 70 + }, + { + "entropy": 1.3747932206839324, + "epoch": 0.14242795148547902, + "grad_norm": 2.234375, + "learning_rate": 5.2666666666666665e-06, + "loss": 1.4403, + "mean_token_accuracy": 0.6936415441334247, + "num_tokens": 2133173.0, + "step": 80 + }, + { + "entropy": 1.1736511185765266, + "epoch": 0.1602314454211639, + "grad_norm": 1.921875, + "learning_rate": 5.933333333333335e-06, + "loss": 1.2902, + "mean_token_accuracy": 0.7222886722534895, + "num_tokens": 2406145.0, + "step": 90 + }, + { + "entropy": 1.1317794900387526, + "epoch": 0.17803493935684878, + "grad_norm": 1.8046875, + "learning_rate": 6.600000000000001e-06, + "loss": 1.2424, + "mean_token_accuracy": 0.7295983772724867, + "num_tokens": 2663709.0, + "step": 100 + }, + { + "epoch": 0.17803493935684878, + "eval_biology_entropy": 2.3133783559799195, + "eval_biology_loss": 2.7910587787628174, + "eval_biology_mean_token_accuracy": 0.5260232911109924, + "eval_biology_num_tokens": 2663709.0, + "eval_biology_runtime": 39.1656, + "eval_biology_samples_per_second": 12.766, + "eval_biology_steps_per_second": 3.192, + "step": 100 + }, + { + "epoch": 0.17803493935684878, + "eval_chemistry_entropy": 1.0788086318969727, + "eval_chemistry_loss": 1.2403336763381958, + "eval_chemistry_mean_token_accuracy": 0.7357268543243408, + "eval_chemistry_num_tokens": 2663709.0, + "eval_chemistry_runtime": 48.7822, + "eval_chemistry_samples_per_second": 10.25, + "eval_chemistry_steps_per_second": 2.562, + "step": 100 + }, + { + "epoch": 0.17803493935684878, + "eval_math_entropy": 0.938195451259613, + "eval_math_loss": 1.3248047828674316, + "eval_math_mean_token_accuracy": 0.7312402629852295, + "eval_math_num_tokens": 2663709.0, + "eval_math_runtime": 50.197, + "eval_math_samples_per_second": 9.961, + "eval_math_steps_per_second": 2.49, + "step": 100 + }, + { + "epoch": 0.17803493935684878, + "eval_physics_entropy": 1.022246531009674, + "eval_physics_loss": 1.176153302192688, + "eval_physics_mean_token_accuracy": 0.7475795021057129, + "eval_physics_num_tokens": 2663709.0, + "eval_physics_runtime": 57.6895, + "eval_physics_samples_per_second": 8.667, + "eval_physics_steps_per_second": 2.167, + "step": 100 + }, + { + "entropy": 1.0380497876554728, + "epoch": 0.19583843329253367, + "grad_norm": 1.7421875, + "learning_rate": 7.266666666666668e-06, + "loss": 1.157, + "mean_token_accuracy": 0.7445634145289659, + "num_tokens": 2934980.0, + "step": 110 + }, + { + "entropy": 1.0149445166811346, + "epoch": 0.21364192722821854, + "grad_norm": 1.6015625, + "learning_rate": 7.933333333333334e-06, + "loss": 1.1167, + "mean_token_accuracy": 0.7485828153789044, + "num_tokens": 3208702.0, + "step": 120 + }, + { + "entropy": 0.9639404848217964, + "epoch": 0.2314454211639034, + "grad_norm": 1.6953125, + "learning_rate": 8.6e-06, + "loss": 1.0979, + "mean_token_accuracy": 0.7553199239075183, + "num_tokens": 3480119.0, + "step": 130 + }, + { + "entropy": 0.9687324531376362, + "epoch": 0.2492489150995883, + "grad_norm": 1.5625, + "learning_rate": 9.266666666666667e-06, + "loss": 1.0762, + "mean_token_accuracy": 0.757096454873681, + "num_tokens": 3746392.0, + "step": 140 + }, + { + "entropy": 0.9682734729722142, + "epoch": 0.26705240903527316, + "grad_norm": 1.875, + "learning_rate": 9.933333333333334e-06, + "loss": 1.0798, + "mean_token_accuracy": 0.7543047085404396, + "num_tokens": 4019487.0, + "step": 150 + }, + { + "entropy": 0.9270784268155694, + "epoch": 0.28485590297095803, + "grad_norm": 1.6015625, + "learning_rate": 1.0600000000000002e-05, + "loss": 1.047, + "mean_token_accuracy": 0.7619738396257162, + "num_tokens": 4298100.0, + "step": 160 + }, + { + "entropy": 0.9063327137380839, + "epoch": 0.30265939690664295, + "grad_norm": 1.5546875, + "learning_rate": 1.1266666666666668e-05, + "loss": 1.0199, + "mean_token_accuracy": 0.7664302848279476, + "num_tokens": 4568116.0, + "step": 170 + }, + { + "entropy": 0.9037324048578739, + "epoch": 0.3204628908423278, + "grad_norm": 1.7109375, + "learning_rate": 1.1933333333333335e-05, + "loss": 1.0144, + "mean_token_accuracy": 0.7657378144562245, + "num_tokens": 4831334.0, + "step": 180 + }, + { + "entropy": 0.8886868219822646, + "epoch": 0.3382663847780127, + "grad_norm": 1.3359375, + "learning_rate": 1.2600000000000001e-05, + "loss": 0.9874, + "mean_token_accuracy": 0.7726409930735827, + "num_tokens": 5103301.0, + "step": 190 + }, + { + "entropy": 0.8936551520600915, + "epoch": 0.35606987871369755, + "grad_norm": 1.3984375, + "learning_rate": 1.3266666666666668e-05, + "loss": 1.019, + "mean_token_accuracy": 0.7648640714585782, + "num_tokens": 5362915.0, + "step": 200 + }, + { + "epoch": 0.35606987871369755, + "eval_biology_entropy": 2.0171296310424807, + "eval_biology_loss": 2.3780677318573, + "eval_biology_mean_token_accuracy": 0.5706260974407196, + "eval_biology_num_tokens": 5362915.0, + "eval_biology_runtime": 39.1203, + "eval_biology_samples_per_second": 12.781, + "eval_biology_steps_per_second": 3.195, + "step": 200 + }, + { + "epoch": 0.35606987871369755, + "eval_chemistry_entropy": 0.8995157589912415, + "eval_chemistry_loss": 1.022930383682251, + "eval_chemistry_mean_token_accuracy": 0.7701377034187317, + "eval_chemistry_num_tokens": 5362915.0, + "eval_chemistry_runtime": 48.7263, + "eval_chemistry_samples_per_second": 10.261, + "eval_chemistry_steps_per_second": 2.565, + "step": 200 + }, + { + "epoch": 0.35606987871369755, + "eval_math_entropy": 0.8352043969631195, + "eval_math_loss": 1.1696399450302124, + "eval_math_mean_token_accuracy": 0.7531440143585205, + "eval_math_num_tokens": 5362915.0, + "eval_math_runtime": 50.1494, + "eval_math_samples_per_second": 9.97, + "eval_math_steps_per_second": 2.493, + "step": 200 + }, + { + "epoch": 0.35606987871369755, + "eval_physics_entropy": 0.8889413847923279, + "eval_physics_loss": 0.9962027072906494, + "eval_physics_mean_token_accuracy": 0.775176504611969, + "eval_physics_num_tokens": 5362915.0, + "eval_physics_runtime": 57.6533, + "eval_physics_samples_per_second": 8.673, + "eval_physics_steps_per_second": 2.168, + "step": 200 + }, + { + "entropy": 0.8509893301874399, + "epoch": 0.3738733726493824, + "grad_norm": 1.3359375, + "learning_rate": 1.3933333333333334e-05, + "loss": 0.9619, + "mean_token_accuracy": 0.7772600032389164, + "num_tokens": 5625455.0, + "step": 210 + }, + { + "entropy": 0.8560685662552714, + "epoch": 0.39167686658506734, + "grad_norm": 1.4609375, + "learning_rate": 1.46e-05, + "loss": 0.9666, + "mean_token_accuracy": 0.7764565829187632, + "num_tokens": 5890806.0, + "step": 220 + }, + { + "entropy": 0.8282079607248306, + "epoch": 0.4094803605207522, + "grad_norm": 1.3828125, + "learning_rate": 1.5266666666666667e-05, + "loss": 0.9353, + "mean_token_accuracy": 0.7817027729004622, + "num_tokens": 6158751.0, + "step": 230 + }, + { + "entropy": 0.8447791969403624, + "epoch": 0.4272838544564371, + "grad_norm": 1.4296875, + "learning_rate": 1.5933333333333336e-05, + "loss": 0.9428, + "mean_token_accuracy": 0.7795210376381874, + "num_tokens": 6423747.0, + "step": 240 + }, + { + "entropy": 0.8478817906230688, + "epoch": 0.44508734839212194, + "grad_norm": 1.8984375, + "learning_rate": 1.66e-05, + "loss": 0.9542, + "mean_token_accuracy": 0.7772787544876337, + "num_tokens": 6684517.0, + "step": 250 + }, + { + "entropy": 0.8135418241843582, + "epoch": 0.4628908423278068, + "grad_norm": 1.453125, + "learning_rate": 1.726666666666667e-05, + "loss": 0.9297, + "mean_token_accuracy": 0.783412617444992, + "num_tokens": 6945659.0, + "step": 260 + }, + { + "entropy": 0.8251218661665917, + "epoch": 0.48069433626349173, + "grad_norm": 1.6640625, + "learning_rate": 1.7933333333333333e-05, + "loss": 0.9307, + "mean_token_accuracy": 0.7831020273268223, + "num_tokens": 7217797.0, + "step": 270 + }, + { + "entropy": 0.7879339458420873, + "epoch": 0.4984978301991766, + "grad_norm": 1.328125, + "learning_rate": 1.86e-05, + "loss": 0.8795, + "mean_token_accuracy": 0.7897050529718399, + "num_tokens": 7498182.0, + "step": 280 + }, + { + "entropy": 0.8145477209240198, + "epoch": 0.5163013241348615, + "grad_norm": 1.265625, + "learning_rate": 1.926666666666667e-05, + "loss": 0.9187, + "mean_token_accuracy": 0.7838029216974973, + "num_tokens": 7761686.0, + "step": 290 + }, + { + "entropy": 0.8231891892850399, + "epoch": 0.5341048180705463, + "grad_norm": 1.5625, + "learning_rate": 1.9933333333333334e-05, + "loss": 0.9271, + "mean_token_accuracy": 0.7819279827177524, + "num_tokens": 8026923.0, + "step": 300 + }, + { + "epoch": 0.5341048180705463, + "eval_biology_entropy": 1.8821040544509888, + "eval_biology_loss": 2.226550340652466, + "eval_biology_mean_token_accuracy": 0.5873979187011719, + "eval_biology_num_tokens": 8026923.0, + "eval_biology_runtime": 39.1096, + "eval_biology_samples_per_second": 12.785, + "eval_biology_steps_per_second": 3.196, + "step": 300 + }, + { + "epoch": 0.5341048180705463, + "eval_chemistry_entropy": 0.7922393922805786, + "eval_chemistry_loss": 0.9438055157661438, + "eval_chemistry_mean_token_accuracy": 0.7841580080986023, + "eval_chemistry_num_tokens": 8026923.0, + "eval_chemistry_runtime": 48.7268, + "eval_chemistry_samples_per_second": 10.261, + "eval_chemistry_steps_per_second": 2.565, + "step": 300 + }, + { + "epoch": 0.5341048180705463, + "eval_math_entropy": 0.7649447858333588, + "eval_math_loss": 1.1145466566085815, + "eval_math_mean_token_accuracy": 0.763204463005066, + "eval_math_num_tokens": 8026923.0, + "eval_math_runtime": 50.1205, + "eval_math_samples_per_second": 9.976, + "eval_math_steps_per_second": 2.494, + "step": 300 + }, + { + "epoch": 0.5341048180705463, + "eval_physics_entropy": 0.7964776389598847, + "eval_physics_loss": 0.9373180866241455, + "eval_physics_mean_token_accuracy": 0.784841582775116, + "eval_physics_num_tokens": 8026923.0, + "eval_physics_runtime": 57.6222, + "eval_physics_samples_per_second": 8.677, + "eval_physics_steps_per_second": 2.169, + "step": 300 + }, + { + "entropy": 0.800599187426269, + "epoch": 0.5519083120062312, + "grad_norm": 1.6640625, + "learning_rate": 1.9933333333333334e-05, + "loss": 0.9071, + "mean_token_accuracy": 0.7856069568544626, + "num_tokens": 8301506.0, + "step": 310 + }, + { + "entropy": 0.7965721761807799, + "epoch": 0.5697118059419161, + "grad_norm": 1.234375, + "learning_rate": 1.985925925925926e-05, + "loss": 0.9139, + "mean_token_accuracy": 0.786777526140213, + "num_tokens": 8559126.0, + "step": 320 + }, + { + "entropy": 0.7925007566809654, + "epoch": 0.587515299877601, + "grad_norm": 1.4453125, + "learning_rate": 1.9785185185185187e-05, + "loss": 0.906, + "mean_token_accuracy": 0.7864235289394855, + "num_tokens": 8821088.0, + "step": 330 + }, + { + "entropy": 0.7728251311928034, + "epoch": 0.6053187938132859, + "grad_norm": 1.2109375, + "learning_rate": 1.971111111111111e-05, + "loss": 0.8764, + "mean_token_accuracy": 0.7919386122375727, + "num_tokens": 9089866.0, + "step": 340 + }, + { + "entropy": 0.7705478269606829, + "epoch": 0.6231222877489707, + "grad_norm": 1.3515625, + "learning_rate": 1.963703703703704e-05, + "loss": 0.8837, + "mean_token_accuracy": 0.7896027479320764, + "num_tokens": 9358866.0, + "step": 350 + }, + { + "entropy": 0.7656249281018972, + "epoch": 0.6409257816846556, + "grad_norm": 1.1875, + "learning_rate": 1.9562962962962964e-05, + "loss": 0.8626, + "mean_token_accuracy": 0.7931958142668009, + "num_tokens": 9629977.0, + "step": 360 + }, + { + "entropy": 0.7598591955378652, + "epoch": 0.6587292756203404, + "grad_norm": 1.296875, + "learning_rate": 1.948888888888889e-05, + "loss": 0.8608, + "mean_token_accuracy": 0.7945109561085701, + "num_tokens": 9897757.0, + "step": 370 + }, + { + "entropy": 0.7398761732503771, + "epoch": 0.6765327695560254, + "grad_norm": 1.265625, + "learning_rate": 1.9414814814814817e-05, + "loss": 0.848, + "mean_token_accuracy": 0.7985429972410202, + "num_tokens": 10168367.0, + "step": 380 + }, + { + "entropy": 0.7331193052232265, + "epoch": 0.6943362634917103, + "grad_norm": 1.4296875, + "learning_rate": 1.9340740740740743e-05, + "loss": 0.8366, + "mean_token_accuracy": 0.8001827634871006, + "num_tokens": 10439039.0, + "step": 390 + }, + { + "entropy": 0.7496613372117281, + "epoch": 0.7121397574273951, + "grad_norm": 1.15625, + "learning_rate": 1.926666666666667e-05, + "loss": 0.8517, + "mean_token_accuracy": 0.7967161379754544, + "num_tokens": 10709549.0, + "step": 400 + }, + { + "epoch": 0.7121397574273951, + "eval_biology_entropy": 1.823248610496521, + "eval_biology_loss": 2.159681558609009, + "eval_biology_mean_token_accuracy": 0.5972214796543122, + "eval_biology_num_tokens": 10709549.0, + "eval_biology_runtime": 39.2024, + "eval_biology_samples_per_second": 12.754, + "eval_biology_steps_per_second": 3.189, + "step": 400 + }, + { + "epoch": 0.7121397574273951, + "eval_chemistry_entropy": 0.7672218294143677, + "eval_chemistry_loss": 0.8971230387687683, + "eval_chemistry_mean_token_accuracy": 0.7928389143943787, + "eval_chemistry_num_tokens": 10709549.0, + "eval_chemistry_runtime": 48.8296, + "eval_chemistry_samples_per_second": 10.24, + "eval_chemistry_steps_per_second": 2.56, + "step": 400 + }, + { + "epoch": 0.7121397574273951, + "eval_math_entropy": 0.7494234163761139, + "eval_math_loss": 1.0849554538726807, + "eval_math_mean_token_accuracy": 0.7666959943771362, + "eval_math_num_tokens": 10709549.0, + "eval_math_runtime": 50.2325, + "eval_math_samples_per_second": 9.954, + "eval_math_steps_per_second": 2.488, + "step": 400 + }, + { + "epoch": 0.7121397574273951, + "eval_physics_entropy": 0.7731240901947022, + "eval_physics_loss": 0.9018430709838867, + "eval_physics_mean_token_accuracy": 0.7909709796905517, + "eval_physics_num_tokens": 10709549.0, + "eval_physics_runtime": 57.7809, + "eval_physics_samples_per_second": 8.653, + "eval_physics_steps_per_second": 2.163, + "step": 400 + }, + { + "entropy": 0.7354462265968322, + "epoch": 0.72994325136308, + "grad_norm": 1.3125, + "learning_rate": 1.9192592592592593e-05, + "loss": 0.8411, + "mean_token_accuracy": 0.8003354988992214, + "num_tokens": 10976011.0, + "step": 410 + }, + { + "entropy": 0.7435122076421976, + "epoch": 0.7477467452987648, + "grad_norm": 1.15625, + "learning_rate": 1.911851851851852e-05, + "loss": 0.8438, + "mean_token_accuracy": 0.7971596572548151, + "num_tokens": 11245251.0, + "step": 420 + }, + { + "entropy": 0.759309609234333, + "epoch": 0.7655502392344498, + "grad_norm": 1.375, + "learning_rate": 1.9044444444444446e-05, + "loss": 0.8579, + "mean_token_accuracy": 0.792990094423294, + "num_tokens": 11518514.0, + "step": 430 + }, + { + "entropy": 0.7545185785740613, + "epoch": 0.7833537331701347, + "grad_norm": 1.3125, + "learning_rate": 1.8970370370370372e-05, + "loss": 0.8574, + "mean_token_accuracy": 0.794682926684618, + "num_tokens": 11787117.0, + "step": 440 + }, + { + "entropy": 0.751254404336214, + "epoch": 0.8011572271058195, + "grad_norm": 1.515625, + "learning_rate": 1.8896296296296295e-05, + "loss": 0.8495, + "mean_token_accuracy": 0.7972703959792853, + "num_tokens": 12050993.0, + "step": 450 + }, + { + "entropy": 0.7543802969157696, + "epoch": 0.8189607210415044, + "grad_norm": 1.28125, + "learning_rate": 1.8822222222222225e-05, + "loss": 0.8527, + "mean_token_accuracy": 0.7960003320127725, + "num_tokens": 12311688.0, + "step": 460 + }, + { + "entropy": 0.7186065271496773, + "epoch": 0.8367642149771892, + "grad_norm": 1.15625, + "learning_rate": 1.874814814814815e-05, + "loss": 0.8096, + "mean_token_accuracy": 0.8027401898056269, + "num_tokens": 12598408.0, + "step": 470 + }, + { + "entropy": 0.7257912319153548, + "epoch": 0.8545677089128741, + "grad_norm": 1.125, + "learning_rate": 1.8674074074074075e-05, + "loss": 0.8299, + "mean_token_accuracy": 0.8012042317539454, + "num_tokens": 12869642.0, + "step": 480 + }, + { + "entropy": 0.7173270899802446, + "epoch": 0.8723712028485591, + "grad_norm": 1.125, + "learning_rate": 1.86e-05, + "loss": 0.8146, + "mean_token_accuracy": 0.8032398864626884, + "num_tokens": 13139555.0, + "step": 490 + }, + { + "entropy": 0.7487573977559805, + "epoch": 0.8901746967842439, + "grad_norm": 1.2421875, + "learning_rate": 1.8525925925925928e-05, + "loss": 0.8562, + "mean_token_accuracy": 0.7955521304160357, + "num_tokens": 13396545.0, + "step": 500 + }, + { + "epoch": 0.8901746967842439, + "eval_biology_entropy": 1.8204833917617798, + "eval_biology_loss": 2.1014022827148438, + "eval_biology_mean_token_accuracy": 0.6051526765823364, + "eval_biology_num_tokens": 13396545.0, + "eval_biology_runtime": 39.1543, + "eval_biology_samples_per_second": 12.77, + "eval_biology_steps_per_second": 3.192, + "step": 500 + }, + { + "epoch": 0.8901746967842439, + "eval_chemistry_entropy": 0.7538458592891694, + "eval_chemistry_loss": 0.8699584007263184, + "eval_chemistry_mean_token_accuracy": 0.7975720787048339, + "eval_chemistry_num_tokens": 13396545.0, + "eval_chemistry_runtime": 48.8229, + "eval_chemistry_samples_per_second": 10.241, + "eval_chemistry_steps_per_second": 2.56, + "step": 500 + }, + { + "epoch": 0.8901746967842439, + "eval_math_entropy": 0.7398368701934814, + "eval_math_loss": 1.0758970975875854, + "eval_math_mean_token_accuracy": 0.7687290472984314, + "eval_math_num_tokens": 13396545.0, + "eval_math_runtime": 50.0916, + "eval_math_samples_per_second": 9.982, + "eval_math_steps_per_second": 2.495, + "step": 500 + }, + { + "epoch": 0.8901746967842439, + "eval_physics_entropy": 0.7607374501228332, + "eval_physics_loss": 0.8848790526390076, + "eval_physics_mean_token_accuracy": 0.7937801394462586, + "eval_physics_num_tokens": 13396545.0, + "eval_physics_runtime": 57.6599, + "eval_physics_samples_per_second": 8.672, + "eval_physics_steps_per_second": 2.168, + "step": 500 + }, + { + "entropy": 0.7129743231460453, + "epoch": 0.9079781907199288, + "grad_norm": 1.1640625, + "learning_rate": 1.8451851851851855e-05, + "loss": 0.8109, + "mean_token_accuracy": 0.8060642663389445, + "num_tokens": 13667857.0, + "step": 510 + }, + { + "entropy": 0.7393450669944286, + "epoch": 0.9257816846556136, + "grad_norm": 1.1484375, + "learning_rate": 1.8377777777777778e-05, + "loss": 0.8325, + "mean_token_accuracy": 0.7997861213982105, + "num_tokens": 13932771.0, + "step": 520 + }, + { + "entropy": 0.7191115407273173, + "epoch": 0.9435851785912985, + "grad_norm": 1.1640625, + "learning_rate": 1.8303703703703704e-05, + "loss": 0.8233, + "mean_token_accuracy": 0.8034567594528198, + "num_tokens": 14194632.0, + "step": 530 + }, + { + "entropy": 0.7434917811304331, + "epoch": 0.9613886725269835, + "grad_norm": 1.1484375, + "learning_rate": 1.822962962962963e-05, + "loss": 0.8435, + "mean_token_accuracy": 0.7969928599894047, + "num_tokens": 14457650.0, + "step": 540 + }, + { + "entropy": 0.7100320702418685, + "epoch": 0.9791921664626683, + "grad_norm": 1.125, + "learning_rate": 1.8155555555555557e-05, + "loss": 0.8149, + "mean_token_accuracy": 0.803739969804883, + "num_tokens": 14724967.0, + "step": 550 + }, + { + "entropy": 0.737933493219316, + "epoch": 0.9969956603983532, + "grad_norm": 1.1875, + "learning_rate": 1.8081481481481484e-05, + "loss": 0.8421, + "mean_token_accuracy": 0.8001800544559956, + "num_tokens": 14993272.0, + "step": 560 + }, + { + "entropy": 0.6655864379098339, + "epoch": 1.014242795148548, + "grad_norm": 1.1171875, + "learning_rate": 1.800740740740741e-05, + "loss": 0.7668, + "mean_token_accuracy": 0.814733551779101, + "num_tokens": 15254936.0, + "step": 570 + }, + { + "entropy": 0.6937626471742988, + "epoch": 1.0320462890842328, + "grad_norm": 1.2578125, + "learning_rate": 1.7933333333333333e-05, + "loss": 0.7942, + "mean_token_accuracy": 0.806296594068408, + "num_tokens": 15515007.0, + "step": 580 + }, + { + "entropy": 0.6596947606652975, + "epoch": 1.0498497830199176, + "grad_norm": 1.2890625, + "learning_rate": 1.785925925925926e-05, + "loss": 0.7623, + "mean_token_accuracy": 0.8152227636426688, + "num_tokens": 15794203.0, + "step": 590 + }, + { + "entropy": 0.6868148937821388, + "epoch": 1.0676532769556026, + "grad_norm": 1.1640625, + "learning_rate": 1.7785185185185186e-05, + "loss": 0.7827, + "mean_token_accuracy": 0.8101296100765467, + "num_tokens": 16063664.0, + "step": 600 + }, + { + "epoch": 1.0676532769556026, + "eval_biology_entropy": 1.6867316131591796, + "eval_biology_loss": 2.077732801437378, + "eval_biology_mean_token_accuracy": 0.6078621828556061, + "eval_biology_num_tokens": 16063664.0, + "eval_biology_runtime": 39.1079, + "eval_biology_samples_per_second": 12.785, + "eval_biology_steps_per_second": 3.196, + "step": 600 + }, + { + "epoch": 1.0676532769556026, + "eval_chemistry_entropy": 0.6882545161247253, + "eval_chemistry_loss": 0.8549924492835999, + "eval_chemistry_mean_token_accuracy": 0.8009334373474121, + "eval_chemistry_num_tokens": 16063664.0, + "eval_chemistry_runtime": 48.732, + "eval_chemistry_samples_per_second": 10.26, + "eval_chemistry_steps_per_second": 2.565, + "step": 600 + }, + { + "epoch": 1.0676532769556026, + "eval_math_entropy": 0.6883157949447631, + "eval_math_loss": 1.075770616531372, + "eval_math_mean_token_accuracy": 0.7705092186927796, + "eval_math_num_tokens": 16063664.0, + "eval_math_runtime": 50.1345, + "eval_math_samples_per_second": 9.973, + "eval_math_steps_per_second": 2.493, + "step": 600 + }, + { + "epoch": 1.0676532769556026, + "eval_physics_entropy": 0.7055722939968109, + "eval_physics_loss": 0.8771266937255859, + "eval_physics_mean_token_accuracy": 0.7955313258171082, + "eval_physics_num_tokens": 16063664.0, + "eval_physics_runtime": 57.6658, + "eval_physics_samples_per_second": 8.671, + "eval_physics_steps_per_second": 2.168, + "step": 600 + }, + { + "entropy": 0.6597639428451657, + "epoch": 1.0854567708912874, + "grad_norm": 1.203125, + "learning_rate": 1.7711111111111113e-05, + "loss": 0.7655, + "mean_token_accuracy": 0.8151189126074314, + "num_tokens": 16338303.0, + "step": 610 + }, + { + "entropy": 0.6991181125864386, + "epoch": 1.1032602648269723, + "grad_norm": 1.2734375, + "learning_rate": 1.763703703703704e-05, + "loss": 0.7934, + "mean_token_accuracy": 0.8071035247296094, + "num_tokens": 16606302.0, + "step": 620 + }, + { + "entropy": 0.6774254593998194, + "epoch": 1.121063758762657, + "grad_norm": 1.28125, + "learning_rate": 1.7562962962962962e-05, + "loss": 0.7706, + "mean_token_accuracy": 0.8098537240177393, + "num_tokens": 16877071.0, + "step": 630 + }, + { + "entropy": 0.6678685389459134, + "epoch": 1.138867252698342, + "grad_norm": 1.203125, + "learning_rate": 1.7488888888888892e-05, + "loss": 0.7714, + "mean_token_accuracy": 0.8127609983086586, + "num_tokens": 17145775.0, + "step": 640 + }, + { + "entropy": 0.6851805068552494, + "epoch": 1.156670746634027, + "grad_norm": 1.2890625, + "learning_rate": 1.7414814814814815e-05, + "loss": 0.777, + "mean_token_accuracy": 0.8110932789742946, + "num_tokens": 17410775.0, + "step": 650 + }, + { + "entropy": 0.7042925998568534, + "epoch": 1.1744742405697117, + "grad_norm": 1.375, + "learning_rate": 1.7340740740740742e-05, + "loss": 0.7979, + "mean_token_accuracy": 0.8038393501192331, + "num_tokens": 17677791.0, + "step": 660 + }, + { + "entropy": 0.6810087833553553, + "epoch": 1.1922777345053968, + "grad_norm": 1.1484375, + "learning_rate": 1.726666666666667e-05, + "loss": 0.7688, + "mean_token_accuracy": 0.8104576140642166, + "num_tokens": 17949441.0, + "step": 670 + }, + { + "entropy": 0.662551742233336, + "epoch": 1.2100812284410816, + "grad_norm": 1.2109375, + "learning_rate": 1.7192592592592595e-05, + "loss": 0.7653, + "mean_token_accuracy": 0.8126230895519256, + "num_tokens": 18217742.0, + "step": 680 + }, + { + "entropy": 0.6803617887198925, + "epoch": 1.2278847223767664, + "grad_norm": 1.2421875, + "learning_rate": 1.711851851851852e-05, + "loss": 0.7771, + "mean_token_accuracy": 0.8095329128205776, + "num_tokens": 18487769.0, + "step": 690 + }, + { + "entropy": 0.6707866100594402, + "epoch": 1.2456882163124514, + "grad_norm": 1.34375, + "learning_rate": 1.7044444444444445e-05, + "loss": 0.7687, + "mean_token_accuracy": 0.8113945126533508, + "num_tokens": 18760296.0, + "step": 700 + }, + { + "epoch": 1.2456882163124514, + "eval_biology_entropy": 1.6589141931533813, + "eval_biology_loss": 2.081825017929077, + "eval_biology_mean_token_accuracy": 0.608892562866211, + "eval_biology_num_tokens": 18760296.0, + "eval_biology_runtime": 39.8083, + "eval_biology_samples_per_second": 12.56, + "eval_biology_steps_per_second": 3.14, + "step": 700 + }, + { + "epoch": 1.2456882163124514, + "eval_chemistry_entropy": 0.6833467047214508, + "eval_chemistry_loss": 0.8464663028717041, + "eval_chemistry_mean_token_accuracy": 0.80278346824646, + "eval_chemistry_num_tokens": 18760296.0, + "eval_chemistry_runtime": 49.1467, + "eval_chemistry_samples_per_second": 10.174, + "eval_chemistry_steps_per_second": 2.543, + "step": 700 + }, + { + "epoch": 1.2456882163124514, + "eval_math_entropy": 0.6947779231071473, + "eval_math_loss": 1.0786137580871582, + "eval_math_mean_token_accuracy": 0.7709644784927369, + "eval_math_num_tokens": 18760296.0, + "eval_math_runtime": 52.5755, + "eval_math_samples_per_second": 9.51, + "eval_math_steps_per_second": 2.378, + "step": 700 + }, + { + "epoch": 1.2456882163124514, + "eval_physics_entropy": 0.704109364271164, + "eval_physics_loss": 0.8724388480186462, + "eval_physics_mean_token_accuracy": 0.796642897605896, + "eval_physics_num_tokens": 18760296.0, + "eval_physics_runtime": 61.3658, + "eval_physics_samples_per_second": 8.148, + "eval_physics_steps_per_second": 2.037, + "step": 700 + }, + { + "entropy": 0.6705936685204505, + "epoch": 1.2634917102481362, + "grad_norm": 1.140625, + "learning_rate": 1.697037037037037e-05, + "loss": 0.7618, + "mean_token_accuracy": 0.8144830856472254, + "num_tokens": 19035733.0, + "step": 710 + }, + { + "entropy": 0.6870324395596981, + "epoch": 1.281295204183821, + "grad_norm": 1.1953125, + "learning_rate": 1.6896296296296298e-05, + "loss": 0.7849, + "mean_token_accuracy": 0.807737946882844, + "num_tokens": 19311432.0, + "step": 720 + }, + { + "entropy": 0.6551721351221204, + "epoch": 1.299098698119506, + "grad_norm": 1.0859375, + "learning_rate": 1.6822222222222224e-05, + "loss": 0.7538, + "mean_token_accuracy": 0.8164622981101275, + "num_tokens": 19593016.0, + "step": 730 + }, + { + "entropy": 0.6645344085991383, + "epoch": 1.3169021920551909, + "grad_norm": 1.1796875, + "learning_rate": 1.6748148148148147e-05, + "loss": 0.759, + "mean_token_accuracy": 0.8134492743760348, + "num_tokens": 19860476.0, + "step": 740 + }, + { + "entropy": 0.6684485200792551, + "epoch": 1.3347056859908757, + "grad_norm": 1.3515625, + "learning_rate": 1.6674074074074077e-05, + "loss": 0.7632, + "mean_token_accuracy": 0.8128790140151978, + "num_tokens": 20124436.0, + "step": 750 + }, + { + "entropy": 0.6736280497163534, + "epoch": 1.3525091799265607, + "grad_norm": 1.25, + "learning_rate": 1.66e-05, + "loss": 0.7771, + "mean_token_accuracy": 0.813029420748353, + "num_tokens": 20388027.0, + "step": 760 + }, + { + "entropy": 0.6706711670383811, + "epoch": 1.3703126738622455, + "grad_norm": 1.265625, + "learning_rate": 1.6525925925925927e-05, + "loss": 0.7697, + "mean_token_accuracy": 0.8115393359214067, + "num_tokens": 20656076.0, + "step": 770 + }, + { + "entropy": 0.6996258163824678, + "epoch": 1.3881161677979303, + "grad_norm": 1.2578125, + "learning_rate": 1.6451851851851853e-05, + "loss": 0.8077, + "mean_token_accuracy": 0.8058754079043865, + "num_tokens": 20911633.0, + "step": 780 + }, + { + "entropy": 0.6905225297436118, + "epoch": 1.4059196617336152, + "grad_norm": 1.171875, + "learning_rate": 1.637777777777778e-05, + "loss": 0.7955, + "mean_token_accuracy": 0.8070559553802014, + "num_tokens": 21176227.0, + "step": 790 + }, + { + "entropy": 0.6718146029859782, + "epoch": 1.4237231556693, + "grad_norm": 1.2890625, + "learning_rate": 1.6303703703703706e-05, + "loss": 0.7716, + "mean_token_accuracy": 0.8126390781253576, + "num_tokens": 21445242.0, + "step": 800 + }, + { + "epoch": 1.4237231556693, + "eval_biology_entropy": 1.6255396161079407, + "eval_biology_loss": 2.0511693954467773, + "eval_biology_mean_token_accuracy": 0.6119143316745758, + "eval_biology_num_tokens": 21445242.0, + "eval_biology_runtime": 42.4445, + "eval_biology_samples_per_second": 11.78, + "eval_biology_steps_per_second": 2.945, + "step": 800 + }, + { + "epoch": 1.4237231556693, + "eval_chemistry_entropy": 0.6763756844997406, + "eval_chemistry_loss": 0.8368268609046936, + "eval_chemistry_mean_token_accuracy": 0.8050474629402161, + "eval_chemistry_num_tokens": 21445242.0, + "eval_chemistry_runtime": 51.5104, + "eval_chemistry_samples_per_second": 9.707, + "eval_chemistry_steps_per_second": 2.427, + "step": 800 + }, + { + "epoch": 1.4237231556693, + "eval_math_entropy": 0.6853804645538331, + "eval_math_loss": 1.0698176622390747, + "eval_math_mean_token_accuracy": 0.7714386968612671, + "eval_math_num_tokens": 21445242.0, + "eval_math_runtime": 54.9388, + "eval_math_samples_per_second": 9.101, + "eval_math_steps_per_second": 2.275, + "step": 800 + }, + { + "epoch": 1.4237231556693, + "eval_physics_entropy": 0.6991692934036254, + "eval_physics_loss": 0.8673914074897766, + "eval_physics_mean_token_accuracy": 0.7979785161018371, + "eval_physics_num_tokens": 21445242.0, + "eval_physics_runtime": 60.9527, + "eval_physics_samples_per_second": 8.203, + "eval_physics_steps_per_second": 2.051, + "step": 800 + }, + { + "entropy": 0.6695453573018313, + "epoch": 1.441526649604985, + "grad_norm": 1.3125, + "learning_rate": 1.622962962962963e-05, + "loss": 0.77, + "mean_token_accuracy": 0.8109262142330408, + "num_tokens": 21718455.0, + "step": 810 + }, + { + "entropy": 0.6753543723374605, + "epoch": 1.4593301435406698, + "grad_norm": 1.2734375, + "learning_rate": 1.6155555555555556e-05, + "loss": 0.7782, + "mean_token_accuracy": 0.8130301378667355, + "num_tokens": 21974076.0, + "step": 820 + }, + { + "entropy": 0.6758085643872619, + "epoch": 1.4771336374763546, + "grad_norm": 1.2265625, + "learning_rate": 1.6081481481481482e-05, + "loss": 0.7768, + "mean_token_accuracy": 0.8110865034162998, + "num_tokens": 22241599.0, + "step": 830 + }, + { + "entropy": 0.6868746858090162, + "epoch": 1.4949371314120397, + "grad_norm": 1.34375, + "learning_rate": 1.600740740740741e-05, + "loss": 0.7818, + "mean_token_accuracy": 0.8090599600225687, + "num_tokens": 22510725.0, + "step": 840 + }, + { + "entropy": 0.6554480012506246, + "epoch": 1.5127406253477245, + "grad_norm": 1.1796875, + "learning_rate": 1.5933333333333336e-05, + "loss": 0.766, + "mean_token_accuracy": 0.8149232871830463, + "num_tokens": 22778514.0, + "step": 850 + }, + { + "entropy": 0.6574800366535782, + "epoch": 1.5305441192834093, + "grad_norm": 1.2734375, + "learning_rate": 1.5859259259259262e-05, + "loss": 0.7517, + "mean_token_accuracy": 0.8147028144448996, + "num_tokens": 23051230.0, + "step": 860 + }, + { + "entropy": 0.6707898294553161, + "epoch": 1.5483476132190943, + "grad_norm": 1.21875, + "learning_rate": 1.5785185185185185e-05, + "loss": 0.776, + "mean_token_accuracy": 0.8121768556535244, + "num_tokens": 23306327.0, + "step": 870 + }, + { + "entropy": 0.6759747974574566, + "epoch": 1.5661511071547791, + "grad_norm": 1.234375, + "learning_rate": 1.571111111111111e-05, + "loss": 0.7717, + "mean_token_accuracy": 0.810706127434969, + "num_tokens": 23573127.0, + "step": 880 + }, + { + "entropy": 0.6696014609187841, + "epoch": 1.583954601090464, + "grad_norm": 1.1953125, + "learning_rate": 1.5637037037037038e-05, + "loss": 0.7635, + "mean_token_accuracy": 0.8123903065919876, + "num_tokens": 23841914.0, + "step": 890 + }, + { + "entropy": 0.6680066948756576, + "epoch": 1.601758095026149, + "grad_norm": 1.28125, + "learning_rate": 1.5562962962962965e-05, + "loss": 0.7763, + "mean_token_accuracy": 0.8117439385503531, + "num_tokens": 24105114.0, + "step": 900 + }, + { + "epoch": 1.601758095026149, + "eval_biology_entropy": 1.6559728245735168, + "eval_biology_loss": 2.0387370586395264, + "eval_biology_mean_token_accuracy": 0.6136598820686341, + "eval_biology_num_tokens": 24105114.0, + "eval_biology_runtime": 41.4479, + "eval_biology_samples_per_second": 12.063, + "eval_biology_steps_per_second": 3.016, + "step": 900 + }, + { + "epoch": 1.601758095026149, + "eval_chemistry_entropy": 0.683923549413681, + "eval_chemistry_loss": 0.8298754692077637, + "eval_chemistry_mean_token_accuracy": 0.8061454033851624, + "eval_chemistry_num_tokens": 24105114.0, + "eval_chemistry_runtime": 50.8984, + "eval_chemistry_samples_per_second": 9.823, + "eval_chemistry_steps_per_second": 2.456, + "step": 900 + }, + { + "epoch": 1.601758095026149, + "eval_math_entropy": 0.6932375190258027, + "eval_math_loss": 1.06475830078125, + "eval_math_mean_token_accuracy": 0.7720690259933471, + "eval_math_num_tokens": 24105114.0, + "eval_math_runtime": 52.5544, + "eval_math_samples_per_second": 9.514, + "eval_math_steps_per_second": 2.378, + "step": 900 + }, + { + "epoch": 1.601758095026149, + "eval_physics_entropy": 0.7051843535900116, + "eval_physics_loss": 0.8616264462471008, + "eval_physics_mean_token_accuracy": 0.7987073979377747, + "eval_physics_num_tokens": 24105114.0, + "eval_physics_runtime": 59.8508, + "eval_physics_samples_per_second": 8.354, + "eval_physics_steps_per_second": 2.089, + "step": 900 + }, + { + "entropy": 0.6430077504366636, + "epoch": 1.6195615889618338, + "grad_norm": 1.1640625, + "learning_rate": 1.548888888888889e-05, + "loss": 0.743, + "mean_token_accuracy": 0.8184590026736259, + "num_tokens": 24384008.0, + "step": 910 + }, + { + "entropy": 0.6775279760360717, + "epoch": 1.6373650828975186, + "grad_norm": 1.3125, + "learning_rate": 1.5414814814814814e-05, + "loss": 0.7704, + "mean_token_accuracy": 0.8114439699798822, + "num_tokens": 24638205.0, + "step": 920 + }, + { + "entropy": 0.6244934551417828, + "epoch": 1.6551685768332036, + "grad_norm": 1.1953125, + "learning_rate": 1.5340740740740744e-05, + "loss": 0.7253, + "mean_token_accuracy": 0.8222480688244105, + "num_tokens": 24910104.0, + "step": 930 + }, + { + "entropy": 0.6539383618161082, + "epoch": 1.6729720707688884, + "grad_norm": 1.2578125, + "learning_rate": 1.5266666666666667e-05, + "loss": 0.7471, + "mean_token_accuracy": 0.8160006206482648, + "num_tokens": 25181306.0, + "step": 940 + }, + { + "entropy": 0.6607510834932327, + "epoch": 1.6907755647045732, + "grad_norm": 1.3203125, + "learning_rate": 1.5192592592592594e-05, + "loss": 0.7589, + "mean_token_accuracy": 0.8142590031027794, + "num_tokens": 25449138.0, + "step": 950 + }, + { + "entropy": 0.6612858925014734, + "epoch": 1.7085790586402583, + "grad_norm": 1.203125, + "learning_rate": 1.5118518518518519e-05, + "loss": 0.766, + "mean_token_accuracy": 0.8137270450592041, + "num_tokens": 25709030.0, + "step": 960 + }, + { + "entropy": 0.6701876953244209, + "epoch": 1.7263825525759429, + "grad_norm": 1.2734375, + "learning_rate": 1.5044444444444445e-05, + "loss": 0.7702, + "mean_token_accuracy": 0.8120660580694675, + "num_tokens": 25972807.0, + "step": 970 + }, + { + "entropy": 0.6566708590835333, + "epoch": 1.744186046511628, + "grad_norm": 1.1640625, + "learning_rate": 1.497037037037037e-05, + "loss": 0.748, + "mean_token_accuracy": 0.8170768674463034, + "num_tokens": 26244955.0, + "step": 980 + }, + { + "entropy": 0.624184824153781, + "epoch": 1.761989540447313, + "grad_norm": 1.1796875, + "learning_rate": 1.4896296296296298e-05, + "loss": 0.7284, + "mean_token_accuracy": 0.8223610106855631, + "num_tokens": 26515548.0, + "step": 990 + }, + { + "entropy": 0.6476762825623155, + "epoch": 1.7797930343829975, + "grad_norm": 1.21875, + "learning_rate": 1.4822222222222225e-05, + "loss": 0.7422, + "mean_token_accuracy": 0.8172390758991241, + "num_tokens": 26782376.0, + "step": 1000 + }, + { + "epoch": 1.7797930343829975, + "eval_biology_entropy": 1.6157320532798767, + "eval_biology_loss": 2.0338852405548096, + "eval_biology_mean_token_accuracy": 0.6143801593780518, + "eval_biology_num_tokens": 26782376.0, + "eval_biology_runtime": 41.5849, + "eval_biology_samples_per_second": 12.024, + "eval_biology_steps_per_second": 3.006, + "step": 1000 + }, + { + "epoch": 1.7797930343829975, + "eval_chemistry_entropy": 0.6670743160247803, + "eval_chemistry_loss": 0.8244208097457886, + "eval_chemistry_mean_token_accuracy": 0.8077364330291747, + "eval_chemistry_num_tokens": 26782376.0, + "eval_chemistry_runtime": 53.0274, + "eval_chemistry_samples_per_second": 9.429, + "eval_chemistry_steps_per_second": 2.357, + "step": 1000 + }, + { + "epoch": 1.7797930343829975, + "eval_math_entropy": 0.6760952143669129, + "eval_math_loss": 1.0642753839492798, + "eval_math_mean_token_accuracy": 0.7728619589805603, + "eval_math_num_tokens": 26782376.0, + "eval_math_runtime": 53.8997, + "eval_math_samples_per_second": 9.276, + "eval_math_steps_per_second": 2.319, + "step": 1000 + }, + { + "epoch": 1.7797930343829975, + "eval_physics_entropy": 0.6914605097770691, + "eval_physics_loss": 0.8597292304039001, + "eval_physics_mean_token_accuracy": 0.7992245469093323, + "eval_physics_num_tokens": 26782376.0, + "eval_physics_runtime": 62.8355, + "eval_physics_samples_per_second": 7.957, + "eval_physics_steps_per_second": 1.989, + "step": 1000 + }, + { + "entropy": 0.660948914475739, + "epoch": 1.7975965283186826, + "grad_norm": 1.359375, + "learning_rate": 1.474814814814815e-05, + "loss": 0.7562, + "mean_token_accuracy": 0.8152279850095511, + "num_tokens": 27049648.0, + "step": 1010 + }, + { + "entropy": 0.6787248050794006, + "epoch": 1.8154000222543676, + "grad_norm": 1.3359375, + "learning_rate": 1.4674074074074076e-05, + "loss": 0.7771, + "mean_token_accuracy": 0.8101395227015018, + "num_tokens": 27315061.0, + "step": 1020 + }, + { + "entropy": 0.6424054227769375, + "epoch": 1.8332035161900522, + "grad_norm": 1.359375, + "learning_rate": 1.46e-05, + "loss": 0.7373, + "mean_token_accuracy": 0.8189616728574037, + "num_tokens": 27587650.0, + "step": 1030 + }, + { + "entropy": 0.6606515791267157, + "epoch": 1.8510070101257372, + "grad_norm": 1.15625, + "learning_rate": 1.4525925925925927e-05, + "loss": 0.7652, + "mean_token_accuracy": 0.8157406080514192, + "num_tokens": 27855458.0, + "step": 1040 + }, + { + "entropy": 0.6464399367570877, + "epoch": 1.868810504061422, + "grad_norm": 1.203125, + "learning_rate": 1.4451851851851852e-05, + "loss": 0.7464, + "mean_token_accuracy": 0.8191553246229887, + "num_tokens": 28127703.0, + "step": 1050 + }, + { + "entropy": 0.633565224148333, + "epoch": 1.8866139979971068, + "grad_norm": 1.140625, + "learning_rate": 1.4377777777777779e-05, + "loss": 0.7265, + "mean_token_accuracy": 0.8215953864157199, + "num_tokens": 28395128.0, + "step": 1060 + }, + { + "entropy": 0.651407066360116, + "epoch": 1.9044174919327919, + "grad_norm": 1.2421875, + "learning_rate": 1.4303703703703703e-05, + "loss": 0.752, + "mean_token_accuracy": 0.8161330189555883, + "num_tokens": 28662344.0, + "step": 1070 + }, + { + "entropy": 0.6537632567808032, + "epoch": 1.9222209858684767, + "grad_norm": 1.3125, + "learning_rate": 1.4229629629629632e-05, + "loss": 0.7551, + "mean_token_accuracy": 0.8160409614443779, + "num_tokens": 28928769.0, + "step": 1080 + }, + { + "entropy": 0.6610247550532222, + "epoch": 1.9400244798041615, + "grad_norm": 1.5625, + "learning_rate": 1.4155555555555556e-05, + "loss": 0.7599, + "mean_token_accuracy": 0.8147491872310638, + "num_tokens": 29186214.0, + "step": 1090 + }, + { + "entropy": 0.6632727082818747, + "epoch": 1.9578279737398465, + "grad_norm": 1.2109375, + "learning_rate": 1.4081481481481483e-05, + "loss": 0.7706, + "mean_token_accuracy": 0.8126637618988752, + "num_tokens": 29446445.0, + "step": 1100 + }, + { + "epoch": 1.9578279737398465, + "eval_biology_entropy": 1.639104124546051, + "eval_biology_loss": 2.0316033363342285, + "eval_biology_mean_token_accuracy": 0.615292895078659, + "eval_biology_num_tokens": 29446445.0, + "eval_biology_runtime": 41.3569, + "eval_biology_samples_per_second": 12.09, + "eval_biology_steps_per_second": 3.022, + "step": 1100 + }, + { + "epoch": 1.9578279737398465, + "eval_chemistry_entropy": 0.6696053767204284, + "eval_chemistry_loss": 0.8191399574279785, + "eval_chemistry_mean_token_accuracy": 0.8085824422836304, + "eval_chemistry_num_tokens": 29446445.0, + "eval_chemistry_runtime": 50.7996, + "eval_chemistry_samples_per_second": 9.843, + "eval_chemistry_steps_per_second": 2.461, + "step": 1100 + }, + { + "epoch": 1.9578279737398465, + "eval_math_entropy": 0.6799165282249451, + "eval_math_loss": 1.0633617639541626, + "eval_math_mean_token_accuracy": 0.7729020061492919, + "eval_math_num_tokens": 29446445.0, + "eval_math_runtime": 52.3664, + "eval_math_samples_per_second": 9.548, + "eval_math_steps_per_second": 2.387, + "step": 1100 + }, + { + "epoch": 1.9578279737398465, + "eval_physics_entropy": 0.6961899037361146, + "eval_physics_loss": 0.8565983176231384, + "eval_physics_mean_token_accuracy": 0.7995611958503723, + "eval_physics_num_tokens": 29446445.0, + "eval_physics_runtime": 59.795, + "eval_physics_samples_per_second": 8.362, + "eval_physics_steps_per_second": 2.09, + "step": 1100 + }, + { + "entropy": 0.6516633328050375, + "epoch": 1.9756314676755313, + "grad_norm": 1.21875, + "learning_rate": 1.400740740740741e-05, + "loss": 0.7439, + "mean_token_accuracy": 0.8187774941325188, + "num_tokens": 29711771.0, + "step": 1110 + }, + { + "entropy": 0.6508602360263467, + "epoch": 1.9934349616112161, + "grad_norm": 1.3125, + "learning_rate": 1.3933333333333334e-05, + "loss": 0.7519, + "mean_token_accuracy": 0.8169589649885893, + "num_tokens": 29979994.0, + "step": 1120 + }, + { + "entropy": 0.6306788632946629, + "epoch": 2.010682096361411, + "grad_norm": 1.2734375, + "learning_rate": 1.385925925925926e-05, + "loss": 0.7224, + "mean_token_accuracy": 0.8239769354943306, + "num_tokens": 30240793.0, + "step": 1130 + }, + { + "entropy": 0.5911945894360542, + "epoch": 2.028485590297096, + "grad_norm": 1.296875, + "learning_rate": 1.3785185185185186e-05, + "loss": 0.7, + "mean_token_accuracy": 0.8281658995896578, + "num_tokens": 30503988.0, + "step": 1140 + }, + { + "entropy": 0.6057484762743115, + "epoch": 2.0462890842327806, + "grad_norm": 1.2890625, + "learning_rate": 1.3711111111111112e-05, + "loss": 0.7031, + "mean_token_accuracy": 0.8266915556043386, + "num_tokens": 30771841.0, + "step": 1150 + }, + { + "entropy": 0.596652439981699, + "epoch": 2.0640925781684656, + "grad_norm": 1.375, + "learning_rate": 1.3637037037037037e-05, + "loss": 0.6926, + "mean_token_accuracy": 0.8293926887214184, + "num_tokens": 31045037.0, + "step": 1160 + }, + { + "entropy": 0.6352694857865572, + "epoch": 2.0818960721041506, + "grad_norm": 1.4296875, + "learning_rate": 1.3562962962962965e-05, + "loss": 0.7356, + "mean_token_accuracy": 0.8186026088893413, + "num_tokens": 31299575.0, + "step": 1170 + }, + { + "entropy": 0.6292696729302406, + "epoch": 2.099699566039835, + "grad_norm": 1.390625, + "learning_rate": 1.3488888888888888e-05, + "loss": 0.7273, + "mean_token_accuracy": 0.8201477468013764, + "num_tokens": 31565963.0, + "step": 1180 + }, + { + "entropy": 0.5940109083428979, + "epoch": 2.1175030599755202, + "grad_norm": 1.2734375, + "learning_rate": 1.3414814814814817e-05, + "loss": 0.7035, + "mean_token_accuracy": 0.8278756473213434, + "num_tokens": 31834212.0, + "step": 1190 + }, + { + "entropy": 0.6235881226137281, + "epoch": 2.1353065539112053, + "grad_norm": 1.375, + "learning_rate": 1.3340740740740741e-05, + "loss": 0.7172, + "mean_token_accuracy": 0.8223393179476262, + "num_tokens": 32103356.0, + "step": 1200 + }, + { + "epoch": 2.1353065539112053, + "eval_biology_entropy": 1.4677531628608704, + "eval_biology_loss": 2.083951234817505, + "eval_biology_mean_token_accuracy": 0.6125459330081939, + "eval_biology_num_tokens": 32103356.0, + "eval_biology_runtime": 41.2722, + "eval_biology_samples_per_second": 12.115, + "eval_biology_steps_per_second": 3.029, + "step": 1200 + }, + { + "epoch": 2.1353065539112053, + "eval_chemistry_entropy": 0.6140689237117767, + "eval_chemistry_loss": 0.8232262134552002, + "eval_chemistry_mean_token_accuracy": 0.8088321657180786, + "eval_chemistry_num_tokens": 32103356.0, + "eval_chemistry_runtime": 50.8871, + "eval_chemistry_samples_per_second": 9.826, + "eval_chemistry_steps_per_second": 2.456, + "step": 1200 + }, + { + "epoch": 2.1353065539112053, + "eval_math_entropy": 0.6312986519336701, + "eval_math_loss": 1.0854209661483765, + "eval_math_mean_token_accuracy": 0.7729385108947754, + "eval_math_num_tokens": 32103356.0, + "eval_math_runtime": 52.4669, + "eval_math_samples_per_second": 9.53, + "eval_math_steps_per_second": 2.382, + "step": 1200 + }, + { + "epoch": 2.1353065539112053, + "eval_physics_entropy": 0.6365497968196869, + "eval_physics_loss": 0.8704355359077454, + "eval_physics_mean_token_accuracy": 0.7990298428535462, + "eval_physics_num_tokens": 32103356.0, + "eval_physics_runtime": 59.7275, + "eval_physics_samples_per_second": 8.371, + "eval_physics_steps_per_second": 2.093, + "step": 1200 + }, + { + "entropy": 0.6137512635439635, + "epoch": 2.15311004784689, + "grad_norm": 1.25, + "learning_rate": 1.3266666666666668e-05, + "loss": 0.7043, + "mean_token_accuracy": 0.8242961999028922, + "num_tokens": 32374830.0, + "step": 1210 + }, + { + "entropy": 0.5896952305920422, + "epoch": 2.170913541782575, + "grad_norm": 1.2734375, + "learning_rate": 1.3192592592592594e-05, + "loss": 0.6922, + "mean_token_accuracy": 0.8302250444889069, + "num_tokens": 32643479.0, + "step": 1220 + }, + { + "entropy": 0.5944842832162976, + "epoch": 2.18871703571826, + "grad_norm": 1.296875, + "learning_rate": 1.311851851851852e-05, + "loss": 0.6959, + "mean_token_accuracy": 0.8287800919264555, + "num_tokens": 32908761.0, + "step": 1230 + }, + { + "entropy": 0.6240290224552154, + "epoch": 2.2065205296539445, + "grad_norm": 1.21875, + "learning_rate": 1.3044444444444446e-05, + "loss": 0.7238, + "mean_token_accuracy": 0.8242842797189951, + "num_tokens": 33166413.0, + "step": 1240 + }, + { + "entropy": 0.6112470507621766, + "epoch": 2.2243240235896296, + "grad_norm": 1.1875, + "learning_rate": 1.297037037037037e-05, + "loss": 0.7149, + "mean_token_accuracy": 0.8240214943885803, + "num_tokens": 33432686.0, + "step": 1250 + }, + { + "entropy": 0.6079921290278435, + "epoch": 2.242127517525314, + "grad_norm": 1.4296875, + "learning_rate": 1.2896296296296299e-05, + "loss": 0.7131, + "mean_token_accuracy": 0.8254695866256953, + "num_tokens": 33693242.0, + "step": 1260 + }, + { + "entropy": 0.5799209821969271, + "epoch": 2.259931011460999, + "grad_norm": 1.390625, + "learning_rate": 1.2822222222222222e-05, + "loss": 0.6722, + "mean_token_accuracy": 0.834067041426897, + "num_tokens": 33970820.0, + "step": 1270 + }, + { + "entropy": 0.594209510087967, + "epoch": 2.277734505396684, + "grad_norm": 1.3671875, + "learning_rate": 1.274814814814815e-05, + "loss": 0.6977, + "mean_token_accuracy": 0.8276573233306408, + "num_tokens": 34243042.0, + "step": 1280 + }, + { + "entropy": 0.6097929472103715, + "epoch": 2.295537999332369, + "grad_norm": 1.296875, + "learning_rate": 1.2674074074074075e-05, + "loss": 0.705, + "mean_token_accuracy": 0.8267240136861801, + "num_tokens": 34515236.0, + "step": 1290 + }, + { + "entropy": 0.5745832217857242, + "epoch": 2.313341493268054, + "grad_norm": 1.234375, + "learning_rate": 1.2600000000000001e-05, + "loss": 0.6783, + "mean_token_accuracy": 0.8345583423972129, + "num_tokens": 34785068.0, + "step": 1300 + }, + { + "epoch": 2.313341493268054, + "eval_biology_entropy": 1.497672739982605, + "eval_biology_loss": 2.0853865146636963, + "eval_biology_mean_token_accuracy": 0.613484436750412, + "eval_biology_num_tokens": 34785068.0, + "eval_biology_runtime": 43.469, + "eval_biology_samples_per_second": 11.502, + "eval_biology_steps_per_second": 2.876, + "step": 1300 + }, + { + "epoch": 2.313341493268054, + "eval_chemistry_entropy": 0.6273940706253052, + "eval_chemistry_loss": 0.8201014995574951, + "eval_chemistry_mean_token_accuracy": 0.8088535733222961, + "eval_chemistry_num_tokens": 34785068.0, + "eval_chemistry_runtime": 52.831, + "eval_chemistry_samples_per_second": 9.464, + "eval_chemistry_steps_per_second": 2.366, + "step": 1300 + }, + { + "epoch": 2.313341493268054, + "eval_math_entropy": 0.646281590461731, + "eval_math_loss": 1.0807055234909058, + "eval_math_mean_token_accuracy": 0.772442822933197, + "eval_math_num_tokens": 34785068.0, + "eval_math_runtime": 55.4339, + "eval_math_samples_per_second": 9.02, + "eval_math_steps_per_second": 2.255, + "step": 1300 + }, + { + "epoch": 2.313341493268054, + "eval_physics_entropy": 0.6572135488986969, + "eval_physics_loss": 0.8676888346672058, + "eval_physics_mean_token_accuracy": 0.7990684523582459, + "eval_physics_num_tokens": 34785068.0, + "eval_physics_runtime": 61.4064, + "eval_physics_samples_per_second": 8.142, + "eval_physics_steps_per_second": 2.036, + "step": 1300 + }, + { + "entropy": 0.6191223243251442, + "epoch": 2.331144987203739, + "grad_norm": 1.2265625, + "learning_rate": 1.2525925925925928e-05, + "loss": 0.7122, + "mean_token_accuracy": 0.8234442934393883, + "num_tokens": 35055845.0, + "step": 1310 + }, + { + "entropy": 0.6000411529093981, + "epoch": 2.3489484811394234, + "grad_norm": 1.296875, + "learning_rate": 1.2451851851851853e-05, + "loss": 0.7023, + "mean_token_accuracy": 0.8272519435733556, + "num_tokens": 35327481.0, + "step": 1320 + }, + { + "entropy": 0.6216201702132821, + "epoch": 2.3667519750751085, + "grad_norm": 1.2578125, + "learning_rate": 1.237777777777778e-05, + "loss": 0.7178, + "mean_token_accuracy": 0.8234907556325197, + "num_tokens": 35596362.0, + "step": 1330 + }, + { + "entropy": 0.5952032478526235, + "epoch": 2.3845554690107935, + "grad_norm": 1.3671875, + "learning_rate": 1.2303703703703704e-05, + "loss": 0.695, + "mean_token_accuracy": 0.827592471987009, + "num_tokens": 35869585.0, + "step": 1340 + }, + { + "entropy": 0.6152689423412084, + "epoch": 2.402358962946478, + "grad_norm": 1.4140625, + "learning_rate": 1.222962962962963e-05, + "loss": 0.7087, + "mean_token_accuracy": 0.8254514247179031, + "num_tokens": 36134072.0, + "step": 1350 + }, + { + "entropy": 0.5919087225571275, + "epoch": 2.420162456882163, + "grad_norm": 1.25, + "learning_rate": 1.2155555555555555e-05, + "loss": 0.6971, + "mean_token_accuracy": 0.829882950335741, + "num_tokens": 36406263.0, + "step": 1360 + }, + { + "entropy": 0.6080563787370921, + "epoch": 2.437965950817848, + "grad_norm": 1.3984375, + "learning_rate": 1.2081481481481484e-05, + "loss": 0.7027, + "mean_token_accuracy": 0.8268357224762439, + "num_tokens": 36676822.0, + "step": 1370 + }, + { + "entropy": 0.6048327511176467, + "epoch": 2.4557694447535328, + "grad_norm": 1.421875, + "learning_rate": 1.2007407407407408e-05, + "loss": 0.7047, + "mean_token_accuracy": 0.8271109748631715, + "num_tokens": 36942837.0, + "step": 1380 + }, + { + "entropy": 0.5988074177876115, + "epoch": 2.473572938689218, + "grad_norm": 1.390625, + "learning_rate": 1.1933333333333335e-05, + "loss": 0.7051, + "mean_token_accuracy": 0.8256583914160729, + "num_tokens": 37210791.0, + "step": 1390 + }, + { + "entropy": 0.5873929725959897, + "epoch": 2.491376432624903, + "grad_norm": 1.3671875, + "learning_rate": 1.185925925925926e-05, + "loss": 0.6871, + "mean_token_accuracy": 0.83187070004642, + "num_tokens": 37485822.0, + "step": 1400 + }, + { + "epoch": 2.491376432624903, + "eval_biology_entropy": 1.4652894883155823, + "eval_biology_loss": 2.099928855895996, + "eval_biology_mean_token_accuracy": 0.6130364212989807, + "eval_biology_num_tokens": 37485822.0, + "eval_biology_runtime": 43.3395, + "eval_biology_samples_per_second": 11.537, + "eval_biology_steps_per_second": 2.884, + "step": 1400 + }, + { + "epoch": 2.491376432624903, + "eval_chemistry_entropy": 0.6137497539520264, + "eval_chemistry_loss": 0.8189122080802917, + "eval_chemistry_mean_token_accuracy": 0.8097302551269532, + "eval_chemistry_num_tokens": 37485822.0, + "eval_chemistry_runtime": 52.903, + "eval_chemistry_samples_per_second": 9.451, + "eval_chemistry_steps_per_second": 2.363, + "step": 1400 + }, + { + "epoch": 2.491376432624903, + "eval_math_entropy": 0.6390214171409607, + "eval_math_loss": 1.082951545715332, + "eval_math_mean_token_accuracy": 0.772076274394989, + "eval_math_num_tokens": 37485822.0, + "eval_math_runtime": 54.1184, + "eval_math_samples_per_second": 9.239, + "eval_math_steps_per_second": 2.31, + "step": 1400 + }, + { + "epoch": 2.491376432624903, + "eval_physics_entropy": 0.6406208200454712, + "eval_physics_loss": 0.8697341084480286, + "eval_physics_mean_token_accuracy": 0.7989166221618652, + "eval_physics_num_tokens": 37485822.0, + "eval_physics_runtime": 61.5427, + "eval_physics_samples_per_second": 8.124, + "eval_physics_steps_per_second": 2.031, + "step": 1400 + }, + { + "entropy": 0.6010861510410905, + "epoch": 2.5091799265605874, + "grad_norm": 1.28125, + "learning_rate": 1.1785185185185186e-05, + "loss": 0.7014, + "mean_token_accuracy": 0.8287522595375776, + "num_tokens": 37746150.0, + "step": 1410 + }, + { + "entropy": 0.5981267247349024, + "epoch": 2.5269834204962724, + "grad_norm": 1.3046875, + "learning_rate": 1.1711111111111113e-05, + "loss": 0.6962, + "mean_token_accuracy": 0.8269948281347752, + "num_tokens": 38016863.0, + "step": 1420 + }, + { + "entropy": 0.6277626631781459, + "epoch": 2.544786914431957, + "grad_norm": 1.5078125, + "learning_rate": 1.1637037037037037e-05, + "loss": 0.7222, + "mean_token_accuracy": 0.8205803319811821, + "num_tokens": 38277754.0, + "step": 1430 + }, + { + "entropy": 0.6118797251954675, + "epoch": 2.562590408367642, + "grad_norm": 1.421875, + "learning_rate": 1.1562962962962964e-05, + "loss": 0.7172, + "mean_token_accuracy": 0.8237183388322592, + "num_tokens": 38542761.0, + "step": 1440 + }, + { + "entropy": 0.6034180456772447, + "epoch": 2.580393902303327, + "grad_norm": 1.453125, + "learning_rate": 1.1488888888888889e-05, + "loss": 0.6941, + "mean_token_accuracy": 0.8272098813205957, + "num_tokens": 38820999.0, + "step": 1450 + }, + { + "entropy": 0.5998265566304326, + "epoch": 2.598197396239012, + "grad_norm": 1.3203125, + "learning_rate": 1.1414814814814817e-05, + "loss": 0.7114, + "mean_token_accuracy": 0.8267886329442262, + "num_tokens": 39080016.0, + "step": 1460 + }, + { + "entropy": 0.6005334880203008, + "epoch": 2.6160008901746967, + "grad_norm": 1.3828125, + "learning_rate": 1.1340740740740742e-05, + "loss": 0.6949, + "mean_token_accuracy": 0.8299154710024595, + "num_tokens": 39345434.0, + "step": 1470 + }, + { + "entropy": 0.6003408120945096, + "epoch": 2.6338043841103818, + "grad_norm": 1.46875, + "learning_rate": 1.1266666666666668e-05, + "loss": 0.7033, + "mean_token_accuracy": 0.8266183461993932, + "num_tokens": 39610041.0, + "step": 1480 + }, + { + "entropy": 0.5847593653947115, + "epoch": 2.6516078780460663, + "grad_norm": 1.4140625, + "learning_rate": 1.1192592592592593e-05, + "loss": 0.6785, + "mean_token_accuracy": 0.8323042623698711, + "num_tokens": 39884185.0, + "step": 1490 + }, + { + "entropy": 0.6105206938460469, + "epoch": 2.6694113719817514, + "grad_norm": 1.5546875, + "learning_rate": 1.111851851851852e-05, + "loss": 0.7201, + "mean_token_accuracy": 0.8220306947827339, + "num_tokens": 40143543.0, + "step": 1500 + }, + { + "epoch": 2.6694113719817514, + "eval_biology_entropy": 1.490638756752014, + "eval_biology_loss": 2.0750932693481445, + "eval_biology_mean_token_accuracy": 0.6143683531284332, + "eval_biology_num_tokens": 40143543.0, + "eval_biology_runtime": 43.2253, + "eval_biology_samples_per_second": 11.567, + "eval_biology_steps_per_second": 2.892, + "step": 1500 + }, + { + "epoch": 2.6694113719817514, + "eval_chemistry_entropy": 0.623803097486496, + "eval_chemistry_loss": 0.8160935044288635, + "eval_chemistry_mean_token_accuracy": 0.8098534879684448, + "eval_chemistry_num_tokens": 40143543.0, + "eval_chemistry_runtime": 50.807, + "eval_chemistry_samples_per_second": 9.841, + "eval_chemistry_steps_per_second": 2.46, + "step": 1500 + }, + { + "epoch": 2.6694113719817514, + "eval_math_entropy": 0.645468202829361, + "eval_math_loss": 1.0840471982955933, + "eval_math_mean_token_accuracy": 0.7726303563117981, + "eval_math_num_tokens": 40143543.0, + "eval_math_runtime": 53.0054, + "eval_math_samples_per_second": 9.433, + "eval_math_steps_per_second": 2.358, + "step": 1500 + }, + { + "epoch": 2.6694113719817514, + "eval_physics_entropy": 0.6492172837257385, + "eval_physics_loss": 0.8670127391815186, + "eval_physics_mean_token_accuracy": 0.7991857075691223, + "eval_physics_num_tokens": 40143543.0, + "eval_physics_runtime": 63.9696, + "eval_physics_samples_per_second": 7.816, + "eval_physics_steps_per_second": 1.954, + "step": 1500 + }, + { + "entropy": 0.6046247070655226, + "epoch": 2.6872148659174364, + "grad_norm": 1.390625, + "learning_rate": 1.1044444444444444e-05, + "loss": 0.7007, + "mean_token_accuracy": 0.828262010216713, + "num_tokens": 40406812.0, + "step": 1510 + }, + { + "entropy": 0.6191225880756974, + "epoch": 2.7050183598531214, + "grad_norm": 1.4140625, + "learning_rate": 1.0970370370370371e-05, + "loss": 0.7269, + "mean_token_accuracy": 0.8213932007551193, + "num_tokens": 40668543.0, + "step": 1520 + }, + { + "entropy": 0.6065074443817139, + "epoch": 2.722821853788806, + "grad_norm": 1.3515625, + "learning_rate": 1.0896296296296298e-05, + "loss": 0.6989, + "mean_token_accuracy": 0.8265019495040178, + "num_tokens": 40942603.0, + "step": 1530 + }, + { + "entropy": 0.6138417391106487, + "epoch": 2.740625347724491, + "grad_norm": 1.34375, + "learning_rate": 1.0822222222222222e-05, + "loss": 0.7057, + "mean_token_accuracy": 0.8261400748044252, + "num_tokens": 41208093.0, + "step": 1540 + }, + { + "entropy": 0.5985930340364576, + "epoch": 2.7584288416601757, + "grad_norm": 1.3125, + "learning_rate": 1.074814814814815e-05, + "loss": 0.7061, + "mean_token_accuracy": 0.8252453569322824, + "num_tokens": 41477562.0, + "step": 1550 + }, + { + "entropy": 0.5930986102670431, + "epoch": 2.7762323355958607, + "grad_norm": 1.5234375, + "learning_rate": 1.0674074074074074e-05, + "loss": 0.6935, + "mean_token_accuracy": 0.8292474631220103, + "num_tokens": 41746571.0, + "step": 1560 + }, + { + "entropy": 0.6006103545427323, + "epoch": 2.7940358295315457, + "grad_norm": 1.4765625, + "learning_rate": 1.0600000000000002e-05, + "loss": 0.7025, + "mean_token_accuracy": 0.8271383382380009, + "num_tokens": 42011633.0, + "step": 1570 + }, + { + "entropy": 0.6129128225147724, + "epoch": 2.8118393234672303, + "grad_norm": 1.328125, + "learning_rate": 1.0525925925925927e-05, + "loss": 0.7076, + "mean_token_accuracy": 0.8252990577369929, + "num_tokens": 42283087.0, + "step": 1580 + }, + { + "entropy": 0.5756963772699237, + "epoch": 2.8296428174029153, + "grad_norm": 1.1953125, + "learning_rate": 1.0451851851851853e-05, + "loss": 0.6725, + "mean_token_accuracy": 0.8329845789819956, + "num_tokens": 42561534.0, + "step": 1590 + }, + { + "entropy": 0.6044865250587463, + "epoch": 2.8474463113386, + "grad_norm": 1.296875, + "learning_rate": 1.0377777777777778e-05, + "loss": 0.705, + "mean_token_accuracy": 0.8281082704663276, + "num_tokens": 42834266.0, + "step": 1600 + }, + { + "epoch": 2.8474463113386, + "eval_biology_entropy": 1.4930514478683472, + "eval_biology_loss": 2.0736353397369385, + "eval_biology_mean_token_accuracy": 0.6145954511165619, + "eval_biology_num_tokens": 42834266.0, + "eval_biology_runtime": 40.9171, + "eval_biology_samples_per_second": 12.22, + "eval_biology_steps_per_second": 3.055, + "step": 1600 + }, + { + "epoch": 2.8474463113386, + "eval_chemistry_entropy": 0.6211945106983184, + "eval_chemistry_loss": 0.8145589828491211, + "eval_chemistry_mean_token_accuracy": 0.810048574924469, + "eval_chemistry_num_tokens": 42834266.0, + "eval_chemistry_runtime": 50.7089, + "eval_chemistry_samples_per_second": 9.86, + "eval_chemistry_steps_per_second": 2.465, + "step": 1600 + }, + { + "epoch": 2.8474463113386, + "eval_math_entropy": 0.6438178873062134, + "eval_math_loss": 1.0831515789031982, + "eval_math_mean_token_accuracy": 0.772301673412323, + "eval_math_num_tokens": 42834266.0, + "eval_math_runtime": 51.7388, + "eval_math_samples_per_second": 9.664, + "eval_math_steps_per_second": 2.416, + "step": 1600 + }, + { + "epoch": 2.8474463113386, + "eval_physics_entropy": 0.649389057636261, + "eval_physics_loss": 0.86627197265625, + "eval_physics_mean_token_accuracy": 0.7995732502937317, + "eval_physics_num_tokens": 42834266.0, + "eval_physics_runtime": 59.549, + "eval_physics_samples_per_second": 8.396, + "eval_physics_steps_per_second": 2.099, + "step": 1600 + }, + { + "entropy": 0.625230243243277, + "epoch": 2.865249805274285, + "grad_norm": 1.46875, + "learning_rate": 1.0303703703703705e-05, + "loss": 0.7228, + "mean_token_accuracy": 0.8225024774670601, + "num_tokens": 43100513.0, + "step": 1610 + }, + { + "entropy": 0.605365008302033, + "epoch": 2.88305329920997, + "grad_norm": 1.359375, + "learning_rate": 1.0229629629629631e-05, + "loss": 0.698, + "mean_token_accuracy": 0.8268981877714395, + "num_tokens": 43375632.0, + "step": 1620 + }, + { + "entropy": 0.5911450985819101, + "epoch": 2.900856793145655, + "grad_norm": 1.3515625, + "learning_rate": 1.0155555555555556e-05, + "loss": 0.6958, + "mean_token_accuracy": 0.829645611718297, + "num_tokens": 43644341.0, + "step": 1630 + }, + { + "entropy": 0.5920769222080707, + "epoch": 2.9186602870813396, + "grad_norm": 1.390625, + "learning_rate": 1.0081481481481484e-05, + "loss": 0.6833, + "mean_token_accuracy": 0.8301208417862653, + "num_tokens": 43917580.0, + "step": 1640 + }, + { + "entropy": 0.5912358593195677, + "epoch": 2.9364637810170247, + "grad_norm": 1.359375, + "learning_rate": 1.0007407407407407e-05, + "loss": 0.6989, + "mean_token_accuracy": 0.8305816825479269, + "num_tokens": 44177325.0, + "step": 1650 + }, + { + "entropy": 0.6072682781144977, + "epoch": 2.9542672749527092, + "grad_norm": 1.3984375, + "learning_rate": 9.933333333333334e-06, + "loss": 0.7079, + "mean_token_accuracy": 0.8247454788535833, + "num_tokens": 44442947.0, + "step": 1660 + }, + { + "entropy": 0.6202316217124462, + "epoch": 2.9720707688883943, + "grad_norm": 1.484375, + "learning_rate": 9.85925925925926e-06, + "loss": 0.7181, + "mean_token_accuracy": 0.8235241327434778, + "num_tokens": 44700601.0, + "step": 1670 + }, + { + "entropy": 0.5890740664675832, + "epoch": 2.9898742628240793, + "grad_norm": 1.3125, + "learning_rate": 9.785185185185187e-06, + "loss": 0.6933, + "mean_token_accuracy": 0.8307324200868607, + "num_tokens": 44971479.0, + "step": 1680 + }, + { + "entropy": 0.587316531712009, + "epoch": 3.007121397574274, + "grad_norm": 1.265625, + "learning_rate": 9.711111111111111e-06, + "loss": 0.6682, + "mean_token_accuracy": 0.8340173459822132, + "num_tokens": 45231796.0, + "step": 1690 + }, + { + "entropy": 0.6013597913086415, + "epoch": 3.0249248915099587, + "grad_norm": 1.5, + "learning_rate": 9.637037037037038e-06, + "loss": 0.7048, + "mean_token_accuracy": 0.8241627678275109, + "num_tokens": 45500317.0, + "step": 1700 + }, + { + "epoch": 3.0249248915099587, + "eval_biology_entropy": 1.4193837118148804, + "eval_biology_loss": 2.11480712890625, + "eval_biology_mean_token_accuracy": 0.6124376230239869, + "eval_biology_num_tokens": 45500317.0, + "eval_biology_runtime": 40.3875, + "eval_biology_samples_per_second": 12.38, + "eval_biology_steps_per_second": 3.095, + "step": 1700 + }, + { + "epoch": 3.0249248915099587, + "eval_chemistry_entropy": 0.5911917226314545, + "eval_chemistry_loss": 0.8190613985061646, + "eval_chemistry_mean_token_accuracy": 0.8099429054260254, + "eval_chemistry_num_tokens": 45500317.0, + "eval_chemistry_runtime": 50.2786, + "eval_chemistry_samples_per_second": 9.945, + "eval_chemistry_steps_per_second": 2.486, + "step": 1700 + }, + { + "epoch": 3.0249248915099587, + "eval_math_entropy": 0.6176868407726288, + "eval_math_loss": 1.0993149280548096, + "eval_math_mean_token_accuracy": 0.7718941388130188, + "eval_math_num_tokens": 45500317.0, + "eval_math_runtime": 51.8195, + "eval_math_samples_per_second": 9.649, + "eval_math_steps_per_second": 2.412, + "step": 1700 + }, + { + "epoch": 3.0249248915099587, + "eval_physics_entropy": 0.6201707119941712, + "eval_physics_loss": 0.8766935467720032, + "eval_physics_mean_token_accuracy": 0.7985301322937012, + "eval_physics_num_tokens": 45500317.0, + "eval_physics_runtime": 59.4252, + "eval_physics_samples_per_second": 8.414, + "eval_physics_steps_per_second": 2.103, + "step": 1700 + }, + { + "entropy": 0.5692525265738368, + "epoch": 3.0427283854456437, + "grad_norm": 1.4140625, + "learning_rate": 9.562962962962965e-06, + "loss": 0.6591, + "mean_token_accuracy": 0.8374071232974529, + "num_tokens": 45762680.0, + "step": 1710 + }, + { + "entropy": 0.5559542520903051, + "epoch": 3.0605318793813288, + "grad_norm": 1.625, + "learning_rate": 9.48888888888889e-06, + "loss": 0.6597, + "mean_token_accuracy": 0.8373042322695256, + "num_tokens": 46029923.0, + "step": 1720 + }, + { + "entropy": 0.5692304980009795, + "epoch": 3.0783353733170133, + "grad_norm": 1.328125, + "learning_rate": 9.414814814814816e-06, + "loss": 0.6629, + "mean_token_accuracy": 0.8344687603414058, + "num_tokens": 46305156.0, + "step": 1730 + }, + { + "entropy": 0.5508972069248557, + "epoch": 3.0961388672526984, + "grad_norm": 1.484375, + "learning_rate": 9.34074074074074e-06, + "loss": 0.6501, + "mean_token_accuracy": 0.8386828090995551, + "num_tokens": 46582874.0, + "step": 1740 + }, + { + "entropy": 0.5854626100510358, + "epoch": 3.1139423611883834, + "grad_norm": 1.640625, + "learning_rate": 9.266666666666667e-06, + "loss": 0.6824, + "mean_token_accuracy": 0.8301110483705998, + "num_tokens": 46850175.0, + "step": 1750 + }, + { + "entropy": 0.5591677505522966, + "epoch": 3.131745855124068, + "grad_norm": 1.3984375, + "learning_rate": 9.192592592592594e-06, + "loss": 0.6564, + "mean_token_accuracy": 0.8384724948555231, + "num_tokens": 47125899.0, + "step": 1760 + }, + { + "entropy": 0.5725911458954215, + "epoch": 3.149549349059753, + "grad_norm": 1.546875, + "learning_rate": 9.118518518518518e-06, + "loss": 0.6752, + "mean_token_accuracy": 0.8343269538134337, + "num_tokens": 47390194.0, + "step": 1770 + }, + { + "entropy": 0.5758154878392816, + "epoch": 3.167352842995438, + "grad_norm": 1.4140625, + "learning_rate": 9.044444444444445e-06, + "loss": 0.6738, + "mean_token_accuracy": 0.8336336623877287, + "num_tokens": 47654784.0, + "step": 1780 + }, + { + "entropy": 0.5612063113600015, + "epoch": 3.1851563369311227, + "grad_norm": 1.53125, + "learning_rate": 8.970370370370372e-06, + "loss": 0.668, + "mean_token_accuracy": 0.8369329560548067, + "num_tokens": 47919400.0, + "step": 1790 + }, + { + "entropy": 0.5750826884061098, + "epoch": 3.2029598308668077, + "grad_norm": 1.5625, + "learning_rate": 8.896296296296298e-06, + "loss": 0.6761, + "mean_token_accuracy": 0.8343805599957704, + "num_tokens": 48187796.0, + "step": 1800 + }, + { + "epoch": 3.2029598308668077, + "eval_biology_entropy": 1.4034225873947144, + "eval_biology_loss": 2.1275665760040283, + "eval_biology_mean_token_accuracy": 0.6119487595558166, + "eval_biology_num_tokens": 48187796.0, + "eval_biology_runtime": 40.734, + "eval_biology_samples_per_second": 12.275, + "eval_biology_steps_per_second": 3.069, + "step": 1800 + }, + { + "epoch": 3.2029598308668077, + "eval_chemistry_entropy": 0.5955355260372162, + "eval_chemistry_loss": 0.8196625709533691, + "eval_chemistry_mean_token_accuracy": 0.8102530264854431, + "eval_chemistry_num_tokens": 48187796.0, + "eval_chemistry_runtime": 50.2147, + "eval_chemistry_samples_per_second": 9.957, + "eval_chemistry_steps_per_second": 2.489, + "step": 1800 + }, + { + "epoch": 3.2029598308668077, + "eval_math_entropy": 0.6184592893123627, + "eval_math_loss": 1.1008187532424927, + "eval_math_mean_token_accuracy": 0.7713511042594909, + "eval_math_num_tokens": 48187796.0, + "eval_math_runtime": 51.7678, + "eval_math_samples_per_second": 9.659, + "eval_math_steps_per_second": 2.415, + "step": 1800 + }, + { + "epoch": 3.2029598308668077, + "eval_physics_entropy": 0.6223777911663055, + "eval_physics_loss": 0.8791524767875671, + "eval_physics_mean_token_accuracy": 0.7979751496315003, + "eval_physics_num_tokens": 48187796.0, + "eval_physics_runtime": 59.0178, + "eval_physics_samples_per_second": 8.472, + "eval_physics_steps_per_second": 2.118, + "step": 1800 + }, + { + "entropy": 0.5663457477465272, + "epoch": 3.2207633248024923, + "grad_norm": 1.3828125, + "learning_rate": 8.822222222222223e-06, + "loss": 0.666, + "mean_token_accuracy": 0.8364688046276569, + "num_tokens": 48457428.0, + "step": 1810 + }, + { + "entropy": 0.6113261075690388, + "epoch": 3.2385668187381773, + "grad_norm": 1.515625, + "learning_rate": 8.74814814814815e-06, + "loss": 0.704, + "mean_token_accuracy": 0.8250963229686021, + "num_tokens": 48716417.0, + "step": 1820 + }, + { + "entropy": 0.5722766594961286, + "epoch": 3.2563703126738623, + "grad_norm": 1.4296875, + "learning_rate": 8.674074074074074e-06, + "loss": 0.6724, + "mean_token_accuracy": 0.8353833951056003, + "num_tokens": 48977306.0, + "step": 1830 + }, + { + "entropy": 0.5715915774926543, + "epoch": 3.274173806609547, + "grad_norm": 1.3359375, + "learning_rate": 8.6e-06, + "loss": 0.6663, + "mean_token_accuracy": 0.8353964131325483, + "num_tokens": 49249667.0, + "step": 1840 + }, + { + "entropy": 0.5664192967116832, + "epoch": 3.291977300545232, + "grad_norm": 1.3828125, + "learning_rate": 8.525925925925927e-06, + "loss": 0.6619, + "mean_token_accuracy": 0.8355418112128973, + "num_tokens": 49520900.0, + "step": 1850 + }, + { + "entropy": 0.5784041214734316, + "epoch": 3.309780794480917, + "grad_norm": 1.484375, + "learning_rate": 8.451851851851852e-06, + "loss": 0.6816, + "mean_token_accuracy": 0.8331022556871176, + "num_tokens": 49786411.0, + "step": 1860 + }, + { + "entropy": 0.5709506403654814, + "epoch": 3.3275842884166016, + "grad_norm": 1.3984375, + "learning_rate": 8.377777777777779e-06, + "loss": 0.6589, + "mean_token_accuracy": 0.8369544640183448, + "num_tokens": 50056207.0, + "step": 1870 + }, + { + "entropy": 0.5751367569901049, + "epoch": 3.3453877823522866, + "grad_norm": 1.390625, + "learning_rate": 8.303703703703705e-06, + "loss": 0.6735, + "mean_token_accuracy": 0.8336179967969656, + "num_tokens": 50330876.0, + "step": 1880 + }, + { + "entropy": 0.577986160852015, + "epoch": 3.3631912762879717, + "grad_norm": 1.6015625, + "learning_rate": 8.229629629629632e-06, + "loss": 0.6774, + "mean_token_accuracy": 0.8319189596921206, + "num_tokens": 50602004.0, + "step": 1890 + }, + { + "entropy": 0.5784968495368957, + "epoch": 3.3809947702236562, + "grad_norm": 1.4140625, + "learning_rate": 8.155555555555556e-06, + "loss": 0.6728, + "mean_token_accuracy": 0.8320122357457876, + "num_tokens": 50873444.0, + "step": 1900 + }, + { + "epoch": 3.3809947702236562, + "eval_biology_entropy": 1.379322585105896, + "eval_biology_loss": 2.1383986473083496, + "eval_biology_mean_token_accuracy": 0.6115301356315613, + "eval_biology_num_tokens": 50873444.0, + "eval_biology_runtime": 40.4711, + "eval_biology_samples_per_second": 12.354, + "eval_biology_steps_per_second": 3.089, + "step": 1900 + }, + { + "epoch": 3.3809947702236562, + "eval_chemistry_entropy": 0.5861142361164093, + "eval_chemistry_loss": 0.8205327391624451, + "eval_chemistry_mean_token_accuracy": 0.8100807156562805, + "eval_chemistry_num_tokens": 50873444.0, + "eval_chemistry_runtime": 50.1133, + "eval_chemistry_samples_per_second": 9.977, + "eval_chemistry_steps_per_second": 2.494, + "step": 1900 + }, + { + "epoch": 3.3809947702236562, + "eval_math_entropy": 0.6114957261085511, + "eval_math_loss": 1.1055864095687866, + "eval_math_mean_token_accuracy": 0.7713269739151001, + "eval_math_num_tokens": 50873444.0, + "eval_math_runtime": 51.5676, + "eval_math_samples_per_second": 9.696, + "eval_math_steps_per_second": 2.424, + "step": 1900 + }, + { + "epoch": 3.3809947702236562, + "eval_physics_entropy": 0.6135163457393646, + "eval_physics_loss": 0.8826585412025452, + "eval_physics_mean_token_accuracy": 0.7979225769042969, + "eval_physics_num_tokens": 50873444.0, + "eval_physics_runtime": 58.9496, + "eval_physics_samples_per_second": 8.482, + "eval_physics_steps_per_second": 2.12, + "step": 1900 + }, + { + "entropy": 0.5896684370934964, + "epoch": 3.3987982641593413, + "grad_norm": 1.5234375, + "learning_rate": 8.081481481481483e-06, + "loss": 0.6862, + "mean_token_accuracy": 0.8290154490619898, + "num_tokens": 51141077.0, + "step": 1910 + }, + { + "entropy": 0.5699383469298482, + "epoch": 3.4166017580950263, + "grad_norm": 1.578125, + "learning_rate": 8.007407407407408e-06, + "loss": 0.6705, + "mean_token_accuracy": 0.8345434635877609, + "num_tokens": 51406483.0, + "step": 1920 + }, + { + "entropy": 0.5792136957868934, + "epoch": 3.434405252030711, + "grad_norm": 1.546875, + "learning_rate": 7.933333333333334e-06, + "loss": 0.6753, + "mean_token_accuracy": 0.8337912186980247, + "num_tokens": 51668458.0, + "step": 1930 + }, + { + "entropy": 0.5839371480047703, + "epoch": 3.452208745966396, + "grad_norm": 1.53125, + "learning_rate": 7.859259259259259e-06, + "loss": 0.6891, + "mean_token_accuracy": 0.8307471171021461, + "num_tokens": 51928199.0, + "step": 1940 + }, + { + "entropy": 0.574542698264122, + "epoch": 3.470012239902081, + "grad_norm": 1.4140625, + "learning_rate": 7.785185185185185e-06, + "loss": 0.6653, + "mean_token_accuracy": 0.8337355487048626, + "num_tokens": 52199244.0, + "step": 1950 + }, + { + "entropy": 0.5655553802847862, + "epoch": 3.4878157338377656, + "grad_norm": 1.5078125, + "learning_rate": 7.711111111111112e-06, + "loss": 0.6606, + "mean_token_accuracy": 0.8357041519135237, + "num_tokens": 52473476.0, + "step": 1960 + }, + { + "entropy": 0.5800750594586134, + "epoch": 3.5056192277734506, + "grad_norm": 1.28125, + "learning_rate": 7.637037037037037e-06, + "loss": 0.6777, + "mean_token_accuracy": 0.832566126063466, + "num_tokens": 52743275.0, + "step": 1970 + }, + { + "entropy": 0.5716741790995001, + "epoch": 3.523422721709135, + "grad_norm": 1.4375, + "learning_rate": 7.562962962962963e-06, + "loss": 0.6637, + "mean_token_accuracy": 0.8349632371217013, + "num_tokens": 53014827.0, + "step": 1980 + }, + { + "entropy": 0.5780819065868854, + "epoch": 3.54122621564482, + "grad_norm": 1.5859375, + "learning_rate": 7.48888888888889e-06, + "loss": 0.6791, + "mean_token_accuracy": 0.8327229652553797, + "num_tokens": 53285577.0, + "step": 1990 + }, + { + "entropy": 0.5571762975305319, + "epoch": 3.5590297095805052, + "grad_norm": 1.5, + "learning_rate": 7.4148148148148155e-06, + "loss": 0.656, + "mean_token_accuracy": 0.8386137288063764, + "num_tokens": 53553391.0, + "step": 2000 + }, + { + "epoch": 3.5590297095805052, + "eval_biology_entropy": 1.381414409160614, + "eval_biology_loss": 2.13814115524292, + "eval_biology_mean_token_accuracy": 0.6120853550434112, + "eval_biology_num_tokens": 53553391.0, + "eval_biology_runtime": 40.2984, + "eval_biology_samples_per_second": 12.407, + "eval_biology_steps_per_second": 3.102, + "step": 2000 + }, + { + "epoch": 3.5590297095805052, + "eval_chemistry_entropy": 0.5889808633327485, + "eval_chemistry_loss": 0.8193114995956421, + "eval_chemistry_mean_token_accuracy": 0.8103628730773926, + "eval_chemistry_num_tokens": 53553391.0, + "eval_chemistry_runtime": 49.8517, + "eval_chemistry_samples_per_second": 10.03, + "eval_chemistry_steps_per_second": 2.507, + "step": 2000 + }, + { + "epoch": 3.5590297095805052, + "eval_math_entropy": 0.6149049291610718, + "eval_math_loss": 1.1041083335876465, + "eval_math_mean_token_accuracy": 0.7715086970329285, + "eval_math_num_tokens": 53553391.0, + "eval_math_runtime": 51.0164, + "eval_math_samples_per_second": 9.801, + "eval_math_steps_per_second": 2.45, + "step": 2000 + }, + { + "epoch": 3.5590297095805052, + "eval_physics_entropy": 0.6177053947448731, + "eval_physics_loss": 0.8811244368553162, + "eval_physics_mean_token_accuracy": 0.798056589126587, + "eval_physics_num_tokens": 53553391.0, + "eval_physics_runtime": 58.0264, + "eval_physics_samples_per_second": 8.617, + "eval_physics_steps_per_second": 2.154, + "step": 2000 + }, + { + "entropy": 0.5717996397987009, + "epoch": 3.5768332035161903, + "grad_norm": 1.4453125, + "learning_rate": 7.340740740740742e-06, + "loss": 0.6683, + "mean_token_accuracy": 0.835101555287838, + "num_tokens": 53826074.0, + "step": 2010 + }, + { + "entropy": 0.5765499664470554, + "epoch": 3.594636697451875, + "grad_norm": 1.5546875, + "learning_rate": 7.266666666666668e-06, + "loss": 0.6758, + "mean_token_accuracy": 0.833604721352458, + "num_tokens": 54090804.0, + "step": 2020 + }, + { + "entropy": 0.5620788568630815, + "epoch": 3.61244019138756, + "grad_norm": 1.4921875, + "learning_rate": 7.192592592592593e-06, + "loss": 0.6652, + "mean_token_accuracy": 0.8382882464677095, + "num_tokens": 54350633.0, + "step": 2030 + }, + { + "entropy": 0.5829729141667486, + "epoch": 3.6302436853232445, + "grad_norm": 1.3671875, + "learning_rate": 7.118518518518519e-06, + "loss": 0.6832, + "mean_token_accuracy": 0.8307051323354244, + "num_tokens": 54623090.0, + "step": 2040 + }, + { + "entropy": 0.5580741995945573, + "epoch": 3.6480471792589295, + "grad_norm": 1.4921875, + "learning_rate": 7.044444444444445e-06, + "loss": 0.6611, + "mean_token_accuracy": 0.8379048384726048, + "num_tokens": 54891961.0, + "step": 2050 + }, + { + "entropy": 0.5711165275424719, + "epoch": 3.6658506731946146, + "grad_norm": 1.3828125, + "learning_rate": 6.97037037037037e-06, + "loss": 0.6686, + "mean_token_accuracy": 0.83506432287395, + "num_tokens": 55160443.0, + "step": 2060 + }, + { + "entropy": 0.5829119691625237, + "epoch": 3.6836541671302996, + "grad_norm": 1.5546875, + "learning_rate": 6.896296296296297e-06, + "loss": 0.679, + "mean_token_accuracy": 0.8321164496243, + "num_tokens": 55425148.0, + "step": 2070 + }, + { + "entropy": 0.577633217908442, + "epoch": 3.701457661065984, + "grad_norm": 1.5078125, + "learning_rate": 6.8222222222222225e-06, + "loss": 0.6756, + "mean_token_accuracy": 0.831652557849884, + "num_tokens": 55701877.0, + "step": 2080 + }, + { + "entropy": 0.5851919570937753, + "epoch": 3.719261155001669, + "grad_norm": 1.4375, + "learning_rate": 6.748148148148149e-06, + "loss": 0.6871, + "mean_token_accuracy": 0.8311607643961907, + "num_tokens": 55969756.0, + "step": 2090 + }, + { + "entropy": 0.5728656422346831, + "epoch": 3.737064648937354, + "grad_norm": 1.3828125, + "learning_rate": 6.674074074074075e-06, + "loss": 0.6697, + "mean_token_accuracy": 0.8338924027979374, + "num_tokens": 56238576.0, + "step": 2100 + }, + { + "epoch": 3.737064648937354, + "eval_biology_entropy": 1.3676475839614868, + "eval_biology_loss": 2.1407225131988525, + "eval_biology_mean_token_accuracy": 0.6122846744060516, + "eval_biology_num_tokens": 56238576.0, + "eval_biology_runtime": 39.0497, + "eval_biology_samples_per_second": 12.804, + "eval_biology_steps_per_second": 3.201, + "step": 2100 + }, + { + "epoch": 3.737064648937354, + "eval_chemistry_entropy": 0.5876963279247284, + "eval_chemistry_loss": 0.8179986476898193, + "eval_chemistry_mean_token_accuracy": 0.8108155965805054, + "eval_chemistry_num_tokens": 56238576.0, + "eval_chemistry_runtime": 48.6816, + "eval_chemistry_samples_per_second": 10.271, + "eval_chemistry_steps_per_second": 2.568, + "step": 2100 + }, + { + "epoch": 3.737064648937354, + "eval_math_entropy": 0.6119424107074738, + "eval_math_loss": 1.103035569190979, + "eval_math_mean_token_accuracy": 0.7718833708763122, + "eval_math_num_tokens": 56238576.0, + "eval_math_runtime": 50.0797, + "eval_math_samples_per_second": 9.984, + "eval_math_steps_per_second": 2.496, + "step": 2100 + }, + { + "epoch": 3.737064648937354, + "eval_physics_entropy": 0.6151737835407257, + "eval_physics_loss": 0.8803703784942627, + "eval_physics_mean_token_accuracy": 0.7982921867370606, + "eval_physics_num_tokens": 56238576.0, + "eval_physics_runtime": 57.6104, + "eval_physics_samples_per_second": 8.679, + "eval_physics_steps_per_second": 2.17, + "step": 2100 + }, + { + "entropy": 0.575149905309081, + "epoch": 3.754868142873039, + "grad_norm": 1.5546875, + "learning_rate": 6.600000000000001e-06, + "loss": 0.6827, + "mean_token_accuracy": 0.8325412474572659, + "num_tokens": 56497905.0, + "step": 2110 + }, + { + "entropy": 0.5853163762018084, + "epoch": 3.772671636808724, + "grad_norm": 1.6796875, + "learning_rate": 6.525925925925927e-06, + "loss": 0.6848, + "mean_token_accuracy": 0.8329160042107105, + "num_tokens": 56757407.0, + "step": 2120 + }, + { + "entropy": 0.5622592825442553, + "epoch": 3.7904751307444084, + "grad_norm": 1.453125, + "learning_rate": 6.4518518518518525e-06, + "loss": 0.666, + "mean_token_accuracy": 0.8372485417872667, + "num_tokens": 57019213.0, + "step": 2130 + }, + { + "entropy": 0.5716784641146659, + "epoch": 3.8082786246800935, + "grad_norm": 1.390625, + "learning_rate": 6.377777777777778e-06, + "loss": 0.6785, + "mean_token_accuracy": 0.8330885298550129, + "num_tokens": 57274629.0, + "step": 2140 + }, + { + "entropy": 0.5762781355530023, + "epoch": 3.826082118615778, + "grad_norm": 1.4375, + "learning_rate": 6.303703703703704e-06, + "loss": 0.6635, + "mean_token_accuracy": 0.8338523168116808, + "num_tokens": 57549791.0, + "step": 2150 + }, + { + "entropy": 0.5586860096082091, + "epoch": 3.843885612551463, + "grad_norm": 1.5703125, + "learning_rate": 6.2296296296296295e-06, + "loss": 0.6637, + "mean_token_accuracy": 0.837235826253891, + "num_tokens": 57815454.0, + "step": 2160 + }, + { + "entropy": 0.5845056502148509, + "epoch": 3.861689106487148, + "grad_norm": 1.4296875, + "learning_rate": 6.155555555555556e-06, + "loss": 0.6887, + "mean_token_accuracy": 0.8297582499682903, + "num_tokens": 58076430.0, + "step": 2170 + }, + { + "entropy": 0.5741686699911952, + "epoch": 3.879492600422833, + "grad_norm": 1.609375, + "learning_rate": 6.081481481481482e-06, + "loss": 0.6757, + "mean_token_accuracy": 0.8332955792546273, + "num_tokens": 58346307.0, + "step": 2180 + }, + { + "entropy": 0.587618819437921, + "epoch": 3.8972960943585178, + "grad_norm": 1.4609375, + "learning_rate": 6.007407407407407e-06, + "loss": 0.6811, + "mean_token_accuracy": 0.8313716314733028, + "num_tokens": 58621624.0, + "step": 2190 + }, + { + "entropy": 0.5831481296569109, + "epoch": 3.915099588294203, + "grad_norm": 1.5, + "learning_rate": 5.933333333333335e-06, + "loss": 0.6864, + "mean_token_accuracy": 0.8321379527449608, + "num_tokens": 58874383.0, + "step": 2200 + }, + { + "epoch": 3.915099588294203, + "eval_biology_entropy": 1.3810388627052308, + "eval_biology_loss": 2.1356613636016846, + "eval_biology_mean_token_accuracy": 0.6122371079921722, + "eval_biology_num_tokens": 58874383.0, + "eval_biology_runtime": 39.0687, + "eval_biology_samples_per_second": 12.798, + "eval_biology_steps_per_second": 3.199, + "step": 2200 + }, + { + "epoch": 3.915099588294203, + "eval_chemistry_entropy": 0.5890640773773194, + "eval_chemistry_loss": 0.8181027770042419, + "eval_chemistry_mean_token_accuracy": 0.8106128220558166, + "eval_chemistry_num_tokens": 58874383.0, + "eval_chemistry_runtime": 48.6446, + "eval_chemistry_samples_per_second": 10.279, + "eval_chemistry_steps_per_second": 2.57, + "step": 2200 + }, + { + "epoch": 3.915099588294203, + "eval_math_entropy": 0.6159807667732239, + "eval_math_loss": 1.1035534143447876, + "eval_math_mean_token_accuracy": 0.7718168897628784, + "eval_math_num_tokens": 58874383.0, + "eval_math_runtime": 50.0657, + "eval_math_samples_per_second": 9.987, + "eval_math_steps_per_second": 2.497, + "step": 2200 + }, + { + "epoch": 3.915099588294203, + "eval_physics_entropy": 0.6170708584785461, + "eval_physics_loss": 0.8807304501533508, + "eval_physics_mean_token_accuracy": 0.7982254939079285, + "eval_physics_num_tokens": 58874383.0, + "eval_physics_runtime": 57.5593, + "eval_physics_samples_per_second": 8.687, + "eval_physics_steps_per_second": 2.172, + "step": 2200 + }, + { + "entropy": 0.5722213637083768, + "epoch": 3.9329030822298874, + "grad_norm": 1.3359375, + "learning_rate": 5.85925925925926e-06, + "loss": 0.669, + "mean_token_accuracy": 0.8334877517074346, + "num_tokens": 59149972.0, + "step": 2210 + }, + { + "entropy": 0.5695188788697123, + "epoch": 3.9507065761655724, + "grad_norm": 1.3828125, + "learning_rate": 5.785185185185186e-06, + "loss": 0.6716, + "mean_token_accuracy": 0.8339714191854, + "num_tokens": 59418591.0, + "step": 2220 + }, + { + "entropy": 0.58336979188025, + "epoch": 3.9685100701012574, + "grad_norm": 1.609375, + "learning_rate": 5.711111111111112e-06, + "loss": 0.6787, + "mean_token_accuracy": 0.8313135091215372, + "num_tokens": 59691322.0, + "step": 2230 + }, + { + "entropy": 0.5696559147909284, + "epoch": 3.9863135640369425, + "grad_norm": 1.625, + "learning_rate": 5.637037037037037e-06, + "loss": 0.674, + "mean_token_accuracy": 0.835492142662406, + "num_tokens": 59953838.0, + "step": 2240 + }, + { + "entropy": 0.5570171371583016, + "epoch": 4.003560698787137, + "grad_norm": 1.34375, + "learning_rate": 5.562962962962963e-06, + "loss": 0.6546, + "mean_token_accuracy": 0.8391922777698886, + "num_tokens": 60216049.0, + "step": 2250 + }, + { + "entropy": 0.5453685067594052, + "epoch": 4.021364192722822, + "grad_norm": 1.3515625, + "learning_rate": 5.4888888888888895e-06, + "loss": 0.6385, + "mean_token_accuracy": 0.8419853840023279, + "num_tokens": 60490937.0, + "step": 2260 + }, + { + "entropy": 0.553492389805615, + "epoch": 4.039167686658507, + "grad_norm": 1.453125, + "learning_rate": 5.414814814814815e-06, + "loss": 0.6556, + "mean_token_accuracy": 0.8393857058137655, + "num_tokens": 60755892.0, + "step": 2270 + }, + { + "entropy": 0.5440751701593399, + "epoch": 4.056971180594192, + "grad_norm": 1.4765625, + "learning_rate": 5.340740740740741e-06, + "loss": 0.6398, + "mean_token_accuracy": 0.8416271641850471, + "num_tokens": 61030893.0, + "step": 2280 + }, + { + "entropy": 0.5338836884126067, + "epoch": 4.074774674529876, + "grad_norm": 1.4453125, + "learning_rate": 5.2666666666666665e-06, + "loss": 0.6349, + "mean_token_accuracy": 0.844103604927659, + "num_tokens": 61301458.0, + "step": 2290 + }, + { + "entropy": 0.5652395199984313, + "epoch": 4.092578168465561, + "grad_norm": 1.5078125, + "learning_rate": 5.192592592592594e-06, + "loss": 0.6572, + "mean_token_accuracy": 0.8368313021957874, + "num_tokens": 61574717.0, + "step": 2300 + }, + { + "epoch": 4.092578168465561, + "eval_biology_entropy": 1.3518374223709106, + "eval_biology_loss": 2.1629726886749268, + "eval_biology_mean_token_accuracy": 0.6107748429775238, + "eval_biology_num_tokens": 61574717.0, + "eval_biology_runtime": 39.0562, + "eval_biology_samples_per_second": 12.802, + "eval_biology_steps_per_second": 3.201, + "step": 2300 + }, + { + "epoch": 4.092578168465561, + "eval_chemistry_entropy": 0.5780029859542847, + "eval_chemistry_loss": 0.8214347958564758, + "eval_chemistry_mean_token_accuracy": 0.810426944732666, + "eval_chemistry_num_tokens": 61574717.0, + "eval_chemistry_runtime": 48.6785, + "eval_chemistry_samples_per_second": 10.271, + "eval_chemistry_steps_per_second": 2.568, + "step": 2300 + }, + { + "epoch": 4.092578168465561, + "eval_math_entropy": 0.6076318190097809, + "eval_math_loss": 1.1111681461334229, + "eval_math_mean_token_accuracy": 0.7712901134490967, + "eval_math_num_tokens": 61574717.0, + "eval_math_runtime": 50.0691, + "eval_math_samples_per_second": 9.986, + "eval_math_steps_per_second": 2.497, + "step": 2300 + }, + { + "epoch": 4.092578168465561, + "eval_physics_entropy": 0.6077080953121186, + "eval_physics_loss": 0.8865928053855896, + "eval_physics_mean_token_accuracy": 0.7976819982528687, + "eval_physics_num_tokens": 61574717.0, + "eval_physics_runtime": 57.5982, + "eval_physics_samples_per_second": 8.681, + "eval_physics_steps_per_second": 2.17, + "step": 2300 + }, + { + "entropy": 0.5665840411558747, + "epoch": 4.110381662401246, + "grad_norm": 1.5859375, + "learning_rate": 5.1185185185185195e-06, + "loss": 0.6708, + "mean_token_accuracy": 0.8358031015843153, + "num_tokens": 61840186.0, + "step": 2310 + }, + { + "entropy": 0.56264589689672, + "epoch": 4.128185156336931, + "grad_norm": 1.453125, + "learning_rate": 5.044444444444445e-06, + "loss": 0.6631, + "mean_token_accuracy": 0.836660572513938, + "num_tokens": 62112844.0, + "step": 2320 + }, + { + "entropy": 0.5638300092890859, + "epoch": 4.145988650272616, + "grad_norm": 1.4296875, + "learning_rate": 4.970370370370371e-06, + "loss": 0.6511, + "mean_token_accuracy": 0.8376240566372871, + "num_tokens": 62382390.0, + "step": 2330 + }, + { + "entropy": 0.5563945654779673, + "epoch": 4.163792144208301, + "grad_norm": 1.3203125, + "learning_rate": 4.8962962962962965e-06, + "loss": 0.6578, + "mean_token_accuracy": 0.8388708371669054, + "num_tokens": 62658026.0, + "step": 2340 + }, + { + "entropy": 0.5789911892265082, + "epoch": 4.181595638143985, + "grad_norm": 1.5078125, + "learning_rate": 4.822222222222222e-06, + "loss": 0.6856, + "mean_token_accuracy": 0.8329568482935429, + "num_tokens": 62916724.0, + "step": 2350 + }, + { + "entropy": 0.555764663591981, + "epoch": 4.19939913207967, + "grad_norm": 1.484375, + "learning_rate": 4.748148148148149e-06, + "loss": 0.6465, + "mean_token_accuracy": 0.8380315884947777, + "num_tokens": 63195271.0, + "step": 2360 + }, + { + "entropy": 0.5678263012319803, + "epoch": 4.2172026260153554, + "grad_norm": 1.453125, + "learning_rate": 4.674074074074074e-06, + "loss": 0.6671, + "mean_token_accuracy": 0.8358955714851618, + "num_tokens": 63466066.0, + "step": 2370 + }, + { + "entropy": 0.554285722412169, + "epoch": 4.2350061199510405, + "grad_norm": 1.4921875, + "learning_rate": 4.600000000000001e-06, + "loss": 0.6573, + "mean_token_accuracy": 0.838936285302043, + "num_tokens": 63739678.0, + "step": 2380 + }, + { + "entropy": 0.5789531769230962, + "epoch": 4.2528096138867255, + "grad_norm": 1.4765625, + "learning_rate": 4.5259259259259265e-06, + "loss": 0.6723, + "mean_token_accuracy": 0.833799697458744, + "num_tokens": 63999364.0, + "step": 2390 + }, + { + "entropy": 0.5556379873305559, + "epoch": 4.2706131078224105, + "grad_norm": 1.4609375, + "learning_rate": 4.451851851851852e-06, + "loss": 0.6557, + "mean_token_accuracy": 0.8391667053103447, + "num_tokens": 64270501.0, + "step": 2400 + }, + { + "epoch": 4.2706131078224105, + "eval_biology_entropy": 1.3456227674484253, + "eval_biology_loss": 2.162630558013916, + "eval_biology_mean_token_accuracy": 0.6108367409706116, + "eval_biology_num_tokens": 64270501.0, + "eval_biology_runtime": 39.0723, + "eval_biology_samples_per_second": 12.797, + "eval_biology_steps_per_second": 3.199, + "step": 2400 + }, + { + "epoch": 4.2706131078224105, + "eval_chemistry_entropy": 0.5770703101158142, + "eval_chemistry_loss": 0.8213785290718079, + "eval_chemistry_mean_token_accuracy": 0.8106174335479737, + "eval_chemistry_num_tokens": 64270501.0, + "eval_chemistry_runtime": 48.65, + "eval_chemistry_samples_per_second": 10.278, + "eval_chemistry_steps_per_second": 2.569, + "step": 2400 + }, + { + "epoch": 4.2706131078224105, + "eval_math_entropy": 0.6059090893268585, + "eval_math_loss": 1.110784888267517, + "eval_math_mean_token_accuracy": 0.771416862487793, + "eval_math_num_tokens": 64270501.0, + "eval_math_runtime": 50.0542, + "eval_math_samples_per_second": 9.989, + "eval_math_steps_per_second": 2.497, + "step": 2400 + }, + { + "epoch": 4.2706131078224105, + "eval_physics_entropy": 0.6061555581092835, + "eval_physics_loss": 0.8872570991516113, + "eval_physics_mean_token_accuracy": 0.797800799369812, + "eval_physics_num_tokens": 64270501.0, + "eval_physics_runtime": 57.5677, + "eval_physics_samples_per_second": 8.685, + "eval_physics_steps_per_second": 2.171, + "step": 2400 + }, + { + "entropy": 0.5603191582486033, + "epoch": 4.288416601758095, + "grad_norm": 1.640625, + "learning_rate": 4.377777777777778e-06, + "loss": 0.6653, + "mean_token_accuracy": 0.8375619545578956, + "num_tokens": 64533223.0, + "step": 2410 + }, + { + "entropy": 0.5656454924494028, + "epoch": 4.30622009569378, + "grad_norm": 1.515625, + "learning_rate": 4.3037037037037035e-06, + "loss": 0.659, + "mean_token_accuracy": 0.8366425581276417, + "num_tokens": 64799113.0, + "step": 2420 + }, + { + "entropy": 0.5550693856552243, + "epoch": 4.324023589629465, + "grad_norm": 1.625, + "learning_rate": 4.22962962962963e-06, + "loss": 0.6538, + "mean_token_accuracy": 0.8389651108533144, + "num_tokens": 65075459.0, + "step": 2430 + }, + { + "entropy": 0.5567959282547236, + "epoch": 4.34182708356515, + "grad_norm": 1.640625, + "learning_rate": 4.155555555555556e-06, + "loss": 0.66, + "mean_token_accuracy": 0.837913642078638, + "num_tokens": 65341896.0, + "step": 2440 + }, + { + "entropy": 0.5572125539183617, + "epoch": 4.359630577500835, + "grad_norm": 1.3828125, + "learning_rate": 4.081481481481482e-06, + "loss": 0.6482, + "mean_token_accuracy": 0.837642902880907, + "num_tokens": 65614487.0, + "step": 2450 + }, + { + "entropy": 0.5893890267238021, + "epoch": 4.37743407143652, + "grad_norm": 1.4375, + "learning_rate": 4.007407407407408e-06, + "loss": 0.6784, + "mean_token_accuracy": 0.830023642629385, + "num_tokens": 65887311.0, + "step": 2460 + }, + { + "entropy": 0.5546840105205775, + "epoch": 4.395237565372204, + "grad_norm": 1.453125, + "learning_rate": 3.9333333333333335e-06, + "loss": 0.6537, + "mean_token_accuracy": 0.8394486285746098, + "num_tokens": 66151098.0, + "step": 2470 + }, + { + "entropy": 0.5494891541078687, + "epoch": 4.413041059307889, + "grad_norm": 1.5703125, + "learning_rate": 3.85925925925926e-06, + "loss": 0.6481, + "mean_token_accuracy": 0.8396681498736143, + "num_tokens": 66417928.0, + "step": 2480 + }, + { + "entropy": 0.5561942363157868, + "epoch": 4.430844553243574, + "grad_norm": 1.796875, + "learning_rate": 3.7851851851851857e-06, + "loss": 0.6614, + "mean_token_accuracy": 0.8361107159405947, + "num_tokens": 66684481.0, + "step": 2490 + }, + { + "entropy": 0.5738992135971784, + "epoch": 4.448648047179259, + "grad_norm": 1.7109375, + "learning_rate": 3.7111111111111113e-06, + "loss": 0.6772, + "mean_token_accuracy": 0.833794866502285, + "num_tokens": 66940215.0, + "step": 2500 + }, + { + "epoch": 4.448648047179259, + "eval_biology_entropy": 1.3500232787132262, + "eval_biology_loss": 2.1630518436431885, + "eval_biology_mean_token_accuracy": 0.6106264660358429, + "eval_biology_num_tokens": 66940215.0, + "eval_biology_runtime": 39.0451, + "eval_biology_samples_per_second": 12.806, + "eval_biology_steps_per_second": 3.201, + "step": 2500 + }, + { + "epoch": 4.448648047179259, + "eval_chemistry_entropy": 0.5779505209922791, + "eval_chemistry_loss": 0.8212056756019592, + "eval_chemistry_mean_token_accuracy": 0.8103259291648864, + "eval_chemistry_num_tokens": 66940215.0, + "eval_chemistry_runtime": 48.6513, + "eval_chemistry_samples_per_second": 10.277, + "eval_chemistry_steps_per_second": 2.569, + "step": 2500 + }, + { + "epoch": 4.448648047179259, + "eval_math_entropy": 0.6078341612815857, + "eval_math_loss": 1.111467719078064, + "eval_math_mean_token_accuracy": 0.771428505897522, + "eval_math_num_tokens": 66940215.0, + "eval_math_runtime": 50.181, + "eval_math_samples_per_second": 9.964, + "eval_math_steps_per_second": 2.491, + "step": 2500 + }, + { + "epoch": 4.448648047179259, + "eval_physics_entropy": 0.6069678666591645, + "eval_physics_loss": 0.8872233629226685, + "eval_physics_mean_token_accuracy": 0.7977854557037354, + "eval_physics_num_tokens": 66940215.0, + "eval_physics_runtime": 57.617, + "eval_physics_samples_per_second": 8.678, + "eval_physics_steps_per_second": 2.169, + "step": 2500 + } + ], + "logging_steps": 10, + "max_steps": 3000, + "num_input_tokens_seen": 0, + "num_train_epochs": 6, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 5.20654490802918e+18, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/checkpoint-2500/training_args.bin b/checkpoint-2500/training_args.bin new file mode 100644 index 0000000..79cad08 --- /dev/null +++ b/checkpoint-2500/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dfc23fa1b9be77c605cba9733debba5f60170f0eadaa2a8866b5aaf4b0e65756 +size 6417 diff --git a/checkpoint-3000/chat_template.jinja b/checkpoint-3000/chat_template.jinja new file mode 100644 index 0000000..923ec25 --- /dev/null +++ b/checkpoint-3000/chat_template.jinja @@ -0,0 +1,4 @@ +{{ bos_token }}{% if messages[0]['role'] == 'system' %}{{ raise_exception('System role not supported') }}{% endif %}{% for message in messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if (message['role'] == 'assistant') %}{% set role = 'model' %}{% else %}{% set role = message['role'] %}{% endif %}{{ '' + role + ' +' + message['content'] | trim + ' +' }}{% endfor %}{% if add_generation_prompt %}{{'model +'}}{% endif %} \ No newline at end of file diff --git a/checkpoint-3000/config.json b/checkpoint-3000/config.json new file mode 100644 index 0000000..29e6cbe --- /dev/null +++ b/checkpoint-3000/config.json @@ -0,0 +1,77 @@ +{ + "architectures": [ + "Gemma2ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "attn_logit_softcapping": 50.0, + "bos_token_id": 2, + "cache_implementation": "hybrid", + "dtype": "bfloat16", + "eos_token_id": 1, + "final_logit_softcapping": 30.0, + "head_dim": 256, + "hidden_act": "gelu_pytorch_tanh", + "hidden_activation": "gelu_pytorch_tanh", + "hidden_size": 3584, + "initializer_range": 0.02, + "intermediate_size": 14336, + "layer_types": [ + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention" + ], + "max_position_embeddings": 8192, + "model_type": "gemma2", + "num_attention_heads": 16, + "num_hidden_layers": 42, + "num_key_value_heads": 8, + "pad_token_id": 0, + "query_pre_attn_scalar": 256, + "rms_norm_eps": 1e-06, + "rope_theta": 10000.0, + "sliding_window": 4096, + "sliding_window_size": 4096, + "transformers_version": "4.56.1", + "use_cache": true, + "vocab_size": 256000 +} diff --git a/checkpoint-3000/generation_config.json b/checkpoint-3000/generation_config.json new file mode 100644 index 0000000..f2c34f2 --- /dev/null +++ b/checkpoint-3000/generation_config.json @@ -0,0 +1,9 @@ +{ + "_from_model_config": true, + "bos_token_id": 2, + "eos_token_id": [ + 1 + ], + "pad_token_id": 0, + "transformers_version": "4.56.1" +} diff --git a/checkpoint-3000/model-00001-of-00004.safetensors b/checkpoint-3000/model-00001-of-00004.safetensors new file mode 100644 index 0000000..4705c7e --- /dev/null +++ b/checkpoint-3000/model-00001-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a0d4eb4fcbddfe01b0dc7a58386a3980826ededaf0ecd67d1ef38995b89615ba +size 4903351912 diff --git a/checkpoint-3000/model-00002-of-00004.safetensors b/checkpoint-3000/model-00002-of-00004.safetensors new file mode 100644 index 0000000..744b59b --- /dev/null +++ b/checkpoint-3000/model-00002-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:712a589c333f49f7b6f3870838a9e0ab4b3c0260abd5130e8c54316e6c99640b +size 4947570872 diff --git a/checkpoint-3000/model-00003-of-00004.safetensors b/checkpoint-3000/model-00003-of-00004.safetensors new file mode 100644 index 0000000..e6eaa38 --- /dev/null +++ b/checkpoint-3000/model-00003-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:123eeb1ba62c1bf1fa49cd38911ea80674b44fe122d80d1ee0aeba8d01bb988e +size 4962221464 diff --git a/checkpoint-3000/model-00004-of-00004.safetensors b/checkpoint-3000/model-00004-of-00004.safetensors new file mode 100644 index 0000000..4707165 --- /dev/null +++ b/checkpoint-3000/model-00004-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6754a347e44efa3115fb68ec8aded7512b7e469a1046d2bad6b7182c5500d23a +size 3670322200 diff --git a/checkpoint-3000/model.safetensors.index.json b/checkpoint-3000/model.safetensors.index.json new file mode 100644 index 0000000..3b569c5 --- /dev/null +++ b/checkpoint-3000/model.safetensors.index.json @@ -0,0 +1,472 @@ +{ + "metadata": { + "total_parameters": 9241705984, + "total_size": 18483411968 + }, + "weight_map": { + "model.embed_tokens.weight": "model-00001-of-00004.safetensors", + "model.layers.0.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.0.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.0.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.1.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.1.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.10.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.10.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.10.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.10.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.11.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.11.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.11.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.12.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.12.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.12.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.13.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.13.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.13.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.14.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.14.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.14.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.15.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.15.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.15.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.16.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.16.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.16.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.17.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.17.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.17.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.18.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.18.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.18.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.19.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.19.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.19.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.2.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.2.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.2.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.20.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.20.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.20.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.20.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.20.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.20.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.20.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.21.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.21.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.21.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.21.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.21.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.22.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.22.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.22.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.22.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.23.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.23.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.23.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.24.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.24.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.24.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.25.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.25.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.25.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.26.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.26.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.26.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.27.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.27.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.27.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.28.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.28.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.28.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.29.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.29.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.29.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.3.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.3.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.3.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.30.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.30.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.30.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.30.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.30.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.31.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.31.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.31.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.32.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.32.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.32.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.32.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.32.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.32.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.32.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.32.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.32.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.32.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.32.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.33.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.33.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.33.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.33.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.33.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.33.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.33.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.33.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.33.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.33.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.33.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.34.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.34.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.34.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.34.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.34.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.34.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.34.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.34.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.34.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.34.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.34.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.35.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.35.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.35.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.35.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.35.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.35.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.35.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.35.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.35.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.35.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.35.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.36.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.36.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.36.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.36.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.36.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.36.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.36.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.36.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.36.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.36.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.36.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.37.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.37.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.37.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.37.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.37.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.37.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.37.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.37.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.37.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.37.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.37.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.38.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.38.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.38.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.38.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.38.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.38.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.38.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.38.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.38.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.38.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.38.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.39.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.39.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.39.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.39.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.39.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.39.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.39.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.39.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.39.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.39.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.39.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.4.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.4.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.4.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.4.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.40.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.40.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.40.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.40.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.40.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.40.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.40.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.40.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.40.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.40.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.40.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.41.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.41.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.41.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.41.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.41.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.41.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.41.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.41.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.41.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.41.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.41.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.5.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.5.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.5.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.5.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.6.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.6.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.6.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.7.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.7.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.7.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.8.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.8.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.8.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.8.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.8.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.8.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.8.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.8.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.8.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.8.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.8.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.9.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.9.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.9.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.9.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.norm.weight": "model-00004-of-00004.safetensors" + } +} diff --git a/checkpoint-3000/optimizer.pt b/checkpoint-3000/optimizer.pt new file mode 100644 index 0000000..82c34b0 --- /dev/null +++ b/checkpoint-3000/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a781711c07ac8dcc8599b6cec02cac464656bd900c73b77043e5421cd287cefc +size 1585596266 diff --git a/checkpoint-3000/rng_state.pth b/checkpoint-3000/rng_state.pth new file mode 100644 index 0000000..5a0bd7e --- /dev/null +++ b/checkpoint-3000/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:df73238e0c82b047e6fcac514e915cb0e150c27053f3ac3761d9a5d5f6a92e4b +size 14645 diff --git a/checkpoint-3000/scheduler.pt b/checkpoint-3000/scheduler.pt new file mode 100644 index 0000000..30995d1 --- /dev/null +++ b/checkpoint-3000/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7adf24c09dad460949bb6a6532283c2b8cee7b11d72400b3fd2ec4ddf2318381 +size 1465 diff --git a/checkpoint-3000/special_tokens_map.json b/checkpoint-3000/special_tokens_map.json new file mode 100644 index 0000000..8d6368f --- /dev/null +++ b/checkpoint-3000/special_tokens_map.json @@ -0,0 +1,34 @@ +{ + "additional_special_tokens": [ + "", + "" + ], + "bos_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "unk_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + } +} diff --git a/checkpoint-3000/tokenizer.json b/checkpoint-3000/tokenizer.json new file mode 100644 index 0000000..a4a305d --- /dev/null +++ b/checkpoint-3000/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5f7eee611703c5ce5d1eee32d9cdcfe465647b8aff0c1dfb3bed7ad7dbb05060 +size 34362873 diff --git a/checkpoint-3000/tokenizer.model b/checkpoint-3000/tokenizer.model new file mode 100644 index 0000000..796efe9 --- /dev/null +++ b/checkpoint-3000/tokenizer.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:61a7b147390c64585d6c3543dd6fc636906c9af3865a5548f27f31aee1d4c8e2 +size 4241003 diff --git a/checkpoint-3000/tokenizer_config.json b/checkpoint-3000/tokenizer_config.json new file mode 100644 index 0000000..3ade6be --- /dev/null +++ b/checkpoint-3000/tokenizer_config.json @@ -0,0 +1,2013 @@ +{ + "add_bos_token": true, + "add_eos_token": false, + "added_tokens_decoder": { + "0": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "1": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "2": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "3": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "4": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "5": { + "content": "<2mass>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "6": { + "content": "[@BOS@]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "7": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "8": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "9": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "10": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "11": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "12": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "13": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "14": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "15": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "16": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "17": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "18": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "19": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "20": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "21": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "22": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "23": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "24": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "25": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "26": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "27": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "28": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "29": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "30": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "31": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "32": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "33": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "34": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "35": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "36": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "37": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "38": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "39": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "40": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "41": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "42": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "43": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "44": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "45": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "46": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "47": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "48": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "49": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "50": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "51": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "52": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "53": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "54": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "55": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "56": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "57": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "58": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "59": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "60": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "61": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "62": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "63": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "64": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "65": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "66": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "67": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "68": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "69": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "70": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "71": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "72": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "73": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "74": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "75": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "76": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "77": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "78": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "79": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "80": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "81": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "82": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "83": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "84": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "85": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "86": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "87": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "88": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "89": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "90": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "91": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "92": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "93": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "94": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "95": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "96": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "97": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "98": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "99": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "100": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "101": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "102": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "103": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "104": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "105": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "106": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "107": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "108": { + "content": "\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "109": { + "content": "\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "110": { + "content": "\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "111": { + "content": "\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "112": { + "content": "\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "113": { + "content": "\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "114": { + "content": "\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "115": { + "content": "\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "116": { + "content": "\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "117": { + "content": "\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "118": { + "content": "\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "119": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "120": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "121": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "122": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "123": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "124": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "125": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "126": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "127": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "128": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "129": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "130": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "131": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "132": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "133": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "134": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "135": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "136": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "137": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "138": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "139": { + "content": "▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "140": { + "content": "▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "141": { + "content": "▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "142": { + "content": "▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "143": { + "content": "▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "144": { + "content": "▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "145": { + "content": "▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "146": { + "content": "▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "147": { + "content": "▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "148": { + "content": "▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "149": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "150": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "152": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "153": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "154": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "155": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "156": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "157": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "158": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "159": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "160": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "161": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "162": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "163": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "164": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "165": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "166": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "167": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "168": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "169": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "170": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "172": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "173": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "174": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "175": { + "content": "
", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "171": { + "content": "
", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "176": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "177": { + "content": "
", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "178": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "179": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "180": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "181": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "182": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "183": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "184": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "185": { + "content": "

", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "186": { + "content": "

", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "187": { + "content": "

", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "188": { + "content": "

", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "189": { + "content": "

", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "190": { + "content": "
", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "191": { + "content": "
", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "192": { + "content": "
", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "193": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "194": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "195": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "196": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "197": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "198": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "199": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "200": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "201": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "202": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "203": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "204": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "205": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "206": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "207": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "208": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "209": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "210": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "211": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "212": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "213": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "214": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "215": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "216": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255968": { + "content": "[toxicity=0]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255969": { + "content": "\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255970": { + "content": "\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255971": { + "content": "\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255972": { + "content": "\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255973": { + "content": "\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255974": { + "content": "\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255975": { + "content": "\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255976": { + "content": "\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255977": { + "content": "\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255978": { + "content": "\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255979": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255980": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255981": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255982": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255983": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255984": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255985": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255986": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255987": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255988": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255989": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255990": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255991": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255992": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255993": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255994": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255995": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255996": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255997": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255998": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255999": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + } + }, + "additional_special_tokens": [ + "", + "" + ], + "bos_token": "", + "clean_up_tokenization_spaces": false, + "eos_token": "", + "extra_special_tokens": {}, + "model_max_length": 1000000000000000019884624838656, + "pad_token": "", + "sp_model_kwargs": {}, + "spaces_between_special_tokens": false, + "tokenizer_class": "GemmaTokenizer", + "unk_token": "", + "use_default_system_prompt": false +} diff --git a/checkpoint-3000/trainer_state.json b/checkpoint-3000/trainer_state.json new file mode 100644 index 0000000..26abc6e --- /dev/null +++ b/checkpoint-3000/trainer_state.json @@ -0,0 +1,4354 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 5.338266384778013, + "eval_steps": 100, + "global_step": 3000, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.3253214355558156, + "epoch": 0.017803493935684877, + "grad_norm": 14.5, + "learning_rate": 6.000000000000001e-07, + "loss": 2.1959, + "mean_token_accuracy": 0.5975642748177051, + "num_tokens": 259048.0, + "step": 10 + }, + { + "entropy": 1.3036176152527332, + "epoch": 0.035606987871369754, + "grad_norm": 12.75, + "learning_rate": 1.2666666666666669e-06, + "loss": 2.1499, + "mean_token_accuracy": 0.6023047739639878, + "num_tokens": 532019.0, + "step": 20 + }, + { + "entropy": 1.3484113823622466, + "epoch": 0.053410481807054634, + "grad_norm": 10.8125, + "learning_rate": 1.9333333333333336e-06, + "loss": 2.108, + "mean_token_accuracy": 0.6078463492915034, + "num_tokens": 807220.0, + "step": 30 + }, + { + "entropy": 1.4311681490391492, + "epoch": 0.07121397574273951, + "grad_norm": 7.875, + "learning_rate": 2.6e-06, + "loss": 2.0349, + "mean_token_accuracy": 0.6107733339071274, + "num_tokens": 1068429.0, + "step": 40 + }, + { + "entropy": 1.479667642340064, + "epoch": 0.08901746967842439, + "grad_norm": 6.4375, + "learning_rate": 3.266666666666667e-06, + "loss": 1.8649, + "mean_token_accuracy": 0.630409375578165, + "num_tokens": 1336698.0, + "step": 50 + }, + { + "entropy": 1.5030938494950532, + "epoch": 0.10682096361410927, + "grad_norm": 4.1875, + "learning_rate": 3.9333333333333335e-06, + "loss": 1.7217, + "mean_token_accuracy": 0.6490663634613156, + "num_tokens": 1608574.0, + "step": 60 + }, + { + "entropy": 1.5066865295171739, + "epoch": 0.12462445754979415, + "grad_norm": 2.71875, + "learning_rate": 4.600000000000001e-06, + "loss": 1.5716, + "mean_token_accuracy": 0.6711439102888107, + "num_tokens": 1872747.0, + "step": 70 + }, + { + "entropy": 1.3747932206839324, + "epoch": 0.14242795148547902, + "grad_norm": 2.234375, + "learning_rate": 5.2666666666666665e-06, + "loss": 1.4403, + "mean_token_accuracy": 0.6936415441334247, + "num_tokens": 2133173.0, + "step": 80 + }, + { + "entropy": 1.1736511185765266, + "epoch": 0.1602314454211639, + "grad_norm": 1.921875, + "learning_rate": 5.933333333333335e-06, + "loss": 1.2902, + "mean_token_accuracy": 0.7222886722534895, + "num_tokens": 2406145.0, + "step": 90 + }, + { + "entropy": 1.1317794900387526, + "epoch": 0.17803493935684878, + "grad_norm": 1.8046875, + "learning_rate": 6.600000000000001e-06, + "loss": 1.2424, + "mean_token_accuracy": 0.7295983772724867, + "num_tokens": 2663709.0, + "step": 100 + }, + { + "epoch": 0.17803493935684878, + "eval_biology_entropy": 2.3133783559799195, + "eval_biology_loss": 2.7910587787628174, + "eval_biology_mean_token_accuracy": 0.5260232911109924, + "eval_biology_num_tokens": 2663709.0, + "eval_biology_runtime": 39.1656, + "eval_biology_samples_per_second": 12.766, + "eval_biology_steps_per_second": 3.192, + "step": 100 + }, + { + "epoch": 0.17803493935684878, + "eval_chemistry_entropy": 1.0788086318969727, + "eval_chemistry_loss": 1.2403336763381958, + "eval_chemistry_mean_token_accuracy": 0.7357268543243408, + "eval_chemistry_num_tokens": 2663709.0, + "eval_chemistry_runtime": 48.7822, + "eval_chemistry_samples_per_second": 10.25, + "eval_chemistry_steps_per_second": 2.562, + "step": 100 + }, + { + "epoch": 0.17803493935684878, + "eval_math_entropy": 0.938195451259613, + "eval_math_loss": 1.3248047828674316, + "eval_math_mean_token_accuracy": 0.7312402629852295, + "eval_math_num_tokens": 2663709.0, + "eval_math_runtime": 50.197, + "eval_math_samples_per_second": 9.961, + "eval_math_steps_per_second": 2.49, + "step": 100 + }, + { + "epoch": 0.17803493935684878, + "eval_physics_entropy": 1.022246531009674, + "eval_physics_loss": 1.176153302192688, + "eval_physics_mean_token_accuracy": 0.7475795021057129, + "eval_physics_num_tokens": 2663709.0, + "eval_physics_runtime": 57.6895, + "eval_physics_samples_per_second": 8.667, + "eval_physics_steps_per_second": 2.167, + "step": 100 + }, + { + "entropy": 1.0380497876554728, + "epoch": 0.19583843329253367, + "grad_norm": 1.7421875, + "learning_rate": 7.266666666666668e-06, + "loss": 1.157, + "mean_token_accuracy": 0.7445634145289659, + "num_tokens": 2934980.0, + "step": 110 + }, + { + "entropy": 1.0149445166811346, + "epoch": 0.21364192722821854, + "grad_norm": 1.6015625, + "learning_rate": 7.933333333333334e-06, + "loss": 1.1167, + "mean_token_accuracy": 0.7485828153789044, + "num_tokens": 3208702.0, + "step": 120 + }, + { + "entropy": 0.9639404848217964, + "epoch": 0.2314454211639034, + "grad_norm": 1.6953125, + "learning_rate": 8.6e-06, + "loss": 1.0979, + "mean_token_accuracy": 0.7553199239075183, + "num_tokens": 3480119.0, + "step": 130 + }, + { + "entropy": 0.9687324531376362, + "epoch": 0.2492489150995883, + "grad_norm": 1.5625, + "learning_rate": 9.266666666666667e-06, + "loss": 1.0762, + "mean_token_accuracy": 0.757096454873681, + "num_tokens": 3746392.0, + "step": 140 + }, + { + "entropy": 0.9682734729722142, + "epoch": 0.26705240903527316, + "grad_norm": 1.875, + "learning_rate": 9.933333333333334e-06, + "loss": 1.0798, + "mean_token_accuracy": 0.7543047085404396, + "num_tokens": 4019487.0, + "step": 150 + }, + { + "entropy": 0.9270784268155694, + "epoch": 0.28485590297095803, + "grad_norm": 1.6015625, + "learning_rate": 1.0600000000000002e-05, + "loss": 1.047, + "mean_token_accuracy": 0.7619738396257162, + "num_tokens": 4298100.0, + "step": 160 + }, + { + "entropy": 0.9063327137380839, + "epoch": 0.30265939690664295, + "grad_norm": 1.5546875, + "learning_rate": 1.1266666666666668e-05, + "loss": 1.0199, + "mean_token_accuracy": 0.7664302848279476, + "num_tokens": 4568116.0, + "step": 170 + }, + { + "entropy": 0.9037324048578739, + "epoch": 0.3204628908423278, + "grad_norm": 1.7109375, + "learning_rate": 1.1933333333333335e-05, + "loss": 1.0144, + "mean_token_accuracy": 0.7657378144562245, + "num_tokens": 4831334.0, + "step": 180 + }, + { + "entropy": 0.8886868219822646, + "epoch": 0.3382663847780127, + "grad_norm": 1.3359375, + "learning_rate": 1.2600000000000001e-05, + "loss": 0.9874, + "mean_token_accuracy": 0.7726409930735827, + "num_tokens": 5103301.0, + "step": 190 + }, + { + "entropy": 0.8936551520600915, + "epoch": 0.35606987871369755, + "grad_norm": 1.3984375, + "learning_rate": 1.3266666666666668e-05, + "loss": 1.019, + "mean_token_accuracy": 0.7648640714585782, + "num_tokens": 5362915.0, + "step": 200 + }, + { + "epoch": 0.35606987871369755, + "eval_biology_entropy": 2.0171296310424807, + "eval_biology_loss": 2.3780677318573, + "eval_biology_mean_token_accuracy": 0.5706260974407196, + "eval_biology_num_tokens": 5362915.0, + "eval_biology_runtime": 39.1203, + "eval_biology_samples_per_second": 12.781, + "eval_biology_steps_per_second": 3.195, + "step": 200 + }, + { + "epoch": 0.35606987871369755, + "eval_chemistry_entropy": 0.8995157589912415, + "eval_chemistry_loss": 1.022930383682251, + "eval_chemistry_mean_token_accuracy": 0.7701377034187317, + "eval_chemistry_num_tokens": 5362915.0, + "eval_chemistry_runtime": 48.7263, + "eval_chemistry_samples_per_second": 10.261, + "eval_chemistry_steps_per_second": 2.565, + "step": 200 + }, + { + "epoch": 0.35606987871369755, + "eval_math_entropy": 0.8352043969631195, + "eval_math_loss": 1.1696399450302124, + "eval_math_mean_token_accuracy": 0.7531440143585205, + "eval_math_num_tokens": 5362915.0, + "eval_math_runtime": 50.1494, + "eval_math_samples_per_second": 9.97, + "eval_math_steps_per_second": 2.493, + "step": 200 + }, + { + "epoch": 0.35606987871369755, + "eval_physics_entropy": 0.8889413847923279, + "eval_physics_loss": 0.9962027072906494, + "eval_physics_mean_token_accuracy": 0.775176504611969, + "eval_physics_num_tokens": 5362915.0, + "eval_physics_runtime": 57.6533, + "eval_physics_samples_per_second": 8.673, + "eval_physics_steps_per_second": 2.168, + "step": 200 + }, + { + "entropy": 0.8509893301874399, + "epoch": 0.3738733726493824, + "grad_norm": 1.3359375, + "learning_rate": 1.3933333333333334e-05, + "loss": 0.9619, + "mean_token_accuracy": 0.7772600032389164, + "num_tokens": 5625455.0, + "step": 210 + }, + { + "entropy": 0.8560685662552714, + "epoch": 0.39167686658506734, + "grad_norm": 1.4609375, + "learning_rate": 1.46e-05, + "loss": 0.9666, + "mean_token_accuracy": 0.7764565829187632, + "num_tokens": 5890806.0, + "step": 220 + }, + { + "entropy": 0.8282079607248306, + "epoch": 0.4094803605207522, + "grad_norm": 1.3828125, + "learning_rate": 1.5266666666666667e-05, + "loss": 0.9353, + "mean_token_accuracy": 0.7817027729004622, + "num_tokens": 6158751.0, + "step": 230 + }, + { + "entropy": 0.8447791969403624, + "epoch": 0.4272838544564371, + "grad_norm": 1.4296875, + "learning_rate": 1.5933333333333336e-05, + "loss": 0.9428, + "mean_token_accuracy": 0.7795210376381874, + "num_tokens": 6423747.0, + "step": 240 + }, + { + "entropy": 0.8478817906230688, + "epoch": 0.44508734839212194, + "grad_norm": 1.8984375, + "learning_rate": 1.66e-05, + "loss": 0.9542, + "mean_token_accuracy": 0.7772787544876337, + "num_tokens": 6684517.0, + "step": 250 + }, + { + "entropy": 0.8135418241843582, + "epoch": 0.4628908423278068, + "grad_norm": 1.453125, + "learning_rate": 1.726666666666667e-05, + "loss": 0.9297, + "mean_token_accuracy": 0.783412617444992, + "num_tokens": 6945659.0, + "step": 260 + }, + { + "entropy": 0.8251218661665917, + "epoch": 0.48069433626349173, + "grad_norm": 1.6640625, + "learning_rate": 1.7933333333333333e-05, + "loss": 0.9307, + "mean_token_accuracy": 0.7831020273268223, + "num_tokens": 7217797.0, + "step": 270 + }, + { + "entropy": 0.7879339458420873, + "epoch": 0.4984978301991766, + "grad_norm": 1.328125, + "learning_rate": 1.86e-05, + "loss": 0.8795, + "mean_token_accuracy": 0.7897050529718399, + "num_tokens": 7498182.0, + "step": 280 + }, + { + "entropy": 0.8145477209240198, + "epoch": 0.5163013241348615, + "grad_norm": 1.265625, + "learning_rate": 1.926666666666667e-05, + "loss": 0.9187, + "mean_token_accuracy": 0.7838029216974973, + "num_tokens": 7761686.0, + "step": 290 + }, + { + "entropy": 0.8231891892850399, + "epoch": 0.5341048180705463, + "grad_norm": 1.5625, + "learning_rate": 1.9933333333333334e-05, + "loss": 0.9271, + "mean_token_accuracy": 0.7819279827177524, + "num_tokens": 8026923.0, + "step": 300 + }, + { + "epoch": 0.5341048180705463, + "eval_biology_entropy": 1.8821040544509888, + "eval_biology_loss": 2.226550340652466, + "eval_biology_mean_token_accuracy": 0.5873979187011719, + "eval_biology_num_tokens": 8026923.0, + "eval_biology_runtime": 39.1096, + "eval_biology_samples_per_second": 12.785, + "eval_biology_steps_per_second": 3.196, + "step": 300 + }, + { + "epoch": 0.5341048180705463, + "eval_chemistry_entropy": 0.7922393922805786, + "eval_chemistry_loss": 0.9438055157661438, + "eval_chemistry_mean_token_accuracy": 0.7841580080986023, + "eval_chemistry_num_tokens": 8026923.0, + "eval_chemistry_runtime": 48.7268, + "eval_chemistry_samples_per_second": 10.261, + "eval_chemistry_steps_per_second": 2.565, + "step": 300 + }, + { + "epoch": 0.5341048180705463, + "eval_math_entropy": 0.7649447858333588, + "eval_math_loss": 1.1145466566085815, + "eval_math_mean_token_accuracy": 0.763204463005066, + "eval_math_num_tokens": 8026923.0, + "eval_math_runtime": 50.1205, + "eval_math_samples_per_second": 9.976, + "eval_math_steps_per_second": 2.494, + "step": 300 + }, + { + "epoch": 0.5341048180705463, + "eval_physics_entropy": 0.7964776389598847, + "eval_physics_loss": 0.9373180866241455, + "eval_physics_mean_token_accuracy": 0.784841582775116, + "eval_physics_num_tokens": 8026923.0, + "eval_physics_runtime": 57.6222, + "eval_physics_samples_per_second": 8.677, + "eval_physics_steps_per_second": 2.169, + "step": 300 + }, + { + "entropy": 0.800599187426269, + "epoch": 0.5519083120062312, + "grad_norm": 1.6640625, + "learning_rate": 1.9933333333333334e-05, + "loss": 0.9071, + "mean_token_accuracy": 0.7856069568544626, + "num_tokens": 8301506.0, + "step": 310 + }, + { + "entropy": 0.7965721761807799, + "epoch": 0.5697118059419161, + "grad_norm": 1.234375, + "learning_rate": 1.985925925925926e-05, + "loss": 0.9139, + "mean_token_accuracy": 0.786777526140213, + "num_tokens": 8559126.0, + "step": 320 + }, + { + "entropy": 0.7925007566809654, + "epoch": 0.587515299877601, + "grad_norm": 1.4453125, + "learning_rate": 1.9785185185185187e-05, + "loss": 0.906, + "mean_token_accuracy": 0.7864235289394855, + "num_tokens": 8821088.0, + "step": 330 + }, + { + "entropy": 0.7728251311928034, + "epoch": 0.6053187938132859, + "grad_norm": 1.2109375, + "learning_rate": 1.971111111111111e-05, + "loss": 0.8764, + "mean_token_accuracy": 0.7919386122375727, + "num_tokens": 9089866.0, + "step": 340 + }, + { + "entropy": 0.7705478269606829, + "epoch": 0.6231222877489707, + "grad_norm": 1.3515625, + "learning_rate": 1.963703703703704e-05, + "loss": 0.8837, + "mean_token_accuracy": 0.7896027479320764, + "num_tokens": 9358866.0, + "step": 350 + }, + { + "entropy": 0.7656249281018972, + "epoch": 0.6409257816846556, + "grad_norm": 1.1875, + "learning_rate": 1.9562962962962964e-05, + "loss": 0.8626, + "mean_token_accuracy": 0.7931958142668009, + "num_tokens": 9629977.0, + "step": 360 + }, + { + "entropy": 0.7598591955378652, + "epoch": 0.6587292756203404, + "grad_norm": 1.296875, + "learning_rate": 1.948888888888889e-05, + "loss": 0.8608, + "mean_token_accuracy": 0.7945109561085701, + "num_tokens": 9897757.0, + "step": 370 + }, + { + "entropy": 0.7398761732503771, + "epoch": 0.6765327695560254, + "grad_norm": 1.265625, + "learning_rate": 1.9414814814814817e-05, + "loss": 0.848, + "mean_token_accuracy": 0.7985429972410202, + "num_tokens": 10168367.0, + "step": 380 + }, + { + "entropy": 0.7331193052232265, + "epoch": 0.6943362634917103, + "grad_norm": 1.4296875, + "learning_rate": 1.9340740740740743e-05, + "loss": 0.8366, + "mean_token_accuracy": 0.8001827634871006, + "num_tokens": 10439039.0, + "step": 390 + }, + { + "entropy": 0.7496613372117281, + "epoch": 0.7121397574273951, + "grad_norm": 1.15625, + "learning_rate": 1.926666666666667e-05, + "loss": 0.8517, + "mean_token_accuracy": 0.7967161379754544, + "num_tokens": 10709549.0, + "step": 400 + }, + { + "epoch": 0.7121397574273951, + "eval_biology_entropy": 1.823248610496521, + "eval_biology_loss": 2.159681558609009, + "eval_biology_mean_token_accuracy": 0.5972214796543122, + "eval_biology_num_tokens": 10709549.0, + "eval_biology_runtime": 39.2024, + "eval_biology_samples_per_second": 12.754, + "eval_biology_steps_per_second": 3.189, + "step": 400 + }, + { + "epoch": 0.7121397574273951, + "eval_chemistry_entropy": 0.7672218294143677, + "eval_chemistry_loss": 0.8971230387687683, + "eval_chemistry_mean_token_accuracy": 0.7928389143943787, + "eval_chemistry_num_tokens": 10709549.0, + "eval_chemistry_runtime": 48.8296, + "eval_chemistry_samples_per_second": 10.24, + "eval_chemistry_steps_per_second": 2.56, + "step": 400 + }, + { + "epoch": 0.7121397574273951, + "eval_math_entropy": 0.7494234163761139, + "eval_math_loss": 1.0849554538726807, + "eval_math_mean_token_accuracy": 0.7666959943771362, + "eval_math_num_tokens": 10709549.0, + "eval_math_runtime": 50.2325, + "eval_math_samples_per_second": 9.954, + "eval_math_steps_per_second": 2.488, + "step": 400 + }, + { + "epoch": 0.7121397574273951, + "eval_physics_entropy": 0.7731240901947022, + "eval_physics_loss": 0.9018430709838867, + "eval_physics_mean_token_accuracy": 0.7909709796905517, + "eval_physics_num_tokens": 10709549.0, + "eval_physics_runtime": 57.7809, + "eval_physics_samples_per_second": 8.653, + "eval_physics_steps_per_second": 2.163, + "step": 400 + }, + { + "entropy": 0.7354462265968322, + "epoch": 0.72994325136308, + "grad_norm": 1.3125, + "learning_rate": 1.9192592592592593e-05, + "loss": 0.8411, + "mean_token_accuracy": 0.8003354988992214, + "num_tokens": 10976011.0, + "step": 410 + }, + { + "entropy": 0.7435122076421976, + "epoch": 0.7477467452987648, + "grad_norm": 1.15625, + "learning_rate": 1.911851851851852e-05, + "loss": 0.8438, + "mean_token_accuracy": 0.7971596572548151, + "num_tokens": 11245251.0, + "step": 420 + }, + { + "entropy": 0.759309609234333, + "epoch": 0.7655502392344498, + "grad_norm": 1.375, + "learning_rate": 1.9044444444444446e-05, + "loss": 0.8579, + "mean_token_accuracy": 0.792990094423294, + "num_tokens": 11518514.0, + "step": 430 + }, + { + "entropy": 0.7545185785740613, + "epoch": 0.7833537331701347, + "grad_norm": 1.3125, + "learning_rate": 1.8970370370370372e-05, + "loss": 0.8574, + "mean_token_accuracy": 0.794682926684618, + "num_tokens": 11787117.0, + "step": 440 + }, + { + "entropy": 0.751254404336214, + "epoch": 0.8011572271058195, + "grad_norm": 1.515625, + "learning_rate": 1.8896296296296295e-05, + "loss": 0.8495, + "mean_token_accuracy": 0.7972703959792853, + "num_tokens": 12050993.0, + "step": 450 + }, + { + "entropy": 0.7543802969157696, + "epoch": 0.8189607210415044, + "grad_norm": 1.28125, + "learning_rate": 1.8822222222222225e-05, + "loss": 0.8527, + "mean_token_accuracy": 0.7960003320127725, + "num_tokens": 12311688.0, + "step": 460 + }, + { + "entropy": 0.7186065271496773, + "epoch": 0.8367642149771892, + "grad_norm": 1.15625, + "learning_rate": 1.874814814814815e-05, + "loss": 0.8096, + "mean_token_accuracy": 0.8027401898056269, + "num_tokens": 12598408.0, + "step": 470 + }, + { + "entropy": 0.7257912319153548, + "epoch": 0.8545677089128741, + "grad_norm": 1.125, + "learning_rate": 1.8674074074074075e-05, + "loss": 0.8299, + "mean_token_accuracy": 0.8012042317539454, + "num_tokens": 12869642.0, + "step": 480 + }, + { + "entropy": 0.7173270899802446, + "epoch": 0.8723712028485591, + "grad_norm": 1.125, + "learning_rate": 1.86e-05, + "loss": 0.8146, + "mean_token_accuracy": 0.8032398864626884, + "num_tokens": 13139555.0, + "step": 490 + }, + { + "entropy": 0.7487573977559805, + "epoch": 0.8901746967842439, + "grad_norm": 1.2421875, + "learning_rate": 1.8525925925925928e-05, + "loss": 0.8562, + "mean_token_accuracy": 0.7955521304160357, + "num_tokens": 13396545.0, + "step": 500 + }, + { + "epoch": 0.8901746967842439, + "eval_biology_entropy": 1.8204833917617798, + "eval_biology_loss": 2.1014022827148438, + "eval_biology_mean_token_accuracy": 0.6051526765823364, + "eval_biology_num_tokens": 13396545.0, + "eval_biology_runtime": 39.1543, + "eval_biology_samples_per_second": 12.77, + "eval_biology_steps_per_second": 3.192, + "step": 500 + }, + { + "epoch": 0.8901746967842439, + "eval_chemistry_entropy": 0.7538458592891694, + "eval_chemistry_loss": 0.8699584007263184, + "eval_chemistry_mean_token_accuracy": 0.7975720787048339, + "eval_chemistry_num_tokens": 13396545.0, + "eval_chemistry_runtime": 48.8229, + "eval_chemistry_samples_per_second": 10.241, + "eval_chemistry_steps_per_second": 2.56, + "step": 500 + }, + { + "epoch": 0.8901746967842439, + "eval_math_entropy": 0.7398368701934814, + "eval_math_loss": 1.0758970975875854, + "eval_math_mean_token_accuracy": 0.7687290472984314, + "eval_math_num_tokens": 13396545.0, + "eval_math_runtime": 50.0916, + "eval_math_samples_per_second": 9.982, + "eval_math_steps_per_second": 2.495, + "step": 500 + }, + { + "epoch": 0.8901746967842439, + "eval_physics_entropy": 0.7607374501228332, + "eval_physics_loss": 0.8848790526390076, + "eval_physics_mean_token_accuracy": 0.7937801394462586, + "eval_physics_num_tokens": 13396545.0, + "eval_physics_runtime": 57.6599, + "eval_physics_samples_per_second": 8.672, + "eval_physics_steps_per_second": 2.168, + "step": 500 + }, + { + "entropy": 0.7129743231460453, + "epoch": 0.9079781907199288, + "grad_norm": 1.1640625, + "learning_rate": 1.8451851851851855e-05, + "loss": 0.8109, + "mean_token_accuracy": 0.8060642663389445, + "num_tokens": 13667857.0, + "step": 510 + }, + { + "entropy": 0.7393450669944286, + "epoch": 0.9257816846556136, + "grad_norm": 1.1484375, + "learning_rate": 1.8377777777777778e-05, + "loss": 0.8325, + "mean_token_accuracy": 0.7997861213982105, + "num_tokens": 13932771.0, + "step": 520 + }, + { + "entropy": 0.7191115407273173, + "epoch": 0.9435851785912985, + "grad_norm": 1.1640625, + "learning_rate": 1.8303703703703704e-05, + "loss": 0.8233, + "mean_token_accuracy": 0.8034567594528198, + "num_tokens": 14194632.0, + "step": 530 + }, + { + "entropy": 0.7434917811304331, + "epoch": 0.9613886725269835, + "grad_norm": 1.1484375, + "learning_rate": 1.822962962962963e-05, + "loss": 0.8435, + "mean_token_accuracy": 0.7969928599894047, + "num_tokens": 14457650.0, + "step": 540 + }, + { + "entropy": 0.7100320702418685, + "epoch": 0.9791921664626683, + "grad_norm": 1.125, + "learning_rate": 1.8155555555555557e-05, + "loss": 0.8149, + "mean_token_accuracy": 0.803739969804883, + "num_tokens": 14724967.0, + "step": 550 + }, + { + "entropy": 0.737933493219316, + "epoch": 0.9969956603983532, + "grad_norm": 1.1875, + "learning_rate": 1.8081481481481484e-05, + "loss": 0.8421, + "mean_token_accuracy": 0.8001800544559956, + "num_tokens": 14993272.0, + "step": 560 + }, + { + "entropy": 0.6655864379098339, + "epoch": 1.014242795148548, + "grad_norm": 1.1171875, + "learning_rate": 1.800740740740741e-05, + "loss": 0.7668, + "mean_token_accuracy": 0.814733551779101, + "num_tokens": 15254936.0, + "step": 570 + }, + { + "entropy": 0.6937626471742988, + "epoch": 1.0320462890842328, + "grad_norm": 1.2578125, + "learning_rate": 1.7933333333333333e-05, + "loss": 0.7942, + "mean_token_accuracy": 0.806296594068408, + "num_tokens": 15515007.0, + "step": 580 + }, + { + "entropy": 0.6596947606652975, + "epoch": 1.0498497830199176, + "grad_norm": 1.2890625, + "learning_rate": 1.785925925925926e-05, + "loss": 0.7623, + "mean_token_accuracy": 0.8152227636426688, + "num_tokens": 15794203.0, + "step": 590 + }, + { + "entropy": 0.6868148937821388, + "epoch": 1.0676532769556026, + "grad_norm": 1.1640625, + "learning_rate": 1.7785185185185186e-05, + "loss": 0.7827, + "mean_token_accuracy": 0.8101296100765467, + "num_tokens": 16063664.0, + "step": 600 + }, + { + "epoch": 1.0676532769556026, + "eval_biology_entropy": 1.6867316131591796, + "eval_biology_loss": 2.077732801437378, + "eval_biology_mean_token_accuracy": 0.6078621828556061, + "eval_biology_num_tokens": 16063664.0, + "eval_biology_runtime": 39.1079, + "eval_biology_samples_per_second": 12.785, + "eval_biology_steps_per_second": 3.196, + "step": 600 + }, + { + "epoch": 1.0676532769556026, + "eval_chemistry_entropy": 0.6882545161247253, + "eval_chemistry_loss": 0.8549924492835999, + "eval_chemistry_mean_token_accuracy": 0.8009334373474121, + "eval_chemistry_num_tokens": 16063664.0, + "eval_chemistry_runtime": 48.732, + "eval_chemistry_samples_per_second": 10.26, + "eval_chemistry_steps_per_second": 2.565, + "step": 600 + }, + { + "epoch": 1.0676532769556026, + "eval_math_entropy": 0.6883157949447631, + "eval_math_loss": 1.075770616531372, + "eval_math_mean_token_accuracy": 0.7705092186927796, + "eval_math_num_tokens": 16063664.0, + "eval_math_runtime": 50.1345, + "eval_math_samples_per_second": 9.973, + "eval_math_steps_per_second": 2.493, + "step": 600 + }, + { + "epoch": 1.0676532769556026, + "eval_physics_entropy": 0.7055722939968109, + "eval_physics_loss": 0.8771266937255859, + "eval_physics_mean_token_accuracy": 0.7955313258171082, + "eval_physics_num_tokens": 16063664.0, + "eval_physics_runtime": 57.6658, + "eval_physics_samples_per_second": 8.671, + "eval_physics_steps_per_second": 2.168, + "step": 600 + }, + { + "entropy": 0.6597639428451657, + "epoch": 1.0854567708912874, + "grad_norm": 1.203125, + "learning_rate": 1.7711111111111113e-05, + "loss": 0.7655, + "mean_token_accuracy": 0.8151189126074314, + "num_tokens": 16338303.0, + "step": 610 + }, + { + "entropy": 0.6991181125864386, + "epoch": 1.1032602648269723, + "grad_norm": 1.2734375, + "learning_rate": 1.763703703703704e-05, + "loss": 0.7934, + "mean_token_accuracy": 0.8071035247296094, + "num_tokens": 16606302.0, + "step": 620 + }, + { + "entropy": 0.6774254593998194, + "epoch": 1.121063758762657, + "grad_norm": 1.28125, + "learning_rate": 1.7562962962962962e-05, + "loss": 0.7706, + "mean_token_accuracy": 0.8098537240177393, + "num_tokens": 16877071.0, + "step": 630 + }, + { + "entropy": 0.6678685389459134, + "epoch": 1.138867252698342, + "grad_norm": 1.203125, + "learning_rate": 1.7488888888888892e-05, + "loss": 0.7714, + "mean_token_accuracy": 0.8127609983086586, + "num_tokens": 17145775.0, + "step": 640 + }, + { + "entropy": 0.6851805068552494, + "epoch": 1.156670746634027, + "grad_norm": 1.2890625, + "learning_rate": 1.7414814814814815e-05, + "loss": 0.777, + "mean_token_accuracy": 0.8110932789742946, + "num_tokens": 17410775.0, + "step": 650 + }, + { + "entropy": 0.7042925998568534, + "epoch": 1.1744742405697117, + "grad_norm": 1.375, + "learning_rate": 1.7340740740740742e-05, + "loss": 0.7979, + "mean_token_accuracy": 0.8038393501192331, + "num_tokens": 17677791.0, + "step": 660 + }, + { + "entropy": 0.6810087833553553, + "epoch": 1.1922777345053968, + "grad_norm": 1.1484375, + "learning_rate": 1.726666666666667e-05, + "loss": 0.7688, + "mean_token_accuracy": 0.8104576140642166, + "num_tokens": 17949441.0, + "step": 670 + }, + { + "entropy": 0.662551742233336, + "epoch": 1.2100812284410816, + "grad_norm": 1.2109375, + "learning_rate": 1.7192592592592595e-05, + "loss": 0.7653, + "mean_token_accuracy": 0.8126230895519256, + "num_tokens": 18217742.0, + "step": 680 + }, + { + "entropy": 0.6803617887198925, + "epoch": 1.2278847223767664, + "grad_norm": 1.2421875, + "learning_rate": 1.711851851851852e-05, + "loss": 0.7771, + "mean_token_accuracy": 0.8095329128205776, + "num_tokens": 18487769.0, + "step": 690 + }, + { + "entropy": 0.6707866100594402, + "epoch": 1.2456882163124514, + "grad_norm": 1.34375, + "learning_rate": 1.7044444444444445e-05, + "loss": 0.7687, + "mean_token_accuracy": 0.8113945126533508, + "num_tokens": 18760296.0, + "step": 700 + }, + { + "epoch": 1.2456882163124514, + "eval_biology_entropy": 1.6589141931533813, + "eval_biology_loss": 2.081825017929077, + "eval_biology_mean_token_accuracy": 0.608892562866211, + "eval_biology_num_tokens": 18760296.0, + "eval_biology_runtime": 39.8083, + "eval_biology_samples_per_second": 12.56, + "eval_biology_steps_per_second": 3.14, + "step": 700 + }, + { + "epoch": 1.2456882163124514, + "eval_chemistry_entropy": 0.6833467047214508, + "eval_chemistry_loss": 0.8464663028717041, + "eval_chemistry_mean_token_accuracy": 0.80278346824646, + "eval_chemistry_num_tokens": 18760296.0, + "eval_chemistry_runtime": 49.1467, + "eval_chemistry_samples_per_second": 10.174, + "eval_chemistry_steps_per_second": 2.543, + "step": 700 + }, + { + "epoch": 1.2456882163124514, + "eval_math_entropy": 0.6947779231071473, + "eval_math_loss": 1.0786137580871582, + "eval_math_mean_token_accuracy": 0.7709644784927369, + "eval_math_num_tokens": 18760296.0, + "eval_math_runtime": 52.5755, + "eval_math_samples_per_second": 9.51, + "eval_math_steps_per_second": 2.378, + "step": 700 + }, + { + "epoch": 1.2456882163124514, + "eval_physics_entropy": 0.704109364271164, + "eval_physics_loss": 0.8724388480186462, + "eval_physics_mean_token_accuracy": 0.796642897605896, + "eval_physics_num_tokens": 18760296.0, + "eval_physics_runtime": 61.3658, + "eval_physics_samples_per_second": 8.148, + "eval_physics_steps_per_second": 2.037, + "step": 700 + }, + { + "entropy": 0.6705936685204505, + "epoch": 1.2634917102481362, + "grad_norm": 1.140625, + "learning_rate": 1.697037037037037e-05, + "loss": 0.7618, + "mean_token_accuracy": 0.8144830856472254, + "num_tokens": 19035733.0, + "step": 710 + }, + { + "entropy": 0.6870324395596981, + "epoch": 1.281295204183821, + "grad_norm": 1.1953125, + "learning_rate": 1.6896296296296298e-05, + "loss": 0.7849, + "mean_token_accuracy": 0.807737946882844, + "num_tokens": 19311432.0, + "step": 720 + }, + { + "entropy": 0.6551721351221204, + "epoch": 1.299098698119506, + "grad_norm": 1.0859375, + "learning_rate": 1.6822222222222224e-05, + "loss": 0.7538, + "mean_token_accuracy": 0.8164622981101275, + "num_tokens": 19593016.0, + "step": 730 + }, + { + "entropy": 0.6645344085991383, + "epoch": 1.3169021920551909, + "grad_norm": 1.1796875, + "learning_rate": 1.6748148148148147e-05, + "loss": 0.759, + "mean_token_accuracy": 0.8134492743760348, + "num_tokens": 19860476.0, + "step": 740 + }, + { + "entropy": 0.6684485200792551, + "epoch": 1.3347056859908757, + "grad_norm": 1.3515625, + "learning_rate": 1.6674074074074077e-05, + "loss": 0.7632, + "mean_token_accuracy": 0.8128790140151978, + "num_tokens": 20124436.0, + "step": 750 + }, + { + "entropy": 0.6736280497163534, + "epoch": 1.3525091799265607, + "grad_norm": 1.25, + "learning_rate": 1.66e-05, + "loss": 0.7771, + "mean_token_accuracy": 0.813029420748353, + "num_tokens": 20388027.0, + "step": 760 + }, + { + "entropy": 0.6706711670383811, + "epoch": 1.3703126738622455, + "grad_norm": 1.265625, + "learning_rate": 1.6525925925925927e-05, + "loss": 0.7697, + "mean_token_accuracy": 0.8115393359214067, + "num_tokens": 20656076.0, + "step": 770 + }, + { + "entropy": 0.6996258163824678, + "epoch": 1.3881161677979303, + "grad_norm": 1.2578125, + "learning_rate": 1.6451851851851853e-05, + "loss": 0.8077, + "mean_token_accuracy": 0.8058754079043865, + "num_tokens": 20911633.0, + "step": 780 + }, + { + "entropy": 0.6905225297436118, + "epoch": 1.4059196617336152, + "grad_norm": 1.171875, + "learning_rate": 1.637777777777778e-05, + "loss": 0.7955, + "mean_token_accuracy": 0.8070559553802014, + "num_tokens": 21176227.0, + "step": 790 + }, + { + "entropy": 0.6718146029859782, + "epoch": 1.4237231556693, + "grad_norm": 1.2890625, + "learning_rate": 1.6303703703703706e-05, + "loss": 0.7716, + "mean_token_accuracy": 0.8126390781253576, + "num_tokens": 21445242.0, + "step": 800 + }, + { + "epoch": 1.4237231556693, + "eval_biology_entropy": 1.6255396161079407, + "eval_biology_loss": 2.0511693954467773, + "eval_biology_mean_token_accuracy": 0.6119143316745758, + "eval_biology_num_tokens": 21445242.0, + "eval_biology_runtime": 42.4445, + "eval_biology_samples_per_second": 11.78, + "eval_biology_steps_per_second": 2.945, + "step": 800 + }, + { + "epoch": 1.4237231556693, + "eval_chemistry_entropy": 0.6763756844997406, + "eval_chemistry_loss": 0.8368268609046936, + "eval_chemistry_mean_token_accuracy": 0.8050474629402161, + "eval_chemistry_num_tokens": 21445242.0, + "eval_chemistry_runtime": 51.5104, + "eval_chemistry_samples_per_second": 9.707, + "eval_chemistry_steps_per_second": 2.427, + "step": 800 + }, + { + "epoch": 1.4237231556693, + "eval_math_entropy": 0.6853804645538331, + "eval_math_loss": 1.0698176622390747, + "eval_math_mean_token_accuracy": 0.7714386968612671, + "eval_math_num_tokens": 21445242.0, + "eval_math_runtime": 54.9388, + "eval_math_samples_per_second": 9.101, + "eval_math_steps_per_second": 2.275, + "step": 800 + }, + { + "epoch": 1.4237231556693, + "eval_physics_entropy": 0.6991692934036254, + "eval_physics_loss": 0.8673914074897766, + "eval_physics_mean_token_accuracy": 0.7979785161018371, + "eval_physics_num_tokens": 21445242.0, + "eval_physics_runtime": 60.9527, + "eval_physics_samples_per_second": 8.203, + "eval_physics_steps_per_second": 2.051, + "step": 800 + }, + { + "entropy": 0.6695453573018313, + "epoch": 1.441526649604985, + "grad_norm": 1.3125, + "learning_rate": 1.622962962962963e-05, + "loss": 0.77, + "mean_token_accuracy": 0.8109262142330408, + "num_tokens": 21718455.0, + "step": 810 + }, + { + "entropy": 0.6753543723374605, + "epoch": 1.4593301435406698, + "grad_norm": 1.2734375, + "learning_rate": 1.6155555555555556e-05, + "loss": 0.7782, + "mean_token_accuracy": 0.8130301378667355, + "num_tokens": 21974076.0, + "step": 820 + }, + { + "entropy": 0.6758085643872619, + "epoch": 1.4771336374763546, + "grad_norm": 1.2265625, + "learning_rate": 1.6081481481481482e-05, + "loss": 0.7768, + "mean_token_accuracy": 0.8110865034162998, + "num_tokens": 22241599.0, + "step": 830 + }, + { + "entropy": 0.6868746858090162, + "epoch": 1.4949371314120397, + "grad_norm": 1.34375, + "learning_rate": 1.600740740740741e-05, + "loss": 0.7818, + "mean_token_accuracy": 0.8090599600225687, + "num_tokens": 22510725.0, + "step": 840 + }, + { + "entropy": 0.6554480012506246, + "epoch": 1.5127406253477245, + "grad_norm": 1.1796875, + "learning_rate": 1.5933333333333336e-05, + "loss": 0.766, + "mean_token_accuracy": 0.8149232871830463, + "num_tokens": 22778514.0, + "step": 850 + }, + { + "entropy": 0.6574800366535782, + "epoch": 1.5305441192834093, + "grad_norm": 1.2734375, + "learning_rate": 1.5859259259259262e-05, + "loss": 0.7517, + "mean_token_accuracy": 0.8147028144448996, + "num_tokens": 23051230.0, + "step": 860 + }, + { + "entropy": 0.6707898294553161, + "epoch": 1.5483476132190943, + "grad_norm": 1.21875, + "learning_rate": 1.5785185185185185e-05, + "loss": 0.776, + "mean_token_accuracy": 0.8121768556535244, + "num_tokens": 23306327.0, + "step": 870 + }, + { + "entropy": 0.6759747974574566, + "epoch": 1.5661511071547791, + "grad_norm": 1.234375, + "learning_rate": 1.571111111111111e-05, + "loss": 0.7717, + "mean_token_accuracy": 0.810706127434969, + "num_tokens": 23573127.0, + "step": 880 + }, + { + "entropy": 0.6696014609187841, + "epoch": 1.583954601090464, + "grad_norm": 1.1953125, + "learning_rate": 1.5637037037037038e-05, + "loss": 0.7635, + "mean_token_accuracy": 0.8123903065919876, + "num_tokens": 23841914.0, + "step": 890 + }, + { + "entropy": 0.6680066948756576, + "epoch": 1.601758095026149, + "grad_norm": 1.28125, + "learning_rate": 1.5562962962962965e-05, + "loss": 0.7763, + "mean_token_accuracy": 0.8117439385503531, + "num_tokens": 24105114.0, + "step": 900 + }, + { + "epoch": 1.601758095026149, + "eval_biology_entropy": 1.6559728245735168, + "eval_biology_loss": 2.0387370586395264, + "eval_biology_mean_token_accuracy": 0.6136598820686341, + "eval_biology_num_tokens": 24105114.0, + "eval_biology_runtime": 41.4479, + "eval_biology_samples_per_second": 12.063, + "eval_biology_steps_per_second": 3.016, + "step": 900 + }, + { + "epoch": 1.601758095026149, + "eval_chemistry_entropy": 0.683923549413681, + "eval_chemistry_loss": 0.8298754692077637, + "eval_chemistry_mean_token_accuracy": 0.8061454033851624, + "eval_chemistry_num_tokens": 24105114.0, + "eval_chemistry_runtime": 50.8984, + "eval_chemistry_samples_per_second": 9.823, + "eval_chemistry_steps_per_second": 2.456, + "step": 900 + }, + { + "epoch": 1.601758095026149, + "eval_math_entropy": 0.6932375190258027, + "eval_math_loss": 1.06475830078125, + "eval_math_mean_token_accuracy": 0.7720690259933471, + "eval_math_num_tokens": 24105114.0, + "eval_math_runtime": 52.5544, + "eval_math_samples_per_second": 9.514, + "eval_math_steps_per_second": 2.378, + "step": 900 + }, + { + "epoch": 1.601758095026149, + "eval_physics_entropy": 0.7051843535900116, + "eval_physics_loss": 0.8616264462471008, + "eval_physics_mean_token_accuracy": 0.7987073979377747, + "eval_physics_num_tokens": 24105114.0, + "eval_physics_runtime": 59.8508, + "eval_physics_samples_per_second": 8.354, + "eval_physics_steps_per_second": 2.089, + "step": 900 + }, + { + "entropy": 0.6430077504366636, + "epoch": 1.6195615889618338, + "grad_norm": 1.1640625, + "learning_rate": 1.548888888888889e-05, + "loss": 0.743, + "mean_token_accuracy": 0.8184590026736259, + "num_tokens": 24384008.0, + "step": 910 + }, + { + "entropy": 0.6775279760360717, + "epoch": 1.6373650828975186, + "grad_norm": 1.3125, + "learning_rate": 1.5414814814814814e-05, + "loss": 0.7704, + "mean_token_accuracy": 0.8114439699798822, + "num_tokens": 24638205.0, + "step": 920 + }, + { + "entropy": 0.6244934551417828, + "epoch": 1.6551685768332036, + "grad_norm": 1.1953125, + "learning_rate": 1.5340740740740744e-05, + "loss": 0.7253, + "mean_token_accuracy": 0.8222480688244105, + "num_tokens": 24910104.0, + "step": 930 + }, + { + "entropy": 0.6539383618161082, + "epoch": 1.6729720707688884, + "grad_norm": 1.2578125, + "learning_rate": 1.5266666666666667e-05, + "loss": 0.7471, + "mean_token_accuracy": 0.8160006206482648, + "num_tokens": 25181306.0, + "step": 940 + }, + { + "entropy": 0.6607510834932327, + "epoch": 1.6907755647045732, + "grad_norm": 1.3203125, + "learning_rate": 1.5192592592592594e-05, + "loss": 0.7589, + "mean_token_accuracy": 0.8142590031027794, + "num_tokens": 25449138.0, + "step": 950 + }, + { + "entropy": 0.6612858925014734, + "epoch": 1.7085790586402583, + "grad_norm": 1.203125, + "learning_rate": 1.5118518518518519e-05, + "loss": 0.766, + "mean_token_accuracy": 0.8137270450592041, + "num_tokens": 25709030.0, + "step": 960 + }, + { + "entropy": 0.6701876953244209, + "epoch": 1.7263825525759429, + "grad_norm": 1.2734375, + "learning_rate": 1.5044444444444445e-05, + "loss": 0.7702, + "mean_token_accuracy": 0.8120660580694675, + "num_tokens": 25972807.0, + "step": 970 + }, + { + "entropy": 0.6566708590835333, + "epoch": 1.744186046511628, + "grad_norm": 1.1640625, + "learning_rate": 1.497037037037037e-05, + "loss": 0.748, + "mean_token_accuracy": 0.8170768674463034, + "num_tokens": 26244955.0, + "step": 980 + }, + { + "entropy": 0.624184824153781, + "epoch": 1.761989540447313, + "grad_norm": 1.1796875, + "learning_rate": 1.4896296296296298e-05, + "loss": 0.7284, + "mean_token_accuracy": 0.8223610106855631, + "num_tokens": 26515548.0, + "step": 990 + }, + { + "entropy": 0.6476762825623155, + "epoch": 1.7797930343829975, + "grad_norm": 1.21875, + "learning_rate": 1.4822222222222225e-05, + "loss": 0.7422, + "mean_token_accuracy": 0.8172390758991241, + "num_tokens": 26782376.0, + "step": 1000 + }, + { + "epoch": 1.7797930343829975, + "eval_biology_entropy": 1.6157320532798767, + "eval_biology_loss": 2.0338852405548096, + "eval_biology_mean_token_accuracy": 0.6143801593780518, + "eval_biology_num_tokens": 26782376.0, + "eval_biology_runtime": 41.5849, + "eval_biology_samples_per_second": 12.024, + "eval_biology_steps_per_second": 3.006, + "step": 1000 + }, + { + "epoch": 1.7797930343829975, + "eval_chemistry_entropy": 0.6670743160247803, + "eval_chemistry_loss": 0.8244208097457886, + "eval_chemistry_mean_token_accuracy": 0.8077364330291747, + "eval_chemistry_num_tokens": 26782376.0, + "eval_chemistry_runtime": 53.0274, + "eval_chemistry_samples_per_second": 9.429, + "eval_chemistry_steps_per_second": 2.357, + "step": 1000 + }, + { + "epoch": 1.7797930343829975, + "eval_math_entropy": 0.6760952143669129, + "eval_math_loss": 1.0642753839492798, + "eval_math_mean_token_accuracy": 0.7728619589805603, + "eval_math_num_tokens": 26782376.0, + "eval_math_runtime": 53.8997, + "eval_math_samples_per_second": 9.276, + "eval_math_steps_per_second": 2.319, + "step": 1000 + }, + { + "epoch": 1.7797930343829975, + "eval_physics_entropy": 0.6914605097770691, + "eval_physics_loss": 0.8597292304039001, + "eval_physics_mean_token_accuracy": 0.7992245469093323, + "eval_physics_num_tokens": 26782376.0, + "eval_physics_runtime": 62.8355, + "eval_physics_samples_per_second": 7.957, + "eval_physics_steps_per_second": 1.989, + "step": 1000 + }, + { + "entropy": 0.660948914475739, + "epoch": 1.7975965283186826, + "grad_norm": 1.359375, + "learning_rate": 1.474814814814815e-05, + "loss": 0.7562, + "mean_token_accuracy": 0.8152279850095511, + "num_tokens": 27049648.0, + "step": 1010 + }, + { + "entropy": 0.6787248050794006, + "epoch": 1.8154000222543676, + "grad_norm": 1.3359375, + "learning_rate": 1.4674074074074076e-05, + "loss": 0.7771, + "mean_token_accuracy": 0.8101395227015018, + "num_tokens": 27315061.0, + "step": 1020 + }, + { + "entropy": 0.6424054227769375, + "epoch": 1.8332035161900522, + "grad_norm": 1.359375, + "learning_rate": 1.46e-05, + "loss": 0.7373, + "mean_token_accuracy": 0.8189616728574037, + "num_tokens": 27587650.0, + "step": 1030 + }, + { + "entropy": 0.6606515791267157, + "epoch": 1.8510070101257372, + "grad_norm": 1.15625, + "learning_rate": 1.4525925925925927e-05, + "loss": 0.7652, + "mean_token_accuracy": 0.8157406080514192, + "num_tokens": 27855458.0, + "step": 1040 + }, + { + "entropy": 0.6464399367570877, + "epoch": 1.868810504061422, + "grad_norm": 1.203125, + "learning_rate": 1.4451851851851852e-05, + "loss": 0.7464, + "mean_token_accuracy": 0.8191553246229887, + "num_tokens": 28127703.0, + "step": 1050 + }, + { + "entropy": 0.633565224148333, + "epoch": 1.8866139979971068, + "grad_norm": 1.140625, + "learning_rate": 1.4377777777777779e-05, + "loss": 0.7265, + "mean_token_accuracy": 0.8215953864157199, + "num_tokens": 28395128.0, + "step": 1060 + }, + { + "entropy": 0.651407066360116, + "epoch": 1.9044174919327919, + "grad_norm": 1.2421875, + "learning_rate": 1.4303703703703703e-05, + "loss": 0.752, + "mean_token_accuracy": 0.8161330189555883, + "num_tokens": 28662344.0, + "step": 1070 + }, + { + "entropy": 0.6537632567808032, + "epoch": 1.9222209858684767, + "grad_norm": 1.3125, + "learning_rate": 1.4229629629629632e-05, + "loss": 0.7551, + "mean_token_accuracy": 0.8160409614443779, + "num_tokens": 28928769.0, + "step": 1080 + }, + { + "entropy": 0.6610247550532222, + "epoch": 1.9400244798041615, + "grad_norm": 1.5625, + "learning_rate": 1.4155555555555556e-05, + "loss": 0.7599, + "mean_token_accuracy": 0.8147491872310638, + "num_tokens": 29186214.0, + "step": 1090 + }, + { + "entropy": 0.6632727082818747, + "epoch": 1.9578279737398465, + "grad_norm": 1.2109375, + "learning_rate": 1.4081481481481483e-05, + "loss": 0.7706, + "mean_token_accuracy": 0.8126637618988752, + "num_tokens": 29446445.0, + "step": 1100 + }, + { + "epoch": 1.9578279737398465, + "eval_biology_entropy": 1.639104124546051, + "eval_biology_loss": 2.0316033363342285, + "eval_biology_mean_token_accuracy": 0.615292895078659, + "eval_biology_num_tokens": 29446445.0, + "eval_biology_runtime": 41.3569, + "eval_biology_samples_per_second": 12.09, + "eval_biology_steps_per_second": 3.022, + "step": 1100 + }, + { + "epoch": 1.9578279737398465, + "eval_chemistry_entropy": 0.6696053767204284, + "eval_chemistry_loss": 0.8191399574279785, + "eval_chemistry_mean_token_accuracy": 0.8085824422836304, + "eval_chemistry_num_tokens": 29446445.0, + "eval_chemistry_runtime": 50.7996, + "eval_chemistry_samples_per_second": 9.843, + "eval_chemistry_steps_per_second": 2.461, + "step": 1100 + }, + { + "epoch": 1.9578279737398465, + "eval_math_entropy": 0.6799165282249451, + "eval_math_loss": 1.0633617639541626, + "eval_math_mean_token_accuracy": 0.7729020061492919, + "eval_math_num_tokens": 29446445.0, + "eval_math_runtime": 52.3664, + "eval_math_samples_per_second": 9.548, + "eval_math_steps_per_second": 2.387, + "step": 1100 + }, + { + "epoch": 1.9578279737398465, + "eval_physics_entropy": 0.6961899037361146, + "eval_physics_loss": 0.8565983176231384, + "eval_physics_mean_token_accuracy": 0.7995611958503723, + "eval_physics_num_tokens": 29446445.0, + "eval_physics_runtime": 59.795, + "eval_physics_samples_per_second": 8.362, + "eval_physics_steps_per_second": 2.09, + "step": 1100 + }, + { + "entropy": 0.6516633328050375, + "epoch": 1.9756314676755313, + "grad_norm": 1.21875, + "learning_rate": 1.400740740740741e-05, + "loss": 0.7439, + "mean_token_accuracy": 0.8187774941325188, + "num_tokens": 29711771.0, + "step": 1110 + }, + { + "entropy": 0.6508602360263467, + "epoch": 1.9934349616112161, + "grad_norm": 1.3125, + "learning_rate": 1.3933333333333334e-05, + "loss": 0.7519, + "mean_token_accuracy": 0.8169589649885893, + "num_tokens": 29979994.0, + "step": 1120 + }, + { + "entropy": 0.6306788632946629, + "epoch": 2.010682096361411, + "grad_norm": 1.2734375, + "learning_rate": 1.385925925925926e-05, + "loss": 0.7224, + "mean_token_accuracy": 0.8239769354943306, + "num_tokens": 30240793.0, + "step": 1130 + }, + { + "entropy": 0.5911945894360542, + "epoch": 2.028485590297096, + "grad_norm": 1.296875, + "learning_rate": 1.3785185185185186e-05, + "loss": 0.7, + "mean_token_accuracy": 0.8281658995896578, + "num_tokens": 30503988.0, + "step": 1140 + }, + { + "entropy": 0.6057484762743115, + "epoch": 2.0462890842327806, + "grad_norm": 1.2890625, + "learning_rate": 1.3711111111111112e-05, + "loss": 0.7031, + "mean_token_accuracy": 0.8266915556043386, + "num_tokens": 30771841.0, + "step": 1150 + }, + { + "entropy": 0.596652439981699, + "epoch": 2.0640925781684656, + "grad_norm": 1.375, + "learning_rate": 1.3637037037037037e-05, + "loss": 0.6926, + "mean_token_accuracy": 0.8293926887214184, + "num_tokens": 31045037.0, + "step": 1160 + }, + { + "entropy": 0.6352694857865572, + "epoch": 2.0818960721041506, + "grad_norm": 1.4296875, + "learning_rate": 1.3562962962962965e-05, + "loss": 0.7356, + "mean_token_accuracy": 0.8186026088893413, + "num_tokens": 31299575.0, + "step": 1170 + }, + { + "entropy": 0.6292696729302406, + "epoch": 2.099699566039835, + "grad_norm": 1.390625, + "learning_rate": 1.3488888888888888e-05, + "loss": 0.7273, + "mean_token_accuracy": 0.8201477468013764, + "num_tokens": 31565963.0, + "step": 1180 + }, + { + "entropy": 0.5940109083428979, + "epoch": 2.1175030599755202, + "grad_norm": 1.2734375, + "learning_rate": 1.3414814814814817e-05, + "loss": 0.7035, + "mean_token_accuracy": 0.8278756473213434, + "num_tokens": 31834212.0, + "step": 1190 + }, + { + "entropy": 0.6235881226137281, + "epoch": 2.1353065539112053, + "grad_norm": 1.375, + "learning_rate": 1.3340740740740741e-05, + "loss": 0.7172, + "mean_token_accuracy": 0.8223393179476262, + "num_tokens": 32103356.0, + "step": 1200 + }, + { + "epoch": 2.1353065539112053, + "eval_biology_entropy": 1.4677531628608704, + "eval_biology_loss": 2.083951234817505, + "eval_biology_mean_token_accuracy": 0.6125459330081939, + "eval_biology_num_tokens": 32103356.0, + "eval_biology_runtime": 41.2722, + "eval_biology_samples_per_second": 12.115, + "eval_biology_steps_per_second": 3.029, + "step": 1200 + }, + { + "epoch": 2.1353065539112053, + "eval_chemistry_entropy": 0.6140689237117767, + "eval_chemistry_loss": 0.8232262134552002, + "eval_chemistry_mean_token_accuracy": 0.8088321657180786, + "eval_chemistry_num_tokens": 32103356.0, + "eval_chemistry_runtime": 50.8871, + "eval_chemistry_samples_per_second": 9.826, + "eval_chemistry_steps_per_second": 2.456, + "step": 1200 + }, + { + "epoch": 2.1353065539112053, + "eval_math_entropy": 0.6312986519336701, + "eval_math_loss": 1.0854209661483765, + "eval_math_mean_token_accuracy": 0.7729385108947754, + "eval_math_num_tokens": 32103356.0, + "eval_math_runtime": 52.4669, + "eval_math_samples_per_second": 9.53, + "eval_math_steps_per_second": 2.382, + "step": 1200 + }, + { + "epoch": 2.1353065539112053, + "eval_physics_entropy": 0.6365497968196869, + "eval_physics_loss": 0.8704355359077454, + "eval_physics_mean_token_accuracy": 0.7990298428535462, + "eval_physics_num_tokens": 32103356.0, + "eval_physics_runtime": 59.7275, + "eval_physics_samples_per_second": 8.371, + "eval_physics_steps_per_second": 2.093, + "step": 1200 + }, + { + "entropy": 0.6137512635439635, + "epoch": 2.15311004784689, + "grad_norm": 1.25, + "learning_rate": 1.3266666666666668e-05, + "loss": 0.7043, + "mean_token_accuracy": 0.8242961999028922, + "num_tokens": 32374830.0, + "step": 1210 + }, + { + "entropy": 0.5896952305920422, + "epoch": 2.170913541782575, + "grad_norm": 1.2734375, + "learning_rate": 1.3192592592592594e-05, + "loss": 0.6922, + "mean_token_accuracy": 0.8302250444889069, + "num_tokens": 32643479.0, + "step": 1220 + }, + { + "entropy": 0.5944842832162976, + "epoch": 2.18871703571826, + "grad_norm": 1.296875, + "learning_rate": 1.311851851851852e-05, + "loss": 0.6959, + "mean_token_accuracy": 0.8287800919264555, + "num_tokens": 32908761.0, + "step": 1230 + }, + { + "entropy": 0.6240290224552154, + "epoch": 2.2065205296539445, + "grad_norm": 1.21875, + "learning_rate": 1.3044444444444446e-05, + "loss": 0.7238, + "mean_token_accuracy": 0.8242842797189951, + "num_tokens": 33166413.0, + "step": 1240 + }, + { + "entropy": 0.6112470507621766, + "epoch": 2.2243240235896296, + "grad_norm": 1.1875, + "learning_rate": 1.297037037037037e-05, + "loss": 0.7149, + "mean_token_accuracy": 0.8240214943885803, + "num_tokens": 33432686.0, + "step": 1250 + }, + { + "entropy": 0.6079921290278435, + "epoch": 2.242127517525314, + "grad_norm": 1.4296875, + "learning_rate": 1.2896296296296299e-05, + "loss": 0.7131, + "mean_token_accuracy": 0.8254695866256953, + "num_tokens": 33693242.0, + "step": 1260 + }, + { + "entropy": 0.5799209821969271, + "epoch": 2.259931011460999, + "grad_norm": 1.390625, + "learning_rate": 1.2822222222222222e-05, + "loss": 0.6722, + "mean_token_accuracy": 0.834067041426897, + "num_tokens": 33970820.0, + "step": 1270 + }, + { + "entropy": 0.594209510087967, + "epoch": 2.277734505396684, + "grad_norm": 1.3671875, + "learning_rate": 1.274814814814815e-05, + "loss": 0.6977, + "mean_token_accuracy": 0.8276573233306408, + "num_tokens": 34243042.0, + "step": 1280 + }, + { + "entropy": 0.6097929472103715, + "epoch": 2.295537999332369, + "grad_norm": 1.296875, + "learning_rate": 1.2674074074074075e-05, + "loss": 0.705, + "mean_token_accuracy": 0.8267240136861801, + "num_tokens": 34515236.0, + "step": 1290 + }, + { + "entropy": 0.5745832217857242, + "epoch": 2.313341493268054, + "grad_norm": 1.234375, + "learning_rate": 1.2600000000000001e-05, + "loss": 0.6783, + "mean_token_accuracy": 0.8345583423972129, + "num_tokens": 34785068.0, + "step": 1300 + }, + { + "epoch": 2.313341493268054, + "eval_biology_entropy": 1.497672739982605, + "eval_biology_loss": 2.0853865146636963, + "eval_biology_mean_token_accuracy": 0.613484436750412, + "eval_biology_num_tokens": 34785068.0, + "eval_biology_runtime": 43.469, + "eval_biology_samples_per_second": 11.502, + "eval_biology_steps_per_second": 2.876, + "step": 1300 + }, + { + "epoch": 2.313341493268054, + "eval_chemistry_entropy": 0.6273940706253052, + "eval_chemistry_loss": 0.8201014995574951, + "eval_chemistry_mean_token_accuracy": 0.8088535733222961, + "eval_chemistry_num_tokens": 34785068.0, + "eval_chemistry_runtime": 52.831, + "eval_chemistry_samples_per_second": 9.464, + "eval_chemistry_steps_per_second": 2.366, + "step": 1300 + }, + { + "epoch": 2.313341493268054, + "eval_math_entropy": 0.646281590461731, + "eval_math_loss": 1.0807055234909058, + "eval_math_mean_token_accuracy": 0.772442822933197, + "eval_math_num_tokens": 34785068.0, + "eval_math_runtime": 55.4339, + "eval_math_samples_per_second": 9.02, + "eval_math_steps_per_second": 2.255, + "step": 1300 + }, + { + "epoch": 2.313341493268054, + "eval_physics_entropy": 0.6572135488986969, + "eval_physics_loss": 0.8676888346672058, + "eval_physics_mean_token_accuracy": 0.7990684523582459, + "eval_physics_num_tokens": 34785068.0, + "eval_physics_runtime": 61.4064, + "eval_physics_samples_per_second": 8.142, + "eval_physics_steps_per_second": 2.036, + "step": 1300 + }, + { + "entropy": 0.6191223243251442, + "epoch": 2.331144987203739, + "grad_norm": 1.2265625, + "learning_rate": 1.2525925925925928e-05, + "loss": 0.7122, + "mean_token_accuracy": 0.8234442934393883, + "num_tokens": 35055845.0, + "step": 1310 + }, + { + "entropy": 0.6000411529093981, + "epoch": 2.3489484811394234, + "grad_norm": 1.296875, + "learning_rate": 1.2451851851851853e-05, + "loss": 0.7023, + "mean_token_accuracy": 0.8272519435733556, + "num_tokens": 35327481.0, + "step": 1320 + }, + { + "entropy": 0.6216201702132821, + "epoch": 2.3667519750751085, + "grad_norm": 1.2578125, + "learning_rate": 1.237777777777778e-05, + "loss": 0.7178, + "mean_token_accuracy": 0.8234907556325197, + "num_tokens": 35596362.0, + "step": 1330 + }, + { + "entropy": 0.5952032478526235, + "epoch": 2.3845554690107935, + "grad_norm": 1.3671875, + "learning_rate": 1.2303703703703704e-05, + "loss": 0.695, + "mean_token_accuracy": 0.827592471987009, + "num_tokens": 35869585.0, + "step": 1340 + }, + { + "entropy": 0.6152689423412084, + "epoch": 2.402358962946478, + "grad_norm": 1.4140625, + "learning_rate": 1.222962962962963e-05, + "loss": 0.7087, + "mean_token_accuracy": 0.8254514247179031, + "num_tokens": 36134072.0, + "step": 1350 + }, + { + "entropy": 0.5919087225571275, + "epoch": 2.420162456882163, + "grad_norm": 1.25, + "learning_rate": 1.2155555555555555e-05, + "loss": 0.6971, + "mean_token_accuracy": 0.829882950335741, + "num_tokens": 36406263.0, + "step": 1360 + }, + { + "entropy": 0.6080563787370921, + "epoch": 2.437965950817848, + "grad_norm": 1.3984375, + "learning_rate": 1.2081481481481484e-05, + "loss": 0.7027, + "mean_token_accuracy": 0.8268357224762439, + "num_tokens": 36676822.0, + "step": 1370 + }, + { + "entropy": 0.6048327511176467, + "epoch": 2.4557694447535328, + "grad_norm": 1.421875, + "learning_rate": 1.2007407407407408e-05, + "loss": 0.7047, + "mean_token_accuracy": 0.8271109748631715, + "num_tokens": 36942837.0, + "step": 1380 + }, + { + "entropy": 0.5988074177876115, + "epoch": 2.473572938689218, + "grad_norm": 1.390625, + "learning_rate": 1.1933333333333335e-05, + "loss": 0.7051, + "mean_token_accuracy": 0.8256583914160729, + "num_tokens": 37210791.0, + "step": 1390 + }, + { + "entropy": 0.5873929725959897, + "epoch": 2.491376432624903, + "grad_norm": 1.3671875, + "learning_rate": 1.185925925925926e-05, + "loss": 0.6871, + "mean_token_accuracy": 0.83187070004642, + "num_tokens": 37485822.0, + "step": 1400 + }, + { + "epoch": 2.491376432624903, + "eval_biology_entropy": 1.4652894883155823, + "eval_biology_loss": 2.099928855895996, + "eval_biology_mean_token_accuracy": 0.6130364212989807, + "eval_biology_num_tokens": 37485822.0, + "eval_biology_runtime": 43.3395, + "eval_biology_samples_per_second": 11.537, + "eval_biology_steps_per_second": 2.884, + "step": 1400 + }, + { + "epoch": 2.491376432624903, + "eval_chemistry_entropy": 0.6137497539520264, + "eval_chemistry_loss": 0.8189122080802917, + "eval_chemistry_mean_token_accuracy": 0.8097302551269532, + "eval_chemistry_num_tokens": 37485822.0, + "eval_chemistry_runtime": 52.903, + "eval_chemistry_samples_per_second": 9.451, + "eval_chemistry_steps_per_second": 2.363, + "step": 1400 + }, + { + "epoch": 2.491376432624903, + "eval_math_entropy": 0.6390214171409607, + "eval_math_loss": 1.082951545715332, + "eval_math_mean_token_accuracy": 0.772076274394989, + "eval_math_num_tokens": 37485822.0, + "eval_math_runtime": 54.1184, + "eval_math_samples_per_second": 9.239, + "eval_math_steps_per_second": 2.31, + "step": 1400 + }, + { + "epoch": 2.491376432624903, + "eval_physics_entropy": 0.6406208200454712, + "eval_physics_loss": 0.8697341084480286, + "eval_physics_mean_token_accuracy": 0.7989166221618652, + "eval_physics_num_tokens": 37485822.0, + "eval_physics_runtime": 61.5427, + "eval_physics_samples_per_second": 8.124, + "eval_physics_steps_per_second": 2.031, + "step": 1400 + }, + { + "entropy": 0.6010861510410905, + "epoch": 2.5091799265605874, + "grad_norm": 1.28125, + "learning_rate": 1.1785185185185186e-05, + "loss": 0.7014, + "mean_token_accuracy": 0.8287522595375776, + "num_tokens": 37746150.0, + "step": 1410 + }, + { + "entropy": 0.5981267247349024, + "epoch": 2.5269834204962724, + "grad_norm": 1.3046875, + "learning_rate": 1.1711111111111113e-05, + "loss": 0.6962, + "mean_token_accuracy": 0.8269948281347752, + "num_tokens": 38016863.0, + "step": 1420 + }, + { + "entropy": 0.6277626631781459, + "epoch": 2.544786914431957, + "grad_norm": 1.5078125, + "learning_rate": 1.1637037037037037e-05, + "loss": 0.7222, + "mean_token_accuracy": 0.8205803319811821, + "num_tokens": 38277754.0, + "step": 1430 + }, + { + "entropy": 0.6118797251954675, + "epoch": 2.562590408367642, + "grad_norm": 1.421875, + "learning_rate": 1.1562962962962964e-05, + "loss": 0.7172, + "mean_token_accuracy": 0.8237183388322592, + "num_tokens": 38542761.0, + "step": 1440 + }, + { + "entropy": 0.6034180456772447, + "epoch": 2.580393902303327, + "grad_norm": 1.453125, + "learning_rate": 1.1488888888888889e-05, + "loss": 0.6941, + "mean_token_accuracy": 0.8272098813205957, + "num_tokens": 38820999.0, + "step": 1450 + }, + { + "entropy": 0.5998265566304326, + "epoch": 2.598197396239012, + "grad_norm": 1.3203125, + "learning_rate": 1.1414814814814817e-05, + "loss": 0.7114, + "mean_token_accuracy": 0.8267886329442262, + "num_tokens": 39080016.0, + "step": 1460 + }, + { + "entropy": 0.6005334880203008, + "epoch": 2.6160008901746967, + "grad_norm": 1.3828125, + "learning_rate": 1.1340740740740742e-05, + "loss": 0.6949, + "mean_token_accuracy": 0.8299154710024595, + "num_tokens": 39345434.0, + "step": 1470 + }, + { + "entropy": 0.6003408120945096, + "epoch": 2.6338043841103818, + "grad_norm": 1.46875, + "learning_rate": 1.1266666666666668e-05, + "loss": 0.7033, + "mean_token_accuracy": 0.8266183461993932, + "num_tokens": 39610041.0, + "step": 1480 + }, + { + "entropy": 0.5847593653947115, + "epoch": 2.6516078780460663, + "grad_norm": 1.4140625, + "learning_rate": 1.1192592592592593e-05, + "loss": 0.6785, + "mean_token_accuracy": 0.8323042623698711, + "num_tokens": 39884185.0, + "step": 1490 + }, + { + "entropy": 0.6105206938460469, + "epoch": 2.6694113719817514, + "grad_norm": 1.5546875, + "learning_rate": 1.111851851851852e-05, + "loss": 0.7201, + "mean_token_accuracy": 0.8220306947827339, + "num_tokens": 40143543.0, + "step": 1500 + }, + { + "epoch": 2.6694113719817514, + "eval_biology_entropy": 1.490638756752014, + "eval_biology_loss": 2.0750932693481445, + "eval_biology_mean_token_accuracy": 0.6143683531284332, + "eval_biology_num_tokens": 40143543.0, + "eval_biology_runtime": 43.2253, + "eval_biology_samples_per_second": 11.567, + "eval_biology_steps_per_second": 2.892, + "step": 1500 + }, + { + "epoch": 2.6694113719817514, + "eval_chemistry_entropy": 0.623803097486496, + "eval_chemistry_loss": 0.8160935044288635, + "eval_chemistry_mean_token_accuracy": 0.8098534879684448, + "eval_chemistry_num_tokens": 40143543.0, + "eval_chemistry_runtime": 50.807, + "eval_chemistry_samples_per_second": 9.841, + "eval_chemistry_steps_per_second": 2.46, + "step": 1500 + }, + { + "epoch": 2.6694113719817514, + "eval_math_entropy": 0.645468202829361, + "eval_math_loss": 1.0840471982955933, + "eval_math_mean_token_accuracy": 0.7726303563117981, + "eval_math_num_tokens": 40143543.0, + "eval_math_runtime": 53.0054, + "eval_math_samples_per_second": 9.433, + "eval_math_steps_per_second": 2.358, + "step": 1500 + }, + { + "epoch": 2.6694113719817514, + "eval_physics_entropy": 0.6492172837257385, + "eval_physics_loss": 0.8670127391815186, + "eval_physics_mean_token_accuracy": 0.7991857075691223, + "eval_physics_num_tokens": 40143543.0, + "eval_physics_runtime": 63.9696, + "eval_physics_samples_per_second": 7.816, + "eval_physics_steps_per_second": 1.954, + "step": 1500 + }, + { + "entropy": 0.6046247070655226, + "epoch": 2.6872148659174364, + "grad_norm": 1.390625, + "learning_rate": 1.1044444444444444e-05, + "loss": 0.7007, + "mean_token_accuracy": 0.828262010216713, + "num_tokens": 40406812.0, + "step": 1510 + }, + { + "entropy": 0.6191225880756974, + "epoch": 2.7050183598531214, + "grad_norm": 1.4140625, + "learning_rate": 1.0970370370370371e-05, + "loss": 0.7269, + "mean_token_accuracy": 0.8213932007551193, + "num_tokens": 40668543.0, + "step": 1520 + }, + { + "entropy": 0.6065074443817139, + "epoch": 2.722821853788806, + "grad_norm": 1.3515625, + "learning_rate": 1.0896296296296298e-05, + "loss": 0.6989, + "mean_token_accuracy": 0.8265019495040178, + "num_tokens": 40942603.0, + "step": 1530 + }, + { + "entropy": 0.6138417391106487, + "epoch": 2.740625347724491, + "grad_norm": 1.34375, + "learning_rate": 1.0822222222222222e-05, + "loss": 0.7057, + "mean_token_accuracy": 0.8261400748044252, + "num_tokens": 41208093.0, + "step": 1540 + }, + { + "entropy": 0.5985930340364576, + "epoch": 2.7584288416601757, + "grad_norm": 1.3125, + "learning_rate": 1.074814814814815e-05, + "loss": 0.7061, + "mean_token_accuracy": 0.8252453569322824, + "num_tokens": 41477562.0, + "step": 1550 + }, + { + "entropy": 0.5930986102670431, + "epoch": 2.7762323355958607, + "grad_norm": 1.5234375, + "learning_rate": 1.0674074074074074e-05, + "loss": 0.6935, + "mean_token_accuracy": 0.8292474631220103, + "num_tokens": 41746571.0, + "step": 1560 + }, + { + "entropy": 0.6006103545427323, + "epoch": 2.7940358295315457, + "grad_norm": 1.4765625, + "learning_rate": 1.0600000000000002e-05, + "loss": 0.7025, + "mean_token_accuracy": 0.8271383382380009, + "num_tokens": 42011633.0, + "step": 1570 + }, + { + "entropy": 0.6129128225147724, + "epoch": 2.8118393234672303, + "grad_norm": 1.328125, + "learning_rate": 1.0525925925925927e-05, + "loss": 0.7076, + "mean_token_accuracy": 0.8252990577369929, + "num_tokens": 42283087.0, + "step": 1580 + }, + { + "entropy": 0.5756963772699237, + "epoch": 2.8296428174029153, + "grad_norm": 1.1953125, + "learning_rate": 1.0451851851851853e-05, + "loss": 0.6725, + "mean_token_accuracy": 0.8329845789819956, + "num_tokens": 42561534.0, + "step": 1590 + }, + { + "entropy": 0.6044865250587463, + "epoch": 2.8474463113386, + "grad_norm": 1.296875, + "learning_rate": 1.0377777777777778e-05, + "loss": 0.705, + "mean_token_accuracy": 0.8281082704663276, + "num_tokens": 42834266.0, + "step": 1600 + }, + { + "epoch": 2.8474463113386, + "eval_biology_entropy": 1.4930514478683472, + "eval_biology_loss": 2.0736353397369385, + "eval_biology_mean_token_accuracy": 0.6145954511165619, + "eval_biology_num_tokens": 42834266.0, + "eval_biology_runtime": 40.9171, + "eval_biology_samples_per_second": 12.22, + "eval_biology_steps_per_second": 3.055, + "step": 1600 + }, + { + "epoch": 2.8474463113386, + "eval_chemistry_entropy": 0.6211945106983184, + "eval_chemistry_loss": 0.8145589828491211, + "eval_chemistry_mean_token_accuracy": 0.810048574924469, + "eval_chemistry_num_tokens": 42834266.0, + "eval_chemistry_runtime": 50.7089, + "eval_chemistry_samples_per_second": 9.86, + "eval_chemistry_steps_per_second": 2.465, + "step": 1600 + }, + { + "epoch": 2.8474463113386, + "eval_math_entropy": 0.6438178873062134, + "eval_math_loss": 1.0831515789031982, + "eval_math_mean_token_accuracy": 0.772301673412323, + "eval_math_num_tokens": 42834266.0, + "eval_math_runtime": 51.7388, + "eval_math_samples_per_second": 9.664, + "eval_math_steps_per_second": 2.416, + "step": 1600 + }, + { + "epoch": 2.8474463113386, + "eval_physics_entropy": 0.649389057636261, + "eval_physics_loss": 0.86627197265625, + "eval_physics_mean_token_accuracy": 0.7995732502937317, + "eval_physics_num_tokens": 42834266.0, + "eval_physics_runtime": 59.549, + "eval_physics_samples_per_second": 8.396, + "eval_physics_steps_per_second": 2.099, + "step": 1600 + }, + { + "entropy": 0.625230243243277, + "epoch": 2.865249805274285, + "grad_norm": 1.46875, + "learning_rate": 1.0303703703703705e-05, + "loss": 0.7228, + "mean_token_accuracy": 0.8225024774670601, + "num_tokens": 43100513.0, + "step": 1610 + }, + { + "entropy": 0.605365008302033, + "epoch": 2.88305329920997, + "grad_norm": 1.359375, + "learning_rate": 1.0229629629629631e-05, + "loss": 0.698, + "mean_token_accuracy": 0.8268981877714395, + "num_tokens": 43375632.0, + "step": 1620 + }, + { + "entropy": 0.5911450985819101, + "epoch": 2.900856793145655, + "grad_norm": 1.3515625, + "learning_rate": 1.0155555555555556e-05, + "loss": 0.6958, + "mean_token_accuracy": 0.829645611718297, + "num_tokens": 43644341.0, + "step": 1630 + }, + { + "entropy": 0.5920769222080707, + "epoch": 2.9186602870813396, + "grad_norm": 1.390625, + "learning_rate": 1.0081481481481484e-05, + "loss": 0.6833, + "mean_token_accuracy": 0.8301208417862653, + "num_tokens": 43917580.0, + "step": 1640 + }, + { + "entropy": 0.5912358593195677, + "epoch": 2.9364637810170247, + "grad_norm": 1.359375, + "learning_rate": 1.0007407407407407e-05, + "loss": 0.6989, + "mean_token_accuracy": 0.8305816825479269, + "num_tokens": 44177325.0, + "step": 1650 + }, + { + "entropy": 0.6072682781144977, + "epoch": 2.9542672749527092, + "grad_norm": 1.3984375, + "learning_rate": 9.933333333333334e-06, + "loss": 0.7079, + "mean_token_accuracy": 0.8247454788535833, + "num_tokens": 44442947.0, + "step": 1660 + }, + { + "entropy": 0.6202316217124462, + "epoch": 2.9720707688883943, + "grad_norm": 1.484375, + "learning_rate": 9.85925925925926e-06, + "loss": 0.7181, + "mean_token_accuracy": 0.8235241327434778, + "num_tokens": 44700601.0, + "step": 1670 + }, + { + "entropy": 0.5890740664675832, + "epoch": 2.9898742628240793, + "grad_norm": 1.3125, + "learning_rate": 9.785185185185187e-06, + "loss": 0.6933, + "mean_token_accuracy": 0.8307324200868607, + "num_tokens": 44971479.0, + "step": 1680 + }, + { + "entropy": 0.587316531712009, + "epoch": 3.007121397574274, + "grad_norm": 1.265625, + "learning_rate": 9.711111111111111e-06, + "loss": 0.6682, + "mean_token_accuracy": 0.8340173459822132, + "num_tokens": 45231796.0, + "step": 1690 + }, + { + "entropy": 0.6013597913086415, + "epoch": 3.0249248915099587, + "grad_norm": 1.5, + "learning_rate": 9.637037037037038e-06, + "loss": 0.7048, + "mean_token_accuracy": 0.8241627678275109, + "num_tokens": 45500317.0, + "step": 1700 + }, + { + "epoch": 3.0249248915099587, + "eval_biology_entropy": 1.4193837118148804, + "eval_biology_loss": 2.11480712890625, + "eval_biology_mean_token_accuracy": 0.6124376230239869, + "eval_biology_num_tokens": 45500317.0, + "eval_biology_runtime": 40.3875, + "eval_biology_samples_per_second": 12.38, + "eval_biology_steps_per_second": 3.095, + "step": 1700 + }, + { + "epoch": 3.0249248915099587, + "eval_chemistry_entropy": 0.5911917226314545, + "eval_chemistry_loss": 0.8190613985061646, + "eval_chemistry_mean_token_accuracy": 0.8099429054260254, + "eval_chemistry_num_tokens": 45500317.0, + "eval_chemistry_runtime": 50.2786, + "eval_chemistry_samples_per_second": 9.945, + "eval_chemistry_steps_per_second": 2.486, + "step": 1700 + }, + { + "epoch": 3.0249248915099587, + "eval_math_entropy": 0.6176868407726288, + "eval_math_loss": 1.0993149280548096, + "eval_math_mean_token_accuracy": 0.7718941388130188, + "eval_math_num_tokens": 45500317.0, + "eval_math_runtime": 51.8195, + "eval_math_samples_per_second": 9.649, + "eval_math_steps_per_second": 2.412, + "step": 1700 + }, + { + "epoch": 3.0249248915099587, + "eval_physics_entropy": 0.6201707119941712, + "eval_physics_loss": 0.8766935467720032, + "eval_physics_mean_token_accuracy": 0.7985301322937012, + "eval_physics_num_tokens": 45500317.0, + "eval_physics_runtime": 59.4252, + "eval_physics_samples_per_second": 8.414, + "eval_physics_steps_per_second": 2.103, + "step": 1700 + }, + { + "entropy": 0.5692525265738368, + "epoch": 3.0427283854456437, + "grad_norm": 1.4140625, + "learning_rate": 9.562962962962965e-06, + "loss": 0.6591, + "mean_token_accuracy": 0.8374071232974529, + "num_tokens": 45762680.0, + "step": 1710 + }, + { + "entropy": 0.5559542520903051, + "epoch": 3.0605318793813288, + "grad_norm": 1.625, + "learning_rate": 9.48888888888889e-06, + "loss": 0.6597, + "mean_token_accuracy": 0.8373042322695256, + "num_tokens": 46029923.0, + "step": 1720 + }, + { + "entropy": 0.5692304980009795, + "epoch": 3.0783353733170133, + "grad_norm": 1.328125, + "learning_rate": 9.414814814814816e-06, + "loss": 0.6629, + "mean_token_accuracy": 0.8344687603414058, + "num_tokens": 46305156.0, + "step": 1730 + }, + { + "entropy": 0.5508972069248557, + "epoch": 3.0961388672526984, + "grad_norm": 1.484375, + "learning_rate": 9.34074074074074e-06, + "loss": 0.6501, + "mean_token_accuracy": 0.8386828090995551, + "num_tokens": 46582874.0, + "step": 1740 + }, + { + "entropy": 0.5854626100510358, + "epoch": 3.1139423611883834, + "grad_norm": 1.640625, + "learning_rate": 9.266666666666667e-06, + "loss": 0.6824, + "mean_token_accuracy": 0.8301110483705998, + "num_tokens": 46850175.0, + "step": 1750 + }, + { + "entropy": 0.5591677505522966, + "epoch": 3.131745855124068, + "grad_norm": 1.3984375, + "learning_rate": 9.192592592592594e-06, + "loss": 0.6564, + "mean_token_accuracy": 0.8384724948555231, + "num_tokens": 47125899.0, + "step": 1760 + }, + { + "entropy": 0.5725911458954215, + "epoch": 3.149549349059753, + "grad_norm": 1.546875, + "learning_rate": 9.118518518518518e-06, + "loss": 0.6752, + "mean_token_accuracy": 0.8343269538134337, + "num_tokens": 47390194.0, + "step": 1770 + }, + { + "entropy": 0.5758154878392816, + "epoch": 3.167352842995438, + "grad_norm": 1.4140625, + "learning_rate": 9.044444444444445e-06, + "loss": 0.6738, + "mean_token_accuracy": 0.8336336623877287, + "num_tokens": 47654784.0, + "step": 1780 + }, + { + "entropy": 0.5612063113600015, + "epoch": 3.1851563369311227, + "grad_norm": 1.53125, + "learning_rate": 8.970370370370372e-06, + "loss": 0.668, + "mean_token_accuracy": 0.8369329560548067, + "num_tokens": 47919400.0, + "step": 1790 + }, + { + "entropy": 0.5750826884061098, + "epoch": 3.2029598308668077, + "grad_norm": 1.5625, + "learning_rate": 8.896296296296298e-06, + "loss": 0.6761, + "mean_token_accuracy": 0.8343805599957704, + "num_tokens": 48187796.0, + "step": 1800 + }, + { + "epoch": 3.2029598308668077, + "eval_biology_entropy": 1.4034225873947144, + "eval_biology_loss": 2.1275665760040283, + "eval_biology_mean_token_accuracy": 0.6119487595558166, + "eval_biology_num_tokens": 48187796.0, + "eval_biology_runtime": 40.734, + "eval_biology_samples_per_second": 12.275, + "eval_biology_steps_per_second": 3.069, + "step": 1800 + }, + { + "epoch": 3.2029598308668077, + "eval_chemistry_entropy": 0.5955355260372162, + "eval_chemistry_loss": 0.8196625709533691, + "eval_chemistry_mean_token_accuracy": 0.8102530264854431, + "eval_chemistry_num_tokens": 48187796.0, + "eval_chemistry_runtime": 50.2147, + "eval_chemistry_samples_per_second": 9.957, + "eval_chemistry_steps_per_second": 2.489, + "step": 1800 + }, + { + "epoch": 3.2029598308668077, + "eval_math_entropy": 0.6184592893123627, + "eval_math_loss": 1.1008187532424927, + "eval_math_mean_token_accuracy": 0.7713511042594909, + "eval_math_num_tokens": 48187796.0, + "eval_math_runtime": 51.7678, + "eval_math_samples_per_second": 9.659, + "eval_math_steps_per_second": 2.415, + "step": 1800 + }, + { + "epoch": 3.2029598308668077, + "eval_physics_entropy": 0.6223777911663055, + "eval_physics_loss": 0.8791524767875671, + "eval_physics_mean_token_accuracy": 0.7979751496315003, + "eval_physics_num_tokens": 48187796.0, + "eval_physics_runtime": 59.0178, + "eval_physics_samples_per_second": 8.472, + "eval_physics_steps_per_second": 2.118, + "step": 1800 + }, + { + "entropy": 0.5663457477465272, + "epoch": 3.2207633248024923, + "grad_norm": 1.3828125, + "learning_rate": 8.822222222222223e-06, + "loss": 0.666, + "mean_token_accuracy": 0.8364688046276569, + "num_tokens": 48457428.0, + "step": 1810 + }, + { + "entropy": 0.6113261075690388, + "epoch": 3.2385668187381773, + "grad_norm": 1.515625, + "learning_rate": 8.74814814814815e-06, + "loss": 0.704, + "mean_token_accuracy": 0.8250963229686021, + "num_tokens": 48716417.0, + "step": 1820 + }, + { + "entropy": 0.5722766594961286, + "epoch": 3.2563703126738623, + "grad_norm": 1.4296875, + "learning_rate": 8.674074074074074e-06, + "loss": 0.6724, + "mean_token_accuracy": 0.8353833951056003, + "num_tokens": 48977306.0, + "step": 1830 + }, + { + "entropy": 0.5715915774926543, + "epoch": 3.274173806609547, + "grad_norm": 1.3359375, + "learning_rate": 8.6e-06, + "loss": 0.6663, + "mean_token_accuracy": 0.8353964131325483, + "num_tokens": 49249667.0, + "step": 1840 + }, + { + "entropy": 0.5664192967116832, + "epoch": 3.291977300545232, + "grad_norm": 1.3828125, + "learning_rate": 8.525925925925927e-06, + "loss": 0.6619, + "mean_token_accuracy": 0.8355418112128973, + "num_tokens": 49520900.0, + "step": 1850 + }, + { + "entropy": 0.5784041214734316, + "epoch": 3.309780794480917, + "grad_norm": 1.484375, + "learning_rate": 8.451851851851852e-06, + "loss": 0.6816, + "mean_token_accuracy": 0.8331022556871176, + "num_tokens": 49786411.0, + "step": 1860 + }, + { + "entropy": 0.5709506403654814, + "epoch": 3.3275842884166016, + "grad_norm": 1.3984375, + "learning_rate": 8.377777777777779e-06, + "loss": 0.6589, + "mean_token_accuracy": 0.8369544640183448, + "num_tokens": 50056207.0, + "step": 1870 + }, + { + "entropy": 0.5751367569901049, + "epoch": 3.3453877823522866, + "grad_norm": 1.390625, + "learning_rate": 8.303703703703705e-06, + "loss": 0.6735, + "mean_token_accuracy": 0.8336179967969656, + "num_tokens": 50330876.0, + "step": 1880 + }, + { + "entropy": 0.577986160852015, + "epoch": 3.3631912762879717, + "grad_norm": 1.6015625, + "learning_rate": 8.229629629629632e-06, + "loss": 0.6774, + "mean_token_accuracy": 0.8319189596921206, + "num_tokens": 50602004.0, + "step": 1890 + }, + { + "entropy": 0.5784968495368957, + "epoch": 3.3809947702236562, + "grad_norm": 1.4140625, + "learning_rate": 8.155555555555556e-06, + "loss": 0.6728, + "mean_token_accuracy": 0.8320122357457876, + "num_tokens": 50873444.0, + "step": 1900 + }, + { + "epoch": 3.3809947702236562, + "eval_biology_entropy": 1.379322585105896, + "eval_biology_loss": 2.1383986473083496, + "eval_biology_mean_token_accuracy": 0.6115301356315613, + "eval_biology_num_tokens": 50873444.0, + "eval_biology_runtime": 40.4711, + "eval_biology_samples_per_second": 12.354, + "eval_biology_steps_per_second": 3.089, + "step": 1900 + }, + { + "epoch": 3.3809947702236562, + "eval_chemistry_entropy": 0.5861142361164093, + "eval_chemistry_loss": 0.8205327391624451, + "eval_chemistry_mean_token_accuracy": 0.8100807156562805, + "eval_chemistry_num_tokens": 50873444.0, + "eval_chemistry_runtime": 50.1133, + "eval_chemistry_samples_per_second": 9.977, + "eval_chemistry_steps_per_second": 2.494, + "step": 1900 + }, + { + "epoch": 3.3809947702236562, + "eval_math_entropy": 0.6114957261085511, + "eval_math_loss": 1.1055864095687866, + "eval_math_mean_token_accuracy": 0.7713269739151001, + "eval_math_num_tokens": 50873444.0, + "eval_math_runtime": 51.5676, + "eval_math_samples_per_second": 9.696, + "eval_math_steps_per_second": 2.424, + "step": 1900 + }, + { + "epoch": 3.3809947702236562, + "eval_physics_entropy": 0.6135163457393646, + "eval_physics_loss": 0.8826585412025452, + "eval_physics_mean_token_accuracy": 0.7979225769042969, + "eval_physics_num_tokens": 50873444.0, + "eval_physics_runtime": 58.9496, + "eval_physics_samples_per_second": 8.482, + "eval_physics_steps_per_second": 2.12, + "step": 1900 + }, + { + "entropy": 0.5896684370934964, + "epoch": 3.3987982641593413, + "grad_norm": 1.5234375, + "learning_rate": 8.081481481481483e-06, + "loss": 0.6862, + "mean_token_accuracy": 0.8290154490619898, + "num_tokens": 51141077.0, + "step": 1910 + }, + { + "entropy": 0.5699383469298482, + "epoch": 3.4166017580950263, + "grad_norm": 1.578125, + "learning_rate": 8.007407407407408e-06, + "loss": 0.6705, + "mean_token_accuracy": 0.8345434635877609, + "num_tokens": 51406483.0, + "step": 1920 + }, + { + "entropy": 0.5792136957868934, + "epoch": 3.434405252030711, + "grad_norm": 1.546875, + "learning_rate": 7.933333333333334e-06, + "loss": 0.6753, + "mean_token_accuracy": 0.8337912186980247, + "num_tokens": 51668458.0, + "step": 1930 + }, + { + "entropy": 0.5839371480047703, + "epoch": 3.452208745966396, + "grad_norm": 1.53125, + "learning_rate": 7.859259259259259e-06, + "loss": 0.6891, + "mean_token_accuracy": 0.8307471171021461, + "num_tokens": 51928199.0, + "step": 1940 + }, + { + "entropy": 0.574542698264122, + "epoch": 3.470012239902081, + "grad_norm": 1.4140625, + "learning_rate": 7.785185185185185e-06, + "loss": 0.6653, + "mean_token_accuracy": 0.8337355487048626, + "num_tokens": 52199244.0, + "step": 1950 + }, + { + "entropy": 0.5655553802847862, + "epoch": 3.4878157338377656, + "grad_norm": 1.5078125, + "learning_rate": 7.711111111111112e-06, + "loss": 0.6606, + "mean_token_accuracy": 0.8357041519135237, + "num_tokens": 52473476.0, + "step": 1960 + }, + { + "entropy": 0.5800750594586134, + "epoch": 3.5056192277734506, + "grad_norm": 1.28125, + "learning_rate": 7.637037037037037e-06, + "loss": 0.6777, + "mean_token_accuracy": 0.832566126063466, + "num_tokens": 52743275.0, + "step": 1970 + }, + { + "entropy": 0.5716741790995001, + "epoch": 3.523422721709135, + "grad_norm": 1.4375, + "learning_rate": 7.562962962962963e-06, + "loss": 0.6637, + "mean_token_accuracy": 0.8349632371217013, + "num_tokens": 53014827.0, + "step": 1980 + }, + { + "entropy": 0.5780819065868854, + "epoch": 3.54122621564482, + "grad_norm": 1.5859375, + "learning_rate": 7.48888888888889e-06, + "loss": 0.6791, + "mean_token_accuracy": 0.8327229652553797, + "num_tokens": 53285577.0, + "step": 1990 + }, + { + "entropy": 0.5571762975305319, + "epoch": 3.5590297095805052, + "grad_norm": 1.5, + "learning_rate": 7.4148148148148155e-06, + "loss": 0.656, + "mean_token_accuracy": 0.8386137288063764, + "num_tokens": 53553391.0, + "step": 2000 + }, + { + "epoch": 3.5590297095805052, + "eval_biology_entropy": 1.381414409160614, + "eval_biology_loss": 2.13814115524292, + "eval_biology_mean_token_accuracy": 0.6120853550434112, + "eval_biology_num_tokens": 53553391.0, + "eval_biology_runtime": 40.2984, + "eval_biology_samples_per_second": 12.407, + "eval_biology_steps_per_second": 3.102, + "step": 2000 + }, + { + "epoch": 3.5590297095805052, + "eval_chemistry_entropy": 0.5889808633327485, + "eval_chemistry_loss": 0.8193114995956421, + "eval_chemistry_mean_token_accuracy": 0.8103628730773926, + "eval_chemistry_num_tokens": 53553391.0, + "eval_chemistry_runtime": 49.8517, + "eval_chemistry_samples_per_second": 10.03, + "eval_chemistry_steps_per_second": 2.507, + "step": 2000 + }, + { + "epoch": 3.5590297095805052, + "eval_math_entropy": 0.6149049291610718, + "eval_math_loss": 1.1041083335876465, + "eval_math_mean_token_accuracy": 0.7715086970329285, + "eval_math_num_tokens": 53553391.0, + "eval_math_runtime": 51.0164, + "eval_math_samples_per_second": 9.801, + "eval_math_steps_per_second": 2.45, + "step": 2000 + }, + { + "epoch": 3.5590297095805052, + "eval_physics_entropy": 0.6177053947448731, + "eval_physics_loss": 0.8811244368553162, + "eval_physics_mean_token_accuracy": 0.798056589126587, + "eval_physics_num_tokens": 53553391.0, + "eval_physics_runtime": 58.0264, + "eval_physics_samples_per_second": 8.617, + "eval_physics_steps_per_second": 2.154, + "step": 2000 + }, + { + "entropy": 0.5717996397987009, + "epoch": 3.5768332035161903, + "grad_norm": 1.4453125, + "learning_rate": 7.340740740740742e-06, + "loss": 0.6683, + "mean_token_accuracy": 0.835101555287838, + "num_tokens": 53826074.0, + "step": 2010 + }, + { + "entropy": 0.5765499664470554, + "epoch": 3.594636697451875, + "grad_norm": 1.5546875, + "learning_rate": 7.266666666666668e-06, + "loss": 0.6758, + "mean_token_accuracy": 0.833604721352458, + "num_tokens": 54090804.0, + "step": 2020 + }, + { + "entropy": 0.5620788568630815, + "epoch": 3.61244019138756, + "grad_norm": 1.4921875, + "learning_rate": 7.192592592592593e-06, + "loss": 0.6652, + "mean_token_accuracy": 0.8382882464677095, + "num_tokens": 54350633.0, + "step": 2030 + }, + { + "entropy": 0.5829729141667486, + "epoch": 3.6302436853232445, + "grad_norm": 1.3671875, + "learning_rate": 7.118518518518519e-06, + "loss": 0.6832, + "mean_token_accuracy": 0.8307051323354244, + "num_tokens": 54623090.0, + "step": 2040 + }, + { + "entropy": 0.5580741995945573, + "epoch": 3.6480471792589295, + "grad_norm": 1.4921875, + "learning_rate": 7.044444444444445e-06, + "loss": 0.6611, + "mean_token_accuracy": 0.8379048384726048, + "num_tokens": 54891961.0, + "step": 2050 + }, + { + "entropy": 0.5711165275424719, + "epoch": 3.6658506731946146, + "grad_norm": 1.3828125, + "learning_rate": 6.97037037037037e-06, + "loss": 0.6686, + "mean_token_accuracy": 0.83506432287395, + "num_tokens": 55160443.0, + "step": 2060 + }, + { + "entropy": 0.5829119691625237, + "epoch": 3.6836541671302996, + "grad_norm": 1.5546875, + "learning_rate": 6.896296296296297e-06, + "loss": 0.679, + "mean_token_accuracy": 0.8321164496243, + "num_tokens": 55425148.0, + "step": 2070 + }, + { + "entropy": 0.577633217908442, + "epoch": 3.701457661065984, + "grad_norm": 1.5078125, + "learning_rate": 6.8222222222222225e-06, + "loss": 0.6756, + "mean_token_accuracy": 0.831652557849884, + "num_tokens": 55701877.0, + "step": 2080 + }, + { + "entropy": 0.5851919570937753, + "epoch": 3.719261155001669, + "grad_norm": 1.4375, + "learning_rate": 6.748148148148149e-06, + "loss": 0.6871, + "mean_token_accuracy": 0.8311607643961907, + "num_tokens": 55969756.0, + "step": 2090 + }, + { + "entropy": 0.5728656422346831, + "epoch": 3.737064648937354, + "grad_norm": 1.3828125, + "learning_rate": 6.674074074074075e-06, + "loss": 0.6697, + "mean_token_accuracy": 0.8338924027979374, + "num_tokens": 56238576.0, + "step": 2100 + }, + { + "epoch": 3.737064648937354, + "eval_biology_entropy": 1.3676475839614868, + "eval_biology_loss": 2.1407225131988525, + "eval_biology_mean_token_accuracy": 0.6122846744060516, + "eval_biology_num_tokens": 56238576.0, + "eval_biology_runtime": 39.0497, + "eval_biology_samples_per_second": 12.804, + "eval_biology_steps_per_second": 3.201, + "step": 2100 + }, + { + "epoch": 3.737064648937354, + "eval_chemistry_entropy": 0.5876963279247284, + "eval_chemistry_loss": 0.8179986476898193, + "eval_chemistry_mean_token_accuracy": 0.8108155965805054, + "eval_chemistry_num_tokens": 56238576.0, + "eval_chemistry_runtime": 48.6816, + "eval_chemistry_samples_per_second": 10.271, + "eval_chemistry_steps_per_second": 2.568, + "step": 2100 + }, + { + "epoch": 3.737064648937354, + "eval_math_entropy": 0.6119424107074738, + "eval_math_loss": 1.103035569190979, + "eval_math_mean_token_accuracy": 0.7718833708763122, + "eval_math_num_tokens": 56238576.0, + "eval_math_runtime": 50.0797, + "eval_math_samples_per_second": 9.984, + "eval_math_steps_per_second": 2.496, + "step": 2100 + }, + { + "epoch": 3.737064648937354, + "eval_physics_entropy": 0.6151737835407257, + "eval_physics_loss": 0.8803703784942627, + "eval_physics_mean_token_accuracy": 0.7982921867370606, + "eval_physics_num_tokens": 56238576.0, + "eval_physics_runtime": 57.6104, + "eval_physics_samples_per_second": 8.679, + "eval_physics_steps_per_second": 2.17, + "step": 2100 + }, + { + "entropy": 0.575149905309081, + "epoch": 3.754868142873039, + "grad_norm": 1.5546875, + "learning_rate": 6.600000000000001e-06, + "loss": 0.6827, + "mean_token_accuracy": 0.8325412474572659, + "num_tokens": 56497905.0, + "step": 2110 + }, + { + "entropy": 0.5853163762018084, + "epoch": 3.772671636808724, + "grad_norm": 1.6796875, + "learning_rate": 6.525925925925927e-06, + "loss": 0.6848, + "mean_token_accuracy": 0.8329160042107105, + "num_tokens": 56757407.0, + "step": 2120 + }, + { + "entropy": 0.5622592825442553, + "epoch": 3.7904751307444084, + "grad_norm": 1.453125, + "learning_rate": 6.4518518518518525e-06, + "loss": 0.666, + "mean_token_accuracy": 0.8372485417872667, + "num_tokens": 57019213.0, + "step": 2130 + }, + { + "entropy": 0.5716784641146659, + "epoch": 3.8082786246800935, + "grad_norm": 1.390625, + "learning_rate": 6.377777777777778e-06, + "loss": 0.6785, + "mean_token_accuracy": 0.8330885298550129, + "num_tokens": 57274629.0, + "step": 2140 + }, + { + "entropy": 0.5762781355530023, + "epoch": 3.826082118615778, + "grad_norm": 1.4375, + "learning_rate": 6.303703703703704e-06, + "loss": 0.6635, + "mean_token_accuracy": 0.8338523168116808, + "num_tokens": 57549791.0, + "step": 2150 + }, + { + "entropy": 0.5586860096082091, + "epoch": 3.843885612551463, + "grad_norm": 1.5703125, + "learning_rate": 6.2296296296296295e-06, + "loss": 0.6637, + "mean_token_accuracy": 0.837235826253891, + "num_tokens": 57815454.0, + "step": 2160 + }, + { + "entropy": 0.5845056502148509, + "epoch": 3.861689106487148, + "grad_norm": 1.4296875, + "learning_rate": 6.155555555555556e-06, + "loss": 0.6887, + "mean_token_accuracy": 0.8297582499682903, + "num_tokens": 58076430.0, + "step": 2170 + }, + { + "entropy": 0.5741686699911952, + "epoch": 3.879492600422833, + "grad_norm": 1.609375, + "learning_rate": 6.081481481481482e-06, + "loss": 0.6757, + "mean_token_accuracy": 0.8332955792546273, + "num_tokens": 58346307.0, + "step": 2180 + }, + { + "entropy": 0.587618819437921, + "epoch": 3.8972960943585178, + "grad_norm": 1.4609375, + "learning_rate": 6.007407407407407e-06, + "loss": 0.6811, + "mean_token_accuracy": 0.8313716314733028, + "num_tokens": 58621624.0, + "step": 2190 + }, + { + "entropy": 0.5831481296569109, + "epoch": 3.915099588294203, + "grad_norm": 1.5, + "learning_rate": 5.933333333333335e-06, + "loss": 0.6864, + "mean_token_accuracy": 0.8321379527449608, + "num_tokens": 58874383.0, + "step": 2200 + }, + { + "epoch": 3.915099588294203, + "eval_biology_entropy": 1.3810388627052308, + "eval_biology_loss": 2.1356613636016846, + "eval_biology_mean_token_accuracy": 0.6122371079921722, + "eval_biology_num_tokens": 58874383.0, + "eval_biology_runtime": 39.0687, + "eval_biology_samples_per_second": 12.798, + "eval_biology_steps_per_second": 3.199, + "step": 2200 + }, + { + "epoch": 3.915099588294203, + "eval_chemistry_entropy": 0.5890640773773194, + "eval_chemistry_loss": 0.8181027770042419, + "eval_chemistry_mean_token_accuracy": 0.8106128220558166, + "eval_chemistry_num_tokens": 58874383.0, + "eval_chemistry_runtime": 48.6446, + "eval_chemistry_samples_per_second": 10.279, + "eval_chemistry_steps_per_second": 2.57, + "step": 2200 + }, + { + "epoch": 3.915099588294203, + "eval_math_entropy": 0.6159807667732239, + "eval_math_loss": 1.1035534143447876, + "eval_math_mean_token_accuracy": 0.7718168897628784, + "eval_math_num_tokens": 58874383.0, + "eval_math_runtime": 50.0657, + "eval_math_samples_per_second": 9.987, + "eval_math_steps_per_second": 2.497, + "step": 2200 + }, + { + "epoch": 3.915099588294203, + "eval_physics_entropy": 0.6170708584785461, + "eval_physics_loss": 0.8807304501533508, + "eval_physics_mean_token_accuracy": 0.7982254939079285, + "eval_physics_num_tokens": 58874383.0, + "eval_physics_runtime": 57.5593, + "eval_physics_samples_per_second": 8.687, + "eval_physics_steps_per_second": 2.172, + "step": 2200 + }, + { + "entropy": 0.5722213637083768, + "epoch": 3.9329030822298874, + "grad_norm": 1.3359375, + "learning_rate": 5.85925925925926e-06, + "loss": 0.669, + "mean_token_accuracy": 0.8334877517074346, + "num_tokens": 59149972.0, + "step": 2210 + }, + { + "entropy": 0.5695188788697123, + "epoch": 3.9507065761655724, + "grad_norm": 1.3828125, + "learning_rate": 5.785185185185186e-06, + "loss": 0.6716, + "mean_token_accuracy": 0.8339714191854, + "num_tokens": 59418591.0, + "step": 2220 + }, + { + "entropy": 0.58336979188025, + "epoch": 3.9685100701012574, + "grad_norm": 1.609375, + "learning_rate": 5.711111111111112e-06, + "loss": 0.6787, + "mean_token_accuracy": 0.8313135091215372, + "num_tokens": 59691322.0, + "step": 2230 + }, + { + "entropy": 0.5696559147909284, + "epoch": 3.9863135640369425, + "grad_norm": 1.625, + "learning_rate": 5.637037037037037e-06, + "loss": 0.674, + "mean_token_accuracy": 0.835492142662406, + "num_tokens": 59953838.0, + "step": 2240 + }, + { + "entropy": 0.5570171371583016, + "epoch": 4.003560698787137, + "grad_norm": 1.34375, + "learning_rate": 5.562962962962963e-06, + "loss": 0.6546, + "mean_token_accuracy": 0.8391922777698886, + "num_tokens": 60216049.0, + "step": 2250 + }, + { + "entropy": 0.5453685067594052, + "epoch": 4.021364192722822, + "grad_norm": 1.3515625, + "learning_rate": 5.4888888888888895e-06, + "loss": 0.6385, + "mean_token_accuracy": 0.8419853840023279, + "num_tokens": 60490937.0, + "step": 2260 + }, + { + "entropy": 0.553492389805615, + "epoch": 4.039167686658507, + "grad_norm": 1.453125, + "learning_rate": 5.414814814814815e-06, + "loss": 0.6556, + "mean_token_accuracy": 0.8393857058137655, + "num_tokens": 60755892.0, + "step": 2270 + }, + { + "entropy": 0.5440751701593399, + "epoch": 4.056971180594192, + "grad_norm": 1.4765625, + "learning_rate": 5.340740740740741e-06, + "loss": 0.6398, + "mean_token_accuracy": 0.8416271641850471, + "num_tokens": 61030893.0, + "step": 2280 + }, + { + "entropy": 0.5338836884126067, + "epoch": 4.074774674529876, + "grad_norm": 1.4453125, + "learning_rate": 5.2666666666666665e-06, + "loss": 0.6349, + "mean_token_accuracy": 0.844103604927659, + "num_tokens": 61301458.0, + "step": 2290 + }, + { + "entropy": 0.5652395199984313, + "epoch": 4.092578168465561, + "grad_norm": 1.5078125, + "learning_rate": 5.192592592592594e-06, + "loss": 0.6572, + "mean_token_accuracy": 0.8368313021957874, + "num_tokens": 61574717.0, + "step": 2300 + }, + { + "epoch": 4.092578168465561, + "eval_biology_entropy": 1.3518374223709106, + "eval_biology_loss": 2.1629726886749268, + "eval_biology_mean_token_accuracy": 0.6107748429775238, + "eval_biology_num_tokens": 61574717.0, + "eval_biology_runtime": 39.0562, + "eval_biology_samples_per_second": 12.802, + "eval_biology_steps_per_second": 3.201, + "step": 2300 + }, + { + "epoch": 4.092578168465561, + "eval_chemistry_entropy": 0.5780029859542847, + "eval_chemistry_loss": 0.8214347958564758, + "eval_chemistry_mean_token_accuracy": 0.810426944732666, + "eval_chemistry_num_tokens": 61574717.0, + "eval_chemistry_runtime": 48.6785, + "eval_chemistry_samples_per_second": 10.271, + "eval_chemistry_steps_per_second": 2.568, + "step": 2300 + }, + { + "epoch": 4.092578168465561, + "eval_math_entropy": 0.6076318190097809, + "eval_math_loss": 1.1111681461334229, + "eval_math_mean_token_accuracy": 0.7712901134490967, + "eval_math_num_tokens": 61574717.0, + "eval_math_runtime": 50.0691, + "eval_math_samples_per_second": 9.986, + "eval_math_steps_per_second": 2.497, + "step": 2300 + }, + { + "epoch": 4.092578168465561, + "eval_physics_entropy": 0.6077080953121186, + "eval_physics_loss": 0.8865928053855896, + "eval_physics_mean_token_accuracy": 0.7976819982528687, + "eval_physics_num_tokens": 61574717.0, + "eval_physics_runtime": 57.5982, + "eval_physics_samples_per_second": 8.681, + "eval_physics_steps_per_second": 2.17, + "step": 2300 + }, + { + "entropy": 0.5665840411558747, + "epoch": 4.110381662401246, + "grad_norm": 1.5859375, + "learning_rate": 5.1185185185185195e-06, + "loss": 0.6708, + "mean_token_accuracy": 0.8358031015843153, + "num_tokens": 61840186.0, + "step": 2310 + }, + { + "entropy": 0.56264589689672, + "epoch": 4.128185156336931, + "grad_norm": 1.453125, + "learning_rate": 5.044444444444445e-06, + "loss": 0.6631, + "mean_token_accuracy": 0.836660572513938, + "num_tokens": 62112844.0, + "step": 2320 + }, + { + "entropy": 0.5638300092890859, + "epoch": 4.145988650272616, + "grad_norm": 1.4296875, + "learning_rate": 4.970370370370371e-06, + "loss": 0.6511, + "mean_token_accuracy": 0.8376240566372871, + "num_tokens": 62382390.0, + "step": 2330 + }, + { + "entropy": 0.5563945654779673, + "epoch": 4.163792144208301, + "grad_norm": 1.3203125, + "learning_rate": 4.8962962962962965e-06, + "loss": 0.6578, + "mean_token_accuracy": 0.8388708371669054, + "num_tokens": 62658026.0, + "step": 2340 + }, + { + "entropy": 0.5789911892265082, + "epoch": 4.181595638143985, + "grad_norm": 1.5078125, + "learning_rate": 4.822222222222222e-06, + "loss": 0.6856, + "mean_token_accuracy": 0.8329568482935429, + "num_tokens": 62916724.0, + "step": 2350 + }, + { + "entropy": 0.555764663591981, + "epoch": 4.19939913207967, + "grad_norm": 1.484375, + "learning_rate": 4.748148148148149e-06, + "loss": 0.6465, + "mean_token_accuracy": 0.8380315884947777, + "num_tokens": 63195271.0, + "step": 2360 + }, + { + "entropy": 0.5678263012319803, + "epoch": 4.2172026260153554, + "grad_norm": 1.453125, + "learning_rate": 4.674074074074074e-06, + "loss": 0.6671, + "mean_token_accuracy": 0.8358955714851618, + "num_tokens": 63466066.0, + "step": 2370 + }, + { + "entropy": 0.554285722412169, + "epoch": 4.2350061199510405, + "grad_norm": 1.4921875, + "learning_rate": 4.600000000000001e-06, + "loss": 0.6573, + "mean_token_accuracy": 0.838936285302043, + "num_tokens": 63739678.0, + "step": 2380 + }, + { + "entropy": 0.5789531769230962, + "epoch": 4.2528096138867255, + "grad_norm": 1.4765625, + "learning_rate": 4.5259259259259265e-06, + "loss": 0.6723, + "mean_token_accuracy": 0.833799697458744, + "num_tokens": 63999364.0, + "step": 2390 + }, + { + "entropy": 0.5556379873305559, + "epoch": 4.2706131078224105, + "grad_norm": 1.4609375, + "learning_rate": 4.451851851851852e-06, + "loss": 0.6557, + "mean_token_accuracy": 0.8391667053103447, + "num_tokens": 64270501.0, + "step": 2400 + }, + { + "epoch": 4.2706131078224105, + "eval_biology_entropy": 1.3456227674484253, + "eval_biology_loss": 2.162630558013916, + "eval_biology_mean_token_accuracy": 0.6108367409706116, + "eval_biology_num_tokens": 64270501.0, + "eval_biology_runtime": 39.0723, + "eval_biology_samples_per_second": 12.797, + "eval_biology_steps_per_second": 3.199, + "step": 2400 + }, + { + "epoch": 4.2706131078224105, + "eval_chemistry_entropy": 0.5770703101158142, + "eval_chemistry_loss": 0.8213785290718079, + "eval_chemistry_mean_token_accuracy": 0.8106174335479737, + "eval_chemistry_num_tokens": 64270501.0, + "eval_chemistry_runtime": 48.65, + "eval_chemistry_samples_per_second": 10.278, + "eval_chemistry_steps_per_second": 2.569, + "step": 2400 + }, + { + "epoch": 4.2706131078224105, + "eval_math_entropy": 0.6059090893268585, + "eval_math_loss": 1.110784888267517, + "eval_math_mean_token_accuracy": 0.771416862487793, + "eval_math_num_tokens": 64270501.0, + "eval_math_runtime": 50.0542, + "eval_math_samples_per_second": 9.989, + "eval_math_steps_per_second": 2.497, + "step": 2400 + }, + { + "epoch": 4.2706131078224105, + "eval_physics_entropy": 0.6061555581092835, + "eval_physics_loss": 0.8872570991516113, + "eval_physics_mean_token_accuracy": 0.797800799369812, + "eval_physics_num_tokens": 64270501.0, + "eval_physics_runtime": 57.5677, + "eval_physics_samples_per_second": 8.685, + "eval_physics_steps_per_second": 2.171, + "step": 2400 + }, + { + "entropy": 0.5603191582486033, + "epoch": 4.288416601758095, + "grad_norm": 1.640625, + "learning_rate": 4.377777777777778e-06, + "loss": 0.6653, + "mean_token_accuracy": 0.8375619545578956, + "num_tokens": 64533223.0, + "step": 2410 + }, + { + "entropy": 0.5656454924494028, + "epoch": 4.30622009569378, + "grad_norm": 1.515625, + "learning_rate": 4.3037037037037035e-06, + "loss": 0.659, + "mean_token_accuracy": 0.8366425581276417, + "num_tokens": 64799113.0, + "step": 2420 + }, + { + "entropy": 0.5550693856552243, + "epoch": 4.324023589629465, + "grad_norm": 1.625, + "learning_rate": 4.22962962962963e-06, + "loss": 0.6538, + "mean_token_accuracy": 0.8389651108533144, + "num_tokens": 65075459.0, + "step": 2430 + }, + { + "entropy": 0.5567959282547236, + "epoch": 4.34182708356515, + "grad_norm": 1.640625, + "learning_rate": 4.155555555555556e-06, + "loss": 0.66, + "mean_token_accuracy": 0.837913642078638, + "num_tokens": 65341896.0, + "step": 2440 + }, + { + "entropy": 0.5572125539183617, + "epoch": 4.359630577500835, + "grad_norm": 1.3828125, + "learning_rate": 4.081481481481482e-06, + "loss": 0.6482, + "mean_token_accuracy": 0.837642902880907, + "num_tokens": 65614487.0, + "step": 2450 + }, + { + "entropy": 0.5893890267238021, + "epoch": 4.37743407143652, + "grad_norm": 1.4375, + "learning_rate": 4.007407407407408e-06, + "loss": 0.6784, + "mean_token_accuracy": 0.830023642629385, + "num_tokens": 65887311.0, + "step": 2460 + }, + { + "entropy": 0.5546840105205775, + "epoch": 4.395237565372204, + "grad_norm": 1.453125, + "learning_rate": 3.9333333333333335e-06, + "loss": 0.6537, + "mean_token_accuracy": 0.8394486285746098, + "num_tokens": 66151098.0, + "step": 2470 + }, + { + "entropy": 0.5494891541078687, + "epoch": 4.413041059307889, + "grad_norm": 1.5703125, + "learning_rate": 3.85925925925926e-06, + "loss": 0.6481, + "mean_token_accuracy": 0.8396681498736143, + "num_tokens": 66417928.0, + "step": 2480 + }, + { + "entropy": 0.5561942363157868, + "epoch": 4.430844553243574, + "grad_norm": 1.796875, + "learning_rate": 3.7851851851851857e-06, + "loss": 0.6614, + "mean_token_accuracy": 0.8361107159405947, + "num_tokens": 66684481.0, + "step": 2490 + }, + { + "entropy": 0.5738992135971784, + "epoch": 4.448648047179259, + "grad_norm": 1.7109375, + "learning_rate": 3.7111111111111113e-06, + "loss": 0.6772, + "mean_token_accuracy": 0.833794866502285, + "num_tokens": 66940215.0, + "step": 2500 + }, + { + "epoch": 4.448648047179259, + "eval_biology_entropy": 1.3500232787132262, + "eval_biology_loss": 2.1630518436431885, + "eval_biology_mean_token_accuracy": 0.6106264660358429, + "eval_biology_num_tokens": 66940215.0, + "eval_biology_runtime": 39.0451, + "eval_biology_samples_per_second": 12.806, + "eval_biology_steps_per_second": 3.201, + "step": 2500 + }, + { + "epoch": 4.448648047179259, + "eval_chemistry_entropy": 0.5779505209922791, + "eval_chemistry_loss": 0.8212056756019592, + "eval_chemistry_mean_token_accuracy": 0.8103259291648864, + "eval_chemistry_num_tokens": 66940215.0, + "eval_chemistry_runtime": 48.6513, + "eval_chemistry_samples_per_second": 10.277, + "eval_chemistry_steps_per_second": 2.569, + "step": 2500 + }, + { + "epoch": 4.448648047179259, + "eval_math_entropy": 0.6078341612815857, + "eval_math_loss": 1.111467719078064, + "eval_math_mean_token_accuracy": 0.771428505897522, + "eval_math_num_tokens": 66940215.0, + "eval_math_runtime": 50.181, + "eval_math_samples_per_second": 9.964, + "eval_math_steps_per_second": 2.491, + "step": 2500 + }, + { + "epoch": 4.448648047179259, + "eval_physics_entropy": 0.6069678666591645, + "eval_physics_loss": 0.8872233629226685, + "eval_physics_mean_token_accuracy": 0.7977854557037354, + "eval_physics_num_tokens": 66940215.0, + "eval_physics_runtime": 57.617, + "eval_physics_samples_per_second": 8.678, + "eval_physics_steps_per_second": 2.169, + "step": 2500 + }, + { + "entropy": 0.5714038230478764, + "epoch": 4.466451541114944, + "grad_norm": 1.546875, + "learning_rate": 3.6370370370370374e-06, + "loss": 0.6744, + "mean_token_accuracy": 0.8347093638032674, + "num_tokens": 67207177.0, + "step": 2510 + }, + { + "entropy": 0.5652401139959693, + "epoch": 4.484255035050628, + "grad_norm": 1.546875, + "learning_rate": 3.562962962962963e-06, + "loss": 0.6596, + "mean_token_accuracy": 0.8372343368828297, + "num_tokens": 67477558.0, + "step": 2520 + }, + { + "entropy": 0.5658701064065099, + "epoch": 4.502058528986313, + "grad_norm": 1.6171875, + "learning_rate": 3.4888888888888896e-06, + "loss": 0.6619, + "mean_token_accuracy": 0.8384524192661047, + "num_tokens": 67741434.0, + "step": 2530 + }, + { + "entropy": 0.5715920325368643, + "epoch": 4.519862022921998, + "grad_norm": 1.46875, + "learning_rate": 3.4148148148148153e-06, + "loss": 0.6756, + "mean_token_accuracy": 0.8340298805385828, + "num_tokens": 68004497.0, + "step": 2540 + }, + { + "entropy": 0.5522453328594565, + "epoch": 4.537665516857683, + "grad_norm": 1.5625, + "learning_rate": 3.340740740740741e-06, + "loss": 0.6489, + "mean_token_accuracy": 0.8395404484122991, + "num_tokens": 68276413.0, + "step": 2550 + }, + { + "entropy": 0.5488388819620014, + "epoch": 4.555469010793368, + "grad_norm": 1.609375, + "learning_rate": 3.266666666666667e-06, + "loss": 0.6414, + "mean_token_accuracy": 0.8398447286337614, + "num_tokens": 68552989.0, + "step": 2560 + }, + { + "entropy": 0.5583368986845016, + "epoch": 4.5732725047290534, + "grad_norm": 1.6171875, + "learning_rate": 3.1925925925925927e-06, + "loss": 0.6641, + "mean_token_accuracy": 0.8369618207216263, + "num_tokens": 68816878.0, + "step": 2570 + }, + { + "entropy": 0.5624300969764591, + "epoch": 4.591075998664738, + "grad_norm": 1.515625, + "learning_rate": 3.1185185185185183e-06, + "loss": 0.6672, + "mean_token_accuracy": 0.8354556068778038, + "num_tokens": 69078503.0, + "step": 2580 + }, + { + "entropy": 0.5367948707193136, + "epoch": 4.608879492600423, + "grad_norm": 1.53125, + "learning_rate": 3.044444444444445e-06, + "loss": 0.6287, + "mean_token_accuracy": 0.8439593967050314, + "num_tokens": 69357964.0, + "step": 2590 + }, + { + "entropy": 0.5852402720600367, + "epoch": 4.626682986536108, + "grad_norm": 1.53125, + "learning_rate": 2.9703703703703705e-06, + "loss": 0.6832, + "mean_token_accuracy": 0.8317147444933652, + "num_tokens": 69615568.0, + "step": 2600 + }, + { + "epoch": 4.626682986536108, + "eval_biology_entropy": 1.3454367084503174, + "eval_biology_loss": 2.164541721343994, + "eval_biology_mean_token_accuracy": 0.6107339887619019, + "eval_biology_num_tokens": 69615568.0, + "eval_biology_runtime": 39.055, + "eval_biology_samples_per_second": 12.802, + "eval_biology_steps_per_second": 3.201, + "step": 2600 + }, + { + "epoch": 4.626682986536108, + "eval_chemistry_entropy": 0.5768348662853241, + "eval_chemistry_loss": 0.8212020397186279, + "eval_chemistry_mean_token_accuracy": 0.8104657735824585, + "eval_chemistry_num_tokens": 69615568.0, + "eval_chemistry_runtime": 48.6662, + "eval_chemistry_samples_per_second": 10.274, + "eval_chemistry_steps_per_second": 2.569, + "step": 2600 + }, + { + "epoch": 4.626682986536108, + "eval_math_entropy": 0.6048591766357422, + "eval_math_loss": 1.1121559143066406, + "eval_math_mean_token_accuracy": 0.7713811802864075, + "eval_math_num_tokens": 69615568.0, + "eval_math_runtime": 50.0753, + "eval_math_samples_per_second": 9.985, + "eval_math_steps_per_second": 2.496, + "step": 2600 + }, + { + "epoch": 4.626682986536108, + "eval_physics_entropy": 0.6050158340930939, + "eval_physics_loss": 0.8875883221626282, + "eval_physics_mean_token_accuracy": 0.7975573592185974, + "eval_physics_num_tokens": 69615568.0, + "eval_physics_runtime": 57.5774, + "eval_physics_samples_per_second": 8.684, + "eval_physics_steps_per_second": 2.171, + "step": 2600 + }, + { + "entropy": 0.5623448735103012, + "epoch": 4.644486480471793, + "grad_norm": 1.5078125, + "learning_rate": 2.8962962962962966e-06, + "loss": 0.6641, + "mean_token_accuracy": 0.8355112373828888, + "num_tokens": 69882933.0, + "step": 2610 + }, + { + "entropy": 0.5704484520480037, + "epoch": 4.662289974407478, + "grad_norm": 1.4765625, + "learning_rate": 2.8222222222222223e-06, + "loss": 0.6722, + "mean_token_accuracy": 0.834565806761384, + "num_tokens": 70147891.0, + "step": 2620 + }, + { + "entropy": 0.5643257100135088, + "epoch": 4.680093468343163, + "grad_norm": 1.6484375, + "learning_rate": 2.748148148148148e-06, + "loss": 0.6613, + "mean_token_accuracy": 0.8358062624931335, + "num_tokens": 70411454.0, + "step": 2630 + }, + { + "entropy": 0.5764127368107438, + "epoch": 4.697896962278847, + "grad_norm": 1.546875, + "learning_rate": 2.6740740740740744e-06, + "loss": 0.6774, + "mean_token_accuracy": 0.8341616597026587, + "num_tokens": 70673950.0, + "step": 2640 + }, + { + "entropy": 0.5694889141246676, + "epoch": 4.715700456214532, + "grad_norm": 1.5625, + "learning_rate": 2.6e-06, + "loss": 0.6686, + "mean_token_accuracy": 0.8337815593928098, + "num_tokens": 70933220.0, + "step": 2650 + }, + { + "entropy": 0.5560853591188788, + "epoch": 4.733503950150217, + "grad_norm": 1.6171875, + "learning_rate": 2.525925925925926e-06, + "loss": 0.6553, + "mean_token_accuracy": 0.8376856446266174, + "num_tokens": 71202429.0, + "step": 2660 + }, + { + "entropy": 0.5625249149277807, + "epoch": 4.751307444085902, + "grad_norm": 1.5625, + "learning_rate": 2.451851851851852e-06, + "loss": 0.6673, + "mean_token_accuracy": 0.8358222220093012, + "num_tokens": 71469322.0, + "step": 2670 + }, + { + "entropy": 0.558993223682046, + "epoch": 4.769110938021587, + "grad_norm": 1.609375, + "learning_rate": 2.377777777777778e-06, + "loss": 0.6581, + "mean_token_accuracy": 0.8376695141196251, + "num_tokens": 71736522.0, + "step": 2680 + }, + { + "entropy": 0.5641753245145082, + "epoch": 4.786914431957271, + "grad_norm": 1.4453125, + "learning_rate": 2.303703703703704e-06, + "loss": 0.665, + "mean_token_accuracy": 0.8369908705353737, + "num_tokens": 71999537.0, + "step": 2690 + }, + { + "entropy": 0.5639397297054529, + "epoch": 4.804717925892956, + "grad_norm": 1.4453125, + "learning_rate": 2.22962962962963e-06, + "loss": 0.6578, + "mean_token_accuracy": 0.8376573950052262, + "num_tokens": 72269735.0, + "step": 2700 + }, + { + "epoch": 4.804717925892956, + "eval_biology_entropy": 1.3441446619033814, + "eval_biology_loss": 2.1645467281341553, + "eval_biology_mean_token_accuracy": 0.6106171884536743, + "eval_biology_num_tokens": 72269735.0, + "eval_biology_runtime": 39.0324, + "eval_biology_samples_per_second": 12.81, + "eval_biology_steps_per_second": 3.202, + "step": 2700 + }, + { + "epoch": 4.804717925892956, + "eval_chemistry_entropy": 0.5765341448783875, + "eval_chemistry_loss": 0.8211090564727783, + "eval_chemistry_mean_token_accuracy": 0.8104948234558106, + "eval_chemistry_num_tokens": 72269735.0, + "eval_chemistry_runtime": 48.6496, + "eval_chemistry_samples_per_second": 10.278, + "eval_chemistry_steps_per_second": 2.569, + "step": 2700 + }, + { + "epoch": 4.804717925892956, + "eval_math_entropy": 0.604786561012268, + "eval_math_loss": 1.1119775772094727, + "eval_math_mean_token_accuracy": 0.7713620710372925, + "eval_math_num_tokens": 72269735.0, + "eval_math_runtime": 50.0581, + "eval_math_samples_per_second": 9.988, + "eval_math_steps_per_second": 2.497, + "step": 2700 + }, + { + "epoch": 4.804717925892956, + "eval_physics_entropy": 0.6049994843006135, + "eval_physics_loss": 0.8874087929725647, + "eval_physics_mean_token_accuracy": 0.7977907452583313, + "eval_physics_num_tokens": 72269735.0, + "eval_physics_runtime": 57.8197, + "eval_physics_samples_per_second": 8.648, + "eval_physics_steps_per_second": 2.162, + "step": 2700 + }, + { + "entropy": 0.5614620108157397, + "epoch": 4.822521419828641, + "grad_norm": 1.421875, + "learning_rate": 2.1555555555555558e-06, + "loss": 0.6562, + "mean_token_accuracy": 0.8374848257750273, + "num_tokens": 72537744.0, + "step": 2710 + }, + { + "entropy": 0.57458830550313, + "epoch": 4.840324913764326, + "grad_norm": 1.5859375, + "learning_rate": 2.0814814814814814e-06, + "loss": 0.6749, + "mean_token_accuracy": 0.8337875984609127, + "num_tokens": 72801656.0, + "step": 2720 + }, + { + "entropy": 0.546775977872312, + "epoch": 4.858128407700011, + "grad_norm": 1.4296875, + "learning_rate": 2.0074074074074075e-06, + "loss": 0.638, + "mean_token_accuracy": 0.8412183981388808, + "num_tokens": 73075773.0, + "step": 2730 + }, + { + "entropy": 0.5441833294928073, + "epoch": 4.875931901635696, + "grad_norm": 1.4140625, + "learning_rate": 1.9333333333333336e-06, + "loss": 0.6409, + "mean_token_accuracy": 0.8414637863636016, + "num_tokens": 73353692.0, + "step": 2740 + }, + { + "entropy": 0.5663061382248997, + "epoch": 4.893735395571381, + "grad_norm": 1.515625, + "learning_rate": 1.8592592592592595e-06, + "loss": 0.6668, + "mean_token_accuracy": 0.8345465075224638, + "num_tokens": 73620577.0, + "step": 2750 + }, + { + "entropy": 0.5377116657793521, + "epoch": 4.9115388895070655, + "grad_norm": 1.3828125, + "learning_rate": 1.7851851851851853e-06, + "loss": 0.642, + "mean_token_accuracy": 0.842970971390605, + "num_tokens": 73888689.0, + "step": 2760 + }, + { + "entropy": 0.5719966486096382, + "epoch": 4.9293423834427506, + "grad_norm": 1.546875, + "learning_rate": 1.7111111111111112e-06, + "loss": 0.6703, + "mean_token_accuracy": 0.8350907273590564, + "num_tokens": 74150672.0, + "step": 2770 + }, + { + "entropy": 0.558096438087523, + "epoch": 4.947145877378436, + "grad_norm": 1.375, + "learning_rate": 1.6370370370370373e-06, + "loss": 0.6561, + "mean_token_accuracy": 0.8376757599413395, + "num_tokens": 74419554.0, + "step": 2780 + }, + { + "entropy": 0.5682699911296367, + "epoch": 4.964949371314121, + "grad_norm": 1.4140625, + "learning_rate": 1.562962962962963e-06, + "loss": 0.6716, + "mean_token_accuracy": 0.8347717296332121, + "num_tokens": 74686704.0, + "step": 2790 + }, + { + "entropy": 0.5673889361321927, + "epoch": 4.982752865249806, + "grad_norm": 1.546875, + "learning_rate": 1.4888888888888888e-06, + "loss": 0.6742, + "mean_token_accuracy": 0.836492333188653, + "num_tokens": 74947391.0, + "step": 2800 + }, + { + "epoch": 4.982752865249806, + "eval_biology_entropy": 1.3449444818496703, + "eval_biology_loss": 2.1643216609954834, + "eval_biology_mean_token_accuracy": 0.6106573867797852, + "eval_biology_num_tokens": 74947391.0, + "eval_biology_runtime": 39.0794, + "eval_biology_samples_per_second": 12.794, + "eval_biology_steps_per_second": 3.199, + "step": 2800 + }, + { + "epoch": 4.982752865249806, + "eval_chemistry_entropy": 0.5765703585147858, + "eval_chemistry_loss": 0.821064293384552, + "eval_chemistry_mean_token_accuracy": 0.8102838611602783, + "eval_chemistry_num_tokens": 74947391.0, + "eval_chemistry_runtime": 48.7235, + "eval_chemistry_samples_per_second": 10.262, + "eval_chemistry_steps_per_second": 2.565, + "step": 2800 + }, + { + "epoch": 4.982752865249806, + "eval_math_entropy": 0.6052222805023193, + "eval_math_loss": 1.1119670867919922, + "eval_math_mean_token_accuracy": 0.7714278025627136, + "eval_math_num_tokens": 74947391.0, + "eval_math_runtime": 50.1289, + "eval_math_samples_per_second": 9.974, + "eval_math_steps_per_second": 2.494, + "step": 2800 + }, + { + "epoch": 4.982752865249806, + "eval_physics_entropy": 0.6050758678913116, + "eval_physics_loss": 0.8875164985656738, + "eval_physics_mean_token_accuracy": 0.7976568508148193, + "eval_physics_num_tokens": 74947391.0, + "eval_physics_runtime": 57.6402, + "eval_physics_samples_per_second": 8.675, + "eval_physics_steps_per_second": 2.169, + "step": 2800 + }, + { + "entropy": 0.5590117814079407, + "epoch": 5.0, + "grad_norm": 1.8359375, + "learning_rate": 1.414814814814815e-06, + "loss": 0.6584, + "mean_token_accuracy": 0.8381629182446387, + "num_tokens": 75199650.0, + "step": 2810 + }, + { + "entropy": 0.576119888573885, + "epoch": 5.017803493935685, + "grad_norm": 1.4453125, + "learning_rate": 1.3407407407407408e-06, + "loss": 0.6684, + "mean_token_accuracy": 0.8363345433026552, + "num_tokens": 75457508.0, + "step": 2820 + }, + { + "entropy": 0.5501403734087944, + "epoch": 5.03560698787137, + "grad_norm": 1.4375, + "learning_rate": 1.2666666666666669e-06, + "loss": 0.6448, + "mean_token_accuracy": 0.840818153694272, + "num_tokens": 75731578.0, + "step": 2830 + }, + { + "entropy": 0.5537693470716476, + "epoch": 5.053410481807054, + "grad_norm": 1.484375, + "learning_rate": 1.1925925925925928e-06, + "loss": 0.6556, + "mean_token_accuracy": 0.8388679180294275, + "num_tokens": 76008875.0, + "step": 2840 + }, + { + "entropy": 0.5640265366062522, + "epoch": 5.071213975742739, + "grad_norm": 1.4296875, + "learning_rate": 1.1185185185185186e-06, + "loss": 0.658, + "mean_token_accuracy": 0.8369051955640316, + "num_tokens": 76281818.0, + "step": 2850 + }, + { + "entropy": 0.5594167856499552, + "epoch": 5.089017469678424, + "grad_norm": 1.3671875, + "learning_rate": 1.0444444444444445e-06, + "loss": 0.6587, + "mean_token_accuracy": 0.8376182932406664, + "num_tokens": 76549100.0, + "step": 2860 + }, + { + "entropy": 0.5736024105921388, + "epoch": 5.106820963614109, + "grad_norm": 1.5859375, + "learning_rate": 9.703703703703704e-07, + "loss": 0.6649, + "mean_token_accuracy": 0.8346774931997061, + "num_tokens": 76815070.0, + "step": 2870 + }, + { + "entropy": 0.5549590906128288, + "epoch": 5.124624457549794, + "grad_norm": 1.46875, + "learning_rate": 8.962962962962964e-07, + "loss": 0.6493, + "mean_token_accuracy": 0.8384062923491001, + "num_tokens": 77090788.0, + "step": 2880 + }, + { + "entropy": 0.5687621856108308, + "epoch": 5.142427951485479, + "grad_norm": 1.4609375, + "learning_rate": 8.222222222222223e-07, + "loss": 0.6695, + "mean_token_accuracy": 0.8359207462519407, + "num_tokens": 77356917.0, + "step": 2890 + }, + { + "entropy": 0.5601389359682798, + "epoch": 5.1602314454211635, + "grad_norm": 1.46875, + "learning_rate": 7.481481481481482e-07, + "loss": 0.6506, + "mean_token_accuracy": 0.8396558336913585, + "num_tokens": 77619491.0, + "step": 2900 + }, + { + "epoch": 5.1602314454211635, + "eval_biology_entropy": 1.3417477326393128, + "eval_biology_loss": 2.16618013381958, + "eval_biology_mean_token_accuracy": 0.6104471139907837, + "eval_biology_num_tokens": 77619491.0, + "eval_biology_runtime": 39.0788, + "eval_biology_samples_per_second": 12.795, + "eval_biology_steps_per_second": 3.199, + "step": 2900 + }, + { + "epoch": 5.1602314454211635, + "eval_chemistry_entropy": 0.5752652714252472, + "eval_chemistry_loss": 0.8214696645736694, + "eval_chemistry_mean_token_accuracy": 0.8104866914749146, + "eval_chemistry_num_tokens": 77619491.0, + "eval_chemistry_runtime": 48.6351, + "eval_chemistry_samples_per_second": 10.281, + "eval_chemistry_steps_per_second": 2.57, + "step": 2900 + }, + { + "epoch": 5.1602314454211635, + "eval_math_entropy": 0.6042537760734558, + "eval_math_loss": 1.1125837564468384, + "eval_math_mean_token_accuracy": 0.7713408164978027, + "eval_math_num_tokens": 77619491.0, + "eval_math_runtime": 50.0516, + "eval_math_samples_per_second": 9.99, + "eval_math_steps_per_second": 2.497, + "step": 2900 + }, + { + "epoch": 5.1602314454211635, + "eval_physics_entropy": 0.6039941041469574, + "eval_physics_loss": 0.8881220817565918, + "eval_physics_mean_token_accuracy": 0.7976104960441589, + "eval_physics_num_tokens": 77619491.0, + "eval_physics_runtime": 57.7291, + "eval_physics_samples_per_second": 8.661, + "eval_physics_steps_per_second": 2.165, + "step": 2900 + }, + { + "entropy": 0.5559898747131229, + "epoch": 5.1780349393568486, + "grad_norm": 1.46875, + "learning_rate": 6.740740740740741e-07, + "loss": 0.6532, + "mean_token_accuracy": 0.8384331479668617, + "num_tokens": 77885239.0, + "step": 2910 + }, + { + "entropy": 0.564581798389554, + "epoch": 5.195838433292534, + "grad_norm": 1.4609375, + "learning_rate": 6.000000000000001e-07, + "loss": 0.6631, + "mean_token_accuracy": 0.8362601526081562, + "num_tokens": 78154475.0, + "step": 2920 + }, + { + "entropy": 0.5659063132479787, + "epoch": 5.213641927228219, + "grad_norm": 1.6328125, + "learning_rate": 5.25925925925926e-07, + "loss": 0.67, + "mean_token_accuracy": 0.8372922617942095, + "num_tokens": 78418518.0, + "step": 2930 + }, + { + "entropy": 0.5593667538836599, + "epoch": 5.231445421163904, + "grad_norm": 1.546875, + "learning_rate": 4.518518518518519e-07, + "loss": 0.6547, + "mean_token_accuracy": 0.8373474027961493, + "num_tokens": 78687867.0, + "step": 2940 + }, + { + "entropy": 0.549276759289205, + "epoch": 5.249248915099589, + "grad_norm": 1.3359375, + "learning_rate": 3.777777777777778e-07, + "loss": 0.6457, + "mean_token_accuracy": 0.8407016690820456, + "num_tokens": 78954512.0, + "step": 2950 + }, + { + "entropy": 0.5572434209287167, + "epoch": 5.267052409035273, + "grad_norm": 1.59375, + "learning_rate": 3.0370370370370374e-07, + "loss": 0.6523, + "mean_token_accuracy": 0.8391760684549808, + "num_tokens": 79219529.0, + "step": 2960 + }, + { + "entropy": 0.5679417058825493, + "epoch": 5.284855902970958, + "grad_norm": 1.3359375, + "learning_rate": 2.2962962962962964e-07, + "loss": 0.6564, + "mean_token_accuracy": 0.8365730907768011, + "num_tokens": 79494122.0, + "step": 2970 + }, + { + "entropy": 0.5557117018848657, + "epoch": 5.302659396906643, + "grad_norm": 1.40625, + "learning_rate": 1.5555555555555556e-07, + "loss": 0.6514, + "mean_token_accuracy": 0.8384904775768518, + "num_tokens": 79766616.0, + "step": 2980 + }, + { + "entropy": 0.5563192063942551, + "epoch": 5.320462890842328, + "grad_norm": 1.46875, + "learning_rate": 8.148148148148148e-08, + "loss": 0.6596, + "mean_token_accuracy": 0.8416946094483138, + "num_tokens": 80028355.0, + "step": 2990 + }, + { + "entropy": 0.5488091288134456, + "epoch": 5.338266384778013, + "grad_norm": 1.515625, + "learning_rate": 7.407407407407408e-09, + "loss": 0.6428, + "mean_token_accuracy": 0.8410037450492382, + "num_tokens": 80296531.0, + "step": 3000 + }, + { + "epoch": 5.338266384778013, + "eval_biology_entropy": 1.3414670877456665, + "eval_biology_loss": 2.1666340827941895, + "eval_biology_mean_token_accuracy": 0.6105899527072907, + "eval_biology_num_tokens": 80296531.0, + "eval_biology_runtime": 39.1562, + "eval_biology_samples_per_second": 12.769, + "eval_biology_steps_per_second": 3.192, + "step": 3000 + }, + { + "epoch": 5.338266384778013, + "eval_chemistry_entropy": 0.5751763939857483, + "eval_chemistry_loss": 0.821597695350647, + "eval_chemistry_mean_token_accuracy": 0.8102493243217468, + "eval_chemistry_num_tokens": 80296531.0, + "eval_chemistry_runtime": 48.8983, + "eval_chemistry_samples_per_second": 10.225, + "eval_chemistry_steps_per_second": 2.556, + "step": 3000 + }, + { + "epoch": 5.338266384778013, + "eval_math_entropy": 0.604233852148056, + "eval_math_loss": 1.1128088235855103, + "eval_math_mean_token_accuracy": 0.7713053269386292, + "eval_math_num_tokens": 80296531.0, + "eval_math_runtime": 50.2197, + "eval_math_samples_per_second": 9.956, + "eval_math_steps_per_second": 2.489, + "step": 3000 + }, + { + "epoch": 5.338266384778013, + "eval_physics_entropy": 0.6038673832416535, + "eval_physics_loss": 0.8882793188095093, + "eval_physics_mean_token_accuracy": 0.797690041065216, + "eval_physics_num_tokens": 80296531.0, + "eval_physics_runtime": 57.6664, + "eval_physics_samples_per_second": 8.671, + "eval_physics_steps_per_second": 2.168, + "step": 3000 + } + ], + "logging_steps": 10, + "max_steps": 3000, + "num_input_tokens_seen": 0, + "num_train_epochs": 6, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 6.243890876917027e+18, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/checkpoint-3000/training_args.bin b/checkpoint-3000/training_args.bin new file mode 100644 index 0000000..79cad08 --- /dev/null +++ b/checkpoint-3000/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dfc23fa1b9be77c605cba9733debba5f60170f0eadaa2a8866b5aaf4b0e65756 +size 6417 diff --git a/checkpoint-500/chat_template.jinja b/checkpoint-500/chat_template.jinja new file mode 100644 index 0000000..923ec25 --- /dev/null +++ b/checkpoint-500/chat_template.jinja @@ -0,0 +1,4 @@ +{{ bos_token }}{% if messages[0]['role'] == 'system' %}{{ raise_exception('System role not supported') }}{% endif %}{% for message in messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if (message['role'] == 'assistant') %}{% set role = 'model' %}{% else %}{% set role = message['role'] %}{% endif %}{{ '' + role + ' +' + message['content'] | trim + ' +' }}{% endfor %}{% if add_generation_prompt %}{{'model +'}}{% endif %} \ No newline at end of file diff --git a/checkpoint-500/config.json b/checkpoint-500/config.json new file mode 100644 index 0000000..29e6cbe --- /dev/null +++ b/checkpoint-500/config.json @@ -0,0 +1,77 @@ +{ + "architectures": [ + "Gemma2ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "attn_logit_softcapping": 50.0, + "bos_token_id": 2, + "cache_implementation": "hybrid", + "dtype": "bfloat16", + "eos_token_id": 1, + "final_logit_softcapping": 30.0, + "head_dim": 256, + "hidden_act": "gelu_pytorch_tanh", + "hidden_activation": "gelu_pytorch_tanh", + "hidden_size": 3584, + "initializer_range": 0.02, + "intermediate_size": 14336, + "layer_types": [ + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention" + ], + "max_position_embeddings": 8192, + "model_type": "gemma2", + "num_attention_heads": 16, + "num_hidden_layers": 42, + "num_key_value_heads": 8, + "pad_token_id": 0, + "query_pre_attn_scalar": 256, + "rms_norm_eps": 1e-06, + "rope_theta": 10000.0, + "sliding_window": 4096, + "sliding_window_size": 4096, + "transformers_version": "4.56.1", + "use_cache": true, + "vocab_size": 256000 +} diff --git a/checkpoint-500/generation_config.json b/checkpoint-500/generation_config.json new file mode 100644 index 0000000..f2c34f2 --- /dev/null +++ b/checkpoint-500/generation_config.json @@ -0,0 +1,9 @@ +{ + "_from_model_config": true, + "bos_token_id": 2, + "eos_token_id": [ + 1 + ], + "pad_token_id": 0, + "transformers_version": "4.56.1" +} diff --git a/checkpoint-500/model-00001-of-00004.safetensors b/checkpoint-500/model-00001-of-00004.safetensors new file mode 100644 index 0000000..4705c7e --- /dev/null +++ b/checkpoint-500/model-00001-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a0d4eb4fcbddfe01b0dc7a58386a3980826ededaf0ecd67d1ef38995b89615ba +size 4903351912 diff --git a/checkpoint-500/model-00002-of-00004.safetensors b/checkpoint-500/model-00002-of-00004.safetensors new file mode 100644 index 0000000..744b59b --- /dev/null +++ b/checkpoint-500/model-00002-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:712a589c333f49f7b6f3870838a9e0ab4b3c0260abd5130e8c54316e6c99640b +size 4947570872 diff --git a/checkpoint-500/model-00003-of-00004.safetensors b/checkpoint-500/model-00003-of-00004.safetensors new file mode 100644 index 0000000..6fcce0d --- /dev/null +++ b/checkpoint-500/model-00003-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9c02d09abebe583ce91b7b324b0fa1b756d5810e64980f7fa42a0a1bfc862cf7 +size 4962221464 diff --git a/checkpoint-500/model-00004-of-00004.safetensors b/checkpoint-500/model-00004-of-00004.safetensors new file mode 100644 index 0000000..9ffc1dc --- /dev/null +++ b/checkpoint-500/model-00004-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cf3d2e24e190e1fc37e2cfc976445af2b9ba48615d1f8c4b58717d920a1f4989 +size 3670322200 diff --git a/checkpoint-500/model.safetensors.index.json b/checkpoint-500/model.safetensors.index.json new file mode 100644 index 0000000..3b569c5 --- /dev/null +++ b/checkpoint-500/model.safetensors.index.json @@ -0,0 +1,472 @@ +{ + "metadata": { + "total_parameters": 9241705984, + "total_size": 18483411968 + }, + "weight_map": { + "model.embed_tokens.weight": "model-00001-of-00004.safetensors", + "model.layers.0.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.0.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.0.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.1.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.1.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.10.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.10.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.10.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.10.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.11.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.11.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.11.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.12.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.12.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.12.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.13.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.13.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.13.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.14.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.14.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.14.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.15.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.15.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.15.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.16.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.16.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.16.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.17.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.17.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.17.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.18.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.18.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.18.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.19.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.19.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.19.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.2.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.2.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.2.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.20.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.20.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.20.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.20.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.20.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.20.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.20.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.21.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.21.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.21.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.21.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.21.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.22.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.22.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.22.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.22.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.23.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.23.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.23.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.24.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.24.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.24.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.25.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.25.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.25.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.26.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.26.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.26.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.27.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.27.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.27.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.28.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.28.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.28.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.29.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.29.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.29.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.3.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.3.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.3.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.30.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.30.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.30.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.30.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.30.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.31.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.31.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.31.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.32.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.32.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.32.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.32.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.32.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.32.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.32.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.32.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.32.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.32.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.32.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.33.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.33.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.33.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.33.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.33.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.33.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.33.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.33.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.33.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.33.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.33.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.34.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.34.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.34.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.34.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.34.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.34.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.34.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.34.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.34.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.34.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.34.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.35.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.35.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.35.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.35.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.35.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.35.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.35.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.35.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.35.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.35.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.35.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.36.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.36.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.36.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.36.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.36.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.36.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.36.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.36.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.36.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.36.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.36.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.37.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.37.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.37.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.37.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.37.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.37.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.37.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.37.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.37.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.37.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.37.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.38.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.38.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.38.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.38.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.38.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.38.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.38.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.38.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.38.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.38.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.38.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.39.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.39.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.39.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.39.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.39.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.39.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.39.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.39.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.39.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.39.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.39.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.4.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.4.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.4.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.4.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.40.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.40.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.40.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.40.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.40.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.40.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.40.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.40.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.40.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.40.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.40.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.41.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.41.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.41.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.41.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.41.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.41.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.41.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.41.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.41.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.41.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.41.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.5.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.5.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.5.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.5.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.6.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.6.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.6.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.7.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.7.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.7.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.8.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.8.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.8.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.8.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.8.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.8.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.8.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.8.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.8.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.8.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.8.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.9.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.9.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.9.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.9.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.norm.weight": "model-00004-of-00004.safetensors" + } +} diff --git a/checkpoint-500/optimizer.pt b/checkpoint-500/optimizer.pt new file mode 100644 index 0000000..4446882 --- /dev/null +++ b/checkpoint-500/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d0e86dd33a09d9589e8b18beb6abd96dd1a50baf221fb85b151165d283486aad +size 1585596266 diff --git a/checkpoint-500/rng_state.pth b/checkpoint-500/rng_state.pth new file mode 100644 index 0000000..74d1d2f --- /dev/null +++ b/checkpoint-500/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fde986f97ecc270a34049538e5bcb495edbc4e7f5914a76b3cad083de287dc73 +size 14645 diff --git a/checkpoint-500/scheduler.pt b/checkpoint-500/scheduler.pt new file mode 100644 index 0000000..23de5b0 --- /dev/null +++ b/checkpoint-500/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ddc00081d4f892d131a0d2b4d4827a214275362bfa62dfb8413f489c1bfad457 +size 1465 diff --git a/checkpoint-500/special_tokens_map.json b/checkpoint-500/special_tokens_map.json new file mode 100644 index 0000000..8d6368f --- /dev/null +++ b/checkpoint-500/special_tokens_map.json @@ -0,0 +1,34 @@ +{ + "additional_special_tokens": [ + "", + "" + ], + "bos_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "unk_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + } +} diff --git a/checkpoint-500/tokenizer.json b/checkpoint-500/tokenizer.json new file mode 100644 index 0000000..a4a305d --- /dev/null +++ b/checkpoint-500/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5f7eee611703c5ce5d1eee32d9cdcfe465647b8aff0c1dfb3bed7ad7dbb05060 +size 34362873 diff --git a/checkpoint-500/tokenizer.model b/checkpoint-500/tokenizer.model new file mode 100644 index 0000000..796efe9 --- /dev/null +++ b/checkpoint-500/tokenizer.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:61a7b147390c64585d6c3543dd6fc636906c9af3865a5548f27f31aee1d4c8e2 +size 4241003 diff --git a/checkpoint-500/tokenizer_config.json b/checkpoint-500/tokenizer_config.json new file mode 100644 index 0000000..3ade6be --- /dev/null +++ b/checkpoint-500/tokenizer_config.json @@ -0,0 +1,2013 @@ +{ + "add_bos_token": true, + "add_eos_token": false, + "added_tokens_decoder": { + "0": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "1": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "2": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "3": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "4": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "5": { + "content": "<2mass>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "6": { + "content": "[@BOS@]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "7": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "8": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "9": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "10": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "11": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "12": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "13": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "14": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "15": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "16": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "17": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "18": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "19": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "20": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "21": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "22": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "23": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "24": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "25": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "26": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "27": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "28": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "29": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "30": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "31": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "32": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "33": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "34": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "35": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "36": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "37": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "38": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "39": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "40": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "41": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "42": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "43": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "44": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "45": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "46": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "47": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "48": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "49": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "50": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "51": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "52": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "53": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "54": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "55": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "56": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "57": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "58": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "59": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "60": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "61": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "62": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "63": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "64": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "65": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "66": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "67": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "68": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "69": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "70": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "71": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "72": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "73": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "74": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "75": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "76": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "77": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "78": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "79": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "80": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "81": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "82": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "83": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "84": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "85": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "86": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "87": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "88": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "89": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "90": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "91": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "92": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "93": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "94": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "95": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "96": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "97": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "98": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "99": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "100": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "101": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "102": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "103": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "104": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "105": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "106": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "107": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "108": { + "content": "\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "109": { + "content": "\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "110": { + "content": "\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "111": { + "content": "\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "112": { + "content": "\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "113": { + "content": "\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "114": { + "content": "\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "115": { + "content": "\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "116": { + "content": "\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "117": { + "content": "\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "118": { + "content": "\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "119": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "120": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "121": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "122": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "123": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "124": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "125": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "126": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "127": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "128": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "129": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "130": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "131": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "132": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "133": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "134": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "135": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "136": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "137": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "138": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "139": { + "content": "▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "140": { + "content": "▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "141": { + "content": "▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "142": { + "content": "▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "143": { + "content": "▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "144": { + "content": "▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "145": { + "content": "▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "146": { + "content": "▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "147": { + "content": "▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "148": { + "content": "▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "149": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "150": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "152": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "153": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "154": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "155": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "156": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "157": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "158": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "159": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "160": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "161": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "162": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "163": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "164": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "165": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "166": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "167": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "168": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "169": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "170": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "172": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "173": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "174": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "175": { + "content": "
", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "171": { + "content": "
", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "176": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "177": { + "content": "
", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "178": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "179": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "180": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "181": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "182": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "183": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "184": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "185": { + "content": "

", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "186": { + "content": "

", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "187": { + "content": "

", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "188": { + "content": "

", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "189": { + "content": "

", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "190": { + "content": "
", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "191": { + "content": "
", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "192": { + "content": "
", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "193": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "194": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "195": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "196": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "197": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "198": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "199": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "200": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "201": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "202": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "203": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "204": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "205": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "206": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "207": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "208": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "209": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "210": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "211": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "212": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "213": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "214": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "215": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "216": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255968": { + "content": "[toxicity=0]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255969": { + "content": "\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255970": { + "content": "\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255971": { + "content": "\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255972": { + "content": "\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255973": { + "content": "\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255974": { + "content": "\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255975": { + "content": "\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255976": { + "content": "\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255977": { + "content": "\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255978": { + "content": "\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255979": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255980": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255981": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255982": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255983": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255984": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255985": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255986": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255987": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255988": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255989": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255990": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255991": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255992": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255993": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255994": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255995": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255996": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255997": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255998": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255999": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + } + }, + "additional_special_tokens": [ + "", + "" + ], + "bos_token": "", + "clean_up_tokenization_spaces": false, + "eos_token": "", + "extra_special_tokens": {}, + "model_max_length": 1000000000000000019884624838656, + "pad_token": "", + "sp_model_kwargs": {}, + "spaces_between_special_tokens": false, + "tokenizer_class": "GemmaTokenizer", + "unk_token": "", + "use_default_system_prompt": false +} diff --git a/checkpoint-500/trainer_state.json b/checkpoint-500/trainer_state.json new file mode 100644 index 0000000..9a3a5a5 --- /dev/null +++ b/checkpoint-500/trainer_state.json @@ -0,0 +1,754 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.8901746967842439, + "eval_steps": 100, + "global_step": 500, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.3253214355558156, + "epoch": 0.017803493935684877, + "grad_norm": 14.5, + "learning_rate": 6.000000000000001e-07, + "loss": 2.1959, + "mean_token_accuracy": 0.5975642748177051, + "num_tokens": 259048.0, + "step": 10 + }, + { + "entropy": 1.3036176152527332, + "epoch": 0.035606987871369754, + "grad_norm": 12.75, + "learning_rate": 1.2666666666666669e-06, + "loss": 2.1499, + "mean_token_accuracy": 0.6023047739639878, + "num_tokens": 532019.0, + "step": 20 + }, + { + "entropy": 1.3484113823622466, + "epoch": 0.053410481807054634, + "grad_norm": 10.8125, + "learning_rate": 1.9333333333333336e-06, + "loss": 2.108, + "mean_token_accuracy": 0.6078463492915034, + "num_tokens": 807220.0, + "step": 30 + }, + { + "entropy": 1.4311681490391492, + "epoch": 0.07121397574273951, + "grad_norm": 7.875, + "learning_rate": 2.6e-06, + "loss": 2.0349, + "mean_token_accuracy": 0.6107733339071274, + "num_tokens": 1068429.0, + "step": 40 + }, + { + "entropy": 1.479667642340064, + "epoch": 0.08901746967842439, + "grad_norm": 6.4375, + "learning_rate": 3.266666666666667e-06, + "loss": 1.8649, + "mean_token_accuracy": 0.630409375578165, + "num_tokens": 1336698.0, + "step": 50 + }, + { + "entropy": 1.5030938494950532, + "epoch": 0.10682096361410927, + "grad_norm": 4.1875, + "learning_rate": 3.9333333333333335e-06, + "loss": 1.7217, + "mean_token_accuracy": 0.6490663634613156, + "num_tokens": 1608574.0, + "step": 60 + }, + { + "entropy": 1.5066865295171739, + "epoch": 0.12462445754979415, + "grad_norm": 2.71875, + "learning_rate": 4.600000000000001e-06, + "loss": 1.5716, + "mean_token_accuracy": 0.6711439102888107, + "num_tokens": 1872747.0, + "step": 70 + }, + { + "entropy": 1.3747932206839324, + "epoch": 0.14242795148547902, + "grad_norm": 2.234375, + "learning_rate": 5.2666666666666665e-06, + "loss": 1.4403, + "mean_token_accuracy": 0.6936415441334247, + "num_tokens": 2133173.0, + "step": 80 + }, + { + "entropy": 1.1736511185765266, + "epoch": 0.1602314454211639, + "grad_norm": 1.921875, + "learning_rate": 5.933333333333335e-06, + "loss": 1.2902, + "mean_token_accuracy": 0.7222886722534895, + "num_tokens": 2406145.0, + "step": 90 + }, + { + "entropy": 1.1317794900387526, + "epoch": 0.17803493935684878, + "grad_norm": 1.8046875, + "learning_rate": 6.600000000000001e-06, + "loss": 1.2424, + "mean_token_accuracy": 0.7295983772724867, + "num_tokens": 2663709.0, + "step": 100 + }, + { + "epoch": 0.17803493935684878, + "eval_biology_entropy": 2.3133783559799195, + "eval_biology_loss": 2.7910587787628174, + "eval_biology_mean_token_accuracy": 0.5260232911109924, + "eval_biology_num_tokens": 2663709.0, + "eval_biology_runtime": 39.1656, + "eval_biology_samples_per_second": 12.766, + "eval_biology_steps_per_second": 3.192, + "step": 100 + }, + { + "epoch": 0.17803493935684878, + "eval_chemistry_entropy": 1.0788086318969727, + "eval_chemistry_loss": 1.2403336763381958, + "eval_chemistry_mean_token_accuracy": 0.7357268543243408, + "eval_chemistry_num_tokens": 2663709.0, + "eval_chemistry_runtime": 48.7822, + "eval_chemistry_samples_per_second": 10.25, + "eval_chemistry_steps_per_second": 2.562, + "step": 100 + }, + { + "epoch": 0.17803493935684878, + "eval_math_entropy": 0.938195451259613, + "eval_math_loss": 1.3248047828674316, + "eval_math_mean_token_accuracy": 0.7312402629852295, + "eval_math_num_tokens": 2663709.0, + "eval_math_runtime": 50.197, + "eval_math_samples_per_second": 9.961, + "eval_math_steps_per_second": 2.49, + "step": 100 + }, + { + "epoch": 0.17803493935684878, + "eval_physics_entropy": 1.022246531009674, + "eval_physics_loss": 1.176153302192688, + "eval_physics_mean_token_accuracy": 0.7475795021057129, + "eval_physics_num_tokens": 2663709.0, + "eval_physics_runtime": 57.6895, + "eval_physics_samples_per_second": 8.667, + "eval_physics_steps_per_second": 2.167, + "step": 100 + }, + { + "entropy": 1.0380497876554728, + "epoch": 0.19583843329253367, + "grad_norm": 1.7421875, + "learning_rate": 7.266666666666668e-06, + "loss": 1.157, + "mean_token_accuracy": 0.7445634145289659, + "num_tokens": 2934980.0, + "step": 110 + }, + { + "entropy": 1.0149445166811346, + "epoch": 0.21364192722821854, + "grad_norm": 1.6015625, + "learning_rate": 7.933333333333334e-06, + "loss": 1.1167, + "mean_token_accuracy": 0.7485828153789044, + "num_tokens": 3208702.0, + "step": 120 + }, + { + "entropy": 0.9639404848217964, + "epoch": 0.2314454211639034, + "grad_norm": 1.6953125, + "learning_rate": 8.6e-06, + "loss": 1.0979, + "mean_token_accuracy": 0.7553199239075183, + "num_tokens": 3480119.0, + "step": 130 + }, + { + "entropy": 0.9687324531376362, + "epoch": 0.2492489150995883, + "grad_norm": 1.5625, + "learning_rate": 9.266666666666667e-06, + "loss": 1.0762, + "mean_token_accuracy": 0.757096454873681, + "num_tokens": 3746392.0, + "step": 140 + }, + { + "entropy": 0.9682734729722142, + "epoch": 0.26705240903527316, + "grad_norm": 1.875, + "learning_rate": 9.933333333333334e-06, + "loss": 1.0798, + "mean_token_accuracy": 0.7543047085404396, + "num_tokens": 4019487.0, + "step": 150 + }, + { + "entropy": 0.9270784268155694, + "epoch": 0.28485590297095803, + "grad_norm": 1.6015625, + "learning_rate": 1.0600000000000002e-05, + "loss": 1.047, + "mean_token_accuracy": 0.7619738396257162, + "num_tokens": 4298100.0, + "step": 160 + }, + { + "entropy": 0.9063327137380839, + "epoch": 0.30265939690664295, + "grad_norm": 1.5546875, + "learning_rate": 1.1266666666666668e-05, + "loss": 1.0199, + "mean_token_accuracy": 0.7664302848279476, + "num_tokens": 4568116.0, + "step": 170 + }, + { + "entropy": 0.9037324048578739, + "epoch": 0.3204628908423278, + "grad_norm": 1.7109375, + "learning_rate": 1.1933333333333335e-05, + "loss": 1.0144, + "mean_token_accuracy": 0.7657378144562245, + "num_tokens": 4831334.0, + "step": 180 + }, + { + "entropy": 0.8886868219822646, + "epoch": 0.3382663847780127, + "grad_norm": 1.3359375, + "learning_rate": 1.2600000000000001e-05, + "loss": 0.9874, + "mean_token_accuracy": 0.7726409930735827, + "num_tokens": 5103301.0, + "step": 190 + }, + { + "entropy": 0.8936551520600915, + "epoch": 0.35606987871369755, + "grad_norm": 1.3984375, + "learning_rate": 1.3266666666666668e-05, + "loss": 1.019, + "mean_token_accuracy": 0.7648640714585782, + "num_tokens": 5362915.0, + "step": 200 + }, + { + "epoch": 0.35606987871369755, + "eval_biology_entropy": 2.0171296310424807, + "eval_biology_loss": 2.3780677318573, + "eval_biology_mean_token_accuracy": 0.5706260974407196, + "eval_biology_num_tokens": 5362915.0, + "eval_biology_runtime": 39.1203, + "eval_biology_samples_per_second": 12.781, + "eval_biology_steps_per_second": 3.195, + "step": 200 + }, + { + "epoch": 0.35606987871369755, + "eval_chemistry_entropy": 0.8995157589912415, + "eval_chemistry_loss": 1.022930383682251, + "eval_chemistry_mean_token_accuracy": 0.7701377034187317, + "eval_chemistry_num_tokens": 5362915.0, + "eval_chemistry_runtime": 48.7263, + "eval_chemistry_samples_per_second": 10.261, + "eval_chemistry_steps_per_second": 2.565, + "step": 200 + }, + { + "epoch": 0.35606987871369755, + "eval_math_entropy": 0.8352043969631195, + "eval_math_loss": 1.1696399450302124, + "eval_math_mean_token_accuracy": 0.7531440143585205, + "eval_math_num_tokens": 5362915.0, + "eval_math_runtime": 50.1494, + "eval_math_samples_per_second": 9.97, + "eval_math_steps_per_second": 2.493, + "step": 200 + }, + { + "epoch": 0.35606987871369755, + "eval_physics_entropy": 0.8889413847923279, + "eval_physics_loss": 0.9962027072906494, + "eval_physics_mean_token_accuracy": 0.775176504611969, + "eval_physics_num_tokens": 5362915.0, + "eval_physics_runtime": 57.6533, + "eval_physics_samples_per_second": 8.673, + "eval_physics_steps_per_second": 2.168, + "step": 200 + }, + { + "entropy": 0.8509893301874399, + "epoch": 0.3738733726493824, + "grad_norm": 1.3359375, + "learning_rate": 1.3933333333333334e-05, + "loss": 0.9619, + "mean_token_accuracy": 0.7772600032389164, + "num_tokens": 5625455.0, + "step": 210 + }, + { + "entropy": 0.8560685662552714, + "epoch": 0.39167686658506734, + "grad_norm": 1.4609375, + "learning_rate": 1.46e-05, + "loss": 0.9666, + "mean_token_accuracy": 0.7764565829187632, + "num_tokens": 5890806.0, + "step": 220 + }, + { + "entropy": 0.8282079607248306, + "epoch": 0.4094803605207522, + "grad_norm": 1.3828125, + "learning_rate": 1.5266666666666667e-05, + "loss": 0.9353, + "mean_token_accuracy": 0.7817027729004622, + "num_tokens": 6158751.0, + "step": 230 + }, + { + "entropy": 0.8447791969403624, + "epoch": 0.4272838544564371, + "grad_norm": 1.4296875, + "learning_rate": 1.5933333333333336e-05, + "loss": 0.9428, + "mean_token_accuracy": 0.7795210376381874, + "num_tokens": 6423747.0, + "step": 240 + }, + { + "entropy": 0.8478817906230688, + "epoch": 0.44508734839212194, + "grad_norm": 1.8984375, + "learning_rate": 1.66e-05, + "loss": 0.9542, + "mean_token_accuracy": 0.7772787544876337, + "num_tokens": 6684517.0, + "step": 250 + }, + { + "entropy": 0.8135418241843582, + "epoch": 0.4628908423278068, + "grad_norm": 1.453125, + "learning_rate": 1.726666666666667e-05, + "loss": 0.9297, + "mean_token_accuracy": 0.783412617444992, + "num_tokens": 6945659.0, + "step": 260 + }, + { + "entropy": 0.8251218661665917, + "epoch": 0.48069433626349173, + "grad_norm": 1.6640625, + "learning_rate": 1.7933333333333333e-05, + "loss": 0.9307, + "mean_token_accuracy": 0.7831020273268223, + "num_tokens": 7217797.0, + "step": 270 + }, + { + "entropy": 0.7879339458420873, + "epoch": 0.4984978301991766, + "grad_norm": 1.328125, + "learning_rate": 1.86e-05, + "loss": 0.8795, + "mean_token_accuracy": 0.7897050529718399, + "num_tokens": 7498182.0, + "step": 280 + }, + { + "entropy": 0.8145477209240198, + "epoch": 0.5163013241348615, + "grad_norm": 1.265625, + "learning_rate": 1.926666666666667e-05, + "loss": 0.9187, + "mean_token_accuracy": 0.7838029216974973, + "num_tokens": 7761686.0, + "step": 290 + }, + { + "entropy": 0.8231891892850399, + "epoch": 0.5341048180705463, + "grad_norm": 1.5625, + "learning_rate": 1.9933333333333334e-05, + "loss": 0.9271, + "mean_token_accuracy": 0.7819279827177524, + "num_tokens": 8026923.0, + "step": 300 + }, + { + "epoch": 0.5341048180705463, + "eval_biology_entropy": 1.8821040544509888, + "eval_biology_loss": 2.226550340652466, + "eval_biology_mean_token_accuracy": 0.5873979187011719, + "eval_biology_num_tokens": 8026923.0, + "eval_biology_runtime": 39.1096, + "eval_biology_samples_per_second": 12.785, + "eval_biology_steps_per_second": 3.196, + "step": 300 + }, + { + "epoch": 0.5341048180705463, + "eval_chemistry_entropy": 0.7922393922805786, + "eval_chemistry_loss": 0.9438055157661438, + "eval_chemistry_mean_token_accuracy": 0.7841580080986023, + "eval_chemistry_num_tokens": 8026923.0, + "eval_chemistry_runtime": 48.7268, + "eval_chemistry_samples_per_second": 10.261, + "eval_chemistry_steps_per_second": 2.565, + "step": 300 + }, + { + "epoch": 0.5341048180705463, + "eval_math_entropy": 0.7649447858333588, + "eval_math_loss": 1.1145466566085815, + "eval_math_mean_token_accuracy": 0.763204463005066, + "eval_math_num_tokens": 8026923.0, + "eval_math_runtime": 50.1205, + "eval_math_samples_per_second": 9.976, + "eval_math_steps_per_second": 2.494, + "step": 300 + }, + { + "epoch": 0.5341048180705463, + "eval_physics_entropy": 0.7964776389598847, + "eval_physics_loss": 0.9373180866241455, + "eval_physics_mean_token_accuracy": 0.784841582775116, + "eval_physics_num_tokens": 8026923.0, + "eval_physics_runtime": 57.6222, + "eval_physics_samples_per_second": 8.677, + "eval_physics_steps_per_second": 2.169, + "step": 300 + }, + { + "entropy": 0.800599187426269, + "epoch": 0.5519083120062312, + "grad_norm": 1.6640625, + "learning_rate": 1.9933333333333334e-05, + "loss": 0.9071, + "mean_token_accuracy": 0.7856069568544626, + "num_tokens": 8301506.0, + "step": 310 + }, + { + "entropy": 0.7965721761807799, + "epoch": 0.5697118059419161, + "grad_norm": 1.234375, + "learning_rate": 1.985925925925926e-05, + "loss": 0.9139, + "mean_token_accuracy": 0.786777526140213, + "num_tokens": 8559126.0, + "step": 320 + }, + { + "entropy": 0.7925007566809654, + "epoch": 0.587515299877601, + "grad_norm": 1.4453125, + "learning_rate": 1.9785185185185187e-05, + "loss": 0.906, + "mean_token_accuracy": 0.7864235289394855, + "num_tokens": 8821088.0, + "step": 330 + }, + { + "entropy": 0.7728251311928034, + "epoch": 0.6053187938132859, + "grad_norm": 1.2109375, + "learning_rate": 1.971111111111111e-05, + "loss": 0.8764, + "mean_token_accuracy": 0.7919386122375727, + "num_tokens": 9089866.0, + "step": 340 + }, + { + "entropy": 0.7705478269606829, + "epoch": 0.6231222877489707, + "grad_norm": 1.3515625, + "learning_rate": 1.963703703703704e-05, + "loss": 0.8837, + "mean_token_accuracy": 0.7896027479320764, + "num_tokens": 9358866.0, + "step": 350 + }, + { + "entropy": 0.7656249281018972, + "epoch": 0.6409257816846556, + "grad_norm": 1.1875, + "learning_rate": 1.9562962962962964e-05, + "loss": 0.8626, + "mean_token_accuracy": 0.7931958142668009, + "num_tokens": 9629977.0, + "step": 360 + }, + { + "entropy": 0.7598591955378652, + "epoch": 0.6587292756203404, + "grad_norm": 1.296875, + "learning_rate": 1.948888888888889e-05, + "loss": 0.8608, + "mean_token_accuracy": 0.7945109561085701, + "num_tokens": 9897757.0, + "step": 370 + }, + { + "entropy": 0.7398761732503771, + "epoch": 0.6765327695560254, + "grad_norm": 1.265625, + "learning_rate": 1.9414814814814817e-05, + "loss": 0.848, + "mean_token_accuracy": 0.7985429972410202, + "num_tokens": 10168367.0, + "step": 380 + }, + { + "entropy": 0.7331193052232265, + "epoch": 0.6943362634917103, + "grad_norm": 1.4296875, + "learning_rate": 1.9340740740740743e-05, + "loss": 0.8366, + "mean_token_accuracy": 0.8001827634871006, + "num_tokens": 10439039.0, + "step": 390 + }, + { + "entropy": 0.7496613372117281, + "epoch": 0.7121397574273951, + "grad_norm": 1.15625, + "learning_rate": 1.926666666666667e-05, + "loss": 0.8517, + "mean_token_accuracy": 0.7967161379754544, + "num_tokens": 10709549.0, + "step": 400 + }, + { + "epoch": 0.7121397574273951, + "eval_biology_entropy": 1.823248610496521, + "eval_biology_loss": 2.159681558609009, + "eval_biology_mean_token_accuracy": 0.5972214796543122, + "eval_biology_num_tokens": 10709549.0, + "eval_biology_runtime": 39.2024, + "eval_biology_samples_per_second": 12.754, + "eval_biology_steps_per_second": 3.189, + "step": 400 + }, + { + "epoch": 0.7121397574273951, + "eval_chemistry_entropy": 0.7672218294143677, + "eval_chemistry_loss": 0.8971230387687683, + "eval_chemistry_mean_token_accuracy": 0.7928389143943787, + "eval_chemistry_num_tokens": 10709549.0, + "eval_chemistry_runtime": 48.8296, + "eval_chemistry_samples_per_second": 10.24, + "eval_chemistry_steps_per_second": 2.56, + "step": 400 + }, + { + "epoch": 0.7121397574273951, + "eval_math_entropy": 0.7494234163761139, + "eval_math_loss": 1.0849554538726807, + "eval_math_mean_token_accuracy": 0.7666959943771362, + "eval_math_num_tokens": 10709549.0, + "eval_math_runtime": 50.2325, + "eval_math_samples_per_second": 9.954, + "eval_math_steps_per_second": 2.488, + "step": 400 + }, + { + "epoch": 0.7121397574273951, + "eval_physics_entropy": 0.7731240901947022, + "eval_physics_loss": 0.9018430709838867, + "eval_physics_mean_token_accuracy": 0.7909709796905517, + "eval_physics_num_tokens": 10709549.0, + "eval_physics_runtime": 57.7809, + "eval_physics_samples_per_second": 8.653, + "eval_physics_steps_per_second": 2.163, + "step": 400 + }, + { + "entropy": 0.7354462265968322, + "epoch": 0.72994325136308, + "grad_norm": 1.3125, + "learning_rate": 1.9192592592592593e-05, + "loss": 0.8411, + "mean_token_accuracy": 0.8003354988992214, + "num_tokens": 10976011.0, + "step": 410 + }, + { + "entropy": 0.7435122076421976, + "epoch": 0.7477467452987648, + "grad_norm": 1.15625, + "learning_rate": 1.911851851851852e-05, + "loss": 0.8438, + "mean_token_accuracy": 0.7971596572548151, + "num_tokens": 11245251.0, + "step": 420 + }, + { + "entropy": 0.759309609234333, + "epoch": 0.7655502392344498, + "grad_norm": 1.375, + "learning_rate": 1.9044444444444446e-05, + "loss": 0.8579, + "mean_token_accuracy": 0.792990094423294, + "num_tokens": 11518514.0, + "step": 430 + }, + { + "entropy": 0.7545185785740613, + "epoch": 0.7833537331701347, + "grad_norm": 1.3125, + "learning_rate": 1.8970370370370372e-05, + "loss": 0.8574, + "mean_token_accuracy": 0.794682926684618, + "num_tokens": 11787117.0, + "step": 440 + }, + { + "entropy": 0.751254404336214, + "epoch": 0.8011572271058195, + "grad_norm": 1.515625, + "learning_rate": 1.8896296296296295e-05, + "loss": 0.8495, + "mean_token_accuracy": 0.7972703959792853, + "num_tokens": 12050993.0, + "step": 450 + }, + { + "entropy": 0.7543802969157696, + "epoch": 0.8189607210415044, + "grad_norm": 1.28125, + "learning_rate": 1.8822222222222225e-05, + "loss": 0.8527, + "mean_token_accuracy": 0.7960003320127725, + "num_tokens": 12311688.0, + "step": 460 + }, + { + "entropy": 0.7186065271496773, + "epoch": 0.8367642149771892, + "grad_norm": 1.15625, + "learning_rate": 1.874814814814815e-05, + "loss": 0.8096, + "mean_token_accuracy": 0.8027401898056269, + "num_tokens": 12598408.0, + "step": 470 + }, + { + "entropy": 0.7257912319153548, + "epoch": 0.8545677089128741, + "grad_norm": 1.125, + "learning_rate": 1.8674074074074075e-05, + "loss": 0.8299, + "mean_token_accuracy": 0.8012042317539454, + "num_tokens": 12869642.0, + "step": 480 + }, + { + "entropy": 0.7173270899802446, + "epoch": 0.8723712028485591, + "grad_norm": 1.125, + "learning_rate": 1.86e-05, + "loss": 0.8146, + "mean_token_accuracy": 0.8032398864626884, + "num_tokens": 13139555.0, + "step": 490 + }, + { + "entropy": 0.7487573977559805, + "epoch": 0.8901746967842439, + "grad_norm": 1.2421875, + "learning_rate": 1.8525925925925928e-05, + "loss": 0.8562, + "mean_token_accuracy": 0.7955521304160357, + "num_tokens": 13396545.0, + "step": 500 + }, + { + "epoch": 0.8901746967842439, + "eval_biology_entropy": 1.8204833917617798, + "eval_biology_loss": 2.1014022827148438, + "eval_biology_mean_token_accuracy": 0.6051526765823364, + "eval_biology_num_tokens": 13396545.0, + "eval_biology_runtime": 39.1543, + "eval_biology_samples_per_second": 12.77, + "eval_biology_steps_per_second": 3.192, + "step": 500 + }, + { + "epoch": 0.8901746967842439, + "eval_chemistry_entropy": 0.7538458592891694, + "eval_chemistry_loss": 0.8699584007263184, + "eval_chemistry_mean_token_accuracy": 0.7975720787048339, + "eval_chemistry_num_tokens": 13396545.0, + "eval_chemistry_runtime": 48.8229, + "eval_chemistry_samples_per_second": 10.241, + "eval_chemistry_steps_per_second": 2.56, + "step": 500 + }, + { + "epoch": 0.8901746967842439, + "eval_math_entropy": 0.7398368701934814, + "eval_math_loss": 1.0758970975875854, + "eval_math_mean_token_accuracy": 0.7687290472984314, + "eval_math_num_tokens": 13396545.0, + "eval_math_runtime": 50.0916, + "eval_math_samples_per_second": 9.982, + "eval_math_steps_per_second": 2.495, + "step": 500 + }, + { + "epoch": 0.8901746967842439, + "eval_physics_entropy": 0.7607374501228332, + "eval_physics_loss": 0.8848790526390076, + "eval_physics_mean_token_accuracy": 0.7937801394462586, + "eval_physics_num_tokens": 13396545.0, + "eval_physics_runtime": 57.6599, + "eval_physics_samples_per_second": 8.672, + "eval_physics_steps_per_second": 2.168, + "step": 500 + } + ], + "logging_steps": 10, + "max_steps": 3000, + "num_input_tokens_seen": 0, + "num_train_epochs": 6, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.0399995579963064e+18, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/checkpoint-500/training_args.bin b/checkpoint-500/training_args.bin new file mode 100644 index 0000000..79cad08 --- /dev/null +++ b/checkpoint-500/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dfc23fa1b9be77c605cba9733debba5f60170f0eadaa2a8866b5aaf4b0e65756 +size 6417 diff --git a/config.json b/config.json new file mode 100644 index 0000000..29e6cbe --- /dev/null +++ b/config.json @@ -0,0 +1,77 @@ +{ + "architectures": [ + "Gemma2ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "attn_logit_softcapping": 50.0, + "bos_token_id": 2, + "cache_implementation": "hybrid", + "dtype": "bfloat16", + "eos_token_id": 1, + "final_logit_softcapping": 30.0, + "head_dim": 256, + "hidden_act": "gelu_pytorch_tanh", + "hidden_activation": "gelu_pytorch_tanh", + "hidden_size": 3584, + "initializer_range": 0.02, + "intermediate_size": 14336, + "layer_types": [ + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention" + ], + "max_position_embeddings": 8192, + "model_type": "gemma2", + "num_attention_heads": 16, + "num_hidden_layers": 42, + "num_key_value_heads": 8, + "pad_token_id": 0, + "query_pre_attn_scalar": 256, + "rms_norm_eps": 1e-06, + "rope_theta": 10000.0, + "sliding_window": 4096, + "sliding_window_size": 4096, + "transformers_version": "4.56.1", + "use_cache": true, + "vocab_size": 256000 +} diff --git a/final/chat_template.jinja b/final/chat_template.jinja new file mode 100644 index 0000000..923ec25 --- /dev/null +++ b/final/chat_template.jinja @@ -0,0 +1,4 @@ +{{ bos_token }}{% if messages[0]['role'] == 'system' %}{{ raise_exception('System role not supported') }}{% endif %}{% for message in messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if (message['role'] == 'assistant') %}{% set role = 'model' %}{% else %}{% set role = message['role'] %}{% endif %}{{ '' + role + ' +' + message['content'] | trim + ' +' }}{% endfor %}{% if add_generation_prompt %}{{'model +'}}{% endif %} \ No newline at end of file diff --git a/final/config.json b/final/config.json new file mode 100644 index 0000000..29e6cbe --- /dev/null +++ b/final/config.json @@ -0,0 +1,77 @@ +{ + "architectures": [ + "Gemma2ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "attn_logit_softcapping": 50.0, + "bos_token_id": 2, + "cache_implementation": "hybrid", + "dtype": "bfloat16", + "eos_token_id": 1, + "final_logit_softcapping": 30.0, + "head_dim": 256, + "hidden_act": "gelu_pytorch_tanh", + "hidden_activation": "gelu_pytorch_tanh", + "hidden_size": 3584, + "initializer_range": 0.02, + "intermediate_size": 14336, + "layer_types": [ + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention" + ], + "max_position_embeddings": 8192, + "model_type": "gemma2", + "num_attention_heads": 16, + "num_hidden_layers": 42, + "num_key_value_heads": 8, + "pad_token_id": 0, + "query_pre_attn_scalar": 256, + "rms_norm_eps": 1e-06, + "rope_theta": 10000.0, + "sliding_window": 4096, + "sliding_window_size": 4096, + "transformers_version": "4.56.1", + "use_cache": true, + "vocab_size": 256000 +} diff --git a/final/generation_config.json b/final/generation_config.json new file mode 100644 index 0000000..f2c34f2 --- /dev/null +++ b/final/generation_config.json @@ -0,0 +1,9 @@ +{ + "_from_model_config": true, + "bos_token_id": 2, + "eos_token_id": [ + 1 + ], + "pad_token_id": 0, + "transformers_version": "4.56.1" +} diff --git a/final/model-00001-of-00004.safetensors b/final/model-00001-of-00004.safetensors new file mode 100644 index 0000000..4705c7e --- /dev/null +++ b/final/model-00001-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a0d4eb4fcbddfe01b0dc7a58386a3980826ededaf0ecd67d1ef38995b89615ba +size 4903351912 diff --git a/final/model-00002-of-00004.safetensors b/final/model-00002-of-00004.safetensors new file mode 100644 index 0000000..744b59b --- /dev/null +++ b/final/model-00002-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:712a589c333f49f7b6f3870838a9e0ab4b3c0260abd5130e8c54316e6c99640b +size 4947570872 diff --git a/final/model-00003-of-00004.safetensors b/final/model-00003-of-00004.safetensors new file mode 100644 index 0000000..e6eaa38 --- /dev/null +++ b/final/model-00003-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:123eeb1ba62c1bf1fa49cd38911ea80674b44fe122d80d1ee0aeba8d01bb988e +size 4962221464 diff --git a/final/model-00004-of-00004.safetensors b/final/model-00004-of-00004.safetensors new file mode 100644 index 0000000..4707165 --- /dev/null +++ b/final/model-00004-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6754a347e44efa3115fb68ec8aded7512b7e469a1046d2bad6b7182c5500d23a +size 3670322200 diff --git a/final/model.safetensors.index.json b/final/model.safetensors.index.json new file mode 100644 index 0000000..3b569c5 --- /dev/null +++ b/final/model.safetensors.index.json @@ -0,0 +1,472 @@ +{ + "metadata": { + "total_parameters": 9241705984, + "total_size": 18483411968 + }, + "weight_map": { + "model.embed_tokens.weight": "model-00001-of-00004.safetensors", + "model.layers.0.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.0.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.0.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.1.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.1.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.10.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.10.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.10.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.10.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.11.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.11.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.11.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.12.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.12.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.12.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.13.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.13.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.13.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.14.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.14.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.14.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.15.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.15.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.15.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.16.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.16.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.16.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.17.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.17.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.17.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.18.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.18.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.18.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.19.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.19.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.19.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.2.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.2.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.2.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.20.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.20.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.20.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.20.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.20.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.20.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.20.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.21.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.21.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.21.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.21.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.21.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.22.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.22.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.22.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.22.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.23.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.23.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.23.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.24.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.24.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.24.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.25.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.25.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.25.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.26.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.26.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.26.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.27.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.27.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.27.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.28.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.28.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.28.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.29.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.29.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.29.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.3.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.3.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.3.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.30.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.30.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.30.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.30.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.30.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.31.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.31.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.31.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.32.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.32.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.32.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.32.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.32.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.32.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.32.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.32.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.32.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.32.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.32.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.33.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.33.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.33.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.33.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.33.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.33.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.33.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.33.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.33.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.33.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.33.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.34.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.34.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.34.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.34.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.34.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.34.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.34.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.34.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.34.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.34.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.34.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.35.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.35.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.35.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.35.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.35.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.35.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.35.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.35.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.35.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.35.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.35.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.36.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.36.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.36.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.36.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.36.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.36.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.36.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.36.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.36.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.36.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.36.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.37.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.37.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.37.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.37.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.37.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.37.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.37.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.37.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.37.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.37.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.37.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.38.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.38.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.38.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.38.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.38.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.38.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.38.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.38.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.38.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.38.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.38.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.39.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.39.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.39.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.39.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.39.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.39.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.39.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.39.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.39.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.39.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.39.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.4.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.4.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.4.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.4.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.40.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.40.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.40.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.40.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.40.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.40.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.40.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.40.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.40.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.40.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.40.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.41.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.41.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.41.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.41.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.41.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.41.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.41.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.41.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.41.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.41.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.41.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.5.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.5.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.5.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.5.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.6.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.6.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.6.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.7.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.7.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.7.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.8.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.8.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.8.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.8.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.8.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.8.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.8.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.8.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.8.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.8.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.8.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.9.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.9.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.9.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.9.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.norm.weight": "model-00004-of-00004.safetensors" + } +} diff --git a/final/special_tokens_map.json b/final/special_tokens_map.json new file mode 100644 index 0000000..8d6368f --- /dev/null +++ b/final/special_tokens_map.json @@ -0,0 +1,34 @@ +{ + "additional_special_tokens": [ + "", + "" + ], + "bos_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "unk_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + } +} diff --git a/final/tokenizer.json b/final/tokenizer.json new file mode 100644 index 0000000..a4a305d --- /dev/null +++ b/final/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5f7eee611703c5ce5d1eee32d9cdcfe465647b8aff0c1dfb3bed7ad7dbb05060 +size 34362873 diff --git a/final/tokenizer.model b/final/tokenizer.model new file mode 100644 index 0000000..796efe9 --- /dev/null +++ b/final/tokenizer.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:61a7b147390c64585d6c3543dd6fc636906c9af3865a5548f27f31aee1d4c8e2 +size 4241003 diff --git a/final/tokenizer_config.json b/final/tokenizer_config.json new file mode 100644 index 0000000..3ade6be --- /dev/null +++ b/final/tokenizer_config.json @@ -0,0 +1,2013 @@ +{ + "add_bos_token": true, + "add_eos_token": false, + "added_tokens_decoder": { + "0": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "1": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "2": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "3": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "4": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "5": { + "content": "<2mass>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "6": { + "content": "[@BOS@]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "7": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "8": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "9": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "10": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "11": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "12": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "13": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "14": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "15": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "16": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "17": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "18": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "19": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "20": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "21": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "22": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "23": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "24": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "25": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "26": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "27": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "28": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "29": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "30": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "31": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "32": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "33": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "34": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "35": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "36": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "37": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "38": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "39": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "40": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "41": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "42": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "43": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "44": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "45": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "46": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "47": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "48": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "49": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "50": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "51": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "52": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "53": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "54": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "55": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "56": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "57": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "58": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "59": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "60": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "61": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "62": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "63": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "64": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "65": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "66": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "67": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "68": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "69": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "70": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "71": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "72": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "73": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "74": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "75": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "76": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "77": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "78": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "79": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "80": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "81": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "82": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "83": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "84": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "85": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "86": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "87": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "88": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "89": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "90": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "91": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "92": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "93": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "94": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "95": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "96": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "97": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "98": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "99": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "100": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "101": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "102": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "103": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "104": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "105": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "106": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "107": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "108": { + "content": "\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "109": { + "content": "\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "110": { + "content": "\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "111": { + "content": "\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "112": { + "content": "\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "113": { + "content": "\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "114": { + "content": "\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "115": { + "content": "\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "116": { + "content": "\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "117": { + "content": "\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "118": { + "content": "\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "119": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "120": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "121": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "122": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "123": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "124": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "125": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "126": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "127": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "128": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "129": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "130": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "131": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "132": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "133": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "134": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "135": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "136": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "137": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "138": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "139": { + "content": "▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "140": { + "content": "▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "141": { + "content": "▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "142": { + "content": "▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "143": { + "content": "▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "144": { + "content": "▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "145": { + "content": "▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "146": { + "content": "▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "147": { + "content": "▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "148": { + "content": "▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "149": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "150": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "152": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "153": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "154": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "155": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "156": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "157": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "158": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "159": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "160": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "161": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "162": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "163": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "164": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "165": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "166": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "167": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "168": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "169": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "170": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "172": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "173": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "174": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "175": { + "content": "
", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "171": { + "content": "
", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "176": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "177": { + "content": "
", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "178": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "179": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "180": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "181": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "182": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "183": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "184": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "185": { + "content": "

", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "186": { + "content": "

", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "187": { + "content": "

", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "188": { + "content": "

", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "189": { + "content": "

", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "190": { + "content": "
", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "191": { + "content": "
", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "192": { + "content": "
", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "193": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "194": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "195": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "196": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "197": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "198": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "199": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "200": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "201": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "202": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "203": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "204": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "205": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "206": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "207": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "208": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "209": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "210": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "211": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "212": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "213": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "214": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "215": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "216": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255968": { + "content": "[toxicity=0]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255969": { + "content": "\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255970": { + "content": "\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255971": { + "content": "\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255972": { + "content": "\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255973": { + "content": "\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255974": { + "content": "\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255975": { + "content": "\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255976": { + "content": "\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255977": { + "content": "\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255978": { + "content": "\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255979": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255980": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255981": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255982": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255983": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255984": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255985": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255986": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255987": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255988": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255989": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255990": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255991": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255992": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255993": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255994": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255995": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255996": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255997": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255998": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255999": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + } + }, + "additional_special_tokens": [ + "", + "" + ], + "bos_token": "", + "clean_up_tokenization_spaces": false, + "eos_token": "", + "extra_special_tokens": {}, + "model_max_length": 1000000000000000019884624838656, + "pad_token": "", + "sp_model_kwargs": {}, + "spaces_between_special_tokens": false, + "tokenizer_class": "GemmaTokenizer", + "unk_token": "", + "use_default_system_prompt": false +} diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..f2c34f2 --- /dev/null +++ b/generation_config.json @@ -0,0 +1,9 @@ +{ + "_from_model_config": true, + "bos_token_id": 2, + "eos_token_id": [ + 1 + ], + "pad_token_id": 0, + "transformers_version": "4.56.1" +} diff --git a/model-00001-of-00004.safetensors b/model-00001-of-00004.safetensors new file mode 100644 index 0000000..4705c7e --- /dev/null +++ b/model-00001-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a0d4eb4fcbddfe01b0dc7a58386a3980826ededaf0ecd67d1ef38995b89615ba +size 4903351912 diff --git a/model-00002-of-00004.safetensors b/model-00002-of-00004.safetensors new file mode 100644 index 0000000..744b59b --- /dev/null +++ b/model-00002-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:712a589c333f49f7b6f3870838a9e0ab4b3c0260abd5130e8c54316e6c99640b +size 4947570872 diff --git a/model-00003-of-00004.safetensors b/model-00003-of-00004.safetensors new file mode 100644 index 0000000..e6eaa38 --- /dev/null +++ b/model-00003-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:123eeb1ba62c1bf1fa49cd38911ea80674b44fe122d80d1ee0aeba8d01bb988e +size 4962221464 diff --git a/model-00004-of-00004.safetensors b/model-00004-of-00004.safetensors new file mode 100644 index 0000000..4707165 --- /dev/null +++ b/model-00004-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6754a347e44efa3115fb68ec8aded7512b7e469a1046d2bad6b7182c5500d23a +size 3670322200 diff --git a/model.safetensors.index.json b/model.safetensors.index.json new file mode 100644 index 0000000..3b569c5 --- /dev/null +++ b/model.safetensors.index.json @@ -0,0 +1,472 @@ +{ + "metadata": { + "total_parameters": 9241705984, + "total_size": 18483411968 + }, + "weight_map": { + "model.embed_tokens.weight": "model-00001-of-00004.safetensors", + "model.layers.0.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.0.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.0.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.1.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.1.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.10.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.10.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.10.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.10.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.11.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.11.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.11.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.12.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.12.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.12.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.13.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.13.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.13.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.14.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.14.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.14.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.15.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.15.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.15.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.16.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.16.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.16.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.17.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.17.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.17.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.18.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.18.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.18.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.19.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.19.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.19.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.2.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.2.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.2.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.20.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.20.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.20.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.20.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.20.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.20.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.20.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.21.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.21.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.21.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.21.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.21.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.22.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.22.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.22.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.22.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.23.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.23.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.23.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.24.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.24.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.24.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.25.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.25.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.25.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.26.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.26.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.26.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.27.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.27.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.27.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.28.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.28.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.28.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.29.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.29.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.29.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.3.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.3.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.3.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.30.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.30.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.30.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.30.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.30.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.31.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.31.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.31.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.32.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.32.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.32.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.32.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.32.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.32.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.32.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.32.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.32.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.32.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.32.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.33.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.33.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.33.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.33.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.33.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.33.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.33.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.33.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.33.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.33.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.33.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.34.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.34.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.34.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.34.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.34.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.34.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.34.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.34.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.34.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.34.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.34.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.35.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.35.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.35.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.35.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.35.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.35.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.35.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.35.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.35.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.35.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.35.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.36.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.36.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.36.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.36.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.36.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.36.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.36.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.36.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.36.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.36.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.36.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.37.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.37.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.37.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.37.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.37.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.37.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.37.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.37.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.37.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.37.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.37.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.38.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.38.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.38.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.38.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.38.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.38.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.38.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.38.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.38.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.38.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.38.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.39.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.39.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.39.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.39.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.39.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.39.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.39.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.39.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.39.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.39.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.39.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.4.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.4.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.4.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.4.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.40.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.40.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.40.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.40.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.40.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.40.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.40.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.40.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.40.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.40.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.40.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.41.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.41.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.41.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.41.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.41.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.41.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.41.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.41.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.41.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.41.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.41.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.5.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.5.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.5.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.5.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.6.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.6.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.6.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.7.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.7.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.7.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.8.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.8.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.8.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.8.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.8.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.8.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.8.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.8.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.8.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.8.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.8.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.9.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.9.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.9.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.9.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.norm.weight": "model-00004-of-00004.safetensors" + } +} diff --git a/special_tokens_map.json b/special_tokens_map.json new file mode 100644 index 0000000..8d6368f --- /dev/null +++ b/special_tokens_map.json @@ -0,0 +1,34 @@ +{ + "additional_special_tokens": [ + "", + "" + ], + "bos_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "unk_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + } +} diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..a4a305d --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5f7eee611703c5ce5d1eee32d9cdcfe465647b8aff0c1dfb3bed7ad7dbb05060 +size 34362873 diff --git a/tokenizer.model b/tokenizer.model new file mode 100644 index 0000000..796efe9 --- /dev/null +++ b/tokenizer.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:61a7b147390c64585d6c3543dd6fc636906c9af3865a5548f27f31aee1d4c8e2 +size 4241003 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..3ade6be --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,2013 @@ +{ + "add_bos_token": true, + "add_eos_token": false, + "added_tokens_decoder": { + "0": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "1": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "2": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "3": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "4": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "5": { + "content": "<2mass>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "6": { + "content": "[@BOS@]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "7": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "8": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "9": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "10": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "11": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "12": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "13": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "14": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "15": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "16": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "17": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "18": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "19": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "20": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "21": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "22": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "23": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "24": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "25": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "26": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "27": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "28": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "29": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "30": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "31": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "32": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "33": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "34": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "35": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "36": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "37": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "38": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "39": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "40": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "41": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "42": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "43": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "44": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "45": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "46": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "47": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "48": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "49": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "50": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "51": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "52": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "53": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "54": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "55": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "56": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "57": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "58": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "59": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "60": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "61": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "62": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "63": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "64": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "65": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "66": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "67": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "68": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "69": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "70": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "71": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "72": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "73": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "74": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "75": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "76": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "77": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "78": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "79": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "80": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "81": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "82": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "83": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "84": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "85": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "86": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "87": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "88": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "89": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "90": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "91": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "92": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "93": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "94": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "95": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "96": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "97": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "98": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "99": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "100": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "101": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "102": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "103": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "104": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "105": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "106": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "107": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "108": { + "content": "\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "109": { + "content": "\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "110": { + "content": "\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "111": { + "content": "\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "112": { + "content": "\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "113": { + "content": "\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "114": { + "content": "\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "115": { + "content": "\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "116": { + "content": "\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "117": { + "content": "\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "118": { + "content": "\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "119": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "120": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "121": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "122": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "123": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "124": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "125": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "126": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "127": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "128": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "129": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "130": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "131": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "132": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "133": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "134": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "135": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "136": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "137": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "138": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "139": { + "content": "▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "140": { + "content": "▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "141": { + "content": "▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "142": { + "content": "▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "143": { + "content": "▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "144": { + "content": "▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "145": { + "content": "▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "146": { + "content": "▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "147": { + "content": "▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "148": { + "content": "▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "149": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "150": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "152": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "153": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "154": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "155": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "156": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "157": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "158": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "159": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "160": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "161": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "162": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "163": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "164": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "165": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "166": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "167": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "168": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "169": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "170": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "172": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "173": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "174": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "175": { + "content": "
", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "171": { + "content": "
", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "176": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "177": { + "content": "
", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "178": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "179": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "180": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "181": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "182": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "183": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "184": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "185": { + "content": "

", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "186": { + "content": "

", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "187": { + "content": "

", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "188": { + "content": "

", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "189": { + "content": "

", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "190": { + "content": "
", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "191": { + "content": "
", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "192": { + "content": "
", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "193": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "194": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "195": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "196": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "197": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "198": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "199": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "200": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "201": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "202": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "203": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "204": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "205": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "206": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "207": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "208": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "209": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "210": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "211": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "212": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "213": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "214": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "215": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "216": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255968": { + "content": "[toxicity=0]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255969": { + "content": "\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255970": { + "content": "\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255971": { + "content": "\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255972": { + "content": "\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255973": { + "content": "\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255974": { + "content": "\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255975": { + "content": "\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255976": { + "content": "\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255977": { + "content": "\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255978": { + "content": "\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255979": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255980": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255981": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255982": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255983": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255984": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255985": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255986": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255987": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255988": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255989": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255990": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255991": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255992": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255993": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255994": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255995": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255996": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255997": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255998": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255999": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + } + }, + "additional_special_tokens": [ + "", + "" + ], + "bos_token": "", + "clean_up_tokenization_spaces": false, + "eos_token": "", + "extra_special_tokens": {}, + "model_max_length": 1000000000000000019884624838656, + "pad_token": "", + "sp_model_kwargs": {}, + "spaces_between_special_tokens": false, + "tokenizer_class": "GemmaTokenizer", + "unk_token": "", + "use_default_system_prompt": false +}