From 8398654810aa787e16a81c1a3a73c7e7d8b9df58 Mon Sep 17 00:00:00 2001 From: ModelHub XC Date: Mon, 7 Sep 2026 15:58:16 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=EF=BC=8C=E7=94=B1ModelHub=20XC=E7=A4=BE=E5=8C=BA=E6=8F=90?= =?UTF-8?q?=E4=BE=9B=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model: roonbug/11sivxlz Source: Original Platform --- .gitattributes | 40 + README.md | 201 ++ chat_template.jinja | 4 + checkpoint-1000/chat_template.jinja | 4 + checkpoint-1000/config.json | 77 + checkpoint-1000/generation_config.json | 9 + .../model-00001-of-00004.safetensors | 3 + .../model-00002-of-00004.safetensors | 3 + .../model-00003-of-00004.safetensors | 3 + .../model-00004-of-00004.safetensors | 3 + checkpoint-1000/model.safetensors.index.json | 472 ++++ checkpoint-1000/optimizer.pt | 3 + checkpoint-1000/rng_state.pth | 3 + checkpoint-1000/scheduler.pt | 3 + checkpoint-1000/special_tokens_map.json | 34 + checkpoint-1000/tokenizer.json | 3 + checkpoint-1000/tokenizer.model | 3 + checkpoint-1000/tokenizer_config.json | 2013 ++++++++++++++ checkpoint-1000/trainer_state.json | 1254 +++++++++ checkpoint-1000/training_args.bin | 3 + checkpoint-1500/chat_template.jinja | 4 + checkpoint-1500/config.json | 77 + checkpoint-1500/generation_config.json | 9 + .../model-00001-of-00004.safetensors | 3 + .../model-00002-of-00004.safetensors | 3 + .../model-00003-of-00004.safetensors | 3 + .../model-00004-of-00004.safetensors | 3 + checkpoint-1500/model.safetensors.index.json | 472 ++++ checkpoint-1500/optimizer.pt | 3 + checkpoint-1500/rng_state.pth | 3 + checkpoint-1500/scheduler.pt | 3 + checkpoint-1500/special_tokens_map.json | 34 + checkpoint-1500/tokenizer.json | 3 + checkpoint-1500/tokenizer.model | 3 + checkpoint-1500/tokenizer_config.json | 2013 ++++++++++++++ checkpoint-1500/trainer_state.json | 1864 +++++++++++++ checkpoint-1500/training_args.bin | 3 + checkpoint-2000/chat_template.jinja | 4 + checkpoint-2000/config.json | 77 + checkpoint-2000/generation_config.json | 9 + .../model-00001-of-00004.safetensors | 3 + .../model-00002-of-00004.safetensors | 3 + .../model-00003-of-00004.safetensors | 3 + .../model-00004-of-00004.safetensors | 3 + checkpoint-2000/model.safetensors.index.json | 472 ++++ checkpoint-2000/optimizer.pt | 3 + checkpoint-2000/rng_state.pth | 3 + checkpoint-2000/scheduler.pt | 3 + checkpoint-2000/special_tokens_map.json | 34 + checkpoint-2000/tokenizer.json | 3 + checkpoint-2000/tokenizer.model | 3 + checkpoint-2000/tokenizer_config.json | 2013 ++++++++++++++ checkpoint-2000/trainer_state.json | 2474 +++++++++++++++++ checkpoint-2000/training_args.bin | 3 + checkpoint-500/chat_template.jinja | 4 + checkpoint-500/config.json | 77 + checkpoint-500/generation_config.json | 9 + .../model-00001-of-00004.safetensors | 3 + .../model-00002-of-00004.safetensors | 3 + .../model-00003-of-00004.safetensors | 3 + .../model-00004-of-00004.safetensors | 3 + checkpoint-500/model.safetensors.index.json | 472 ++++ checkpoint-500/optimizer.pt | 3 + checkpoint-500/rng_state.pth | 3 + checkpoint-500/scheduler.pt | 3 + checkpoint-500/special_tokens_map.json | 34 + checkpoint-500/tokenizer.json | 3 + checkpoint-500/tokenizer.model | 3 + checkpoint-500/tokenizer_config.json | 2013 ++++++++++++++ checkpoint-500/trainer_state.json | 644 +++++ checkpoint-500/training_args.bin | 3 + config.json | 77 + generation_config.json | 9 + model-00001-of-00004.safetensors | 3 + model-00002-of-00004.safetensors | 3 + model-00003-of-00004.safetensors | 3 + model-00004-of-00004.safetensors | 3 + model.safetensors.index.json | 472 ++++ special_tokens_map.json | 34 + tokenizer.json | 3 + tokenizer.model | 3 + tokenizer_config.json | 2013 ++++++++++++++ 82 files changed, 19660 insertions(+) create mode 100644 .gitattributes create mode 100644 README.md create mode 100644 chat_template.jinja create mode 100644 checkpoint-1000/chat_template.jinja create mode 100644 checkpoint-1000/config.json create mode 100644 checkpoint-1000/generation_config.json create mode 100644 checkpoint-1000/model-00001-of-00004.safetensors create mode 100644 checkpoint-1000/model-00002-of-00004.safetensors create mode 100644 checkpoint-1000/model-00003-of-00004.safetensors create mode 100644 checkpoint-1000/model-00004-of-00004.safetensors create mode 100644 checkpoint-1000/model.safetensors.index.json create mode 100644 checkpoint-1000/optimizer.pt create mode 100644 checkpoint-1000/rng_state.pth create mode 100644 checkpoint-1000/scheduler.pt create mode 100644 checkpoint-1000/special_tokens_map.json create mode 100644 checkpoint-1000/tokenizer.json create mode 100644 checkpoint-1000/tokenizer.model create mode 100644 checkpoint-1000/tokenizer_config.json create mode 100644 checkpoint-1000/trainer_state.json create mode 100644 checkpoint-1000/training_args.bin create mode 100644 checkpoint-1500/chat_template.jinja create mode 100644 checkpoint-1500/config.json create mode 100644 checkpoint-1500/generation_config.json create mode 100644 checkpoint-1500/model-00001-of-00004.safetensors create mode 100644 checkpoint-1500/model-00002-of-00004.safetensors create mode 100644 checkpoint-1500/model-00003-of-00004.safetensors create mode 100644 checkpoint-1500/model-00004-of-00004.safetensors create mode 100644 checkpoint-1500/model.safetensors.index.json create mode 100644 checkpoint-1500/optimizer.pt create mode 100644 checkpoint-1500/rng_state.pth create mode 100644 checkpoint-1500/scheduler.pt create mode 100644 checkpoint-1500/special_tokens_map.json create mode 100644 checkpoint-1500/tokenizer.json create mode 100644 checkpoint-1500/tokenizer.model create mode 100644 checkpoint-1500/tokenizer_config.json create mode 100644 checkpoint-1500/trainer_state.json create mode 100644 checkpoint-1500/training_args.bin create mode 100644 checkpoint-2000/chat_template.jinja create mode 100644 checkpoint-2000/config.json create mode 100644 checkpoint-2000/generation_config.json create mode 100644 checkpoint-2000/model-00001-of-00004.safetensors create mode 100644 checkpoint-2000/model-00002-of-00004.safetensors create mode 100644 checkpoint-2000/model-00003-of-00004.safetensors create mode 100644 checkpoint-2000/model-00004-of-00004.safetensors create mode 100644 checkpoint-2000/model.safetensors.index.json create mode 100644 checkpoint-2000/optimizer.pt create mode 100644 checkpoint-2000/rng_state.pth create mode 100644 checkpoint-2000/scheduler.pt create mode 100644 checkpoint-2000/special_tokens_map.json create mode 100644 checkpoint-2000/tokenizer.json create mode 100644 checkpoint-2000/tokenizer.model create mode 100644 checkpoint-2000/tokenizer_config.json create mode 100644 checkpoint-2000/trainer_state.json create mode 100644 checkpoint-2000/training_args.bin create mode 100644 checkpoint-500/chat_template.jinja create mode 100644 checkpoint-500/config.json create mode 100644 checkpoint-500/generation_config.json create mode 100644 checkpoint-500/model-00001-of-00004.safetensors create mode 100644 checkpoint-500/model-00002-of-00004.safetensors create mode 100644 checkpoint-500/model-00003-of-00004.safetensors create mode 100644 checkpoint-500/model-00004-of-00004.safetensors create mode 100644 checkpoint-500/model.safetensors.index.json create mode 100644 checkpoint-500/optimizer.pt create mode 100644 checkpoint-500/rng_state.pth create mode 100644 checkpoint-500/scheduler.pt create mode 100644 checkpoint-500/special_tokens_map.json create mode 100644 checkpoint-500/tokenizer.json create mode 100644 checkpoint-500/tokenizer.model create mode 100644 checkpoint-500/tokenizer_config.json create mode 100644 checkpoint-500/trainer_state.json create mode 100644 checkpoint-500/training_args.bin create mode 100644 config.json create mode 100644 generation_config.json create mode 100644 model-00001-of-00004.safetensors create mode 100644 model-00002-of-00004.safetensors create mode 100644 model-00003-of-00004.safetensors create mode 100644 model-00004-of-00004.safetensors create mode 100644 model.safetensors.index.json create mode 100644 special_tokens_map.json create mode 100644 tokenizer.json create mode 100644 tokenizer.model create mode 100644 tokenizer_config.json diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..720c030 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,40 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +checkpoint-500/tokenizer.json filter=lfs diff=lfs merge=lfs -text +checkpoint-1000/tokenizer.json filter=lfs diff=lfs merge=lfs -text +checkpoint-1500/tokenizer.json filter=lfs diff=lfs merge=lfs -text +checkpoint-2000/tokenizer.json filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..2480968 --- /dev/null +++ b/README.md @@ -0,0 +1,201 @@ +--- +library_name: transformers +tags: +- trl +- sft +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + +This is the model card of a 🤗 transformers model that has been pushed on the Hub. This model card has been automatically generated. + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] \ No newline at end of file diff --git a/chat_template.jinja b/chat_template.jinja new file mode 100644 index 0000000..923ec25 --- /dev/null +++ b/chat_template.jinja @@ -0,0 +1,4 @@ +{{ bos_token }}{% if messages[0]['role'] == 'system' %}{{ raise_exception('System role not supported') }}{% endif %}{% for message in messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if (message['role'] == 'assistant') %}{% set role = 'model' %}{% else %}{% set role = message['role'] %}{% endif %}{{ '' + role + ' +' + message['content'] | trim + ' +' }}{% endfor %}{% if add_generation_prompt %}{{'model +'}}{% endif %} \ No newline at end of file diff --git a/checkpoint-1000/chat_template.jinja b/checkpoint-1000/chat_template.jinja new file mode 100644 index 0000000..923ec25 --- /dev/null +++ b/checkpoint-1000/chat_template.jinja @@ -0,0 +1,4 @@ +{{ bos_token }}{% if messages[0]['role'] == 'system' %}{{ raise_exception('System role not supported') }}{% endif %}{% for message in messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if (message['role'] == 'assistant') %}{% set role = 'model' %}{% else %}{% set role = message['role'] %}{% endif %}{{ '' + role + ' +' + message['content'] | trim + ' +' }}{% endfor %}{% if add_generation_prompt %}{{'model +'}}{% endif %} \ No newline at end of file diff --git a/checkpoint-1000/config.json b/checkpoint-1000/config.json new file mode 100644 index 0000000..29e6cbe --- /dev/null +++ b/checkpoint-1000/config.json @@ -0,0 +1,77 @@ +{ + "architectures": [ + "Gemma2ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "attn_logit_softcapping": 50.0, + "bos_token_id": 2, + "cache_implementation": "hybrid", + "dtype": "bfloat16", + "eos_token_id": 1, + "final_logit_softcapping": 30.0, + "head_dim": 256, + "hidden_act": "gelu_pytorch_tanh", + "hidden_activation": "gelu_pytorch_tanh", + "hidden_size": 3584, + "initializer_range": 0.02, + "intermediate_size": 14336, + "layer_types": [ + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention" + ], + "max_position_embeddings": 8192, + "model_type": "gemma2", + "num_attention_heads": 16, + "num_hidden_layers": 42, + "num_key_value_heads": 8, + "pad_token_id": 0, + "query_pre_attn_scalar": 256, + "rms_norm_eps": 1e-06, + "rope_theta": 10000.0, + "sliding_window": 4096, + "sliding_window_size": 4096, + "transformers_version": "4.56.1", + "use_cache": true, + "vocab_size": 256000 +} diff --git a/checkpoint-1000/generation_config.json b/checkpoint-1000/generation_config.json new file mode 100644 index 0000000..f2c34f2 --- /dev/null +++ b/checkpoint-1000/generation_config.json @@ -0,0 +1,9 @@ +{ + "_from_model_config": true, + "bos_token_id": 2, + "eos_token_id": [ + 1 + ], + "pad_token_id": 0, + "transformers_version": "4.56.1" +} diff --git a/checkpoint-1000/model-00001-of-00004.safetensors b/checkpoint-1000/model-00001-of-00004.safetensors new file mode 100644 index 0000000..4705c7e --- /dev/null +++ b/checkpoint-1000/model-00001-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a0d4eb4fcbddfe01b0dc7a58386a3980826ededaf0ecd67d1ef38995b89615ba +size 4903351912 diff --git a/checkpoint-1000/model-00002-of-00004.safetensors b/checkpoint-1000/model-00002-of-00004.safetensors new file mode 100644 index 0000000..744b59b --- /dev/null +++ b/checkpoint-1000/model-00002-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:712a589c333f49f7b6f3870838a9e0ab4b3c0260abd5130e8c54316e6c99640b +size 4947570872 diff --git a/checkpoint-1000/model-00003-of-00004.safetensors b/checkpoint-1000/model-00003-of-00004.safetensors new file mode 100644 index 0000000..11e060c --- /dev/null +++ b/checkpoint-1000/model-00003-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cb79310b4ea4bcbcea914d5defb8d7fdd827bbe9bb95a6e34e1d26e296c192c1 +size 4962221464 diff --git a/checkpoint-1000/model-00004-of-00004.safetensors b/checkpoint-1000/model-00004-of-00004.safetensors new file mode 100644 index 0000000..31007b1 --- /dev/null +++ b/checkpoint-1000/model-00004-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c88167eb3924f3cb26a486123b1613c1b82b29d2f01fbae44b18ed99d610fe59 +size 3670322200 diff --git a/checkpoint-1000/model.safetensors.index.json b/checkpoint-1000/model.safetensors.index.json new file mode 100644 index 0000000..3b569c5 --- /dev/null +++ b/checkpoint-1000/model.safetensors.index.json @@ -0,0 +1,472 @@ +{ + "metadata": { + "total_parameters": 9241705984, + "total_size": 18483411968 + }, + "weight_map": { + "model.embed_tokens.weight": "model-00001-of-00004.safetensors", + "model.layers.0.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.0.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.0.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.1.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.1.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.10.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.10.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.10.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.10.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.11.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.11.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.11.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.12.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.12.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.12.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.13.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.13.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.13.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.14.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.14.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.14.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.15.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.15.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.15.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.16.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.16.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.16.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.17.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.17.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.17.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.18.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.18.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.18.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.19.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.19.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.19.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.2.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.2.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.2.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.20.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.20.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.20.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.20.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.20.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.20.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.20.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.21.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.21.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.21.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.21.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.21.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.22.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.22.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.22.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.22.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.23.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.23.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.23.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.24.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.24.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.24.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.25.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.25.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.25.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.26.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.26.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.26.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.27.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.27.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.27.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.28.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.28.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.28.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.29.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.29.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.29.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.3.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.3.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.3.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.30.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.30.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.30.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.30.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.30.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.31.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.31.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.31.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.32.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.32.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.32.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.32.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.32.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.32.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.32.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.32.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.32.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.32.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.32.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.33.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.33.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.33.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.33.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.33.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.33.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.33.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.33.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.33.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.33.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.33.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.34.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.34.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.34.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.34.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.34.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.34.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.34.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.34.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.34.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.34.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.34.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.35.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.35.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.35.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.35.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.35.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.35.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.35.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.35.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.35.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.35.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.35.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.36.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.36.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.36.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.36.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.36.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.36.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.36.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.36.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.36.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.36.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.36.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.37.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.37.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.37.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.37.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.37.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.37.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.37.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.37.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.37.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.37.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.37.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.38.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.38.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.38.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.38.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.38.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.38.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.38.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.38.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.38.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.38.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.38.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.39.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.39.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.39.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.39.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.39.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.39.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.39.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.39.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.39.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.39.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.39.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.4.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.4.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.4.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.4.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.40.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.40.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.40.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.40.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.40.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.40.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.40.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.40.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.40.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.40.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.40.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.41.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.41.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.41.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.41.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.41.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.41.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.41.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.41.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.41.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.41.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.41.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.5.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.5.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.5.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.5.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.6.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.6.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.6.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.7.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.7.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.7.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.8.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.8.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.8.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.8.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.8.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.8.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.8.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.8.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.8.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.8.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.8.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.9.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.9.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.9.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.9.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.norm.weight": "model-00004-of-00004.safetensors" + } +} diff --git a/checkpoint-1000/optimizer.pt b/checkpoint-1000/optimizer.pt new file mode 100644 index 0000000..7d822b2 --- /dev/null +++ b/checkpoint-1000/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bb0e6f5c2589f42abf1089f222d68f82aec2f11fba3855b8d3a4a4d29f872abc +size 7927918951 diff --git a/checkpoint-1000/rng_state.pth b/checkpoint-1000/rng_state.pth new file mode 100644 index 0000000..f8e3430 --- /dev/null +++ b/checkpoint-1000/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e7395a0d10191272f767d528e3eb0a8dc22c36db5202733defae76f7df421c40 +size 14645 diff --git a/checkpoint-1000/scheduler.pt b/checkpoint-1000/scheduler.pt new file mode 100644 index 0000000..a2f0e65 --- /dev/null +++ b/checkpoint-1000/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:44166fa71336dff87d04b2669a2f6008091d277a745ea6bf7e192826bbf58a35 +size 1465 diff --git a/checkpoint-1000/special_tokens_map.json b/checkpoint-1000/special_tokens_map.json new file mode 100644 index 0000000..8d6368f --- /dev/null +++ b/checkpoint-1000/special_tokens_map.json @@ -0,0 +1,34 @@ +{ + "additional_special_tokens": [ + "", + "" + ], + "bos_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "unk_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + } +} diff --git a/checkpoint-1000/tokenizer.json b/checkpoint-1000/tokenizer.json new file mode 100644 index 0000000..a4a305d --- /dev/null +++ b/checkpoint-1000/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5f7eee611703c5ce5d1eee32d9cdcfe465647b8aff0c1dfb3bed7ad7dbb05060 +size 34362873 diff --git a/checkpoint-1000/tokenizer.model b/checkpoint-1000/tokenizer.model new file mode 100644 index 0000000..796efe9 --- /dev/null +++ b/checkpoint-1000/tokenizer.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:61a7b147390c64585d6c3543dd6fc636906c9af3865a5548f27f31aee1d4c8e2 +size 4241003 diff --git a/checkpoint-1000/tokenizer_config.json b/checkpoint-1000/tokenizer_config.json new file mode 100644 index 0000000..3ade6be --- /dev/null +++ b/checkpoint-1000/tokenizer_config.json @@ -0,0 +1,2013 @@ +{ + "add_bos_token": true, + "add_eos_token": false, + "added_tokens_decoder": { + "0": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "1": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "2": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "3": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "4": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "5": { + "content": "<2mass>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "6": { + "content": "[@BOS@]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "7": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "8": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "9": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "10": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "11": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "12": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "13": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "14": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "15": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "16": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "17": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "18": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "19": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "20": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "21": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "22": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "23": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "24": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "25": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "26": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "27": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "28": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "29": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "30": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "31": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "32": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "33": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "34": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "35": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "36": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "37": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "38": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "39": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "40": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "41": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "42": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "43": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "44": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "45": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "46": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "47": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "48": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "49": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "50": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "51": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "52": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "53": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "54": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "55": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "56": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "57": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "58": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "59": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "60": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "61": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "62": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "63": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "64": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "65": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "66": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "67": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "68": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "69": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "70": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "71": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "72": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "73": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "74": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "75": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "76": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "77": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "78": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "79": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "80": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "81": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "82": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "83": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "84": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "85": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "86": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "87": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "88": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "89": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "90": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "91": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "92": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "93": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "94": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "95": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "96": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "97": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "98": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "99": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "100": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "101": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "102": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "103": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "104": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "105": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "106": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "107": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "108": { + "content": "\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "109": { + "content": "\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "110": { + "content": "\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "111": { + "content": "\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "112": { + "content": "\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "113": { + "content": "\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "114": { + "content": "\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "115": { + "content": "\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "116": { + "content": "\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "117": { + "content": "\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "118": { + "content": "\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "119": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "120": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "121": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "122": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "123": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "124": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "125": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "126": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "127": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "128": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "129": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "130": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "131": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "132": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "133": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "134": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "135": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "136": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "137": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "138": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "139": { + "content": "▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "140": { + "content": "▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "141": { + "content": "▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "142": { + "content": "▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "143": { + "content": "▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "144": { + "content": "▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "145": { + "content": "▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "146": { + "content": "▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "147": { + "content": "▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "148": { + "content": "▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "149": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "150": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "152": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "153": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "154": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "155": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "156": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "157": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "158": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "159": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "160": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "161": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "162": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "163": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "164": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "165": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "166": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "167": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "168": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "169": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "170": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "172": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "173": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "174": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "175": { + "content": "
", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "171": { + "content": "
", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "176": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "177": { + "content": "
", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "178": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "179": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "180": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "181": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "182": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "183": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "184": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "185": { + "content": "

", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "186": { + "content": "

", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "187": { + "content": "

", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "188": { + "content": "

", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "189": { + "content": "

", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "190": { + "content": "
", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "191": { + "content": "
", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "192": { + "content": "
", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "193": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "194": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "195": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "196": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "197": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "198": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "199": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "200": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "201": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "202": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "203": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "204": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "205": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "206": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "207": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "208": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "209": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "210": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "211": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "212": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "213": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "214": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "215": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "216": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255968": { + "content": "[toxicity=0]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255969": { + "content": "\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255970": { + "content": "\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255971": { + "content": "\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255972": { + "content": "\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255973": { + "content": "\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255974": { + "content": "\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255975": { + "content": "\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255976": { + "content": "\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255977": { + "content": "\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255978": { + "content": "\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255979": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255980": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255981": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255982": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255983": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255984": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255985": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255986": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255987": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255988": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255989": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255990": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255991": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255992": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255993": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255994": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255995": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255996": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255997": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255998": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255999": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + } + }, + "additional_special_tokens": [ + "", + "" + ], + "bos_token": "", + "clean_up_tokenization_spaces": false, + "eos_token": "", + "extra_special_tokens": {}, + "model_max_length": 1000000000000000019884624838656, + "pad_token": "", + "sp_model_kwargs": {}, + "spaces_between_special_tokens": false, + "tokenizer_class": "GemmaTokenizer", + "unk_token": "", + "use_default_system_prompt": false +} diff --git a/checkpoint-1000/trainer_state.json b/checkpoint-1000/trainer_state.json new file mode 100644 index 0000000..e6524e8 --- /dev/null +++ b/checkpoint-1000/trainer_state.json @@ -0,0 +1,1254 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.6, + "eval_steps": 100, + "global_step": 1000, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 0.08180969459936023, + "epoch": 0.016, + "grad_norm": 0.67578125, + "learning_rate": 9.000000000000001e-07, + "loss": 0.0453, + "mean_token_accuracy": 0.9865816306322813, + "num_tokens": 195510.0, + "step": 10 + }, + { + "entropy": 0.08111731857061386, + "epoch": 0.032, + "grad_norm": 0.60546875, + "learning_rate": 1.9000000000000002e-06, + "loss": 0.0452, + "mean_token_accuracy": 0.986666041240096, + "num_tokens": 391008.0, + "step": 20 + }, + { + "entropy": 0.07587501825764775, + "epoch": 0.048, + "grad_norm": 0.6953125, + "learning_rate": 2.9e-06, + "loss": 0.0445, + "mean_token_accuracy": 0.9873391918838024, + "num_tokens": 590136.0, + "step": 30 + }, + { + "entropy": 0.07685074284672737, + "epoch": 0.064, + "grad_norm": 0.625, + "learning_rate": 3.900000000000001e-06, + "loss": 0.0441, + "mean_token_accuracy": 0.987066001445055, + "num_tokens": 791432.0, + "step": 40 + }, + { + "entropy": 0.07894687210209668, + "epoch": 0.08, + "grad_norm": 0.75390625, + "learning_rate": 4.9000000000000005e-06, + "loss": 0.0447, + "mean_token_accuracy": 0.9865922331809998, + "num_tokens": 990342.0, + "step": 50 + }, + { + "entropy": 0.07919039381667972, + "epoch": 0.096, + "grad_norm": 0.66015625, + "learning_rate": 5.9e-06, + "loss": 0.0466, + "mean_token_accuracy": 0.9863619733601808, + "num_tokens": 1182339.0, + "step": 60 + }, + { + "entropy": 0.07967684981413185, + "epoch": 0.112, + "grad_norm": 0.5859375, + "learning_rate": 6.9e-06, + "loss": 0.0447, + "mean_token_accuracy": 0.9870311982929707, + "num_tokens": 1386193.0, + "step": 70 + }, + { + "entropy": 0.08054462079890072, + "epoch": 0.128, + "grad_norm": 0.65234375, + "learning_rate": 7.9e-06, + "loss": 0.0466, + "mean_token_accuracy": 0.9863289542496204, + "num_tokens": 1583261.0, + "step": 80 + }, + { + "entropy": 0.08421374012250453, + "epoch": 0.144, + "grad_norm": 0.5546875, + "learning_rate": 8.900000000000001e-06, + "loss": 0.0489, + "mean_token_accuracy": 0.9860724166035653, + "num_tokens": 1774826.0, + "step": 90 + }, + { + "entropy": 0.08307966932188719, + "epoch": 0.16, + "grad_norm": 0.71484375, + "learning_rate": 9.9e-06, + "loss": 0.0489, + "mean_token_accuracy": 0.986110083758831, + "num_tokens": 1971310.0, + "step": 100 + }, + { + "epoch": 0.16, + "eval_biology_entropy": 0.3792700763940811, + "eval_biology_loss": 2.260571241378784, + "eval_biology_mean_token_accuracy": 0.6872955484390258, + "eval_biology_num_tokens": 1971310.0, + "eval_biology_runtime": 39.5255, + "eval_biology_samples_per_second": 12.65, + "eval_biology_steps_per_second": 3.163, + "step": 100 + }, + { + "epoch": 0.16, + "eval_chemistry_entropy": 0.23563545274734496, + "eval_chemistry_loss": 1.4561725854873657, + "eval_chemistry_mean_token_accuracy": 0.7805930047035218, + "eval_chemistry_num_tokens": 1971310.0, + "eval_chemistry_runtime": 49.2319, + "eval_chemistry_samples_per_second": 10.156, + "eval_chemistry_steps_per_second": 2.539, + "step": 100 + }, + { + "entropy": 0.08481427943333983, + "epoch": 0.176, + "grad_norm": 0.734375, + "learning_rate": 1.0900000000000002e-05, + "loss": 0.0497, + "mean_token_accuracy": 0.9861054014414549, + "num_tokens": 2169453.0, + "step": 110 + }, + { + "entropy": 0.0882645736914128, + "epoch": 0.192, + "grad_norm": 0.79296875, + "learning_rate": 1.1900000000000001e-05, + "loss": 0.0512, + "mean_token_accuracy": 0.9859256945550442, + "num_tokens": 2366844.0, + "step": 120 + }, + { + "entropy": 0.09435517250094563, + "epoch": 0.208, + "grad_norm": 1.2421875, + "learning_rate": 1.2900000000000002e-05, + "loss": 0.0547, + "mean_token_accuracy": 0.9849376413971186, + "num_tokens": 2559708.0, + "step": 130 + }, + { + "entropy": 0.09682020233012736, + "epoch": 0.224, + "grad_norm": 1.3671875, + "learning_rate": 1.39e-05, + "loss": 0.057, + "mean_token_accuracy": 0.985217172652483, + "num_tokens": 2756533.0, + "step": 140 + }, + { + "entropy": 0.11111663887277246, + "epoch": 0.24, + "grad_norm": 2.84375, + "learning_rate": 1.4900000000000001e-05, + "loss": 0.0657, + "mean_token_accuracy": 0.9836616575717926, + "num_tokens": 2948600.0, + "step": 150 + }, + { + "entropy": 0.12251729485578836, + "epoch": 0.256, + "grad_norm": 2.453125, + "learning_rate": 1.5900000000000004e-05, + "loss": 0.0728, + "mean_token_accuracy": 0.9814188834279776, + "num_tokens": 3141473.0, + "step": 160 + }, + { + "entropy": 0.12813111576251685, + "epoch": 0.272, + "grad_norm": 3.03125, + "learning_rate": 1.69e-05, + "loss": 0.0758, + "mean_token_accuracy": 0.9808279071003199, + "num_tokens": 3337285.0, + "step": 170 + }, + { + "entropy": 0.13337696841917931, + "epoch": 0.288, + "grad_norm": 2.984375, + "learning_rate": 1.79e-05, + "loss": 0.0771, + "mean_token_accuracy": 0.9798636745661498, + "num_tokens": 3541221.0, + "step": 180 + }, + { + "entropy": 0.1436500378418714, + "epoch": 0.304, + "grad_norm": 2.78125, + "learning_rate": 1.8900000000000002e-05, + "loss": 0.0851, + "mean_token_accuracy": 0.9780430492013693, + "num_tokens": 3734952.0, + "step": 190 + }, + { + "entropy": 0.1549748638179153, + "epoch": 0.32, + "grad_norm": 3.390625, + "learning_rate": 1.9900000000000003e-05, + "loss": 0.0946, + "mean_token_accuracy": 0.9750579129904509, + "num_tokens": 3922209.0, + "step": 200 + }, + { + "epoch": 0.32, + "eval_biology_entropy": 0.4169517788887024, + "eval_biology_loss": 2.160527229309082, + "eval_biology_mean_token_accuracy": 0.6808431892395019, + "eval_biology_num_tokens": 3922209.0, + "eval_biology_runtime": 39.5121, + "eval_biology_samples_per_second": 12.654, + "eval_biology_steps_per_second": 3.164, + "step": 200 + }, + { + "epoch": 0.32, + "eval_chemistry_entropy": 0.25474807441234587, + "eval_chemistry_loss": 1.3652101755142212, + "eval_chemistry_mean_token_accuracy": 0.7794562330245972, + "eval_chemistry_num_tokens": 3922209.0, + "eval_chemistry_runtime": 49.2263, + "eval_chemistry_samples_per_second": 10.157, + "eval_chemistry_steps_per_second": 2.539, + "step": 200 + }, + { + "entropy": 0.16029730178415774, + "epoch": 0.336, + "grad_norm": 3.515625, + "learning_rate": 1.9900000000000003e-05, + "loss": 0.097, + "mean_token_accuracy": 0.9749548710882664, + "num_tokens": 4115941.0, + "step": 210 + }, + { + "entropy": 0.16590779088437557, + "epoch": 0.352, + "grad_norm": 3.78125, + "learning_rate": 1.978888888888889e-05, + "loss": 0.1023, + "mean_token_accuracy": 0.9726595059037209, + "num_tokens": 4308755.0, + "step": 220 + }, + { + "entropy": 0.1707212173845619, + "epoch": 0.368, + "grad_norm": 3.390625, + "learning_rate": 1.967777777777778e-05, + "loss": 0.1023, + "mean_token_accuracy": 0.9726378936320543, + "num_tokens": 4505910.0, + "step": 230 + }, + { + "entropy": 0.1765518521424383, + "epoch": 0.384, + "grad_norm": 3.671875, + "learning_rate": 1.956666666666667e-05, + "loss": 0.1083, + "mean_token_accuracy": 0.97143131531775, + "num_tokens": 4696101.0, + "step": 240 + }, + { + "entropy": 0.17895571915432812, + "epoch": 0.4, + "grad_norm": 3.5, + "learning_rate": 1.9455555555555555e-05, + "loss": 0.11, + "mean_token_accuracy": 0.9705049484968186, + "num_tokens": 4889445.0, + "step": 250 + }, + { + "entropy": 0.18151565054431557, + "epoch": 0.416, + "grad_norm": 3.578125, + "learning_rate": 1.9344444444444446e-05, + "loss": 0.1091, + "mean_token_accuracy": 0.970859233289957, + "num_tokens": 5087624.0, + "step": 260 + }, + { + "entropy": 0.18663504091091454, + "epoch": 0.432, + "grad_norm": 3.859375, + "learning_rate": 1.9233333333333334e-05, + "loss": 0.1135, + "mean_token_accuracy": 0.9693388067185879, + "num_tokens": 5273794.0, + "step": 270 + }, + { + "entropy": 0.18451756797730923, + "epoch": 0.448, + "grad_norm": 4.125, + "learning_rate": 1.9122222222222226e-05, + "loss": 0.1144, + "mean_token_accuracy": 0.9698665160685778, + "num_tokens": 5463288.0, + "step": 280 + }, + { + "entropy": 0.17932639280334114, + "epoch": 0.464, + "grad_norm": 3.5625, + "learning_rate": 1.9011111111111114e-05, + "loss": 0.1081, + "mean_token_accuracy": 0.9717006407678127, + "num_tokens": 5656827.0, + "step": 290 + }, + { + "entropy": 0.18305762661620975, + "epoch": 0.48, + "grad_norm": 3.90625, + "learning_rate": 1.8900000000000002e-05, + "loss": 0.1107, + "mean_token_accuracy": 0.9712053131312132, + "num_tokens": 5853311.0, + "step": 300 + }, + { + "epoch": 0.48, + "eval_biology_entropy": 0.42041616535186765, + "eval_biology_loss": 2.1313436031341553, + "eval_biology_mean_token_accuracy": 0.6798483271598816, + "eval_biology_num_tokens": 5853311.0, + "eval_biology_runtime": 39.5137, + "eval_biology_samples_per_second": 12.654, + "eval_biology_steps_per_second": 3.163, + "step": 300 + }, + { + "epoch": 0.48, + "eval_chemistry_entropy": 0.259404793381691, + "eval_chemistry_loss": 1.3463311195373535, + "eval_chemistry_mean_token_accuracy": 0.7787248449325561, + "eval_chemistry_num_tokens": 5853311.0, + "eval_chemistry_runtime": 49.201, + "eval_chemistry_samples_per_second": 10.162, + "eval_chemistry_steps_per_second": 2.541, + "step": 300 + }, + { + "entropy": 0.17875824291259051, + "epoch": 0.496, + "grad_norm": 3.453125, + "learning_rate": 1.878888888888889e-05, + "loss": 0.1101, + "mean_token_accuracy": 0.9710387323051691, + "num_tokens": 6049788.0, + "step": 310 + }, + { + "entropy": 0.1863187742419541, + "epoch": 0.512, + "grad_norm": 3.4375, + "learning_rate": 1.8677777777777778e-05, + "loss": 0.113, + "mean_token_accuracy": 0.9698710650205612, + "num_tokens": 6243825.0, + "step": 320 + }, + { + "entropy": 0.18619006094522775, + "epoch": 0.528, + "grad_norm": 3.890625, + "learning_rate": 1.856666666666667e-05, + "loss": 0.1135, + "mean_token_accuracy": 0.9703223951160908, + "num_tokens": 6434188.0, + "step": 330 + }, + { + "entropy": 0.18118073977530003, + "epoch": 0.544, + "grad_norm": 3.484375, + "learning_rate": 1.8455555555555558e-05, + "loss": 0.1114, + "mean_token_accuracy": 0.9708513412624598, + "num_tokens": 6629832.0, + "step": 340 + }, + { + "entropy": 0.1808169964235276, + "epoch": 0.56, + "grad_norm": 3.28125, + "learning_rate": 1.8344444444444446e-05, + "loss": 0.1079, + "mean_token_accuracy": 0.9716686312109232, + "num_tokens": 6824668.0, + "step": 350 + }, + { + "entropy": 0.18033515606075526, + "epoch": 0.576, + "grad_norm": 3.53125, + "learning_rate": 1.8233333333333334e-05, + "loss": 0.106, + "mean_token_accuracy": 0.9722726982086897, + "num_tokens": 7025923.0, + "step": 360 + }, + { + "entropy": 0.1852203646209091, + "epoch": 0.592, + "grad_norm": 3.625, + "learning_rate": 1.8122222222222222e-05, + "loss": 0.111, + "mean_token_accuracy": 0.9705461781471968, + "num_tokens": 7218224.0, + "step": 370 + }, + { + "entropy": 0.17758405841886998, + "epoch": 0.608, + "grad_norm": 3.65625, + "learning_rate": 1.8011111111111113e-05, + "loss": 0.1066, + "mean_token_accuracy": 0.9723859451711178, + "num_tokens": 7421216.0, + "step": 380 + }, + { + "entropy": 0.18370326030999423, + "epoch": 0.624, + "grad_norm": 3.765625, + "learning_rate": 1.79e-05, + "loss": 0.1096, + "mean_token_accuracy": 0.9708739675581455, + "num_tokens": 7617258.0, + "step": 390 + }, + { + "entropy": 0.1831163068767637, + "epoch": 0.64, + "grad_norm": 3.578125, + "learning_rate": 1.778888888888889e-05, + "loss": 0.1072, + "mean_token_accuracy": 0.9717655830085278, + "num_tokens": 7806173.0, + "step": 400 + }, + { + "epoch": 0.64, + "eval_biology_entropy": 0.42119112753868104, + "eval_biology_loss": 2.1267058849334717, + "eval_biology_mean_token_accuracy": 0.6801503438949585, + "eval_biology_num_tokens": 7806173.0, + "eval_biology_runtime": 39.5121, + "eval_biology_samples_per_second": 12.654, + "eval_biology_steps_per_second": 3.164, + "step": 400 + }, + { + "epoch": 0.64, + "eval_chemistry_entropy": 0.2595027017593384, + "eval_chemistry_loss": 1.3404533863067627, + "eval_chemistry_mean_token_accuracy": 0.7790702543258667, + "eval_chemistry_num_tokens": 7806173.0, + "eval_chemistry_runtime": 49.2002, + "eval_chemistry_samples_per_second": 10.163, + "eval_chemistry_steps_per_second": 2.541, + "step": 400 + }, + { + "entropy": 0.1827191138640046, + "epoch": 0.656, + "grad_norm": 3.84375, + "learning_rate": 1.7677777777777778e-05, + "loss": 0.1088, + "mean_token_accuracy": 0.9715055178850889, + "num_tokens": 8000514.0, + "step": 410 + }, + { + "entropy": 0.18661352493800223, + "epoch": 0.672, + "grad_norm": 3.796875, + "learning_rate": 1.756666666666667e-05, + "loss": 0.1114, + "mean_token_accuracy": 0.9704575203359127, + "num_tokens": 8187884.0, + "step": 420 + }, + { + "entropy": 0.17729734890162946, + "epoch": 0.688, + "grad_norm": 3.09375, + "learning_rate": 1.7455555555555557e-05, + "loss": 0.1046, + "mean_token_accuracy": 0.9725840430706739, + "num_tokens": 8390643.0, + "step": 430 + }, + { + "entropy": 0.18038612157106398, + "epoch": 0.704, + "grad_norm": 3.46875, + "learning_rate": 1.7344444444444445e-05, + "loss": 0.1072, + "mean_token_accuracy": 0.9721495136618614, + "num_tokens": 8583266.0, + "step": 440 + }, + { + "entropy": 0.17609651018865408, + "epoch": 0.72, + "grad_norm": 3.734375, + "learning_rate": 1.7233333333333337e-05, + "loss": 0.1045, + "mean_token_accuracy": 0.9718786559998989, + "num_tokens": 8786233.0, + "step": 450 + }, + { + "entropy": 0.1797325788065791, + "epoch": 0.736, + "grad_norm": 3.390625, + "learning_rate": 1.7122222222222225e-05, + "loss": 0.107, + "mean_token_accuracy": 0.9716725051403046, + "num_tokens": 8982898.0, + "step": 460 + }, + { + "entropy": 0.17950535458512604, + "epoch": 0.752, + "grad_norm": 3.5625, + "learning_rate": 1.7011111111111113e-05, + "loss": 0.1072, + "mean_token_accuracy": 0.9725060146301985, + "num_tokens": 9174361.0, + "step": 470 + }, + { + "entropy": 0.17792485035024583, + "epoch": 0.768, + "grad_norm": 3.375, + "learning_rate": 1.69e-05, + "loss": 0.1025, + "mean_token_accuracy": 0.973187630996108, + "num_tokens": 9373430.0, + "step": 480 + }, + { + "entropy": 0.17844869694672524, + "epoch": 0.784, + "grad_norm": 3.515625, + "learning_rate": 1.678888888888889e-05, + "loss": 0.1073, + "mean_token_accuracy": 0.9717021223157645, + "num_tokens": 9570409.0, + "step": 490 + }, + { + "entropy": 0.17820251639932394, + "epoch": 0.8, + "grad_norm": 3.765625, + "learning_rate": 1.667777777777778e-05, + "loss": 0.1032, + "mean_token_accuracy": 0.9729967709630728, + "num_tokens": 9766360.0, + "step": 500 + }, + { + "epoch": 0.8, + "eval_biology_entropy": 0.41608734560012817, + "eval_biology_loss": 2.139784097671509, + "eval_biology_mean_token_accuracy": 0.6806383662223816, + "eval_biology_num_tokens": 9766360.0, + "eval_biology_runtime": 39.5271, + "eval_biology_samples_per_second": 12.65, + "eval_biology_steps_per_second": 3.162, + "step": 500 + }, + { + "epoch": 0.8, + "eval_chemistry_entropy": 0.25711704087257387, + "eval_chemistry_loss": 1.3562349081039429, + "eval_chemistry_mean_token_accuracy": 0.7787247653007507, + "eval_chemistry_num_tokens": 9766360.0, + "eval_chemistry_runtime": 48.8194, + "eval_chemistry_samples_per_second": 10.242, + "eval_chemistry_steps_per_second": 2.56, + "step": 500 + }, + { + "entropy": 0.1748917239718139, + "epoch": 0.816, + "grad_norm": 3.515625, + "learning_rate": 1.656666666666667e-05, + "loss": 0.1004, + "mean_token_accuracy": 0.9740771554410458, + "num_tokens": 9963834.0, + "step": 510 + }, + { + "entropy": 0.17268717400729655, + "epoch": 0.832, + "grad_norm": 3.125, + "learning_rate": 1.6455555555555556e-05, + "loss": 0.1023, + "mean_token_accuracy": 0.9740318663418293, + "num_tokens": 10160950.0, + "step": 520 + }, + { + "entropy": 0.17282785503193737, + "epoch": 0.848, + "grad_norm": 3.671875, + "learning_rate": 1.6344444444444445e-05, + "loss": 0.0991, + "mean_token_accuracy": 0.9744720965623855, + "num_tokens": 10363131.0, + "step": 530 + }, + { + "entropy": 0.17644098605960606, + "epoch": 0.864, + "grad_norm": 3.34375, + "learning_rate": 1.6233333333333333e-05, + "loss": 0.1026, + "mean_token_accuracy": 0.972842201218009, + "num_tokens": 10558003.0, + "step": 540 + }, + { + "entropy": 0.17435273211449384, + "epoch": 0.88, + "grad_norm": 3.34375, + "learning_rate": 1.6122222222222224e-05, + "loss": 0.1004, + "mean_token_accuracy": 0.9740745916962623, + "num_tokens": 10754470.0, + "step": 550 + }, + { + "entropy": 0.1750097878277302, + "epoch": 0.896, + "grad_norm": 3.453125, + "learning_rate": 1.6011111111111112e-05, + "loss": 0.1007, + "mean_token_accuracy": 0.9743314851075411, + "num_tokens": 10948943.0, + "step": 560 + }, + { + "entropy": 0.17419813955202698, + "epoch": 0.912, + "grad_norm": 3.234375, + "learning_rate": 1.5900000000000004e-05, + "loss": 0.1005, + "mean_token_accuracy": 0.974232492223382, + "num_tokens": 11142613.0, + "step": 570 + }, + { + "entropy": 0.17595540760084988, + "epoch": 0.928, + "grad_norm": 3.546875, + "learning_rate": 1.5788888888888888e-05, + "loss": 0.0997, + "mean_token_accuracy": 0.974004115909338, + "num_tokens": 11336779.0, + "step": 580 + }, + { + "entropy": 0.16677158032543957, + "epoch": 0.944, + "grad_norm": 3.140625, + "learning_rate": 1.567777777777778e-05, + "loss": 0.0982, + "mean_token_accuracy": 0.9750826183706522, + "num_tokens": 11536381.0, + "step": 590 + }, + { + "entropy": 0.16921130288392305, + "epoch": 0.96, + "grad_norm": 3.546875, + "learning_rate": 1.5566666666666668e-05, + "loss": 0.0966, + "mean_token_accuracy": 0.9753067292273044, + "num_tokens": 11735527.0, + "step": 600 + }, + { + "epoch": 0.96, + "eval_biology_entropy": 0.4160035889148712, + "eval_biology_loss": 2.1477200984954834, + "eval_biology_mean_token_accuracy": 0.68089803981781, + "eval_biology_num_tokens": 11735527.0, + "eval_biology_runtime": 39.5581, + "eval_biology_samples_per_second": 12.64, + "eval_biology_steps_per_second": 3.16, + "step": 600 + }, + { + "epoch": 0.96, + "eval_chemistry_entropy": 0.2561914825439453, + "eval_chemistry_loss": 1.3602544069290161, + "eval_chemistry_mean_token_accuracy": 0.779716450214386, + "eval_chemistry_num_tokens": 11735527.0, + "eval_chemistry_runtime": 49.254, + "eval_chemistry_samples_per_second": 10.151, + "eval_chemistry_steps_per_second": 2.538, + "step": 600 + }, + { + "entropy": 0.16858043959364294, + "epoch": 0.976, + "grad_norm": 3.65625, + "learning_rate": 1.5455555555555556e-05, + "loss": 0.0985, + "mean_token_accuracy": 0.9748617243021727, + "num_tokens": 11930935.0, + "step": 610 + }, + { + "entropy": 0.16747406986542046, + "epoch": 0.992, + "grad_norm": 3.15625, + "learning_rate": 1.5344444444444447e-05, + "loss": 0.0955, + "mean_token_accuracy": 0.9756584648042917, + "num_tokens": 12129585.0, + "step": 620 + }, + { + "entropy": 0.15082475617527963, + "epoch": 1.008, + "grad_norm": 2.28125, + "learning_rate": 1.5233333333333335e-05, + "loss": 0.0856, + "mean_token_accuracy": 0.9783346977084875, + "num_tokens": 12325847.0, + "step": 630 + }, + { + "entropy": 0.11756393420509995, + "epoch": 1.024, + "grad_norm": 2.3125, + "learning_rate": 1.5122222222222222e-05, + "loss": 0.07, + "mean_token_accuracy": 0.9816608231514692, + "num_tokens": 12530820.0, + "step": 640 + }, + { + "entropy": 0.12562460631597788, + "epoch": 1.04, + "grad_norm": 2.796875, + "learning_rate": 1.5011111111111112e-05, + "loss": 0.0744, + "mean_token_accuracy": 0.9802271891385317, + "num_tokens": 12725384.0, + "step": 650 + }, + { + "entropy": 0.12352396142669023, + "epoch": 1.056, + "grad_norm": 2.609375, + "learning_rate": 1.4900000000000001e-05, + "loss": 0.0744, + "mean_token_accuracy": 0.9802641972899437, + "num_tokens": 12924592.0, + "step": 660 + }, + { + "entropy": 0.12865133616141974, + "epoch": 1.072, + "grad_norm": 2.671875, + "learning_rate": 1.4788888888888891e-05, + "loss": 0.0774, + "mean_token_accuracy": 0.9798023611307144, + "num_tokens": 13112848.0, + "step": 670 + }, + { + "entropy": 0.12567441049031913, + "epoch": 1.088, + "grad_norm": 2.53125, + "learning_rate": 1.467777777777778e-05, + "loss": 0.0756, + "mean_token_accuracy": 0.9806226130574942, + "num_tokens": 13305725.0, + "step": 680 + }, + { + "entropy": 0.12565653487108647, + "epoch": 1.104, + "grad_norm": 2.234375, + "learning_rate": 1.4566666666666669e-05, + "loss": 0.0745, + "mean_token_accuracy": 0.980208383128047, + "num_tokens": 13498685.0, + "step": 690 + }, + { + "entropy": 0.1318228157237172, + "epoch": 1.12, + "grad_norm": 2.5, + "learning_rate": 1.4455555555555555e-05, + "loss": 0.0778, + "mean_token_accuracy": 0.97950509339571, + "num_tokens": 13682146.0, + "step": 700 + }, + { + "epoch": 1.12, + "eval_biology_entropy": 0.4004362862110138, + "eval_biology_loss": 2.177130699157715, + "eval_biology_mean_token_accuracy": 0.6821538486480713, + "eval_biology_num_tokens": 13682146.0, + "eval_biology_runtime": 39.5089, + "eval_biology_samples_per_second": 12.655, + "eval_biology_steps_per_second": 3.164, + "step": 700 + }, + { + "epoch": 1.12, + "eval_chemistry_entropy": 0.24954386246204377, + "eval_chemistry_loss": 1.3777952194213867, + "eval_chemistry_mean_token_accuracy": 0.7797037825584412, + "eval_chemistry_num_tokens": 13682146.0, + "eval_chemistry_runtime": 49.2017, + "eval_chemistry_samples_per_second": 10.162, + "eval_chemistry_steps_per_second": 2.541, + "step": 700 + }, + { + "entropy": 0.12691126235295086, + "epoch": 1.1360000000000001, + "grad_norm": 2.6875, + "learning_rate": 1.4344444444444445e-05, + "loss": 0.0757, + "mean_token_accuracy": 0.980040468648076, + "num_tokens": 13876656.0, + "step": 710 + }, + { + "entropy": 0.12381730475462974, + "epoch": 1.152, + "grad_norm": 2.359375, + "learning_rate": 1.4233333333333335e-05, + "loss": 0.0743, + "mean_token_accuracy": 0.980041703954339, + "num_tokens": 14086104.0, + "step": 720 + }, + { + "entropy": 0.1285357222892344, + "epoch": 1.168, + "grad_norm": 2.5625, + "learning_rate": 1.4122222222222225e-05, + "loss": 0.0763, + "mean_token_accuracy": 0.9799800109118223, + "num_tokens": 14274520.0, + "step": 730 + }, + { + "entropy": 0.12321301018819213, + "epoch": 1.184, + "grad_norm": 2.375, + "learning_rate": 1.4011111111111113e-05, + "loss": 0.0734, + "mean_token_accuracy": 0.9810513865202666, + "num_tokens": 14473623.0, + "step": 740 + }, + { + "entropy": 0.1292376267723739, + "epoch": 1.2, + "grad_norm": 2.609375, + "learning_rate": 1.39e-05, + "loss": 0.0784, + "mean_token_accuracy": 0.9795571226626635, + "num_tokens": 14661238.0, + "step": 750 + }, + { + "entropy": 0.12086025532335043, + "epoch": 1.216, + "grad_norm": 2.640625, + "learning_rate": 1.3788888888888889e-05, + "loss": 0.073, + "mean_token_accuracy": 0.9810031026601791, + "num_tokens": 14865915.0, + "step": 760 + }, + { + "entropy": 0.1262558945454657, + "epoch": 1.232, + "grad_norm": 2.40625, + "learning_rate": 1.3677777777777779e-05, + "loss": 0.0786, + "mean_token_accuracy": 0.9797264195978641, + "num_tokens": 15055682.0, + "step": 770 + }, + { + "entropy": 0.12587894778698683, + "epoch": 1.248, + "grad_norm": 2.484375, + "learning_rate": 1.3566666666666668e-05, + "loss": 0.0744, + "mean_token_accuracy": 0.9802309483289718, + "num_tokens": 15249576.0, + "step": 780 + }, + { + "entropy": 0.12361958674155175, + "epoch": 1.264, + "grad_norm": 2.1875, + "learning_rate": 1.3455555555555556e-05, + "loss": 0.0726, + "mean_token_accuracy": 0.9805784326046705, + "num_tokens": 15446149.0, + "step": 790 + }, + { + "entropy": 0.12549991067498922, + "epoch": 1.28, + "grad_norm": 2.546875, + "learning_rate": 1.3344444444444446e-05, + "loss": 0.0743, + "mean_token_accuracy": 0.9805208504199981, + "num_tokens": 15639317.0, + "step": 800 + }, + { + "epoch": 1.28, + "eval_biology_entropy": 0.4032972083091736, + "eval_biology_loss": 2.1712591648101807, + "eval_biology_mean_token_accuracy": 0.6816940937042236, + "eval_biology_num_tokens": 15639317.0, + "eval_biology_runtime": 39.5209, + "eval_biology_samples_per_second": 12.652, + "eval_biology_steps_per_second": 3.163, + "step": 800 + }, + { + "epoch": 1.28, + "eval_chemistry_entropy": 0.25105089938640596, + "eval_chemistry_loss": 1.3797656297683716, + "eval_chemistry_mean_token_accuracy": 0.7792046613693238, + "eval_chemistry_num_tokens": 15639317.0, + "eval_chemistry_runtime": 49.2346, + "eval_chemistry_samples_per_second": 10.155, + "eval_chemistry_steps_per_second": 2.539, + "step": 800 + }, + { + "entropy": 0.12616121221799403, + "epoch": 1.296, + "grad_norm": 2.203125, + "learning_rate": 1.3233333333333334e-05, + "loss": 0.075, + "mean_token_accuracy": 0.9805853247642518, + "num_tokens": 15835805.0, + "step": 810 + }, + { + "entropy": 0.12473565563559533, + "epoch": 1.312, + "grad_norm": 2.296875, + "learning_rate": 1.3122222222222222e-05, + "loss": 0.0738, + "mean_token_accuracy": 0.9808966059237718, + "num_tokens": 16028375.0, + "step": 820 + }, + { + "entropy": 0.12206203136593104, + "epoch": 1.328, + "grad_norm": 2.265625, + "learning_rate": 1.3011111111111112e-05, + "loss": 0.0712, + "mean_token_accuracy": 0.9813739657402039, + "num_tokens": 16230555.0, + "step": 830 + }, + { + "entropy": 0.11784348418004811, + "epoch": 1.3439999999999999, + "grad_norm": 2.359375, + "learning_rate": 1.2900000000000002e-05, + "loss": 0.0692, + "mean_token_accuracy": 0.9817070845514536, + "num_tokens": 16436297.0, + "step": 840 + }, + { + "entropy": 0.12049585804343224, + "epoch": 1.3599999999999999, + "grad_norm": 2.234375, + "learning_rate": 1.278888888888889e-05, + "loss": 0.0719, + "mean_token_accuracy": 0.9809176307171583, + "num_tokens": 16630563.0, + "step": 850 + }, + { + "entropy": 0.12437316868454218, + "epoch": 1.376, + "grad_norm": 2.8125, + "learning_rate": 1.2677777777777778e-05, + "loss": 0.0751, + "mean_token_accuracy": 0.9806294579058885, + "num_tokens": 16822602.0, + "step": 860 + }, + { + "entropy": 0.12243083966895937, + "epoch": 1.392, + "grad_norm": 2.34375, + "learning_rate": 1.2566666666666668e-05, + "loss": 0.0729, + "mean_token_accuracy": 0.9806884177029133, + "num_tokens": 17015750.0, + "step": 870 + }, + { + "entropy": 0.12194055491127073, + "epoch": 1.408, + "grad_norm": 2.21875, + "learning_rate": 1.2455555555555556e-05, + "loss": 0.0738, + "mean_token_accuracy": 0.9811285879462958, + "num_tokens": 17207248.0, + "step": 880 + }, + { + "entropy": 0.12085925284773111, + "epoch": 1.424, + "grad_norm": 2.40625, + "learning_rate": 1.2344444444444446e-05, + "loss": 0.0714, + "mean_token_accuracy": 0.9814036220312119, + "num_tokens": 17403869.0, + "step": 890 + }, + { + "entropy": 0.12371515287086368, + "epoch": 1.44, + "grad_norm": 2.671875, + "learning_rate": 1.2233333333333335e-05, + "loss": 0.0732, + "mean_token_accuracy": 0.9810383848845958, + "num_tokens": 17597428.0, + "step": 900 + }, + { + "epoch": 1.44, + "eval_biology_entropy": 0.3943380331993103, + "eval_biology_loss": 2.181544542312622, + "eval_biology_mean_token_accuracy": 0.6840425395965576, + "eval_biology_num_tokens": 17597428.0, + "eval_biology_runtime": 39.5119, + "eval_biology_samples_per_second": 12.654, + "eval_biology_steps_per_second": 3.164, + "step": 900 + }, + { + "epoch": 1.44, + "eval_chemistry_entropy": 0.24567515325546266, + "eval_chemistry_loss": 1.3865065574645996, + "eval_chemistry_mean_token_accuracy": 0.7803217101097107, + "eval_chemistry_num_tokens": 17597428.0, + "eval_chemistry_runtime": 49.2083, + "eval_chemistry_samples_per_second": 10.161, + "eval_chemistry_steps_per_second": 2.54, + "step": 900 + }, + { + "entropy": 0.11841928996145726, + "epoch": 1.456, + "grad_norm": 2.421875, + "learning_rate": 1.2122222222222223e-05, + "loss": 0.0692, + "mean_token_accuracy": 0.9820894330739975, + "num_tokens": 17798083.0, + "step": 910 + }, + { + "entropy": 0.12107395366765558, + "epoch": 1.472, + "grad_norm": 2.09375, + "learning_rate": 1.2011111111111111e-05, + "loss": 0.0727, + "mean_token_accuracy": 0.980775249004364, + "num_tokens": 17993802.0, + "step": 920 + }, + { + "entropy": 0.11666952911764383, + "epoch": 1.488, + "grad_norm": 2.234375, + "learning_rate": 1.1900000000000001e-05, + "loss": 0.0711, + "mean_token_accuracy": 0.9814896911382676, + "num_tokens": 18185570.0, + "step": 930 + }, + { + "entropy": 0.12296288679353892, + "epoch": 1.504, + "grad_norm": 2.203125, + "learning_rate": 1.178888888888889e-05, + "loss": 0.0721, + "mean_token_accuracy": 0.9813314471393824, + "num_tokens": 18377903.0, + "step": 940 + }, + { + "entropy": 0.11539274472743273, + "epoch": 1.52, + "grad_norm": 2.21875, + "learning_rate": 1.1677777777777779e-05, + "loss": 0.0691, + "mean_token_accuracy": 0.9822911247611046, + "num_tokens": 18579284.0, + "step": 950 + }, + { + "entropy": 0.12136667580343782, + "epoch": 1.536, + "grad_norm": 2.140625, + "learning_rate": 1.1566666666666669e-05, + "loss": 0.0725, + "mean_token_accuracy": 0.9812386609613896, + "num_tokens": 18765324.0, + "step": 960 + }, + { + "entropy": 0.11671084221452474, + "epoch": 1.552, + "grad_norm": 2.15625, + "learning_rate": 1.1455555555555557e-05, + "loss": 0.0676, + "mean_token_accuracy": 0.9818577177822589, + "num_tokens": 18966382.0, + "step": 970 + }, + { + "entropy": 0.11611056034453213, + "epoch": 1.568, + "grad_norm": 2.296875, + "learning_rate": 1.1344444444444445e-05, + "loss": 0.0701, + "mean_token_accuracy": 0.9818583976477384, + "num_tokens": 19160509.0, + "step": 980 + }, + { + "entropy": 0.11777197350747884, + "epoch": 1.584, + "grad_norm": 2.1875, + "learning_rate": 1.1233333333333333e-05, + "loss": 0.0697, + "mean_token_accuracy": 0.9817087065428496, + "num_tokens": 19354662.0, + "step": 990 + }, + { + "entropy": 0.12017350159585476, + "epoch": 1.6, + "grad_norm": 2.28125, + "learning_rate": 1.1122222222222223e-05, + "loss": 0.0718, + "mean_token_accuracy": 0.9814203523099423, + "num_tokens": 19543088.0, + "step": 1000 + }, + { + "epoch": 1.6, + "eval_biology_entropy": 0.3948675305843353, + "eval_biology_loss": 2.1792049407958984, + "eval_biology_mean_token_accuracy": 0.6840984811782836, + "eval_biology_num_tokens": 19543088.0, + "eval_biology_runtime": 39.6119, + "eval_biology_samples_per_second": 12.622, + "eval_biology_steps_per_second": 3.156, + "step": 1000 + }, + { + "epoch": 1.6, + "eval_chemistry_entropy": 0.24652056658267976, + "eval_chemistry_loss": 1.388296365737915, + "eval_chemistry_mean_token_accuracy": 0.7802323937416077, + "eval_chemistry_num_tokens": 19543088.0, + "eval_chemistry_runtime": 48.7687, + "eval_chemistry_samples_per_second": 10.252, + "eval_chemistry_steps_per_second": 2.563, + "step": 1000 + } + ], + "logging_steps": 10, + "max_steps": 2000, + "num_input_tokens_seen": 0, + "num_train_epochs": 4, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.6380077645630792e+18, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/checkpoint-1000/training_args.bin b/checkpoint-1000/training_args.bin new file mode 100644 index 0000000..4c8c7f4 --- /dev/null +++ b/checkpoint-1000/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:387a0fa8295895440900c3b62c83fa1f8ff3d01320d6ee63d02eb78146f15c65 +size 6545 diff --git a/checkpoint-1500/chat_template.jinja b/checkpoint-1500/chat_template.jinja new file mode 100644 index 0000000..923ec25 --- /dev/null +++ b/checkpoint-1500/chat_template.jinja @@ -0,0 +1,4 @@ +{{ bos_token }}{% if messages[0]['role'] == 'system' %}{{ raise_exception('System role not supported') }}{% endif %}{% for message in messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if (message['role'] == 'assistant') %}{% set role = 'model' %}{% else %}{% set role = message['role'] %}{% endif %}{{ '' + role + ' +' + message['content'] | trim + ' +' }}{% endfor %}{% if add_generation_prompt %}{{'model +'}}{% endif %} \ No newline at end of file diff --git a/checkpoint-1500/config.json b/checkpoint-1500/config.json new file mode 100644 index 0000000..29e6cbe --- /dev/null +++ b/checkpoint-1500/config.json @@ -0,0 +1,77 @@ +{ + "architectures": [ + "Gemma2ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "attn_logit_softcapping": 50.0, + "bos_token_id": 2, + "cache_implementation": "hybrid", + "dtype": "bfloat16", + "eos_token_id": 1, + "final_logit_softcapping": 30.0, + "head_dim": 256, + "hidden_act": "gelu_pytorch_tanh", + "hidden_activation": "gelu_pytorch_tanh", + "hidden_size": 3584, + "initializer_range": 0.02, + "intermediate_size": 14336, + "layer_types": [ + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention" + ], + "max_position_embeddings": 8192, + "model_type": "gemma2", + "num_attention_heads": 16, + "num_hidden_layers": 42, + "num_key_value_heads": 8, + "pad_token_id": 0, + "query_pre_attn_scalar": 256, + "rms_norm_eps": 1e-06, + "rope_theta": 10000.0, + "sliding_window": 4096, + "sliding_window_size": 4096, + "transformers_version": "4.56.1", + "use_cache": true, + "vocab_size": 256000 +} diff --git a/checkpoint-1500/generation_config.json b/checkpoint-1500/generation_config.json new file mode 100644 index 0000000..f2c34f2 --- /dev/null +++ b/checkpoint-1500/generation_config.json @@ -0,0 +1,9 @@ +{ + "_from_model_config": true, + "bos_token_id": 2, + "eos_token_id": [ + 1 + ], + "pad_token_id": 0, + "transformers_version": "4.56.1" +} diff --git a/checkpoint-1500/model-00001-of-00004.safetensors b/checkpoint-1500/model-00001-of-00004.safetensors new file mode 100644 index 0000000..4705c7e --- /dev/null +++ b/checkpoint-1500/model-00001-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a0d4eb4fcbddfe01b0dc7a58386a3980826ededaf0ecd67d1ef38995b89615ba +size 4903351912 diff --git a/checkpoint-1500/model-00002-of-00004.safetensors b/checkpoint-1500/model-00002-of-00004.safetensors new file mode 100644 index 0000000..744b59b --- /dev/null +++ b/checkpoint-1500/model-00002-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:712a589c333f49f7b6f3870838a9e0ab4b3c0260abd5130e8c54316e6c99640b +size 4947570872 diff --git a/checkpoint-1500/model-00003-of-00004.safetensors b/checkpoint-1500/model-00003-of-00004.safetensors new file mode 100644 index 0000000..7bd98b3 --- /dev/null +++ b/checkpoint-1500/model-00003-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ebfce8157514e2cc5d0cfc593d5265773f1c801b57e010d3626c91b25f7d634f +size 4962221464 diff --git a/checkpoint-1500/model-00004-of-00004.safetensors b/checkpoint-1500/model-00004-of-00004.safetensors new file mode 100644 index 0000000..d25fbf1 --- /dev/null +++ b/checkpoint-1500/model-00004-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a98342ce9a7ba76cc7fedbd7facc8c87027308867c82c5aa0f4ffca7174504dd +size 3670322200 diff --git a/checkpoint-1500/model.safetensors.index.json b/checkpoint-1500/model.safetensors.index.json new file mode 100644 index 0000000..3b569c5 --- /dev/null +++ b/checkpoint-1500/model.safetensors.index.json @@ -0,0 +1,472 @@ +{ + "metadata": { + "total_parameters": 9241705984, + "total_size": 18483411968 + }, + "weight_map": { + "model.embed_tokens.weight": "model-00001-of-00004.safetensors", + "model.layers.0.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.0.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.0.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.1.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.1.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.10.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.10.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.10.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.10.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.11.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.11.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.11.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.12.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.12.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.12.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.13.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.13.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.13.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.14.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.14.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.14.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.15.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.15.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.15.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.16.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.16.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.16.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.17.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.17.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.17.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.18.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.18.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.18.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.19.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.19.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.19.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.2.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.2.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.2.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.20.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.20.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.20.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.20.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.20.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.20.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.20.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.21.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.21.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.21.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.21.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.21.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.22.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.22.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.22.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.22.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.23.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.23.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.23.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.24.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.24.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.24.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.25.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.25.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.25.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.26.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.26.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.26.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.27.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.27.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.27.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.28.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.28.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.28.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.29.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.29.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.29.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.3.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.3.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.3.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.30.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.30.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.30.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.30.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.30.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.31.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.31.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.31.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.32.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.32.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.32.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.32.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.32.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.32.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.32.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.32.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.32.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.32.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.32.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.33.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.33.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.33.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.33.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.33.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.33.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.33.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.33.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.33.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.33.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.33.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.34.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.34.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.34.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.34.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.34.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.34.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.34.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.34.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.34.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.34.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.34.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.35.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.35.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.35.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.35.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.35.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.35.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.35.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.35.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.35.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.35.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.35.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.36.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.36.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.36.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.36.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.36.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.36.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.36.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.36.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.36.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.36.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.36.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.37.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.37.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.37.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.37.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.37.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.37.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.37.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.37.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.37.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.37.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.37.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.38.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.38.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.38.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.38.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.38.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.38.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.38.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.38.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.38.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.38.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.38.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.39.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.39.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.39.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.39.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.39.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.39.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.39.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.39.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.39.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.39.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.39.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.4.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.4.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.4.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.4.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.40.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.40.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.40.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.40.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.40.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.40.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.40.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.40.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.40.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.40.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.40.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.41.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.41.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.41.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.41.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.41.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.41.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.41.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.41.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.41.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.41.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.41.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.5.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.5.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.5.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.5.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.6.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.6.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.6.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.7.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.7.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.7.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.8.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.8.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.8.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.8.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.8.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.8.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.8.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.8.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.8.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.8.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.8.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.9.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.9.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.9.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.9.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.norm.weight": "model-00004-of-00004.safetensors" + } +} diff --git a/checkpoint-1500/optimizer.pt b/checkpoint-1500/optimizer.pt new file mode 100644 index 0000000..6326b83 --- /dev/null +++ b/checkpoint-1500/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2ad5302535aa3b2e70fc7d743b1b94d6069fe8d2e49d6973064e8eb4d3d02fcb +size 7927918951 diff --git a/checkpoint-1500/rng_state.pth b/checkpoint-1500/rng_state.pth new file mode 100644 index 0000000..72b9acd --- /dev/null +++ b/checkpoint-1500/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7905839719e409e7a3dc16880442546c5525350d5767488905f5a02ad96619e2 +size 14645 diff --git a/checkpoint-1500/scheduler.pt b/checkpoint-1500/scheduler.pt new file mode 100644 index 0000000..d36cd39 --- /dev/null +++ b/checkpoint-1500/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:036fce910c5bad6117172c92fb1bde4b13f0cf9d80f30a2e24e4e934ac3e98f8 +size 1465 diff --git a/checkpoint-1500/special_tokens_map.json b/checkpoint-1500/special_tokens_map.json new file mode 100644 index 0000000..8d6368f --- /dev/null +++ b/checkpoint-1500/special_tokens_map.json @@ -0,0 +1,34 @@ +{ + "additional_special_tokens": [ + "", + "" + ], + "bos_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "unk_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + } +} diff --git a/checkpoint-1500/tokenizer.json b/checkpoint-1500/tokenizer.json new file mode 100644 index 0000000..a4a305d --- /dev/null +++ b/checkpoint-1500/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5f7eee611703c5ce5d1eee32d9cdcfe465647b8aff0c1dfb3bed7ad7dbb05060 +size 34362873 diff --git a/checkpoint-1500/tokenizer.model b/checkpoint-1500/tokenizer.model new file mode 100644 index 0000000..796efe9 --- /dev/null +++ b/checkpoint-1500/tokenizer.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:61a7b147390c64585d6c3543dd6fc636906c9af3865a5548f27f31aee1d4c8e2 +size 4241003 diff --git a/checkpoint-1500/tokenizer_config.json b/checkpoint-1500/tokenizer_config.json new file mode 100644 index 0000000..3ade6be --- /dev/null +++ b/checkpoint-1500/tokenizer_config.json @@ -0,0 +1,2013 @@ +{ + "add_bos_token": true, + "add_eos_token": false, + "added_tokens_decoder": { + "0": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "1": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "2": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "3": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "4": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "5": { + "content": "<2mass>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "6": { + "content": "[@BOS@]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "7": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "8": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "9": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "10": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "11": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "12": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "13": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "14": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "15": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "16": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "17": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "18": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "19": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "20": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "21": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "22": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "23": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "24": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "25": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "26": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "27": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "28": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "29": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "30": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "31": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "32": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "33": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "34": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "35": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "36": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "37": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "38": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "39": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "40": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "41": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "42": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "43": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "44": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "45": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "46": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "47": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "48": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "49": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "50": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "51": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "52": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "53": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "54": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "55": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "56": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "57": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "58": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "59": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "60": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "61": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "62": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "63": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "64": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "65": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "66": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "67": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "68": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "69": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "70": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "71": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "72": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "73": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "74": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "75": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "76": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "77": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "78": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "79": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "80": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "81": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "82": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "83": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "84": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "85": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "86": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "87": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "88": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "89": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "90": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "91": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "92": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "93": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "94": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "95": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "96": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "97": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "98": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "99": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "100": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "101": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "102": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "103": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "104": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "105": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "106": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "107": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "108": { + "content": "\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "109": { + "content": "\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "110": { + "content": "\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "111": { + "content": "\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "112": { + "content": "\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "113": { + "content": "\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "114": { + "content": "\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "115": { + "content": "\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "116": { + "content": "\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "117": { + "content": "\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "118": { + "content": "\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "119": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "120": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "121": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "122": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "123": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "124": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "125": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "126": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "127": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "128": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "129": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "130": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "131": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "132": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "133": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "134": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "135": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "136": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "137": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "138": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "139": { + "content": "▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "140": { + "content": "▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "141": { + "content": "▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "142": { + "content": "▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "143": { + "content": "▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "144": { + "content": "▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "145": { + "content": "▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "146": { + "content": "▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "147": { + "content": "▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "148": { + "content": "▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "149": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "150": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "152": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "153": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "154": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "155": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "156": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "157": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "158": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "159": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "160": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "161": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "162": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "163": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "164": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "165": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "166": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "167": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "168": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "169": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "170": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "172": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "173": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "174": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "175": { + "content": "
", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "171": { + "content": "
", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "176": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "177": { + "content": "
", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "178": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "179": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "180": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "181": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "182": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "183": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "184": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "185": { + "content": "

", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "186": { + "content": "

", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "187": { + "content": "

", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "188": { + "content": "

", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "189": { + "content": "

", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "190": { + "content": "
", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "191": { + "content": "
", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "192": { + "content": "
", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "193": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "194": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "195": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "196": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "197": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "198": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "199": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "200": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "201": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "202": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "203": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "204": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "205": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "206": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "207": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "208": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "209": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "210": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "211": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "212": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "213": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "214": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "215": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "216": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255968": { + "content": "[toxicity=0]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255969": { + "content": "\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255970": { + "content": "\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255971": { + "content": "\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255972": { + "content": "\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255973": { + "content": "\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255974": { + "content": "\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255975": { + "content": "\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255976": { + "content": "\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255977": { + "content": "\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255978": { + "content": "\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255979": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255980": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255981": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255982": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255983": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255984": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255985": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255986": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255987": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255988": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255989": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255990": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255991": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255992": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255993": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255994": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255995": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255996": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255997": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255998": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255999": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + } + }, + "additional_special_tokens": [ + "", + "" + ], + "bos_token": "", + "clean_up_tokenization_spaces": false, + "eos_token": "", + "extra_special_tokens": {}, + "model_max_length": 1000000000000000019884624838656, + "pad_token": "", + "sp_model_kwargs": {}, + "spaces_between_special_tokens": false, + "tokenizer_class": "GemmaTokenizer", + "unk_token": "", + "use_default_system_prompt": false +} diff --git a/checkpoint-1500/trainer_state.json b/checkpoint-1500/trainer_state.json new file mode 100644 index 0000000..02b71b8 --- /dev/null +++ b/checkpoint-1500/trainer_state.json @@ -0,0 +1,1864 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 2.4, + "eval_steps": 100, + "global_step": 1500, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 0.08180969459936023, + "epoch": 0.016, + "grad_norm": 0.67578125, + "learning_rate": 9.000000000000001e-07, + "loss": 0.0453, + "mean_token_accuracy": 0.9865816306322813, + "num_tokens": 195510.0, + "step": 10 + }, + { + "entropy": 0.08111731857061386, + "epoch": 0.032, + "grad_norm": 0.60546875, + "learning_rate": 1.9000000000000002e-06, + "loss": 0.0452, + "mean_token_accuracy": 0.986666041240096, + "num_tokens": 391008.0, + "step": 20 + }, + { + "entropy": 0.07587501825764775, + "epoch": 0.048, + "grad_norm": 0.6953125, + "learning_rate": 2.9e-06, + "loss": 0.0445, + "mean_token_accuracy": 0.9873391918838024, + "num_tokens": 590136.0, + "step": 30 + }, + { + "entropy": 0.07685074284672737, + "epoch": 0.064, + "grad_norm": 0.625, + "learning_rate": 3.900000000000001e-06, + "loss": 0.0441, + "mean_token_accuracy": 0.987066001445055, + "num_tokens": 791432.0, + "step": 40 + }, + { + "entropy": 0.07894687210209668, + "epoch": 0.08, + "grad_norm": 0.75390625, + "learning_rate": 4.9000000000000005e-06, + "loss": 0.0447, + "mean_token_accuracy": 0.9865922331809998, + "num_tokens": 990342.0, + "step": 50 + }, + { + "entropy": 0.07919039381667972, + "epoch": 0.096, + "grad_norm": 0.66015625, + "learning_rate": 5.9e-06, + "loss": 0.0466, + "mean_token_accuracy": 0.9863619733601808, + "num_tokens": 1182339.0, + "step": 60 + }, + { + "entropy": 0.07967684981413185, + "epoch": 0.112, + "grad_norm": 0.5859375, + "learning_rate": 6.9e-06, + "loss": 0.0447, + "mean_token_accuracy": 0.9870311982929707, + "num_tokens": 1386193.0, + "step": 70 + }, + { + "entropy": 0.08054462079890072, + "epoch": 0.128, + "grad_norm": 0.65234375, + "learning_rate": 7.9e-06, + "loss": 0.0466, + "mean_token_accuracy": 0.9863289542496204, + "num_tokens": 1583261.0, + "step": 80 + }, + { + "entropy": 0.08421374012250453, + "epoch": 0.144, + "grad_norm": 0.5546875, + "learning_rate": 8.900000000000001e-06, + "loss": 0.0489, + "mean_token_accuracy": 0.9860724166035653, + "num_tokens": 1774826.0, + "step": 90 + }, + { + "entropy": 0.08307966932188719, + "epoch": 0.16, + "grad_norm": 0.71484375, + "learning_rate": 9.9e-06, + "loss": 0.0489, + "mean_token_accuracy": 0.986110083758831, + "num_tokens": 1971310.0, + "step": 100 + }, + { + "epoch": 0.16, + "eval_biology_entropy": 0.3792700763940811, + "eval_biology_loss": 2.260571241378784, + "eval_biology_mean_token_accuracy": 0.6872955484390258, + "eval_biology_num_tokens": 1971310.0, + "eval_biology_runtime": 39.5255, + "eval_biology_samples_per_second": 12.65, + "eval_biology_steps_per_second": 3.163, + "step": 100 + }, + { + "epoch": 0.16, + "eval_chemistry_entropy": 0.23563545274734496, + "eval_chemistry_loss": 1.4561725854873657, + "eval_chemistry_mean_token_accuracy": 0.7805930047035218, + "eval_chemistry_num_tokens": 1971310.0, + "eval_chemistry_runtime": 49.2319, + "eval_chemistry_samples_per_second": 10.156, + "eval_chemistry_steps_per_second": 2.539, + "step": 100 + }, + { + "entropy": 0.08481427943333983, + "epoch": 0.176, + "grad_norm": 0.734375, + "learning_rate": 1.0900000000000002e-05, + "loss": 0.0497, + "mean_token_accuracy": 0.9861054014414549, + "num_tokens": 2169453.0, + "step": 110 + }, + { + "entropy": 0.0882645736914128, + "epoch": 0.192, + "grad_norm": 0.79296875, + "learning_rate": 1.1900000000000001e-05, + "loss": 0.0512, + "mean_token_accuracy": 0.9859256945550442, + "num_tokens": 2366844.0, + "step": 120 + }, + { + "entropy": 0.09435517250094563, + "epoch": 0.208, + "grad_norm": 1.2421875, + "learning_rate": 1.2900000000000002e-05, + "loss": 0.0547, + "mean_token_accuracy": 0.9849376413971186, + "num_tokens": 2559708.0, + "step": 130 + }, + { + "entropy": 0.09682020233012736, + "epoch": 0.224, + "grad_norm": 1.3671875, + "learning_rate": 1.39e-05, + "loss": 0.057, + "mean_token_accuracy": 0.985217172652483, + "num_tokens": 2756533.0, + "step": 140 + }, + { + "entropy": 0.11111663887277246, + "epoch": 0.24, + "grad_norm": 2.84375, + "learning_rate": 1.4900000000000001e-05, + "loss": 0.0657, + "mean_token_accuracy": 0.9836616575717926, + "num_tokens": 2948600.0, + "step": 150 + }, + { + "entropy": 0.12251729485578836, + "epoch": 0.256, + "grad_norm": 2.453125, + "learning_rate": 1.5900000000000004e-05, + "loss": 0.0728, + "mean_token_accuracy": 0.9814188834279776, + "num_tokens": 3141473.0, + "step": 160 + }, + { + "entropy": 0.12813111576251685, + "epoch": 0.272, + "grad_norm": 3.03125, + "learning_rate": 1.69e-05, + "loss": 0.0758, + "mean_token_accuracy": 0.9808279071003199, + "num_tokens": 3337285.0, + "step": 170 + }, + { + "entropy": 0.13337696841917931, + "epoch": 0.288, + "grad_norm": 2.984375, + "learning_rate": 1.79e-05, + "loss": 0.0771, + "mean_token_accuracy": 0.9798636745661498, + "num_tokens": 3541221.0, + "step": 180 + }, + { + "entropy": 0.1436500378418714, + "epoch": 0.304, + "grad_norm": 2.78125, + "learning_rate": 1.8900000000000002e-05, + "loss": 0.0851, + "mean_token_accuracy": 0.9780430492013693, + "num_tokens": 3734952.0, + "step": 190 + }, + { + "entropy": 0.1549748638179153, + "epoch": 0.32, + "grad_norm": 3.390625, + "learning_rate": 1.9900000000000003e-05, + "loss": 0.0946, + "mean_token_accuracy": 0.9750579129904509, + "num_tokens": 3922209.0, + "step": 200 + }, + { + "epoch": 0.32, + "eval_biology_entropy": 0.4169517788887024, + "eval_biology_loss": 2.160527229309082, + "eval_biology_mean_token_accuracy": 0.6808431892395019, + "eval_biology_num_tokens": 3922209.0, + "eval_biology_runtime": 39.5121, + "eval_biology_samples_per_second": 12.654, + "eval_biology_steps_per_second": 3.164, + "step": 200 + }, + { + "epoch": 0.32, + "eval_chemistry_entropy": 0.25474807441234587, + "eval_chemistry_loss": 1.3652101755142212, + "eval_chemistry_mean_token_accuracy": 0.7794562330245972, + "eval_chemistry_num_tokens": 3922209.0, + "eval_chemistry_runtime": 49.2263, + "eval_chemistry_samples_per_second": 10.157, + "eval_chemistry_steps_per_second": 2.539, + "step": 200 + }, + { + "entropy": 0.16029730178415774, + "epoch": 0.336, + "grad_norm": 3.515625, + "learning_rate": 1.9900000000000003e-05, + "loss": 0.097, + "mean_token_accuracy": 0.9749548710882664, + "num_tokens": 4115941.0, + "step": 210 + }, + { + "entropy": 0.16590779088437557, + "epoch": 0.352, + "grad_norm": 3.78125, + "learning_rate": 1.978888888888889e-05, + "loss": 0.1023, + "mean_token_accuracy": 0.9726595059037209, + "num_tokens": 4308755.0, + "step": 220 + }, + { + "entropy": 0.1707212173845619, + "epoch": 0.368, + "grad_norm": 3.390625, + "learning_rate": 1.967777777777778e-05, + "loss": 0.1023, + "mean_token_accuracy": 0.9726378936320543, + "num_tokens": 4505910.0, + "step": 230 + }, + { + "entropy": 0.1765518521424383, + "epoch": 0.384, + "grad_norm": 3.671875, + "learning_rate": 1.956666666666667e-05, + "loss": 0.1083, + "mean_token_accuracy": 0.97143131531775, + "num_tokens": 4696101.0, + "step": 240 + }, + { + "entropy": 0.17895571915432812, + "epoch": 0.4, + "grad_norm": 3.5, + "learning_rate": 1.9455555555555555e-05, + "loss": 0.11, + "mean_token_accuracy": 0.9705049484968186, + "num_tokens": 4889445.0, + "step": 250 + }, + { + "entropy": 0.18151565054431557, + "epoch": 0.416, + "grad_norm": 3.578125, + "learning_rate": 1.9344444444444446e-05, + "loss": 0.1091, + "mean_token_accuracy": 0.970859233289957, + "num_tokens": 5087624.0, + "step": 260 + }, + { + "entropy": 0.18663504091091454, + "epoch": 0.432, + "grad_norm": 3.859375, + "learning_rate": 1.9233333333333334e-05, + "loss": 0.1135, + "mean_token_accuracy": 0.9693388067185879, + "num_tokens": 5273794.0, + "step": 270 + }, + { + "entropy": 0.18451756797730923, + "epoch": 0.448, + "grad_norm": 4.125, + "learning_rate": 1.9122222222222226e-05, + "loss": 0.1144, + "mean_token_accuracy": 0.9698665160685778, + "num_tokens": 5463288.0, + "step": 280 + }, + { + "entropy": 0.17932639280334114, + "epoch": 0.464, + "grad_norm": 3.5625, + "learning_rate": 1.9011111111111114e-05, + "loss": 0.1081, + "mean_token_accuracy": 0.9717006407678127, + "num_tokens": 5656827.0, + "step": 290 + }, + { + "entropy": 0.18305762661620975, + "epoch": 0.48, + "grad_norm": 3.90625, + "learning_rate": 1.8900000000000002e-05, + "loss": 0.1107, + "mean_token_accuracy": 0.9712053131312132, + "num_tokens": 5853311.0, + "step": 300 + }, + { + "epoch": 0.48, + "eval_biology_entropy": 0.42041616535186765, + "eval_biology_loss": 2.1313436031341553, + "eval_biology_mean_token_accuracy": 0.6798483271598816, + "eval_biology_num_tokens": 5853311.0, + "eval_biology_runtime": 39.5137, + "eval_biology_samples_per_second": 12.654, + "eval_biology_steps_per_second": 3.163, + "step": 300 + }, + { + "epoch": 0.48, + "eval_chemistry_entropy": 0.259404793381691, + "eval_chemistry_loss": 1.3463311195373535, + "eval_chemistry_mean_token_accuracy": 0.7787248449325561, + "eval_chemistry_num_tokens": 5853311.0, + "eval_chemistry_runtime": 49.201, + "eval_chemistry_samples_per_second": 10.162, + "eval_chemistry_steps_per_second": 2.541, + "step": 300 + }, + { + "entropy": 0.17875824291259051, + "epoch": 0.496, + "grad_norm": 3.453125, + "learning_rate": 1.878888888888889e-05, + "loss": 0.1101, + "mean_token_accuracy": 0.9710387323051691, + "num_tokens": 6049788.0, + "step": 310 + }, + { + "entropy": 0.1863187742419541, + "epoch": 0.512, + "grad_norm": 3.4375, + "learning_rate": 1.8677777777777778e-05, + "loss": 0.113, + "mean_token_accuracy": 0.9698710650205612, + "num_tokens": 6243825.0, + "step": 320 + }, + { + "entropy": 0.18619006094522775, + "epoch": 0.528, + "grad_norm": 3.890625, + "learning_rate": 1.856666666666667e-05, + "loss": 0.1135, + "mean_token_accuracy": 0.9703223951160908, + "num_tokens": 6434188.0, + "step": 330 + }, + { + "entropy": 0.18118073977530003, + "epoch": 0.544, + "grad_norm": 3.484375, + "learning_rate": 1.8455555555555558e-05, + "loss": 0.1114, + "mean_token_accuracy": 0.9708513412624598, + "num_tokens": 6629832.0, + "step": 340 + }, + { + "entropy": 0.1808169964235276, + "epoch": 0.56, + "grad_norm": 3.28125, + "learning_rate": 1.8344444444444446e-05, + "loss": 0.1079, + "mean_token_accuracy": 0.9716686312109232, + "num_tokens": 6824668.0, + "step": 350 + }, + { + "entropy": 0.18033515606075526, + "epoch": 0.576, + "grad_norm": 3.53125, + "learning_rate": 1.8233333333333334e-05, + "loss": 0.106, + "mean_token_accuracy": 0.9722726982086897, + "num_tokens": 7025923.0, + "step": 360 + }, + { + "entropy": 0.1852203646209091, + "epoch": 0.592, + "grad_norm": 3.625, + "learning_rate": 1.8122222222222222e-05, + "loss": 0.111, + "mean_token_accuracy": 0.9705461781471968, + "num_tokens": 7218224.0, + "step": 370 + }, + { + "entropy": 0.17758405841886998, + "epoch": 0.608, + "grad_norm": 3.65625, + "learning_rate": 1.8011111111111113e-05, + "loss": 0.1066, + "mean_token_accuracy": 0.9723859451711178, + "num_tokens": 7421216.0, + "step": 380 + }, + { + "entropy": 0.18370326030999423, + "epoch": 0.624, + "grad_norm": 3.765625, + "learning_rate": 1.79e-05, + "loss": 0.1096, + "mean_token_accuracy": 0.9708739675581455, + "num_tokens": 7617258.0, + "step": 390 + }, + { + "entropy": 0.1831163068767637, + "epoch": 0.64, + "grad_norm": 3.578125, + "learning_rate": 1.778888888888889e-05, + "loss": 0.1072, + "mean_token_accuracy": 0.9717655830085278, + "num_tokens": 7806173.0, + "step": 400 + }, + { + "epoch": 0.64, + "eval_biology_entropy": 0.42119112753868104, + "eval_biology_loss": 2.1267058849334717, + "eval_biology_mean_token_accuracy": 0.6801503438949585, + "eval_biology_num_tokens": 7806173.0, + "eval_biology_runtime": 39.5121, + "eval_biology_samples_per_second": 12.654, + "eval_biology_steps_per_second": 3.164, + "step": 400 + }, + { + "epoch": 0.64, + "eval_chemistry_entropy": 0.2595027017593384, + "eval_chemistry_loss": 1.3404533863067627, + "eval_chemistry_mean_token_accuracy": 0.7790702543258667, + "eval_chemistry_num_tokens": 7806173.0, + "eval_chemistry_runtime": 49.2002, + "eval_chemistry_samples_per_second": 10.163, + "eval_chemistry_steps_per_second": 2.541, + "step": 400 + }, + { + "entropy": 0.1827191138640046, + "epoch": 0.656, + "grad_norm": 3.84375, + "learning_rate": 1.7677777777777778e-05, + "loss": 0.1088, + "mean_token_accuracy": 0.9715055178850889, + "num_tokens": 8000514.0, + "step": 410 + }, + { + "entropy": 0.18661352493800223, + "epoch": 0.672, + "grad_norm": 3.796875, + "learning_rate": 1.756666666666667e-05, + "loss": 0.1114, + "mean_token_accuracy": 0.9704575203359127, + "num_tokens": 8187884.0, + "step": 420 + }, + { + "entropy": 0.17729734890162946, + "epoch": 0.688, + "grad_norm": 3.09375, + "learning_rate": 1.7455555555555557e-05, + "loss": 0.1046, + "mean_token_accuracy": 0.9725840430706739, + "num_tokens": 8390643.0, + "step": 430 + }, + { + "entropy": 0.18038612157106398, + "epoch": 0.704, + "grad_norm": 3.46875, + "learning_rate": 1.7344444444444445e-05, + "loss": 0.1072, + "mean_token_accuracy": 0.9721495136618614, + "num_tokens": 8583266.0, + "step": 440 + }, + { + "entropy": 0.17609651018865408, + "epoch": 0.72, + "grad_norm": 3.734375, + "learning_rate": 1.7233333333333337e-05, + "loss": 0.1045, + "mean_token_accuracy": 0.9718786559998989, + "num_tokens": 8786233.0, + "step": 450 + }, + { + "entropy": 0.1797325788065791, + "epoch": 0.736, + "grad_norm": 3.390625, + "learning_rate": 1.7122222222222225e-05, + "loss": 0.107, + "mean_token_accuracy": 0.9716725051403046, + "num_tokens": 8982898.0, + "step": 460 + }, + { + "entropy": 0.17950535458512604, + "epoch": 0.752, + "grad_norm": 3.5625, + "learning_rate": 1.7011111111111113e-05, + "loss": 0.1072, + "mean_token_accuracy": 0.9725060146301985, + "num_tokens": 9174361.0, + "step": 470 + }, + { + "entropy": 0.17792485035024583, + "epoch": 0.768, + "grad_norm": 3.375, + "learning_rate": 1.69e-05, + "loss": 0.1025, + "mean_token_accuracy": 0.973187630996108, + "num_tokens": 9373430.0, + "step": 480 + }, + { + "entropy": 0.17844869694672524, + "epoch": 0.784, + "grad_norm": 3.515625, + "learning_rate": 1.678888888888889e-05, + "loss": 0.1073, + "mean_token_accuracy": 0.9717021223157645, + "num_tokens": 9570409.0, + "step": 490 + }, + { + "entropy": 0.17820251639932394, + "epoch": 0.8, + "grad_norm": 3.765625, + "learning_rate": 1.667777777777778e-05, + "loss": 0.1032, + "mean_token_accuracy": 0.9729967709630728, + "num_tokens": 9766360.0, + "step": 500 + }, + { + "epoch": 0.8, + "eval_biology_entropy": 0.41608734560012817, + "eval_biology_loss": 2.139784097671509, + "eval_biology_mean_token_accuracy": 0.6806383662223816, + "eval_biology_num_tokens": 9766360.0, + "eval_biology_runtime": 39.5271, + "eval_biology_samples_per_second": 12.65, + "eval_biology_steps_per_second": 3.162, + "step": 500 + }, + { + "epoch": 0.8, + "eval_chemistry_entropy": 0.25711704087257387, + "eval_chemistry_loss": 1.3562349081039429, + "eval_chemistry_mean_token_accuracy": 0.7787247653007507, + "eval_chemistry_num_tokens": 9766360.0, + "eval_chemistry_runtime": 48.8194, + "eval_chemistry_samples_per_second": 10.242, + "eval_chemistry_steps_per_second": 2.56, + "step": 500 + }, + { + "entropy": 0.1748917239718139, + "epoch": 0.816, + "grad_norm": 3.515625, + "learning_rate": 1.656666666666667e-05, + "loss": 0.1004, + "mean_token_accuracy": 0.9740771554410458, + "num_tokens": 9963834.0, + "step": 510 + }, + { + "entropy": 0.17268717400729655, + "epoch": 0.832, + "grad_norm": 3.125, + "learning_rate": 1.6455555555555556e-05, + "loss": 0.1023, + "mean_token_accuracy": 0.9740318663418293, + "num_tokens": 10160950.0, + "step": 520 + }, + { + "entropy": 0.17282785503193737, + "epoch": 0.848, + "grad_norm": 3.671875, + "learning_rate": 1.6344444444444445e-05, + "loss": 0.0991, + "mean_token_accuracy": 0.9744720965623855, + "num_tokens": 10363131.0, + "step": 530 + }, + { + "entropy": 0.17644098605960606, + "epoch": 0.864, + "grad_norm": 3.34375, + "learning_rate": 1.6233333333333333e-05, + "loss": 0.1026, + "mean_token_accuracy": 0.972842201218009, + "num_tokens": 10558003.0, + "step": 540 + }, + { + "entropy": 0.17435273211449384, + "epoch": 0.88, + "grad_norm": 3.34375, + "learning_rate": 1.6122222222222224e-05, + "loss": 0.1004, + "mean_token_accuracy": 0.9740745916962623, + "num_tokens": 10754470.0, + "step": 550 + }, + { + "entropy": 0.1750097878277302, + "epoch": 0.896, + "grad_norm": 3.453125, + "learning_rate": 1.6011111111111112e-05, + "loss": 0.1007, + "mean_token_accuracy": 0.9743314851075411, + "num_tokens": 10948943.0, + "step": 560 + }, + { + "entropy": 0.17419813955202698, + "epoch": 0.912, + "grad_norm": 3.234375, + "learning_rate": 1.5900000000000004e-05, + "loss": 0.1005, + "mean_token_accuracy": 0.974232492223382, + "num_tokens": 11142613.0, + "step": 570 + }, + { + "entropy": 0.17595540760084988, + "epoch": 0.928, + "grad_norm": 3.546875, + "learning_rate": 1.5788888888888888e-05, + "loss": 0.0997, + "mean_token_accuracy": 0.974004115909338, + "num_tokens": 11336779.0, + "step": 580 + }, + { + "entropy": 0.16677158032543957, + "epoch": 0.944, + "grad_norm": 3.140625, + "learning_rate": 1.567777777777778e-05, + "loss": 0.0982, + "mean_token_accuracy": 0.9750826183706522, + "num_tokens": 11536381.0, + "step": 590 + }, + { + "entropy": 0.16921130288392305, + "epoch": 0.96, + "grad_norm": 3.546875, + "learning_rate": 1.5566666666666668e-05, + "loss": 0.0966, + "mean_token_accuracy": 0.9753067292273044, + "num_tokens": 11735527.0, + "step": 600 + }, + { + "epoch": 0.96, + "eval_biology_entropy": 0.4160035889148712, + "eval_biology_loss": 2.1477200984954834, + "eval_biology_mean_token_accuracy": 0.68089803981781, + "eval_biology_num_tokens": 11735527.0, + "eval_biology_runtime": 39.5581, + "eval_biology_samples_per_second": 12.64, + "eval_biology_steps_per_second": 3.16, + "step": 600 + }, + { + "epoch": 0.96, + "eval_chemistry_entropy": 0.2561914825439453, + "eval_chemistry_loss": 1.3602544069290161, + "eval_chemistry_mean_token_accuracy": 0.779716450214386, + "eval_chemistry_num_tokens": 11735527.0, + "eval_chemistry_runtime": 49.254, + "eval_chemistry_samples_per_second": 10.151, + "eval_chemistry_steps_per_second": 2.538, + "step": 600 + }, + { + "entropy": 0.16858043959364294, + "epoch": 0.976, + "grad_norm": 3.65625, + "learning_rate": 1.5455555555555556e-05, + "loss": 0.0985, + "mean_token_accuracy": 0.9748617243021727, + "num_tokens": 11930935.0, + "step": 610 + }, + { + "entropy": 0.16747406986542046, + "epoch": 0.992, + "grad_norm": 3.15625, + "learning_rate": 1.5344444444444447e-05, + "loss": 0.0955, + "mean_token_accuracy": 0.9756584648042917, + "num_tokens": 12129585.0, + "step": 620 + }, + { + "entropy": 0.15082475617527963, + "epoch": 1.008, + "grad_norm": 2.28125, + "learning_rate": 1.5233333333333335e-05, + "loss": 0.0856, + "mean_token_accuracy": 0.9783346977084875, + "num_tokens": 12325847.0, + "step": 630 + }, + { + "entropy": 0.11756393420509995, + "epoch": 1.024, + "grad_norm": 2.3125, + "learning_rate": 1.5122222222222222e-05, + "loss": 0.07, + "mean_token_accuracy": 0.9816608231514692, + "num_tokens": 12530820.0, + "step": 640 + }, + { + "entropy": 0.12562460631597788, + "epoch": 1.04, + "grad_norm": 2.796875, + "learning_rate": 1.5011111111111112e-05, + "loss": 0.0744, + "mean_token_accuracy": 0.9802271891385317, + "num_tokens": 12725384.0, + "step": 650 + }, + { + "entropy": 0.12352396142669023, + "epoch": 1.056, + "grad_norm": 2.609375, + "learning_rate": 1.4900000000000001e-05, + "loss": 0.0744, + "mean_token_accuracy": 0.9802641972899437, + "num_tokens": 12924592.0, + "step": 660 + }, + { + "entropy": 0.12865133616141974, + "epoch": 1.072, + "grad_norm": 2.671875, + "learning_rate": 1.4788888888888891e-05, + "loss": 0.0774, + "mean_token_accuracy": 0.9798023611307144, + "num_tokens": 13112848.0, + "step": 670 + }, + { + "entropy": 0.12567441049031913, + "epoch": 1.088, + "grad_norm": 2.53125, + "learning_rate": 1.467777777777778e-05, + "loss": 0.0756, + "mean_token_accuracy": 0.9806226130574942, + "num_tokens": 13305725.0, + "step": 680 + }, + { + "entropy": 0.12565653487108647, + "epoch": 1.104, + "grad_norm": 2.234375, + "learning_rate": 1.4566666666666669e-05, + "loss": 0.0745, + "mean_token_accuracy": 0.980208383128047, + "num_tokens": 13498685.0, + "step": 690 + }, + { + "entropy": 0.1318228157237172, + "epoch": 1.12, + "grad_norm": 2.5, + "learning_rate": 1.4455555555555555e-05, + "loss": 0.0778, + "mean_token_accuracy": 0.97950509339571, + "num_tokens": 13682146.0, + "step": 700 + }, + { + "epoch": 1.12, + "eval_biology_entropy": 0.4004362862110138, + "eval_biology_loss": 2.177130699157715, + "eval_biology_mean_token_accuracy": 0.6821538486480713, + "eval_biology_num_tokens": 13682146.0, + "eval_biology_runtime": 39.5089, + "eval_biology_samples_per_second": 12.655, + "eval_biology_steps_per_second": 3.164, + "step": 700 + }, + { + "epoch": 1.12, + "eval_chemistry_entropy": 0.24954386246204377, + "eval_chemistry_loss": 1.3777952194213867, + "eval_chemistry_mean_token_accuracy": 0.7797037825584412, + "eval_chemistry_num_tokens": 13682146.0, + "eval_chemistry_runtime": 49.2017, + "eval_chemistry_samples_per_second": 10.162, + "eval_chemistry_steps_per_second": 2.541, + "step": 700 + }, + { + "entropy": 0.12691126235295086, + "epoch": 1.1360000000000001, + "grad_norm": 2.6875, + "learning_rate": 1.4344444444444445e-05, + "loss": 0.0757, + "mean_token_accuracy": 0.980040468648076, + "num_tokens": 13876656.0, + "step": 710 + }, + { + "entropy": 0.12381730475462974, + "epoch": 1.152, + "grad_norm": 2.359375, + "learning_rate": 1.4233333333333335e-05, + "loss": 0.0743, + "mean_token_accuracy": 0.980041703954339, + "num_tokens": 14086104.0, + "step": 720 + }, + { + "entropy": 0.1285357222892344, + "epoch": 1.168, + "grad_norm": 2.5625, + "learning_rate": 1.4122222222222225e-05, + "loss": 0.0763, + "mean_token_accuracy": 0.9799800109118223, + "num_tokens": 14274520.0, + "step": 730 + }, + { + "entropy": 0.12321301018819213, + "epoch": 1.184, + "grad_norm": 2.375, + "learning_rate": 1.4011111111111113e-05, + "loss": 0.0734, + "mean_token_accuracy": 0.9810513865202666, + "num_tokens": 14473623.0, + "step": 740 + }, + { + "entropy": 0.1292376267723739, + "epoch": 1.2, + "grad_norm": 2.609375, + "learning_rate": 1.39e-05, + "loss": 0.0784, + "mean_token_accuracy": 0.9795571226626635, + "num_tokens": 14661238.0, + "step": 750 + }, + { + "entropy": 0.12086025532335043, + "epoch": 1.216, + "grad_norm": 2.640625, + "learning_rate": 1.3788888888888889e-05, + "loss": 0.073, + "mean_token_accuracy": 0.9810031026601791, + "num_tokens": 14865915.0, + "step": 760 + }, + { + "entropy": 0.1262558945454657, + "epoch": 1.232, + "grad_norm": 2.40625, + "learning_rate": 1.3677777777777779e-05, + "loss": 0.0786, + "mean_token_accuracy": 0.9797264195978641, + "num_tokens": 15055682.0, + "step": 770 + }, + { + "entropy": 0.12587894778698683, + "epoch": 1.248, + "grad_norm": 2.484375, + "learning_rate": 1.3566666666666668e-05, + "loss": 0.0744, + "mean_token_accuracy": 0.9802309483289718, + "num_tokens": 15249576.0, + "step": 780 + }, + { + "entropy": 0.12361958674155175, + "epoch": 1.264, + "grad_norm": 2.1875, + "learning_rate": 1.3455555555555556e-05, + "loss": 0.0726, + "mean_token_accuracy": 0.9805784326046705, + "num_tokens": 15446149.0, + "step": 790 + }, + { + "entropy": 0.12549991067498922, + "epoch": 1.28, + "grad_norm": 2.546875, + "learning_rate": 1.3344444444444446e-05, + "loss": 0.0743, + "mean_token_accuracy": 0.9805208504199981, + "num_tokens": 15639317.0, + "step": 800 + }, + { + "epoch": 1.28, + "eval_biology_entropy": 0.4032972083091736, + "eval_biology_loss": 2.1712591648101807, + "eval_biology_mean_token_accuracy": 0.6816940937042236, + "eval_biology_num_tokens": 15639317.0, + "eval_biology_runtime": 39.5209, + "eval_biology_samples_per_second": 12.652, + "eval_biology_steps_per_second": 3.163, + "step": 800 + }, + { + "epoch": 1.28, + "eval_chemistry_entropy": 0.25105089938640596, + "eval_chemistry_loss": 1.3797656297683716, + "eval_chemistry_mean_token_accuracy": 0.7792046613693238, + "eval_chemistry_num_tokens": 15639317.0, + "eval_chemistry_runtime": 49.2346, + "eval_chemistry_samples_per_second": 10.155, + "eval_chemistry_steps_per_second": 2.539, + "step": 800 + }, + { + "entropy": 0.12616121221799403, + "epoch": 1.296, + "grad_norm": 2.203125, + "learning_rate": 1.3233333333333334e-05, + "loss": 0.075, + "mean_token_accuracy": 0.9805853247642518, + "num_tokens": 15835805.0, + "step": 810 + }, + { + "entropy": 0.12473565563559533, + "epoch": 1.312, + "grad_norm": 2.296875, + "learning_rate": 1.3122222222222222e-05, + "loss": 0.0738, + "mean_token_accuracy": 0.9808966059237718, + "num_tokens": 16028375.0, + "step": 820 + }, + { + "entropy": 0.12206203136593104, + "epoch": 1.328, + "grad_norm": 2.265625, + "learning_rate": 1.3011111111111112e-05, + "loss": 0.0712, + "mean_token_accuracy": 0.9813739657402039, + "num_tokens": 16230555.0, + "step": 830 + }, + { + "entropy": 0.11784348418004811, + "epoch": 1.3439999999999999, + "grad_norm": 2.359375, + "learning_rate": 1.2900000000000002e-05, + "loss": 0.0692, + "mean_token_accuracy": 0.9817070845514536, + "num_tokens": 16436297.0, + "step": 840 + }, + { + "entropy": 0.12049585804343224, + "epoch": 1.3599999999999999, + "grad_norm": 2.234375, + "learning_rate": 1.278888888888889e-05, + "loss": 0.0719, + "mean_token_accuracy": 0.9809176307171583, + "num_tokens": 16630563.0, + "step": 850 + }, + { + "entropy": 0.12437316868454218, + "epoch": 1.376, + "grad_norm": 2.8125, + "learning_rate": 1.2677777777777778e-05, + "loss": 0.0751, + "mean_token_accuracy": 0.9806294579058885, + "num_tokens": 16822602.0, + "step": 860 + }, + { + "entropy": 0.12243083966895937, + "epoch": 1.392, + "grad_norm": 2.34375, + "learning_rate": 1.2566666666666668e-05, + "loss": 0.0729, + "mean_token_accuracy": 0.9806884177029133, + "num_tokens": 17015750.0, + "step": 870 + }, + { + "entropy": 0.12194055491127073, + "epoch": 1.408, + "grad_norm": 2.21875, + "learning_rate": 1.2455555555555556e-05, + "loss": 0.0738, + "mean_token_accuracy": 0.9811285879462958, + "num_tokens": 17207248.0, + "step": 880 + }, + { + "entropy": 0.12085925284773111, + "epoch": 1.424, + "grad_norm": 2.40625, + "learning_rate": 1.2344444444444446e-05, + "loss": 0.0714, + "mean_token_accuracy": 0.9814036220312119, + "num_tokens": 17403869.0, + "step": 890 + }, + { + "entropy": 0.12371515287086368, + "epoch": 1.44, + "grad_norm": 2.671875, + "learning_rate": 1.2233333333333335e-05, + "loss": 0.0732, + "mean_token_accuracy": 0.9810383848845958, + "num_tokens": 17597428.0, + "step": 900 + }, + { + "epoch": 1.44, + "eval_biology_entropy": 0.3943380331993103, + "eval_biology_loss": 2.181544542312622, + "eval_biology_mean_token_accuracy": 0.6840425395965576, + "eval_biology_num_tokens": 17597428.0, + "eval_biology_runtime": 39.5119, + "eval_biology_samples_per_second": 12.654, + "eval_biology_steps_per_second": 3.164, + "step": 900 + }, + { + "epoch": 1.44, + "eval_chemistry_entropy": 0.24567515325546266, + "eval_chemistry_loss": 1.3865065574645996, + "eval_chemistry_mean_token_accuracy": 0.7803217101097107, + "eval_chemistry_num_tokens": 17597428.0, + "eval_chemistry_runtime": 49.2083, + "eval_chemistry_samples_per_second": 10.161, + "eval_chemistry_steps_per_second": 2.54, + "step": 900 + }, + { + "entropy": 0.11841928996145726, + "epoch": 1.456, + "grad_norm": 2.421875, + "learning_rate": 1.2122222222222223e-05, + "loss": 0.0692, + "mean_token_accuracy": 0.9820894330739975, + "num_tokens": 17798083.0, + "step": 910 + }, + { + "entropy": 0.12107395366765558, + "epoch": 1.472, + "grad_norm": 2.09375, + "learning_rate": 1.2011111111111111e-05, + "loss": 0.0727, + "mean_token_accuracy": 0.980775249004364, + "num_tokens": 17993802.0, + "step": 920 + }, + { + "entropy": 0.11666952911764383, + "epoch": 1.488, + "grad_norm": 2.234375, + "learning_rate": 1.1900000000000001e-05, + "loss": 0.0711, + "mean_token_accuracy": 0.9814896911382676, + "num_tokens": 18185570.0, + "step": 930 + }, + { + "entropy": 0.12296288679353892, + "epoch": 1.504, + "grad_norm": 2.203125, + "learning_rate": 1.178888888888889e-05, + "loss": 0.0721, + "mean_token_accuracy": 0.9813314471393824, + "num_tokens": 18377903.0, + "step": 940 + }, + { + "entropy": 0.11539274472743273, + "epoch": 1.52, + "grad_norm": 2.21875, + "learning_rate": 1.1677777777777779e-05, + "loss": 0.0691, + "mean_token_accuracy": 0.9822911247611046, + "num_tokens": 18579284.0, + "step": 950 + }, + { + "entropy": 0.12136667580343782, + "epoch": 1.536, + "grad_norm": 2.140625, + "learning_rate": 1.1566666666666669e-05, + "loss": 0.0725, + "mean_token_accuracy": 0.9812386609613896, + "num_tokens": 18765324.0, + "step": 960 + }, + { + "entropy": 0.11671084221452474, + "epoch": 1.552, + "grad_norm": 2.15625, + "learning_rate": 1.1455555555555557e-05, + "loss": 0.0676, + "mean_token_accuracy": 0.9818577177822589, + "num_tokens": 18966382.0, + "step": 970 + }, + { + "entropy": 0.11611056034453213, + "epoch": 1.568, + "grad_norm": 2.296875, + "learning_rate": 1.1344444444444445e-05, + "loss": 0.0701, + "mean_token_accuracy": 0.9818583976477384, + "num_tokens": 19160509.0, + "step": 980 + }, + { + "entropy": 0.11777197350747884, + "epoch": 1.584, + "grad_norm": 2.1875, + "learning_rate": 1.1233333333333333e-05, + "loss": 0.0697, + "mean_token_accuracy": 0.9817087065428496, + "num_tokens": 19354662.0, + "step": 990 + }, + { + "entropy": 0.12017350159585476, + "epoch": 1.6, + "grad_norm": 2.28125, + "learning_rate": 1.1122222222222223e-05, + "loss": 0.0718, + "mean_token_accuracy": 0.9814203523099423, + "num_tokens": 19543088.0, + "step": 1000 + }, + { + "epoch": 1.6, + "eval_biology_entropy": 0.3948675305843353, + "eval_biology_loss": 2.1792049407958984, + "eval_biology_mean_token_accuracy": 0.6840984811782836, + "eval_biology_num_tokens": 19543088.0, + "eval_biology_runtime": 39.6119, + "eval_biology_samples_per_second": 12.622, + "eval_biology_steps_per_second": 3.156, + "step": 1000 + }, + { + "epoch": 1.6, + "eval_chemistry_entropy": 0.24652056658267976, + "eval_chemistry_loss": 1.388296365737915, + "eval_chemistry_mean_token_accuracy": 0.7802323937416077, + "eval_chemistry_num_tokens": 19543088.0, + "eval_chemistry_runtime": 48.7687, + "eval_chemistry_samples_per_second": 10.252, + "eval_chemistry_steps_per_second": 2.563, + "step": 1000 + }, + { + "entropy": 0.11641141260042787, + "epoch": 1.616, + "grad_norm": 2.109375, + "learning_rate": 1.1011111111111113e-05, + "loss": 0.0685, + "mean_token_accuracy": 0.9818709082901478, + "num_tokens": 19743506.0, + "step": 1010 + }, + { + "entropy": 0.11852226112969219, + "epoch": 1.6320000000000001, + "grad_norm": 2.40625, + "learning_rate": 1.0900000000000002e-05, + "loss": 0.0679, + "mean_token_accuracy": 0.9817365873605013, + "num_tokens": 19937567.0, + "step": 1020 + }, + { + "entropy": 0.11748991990461946, + "epoch": 1.6480000000000001, + "grad_norm": 2.125, + "learning_rate": 1.0788888888888889e-05, + "loss": 0.0699, + "mean_token_accuracy": 0.9814462032169103, + "num_tokens": 20129743.0, + "step": 1030 + }, + { + "entropy": 0.11318188132718206, + "epoch": 1.6640000000000001, + "grad_norm": 2.109375, + "learning_rate": 1.0677777777777779e-05, + "loss": 0.068, + "mean_token_accuracy": 0.9825476247817277, + "num_tokens": 20331548.0, + "step": 1040 + }, + { + "entropy": 0.11528927914332598, + "epoch": 1.6800000000000002, + "grad_norm": 2.078125, + "learning_rate": 1.0566666666666667e-05, + "loss": 0.0684, + "mean_token_accuracy": 0.9821649711579085, + "num_tokens": 20524247.0, + "step": 1050 + }, + { + "entropy": 0.11758545534685254, + "epoch": 1.696, + "grad_norm": 1.9296875, + "learning_rate": 1.0455555555555556e-05, + "loss": 0.0688, + "mean_token_accuracy": 0.9817702580243349, + "num_tokens": 20718121.0, + "step": 1060 + }, + { + "entropy": 0.11215258438605816, + "epoch": 1.712, + "grad_norm": 2.109375, + "learning_rate": 1.0344444444444446e-05, + "loss": 0.066, + "mean_token_accuracy": 0.9826954051852226, + "num_tokens": 20921419.0, + "step": 1070 + }, + { + "entropy": 0.11761863068677485, + "epoch": 1.728, + "grad_norm": 2.078125, + "learning_rate": 1.0233333333333336e-05, + "loss": 0.0673, + "mean_token_accuracy": 0.9817208234220743, + "num_tokens": 21118581.0, + "step": 1080 + }, + { + "entropy": 0.11461647190153598, + "epoch": 1.744, + "grad_norm": 2.34375, + "learning_rate": 1.0122222222222222e-05, + "loss": 0.0683, + "mean_token_accuracy": 0.9821706589311361, + "num_tokens": 21315128.0, + "step": 1090 + }, + { + "entropy": 0.11213490981608629, + "epoch": 1.76, + "grad_norm": 1.921875, + "learning_rate": 1.0011111111111112e-05, + "loss": 0.0668, + "mean_token_accuracy": 0.9826094429939985, + "num_tokens": 21511066.0, + "step": 1100 + }, + { + "epoch": 1.76, + "eval_biology_entropy": 0.39157771730422974, + "eval_biology_loss": 2.183682680130005, + "eval_biology_mean_token_accuracy": 0.6850721926689148, + "eval_biology_num_tokens": 21511066.0, + "eval_biology_runtime": 39.4964, + "eval_biology_samples_per_second": 12.659, + "eval_biology_steps_per_second": 3.165, + "step": 1100 + }, + { + "epoch": 1.76, + "eval_chemistry_entropy": 0.24445027112960815, + "eval_chemistry_loss": 1.3950114250183105, + "eval_chemistry_mean_token_accuracy": 0.7797145857810974, + "eval_chemistry_num_tokens": 21511066.0, + "eval_chemistry_runtime": 49.2195, + "eval_chemistry_samples_per_second": 10.159, + "eval_chemistry_steps_per_second": 2.54, + "step": 1100 + }, + { + "entropy": 0.11297063264064491, + "epoch": 1.776, + "grad_norm": 2.078125, + "learning_rate": 9.9e-06, + "loss": 0.0678, + "mean_token_accuracy": 0.9824550054967404, + "num_tokens": 21704368.0, + "step": 1110 + }, + { + "entropy": 0.11269787652418017, + "epoch": 1.792, + "grad_norm": 1.8359375, + "learning_rate": 9.78888888888889e-06, + "loss": 0.0652, + "mean_token_accuracy": 0.9826549135148526, + "num_tokens": 21905784.0, + "step": 1120 + }, + { + "entropy": 0.11504443329758943, + "epoch": 1.808, + "grad_norm": 1.9765625, + "learning_rate": 9.677777777777778e-06, + "loss": 0.0673, + "mean_token_accuracy": 0.9823829177767038, + "num_tokens": 22094159.0, + "step": 1130 + }, + { + "entropy": 0.11632196293212474, + "epoch": 1.8239999999999998, + "grad_norm": 1.9921875, + "learning_rate": 9.566666666666668e-06, + "loss": 0.0672, + "mean_token_accuracy": 0.9823333639651537, + "num_tokens": 22290654.0, + "step": 1140 + }, + { + "entropy": 0.11296359319239854, + "epoch": 1.8399999999999999, + "grad_norm": 2.0625, + "learning_rate": 9.455555555555557e-06, + "loss": 0.0675, + "mean_token_accuracy": 0.9825551677495241, + "num_tokens": 22485675.0, + "step": 1150 + }, + { + "entropy": 0.10986362439580262, + "epoch": 1.8559999999999999, + "grad_norm": 2.265625, + "learning_rate": 9.344444444444446e-06, + "loss": 0.0637, + "mean_token_accuracy": 0.9831937052309513, + "num_tokens": 22689900.0, + "step": 1160 + }, + { + "entropy": 0.11179361823014915, + "epoch": 1.8719999999999999, + "grad_norm": 1.828125, + "learning_rate": 9.233333333333334e-06, + "loss": 0.065, + "mean_token_accuracy": 0.9827314846217632, + "num_tokens": 22887275.0, + "step": 1170 + }, + { + "entropy": 0.11337793134152889, + "epoch": 1.888, + "grad_norm": 1.875, + "learning_rate": 9.122222222222223e-06, + "loss": 0.0663, + "mean_token_accuracy": 0.9825703386217356, + "num_tokens": 23081241.0, + "step": 1180 + }, + { + "entropy": 0.11284571103751659, + "epoch": 1.904, + "grad_norm": 2.296875, + "learning_rate": 9.011111111111111e-06, + "loss": 0.0655, + "mean_token_accuracy": 0.9827509298920631, + "num_tokens": 23276417.0, + "step": 1190 + }, + { + "entropy": 0.1114346832036972, + "epoch": 1.92, + "grad_norm": 1.9296875, + "learning_rate": 8.900000000000001e-06, + "loss": 0.065, + "mean_token_accuracy": 0.9832344222813845, + "num_tokens": 23476340.0, + "step": 1200 + }, + { + "epoch": 1.92, + "eval_biology_entropy": 0.3916603057384491, + "eval_biology_loss": 2.1840121746063232, + "eval_biology_mean_token_accuracy": 0.6849033751487732, + "eval_biology_num_tokens": 23476340.0, + "eval_biology_runtime": 39.5552, + "eval_biology_samples_per_second": 12.641, + "eval_biology_steps_per_second": 3.16, + "step": 1200 + }, + { + "epoch": 1.92, + "eval_chemistry_entropy": 0.24395500665903092, + "eval_chemistry_loss": 1.3961467742919922, + "eval_chemistry_mean_token_accuracy": 0.7803040590286255, + "eval_chemistry_num_tokens": 23476340.0, + "eval_chemistry_runtime": 49.2524, + "eval_chemistry_samples_per_second": 10.152, + "eval_chemistry_steps_per_second": 2.538, + "step": 1200 + }, + { + "entropy": 0.10977238831110299, + "epoch": 1.936, + "grad_norm": 2.078125, + "learning_rate": 8.788888888888891e-06, + "loss": 0.0642, + "mean_token_accuracy": 0.9830574735999107, + "num_tokens": 23673208.0, + "step": 1210 + }, + { + "entropy": 0.10960933691821992, + "epoch": 1.952, + "grad_norm": 1.7109375, + "learning_rate": 8.677777777777779e-06, + "loss": 0.0637, + "mean_token_accuracy": 0.9831824779510498, + "num_tokens": 23870993.0, + "step": 1220 + }, + { + "entropy": 0.11105740028433501, + "epoch": 1.968, + "grad_norm": 1.8984375, + "learning_rate": 8.566666666666667e-06, + "loss": 0.0648, + "mean_token_accuracy": 0.9832170728594065, + "num_tokens": 24066234.0, + "step": 1230 + }, + { + "entropy": 0.10861094021238386, + "epoch": 1.984, + "grad_norm": 1.9296875, + "learning_rate": 8.455555555555555e-06, + "loss": 0.0642, + "mean_token_accuracy": 0.9830795601010323, + "num_tokens": 24262389.0, + "step": 1240 + }, + { + "entropy": 0.11186399965081364, + "epoch": 2.0, + "grad_norm": 2.15625, + "learning_rate": 8.344444444444445e-06, + "loss": 0.0659, + "mean_token_accuracy": 0.9825448121875524, + "num_tokens": 24455544.0, + "step": 1250 + }, + { + "entropy": 0.088846565480344, + "epoch": 2.016, + "grad_norm": 1.1328125, + "learning_rate": 8.233333333333335e-06, + "loss": 0.0505, + "mean_token_accuracy": 0.9856560029089451, + "num_tokens": 24645425.0, + "step": 1260 + }, + { + "entropy": 0.08578005221206694, + "epoch": 2.032, + "grad_norm": 0.98828125, + "learning_rate": 8.122222222222223e-06, + "loss": 0.0495, + "mean_token_accuracy": 0.9862964119762182, + "num_tokens": 24835956.0, + "step": 1270 + }, + { + "entropy": 0.08365524045657366, + "epoch": 2.048, + "grad_norm": 0.890625, + "learning_rate": 8.011111111111113e-06, + "loss": 0.0482, + "mean_token_accuracy": 0.9864422902464867, + "num_tokens": 25036470.0, + "step": 1280 + }, + { + "entropy": 0.08349559165071696, + "epoch": 2.064, + "grad_norm": 0.8671875, + "learning_rate": 7.9e-06, + "loss": 0.0489, + "mean_token_accuracy": 0.9865494292229414, + "num_tokens": 25230959.0, + "step": 1290 + }, + { + "entropy": 0.08540160392876714, + "epoch": 2.08, + "grad_norm": 0.98828125, + "learning_rate": 7.788888888888889e-06, + "loss": 0.0498, + "mean_token_accuracy": 0.9862173371016979, + "num_tokens": 25422625.0, + "step": 1300 + }, + { + "epoch": 2.08, + "eval_biology_entropy": 0.37641260612010957, + "eval_biology_loss": 2.236539125442505, + "eval_biology_mean_token_accuracy": 0.6860877084732055, + "eval_biology_num_tokens": 25422625.0, + "eval_biology_runtime": 39.5503, + "eval_biology_samples_per_second": 12.642, + "eval_biology_steps_per_second": 3.161, + "step": 1300 + }, + { + "epoch": 2.08, + "eval_chemistry_entropy": 0.2347547177672386, + "eval_chemistry_loss": 1.4374333620071411, + "eval_chemistry_mean_token_accuracy": 0.7814928584098816, + "eval_chemistry_num_tokens": 25422625.0, + "eval_chemistry_runtime": 49.2536, + "eval_chemistry_samples_per_second": 10.152, + "eval_chemistry_steps_per_second": 2.538, + "step": 1300 + }, + { + "entropy": 0.08387249778024852, + "epoch": 2.096, + "grad_norm": 0.9296875, + "learning_rate": 7.677777777777778e-06, + "loss": 0.0496, + "mean_token_accuracy": 0.9860624361783266, + "num_tokens": 25614286.0, + "step": 1310 + }, + { + "entropy": 0.08452012583147735, + "epoch": 2.112, + "grad_norm": 0.88671875, + "learning_rate": 7.566666666666667e-06, + "loss": 0.0489, + "mean_token_accuracy": 0.986203158274293, + "num_tokens": 25810384.0, + "step": 1320 + }, + { + "entropy": 0.08262390322051942, + "epoch": 2.128, + "grad_norm": 0.84765625, + "learning_rate": 7.455555555555556e-06, + "loss": 0.0493, + "mean_token_accuracy": 0.9862308964133263, + "num_tokens": 26005047.0, + "step": 1330 + }, + { + "entropy": 0.08097941202577204, + "epoch": 2.144, + "grad_norm": 1.1796875, + "learning_rate": 7.344444444444445e-06, + "loss": 0.0481, + "mean_token_accuracy": 0.9865331679582596, + "num_tokens": 26207169.0, + "step": 1340 + }, + { + "entropy": 0.08371762945316732, + "epoch": 2.16, + "grad_norm": 0.90234375, + "learning_rate": 7.233333333333334e-06, + "loss": 0.0511, + "mean_token_accuracy": 0.985823930054903, + "num_tokens": 26402150.0, + "step": 1350 + }, + { + "entropy": 0.08566265315748751, + "epoch": 2.176, + "grad_norm": 0.8125, + "learning_rate": 7.122222222222222e-06, + "loss": 0.0487, + "mean_token_accuracy": 0.9863390911370515, + "num_tokens": 26599464.0, + "step": 1360 + }, + { + "entropy": 0.08372032032348216, + "epoch": 2.192, + "grad_norm": 0.9375, + "learning_rate": 7.011111111111112e-06, + "loss": 0.0503, + "mean_token_accuracy": 0.9858050804585219, + "num_tokens": 26790601.0, + "step": 1370 + }, + { + "entropy": 0.08293257770128548, + "epoch": 2.208, + "grad_norm": 0.93359375, + "learning_rate": 6.9e-06, + "loss": 0.049, + "mean_token_accuracy": 0.9863026011735201, + "num_tokens": 26988598.0, + "step": 1380 + }, + { + "entropy": 0.08349582962691784, + "epoch": 2.224, + "grad_norm": 0.76171875, + "learning_rate": 6.788888888888889e-06, + "loss": 0.0492, + "mean_token_accuracy": 0.9860329531133175, + "num_tokens": 27181933.0, + "step": 1390 + }, + { + "entropy": 0.08361051741521805, + "epoch": 2.24, + "grad_norm": 0.89453125, + "learning_rate": 6.677777777777779e-06, + "loss": 0.0496, + "mean_token_accuracy": 0.985880882665515, + "num_tokens": 27378686.0, + "step": 1400 + }, + { + "epoch": 2.24, + "eval_biology_entropy": 0.3741675862073898, + "eval_biology_loss": 2.242098569869995, + "eval_biology_mean_token_accuracy": 0.6867531027793884, + "eval_biology_num_tokens": 27378686.0, + "eval_biology_runtime": 39.7857, + "eval_biology_samples_per_second": 12.567, + "eval_biology_steps_per_second": 3.142, + "step": 1400 + }, + { + "epoch": 2.24, + "eval_chemistry_entropy": 0.23381740498542786, + "eval_chemistry_loss": 1.4444365501403809, + "eval_chemistry_mean_token_accuracy": 0.7810734486579896, + "eval_chemistry_num_tokens": 27378686.0, + "eval_chemistry_runtime": 49.2506, + "eval_chemistry_samples_per_second": 10.152, + "eval_chemistry_steps_per_second": 2.538, + "step": 1400 + }, + { + "entropy": 0.0837663013022393, + "epoch": 2.2560000000000002, + "grad_norm": 0.91015625, + "learning_rate": 6.566666666666667e-06, + "loss": 0.0495, + "mean_token_accuracy": 0.9859327003359795, + "num_tokens": 27571908.0, + "step": 1410 + }, + { + "entropy": 0.08184252071660011, + "epoch": 2.2720000000000002, + "grad_norm": 0.80859375, + "learning_rate": 6.455555555555556e-06, + "loss": 0.0485, + "mean_token_accuracy": 0.9862752124667168, + "num_tokens": 27770478.0, + "step": 1420 + }, + { + "entropy": 0.0840547483181581, + "epoch": 2.288, + "grad_norm": 0.9296875, + "learning_rate": 6.3444444444444454e-06, + "loss": 0.0495, + "mean_token_accuracy": 0.9857813119888306, + "num_tokens": 27965558.0, + "step": 1430 + }, + { + "entropy": 0.08257126114331186, + "epoch": 2.304, + "grad_norm": 0.7734375, + "learning_rate": 6.2333333333333335e-06, + "loss": 0.0481, + "mean_token_accuracy": 0.9863984134048224, + "num_tokens": 28165624.0, + "step": 1440 + }, + { + "entropy": 0.08313491414301097, + "epoch": 2.32, + "grad_norm": 0.9453125, + "learning_rate": 6.1222222222222224e-06, + "loss": 0.0492, + "mean_token_accuracy": 0.9862582210451365, + "num_tokens": 28363594.0, + "step": 1450 + }, + { + "entropy": 0.08305449401959777, + "epoch": 2.336, + "grad_norm": 1.0234375, + "learning_rate": 6.011111111111112e-06, + "loss": 0.0495, + "mean_token_accuracy": 0.9860414322465658, + "num_tokens": 28558104.0, + "step": 1460 + }, + { + "entropy": 0.08424469081219285, + "epoch": 2.352, + "grad_norm": 0.89453125, + "learning_rate": 5.9e-06, + "loss": 0.0509, + "mean_token_accuracy": 0.9856379702687263, + "num_tokens": 28747265.0, + "step": 1470 + }, + { + "entropy": 0.08349237225484103, + "epoch": 2.368, + "grad_norm": 0.89453125, + "learning_rate": 5.788888888888889e-06, + "loss": 0.0488, + "mean_token_accuracy": 0.9860499382019043, + "num_tokens": 28944694.0, + "step": 1480 + }, + { + "entropy": 0.08621921115554869, + "epoch": 2.384, + "grad_norm": 0.875, + "learning_rate": 5.677777777777779e-06, + "loss": 0.0506, + "mean_token_accuracy": 0.9857000894844532, + "num_tokens": 29136617.0, + "step": 1490 + }, + { + "entropy": 0.08343604276888072, + "epoch": 2.4, + "grad_norm": 0.76953125, + "learning_rate": 5.566666666666667e-06, + "loss": 0.0493, + "mean_token_accuracy": 0.986125298216939, + "num_tokens": 29333526.0, + "step": 1500 + }, + { + "epoch": 2.4, + "eval_biology_entropy": 0.37332249569892884, + "eval_biology_loss": 2.2458555698394775, + "eval_biology_mean_token_accuracy": 0.6866299691200256, + "eval_biology_num_tokens": 29333526.0, + "eval_biology_runtime": 39.6765, + "eval_biology_samples_per_second": 12.602, + "eval_biology_steps_per_second": 3.15, + "step": 1500 + }, + { + "epoch": 2.4, + "eval_chemistry_entropy": 0.2335134366750717, + "eval_chemistry_loss": 1.446710228919983, + "eval_chemistry_mean_token_accuracy": 0.7809953107833862, + "eval_chemistry_num_tokens": 29333526.0, + "eval_chemistry_runtime": 48.8671, + "eval_chemistry_samples_per_second": 10.232, + "eval_chemistry_steps_per_second": 2.558, + "step": 1500 + } + ], + "logging_steps": 10, + "max_steps": 2000, + "num_input_tokens_seen": 0, + "num_train_epochs": 4, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 2.458799785321206e+18, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/checkpoint-1500/training_args.bin b/checkpoint-1500/training_args.bin new file mode 100644 index 0000000..4c8c7f4 --- /dev/null +++ b/checkpoint-1500/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:387a0fa8295895440900c3b62c83fa1f8ff3d01320d6ee63d02eb78146f15c65 +size 6545 diff --git a/checkpoint-2000/chat_template.jinja b/checkpoint-2000/chat_template.jinja new file mode 100644 index 0000000..923ec25 --- /dev/null +++ b/checkpoint-2000/chat_template.jinja @@ -0,0 +1,4 @@ +{{ bos_token }}{% if messages[0]['role'] == 'system' %}{{ raise_exception('System role not supported') }}{% endif %}{% for message in messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if (message['role'] == 'assistant') %}{% set role = 'model' %}{% else %}{% set role = message['role'] %}{% endif %}{{ '' + role + ' +' + message['content'] | trim + ' +' }}{% endfor %}{% if add_generation_prompt %}{{'model +'}}{% endif %} \ No newline at end of file diff --git a/checkpoint-2000/config.json b/checkpoint-2000/config.json new file mode 100644 index 0000000..29e6cbe --- /dev/null +++ b/checkpoint-2000/config.json @@ -0,0 +1,77 @@ +{ + "architectures": [ + "Gemma2ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "attn_logit_softcapping": 50.0, + "bos_token_id": 2, + "cache_implementation": "hybrid", + "dtype": "bfloat16", + "eos_token_id": 1, + "final_logit_softcapping": 30.0, + "head_dim": 256, + "hidden_act": "gelu_pytorch_tanh", + "hidden_activation": "gelu_pytorch_tanh", + "hidden_size": 3584, + "initializer_range": 0.02, + "intermediate_size": 14336, + "layer_types": [ + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention" + ], + "max_position_embeddings": 8192, + "model_type": "gemma2", + "num_attention_heads": 16, + "num_hidden_layers": 42, + "num_key_value_heads": 8, + "pad_token_id": 0, + "query_pre_attn_scalar": 256, + "rms_norm_eps": 1e-06, + "rope_theta": 10000.0, + "sliding_window": 4096, + "sliding_window_size": 4096, + "transformers_version": "4.56.1", + "use_cache": true, + "vocab_size": 256000 +} diff --git a/checkpoint-2000/generation_config.json b/checkpoint-2000/generation_config.json new file mode 100644 index 0000000..f2c34f2 --- /dev/null +++ b/checkpoint-2000/generation_config.json @@ -0,0 +1,9 @@ +{ + "_from_model_config": true, + "bos_token_id": 2, + "eos_token_id": [ + 1 + ], + "pad_token_id": 0, + "transformers_version": "4.56.1" +} diff --git a/checkpoint-2000/model-00001-of-00004.safetensors b/checkpoint-2000/model-00001-of-00004.safetensors new file mode 100644 index 0000000..4705c7e --- /dev/null +++ b/checkpoint-2000/model-00001-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a0d4eb4fcbddfe01b0dc7a58386a3980826ededaf0ecd67d1ef38995b89615ba +size 4903351912 diff --git a/checkpoint-2000/model-00002-of-00004.safetensors b/checkpoint-2000/model-00002-of-00004.safetensors new file mode 100644 index 0000000..744b59b --- /dev/null +++ b/checkpoint-2000/model-00002-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:712a589c333f49f7b6f3870838a9e0ab4b3c0260abd5130e8c54316e6c99640b +size 4947570872 diff --git a/checkpoint-2000/model-00003-of-00004.safetensors b/checkpoint-2000/model-00003-of-00004.safetensors new file mode 100644 index 0000000..a394d39 --- /dev/null +++ b/checkpoint-2000/model-00003-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d02617b963ba5ff0a2f0f95663f02ae6d6b971bba52293a6876ba159520e3361 +size 4962221464 diff --git a/checkpoint-2000/model-00004-of-00004.safetensors b/checkpoint-2000/model-00004-of-00004.safetensors new file mode 100644 index 0000000..f1a23f8 --- /dev/null +++ b/checkpoint-2000/model-00004-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:326631a22f10135d46fc892c27d9e64f5aaf558af084c67b2f6ce95e612cd3f8 +size 3670322200 diff --git a/checkpoint-2000/model.safetensors.index.json b/checkpoint-2000/model.safetensors.index.json new file mode 100644 index 0000000..3b569c5 --- /dev/null +++ b/checkpoint-2000/model.safetensors.index.json @@ -0,0 +1,472 @@ +{ + "metadata": { + "total_parameters": 9241705984, + "total_size": 18483411968 + }, + "weight_map": { + "model.embed_tokens.weight": "model-00001-of-00004.safetensors", + "model.layers.0.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.0.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.0.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.1.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.1.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.10.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.10.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.10.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.10.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.11.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.11.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.11.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.12.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.12.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.12.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.13.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.13.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.13.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.14.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.14.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.14.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.15.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.15.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.15.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.16.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.16.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.16.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.17.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.17.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.17.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.18.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.18.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.18.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.19.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.19.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.19.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.2.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.2.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.2.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.20.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.20.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.20.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.20.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.20.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.20.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.20.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.21.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.21.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.21.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.21.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.21.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.22.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.22.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.22.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.22.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.23.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.23.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.23.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.24.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.24.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.24.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.25.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.25.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.25.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.26.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.26.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.26.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.27.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.27.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.27.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.28.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.28.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.28.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.29.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.29.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.29.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.3.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.3.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.3.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.30.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.30.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.30.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.30.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.30.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.31.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.31.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.31.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.32.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.32.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.32.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.32.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.32.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.32.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.32.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.32.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.32.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.32.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.32.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.33.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.33.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.33.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.33.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.33.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.33.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.33.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.33.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.33.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.33.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.33.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.34.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.34.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.34.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.34.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.34.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.34.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.34.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.34.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.34.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.34.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.34.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.35.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.35.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.35.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.35.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.35.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.35.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.35.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.35.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.35.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.35.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.35.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.36.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.36.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.36.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.36.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.36.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.36.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.36.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.36.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.36.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.36.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.36.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.37.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.37.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.37.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.37.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.37.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.37.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.37.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.37.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.37.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.37.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.37.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.38.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.38.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.38.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.38.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.38.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.38.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.38.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.38.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.38.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.38.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.38.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.39.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.39.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.39.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.39.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.39.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.39.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.39.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.39.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.39.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.39.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.39.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.4.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.4.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.4.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.4.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.40.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.40.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.40.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.40.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.40.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.40.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.40.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.40.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.40.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.40.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.40.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.41.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.41.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.41.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.41.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.41.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.41.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.41.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.41.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.41.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.41.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.41.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.5.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.5.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.5.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.5.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.6.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.6.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.6.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.7.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.7.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.7.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.8.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.8.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.8.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.8.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.8.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.8.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.8.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.8.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.8.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.8.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.8.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.9.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.9.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.9.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.9.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.norm.weight": "model-00004-of-00004.safetensors" + } +} diff --git a/checkpoint-2000/optimizer.pt b/checkpoint-2000/optimizer.pt new file mode 100644 index 0000000..d40135a --- /dev/null +++ b/checkpoint-2000/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1ed5c7b4fdbf3ec0730fdd117ed72963f318d37809c63e221bdda7c4952a529e +size 7927918951 diff --git a/checkpoint-2000/rng_state.pth b/checkpoint-2000/rng_state.pth new file mode 100644 index 0000000..1d85800 --- /dev/null +++ b/checkpoint-2000/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:039457aabfa4625f7b62a58eb9be9b588c4dd078fed6a4df33522308f5868222 +size 14645 diff --git a/checkpoint-2000/scheduler.pt b/checkpoint-2000/scheduler.pt new file mode 100644 index 0000000..94f6631 --- /dev/null +++ b/checkpoint-2000/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e2264fb7d318ae863681c0024e73708779a1fae7dff93c0490c6b438053f319b +size 1465 diff --git a/checkpoint-2000/special_tokens_map.json b/checkpoint-2000/special_tokens_map.json new file mode 100644 index 0000000..8d6368f --- /dev/null +++ b/checkpoint-2000/special_tokens_map.json @@ -0,0 +1,34 @@ +{ + "additional_special_tokens": [ + "", + "" + ], + "bos_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "unk_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + } +} diff --git a/checkpoint-2000/tokenizer.json b/checkpoint-2000/tokenizer.json new file mode 100644 index 0000000..a4a305d --- /dev/null +++ b/checkpoint-2000/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5f7eee611703c5ce5d1eee32d9cdcfe465647b8aff0c1dfb3bed7ad7dbb05060 +size 34362873 diff --git a/checkpoint-2000/tokenizer.model b/checkpoint-2000/tokenizer.model new file mode 100644 index 0000000..796efe9 --- /dev/null +++ b/checkpoint-2000/tokenizer.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:61a7b147390c64585d6c3543dd6fc636906c9af3865a5548f27f31aee1d4c8e2 +size 4241003 diff --git a/checkpoint-2000/tokenizer_config.json b/checkpoint-2000/tokenizer_config.json new file mode 100644 index 0000000..3ade6be --- /dev/null +++ b/checkpoint-2000/tokenizer_config.json @@ -0,0 +1,2013 @@ +{ + "add_bos_token": true, + "add_eos_token": false, + "added_tokens_decoder": { + "0": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "1": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "2": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "3": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "4": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "5": { + "content": "<2mass>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "6": { + "content": "[@BOS@]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "7": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "8": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "9": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "10": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "11": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "12": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "13": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "14": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "15": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "16": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "17": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "18": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "19": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "20": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "21": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "22": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "23": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "24": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "25": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "26": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "27": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "28": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "29": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "30": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "31": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "32": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "33": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "34": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "35": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "36": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "37": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "38": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "39": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "40": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "41": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "42": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "43": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "44": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "45": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "46": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "47": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "48": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "49": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "50": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "51": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "52": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "53": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "54": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "55": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "56": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "57": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "58": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "59": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "60": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "61": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "62": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "63": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "64": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "65": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "66": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "67": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "68": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "69": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "70": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "71": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "72": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "73": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "74": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "75": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "76": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "77": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "78": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "79": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "80": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "81": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "82": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "83": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "84": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "85": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "86": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "87": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "88": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "89": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "90": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "91": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "92": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "93": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "94": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "95": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "96": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "97": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "98": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "99": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "100": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "101": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "102": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "103": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "104": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "105": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "106": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "107": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "108": { + "content": "\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "109": { + "content": "\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "110": { + "content": "\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "111": { + "content": "\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "112": { + "content": "\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "113": { + "content": "\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "114": { + "content": "\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "115": { + "content": "\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "116": { + "content": "\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "117": { + "content": "\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "118": { + "content": "\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "119": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "120": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "121": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "122": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "123": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "124": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "125": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "126": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "127": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "128": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "129": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "130": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "131": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "132": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "133": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "134": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "135": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "136": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "137": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "138": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "139": { + "content": "▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "140": { + "content": "▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "141": { + "content": "▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "142": { + "content": "▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "143": { + "content": "▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "144": { + "content": "▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "145": { + "content": "▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "146": { + "content": "▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "147": { + "content": "▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "148": { + "content": "▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "149": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "150": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "152": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "153": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "154": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "155": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "156": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "157": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "158": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "159": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "160": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "161": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "162": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "163": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "164": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "165": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "166": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "167": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "168": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "169": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "170": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "172": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "173": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "174": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "175": { + "content": "
", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "171": { + "content": "
", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "176": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "177": { + "content": "
", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "178": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "179": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "180": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "181": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "182": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "183": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "184": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "185": { + "content": "

", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "186": { + "content": "

", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "187": { + "content": "

", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "188": { + "content": "

", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "189": { + "content": "

", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "190": { + "content": "
", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "191": { + "content": "
", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "192": { + "content": "
", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "193": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "194": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "195": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "196": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "197": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "198": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "199": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "200": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "201": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "202": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "203": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "204": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "205": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "206": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "207": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "208": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "209": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "210": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "211": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "212": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "213": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "214": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "215": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "216": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255968": { + "content": "[toxicity=0]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255969": { + "content": "\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255970": { + "content": "\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255971": { + "content": "\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255972": { + "content": "\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255973": { + "content": "\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255974": { + "content": "\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255975": { + "content": "\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255976": { + "content": "\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255977": { + "content": "\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255978": { + "content": "\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255979": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255980": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255981": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255982": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255983": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255984": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255985": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255986": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255987": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255988": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255989": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255990": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255991": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255992": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255993": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255994": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255995": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255996": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255997": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255998": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255999": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + } + }, + "additional_special_tokens": [ + "", + "" + ], + "bos_token": "", + "clean_up_tokenization_spaces": false, + "eos_token": "", + "extra_special_tokens": {}, + "model_max_length": 1000000000000000019884624838656, + "pad_token": "", + "sp_model_kwargs": {}, + "spaces_between_special_tokens": false, + "tokenizer_class": "GemmaTokenizer", + "unk_token": "", + "use_default_system_prompt": false +} diff --git a/checkpoint-2000/trainer_state.json b/checkpoint-2000/trainer_state.json new file mode 100644 index 0000000..e63f884 --- /dev/null +++ b/checkpoint-2000/trainer_state.json @@ -0,0 +1,2474 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 3.2, + "eval_steps": 100, + "global_step": 2000, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 0.08180969459936023, + "epoch": 0.016, + "grad_norm": 0.67578125, + "learning_rate": 9.000000000000001e-07, + "loss": 0.0453, + "mean_token_accuracy": 0.9865816306322813, + "num_tokens": 195510.0, + "step": 10 + }, + { + "entropy": 0.08111731857061386, + "epoch": 0.032, + "grad_norm": 0.60546875, + "learning_rate": 1.9000000000000002e-06, + "loss": 0.0452, + "mean_token_accuracy": 0.986666041240096, + "num_tokens": 391008.0, + "step": 20 + }, + { + "entropy": 0.07587501825764775, + "epoch": 0.048, + "grad_norm": 0.6953125, + "learning_rate": 2.9e-06, + "loss": 0.0445, + "mean_token_accuracy": 0.9873391918838024, + "num_tokens": 590136.0, + "step": 30 + }, + { + "entropy": 0.07685074284672737, + "epoch": 0.064, + "grad_norm": 0.625, + "learning_rate": 3.900000000000001e-06, + "loss": 0.0441, + "mean_token_accuracy": 0.987066001445055, + "num_tokens": 791432.0, + "step": 40 + }, + { + "entropy": 0.07894687210209668, + "epoch": 0.08, + "grad_norm": 0.75390625, + "learning_rate": 4.9000000000000005e-06, + "loss": 0.0447, + "mean_token_accuracy": 0.9865922331809998, + "num_tokens": 990342.0, + "step": 50 + }, + { + "entropy": 0.07919039381667972, + "epoch": 0.096, + "grad_norm": 0.66015625, + "learning_rate": 5.9e-06, + "loss": 0.0466, + "mean_token_accuracy": 0.9863619733601808, + "num_tokens": 1182339.0, + "step": 60 + }, + { + "entropy": 0.07967684981413185, + "epoch": 0.112, + "grad_norm": 0.5859375, + "learning_rate": 6.9e-06, + "loss": 0.0447, + "mean_token_accuracy": 0.9870311982929707, + "num_tokens": 1386193.0, + "step": 70 + }, + { + "entropy": 0.08054462079890072, + "epoch": 0.128, + "grad_norm": 0.65234375, + "learning_rate": 7.9e-06, + "loss": 0.0466, + "mean_token_accuracy": 0.9863289542496204, + "num_tokens": 1583261.0, + "step": 80 + }, + { + "entropy": 0.08421374012250453, + "epoch": 0.144, + "grad_norm": 0.5546875, + "learning_rate": 8.900000000000001e-06, + "loss": 0.0489, + "mean_token_accuracy": 0.9860724166035653, + "num_tokens": 1774826.0, + "step": 90 + }, + { + "entropy": 0.08307966932188719, + "epoch": 0.16, + "grad_norm": 0.71484375, + "learning_rate": 9.9e-06, + "loss": 0.0489, + "mean_token_accuracy": 0.986110083758831, + "num_tokens": 1971310.0, + "step": 100 + }, + { + "epoch": 0.16, + "eval_biology_entropy": 0.3792700763940811, + "eval_biology_loss": 2.260571241378784, + "eval_biology_mean_token_accuracy": 0.6872955484390258, + "eval_biology_num_tokens": 1971310.0, + "eval_biology_runtime": 39.5255, + "eval_biology_samples_per_second": 12.65, + "eval_biology_steps_per_second": 3.163, + "step": 100 + }, + { + "epoch": 0.16, + "eval_chemistry_entropy": 0.23563545274734496, + "eval_chemistry_loss": 1.4561725854873657, + "eval_chemistry_mean_token_accuracy": 0.7805930047035218, + "eval_chemistry_num_tokens": 1971310.0, + "eval_chemistry_runtime": 49.2319, + "eval_chemistry_samples_per_second": 10.156, + "eval_chemistry_steps_per_second": 2.539, + "step": 100 + }, + { + "entropy": 0.08481427943333983, + "epoch": 0.176, + "grad_norm": 0.734375, + "learning_rate": 1.0900000000000002e-05, + "loss": 0.0497, + "mean_token_accuracy": 0.9861054014414549, + "num_tokens": 2169453.0, + "step": 110 + }, + { + "entropy": 0.0882645736914128, + "epoch": 0.192, + "grad_norm": 0.79296875, + "learning_rate": 1.1900000000000001e-05, + "loss": 0.0512, + "mean_token_accuracy": 0.9859256945550442, + "num_tokens": 2366844.0, + "step": 120 + }, + { + "entropy": 0.09435517250094563, + "epoch": 0.208, + "grad_norm": 1.2421875, + "learning_rate": 1.2900000000000002e-05, + "loss": 0.0547, + "mean_token_accuracy": 0.9849376413971186, + "num_tokens": 2559708.0, + "step": 130 + }, + { + "entropy": 0.09682020233012736, + "epoch": 0.224, + "grad_norm": 1.3671875, + "learning_rate": 1.39e-05, + "loss": 0.057, + "mean_token_accuracy": 0.985217172652483, + "num_tokens": 2756533.0, + "step": 140 + }, + { + "entropy": 0.11111663887277246, + "epoch": 0.24, + "grad_norm": 2.84375, + "learning_rate": 1.4900000000000001e-05, + "loss": 0.0657, + "mean_token_accuracy": 0.9836616575717926, + "num_tokens": 2948600.0, + "step": 150 + }, + { + "entropy": 0.12251729485578836, + "epoch": 0.256, + "grad_norm": 2.453125, + "learning_rate": 1.5900000000000004e-05, + "loss": 0.0728, + "mean_token_accuracy": 0.9814188834279776, + "num_tokens": 3141473.0, + "step": 160 + }, + { + "entropy": 0.12813111576251685, + "epoch": 0.272, + "grad_norm": 3.03125, + "learning_rate": 1.69e-05, + "loss": 0.0758, + "mean_token_accuracy": 0.9808279071003199, + "num_tokens": 3337285.0, + "step": 170 + }, + { + "entropy": 0.13337696841917931, + "epoch": 0.288, + "grad_norm": 2.984375, + "learning_rate": 1.79e-05, + "loss": 0.0771, + "mean_token_accuracy": 0.9798636745661498, + "num_tokens": 3541221.0, + "step": 180 + }, + { + "entropy": 0.1436500378418714, + "epoch": 0.304, + "grad_norm": 2.78125, + "learning_rate": 1.8900000000000002e-05, + "loss": 0.0851, + "mean_token_accuracy": 0.9780430492013693, + "num_tokens": 3734952.0, + "step": 190 + }, + { + "entropy": 0.1549748638179153, + "epoch": 0.32, + "grad_norm": 3.390625, + "learning_rate": 1.9900000000000003e-05, + "loss": 0.0946, + "mean_token_accuracy": 0.9750579129904509, + "num_tokens": 3922209.0, + "step": 200 + }, + { + "epoch": 0.32, + "eval_biology_entropy": 0.4169517788887024, + "eval_biology_loss": 2.160527229309082, + "eval_biology_mean_token_accuracy": 0.6808431892395019, + "eval_biology_num_tokens": 3922209.0, + "eval_biology_runtime": 39.5121, + "eval_biology_samples_per_second": 12.654, + "eval_biology_steps_per_second": 3.164, + "step": 200 + }, + { + "epoch": 0.32, + "eval_chemistry_entropy": 0.25474807441234587, + "eval_chemistry_loss": 1.3652101755142212, + "eval_chemistry_mean_token_accuracy": 0.7794562330245972, + "eval_chemistry_num_tokens": 3922209.0, + "eval_chemistry_runtime": 49.2263, + "eval_chemistry_samples_per_second": 10.157, + "eval_chemistry_steps_per_second": 2.539, + "step": 200 + }, + { + "entropy": 0.16029730178415774, + "epoch": 0.336, + "grad_norm": 3.515625, + "learning_rate": 1.9900000000000003e-05, + "loss": 0.097, + "mean_token_accuracy": 0.9749548710882664, + "num_tokens": 4115941.0, + "step": 210 + }, + { + "entropy": 0.16590779088437557, + "epoch": 0.352, + "grad_norm": 3.78125, + "learning_rate": 1.978888888888889e-05, + "loss": 0.1023, + "mean_token_accuracy": 0.9726595059037209, + "num_tokens": 4308755.0, + "step": 220 + }, + { + "entropy": 0.1707212173845619, + "epoch": 0.368, + "grad_norm": 3.390625, + "learning_rate": 1.967777777777778e-05, + "loss": 0.1023, + "mean_token_accuracy": 0.9726378936320543, + "num_tokens": 4505910.0, + "step": 230 + }, + { + "entropy": 0.1765518521424383, + "epoch": 0.384, + "grad_norm": 3.671875, + "learning_rate": 1.956666666666667e-05, + "loss": 0.1083, + "mean_token_accuracy": 0.97143131531775, + "num_tokens": 4696101.0, + "step": 240 + }, + { + "entropy": 0.17895571915432812, + "epoch": 0.4, + "grad_norm": 3.5, + "learning_rate": 1.9455555555555555e-05, + "loss": 0.11, + "mean_token_accuracy": 0.9705049484968186, + "num_tokens": 4889445.0, + "step": 250 + }, + { + "entropy": 0.18151565054431557, + "epoch": 0.416, + "grad_norm": 3.578125, + "learning_rate": 1.9344444444444446e-05, + "loss": 0.1091, + "mean_token_accuracy": 0.970859233289957, + "num_tokens": 5087624.0, + "step": 260 + }, + { + "entropy": 0.18663504091091454, + "epoch": 0.432, + "grad_norm": 3.859375, + "learning_rate": 1.9233333333333334e-05, + "loss": 0.1135, + "mean_token_accuracy": 0.9693388067185879, + "num_tokens": 5273794.0, + "step": 270 + }, + { + "entropy": 0.18451756797730923, + "epoch": 0.448, + "grad_norm": 4.125, + "learning_rate": 1.9122222222222226e-05, + "loss": 0.1144, + "mean_token_accuracy": 0.9698665160685778, + "num_tokens": 5463288.0, + "step": 280 + }, + { + "entropy": 0.17932639280334114, + "epoch": 0.464, + "grad_norm": 3.5625, + "learning_rate": 1.9011111111111114e-05, + "loss": 0.1081, + "mean_token_accuracy": 0.9717006407678127, + "num_tokens": 5656827.0, + "step": 290 + }, + { + "entropy": 0.18305762661620975, + "epoch": 0.48, + "grad_norm": 3.90625, + "learning_rate": 1.8900000000000002e-05, + "loss": 0.1107, + "mean_token_accuracy": 0.9712053131312132, + "num_tokens": 5853311.0, + "step": 300 + }, + { + "epoch": 0.48, + "eval_biology_entropy": 0.42041616535186765, + "eval_biology_loss": 2.1313436031341553, + "eval_biology_mean_token_accuracy": 0.6798483271598816, + "eval_biology_num_tokens": 5853311.0, + "eval_biology_runtime": 39.5137, + "eval_biology_samples_per_second": 12.654, + "eval_biology_steps_per_second": 3.163, + "step": 300 + }, + { + "epoch": 0.48, + "eval_chemistry_entropy": 0.259404793381691, + "eval_chemistry_loss": 1.3463311195373535, + "eval_chemistry_mean_token_accuracy": 0.7787248449325561, + "eval_chemistry_num_tokens": 5853311.0, + "eval_chemistry_runtime": 49.201, + "eval_chemistry_samples_per_second": 10.162, + "eval_chemistry_steps_per_second": 2.541, + "step": 300 + }, + { + "entropy": 0.17875824291259051, + "epoch": 0.496, + "grad_norm": 3.453125, + "learning_rate": 1.878888888888889e-05, + "loss": 0.1101, + "mean_token_accuracy": 0.9710387323051691, + "num_tokens": 6049788.0, + "step": 310 + }, + { + "entropy": 0.1863187742419541, + "epoch": 0.512, + "grad_norm": 3.4375, + "learning_rate": 1.8677777777777778e-05, + "loss": 0.113, + "mean_token_accuracy": 0.9698710650205612, + "num_tokens": 6243825.0, + "step": 320 + }, + { + "entropy": 0.18619006094522775, + "epoch": 0.528, + "grad_norm": 3.890625, + "learning_rate": 1.856666666666667e-05, + "loss": 0.1135, + "mean_token_accuracy": 0.9703223951160908, + "num_tokens": 6434188.0, + "step": 330 + }, + { + "entropy": 0.18118073977530003, + "epoch": 0.544, + "grad_norm": 3.484375, + "learning_rate": 1.8455555555555558e-05, + "loss": 0.1114, + "mean_token_accuracy": 0.9708513412624598, + "num_tokens": 6629832.0, + "step": 340 + }, + { + "entropy": 0.1808169964235276, + "epoch": 0.56, + "grad_norm": 3.28125, + "learning_rate": 1.8344444444444446e-05, + "loss": 0.1079, + "mean_token_accuracy": 0.9716686312109232, + "num_tokens": 6824668.0, + "step": 350 + }, + { + "entropy": 0.18033515606075526, + "epoch": 0.576, + "grad_norm": 3.53125, + "learning_rate": 1.8233333333333334e-05, + "loss": 0.106, + "mean_token_accuracy": 0.9722726982086897, + "num_tokens": 7025923.0, + "step": 360 + }, + { + "entropy": 0.1852203646209091, + "epoch": 0.592, + "grad_norm": 3.625, + "learning_rate": 1.8122222222222222e-05, + "loss": 0.111, + "mean_token_accuracy": 0.9705461781471968, + "num_tokens": 7218224.0, + "step": 370 + }, + { + "entropy": 0.17758405841886998, + "epoch": 0.608, + "grad_norm": 3.65625, + "learning_rate": 1.8011111111111113e-05, + "loss": 0.1066, + "mean_token_accuracy": 0.9723859451711178, + "num_tokens": 7421216.0, + "step": 380 + }, + { + "entropy": 0.18370326030999423, + "epoch": 0.624, + "grad_norm": 3.765625, + "learning_rate": 1.79e-05, + "loss": 0.1096, + "mean_token_accuracy": 0.9708739675581455, + "num_tokens": 7617258.0, + "step": 390 + }, + { + "entropy": 0.1831163068767637, + "epoch": 0.64, + "grad_norm": 3.578125, + "learning_rate": 1.778888888888889e-05, + "loss": 0.1072, + "mean_token_accuracy": 0.9717655830085278, + "num_tokens": 7806173.0, + "step": 400 + }, + { + "epoch": 0.64, + "eval_biology_entropy": 0.42119112753868104, + "eval_biology_loss": 2.1267058849334717, + "eval_biology_mean_token_accuracy": 0.6801503438949585, + "eval_biology_num_tokens": 7806173.0, + "eval_biology_runtime": 39.5121, + "eval_biology_samples_per_second": 12.654, + "eval_biology_steps_per_second": 3.164, + "step": 400 + }, + { + "epoch": 0.64, + "eval_chemistry_entropy": 0.2595027017593384, + "eval_chemistry_loss": 1.3404533863067627, + "eval_chemistry_mean_token_accuracy": 0.7790702543258667, + "eval_chemistry_num_tokens": 7806173.0, + "eval_chemistry_runtime": 49.2002, + "eval_chemistry_samples_per_second": 10.163, + "eval_chemistry_steps_per_second": 2.541, + "step": 400 + }, + { + "entropy": 0.1827191138640046, + "epoch": 0.656, + "grad_norm": 3.84375, + "learning_rate": 1.7677777777777778e-05, + "loss": 0.1088, + "mean_token_accuracy": 0.9715055178850889, + "num_tokens": 8000514.0, + "step": 410 + }, + { + "entropy": 0.18661352493800223, + "epoch": 0.672, + "grad_norm": 3.796875, + "learning_rate": 1.756666666666667e-05, + "loss": 0.1114, + "mean_token_accuracy": 0.9704575203359127, + "num_tokens": 8187884.0, + "step": 420 + }, + { + "entropy": 0.17729734890162946, + "epoch": 0.688, + "grad_norm": 3.09375, + "learning_rate": 1.7455555555555557e-05, + "loss": 0.1046, + "mean_token_accuracy": 0.9725840430706739, + "num_tokens": 8390643.0, + "step": 430 + }, + { + "entropy": 0.18038612157106398, + "epoch": 0.704, + "grad_norm": 3.46875, + "learning_rate": 1.7344444444444445e-05, + "loss": 0.1072, + "mean_token_accuracy": 0.9721495136618614, + "num_tokens": 8583266.0, + "step": 440 + }, + { + "entropy": 0.17609651018865408, + "epoch": 0.72, + "grad_norm": 3.734375, + "learning_rate": 1.7233333333333337e-05, + "loss": 0.1045, + "mean_token_accuracy": 0.9718786559998989, + "num_tokens": 8786233.0, + "step": 450 + }, + { + "entropy": 0.1797325788065791, + "epoch": 0.736, + "grad_norm": 3.390625, + "learning_rate": 1.7122222222222225e-05, + "loss": 0.107, + "mean_token_accuracy": 0.9716725051403046, + "num_tokens": 8982898.0, + "step": 460 + }, + { + "entropy": 0.17950535458512604, + "epoch": 0.752, + "grad_norm": 3.5625, + "learning_rate": 1.7011111111111113e-05, + "loss": 0.1072, + "mean_token_accuracy": 0.9725060146301985, + "num_tokens": 9174361.0, + "step": 470 + }, + { + "entropy": 0.17792485035024583, + "epoch": 0.768, + "grad_norm": 3.375, + "learning_rate": 1.69e-05, + "loss": 0.1025, + "mean_token_accuracy": 0.973187630996108, + "num_tokens": 9373430.0, + "step": 480 + }, + { + "entropy": 0.17844869694672524, + "epoch": 0.784, + "grad_norm": 3.515625, + "learning_rate": 1.678888888888889e-05, + "loss": 0.1073, + "mean_token_accuracy": 0.9717021223157645, + "num_tokens": 9570409.0, + "step": 490 + }, + { + "entropy": 0.17820251639932394, + "epoch": 0.8, + "grad_norm": 3.765625, + "learning_rate": 1.667777777777778e-05, + "loss": 0.1032, + "mean_token_accuracy": 0.9729967709630728, + "num_tokens": 9766360.0, + "step": 500 + }, + { + "epoch": 0.8, + "eval_biology_entropy": 0.41608734560012817, + "eval_biology_loss": 2.139784097671509, + "eval_biology_mean_token_accuracy": 0.6806383662223816, + "eval_biology_num_tokens": 9766360.0, + "eval_biology_runtime": 39.5271, + "eval_biology_samples_per_second": 12.65, + "eval_biology_steps_per_second": 3.162, + "step": 500 + }, + { + "epoch": 0.8, + "eval_chemistry_entropy": 0.25711704087257387, + "eval_chemistry_loss": 1.3562349081039429, + "eval_chemistry_mean_token_accuracy": 0.7787247653007507, + "eval_chemistry_num_tokens": 9766360.0, + "eval_chemistry_runtime": 48.8194, + "eval_chemistry_samples_per_second": 10.242, + "eval_chemistry_steps_per_second": 2.56, + "step": 500 + }, + { + "entropy": 0.1748917239718139, + "epoch": 0.816, + "grad_norm": 3.515625, + "learning_rate": 1.656666666666667e-05, + "loss": 0.1004, + "mean_token_accuracy": 0.9740771554410458, + "num_tokens": 9963834.0, + "step": 510 + }, + { + "entropy": 0.17268717400729655, + "epoch": 0.832, + "grad_norm": 3.125, + "learning_rate": 1.6455555555555556e-05, + "loss": 0.1023, + "mean_token_accuracy": 0.9740318663418293, + "num_tokens": 10160950.0, + "step": 520 + }, + { + "entropy": 0.17282785503193737, + "epoch": 0.848, + "grad_norm": 3.671875, + "learning_rate": 1.6344444444444445e-05, + "loss": 0.0991, + "mean_token_accuracy": 0.9744720965623855, + "num_tokens": 10363131.0, + "step": 530 + }, + { + "entropy": 0.17644098605960606, + "epoch": 0.864, + "grad_norm": 3.34375, + "learning_rate": 1.6233333333333333e-05, + "loss": 0.1026, + "mean_token_accuracy": 0.972842201218009, + "num_tokens": 10558003.0, + "step": 540 + }, + { + "entropy": 0.17435273211449384, + "epoch": 0.88, + "grad_norm": 3.34375, + "learning_rate": 1.6122222222222224e-05, + "loss": 0.1004, + "mean_token_accuracy": 0.9740745916962623, + "num_tokens": 10754470.0, + "step": 550 + }, + { + "entropy": 0.1750097878277302, + "epoch": 0.896, + "grad_norm": 3.453125, + "learning_rate": 1.6011111111111112e-05, + "loss": 0.1007, + "mean_token_accuracy": 0.9743314851075411, + "num_tokens": 10948943.0, + "step": 560 + }, + { + "entropy": 0.17419813955202698, + "epoch": 0.912, + "grad_norm": 3.234375, + "learning_rate": 1.5900000000000004e-05, + "loss": 0.1005, + "mean_token_accuracy": 0.974232492223382, + "num_tokens": 11142613.0, + "step": 570 + }, + { + "entropy": 0.17595540760084988, + "epoch": 0.928, + "grad_norm": 3.546875, + "learning_rate": 1.5788888888888888e-05, + "loss": 0.0997, + "mean_token_accuracy": 0.974004115909338, + "num_tokens": 11336779.0, + "step": 580 + }, + { + "entropy": 0.16677158032543957, + "epoch": 0.944, + "grad_norm": 3.140625, + "learning_rate": 1.567777777777778e-05, + "loss": 0.0982, + "mean_token_accuracy": 0.9750826183706522, + "num_tokens": 11536381.0, + "step": 590 + }, + { + "entropy": 0.16921130288392305, + "epoch": 0.96, + "grad_norm": 3.546875, + "learning_rate": 1.5566666666666668e-05, + "loss": 0.0966, + "mean_token_accuracy": 0.9753067292273044, + "num_tokens": 11735527.0, + "step": 600 + }, + { + "epoch": 0.96, + "eval_biology_entropy": 0.4160035889148712, + "eval_biology_loss": 2.1477200984954834, + "eval_biology_mean_token_accuracy": 0.68089803981781, + "eval_biology_num_tokens": 11735527.0, + "eval_biology_runtime": 39.5581, + "eval_biology_samples_per_second": 12.64, + "eval_biology_steps_per_second": 3.16, + "step": 600 + }, + { + "epoch": 0.96, + "eval_chemistry_entropy": 0.2561914825439453, + "eval_chemistry_loss": 1.3602544069290161, + "eval_chemistry_mean_token_accuracy": 0.779716450214386, + "eval_chemistry_num_tokens": 11735527.0, + "eval_chemistry_runtime": 49.254, + "eval_chemistry_samples_per_second": 10.151, + "eval_chemistry_steps_per_second": 2.538, + "step": 600 + }, + { + "entropy": 0.16858043959364294, + "epoch": 0.976, + "grad_norm": 3.65625, + "learning_rate": 1.5455555555555556e-05, + "loss": 0.0985, + "mean_token_accuracy": 0.9748617243021727, + "num_tokens": 11930935.0, + "step": 610 + }, + { + "entropy": 0.16747406986542046, + "epoch": 0.992, + "grad_norm": 3.15625, + "learning_rate": 1.5344444444444447e-05, + "loss": 0.0955, + "mean_token_accuracy": 0.9756584648042917, + "num_tokens": 12129585.0, + "step": 620 + }, + { + "entropy": 0.15082475617527963, + "epoch": 1.008, + "grad_norm": 2.28125, + "learning_rate": 1.5233333333333335e-05, + "loss": 0.0856, + "mean_token_accuracy": 0.9783346977084875, + "num_tokens": 12325847.0, + "step": 630 + }, + { + "entropy": 0.11756393420509995, + "epoch": 1.024, + "grad_norm": 2.3125, + "learning_rate": 1.5122222222222222e-05, + "loss": 0.07, + "mean_token_accuracy": 0.9816608231514692, + "num_tokens": 12530820.0, + "step": 640 + }, + { + "entropy": 0.12562460631597788, + "epoch": 1.04, + "grad_norm": 2.796875, + "learning_rate": 1.5011111111111112e-05, + "loss": 0.0744, + "mean_token_accuracy": 0.9802271891385317, + "num_tokens": 12725384.0, + "step": 650 + }, + { + "entropy": 0.12352396142669023, + "epoch": 1.056, + "grad_norm": 2.609375, + "learning_rate": 1.4900000000000001e-05, + "loss": 0.0744, + "mean_token_accuracy": 0.9802641972899437, + "num_tokens": 12924592.0, + "step": 660 + }, + { + "entropy": 0.12865133616141974, + "epoch": 1.072, + "grad_norm": 2.671875, + "learning_rate": 1.4788888888888891e-05, + "loss": 0.0774, + "mean_token_accuracy": 0.9798023611307144, + "num_tokens": 13112848.0, + "step": 670 + }, + { + "entropy": 0.12567441049031913, + "epoch": 1.088, + "grad_norm": 2.53125, + "learning_rate": 1.467777777777778e-05, + "loss": 0.0756, + "mean_token_accuracy": 0.9806226130574942, + "num_tokens": 13305725.0, + "step": 680 + }, + { + "entropy": 0.12565653487108647, + "epoch": 1.104, + "grad_norm": 2.234375, + "learning_rate": 1.4566666666666669e-05, + "loss": 0.0745, + "mean_token_accuracy": 0.980208383128047, + "num_tokens": 13498685.0, + "step": 690 + }, + { + "entropy": 0.1318228157237172, + "epoch": 1.12, + "grad_norm": 2.5, + "learning_rate": 1.4455555555555555e-05, + "loss": 0.0778, + "mean_token_accuracy": 0.97950509339571, + "num_tokens": 13682146.0, + "step": 700 + }, + { + "epoch": 1.12, + "eval_biology_entropy": 0.4004362862110138, + "eval_biology_loss": 2.177130699157715, + "eval_biology_mean_token_accuracy": 0.6821538486480713, + "eval_biology_num_tokens": 13682146.0, + "eval_biology_runtime": 39.5089, + "eval_biology_samples_per_second": 12.655, + "eval_biology_steps_per_second": 3.164, + "step": 700 + }, + { + "epoch": 1.12, + "eval_chemistry_entropy": 0.24954386246204377, + "eval_chemistry_loss": 1.3777952194213867, + "eval_chemistry_mean_token_accuracy": 0.7797037825584412, + "eval_chemistry_num_tokens": 13682146.0, + "eval_chemistry_runtime": 49.2017, + "eval_chemistry_samples_per_second": 10.162, + "eval_chemistry_steps_per_second": 2.541, + "step": 700 + }, + { + "entropy": 0.12691126235295086, + "epoch": 1.1360000000000001, + "grad_norm": 2.6875, + "learning_rate": 1.4344444444444445e-05, + "loss": 0.0757, + "mean_token_accuracy": 0.980040468648076, + "num_tokens": 13876656.0, + "step": 710 + }, + { + "entropy": 0.12381730475462974, + "epoch": 1.152, + "grad_norm": 2.359375, + "learning_rate": 1.4233333333333335e-05, + "loss": 0.0743, + "mean_token_accuracy": 0.980041703954339, + "num_tokens": 14086104.0, + "step": 720 + }, + { + "entropy": 0.1285357222892344, + "epoch": 1.168, + "grad_norm": 2.5625, + "learning_rate": 1.4122222222222225e-05, + "loss": 0.0763, + "mean_token_accuracy": 0.9799800109118223, + "num_tokens": 14274520.0, + "step": 730 + }, + { + "entropy": 0.12321301018819213, + "epoch": 1.184, + "grad_norm": 2.375, + "learning_rate": 1.4011111111111113e-05, + "loss": 0.0734, + "mean_token_accuracy": 0.9810513865202666, + "num_tokens": 14473623.0, + "step": 740 + }, + { + "entropy": 0.1292376267723739, + "epoch": 1.2, + "grad_norm": 2.609375, + "learning_rate": 1.39e-05, + "loss": 0.0784, + "mean_token_accuracy": 0.9795571226626635, + "num_tokens": 14661238.0, + "step": 750 + }, + { + "entropy": 0.12086025532335043, + "epoch": 1.216, + "grad_norm": 2.640625, + "learning_rate": 1.3788888888888889e-05, + "loss": 0.073, + "mean_token_accuracy": 0.9810031026601791, + "num_tokens": 14865915.0, + "step": 760 + }, + { + "entropy": 0.1262558945454657, + "epoch": 1.232, + "grad_norm": 2.40625, + "learning_rate": 1.3677777777777779e-05, + "loss": 0.0786, + "mean_token_accuracy": 0.9797264195978641, + "num_tokens": 15055682.0, + "step": 770 + }, + { + "entropy": 0.12587894778698683, + "epoch": 1.248, + "grad_norm": 2.484375, + "learning_rate": 1.3566666666666668e-05, + "loss": 0.0744, + "mean_token_accuracy": 0.9802309483289718, + "num_tokens": 15249576.0, + "step": 780 + }, + { + "entropy": 0.12361958674155175, + "epoch": 1.264, + "grad_norm": 2.1875, + "learning_rate": 1.3455555555555556e-05, + "loss": 0.0726, + "mean_token_accuracy": 0.9805784326046705, + "num_tokens": 15446149.0, + "step": 790 + }, + { + "entropy": 0.12549991067498922, + "epoch": 1.28, + "grad_norm": 2.546875, + "learning_rate": 1.3344444444444446e-05, + "loss": 0.0743, + "mean_token_accuracy": 0.9805208504199981, + "num_tokens": 15639317.0, + "step": 800 + }, + { + "epoch": 1.28, + "eval_biology_entropy": 0.4032972083091736, + "eval_biology_loss": 2.1712591648101807, + "eval_biology_mean_token_accuracy": 0.6816940937042236, + "eval_biology_num_tokens": 15639317.0, + "eval_biology_runtime": 39.5209, + "eval_biology_samples_per_second": 12.652, + "eval_biology_steps_per_second": 3.163, + "step": 800 + }, + { + "epoch": 1.28, + "eval_chemistry_entropy": 0.25105089938640596, + "eval_chemistry_loss": 1.3797656297683716, + "eval_chemistry_mean_token_accuracy": 0.7792046613693238, + "eval_chemistry_num_tokens": 15639317.0, + "eval_chemistry_runtime": 49.2346, + "eval_chemistry_samples_per_second": 10.155, + "eval_chemistry_steps_per_second": 2.539, + "step": 800 + }, + { + "entropy": 0.12616121221799403, + "epoch": 1.296, + "grad_norm": 2.203125, + "learning_rate": 1.3233333333333334e-05, + "loss": 0.075, + "mean_token_accuracy": 0.9805853247642518, + "num_tokens": 15835805.0, + "step": 810 + }, + { + "entropy": 0.12473565563559533, + "epoch": 1.312, + "grad_norm": 2.296875, + "learning_rate": 1.3122222222222222e-05, + "loss": 0.0738, + "mean_token_accuracy": 0.9808966059237718, + "num_tokens": 16028375.0, + "step": 820 + }, + { + "entropy": 0.12206203136593104, + "epoch": 1.328, + "grad_norm": 2.265625, + "learning_rate": 1.3011111111111112e-05, + "loss": 0.0712, + "mean_token_accuracy": 0.9813739657402039, + "num_tokens": 16230555.0, + "step": 830 + }, + { + "entropy": 0.11784348418004811, + "epoch": 1.3439999999999999, + "grad_norm": 2.359375, + "learning_rate": 1.2900000000000002e-05, + "loss": 0.0692, + "mean_token_accuracy": 0.9817070845514536, + "num_tokens": 16436297.0, + "step": 840 + }, + { + "entropy": 0.12049585804343224, + "epoch": 1.3599999999999999, + "grad_norm": 2.234375, + "learning_rate": 1.278888888888889e-05, + "loss": 0.0719, + "mean_token_accuracy": 0.9809176307171583, + "num_tokens": 16630563.0, + "step": 850 + }, + { + "entropy": 0.12437316868454218, + "epoch": 1.376, + "grad_norm": 2.8125, + "learning_rate": 1.2677777777777778e-05, + "loss": 0.0751, + "mean_token_accuracy": 0.9806294579058885, + "num_tokens": 16822602.0, + "step": 860 + }, + { + "entropy": 0.12243083966895937, + "epoch": 1.392, + "grad_norm": 2.34375, + "learning_rate": 1.2566666666666668e-05, + "loss": 0.0729, + "mean_token_accuracy": 0.9806884177029133, + "num_tokens": 17015750.0, + "step": 870 + }, + { + "entropy": 0.12194055491127073, + "epoch": 1.408, + "grad_norm": 2.21875, + "learning_rate": 1.2455555555555556e-05, + "loss": 0.0738, + "mean_token_accuracy": 0.9811285879462958, + "num_tokens": 17207248.0, + "step": 880 + }, + { + "entropy": 0.12085925284773111, + "epoch": 1.424, + "grad_norm": 2.40625, + "learning_rate": 1.2344444444444446e-05, + "loss": 0.0714, + "mean_token_accuracy": 0.9814036220312119, + "num_tokens": 17403869.0, + "step": 890 + }, + { + "entropy": 0.12371515287086368, + "epoch": 1.44, + "grad_norm": 2.671875, + "learning_rate": 1.2233333333333335e-05, + "loss": 0.0732, + "mean_token_accuracy": 0.9810383848845958, + "num_tokens": 17597428.0, + "step": 900 + }, + { + "epoch": 1.44, + "eval_biology_entropy": 0.3943380331993103, + "eval_biology_loss": 2.181544542312622, + "eval_biology_mean_token_accuracy": 0.6840425395965576, + "eval_biology_num_tokens": 17597428.0, + "eval_biology_runtime": 39.5119, + "eval_biology_samples_per_second": 12.654, + "eval_biology_steps_per_second": 3.164, + "step": 900 + }, + { + "epoch": 1.44, + "eval_chemistry_entropy": 0.24567515325546266, + "eval_chemistry_loss": 1.3865065574645996, + "eval_chemistry_mean_token_accuracy": 0.7803217101097107, + "eval_chemistry_num_tokens": 17597428.0, + "eval_chemistry_runtime": 49.2083, + "eval_chemistry_samples_per_second": 10.161, + "eval_chemistry_steps_per_second": 2.54, + "step": 900 + }, + { + "entropy": 0.11841928996145726, + "epoch": 1.456, + "grad_norm": 2.421875, + "learning_rate": 1.2122222222222223e-05, + "loss": 0.0692, + "mean_token_accuracy": 0.9820894330739975, + "num_tokens": 17798083.0, + "step": 910 + }, + { + "entropy": 0.12107395366765558, + "epoch": 1.472, + "grad_norm": 2.09375, + "learning_rate": 1.2011111111111111e-05, + "loss": 0.0727, + "mean_token_accuracy": 0.980775249004364, + "num_tokens": 17993802.0, + "step": 920 + }, + { + "entropy": 0.11666952911764383, + "epoch": 1.488, + "grad_norm": 2.234375, + "learning_rate": 1.1900000000000001e-05, + "loss": 0.0711, + "mean_token_accuracy": 0.9814896911382676, + "num_tokens": 18185570.0, + "step": 930 + }, + { + "entropy": 0.12296288679353892, + "epoch": 1.504, + "grad_norm": 2.203125, + "learning_rate": 1.178888888888889e-05, + "loss": 0.0721, + "mean_token_accuracy": 0.9813314471393824, + "num_tokens": 18377903.0, + "step": 940 + }, + { + "entropy": 0.11539274472743273, + "epoch": 1.52, + "grad_norm": 2.21875, + "learning_rate": 1.1677777777777779e-05, + "loss": 0.0691, + "mean_token_accuracy": 0.9822911247611046, + "num_tokens": 18579284.0, + "step": 950 + }, + { + "entropy": 0.12136667580343782, + "epoch": 1.536, + "grad_norm": 2.140625, + "learning_rate": 1.1566666666666669e-05, + "loss": 0.0725, + "mean_token_accuracy": 0.9812386609613896, + "num_tokens": 18765324.0, + "step": 960 + }, + { + "entropy": 0.11671084221452474, + "epoch": 1.552, + "grad_norm": 2.15625, + "learning_rate": 1.1455555555555557e-05, + "loss": 0.0676, + "mean_token_accuracy": 0.9818577177822589, + "num_tokens": 18966382.0, + "step": 970 + }, + { + "entropy": 0.11611056034453213, + "epoch": 1.568, + "grad_norm": 2.296875, + "learning_rate": 1.1344444444444445e-05, + "loss": 0.0701, + "mean_token_accuracy": 0.9818583976477384, + "num_tokens": 19160509.0, + "step": 980 + }, + { + "entropy": 0.11777197350747884, + "epoch": 1.584, + "grad_norm": 2.1875, + "learning_rate": 1.1233333333333333e-05, + "loss": 0.0697, + "mean_token_accuracy": 0.9817087065428496, + "num_tokens": 19354662.0, + "step": 990 + }, + { + "entropy": 0.12017350159585476, + "epoch": 1.6, + "grad_norm": 2.28125, + "learning_rate": 1.1122222222222223e-05, + "loss": 0.0718, + "mean_token_accuracy": 0.9814203523099423, + "num_tokens": 19543088.0, + "step": 1000 + }, + { + "epoch": 1.6, + "eval_biology_entropy": 0.3948675305843353, + "eval_biology_loss": 2.1792049407958984, + "eval_biology_mean_token_accuracy": 0.6840984811782836, + "eval_biology_num_tokens": 19543088.0, + "eval_biology_runtime": 39.6119, + "eval_biology_samples_per_second": 12.622, + "eval_biology_steps_per_second": 3.156, + "step": 1000 + }, + { + "epoch": 1.6, + "eval_chemistry_entropy": 0.24652056658267976, + "eval_chemistry_loss": 1.388296365737915, + "eval_chemistry_mean_token_accuracy": 0.7802323937416077, + "eval_chemistry_num_tokens": 19543088.0, + "eval_chemistry_runtime": 48.7687, + "eval_chemistry_samples_per_second": 10.252, + "eval_chemistry_steps_per_second": 2.563, + "step": 1000 + }, + { + "entropy": 0.11641141260042787, + "epoch": 1.616, + "grad_norm": 2.109375, + "learning_rate": 1.1011111111111113e-05, + "loss": 0.0685, + "mean_token_accuracy": 0.9818709082901478, + "num_tokens": 19743506.0, + "step": 1010 + }, + { + "entropy": 0.11852226112969219, + "epoch": 1.6320000000000001, + "grad_norm": 2.40625, + "learning_rate": 1.0900000000000002e-05, + "loss": 0.0679, + "mean_token_accuracy": 0.9817365873605013, + "num_tokens": 19937567.0, + "step": 1020 + }, + { + "entropy": 0.11748991990461946, + "epoch": 1.6480000000000001, + "grad_norm": 2.125, + "learning_rate": 1.0788888888888889e-05, + "loss": 0.0699, + "mean_token_accuracy": 0.9814462032169103, + "num_tokens": 20129743.0, + "step": 1030 + }, + { + "entropy": 0.11318188132718206, + "epoch": 1.6640000000000001, + "grad_norm": 2.109375, + "learning_rate": 1.0677777777777779e-05, + "loss": 0.068, + "mean_token_accuracy": 0.9825476247817277, + "num_tokens": 20331548.0, + "step": 1040 + }, + { + "entropy": 0.11528927914332598, + "epoch": 1.6800000000000002, + "grad_norm": 2.078125, + "learning_rate": 1.0566666666666667e-05, + "loss": 0.0684, + "mean_token_accuracy": 0.9821649711579085, + "num_tokens": 20524247.0, + "step": 1050 + }, + { + "entropy": 0.11758545534685254, + "epoch": 1.696, + "grad_norm": 1.9296875, + "learning_rate": 1.0455555555555556e-05, + "loss": 0.0688, + "mean_token_accuracy": 0.9817702580243349, + "num_tokens": 20718121.0, + "step": 1060 + }, + { + "entropy": 0.11215258438605816, + "epoch": 1.712, + "grad_norm": 2.109375, + "learning_rate": 1.0344444444444446e-05, + "loss": 0.066, + "mean_token_accuracy": 0.9826954051852226, + "num_tokens": 20921419.0, + "step": 1070 + }, + { + "entropy": 0.11761863068677485, + "epoch": 1.728, + "grad_norm": 2.078125, + "learning_rate": 1.0233333333333336e-05, + "loss": 0.0673, + "mean_token_accuracy": 0.9817208234220743, + "num_tokens": 21118581.0, + "step": 1080 + }, + { + "entropy": 0.11461647190153598, + "epoch": 1.744, + "grad_norm": 2.34375, + "learning_rate": 1.0122222222222222e-05, + "loss": 0.0683, + "mean_token_accuracy": 0.9821706589311361, + "num_tokens": 21315128.0, + "step": 1090 + }, + { + "entropy": 0.11213490981608629, + "epoch": 1.76, + "grad_norm": 1.921875, + "learning_rate": 1.0011111111111112e-05, + "loss": 0.0668, + "mean_token_accuracy": 0.9826094429939985, + "num_tokens": 21511066.0, + "step": 1100 + }, + { + "epoch": 1.76, + "eval_biology_entropy": 0.39157771730422974, + "eval_biology_loss": 2.183682680130005, + "eval_biology_mean_token_accuracy": 0.6850721926689148, + "eval_biology_num_tokens": 21511066.0, + "eval_biology_runtime": 39.4964, + "eval_biology_samples_per_second": 12.659, + "eval_biology_steps_per_second": 3.165, + "step": 1100 + }, + { + "epoch": 1.76, + "eval_chemistry_entropy": 0.24445027112960815, + "eval_chemistry_loss": 1.3950114250183105, + "eval_chemistry_mean_token_accuracy": 0.7797145857810974, + "eval_chemistry_num_tokens": 21511066.0, + "eval_chemistry_runtime": 49.2195, + "eval_chemistry_samples_per_second": 10.159, + "eval_chemistry_steps_per_second": 2.54, + "step": 1100 + }, + { + "entropy": 0.11297063264064491, + "epoch": 1.776, + "grad_norm": 2.078125, + "learning_rate": 9.9e-06, + "loss": 0.0678, + "mean_token_accuracy": 0.9824550054967404, + "num_tokens": 21704368.0, + "step": 1110 + }, + { + "entropy": 0.11269787652418017, + "epoch": 1.792, + "grad_norm": 1.8359375, + "learning_rate": 9.78888888888889e-06, + "loss": 0.0652, + "mean_token_accuracy": 0.9826549135148526, + "num_tokens": 21905784.0, + "step": 1120 + }, + { + "entropy": 0.11504443329758943, + "epoch": 1.808, + "grad_norm": 1.9765625, + "learning_rate": 9.677777777777778e-06, + "loss": 0.0673, + "mean_token_accuracy": 0.9823829177767038, + "num_tokens": 22094159.0, + "step": 1130 + }, + { + "entropy": 0.11632196293212474, + "epoch": 1.8239999999999998, + "grad_norm": 1.9921875, + "learning_rate": 9.566666666666668e-06, + "loss": 0.0672, + "mean_token_accuracy": 0.9823333639651537, + "num_tokens": 22290654.0, + "step": 1140 + }, + { + "entropy": 0.11296359319239854, + "epoch": 1.8399999999999999, + "grad_norm": 2.0625, + "learning_rate": 9.455555555555557e-06, + "loss": 0.0675, + "mean_token_accuracy": 0.9825551677495241, + "num_tokens": 22485675.0, + "step": 1150 + }, + { + "entropy": 0.10986362439580262, + "epoch": 1.8559999999999999, + "grad_norm": 2.265625, + "learning_rate": 9.344444444444446e-06, + "loss": 0.0637, + "mean_token_accuracy": 0.9831937052309513, + "num_tokens": 22689900.0, + "step": 1160 + }, + { + "entropy": 0.11179361823014915, + "epoch": 1.8719999999999999, + "grad_norm": 1.828125, + "learning_rate": 9.233333333333334e-06, + "loss": 0.065, + "mean_token_accuracy": 0.9827314846217632, + "num_tokens": 22887275.0, + "step": 1170 + }, + { + "entropy": 0.11337793134152889, + "epoch": 1.888, + "grad_norm": 1.875, + "learning_rate": 9.122222222222223e-06, + "loss": 0.0663, + "mean_token_accuracy": 0.9825703386217356, + "num_tokens": 23081241.0, + "step": 1180 + }, + { + "entropy": 0.11284571103751659, + "epoch": 1.904, + "grad_norm": 2.296875, + "learning_rate": 9.011111111111111e-06, + "loss": 0.0655, + "mean_token_accuracy": 0.9827509298920631, + "num_tokens": 23276417.0, + "step": 1190 + }, + { + "entropy": 0.1114346832036972, + "epoch": 1.92, + "grad_norm": 1.9296875, + "learning_rate": 8.900000000000001e-06, + "loss": 0.065, + "mean_token_accuracy": 0.9832344222813845, + "num_tokens": 23476340.0, + "step": 1200 + }, + { + "epoch": 1.92, + "eval_biology_entropy": 0.3916603057384491, + "eval_biology_loss": 2.1840121746063232, + "eval_biology_mean_token_accuracy": 0.6849033751487732, + "eval_biology_num_tokens": 23476340.0, + "eval_biology_runtime": 39.5552, + "eval_biology_samples_per_second": 12.641, + "eval_biology_steps_per_second": 3.16, + "step": 1200 + }, + { + "epoch": 1.92, + "eval_chemistry_entropy": 0.24395500665903092, + "eval_chemistry_loss": 1.3961467742919922, + "eval_chemistry_mean_token_accuracy": 0.7803040590286255, + "eval_chemistry_num_tokens": 23476340.0, + "eval_chemistry_runtime": 49.2524, + "eval_chemistry_samples_per_second": 10.152, + "eval_chemistry_steps_per_second": 2.538, + "step": 1200 + }, + { + "entropy": 0.10977238831110299, + "epoch": 1.936, + "grad_norm": 2.078125, + "learning_rate": 8.788888888888891e-06, + "loss": 0.0642, + "mean_token_accuracy": 0.9830574735999107, + "num_tokens": 23673208.0, + "step": 1210 + }, + { + "entropy": 0.10960933691821992, + "epoch": 1.952, + "grad_norm": 1.7109375, + "learning_rate": 8.677777777777779e-06, + "loss": 0.0637, + "mean_token_accuracy": 0.9831824779510498, + "num_tokens": 23870993.0, + "step": 1220 + }, + { + "entropy": 0.11105740028433501, + "epoch": 1.968, + "grad_norm": 1.8984375, + "learning_rate": 8.566666666666667e-06, + "loss": 0.0648, + "mean_token_accuracy": 0.9832170728594065, + "num_tokens": 24066234.0, + "step": 1230 + }, + { + "entropy": 0.10861094021238386, + "epoch": 1.984, + "grad_norm": 1.9296875, + "learning_rate": 8.455555555555555e-06, + "loss": 0.0642, + "mean_token_accuracy": 0.9830795601010323, + "num_tokens": 24262389.0, + "step": 1240 + }, + { + "entropy": 0.11186399965081364, + "epoch": 2.0, + "grad_norm": 2.15625, + "learning_rate": 8.344444444444445e-06, + "loss": 0.0659, + "mean_token_accuracy": 0.9825448121875524, + "num_tokens": 24455544.0, + "step": 1250 + }, + { + "entropy": 0.088846565480344, + "epoch": 2.016, + "grad_norm": 1.1328125, + "learning_rate": 8.233333333333335e-06, + "loss": 0.0505, + "mean_token_accuracy": 0.9856560029089451, + "num_tokens": 24645425.0, + "step": 1260 + }, + { + "entropy": 0.08578005221206694, + "epoch": 2.032, + "grad_norm": 0.98828125, + "learning_rate": 8.122222222222223e-06, + "loss": 0.0495, + "mean_token_accuracy": 0.9862964119762182, + "num_tokens": 24835956.0, + "step": 1270 + }, + { + "entropy": 0.08365524045657366, + "epoch": 2.048, + "grad_norm": 0.890625, + "learning_rate": 8.011111111111113e-06, + "loss": 0.0482, + "mean_token_accuracy": 0.9864422902464867, + "num_tokens": 25036470.0, + "step": 1280 + }, + { + "entropy": 0.08349559165071696, + "epoch": 2.064, + "grad_norm": 0.8671875, + "learning_rate": 7.9e-06, + "loss": 0.0489, + "mean_token_accuracy": 0.9865494292229414, + "num_tokens": 25230959.0, + "step": 1290 + }, + { + "entropy": 0.08540160392876714, + "epoch": 2.08, + "grad_norm": 0.98828125, + "learning_rate": 7.788888888888889e-06, + "loss": 0.0498, + "mean_token_accuracy": 0.9862173371016979, + "num_tokens": 25422625.0, + "step": 1300 + }, + { + "epoch": 2.08, + "eval_biology_entropy": 0.37641260612010957, + "eval_biology_loss": 2.236539125442505, + "eval_biology_mean_token_accuracy": 0.6860877084732055, + "eval_biology_num_tokens": 25422625.0, + "eval_biology_runtime": 39.5503, + "eval_biology_samples_per_second": 12.642, + "eval_biology_steps_per_second": 3.161, + "step": 1300 + }, + { + "epoch": 2.08, + "eval_chemistry_entropy": 0.2347547177672386, + "eval_chemistry_loss": 1.4374333620071411, + "eval_chemistry_mean_token_accuracy": 0.7814928584098816, + "eval_chemistry_num_tokens": 25422625.0, + "eval_chemistry_runtime": 49.2536, + "eval_chemistry_samples_per_second": 10.152, + "eval_chemistry_steps_per_second": 2.538, + "step": 1300 + }, + { + "entropy": 0.08387249778024852, + "epoch": 2.096, + "grad_norm": 0.9296875, + "learning_rate": 7.677777777777778e-06, + "loss": 0.0496, + "mean_token_accuracy": 0.9860624361783266, + "num_tokens": 25614286.0, + "step": 1310 + }, + { + "entropy": 0.08452012583147735, + "epoch": 2.112, + "grad_norm": 0.88671875, + "learning_rate": 7.566666666666667e-06, + "loss": 0.0489, + "mean_token_accuracy": 0.986203158274293, + "num_tokens": 25810384.0, + "step": 1320 + }, + { + "entropy": 0.08262390322051942, + "epoch": 2.128, + "grad_norm": 0.84765625, + "learning_rate": 7.455555555555556e-06, + "loss": 0.0493, + "mean_token_accuracy": 0.9862308964133263, + "num_tokens": 26005047.0, + "step": 1330 + }, + { + "entropy": 0.08097941202577204, + "epoch": 2.144, + "grad_norm": 1.1796875, + "learning_rate": 7.344444444444445e-06, + "loss": 0.0481, + "mean_token_accuracy": 0.9865331679582596, + "num_tokens": 26207169.0, + "step": 1340 + }, + { + "entropy": 0.08371762945316732, + "epoch": 2.16, + "grad_norm": 0.90234375, + "learning_rate": 7.233333333333334e-06, + "loss": 0.0511, + "mean_token_accuracy": 0.985823930054903, + "num_tokens": 26402150.0, + "step": 1350 + }, + { + "entropy": 0.08566265315748751, + "epoch": 2.176, + "grad_norm": 0.8125, + "learning_rate": 7.122222222222222e-06, + "loss": 0.0487, + "mean_token_accuracy": 0.9863390911370515, + "num_tokens": 26599464.0, + "step": 1360 + }, + { + "entropy": 0.08372032032348216, + "epoch": 2.192, + "grad_norm": 0.9375, + "learning_rate": 7.011111111111112e-06, + "loss": 0.0503, + "mean_token_accuracy": 0.9858050804585219, + "num_tokens": 26790601.0, + "step": 1370 + }, + { + "entropy": 0.08293257770128548, + "epoch": 2.208, + "grad_norm": 0.93359375, + "learning_rate": 6.9e-06, + "loss": 0.049, + "mean_token_accuracy": 0.9863026011735201, + "num_tokens": 26988598.0, + "step": 1380 + }, + { + "entropy": 0.08349582962691784, + "epoch": 2.224, + "grad_norm": 0.76171875, + "learning_rate": 6.788888888888889e-06, + "loss": 0.0492, + "mean_token_accuracy": 0.9860329531133175, + "num_tokens": 27181933.0, + "step": 1390 + }, + { + "entropy": 0.08361051741521805, + "epoch": 2.24, + "grad_norm": 0.89453125, + "learning_rate": 6.677777777777779e-06, + "loss": 0.0496, + "mean_token_accuracy": 0.985880882665515, + "num_tokens": 27378686.0, + "step": 1400 + }, + { + "epoch": 2.24, + "eval_biology_entropy": 0.3741675862073898, + "eval_biology_loss": 2.242098569869995, + "eval_biology_mean_token_accuracy": 0.6867531027793884, + "eval_biology_num_tokens": 27378686.0, + "eval_biology_runtime": 39.7857, + "eval_biology_samples_per_second": 12.567, + "eval_biology_steps_per_second": 3.142, + "step": 1400 + }, + { + "epoch": 2.24, + "eval_chemistry_entropy": 0.23381740498542786, + "eval_chemistry_loss": 1.4444365501403809, + "eval_chemistry_mean_token_accuracy": 0.7810734486579896, + "eval_chemistry_num_tokens": 27378686.0, + "eval_chemistry_runtime": 49.2506, + "eval_chemistry_samples_per_second": 10.152, + "eval_chemistry_steps_per_second": 2.538, + "step": 1400 + }, + { + "entropy": 0.0837663013022393, + "epoch": 2.2560000000000002, + "grad_norm": 0.91015625, + "learning_rate": 6.566666666666667e-06, + "loss": 0.0495, + "mean_token_accuracy": 0.9859327003359795, + "num_tokens": 27571908.0, + "step": 1410 + }, + { + "entropy": 0.08184252071660011, + "epoch": 2.2720000000000002, + "grad_norm": 0.80859375, + "learning_rate": 6.455555555555556e-06, + "loss": 0.0485, + "mean_token_accuracy": 0.9862752124667168, + "num_tokens": 27770478.0, + "step": 1420 + }, + { + "entropy": 0.0840547483181581, + "epoch": 2.288, + "grad_norm": 0.9296875, + "learning_rate": 6.3444444444444454e-06, + "loss": 0.0495, + "mean_token_accuracy": 0.9857813119888306, + "num_tokens": 27965558.0, + "step": 1430 + }, + { + "entropy": 0.08257126114331186, + "epoch": 2.304, + "grad_norm": 0.7734375, + "learning_rate": 6.2333333333333335e-06, + "loss": 0.0481, + "mean_token_accuracy": 0.9863984134048224, + "num_tokens": 28165624.0, + "step": 1440 + }, + { + "entropy": 0.08313491414301097, + "epoch": 2.32, + "grad_norm": 0.9453125, + "learning_rate": 6.1222222222222224e-06, + "loss": 0.0492, + "mean_token_accuracy": 0.9862582210451365, + "num_tokens": 28363594.0, + "step": 1450 + }, + { + "entropy": 0.08305449401959777, + "epoch": 2.336, + "grad_norm": 1.0234375, + "learning_rate": 6.011111111111112e-06, + "loss": 0.0495, + "mean_token_accuracy": 0.9860414322465658, + "num_tokens": 28558104.0, + "step": 1460 + }, + { + "entropy": 0.08424469081219285, + "epoch": 2.352, + "grad_norm": 0.89453125, + "learning_rate": 5.9e-06, + "loss": 0.0509, + "mean_token_accuracy": 0.9856379702687263, + "num_tokens": 28747265.0, + "step": 1470 + }, + { + "entropy": 0.08349237225484103, + "epoch": 2.368, + "grad_norm": 0.89453125, + "learning_rate": 5.788888888888889e-06, + "loss": 0.0488, + "mean_token_accuracy": 0.9860499382019043, + "num_tokens": 28944694.0, + "step": 1480 + }, + { + "entropy": 0.08621921115554869, + "epoch": 2.384, + "grad_norm": 0.875, + "learning_rate": 5.677777777777779e-06, + "loss": 0.0506, + "mean_token_accuracy": 0.9857000894844532, + "num_tokens": 29136617.0, + "step": 1490 + }, + { + "entropy": 0.08343604276888072, + "epoch": 2.4, + "grad_norm": 0.76953125, + "learning_rate": 5.566666666666667e-06, + "loss": 0.0493, + "mean_token_accuracy": 0.986125298216939, + "num_tokens": 29333526.0, + "step": 1500 + }, + { + "epoch": 2.4, + "eval_biology_entropy": 0.37332249569892884, + "eval_biology_loss": 2.2458555698394775, + "eval_biology_mean_token_accuracy": 0.6866299691200256, + "eval_biology_num_tokens": 29333526.0, + "eval_biology_runtime": 39.6765, + "eval_biology_samples_per_second": 12.602, + "eval_biology_steps_per_second": 3.15, + "step": 1500 + }, + { + "epoch": 2.4, + "eval_chemistry_entropy": 0.2335134366750717, + "eval_chemistry_loss": 1.446710228919983, + "eval_chemistry_mean_token_accuracy": 0.7809953107833862, + "eval_chemistry_num_tokens": 29333526.0, + "eval_chemistry_runtime": 48.8671, + "eval_chemistry_samples_per_second": 10.232, + "eval_chemistry_steps_per_second": 2.558, + "step": 1500 + }, + { + "entropy": 0.08370916359126568, + "epoch": 2.416, + "grad_norm": 0.93359375, + "learning_rate": 5.455555555555556e-06, + "loss": 0.0487, + "mean_token_accuracy": 0.9860077023506164, + "num_tokens": 29534153.0, + "step": 1510 + }, + { + "entropy": 0.08138901412021368, + "epoch": 2.432, + "grad_norm": 1.03125, + "learning_rate": 5.344444444444446e-06, + "loss": 0.0479, + "mean_token_accuracy": 0.9862620770931244, + "num_tokens": 29736571.0, + "step": 1520 + }, + { + "entropy": 0.08320502748247235, + "epoch": 2.448, + "grad_norm": 1.1484375, + "learning_rate": 5.233333333333334e-06, + "loss": 0.0495, + "mean_token_accuracy": 0.9859858397394419, + "num_tokens": 29932423.0, + "step": 1530 + }, + { + "entropy": 0.08031986611895263, + "epoch": 2.464, + "grad_norm": 1.203125, + "learning_rate": 5.122222222222223e-06, + "loss": 0.0479, + "mean_token_accuracy": 0.9863451778888702, + "num_tokens": 30137125.0, + "step": 1540 + }, + { + "entropy": 0.08042037494014949, + "epoch": 2.48, + "grad_norm": 0.74609375, + "learning_rate": 5.011111111111111e-06, + "loss": 0.0484, + "mean_token_accuracy": 0.9864198382943868, + "num_tokens": 30341656.0, + "step": 1550 + }, + { + "entropy": 0.08549262350425124, + "epoch": 2.496, + "grad_norm": 0.69140625, + "learning_rate": 4.9000000000000005e-06, + "loss": 0.0498, + "mean_token_accuracy": 0.985682675614953, + "num_tokens": 30533732.0, + "step": 1560 + }, + { + "entropy": 0.08396681451704353, + "epoch": 2.512, + "grad_norm": 1.28125, + "learning_rate": 4.7888888888888894e-06, + "loss": 0.0494, + "mean_token_accuracy": 0.9859193779528141, + "num_tokens": 30730599.0, + "step": 1570 + }, + { + "entropy": 0.08237075076904148, + "epoch": 2.528, + "grad_norm": 0.875, + "learning_rate": 4.677777777777778e-06, + "loss": 0.0487, + "mean_token_accuracy": 0.9857875861227512, + "num_tokens": 30927950.0, + "step": 1580 + }, + { + "entropy": 0.08105518384836614, + "epoch": 2.544, + "grad_norm": 1.09375, + "learning_rate": 4.566666666666667e-06, + "loss": 0.0494, + "mean_token_accuracy": 0.9860106244683265, + "num_tokens": 31125667.0, + "step": 1590 + }, + { + "entropy": 0.08552559302188456, + "epoch": 2.56, + "grad_norm": 0.8125, + "learning_rate": 4.455555555555555e-06, + "loss": 0.0508, + "mean_token_accuracy": 0.9855562083423137, + "num_tokens": 31315742.0, + "step": 1600 + }, + { + "epoch": 2.56, + "eval_biology_entropy": 0.372919380903244, + "eval_biology_loss": 2.248323678970337, + "eval_biology_mean_token_accuracy": 0.6864355983734131, + "eval_biology_num_tokens": 31315742.0, + "eval_biology_runtime": 39.5052, + "eval_biology_samples_per_second": 12.657, + "eval_biology_steps_per_second": 3.164, + "step": 1600 + }, + { + "epoch": 2.56, + "eval_chemistry_entropy": 0.23327860993146896, + "eval_chemistry_loss": 1.4478672742843628, + "eval_chemistry_mean_token_accuracy": 0.7808715200424194, + "eval_chemistry_num_tokens": 31315742.0, + "eval_chemistry_runtime": 49.2432, + "eval_chemistry_samples_per_second": 10.154, + "eval_chemistry_steps_per_second": 2.538, + "step": 1600 + }, + { + "entropy": 0.08430088346358389, + "epoch": 2.576, + "grad_norm": 0.77734375, + "learning_rate": 4.344444444444445e-06, + "loss": 0.0507, + "mean_token_accuracy": 0.9855997852981091, + "num_tokens": 31505374.0, + "step": 1610 + }, + { + "entropy": 0.08420886148232967, + "epoch": 2.592, + "grad_norm": 0.69921875, + "learning_rate": 4.233333333333334e-06, + "loss": 0.0494, + "mean_token_accuracy": 0.9857381645590066, + "num_tokens": 31701680.0, + "step": 1620 + }, + { + "entropy": 0.08595787608064712, + "epoch": 2.608, + "grad_norm": 0.984375, + "learning_rate": 4.122222222222222e-06, + "loss": 0.0504, + "mean_token_accuracy": 0.9856818422675133, + "num_tokens": 31893221.0, + "step": 1630 + }, + { + "entropy": 0.08458381698001176, + "epoch": 2.624, + "grad_norm": 0.921875, + "learning_rate": 4.011111111111111e-06, + "loss": 0.0503, + "mean_token_accuracy": 0.9855606179684401, + "num_tokens": 32087176.0, + "step": 1640 + }, + { + "entropy": 0.08533325386233628, + "epoch": 2.64, + "grad_norm": 0.921875, + "learning_rate": 3.900000000000001e-06, + "loss": 0.05, + "mean_token_accuracy": 0.9852504417300224, + "num_tokens": 32280689.0, + "step": 1650 + }, + { + "entropy": 0.07994957917835563, + "epoch": 2.656, + "grad_norm": 0.81640625, + "learning_rate": 3.7888888888888893e-06, + "loss": 0.0478, + "mean_token_accuracy": 0.9864356804639101, + "num_tokens": 32483392.0, + "step": 1660 + }, + { + "entropy": 0.08173574756365269, + "epoch": 2.672, + "grad_norm": 0.7578125, + "learning_rate": 3.6777777777777778e-06, + "loss": 0.048, + "mean_token_accuracy": 0.9862112771719694, + "num_tokens": 32683246.0, + "step": 1670 + }, + { + "entropy": 0.07958637131378055, + "epoch": 2.6879999999999997, + "grad_norm": 0.8046875, + "learning_rate": 3.566666666666667e-06, + "loss": 0.0474, + "mean_token_accuracy": 0.986869765073061, + "num_tokens": 32884993.0, + "step": 1680 + }, + { + "entropy": 0.08010483384132386, + "epoch": 2.7039999999999997, + "grad_norm": 0.890625, + "learning_rate": 3.455555555555556e-06, + "loss": 0.0476, + "mean_token_accuracy": 0.9867513310164213, + "num_tokens": 33088388.0, + "step": 1690 + }, + { + "entropy": 0.08223347943276167, + "epoch": 2.7199999999999998, + "grad_norm": 0.94140625, + "learning_rate": 3.3444444444444445e-06, + "loss": 0.049, + "mean_token_accuracy": 0.9861364755779505, + "num_tokens": 33287208.0, + "step": 1700 + }, + { + "epoch": 2.7199999999999998, + "eval_biology_entropy": 0.37285495436191557, + "eval_biology_loss": 2.2501490116119385, + "eval_biology_mean_token_accuracy": 0.6862905750274658, + "eval_biology_num_tokens": 33287208.0, + "eval_biology_runtime": 39.5673, + "eval_biology_samples_per_second": 12.637, + "eval_biology_steps_per_second": 3.159, + "step": 1700 + }, + { + "epoch": 2.7199999999999998, + "eval_chemistry_entropy": 0.23335686594247818, + "eval_chemistry_loss": 1.4490605592727661, + "eval_chemistry_mean_token_accuracy": 0.7807522406578064, + "eval_chemistry_num_tokens": 33287208.0, + "eval_chemistry_runtime": 49.3763, + "eval_chemistry_samples_per_second": 10.126, + "eval_chemistry_steps_per_second": 2.532, + "step": 1700 + }, + { + "entropy": 0.0847947500878945, + "epoch": 2.7359999999999998, + "grad_norm": 0.94921875, + "learning_rate": 3.2333333333333334e-06, + "loss": 0.0508, + "mean_token_accuracy": 0.9851035326719284, + "num_tokens": 33474750.0, + "step": 1710 + }, + { + "entropy": 0.0839041729690507, + "epoch": 2.752, + "grad_norm": 1.1796875, + "learning_rate": 3.1222222222222228e-06, + "loss": 0.0508, + "mean_token_accuracy": 0.9855955693870783, + "num_tokens": 33664302.0, + "step": 1720 + }, + { + "entropy": 0.08604868296533823, + "epoch": 2.768, + "grad_norm": 1.125, + "learning_rate": 3.0111111111111113e-06, + "loss": 0.0505, + "mean_token_accuracy": 0.9854719124734401, + "num_tokens": 33854081.0, + "step": 1730 + }, + { + "entropy": 0.08103844472207129, + "epoch": 2.784, + "grad_norm": 0.8046875, + "learning_rate": 2.9e-06, + "loss": 0.0484, + "mean_token_accuracy": 0.9861982863396406, + "num_tokens": 34053149.0, + "step": 1740 + }, + { + "entropy": 0.08212419885676354, + "epoch": 2.8, + "grad_norm": 0.87109375, + "learning_rate": 2.788888888888889e-06, + "loss": 0.0479, + "mean_token_accuracy": 0.9861491050571203, + "num_tokens": 34254105.0, + "step": 1750 + }, + { + "entropy": 0.08174366024322807, + "epoch": 2.816, + "grad_norm": 0.8359375, + "learning_rate": 2.677777777777778e-06, + "loss": 0.0483, + "mean_token_accuracy": 0.9865252107381821, + "num_tokens": 34453342.0, + "step": 1760 + }, + { + "entropy": 0.08624515237752348, + "epoch": 2.832, + "grad_norm": 0.8984375, + "learning_rate": 2.566666666666667e-06, + "loss": 0.0508, + "mean_token_accuracy": 0.9852249193936586, + "num_tokens": 34644113.0, + "step": 1770 + }, + { + "entropy": 0.08373226635158063, + "epoch": 2.848, + "grad_norm": 0.9765625, + "learning_rate": 2.455555555555556e-06, + "loss": 0.0509, + "mean_token_accuracy": 0.9856173813343048, + "num_tokens": 34835062.0, + "step": 1780 + }, + { + "entropy": 0.0821458526654169, + "epoch": 2.864, + "grad_norm": 0.671875, + "learning_rate": 2.3444444444444448e-06, + "loss": 0.049, + "mean_token_accuracy": 0.9862217791378498, + "num_tokens": 35032361.0, + "step": 1790 + }, + { + "entropy": 0.0835355838527903, + "epoch": 2.88, + "grad_norm": 0.984375, + "learning_rate": 2.2333333333333333e-06, + "loss": 0.0499, + "mean_token_accuracy": 0.9855714619159699, + "num_tokens": 35226653.0, + "step": 1800 + }, + { + "epoch": 2.88, + "eval_biology_entropy": 0.37314171600341794, + "eval_biology_loss": 2.2503252029418945, + "eval_biology_mean_token_accuracy": 0.6864723601341247, + "eval_biology_num_tokens": 35226653.0, + "eval_biology_runtime": 39.5746, + "eval_biology_samples_per_second": 12.634, + "eval_biology_steps_per_second": 3.159, + "step": 1800 + }, + { + "epoch": 2.88, + "eval_chemistry_entropy": 0.23350779289007187, + "eval_chemistry_loss": 1.449589729309082, + "eval_chemistry_mean_token_accuracy": 0.7807379026412964, + "eval_chemistry_num_tokens": 35226653.0, + "eval_chemistry_runtime": 49.2909, + "eval_chemistry_samples_per_second": 10.144, + "eval_chemistry_steps_per_second": 2.536, + "step": 1800 + }, + { + "entropy": 0.08435964593663811, + "epoch": 2.896, + "grad_norm": 0.84375, + "learning_rate": 2.1222222222222226e-06, + "loss": 0.0501, + "mean_token_accuracy": 0.9855885654687881, + "num_tokens": 35421873.0, + "step": 1810 + }, + { + "entropy": 0.08406312065199018, + "epoch": 2.912, + "grad_norm": 0.70703125, + "learning_rate": 2.011111111111111e-06, + "loss": 0.0497, + "mean_token_accuracy": 0.9859340194612741, + "num_tokens": 35614590.0, + "step": 1820 + }, + { + "entropy": 0.08338865605182946, + "epoch": 2.928, + "grad_norm": 0.859375, + "learning_rate": 1.9000000000000002e-06, + "loss": 0.05, + "mean_token_accuracy": 0.9860526237636804, + "num_tokens": 35806494.0, + "step": 1830 + }, + { + "entropy": 0.08239444589708, + "epoch": 2.944, + "grad_norm": 0.88671875, + "learning_rate": 1.788888888888889e-06, + "loss": 0.0494, + "mean_token_accuracy": 0.9860717881470918, + "num_tokens": 36001867.0, + "step": 1840 + }, + { + "entropy": 0.08404006839264185, + "epoch": 2.96, + "grad_norm": 0.7265625, + "learning_rate": 1.6777777777777779e-06, + "loss": 0.0489, + "mean_token_accuracy": 0.9858945500105619, + "num_tokens": 36198806.0, + "step": 1850 + }, + { + "entropy": 0.08261097352951766, + "epoch": 2.976, + "grad_norm": 1.0078125, + "learning_rate": 1.566666666666667e-06, + "loss": 0.0493, + "mean_token_accuracy": 0.9861804407089949, + "num_tokens": 36393098.0, + "step": 1860 + }, + { + "entropy": 0.08398979662451893, + "epoch": 2.992, + "grad_norm": 1.15625, + "learning_rate": 1.4555555555555557e-06, + "loss": 0.0497, + "mean_token_accuracy": 0.9860749129205942, + "num_tokens": 36589074.0, + "step": 1870 + }, + { + "entropy": 0.0843541685724631, + "epoch": 3.008, + "grad_norm": 0.56640625, + "learning_rate": 1.3444444444444446e-06, + "loss": 0.0491, + "mean_token_accuracy": 0.985735047981143, + "num_tokens": 36778710.0, + "step": 1880 + }, + { + "entropy": 0.08233831936959177, + "epoch": 3.024, + "grad_norm": 0.45703125, + "learning_rate": 1.2333333333333335e-06, + "loss": 0.0476, + "mean_token_accuracy": 0.985984667390585, + "num_tokens": 36966789.0, + "step": 1890 + }, + { + "entropy": 0.08126567860599607, + "epoch": 3.04, + "grad_norm": 0.546875, + "learning_rate": 1.1222222222222222e-06, + "loss": 0.047, + "mean_token_accuracy": 0.9863237645477057, + "num_tokens": 37156529.0, + "step": 1900 + }, + { + "epoch": 3.04, + "eval_biology_entropy": 0.37237140417099, + "eval_biology_loss": 2.251655101776123, + "eval_biology_mean_token_accuracy": 0.68663911485672, + "eval_biology_num_tokens": 37156529.0, + "eval_biology_runtime": 39.9216, + "eval_biology_samples_per_second": 12.525, + "eval_biology_steps_per_second": 3.131, + "step": 1900 + }, + { + "epoch": 3.04, + "eval_chemistry_entropy": 0.23310688173770905, + "eval_chemistry_loss": 1.450459361076355, + "eval_chemistry_mean_token_accuracy": 0.7806414008140564, + "eval_chemistry_num_tokens": 37156529.0, + "eval_chemistry_runtime": 49.2616, + "eval_chemistry_samples_per_second": 10.15, + "eval_chemistry_steps_per_second": 2.537, + "step": 1900 + }, + { + "entropy": 0.07634997691493481, + "epoch": 3.056, + "grad_norm": 0.69140625, + "learning_rate": 1.0111111111111111e-06, + "loss": 0.046, + "mean_token_accuracy": 0.9868301976472139, + "num_tokens": 37351325.0, + "step": 1910 + }, + { + "entropy": 0.07988243501167744, + "epoch": 3.072, + "grad_norm": 0.55078125, + "learning_rate": 9.000000000000001e-07, + "loss": 0.047, + "mean_token_accuracy": 0.9863556236028671, + "num_tokens": 37541069.0, + "step": 1920 + }, + { + "entropy": 0.07848384822718799, + "epoch": 3.088, + "grad_norm": 0.55078125, + "learning_rate": 7.888888888888889e-07, + "loss": 0.0467, + "mean_token_accuracy": 0.9864400655031205, + "num_tokens": 37733440.0, + "step": 1930 + }, + { + "entropy": 0.07745544153731317, + "epoch": 3.104, + "grad_norm": 0.5703125, + "learning_rate": 6.777777777777779e-07, + "loss": 0.0445, + "mean_token_accuracy": 0.9870091959834099, + "num_tokens": 37933661.0, + "step": 1940 + }, + { + "entropy": 0.0795532870106399, + "epoch": 3.12, + "grad_norm": 0.56640625, + "learning_rate": 5.666666666666667e-07, + "loss": 0.0462, + "mean_token_accuracy": 0.9865683149546385, + "num_tokens": 38126905.0, + "step": 1950 + }, + { + "entropy": 0.08223398483823985, + "epoch": 3.136, + "grad_norm": 0.80078125, + "learning_rate": 4.5555555555555563e-07, + "loss": 0.047, + "mean_token_accuracy": 0.9862224843353034, + "num_tokens": 38316042.0, + "step": 1960 + }, + { + "entropy": 0.07699157926253974, + "epoch": 3.152, + "grad_norm": 0.59375, + "learning_rate": 3.444444444444445e-07, + "loss": 0.0453, + "mean_token_accuracy": 0.9869033012539148, + "num_tokens": 38514212.0, + "step": 1970 + }, + { + "entropy": 0.07890783820766956, + "epoch": 3.168, + "grad_norm": 0.5390625, + "learning_rate": 2.3333333333333336e-07, + "loss": 0.0459, + "mean_token_accuracy": 0.9864298563450575, + "num_tokens": 38708822.0, + "step": 1980 + }, + { + "entropy": 0.0802168081048876, + "epoch": 3.184, + "grad_norm": 0.58984375, + "learning_rate": 1.2222222222222225e-07, + "loss": 0.0468, + "mean_token_accuracy": 0.9863268151879311, + "num_tokens": 38900708.0, + "step": 1990 + }, + { + "entropy": 0.07599673259537668, + "epoch": 3.2, + "grad_norm": 0.478515625, + "learning_rate": 1.1111111111111112e-08, + "loss": 0.0446, + "mean_token_accuracy": 0.9873865962028503, + "num_tokens": 39102325.0, + "step": 2000 + }, + { + "epoch": 3.2, + "eval_biology_entropy": 0.372367325425148, + "eval_biology_loss": 2.25302791595459, + "eval_biology_mean_token_accuracy": 0.6864387440681458, + "eval_biology_num_tokens": 39102325.0, + "eval_biology_runtime": 39.5689, + "eval_biology_samples_per_second": 12.636, + "eval_biology_steps_per_second": 3.159, + "step": 2000 + }, + { + "epoch": 3.2, + "eval_chemistry_entropy": 0.23305992305278778, + "eval_chemistry_loss": 1.4514989852905273, + "eval_chemistry_mean_token_accuracy": 0.7805765233039856, + "eval_chemistry_num_tokens": 39102325.0, + "eval_chemistry_runtime": 48.8599, + "eval_chemistry_samples_per_second": 10.233, + "eval_chemistry_steps_per_second": 2.558, + "step": 2000 + } + ], + "logging_steps": 10, + "max_steps": 2000, + "num_input_tokens_seen": 0, + "num_train_epochs": 4, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 3.279038413131436e+18, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/checkpoint-2000/training_args.bin b/checkpoint-2000/training_args.bin new file mode 100644 index 0000000..4c8c7f4 --- /dev/null +++ b/checkpoint-2000/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:387a0fa8295895440900c3b62c83fa1f8ff3d01320d6ee63d02eb78146f15c65 +size 6545 diff --git a/checkpoint-500/chat_template.jinja b/checkpoint-500/chat_template.jinja new file mode 100644 index 0000000..923ec25 --- /dev/null +++ b/checkpoint-500/chat_template.jinja @@ -0,0 +1,4 @@ +{{ bos_token }}{% if messages[0]['role'] == 'system' %}{{ raise_exception('System role not supported') }}{% endif %}{% for message in messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if (message['role'] == 'assistant') %}{% set role = 'model' %}{% else %}{% set role = message['role'] %}{% endif %}{{ '' + role + ' +' + message['content'] | trim + ' +' }}{% endfor %}{% if add_generation_prompt %}{{'model +'}}{% endif %} \ No newline at end of file diff --git a/checkpoint-500/config.json b/checkpoint-500/config.json new file mode 100644 index 0000000..29e6cbe --- /dev/null +++ b/checkpoint-500/config.json @@ -0,0 +1,77 @@ +{ + "architectures": [ + "Gemma2ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "attn_logit_softcapping": 50.0, + "bos_token_id": 2, + "cache_implementation": "hybrid", + "dtype": "bfloat16", + "eos_token_id": 1, + "final_logit_softcapping": 30.0, + "head_dim": 256, + "hidden_act": "gelu_pytorch_tanh", + "hidden_activation": "gelu_pytorch_tanh", + "hidden_size": 3584, + "initializer_range": 0.02, + "intermediate_size": 14336, + "layer_types": [ + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention" + ], + "max_position_embeddings": 8192, + "model_type": "gemma2", + "num_attention_heads": 16, + "num_hidden_layers": 42, + "num_key_value_heads": 8, + "pad_token_id": 0, + "query_pre_attn_scalar": 256, + "rms_norm_eps": 1e-06, + "rope_theta": 10000.0, + "sliding_window": 4096, + "sliding_window_size": 4096, + "transformers_version": "4.56.1", + "use_cache": true, + "vocab_size": 256000 +} diff --git a/checkpoint-500/generation_config.json b/checkpoint-500/generation_config.json new file mode 100644 index 0000000..f2c34f2 --- /dev/null +++ b/checkpoint-500/generation_config.json @@ -0,0 +1,9 @@ +{ + "_from_model_config": true, + "bos_token_id": 2, + "eos_token_id": [ + 1 + ], + "pad_token_id": 0, + "transformers_version": "4.56.1" +} diff --git a/checkpoint-500/model-00001-of-00004.safetensors b/checkpoint-500/model-00001-of-00004.safetensors new file mode 100644 index 0000000..4705c7e --- /dev/null +++ b/checkpoint-500/model-00001-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a0d4eb4fcbddfe01b0dc7a58386a3980826ededaf0ecd67d1ef38995b89615ba +size 4903351912 diff --git a/checkpoint-500/model-00002-of-00004.safetensors b/checkpoint-500/model-00002-of-00004.safetensors new file mode 100644 index 0000000..744b59b --- /dev/null +++ b/checkpoint-500/model-00002-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:712a589c333f49f7b6f3870838a9e0ab4b3c0260abd5130e8c54316e6c99640b +size 4947570872 diff --git a/checkpoint-500/model-00003-of-00004.safetensors b/checkpoint-500/model-00003-of-00004.safetensors new file mode 100644 index 0000000..2fd7d62 --- /dev/null +++ b/checkpoint-500/model-00003-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c5d2ecd862dc6da5008eec12892332d1f71c4f635664c3ded722902d7543e06f +size 4962221464 diff --git a/checkpoint-500/model-00004-of-00004.safetensors b/checkpoint-500/model-00004-of-00004.safetensors new file mode 100644 index 0000000..0a8396a --- /dev/null +++ b/checkpoint-500/model-00004-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:26a017ddf3ebf154da8f0eb76738252be29c0b8aa6ba788d003942f9ef80121e +size 3670322200 diff --git a/checkpoint-500/model.safetensors.index.json b/checkpoint-500/model.safetensors.index.json new file mode 100644 index 0000000..3b569c5 --- /dev/null +++ b/checkpoint-500/model.safetensors.index.json @@ -0,0 +1,472 @@ +{ + "metadata": { + "total_parameters": 9241705984, + "total_size": 18483411968 + }, + "weight_map": { + "model.embed_tokens.weight": "model-00001-of-00004.safetensors", + "model.layers.0.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.0.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.0.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.1.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.1.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.10.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.10.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.10.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.10.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.11.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.11.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.11.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.12.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.12.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.12.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.13.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.13.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.13.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.14.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.14.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.14.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.15.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.15.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.15.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.16.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.16.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.16.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.17.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.17.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.17.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.18.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.18.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.18.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.19.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.19.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.19.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.2.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.2.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.2.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.20.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.20.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.20.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.20.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.20.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.20.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.20.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.21.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.21.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.21.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.21.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.21.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.22.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.22.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.22.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.22.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.23.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.23.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.23.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.24.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.24.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.24.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.25.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.25.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.25.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.26.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.26.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.26.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.27.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.27.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.27.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.28.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.28.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.28.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.29.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.29.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.29.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.3.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.3.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.3.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.30.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.30.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.30.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.30.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.30.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.31.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.31.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.31.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.32.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.32.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.32.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.32.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.32.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.32.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.32.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.32.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.32.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.32.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.32.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.33.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.33.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.33.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.33.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.33.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.33.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.33.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.33.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.33.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.33.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.33.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.34.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.34.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.34.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.34.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.34.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.34.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.34.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.34.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.34.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.34.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.34.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.35.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.35.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.35.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.35.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.35.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.35.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.35.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.35.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.35.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.35.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.35.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.36.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.36.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.36.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.36.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.36.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.36.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.36.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.36.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.36.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.36.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.36.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.37.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.37.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.37.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.37.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.37.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.37.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.37.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.37.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.37.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.37.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.37.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.38.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.38.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.38.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.38.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.38.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.38.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.38.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.38.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.38.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.38.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.38.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.39.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.39.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.39.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.39.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.39.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.39.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.39.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.39.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.39.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.39.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.39.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.4.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.4.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.4.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.4.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.40.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.40.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.40.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.40.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.40.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.40.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.40.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.40.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.40.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.40.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.40.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.41.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.41.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.41.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.41.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.41.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.41.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.41.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.41.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.41.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.41.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.41.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.5.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.5.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.5.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.5.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.6.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.6.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.6.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.7.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.7.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.7.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.8.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.8.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.8.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.8.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.8.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.8.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.8.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.8.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.8.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.8.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.8.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.9.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.9.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.9.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.9.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.norm.weight": "model-00004-of-00004.safetensors" + } +} diff --git a/checkpoint-500/optimizer.pt b/checkpoint-500/optimizer.pt new file mode 100644 index 0000000..2e422e9 --- /dev/null +++ b/checkpoint-500/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29ad885eff082f1ab6fa626931df3bc22de0ed5789aba90ea58fc76e18cf47f8 +size 7927918951 diff --git a/checkpoint-500/rng_state.pth b/checkpoint-500/rng_state.pth new file mode 100644 index 0000000..73a1a8c --- /dev/null +++ b/checkpoint-500/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:69b6509f8bbdd943c9f8984cdc19d5d17ca63b6e47cd6e28080ebf502da06ddc +size 14645 diff --git a/checkpoint-500/scheduler.pt b/checkpoint-500/scheduler.pt new file mode 100644 index 0000000..20f843b --- /dev/null +++ b/checkpoint-500/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e1e078d7b18ee07da9a874a505bffc02d481132bce9698ce24abe78fa0fbbf05 +size 1465 diff --git a/checkpoint-500/special_tokens_map.json b/checkpoint-500/special_tokens_map.json new file mode 100644 index 0000000..8d6368f --- /dev/null +++ b/checkpoint-500/special_tokens_map.json @@ -0,0 +1,34 @@ +{ + "additional_special_tokens": [ + "", + "" + ], + "bos_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "unk_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + } +} diff --git a/checkpoint-500/tokenizer.json b/checkpoint-500/tokenizer.json new file mode 100644 index 0000000..a4a305d --- /dev/null +++ b/checkpoint-500/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5f7eee611703c5ce5d1eee32d9cdcfe465647b8aff0c1dfb3bed7ad7dbb05060 +size 34362873 diff --git a/checkpoint-500/tokenizer.model b/checkpoint-500/tokenizer.model new file mode 100644 index 0000000..796efe9 --- /dev/null +++ b/checkpoint-500/tokenizer.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:61a7b147390c64585d6c3543dd6fc636906c9af3865a5548f27f31aee1d4c8e2 +size 4241003 diff --git a/checkpoint-500/tokenizer_config.json b/checkpoint-500/tokenizer_config.json new file mode 100644 index 0000000..3ade6be --- /dev/null +++ b/checkpoint-500/tokenizer_config.json @@ -0,0 +1,2013 @@ +{ + "add_bos_token": true, + "add_eos_token": false, + "added_tokens_decoder": { + "0": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "1": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "2": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "3": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "4": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "5": { + "content": "<2mass>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "6": { + "content": "[@BOS@]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "7": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "8": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "9": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "10": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "11": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "12": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "13": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "14": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "15": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "16": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "17": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "18": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "19": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "20": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "21": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "22": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "23": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "24": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "25": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "26": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "27": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "28": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "29": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "30": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "31": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "32": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "33": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "34": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "35": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "36": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "37": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "38": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "39": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "40": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "41": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "42": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "43": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "44": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "45": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "46": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "47": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "48": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "49": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "50": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "51": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "52": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "53": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "54": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "55": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "56": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "57": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "58": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "59": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "60": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "61": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "62": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "63": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "64": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "65": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "66": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "67": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "68": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "69": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "70": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "71": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "72": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "73": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "74": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "75": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "76": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "77": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "78": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "79": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "80": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "81": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "82": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "83": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "84": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "85": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "86": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "87": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "88": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "89": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "90": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "91": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "92": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "93": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "94": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "95": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "96": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "97": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "98": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "99": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "100": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "101": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "102": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "103": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "104": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "105": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "106": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "107": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "108": { + "content": "\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "109": { + "content": "\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "110": { + "content": "\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "111": { + "content": "\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "112": { + "content": "\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "113": { + "content": "\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "114": { + "content": "\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "115": { + "content": "\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "116": { + "content": "\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "117": { + "content": "\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "118": { + "content": "\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "119": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "120": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "121": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "122": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "123": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "124": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "125": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "126": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "127": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "128": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "129": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "130": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "131": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "132": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "133": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "134": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "135": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "136": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "137": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "138": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "139": { + "content": "▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "140": { + "content": "▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "141": { + "content": "▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "142": { + "content": "▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "143": { + "content": "▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "144": { + "content": "▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "145": { + "content": "▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "146": { + "content": "▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "147": { + "content": "▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "148": { + "content": "▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "149": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "150": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "152": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "153": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "154": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "155": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "156": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "157": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "158": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "159": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "160": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "161": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "162": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "163": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "164": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "165": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "166": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "167": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "168": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "169": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "170": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "172": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "173": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "174": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "175": { + "content": "
", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "171": { + "content": "
", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "176": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "177": { + "content": "
", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "178": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "179": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "180": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "181": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "182": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "183": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "184": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "185": { + "content": "

", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "186": { + "content": "

", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "187": { + "content": "

", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "188": { + "content": "

", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "189": { + "content": "

", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "190": { + "content": "
", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "191": { + "content": "
", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "192": { + "content": "
", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "193": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "194": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "195": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "196": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "197": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "198": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "199": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "200": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "201": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "202": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "203": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "204": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "205": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "206": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "207": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "208": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "209": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "210": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "211": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "212": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "213": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "214": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "215": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "216": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255968": { + "content": "[toxicity=0]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255969": { + "content": "\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255970": { + "content": "\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255971": { + "content": "\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255972": { + "content": "\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255973": { + "content": "\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255974": { + "content": "\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255975": { + "content": "\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255976": { + "content": "\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255977": { + "content": "\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255978": { + "content": "\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255979": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255980": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255981": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255982": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255983": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255984": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255985": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255986": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255987": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255988": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255989": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255990": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255991": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255992": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255993": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255994": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255995": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255996": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255997": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255998": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255999": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + } + }, + "additional_special_tokens": [ + "", + "" + ], + "bos_token": "", + "clean_up_tokenization_spaces": false, + "eos_token": "", + "extra_special_tokens": {}, + "model_max_length": 1000000000000000019884624838656, + "pad_token": "", + "sp_model_kwargs": {}, + "spaces_between_special_tokens": false, + "tokenizer_class": "GemmaTokenizer", + "unk_token": "", + "use_default_system_prompt": false +} diff --git a/checkpoint-500/trainer_state.json b/checkpoint-500/trainer_state.json new file mode 100644 index 0000000..64f4ac8 --- /dev/null +++ b/checkpoint-500/trainer_state.json @@ -0,0 +1,644 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.8, + "eval_steps": 100, + "global_step": 500, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 0.08180969459936023, + "epoch": 0.016, + "grad_norm": 0.67578125, + "learning_rate": 9.000000000000001e-07, + "loss": 0.0453, + "mean_token_accuracy": 0.9865816306322813, + "num_tokens": 195510.0, + "step": 10 + }, + { + "entropy": 0.08111731857061386, + "epoch": 0.032, + "grad_norm": 0.60546875, + "learning_rate": 1.9000000000000002e-06, + "loss": 0.0452, + "mean_token_accuracy": 0.986666041240096, + "num_tokens": 391008.0, + "step": 20 + }, + { + "entropy": 0.07587501825764775, + "epoch": 0.048, + "grad_norm": 0.6953125, + "learning_rate": 2.9e-06, + "loss": 0.0445, + "mean_token_accuracy": 0.9873391918838024, + "num_tokens": 590136.0, + "step": 30 + }, + { + "entropy": 0.07685074284672737, + "epoch": 0.064, + "grad_norm": 0.625, + "learning_rate": 3.900000000000001e-06, + "loss": 0.0441, + "mean_token_accuracy": 0.987066001445055, + "num_tokens": 791432.0, + "step": 40 + }, + { + "entropy": 0.07894687210209668, + "epoch": 0.08, + "grad_norm": 0.75390625, + "learning_rate": 4.9000000000000005e-06, + "loss": 0.0447, + "mean_token_accuracy": 0.9865922331809998, + "num_tokens": 990342.0, + "step": 50 + }, + { + "entropy": 0.07919039381667972, + "epoch": 0.096, + "grad_norm": 0.66015625, + "learning_rate": 5.9e-06, + "loss": 0.0466, + "mean_token_accuracy": 0.9863619733601808, + "num_tokens": 1182339.0, + "step": 60 + }, + { + "entropy": 0.07967684981413185, + "epoch": 0.112, + "grad_norm": 0.5859375, + "learning_rate": 6.9e-06, + "loss": 0.0447, + "mean_token_accuracy": 0.9870311982929707, + "num_tokens": 1386193.0, + "step": 70 + }, + { + "entropy": 0.08054462079890072, + "epoch": 0.128, + "grad_norm": 0.65234375, + "learning_rate": 7.9e-06, + "loss": 0.0466, + "mean_token_accuracy": 0.9863289542496204, + "num_tokens": 1583261.0, + "step": 80 + }, + { + "entropy": 0.08421374012250453, + "epoch": 0.144, + "grad_norm": 0.5546875, + "learning_rate": 8.900000000000001e-06, + "loss": 0.0489, + "mean_token_accuracy": 0.9860724166035653, + "num_tokens": 1774826.0, + "step": 90 + }, + { + "entropy": 0.08307966932188719, + "epoch": 0.16, + "grad_norm": 0.71484375, + "learning_rate": 9.9e-06, + "loss": 0.0489, + "mean_token_accuracy": 0.986110083758831, + "num_tokens": 1971310.0, + "step": 100 + }, + { + "epoch": 0.16, + "eval_biology_entropy": 0.3792700763940811, + "eval_biology_loss": 2.260571241378784, + "eval_biology_mean_token_accuracy": 0.6872955484390258, + "eval_biology_num_tokens": 1971310.0, + "eval_biology_runtime": 39.5255, + "eval_biology_samples_per_second": 12.65, + "eval_biology_steps_per_second": 3.163, + "step": 100 + }, + { + "epoch": 0.16, + "eval_chemistry_entropy": 0.23563545274734496, + "eval_chemistry_loss": 1.4561725854873657, + "eval_chemistry_mean_token_accuracy": 0.7805930047035218, + "eval_chemistry_num_tokens": 1971310.0, + "eval_chemistry_runtime": 49.2319, + "eval_chemistry_samples_per_second": 10.156, + "eval_chemistry_steps_per_second": 2.539, + "step": 100 + }, + { + "entropy": 0.08481427943333983, + "epoch": 0.176, + "grad_norm": 0.734375, + "learning_rate": 1.0900000000000002e-05, + "loss": 0.0497, + "mean_token_accuracy": 0.9861054014414549, + "num_tokens": 2169453.0, + "step": 110 + }, + { + "entropy": 0.0882645736914128, + "epoch": 0.192, + "grad_norm": 0.79296875, + "learning_rate": 1.1900000000000001e-05, + "loss": 0.0512, + "mean_token_accuracy": 0.9859256945550442, + "num_tokens": 2366844.0, + "step": 120 + }, + { + "entropy": 0.09435517250094563, + "epoch": 0.208, + "grad_norm": 1.2421875, + "learning_rate": 1.2900000000000002e-05, + "loss": 0.0547, + "mean_token_accuracy": 0.9849376413971186, + "num_tokens": 2559708.0, + "step": 130 + }, + { + "entropy": 0.09682020233012736, + "epoch": 0.224, + "grad_norm": 1.3671875, + "learning_rate": 1.39e-05, + "loss": 0.057, + "mean_token_accuracy": 0.985217172652483, + "num_tokens": 2756533.0, + "step": 140 + }, + { + "entropy": 0.11111663887277246, + "epoch": 0.24, + "grad_norm": 2.84375, + "learning_rate": 1.4900000000000001e-05, + "loss": 0.0657, + "mean_token_accuracy": 0.9836616575717926, + "num_tokens": 2948600.0, + "step": 150 + }, + { + "entropy": 0.12251729485578836, + "epoch": 0.256, + "grad_norm": 2.453125, + "learning_rate": 1.5900000000000004e-05, + "loss": 0.0728, + "mean_token_accuracy": 0.9814188834279776, + "num_tokens": 3141473.0, + "step": 160 + }, + { + "entropy": 0.12813111576251685, + "epoch": 0.272, + "grad_norm": 3.03125, + "learning_rate": 1.69e-05, + "loss": 0.0758, + "mean_token_accuracy": 0.9808279071003199, + "num_tokens": 3337285.0, + "step": 170 + }, + { + "entropy": 0.13337696841917931, + "epoch": 0.288, + "grad_norm": 2.984375, + "learning_rate": 1.79e-05, + "loss": 0.0771, + "mean_token_accuracy": 0.9798636745661498, + "num_tokens": 3541221.0, + "step": 180 + }, + { + "entropy": 0.1436500378418714, + "epoch": 0.304, + "grad_norm": 2.78125, + "learning_rate": 1.8900000000000002e-05, + "loss": 0.0851, + "mean_token_accuracy": 0.9780430492013693, + "num_tokens": 3734952.0, + "step": 190 + }, + { + "entropy": 0.1549748638179153, + "epoch": 0.32, + "grad_norm": 3.390625, + "learning_rate": 1.9900000000000003e-05, + "loss": 0.0946, + "mean_token_accuracy": 0.9750579129904509, + "num_tokens": 3922209.0, + "step": 200 + }, + { + "epoch": 0.32, + "eval_biology_entropy": 0.4169517788887024, + "eval_biology_loss": 2.160527229309082, + "eval_biology_mean_token_accuracy": 0.6808431892395019, + "eval_biology_num_tokens": 3922209.0, + "eval_biology_runtime": 39.5121, + "eval_biology_samples_per_second": 12.654, + "eval_biology_steps_per_second": 3.164, + "step": 200 + }, + { + "epoch": 0.32, + "eval_chemistry_entropy": 0.25474807441234587, + "eval_chemistry_loss": 1.3652101755142212, + "eval_chemistry_mean_token_accuracy": 0.7794562330245972, + "eval_chemistry_num_tokens": 3922209.0, + "eval_chemistry_runtime": 49.2263, + "eval_chemistry_samples_per_second": 10.157, + "eval_chemistry_steps_per_second": 2.539, + "step": 200 + }, + { + "entropy": 0.16029730178415774, + "epoch": 0.336, + "grad_norm": 3.515625, + "learning_rate": 1.9900000000000003e-05, + "loss": 0.097, + "mean_token_accuracy": 0.9749548710882664, + "num_tokens": 4115941.0, + "step": 210 + }, + { + "entropy": 0.16590779088437557, + "epoch": 0.352, + "grad_norm": 3.78125, + "learning_rate": 1.978888888888889e-05, + "loss": 0.1023, + "mean_token_accuracy": 0.9726595059037209, + "num_tokens": 4308755.0, + "step": 220 + }, + { + "entropy": 0.1707212173845619, + "epoch": 0.368, + "grad_norm": 3.390625, + "learning_rate": 1.967777777777778e-05, + "loss": 0.1023, + "mean_token_accuracy": 0.9726378936320543, + "num_tokens": 4505910.0, + "step": 230 + }, + { + "entropy": 0.1765518521424383, + "epoch": 0.384, + "grad_norm": 3.671875, + "learning_rate": 1.956666666666667e-05, + "loss": 0.1083, + "mean_token_accuracy": 0.97143131531775, + "num_tokens": 4696101.0, + "step": 240 + }, + { + "entropy": 0.17895571915432812, + "epoch": 0.4, + "grad_norm": 3.5, + "learning_rate": 1.9455555555555555e-05, + "loss": 0.11, + "mean_token_accuracy": 0.9705049484968186, + "num_tokens": 4889445.0, + "step": 250 + }, + { + "entropy": 0.18151565054431557, + "epoch": 0.416, + "grad_norm": 3.578125, + "learning_rate": 1.9344444444444446e-05, + "loss": 0.1091, + "mean_token_accuracy": 0.970859233289957, + "num_tokens": 5087624.0, + "step": 260 + }, + { + "entropy": 0.18663504091091454, + "epoch": 0.432, + "grad_norm": 3.859375, + "learning_rate": 1.9233333333333334e-05, + "loss": 0.1135, + "mean_token_accuracy": 0.9693388067185879, + "num_tokens": 5273794.0, + "step": 270 + }, + { + "entropy": 0.18451756797730923, + "epoch": 0.448, + "grad_norm": 4.125, + "learning_rate": 1.9122222222222226e-05, + "loss": 0.1144, + "mean_token_accuracy": 0.9698665160685778, + "num_tokens": 5463288.0, + "step": 280 + }, + { + "entropy": 0.17932639280334114, + "epoch": 0.464, + "grad_norm": 3.5625, + "learning_rate": 1.9011111111111114e-05, + "loss": 0.1081, + "mean_token_accuracy": 0.9717006407678127, + "num_tokens": 5656827.0, + "step": 290 + }, + { + "entropy": 0.18305762661620975, + "epoch": 0.48, + "grad_norm": 3.90625, + "learning_rate": 1.8900000000000002e-05, + "loss": 0.1107, + "mean_token_accuracy": 0.9712053131312132, + "num_tokens": 5853311.0, + "step": 300 + }, + { + "epoch": 0.48, + "eval_biology_entropy": 0.42041616535186765, + "eval_biology_loss": 2.1313436031341553, + "eval_biology_mean_token_accuracy": 0.6798483271598816, + "eval_biology_num_tokens": 5853311.0, + "eval_biology_runtime": 39.5137, + "eval_biology_samples_per_second": 12.654, + "eval_biology_steps_per_second": 3.163, + "step": 300 + }, + { + "epoch": 0.48, + "eval_chemistry_entropy": 0.259404793381691, + "eval_chemistry_loss": 1.3463311195373535, + "eval_chemistry_mean_token_accuracy": 0.7787248449325561, + "eval_chemistry_num_tokens": 5853311.0, + "eval_chemistry_runtime": 49.201, + "eval_chemistry_samples_per_second": 10.162, + "eval_chemistry_steps_per_second": 2.541, + "step": 300 + }, + { + "entropy": 0.17875824291259051, + "epoch": 0.496, + "grad_norm": 3.453125, + "learning_rate": 1.878888888888889e-05, + "loss": 0.1101, + "mean_token_accuracy": 0.9710387323051691, + "num_tokens": 6049788.0, + "step": 310 + }, + { + "entropy": 0.1863187742419541, + "epoch": 0.512, + "grad_norm": 3.4375, + "learning_rate": 1.8677777777777778e-05, + "loss": 0.113, + "mean_token_accuracy": 0.9698710650205612, + "num_tokens": 6243825.0, + "step": 320 + }, + { + "entropy": 0.18619006094522775, + "epoch": 0.528, + "grad_norm": 3.890625, + "learning_rate": 1.856666666666667e-05, + "loss": 0.1135, + "mean_token_accuracy": 0.9703223951160908, + "num_tokens": 6434188.0, + "step": 330 + }, + { + "entropy": 0.18118073977530003, + "epoch": 0.544, + "grad_norm": 3.484375, + "learning_rate": 1.8455555555555558e-05, + "loss": 0.1114, + "mean_token_accuracy": 0.9708513412624598, + "num_tokens": 6629832.0, + "step": 340 + }, + { + "entropy": 0.1808169964235276, + "epoch": 0.56, + "grad_norm": 3.28125, + "learning_rate": 1.8344444444444446e-05, + "loss": 0.1079, + "mean_token_accuracy": 0.9716686312109232, + "num_tokens": 6824668.0, + "step": 350 + }, + { + "entropy": 0.18033515606075526, + "epoch": 0.576, + "grad_norm": 3.53125, + "learning_rate": 1.8233333333333334e-05, + "loss": 0.106, + "mean_token_accuracy": 0.9722726982086897, + "num_tokens": 7025923.0, + "step": 360 + }, + { + "entropy": 0.1852203646209091, + "epoch": 0.592, + "grad_norm": 3.625, + "learning_rate": 1.8122222222222222e-05, + "loss": 0.111, + "mean_token_accuracy": 0.9705461781471968, + "num_tokens": 7218224.0, + "step": 370 + }, + { + "entropy": 0.17758405841886998, + "epoch": 0.608, + "grad_norm": 3.65625, + "learning_rate": 1.8011111111111113e-05, + "loss": 0.1066, + "mean_token_accuracy": 0.9723859451711178, + "num_tokens": 7421216.0, + "step": 380 + }, + { + "entropy": 0.18370326030999423, + "epoch": 0.624, + "grad_norm": 3.765625, + "learning_rate": 1.79e-05, + "loss": 0.1096, + "mean_token_accuracy": 0.9708739675581455, + "num_tokens": 7617258.0, + "step": 390 + }, + { + "entropy": 0.1831163068767637, + "epoch": 0.64, + "grad_norm": 3.578125, + "learning_rate": 1.778888888888889e-05, + "loss": 0.1072, + "mean_token_accuracy": 0.9717655830085278, + "num_tokens": 7806173.0, + "step": 400 + }, + { + "epoch": 0.64, + "eval_biology_entropy": 0.42119112753868104, + "eval_biology_loss": 2.1267058849334717, + "eval_biology_mean_token_accuracy": 0.6801503438949585, + "eval_biology_num_tokens": 7806173.0, + "eval_biology_runtime": 39.5121, + "eval_biology_samples_per_second": 12.654, + "eval_biology_steps_per_second": 3.164, + "step": 400 + }, + { + "epoch": 0.64, + "eval_chemistry_entropy": 0.2595027017593384, + "eval_chemistry_loss": 1.3404533863067627, + "eval_chemistry_mean_token_accuracy": 0.7790702543258667, + "eval_chemistry_num_tokens": 7806173.0, + "eval_chemistry_runtime": 49.2002, + "eval_chemistry_samples_per_second": 10.163, + "eval_chemistry_steps_per_second": 2.541, + "step": 400 + }, + { + "entropy": 0.1827191138640046, + "epoch": 0.656, + "grad_norm": 3.84375, + "learning_rate": 1.7677777777777778e-05, + "loss": 0.1088, + "mean_token_accuracy": 0.9715055178850889, + "num_tokens": 8000514.0, + "step": 410 + }, + { + "entropy": 0.18661352493800223, + "epoch": 0.672, + "grad_norm": 3.796875, + "learning_rate": 1.756666666666667e-05, + "loss": 0.1114, + "mean_token_accuracy": 0.9704575203359127, + "num_tokens": 8187884.0, + "step": 420 + }, + { + "entropy": 0.17729734890162946, + "epoch": 0.688, + "grad_norm": 3.09375, + "learning_rate": 1.7455555555555557e-05, + "loss": 0.1046, + "mean_token_accuracy": 0.9725840430706739, + "num_tokens": 8390643.0, + "step": 430 + }, + { + "entropy": 0.18038612157106398, + "epoch": 0.704, + "grad_norm": 3.46875, + "learning_rate": 1.7344444444444445e-05, + "loss": 0.1072, + "mean_token_accuracy": 0.9721495136618614, + "num_tokens": 8583266.0, + "step": 440 + }, + { + "entropy": 0.17609651018865408, + "epoch": 0.72, + "grad_norm": 3.734375, + "learning_rate": 1.7233333333333337e-05, + "loss": 0.1045, + "mean_token_accuracy": 0.9718786559998989, + "num_tokens": 8786233.0, + "step": 450 + }, + { + "entropy": 0.1797325788065791, + "epoch": 0.736, + "grad_norm": 3.390625, + "learning_rate": 1.7122222222222225e-05, + "loss": 0.107, + "mean_token_accuracy": 0.9716725051403046, + "num_tokens": 8982898.0, + "step": 460 + }, + { + "entropy": 0.17950535458512604, + "epoch": 0.752, + "grad_norm": 3.5625, + "learning_rate": 1.7011111111111113e-05, + "loss": 0.1072, + "mean_token_accuracy": 0.9725060146301985, + "num_tokens": 9174361.0, + "step": 470 + }, + { + "entropy": 0.17792485035024583, + "epoch": 0.768, + "grad_norm": 3.375, + "learning_rate": 1.69e-05, + "loss": 0.1025, + "mean_token_accuracy": 0.973187630996108, + "num_tokens": 9373430.0, + "step": 480 + }, + { + "entropy": 0.17844869694672524, + "epoch": 0.784, + "grad_norm": 3.515625, + "learning_rate": 1.678888888888889e-05, + "loss": 0.1073, + "mean_token_accuracy": 0.9717021223157645, + "num_tokens": 9570409.0, + "step": 490 + }, + { + "entropy": 0.17820251639932394, + "epoch": 0.8, + "grad_norm": 3.765625, + "learning_rate": 1.667777777777778e-05, + "loss": 0.1032, + "mean_token_accuracy": 0.9729967709630728, + "num_tokens": 9766360.0, + "step": 500 + }, + { + "epoch": 0.8, + "eval_biology_entropy": 0.41608734560012817, + "eval_biology_loss": 2.139784097671509, + "eval_biology_mean_token_accuracy": 0.6806383662223816, + "eval_biology_num_tokens": 9766360.0, + "eval_biology_runtime": 39.5271, + "eval_biology_samples_per_second": 12.65, + "eval_biology_steps_per_second": 3.162, + "step": 500 + }, + { + "epoch": 0.8, + "eval_chemistry_entropy": 0.25711704087257387, + "eval_chemistry_loss": 1.3562349081039429, + "eval_chemistry_mean_token_accuracy": 0.7787247653007507, + "eval_chemistry_num_tokens": 9766360.0, + "eval_chemistry_runtime": 48.8194, + "eval_chemistry_samples_per_second": 10.242, + "eval_chemistry_steps_per_second": 2.56, + "step": 500 + } + ], + "logging_steps": 10, + "max_steps": 2000, + "num_input_tokens_seen": 0, + "num_train_epochs": 4, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 8.165294965933916e+17, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/checkpoint-500/training_args.bin b/checkpoint-500/training_args.bin new file mode 100644 index 0000000..4c8c7f4 --- /dev/null +++ b/checkpoint-500/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:387a0fa8295895440900c3b62c83fa1f8ff3d01320d6ee63d02eb78146f15c65 +size 6545 diff --git a/config.json b/config.json new file mode 100644 index 0000000..29e6cbe --- /dev/null +++ b/config.json @@ -0,0 +1,77 @@ +{ + "architectures": [ + "Gemma2ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "attn_logit_softcapping": 50.0, + "bos_token_id": 2, + "cache_implementation": "hybrid", + "dtype": "bfloat16", + "eos_token_id": 1, + "final_logit_softcapping": 30.0, + "head_dim": 256, + "hidden_act": "gelu_pytorch_tanh", + "hidden_activation": "gelu_pytorch_tanh", + "hidden_size": 3584, + "initializer_range": 0.02, + "intermediate_size": 14336, + "layer_types": [ + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention" + ], + "max_position_embeddings": 8192, + "model_type": "gemma2", + "num_attention_heads": 16, + "num_hidden_layers": 42, + "num_key_value_heads": 8, + "pad_token_id": 0, + "query_pre_attn_scalar": 256, + "rms_norm_eps": 1e-06, + "rope_theta": 10000.0, + "sliding_window": 4096, + "sliding_window_size": 4096, + "transformers_version": "4.56.1", + "use_cache": true, + "vocab_size": 256000 +} diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..f2c34f2 --- /dev/null +++ b/generation_config.json @@ -0,0 +1,9 @@ +{ + "_from_model_config": true, + "bos_token_id": 2, + "eos_token_id": [ + 1 + ], + "pad_token_id": 0, + "transformers_version": "4.56.1" +} diff --git a/model-00001-of-00004.safetensors b/model-00001-of-00004.safetensors new file mode 100644 index 0000000..4705c7e --- /dev/null +++ b/model-00001-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a0d4eb4fcbddfe01b0dc7a58386a3980826ededaf0ecd67d1ef38995b89615ba +size 4903351912 diff --git a/model-00002-of-00004.safetensors b/model-00002-of-00004.safetensors new file mode 100644 index 0000000..744b59b --- /dev/null +++ b/model-00002-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:712a589c333f49f7b6f3870838a9e0ab4b3c0260abd5130e8c54316e6c99640b +size 4947570872 diff --git a/model-00003-of-00004.safetensors b/model-00003-of-00004.safetensors new file mode 100644 index 0000000..a394d39 --- /dev/null +++ b/model-00003-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d02617b963ba5ff0a2f0f95663f02ae6d6b971bba52293a6876ba159520e3361 +size 4962221464 diff --git a/model-00004-of-00004.safetensors b/model-00004-of-00004.safetensors new file mode 100644 index 0000000..f1a23f8 --- /dev/null +++ b/model-00004-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:326631a22f10135d46fc892c27d9e64f5aaf558af084c67b2f6ce95e612cd3f8 +size 3670322200 diff --git a/model.safetensors.index.json b/model.safetensors.index.json new file mode 100644 index 0000000..3b569c5 --- /dev/null +++ b/model.safetensors.index.json @@ -0,0 +1,472 @@ +{ + "metadata": { + "total_parameters": 9241705984, + "total_size": 18483411968 + }, + "weight_map": { + "model.embed_tokens.weight": "model-00001-of-00004.safetensors", + "model.layers.0.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.0.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.0.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.1.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.1.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.10.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.10.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.10.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.10.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.11.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.11.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.11.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.12.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.12.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.12.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.13.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.13.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.13.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.14.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.14.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.14.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.15.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.15.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.15.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.16.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.16.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.16.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.17.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.17.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.17.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.18.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.18.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.18.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.19.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.19.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.19.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.2.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.2.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.2.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.20.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.20.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.20.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.20.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.20.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.20.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.20.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.21.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.21.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.21.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.21.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.21.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.22.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.22.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.22.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.22.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.23.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.23.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.23.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.24.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.24.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.24.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.25.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.25.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.25.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.26.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.26.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.26.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.27.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.27.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.27.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.28.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.28.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.28.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.29.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.29.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.29.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.3.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.3.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.3.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.30.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.30.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.30.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.30.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.30.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.31.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.31.post_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.31.pre_feedforward_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.32.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.32.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.32.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.32.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.32.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.32.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.32.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.32.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.32.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.32.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.32.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.33.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.33.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.33.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.33.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.33.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.33.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.33.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.33.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.33.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.33.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.33.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.34.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.34.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.34.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.34.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.34.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.34.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.34.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.34.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.34.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.34.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.34.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.35.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.35.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.35.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.35.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.35.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.35.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.35.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.35.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.35.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.35.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.35.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.36.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.36.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.36.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.36.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.36.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.36.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.36.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.36.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.36.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.36.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.36.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.37.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.37.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.37.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.37.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.37.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.37.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.37.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.37.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.37.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.37.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.37.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.38.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.38.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.38.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.38.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.38.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.38.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.38.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.38.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.38.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.38.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.38.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.39.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.39.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.39.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.39.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.39.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.39.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.39.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.39.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.39.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.39.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.39.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.4.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.4.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.4.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.4.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.40.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.40.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.40.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.40.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.40.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.40.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.40.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.40.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.40.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.40.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.40.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.41.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.41.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.41.mlp.gate_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.41.mlp.up_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.41.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.41.post_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.41.pre_feedforward_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.41.self_attn.k_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.41.self_attn.o_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.41.self_attn.q_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.41.self_attn.v_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.5.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.5.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.5.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.5.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.6.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.6.post_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.6.pre_feedforward_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.7.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.7.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.7.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.8.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.8.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.8.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.8.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.8.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.8.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.8.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.8.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.8.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.8.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.8.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.9.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.9.post_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.9.pre_feedforward_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.9.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.norm.weight": "model-00004-of-00004.safetensors" + } +} diff --git a/special_tokens_map.json b/special_tokens_map.json new file mode 100644 index 0000000..8d6368f --- /dev/null +++ b/special_tokens_map.json @@ -0,0 +1,34 @@ +{ + "additional_special_tokens": [ + "", + "" + ], + "bos_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "unk_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + } +} diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..a4a305d --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5f7eee611703c5ce5d1eee32d9cdcfe465647b8aff0c1dfb3bed7ad7dbb05060 +size 34362873 diff --git a/tokenizer.model b/tokenizer.model new file mode 100644 index 0000000..796efe9 --- /dev/null +++ b/tokenizer.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:61a7b147390c64585d6c3543dd6fc636906c9af3865a5548f27f31aee1d4c8e2 +size 4241003 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..3ade6be --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,2013 @@ +{ + "add_bos_token": true, + "add_eos_token": false, + "added_tokens_decoder": { + "0": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "1": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "2": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "3": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "4": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "5": { + "content": "<2mass>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "6": { + "content": "[@BOS@]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "7": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "8": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "9": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "10": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "11": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "12": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "13": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "14": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "15": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "16": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "17": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "18": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "19": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "20": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "21": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "22": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "23": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "24": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "25": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "26": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "27": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "28": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "29": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "30": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "31": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "32": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "33": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "34": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "35": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "36": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "37": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "38": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "39": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "40": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "41": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "42": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "43": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "44": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "45": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "46": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "47": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "48": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "49": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "50": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "51": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "52": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "53": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "54": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "55": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "56": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "57": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "58": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "59": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "60": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "61": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "62": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "63": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "64": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "65": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "66": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "67": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "68": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "69": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "70": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "71": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "72": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "73": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "74": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "75": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "76": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "77": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "78": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "79": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "80": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "81": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "82": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "83": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "84": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "85": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "86": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "87": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "88": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "89": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "90": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "91": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "92": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "93": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "94": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "95": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "96": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "97": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "98": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "99": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "100": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "101": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "102": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "103": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "104": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "105": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "106": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "107": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "108": { + "content": "\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "109": { + "content": "\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "110": { + "content": "\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "111": { + "content": "\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "112": { + "content": "\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "113": { + "content": "\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "114": { + "content": "\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "115": { + "content": "\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "116": { + "content": "\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "117": { + "content": "\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "118": { + "content": "\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "119": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "120": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "121": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "122": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "123": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "124": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "125": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "126": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "127": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "128": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "129": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "130": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "131": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "132": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "133": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "134": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "135": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "136": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "137": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "138": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "139": { + "content": "▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "140": { + "content": "▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "141": { + "content": "▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "142": { + "content": "▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "143": { + "content": "▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "144": { + "content": "▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "145": { + "content": "▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "146": { + "content": "▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "147": { + "content": "▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "148": { + "content": "▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "149": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "150": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "152": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "153": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "154": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "155": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "156": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "157": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "158": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "159": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "160": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "161": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "162": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "163": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "164": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "165": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "166": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "167": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "168": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "169": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "170": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "172": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "173": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "174": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "175": { + "content": "
", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "171": { + "content": "
", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "176": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "177": { + "content": "
", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "178": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "179": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "180": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "181": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "182": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "183": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "184": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "185": { + "content": "

", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "186": { + "content": "

", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "187": { + "content": "

", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "188": { + "content": "

", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "189": { + "content": "

", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "190": { + "content": "
", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "191": { + "content": "
", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "192": { + "content": "
", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "193": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "194": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "195": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "196": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "197": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "198": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "199": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "200": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "201": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "202": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "203": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "204": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "205": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "206": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "207": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "208": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "209": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "210": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "211": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "212": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "213": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "214": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "215": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "216": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255968": { + "content": "[toxicity=0]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255969": { + "content": "\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255970": { + "content": "\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255971": { + "content": "\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255972": { + "content": "\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255973": { + "content": "\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255974": { + "content": "\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255975": { + "content": "\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255976": { + "content": "\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255977": { + "content": "\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255978": { + "content": "\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255979": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255980": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255981": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255982": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255983": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255984": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255985": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255986": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255987": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255988": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255989": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255990": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255991": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255992": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255993": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255994": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255995": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255996": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255997": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255998": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255999": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + } + }, + "additional_special_tokens": [ + "", + "" + ], + "bos_token": "", + "clean_up_tokenization_spaces": false, + "eos_token": "", + "extra_special_tokens": {}, + "model_max_length": 1000000000000000019884624838656, + "pad_token": "", + "sp_model_kwargs": {}, + "spaces_between_special_tokens": false, + "tokenizer_class": "GemmaTokenizer", + "unk_token": "", + "use_default_system_prompt": false +}