commit 15750674d956bda577230574858d57d2e8749a6b Author: ModelHub XC Date: Fri Aug 7 20:13:16 2026 +0800 初始化项目，由ModelHub XC社区提供模型 Model: tzchen07/SG_X9e Source: Original Platform diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..e16daa5 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,37 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +checkpoint-498/tokenizer.json filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..717e816 --- /dev/null +++ b/README.md @@ -0,0 +1,58 @@ +--- +library_name: transformers +license: other +base_model: jxm/shieldgemma-2b +tags: +- llama-factory +- full +- generated_from_trainer +model-index: +- name: sg_x9e + results: [] +--- + + + +# sg_x9e + +This model is a fine-tuned version of [jxm/shieldgemma-2b](https://huggingface.co/jxm/shieldgemma-2b) on the v1.6, the v1.6b, the v1.6c, the v1.6d and the v1.6e datasets. + +## Model description + +More information needed + +## Intended uses & limitations + +More information needed + +## Training and evaluation data + +More information needed + +## Training procedure + +### Training hyperparameters + +The following hyperparameters were used during training: +- learning_rate: 5e-06 +- train_batch_size: 4 +- eval_batch_size: 8 +- seed: 42 +- gradient_accumulation_steps: 16 +- total_train_batch_size: 64 +- optimizer: Use OptimizerNames.ADAMW_TORCH_FUSED with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments +- lr_scheduler_type: cosine +- lr_scheduler_warmup_ratio: 0.1 +- num_epochs: 2 + +### Training results + + + +### Framework versions + +- Transformers 4.57.1 +- Pytorch 2.10.0+cu129 +- Datasets 4.0.0 +- Tokenizers 0.22.2 diff --git a/all_results.json b/all_results.json new file mode 100644 index 0000000..820de62 --- /dev/null +++ b/all_results.json @@ -0,0 +1,8 @@ +{ + "epoch": 2.0, + "total_flos": 1.28494219771102e+18, + "train_loss": 0.03900693407858232, + "train_runtime": 6158.7829, + "train_samples_per_second": 5.17, + "train_steps_per_second": 0.081 +} \ No newline at end of file diff --git a/chat_template.jinja b/chat_template.jinja new file mode 100644 index 0000000..b017fbd --- /dev/null +++ b/chat_template.jinja @@ -0,0 +1,29 @@ +{%- if not guideline and messages[0].role == "system" %} + {%- set guideline = messages[0].content %} + {%- set messages = messages[1:] %} +{%- endif %} + +{{- bos_token }} +{%- if messages[-1].role == 'user' %} + {{- "You are a policy expert trying to help determine whether a user prompt is in violation of the defined safety policies.\n\n" }} + {{- "\n" }} + {{- "Human Question: " + messages[-1].content }} + {{- "\n\n\n" }} + {{- "Our safety principle is defined in the below:\n\n" }} + {{- "* " + guideline + "\n" }} + {{- "\n===\n\n" }} + {{- "Does the human question violate the above principle? Your answer must start with 'Yes' or 'No'. And then walk through step by step to be sure we answer correctly.\n\n" }} +{%- else %} + {{- "You are a policy expert trying to help determine whether an AI response to a prompt is in violation of the defined safety policies.\n\n" }} + {{- "\n" }} + {{- "Human Question: " + messages[-2].content }} + {{- "\n\n" }} + {{- "\n" }} + {{- "Chatbot Response: " + messages[-1].content }} + {{- "\n\n\n" }} + {{- "Our safety principle is defined in the below:\n\n" }} + {{- "* " + guideline + "\n" }} + {{- "\n===\n\n" }} + {{- "Does the Chatbot Response violate the above principle? Your answer must start with 'Yes' or 'No'. And then walk through step by step to be sure we answer correctly.\n\n" }} +{%- endif %} + diff --git a/checkpoint-498/chat_template.jinja b/checkpoint-498/chat_template.jinja new file mode 100644 index 0000000..b017fbd --- /dev/null +++ b/checkpoint-498/chat_template.jinja @@ -0,0 +1,29 @@ +{%- if not guideline and messages[0].role == "system" %} + {%- set guideline = messages[0].content %} + {%- set messages = messages[1:] %} +{%- endif %} + +{{- bos_token }} +{%- if messages[-1].role == 'user' %} + {{- "You are a policy expert trying to help determine whether a user prompt is in violation of the defined safety policies.\n\n" }} + {{- "\n" }} + {{- "Human Question: " + messages[-1].content }} + {{- "\n\n\n" }} + {{- "Our safety principle is defined in the below:\n\n" }} + {{- "* " + guideline + "\n" }} + {{- "\n===\n\n" }} + {{- "Does the human question violate the above principle? Your answer must start with 'Yes' or 'No'. And then walk through step by step to be sure we answer correctly.\n\n" }} +{%- else %} + {{- "You are a policy expert trying to help determine whether an AI response to a prompt is in violation of the defined safety policies.\n\n" }} + {{- "\n" }} + {{- "Human Question: " + messages[-2].content }} + {{- "\n\n" }} + {{- "\n" }} + {{- "Chatbot Response: " + messages[-1].content }} + {{- "\n\n\n" }} + {{- "Our safety principle is defined in the below:\n\n" }} + {{- "* " + guideline + "\n" }} + {{- "\n===\n\n" }} + {{- "Does the Chatbot Response violate the above principle? Your answer must start with 'Yes' or 'No'. And then walk through step by step to be sure we answer correctly.\n\n" }} +{%- endif %} + diff --git a/checkpoint-498/config.json b/checkpoint-498/config.json new file mode 100644 index 0000000..ecfcdd0 --- /dev/null +++ b/checkpoint-498/config.json @@ -0,0 +1,60 @@ +{ + "architectures": [ + "Gemma2ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "attn_logit_softcapping": 50.0, + "bos_token_id": 2, + "cache_implementation": "hybrid", + "dtype": "float32", + "eos_token_id": 107, + "final_logit_softcapping": 30.0, + "head_dim": 256, + "hidden_act": "gelu_pytorch_tanh", + "hidden_activation": "gelu_pytorch_tanh", + "hidden_size": 2304, + "initializer_range": 0.02, + "intermediate_size": 9216, + "layer_types": [ + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention" + ], + "max_position_embeddings": 8192, + "model_type": "gemma2", + "num_attention_heads": 8, + "num_hidden_layers": 26, + "num_key_value_heads": 4, + "pad_token_id": 0, + "query_pre_attn_scalar": 224, + "rms_norm_eps": 1e-06, + "rope_theta": 10000.0, + "sliding_window": 4096, + "transformers_version": "4.57.1", + "use_cache": false, + "vocab_size": 256000 +} diff --git a/checkpoint-498/generation_config.json b/checkpoint-498/generation_config.json new file mode 100644 index 0000000..4700366 --- /dev/null +++ b/checkpoint-498/generation_config.json @@ -0,0 +1,11 @@ +{ + "_from_model_config": true, + "bos_token_id": 2, + "cache_implementation": "hybrid", + "eos_token_id": [ + 107, + 1 + ], + "pad_token_id": 0, + "transformers_version": "4.57.1" +} diff --git a/checkpoint-498/model-00001-of-00003.safetensors b/checkpoint-498/model-00001-of-00003.safetensors new file mode 100644 index 0000000..b03663e --- /dev/null +++ b/checkpoint-498/model-00001-of-00003.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:27f3efcf7fa387005a2447be0b4ae55681e84d5d0385c0be339b8e43b61ecd58 +size 4992576136 diff --git a/checkpoint-498/model-00002-of-00003.safetensors b/checkpoint-498/model-00002-of-00003.safetensors new file mode 100644 index 0000000..8d018f9 --- /dev/null +++ b/checkpoint-498/model-00002-of-00003.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f49c8916c976384e6cf23c4dfebbf04618e1741ca8838c69a4b3b55267b3b65c +size 4983443424 diff --git a/checkpoint-498/model-00003-of-00003.safetensors b/checkpoint-498/model-00003-of-00003.safetensors new file mode 100644 index 0000000..7874829 --- /dev/null +++ b/checkpoint-498/model-00003-of-00003.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bcb13e57128059be2ef3f6db700237dec8ad73862fdcf2665b0cc904b48b23c0 +size 481381384 diff --git a/checkpoint-498/model.safetensors.index.json b/checkpoint-498/model.safetensors.index.json new file mode 100644 index 0000000..1a43cf8 --- /dev/null +++ b/checkpoint-498/model.safetensors.index.json @@ -0,0 +1,296 @@ +{ + "metadata": { + "total_parameters": 2614341888, + "total_size": 10457367552 + }, + "weight_map": { + "model.embed_tokens.weight": "model-00001-of-00003.safetensors", + "model.layers.0.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.0.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.0.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.0.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.0.pre_feedforward_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.1.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.1.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.1.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.1.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.1.pre_feedforward_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.10.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.10.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.10.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.10.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.10.pre_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.11.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.11.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.11.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.11.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.11.pre_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.12.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.12.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.12.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.12.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.12.pre_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.13.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.13.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.13.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.13.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.13.pre_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.14.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.14.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.14.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.14.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.14.pre_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.15.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.15.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.15.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.15.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.15.pre_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.16.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.16.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.16.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.16.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.16.pre_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.17.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.17.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.17.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.17.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.17.pre_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.18.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.18.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.18.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.18.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.18.pre_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.19.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.19.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.19.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.19.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.19.pre_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.2.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.2.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.2.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.2.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.2.pre_feedforward_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.20.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.20.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.20.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.20.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.20.pre_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.21.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.21.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.21.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.21.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.21.pre_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.22.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.22.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.22.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.22.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.22.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.22.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.22.pre_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.23.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.23.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.23.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.23.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.23.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.23.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.23.pre_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.23.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.23.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.23.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.23.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.24.input_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.24.mlp.down_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.24.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.24.mlp.up_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.24.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.24.pre_feedforward_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.24.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.24.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.24.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.24.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.25.input_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.25.mlp.down_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.25.mlp.up_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.25.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.25.pre_feedforward_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.3.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.3.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.3.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.3.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.3.pre_feedforward_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.4.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.4.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.4.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.4.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.4.pre_feedforward_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.5.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.5.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.5.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.5.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.5.pre_feedforward_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.6.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.6.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.6.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.6.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.6.pre_feedforward_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.7.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.7.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.7.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.7.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.7.pre_feedforward_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.8.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.8.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.8.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.8.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.8.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.8.pre_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.9.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.9.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.9.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.9.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.9.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.9.pre_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.9.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.9.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.9.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.norm.weight": "model-00003-of-00003.safetensors" + } +} diff --git a/checkpoint-498/optimizer.pt b/checkpoint-498/optimizer.pt new file mode 100644 index 0000000..bf7acb7 --- /dev/null +++ b/checkpoint-498/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a0778ad4e60fc37022ea892da860654bacaa543bb3c603595bb0e74551f2e324 +size 20914991879 diff --git a/checkpoint-498/rng_state.pth b/checkpoint-498/rng_state.pth new file mode 100644 index 0000000..1feba1a --- /dev/null +++ b/checkpoint-498/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:098b29492211804ab324a36f37466821d948280bb74fce4ba895c03f13ecd878 +size 14645 diff --git a/checkpoint-498/scheduler.pt b/checkpoint-498/scheduler.pt new file mode 100644 index 0000000..0210aae --- /dev/null +++ b/checkpoint-498/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1bb0f8caa33fed2c1de44d000c78a6e40fc301c49dd2a1b8e9ab8bb30c5f3dfd +size 1465 diff --git a/checkpoint-498/special_tokens_map.json b/checkpoint-498/special_tokens_map.json new file mode 100644 index 0000000..60a73bc --- /dev/null +++ b/checkpoint-498/special_tokens_map.json @@ -0,0 +1,30 @@ +{ + "bos_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "unk_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + } +} diff --git a/checkpoint-498/tokenizer.json b/checkpoint-498/tokenizer.json new file mode 100644 index 0000000..a4a305d --- /dev/null +++ b/checkpoint-498/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5f7eee611703c5ce5d1eee32d9cdcfe465647b8aff0c1dfb3bed7ad7dbb05060 +size 34362873 diff --git a/checkpoint-498/tokenizer.model b/checkpoint-498/tokenizer.model new file mode 100644 index 0000000..796efe9 --- /dev/null +++ b/checkpoint-498/tokenizer.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:61a7b147390c64585d6c3543dd6fc636906c9af3865a5548f27f31aee1d4c8e2 +size 4241003 diff --git a/checkpoint-498/tokenizer_config.json b/checkpoint-498/tokenizer_config.json new file mode 100644 index 0000000..789c0ca --- /dev/null +++ b/checkpoint-498/tokenizer_config.json @@ -0,0 +1,2011 @@ +{ + "add_bos_token": true, + "add_eos_token": false, + "added_tokens_decoder": { + "0": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "1": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "2": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "3": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "4": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "5": { + "content": "<2mass>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "6": { + "content": "[@BOS@]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "7": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "8": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "9": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "10": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "11": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "12": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "13": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "14": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "15": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "16": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "17": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "18": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "19": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "20": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "21": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "22": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "23": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "24": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "25": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "26": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "27": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "28": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "29": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "30": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "31": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "32": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "33": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "34": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "35": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "36": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "37": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "38": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "39": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "40": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "41": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "42": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "43": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "44": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "45": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "46": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "47": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "48": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "49": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "50": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "51": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "52": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "53": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "54": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "55": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "56": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "57": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "58": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "59": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "60": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "61": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "62": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "63": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "64": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "65": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "66": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "67": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "68": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "69": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "70": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "71": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "72": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "73": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "74": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "75": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "76": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "77": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "78": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "79": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "80": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "81": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "82": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "83": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "84": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "85": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "86": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "87": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "88": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "89": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "90": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "91": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "92": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "93": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "94": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "95": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "96": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "97": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "98": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "99": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "100": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "101": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "102": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "103": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "104": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "105": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "106": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "107": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "108": { + "content": "\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "109": { + "content": "\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "110": { + "content": "\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "111": { + "content": "\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "112": { + "content": "\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "113": { + "content": "\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "114": { + "content": "\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "115": { + "content": "\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "116": { + "content": "\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "117": { + "content": "\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "118": { + "content": "\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "119": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "120": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "121": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "122": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "123": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "124": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "125": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "126": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "127": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "128": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "129": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "130": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "131": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "132": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "133": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "134": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "135": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "136": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "137": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "138": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "139": { + "content": "▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "140": { + "content": "▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "141": { + "content": "▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "142": { + "content": "▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "143": { + "content": "▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "144": { + "content": "▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "145": { + "content": "▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "146": { + "content": "▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "147": { + "content": "▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "148": { + "content": "▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "149": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "150": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "152": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "153": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "154": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "155": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "156": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "157": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "158": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "159": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "160": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "161": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "162": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "163": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "164": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "165": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "166": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "167": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "168": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "169": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "170": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "172": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "173": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "174": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "175": { + "content": "
", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "171": { + "content": "
", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "176": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "177": { + "content": "
", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "178": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "179": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "180": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "181": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "182": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "183": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "184": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "185": { + "content": "

", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "186": { + "content": "

", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "187": { + "content": "

", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "188": { + "content": "

", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "189": { + "content": "

", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "190": { + "content": "
", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "191": { + "content": "
", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "192": { + "content": "
", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "193": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "194": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "195": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "196": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "197": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "198": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "199": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "200": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "201": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "202": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "203": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "204": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "205": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "206": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "207": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "208": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "209": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "210": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "211": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "212": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "213": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "214": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "215": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "216": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255968": { + "content": "[toxicity=0]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255969": { + "content": "\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255970": { + "content": "\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255971": { + "content": "\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255972": { + "content": "\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255973": { + "content": "\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255974": { + "content": "\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255975": { + "content": "\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255976": { + "content": "\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255977": { + "content": "\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255978": { + "content": "\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255979": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255980": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255981": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255982": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255983": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255984": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255985": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255986": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255987": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255988": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255989": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255990": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255991": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255992": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255993": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255994": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255995": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255996": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255997": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255998": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255999": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + } + }, + "bos_token": "", + "clean_up_tokenization_spaces": false, + "eos_token": "", + "extra_special_tokens": {}, + "model_max_length": 1000000000000000019884624838656, + "pad_token": "", + "padding_side": "right", + "sp_model_kwargs": {}, + "spaces_between_special_tokens": false, + "split_special_tokens": false, + "tokenizer_class": "GemmaTokenizer", + "unk_token": "", + "use_default_system_prompt": false +} diff --git a/checkpoint-498/trainer_state.json b/checkpoint-498/trainer_state.json new file mode 100644 index 0000000..73efbd3 --- /dev/null +++ b/checkpoint-498/trainer_state.json @@ -0,0 +1,727 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 2.0, + "eval_steps": 500, + "global_step": 498, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.02009545340366742, + "grad_norm": 101.4629135131836, + "learning_rate": 4.0000000000000003e-07, + "loss": 2.9256, + "step": 5 + }, + { + "epoch": 0.04019090680733484, + "grad_norm": 83.51904296875, + "learning_rate": 9.000000000000001e-07, + "loss": 2.7459, + "step": 10 + }, + { + "epoch": 0.06028636021100226, + "grad_norm": 25.10388946533203, + "learning_rate": 1.4000000000000001e-06, + "loss": 1.637, + "step": 15 + }, + { + "epoch": 0.08038181361466969, + "grad_norm": 13.024300575256348, + "learning_rate": 1.9000000000000002e-06, + "loss": 1.1459, + "step": 20 + }, + { + "epoch": 0.1004772670183371, + "grad_norm": 8.315956115722656, + "learning_rate": 2.4000000000000003e-06, + "loss": 0.7456, + "step": 25 + }, + { + "epoch": 0.12057272042200452, + "grad_norm": 8.930346488952637, + "learning_rate": 2.9e-06, + "loss": 0.5717, + "step": 30 + }, + { + "epoch": 0.14066817382567195, + "grad_norm": 7.356456756591797, + "learning_rate": 3.4000000000000005e-06, + "loss": 0.414, + "step": 35 + }, + { + "epoch": 0.16076362722933937, + "grad_norm": 5.648008346557617, + "learning_rate": 3.900000000000001e-06, + "loss": 0.2577, + "step": 40 + }, + { + "epoch": 0.1808590806330068, + "grad_norm": 6.329988956451416, + "learning_rate": 4.4e-06, + "loss": 0.2394, + "step": 45 + }, + { + "epoch": 0.2009545340366742, + "grad_norm": 5.1738080978393555, + "learning_rate": 4.9000000000000005e-06, + "loss": 0.1848, + "step": 50 + }, + { + "epoch": 0.22104998744034163, + "grad_norm": 4.86478853225708, + "learning_rate": 4.999016565957633e-06, + "loss": 0.2195, + "step": 55 + }, + { + "epoch": 0.24114544084400905, + "grad_norm": 4.442976474761963, + "learning_rate": 4.995022691166621e-06, + "loss": 0.1582, + "step": 60 + }, + { + "epoch": 0.26124089424767644, + "grad_norm": 3.704345941543579, + "learning_rate": 4.987961816680493e-06, + "loss": 0.183, + "step": 65 + }, + { + "epoch": 0.2813363476513439, + "grad_norm": 3.772160768508911, + "learning_rate": 4.977842622059546e-06, + "loss": 0.164, + "step": 70 + }, + { + "epoch": 0.30143180105501133, + "grad_norm": 3.2532081604003906, + "learning_rate": 4.96467754629559e-06, + "loss": 0.1781, + "step": 75 + }, + { + "epoch": 0.32152725445867875, + "grad_norm": 2.893615484237671, + "learning_rate": 4.948482772521346e-06, + "loss": 0.1288, + "step": 80 + }, + { + "epoch": 0.34162270786234616, + "grad_norm": 4.654979705810547, + "learning_rate": 4.929278208117378e-06, + "loss": 0.1444, + "step": 85 + }, + { + "epoch": 0.3617181612660136, + "grad_norm": 2.8989882469177246, + "learning_rate": 4.907087460240999e-06, + "loss": 0.1151, + "step": 90 + }, + { + "epoch": 0.381813614669681, + "grad_norm": 3.7547988891601562, + "learning_rate": 4.881937806807241e-06, + "loss": 0.1197, + "step": 95 + }, + { + "epoch": 0.4019090680733484, + "grad_norm": 2.273831605911255, + "learning_rate": 4.8538601629575525e-06, + "loss": 0.0966, + "step": 100 + }, + { + "epoch": 0.42200452147701584, + "grad_norm": 2.4642996788024902, + "learning_rate": 4.822889043057446e-06, + "loss": 0.1088, + "step": 105 + }, + { + "epoch": 0.44209997488068326, + "grad_norm": 3.29301381111145, + "learning_rate": 4.789062518269812e-06, + "loss": 0.1163, + "step": 110 + }, + { + "epoch": 0.4621954282843507, + "grad_norm": 2.45603084564209, + "learning_rate": 4.752422169756048e-06, + "loss": 0.1151, + "step": 115 + }, + { + "epoch": 0.4822908816880181, + "grad_norm": 3.199894666671753, + "learning_rate": 4.713013037562531e-06, + "loss": 0.098, + "step": 120 + }, + { + "epoch": 0.5023863350916855, + "grad_norm": 1.7128061056137085, + "learning_rate": 4.670883565255264e-06, + "loss": 0.0968, + "step": 125 + }, + { + "epoch": 0.5224817884953529, + "grad_norm": 2.645235776901245, + "learning_rate": 4.6260855403707625e-06, + "loss": 0.1068, + "step": 130 + }, + { + "epoch": 0.5425772418990203, + "grad_norm": 2.3140530586242676, + "learning_rate": 4.578674030756364e-06, + "loss": 0.0914, + "step": 135 + }, + { + "epoch": 0.5626726953026878, + "grad_norm": 1.8588067293167114, + "learning_rate": 4.528707316878233e-06, + "loss": 0.0995, + "step": 140 + }, + { + "epoch": 0.5827681487063552, + "grad_norm": 1.9172967672348022, + "learning_rate": 4.476246820180259e-06, + "loss": 0.08, + "step": 145 + }, + { + "epoch": 0.6028636021100227, + "grad_norm": 2.9082584381103516, + "learning_rate": 4.421357027581921e-06, + "loss": 0.1066, + "step": 150 + }, + { + "epoch": 0.62295905551369, + "grad_norm": 3.6944031715393066, + "learning_rate": 4.364105412207914e-06, + "loss": 0.1124, + "step": 155 + }, + { + "epoch": 0.6430545089173575, + "grad_norm": 1.5622600317001343, + "learning_rate": 4.304562350447008e-06, + "loss": 0.0856, + "step": 160 + }, + { + "epoch": 0.6631499623210249, + "grad_norm": 2.388239860534668, + "learning_rate": 4.242801035442059e-06, + "loss": 0.0766, + "step": 165 + }, + { + "epoch": 0.6832454157246923, + "grad_norm": 1.7182718515396118, + "learning_rate": 4.178897387117547e-06, + "loss": 0.104, + "step": 170 + }, + { + "epoch": 0.7033408691283597, + "grad_norm": 2.2367119789123535, + "learning_rate": 4.11292995885522e-06, + "loss": 0.0737, + "step": 175 + }, + { + "epoch": 0.7234363225320272, + "grad_norm": 2.007200241088867, + "learning_rate": 4.044979840932567e-06, + "loss": 0.0893, + "step": 180 + }, + { + "epoch": 0.7435317759356945, + "grad_norm": 2.6910905838012695, + "learning_rate": 3.975130560842821e-06, + "loss": 0.0944, + "step": 185 + }, + { + "epoch": 0.763627229339362, + "grad_norm": 2.042320489883423, + "learning_rate": 3.9034679806190204e-06, + "loss": 0.0843, + "step": 190 + }, + { + "epoch": 0.7837226827430294, + "grad_norm": 2.700977087020874, + "learning_rate": 3.830080191288342e-06, + "loss": 0.0669, + "step": 195 + }, + { + "epoch": 0.8038181361466968, + "grad_norm": 3.0307390689849854, + "learning_rate": 3.755057404586455e-06, + "loss": 0.0955, + "step": 200 + }, + { + "epoch": 0.8239135895503642, + "grad_norm": 1.11714506149292, + "learning_rate": 4.0000000000000003e-07, + "loss": 0.0956, + "step": 205 + }, + { + "epoch": 0.8440090429540317, + "grad_norm": 1.5246130228042603, + "learning_rate": 9.000000000000001e-07, + "loss": 0.0755, + "step": 210 + }, + { + "epoch": 0.864104496357699, + "grad_norm": 2.0117733478546143, + "learning_rate": 1.4000000000000001e-06, + "loss": 0.0894, + "step": 215 + }, + { + "epoch": 0.8841999497613665, + "grad_norm": 2.038092613220215, + "learning_rate": 1.9000000000000002e-06, + "loss": 0.074, + "step": 220 + }, + { + "epoch": 0.9042954031650339, + "grad_norm": 1.607172966003418, + "learning_rate": 2.4000000000000003e-06, + "loss": 0.0612, + "step": 225 + }, + { + "epoch": 0.9243908565687013, + "grad_norm": 1.643001914024353, + "learning_rate": 2.9e-06, + "loss": 0.0967, + "step": 230 + }, + { + "epoch": 0.9444863099723687, + "grad_norm": 1.5271589756011963, + "learning_rate": 3.4000000000000005e-06, + "loss": 0.1008, + "step": 235 + }, + { + "epoch": 0.9645817633760362, + "grad_norm": 2.2744393348693848, + "learning_rate": 3.900000000000001e-06, + "loss": 0.0846, + "step": 240 + }, + { + "epoch": 0.9846772167797035, + "grad_norm": 1.2268872261047363, + "learning_rate": 4.4e-06, + "loss": 0.0889, + "step": 245 + }, + { + "epoch": 1.0040190906807336, + "grad_norm": 1.971510648727417, + "learning_rate": 4.9000000000000005e-06, + "loss": 0.1021, + "step": 250 + }, + { + "epoch": 1.0241145440844008, + "grad_norm": 0.6695238947868347, + "learning_rate": 4.999016565957633e-06, + "loss": 0.057, + "step": 255 + }, + { + "epoch": 1.0442099974880683, + "grad_norm": 3.379134178161621, + "learning_rate": 4.995022691166621e-06, + "loss": 0.0695, + "step": 260 + }, + { + "epoch": 1.0643054508917358, + "grad_norm": 2.573274850845337, + "learning_rate": 4.987961816680493e-06, + "loss": 0.0713, + "step": 265 + }, + { + "epoch": 1.0844009042954033, + "grad_norm": 1.3487026691436768, + "learning_rate": 4.977842622059546e-06, + "loss": 0.0704, + "step": 270 + }, + { + "epoch": 1.1044963576990705, + "grad_norm": 1.7525826692581177, + "learning_rate": 4.96467754629559e-06, + "loss": 0.0615, + "step": 275 + }, + { + "epoch": 1.124591811102738, + "grad_norm": 1.1195745468139648, + "learning_rate": 4.948482772521346e-06, + "loss": 0.0616, + "step": 280 + }, + { + "epoch": 1.1446872645064055, + "grad_norm": 2.203016996383667, + "learning_rate": 4.929278208117378e-06, + "loss": 0.0627, + "step": 285 + }, + { + "epoch": 1.164782717910073, + "grad_norm": 3.2196788787841797, + "learning_rate": 4.907087460240999e-06, + "loss": 0.0707, + "step": 290 + }, + { + "epoch": 1.1848781713137402, + "grad_norm": 2.4914467334747314, + "learning_rate": 4.881937806807241e-06, + "loss": 0.0758, + "step": 295 + }, + { + "epoch": 1.2049736247174077, + "grad_norm": 2.223809242248535, + "learning_rate": 4.8538601629575525e-06, + "loss": 0.0698, + "step": 300 + }, + { + "epoch": 1.2250690781210751, + "grad_norm": 1.5360513925552368, + "learning_rate": 4.822889043057446e-06, + "loss": 0.0593, + "step": 305 + }, + { + "epoch": 1.2451645315247426, + "grad_norm": 1.8444727659225464, + "learning_rate": 4.789062518269812e-06, + "loss": 0.0635, + "step": 310 + }, + { + "epoch": 1.2652599849284099, + "grad_norm": 1.5122318267822266, + "learning_rate": 4.752422169756048e-06, + "loss": 0.0737, + "step": 315 + }, + { + "epoch": 1.2853554383320773, + "grad_norm": 2.0612282752990723, + "learning_rate": 4.713013037562531e-06, + "loss": 0.065, + "step": 320 + }, + { + "epoch": 1.3054508917357448, + "grad_norm": 1.4917356967926025, + "learning_rate": 4.670883565255264e-06, + "loss": 0.0563, + "step": 325 + }, + { + "epoch": 1.3255463451394123, + "grad_norm": 1.861807942390442, + "learning_rate": 4.6260855403707625e-06, + "loss": 0.0654, + "step": 330 + }, + { + "epoch": 1.3456417985430797, + "grad_norm": 2.3603355884552, + "learning_rate": 4.578674030756364e-06, + "loss": 0.0616, + "step": 335 + }, + { + "epoch": 1.365737251946747, + "grad_norm": 1.4312632083892822, + "learning_rate": 4.528707316878233e-06, + "loss": 0.0608, + "step": 340 + }, + { + "epoch": 1.3858327053504145, + "grad_norm": 3.3819150924682617, + "learning_rate": 4.476246820180259e-06, + "loss": 0.0573, + "step": 345 + }, + { + "epoch": 1.405928158754082, + "grad_norm": 2.076028347015381, + "learning_rate": 4.421357027581921e-06, + "loss": 0.0648, + "step": 350 + }, + { + "epoch": 1.4260236121577492, + "grad_norm": 2.0433411598205566, + "learning_rate": 4.364105412207914e-06, + "loss": 0.0679, + "step": 355 + }, + { + "epoch": 1.4461190655614167, + "grad_norm": 1.695204496383667, + "learning_rate": 4.304562350447008e-06, + "loss": 0.0655, + "step": 360 + }, + { + "epoch": 1.4662145189650841, + "grad_norm": 0.7300482988357544, + "learning_rate": 4.242801035442059e-06, + "loss": 0.0662, + "step": 365 + }, + { + "epoch": 1.4863099723687516, + "grad_norm": 1.8220871686935425, + "learning_rate": 4.178897387117547e-06, + "loss": 0.0659, + "step": 370 + }, + { + "epoch": 1.506405425772419, + "grad_norm": 1.4738329648971558, + "learning_rate": 4.11292995885522e-06, + "loss": 0.0605, + "step": 375 + }, + { + "epoch": 1.5265008791760866, + "grad_norm": 1.562551736831665, + "learning_rate": 4.044979840932567e-06, + "loss": 0.0741, + "step": 380 + }, + { + "epoch": 1.5465963325797538, + "grad_norm": 1.5520535707473755, + "learning_rate": 3.975130560842821e-06, + "loss": 0.058, + "step": 385 + }, + { + "epoch": 1.5666917859834213, + "grad_norm": 1.29172682762146, + "learning_rate": 3.9034679806190204e-06, + "loss": 0.0584, + "step": 390 + }, + { + "epoch": 1.5867872393870885, + "grad_norm": 1.9488056898117065, + "learning_rate": 3.830080191288342e-06, + "loss": 0.0582, + "step": 395 + }, + { + "epoch": 1.606882692790756, + "grad_norm": 1.6518131494522095, + "learning_rate": 3.755057404586455e-06, + "loss": 0.0634, + "step": 400 + }, + { + "epoch": 1.6269781461944235, + "grad_norm": 1.5398550033569336, + "learning_rate": 3.6784918420649952e-06, + "loss": 0.0525, + "step": 405 + }, + { + "epoch": 1.647073599598091, + "grad_norm": 1.6156771183013916, + "learning_rate": 3.600477621728473e-06, + "loss": 0.0645, + "step": 410 + }, + { + "epoch": 1.6671690530017584, + "grad_norm": 1.564670205116272, + "learning_rate": 3.521110642339991e-06, + "loss": 0.0526, + "step": 415 + }, + { + "epoch": 1.687264506405426, + "grad_norm": 1.2277319431304932, + "learning_rate": 3.4404884655379433e-06, + "loss": 0.0616, + "step": 420 + }, + { + "epoch": 1.7073599598090932, + "grad_norm": 1.1757029294967651, + "learning_rate": 3.358710195908653e-06, + "loss": 0.0537, + "step": 425 + }, + { + "epoch": 1.7274554132127606, + "grad_norm": 1.0405977964401245, + "learning_rate": 3.27587635916233e-06, + "loss": 0.0619, + "step": 430 + }, + { + "epoch": 1.747550866616428, + "grad_norm": 1.5975450277328491, + "learning_rate": 3.1920887785621233e-06, + "loss": 0.0603, + "step": 435 + }, + { + "epoch": 1.7676463200200954, + "grad_norm": 1.7237313985824585, + "learning_rate": 3.10745044975816e-06, + "loss": 0.0594, + "step": 440 + }, + { + "epoch": 1.7877417734237628, + "grad_norm": 1.7574973106384277, + "learning_rate": 3.022065414180425e-06, + "loss": 0.0608, + "step": 445 + }, + { + "epoch": 1.8078372268274303, + "grad_norm": 1.450028419494629, + "learning_rate": 2.9360386311461276e-06, + "loss": 0.0505, + "step": 450 + }, + { + "epoch": 1.8279326802310978, + "grad_norm": 1.0331708192825317, + "learning_rate": 2.849475848838749e-06, + "loss": 0.0497, + "step": 455 + }, + { + "epoch": 1.8480281336347653, + "grad_norm": 1.5370794534683228, + "learning_rate": 2.762483474317387e-06, + "loss": 0.0625, + "step": 460 + }, + { + "epoch": 1.8681235870384325, + "grad_norm": 2.073357582092285, + "learning_rate": 2.6751684427161684e-06, + "loss": 0.0448, + "step": 465 + }, + { + "epoch": 1.8882190404421, + "grad_norm": 1.4379805326461792, + "learning_rate": 2.587638085794541e-06, + "loss": 0.0588, + "step": 470 + }, + { + "epoch": 1.9083144938457672, + "grad_norm": 1.8600538969039917, + "learning_rate": 2.5e-06, + "loss": 0.0511, + "step": 475 + }, + { + "epoch": 1.9284099472494347, + "grad_norm": 1.2249418497085571, + "learning_rate": 2.41236191420546e-06, + "loss": 0.0497, + "step": 480 + }, + { + "epoch": 1.9485054006531022, + "grad_norm": 0.8812041282653809, + "learning_rate": 2.3248315572838316e-06, + "loss": 0.0498, + "step": 485 + }, + { + "epoch": 1.9686008540567697, + "grad_norm": 0.8966412544250488, + "learning_rate": 2.237516525682614e-06, + "loss": 0.0459, + "step": 490 + }, + { + "epoch": 1.9886963074604371, + "grad_norm": 1.238738775253296, + "learning_rate": 2.1505241511612522e-06, + "loss": 0.052, + "step": 495 + } + ], + "logging_steps": 5, + "max_steps": 498, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 200, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 1.28494219771102e+18, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/checkpoint-498/training_args.bin b/checkpoint-498/training_args.bin new file mode 100644 index 0000000..e9a2513 --- /dev/null +++ b/checkpoint-498/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5acbccb5459f9c0d139eb9e75792351bde8d66e4f32093b31c72148828bf3e33 +size 6225 diff --git a/config.json b/config.json new file mode 100644 index 0000000..ecfcdd0 --- /dev/null +++ b/config.json @@ -0,0 +1,60 @@ +{ + "architectures": [ + "Gemma2ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "attn_logit_softcapping": 50.0, + "bos_token_id": 2, + "cache_implementation": "hybrid", + "dtype": "float32", + "eos_token_id": 107, + "final_logit_softcapping": 30.0, + "head_dim": 256, + "hidden_act": "gelu_pytorch_tanh", + "hidden_activation": "gelu_pytorch_tanh", + "hidden_size": 2304, + "initializer_range": 0.02, + "intermediate_size": 9216, + "layer_types": [ + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention" + ], + "max_position_embeddings": 8192, + "model_type": "gemma2", + "num_attention_heads": 8, + "num_hidden_layers": 26, + "num_key_value_heads": 4, + "pad_token_id": 0, + "query_pre_attn_scalar": 224, + "rms_norm_eps": 1e-06, + "rope_theta": 10000.0, + "sliding_window": 4096, + "transformers_version": "4.57.1", + "use_cache": false, + "vocab_size": 256000 +} diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..4700366 --- /dev/null +++ b/generation_config.json @@ -0,0 +1,11 @@ +{ + "_from_model_config": true, + "bos_token_id": 2, + "cache_implementation": "hybrid", + "eos_token_id": [ + 107, + 1 + ], + "pad_token_id": 0, + "transformers_version": "4.57.1" +} diff --git a/model-00001-of-00003.safetensors b/model-00001-of-00003.safetensors new file mode 100644 index 0000000..b03663e --- /dev/null +++ b/model-00001-of-00003.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:27f3efcf7fa387005a2447be0b4ae55681e84d5d0385c0be339b8e43b61ecd58 +size 4992576136 diff --git a/model-00002-of-00003.safetensors b/model-00002-of-00003.safetensors new file mode 100644 index 0000000..8d018f9 --- /dev/null +++ b/model-00002-of-00003.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f49c8916c976384e6cf23c4dfebbf04618e1741ca8838c69a4b3b55267b3b65c +size 4983443424 diff --git a/model-00003-of-00003.safetensors b/model-00003-of-00003.safetensors new file mode 100644 index 0000000..7874829 --- /dev/null +++ b/model-00003-of-00003.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bcb13e57128059be2ef3f6db700237dec8ad73862fdcf2665b0cc904b48b23c0 +size 481381384 diff --git a/model.safetensors.index.json b/model.safetensors.index.json new file mode 100644 index 0000000..1a43cf8 --- /dev/null +++ b/model.safetensors.index.json @@ -0,0 +1,296 @@ +{ + "metadata": { + "total_parameters": 2614341888, + "total_size": 10457367552 + }, + "weight_map": { + "model.embed_tokens.weight": "model-00001-of-00003.safetensors", + "model.layers.0.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.0.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.0.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.0.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.0.pre_feedforward_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.1.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.1.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.1.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.1.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.1.pre_feedforward_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.10.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.10.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.10.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.10.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.10.pre_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.11.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.11.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.11.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.11.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.11.pre_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.12.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.12.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.12.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.12.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.12.pre_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.13.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.13.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.13.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.13.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.13.pre_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.14.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.14.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.14.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.14.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.14.pre_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.15.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.15.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.15.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.15.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.15.pre_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.16.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.16.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.16.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.16.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.16.pre_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.17.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.17.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.17.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.17.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.17.pre_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.18.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.18.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.18.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.18.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.18.pre_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.19.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.19.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.19.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.19.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.19.pre_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.2.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.2.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.2.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.2.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.2.pre_feedforward_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.20.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.20.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.20.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.20.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.20.pre_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.21.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.21.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.21.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.21.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.21.pre_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.22.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.22.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.22.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.22.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.22.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.22.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.22.pre_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.23.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.23.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.23.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.23.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.23.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.23.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.23.pre_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.23.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.23.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.23.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.23.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.24.input_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.24.mlp.down_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.24.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.24.mlp.up_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.24.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.24.pre_feedforward_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.24.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.24.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.24.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.24.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.25.input_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.25.mlp.down_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.25.mlp.up_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.25.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.25.pre_feedforward_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.3.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.3.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.3.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.3.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.3.pre_feedforward_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.4.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.4.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.4.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.4.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.4.pre_feedforward_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.5.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.5.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.5.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.5.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.5.pre_feedforward_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.6.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.6.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.6.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.6.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.6.pre_feedforward_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.7.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.7.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.7.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.7.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.7.pre_feedforward_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.8.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.8.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.8.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.8.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.8.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.8.pre_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.9.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.9.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.9.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.9.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.9.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.9.pre_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.9.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.9.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.9.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.norm.weight": "model-00003-of-00003.safetensors" + } +} diff --git a/special_tokens_map.json b/special_tokens_map.json new file mode 100644 index 0000000..60a73bc --- /dev/null +++ b/special_tokens_map.json @@ -0,0 +1,30 @@ +{ + "bos_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "unk_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + } +} diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..a4a305d --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5f7eee611703c5ce5d1eee32d9cdcfe465647b8aff0c1dfb3bed7ad7dbb05060 +size 34362873 diff --git a/tokenizer.model b/tokenizer.model new file mode 100644 index 0000000..796efe9 --- /dev/null +++ b/tokenizer.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:61a7b147390c64585d6c3543dd6fc636906c9af3865a5548f27f31aee1d4c8e2 +size 4241003 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..789c0ca --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,2011 @@ +{ + "add_bos_token": true, + "add_eos_token": false, + "added_tokens_decoder": { + "0": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "1": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "2": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "3": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "4": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "5": { + "content": "<2mass>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "6": { + "content": "[@BOS@]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "7": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "8": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "9": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "10": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "11": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "12": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "13": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "14": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "15": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "16": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "17": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "18": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "19": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "20": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "21": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "22": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "23": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "24": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "25": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "26": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "27": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "28": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "29": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "30": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "31": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "32": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "33": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "34": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "35": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "36": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "37": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "38": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "39": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "40": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "41": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "42": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "43": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "44": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "45": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "46": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "47": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "48": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "49": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "50": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "51": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "52": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "53": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "54": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "55": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "56": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "57": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "58": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "59": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "60": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "61": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "62": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "63": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "64": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "65": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "66": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "67": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "68": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "69": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "70": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "71": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "72": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "73": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "74": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "75": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "76": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "77": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "78": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "79": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "80": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "81": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "82": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "83": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "84": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "85": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "86": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "87": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "88": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "89": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "90": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "91": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "92": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "93": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "94": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "95": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "96": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "97": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "98": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "99": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "100": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "101": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "102": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "103": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "104": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "105": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "106": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "107": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "108": { + "content": "\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "109": { + "content": "\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "110": { + "content": "\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "111": { + "content": "\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "112": { + "content": "\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "113": { + "content": "\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "114": { + "content": "\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "115": { + "content": "\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "116": { + "content": "\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "117": { + "content": "\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "118": { + "content": "\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "119": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "120": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "121": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "122": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "123": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "124": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "125": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "126": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "127": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "128": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "129": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "130": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "131": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "132": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "133": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "134": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "135": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "136": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "137": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "138": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "139": { + "content": "▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "140": { + "content": "▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "141": { + "content": "▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "142": { + "content": "▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "143": { + "content": "▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "144": { + "content": "▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "145": { + "content": "▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "146": { + "content": "▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "147": { + "content": "▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "148": { + "content": "▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "149": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "150": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "152": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "153": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "154": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "155": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "156": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "157": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "158": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "159": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "160": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "161": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "162": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "163": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "164": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "165": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "166": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "167": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "168": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "169": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "170": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "172": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "173": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "174": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "175": { + "content": "
", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "171": { + "content": "
", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "176": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "177": { + "content": "
", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "178": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "179": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "180": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "181": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "182": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "183": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "184": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "185": { + "content": "

", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "186": { + "content": "

", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "187": { + "content": "

", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "188": { + "content": "

", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "189": { + "content": "

", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "190": { + "content": "
", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "191": { + "content": "
", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "192": { + "content": "
", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "193": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "194": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "195": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "196": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "197": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "198": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "199": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "200": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "201": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "202": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "203": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "204": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "205": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "206": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "207": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "208": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "209": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "210": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "211": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "212": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "213": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "214": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "215": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "216": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255968": { + "content": "[toxicity=0]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255969": { + "content": "\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255970": { + "content": "\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255971": { + "content": "\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255972": { + "content": "\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255973": { + "content": "\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255974": { + "content": "\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255975": { + "content": "\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255976": { + "content": "\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255977": { + "content": "\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255978": { + "content": "\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255979": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255980": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255981": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255982": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255983": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255984": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255985": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255986": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255987": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255988": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255989": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255990": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255991": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255992": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255993": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255994": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255995": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255996": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255997": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255998": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255999": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + } + }, + "bos_token": "", + "clean_up_tokenization_spaces": false, + "eos_token": "", + "extra_special_tokens": {}, + "model_max_length": 1000000000000000019884624838656, + "pad_token": "", + "padding_side": "right", + "sp_model_kwargs": {}, + "spaces_between_special_tokens": false, + "split_special_tokens": false, + "tokenizer_class": "GemmaTokenizer", + "unk_token": "", + "use_default_system_prompt": false +} diff --git a/train_results.json b/train_results.json new file mode 100644 index 0000000..820de62 --- /dev/null +++ b/train_results.json @@ -0,0 +1,8 @@ +{ + "epoch": 2.0, + "total_flos": 1.28494219771102e+18, + "train_loss": 0.03900693407858232, + "train_runtime": 6158.7829, + "train_samples_per_second": 5.17, + "train_steps_per_second": 0.081 +} \ No newline at end of file diff --git a/trainer_log.jsonl b/trainer_log.jsonl new file mode 100644 index 0000000..31fd0fd --- /dev/null +++ b/trainer_log.jsonl @@ -0,0 +1,60 @@ +{"current_steps": 205, "total_steps": 498, "loss": 0.0956, "lr": 4.0000000000000003e-07, "epoch": 0.8239135895503642, "percentage": 41.16, "elapsed_time": "0:01:41", "remaining_time": "0:02:24"} +{"current_steps": 210, "total_steps": 498, "loss": 0.0755, "lr": 9.000000000000001e-07, "epoch": 0.8440090429540317, "percentage": 42.17, "elapsed_time": "0:03:21", "remaining_time": "0:04:36"} +{"current_steps": 215, "total_steps": 498, "loss": 0.0894, "lr": 1.4000000000000001e-06, "epoch": 0.864104496357699, "percentage": 43.17, "elapsed_time": "0:05:03", "remaining_time": "0:06:38"} +{"current_steps": 220, "total_steps": 498, "loss": 0.074, "lr": 1.9000000000000002e-06, "epoch": 0.8841999497613665, "percentage": 44.18, "elapsed_time": "0:06:43", "remaining_time": "0:08:29"} +{"current_steps": 225, "total_steps": 498, "loss": 0.0612, "lr": 2.4000000000000003e-06, "epoch": 0.9042954031650339, "percentage": 45.18, "elapsed_time": "0:08:26", "remaining_time": "0:10:14"} +{"current_steps": 230, "total_steps": 498, "loss": 0.0967, "lr": 2.9e-06, "epoch": 0.9243908565687013, "percentage": 46.18, "elapsed_time": "0:10:07", "remaining_time": "0:11:47"} +{"current_steps": 235, "total_steps": 498, "loss": 0.1008, "lr": 3.4000000000000005e-06, "epoch": 0.9444863099723687, "percentage": 47.19, "elapsed_time": "0:11:50", "remaining_time": "0:13:15"} +{"current_steps": 240, "total_steps": 498, "loss": 0.0846, "lr": 3.900000000000001e-06, "epoch": 0.9645817633760362, "percentage": 48.19, "elapsed_time": "0:13:32", "remaining_time": "0:14:33"} +{"current_steps": 245, "total_steps": 498, "loss": 0.0889, "lr": 4.4e-06, "epoch": 0.9846772167797035, "percentage": 49.2, "elapsed_time": "0:15:11", "remaining_time": "0:15:41"} +{"current_steps": 250, "total_steps": 498, "loss": 0.1021, "lr": 4.9000000000000005e-06, "epoch": 1.0040190906807336, "percentage": 50.2, "elapsed_time": "0:16:50", "remaining_time": "0:16:42"} +{"current_steps": 255, "total_steps": 498, "loss": 0.057, "lr": 4.999016565957633e-06, "epoch": 1.0241145440844008, "percentage": 51.2, "elapsed_time": "0:18:29", "remaining_time": "0:17:36"} +{"current_steps": 260, "total_steps": 498, "loss": 0.0695, "lr": 4.995022691166621e-06, "epoch": 1.0442099974880683, "percentage": 52.21, "elapsed_time": "0:20:08", "remaining_time": "0:18:25"} +{"current_steps": 265, "total_steps": 498, "loss": 0.0713, "lr": 4.987961816680493e-06, "epoch": 1.0643054508917358, "percentage": 53.21, "elapsed_time": "0:21:49", "remaining_time": "0:19:11"} +{"current_steps": 270, "total_steps": 498, "loss": 0.0704, "lr": 4.977842622059546e-06, "epoch": 1.0844009042954033, "percentage": 54.22, "elapsed_time": "0:23:30", "remaining_time": "0:19:50"} +{"current_steps": 275, "total_steps": 498, "loss": 0.0615, "lr": 4.96467754629559e-06, "epoch": 1.1044963576990705, "percentage": 55.22, "elapsed_time": "0:25:10", "remaining_time": "0:20:24"} +{"current_steps": 280, "total_steps": 498, "loss": 0.0616, "lr": 4.948482772521346e-06, "epoch": 1.124591811102738, "percentage": 56.22, "elapsed_time": "0:26:50", "remaining_time": "0:20:54"} +{"current_steps": 285, "total_steps": 498, "loss": 0.0627, "lr": 4.929278208117378e-06, "epoch": 1.1446872645064055, "percentage": 57.23, "elapsed_time": "0:28:31", "remaining_time": "0:21:19"} +{"current_steps": 290, "total_steps": 498, "loss": 0.0707, "lr": 4.907087460240999e-06, "epoch": 1.164782717910073, "percentage": 58.23, "elapsed_time": "0:30:11", "remaining_time": "0:21:39"} +{"current_steps": 295, "total_steps": 498, "loss": 0.0758, "lr": 4.881937806807241e-06, "epoch": 1.1848781713137402, "percentage": 59.24, "elapsed_time": "0:31:55", "remaining_time": "0:21:58"} +{"current_steps": 300, "total_steps": 498, "loss": 0.0698, "lr": 4.8538601629575525e-06, "epoch": 1.2049736247174077, "percentage": 60.24, "elapsed_time": "0:33:39", "remaining_time": "0:22:13"} +{"current_steps": 305, "total_steps": 498, "loss": 0.0593, "lr": 4.822889043057446e-06, "epoch": 1.2250690781210751, "percentage": 61.24, "elapsed_time": "0:35:22", "remaining_time": "0:22:23"} +{"current_steps": 310, "total_steps": 498, "loss": 0.0635, "lr": 4.789062518269812e-06, "epoch": 1.2451645315247426, "percentage": 62.25, "elapsed_time": "0:37:04", "remaining_time": "0:22:29"} +{"current_steps": 315, "total_steps": 498, "loss": 0.0737, "lr": 4.752422169756048e-06, "epoch": 1.2652599849284099, "percentage": 63.25, "elapsed_time": "0:38:46", "remaining_time": "0:22:31"} +{"current_steps": 320, "total_steps": 498, "loss": 0.065, "lr": 4.713013037562531e-06, "epoch": 1.2853554383320773, "percentage": 64.26, "elapsed_time": "0:40:24", "remaining_time": "0:22:28"} +{"current_steps": 325, "total_steps": 498, "loss": 0.0563, "lr": 4.670883565255264e-06, "epoch": 1.3054508917357448, "percentage": 65.26, "elapsed_time": "0:42:06", "remaining_time": "0:22:24"} +{"current_steps": 330, "total_steps": 498, "loss": 0.0654, "lr": 4.6260855403707625e-06, "epoch": 1.3255463451394123, "percentage": 66.27, "elapsed_time": "0:43:52", "remaining_time": "0:22:19"} +{"current_steps": 335, "total_steps": 498, "loss": 0.0616, "lr": 4.578674030756364e-06, "epoch": 1.3456417985430797, "percentage": 67.27, "elapsed_time": "0:45:34", "remaining_time": "0:22:10"} +{"current_steps": 340, "total_steps": 498, "loss": 0.0608, "lr": 4.528707316878233e-06, "epoch": 1.365737251946747, "percentage": 68.27, "elapsed_time": "0:47:13", "remaining_time": "0:21:56"} +{"current_steps": 345, "total_steps": 498, "loss": 0.0573, "lr": 4.476246820180259e-06, "epoch": 1.3858327053504145, "percentage": 69.28, "elapsed_time": "0:48:55", "remaining_time": "0:21:41"} +{"current_steps": 350, "total_steps": 498, "loss": 0.0648, "lr": 4.421357027581921e-06, "epoch": 1.405928158754082, "percentage": 70.28, "elapsed_time": "0:50:35", "remaining_time": "0:21:23"} +{"current_steps": 355, "total_steps": 498, "loss": 0.0679, "lr": 4.364105412207914e-06, "epoch": 1.4260236121577492, "percentage": 71.29, "elapsed_time": "0:52:17", "remaining_time": "0:21:03"} +{"current_steps": 360, "total_steps": 498, "loss": 0.0655, "lr": 4.304562350447008e-06, "epoch": 1.4461190655614167, "percentage": 72.29, "elapsed_time": "0:53:59", "remaining_time": "0:20:41"} +{"current_steps": 365, "total_steps": 498, "loss": 0.0662, "lr": 4.242801035442059e-06, "epoch": 1.4662145189650841, "percentage": 73.29, "elapsed_time": "0:55:41", "remaining_time": "0:20:17"} +{"current_steps": 370, "total_steps": 498, "loss": 0.0659, "lr": 4.178897387117547e-06, "epoch": 1.4863099723687516, "percentage": 74.3, "elapsed_time": "0:57:25", "remaining_time": "0:19:51"} +{"current_steps": 375, "total_steps": 498, "loss": 0.0605, "lr": 4.11292995885522e-06, "epoch": 1.506405425772419, "percentage": 75.3, "elapsed_time": "0:59:11", "remaining_time": "0:19:24"} +{"current_steps": 380, "total_steps": 498, "loss": 0.0741, "lr": 4.044979840932567e-06, "epoch": 1.5265008791760866, "percentage": 76.31, "elapsed_time": "1:00:56", "remaining_time": "0:18:55"} +{"current_steps": 385, "total_steps": 498, "loss": 0.058, "lr": 3.975130560842821e-06, "epoch": 1.5465963325797538, "percentage": 77.31, "elapsed_time": "1:02:38", "remaining_time": "0:18:23"} +{"current_steps": 390, "total_steps": 498, "loss": 0.0584, "lr": 3.9034679806190204e-06, "epoch": 1.5666917859834213, "percentage": 78.31, "elapsed_time": "1:04:18", "remaining_time": "0:17:48"} +{"current_steps": 395, "total_steps": 498, "loss": 0.0582, "lr": 3.830080191288342e-06, "epoch": 1.5867872393870885, "percentage": 79.32, "elapsed_time": "1:06:00", "remaining_time": "0:17:12"} +{"current_steps": 400, "total_steps": 498, "loss": 0.0634, "lr": 3.755057404586455e-06, "epoch": 1.606882692790756, "percentage": 80.32, "elapsed_time": "1:07:43", "remaining_time": "0:16:35"} +{"current_steps": 405, "total_steps": 498, "loss": 0.0525, "lr": 3.6784918420649952e-06, "epoch": 1.6269781461944235, "percentage": 81.33, "elapsed_time": "1:10:09", "remaining_time": "0:16:06"} +{"current_steps": 410, "total_steps": 498, "loss": 0.0645, "lr": 3.600477621728473e-06, "epoch": 1.647073599598091, "percentage": 82.33, "elapsed_time": "1:11:49", "remaining_time": "0:15:24"} +{"current_steps": 415, "total_steps": 498, "loss": 0.0526, "lr": 3.521110642339991e-06, "epoch": 1.6671690530017584, "percentage": 83.33, "elapsed_time": "1:13:32", "remaining_time": "0:14:42"} +{"current_steps": 420, "total_steps": 498, "loss": 0.0616, "lr": 3.4404884655379433e-06, "epoch": 1.687264506405426, "percentage": 84.34, "elapsed_time": "1:15:15", "remaining_time": "0:13:58"} +{"current_steps": 425, "total_steps": 498, "loss": 0.0537, "lr": 3.358710195908653e-06, "epoch": 1.7073599598090932, "percentage": 85.34, "elapsed_time": "1:16:58", "remaining_time": "0:13:13"} +{"current_steps": 430, "total_steps": 498, "loss": 0.0619, "lr": 3.27587635916233e-06, "epoch": 1.7274554132127606, "percentage": 86.35, "elapsed_time": "1:18:38", "remaining_time": "0:12:26"} +{"current_steps": 435, "total_steps": 498, "loss": 0.0603, "lr": 3.1920887785621233e-06, "epoch": 1.747550866616428, "percentage": 87.35, "elapsed_time": "1:20:20", "remaining_time": "0:11:38"} +{"current_steps": 440, "total_steps": 498, "loss": 0.0594, "lr": 3.10745044975816e-06, "epoch": 1.7676463200200954, "percentage": 88.35, "elapsed_time": "1:22:02", "remaining_time": "0:10:48"} +{"current_steps": 445, "total_steps": 498, "loss": 0.0608, "lr": 3.022065414180425e-06, "epoch": 1.7877417734237628, "percentage": 89.36, "elapsed_time": "1:23:44", "remaining_time": "0:09:58"} +{"current_steps": 450, "total_steps": 498, "loss": 0.0505, "lr": 2.9360386311461276e-06, "epoch": 1.8078372268274303, "percentage": 90.36, "elapsed_time": "1:25:26", "remaining_time": "0:09:06"} +{"current_steps": 455, "total_steps": 498, "loss": 0.0497, "lr": 2.849475848838749e-06, "epoch": 1.8279326802310978, "percentage": 91.37, "elapsed_time": "1:27:05", "remaining_time": "0:08:13"} +{"current_steps": 460, "total_steps": 498, "loss": 0.0625, "lr": 2.762483474317387e-06, "epoch": 1.8480281336347653, "percentage": 92.37, "elapsed_time": "1:28:52", "remaining_time": "0:07:20"} +{"current_steps": 465, "total_steps": 498, "loss": 0.0448, "lr": 2.6751684427161684e-06, "epoch": 1.8681235870384325, "percentage": 93.37, "elapsed_time": "1:30:36", "remaining_time": "0:06:25"} +{"current_steps": 470, "total_steps": 498, "loss": 0.0588, "lr": 2.587638085794541e-06, "epoch": 1.8882190404421, "percentage": 94.38, "elapsed_time": "1:32:15", "remaining_time": "0:05:29"} +{"current_steps": 475, "total_steps": 498, "loss": 0.0511, "lr": 2.5e-06, "epoch": 1.9083144938457672, "percentage": 95.38, "elapsed_time": "1:33:57", "remaining_time": "0:04:32"} +{"current_steps": 480, "total_steps": 498, "loss": 0.0497, "lr": 2.41236191420546e-06, "epoch": 1.9284099472494347, "percentage": 96.39, "elapsed_time": "1:35:43", "remaining_time": "0:03:35"} +{"current_steps": 485, "total_steps": 498, "loss": 0.0498, "lr": 2.3248315572838316e-06, "epoch": 1.9485054006531022, "percentage": 97.39, "elapsed_time": "1:37:25", "remaining_time": "0:02:36"} +{"current_steps": 490, "total_steps": 498, "loss": 0.0459, "lr": 2.237516525682614e-06, "epoch": 1.9686008540567697, "percentage": 98.39, "elapsed_time": "1:39:07", "remaining_time": "0:01:37"} +{"current_steps": 495, "total_steps": 498, "loss": 0.052, "lr": 2.1505241511612522e-06, "epoch": 1.9886963074604371, "percentage": 99.4, "elapsed_time": "1:40:50", "remaining_time": "0:00:36"} +{"current_steps": 498, "total_steps": 498, "epoch": 2.0, "percentage": 100.0, "elapsed_time": "1:42:38", "remaining_time": "0:00:00"} diff --git a/trainer_state.json b/trainer_state.json new file mode 100644 index 0000000..db1d991 --- /dev/null +++ b/trainer_state.json @@ -0,0 +1,736 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 2.0, + "eval_steps": 500, + "global_step": 498, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.02009545340366742, + "grad_norm": 101.4629135131836, + "learning_rate": 4.0000000000000003e-07, + "loss": 2.9256, + "step": 5 + }, + { + "epoch": 0.04019090680733484, + "grad_norm": 83.51904296875, + "learning_rate": 9.000000000000001e-07, + "loss": 2.7459, + "step": 10 + }, + { + "epoch": 0.06028636021100226, + "grad_norm": 25.10388946533203, + "learning_rate": 1.4000000000000001e-06, + "loss": 1.637, + "step": 15 + }, + { + "epoch": 0.08038181361466969, + "grad_norm": 13.024300575256348, + "learning_rate": 1.9000000000000002e-06, + "loss": 1.1459, + "step": 20 + }, + { + "epoch": 0.1004772670183371, + "grad_norm": 8.315956115722656, + "learning_rate": 2.4000000000000003e-06, + "loss": 0.7456, + "step": 25 + }, + { + "epoch": 0.12057272042200452, + "grad_norm": 8.930346488952637, + "learning_rate": 2.9e-06, + "loss": 0.5717, + "step": 30 + }, + { + "epoch": 0.14066817382567195, + "grad_norm": 7.356456756591797, + "learning_rate": 3.4000000000000005e-06, + "loss": 0.414, + "step": 35 + }, + { + "epoch": 0.16076362722933937, + "grad_norm": 5.648008346557617, + "learning_rate": 3.900000000000001e-06, + "loss": 0.2577, + "step": 40 + }, + { + "epoch": 0.1808590806330068, + "grad_norm": 6.329988956451416, + "learning_rate": 4.4e-06, + "loss": 0.2394, + "step": 45 + }, + { + "epoch": 0.2009545340366742, + "grad_norm": 5.1738080978393555, + "learning_rate": 4.9000000000000005e-06, + "loss": 0.1848, + "step": 50 + }, + { + "epoch": 0.22104998744034163, + "grad_norm": 4.86478853225708, + "learning_rate": 4.999016565957633e-06, + "loss": 0.2195, + "step": 55 + }, + { + "epoch": 0.24114544084400905, + "grad_norm": 4.442976474761963, + "learning_rate": 4.995022691166621e-06, + "loss": 0.1582, + "step": 60 + }, + { + "epoch": 0.26124089424767644, + "grad_norm": 3.704345941543579, + "learning_rate": 4.987961816680493e-06, + "loss": 0.183, + "step": 65 + }, + { + "epoch": 0.2813363476513439, + "grad_norm": 3.772160768508911, + "learning_rate": 4.977842622059546e-06, + "loss": 0.164, + "step": 70 + }, + { + "epoch": 0.30143180105501133, + "grad_norm": 3.2532081604003906, + "learning_rate": 4.96467754629559e-06, + "loss": 0.1781, + "step": 75 + }, + { + "epoch": 0.32152725445867875, + "grad_norm": 2.893615484237671, + "learning_rate": 4.948482772521346e-06, + "loss": 0.1288, + "step": 80 + }, + { + "epoch": 0.34162270786234616, + "grad_norm": 4.654979705810547, + "learning_rate": 4.929278208117378e-06, + "loss": 0.1444, + "step": 85 + }, + { + "epoch": 0.3617181612660136, + "grad_norm": 2.8989882469177246, + "learning_rate": 4.907087460240999e-06, + "loss": 0.1151, + "step": 90 + }, + { + "epoch": 0.381813614669681, + "grad_norm": 3.7547988891601562, + "learning_rate": 4.881937806807241e-06, + "loss": 0.1197, + "step": 95 + }, + { + "epoch": 0.4019090680733484, + "grad_norm": 2.273831605911255, + "learning_rate": 4.8538601629575525e-06, + "loss": 0.0966, + "step": 100 + }, + { + "epoch": 0.42200452147701584, + "grad_norm": 2.4642996788024902, + "learning_rate": 4.822889043057446e-06, + "loss": 0.1088, + "step": 105 + }, + { + "epoch": 0.44209997488068326, + "grad_norm": 3.29301381111145, + "learning_rate": 4.789062518269812e-06, + "loss": 0.1163, + "step": 110 + }, + { + "epoch": 0.4621954282843507, + "grad_norm": 2.45603084564209, + "learning_rate": 4.752422169756048e-06, + "loss": 0.1151, + "step": 115 + }, + { + "epoch": 0.4822908816880181, + "grad_norm": 3.199894666671753, + "learning_rate": 4.713013037562531e-06, + "loss": 0.098, + "step": 120 + }, + { + "epoch": 0.5023863350916855, + "grad_norm": 1.7128061056137085, + "learning_rate": 4.670883565255264e-06, + "loss": 0.0968, + "step": 125 + }, + { + "epoch": 0.5224817884953529, + "grad_norm": 2.645235776901245, + "learning_rate": 4.6260855403707625e-06, + "loss": 0.1068, + "step": 130 + }, + { + "epoch": 0.5425772418990203, + "grad_norm": 2.3140530586242676, + "learning_rate": 4.578674030756364e-06, + "loss": 0.0914, + "step": 135 + }, + { + "epoch": 0.5626726953026878, + "grad_norm": 1.8588067293167114, + "learning_rate": 4.528707316878233e-06, + "loss": 0.0995, + "step": 140 + }, + { + "epoch": 0.5827681487063552, + "grad_norm": 1.9172967672348022, + "learning_rate": 4.476246820180259e-06, + "loss": 0.08, + "step": 145 + }, + { + "epoch": 0.6028636021100227, + "grad_norm": 2.9082584381103516, + "learning_rate": 4.421357027581921e-06, + "loss": 0.1066, + "step": 150 + }, + { + "epoch": 0.62295905551369, + "grad_norm": 3.6944031715393066, + "learning_rate": 4.364105412207914e-06, + "loss": 0.1124, + "step": 155 + }, + { + "epoch": 0.6430545089173575, + "grad_norm": 1.5622600317001343, + "learning_rate": 4.304562350447008e-06, + "loss": 0.0856, + "step": 160 + }, + { + "epoch": 0.6631499623210249, + "grad_norm": 2.388239860534668, + "learning_rate": 4.242801035442059e-06, + "loss": 0.0766, + "step": 165 + }, + { + "epoch": 0.6832454157246923, + "grad_norm": 1.7182718515396118, + "learning_rate": 4.178897387117547e-06, + "loss": 0.104, + "step": 170 + }, + { + "epoch": 0.7033408691283597, + "grad_norm": 2.2367119789123535, + "learning_rate": 4.11292995885522e-06, + "loss": 0.0737, + "step": 175 + }, + { + "epoch": 0.7234363225320272, + "grad_norm": 2.007200241088867, + "learning_rate": 4.044979840932567e-06, + "loss": 0.0893, + "step": 180 + }, + { + "epoch": 0.7435317759356945, + "grad_norm": 2.6910905838012695, + "learning_rate": 3.975130560842821e-06, + "loss": 0.0944, + "step": 185 + }, + { + "epoch": 0.763627229339362, + "grad_norm": 2.042320489883423, + "learning_rate": 3.9034679806190204e-06, + "loss": 0.0843, + "step": 190 + }, + { + "epoch": 0.7837226827430294, + "grad_norm": 2.700977087020874, + "learning_rate": 3.830080191288342e-06, + "loss": 0.0669, + "step": 195 + }, + { + "epoch": 0.8038181361466968, + "grad_norm": 3.0307390689849854, + "learning_rate": 3.755057404586455e-06, + "loss": 0.0955, + "step": 200 + }, + { + "epoch": 0.8239135895503642, + "grad_norm": 1.11714506149292, + "learning_rate": 4.0000000000000003e-07, + "loss": 0.0956, + "step": 205 + }, + { + "epoch": 0.8440090429540317, + "grad_norm": 1.5246130228042603, + "learning_rate": 9.000000000000001e-07, + "loss": 0.0755, + "step": 210 + }, + { + "epoch": 0.864104496357699, + "grad_norm": 2.0117733478546143, + "learning_rate": 1.4000000000000001e-06, + "loss": 0.0894, + "step": 215 + }, + { + "epoch": 0.8841999497613665, + "grad_norm": 2.038092613220215, + "learning_rate": 1.9000000000000002e-06, + "loss": 0.074, + "step": 220 + }, + { + "epoch": 0.9042954031650339, + "grad_norm": 1.607172966003418, + "learning_rate": 2.4000000000000003e-06, + "loss": 0.0612, + "step": 225 + }, + { + "epoch": 0.9243908565687013, + "grad_norm": 1.643001914024353, + "learning_rate": 2.9e-06, + "loss": 0.0967, + "step": 230 + }, + { + "epoch": 0.9444863099723687, + "grad_norm": 1.5271589756011963, + "learning_rate": 3.4000000000000005e-06, + "loss": 0.1008, + "step": 235 + }, + { + "epoch": 0.9645817633760362, + "grad_norm": 2.2744393348693848, + "learning_rate": 3.900000000000001e-06, + "loss": 0.0846, + "step": 240 + }, + { + "epoch": 0.9846772167797035, + "grad_norm": 1.2268872261047363, + "learning_rate": 4.4e-06, + "loss": 0.0889, + "step": 245 + }, + { + "epoch": 1.0040190906807336, + "grad_norm": 1.971510648727417, + "learning_rate": 4.9000000000000005e-06, + "loss": 0.1021, + "step": 250 + }, + { + "epoch": 1.0241145440844008, + "grad_norm": 0.6695238947868347, + "learning_rate": 4.999016565957633e-06, + "loss": 0.057, + "step": 255 + }, + { + "epoch": 1.0442099974880683, + "grad_norm": 3.379134178161621, + "learning_rate": 4.995022691166621e-06, + "loss": 0.0695, + "step": 260 + }, + { + "epoch": 1.0643054508917358, + "grad_norm": 2.573274850845337, + "learning_rate": 4.987961816680493e-06, + "loss": 0.0713, + "step": 265 + }, + { + "epoch": 1.0844009042954033, + "grad_norm": 1.3487026691436768, + "learning_rate": 4.977842622059546e-06, + "loss": 0.0704, + "step": 270 + }, + { + "epoch": 1.1044963576990705, + "grad_norm": 1.7525826692581177, + "learning_rate": 4.96467754629559e-06, + "loss": 0.0615, + "step": 275 + }, + { + "epoch": 1.124591811102738, + "grad_norm": 1.1195745468139648, + "learning_rate": 4.948482772521346e-06, + "loss": 0.0616, + "step": 280 + }, + { + "epoch": 1.1446872645064055, + "grad_norm": 2.203016996383667, + "learning_rate": 4.929278208117378e-06, + "loss": 0.0627, + "step": 285 + }, + { + "epoch": 1.164782717910073, + "grad_norm": 3.2196788787841797, + "learning_rate": 4.907087460240999e-06, + "loss": 0.0707, + "step": 290 + }, + { + "epoch": 1.1848781713137402, + "grad_norm": 2.4914467334747314, + "learning_rate": 4.881937806807241e-06, + "loss": 0.0758, + "step": 295 + }, + { + "epoch": 1.2049736247174077, + "grad_norm": 2.223809242248535, + "learning_rate": 4.8538601629575525e-06, + "loss": 0.0698, + "step": 300 + }, + { + "epoch": 1.2250690781210751, + "grad_norm": 1.5360513925552368, + "learning_rate": 4.822889043057446e-06, + "loss": 0.0593, + "step": 305 + }, + { + "epoch": 1.2451645315247426, + "grad_norm": 1.8444727659225464, + "learning_rate": 4.789062518269812e-06, + "loss": 0.0635, + "step": 310 + }, + { + "epoch": 1.2652599849284099, + "grad_norm": 1.5122318267822266, + "learning_rate": 4.752422169756048e-06, + "loss": 0.0737, + "step": 315 + }, + { + "epoch": 1.2853554383320773, + "grad_norm": 2.0612282752990723, + "learning_rate": 4.713013037562531e-06, + "loss": 0.065, + "step": 320 + }, + { + "epoch": 1.3054508917357448, + "grad_norm": 1.4917356967926025, + "learning_rate": 4.670883565255264e-06, + "loss": 0.0563, + "step": 325 + }, + { + "epoch": 1.3255463451394123, + "grad_norm": 1.861807942390442, + "learning_rate": 4.6260855403707625e-06, + "loss": 0.0654, + "step": 330 + }, + { + "epoch": 1.3456417985430797, + "grad_norm": 2.3603355884552, + "learning_rate": 4.578674030756364e-06, + "loss": 0.0616, + "step": 335 + }, + { + "epoch": 1.365737251946747, + "grad_norm": 1.4312632083892822, + "learning_rate": 4.528707316878233e-06, + "loss": 0.0608, + "step": 340 + }, + { + "epoch": 1.3858327053504145, + "grad_norm": 3.3819150924682617, + "learning_rate": 4.476246820180259e-06, + "loss": 0.0573, + "step": 345 + }, + { + "epoch": 1.405928158754082, + "grad_norm": 2.076028347015381, + "learning_rate": 4.421357027581921e-06, + "loss": 0.0648, + "step": 350 + }, + { + "epoch": 1.4260236121577492, + "grad_norm": 2.0433411598205566, + "learning_rate": 4.364105412207914e-06, + "loss": 0.0679, + "step": 355 + }, + { + "epoch": 1.4461190655614167, + "grad_norm": 1.695204496383667, + "learning_rate": 4.304562350447008e-06, + "loss": 0.0655, + "step": 360 + }, + { + "epoch": 1.4662145189650841, + "grad_norm": 0.7300482988357544, + "learning_rate": 4.242801035442059e-06, + "loss": 0.0662, + "step": 365 + }, + { + "epoch": 1.4863099723687516, + "grad_norm": 1.8220871686935425, + "learning_rate": 4.178897387117547e-06, + "loss": 0.0659, + "step": 370 + }, + { + "epoch": 1.506405425772419, + "grad_norm": 1.4738329648971558, + "learning_rate": 4.11292995885522e-06, + "loss": 0.0605, + "step": 375 + }, + { + "epoch": 1.5265008791760866, + "grad_norm": 1.562551736831665, + "learning_rate": 4.044979840932567e-06, + "loss": 0.0741, + "step": 380 + }, + { + "epoch": 1.5465963325797538, + "grad_norm": 1.5520535707473755, + "learning_rate": 3.975130560842821e-06, + "loss": 0.058, + "step": 385 + }, + { + "epoch": 1.5666917859834213, + "grad_norm": 1.29172682762146, + "learning_rate": 3.9034679806190204e-06, + "loss": 0.0584, + "step": 390 + }, + { + "epoch": 1.5867872393870885, + "grad_norm": 1.9488056898117065, + "learning_rate": 3.830080191288342e-06, + "loss": 0.0582, + "step": 395 + }, + { + "epoch": 1.606882692790756, + "grad_norm": 1.6518131494522095, + "learning_rate": 3.755057404586455e-06, + "loss": 0.0634, + "step": 400 + }, + { + "epoch": 1.6269781461944235, + "grad_norm": 1.5398550033569336, + "learning_rate": 3.6784918420649952e-06, + "loss": 0.0525, + "step": 405 + }, + { + "epoch": 1.647073599598091, + "grad_norm": 1.6156771183013916, + "learning_rate": 3.600477621728473e-06, + "loss": 0.0645, + "step": 410 + }, + { + "epoch": 1.6671690530017584, + "grad_norm": 1.564670205116272, + "learning_rate": 3.521110642339991e-06, + "loss": 0.0526, + "step": 415 + }, + { + "epoch": 1.687264506405426, + "grad_norm": 1.2277319431304932, + "learning_rate": 3.4404884655379433e-06, + "loss": 0.0616, + "step": 420 + }, + { + "epoch": 1.7073599598090932, + "grad_norm": 1.1757029294967651, + "learning_rate": 3.358710195908653e-06, + "loss": 0.0537, + "step": 425 + }, + { + "epoch": 1.7274554132127606, + "grad_norm": 1.0405977964401245, + "learning_rate": 3.27587635916233e-06, + "loss": 0.0619, + "step": 430 + }, + { + "epoch": 1.747550866616428, + "grad_norm": 1.5975450277328491, + "learning_rate": 3.1920887785621233e-06, + "loss": 0.0603, + "step": 435 + }, + { + "epoch": 1.7676463200200954, + "grad_norm": 1.7237313985824585, + "learning_rate": 3.10745044975816e-06, + "loss": 0.0594, + "step": 440 + }, + { + "epoch": 1.7877417734237628, + "grad_norm": 1.7574973106384277, + "learning_rate": 3.022065414180425e-06, + "loss": 0.0608, + "step": 445 + }, + { + "epoch": 1.8078372268274303, + "grad_norm": 1.450028419494629, + "learning_rate": 2.9360386311461276e-06, + "loss": 0.0505, + "step": 450 + }, + { + "epoch": 1.8279326802310978, + "grad_norm": 1.0331708192825317, + "learning_rate": 2.849475848838749e-06, + "loss": 0.0497, + "step": 455 + }, + { + "epoch": 1.8480281336347653, + "grad_norm": 1.5370794534683228, + "learning_rate": 2.762483474317387e-06, + "loss": 0.0625, + "step": 460 + }, + { + "epoch": 1.8681235870384325, + "grad_norm": 2.073357582092285, + "learning_rate": 2.6751684427161684e-06, + "loss": 0.0448, + "step": 465 + }, + { + "epoch": 1.8882190404421, + "grad_norm": 1.4379805326461792, + "learning_rate": 2.587638085794541e-06, + "loss": 0.0588, + "step": 470 + }, + { + "epoch": 1.9083144938457672, + "grad_norm": 1.8600538969039917, + "learning_rate": 2.5e-06, + "loss": 0.0511, + "step": 475 + }, + { + "epoch": 1.9284099472494347, + "grad_norm": 1.2249418497085571, + "learning_rate": 2.41236191420546e-06, + "loss": 0.0497, + "step": 480 + }, + { + "epoch": 1.9485054006531022, + "grad_norm": 0.8812041282653809, + "learning_rate": 2.3248315572838316e-06, + "loss": 0.0498, + "step": 485 + }, + { + "epoch": 1.9686008540567697, + "grad_norm": 0.8966412544250488, + "learning_rate": 2.237516525682614e-06, + "loss": 0.0459, + "step": 490 + }, + { + "epoch": 1.9886963074604371, + "grad_norm": 1.238738775253296, + "learning_rate": 2.1505241511612522e-06, + "loss": 0.052, + "step": 495 + }, + { + "epoch": 2.0, + "step": 498, + "total_flos": 1.28494219771102e+18, + "train_loss": 0.03900693407858232, + "train_runtime": 6158.7829, + "train_samples_per_second": 5.17, + "train_steps_per_second": 0.081 + } + ], + "logging_steps": 5, + "max_steps": 498, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 200, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 1.28494219771102e+18, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/training_args.bin b/training_args.bin new file mode 100644 index 0000000..e9a2513 --- /dev/null +++ b/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5acbccb5459f9c0d139eb9e75792351bde8d66e4f32093b31c72148828bf3e33 +size 6225 diff --git a/training_loss.png b/training_loss.png new file mode 100644 index 0000000..eccf28a Binary files /dev/null and b/training_loss.png differ