From 16be0b7ad5938a2cb6d3a9f3b12dd564366da713 Mon Sep 17 00:00:00 2001 From: ModelHub XC Date: Tue, 11 Aug 2026 13:15:16 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=EF=BC=8C=E7=94=B1ModelHub=20XC=E7=A4=BE=E5=8C=BA=E6=8F=90?= =?UTF-8?q?=E4=BE=9B=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model: tzchen07/g2_X9e Source: Original Platform --- .gitattributes | 37 + README.md | 58 + all_results.json | 8 + chat_template.jinja | 4 + checkpoint-498/chat_template.jinja | 4 + checkpoint-498/config.json | 61 + checkpoint-498/generation_config.json | 13 + .../model-00001-of-00003.safetensors | 3 + .../model-00002-of-00003.safetensors | 3 + .../model-00003-of-00003.safetensors | 3 + checkpoint-498/model.safetensors.index.json | 296 +++ checkpoint-498/optimizer.pt | 3 + checkpoint-498/rng_state.pth | 3 + checkpoint-498/scheduler.pt | 3 + checkpoint-498/special_tokens_map.json | 34 + checkpoint-498/tokenizer.json | 3 + checkpoint-498/tokenizer.model | 3 + checkpoint-498/tokenizer_config.json | 2015 +++++++++++++++++ checkpoint-498/trainer_state.json | 727 ++++++ checkpoint-498/training_args.bin | 3 + config.json | 61 + generation_config.json | 13 + model-00001-of-00003.safetensors | 3 + model-00002-of-00003.safetensors | 3 + model-00003-of-00003.safetensors | 3 + model.safetensors.index.json | 296 +++ special_tokens_map.json | 34 + tokenizer.json | 3 + tokenizer.model | 3 + tokenizer_config.json | 2015 +++++++++++++++++ train_results.json | 8 + trainer_log.jsonl | 60 + trainer_state.json | 736 ++++++ training_args.bin | 3 + training_loss.png | Bin 0 -> 31316 bytes 35 files changed, 6525 insertions(+) create mode 100644 .gitattributes create mode 100644 README.md create mode 100644 all_results.json create mode 100644 chat_template.jinja create mode 100644 checkpoint-498/chat_template.jinja create mode 100644 checkpoint-498/config.json create mode 100644 checkpoint-498/generation_config.json create mode 100644 checkpoint-498/model-00001-of-00003.safetensors create mode 100644 checkpoint-498/model-00002-of-00003.safetensors create mode 100644 checkpoint-498/model-00003-of-00003.safetensors create mode 100644 checkpoint-498/model.safetensors.index.json create mode 100644 checkpoint-498/optimizer.pt create mode 100644 checkpoint-498/rng_state.pth create mode 100644 checkpoint-498/scheduler.pt create mode 100644 checkpoint-498/special_tokens_map.json create mode 100644 checkpoint-498/tokenizer.json create mode 100644 checkpoint-498/tokenizer.model create mode 100644 checkpoint-498/tokenizer_config.json create mode 100644 checkpoint-498/trainer_state.json create mode 100644 checkpoint-498/training_args.bin create mode 100644 config.json create mode 100644 generation_config.json create mode 100644 model-00001-of-00003.safetensors create mode 100644 model-00002-of-00003.safetensors create mode 100644 model-00003-of-00003.safetensors create mode 100644 model.safetensors.index.json create mode 100644 special_tokens_map.json create mode 100644 tokenizer.json create mode 100644 tokenizer.model create mode 100644 tokenizer_config.json create mode 100644 train_results.json create mode 100644 trainer_log.jsonl create mode 100644 trainer_state.json create mode 100644 training_args.bin create mode 100644 training_loss.png diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..e16daa5 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,37 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +checkpoint-498/tokenizer.json filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..9964e8b --- /dev/null +++ b/README.md @@ -0,0 +1,58 @@ +--- +library_name: transformers +license: other +base_model: unsloth/gemma-2-2b-it +tags: +- llama-factory +- full +- generated_from_trainer +model-index: +- name: x9e + results: [] +--- + + + +# x9e + +This model is a fine-tuned version of [unsloth/gemma-2-2b-it](https://huggingface.co/unsloth/gemma-2-2b-it) on the v1.6, the v1.6b, the v1.6c, the v1.6d and the v1.6e datasets. + +## Model description + +More information needed + +## Intended uses & limitations + +More information needed + +## Training and evaluation data + +More information needed + +## Training procedure + +### Training hyperparameters + +The following hyperparameters were used during training: +- learning_rate: 5e-06 +- train_batch_size: 4 +- eval_batch_size: 8 +- seed: 42 +- gradient_accumulation_steps: 16 +- total_train_batch_size: 64 +- optimizer: Use OptimizerNames.ADAMW_TORCH_FUSED with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments +- lr_scheduler_type: cosine +- lr_scheduler_warmup_ratio: 0.1 +- num_epochs: 2.0 + +### Training results + + + +### Framework versions + +- Transformers 4.57.1 +- Pytorch 2.10.0+cu129 +- Datasets 4.0.0 +- Tokenizers 0.22.2 diff --git a/all_results.json b/all_results.json new file mode 100644 index 0000000..2e39d80 --- /dev/null +++ b/all_results.json @@ -0,0 +1,8 @@ +{ + "epoch": 2.0, + "total_flos": 1.28494219771102e+18, + "train_loss": 0.03479443759803312, + "train_runtime": 6147.788, + "train_samples_per_second": 5.179, + "train_steps_per_second": 0.081 +} \ No newline at end of file diff --git a/chat_template.jinja b/chat_template.jinja new file mode 100644 index 0000000..923ec25 --- /dev/null +++ b/chat_template.jinja @@ -0,0 +1,4 @@ +{{ bos_token }}{% if messages[0]['role'] == 'system' %}{{ raise_exception('System role not supported') }}{% endif %}{% for message in messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if (message['role'] == 'assistant') %}{% set role = 'model' %}{% else %}{% set role = message['role'] %}{% endif %}{{ '' + role + ' +' + message['content'] | trim + ' +' }}{% endfor %}{% if add_generation_prompt %}{{'model +'}}{% endif %} \ No newline at end of file diff --git a/checkpoint-498/chat_template.jinja b/checkpoint-498/chat_template.jinja new file mode 100644 index 0000000..923ec25 --- /dev/null +++ b/checkpoint-498/chat_template.jinja @@ -0,0 +1,4 @@ +{{ bos_token }}{% if messages[0]['role'] == 'system' %}{{ raise_exception('System role not supported') }}{% endif %}{% for message in messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if (message['role'] == 'assistant') %}{% set role = 'model' %}{% else %}{% set role = message['role'] %}{% endif %}{{ '' + role + ' +' + message['content'] | trim + ' +' }}{% endfor %}{% if add_generation_prompt %}{{'model +'}}{% endif %} \ No newline at end of file diff --git a/checkpoint-498/config.json b/checkpoint-498/config.json new file mode 100644 index 0000000..011fcae --- /dev/null +++ b/checkpoint-498/config.json @@ -0,0 +1,61 @@ +{ + "architectures": [ + "Gemma2ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "attn_logit_softcapping": 50.0, + "bos_token_id": 2, + "cache_implementation": "hybrid", + "dtype": "float32", + "eos_token_id": 107, + "final_logit_softcapping": 30.0, + "head_dim": 256, + "hidden_act": "gelu_pytorch_tanh", + "hidden_activation": "gelu_pytorch_tanh", + "hidden_size": 2304, + "initializer_range": 0.02, + "intermediate_size": 9216, + "layer_types": [ + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention" + ], + "max_position_embeddings": 8192, + "model_type": "gemma2", + "num_attention_heads": 8, + "num_hidden_layers": 26, + "num_key_value_heads": 4, + "pad_token_id": 0, + "query_pre_attn_scalar": 256, + "rms_norm_eps": 1e-06, + "rope_theta": 10000.0, + "sliding_window": 4096, + "transformers_version": "4.57.1", + "unsloth_version": "2024.9", + "use_cache": false, + "vocab_size": 256000 +} diff --git a/checkpoint-498/generation_config.json b/checkpoint-498/generation_config.json new file mode 100644 index 0000000..014175e --- /dev/null +++ b/checkpoint-498/generation_config.json @@ -0,0 +1,13 @@ +{ + "_from_model_config": true, + "bos_token_id": 2, + "cache_implementation": "hybrid", + "eos_token_id": [ + 107, + 1, + 107 + ], + "max_length": 8192, + "pad_token_id": 0, + "transformers_version": "4.57.1" +} diff --git a/checkpoint-498/model-00001-of-00003.safetensors b/checkpoint-498/model-00001-of-00003.safetensors new file mode 100644 index 0000000..671636c --- /dev/null +++ b/checkpoint-498/model-00001-of-00003.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:883757107ed628ac68adfc4776742299a9f4152a51109595cf5c4447d889d856 +size 4992576136 diff --git a/checkpoint-498/model-00002-of-00003.safetensors b/checkpoint-498/model-00002-of-00003.safetensors new file mode 100644 index 0000000..97ecfd5 --- /dev/null +++ b/checkpoint-498/model-00002-of-00003.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cbfeb36073f11805e2660ec76cc0b38985887e07581fcf923570b5a89d78c536 +size 4983443424 diff --git a/checkpoint-498/model-00003-of-00003.safetensors b/checkpoint-498/model-00003-of-00003.safetensors new file mode 100644 index 0000000..df38a2d --- /dev/null +++ b/checkpoint-498/model-00003-of-00003.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8e04f476d50568d217d30f29db34ce1a63951758fede234cafef5b20b72af7d3 +size 481381384 diff --git a/checkpoint-498/model.safetensors.index.json b/checkpoint-498/model.safetensors.index.json new file mode 100644 index 0000000..1a43cf8 --- /dev/null +++ b/checkpoint-498/model.safetensors.index.json @@ -0,0 +1,296 @@ +{ + "metadata": { + "total_parameters": 2614341888, + "total_size": 10457367552 + }, + "weight_map": { + "model.embed_tokens.weight": "model-00001-of-00003.safetensors", + "model.layers.0.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.0.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.0.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.0.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.0.pre_feedforward_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.1.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.1.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.1.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.1.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.1.pre_feedforward_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.10.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.10.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.10.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.10.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.10.pre_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.11.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.11.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.11.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.11.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.11.pre_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.12.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.12.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.12.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.12.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.12.pre_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.13.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.13.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.13.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.13.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.13.pre_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.14.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.14.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.14.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.14.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.14.pre_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.15.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.15.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.15.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.15.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.15.pre_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.16.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.16.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.16.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.16.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.16.pre_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.17.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.17.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.17.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.17.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.17.pre_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.18.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.18.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.18.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.18.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.18.pre_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.19.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.19.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.19.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.19.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.19.pre_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.2.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.2.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.2.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.2.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.2.pre_feedforward_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.20.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.20.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.20.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.20.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.20.pre_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.21.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.21.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.21.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.21.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.21.pre_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.22.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.22.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.22.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.22.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.22.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.22.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.22.pre_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.23.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.23.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.23.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.23.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.23.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.23.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.23.pre_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.23.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.23.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.23.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.23.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.24.input_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.24.mlp.down_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.24.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.24.mlp.up_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.24.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.24.pre_feedforward_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.24.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.24.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.24.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.24.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.25.input_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.25.mlp.down_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.25.mlp.up_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.25.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.25.pre_feedforward_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.3.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.3.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.3.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.3.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.3.pre_feedforward_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.4.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.4.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.4.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.4.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.4.pre_feedforward_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.5.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.5.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.5.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.5.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.5.pre_feedforward_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.6.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.6.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.6.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.6.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.6.pre_feedforward_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.7.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.7.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.7.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.7.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.7.pre_feedforward_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.8.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.8.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.8.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.8.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.8.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.8.pre_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.9.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.9.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.9.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.9.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.9.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.9.pre_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.9.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.9.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.9.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.norm.weight": "model-00003-of-00003.safetensors" + } +} diff --git a/checkpoint-498/optimizer.pt b/checkpoint-498/optimizer.pt new file mode 100644 index 0000000..3fb0de1 --- /dev/null +++ b/checkpoint-498/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ec9b92be30bc1d929984a3428f1bb2ec8980159209b8f286ad73c39e778d9f9d +size 20914991879 diff --git a/checkpoint-498/rng_state.pth b/checkpoint-498/rng_state.pth new file mode 100644 index 0000000..1feba1a --- /dev/null +++ b/checkpoint-498/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:098b29492211804ab324a36f37466821d948280bb74fce4ba895c03f13ecd878 +size 14645 diff --git a/checkpoint-498/scheduler.pt b/checkpoint-498/scheduler.pt new file mode 100644 index 0000000..af2c1f0 --- /dev/null +++ b/checkpoint-498/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a1dab4ab977132a8fa31885c15cbac51ab780724467e4072a01a4b087b2d0314 +size 1465 diff --git a/checkpoint-498/special_tokens_map.json b/checkpoint-498/special_tokens_map.json new file mode 100644 index 0000000..07f6ba6 --- /dev/null +++ b/checkpoint-498/special_tokens_map.json @@ -0,0 +1,34 @@ +{ + "additional_special_tokens": [ + "", + "" + ], + "bos_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "unk_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + } +} diff --git a/checkpoint-498/tokenizer.json b/checkpoint-498/tokenizer.json new file mode 100644 index 0000000..a4a305d --- /dev/null +++ b/checkpoint-498/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5f7eee611703c5ce5d1eee32d9cdcfe465647b8aff0c1dfb3bed7ad7dbb05060 +size 34362873 diff --git a/checkpoint-498/tokenizer.model b/checkpoint-498/tokenizer.model new file mode 100644 index 0000000..796efe9 --- /dev/null +++ b/checkpoint-498/tokenizer.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:61a7b147390c64585d6c3543dd6fc636906c9af3865a5548f27f31aee1d4c8e2 +size 4241003 diff --git a/checkpoint-498/tokenizer_config.json b/checkpoint-498/tokenizer_config.json new file mode 100644 index 0000000..99660b4 --- /dev/null +++ b/checkpoint-498/tokenizer_config.json @@ -0,0 +1,2015 @@ +{ + "add_bos_token": true, + "add_eos_token": false, + "added_tokens_decoder": { + "0": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "1": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "2": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "3": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "4": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "5": { + "content": "<2mass>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "6": { + "content": "[@BOS@]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "7": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "8": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "9": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "10": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "11": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "12": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "13": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "14": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "15": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "16": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "17": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "18": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "19": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "20": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "21": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "22": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "23": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "24": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "25": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "26": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "27": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "28": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "29": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "30": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "31": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "32": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "33": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "34": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "35": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "36": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "37": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "38": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "39": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "40": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "41": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "42": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "43": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "44": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "45": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "46": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "47": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "48": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "49": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "50": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "51": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "52": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "53": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "54": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "55": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "56": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "57": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "58": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "59": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "60": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "61": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "62": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "63": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "64": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "65": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "66": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "67": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "68": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "69": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "70": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "71": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "72": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "73": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "74": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "75": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "76": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "77": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "78": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "79": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "80": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "81": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "82": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "83": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "84": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "85": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "86": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "87": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "88": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "89": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "90": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "91": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "92": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "93": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "94": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "95": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "96": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "97": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "98": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "99": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "100": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "101": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "102": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "103": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "104": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "105": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "106": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "107": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "108": { + "content": "\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "109": { + "content": "\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "110": { + "content": "\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "111": { + "content": "\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "112": { + "content": "\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "113": { + "content": "\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "114": { + "content": "\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "115": { + "content": "\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "116": { + "content": "\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "117": { + "content": "\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "118": { + "content": "\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "119": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "120": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "121": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "122": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "123": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "124": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "125": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "126": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "127": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "128": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "129": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "130": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "131": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "132": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "133": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "134": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "135": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "136": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "137": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "138": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "139": { + "content": "▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "140": { + "content": "▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "141": { + "content": "▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "142": { + "content": "▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "143": { + "content": "▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "144": { + "content": "▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "145": { + "content": "▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "146": { + "content": "▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "147": { + "content": "▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "148": { + "content": "▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "149": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "150": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "152": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "153": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "154": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "155": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "156": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "157": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "158": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "159": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "160": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "161": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "162": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "163": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "164": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "165": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "166": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "167": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "168": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "169": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "170": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "172": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "173": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "174": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "175": { + "content": "
", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "171": { + "content": "
", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "176": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "177": { + "content": "
", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "178": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "179": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "180": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "181": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "182": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "183": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "184": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "185": { + "content": "

", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "186": { + "content": "

", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "187": { + "content": "

", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "188": { + "content": "

", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "189": { + "content": "

", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "190": { + "content": "
", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "191": { + "content": "
", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "192": { + "content": "
", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "193": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "194": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "195": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "196": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "197": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "198": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "199": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "200": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "201": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "202": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "203": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "204": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "205": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "206": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "207": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "208": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "209": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "210": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "211": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "212": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "213": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "214": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "215": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "216": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255968": { + "content": "[toxicity=0]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255969": { + "content": "\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255970": { + "content": "\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255971": { + "content": "\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255972": { + "content": "\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255973": { + "content": "\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255974": { + "content": "\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255975": { + "content": "\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255976": { + "content": "\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255977": { + "content": "\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255978": { + "content": "\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255979": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255980": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255981": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255982": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255983": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255984": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255985": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255986": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255987": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255988": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255989": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255990": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255991": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255992": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255993": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255994": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255995": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255996": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255997": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255998": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255999": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + } + }, + "additional_special_tokens": [ + "", + "" + ], + "bos_token": "", + "clean_up_tokenization_spaces": false, + "eos_token": "", + "extra_special_tokens": {}, + "model_max_length": 1000000000000000019884624838656, + "pad_token": "", + "padding_side": "right", + "sp_model_kwargs": {}, + "spaces_between_special_tokens": false, + "split_special_tokens": false, + "tokenizer_class": "GemmaTokenizer", + "unk_token": "", + "use_default_system_prompt": false +} diff --git a/checkpoint-498/trainer_state.json b/checkpoint-498/trainer_state.json new file mode 100644 index 0000000..5b6b9cb --- /dev/null +++ b/checkpoint-498/trainer_state.json @@ -0,0 +1,727 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 2.0, + "eval_steps": 500, + "global_step": 498, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.02009545340366742, + "grad_norm": 101.4629135131836, + "learning_rate": 4.0000000000000003e-07, + "loss": 2.9256, + "step": 5 + }, + { + "epoch": 0.04019090680733484, + "grad_norm": 83.51904296875, + "learning_rate": 9.000000000000001e-07, + "loss": 2.7459, + "step": 10 + }, + { + "epoch": 0.06028636021100226, + "grad_norm": 25.10388946533203, + "learning_rate": 1.4000000000000001e-06, + "loss": 1.637, + "step": 15 + }, + { + "epoch": 0.08038181361466969, + "grad_norm": 13.024300575256348, + "learning_rate": 1.9000000000000002e-06, + "loss": 1.1459, + "step": 20 + }, + { + "epoch": 0.1004772670183371, + "grad_norm": 8.315956115722656, + "learning_rate": 2.4000000000000003e-06, + "loss": 0.7456, + "step": 25 + }, + { + "epoch": 0.12057272042200452, + "grad_norm": 8.930346488952637, + "learning_rate": 2.9e-06, + "loss": 0.5717, + "step": 30 + }, + { + "epoch": 0.14066817382567195, + "grad_norm": 7.356456756591797, + "learning_rate": 3.4000000000000005e-06, + "loss": 0.414, + "step": 35 + }, + { + "epoch": 0.16076362722933937, + "grad_norm": 5.648008346557617, + "learning_rate": 3.900000000000001e-06, + "loss": 0.2577, + "step": 40 + }, + { + "epoch": 0.1808590806330068, + "grad_norm": 6.329988956451416, + "learning_rate": 4.4e-06, + "loss": 0.2394, + "step": 45 + }, + { + "epoch": 0.2009545340366742, + "grad_norm": 5.1738080978393555, + "learning_rate": 4.9000000000000005e-06, + "loss": 0.1848, + "step": 50 + }, + { + "epoch": 0.22104998744034163, + "grad_norm": 4.86478853225708, + "learning_rate": 4.999016565957633e-06, + "loss": 0.2195, + "step": 55 + }, + { + "epoch": 0.24114544084400905, + "grad_norm": 4.442976474761963, + "learning_rate": 4.995022691166621e-06, + "loss": 0.1582, + "step": 60 + }, + { + "epoch": 0.26124089424767644, + "grad_norm": 3.704345941543579, + "learning_rate": 4.987961816680493e-06, + "loss": 0.183, + "step": 65 + }, + { + "epoch": 0.2813363476513439, + "grad_norm": 3.772160768508911, + "learning_rate": 4.977842622059546e-06, + "loss": 0.164, + "step": 70 + }, + { + "epoch": 0.30143180105501133, + "grad_norm": 3.2532081604003906, + "learning_rate": 4.96467754629559e-06, + "loss": 0.1781, + "step": 75 + }, + { + "epoch": 0.32152725445867875, + "grad_norm": 2.893615484237671, + "learning_rate": 4.948482772521346e-06, + "loss": 0.1288, + "step": 80 + }, + { + "epoch": 0.34162270786234616, + "grad_norm": 4.654979705810547, + "learning_rate": 4.929278208117378e-06, + "loss": 0.1444, + "step": 85 + }, + { + "epoch": 0.3617181612660136, + "grad_norm": 2.8989882469177246, + "learning_rate": 4.907087460240999e-06, + "loss": 0.1151, + "step": 90 + }, + { + "epoch": 0.381813614669681, + "grad_norm": 3.7547988891601562, + "learning_rate": 4.881937806807241e-06, + "loss": 0.1197, + "step": 95 + }, + { + "epoch": 0.4019090680733484, + "grad_norm": 2.273831605911255, + "learning_rate": 4.8538601629575525e-06, + "loss": 0.0966, + "step": 100 + }, + { + "epoch": 0.42200452147701584, + "grad_norm": 2.4642996788024902, + "learning_rate": 4.822889043057446e-06, + "loss": 0.1088, + "step": 105 + }, + { + "epoch": 0.44209997488068326, + "grad_norm": 3.29301381111145, + "learning_rate": 4.789062518269812e-06, + "loss": 0.1163, + "step": 110 + }, + { + "epoch": 0.4621954282843507, + "grad_norm": 2.45603084564209, + "learning_rate": 4.752422169756048e-06, + "loss": 0.1151, + "step": 115 + }, + { + "epoch": 0.4822908816880181, + "grad_norm": 3.199894666671753, + "learning_rate": 4.713013037562531e-06, + "loss": 0.098, + "step": 120 + }, + { + "epoch": 0.5023863350916855, + "grad_norm": 1.7128061056137085, + "learning_rate": 4.670883565255264e-06, + "loss": 0.0968, + "step": 125 + }, + { + "epoch": 0.5224817884953529, + "grad_norm": 2.645235776901245, + "learning_rate": 4.6260855403707625e-06, + "loss": 0.1068, + "step": 130 + }, + { + "epoch": 0.5425772418990203, + "grad_norm": 2.3140530586242676, + "learning_rate": 4.578674030756364e-06, + "loss": 0.0914, + "step": 135 + }, + { + "epoch": 0.5626726953026878, + "grad_norm": 1.8588067293167114, + "learning_rate": 4.528707316878233e-06, + "loss": 0.0995, + "step": 140 + }, + { + "epoch": 0.5827681487063552, + "grad_norm": 1.9172967672348022, + "learning_rate": 4.476246820180259e-06, + "loss": 0.08, + "step": 145 + }, + { + "epoch": 0.6028636021100227, + "grad_norm": 2.9082584381103516, + "learning_rate": 4.421357027581921e-06, + "loss": 0.1066, + "step": 150 + }, + { + "epoch": 0.62295905551369, + "grad_norm": 3.6944031715393066, + "learning_rate": 4.364105412207914e-06, + "loss": 0.1124, + "step": 155 + }, + { + "epoch": 0.6430545089173575, + "grad_norm": 1.5622600317001343, + "learning_rate": 4.304562350447008e-06, + "loss": 0.0856, + "step": 160 + }, + { + "epoch": 0.6631499623210249, + "grad_norm": 2.388239860534668, + "learning_rate": 4.242801035442059e-06, + "loss": 0.0766, + "step": 165 + }, + { + "epoch": 0.6832454157246923, + "grad_norm": 1.7182718515396118, + "learning_rate": 4.178897387117547e-06, + "loss": 0.104, + "step": 170 + }, + { + "epoch": 0.7033408691283597, + "grad_norm": 2.2367119789123535, + "learning_rate": 4.11292995885522e-06, + "loss": 0.0737, + "step": 175 + }, + { + "epoch": 0.7234363225320272, + "grad_norm": 2.007200241088867, + "learning_rate": 4.044979840932567e-06, + "loss": 0.0893, + "step": 180 + }, + { + "epoch": 0.7435317759356945, + "grad_norm": 2.6910905838012695, + "learning_rate": 3.975130560842821e-06, + "loss": 0.0944, + "step": 185 + }, + { + "epoch": 0.763627229339362, + "grad_norm": 2.042320489883423, + "learning_rate": 3.9034679806190204e-06, + "loss": 0.0843, + "step": 190 + }, + { + "epoch": 0.7837226827430294, + "grad_norm": 2.700977087020874, + "learning_rate": 3.830080191288342e-06, + "loss": 0.0669, + "step": 195 + }, + { + "epoch": 0.8038181361466968, + "grad_norm": 3.0307390689849854, + "learning_rate": 3.755057404586455e-06, + "loss": 0.0955, + "step": 200 + }, + { + "epoch": 0.8239135895503642, + "grad_norm": 1.0048880577087402, + "learning_rate": 3.6784918420649952e-06, + "loss": 0.0977, + "step": 205 + }, + { + "epoch": 0.8440090429540317, + "grad_norm": 1.3049492835998535, + "learning_rate": 3.600477621728473e-06, + "loss": 0.0731, + "step": 210 + }, + { + "epoch": 0.864104496357699, + "grad_norm": 2.020117998123169, + "learning_rate": 3.521110642339991e-06, + "loss": 0.0939, + "step": 215 + }, + { + "epoch": 0.8841999497613665, + "grad_norm": 2.2311348915100098, + "learning_rate": 3.4404884655379433e-06, + "loss": 0.0747, + "step": 220 + }, + { + "epoch": 0.9042954031650339, + "grad_norm": 2.1661417484283447, + "learning_rate": 3.358710195908653e-06, + "loss": 0.0625, + "step": 225 + }, + { + "epoch": 0.9243908565687013, + "grad_norm": 2.463257074356079, + "learning_rate": 3.27587635916233e-06, + "loss": 0.1004, + "step": 230 + }, + { + "epoch": 0.9444863099723687, + "grad_norm": 1.5093741416931152, + "learning_rate": 3.1920887785621233e-06, + "loss": 0.1034, + "step": 235 + }, + { + "epoch": 0.9645817633760362, + "grad_norm": 2.1394777297973633, + "learning_rate": 3.10745044975816e-06, + "loss": 0.0815, + "step": 240 + }, + { + "epoch": 0.9846772167797035, + "grad_norm": 1.6689902544021606, + "learning_rate": 3.022065414180425e-06, + "loss": 0.0887, + "step": 245 + }, + { + "epoch": 1.0040190906807336, + "grad_norm": 1.9693433046340942, + "learning_rate": 2.9360386311461276e-06, + "loss": 0.0972, + "step": 250 + }, + { + "epoch": 1.0241145440844008, + "grad_norm": 0.750286340713501, + "learning_rate": 2.849475848838749e-06, + "loss": 0.0546, + "step": 255 + }, + { + "epoch": 1.0442099974880683, + "grad_norm": 3.802541732788086, + "learning_rate": 2.762483474317387e-06, + "loss": 0.0601, + "step": 260 + }, + { + "epoch": 1.0643054508917358, + "grad_norm": 1.3461735248565674, + "learning_rate": 2.6751684427161684e-06, + "loss": 0.0596, + "step": 265 + }, + { + "epoch": 1.0844009042954033, + "grad_norm": 2.324124813079834, + "learning_rate": 2.587638085794541e-06, + "loss": 0.0577, + "step": 270 + }, + { + "epoch": 1.1044963576990705, + "grad_norm": 1.4222259521484375, + "learning_rate": 2.5e-06, + "loss": 0.0527, + "step": 275 + }, + { + "epoch": 1.124591811102738, + "grad_norm": 0.48392441868782043, + "learning_rate": 2.41236191420546e-06, + "loss": 0.0592, + "step": 280 + }, + { + "epoch": 1.1446872645064055, + "grad_norm": 1.6113179922103882, + "learning_rate": 2.3248315572838316e-06, + "loss": 0.0552, + "step": 285 + }, + { + "epoch": 1.164782717910073, + "grad_norm": 2.1131763458251953, + "learning_rate": 2.237516525682614e-06, + "loss": 0.0601, + "step": 290 + }, + { + "epoch": 1.1848781713137402, + "grad_norm": 1.010076880455017, + "learning_rate": 2.1505241511612522e-06, + "loss": 0.0601, + "step": 295 + }, + { + "epoch": 1.2049736247174077, + "grad_norm": 1.5603766441345215, + "learning_rate": 2.0639613688538733e-06, + "loss": 0.0626, + "step": 300 + }, + { + "epoch": 1.2250690781210751, + "grad_norm": 0.9277430176734924, + "learning_rate": 1.9779345858195757e-06, + "loss": 0.049, + "step": 305 + }, + { + "epoch": 1.2451645315247426, + "grad_norm": 1.768437147140503, + "learning_rate": 1.8925495502418407e-06, + "loss": 0.055, + "step": 310 + }, + { + "epoch": 1.2652599849284099, + "grad_norm": 1.4469853639602661, + "learning_rate": 1.8079112214378769e-06, + "loss": 0.0608, + "step": 315 + }, + { + "epoch": 1.2853554383320773, + "grad_norm": 1.503761649131775, + "learning_rate": 1.7241236408376702e-06, + "loss": 0.0526, + "step": 320 + }, + { + "epoch": 1.3054508917357448, + "grad_norm": 1.1352767944335938, + "learning_rate": 1.6412898040913471e-06, + "loss": 0.0509, + "step": 325 + }, + { + "epoch": 1.3255463451394123, + "grad_norm": 2.6707780361175537, + "learning_rate": 1.5595115344620575e-06, + "loss": 0.0631, + "step": 330 + }, + { + "epoch": 1.3456417985430797, + "grad_norm": 1.1756007671356201, + "learning_rate": 1.47888935766001e-06, + "loss": 0.0524, + "step": 335 + }, + { + "epoch": 1.365737251946747, + "grad_norm": 1.0455981492996216, + "learning_rate": 1.399522378271527e-06, + "loss": 0.0507, + "step": 340 + }, + { + "epoch": 1.3858327053504145, + "grad_norm": 0.7072739601135254, + "learning_rate": 1.3215081579350058e-06, + "loss": 0.0487, + "step": 345 + }, + { + "epoch": 1.405928158754082, + "grad_norm": 1.145075798034668, + "learning_rate": 1.2449425954135452e-06, + "loss": 0.0544, + "step": 350 + }, + { + "epoch": 1.4260236121577492, + "grad_norm": 1.4399057626724243, + "learning_rate": 1.169919808711659e-06, + "loss": 0.0606, + "step": 355 + }, + { + "epoch": 1.4461190655614167, + "grad_norm": 1.0342814922332764, + "learning_rate": 1.0965320193809808e-06, + "loss": 0.0547, + "step": 360 + }, + { + "epoch": 1.4662145189650841, + "grad_norm": 0.5163133144378662, + "learning_rate": 1.02486943915718e-06, + "loss": 0.0534, + "step": 365 + }, + { + "epoch": 1.4863099723687516, + "grad_norm": 1.0042380094528198, + "learning_rate": 9.550201590674343e-07, + "loss": 0.0541, + "step": 370 + }, + { + "epoch": 1.506405425772419, + "grad_norm": 1.1944531202316284, + "learning_rate": 8.870700411447817e-07, + "loss": 0.0501, + "step": 375 + }, + { + "epoch": 1.5265008791760866, + "grad_norm": 1.3594350814819336, + "learning_rate": 8.21102612882454e-07, + "loss": 0.0611, + "step": 380 + }, + { + "epoch": 1.5465963325797538, + "grad_norm": 1.2450895309448242, + "learning_rate": 7.57198964557942e-07, + "loss": 0.0505, + "step": 385 + }, + { + "epoch": 1.5666917859834213, + "grad_norm": 1.6389065980911255, + "learning_rate": 6.954376495529927e-07, + "loss": 0.0473, + "step": 390 + }, + { + "epoch": 1.5867872393870885, + "grad_norm": 2.077847719192505, + "learning_rate": 6.358945877920861e-07, + "loss": 0.0519, + "step": 395 + }, + { + "epoch": 1.606882692790756, + "grad_norm": 1.5446641445159912, + "learning_rate": 5.786429724180801e-07, + "loss": 0.0491, + "step": 400 + }, + { + "epoch": 1.6269781461944235, + "grad_norm": 1.4639307260513306, + "learning_rate": 5.237531798197415e-07, + "loss": 0.0441, + "step": 405 + }, + { + "epoch": 1.647073599598091, + "grad_norm": 1.4568461179733276, + "learning_rate": 4.712926831217685e-07, + "loss": 0.054, + "step": 410 + }, + { + "epoch": 1.6671690530017584, + "grad_norm": 1.6878377199172974, + "learning_rate": 4.2132596924363666e-07, + "loss": 0.047, + "step": 415 + }, + { + "epoch": 1.687264506405426, + "grad_norm": 0.9864447116851807, + "learning_rate": 3.7391445962923824e-07, + "loss": 0.0503, + "step": 420 + }, + { + "epoch": 1.7073599598090932, + "grad_norm": 0.9351651668548584, + "learning_rate": 3.2911643474473647e-07, + "loss": 0.0456, + "step": 425 + }, + { + "epoch": 1.7274554132127606, + "grad_norm": 0.9683827757835388, + "learning_rate": 2.869869624374699e-07, + "loss": 0.0535, + "step": 430 + }, + { + "epoch": 1.747550866616428, + "grad_norm": 1.0934531688690186, + "learning_rate": 2.4757783024395244e-07, + "loss": 0.0481, + "step": 435 + }, + { + "epoch": 1.7676463200200954, + "grad_norm": 1.5016802549362183, + "learning_rate": 2.1093748173018846e-07, + "loss": 0.0519, + "step": 440 + }, + { + "epoch": 1.7877417734237628, + "grad_norm": 1.4125523567199707, + "learning_rate": 1.771109569425547e-07, + "loss": 0.0523, + "step": 445 + }, + { + "epoch": 1.8078372268274303, + "grad_norm": 1.1559640169143677, + "learning_rate": 1.4613983704244826e-07, + "loss": 0.0454, + "step": 450 + }, + { + "epoch": 1.8279326802310978, + "grad_norm": 1.0382763147354126, + "learning_rate": 1.1806219319275918e-07, + "loss": 0.046, + "step": 455 + }, + { + "epoch": 1.8480281336347653, + "grad_norm": 1.4664262533187866, + "learning_rate": 9.291253975900138e-08, + "loss": 0.0529, + "step": 460 + }, + { + "epoch": 1.8681235870384325, + "grad_norm": 0.8224850296974182, + "learning_rate": 7.072179188262252e-08, + "loss": 0.0389, + "step": 465 + }, + { + "epoch": 1.8882190404421, + "grad_norm": 0.5471680760383606, + "learning_rate": 5.151722747865434e-08, + "loss": 0.0489, + "step": 470 + }, + { + "epoch": 1.9083144938457672, + "grad_norm": 0.9190619587898254, + "learning_rate": 3.5322453704410286e-08, + "loss": 0.0438, + "step": 475 + }, + { + "epoch": 1.9284099472494347, + "grad_norm": 0.8270437121391296, + "learning_rate": 2.215737794045436e-08, + "loss": 0.0448, + "step": 480 + }, + { + "epoch": 1.9485054006531022, + "grad_norm": 0.6661733388900757, + "learning_rate": 1.2038183319507957e-08, + "loss": 0.0436, + "step": 485 + }, + { + "epoch": 1.9686008540567697, + "grad_norm": 0.7018892168998718, + "learning_rate": 4.977308833379324e-09, + "loss": 0.0419, + "step": 490 + }, + { + "epoch": 1.9886963074604371, + "grad_norm": 0.8652178645133972, + "learning_rate": 9.834340423678368e-10, + "loss": 0.048, + "step": 495 + } + ], + "logging_steps": 5, + "max_steps": 498, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 200, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 1.28494219771102e+18, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/checkpoint-498/training_args.bin b/checkpoint-498/training_args.bin new file mode 100644 index 0000000..b697238 --- /dev/null +++ b/checkpoint-498/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:62f0adb9f98878d5918d58cd798bf85ae88dd0ed2328bb63223e8143e2502520 +size 6225 diff --git a/config.json b/config.json new file mode 100644 index 0000000..011fcae --- /dev/null +++ b/config.json @@ -0,0 +1,61 @@ +{ + "architectures": [ + "Gemma2ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "attn_logit_softcapping": 50.0, + "bos_token_id": 2, + "cache_implementation": "hybrid", + "dtype": "float32", + "eos_token_id": 107, + "final_logit_softcapping": 30.0, + "head_dim": 256, + "hidden_act": "gelu_pytorch_tanh", + "hidden_activation": "gelu_pytorch_tanh", + "hidden_size": 2304, + "initializer_range": 0.02, + "intermediate_size": 9216, + "layer_types": [ + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention" + ], + "max_position_embeddings": 8192, + "model_type": "gemma2", + "num_attention_heads": 8, + "num_hidden_layers": 26, + "num_key_value_heads": 4, + "pad_token_id": 0, + "query_pre_attn_scalar": 256, + "rms_norm_eps": 1e-06, + "rope_theta": 10000.0, + "sliding_window": 4096, + "transformers_version": "4.57.1", + "unsloth_version": "2024.9", + "use_cache": false, + "vocab_size": 256000 +} diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..014175e --- /dev/null +++ b/generation_config.json @@ -0,0 +1,13 @@ +{ + "_from_model_config": true, + "bos_token_id": 2, + "cache_implementation": "hybrid", + "eos_token_id": [ + 107, + 1, + 107 + ], + "max_length": 8192, + "pad_token_id": 0, + "transformers_version": "4.57.1" +} diff --git a/model-00001-of-00003.safetensors b/model-00001-of-00003.safetensors new file mode 100644 index 0000000..671636c --- /dev/null +++ b/model-00001-of-00003.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:883757107ed628ac68adfc4776742299a9f4152a51109595cf5c4447d889d856 +size 4992576136 diff --git a/model-00002-of-00003.safetensors b/model-00002-of-00003.safetensors new file mode 100644 index 0000000..97ecfd5 --- /dev/null +++ b/model-00002-of-00003.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cbfeb36073f11805e2660ec76cc0b38985887e07581fcf923570b5a89d78c536 +size 4983443424 diff --git a/model-00003-of-00003.safetensors b/model-00003-of-00003.safetensors new file mode 100644 index 0000000..df38a2d --- /dev/null +++ b/model-00003-of-00003.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8e04f476d50568d217d30f29db34ce1a63951758fede234cafef5b20b72af7d3 +size 481381384 diff --git a/model.safetensors.index.json b/model.safetensors.index.json new file mode 100644 index 0000000..1a43cf8 --- /dev/null +++ b/model.safetensors.index.json @@ -0,0 +1,296 @@ +{ + "metadata": { + "total_parameters": 2614341888, + "total_size": 10457367552 + }, + "weight_map": { + "model.embed_tokens.weight": "model-00001-of-00003.safetensors", + "model.layers.0.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.0.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.0.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.0.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.0.pre_feedforward_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.1.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.1.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.1.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.1.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.1.pre_feedforward_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.10.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.10.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.10.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.10.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.10.pre_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.11.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.11.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.11.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.11.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.11.pre_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.12.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.12.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.12.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.12.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.12.pre_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.13.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.13.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.13.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.13.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.13.pre_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.14.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.14.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.14.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.14.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.14.pre_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.15.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.15.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.15.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.15.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.15.pre_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.16.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.16.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.16.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.16.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.16.pre_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.17.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.17.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.17.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.17.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.17.pre_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.18.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.18.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.18.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.18.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.18.pre_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.19.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.19.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.19.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.19.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.19.pre_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.2.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.2.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.2.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.2.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.2.pre_feedforward_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.20.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.20.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.20.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.20.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.20.pre_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.21.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.21.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.21.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.21.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.21.pre_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.22.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.22.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.22.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.22.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.22.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.22.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.22.pre_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.23.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.23.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.23.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.23.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.23.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.23.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.23.pre_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.23.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.23.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.23.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.23.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.24.input_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.24.mlp.down_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.24.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.24.mlp.up_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.24.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.24.pre_feedforward_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.24.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.24.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.24.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.24.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.25.input_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.25.mlp.down_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.25.mlp.up_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.25.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.25.pre_feedforward_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.3.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.3.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.3.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.3.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.3.pre_feedforward_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.4.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.4.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.4.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.4.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.4.pre_feedforward_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.5.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.5.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.5.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.5.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.5.pre_feedforward_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.6.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.6.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.6.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.6.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.6.pre_feedforward_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.7.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.7.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.7.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.7.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.7.pre_feedforward_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.8.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.8.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.8.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.8.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.8.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.8.pre_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.9.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.9.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.9.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.9.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.9.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.9.pre_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.9.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.9.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.9.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.norm.weight": "model-00003-of-00003.safetensors" + } +} diff --git a/special_tokens_map.json b/special_tokens_map.json new file mode 100644 index 0000000..07f6ba6 --- /dev/null +++ b/special_tokens_map.json @@ -0,0 +1,34 @@ +{ + "additional_special_tokens": [ + "", + "" + ], + "bos_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "unk_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + } +} diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..a4a305d --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5f7eee611703c5ce5d1eee32d9cdcfe465647b8aff0c1dfb3bed7ad7dbb05060 +size 34362873 diff --git a/tokenizer.model b/tokenizer.model new file mode 100644 index 0000000..796efe9 --- /dev/null +++ b/tokenizer.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:61a7b147390c64585d6c3543dd6fc636906c9af3865a5548f27f31aee1d4c8e2 +size 4241003 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..99660b4 --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,2015 @@ +{ + "add_bos_token": true, + "add_eos_token": false, + "added_tokens_decoder": { + "0": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "1": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "2": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "3": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "4": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "5": { + "content": "<2mass>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "6": { + "content": "[@BOS@]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "7": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "8": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "9": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "10": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "11": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "12": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "13": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "14": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "15": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "16": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "17": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "18": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "19": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "20": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "21": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "22": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "23": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "24": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "25": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "26": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "27": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "28": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "29": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "30": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "31": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "32": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "33": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "34": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "35": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "36": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "37": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "38": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "39": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "40": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "41": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "42": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "43": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "44": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "45": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "46": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "47": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "48": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "49": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "50": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "51": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "52": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "53": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "54": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "55": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "56": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "57": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "58": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "59": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "60": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "61": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "62": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "63": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "64": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "65": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "66": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "67": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "68": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "69": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "70": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "71": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "72": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "73": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "74": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "75": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "76": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "77": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "78": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "79": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "80": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "81": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "82": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "83": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "84": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "85": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "86": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "87": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "88": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "89": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "90": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "91": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "92": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "93": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "94": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "95": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "96": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "97": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "98": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "99": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "100": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "101": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "102": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "103": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "104": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "105": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "106": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "107": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "108": { + "content": "\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "109": { + "content": "\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "110": { + "content": "\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "111": { + "content": "\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "112": { + "content": "\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "113": { + "content": "\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "114": { + "content": "\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "115": { + "content": "\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "116": { + "content": "\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "117": { + "content": "\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "118": { + "content": "\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "119": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "120": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "121": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "122": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "123": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "124": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "125": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "126": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "127": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "128": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "129": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "130": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "131": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "132": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "133": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "134": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "135": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "136": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "137": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "138": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "139": { + "content": "▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "140": { + "content": "▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "141": { + "content": "▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "142": { + "content": "▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "143": { + "content": "▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "144": { + "content": "▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "145": { + "content": "▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "146": { + "content": "▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "147": { + "content": "▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "148": { + "content": "▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "149": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "150": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "152": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "153": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "154": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "155": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "156": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "157": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "158": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "159": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "160": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "161": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "162": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "163": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "164": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "165": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "166": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "167": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "168": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "169": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "170": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "172": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "173": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "174": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "175": { + "content": "
", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "171": { + "content": "
", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "176": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "177": { + "content": "
", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "178": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "179": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "180": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "181": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "182": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "183": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "184": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "185": { + "content": "

", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "186": { + "content": "

", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "187": { + "content": "

", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "188": { + "content": "

", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "189": { + "content": "

", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "190": { + "content": "
", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "191": { + "content": "
", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "192": { + "content": "
", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "193": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "194": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "195": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "196": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "197": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "198": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "199": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "200": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "201": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "202": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "203": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "204": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "205": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "206": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "207": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "208": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "209": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "210": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "211": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "212": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "213": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "214": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "215": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "216": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255968": { + "content": "[toxicity=0]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255969": { + "content": "\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255970": { + "content": "\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255971": { + "content": "\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255972": { + "content": "\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255973": { + "content": "\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255974": { + "content": "\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255975": { + "content": "\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255976": { + "content": "\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255977": { + "content": "\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255978": { + "content": "\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255979": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255980": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255981": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255982": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255983": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255984": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255985": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255986": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255987": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255988": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255989": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255990": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255991": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255992": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255993": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255994": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255995": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255996": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255997": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255998": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255999": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + } + }, + "additional_special_tokens": [ + "", + "" + ], + "bos_token": "", + "clean_up_tokenization_spaces": false, + "eos_token": "", + "extra_special_tokens": {}, + "model_max_length": 1000000000000000019884624838656, + "pad_token": "", + "padding_side": "right", + "sp_model_kwargs": {}, + "spaces_between_special_tokens": false, + "split_special_tokens": false, + "tokenizer_class": "GemmaTokenizer", + "unk_token": "", + "use_default_system_prompt": false +} diff --git a/train_results.json b/train_results.json new file mode 100644 index 0000000..2e39d80 --- /dev/null +++ b/train_results.json @@ -0,0 +1,8 @@ +{ + "epoch": 2.0, + "total_flos": 1.28494219771102e+18, + "train_loss": 0.03479443759803312, + "train_runtime": 6147.788, + "train_samples_per_second": 5.179, + "train_steps_per_second": 0.081 +} \ No newline at end of file diff --git a/trainer_log.jsonl b/trainer_log.jsonl new file mode 100644 index 0000000..1d4f424 --- /dev/null +++ b/trainer_log.jsonl @@ -0,0 +1,60 @@ +{"current_steps": 205, "total_steps": 498, "loss": 0.0977, "lr": 3.6784918420649952e-06, "epoch": 0.8239135895503642, "percentage": 41.16, "elapsed_time": "0:01:41", "remaining_time": "0:02:24"} +{"current_steps": 210, "total_steps": 498, "loss": 0.0731, "lr": 3.600477621728473e-06, "epoch": 0.8440090429540317, "percentage": 42.17, "elapsed_time": "0:03:21", "remaining_time": "0:04:36"} +{"current_steps": 215, "total_steps": 498, "loss": 0.0939, "lr": 3.521110642339991e-06, "epoch": 0.864104496357699, "percentage": 43.17, "elapsed_time": "0:05:03", "remaining_time": "0:06:39"} +{"current_steps": 220, "total_steps": 498, "loss": 0.0747, "lr": 3.4404884655379433e-06, "epoch": 0.8841999497613665, "percentage": 44.18, "elapsed_time": "0:06:43", "remaining_time": "0:08:30"} +{"current_steps": 225, "total_steps": 498, "loss": 0.0625, "lr": 3.358710195908653e-06, "epoch": 0.9042954031650339, "percentage": 45.18, "elapsed_time": "0:08:26", "remaining_time": "0:10:15"} +{"current_steps": 230, "total_steps": 498, "loss": 0.1004, "lr": 3.27587635916233e-06, "epoch": 0.9243908565687013, "percentage": 46.18, "elapsed_time": "0:10:07", "remaining_time": "0:11:48"} +{"current_steps": 235, "total_steps": 498, "loss": 0.1034, "lr": 3.1920887785621233e-06, "epoch": 0.9444863099723687, "percentage": 47.19, "elapsed_time": "0:11:51", "remaining_time": "0:13:16"} +{"current_steps": 240, "total_steps": 498, "loss": 0.0815, "lr": 3.10745044975816e-06, "epoch": 0.9645817633760362, "percentage": 48.19, "elapsed_time": "0:13:33", "remaining_time": "0:14:34"} +{"current_steps": 245, "total_steps": 498, "loss": 0.0887, "lr": 3.022065414180425e-06, "epoch": 0.9846772167797035, "percentage": 49.2, "elapsed_time": "0:15:13", "remaining_time": "0:15:42"} +{"current_steps": 250, "total_steps": 498, "loss": 0.0972, "lr": 2.9360386311461276e-06, "epoch": 1.0040190906807336, "percentage": 50.2, "elapsed_time": "0:16:52", "remaining_time": "0:16:43"} +{"current_steps": 255, "total_steps": 498, "loss": 0.0546, "lr": 2.849475848838749e-06, "epoch": 1.0241145440844008, "percentage": 51.2, "elapsed_time": "0:18:30", "remaining_time": "0:17:38"} +{"current_steps": 260, "total_steps": 498, "loss": 0.0601, "lr": 2.762483474317387e-06, "epoch": 1.0442099974880683, "percentage": 52.21, "elapsed_time": "0:20:09", "remaining_time": "0:18:27"} +{"current_steps": 265, "total_steps": 498, "loss": 0.0596, "lr": 2.6751684427161684e-06, "epoch": 1.0643054508917358, "percentage": 53.21, "elapsed_time": "0:21:51", "remaining_time": "0:19:12"} +{"current_steps": 270, "total_steps": 498, "loss": 0.0577, "lr": 2.587638085794541e-06, "epoch": 1.0844009042954033, "percentage": 54.22, "elapsed_time": "0:23:31", "remaining_time": "0:19:52"} +{"current_steps": 275, "total_steps": 498, "loss": 0.0527, "lr": 2.5e-06, "epoch": 1.1044963576990705, "percentage": 55.22, "elapsed_time": "0:25:11", "remaining_time": "0:20:25"} +{"current_steps": 280, "total_steps": 498, "loss": 0.0592, "lr": 2.41236191420546e-06, "epoch": 1.124591811102738, "percentage": 56.22, "elapsed_time": "0:26:52", "remaining_time": "0:20:55"} +{"current_steps": 285, "total_steps": 498, "loss": 0.0552, "lr": 2.3248315572838316e-06, "epoch": 1.1446872645064055, "percentage": 57.23, "elapsed_time": "0:28:33", "remaining_time": "0:21:20"} +{"current_steps": 290, "total_steps": 498, "loss": 0.0601, "lr": 2.237516525682614e-06, "epoch": 1.164782717910073, "percentage": 58.23, "elapsed_time": "0:30:13", "remaining_time": "0:21:40"} +{"current_steps": 295, "total_steps": 498, "loss": 0.0601, "lr": 2.1505241511612522e-06, "epoch": 1.1848781713137402, "percentage": 59.24, "elapsed_time": "0:31:57", "remaining_time": "0:21:59"} +{"current_steps": 300, "total_steps": 498, "loss": 0.0626, "lr": 2.0639613688538733e-06, "epoch": 1.2049736247174077, "percentage": 60.24, "elapsed_time": "0:33:42", "remaining_time": "0:22:14"} +{"current_steps": 305, "total_steps": 498, "loss": 0.049, "lr": 1.9779345858195757e-06, "epoch": 1.2250690781210751, "percentage": 61.24, "elapsed_time": "0:35:25", "remaining_time": "0:22:24"} +{"current_steps": 310, "total_steps": 498, "loss": 0.055, "lr": 1.8925495502418407e-06, "epoch": 1.2451645315247426, "percentage": 62.25, "elapsed_time": "0:37:07", "remaining_time": "0:22:30"} +{"current_steps": 315, "total_steps": 498, "loss": 0.0608, "lr": 1.8079112214378769e-06, "epoch": 1.2652599849284099, "percentage": 63.25, "elapsed_time": "0:38:48", "remaining_time": "0:22:32"} +{"current_steps": 320, "total_steps": 498, "loss": 0.0526, "lr": 1.7241236408376702e-06, "epoch": 1.2853554383320773, "percentage": 64.26, "elapsed_time": "0:40:27", "remaining_time": "0:22:30"} +{"current_steps": 325, "total_steps": 498, "loss": 0.0509, "lr": 1.6412898040913471e-06, "epoch": 1.3054508917357448, "percentage": 65.26, "elapsed_time": "0:42:09", "remaining_time": "0:22:26"} +{"current_steps": 330, "total_steps": 498, "loss": 0.0631, "lr": 1.5595115344620575e-06, "epoch": 1.3255463451394123, "percentage": 66.27, "elapsed_time": "0:43:54", "remaining_time": "0:22:21"} +{"current_steps": 335, "total_steps": 498, "loss": 0.0524, "lr": 1.47888935766001e-06, "epoch": 1.3456417985430797, "percentage": 67.27, "elapsed_time": "0:45:37", "remaining_time": "0:22:11"} +{"current_steps": 340, "total_steps": 498, "loss": 0.0507, "lr": 1.399522378271527e-06, "epoch": 1.365737251946747, "percentage": 68.27, "elapsed_time": "0:47:16", "remaining_time": "0:21:58"} +{"current_steps": 345, "total_steps": 498, "loss": 0.0487, "lr": 1.3215081579350058e-06, "epoch": 1.3858327053504145, "percentage": 69.28, "elapsed_time": "0:48:58", "remaining_time": "0:21:43"} +{"current_steps": 350, "total_steps": 498, "loss": 0.0544, "lr": 1.2449425954135452e-06, "epoch": 1.405928158754082, "percentage": 70.28, "elapsed_time": "0:50:38", "remaining_time": "0:21:25"} +{"current_steps": 355, "total_steps": 498, "loss": 0.0606, "lr": 1.169919808711659e-06, "epoch": 1.4260236121577492, "percentage": 71.29, "elapsed_time": "0:52:20", "remaining_time": "0:21:05"} +{"current_steps": 360, "total_steps": 498, "loss": 0.0547, "lr": 1.0965320193809808e-06, "epoch": 1.4461190655614167, "percentage": 72.29, "elapsed_time": "0:54:03", "remaining_time": "0:20:43"} +{"current_steps": 365, "total_steps": 498, "loss": 0.0534, "lr": 1.02486943915718e-06, "epoch": 1.4662145189650841, "percentage": 73.29, "elapsed_time": "0:55:45", "remaining_time": "0:20:18"} +{"current_steps": 370, "total_steps": 498, "loss": 0.0541, "lr": 9.550201590674343e-07, "epoch": 1.4863099723687516, "percentage": 74.3, "elapsed_time": "0:57:29", "remaining_time": "0:19:53"} +{"current_steps": 375, "total_steps": 498, "loss": 0.0501, "lr": 8.870700411447817e-07, "epoch": 1.506405425772419, "percentage": 75.3, "elapsed_time": "0:59:15", "remaining_time": "0:19:26"} +{"current_steps": 380, "total_steps": 498, "loss": 0.0611, "lr": 8.21102612882454e-07, "epoch": 1.5265008791760866, "percentage": 76.31, "elapsed_time": "1:01:00", "remaining_time": "0:18:56"} +{"current_steps": 385, "total_steps": 498, "loss": 0.0505, "lr": 7.57198964557942e-07, "epoch": 1.5465963325797538, "percentage": 77.31, "elapsed_time": "1:02:43", "remaining_time": "0:18:24"} +{"current_steps": 390, "total_steps": 498, "loss": 0.0473, "lr": 6.954376495529927e-07, "epoch": 1.5666917859834213, "percentage": 78.31, "elapsed_time": "1:04:23", "remaining_time": "0:17:49"} +{"current_steps": 395, "total_steps": 498, "loss": 0.0519, "lr": 6.358945877920861e-07, "epoch": 1.5867872393870885, "percentage": 79.32, "elapsed_time": "1:06:04", "remaining_time": "0:17:13"} +{"current_steps": 400, "total_steps": 498, "loss": 0.0491, "lr": 5.786429724180801e-07, "epoch": 1.606882692790756, "percentage": 80.32, "elapsed_time": "1:07:48", "remaining_time": "0:16:36"} +{"current_steps": 405, "total_steps": 498, "loss": 0.0441, "lr": 5.237531798197415e-07, "epoch": 1.6269781461944235, "percentage": 81.33, "elapsed_time": "1:10:07", "remaining_time": "0:16:06"} +{"current_steps": 410, "total_steps": 498, "loss": 0.054, "lr": 4.712926831217685e-07, "epoch": 1.647073599598091, "percentage": 82.33, "elapsed_time": "1:11:47", "remaining_time": "0:15:24"} +{"current_steps": 415, "total_steps": 498, "loss": 0.047, "lr": 4.2132596924363666e-07, "epoch": 1.6671690530017584, "percentage": 83.33, "elapsed_time": "1:13:30", "remaining_time": "0:14:42"} +{"current_steps": 420, "total_steps": 498, "loss": 0.0503, "lr": 3.7391445962923824e-07, "epoch": 1.687264506405426, "percentage": 84.34, "elapsed_time": "1:15:14", "remaining_time": "0:13:58"} +{"current_steps": 425, "total_steps": 498, "loss": 0.0456, "lr": 3.2911643474473647e-07, "epoch": 1.7073599598090932, "percentage": 85.34, "elapsed_time": "1:16:56", "remaining_time": "0:13:12"} +{"current_steps": 430, "total_steps": 498, "loss": 0.0535, "lr": 2.869869624374699e-07, "epoch": 1.7274554132127606, "percentage": 86.35, "elapsed_time": "1:18:36", "remaining_time": "0:12:25"} +{"current_steps": 435, "total_steps": 498, "loss": 0.0481, "lr": 2.4757783024395244e-07, "epoch": 1.747550866616428, "percentage": 87.35, "elapsed_time": "1:20:19", "remaining_time": "0:11:37"} +{"current_steps": 440, "total_steps": 498, "loss": 0.0519, "lr": 2.1093748173018846e-07, "epoch": 1.7676463200200954, "percentage": 88.35, "elapsed_time": "1:22:01", "remaining_time": "0:10:48"} +{"current_steps": 445, "total_steps": 498, "loss": 0.0523, "lr": 1.771109569425547e-07, "epoch": 1.7877417734237628, "percentage": 89.36, "elapsed_time": "1:23:43", "remaining_time": "0:09:58"} +{"current_steps": 450, "total_steps": 498, "loss": 0.0454, "lr": 1.4613983704244826e-07, "epoch": 1.8078372268274303, "percentage": 90.36, "elapsed_time": "1:25:25", "remaining_time": "0:09:06"} +{"current_steps": 455, "total_steps": 498, "loss": 0.046, "lr": 1.1806219319275918e-07, "epoch": 1.8279326802310978, "percentage": 91.37, "elapsed_time": "1:27:04", "remaining_time": "0:08:13"} +{"current_steps": 460, "total_steps": 498, "loss": 0.0529, "lr": 9.291253975900138e-08, "epoch": 1.8480281336347653, "percentage": 92.37, "elapsed_time": "1:28:51", "remaining_time": "0:07:20"} +{"current_steps": 465, "total_steps": 498, "loss": 0.0389, "lr": 7.072179188262252e-08, "epoch": 1.8681235870384325, "percentage": 93.37, "elapsed_time": "1:30:35", "remaining_time": "0:06:25"} +{"current_steps": 470, "total_steps": 498, "loss": 0.0489, "lr": 5.151722747865434e-08, "epoch": 1.8882190404421, "percentage": 94.38, "elapsed_time": "1:32:14", "remaining_time": "0:05:29"} +{"current_steps": 475, "total_steps": 498, "loss": 0.0438, "lr": 3.5322453704410286e-08, "epoch": 1.9083144938457672, "percentage": 95.38, "elapsed_time": "1:33:56", "remaining_time": "0:04:32"} +{"current_steps": 480, "total_steps": 498, "loss": 0.0448, "lr": 2.215737794045436e-08, "epoch": 1.9284099472494347, "percentage": 96.39, "elapsed_time": "1:35:43", "remaining_time": "0:03:35"} +{"current_steps": 485, "total_steps": 498, "loss": 0.0436, "lr": 1.2038183319507957e-08, "epoch": 1.9485054006531022, "percentage": 97.39, "elapsed_time": "1:37:25", "remaining_time": "0:02:36"} +{"current_steps": 490, "total_steps": 498, "loss": 0.0419, "lr": 4.977308833379324e-09, "epoch": 1.9686008540567697, "percentage": 98.39, "elapsed_time": "1:39:07", "remaining_time": "0:01:37"} +{"current_steps": 495, "total_steps": 498, "loss": 0.048, "lr": 9.834340423678368e-10, "epoch": 1.9886963074604371, "percentage": 99.4, "elapsed_time": "1:40:50", "remaining_time": "0:00:36"} +{"current_steps": 498, "total_steps": 498, "epoch": 2.0, "percentage": 100.0, "elapsed_time": "1:42:27", "remaining_time": "0:00:00"} diff --git a/trainer_state.json b/trainer_state.json new file mode 100644 index 0000000..c073945 --- /dev/null +++ b/trainer_state.json @@ -0,0 +1,736 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 2.0, + "eval_steps": 500, + "global_step": 498, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.02009545340366742, + "grad_norm": 101.4629135131836, + "learning_rate": 4.0000000000000003e-07, + "loss": 2.9256, + "step": 5 + }, + { + "epoch": 0.04019090680733484, + "grad_norm": 83.51904296875, + "learning_rate": 9.000000000000001e-07, + "loss": 2.7459, + "step": 10 + }, + { + "epoch": 0.06028636021100226, + "grad_norm": 25.10388946533203, + "learning_rate": 1.4000000000000001e-06, + "loss": 1.637, + "step": 15 + }, + { + "epoch": 0.08038181361466969, + "grad_norm": 13.024300575256348, + "learning_rate": 1.9000000000000002e-06, + "loss": 1.1459, + "step": 20 + }, + { + "epoch": 0.1004772670183371, + "grad_norm": 8.315956115722656, + "learning_rate": 2.4000000000000003e-06, + "loss": 0.7456, + "step": 25 + }, + { + "epoch": 0.12057272042200452, + "grad_norm": 8.930346488952637, + "learning_rate": 2.9e-06, + "loss": 0.5717, + "step": 30 + }, + { + "epoch": 0.14066817382567195, + "grad_norm": 7.356456756591797, + "learning_rate": 3.4000000000000005e-06, + "loss": 0.414, + "step": 35 + }, + { + "epoch": 0.16076362722933937, + "grad_norm": 5.648008346557617, + "learning_rate": 3.900000000000001e-06, + "loss": 0.2577, + "step": 40 + }, + { + "epoch": 0.1808590806330068, + "grad_norm": 6.329988956451416, + "learning_rate": 4.4e-06, + "loss": 0.2394, + "step": 45 + }, + { + "epoch": 0.2009545340366742, + "grad_norm": 5.1738080978393555, + "learning_rate": 4.9000000000000005e-06, + "loss": 0.1848, + "step": 50 + }, + { + "epoch": 0.22104998744034163, + "grad_norm": 4.86478853225708, + "learning_rate": 4.999016565957633e-06, + "loss": 0.2195, + "step": 55 + }, + { + "epoch": 0.24114544084400905, + "grad_norm": 4.442976474761963, + "learning_rate": 4.995022691166621e-06, + "loss": 0.1582, + "step": 60 + }, + { + "epoch": 0.26124089424767644, + "grad_norm": 3.704345941543579, + "learning_rate": 4.987961816680493e-06, + "loss": 0.183, + "step": 65 + }, + { + "epoch": 0.2813363476513439, + "grad_norm": 3.772160768508911, + "learning_rate": 4.977842622059546e-06, + "loss": 0.164, + "step": 70 + }, + { + "epoch": 0.30143180105501133, + "grad_norm": 3.2532081604003906, + "learning_rate": 4.96467754629559e-06, + "loss": 0.1781, + "step": 75 + }, + { + "epoch": 0.32152725445867875, + "grad_norm": 2.893615484237671, + "learning_rate": 4.948482772521346e-06, + "loss": 0.1288, + "step": 80 + }, + { + "epoch": 0.34162270786234616, + "grad_norm": 4.654979705810547, + "learning_rate": 4.929278208117378e-06, + "loss": 0.1444, + "step": 85 + }, + { + "epoch": 0.3617181612660136, + "grad_norm": 2.8989882469177246, + "learning_rate": 4.907087460240999e-06, + "loss": 0.1151, + "step": 90 + }, + { + "epoch": 0.381813614669681, + "grad_norm": 3.7547988891601562, + "learning_rate": 4.881937806807241e-06, + "loss": 0.1197, + "step": 95 + }, + { + "epoch": 0.4019090680733484, + "grad_norm": 2.273831605911255, + "learning_rate": 4.8538601629575525e-06, + "loss": 0.0966, + "step": 100 + }, + { + "epoch": 0.42200452147701584, + "grad_norm": 2.4642996788024902, + "learning_rate": 4.822889043057446e-06, + "loss": 0.1088, + "step": 105 + }, + { + "epoch": 0.44209997488068326, + "grad_norm": 3.29301381111145, + "learning_rate": 4.789062518269812e-06, + "loss": 0.1163, + "step": 110 + }, + { + "epoch": 0.4621954282843507, + "grad_norm": 2.45603084564209, + "learning_rate": 4.752422169756048e-06, + "loss": 0.1151, + "step": 115 + }, + { + "epoch": 0.4822908816880181, + "grad_norm": 3.199894666671753, + "learning_rate": 4.713013037562531e-06, + "loss": 0.098, + "step": 120 + }, + { + "epoch": 0.5023863350916855, + "grad_norm": 1.7128061056137085, + "learning_rate": 4.670883565255264e-06, + "loss": 0.0968, + "step": 125 + }, + { + "epoch": 0.5224817884953529, + "grad_norm": 2.645235776901245, + "learning_rate": 4.6260855403707625e-06, + "loss": 0.1068, + "step": 130 + }, + { + "epoch": 0.5425772418990203, + "grad_norm": 2.3140530586242676, + "learning_rate": 4.578674030756364e-06, + "loss": 0.0914, + "step": 135 + }, + { + "epoch": 0.5626726953026878, + "grad_norm": 1.8588067293167114, + "learning_rate": 4.528707316878233e-06, + "loss": 0.0995, + "step": 140 + }, + { + "epoch": 0.5827681487063552, + "grad_norm": 1.9172967672348022, + "learning_rate": 4.476246820180259e-06, + "loss": 0.08, + "step": 145 + }, + { + "epoch": 0.6028636021100227, + "grad_norm": 2.9082584381103516, + "learning_rate": 4.421357027581921e-06, + "loss": 0.1066, + "step": 150 + }, + { + "epoch": 0.62295905551369, + "grad_norm": 3.6944031715393066, + "learning_rate": 4.364105412207914e-06, + "loss": 0.1124, + "step": 155 + }, + { + "epoch": 0.6430545089173575, + "grad_norm": 1.5622600317001343, + "learning_rate": 4.304562350447008e-06, + "loss": 0.0856, + "step": 160 + }, + { + "epoch": 0.6631499623210249, + "grad_norm": 2.388239860534668, + "learning_rate": 4.242801035442059e-06, + "loss": 0.0766, + "step": 165 + }, + { + "epoch": 0.6832454157246923, + "grad_norm": 1.7182718515396118, + "learning_rate": 4.178897387117547e-06, + "loss": 0.104, + "step": 170 + }, + { + "epoch": 0.7033408691283597, + "grad_norm": 2.2367119789123535, + "learning_rate": 4.11292995885522e-06, + "loss": 0.0737, + "step": 175 + }, + { + "epoch": 0.7234363225320272, + "grad_norm": 2.007200241088867, + "learning_rate": 4.044979840932567e-06, + "loss": 0.0893, + "step": 180 + }, + { + "epoch": 0.7435317759356945, + "grad_norm": 2.6910905838012695, + "learning_rate": 3.975130560842821e-06, + "loss": 0.0944, + "step": 185 + }, + { + "epoch": 0.763627229339362, + "grad_norm": 2.042320489883423, + "learning_rate": 3.9034679806190204e-06, + "loss": 0.0843, + "step": 190 + }, + { + "epoch": 0.7837226827430294, + "grad_norm": 2.700977087020874, + "learning_rate": 3.830080191288342e-06, + "loss": 0.0669, + "step": 195 + }, + { + "epoch": 0.8038181361466968, + "grad_norm": 3.0307390689849854, + "learning_rate": 3.755057404586455e-06, + "loss": 0.0955, + "step": 200 + }, + { + "epoch": 0.8239135895503642, + "grad_norm": 1.0048880577087402, + "learning_rate": 3.6784918420649952e-06, + "loss": 0.0977, + "step": 205 + }, + { + "epoch": 0.8440090429540317, + "grad_norm": 1.3049492835998535, + "learning_rate": 3.600477621728473e-06, + "loss": 0.0731, + "step": 210 + }, + { + "epoch": 0.864104496357699, + "grad_norm": 2.020117998123169, + "learning_rate": 3.521110642339991e-06, + "loss": 0.0939, + "step": 215 + }, + { + "epoch": 0.8841999497613665, + "grad_norm": 2.2311348915100098, + "learning_rate": 3.4404884655379433e-06, + "loss": 0.0747, + "step": 220 + }, + { + "epoch": 0.9042954031650339, + "grad_norm": 2.1661417484283447, + "learning_rate": 3.358710195908653e-06, + "loss": 0.0625, + "step": 225 + }, + { + "epoch": 0.9243908565687013, + "grad_norm": 2.463257074356079, + "learning_rate": 3.27587635916233e-06, + "loss": 0.1004, + "step": 230 + }, + { + "epoch": 0.9444863099723687, + "grad_norm": 1.5093741416931152, + "learning_rate": 3.1920887785621233e-06, + "loss": 0.1034, + "step": 235 + }, + { + "epoch": 0.9645817633760362, + "grad_norm": 2.1394777297973633, + "learning_rate": 3.10745044975816e-06, + "loss": 0.0815, + "step": 240 + }, + { + "epoch": 0.9846772167797035, + "grad_norm": 1.6689902544021606, + "learning_rate": 3.022065414180425e-06, + "loss": 0.0887, + "step": 245 + }, + { + "epoch": 1.0040190906807336, + "grad_norm": 1.9693433046340942, + "learning_rate": 2.9360386311461276e-06, + "loss": 0.0972, + "step": 250 + }, + { + "epoch": 1.0241145440844008, + "grad_norm": 0.750286340713501, + "learning_rate": 2.849475848838749e-06, + "loss": 0.0546, + "step": 255 + }, + { + "epoch": 1.0442099974880683, + "grad_norm": 3.802541732788086, + "learning_rate": 2.762483474317387e-06, + "loss": 0.0601, + "step": 260 + }, + { + "epoch": 1.0643054508917358, + "grad_norm": 1.3461735248565674, + "learning_rate": 2.6751684427161684e-06, + "loss": 0.0596, + "step": 265 + }, + { + "epoch": 1.0844009042954033, + "grad_norm": 2.324124813079834, + "learning_rate": 2.587638085794541e-06, + "loss": 0.0577, + "step": 270 + }, + { + "epoch": 1.1044963576990705, + "grad_norm": 1.4222259521484375, + "learning_rate": 2.5e-06, + "loss": 0.0527, + "step": 275 + }, + { + "epoch": 1.124591811102738, + "grad_norm": 0.48392441868782043, + "learning_rate": 2.41236191420546e-06, + "loss": 0.0592, + "step": 280 + }, + { + "epoch": 1.1446872645064055, + "grad_norm": 1.6113179922103882, + "learning_rate": 2.3248315572838316e-06, + "loss": 0.0552, + "step": 285 + }, + { + "epoch": 1.164782717910073, + "grad_norm": 2.1131763458251953, + "learning_rate": 2.237516525682614e-06, + "loss": 0.0601, + "step": 290 + }, + { + "epoch": 1.1848781713137402, + "grad_norm": 1.010076880455017, + "learning_rate": 2.1505241511612522e-06, + "loss": 0.0601, + "step": 295 + }, + { + "epoch": 1.2049736247174077, + "grad_norm": 1.5603766441345215, + "learning_rate": 2.0639613688538733e-06, + "loss": 0.0626, + "step": 300 + }, + { + "epoch": 1.2250690781210751, + "grad_norm": 0.9277430176734924, + "learning_rate": 1.9779345858195757e-06, + "loss": 0.049, + "step": 305 + }, + { + "epoch": 1.2451645315247426, + "grad_norm": 1.768437147140503, + "learning_rate": 1.8925495502418407e-06, + "loss": 0.055, + "step": 310 + }, + { + "epoch": 1.2652599849284099, + "grad_norm": 1.4469853639602661, + "learning_rate": 1.8079112214378769e-06, + "loss": 0.0608, + "step": 315 + }, + { + "epoch": 1.2853554383320773, + "grad_norm": 1.503761649131775, + "learning_rate": 1.7241236408376702e-06, + "loss": 0.0526, + "step": 320 + }, + { + "epoch": 1.3054508917357448, + "grad_norm": 1.1352767944335938, + "learning_rate": 1.6412898040913471e-06, + "loss": 0.0509, + "step": 325 + }, + { + "epoch": 1.3255463451394123, + "grad_norm": 2.6707780361175537, + "learning_rate": 1.5595115344620575e-06, + "loss": 0.0631, + "step": 330 + }, + { + "epoch": 1.3456417985430797, + "grad_norm": 1.1756007671356201, + "learning_rate": 1.47888935766001e-06, + "loss": 0.0524, + "step": 335 + }, + { + "epoch": 1.365737251946747, + "grad_norm": 1.0455981492996216, + "learning_rate": 1.399522378271527e-06, + "loss": 0.0507, + "step": 340 + }, + { + "epoch": 1.3858327053504145, + "grad_norm": 0.7072739601135254, + "learning_rate": 1.3215081579350058e-06, + "loss": 0.0487, + "step": 345 + }, + { + "epoch": 1.405928158754082, + "grad_norm": 1.145075798034668, + "learning_rate": 1.2449425954135452e-06, + "loss": 0.0544, + "step": 350 + }, + { + "epoch": 1.4260236121577492, + "grad_norm": 1.4399057626724243, + "learning_rate": 1.169919808711659e-06, + "loss": 0.0606, + "step": 355 + }, + { + "epoch": 1.4461190655614167, + "grad_norm": 1.0342814922332764, + "learning_rate": 1.0965320193809808e-06, + "loss": 0.0547, + "step": 360 + }, + { + "epoch": 1.4662145189650841, + "grad_norm": 0.5163133144378662, + "learning_rate": 1.02486943915718e-06, + "loss": 0.0534, + "step": 365 + }, + { + "epoch": 1.4863099723687516, + "grad_norm": 1.0042380094528198, + "learning_rate": 9.550201590674343e-07, + "loss": 0.0541, + "step": 370 + }, + { + "epoch": 1.506405425772419, + "grad_norm": 1.1944531202316284, + "learning_rate": 8.870700411447817e-07, + "loss": 0.0501, + "step": 375 + }, + { + "epoch": 1.5265008791760866, + "grad_norm": 1.3594350814819336, + "learning_rate": 8.21102612882454e-07, + "loss": 0.0611, + "step": 380 + }, + { + "epoch": 1.5465963325797538, + "grad_norm": 1.2450895309448242, + "learning_rate": 7.57198964557942e-07, + "loss": 0.0505, + "step": 385 + }, + { + "epoch": 1.5666917859834213, + "grad_norm": 1.6389065980911255, + "learning_rate": 6.954376495529927e-07, + "loss": 0.0473, + "step": 390 + }, + { + "epoch": 1.5867872393870885, + "grad_norm": 2.077847719192505, + "learning_rate": 6.358945877920861e-07, + "loss": 0.0519, + "step": 395 + }, + { + "epoch": 1.606882692790756, + "grad_norm": 1.5446641445159912, + "learning_rate": 5.786429724180801e-07, + "loss": 0.0491, + "step": 400 + }, + { + "epoch": 1.6269781461944235, + "grad_norm": 1.4639307260513306, + "learning_rate": 5.237531798197415e-07, + "loss": 0.0441, + "step": 405 + }, + { + "epoch": 1.647073599598091, + "grad_norm": 1.4568461179733276, + "learning_rate": 4.712926831217685e-07, + "loss": 0.054, + "step": 410 + }, + { + "epoch": 1.6671690530017584, + "grad_norm": 1.6878377199172974, + "learning_rate": 4.2132596924363666e-07, + "loss": 0.047, + "step": 415 + }, + { + "epoch": 1.687264506405426, + "grad_norm": 0.9864447116851807, + "learning_rate": 3.7391445962923824e-07, + "loss": 0.0503, + "step": 420 + }, + { + "epoch": 1.7073599598090932, + "grad_norm": 0.9351651668548584, + "learning_rate": 3.2911643474473647e-07, + "loss": 0.0456, + "step": 425 + }, + { + "epoch": 1.7274554132127606, + "grad_norm": 0.9683827757835388, + "learning_rate": 2.869869624374699e-07, + "loss": 0.0535, + "step": 430 + }, + { + "epoch": 1.747550866616428, + "grad_norm": 1.0934531688690186, + "learning_rate": 2.4757783024395244e-07, + "loss": 0.0481, + "step": 435 + }, + { + "epoch": 1.7676463200200954, + "grad_norm": 1.5016802549362183, + "learning_rate": 2.1093748173018846e-07, + "loss": 0.0519, + "step": 440 + }, + { + "epoch": 1.7877417734237628, + "grad_norm": 1.4125523567199707, + "learning_rate": 1.771109569425547e-07, + "loss": 0.0523, + "step": 445 + }, + { + "epoch": 1.8078372268274303, + "grad_norm": 1.1559640169143677, + "learning_rate": 1.4613983704244826e-07, + "loss": 0.0454, + "step": 450 + }, + { + "epoch": 1.8279326802310978, + "grad_norm": 1.0382763147354126, + "learning_rate": 1.1806219319275918e-07, + "loss": 0.046, + "step": 455 + }, + { + "epoch": 1.8480281336347653, + "grad_norm": 1.4664262533187866, + "learning_rate": 9.291253975900138e-08, + "loss": 0.0529, + "step": 460 + }, + { + "epoch": 1.8681235870384325, + "grad_norm": 0.8224850296974182, + "learning_rate": 7.072179188262252e-08, + "loss": 0.0389, + "step": 465 + }, + { + "epoch": 1.8882190404421, + "grad_norm": 0.5471680760383606, + "learning_rate": 5.151722747865434e-08, + "loss": 0.0489, + "step": 470 + }, + { + "epoch": 1.9083144938457672, + "grad_norm": 0.9190619587898254, + "learning_rate": 3.5322453704410286e-08, + "loss": 0.0438, + "step": 475 + }, + { + "epoch": 1.9284099472494347, + "grad_norm": 0.8270437121391296, + "learning_rate": 2.215737794045436e-08, + "loss": 0.0448, + "step": 480 + }, + { + "epoch": 1.9485054006531022, + "grad_norm": 0.6661733388900757, + "learning_rate": 1.2038183319507957e-08, + "loss": 0.0436, + "step": 485 + }, + { + "epoch": 1.9686008540567697, + "grad_norm": 0.7018892168998718, + "learning_rate": 4.977308833379324e-09, + "loss": 0.0419, + "step": 490 + }, + { + "epoch": 1.9886963074604371, + "grad_norm": 0.8652178645133972, + "learning_rate": 9.834340423678368e-10, + "loss": 0.048, + "step": 495 + }, + { + "epoch": 2.0, + "step": 498, + "total_flos": 1.28494219771102e+18, + "train_loss": 0.03479443759803312, + "train_runtime": 6147.788, + "train_samples_per_second": 5.179, + "train_steps_per_second": 0.081 + } + ], + "logging_steps": 5, + "max_steps": 498, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 200, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 1.28494219771102e+18, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/training_args.bin b/training_args.bin new file mode 100644 index 0000000..b697238 --- /dev/null +++ b/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:62f0adb9f98878d5918d58cd798bf85ae88dd0ed2328bb63223e8143e2502520 +size 6225 diff --git a/training_loss.png b/training_loss.png new file mode 100644 index 0000000000000000000000000000000000000000..96135df90254b8ec5179083f2d6c482e8e707ed1 GIT binary patch literal 31316 zcmdqJg!n52k^Gzv&}$3ZDY$^ay!q(zXB4hca}LJ5(S zmJR`t`pw1r-S7VHzi@r@dEe*6-fOQl*BE1tIoEltu6mi8f|&xtFzTyUE^1;J0UpC} zqhuuTibU7o1pG(F`I3(FO*=DZH)F^9n2NEpy|taQ^#c1(?HtWTPAY%D16LupzjDV3!)T4sKU|hV<^v4#sJnXcyq0^y!jPNy zbC)sF<&Nixk}8`4nyoJgPPg*&C375c7_I-vIx2hQV&K|~b4R+*%}F(Sq`DJq-n@6p zJ(!H^=Dnq`r%II0ov+&8T&<~9_$Dh}92gxg{&{-$w8;E(bMH0Tt82gA~j@;_4p~Q!albWf%WkBjWa}p7=|w7~%HDLas|9+zW<2|Koj{!FGFVYdv>_?8A@{ zvYByW-0x_y8mIHp&u*5$s3C5>gs7r`+rx))%o5n*o7zg z?UhGeA3xsO+FIZ;r@gaa=hyJ&>hsxRl}ne1d*9x9^y0;flfwFKYvU0LCXFHV>bX5= z>;`_jE>x^ChiwKvxVUxAPOAC+V+f*UUo5;q=AZf6lLB79l&hULTb2J(;P%*eH3geO zIvs6ol9xHghL>*T-@U0*V1%P#l?j+m@~gSJVb~K$$+-M)E|GckZ|^mQTYB1~!ESVkTXwm&TU-_A4!Va$a^?mW;v79pzO`$a0-mli#}7*xXE+ zsLK0qV?jXD=BJiTQb~E$pc0wGZ1*XL5ueS+qoYPCeRG%g_ja3}91+N|+3vc`{x~3j z2)(GT9(m=)jl(0}YhgQ8pYI*&klRcM2@Pdn4pR;Y4i5d*5F9#G>1J4>=e03UUpnlW zOGZXUx3{-62wNJcJ7gtl@*h{oh{+*_OD-$+{{!xGF0Kh0GH?D;7}a& zoXazT&k4^-=P#9x_zLW8l-53ujb-Az`mCj+gHeg>z0H7RR6>H_Q{}YIxpkA}(T`|G zdkot0Yxns7{rSyZtea=i_bxG3&g+A>f$-+Y{@(2?&v;IsIWxIAH5mbO+cuZ4SFTCL zCac3lMM6(U#{u6OceDwnI>uGuy1-AvE-x5F!^;21vpVGQWBR44HPsT+n3$N*J_9e6 zV!IIo3vuSI3Ky%Du~3mGtE-MN>j_B>_uHZ$ztgXz2vg=hbm&l{n!>Kuu9Csj@64p5 z&v?r9N^_4X_FJ?vV>pizGEe%%( zc3B+u_4OSKI&Moh*PEB?N`;xgAXhuu=bXkG$x$@XGcobO6vT=eKjb@gD*J;^&)k$& zOh@J#v!Sni!jjx{3ESy=SMrj$rz6(HDRXSvxjaK>9CwZ~8I~4Zk3#;BGiLL6r#qnTua+`yXM*Q|!M|`)1%)Z7278HnLitg@Z zRqX)H1@0>*H{!&}A^IJQegw1ohMZffVlZtE4{S2|SRY8~u)n)$3L)!~bIEg~-<+O> z<>~M5nd4txo%uWZh*tE$Hy%OV;+dR*n%#{7b-u%Y*OK<1*4HalZ7)_bopTlZU^gP1 zzt(S_Wc=;T)v}dO$5-kpMMmMDtB7_GxukJRYf+Iq}pXBIgB)FlsO_fuu=z15gn#XRltwGM@5V72!Sy?x$8+;*VCykN#>R#~3=&|fN%CQnZ876}+e??dy=4mv3zxU1lT2G8 zIT06_n3(9bt^%6CJ{|4w+dK2c?2I@wb#LC?;3)3PTw-GM-!s*Ml~U>k%bgt7zMdTi zD!NqbyNgNvyBfwMM)2#`uUIitlDFCd8EQ%4EiEm&hHfS%TwL7TV(!aEH(tvgWB2(( zzxS`NHWg+QjIgaYE3iI9}$?QN%xq`g19 zdS&)FtKPRC_jgD3d&=73IW-In!g-Ut59#I|<>cfPeRbpSYMaPP=LOndk9l7!YG}~! z?JoIoOG_U+=Q^KN@}MY3%=8n7ygb{fQ>T6unbX$ntX>C+HVPwkxRXEYLO8p(^)|P5 zcfpQNP_PkTU(~8s5X;(-{@1Lqhr9Z08J>{y+_~fI-pe?k;c%brjq~|^eOo&E70w@z z+g14k)KCM;^Vn9cC}VYX{)Ce*+W7|5SDzsjgw+Y;S3TT^5xLL5Z${HWL_}n?KHGz0 zcG!DO<NyZ}L9GcBtZUyj@NIeGeN6d_yoTqidzU!5+jZS-R#a3JQve`z1Y?>+f%u zV!xVsY{3TR!$LsZP;dr_#((7!7uWi;FmPK-ORHrL)=Uj6%V4ep(O`hl)$aYvt$Lf4 zbp@y^6r>(S{MVBKbYt%-7Z2%Ix>k?X5E2qLH9ls@*9~e515yN-MQrBL4_f1*q9UG2 z*1e$Wt)-BPeuBKu&-4v$1%+DGf##+ra?AwaZMATI-a4hnvZ*`!=H&g!$;s8NY+Yrb z|6%uWYL8cI|2|mtwNLKeFCDnl9L{>l(Xp`2E2Fqr3ytQkZTTewNzXqPDc^rLf}EMW ze>t3m{#ttBC+0d}*Rt-5LU#(^=;kHSDS>_r!Q-*6T0cJxZSAR5y`Iz3R^fX9LoIE9 z;2}?*9D==@Up2oEsxm++#oj}KoU?JhzX(8oz^?Xl8%72kq54|(l*mA>UoFNVzxQ|C zGXL{$zy0kbjAL&-->;`a)#jvXg3kDsV@>H(Wz!O7uiT#5)$+sjD+W)VK0O>E|1Yb} zv0IbSeaq8r#HW;unmSKAyeB>eZMUL4=0ZlE>R$%X#(KuzO6<$M_1g*T$+P$x~xLN~{j$SOq{^x|>H19}ZX; zC>;m>aFMV1YSsXh^t3DOavB8I>gvplh3w99!;dPDGdE6tm#8{@d3v?3PI0T`_qrzp zne~qX+}76C?LkRS&hzs7__U2OxaevL#ksp`8HtY4`u%;hQ>2`B9za*G{qwUE*E`d% zGwQhwxvb88{`@&=`$3*z?eP61*qoOkxtL4><>Sc|rX1sAVEgeQF-U%8nWn`EIZojkc9{;i6`c}6j=jV1YC_wk!^}dzK7(*_AL}_X1SOveDwxz0O zc0ZPD>4y-EL$+lPoA=F|H%6e2F_uRT_zJ&IEo)+!W@1xwb6=S^a6FT!5#}tz3WAxA zjV-)Gew!*VFwo_vU5|(b<|taJJDSz@p`ETj^qLc#xo|`EsIJi<4@c_W$;iE^##+pL;1O(8qau>Ee({Q7_5V!PC>Hj_AUOG62gp1;=ezr zN?Tp6$t9na$1TI&p#+X6-`0LG{G?tfSDd z=bTX0zu8>(KV6qsK+$mV@DNc162R{F4lALEshv)kLpLbd;0E*ndUEmnstVEE2!iA1 z48z5~$pnaM=cQrRrP_Vpu(?m4KJ|Jf-4X(S;p*l6-alTy=JIPlfwB?>YCtmx2`*kU z`{K20Pgs)kGlcVtIQ#NG*jP*6t={bE9PioC6+!Z?7zEp==vYuuZNC;qKR%MM^@E-C ztNq+k9c)5DYWCkT?v7^cN}<)(iGQFx&u4x&1}Yc+UiBsMT&Y;7 z>({ZdF-(etZ_xR>%Bo{-&l5o6QCPd$3?DWH->RRhpq2D(Zp(zpIEf8{*!py3iAEE&076v#>0nO z2qWG#_WyYx&Yb(z>GmymcPU7pb^>*JCa3d~ld-!Wtg|K6Ol*=!&5wbNpp|E~r3Z5^7qPlGZCbKrh=piy>QP2zmZBvs#K9DB+4 z^4gVW6RWq^;NjyYTuu8@fwO>dn-Zj*j7oGTCt=(wiE*)3jsrHO#<1OgRJef5dGm(I z(0e7Y&2YNByj*wy=7oocN0H1Q#OM#}50)yYz`}#aSaE9kh9IP=_Yh4wqW$MQb^%AP(G5)!)P;Zfe^3EXp8QBkqlsz2d=ZB>RPFoa9Z znM@o8 zW4Jk__lno9T`O^N5|ZdZ=)JWzEoR=HVA=8V<;xVn@ZoLe3h-lmwH z^o~&C=eB;-_*Ax;XQWtG+Pds|%RFiafj_YhoLK|g#j<&IbF? z6^Fqo5ca!K0;@mp!RAquNmNXp&ihndx3Hh>Zra*MH|G0qnyxKvFV#i}-Tg2cDpFGi za`Z8zYh#d2E32xi=G`Y@`RD?lD}x+xlDV$J7l`FON#wo9j~` zVR%L2n1towVZVJJWG%S={S_RQm>356`C$ZjaMifnnUZf7;wYtg4NlUa_u5yWPpb9= z=|>)GQz;s&M0mAY_Efm)SnT<2it^gC{yGEWvR*Rh@4c;@o7SB?_ftY@ou5Eqf@YJCd6J ztYGq2R{@6uw(*E8+Oxpj~|e>u}p2i_B~mvY}bef{Bh*iziKWxq(kimhPDHSc(> z&i@htxVnxGVn2?ljEuZsj?MJtC;eS09l2;?l3KRjdpF-x^_@7qGnj&%znxPJYkhl8#_jH` zbuEy&Ig7@8U-$PY39NDL3ixqPLwY9ZmR8)s|w>M*WQ}R5+ zmBW~C+0@1umV&D$<}#~A&&nzUG{Piq5ecYjm6)IZeYo1I38;-bX#lyZW?z`zyDsR) ztV`Ms@*QGe2vknXcHpw891@1gNF!*EyW@~AyyE^@V#TY*dUIWF3%P+*EK=&=yFR&gX2JAYStG7i7j5gY@xLl4n&L^n?xI3={P@!|^Xinb+2PRo6kUdg5w z2rEeV10XmhYHBLrKV#rI9RX`Z>De1*-iI{_L$-`|)$7qaujYDkV|3n|Tnfu)ExAGH zv@&)nW<5uBz_Dab8|*r#YH(XAAxj1OqzSau?f2lAS@u(k@p+TD%2 zVi3i!vpCU&fT7y<=YlFIffu}a>}YyQtZs)feA@>WSq<23A2A2>I;m{o!+>6Ik#1h9 z-AGL;R7Hfpn~^&_xc2r^=+Vz@Y~IV~>5m^55N8fUHYZ93T3TCkR#-jtu6xv#)P&N zzfTI@DI|h%7(Qohwk!c>;4ru)?o+SK145GCn zQ(fo=(1IBFB}o*1sxXB|8>IpA`nJFq-{J+N64FA-D(yItQ_3+RYkzS#^nZ$Q_H<;h7*L~Q zuOoqUqRjpm0sQfH{KEsU@*Yx3&dMSa6BApV7_AAy72t%ya#GpUI{{(2d0kEIGKknA zeX3F;%o-170v3L@vt-cgCCJOTq|f)1tgYYf$SsC4VaMzG=dHw{Ai9k<}+b zODInQMPAL)`0Uv;l(gtO#GkkxJqkGRNHy~Um_6?uCtJ<E?&4 zk5{@b)I+7?Y2{)CWiT!KJBM9FvdDo%W+K!qOdwrFUMZ-w?k^{JVZfHY2Ipvvu3RBQ z1rFJbAErI8y+CEEYid3`BB0&UMnc7ehq*`a>mnZn!#6VGprEAeC%OSOP0qi&3xPhE zjpXrY!8;G1Jv+p6;sjW3pfbhow;qnU5BaQ4jkm{{7f{>OW#%M;?9^H^zA{uPOfg$a zDP42RnY}i5B@S~39)qu@K=}s}K0uZWkTr(QnVFeER`u=voqP9?!=LX?Rn!lE1^2B_ zzhXA9=i+h{2yJ@!^eGiEPX#Lb?dDk-&p9)&QcVD?{ZtC@`cgf55&$W+KJ} zQ4gdN_~ON3tj?)JE|~K|kP*Noz{}yI!g?V9fWJU~*-y7|pfEUUKp#<5R3v6QDCN93 zc&s6ax&ai-d(U;tLB}>B3=iHkphC+WEsGQ#R(JkAXGq<|Uow;ISZ!|LTt>!{Y-G05Ce2y!>2R3CYqR<1{MjHxgxn zzq2JaAPpE^J{kNh03Z1pG~Um&x2G)lGm1pJ^ANp;3V7hi?MD?^XPH6!25Kq+RHf4vNaR0sisBE)H z=6LKs9jF={;{AW?DNKzPG~Z{$RpoX6WRPp(cnp zUF~?*P`5;QhB}8=e!(vygOzN3W8+L?Yn*Iyf9_}-tHdopv z^PWI|#SmyWbqx*w^N&bAw{=@efcOngyByKlhJ3L(dBCnH(9^!V|30_9xOM^-KHqaE zr{wW79!`FKivIro)j}Xp)zXmsGl7Z(@H)XH0WzQtMl7xSNZ>e-y`F_l5% zj{ceK3A20_{S}jT$Nf)```H-alyliHD4?$O)27+h8Y-V*HEMPAlDW@#$XzSbj(Gt) zpjs9x+Vk-8q80?ut&OmH@oxm3ZQvP|0{>azu<|CeVv293&iPDOT?PZ=&>cZ8uKY|* z{=wC(h>eAT3sBwxZSy?lmX;PmI;4%i%M%b1(jPl^ik1b3f!>z$+UU*|j2S4rPYE`| za%t@@HS|H82Rw6tC?i>T&dhFw-OXCYs5PCgMc$k*OB@PgZNRZKeE43@mtSx`Q@f%R zx>F`3;UQ4%9MwdT31joK_#?=#P$IB{bQ!T~urDDiUX(+>to%>>3PbzIO} z(EV4gUvB{a7KH}tB>;n{Ut%dDjs)Fb^idj#Xyy+BPNH|A6p9z|M<VRN2PqOh(VHY$t4}rx1NyKsSQya+9 z0!4v)eo{rLXj!qIEbcmkNk zF_scMCR%or2{RgBnkR*hA8NOa`8%+xq)^E^N5du?l9}Yo3_OF9)FFLwOA{)*kWSVh zm3KCKG7ql#bZDdvzneKgTMH%MVTv#DXK6u}Hzl8cgi1i*AdAd=3(UJQ9R5gAx1gYv zxo!~t(fLtVG!c!BZYk|}a*=MHtT4|ZfSw|n_wU~eb3gv_kI!N-!7SG#hL9 zxu5FGZzIqlGr9muMlvObND|YYNtQNtQOTB>N@1gY4AKPy(!Rm)k=Y1bK_KOtn+?e# zD}q^oupIx z{y3evf}$$v1(4ExcyzUm?~2)&p!T^NliFC_sA{+`zOrK~?T59C`3kxyA-X6=#KNvm#cOl9t9k+WQ?!+0dKy&luilCr_bl zfc{5ZjyQ;n1AfpHf3Nis77wYj*@p>a*Gl3NV z3ZBfaY6X=`4ZK$_fpL%wN<8+Q+qHUQc$#7o+YPEQBJo;+S&{uX;cMxscY+{c%4Q-Fg=T}zsdr#tylJe>ko z=kG*pE67S8wMnOx+->60)*$`;fbZSyj-iwP-aB> z4~pzi$G9e>%~SQ^r_g=rRprw=;yE&TmF696JSR_%!6t|P8Yd51FaePKf&x8tV7*3% z(~wIaCl?n1tPTP8!&(wdm9aLF+Qvt;@*&brQ={oI21=fuGTB|0mtdn zhw^kw>Y-S=(pS58OI{oD*iHaWZ-8Jarzxtj@$vU=O9p>HJL-ChXQyYIwh7zpzLcYdpX8%ASZ5Z zyE5H`IyqSV6c2_U@_le8U-CSL_xVly)euyoV?0uP@Q5?R)#nz$R7daz7{9s(V)A7J zI|e%<6-xW79Z)BN@_b8sI|+t#i=5}5NO&6+B0y|6N3aKfz6YzwhzzwDNAxmym(`XY zkiYN;D_{`%(GflXdoXFvv)xbxI`4~m!7^17pSAS?+r;>sJaIyQy-cgJVmJG|ZRT4l zd@nzF!R_Oh73!)VY|$yf-YOC_v&jUc=D3!-!d+W*Gw z{I-unPYFbCZjbQX^=De+r@eY~15Y}qn=&#oUV!E=$hU+!=`kvj3q>zF7{xG8vUII# zl7j2bnD{l~)$0P0(YK$JsqgE4ILVb-5y_L?{Aog&?Y^(rvJBI-#(I;g!c@3!&`X zGxwv~bN%60vkTBD0!;&-blaw>N>w3~&h2w(ITDYT3(&k%O`@KwnQA3)c_gs<0r4W= zQ9!ScAQV=5@7fuIyN4JUby9>filEMozT#SFp;Ll-1c+NyO+x|<^`ju~4m1Qg=DyOz zxB_JjBe3aE(Z|4d>plSLLB9+J(Lmr?%?6cZC2(~iq~@IGAry_l211EP4RmiH(+)QG zm8(~yws%yZaRfeGudu(RmH;(1C>%d-BfFQQ07&)gEv~p_Wt$B1%hJ+XYAuP(Pe8#3XWLCtQ%sKXo(1 z2Sy4`#{sn{U!N)pvven@$+*}lz~6lf^eM|_!4|D5v%txJmhkw zA}R;G>MVi2bW1vAdQt*kmrkCt>8NA@7t0|;^P12(WSPr}X>Sk?HfJN=&!S<%A%GOAIt>M8-pz+-k=o+?{;|z`wCI?4e#`P@+(R5~ao#MhUE8OFd*#?i$ zLXdlBV*;4t*WwI(kIOhNKiwWufBC@3vc}G|;|B*z2Y>zWAA+Yd%smO{f;xZ!=g*%< z7%&=3_iFRjrnM3oBFvr(V$ofvH%rc+!%wI{dM!qzK+}y zg>7n{T+}N{Muf>+WE#!aNNqG}Sh{mbxHOcTD>e5f|4S6ry68H@nqB_%Kmu|m3h1$ADKEbPpU#@BPUs~W4mH&cO>pm`|IIijRB)-TUzBO^a ziVMql^ClUn3)N>XUL;bEU<-rp14D;+AiML=cw&N8Gp#ZX^(E!B77I%ZQ4`ry5tR%7 zbhu0>NytD?uVDjMumwL-v7(u}Y6iWG+E7Ze)QCE?}@KtN<-_27Z z`Z)p;ONH#H{aion_7}o35Far~(||~j7$f@#8hzfmoV)r=)-u{Yz21R0Qh-#k`N}Bw zL5sQ;WquZ}O3ygOl=%fTgb2L5t*3{YUmHJtLR~)f;B>~Px%b`3w%2=C5k@2&kd>Z| z6rp!~AF(0`AJ}zlk4UJMN^OTM_4C2f{tLc2s*8<6(J^Sqwi(qk76;39OU2^9LP{$f zF*RXLT%$>Cl-Yvu9XW)+Rzqj_#tCkiU$rq8oc&niV+344dG1Z?;FgKPHU?5;01*9% zkZyU|-g|Fry2SSRvu6)cO9&XE13#LeaZLg*F+$6PIa{8PQD(vl4yL5%OD@4EG+P&# zG*P@YtYrg{_O9=PWeGDtZ$0E3R{G{YiyuJ8*9-RFzkgE;Iace%Q7*U6ozQxKAd9Oa z;Wm*XClS&h&@O5WU9TY#5&4_)P%T4BaAsykx39~x1mp*LvINTIYiP&%bNr`7nRD6PdI5ud# z35861&?bM}a#ojchJ~y?sty88iy@y34?Wb!Fr%5y6yY|Yn^1CwlNk2x+qYzOy`1GQ z(4ohmChHG31MS8Q{ee);JrDMq(~-*I7p)aNm`psrgYgAtZR0~DKb7Y(VX&sM!MBO( zDsH$H8aI%{B$-1OD0I6@XgrP!Dl8=bYq_~4G23myFcw@~r3tyQv3hX96dxOeod;R| z4B?DGf=XS<5XHNkcPwN|Fvb#)=NzrAm_=qyONye~TydH~Y_xXwp>6B>0cxnFPc@p#C@ zUYPEuLNI}#?oEvZ9_!32JodNKgN6u{Ar~p2v}TeAmRM0TCWjugGXESqHW0XVAy+dQ;Oy1lvkZP_G^uoOmDGx$lSVH!{e`I4gnMH;>wYxRnf4y$P8_W#dQ zo`d=Qk}i+m@U-bWA%jtnm+9P7{*YaD>OUfUi0f>C;m^I=AI;QA@hQV$(}b|vt1d9X zpQIU$eBjagwveUX;s$b!X;NYXM}&VT_TR$7hc2@bIa0g*i6nhk0RDeSOD6hMU=jHb z?R_j({SKjfO)}NiiD+dZMl`oHFt^;a1m!`@b&A!@4k^Bv`(-*scpQ4<_O$nh42cS`PJel1NlyXfa1Oy){{l&rhf2 zz?|~|4yT65*jk;Rp9Pi|&#BO`4)?Z#*o~zH6NC({s&)=XH0`&@k4cG-%{mekA2$WywAaQioXAiaz)FZioAA>;qC#R+oa4SDmyn6L&-hg#~ zAu+AoA38R#c`@XfLgtIap98?8nc!B5hQv25(RM60?q z(f=rO;3lDE(;Jtem0iZ5y#mpOM?Vw}JB!TKpu$_J+AQ|!1{6Rq8XI#$n`hdKYT>=g zeS#-#d1WU!5BZbHxi1etf{?w=zqvI?dlmGo;STN zA8d!70K--aURPE91oaiGp-b7DCV2xECAt)hn!wS_PvMtL20wQzhNdab(4Z{A4 z`eAkF4;aov4&bUgHVvPcKzN=eLBOKe2kxuU)`-pcYf&`cxB7z3B+xEn+eqjheE7}V zD$B4QQabFN9V-!D1xbb)Jg$@Qlz`!0VxmlB7nkin( zcDRfq#Mf#E($JG&2{b`~A_IUT8~`|EZ+>kOyu0?AF75z_a^dCV>)-3d7k#}N9TDS3 zi2&m(Qk0XI0elE7kvY~BsJh)ogxG;b%Dk;@ud1IK1}4PB;k))Jkq5w4g{aYS16TZ$ z`7Acuea$%>4wJInJA z=dmn}Q=KP%l3_nO%#2{b`7q#=gc5w%5D^t7#~<%6eF#Kv%Q<*vzFU)8q8lg4@L(`* zW)>Zg&2jWPNbBEmsU!ur;~_Ns*NC$$=C{8BrbM9$%*sfhEYOd|o%gkTar=kE#KY{a zO~`dFE;gYXL9JCq^j4@6XNp_A5mvWYv=vt42g7g_frW=mO1XwL;iie+ee8xQI$pXe z37~!LLgahEs&h$?B=8lEuaW#|opXEfW5?h@;E;NtMuduNiEB)Tw3|nby~1|3d`1Km>f3 z7|!AdV+cmOo^rj2%~Nq@fiYSdz}Iu6EwMDh^$_yzk7kTNk_&pjPo6V|#{bcQnd_E?<$ZDOSDCYlGzAl;b&c5b6A2D;O*B4;owE>&>eteX zND3Y-66?K`W#=G$2vp-ij4pXgS~#))Cl#(Bj~iS3bwFqNb!Z`Q1y%KM4t17wjtrkI zeWK-*A`VrM--E1alHjurIda`PTo;RtuTXPAt!x`S}BLth5F{Kd=L`Blf_{I|7K z$*RvtT8n8i{J?+R0U)JFt=j!LcE>M(5D+cE3k(N^$ITjs5^cG$dOKdt@o|K?r1l(Y`HhhX?%=v*o? zBHf`}Nq_Wc5Y$*ceoeRPk8`3U!~WR8y2v`$&^4J^6|p>r`Cv|J9U_Ky?90#ki}LwPkxr;#Yf z*|Ut;GBgd3N8h!@VNk|>@$zMlhXQ!3^~g5`Kj+G`0H_!nI_y#(msEWA#xml|moHYn zLSOTwN&TgyFB96GOB}FnMpW}@aszX36Dl1A53m`m&~S8~t#rusI5rBWWh}r~OWeE( zem1zBqtJno*S8YEt`H7=23rto8u<%D+2vN4KmQu;G=g;x!P2uhKQtwiFKFLKs=mBU#cL;baP`!8I9jDjL((^bfSkhI_qVeXl$8%RFs;9#Nxycqx1+vS%v z0SOkG=V6IEarGI@uiE{C4aMu|g?WQ_I2l(Ah63s^My*PbadA(;m=o`JIOn$bRAKkG z8XQt>gq(k+=eD8oXq`W*N#_mR_`j6K=hoYd&v-s_7D$sml-m9;eTg!M@!MNk^20ey z3_~NdgyHZ(c;+yY^z`)qN)R9pF${`0;$G~PP)LB|5fyhL`qg7jKC-RBm>6e+ zucRG48qezWE$-c`*RWJ^K?7Ep9rR1ycy*c>1#~#;xdMwWF02kUa)J)(^H6ANYLfC; zeY4{Pg9bY`7#aYPxrjP9pb>WkY9gc-d(V@wS&lk0yR3u2$~zi!vM#?Y7Wr+tnMJu zUXI0Ak^l+T%A!(Td0BSR^x%2e8ZCY0)#QICK0luUC;d~{@nD!`HrTszd&|KJ$#BeD z%4PONWoX}+e#P-CYwI{7mR&4ZE2t){SwwQ~Jwg-XK{kvREMiy)(6X#>i0LD=szgRd z2SE)E4mb7>HCe4Vw=>t7u@KLT7duH|M!PHXW(tz9ZS~K=KDPtM4e;T@m6r59D=2Ze zgIS;Nepk}bz88x=>qO0gQO(kklxWuX4{E-ddch(8rMACK5n^q9dV?8GiiW19rdqaz zhK9ml~zbVlYb70xP>U@f=#P&jrxYDH*|C-Qi%W zJ#^z%RJsejK7;)vHhM7S&rz3dHcI_p`iZ`%0KK57`5uxSXe)xhOj7n7GR`x2TKT_q z=<$F6wJ69H*Jz1w91(j%o-&O0qo)+);R$<^2DUL0Vq);0d{NK+9PHI&b&4@)MZqtJ9NDI`R!`4fgiI zhO5mE9bYNtC%(o{XyoK{gOnkhc;I`%FLz1wI~Y-<_SdmNIIob{`8-bQ^Nlz)J>JNp$jQmAy&d<#?JEJzIC^#7ba%5OMkf`CQOYE$8}P+ zxDh(sWQXwf#t%rbQUOGUf}kOukjLA;#W|~zn!f+}e%(2Ip!Gal={9<+SSXoX2hM#T zwciyxkliKssPG1gY(!(9P8wlMy-i)T_6uEtWSoP#N71;@-R)@zvJ1$6EG$wQ5Do;N z&(|JPGL{a%7dxnphF$?gEt0IkKbqjxpZK*73iD@OsR*&R($6M8ID#1IdI_omPL@vi zb1bruQ(+AZTsW+jM?jvm#2j6{s z{pQEAQWDWG$(NQSKLXf=n^l$w!UJiYfjg(%EPaY<<4w>Xa9}409U@GQv2Q1yS%myU zv?`Pvvz2=pYEsnmH62~C4P7zX{w!VQ#;?5)+O^;&v*we-$3ggJH9>N42=LO?4|~Bw z-XOpmXog5ZqYY%r3Efr>I$rZL%N67O1wRMbufdL-IRy9bdWIYm6Ft#~JKcPIc}UhS z3e!M9-U3?yzICxvl|P-}tlK+llBqYHXhA8^_`(S!?=H~Q`)m@vYxTGtDRH?__r@zg zb&=#QGEse*2T9hj|M_hxCyW%>mJV;spgce>zd}PQemEJaXuCp|&pa?HkEMtZrGDe)&5e z076$8|BdWNM!jfzcgGN<2{Yg^*ej>87LB(l6&0$6)!lWZ?s2Z*hV}#c#1JZGO=l zFLBIht7gqsm>OmYR)c<;Z!}sP*57UYDGqxp|Dt~^E(MXI3~rqXP=Ri!N!76^BMk9C`ZONqjkIoKZtZ z_t(@nG#X(vT=gqLj;Ad$+cj`y!u%MJq5^BHy_*BfzBkxiF%PnDev}%z(@#}pEpjk# zgMqltHmP|3>vw->iK?b~eR#-&8@Rb<58Wq2O%PP8S78sKls5$@dh$xdN9aSsLfz6U z57qgU%_j`8V(?vLf0jR7uK&Zf?f@Db^=##7Gsd zKUyi5a0t@xEa@$43?%Qxd-vY76}GfoiM??WY2>o7erxF4b@V6}Z$D`i&nkNNt}B^$ za)ha9%b5Oep~YSo2>9VI=7ggqP^vv|WR$!EZK^JAQ8^m?JQfLVgzt;z=|7~siwLS! ze%zgKX#UDqr~f7`sDq^L|8jXAc3A`%%O%W{@f!y zO{C|+M~E77P_ReY!HVlZ$G6BL7 zgW!kXD8B4n;IdsfI=RRN&&_C+Ju>q3vy~N@|82}FW&i_6&?%*(+c-nvZ&?kAax9yqdz({Q zXJe3oQwiIi6Snn4`l$W-l^Th{xZkUvV_7%a_cow_Hn*e+n`B8m#~8 zzr|l4!*HXE26=R|`5(6)p5Ad6ZTJsBWB@=~88RO_e9M06UR-!|xrjQmF1`m4O9CO* zWVTA3svi>_=_BnxDCp?;$qxMYJ%02(aLKy~p>)8Zo*!*TyV)Of^S^L&13uG~q0I*^ z$#B8y1;xDV^N2+A)xiTE$p?B$-{H5y@+AE!Xj!iwH}3v5+}Mj9sJQM}^eDR7_-nsB z+z87DlSoP*DG~ItwtecNdJY}nz%obR6R}SGOlmCT^WBnP*tsh&6Jlm|_k=(hX_$g9 zVd87hNlS;lX8Xp5E!XZcMn>PA{co-1;fXRfM?WX$&i3-tJvtZIAdK&jeElDs$%0+L z_}yb>7jJcXYl;TLH&wR(y65^n{yMsAKe}tC8j1d`qz{}-!zT_&R@QmtMwC}d!6dVV z;W|U5lcd>7@=^18V5ba%&C(zhp8tVS-A;CGt>kzj{B2h z!`wjsGtB7iYQ}Y(v!|~+?@Twvyi9xq5c7ZY6#=x`v{QWT${}@Q1_K6yGy)fQXLVU@ zbf~AFHf<&BEZVnScu8v0(&Zkn9}2&MNp3X{ZWmPLiX1V3bh8;!W?V8QzKa`fk? z{m0cR|9f1-N`bd~v8foHs+`&c?es=NyEy{vulM+=gtlp+G2pY=C-9kL#r!i86hDX~ zZ(BOJIWYIM*lI9!3KN;-vYrx9f;uRVKo4pf5l2S-* z5qq**FG2Xi2uUJ0VWgq1|7PFL;sMf?SWw~RxxOUBA?3qf<_odcvg`hwgM@m&bV1NH z@w1Z?XonD8*f#9Hz7n2KAaNyY*nOPGNwgf3PO5J3i*-{0j8RK#xJbP;r}Oy-#DdyO ztf#hH7sx}zEgfVs15`f31Eaz&{Xv&01r~XCthxm6?l#2xQrvv%R6$}4bZn{_ zroe~kHOn9kw!FGY8hL*;u}>8h=ixRCmYqu3SgW9G(hTX0_=8k9%=|dX8Rl1Uc{e?5Msc=96 zvOaWd3w{njW3B>3$0MGr!tkR4eNWR}lCGqUyO_Q5PGe8L8B9iKaO@?9xtsTh0kBOU ztVVe-Xd%=B-}>Sd{4$yhI*$NG7&?yt-SH*1rsB--Q&XG@&~Y&E^n~mG>Fm7Yss8`J z|K=c$U1V=bR+ODGvQqX;wofV~yL=+^7#Sf#5|SL*$tEMRM`avJky&Y35pq3WeLuh7 zb^Wg2?|c1mb*o#&anAd^&uc#)kNblUG`{vrvd?O_LhS7xr@_QJC0$4%^Fg;n|WpB{8zO0#=pBq|-a;Rzp>{JPwiV0Gd4nCRk3u^QGUujmzw} zD4{z1r#vP%xHSs=lt7?$AteU#fj;-BafTx*MKD~Hbp9hxVKkgu+OsIWLS|QLp2%Cd zn@A12e#59E0nM-!Ts~ySjFoADA{`x5ApaELnUK1b8KE##cN_ofVOm9dym;GU!w#19 z>nVM!>esK(Lp;CGAVsP|8nj1Z02HI>XlcU%Pheqjr+2xySVB`vYvdB&ztyLR)f^-I z+VW6sdHiBEro&mG9fv)Vn21{IbmjZA!;yrlnJf6#MMRvSy0&}-6gpry5(L=8b=+$H7WtTViP_ZcsOvK?1CCY9nMsNCO~w!m z8Pw7)4wlS;o$qnHaKzL$+gR6suGx6Ux!o&iB_^{NGI`iXj&p|IaJ|ab{Y9_OV*AJY zH^@NaNJalFAP`g5p#+vBjOx)UJ6jDh*Yx*Ek7S=#RU0LT&198WU(iFoy$|BM4pmQu z9|2woH7D@oaL>M%VX{(88oG_(>j6fzZWEj}TFg(PR9vJ6O@n!qkl=#+cfo0_XcJ3;Vx5PiwwU z)oB0=(1TD>(xiZM*dy4eW{H`@X!`wh>2gVI>YFNW2bXw4i^~OS7>A&Or<*p9lg&6= zX--SfXoG3xd(E4A44eP$Pj!3$l6nB4zaMG1kn~f-YUD=9i9q|QBG8x^(s@MP;{8Mk zmY!^e^B)-P0caD`F{yJhe`J(UL&D-N0U!_2hV_X7H9^$5|_KvOg1`D2Mkl}N5naHXZlxlYt z@Lq3pnm@$q7bu#;QB##JU5Icj-BeI0*M~;~%vNjL@$9b-aqlnUxl*DMxpm=6>&Mt) zdUO#4vc$iVKxP7C&b9alxV7}3y7rAeSFW-@6@uzE2oL6?QuCgMdM{bI7wmClf}DT!4hyjR z1+TBXQgFZuvzag8p88O@<;74BO}~b&*6S6-#6vv!w;hLw zBq{dg?nKz-0-s4+6&N&ug*z)qGNi9E2^$efwCuV5R=#?gCTZh*50i>&qtLdj-jd1=(^=&|Ix@d^l6e`42^Sx`6Q9KA z(UzUI)Cs$Vt86g6H*(|n&1dLBnlG?UYg#ZD9KIwtO=jJwRL?cqD9v8ucPq>mJAF0# zx&H3kkQxhKEiyNUm}p{*ZdlV$S;l`lU_rG#X?suG)TP34ZTd`#MS|JLW0pm3){ulG zs*C1&3TXq^zv%aNzES0QzCNsisf8F2w)UQLhCSON6Y~`EHU%(V+4)_{NYz61eO@7P z#HAA)f{J%8X~c{(Q<&?~oIDmK^k6wbV0a^fiS&I-t8wME40qA1gXQ*bm{i_2)5F$~ zGV89uy=bC9II&J{QiQ(K%l53f3|X23Khm2yR-(lWOX@jPn1f@qq7rAZ zvX<^8r>Bexuut+od8oRC&&``8O_&Yt4~XuQNc`BKH-0J9?Z`j}mvv`Ne=eWd{-OoBhi^mo>IM?Va4-W|BBd!d5*b4#276g4_G|a*migW*>;->6doA zVojEUpBgQFNxf<+Bw539&Tt{;lf!hWjP`0Yz4tk#rmTJ`D4s~&AYCOBXRuK_H5bBj zikzy7@%dzvwLslD7CP(H%+q?B!^;~Ve*T>Mb4%Z=4XEQeDCek`O@*X}8KKHShLwb` zA$_A{fj6ilW=oB!*T-t>+TWE8J9RuhnPK(hGk3K`T~V|_iJJHMeKK3Ldzo!N#J+!@ z=8P^i2$yO}g11H0BQ9mW>6dUM2a5U7nd}X1G@)B)Y>%)fF;QxzK5}TVoeEUvfcrwR zDZ?#az1nCZ@#Z>j@A`cmSA&>Fn=%VQ87cOi4=0BXpP+#A48^ zHdT+P#PeUyY-Cu|1V!%Y)%-DgXsysYVOU68Sk{vuCEbl}OvYnsMGnX7U$BFIvggg+ z68VPbVY3}|E$@EbHHu<;u3vh#z4^Urtvf=klOFO^-+b$ve)BObofUdcvH`@C*Z9BHN8hd+Dz4wf9SsP4s?}Z@`S&lmINz% zm1$mT(|u)0Xs>lx_1G6N#94X>v>xBrzFh&6nfeA*{S&{=Cy8elIb|mzf8}1ak%s+J zf{paFyGP9FDO?E1_?WdN4gO-qVN`z9S~kRBAJ#XYX#|k%tC_q1#bKSw@`I9mX}kNQ zY1dA|Y55|jv!=#VW)P$-4W81p3xEHfE`+{KVJMX~)pDvJJfnPql1`Zw?v%D~j+?1- z%=k?q4w2dY=a?=HKe$Y6=9YyW$W#~QZJXFG?3Ju{Y`+!MH2fdUI~rP_F!+mYtky2Y zI7;dz*&^q@XlSUu@idz5(f3yv_a=2t|5U&^gny^^^@Cm_Dp8^qn67HJ#!V25Xn_;i zBx5Uv;F7)FwmdcJmE)AJR`g)!#SW84&_v`giB5m)QF$@>)+^fb!zGx`5xkdd9`&Vw ze7RDt9MkDvqoIqeTyb`n!-k6Vr8U@xV~B*Ae_m~PwaD>?JVMy{=cQI!rQug(q`=4ozu z;y?sBJ5iX<&qMW;wnV-uhAFH+MZd+VYDUwadA)$m>5BD<^+#+|e@XUtnxEPWr$Fj( zc=nW5A;-2>b%Mab1XWt2V0yUdAtF0d4DsXlt;|2o!u7<*fP9jlJY()}>!-J*`9+Kl zM%C{-s|DLo#>`1(x;&hpXcfuGsp7$~dw7UiXBXxhOVj97UjByiW=~1i;VS;J;V|!5 zdOU$5Un5K-Imbkons%t3`%M-MU2wzbtj%LZ88MbS7E>1=(XiorK&>vdn`vv75Ub?^ zIwpsPz+ppr@~2`HTW+Xn_?7d)aXcDYN8xTkQ#+G*zlKxzf!m93S75N5<|58@9jE?zJ7}nth7CAcZOcR97-q1s3?O`jM=WC6uBHLc!)2v@iP%N4)vT%g>Pye=TY=|6p zJz3oy*Y*S-!SNLK<7G-jXx>l&Is3>Pp3B(GPQb98$90ImFi12`*)MDTd<|`iRWUVj z=2v{T2OD~;ugwzRsQ2D8t1!@7L#v`#Snf44axIh>YZT$kC%>$*GbBIEH%-P#WWmF_ z2ox=5<{AwYEmBy#wMKe#9<6#}HYZ4!Hr&jhL^54q3|``==}h{X`nUe+K1RJJj+HZ- z7BWj3UG_^itPj9VhBZE;X@(GY1TI`F$Y^h002~i!c zIxxwJi7iN*9Xjxoel03nn}#q;&7()@AXX#z#!Sla+LzffKiP9Nv_Ii{l(8cT+a|Q$ zv67;qmHZ^ba0cc`*W4#0_m1_~bzMj53<`0IcTEXc3O?t8yR@$nWrDkHxaRs3$4K^ z%r{#06DcjJEBwblZFZU4zoF$vu^_jK{o)atDm?)KF{3Dp*1se6Tx9l4ogc<$@?7xN z^HeF)Wk|7f8ZKfYS_)N`+b|>~Q*5}Ii|77gwvridP=;e2)8i5T+ad33k`LSf!RA#XAEe-;Neg;Zfs>2Ivjg|SWZhl zeXXAyipwQZRK>sj+XZuHdKRi;n9&iLmmy+xnOsI)S(C$W%q&(YLQUII=?K1ej#Hz7 z3C8eM*8_s&+^uS5UUa$@6Wmo%=!nTFwZU`4EtPt0tc_?nea54Z8!snD2`gjon>;n1 zw2VVeY7L3jnU@X!Vs*T7SLh-PNJ|CU>FTl4;rpqw@LR9&vkk%R6d#99ybZCvuwvv= zaWX$en*RigjCwt1@3jNT6qc! zeOV^%Z`NN`^y*L-gd!%00%2$8CVli1kGix}d)W*ARAZ}1`{2RQ5h)m1KKvS0A1Y)W zdYl4>^nbBsR47Q=m!__bi8&5EPCVzN9y5-Q9bs@g-~X`bh@7jL-YiRIlD$)?dVvNk z&fsJ1I+Ed4L9!`$7H0rELl{fj=&a%Vt@Jce9hYZ7nP*3zXArN}RKLtfMNht#kgdcX zNxbwQqwE7WN3k-s!tqFc)vDg;^RR$%w>tzLsuVZ2RD36k0vtq^PUw^n8_AEEthl<1 zQE^vgHO+**6llX*m?LZM^(2j19uls5WRM(^&VRg(j+5fz$oDa^=`wS5YW`lcuh&>I zY+Yq=Ypx*w1odgmmx2{&72T$*Hf-ii+t3+c?zS%_ynf8${r%`C^Tqx9q91sRC!D`K zJ7F4bK&XYO0zWD=Y!7-)PpM{>Ov71lj4bl=cq}nlI1{g~=c#y>w&Z`bHy$76dT1&1 z^hi1-Y@`xH-W$vdN|v#R5h5}hG`m*xCzi+o(C}AAU)PaTp#oCeJ*T5m`<2=p4Js(% z&>CicBP$Ez5(&S2Q*p4Lsh1h|?EboxGE|g0WVvZsWTKYmO<@Z?7&-SU$0Oy*NT=?JdHtmR6uXRF)pp$hnC+A_raO9atc+Yk3;Xb< z0hDk(q5vT%?7JCeLVL4s)U{^8&D}b+P3EHZp>Es-U)S^Jjm*p#61*t1RWD2pi{@IG zJu|}Q@j9mU%@*-Y!E(?$0le`vwW^!+MLu2~3a8Y?RmMLWh;Cx;b3A zZdaBSHd~l~J5e&rv$4~Uk@y2~Ni|)C?yusgYpaubU<88o=*-Vz(4}4m>4Hvnb*tZR z?|w=7=Vl(@k#Dwso(?;)y6q$Pinyz6s}5gyW28!Vk!?bFXQ^2J*eGwWwnqge+$fpQ zzgaR%M8#?z3_BJ19V1W>Q}9G_or7DKxcdiwTeH%o;^NR!Mh6yAuK60Jxs|j!YEq&l zYU+h~A{Bif*ma!?xs9AFlz&%EP!`J@Zp)uf6PU`2A=1L#l{P0{lS(pgh`!rWA@%K^ZSuIk`Nyc`L;geE1M!G|VDXDVB z3DzD}dSoT9m=oku1x92ZzcuC+;I@^O)ze9%O4UdngU#yDW1dbQh9U1Wht0`i`x#E2 zHw6giHE~vxxzD#}TXlMRb_!u?$5XQ4&DJmZ4KHW*i07D#Gv$~+H!KP>d2AxD_^xcu z;+VTUU~n2HD{RPN`cW@1?q1M=i%W53oO-m3CoPc+q*i@-7xB-u&y*zIV4L&SH9T9} zZ!MLp#f7cwh|)aAaA7Jev)x}E7KKr;N~hA;$ht{sY*XvKjnrUmGKEqIF*U)##K=Sb!}3&Myp9n+HE=^0w^3Cdjhmnl zTUzf?`=ppq)w7JxBwFsCa^)Ss#Eg_e9_vwCiXXTr?&i58D@J5V-hNkM@hteULB~U5 zvDu+wBO83pKuD2^u94UQB1_tDH4||bqatAvVTMsJc0{CkkB)zATlmg|?5EFjKNW_a zHs{B56^OE!+>TOhks%6#jv<|YXT;Kj!a8-Yf?q;Q65T#oJ=iViNZN%Mn3wPsU=FFc zp?u8TsrmuQy=Z$3PY`Y;nhFdJ8jG{!m_Gl+UE+?WyrO%rbj{EDEMB}{(n~?opoUT1 zOTLB$R`cHd+17^yNcOZaCSzH6GG)2A)ZI^Vdg&$dvuS4hHM1tWS_^Jo6k0mzaWt=T z{Sx^eg!_6+UQsWkMjP`7mWml4{X>sK{LU z++}HV`CDvv&KbE@cRRjvYLLCwdDcyIY6I6Az}qWP-Sd@(TszCz(=57GX6Vtd)9o$S z`BW=7wVhSB9k4cU@Z>Raxji1ISme*=+2>Ll~E?N~Fm_*gH zZ+G>4b4p+0_yf+;PXS-$CU9aH!QcX#!aYDE(>_)G z#-nKE#|9e(bCoZgnAjCkLRp0>{;_)h?0S#!{955 zW+vKAGYQAZtXX@+8#q%^&Vfa;^@IEM7x{NJb!w~g+N2yRY0|#ZuJ<@dw*JtGTxo<; zD!Q-ikuagjPuUvrG&e&bRm%D)V(B~w!7l<`27)pbu{;t${|slVuAi@fYYJ4o+Io6; zB)kPeGzJkW45&y1(E=kM(*tpER$K-b)Fmnru8|DeK8>RKUuqTg-oG6|;db}@z?mDt z*Nridn5sg&O)OHbVh%NLz?&P3@B@Im@~&FrgVTM;I&uJv`0^7n990e!2%v1_pOw(9 z02Q&M1=tx+vCh1g+)wNO?Oi-Lqk4OReu2^!z_&YE+Yj1X+tEFL*xF+VbGN+TPKFwA zOMB!q>64O5gD8N==moF>&YJGNzGAN(5YAZNa+c3@l()b8*Vtc2gsAF80U~qT@J1fjmwiUn=*U z)<7O71S>vfkGvhmw^#0B8;B$gaJy??s>TvCGGb7D3XrhNYPSF#BFd5`rm}~DJDQSq z|5U1_T`S^N`Yw@}LA+nK!X1Kc%l#g=c{1NU148ic*#Lk59tipqZbtn2{|-5O;r1xh zYtI7a`feX`$@nNX*8~L!vcPs1LZHa8e|;4PiCn;H8Jg_yq#eMKFt||wp#dZ^CVZ-X zeg|YtU^y2ð!6f{?3$%KQG|U|`Ngf79YQAC;MJ86n;v%m{G8m2&GnJHst4c4HK8|l0*VLlVWAkdY77+QC@aOH z$nUCgAo+j&Ie0x@26E7?!Kw8h2lt=T>pxHN-;>-FwxAM*63xK2fZ|~wX2kPqlFBRK z>UehYT;&%Lp@KNG&E+|%L0W`=QiU@Dg@7-hllM9u zsLJ~{#8h;kAEI6q5`w{NJAi%-NFBAk%0R9`zDMBKLA1f2z!Y)u-t$@T@y2n=;URno z-ee2V|EP;bChi^X5DJ7ap4#8PKF@Vu0kixcR{r3MjY0`PO1P4Rr-7G}miFVG!hMdT zE`U}^>xA;^K~(_U$g*RT7l99we&jqJ<;8%P09oO54GgsX{i_>d=e@uhIh3cvh%6&= zCw2h>_W>a|V0ftr+VTgG5sF^eUj%&>!f+D#Adu!x z_Dr3JVA$J7K+|;s2*L88e5O7$xM)BvD%|N=A!7o@%AsVJZjBo6@wVL{^nI531Eg$XdtSE`>}e+=mWWNPq!jj5*FOd8P;;f9AyDX4kQ`JLEWItq<~Fz)X0j%_LwILsY@p6>RUjJSF8rY6jF9e3s2 z+9AQC9d60Nh^eiH3jOR8w$M!SUMyTI&17)HCfl7VP=pNoiNM;%Gr2|%fVe`h#C3W; z2*RwAGBQ4TmqK<6nhiE>u1wam0+eI%b-}I10r1M?8i|&J63^vuWx*i{2@2$$z%Vo- z1CS%l=LP_fa)Es;wa^Gupy;@`l(AC=1|UXEcB>{TgZ>{7Ko6im3{rKKG5w2hE7=kI zwvKY+w+9~bH`LCYp@Xq93N(Bu@CO_8Gc^IT=`*oE<9PoZJuL>Z0Y(3%^h~|EEVv_N zIWK&^1WJMR6NWme!6Gv;xS+1jM*U0v$L<5B6Oo1h%z&Fx%m&*Byb-C-b~gsx>O}?? zx<(-X00Q1308R8mYaR7+1xSsIs+lvvn8>l_O7qqpt zsSwNWE#w{`90few2@`%ml>z+^w5PMLE<|zK19suRqevIOItApfuKKD#T$EL%3%##$ zRzvK}8hC{c?e2o=;1=Ww2uVpXfOIj^uI#FFcOzG3$fU4@1RZ48Oz1xPmk6M)(Cl4z zz=<;7Qb3SCrhispT>@mBnYG%mQ@2GNVFE<}ZW#Y058ncyVTTjE>(Jg`Y1u1xOJi1) zyz~D>GzA?1g7+aoBZ&XId?qQ-rwFYP0ko4lHh&Fw1HuE_DFKfr$-QI1m+8kP6QMJo zqoAaW0X&d8x1y&S5}g3l)DBd)-CKxErUg3EcCebD_%K9~?KsJJe6}=g_`&g=)z^a3 z(ni1$S4ULUTY%OH$bzUwJ+KC8gZ0M%X6@6kDu{u*v>8w|A_ja_5$!m30eH59;O;p9 z32hzl*tb8Dt)D`dKZ}M80507GPjWQ+KV;(FfB+5wia|dgqB7o!`u@rbS%7X=0I>rF z9YVCGf(!^`z``d2kr4zY=zyT2>PM?)M3V)3iYCHmgLD3^t?;d~4Dj*9D)+k1nBG(J&J$G~T^a_Q;1`4H7Rb z?Fx#E�{Q9TikmwA$F*j6oD{xC6CI3m_ap?0Su|F%-gr0O5eqec4y`Xzn`Xe`wj* z+#9+mE2yGkgIfkzP?Z4`zWCB17u_{fypN&d!f#Q69QRl z3%(*zSM=M^f2~sf|07tt3iKgq0!RV=S7pHc#fmH}|3tZFqr^BAHUjfD_<~?KRYXL< zzHC)K)$Q*%;ZtG4rw|+t;|JdnAm-H(URFcHbIkVYG`#iY-}mIAAn8XLK3Evhe+-)% zwmMbRE>P}e9tj5vCBLAnE11r&E#^jjN0TW0(Ka?_zOl1r8ONo7SZ{#L0q!6qvH_qI zR#0dP2Kf}>nbzfgV%*pMK^5esjH-yz_gOG!(cfebLIt+kbv+rb8UdK%E9U%h^vxw^*0EXU%8 z#dLi>l&Xi+*#NR>1+7&aB9R7fy?KvjD$cRPQc~JZPJ++iDqIH%QBewh;4{4ok^u1M zgKQI+322r;vLblT>0H1_lGK0?xAz|Cv+Wxc$uzYHPTM&|R zLg4Ms0HJL82JCbYsRD&#Ay(k($Ja8Dmbktv^kDxjxe(3SHAo{w@C9&rVIsF_yu7^9 zm3%~y?-cSy5a}4^Y(@~sRX|ThS+Qt+h2W_unNLeg3$caZGnh9KQgkOtqvbyA@*>CG zU%-$R0js{+Zv<$GNx~6KU!^FRlL|7V9m#n!zopPKRf+VDY xbnt=v`TPIqD*sY?{F}8h+VuS2smFc0IL)AMTgJKSqZs@;rDdR5s^Jj!zW_5rf%^ae literal 0 HcmV?d00001