From 15750674d956bda577230574858d57d2e8749a6b Mon Sep 17 00:00:00 2001 From: ModelHub XC Date: Fri, 7 Aug 2026 20:13:16 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=EF=BC=8C=E7=94=B1ModelHub=20XC=E7=A4=BE=E5=8C=BA=E6=8F=90?= =?UTF-8?q?=E4=BE=9B=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model: tzchen07/SG_X9e Source: Original Platform --- .gitattributes | 37 + README.md | 58 + all_results.json | 8 + chat_template.jinja | 29 + checkpoint-498/chat_template.jinja | 29 + checkpoint-498/config.json | 60 + checkpoint-498/generation_config.json | 11 + .../model-00001-of-00003.safetensors | 3 + .../model-00002-of-00003.safetensors | 3 + .../model-00003-of-00003.safetensors | 3 + checkpoint-498/model.safetensors.index.json | 296 +++ checkpoint-498/optimizer.pt | 3 + checkpoint-498/rng_state.pth | 3 + checkpoint-498/scheduler.pt | 3 + checkpoint-498/special_tokens_map.json | 30 + checkpoint-498/tokenizer.json | 3 + checkpoint-498/tokenizer.model | 3 + checkpoint-498/tokenizer_config.json | 2011 +++++++++++++++++ checkpoint-498/trainer_state.json | 727 ++++++ checkpoint-498/training_args.bin | 3 + config.json | 60 + generation_config.json | 11 + model-00001-of-00003.safetensors | 3 + model-00002-of-00003.safetensors | 3 + model-00003-of-00003.safetensors | 3 + model.safetensors.index.json | 296 +++ special_tokens_map.json | 30 + tokenizer.json | 3 + tokenizer.model | 3 + tokenizer_config.json | 2011 +++++++++++++++++ train_results.json | 8 + trainer_log.jsonl | 60 + trainer_state.json | 736 ++++++ training_args.bin | 3 + training_loss.png | Bin 0 -> 31982 bytes 35 files changed, 6553 insertions(+) create mode 100644 .gitattributes create mode 100644 README.md create mode 100644 all_results.json create mode 100644 chat_template.jinja create mode 100644 checkpoint-498/chat_template.jinja create mode 100644 checkpoint-498/config.json create mode 100644 checkpoint-498/generation_config.json create mode 100644 checkpoint-498/model-00001-of-00003.safetensors create mode 100644 checkpoint-498/model-00002-of-00003.safetensors create mode 100644 checkpoint-498/model-00003-of-00003.safetensors create mode 100644 checkpoint-498/model.safetensors.index.json create mode 100644 checkpoint-498/optimizer.pt create mode 100644 checkpoint-498/rng_state.pth create mode 100644 checkpoint-498/scheduler.pt create mode 100644 checkpoint-498/special_tokens_map.json create mode 100644 checkpoint-498/tokenizer.json create mode 100644 checkpoint-498/tokenizer.model create mode 100644 checkpoint-498/tokenizer_config.json create mode 100644 checkpoint-498/trainer_state.json create mode 100644 checkpoint-498/training_args.bin create mode 100644 config.json create mode 100644 generation_config.json create mode 100644 model-00001-of-00003.safetensors create mode 100644 model-00002-of-00003.safetensors create mode 100644 model-00003-of-00003.safetensors create mode 100644 model.safetensors.index.json create mode 100644 special_tokens_map.json create mode 100644 tokenizer.json create mode 100644 tokenizer.model create mode 100644 tokenizer_config.json create mode 100644 train_results.json create mode 100644 trainer_log.jsonl create mode 100644 trainer_state.json create mode 100644 training_args.bin create mode 100644 training_loss.png diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..e16daa5 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,37 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +checkpoint-498/tokenizer.json filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..717e816 --- /dev/null +++ b/README.md @@ -0,0 +1,58 @@ +--- +library_name: transformers +license: other +base_model: jxm/shieldgemma-2b +tags: +- llama-factory +- full +- generated_from_trainer +model-index: +- name: sg_x9e + results: [] +--- + + + +# sg_x9e + +This model is a fine-tuned version of [jxm/shieldgemma-2b](https://huggingface.co/jxm/shieldgemma-2b) on the v1.6, the v1.6b, the v1.6c, the v1.6d and the v1.6e datasets. + +## Model description + +More information needed + +## Intended uses & limitations + +More information needed + +## Training and evaluation data + +More information needed + +## Training procedure + +### Training hyperparameters + +The following hyperparameters were used during training: +- learning_rate: 5e-06 +- train_batch_size: 4 +- eval_batch_size: 8 +- seed: 42 +- gradient_accumulation_steps: 16 +- total_train_batch_size: 64 +- optimizer: Use OptimizerNames.ADAMW_TORCH_FUSED with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments +- lr_scheduler_type: cosine +- lr_scheduler_warmup_ratio: 0.1 +- num_epochs: 2 + +### Training results + + + +### Framework versions + +- Transformers 4.57.1 +- Pytorch 2.10.0+cu129 +- Datasets 4.0.0 +- Tokenizers 0.22.2 diff --git a/all_results.json b/all_results.json new file mode 100644 index 0000000..820de62 --- /dev/null +++ b/all_results.json @@ -0,0 +1,8 @@ +{ + "epoch": 2.0, + "total_flos": 1.28494219771102e+18, + "train_loss": 0.03900693407858232, + "train_runtime": 6158.7829, + "train_samples_per_second": 5.17, + "train_steps_per_second": 0.081 +} \ No newline at end of file diff --git a/chat_template.jinja b/chat_template.jinja new file mode 100644 index 0000000..b017fbd --- /dev/null +++ b/chat_template.jinja @@ -0,0 +1,29 @@ +{%- if not guideline and messages[0].role == "system" %} + {%- set guideline = messages[0].content %} + {%- set messages = messages[1:] %} +{%- endif %} + +{{- bos_token }} +{%- if messages[-1].role == 'user' %} + {{- "You are a policy expert trying to help determine whether a user prompt is in violation of the defined safety policies.\n\n" }} + {{- "\n" }} + {{- "Human Question: " + messages[-1].content }} + {{- "\n\n\n" }} + {{- "Our safety principle is defined in the below:\n\n" }} + {{- "* " + guideline + "\n" }} + {{- "\n===\n\n" }} + {{- "Does the human question violate the above principle? Your answer must start with 'Yes' or 'No'. And then walk through step by step to be sure we answer correctly.\n\n" }} +{%- else %} + {{- "You are a policy expert trying to help determine whether an AI response to a prompt is in violation of the defined safety policies.\n\n" }} + {{- "\n" }} + {{- "Human Question: " + messages[-2].content }} + {{- "\n\n" }} + {{- "\n" }} + {{- "Chatbot Response: " + messages[-1].content }} + {{- "\n\n\n" }} + {{- "Our safety principle is defined in the below:\n\n" }} + {{- "* " + guideline + "\n" }} + {{- "\n===\n\n" }} + {{- "Does the Chatbot Response violate the above principle? Your answer must start with 'Yes' or 'No'. And then walk through step by step to be sure we answer correctly.\n\n" }} +{%- endif %} + diff --git a/checkpoint-498/chat_template.jinja b/checkpoint-498/chat_template.jinja new file mode 100644 index 0000000..b017fbd --- /dev/null +++ b/checkpoint-498/chat_template.jinja @@ -0,0 +1,29 @@ +{%- if not guideline and messages[0].role == "system" %} + {%- set guideline = messages[0].content %} + {%- set messages = messages[1:] %} +{%- endif %} + +{{- bos_token }} +{%- if messages[-1].role == 'user' %} + {{- "You are a policy expert trying to help determine whether a user prompt is in violation of the defined safety policies.\n\n" }} + {{- "\n" }} + {{- "Human Question: " + messages[-1].content }} + {{- "\n\n\n" }} + {{- "Our safety principle is defined in the below:\n\n" }} + {{- "* " + guideline + "\n" }} + {{- "\n===\n\n" }} + {{- "Does the human question violate the above principle? Your answer must start with 'Yes' or 'No'. And then walk through step by step to be sure we answer correctly.\n\n" }} +{%- else %} + {{- "You are a policy expert trying to help determine whether an AI response to a prompt is in violation of the defined safety policies.\n\n" }} + {{- "\n" }} + {{- "Human Question: " + messages[-2].content }} + {{- "\n\n" }} + {{- "\n" }} + {{- "Chatbot Response: " + messages[-1].content }} + {{- "\n\n\n" }} + {{- "Our safety principle is defined in the below:\n\n" }} + {{- "* " + guideline + "\n" }} + {{- "\n===\n\n" }} + {{- "Does the Chatbot Response violate the above principle? Your answer must start with 'Yes' or 'No'. And then walk through step by step to be sure we answer correctly.\n\n" }} +{%- endif %} + diff --git a/checkpoint-498/config.json b/checkpoint-498/config.json new file mode 100644 index 0000000..ecfcdd0 --- /dev/null +++ b/checkpoint-498/config.json @@ -0,0 +1,60 @@ +{ + "architectures": [ + "Gemma2ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "attn_logit_softcapping": 50.0, + "bos_token_id": 2, + "cache_implementation": "hybrid", + "dtype": "float32", + "eos_token_id": 107, + "final_logit_softcapping": 30.0, + "head_dim": 256, + "hidden_act": "gelu_pytorch_tanh", + "hidden_activation": "gelu_pytorch_tanh", + "hidden_size": 2304, + "initializer_range": 0.02, + "intermediate_size": 9216, + "layer_types": [ + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention" + ], + "max_position_embeddings": 8192, + "model_type": "gemma2", + "num_attention_heads": 8, + "num_hidden_layers": 26, + "num_key_value_heads": 4, + "pad_token_id": 0, + "query_pre_attn_scalar": 224, + "rms_norm_eps": 1e-06, + "rope_theta": 10000.0, + "sliding_window": 4096, + "transformers_version": "4.57.1", + "use_cache": false, + "vocab_size": 256000 +} diff --git a/checkpoint-498/generation_config.json b/checkpoint-498/generation_config.json new file mode 100644 index 0000000..4700366 --- /dev/null +++ b/checkpoint-498/generation_config.json @@ -0,0 +1,11 @@ +{ + "_from_model_config": true, + "bos_token_id": 2, + "cache_implementation": "hybrid", + "eos_token_id": [ + 107, + 1 + ], + "pad_token_id": 0, + "transformers_version": "4.57.1" +} diff --git a/checkpoint-498/model-00001-of-00003.safetensors b/checkpoint-498/model-00001-of-00003.safetensors new file mode 100644 index 0000000..b03663e --- /dev/null +++ b/checkpoint-498/model-00001-of-00003.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:27f3efcf7fa387005a2447be0b4ae55681e84d5d0385c0be339b8e43b61ecd58 +size 4992576136 diff --git a/checkpoint-498/model-00002-of-00003.safetensors b/checkpoint-498/model-00002-of-00003.safetensors new file mode 100644 index 0000000..8d018f9 --- /dev/null +++ b/checkpoint-498/model-00002-of-00003.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f49c8916c976384e6cf23c4dfebbf04618e1741ca8838c69a4b3b55267b3b65c +size 4983443424 diff --git a/checkpoint-498/model-00003-of-00003.safetensors b/checkpoint-498/model-00003-of-00003.safetensors new file mode 100644 index 0000000..7874829 --- /dev/null +++ b/checkpoint-498/model-00003-of-00003.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bcb13e57128059be2ef3f6db700237dec8ad73862fdcf2665b0cc904b48b23c0 +size 481381384 diff --git a/checkpoint-498/model.safetensors.index.json b/checkpoint-498/model.safetensors.index.json new file mode 100644 index 0000000..1a43cf8 --- /dev/null +++ b/checkpoint-498/model.safetensors.index.json @@ -0,0 +1,296 @@ +{ + "metadata": { + "total_parameters": 2614341888, + "total_size": 10457367552 + }, + "weight_map": { + "model.embed_tokens.weight": "model-00001-of-00003.safetensors", + "model.layers.0.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.0.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.0.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.0.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.0.pre_feedforward_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.1.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.1.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.1.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.1.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.1.pre_feedforward_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.10.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.10.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.10.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.10.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.10.pre_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.11.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.11.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.11.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.11.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.11.pre_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.12.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.12.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.12.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.12.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.12.pre_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.13.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.13.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.13.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.13.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.13.pre_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.14.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.14.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.14.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.14.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.14.pre_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.15.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.15.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.15.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.15.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.15.pre_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.16.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.16.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.16.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.16.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.16.pre_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.17.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.17.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.17.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.17.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.17.pre_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.18.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.18.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.18.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.18.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.18.pre_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.19.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.19.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.19.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.19.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.19.pre_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.2.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.2.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.2.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.2.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.2.pre_feedforward_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.20.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.20.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.20.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.20.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.20.pre_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.21.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.21.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.21.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.21.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.21.pre_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.22.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.22.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.22.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.22.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.22.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.22.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.22.pre_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.23.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.23.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.23.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.23.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.23.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.23.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.23.pre_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.23.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.23.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.23.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.23.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.24.input_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.24.mlp.down_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.24.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.24.mlp.up_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.24.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.24.pre_feedforward_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.24.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.24.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.24.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.24.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.25.input_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.25.mlp.down_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.25.mlp.up_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.25.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.25.pre_feedforward_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.3.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.3.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.3.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.3.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.3.pre_feedforward_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.4.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.4.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.4.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.4.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.4.pre_feedforward_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.5.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.5.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.5.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.5.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.5.pre_feedforward_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.6.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.6.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.6.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.6.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.6.pre_feedforward_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.7.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.7.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.7.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.7.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.7.pre_feedforward_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.8.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.8.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.8.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.8.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.8.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.8.pre_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.9.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.9.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.9.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.9.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.9.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.9.pre_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.9.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.9.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.9.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.norm.weight": "model-00003-of-00003.safetensors" + } +} diff --git a/checkpoint-498/optimizer.pt b/checkpoint-498/optimizer.pt new file mode 100644 index 0000000..bf7acb7 --- /dev/null +++ b/checkpoint-498/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a0778ad4e60fc37022ea892da860654bacaa543bb3c603595bb0e74551f2e324 +size 20914991879 diff --git a/checkpoint-498/rng_state.pth b/checkpoint-498/rng_state.pth new file mode 100644 index 0000000..1feba1a --- /dev/null +++ b/checkpoint-498/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:098b29492211804ab324a36f37466821d948280bb74fce4ba895c03f13ecd878 +size 14645 diff --git a/checkpoint-498/scheduler.pt b/checkpoint-498/scheduler.pt new file mode 100644 index 0000000..0210aae --- /dev/null +++ b/checkpoint-498/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1bb0f8caa33fed2c1de44d000c78a6e40fc301c49dd2a1b8e9ab8bb30c5f3dfd +size 1465 diff --git a/checkpoint-498/special_tokens_map.json b/checkpoint-498/special_tokens_map.json new file mode 100644 index 0000000..60a73bc --- /dev/null +++ b/checkpoint-498/special_tokens_map.json @@ -0,0 +1,30 @@ +{ + "bos_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "unk_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + } +} diff --git a/checkpoint-498/tokenizer.json b/checkpoint-498/tokenizer.json new file mode 100644 index 0000000..a4a305d --- /dev/null +++ b/checkpoint-498/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5f7eee611703c5ce5d1eee32d9cdcfe465647b8aff0c1dfb3bed7ad7dbb05060 +size 34362873 diff --git a/checkpoint-498/tokenizer.model b/checkpoint-498/tokenizer.model new file mode 100644 index 0000000..796efe9 --- /dev/null +++ b/checkpoint-498/tokenizer.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:61a7b147390c64585d6c3543dd6fc636906c9af3865a5548f27f31aee1d4c8e2 +size 4241003 diff --git a/checkpoint-498/tokenizer_config.json b/checkpoint-498/tokenizer_config.json new file mode 100644 index 0000000..789c0ca --- /dev/null +++ b/checkpoint-498/tokenizer_config.json @@ -0,0 +1,2011 @@ +{ + "add_bos_token": true, + "add_eos_token": false, + "added_tokens_decoder": { + "0": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "1": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "2": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "3": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "4": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "5": { + "content": "<2mass>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "6": { + "content": "[@BOS@]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "7": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "8": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "9": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "10": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "11": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "12": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "13": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "14": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "15": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "16": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "17": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "18": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "19": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "20": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "21": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "22": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "23": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "24": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "25": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "26": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "27": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "28": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "29": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "30": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "31": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "32": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "33": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "34": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "35": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "36": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "37": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "38": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "39": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "40": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "41": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "42": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "43": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "44": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "45": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "46": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "47": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "48": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "49": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "50": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "51": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "52": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "53": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "54": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "55": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "56": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "57": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "58": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "59": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "60": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "61": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "62": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "63": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "64": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "65": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "66": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "67": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "68": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "69": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "70": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "71": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "72": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "73": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "74": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "75": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "76": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "77": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "78": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "79": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "80": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "81": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "82": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "83": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "84": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "85": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "86": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "87": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "88": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "89": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "90": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "91": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "92": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "93": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "94": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "95": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "96": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "97": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "98": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "99": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "100": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "101": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "102": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "103": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "104": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "105": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "106": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "107": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "108": { + "content": "\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "109": { + "content": "\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "110": { + "content": "\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "111": { + "content": "\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "112": { + "content": "\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "113": { + "content": "\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "114": { + "content": "\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "115": { + "content": "\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "116": { + "content": "\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "117": { + "content": "\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "118": { + "content": "\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "119": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "120": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "121": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "122": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "123": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "124": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "125": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "126": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "127": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "128": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "129": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "130": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "131": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "132": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "133": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "134": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "135": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "136": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "137": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "138": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "139": { + "content": "▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "140": { + "content": "▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "141": { + "content": "▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "142": { + "content": "▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "143": { + "content": "▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "144": { + "content": "▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "145": { + "content": "▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "146": { + "content": "▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "147": { + "content": "▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "148": { + "content": "▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "149": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "150": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "152": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "153": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "154": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "155": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "156": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "157": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "158": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "159": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "160": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "161": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "162": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "163": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "164": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "165": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "166": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "167": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "168": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "169": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "170": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "172": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "173": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "174": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "175": { + "content": "
", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "171": { + "content": "
", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "176": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "177": { + "content": "
", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "178": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "179": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "180": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "181": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "182": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "183": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "184": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "185": { + "content": "

", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "186": { + "content": "

", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "187": { + "content": "

", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "188": { + "content": "

", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "189": { + "content": "

", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "190": { + "content": "
", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "191": { + "content": "
", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "192": { + "content": "
", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "193": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "194": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "195": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "196": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "197": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "198": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "199": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "200": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "201": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "202": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "203": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "204": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "205": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "206": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "207": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "208": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "209": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "210": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "211": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "212": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "213": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "214": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "215": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "216": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255968": { + "content": "[toxicity=0]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255969": { + "content": "\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255970": { + "content": "\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255971": { + "content": "\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255972": { + "content": "\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255973": { + "content": "\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255974": { + "content": "\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255975": { + "content": "\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255976": { + "content": "\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255977": { + "content": "\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255978": { + "content": "\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255979": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255980": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255981": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255982": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255983": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255984": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255985": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255986": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255987": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255988": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255989": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255990": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255991": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255992": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255993": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255994": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255995": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255996": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255997": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255998": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255999": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + } + }, + "bos_token": "", + "clean_up_tokenization_spaces": false, + "eos_token": "", + "extra_special_tokens": {}, + "model_max_length": 1000000000000000019884624838656, + "pad_token": "", + "padding_side": "right", + "sp_model_kwargs": {}, + "spaces_between_special_tokens": false, + "split_special_tokens": false, + "tokenizer_class": "GemmaTokenizer", + "unk_token": "", + "use_default_system_prompt": false +} diff --git a/checkpoint-498/trainer_state.json b/checkpoint-498/trainer_state.json new file mode 100644 index 0000000..73efbd3 --- /dev/null +++ b/checkpoint-498/trainer_state.json @@ -0,0 +1,727 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 2.0, + "eval_steps": 500, + "global_step": 498, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.02009545340366742, + "grad_norm": 101.4629135131836, + "learning_rate": 4.0000000000000003e-07, + "loss": 2.9256, + "step": 5 + }, + { + "epoch": 0.04019090680733484, + "grad_norm": 83.51904296875, + "learning_rate": 9.000000000000001e-07, + "loss": 2.7459, + "step": 10 + }, + { + "epoch": 0.06028636021100226, + "grad_norm": 25.10388946533203, + "learning_rate": 1.4000000000000001e-06, + "loss": 1.637, + "step": 15 + }, + { + "epoch": 0.08038181361466969, + "grad_norm": 13.024300575256348, + "learning_rate": 1.9000000000000002e-06, + "loss": 1.1459, + "step": 20 + }, + { + "epoch": 0.1004772670183371, + "grad_norm": 8.315956115722656, + "learning_rate": 2.4000000000000003e-06, + "loss": 0.7456, + "step": 25 + }, + { + "epoch": 0.12057272042200452, + "grad_norm": 8.930346488952637, + "learning_rate": 2.9e-06, + "loss": 0.5717, + "step": 30 + }, + { + "epoch": 0.14066817382567195, + "grad_norm": 7.356456756591797, + "learning_rate": 3.4000000000000005e-06, + "loss": 0.414, + "step": 35 + }, + { + "epoch": 0.16076362722933937, + "grad_norm": 5.648008346557617, + "learning_rate": 3.900000000000001e-06, + "loss": 0.2577, + "step": 40 + }, + { + "epoch": 0.1808590806330068, + "grad_norm": 6.329988956451416, + "learning_rate": 4.4e-06, + "loss": 0.2394, + "step": 45 + }, + { + "epoch": 0.2009545340366742, + "grad_norm": 5.1738080978393555, + "learning_rate": 4.9000000000000005e-06, + "loss": 0.1848, + "step": 50 + }, + { + "epoch": 0.22104998744034163, + "grad_norm": 4.86478853225708, + "learning_rate": 4.999016565957633e-06, + "loss": 0.2195, + "step": 55 + }, + { + "epoch": 0.24114544084400905, + "grad_norm": 4.442976474761963, + "learning_rate": 4.995022691166621e-06, + "loss": 0.1582, + "step": 60 + }, + { + "epoch": 0.26124089424767644, + "grad_norm": 3.704345941543579, + "learning_rate": 4.987961816680493e-06, + "loss": 0.183, + "step": 65 + }, + { + "epoch": 0.2813363476513439, + "grad_norm": 3.772160768508911, + "learning_rate": 4.977842622059546e-06, + "loss": 0.164, + "step": 70 + }, + { + "epoch": 0.30143180105501133, + "grad_norm": 3.2532081604003906, + "learning_rate": 4.96467754629559e-06, + "loss": 0.1781, + "step": 75 + }, + { + "epoch": 0.32152725445867875, + "grad_norm": 2.893615484237671, + "learning_rate": 4.948482772521346e-06, + "loss": 0.1288, + "step": 80 + }, + { + "epoch": 0.34162270786234616, + "grad_norm": 4.654979705810547, + "learning_rate": 4.929278208117378e-06, + "loss": 0.1444, + "step": 85 + }, + { + "epoch": 0.3617181612660136, + "grad_norm": 2.8989882469177246, + "learning_rate": 4.907087460240999e-06, + "loss": 0.1151, + "step": 90 + }, + { + "epoch": 0.381813614669681, + "grad_norm": 3.7547988891601562, + "learning_rate": 4.881937806807241e-06, + "loss": 0.1197, + "step": 95 + }, + { + "epoch": 0.4019090680733484, + "grad_norm": 2.273831605911255, + "learning_rate": 4.8538601629575525e-06, + "loss": 0.0966, + "step": 100 + }, + { + "epoch": 0.42200452147701584, + "grad_norm": 2.4642996788024902, + "learning_rate": 4.822889043057446e-06, + "loss": 0.1088, + "step": 105 + }, + { + "epoch": 0.44209997488068326, + "grad_norm": 3.29301381111145, + "learning_rate": 4.789062518269812e-06, + "loss": 0.1163, + "step": 110 + }, + { + "epoch": 0.4621954282843507, + "grad_norm": 2.45603084564209, + "learning_rate": 4.752422169756048e-06, + "loss": 0.1151, + "step": 115 + }, + { + "epoch": 0.4822908816880181, + "grad_norm": 3.199894666671753, + "learning_rate": 4.713013037562531e-06, + "loss": 0.098, + "step": 120 + }, + { + "epoch": 0.5023863350916855, + "grad_norm": 1.7128061056137085, + "learning_rate": 4.670883565255264e-06, + "loss": 0.0968, + "step": 125 + }, + { + "epoch": 0.5224817884953529, + "grad_norm": 2.645235776901245, + "learning_rate": 4.6260855403707625e-06, + "loss": 0.1068, + "step": 130 + }, + { + "epoch": 0.5425772418990203, + "grad_norm": 2.3140530586242676, + "learning_rate": 4.578674030756364e-06, + "loss": 0.0914, + "step": 135 + }, + { + "epoch": 0.5626726953026878, + "grad_norm": 1.8588067293167114, + "learning_rate": 4.528707316878233e-06, + "loss": 0.0995, + "step": 140 + }, + { + "epoch": 0.5827681487063552, + "grad_norm": 1.9172967672348022, + "learning_rate": 4.476246820180259e-06, + "loss": 0.08, + "step": 145 + }, + { + "epoch": 0.6028636021100227, + "grad_norm": 2.9082584381103516, + "learning_rate": 4.421357027581921e-06, + "loss": 0.1066, + "step": 150 + }, + { + "epoch": 0.62295905551369, + "grad_norm": 3.6944031715393066, + "learning_rate": 4.364105412207914e-06, + "loss": 0.1124, + "step": 155 + }, + { + "epoch": 0.6430545089173575, + "grad_norm": 1.5622600317001343, + "learning_rate": 4.304562350447008e-06, + "loss": 0.0856, + "step": 160 + }, + { + "epoch": 0.6631499623210249, + "grad_norm": 2.388239860534668, + "learning_rate": 4.242801035442059e-06, + "loss": 0.0766, + "step": 165 + }, + { + "epoch": 0.6832454157246923, + "grad_norm": 1.7182718515396118, + "learning_rate": 4.178897387117547e-06, + "loss": 0.104, + "step": 170 + }, + { + "epoch": 0.7033408691283597, + "grad_norm": 2.2367119789123535, + "learning_rate": 4.11292995885522e-06, + "loss": 0.0737, + "step": 175 + }, + { + "epoch": 0.7234363225320272, + "grad_norm": 2.007200241088867, + "learning_rate": 4.044979840932567e-06, + "loss": 0.0893, + "step": 180 + }, + { + "epoch": 0.7435317759356945, + "grad_norm": 2.6910905838012695, + "learning_rate": 3.975130560842821e-06, + "loss": 0.0944, + "step": 185 + }, + { + "epoch": 0.763627229339362, + "grad_norm": 2.042320489883423, + "learning_rate": 3.9034679806190204e-06, + "loss": 0.0843, + "step": 190 + }, + { + "epoch": 0.7837226827430294, + "grad_norm": 2.700977087020874, + "learning_rate": 3.830080191288342e-06, + "loss": 0.0669, + "step": 195 + }, + { + "epoch": 0.8038181361466968, + "grad_norm": 3.0307390689849854, + "learning_rate": 3.755057404586455e-06, + "loss": 0.0955, + "step": 200 + }, + { + "epoch": 0.8239135895503642, + "grad_norm": 1.11714506149292, + "learning_rate": 4.0000000000000003e-07, + "loss": 0.0956, + "step": 205 + }, + { + "epoch": 0.8440090429540317, + "grad_norm": 1.5246130228042603, + "learning_rate": 9.000000000000001e-07, + "loss": 0.0755, + "step": 210 + }, + { + "epoch": 0.864104496357699, + "grad_norm": 2.0117733478546143, + "learning_rate": 1.4000000000000001e-06, + "loss": 0.0894, + "step": 215 + }, + { + "epoch": 0.8841999497613665, + "grad_norm": 2.038092613220215, + "learning_rate": 1.9000000000000002e-06, + "loss": 0.074, + "step": 220 + }, + { + "epoch": 0.9042954031650339, + "grad_norm": 1.607172966003418, + "learning_rate": 2.4000000000000003e-06, + "loss": 0.0612, + "step": 225 + }, + { + "epoch": 0.9243908565687013, + "grad_norm": 1.643001914024353, + "learning_rate": 2.9e-06, + "loss": 0.0967, + "step": 230 + }, + { + "epoch": 0.9444863099723687, + "grad_norm": 1.5271589756011963, + "learning_rate": 3.4000000000000005e-06, + "loss": 0.1008, + "step": 235 + }, + { + "epoch": 0.9645817633760362, + "grad_norm": 2.2744393348693848, + "learning_rate": 3.900000000000001e-06, + "loss": 0.0846, + "step": 240 + }, + { + "epoch": 0.9846772167797035, + "grad_norm": 1.2268872261047363, + "learning_rate": 4.4e-06, + "loss": 0.0889, + "step": 245 + }, + { + "epoch": 1.0040190906807336, + "grad_norm": 1.971510648727417, + "learning_rate": 4.9000000000000005e-06, + "loss": 0.1021, + "step": 250 + }, + { + "epoch": 1.0241145440844008, + "grad_norm": 0.6695238947868347, + "learning_rate": 4.999016565957633e-06, + "loss": 0.057, + "step": 255 + }, + { + "epoch": 1.0442099974880683, + "grad_norm": 3.379134178161621, + "learning_rate": 4.995022691166621e-06, + "loss": 0.0695, + "step": 260 + }, + { + "epoch": 1.0643054508917358, + "grad_norm": 2.573274850845337, + "learning_rate": 4.987961816680493e-06, + "loss": 0.0713, + "step": 265 + }, + { + "epoch": 1.0844009042954033, + "grad_norm": 1.3487026691436768, + "learning_rate": 4.977842622059546e-06, + "loss": 0.0704, + "step": 270 + }, + { + "epoch": 1.1044963576990705, + "grad_norm": 1.7525826692581177, + "learning_rate": 4.96467754629559e-06, + "loss": 0.0615, + "step": 275 + }, + { + "epoch": 1.124591811102738, + "grad_norm": 1.1195745468139648, + "learning_rate": 4.948482772521346e-06, + "loss": 0.0616, + "step": 280 + }, + { + "epoch": 1.1446872645064055, + "grad_norm": 2.203016996383667, + "learning_rate": 4.929278208117378e-06, + "loss": 0.0627, + "step": 285 + }, + { + "epoch": 1.164782717910073, + "grad_norm": 3.2196788787841797, + "learning_rate": 4.907087460240999e-06, + "loss": 0.0707, + "step": 290 + }, + { + "epoch": 1.1848781713137402, + "grad_norm": 2.4914467334747314, + "learning_rate": 4.881937806807241e-06, + "loss": 0.0758, + "step": 295 + }, + { + "epoch": 1.2049736247174077, + "grad_norm": 2.223809242248535, + "learning_rate": 4.8538601629575525e-06, + "loss": 0.0698, + "step": 300 + }, + { + "epoch": 1.2250690781210751, + "grad_norm": 1.5360513925552368, + "learning_rate": 4.822889043057446e-06, + "loss": 0.0593, + "step": 305 + }, + { + "epoch": 1.2451645315247426, + "grad_norm": 1.8444727659225464, + "learning_rate": 4.789062518269812e-06, + "loss": 0.0635, + "step": 310 + }, + { + "epoch": 1.2652599849284099, + "grad_norm": 1.5122318267822266, + "learning_rate": 4.752422169756048e-06, + "loss": 0.0737, + "step": 315 + }, + { + "epoch": 1.2853554383320773, + "grad_norm": 2.0612282752990723, + "learning_rate": 4.713013037562531e-06, + "loss": 0.065, + "step": 320 + }, + { + "epoch": 1.3054508917357448, + "grad_norm": 1.4917356967926025, + "learning_rate": 4.670883565255264e-06, + "loss": 0.0563, + "step": 325 + }, + { + "epoch": 1.3255463451394123, + "grad_norm": 1.861807942390442, + "learning_rate": 4.6260855403707625e-06, + "loss": 0.0654, + "step": 330 + }, + { + "epoch": 1.3456417985430797, + "grad_norm": 2.3603355884552, + "learning_rate": 4.578674030756364e-06, + "loss": 0.0616, + "step": 335 + }, + { + "epoch": 1.365737251946747, + "grad_norm": 1.4312632083892822, + "learning_rate": 4.528707316878233e-06, + "loss": 0.0608, + "step": 340 + }, + { + "epoch": 1.3858327053504145, + "grad_norm": 3.3819150924682617, + "learning_rate": 4.476246820180259e-06, + "loss": 0.0573, + "step": 345 + }, + { + "epoch": 1.405928158754082, + "grad_norm": 2.076028347015381, + "learning_rate": 4.421357027581921e-06, + "loss": 0.0648, + "step": 350 + }, + { + "epoch": 1.4260236121577492, + "grad_norm": 2.0433411598205566, + "learning_rate": 4.364105412207914e-06, + "loss": 0.0679, + "step": 355 + }, + { + "epoch": 1.4461190655614167, + "grad_norm": 1.695204496383667, + "learning_rate": 4.304562350447008e-06, + "loss": 0.0655, + "step": 360 + }, + { + "epoch": 1.4662145189650841, + "grad_norm": 0.7300482988357544, + "learning_rate": 4.242801035442059e-06, + "loss": 0.0662, + "step": 365 + }, + { + "epoch": 1.4863099723687516, + "grad_norm": 1.8220871686935425, + "learning_rate": 4.178897387117547e-06, + "loss": 0.0659, + "step": 370 + }, + { + "epoch": 1.506405425772419, + "grad_norm": 1.4738329648971558, + "learning_rate": 4.11292995885522e-06, + "loss": 0.0605, + "step": 375 + }, + { + "epoch": 1.5265008791760866, + "grad_norm": 1.562551736831665, + "learning_rate": 4.044979840932567e-06, + "loss": 0.0741, + "step": 380 + }, + { + "epoch": 1.5465963325797538, + "grad_norm": 1.5520535707473755, + "learning_rate": 3.975130560842821e-06, + "loss": 0.058, + "step": 385 + }, + { + "epoch": 1.5666917859834213, + "grad_norm": 1.29172682762146, + "learning_rate": 3.9034679806190204e-06, + "loss": 0.0584, + "step": 390 + }, + { + "epoch": 1.5867872393870885, + "grad_norm": 1.9488056898117065, + "learning_rate": 3.830080191288342e-06, + "loss": 0.0582, + "step": 395 + }, + { + "epoch": 1.606882692790756, + "grad_norm": 1.6518131494522095, + "learning_rate": 3.755057404586455e-06, + "loss": 0.0634, + "step": 400 + }, + { + "epoch": 1.6269781461944235, + "grad_norm": 1.5398550033569336, + "learning_rate": 3.6784918420649952e-06, + "loss": 0.0525, + "step": 405 + }, + { + "epoch": 1.647073599598091, + "grad_norm": 1.6156771183013916, + "learning_rate": 3.600477621728473e-06, + "loss": 0.0645, + "step": 410 + }, + { + "epoch": 1.6671690530017584, + "grad_norm": 1.564670205116272, + "learning_rate": 3.521110642339991e-06, + "loss": 0.0526, + "step": 415 + }, + { + "epoch": 1.687264506405426, + "grad_norm": 1.2277319431304932, + "learning_rate": 3.4404884655379433e-06, + "loss": 0.0616, + "step": 420 + }, + { + "epoch": 1.7073599598090932, + "grad_norm": 1.1757029294967651, + "learning_rate": 3.358710195908653e-06, + "loss": 0.0537, + "step": 425 + }, + { + "epoch": 1.7274554132127606, + "grad_norm": 1.0405977964401245, + "learning_rate": 3.27587635916233e-06, + "loss": 0.0619, + "step": 430 + }, + { + "epoch": 1.747550866616428, + "grad_norm": 1.5975450277328491, + "learning_rate": 3.1920887785621233e-06, + "loss": 0.0603, + "step": 435 + }, + { + "epoch": 1.7676463200200954, + "grad_norm": 1.7237313985824585, + "learning_rate": 3.10745044975816e-06, + "loss": 0.0594, + "step": 440 + }, + { + "epoch": 1.7877417734237628, + "grad_norm": 1.7574973106384277, + "learning_rate": 3.022065414180425e-06, + "loss": 0.0608, + "step": 445 + }, + { + "epoch": 1.8078372268274303, + "grad_norm": 1.450028419494629, + "learning_rate": 2.9360386311461276e-06, + "loss": 0.0505, + "step": 450 + }, + { + "epoch": 1.8279326802310978, + "grad_norm": 1.0331708192825317, + "learning_rate": 2.849475848838749e-06, + "loss": 0.0497, + "step": 455 + }, + { + "epoch": 1.8480281336347653, + "grad_norm": 1.5370794534683228, + "learning_rate": 2.762483474317387e-06, + "loss": 0.0625, + "step": 460 + }, + { + "epoch": 1.8681235870384325, + "grad_norm": 2.073357582092285, + "learning_rate": 2.6751684427161684e-06, + "loss": 0.0448, + "step": 465 + }, + { + "epoch": 1.8882190404421, + "grad_norm": 1.4379805326461792, + "learning_rate": 2.587638085794541e-06, + "loss": 0.0588, + "step": 470 + }, + { + "epoch": 1.9083144938457672, + "grad_norm": 1.8600538969039917, + "learning_rate": 2.5e-06, + "loss": 0.0511, + "step": 475 + }, + { + "epoch": 1.9284099472494347, + "grad_norm": 1.2249418497085571, + "learning_rate": 2.41236191420546e-06, + "loss": 0.0497, + "step": 480 + }, + { + "epoch": 1.9485054006531022, + "grad_norm": 0.8812041282653809, + "learning_rate": 2.3248315572838316e-06, + "loss": 0.0498, + "step": 485 + }, + { + "epoch": 1.9686008540567697, + "grad_norm": 0.8966412544250488, + "learning_rate": 2.237516525682614e-06, + "loss": 0.0459, + "step": 490 + }, + { + "epoch": 1.9886963074604371, + "grad_norm": 1.238738775253296, + "learning_rate": 2.1505241511612522e-06, + "loss": 0.052, + "step": 495 + } + ], + "logging_steps": 5, + "max_steps": 498, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 200, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 1.28494219771102e+18, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/checkpoint-498/training_args.bin b/checkpoint-498/training_args.bin new file mode 100644 index 0000000..e9a2513 --- /dev/null +++ b/checkpoint-498/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5acbccb5459f9c0d139eb9e75792351bde8d66e4f32093b31c72148828bf3e33 +size 6225 diff --git a/config.json b/config.json new file mode 100644 index 0000000..ecfcdd0 --- /dev/null +++ b/config.json @@ -0,0 +1,60 @@ +{ + "architectures": [ + "Gemma2ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "attn_logit_softcapping": 50.0, + "bos_token_id": 2, + "cache_implementation": "hybrid", + "dtype": "float32", + "eos_token_id": 107, + "final_logit_softcapping": 30.0, + "head_dim": 256, + "hidden_act": "gelu_pytorch_tanh", + "hidden_activation": "gelu_pytorch_tanh", + "hidden_size": 2304, + "initializer_range": 0.02, + "intermediate_size": 9216, + "layer_types": [ + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "full_attention" + ], + "max_position_embeddings": 8192, + "model_type": "gemma2", + "num_attention_heads": 8, + "num_hidden_layers": 26, + "num_key_value_heads": 4, + "pad_token_id": 0, + "query_pre_attn_scalar": 224, + "rms_norm_eps": 1e-06, + "rope_theta": 10000.0, + "sliding_window": 4096, + "transformers_version": "4.57.1", + "use_cache": false, + "vocab_size": 256000 +} diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..4700366 --- /dev/null +++ b/generation_config.json @@ -0,0 +1,11 @@ +{ + "_from_model_config": true, + "bos_token_id": 2, + "cache_implementation": "hybrid", + "eos_token_id": [ + 107, + 1 + ], + "pad_token_id": 0, + "transformers_version": "4.57.1" +} diff --git a/model-00001-of-00003.safetensors b/model-00001-of-00003.safetensors new file mode 100644 index 0000000..b03663e --- /dev/null +++ b/model-00001-of-00003.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:27f3efcf7fa387005a2447be0b4ae55681e84d5d0385c0be339b8e43b61ecd58 +size 4992576136 diff --git a/model-00002-of-00003.safetensors b/model-00002-of-00003.safetensors new file mode 100644 index 0000000..8d018f9 --- /dev/null +++ b/model-00002-of-00003.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f49c8916c976384e6cf23c4dfebbf04618e1741ca8838c69a4b3b55267b3b65c +size 4983443424 diff --git a/model-00003-of-00003.safetensors b/model-00003-of-00003.safetensors new file mode 100644 index 0000000..7874829 --- /dev/null +++ b/model-00003-of-00003.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bcb13e57128059be2ef3f6db700237dec8ad73862fdcf2665b0cc904b48b23c0 +size 481381384 diff --git a/model.safetensors.index.json b/model.safetensors.index.json new file mode 100644 index 0000000..1a43cf8 --- /dev/null +++ b/model.safetensors.index.json @@ -0,0 +1,296 @@ +{ + "metadata": { + "total_parameters": 2614341888, + "total_size": 10457367552 + }, + "weight_map": { + "model.embed_tokens.weight": "model-00001-of-00003.safetensors", + "model.layers.0.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.0.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.0.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.0.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.0.pre_feedforward_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.1.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.1.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.1.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.1.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.1.pre_feedforward_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.10.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.10.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.10.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.10.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.10.pre_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.11.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.11.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.11.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.11.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.11.pre_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.12.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.12.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.12.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.12.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.12.pre_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.13.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.13.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.13.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.13.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.13.pre_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.14.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.14.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.14.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.14.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.14.pre_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.15.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.15.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.15.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.15.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.15.pre_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.16.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.16.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.16.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.16.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.16.pre_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.17.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.17.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.17.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.17.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.17.pre_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.18.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.18.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.18.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.18.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.18.pre_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.19.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.19.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.19.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.19.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.19.pre_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.2.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.2.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.2.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.2.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.2.pre_feedforward_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.20.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.20.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.20.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.20.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.20.pre_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.21.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.21.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.21.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.21.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.21.pre_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.22.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.22.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.22.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.22.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.22.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.22.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.22.pre_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.23.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.23.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.23.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.23.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.23.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.23.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.23.pre_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.23.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.23.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.23.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.23.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.24.input_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.24.mlp.down_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.24.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.24.mlp.up_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.24.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.24.pre_feedforward_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.24.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.24.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.24.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.24.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.25.input_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.25.mlp.down_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.25.mlp.up_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.25.post_feedforward_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.25.pre_feedforward_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.3.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.3.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.3.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.3.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.3.pre_feedforward_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.4.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.4.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.4.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.4.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.4.pre_feedforward_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.5.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.5.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.5.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.5.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.5.pre_feedforward_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.6.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.6.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.6.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.6.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.6.pre_feedforward_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.7.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.7.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.7.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.7.post_feedforward_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.7.pre_feedforward_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.8.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.8.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.8.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.8.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.8.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.8.pre_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.9.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.9.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.9.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.9.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.9.post_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.9.pre_feedforward_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.9.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.9.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.9.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.norm.weight": "model-00003-of-00003.safetensors" + } +} diff --git a/special_tokens_map.json b/special_tokens_map.json new file mode 100644 index 0000000..60a73bc --- /dev/null +++ b/special_tokens_map.json @@ -0,0 +1,30 @@ +{ + "bos_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "unk_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + } +} diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..a4a305d --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5f7eee611703c5ce5d1eee32d9cdcfe465647b8aff0c1dfb3bed7ad7dbb05060 +size 34362873 diff --git a/tokenizer.model b/tokenizer.model new file mode 100644 index 0000000..796efe9 --- /dev/null +++ b/tokenizer.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:61a7b147390c64585d6c3543dd6fc636906c9af3865a5548f27f31aee1d4c8e2 +size 4241003 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..789c0ca --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,2011 @@ +{ + "add_bos_token": true, + "add_eos_token": false, + "added_tokens_decoder": { + "0": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "1": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "2": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "3": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "4": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "5": { + "content": "<2mass>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "6": { + "content": "[@BOS@]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "7": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "8": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "9": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "10": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "11": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "12": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "13": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "14": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "15": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "16": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "17": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "18": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "19": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "20": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "21": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "22": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "23": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "24": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "25": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "26": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "27": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "28": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "29": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "30": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "31": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "32": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "33": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "34": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "35": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "36": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "37": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "38": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "39": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "40": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "41": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "42": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "43": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "44": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "45": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "46": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "47": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "48": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "49": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "50": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "51": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "52": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "53": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "54": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "55": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "56": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "57": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "58": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "59": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "60": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "61": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "62": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "63": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "64": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "65": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "66": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "67": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "68": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "69": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "70": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "71": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "72": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "73": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "74": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "75": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "76": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "77": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "78": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "79": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "80": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "81": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "82": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "83": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "84": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "85": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "86": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "87": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "88": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "89": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "90": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "91": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "92": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "93": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "94": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "95": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "96": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "97": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "98": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "99": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "100": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "101": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "102": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "103": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "104": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "105": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "106": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "107": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "108": { + "content": "\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "109": { + "content": "\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "110": { + "content": "\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "111": { + "content": "\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "112": { + "content": "\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "113": { + "content": "\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "114": { + "content": "\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "115": { + "content": "\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "116": { + "content": "\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "117": { + "content": "\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "118": { + "content": "\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "119": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "120": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "121": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "122": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "123": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "124": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "125": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "126": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "127": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "128": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "129": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "130": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "131": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "132": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "133": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "134": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "135": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "136": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "137": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "138": { + "content": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "139": { + "content": "▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "140": { + "content": "▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "141": { + "content": "▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "142": { + "content": "▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "143": { + "content": "▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "144": { + "content": "▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "145": { + "content": "▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "146": { + "content": "▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "147": { + "content": "▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "148": { + "content": "▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "149": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "150": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "152": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "153": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "154": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "155": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "156": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "157": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "158": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "159": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "160": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "161": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "162": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "163": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "164": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "165": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "166": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "167": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "168": { + "content": "▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "169": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "170": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "172": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "173": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "174": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "175": { + "content": "
", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "171": { + "content": "
", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "176": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "177": { + "content": "
", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "178": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "179": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "180": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "181": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "182": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "183": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "184": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "185": { + "content": "

", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "186": { + "content": "

", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "187": { + "content": "

", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "188": { + "content": "

", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "189": { + "content": "

", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "190": { + "content": "
", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "191": { + "content": "
", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "192": { + "content": "
", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "193": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "194": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "195": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "196": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "197": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "198": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "199": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "200": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "201": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "202": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "203": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "204": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "205": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "206": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "207": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "208": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "209": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "210": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "211": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "212": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "213": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "214": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "215": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "216": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255968": { + "content": "[toxicity=0]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255969": { + "content": "\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255970": { + "content": "\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255971": { + "content": "\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255972": { + "content": "\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255973": { + "content": "\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255974": { + "content": "\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255975": { + "content": "\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255976": { + "content": "\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255977": { + "content": "\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255978": { + "content": "\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255979": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255980": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255981": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255982": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255983": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255984": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255985": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255986": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255987": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255988": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255989": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255990": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255991": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255992": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255993": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255994": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255995": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255996": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255997": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255998": { + "content": "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "255999": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + } + }, + "bos_token": "", + "clean_up_tokenization_spaces": false, + "eos_token": "", + "extra_special_tokens": {}, + "model_max_length": 1000000000000000019884624838656, + "pad_token": "", + "padding_side": "right", + "sp_model_kwargs": {}, + "spaces_between_special_tokens": false, + "split_special_tokens": false, + "tokenizer_class": "GemmaTokenizer", + "unk_token": "", + "use_default_system_prompt": false +} diff --git a/train_results.json b/train_results.json new file mode 100644 index 0000000..820de62 --- /dev/null +++ b/train_results.json @@ -0,0 +1,8 @@ +{ + "epoch": 2.0, + "total_flos": 1.28494219771102e+18, + "train_loss": 0.03900693407858232, + "train_runtime": 6158.7829, + "train_samples_per_second": 5.17, + "train_steps_per_second": 0.081 +} \ No newline at end of file diff --git a/trainer_log.jsonl b/trainer_log.jsonl new file mode 100644 index 0000000..31fd0fd --- /dev/null +++ b/trainer_log.jsonl @@ -0,0 +1,60 @@ +{"current_steps": 205, "total_steps": 498, "loss": 0.0956, "lr": 4.0000000000000003e-07, "epoch": 0.8239135895503642, "percentage": 41.16, "elapsed_time": "0:01:41", "remaining_time": "0:02:24"} +{"current_steps": 210, "total_steps": 498, "loss": 0.0755, "lr": 9.000000000000001e-07, "epoch": 0.8440090429540317, "percentage": 42.17, "elapsed_time": "0:03:21", "remaining_time": "0:04:36"} +{"current_steps": 215, "total_steps": 498, "loss": 0.0894, "lr": 1.4000000000000001e-06, "epoch": 0.864104496357699, "percentage": 43.17, "elapsed_time": "0:05:03", "remaining_time": "0:06:38"} +{"current_steps": 220, "total_steps": 498, "loss": 0.074, "lr": 1.9000000000000002e-06, "epoch": 0.8841999497613665, "percentage": 44.18, "elapsed_time": "0:06:43", "remaining_time": "0:08:29"} +{"current_steps": 225, "total_steps": 498, "loss": 0.0612, "lr": 2.4000000000000003e-06, "epoch": 0.9042954031650339, "percentage": 45.18, "elapsed_time": "0:08:26", "remaining_time": "0:10:14"} +{"current_steps": 230, "total_steps": 498, "loss": 0.0967, "lr": 2.9e-06, "epoch": 0.9243908565687013, "percentage": 46.18, "elapsed_time": "0:10:07", "remaining_time": "0:11:47"} +{"current_steps": 235, "total_steps": 498, "loss": 0.1008, "lr": 3.4000000000000005e-06, "epoch": 0.9444863099723687, "percentage": 47.19, "elapsed_time": "0:11:50", "remaining_time": "0:13:15"} +{"current_steps": 240, "total_steps": 498, "loss": 0.0846, "lr": 3.900000000000001e-06, "epoch": 0.9645817633760362, "percentage": 48.19, "elapsed_time": "0:13:32", "remaining_time": "0:14:33"} +{"current_steps": 245, "total_steps": 498, "loss": 0.0889, "lr": 4.4e-06, "epoch": 0.9846772167797035, "percentage": 49.2, "elapsed_time": "0:15:11", "remaining_time": "0:15:41"} +{"current_steps": 250, "total_steps": 498, "loss": 0.1021, "lr": 4.9000000000000005e-06, "epoch": 1.0040190906807336, "percentage": 50.2, "elapsed_time": "0:16:50", "remaining_time": "0:16:42"} +{"current_steps": 255, "total_steps": 498, "loss": 0.057, "lr": 4.999016565957633e-06, "epoch": 1.0241145440844008, "percentage": 51.2, "elapsed_time": "0:18:29", "remaining_time": "0:17:36"} +{"current_steps": 260, "total_steps": 498, "loss": 0.0695, "lr": 4.995022691166621e-06, "epoch": 1.0442099974880683, "percentage": 52.21, "elapsed_time": "0:20:08", "remaining_time": "0:18:25"} +{"current_steps": 265, "total_steps": 498, "loss": 0.0713, "lr": 4.987961816680493e-06, "epoch": 1.0643054508917358, "percentage": 53.21, "elapsed_time": "0:21:49", "remaining_time": "0:19:11"} +{"current_steps": 270, "total_steps": 498, "loss": 0.0704, "lr": 4.977842622059546e-06, "epoch": 1.0844009042954033, "percentage": 54.22, "elapsed_time": "0:23:30", "remaining_time": "0:19:50"} +{"current_steps": 275, "total_steps": 498, "loss": 0.0615, "lr": 4.96467754629559e-06, "epoch": 1.1044963576990705, "percentage": 55.22, "elapsed_time": "0:25:10", "remaining_time": "0:20:24"} +{"current_steps": 280, "total_steps": 498, "loss": 0.0616, "lr": 4.948482772521346e-06, "epoch": 1.124591811102738, "percentage": 56.22, "elapsed_time": "0:26:50", "remaining_time": "0:20:54"} +{"current_steps": 285, "total_steps": 498, "loss": 0.0627, "lr": 4.929278208117378e-06, "epoch": 1.1446872645064055, "percentage": 57.23, "elapsed_time": "0:28:31", "remaining_time": "0:21:19"} +{"current_steps": 290, "total_steps": 498, "loss": 0.0707, "lr": 4.907087460240999e-06, "epoch": 1.164782717910073, "percentage": 58.23, "elapsed_time": "0:30:11", "remaining_time": "0:21:39"} +{"current_steps": 295, "total_steps": 498, "loss": 0.0758, "lr": 4.881937806807241e-06, "epoch": 1.1848781713137402, "percentage": 59.24, "elapsed_time": "0:31:55", "remaining_time": "0:21:58"} +{"current_steps": 300, "total_steps": 498, "loss": 0.0698, "lr": 4.8538601629575525e-06, "epoch": 1.2049736247174077, "percentage": 60.24, "elapsed_time": "0:33:39", "remaining_time": "0:22:13"} +{"current_steps": 305, "total_steps": 498, "loss": 0.0593, "lr": 4.822889043057446e-06, "epoch": 1.2250690781210751, "percentage": 61.24, "elapsed_time": "0:35:22", "remaining_time": "0:22:23"} +{"current_steps": 310, "total_steps": 498, "loss": 0.0635, "lr": 4.789062518269812e-06, "epoch": 1.2451645315247426, "percentage": 62.25, "elapsed_time": "0:37:04", "remaining_time": "0:22:29"} +{"current_steps": 315, "total_steps": 498, "loss": 0.0737, "lr": 4.752422169756048e-06, "epoch": 1.2652599849284099, "percentage": 63.25, "elapsed_time": "0:38:46", "remaining_time": "0:22:31"} +{"current_steps": 320, "total_steps": 498, "loss": 0.065, "lr": 4.713013037562531e-06, "epoch": 1.2853554383320773, "percentage": 64.26, "elapsed_time": "0:40:24", "remaining_time": "0:22:28"} +{"current_steps": 325, "total_steps": 498, "loss": 0.0563, "lr": 4.670883565255264e-06, "epoch": 1.3054508917357448, "percentage": 65.26, "elapsed_time": "0:42:06", "remaining_time": "0:22:24"} +{"current_steps": 330, "total_steps": 498, "loss": 0.0654, "lr": 4.6260855403707625e-06, "epoch": 1.3255463451394123, "percentage": 66.27, "elapsed_time": "0:43:52", "remaining_time": "0:22:19"} +{"current_steps": 335, "total_steps": 498, "loss": 0.0616, "lr": 4.578674030756364e-06, "epoch": 1.3456417985430797, "percentage": 67.27, "elapsed_time": "0:45:34", "remaining_time": "0:22:10"} +{"current_steps": 340, "total_steps": 498, "loss": 0.0608, "lr": 4.528707316878233e-06, "epoch": 1.365737251946747, "percentage": 68.27, "elapsed_time": "0:47:13", "remaining_time": "0:21:56"} +{"current_steps": 345, "total_steps": 498, "loss": 0.0573, "lr": 4.476246820180259e-06, "epoch": 1.3858327053504145, "percentage": 69.28, "elapsed_time": "0:48:55", "remaining_time": "0:21:41"} +{"current_steps": 350, "total_steps": 498, "loss": 0.0648, "lr": 4.421357027581921e-06, "epoch": 1.405928158754082, "percentage": 70.28, "elapsed_time": "0:50:35", "remaining_time": "0:21:23"} +{"current_steps": 355, "total_steps": 498, "loss": 0.0679, "lr": 4.364105412207914e-06, "epoch": 1.4260236121577492, "percentage": 71.29, "elapsed_time": "0:52:17", "remaining_time": "0:21:03"} +{"current_steps": 360, "total_steps": 498, "loss": 0.0655, "lr": 4.304562350447008e-06, "epoch": 1.4461190655614167, "percentage": 72.29, "elapsed_time": "0:53:59", "remaining_time": "0:20:41"} +{"current_steps": 365, "total_steps": 498, "loss": 0.0662, "lr": 4.242801035442059e-06, "epoch": 1.4662145189650841, "percentage": 73.29, "elapsed_time": "0:55:41", "remaining_time": "0:20:17"} +{"current_steps": 370, "total_steps": 498, "loss": 0.0659, "lr": 4.178897387117547e-06, "epoch": 1.4863099723687516, "percentage": 74.3, "elapsed_time": "0:57:25", "remaining_time": "0:19:51"} +{"current_steps": 375, "total_steps": 498, "loss": 0.0605, "lr": 4.11292995885522e-06, "epoch": 1.506405425772419, "percentage": 75.3, "elapsed_time": "0:59:11", "remaining_time": "0:19:24"} +{"current_steps": 380, "total_steps": 498, "loss": 0.0741, "lr": 4.044979840932567e-06, "epoch": 1.5265008791760866, "percentage": 76.31, "elapsed_time": "1:00:56", "remaining_time": "0:18:55"} +{"current_steps": 385, "total_steps": 498, "loss": 0.058, "lr": 3.975130560842821e-06, "epoch": 1.5465963325797538, "percentage": 77.31, "elapsed_time": "1:02:38", "remaining_time": "0:18:23"} +{"current_steps": 390, "total_steps": 498, "loss": 0.0584, "lr": 3.9034679806190204e-06, "epoch": 1.5666917859834213, "percentage": 78.31, "elapsed_time": "1:04:18", "remaining_time": "0:17:48"} +{"current_steps": 395, "total_steps": 498, "loss": 0.0582, "lr": 3.830080191288342e-06, "epoch": 1.5867872393870885, "percentage": 79.32, "elapsed_time": "1:06:00", "remaining_time": "0:17:12"} +{"current_steps": 400, "total_steps": 498, "loss": 0.0634, "lr": 3.755057404586455e-06, "epoch": 1.606882692790756, "percentage": 80.32, "elapsed_time": "1:07:43", "remaining_time": "0:16:35"} +{"current_steps": 405, "total_steps": 498, "loss": 0.0525, "lr": 3.6784918420649952e-06, "epoch": 1.6269781461944235, "percentage": 81.33, "elapsed_time": "1:10:09", "remaining_time": "0:16:06"} +{"current_steps": 410, "total_steps": 498, "loss": 0.0645, "lr": 3.600477621728473e-06, "epoch": 1.647073599598091, "percentage": 82.33, "elapsed_time": "1:11:49", "remaining_time": "0:15:24"} +{"current_steps": 415, "total_steps": 498, "loss": 0.0526, "lr": 3.521110642339991e-06, "epoch": 1.6671690530017584, "percentage": 83.33, "elapsed_time": "1:13:32", "remaining_time": "0:14:42"} +{"current_steps": 420, "total_steps": 498, "loss": 0.0616, "lr": 3.4404884655379433e-06, "epoch": 1.687264506405426, "percentage": 84.34, "elapsed_time": "1:15:15", "remaining_time": "0:13:58"} +{"current_steps": 425, "total_steps": 498, "loss": 0.0537, "lr": 3.358710195908653e-06, "epoch": 1.7073599598090932, "percentage": 85.34, "elapsed_time": "1:16:58", "remaining_time": "0:13:13"} +{"current_steps": 430, "total_steps": 498, "loss": 0.0619, "lr": 3.27587635916233e-06, "epoch": 1.7274554132127606, "percentage": 86.35, "elapsed_time": "1:18:38", "remaining_time": "0:12:26"} +{"current_steps": 435, "total_steps": 498, "loss": 0.0603, "lr": 3.1920887785621233e-06, "epoch": 1.747550866616428, "percentage": 87.35, "elapsed_time": "1:20:20", "remaining_time": "0:11:38"} +{"current_steps": 440, "total_steps": 498, "loss": 0.0594, "lr": 3.10745044975816e-06, "epoch": 1.7676463200200954, "percentage": 88.35, "elapsed_time": "1:22:02", "remaining_time": "0:10:48"} +{"current_steps": 445, "total_steps": 498, "loss": 0.0608, "lr": 3.022065414180425e-06, "epoch": 1.7877417734237628, "percentage": 89.36, "elapsed_time": "1:23:44", "remaining_time": "0:09:58"} +{"current_steps": 450, "total_steps": 498, "loss": 0.0505, "lr": 2.9360386311461276e-06, "epoch": 1.8078372268274303, "percentage": 90.36, "elapsed_time": "1:25:26", "remaining_time": "0:09:06"} +{"current_steps": 455, "total_steps": 498, "loss": 0.0497, "lr": 2.849475848838749e-06, "epoch": 1.8279326802310978, "percentage": 91.37, "elapsed_time": "1:27:05", "remaining_time": "0:08:13"} +{"current_steps": 460, "total_steps": 498, "loss": 0.0625, "lr": 2.762483474317387e-06, "epoch": 1.8480281336347653, "percentage": 92.37, "elapsed_time": "1:28:52", "remaining_time": "0:07:20"} +{"current_steps": 465, "total_steps": 498, "loss": 0.0448, "lr": 2.6751684427161684e-06, "epoch": 1.8681235870384325, "percentage": 93.37, "elapsed_time": "1:30:36", "remaining_time": "0:06:25"} +{"current_steps": 470, "total_steps": 498, "loss": 0.0588, "lr": 2.587638085794541e-06, "epoch": 1.8882190404421, "percentage": 94.38, "elapsed_time": "1:32:15", "remaining_time": "0:05:29"} +{"current_steps": 475, "total_steps": 498, "loss": 0.0511, "lr": 2.5e-06, "epoch": 1.9083144938457672, "percentage": 95.38, "elapsed_time": "1:33:57", "remaining_time": "0:04:32"} +{"current_steps": 480, "total_steps": 498, "loss": 0.0497, "lr": 2.41236191420546e-06, "epoch": 1.9284099472494347, "percentage": 96.39, "elapsed_time": "1:35:43", "remaining_time": "0:03:35"} +{"current_steps": 485, "total_steps": 498, "loss": 0.0498, "lr": 2.3248315572838316e-06, "epoch": 1.9485054006531022, "percentage": 97.39, "elapsed_time": "1:37:25", "remaining_time": "0:02:36"} +{"current_steps": 490, "total_steps": 498, "loss": 0.0459, "lr": 2.237516525682614e-06, "epoch": 1.9686008540567697, "percentage": 98.39, "elapsed_time": "1:39:07", "remaining_time": "0:01:37"} +{"current_steps": 495, "total_steps": 498, "loss": 0.052, "lr": 2.1505241511612522e-06, "epoch": 1.9886963074604371, "percentage": 99.4, "elapsed_time": "1:40:50", "remaining_time": "0:00:36"} +{"current_steps": 498, "total_steps": 498, "epoch": 2.0, "percentage": 100.0, "elapsed_time": "1:42:38", "remaining_time": "0:00:00"} diff --git a/trainer_state.json b/trainer_state.json new file mode 100644 index 0000000..db1d991 --- /dev/null +++ b/trainer_state.json @@ -0,0 +1,736 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 2.0, + "eval_steps": 500, + "global_step": 498, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.02009545340366742, + "grad_norm": 101.4629135131836, + "learning_rate": 4.0000000000000003e-07, + "loss": 2.9256, + "step": 5 + }, + { + "epoch": 0.04019090680733484, + "grad_norm": 83.51904296875, + "learning_rate": 9.000000000000001e-07, + "loss": 2.7459, + "step": 10 + }, + { + "epoch": 0.06028636021100226, + "grad_norm": 25.10388946533203, + "learning_rate": 1.4000000000000001e-06, + "loss": 1.637, + "step": 15 + }, + { + "epoch": 0.08038181361466969, + "grad_norm": 13.024300575256348, + "learning_rate": 1.9000000000000002e-06, + "loss": 1.1459, + "step": 20 + }, + { + "epoch": 0.1004772670183371, + "grad_norm": 8.315956115722656, + "learning_rate": 2.4000000000000003e-06, + "loss": 0.7456, + "step": 25 + }, + { + "epoch": 0.12057272042200452, + "grad_norm": 8.930346488952637, + "learning_rate": 2.9e-06, + "loss": 0.5717, + "step": 30 + }, + { + "epoch": 0.14066817382567195, + "grad_norm": 7.356456756591797, + "learning_rate": 3.4000000000000005e-06, + "loss": 0.414, + "step": 35 + }, + { + "epoch": 0.16076362722933937, + "grad_norm": 5.648008346557617, + "learning_rate": 3.900000000000001e-06, + "loss": 0.2577, + "step": 40 + }, + { + "epoch": 0.1808590806330068, + "grad_norm": 6.329988956451416, + "learning_rate": 4.4e-06, + "loss": 0.2394, + "step": 45 + }, + { + "epoch": 0.2009545340366742, + "grad_norm": 5.1738080978393555, + "learning_rate": 4.9000000000000005e-06, + "loss": 0.1848, + "step": 50 + }, + { + "epoch": 0.22104998744034163, + "grad_norm": 4.86478853225708, + "learning_rate": 4.999016565957633e-06, + "loss": 0.2195, + "step": 55 + }, + { + "epoch": 0.24114544084400905, + "grad_norm": 4.442976474761963, + "learning_rate": 4.995022691166621e-06, + "loss": 0.1582, + "step": 60 + }, + { + "epoch": 0.26124089424767644, + "grad_norm": 3.704345941543579, + "learning_rate": 4.987961816680493e-06, + "loss": 0.183, + "step": 65 + }, + { + "epoch": 0.2813363476513439, + "grad_norm": 3.772160768508911, + "learning_rate": 4.977842622059546e-06, + "loss": 0.164, + "step": 70 + }, + { + "epoch": 0.30143180105501133, + "grad_norm": 3.2532081604003906, + "learning_rate": 4.96467754629559e-06, + "loss": 0.1781, + "step": 75 + }, + { + "epoch": 0.32152725445867875, + "grad_norm": 2.893615484237671, + "learning_rate": 4.948482772521346e-06, + "loss": 0.1288, + "step": 80 + }, + { + "epoch": 0.34162270786234616, + "grad_norm": 4.654979705810547, + "learning_rate": 4.929278208117378e-06, + "loss": 0.1444, + "step": 85 + }, + { + "epoch": 0.3617181612660136, + "grad_norm": 2.8989882469177246, + "learning_rate": 4.907087460240999e-06, + "loss": 0.1151, + "step": 90 + }, + { + "epoch": 0.381813614669681, + "grad_norm": 3.7547988891601562, + "learning_rate": 4.881937806807241e-06, + "loss": 0.1197, + "step": 95 + }, + { + "epoch": 0.4019090680733484, + "grad_norm": 2.273831605911255, + "learning_rate": 4.8538601629575525e-06, + "loss": 0.0966, + "step": 100 + }, + { + "epoch": 0.42200452147701584, + "grad_norm": 2.4642996788024902, + "learning_rate": 4.822889043057446e-06, + "loss": 0.1088, + "step": 105 + }, + { + "epoch": 0.44209997488068326, + "grad_norm": 3.29301381111145, + "learning_rate": 4.789062518269812e-06, + "loss": 0.1163, + "step": 110 + }, + { + "epoch": 0.4621954282843507, + "grad_norm": 2.45603084564209, + "learning_rate": 4.752422169756048e-06, + "loss": 0.1151, + "step": 115 + }, + { + "epoch": 0.4822908816880181, + "grad_norm": 3.199894666671753, + "learning_rate": 4.713013037562531e-06, + "loss": 0.098, + "step": 120 + }, + { + "epoch": 0.5023863350916855, + "grad_norm": 1.7128061056137085, + "learning_rate": 4.670883565255264e-06, + "loss": 0.0968, + "step": 125 + }, + { + "epoch": 0.5224817884953529, + "grad_norm": 2.645235776901245, + "learning_rate": 4.6260855403707625e-06, + "loss": 0.1068, + "step": 130 + }, + { + "epoch": 0.5425772418990203, + "grad_norm": 2.3140530586242676, + "learning_rate": 4.578674030756364e-06, + "loss": 0.0914, + "step": 135 + }, + { + "epoch": 0.5626726953026878, + "grad_norm": 1.8588067293167114, + "learning_rate": 4.528707316878233e-06, + "loss": 0.0995, + "step": 140 + }, + { + "epoch": 0.5827681487063552, + "grad_norm": 1.9172967672348022, + "learning_rate": 4.476246820180259e-06, + "loss": 0.08, + "step": 145 + }, + { + "epoch": 0.6028636021100227, + "grad_norm": 2.9082584381103516, + "learning_rate": 4.421357027581921e-06, + "loss": 0.1066, + "step": 150 + }, + { + "epoch": 0.62295905551369, + "grad_norm": 3.6944031715393066, + "learning_rate": 4.364105412207914e-06, + "loss": 0.1124, + "step": 155 + }, + { + "epoch": 0.6430545089173575, + "grad_norm": 1.5622600317001343, + "learning_rate": 4.304562350447008e-06, + "loss": 0.0856, + "step": 160 + }, + { + "epoch": 0.6631499623210249, + "grad_norm": 2.388239860534668, + "learning_rate": 4.242801035442059e-06, + "loss": 0.0766, + "step": 165 + }, + { + "epoch": 0.6832454157246923, + "grad_norm": 1.7182718515396118, + "learning_rate": 4.178897387117547e-06, + "loss": 0.104, + "step": 170 + }, + { + "epoch": 0.7033408691283597, + "grad_norm": 2.2367119789123535, + "learning_rate": 4.11292995885522e-06, + "loss": 0.0737, + "step": 175 + }, + { + "epoch": 0.7234363225320272, + "grad_norm": 2.007200241088867, + "learning_rate": 4.044979840932567e-06, + "loss": 0.0893, + "step": 180 + }, + { + "epoch": 0.7435317759356945, + "grad_norm": 2.6910905838012695, + "learning_rate": 3.975130560842821e-06, + "loss": 0.0944, + "step": 185 + }, + { + "epoch": 0.763627229339362, + "grad_norm": 2.042320489883423, + "learning_rate": 3.9034679806190204e-06, + "loss": 0.0843, + "step": 190 + }, + { + "epoch": 0.7837226827430294, + "grad_norm": 2.700977087020874, + "learning_rate": 3.830080191288342e-06, + "loss": 0.0669, + "step": 195 + }, + { + "epoch": 0.8038181361466968, + "grad_norm": 3.0307390689849854, + "learning_rate": 3.755057404586455e-06, + "loss": 0.0955, + "step": 200 + }, + { + "epoch": 0.8239135895503642, + "grad_norm": 1.11714506149292, + "learning_rate": 4.0000000000000003e-07, + "loss": 0.0956, + "step": 205 + }, + { + "epoch": 0.8440090429540317, + "grad_norm": 1.5246130228042603, + "learning_rate": 9.000000000000001e-07, + "loss": 0.0755, + "step": 210 + }, + { + "epoch": 0.864104496357699, + "grad_norm": 2.0117733478546143, + "learning_rate": 1.4000000000000001e-06, + "loss": 0.0894, + "step": 215 + }, + { + "epoch": 0.8841999497613665, + "grad_norm": 2.038092613220215, + "learning_rate": 1.9000000000000002e-06, + "loss": 0.074, + "step": 220 + }, + { + "epoch": 0.9042954031650339, + "grad_norm": 1.607172966003418, + "learning_rate": 2.4000000000000003e-06, + "loss": 0.0612, + "step": 225 + }, + { + "epoch": 0.9243908565687013, + "grad_norm": 1.643001914024353, + "learning_rate": 2.9e-06, + "loss": 0.0967, + "step": 230 + }, + { + "epoch": 0.9444863099723687, + "grad_norm": 1.5271589756011963, + "learning_rate": 3.4000000000000005e-06, + "loss": 0.1008, + "step": 235 + }, + { + "epoch": 0.9645817633760362, + "grad_norm": 2.2744393348693848, + "learning_rate": 3.900000000000001e-06, + "loss": 0.0846, + "step": 240 + }, + { + "epoch": 0.9846772167797035, + "grad_norm": 1.2268872261047363, + "learning_rate": 4.4e-06, + "loss": 0.0889, + "step": 245 + }, + { + "epoch": 1.0040190906807336, + "grad_norm": 1.971510648727417, + "learning_rate": 4.9000000000000005e-06, + "loss": 0.1021, + "step": 250 + }, + { + "epoch": 1.0241145440844008, + "grad_norm": 0.6695238947868347, + "learning_rate": 4.999016565957633e-06, + "loss": 0.057, + "step": 255 + }, + { + "epoch": 1.0442099974880683, + "grad_norm": 3.379134178161621, + "learning_rate": 4.995022691166621e-06, + "loss": 0.0695, + "step": 260 + }, + { + "epoch": 1.0643054508917358, + "grad_norm": 2.573274850845337, + "learning_rate": 4.987961816680493e-06, + "loss": 0.0713, + "step": 265 + }, + { + "epoch": 1.0844009042954033, + "grad_norm": 1.3487026691436768, + "learning_rate": 4.977842622059546e-06, + "loss": 0.0704, + "step": 270 + }, + { + "epoch": 1.1044963576990705, + "grad_norm": 1.7525826692581177, + "learning_rate": 4.96467754629559e-06, + "loss": 0.0615, + "step": 275 + }, + { + "epoch": 1.124591811102738, + "grad_norm": 1.1195745468139648, + "learning_rate": 4.948482772521346e-06, + "loss": 0.0616, + "step": 280 + }, + { + "epoch": 1.1446872645064055, + "grad_norm": 2.203016996383667, + "learning_rate": 4.929278208117378e-06, + "loss": 0.0627, + "step": 285 + }, + { + "epoch": 1.164782717910073, + "grad_norm": 3.2196788787841797, + "learning_rate": 4.907087460240999e-06, + "loss": 0.0707, + "step": 290 + }, + { + "epoch": 1.1848781713137402, + "grad_norm": 2.4914467334747314, + "learning_rate": 4.881937806807241e-06, + "loss": 0.0758, + "step": 295 + }, + { + "epoch": 1.2049736247174077, + "grad_norm": 2.223809242248535, + "learning_rate": 4.8538601629575525e-06, + "loss": 0.0698, + "step": 300 + }, + { + "epoch": 1.2250690781210751, + "grad_norm": 1.5360513925552368, + "learning_rate": 4.822889043057446e-06, + "loss": 0.0593, + "step": 305 + }, + { + "epoch": 1.2451645315247426, + "grad_norm": 1.8444727659225464, + "learning_rate": 4.789062518269812e-06, + "loss": 0.0635, + "step": 310 + }, + { + "epoch": 1.2652599849284099, + "grad_norm": 1.5122318267822266, + "learning_rate": 4.752422169756048e-06, + "loss": 0.0737, + "step": 315 + }, + { + "epoch": 1.2853554383320773, + "grad_norm": 2.0612282752990723, + "learning_rate": 4.713013037562531e-06, + "loss": 0.065, + "step": 320 + }, + { + "epoch": 1.3054508917357448, + "grad_norm": 1.4917356967926025, + "learning_rate": 4.670883565255264e-06, + "loss": 0.0563, + "step": 325 + }, + { + "epoch": 1.3255463451394123, + "grad_norm": 1.861807942390442, + "learning_rate": 4.6260855403707625e-06, + "loss": 0.0654, + "step": 330 + }, + { + "epoch": 1.3456417985430797, + "grad_norm": 2.3603355884552, + "learning_rate": 4.578674030756364e-06, + "loss": 0.0616, + "step": 335 + }, + { + "epoch": 1.365737251946747, + "grad_norm": 1.4312632083892822, + "learning_rate": 4.528707316878233e-06, + "loss": 0.0608, + "step": 340 + }, + { + "epoch": 1.3858327053504145, + "grad_norm": 3.3819150924682617, + "learning_rate": 4.476246820180259e-06, + "loss": 0.0573, + "step": 345 + }, + { + "epoch": 1.405928158754082, + "grad_norm": 2.076028347015381, + "learning_rate": 4.421357027581921e-06, + "loss": 0.0648, + "step": 350 + }, + { + "epoch": 1.4260236121577492, + "grad_norm": 2.0433411598205566, + "learning_rate": 4.364105412207914e-06, + "loss": 0.0679, + "step": 355 + }, + { + "epoch": 1.4461190655614167, + "grad_norm": 1.695204496383667, + "learning_rate": 4.304562350447008e-06, + "loss": 0.0655, + "step": 360 + }, + { + "epoch": 1.4662145189650841, + "grad_norm": 0.7300482988357544, + "learning_rate": 4.242801035442059e-06, + "loss": 0.0662, + "step": 365 + }, + { + "epoch": 1.4863099723687516, + "grad_norm": 1.8220871686935425, + "learning_rate": 4.178897387117547e-06, + "loss": 0.0659, + "step": 370 + }, + { + "epoch": 1.506405425772419, + "grad_norm": 1.4738329648971558, + "learning_rate": 4.11292995885522e-06, + "loss": 0.0605, + "step": 375 + }, + { + "epoch": 1.5265008791760866, + "grad_norm": 1.562551736831665, + "learning_rate": 4.044979840932567e-06, + "loss": 0.0741, + "step": 380 + }, + { + "epoch": 1.5465963325797538, + "grad_norm": 1.5520535707473755, + "learning_rate": 3.975130560842821e-06, + "loss": 0.058, + "step": 385 + }, + { + "epoch": 1.5666917859834213, + "grad_norm": 1.29172682762146, + "learning_rate": 3.9034679806190204e-06, + "loss": 0.0584, + "step": 390 + }, + { + "epoch": 1.5867872393870885, + "grad_norm": 1.9488056898117065, + "learning_rate": 3.830080191288342e-06, + "loss": 0.0582, + "step": 395 + }, + { + "epoch": 1.606882692790756, + "grad_norm": 1.6518131494522095, + "learning_rate": 3.755057404586455e-06, + "loss": 0.0634, + "step": 400 + }, + { + "epoch": 1.6269781461944235, + "grad_norm": 1.5398550033569336, + "learning_rate": 3.6784918420649952e-06, + "loss": 0.0525, + "step": 405 + }, + { + "epoch": 1.647073599598091, + "grad_norm": 1.6156771183013916, + "learning_rate": 3.600477621728473e-06, + "loss": 0.0645, + "step": 410 + }, + { + "epoch": 1.6671690530017584, + "grad_norm": 1.564670205116272, + "learning_rate": 3.521110642339991e-06, + "loss": 0.0526, + "step": 415 + }, + { + "epoch": 1.687264506405426, + "grad_norm": 1.2277319431304932, + "learning_rate": 3.4404884655379433e-06, + "loss": 0.0616, + "step": 420 + }, + { + "epoch": 1.7073599598090932, + "grad_norm": 1.1757029294967651, + "learning_rate": 3.358710195908653e-06, + "loss": 0.0537, + "step": 425 + }, + { + "epoch": 1.7274554132127606, + "grad_norm": 1.0405977964401245, + "learning_rate": 3.27587635916233e-06, + "loss": 0.0619, + "step": 430 + }, + { + "epoch": 1.747550866616428, + "grad_norm": 1.5975450277328491, + "learning_rate": 3.1920887785621233e-06, + "loss": 0.0603, + "step": 435 + }, + { + "epoch": 1.7676463200200954, + "grad_norm": 1.7237313985824585, + "learning_rate": 3.10745044975816e-06, + "loss": 0.0594, + "step": 440 + }, + { + "epoch": 1.7877417734237628, + "grad_norm": 1.7574973106384277, + "learning_rate": 3.022065414180425e-06, + "loss": 0.0608, + "step": 445 + }, + { + "epoch": 1.8078372268274303, + "grad_norm": 1.450028419494629, + "learning_rate": 2.9360386311461276e-06, + "loss": 0.0505, + "step": 450 + }, + { + "epoch": 1.8279326802310978, + "grad_norm": 1.0331708192825317, + "learning_rate": 2.849475848838749e-06, + "loss": 0.0497, + "step": 455 + }, + { + "epoch": 1.8480281336347653, + "grad_norm": 1.5370794534683228, + "learning_rate": 2.762483474317387e-06, + "loss": 0.0625, + "step": 460 + }, + { + "epoch": 1.8681235870384325, + "grad_norm": 2.073357582092285, + "learning_rate": 2.6751684427161684e-06, + "loss": 0.0448, + "step": 465 + }, + { + "epoch": 1.8882190404421, + "grad_norm": 1.4379805326461792, + "learning_rate": 2.587638085794541e-06, + "loss": 0.0588, + "step": 470 + }, + { + "epoch": 1.9083144938457672, + "grad_norm": 1.8600538969039917, + "learning_rate": 2.5e-06, + "loss": 0.0511, + "step": 475 + }, + { + "epoch": 1.9284099472494347, + "grad_norm": 1.2249418497085571, + "learning_rate": 2.41236191420546e-06, + "loss": 0.0497, + "step": 480 + }, + { + "epoch": 1.9485054006531022, + "grad_norm": 0.8812041282653809, + "learning_rate": 2.3248315572838316e-06, + "loss": 0.0498, + "step": 485 + }, + { + "epoch": 1.9686008540567697, + "grad_norm": 0.8966412544250488, + "learning_rate": 2.237516525682614e-06, + "loss": 0.0459, + "step": 490 + }, + { + "epoch": 1.9886963074604371, + "grad_norm": 1.238738775253296, + "learning_rate": 2.1505241511612522e-06, + "loss": 0.052, + "step": 495 + }, + { + "epoch": 2.0, + "step": 498, + "total_flos": 1.28494219771102e+18, + "train_loss": 0.03900693407858232, + "train_runtime": 6158.7829, + "train_samples_per_second": 5.17, + "train_steps_per_second": 0.081 + } + ], + "logging_steps": 5, + "max_steps": 498, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 200, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 1.28494219771102e+18, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/training_args.bin b/training_args.bin new file mode 100644 index 0000000..e9a2513 --- /dev/null +++ b/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5acbccb5459f9c0d139eb9e75792351bde8d66e4f32093b31c72148828bf3e33 +size 6225 diff --git a/training_loss.png b/training_loss.png new file mode 100644 index 0000000000000000000000000000000000000000..eccf28a71e73e2d65b8150928a6f94cc5f8d6845 GIT binary patch literal 31982 zcmdqJ`9GFh_%?jq=9wZSAqmM?5i%t-ks%>76*ABB7$P#3Arz8KnKEb27@0ChC`1_} z^Yos}{yxv=`Qd$^Kj7`N`|Q2%;kwqf*168}IF9pJ_fwVI*C@%D$S@3}ym4Jl4a4w! zF$_0BLIke}_l!=#zr-EowH)u*m^iu^*+0aTj2vw(Z5%Bh8M8S*w0C%9V|{^NNRVHQ zm(9%4(bhp+K)~vMe}dn}-c*27q4yqKh1B-?JqHY_^zU0qf6Wqo22 zys=Oc_B-$1w<##3$}l*lUs&DSi~7~wP5-f%qibXJ^ED6q=EUbOB)l>>0%+LS;ZH=W zturnd{$#odVMmV~IpW4hB!k|dZosa~$;nL_F=9{QZ3zl|Hh4EG$Oz+rcUJ@{eBoU= zmM6GKc(=7d@c%Dgb{hETWHIVQOuqL4-b#+ z#!-zt-Qn3mv+o~;t!uV^DM;Jgi5Iq@=eS|MGDmSzoKCOO*=l93T>M6~pv3^MDT9Bo z`BPjsrPX-N$s&^u*(+E4ib7?>%zidL6?I)^8*`nc+?|m=xRNa8mFM>5mCM-EMC*@P z$8QdP@ch?vjf8~6_w$2KQKzq=izSIUG!Hjr7ws(7vPN_CZgU!E_D%^iI${GY}cfYo# z|D|~N@ZnJ5ePVsjzw(MN&!=mB>X*D0eaGVQYd`L#qsdhZ{tLO3(2fPetAm_0#$ z%{I>ys*@oDxjL6C&74+-?nf9k1|Ojs37~P^{oB$56W`d_*te~lqnY1a<+hr@OAs3q zb0yt;Nb~0OM($b9orN+{$61=!-rK)9#e5zfKYrYGtzQ#;@8A4z(Kj!6ohu+HD9vWP zMoItmwswj6G3&|&k$;QdPuNti(f@Ool-iuYr(n8Bva_*tO-n1jDD?VTBR2cvy~ep` z3a5*;t#TuJ%r36gyw=Xm+bDcJ|FbC+Q^}AyfcZ?#%p|$r{Or*iwSHwcH?cl*r|N^? z^4PCm)hx%VjwgGr+<2Z;D|Uv9>$I5Iu+a!SqpG%c$j)MA{^vXkkwl{s?YVzD+HpQt zvnOva=~(6JS2Lwui?EK(qgY$pUF}L!N)#C?wMyJ|E1!44u0A zBpCj-O52Q?tDJnkeHoIs%RA2<5N_#hN?{5ohXg~l#YQMSsc#T(6 zET2wxQW2+StPaVk+pomWpeQ;*N5@fBT`hmGzt@y~Hz%8U=SL802*2J(5*r(vBaDpP z(b3V3m&QF$OXm;HHQiB`Jo9sMGEh60-pI(v7$U-WX{>sXcd*85r!iACEuglxc6n{> zSi=)a+RM)Kk;(asl3@`MwZGglloB~VI{s!qNP7A5>|*VK_Yqp!(|zV8!Ox!2m5sUQ z><$--ySaXHw$0SakHtiBZ=S)J*MTWXKl-x)t3yAds?p%J?~T(weYH>u!u(4V6j zTJKLxE-fwnV`)iX``_=$03pIt;mLor!O??qhP<$2QsN!>!@!0U>Nbmk@6j=#Uk??OD?9dA7+@wbsy zHku=7V?mqqc9PbD7maLiaq&f*OcjKs@#1L38wfhtRF&YpoegsTdNzIUt%p68E;gyX zzXKv7A{Z~4MZ&UQjj_OJE|_+{@F}g8=DM9kL(6rW4u`|ZX7oOV9Aog}#S3+@nw1?B z&BSr#9S_gxy#k;0`K7hUQ&-J*B|j`xy8bnWHK2g%amv%vvyrcCl>KXO@5xS`@3p7b zS4R#`en?(wYxxvNKE1cPx*DjKMK|tqP+hf9xl}`2?XeLn9SXVh804&)2B-baNopb2 zWkVEw5SgxfYlBU32Hy7;Br|V3dXzOgRG5bTa`W-Ul%e%)ZM`0ql*IeRq^Px{;~3WX z?{{xb(rF@|E?X99@g7dYtvG(!w4UJQW!t~AX%Qh+RaM$Cp$QBY*!M3dT}J(+ z4|ZwCy*5t%>U|m;|X7v83 zn0f6V7snvA_wNoWbBUL16kwa%wVZTe6BML@)I!l5!5RU@i`3iOd(hJha=9~Ox82Mu zm!Hc&8eHLVP`;j;nlk+U(doLfvWpp$q}vJyN*faYy06BI+OgN{uG~Tu`1;M8vc|^j z*aWPaR)zhU#hM*!4h{~s%a>0idvBk8LUH`@>1*L^;^N;@dGl-c#rw>K_xCqy8w>?y zay9Nlx?q%azX+*D(BYRNHys(TZaEFS*kDt;_r!Z|8&|OQoC&J=r%#_yQH}2~L8}8% zDqOSp_ALj18^h&^`gn0C9;w~stGnX|d#vIvixk+34+Ece04z7n)2C14h3*sK;o}$j z9C*LKSES%=gJYAHW=&HF`-B3=ZnD`3rqy+SYo;ex=h8D4NhaT1k-Y&On<>vI z3PzQTU-K<29u!aqwaTnd?yn{vs8@EuMpD(+55LChMaL9IR9|1;J-4^eXhorIL;Je0q(L)5pw zQB;bmJPOa)QzJE9MlbTS?_NCWt8$P_p}LTkqogKDgBX^ z@fr`)Ew{y88nwj-#PaO;g^9Vj&@SnNs?OZ=V`F2R08IpKCuFKNe}=j)Rc~O}b1r2C zBcoIFeA)!*YNU|PmVC~zva&Kuc{1L+cMnPef#FUPpLQ_`wZt;^?)Iw?NQIsNJBDg~ zq)V+v1|BHws4I8gPU+hmgWbn;z(l}CM;0jRxoI0S!*g(kJAg(ZO%>G&iW4GVj69Tt zeNQVqqtVE@JGTGjLKjK8+F3F(vMDR!e$07s^d2Jxk=qUg2^$vdPfSgADnO8tf}Mjy z4pkvQNp7|Bq4cEKS~kve`*#K*_7@Pw=aG<*5II@d$ABqI3bc4-Nlqd35)~is+V4P2 zKoI=-IhR2Zg(lNw=l4(qe)eVESt)(?i~`jVB5D?1n;hL+cTQv|Cp%~EwRUyo_y2`I zc)mj)9)?!&adBOP;`evHsH<(Ht-XDE=T5~lrZ9!7^=}p{B0D=H@03mib;b3YA6pnH zV>o;EELHMLfHtUDT37v5a2o-f7pReQ%(UxuOxZ~>dpYiyHggM1r^I|(g|-BW4o``< zEQzX$3LC^Ms{8Wy?!9bqh-rV2FSfl=d%$ON0iOT-d9_iunO7^N2Y_{<@Q8Z84xlmK z-`#>y0`{q*^hM7#_4)Zqsr3HlRg8w!>z``O-~+h8o-#&rusv97KAOP^E1kFQJuIyw z{wnWgSoN@_^JX>Or}cXw!ND}tQky9a&V4HQuG8LS~9-10|-XTr-p0hSV1yw9>#trnjxP z`AkpG%%l|*&NVORU7Q17E-)RMGej^rYg;$3~L)H`Ke%9B2nu%e=3 zd3E(_OZds0y(+~d(VicxF$oDF04OHLJeK6Gt@!~X20%4VefyRGfKUrereTS;QK43W z^MdxXlb7jpwM(KSmPRWssHv%C>&|qiUWF2pe>ZIHqtl#g87eKHy3=Ba~W{n2k z-_zV7w6hCF_tJVMpw2WyX6L;WRs(a~+SZnBKOeJ1 zXy_r$QyT$E2g->2&iX=Xj~NXUQ%FI9AhW1#%D1lenB)q0c&vc10bUDgnMwAc@~`*( z^hs8?%Jm3vQ#0TvF%B9o^$ifgmMrbh3eB>yT2wNCof;$-cC-= zz~2aYQ731W2QqjW`Kg3|L+=J)S1?O^GXq%tlpt)e+(0coVQ{d`pzk()tZMsD4raNR zgfh>2twOxo-4${uka|^n>mynT0Z@$;7q=!d3 zZ_@QTjPis?ie|q4ig$^dz?kQ%Qq}HCyX$g2@n0aary&H%0`Zik_b$HIu!y;zzx!o) z!wqn>#r!qMR$LobvU7nl-S5j(4U@|LAhOYDUh)Qbti4b6xu&M3{;i~#T-}NfH?AFM zY%G0(lo^JE5lCB2FkeSlSg6v~oWUSH(}P)bu+(YckGStzdGmPSBr=(a6^i zLy|;sBY>XM!otU3{hA;ipC$Y3A4l^QKuWvZuW2BBb2z66@(@^FgklTcH)0Mos*;3TT2*xrE6KKZj6F-tAIi{)yB zmRJmmJ96un+Zx@c&m98}j2j~29<^Q?#baY}O?>9>WJ`{vwDWv?gD|dc zaFAhR)PZ`jVupn=Okos|w=2LfK`1{JTY@}1WY0O3KE1v1Jo5Q-|FLQhcxx+h6b-PU~gy0Q(R^Tfge-yoxg^R(AFZtgm$RXJatYxOSq zS5pL0oy>NjY>S!MnT(JY^xRzLd^s?gXj9{toXl!ny*A)}(jfUfpH3+S(B=A->GqT1 z$zl#B)om)T62?%q{$@d!pR`%?k?E7gcmH$Jrm`eS5^i6AjK`&-SPn68s}0X%Oi$DY`ed-*IWYFZ*eQ z_@(cTwB84ME+DyF9n(-#dkk~_Tsot7c`v*Gk<%Vnws8+q<(ihOMh$W znQUsLf5x86MR)+vg>1F-I0OXpo3+bwgkCC^AMEddei8O$e&c zU1~Kaz>fX^fv6F|tKJw3O-f?^l6Z+4ie!TyK5;_Mtgu>6Kazo8gB}Pxfda1I@bZFb zfmsiGG4LAwk`n?|%Xl?Aijpd4onR%=(g3$wUR;DW4BqIyEj8V(DY3Q!9oK&#N05QSA-?QVMKVENXs#OVZo z96r&+M7{s!uS6SWbD2DaRF#Rc*i`JLB)~FJ#uuCIMi)wX?GsNjo5{`~Valwc!TOQfN7R{uz@n843l!Z_6+> zmnmj(mRE!T{DDjuR9h?U>E+c^>*Hhjt(a6hcOdmlgZEffl>`c)!oosPk9A&v|EFQn z?ksL>jyiPxfHiN1#2(C*?5Xye=-u5Mjn?+|oZec0B-sN04q2NUY#1MB{a)$PaxTB> zah$$8smph#INhsTaQjIJv^rSZX`d0KVWIR_4!~=AmGby`f(XfVMpUsOFS^OcA#e$^ zvpL~UZ9Ayv(*M!3k0R;fc6`DR41wR+*f>BoHPe}W_K)Y^CWbd&JL^bC0~z>fAUT7^ zJg*bKktk(A2G$B<4P)3RBV+a6Obw2RU;q;OZ*EZ8a->v=Mn0&luGkSQ#6x{%*x|1u|bgl0=r**XhMy-9~fa3D}1!zxv}NUnJW;ym=(8hZ}TYE7*A47n#Cf zQyjf<bBr(zgvVFTATD^3qVRc>s*r@$OcWTgBB(Vt;$FoqM12}H<-zM-nlhe6f;{_MN`HO_s4lwX=c8BVU}NLaO(K+)ax{ zV|lq2+K}vmf(756rqn_LJJfVpSolHcZvq*N`5o(r#0#e35|8#^TQ&iX$bJ>M4?Fb* z*J#;+hck@tu%*=|7RRT<3Bu1=z^zF02TFGof-b)G%NNHX8;}lu0`@>*0s3YL;I>cz zOQ5;N%k7x2FMdC#&J%+~#FM3o9VRGEAdy^opvGd5AL+uV^MKkPJ!wJl(mpi}>NjG% zAknj*J$u8|!-E{C7!uFjhm(+CLA;rn|#lb~Yu-?o$EyK~EJT*T^IzzOSe90ZZoS8W~uZ4jSd zUQ3<;5N2MyiqP=pzg}fkU0vNdKW@2s{OkN_Kq+sIzxO%V8HYXF0C@4$olN?7(24_9 zE$>>ObS1#W=Rdj#-M;R`#>EAhzIJ=pnG>&WlI+sOf9Vp#!B&@#Amp1T(%G~FSadsr zDNxytkAi3gVn;nRO_rM&3{rqJ4S|;biaw9ao*^(W@UCoq%=|4U)wP5iq5>h5fUgEo z$LomicVo<0y{vt!kj z%#G2{pZ|r@XRsEZek<+`)a-8alG%aW6Tln3E2f`;;o0+-k@!NUcAd*DD{xuThh`)( zQkt#GzCQsOL$mS#($S&*3Sr)6cAU_08b&))+o1+?CjA*;!5;rTE+# z8t7qE+$%Ck1+~^m+yq<t}!r^zlXT+-9&mY!2x29frh)6j(@VB$K|2y;4F5|AS| zx3=m*aK7CF0end{tM%pR8`7cgmg2Adk1Sa=N00Z`lWDWFXNT7t)`4h^C^?YdEB-`A zfLk$NXhL3z=0HtLX!txkPWbs_SvbnR91`z1(b~m(8R^f5+q0iV2iZ)Ui1YasDxn0f-XT)?;1j^q1=xaJE zQM=2wm{5y9qO51{90lH%C&R{0ZE1P0Jr>0P3sEDTr@&@h3WOXgkisENKi_cs5y-F% zjhi2^(2y29VjPeFg3xgRxGw;`P`X#wXyh1oyu3gRsjMCb+aOTLmZ& zS`z`#umUV4Rri>XT+n54RJ>NfT-*}R+V3^%cTx>IipsC{g&USqi+Tm*pRHtE4` z`*~W2n@+Ra>zITd8($JzdSMYUM@t^_!FlaGWCtYNA$Ec21Xf^JkB9sh2A+Qjx7QcQ zG1PsT!bSCOL8}QV1PJ(Iz7f*sRy63q2? zdfb#3Xe?m>PnH1x3c=h|Xje;_yT8X-`h5%B`SH$L)pfl&ygc9MTxJZ}yy&aYTF+A2 z-xa#qUtCWC&o;~|J zVK84ek^9C`*tSv6pP#nT`S5@g_;xTfa3Q*Wq{r#Yy1TpQ8n6mW4q&A+Ofg5rZANr1 zWSL%8{Fr)@Wy_p8{K22Ub3K+4qTKbL&v3BYEAyn|O>Q@eFMJE@7#u9?=^`EaWb!bY zqu#tk`2NyRAwKHV>U*u-1$w{+*))_vpc$Ms32%ATrQsn{Wj%#4-%(?HW4vlRoh~!1 z&W4L$yhuw<&RRKLU36NN0r(u?mu8^UQ>ZBi3c3P_%H~etAc{k}Y{kEZ)fpEW0$+9F zX^$zj84m{$0`;qAdo!rNmDxl<`V*N=>f40S%oOwy#wOGXt;8zpPkzF%%gQtLhNYsp zx~P(Y)ZqH_DVLzvwj(qdiLQk+*Po7%cnrxFiGw9lU8uVZ8D0ZP$;UnFy5EAx>cR;sejRClXMf#g*8*{`fd`%Euh@(y`$ zj(<)+&CYJ>0maM~wQX&Dc22l&EQUeb#Tb|_30d}@VLixRER!t}Es)Lnj`80zf*kq7>P+B+`4W}$IW?3nKS8&{z9UHKs9cT zfX(3qKLL%4_|#cuL}M#lGlU%MZ=PHRzC_QfFgSbrAJKI$sU9O)ZL2 zDw5OxF=Zt&e!qFoKTk-U`D04U0BxM@HJcg5yE1Zei>LVcJB>Jm+^8T%4O5Q^`vyMb zWb7#2iXkd16UIMqe%w(9k)bY4;ma34aEsI+>?jtgX`Dk?rBCI{U{qqiO9fS+|0KR4 zIm0j0WUA%?YQeKoBsJz|_>Y~i5xG{*cWh@imXZ`)qX;D>xnx9=tz6 zxPrh+do~i=isP>B$-4C6=*4h&k$gqs9S86%@7^zJl1UJHCKLgLl{I?QvNZ+C?9`iy;(6+-setQh?j)0I56;rT$o%P;Z z(*#?Je9bRpNfC4SHGu>_DrQk%_}r-bT&}3yPXcTLbQEH27pNBSu11hwr|(~~9t{R` zYXm~YVFiSh=G88~?(N<7;w-ZHAxjHVPQe$|47RK(hb0&c>P4Wf4srn!u0ej-SFOGD z62qc1+quG-XV=^b2nbRj-$SqVM*iBk^m!Fs>*iqoA0!)tc-TA7O6nY{2gdFOI#_4sJae6Bl=ykB^d+loWtAw{SwwbWF+Zv+YrXY7=Q1{pRDKGMqYlmJ}M8S3Etd z8XVwBq!AJ8%}~CYTF}1={XNnjKYl>pn;a!zl-ohc7eiW}Vf|x5SHNzOQBMIpQFJf6F-3>RFe<0R@OfX1V z-r9<*B|l0_JH1SdK~jLX5_gN#vg9Ckp@)TG$h?CT5qKLGEdaW1P#K`#4bq|?XqDnG z?oZVDK~0_&ZchYVAxIj7$dAtVTAsK9h|Y1g8wU#4%6)ivQKvsIbZY?6Tu!dtZOc7+z)ORBUEe2Fg z`RdMoXl+3hlK@n*=bM09oub0s4bsi5qd4%!`=BE+o_`SRISTJ|!+K-B+EynD%nG&s zI4Txrdcfez01J&F%|LIu1-6#m-=Ff10I&3Cz!A2S(r+#V90)Wh5~N|G+Fc9ECWIe` zfvN8#5=CQq?>&a{(@aQOE9LQR{JefI#dFP^Q?YFjx7|YK|K*+c9HhLvH%PEGlEEKq=;Mxi;uK=m3BaI&7aEc+`X`AxAv zS)Ja9%|^3A=0;ag@@?jv0I6vSnVSzhH+b1;jvWhx#uPZ15P$XqBcR28o*!IP7gVJ? zZ>5X%$>@%M5%tPqT==w?SmX-$LODzQ{Kd$Y2{sxAyAn4HCzGz8;-Scf85+1>xex=M zonYM~_$i?(H#Mo@v4lA>%$DQoIPXK4Y4$-dXS2U(vmmZ(G&_k+*eVzmy+EfCbZOxm znM#y~#?SaLmJZ%*yN`_M^JqT1@VU)jf3LENXYJ!5452i6J7cGI2j0tOL3g>EFp&N( zs!snVp}71vg##87X|$x-emVqofS{3c9Q37Yu+@-616vSOA8={`KMe}f$gz5Nbnw*& zHSEFRm&{2(^#T85GZ>Z%9cFGL<^?u`}Zh`@c+42H>8PWnF}3L zoZRwqW#qn~sG+ty%9@b22FeXUGWt6mXYdm-3CMbsIsGB1 z5d@w;VU`OX>y}W=TO-QL%ip<@3ajy0+6?yNr9@~4ilo+HTIA@wNWViRChE5G0Hrrz zR5gFR!P0_uy8HV4Q}CA?g0+!^oV)>gLE4i-pqzk%A$3>OWl{G&!$G$B1qL>ZL%~6w z1}(}{iqJZ>90;2JQhYxWWw1=VFO^KnWfzF2@Ob1qHkzf5}F0YabO5N2f zS8&J~0p=`vAX){z`Yk0x7FdVI+}({iH6ezH;Ze2OcBSq|)d^LWaHA>l(KkfPkpHiM zi36RB$js#4Dtz$ys9W;Ce~zF#w@J^;%uGTf=?Z9>cD+iMuH+Bi2R=3XH9ho#zHF|p zKGotMVc+^1b2-8x&JwBEdwo;WFmddIC_sdB!*qrbR`q@jc1w%cXX2?KTK<-Q z31;cbCwt~jZn@fk9Oa;^Fr3#-eCC*B;3pO=zlWG+BA^>yQ@;Q^>P_}TT0s>zCBwsm zLH4jEW$BzO@rZnBhvfz1`INNz;gVnxy}SK@;gVAi?AO*n zLF^;V!;^R^95-flQ}9vw{ELD~|uYLMw=IG~2R~K%Cxfg8RK7 z8yUH~!z44JE_m(2TEV~y+U@o}=$2>uz3Bw8V$!uhg>;#xJ&!L`oKd=m+86ex(A!zX zmv9DNe1?}})ciV+;a{ODJS0mzFrLVEFU-P6QPV@Zj+~hUBr!P=^o>)OaD!4f=iFIh z*WlBBkZm&Y&?wv0Y@MIR3=q&ei6UY($^b7$#XT;pmt`x?B*64_24@KR1DU$e+E0OS|1s0UUH4A04NH(a?ENvZ<75*R|~(p0{Ge z^nxMNykB_GT{q2CcQuLZ3EUO8);JaYBLR2n zp&q(PzcEogcIW2+k>x2qq+L}>z$5Cqtb(M_yeTm*+7w}S(Aii+-i80JkkNkfZQ z5erNWekc^s5c!GgKC)UQlRY^cpTr5s*A)^+9KX}1c(GmomVWdF=a)UD z;rAWv$(998YOf5*gc)ZdS}loV@U^TiHQN}y*E9)-kN-LZo^n>-9L&mX+%%p*1{vm@BRRj#)cwVV)?KmL?7R_-t;eXvX7=2;h0ylU8!;`t>#zun9&_!`U zbrrh#upz-0S3m@(=UN-kBt67W^u?uAO3=JzoaUq=fUd{NMKtO9q;h@I*f-+GTJInK zH!n13UaIJHzYpS-P96@Rs~H9`l|I3{mJbkwl0dBMu$A!$Eew1RKf%|p9si2xQIH8* zp|kQazFn!#*Gz__co6$qSbQfA=@%GuG9X4r>f(5JRU&=Bk*$n^x2(GGmw7A$3szts>YgI*t4{u{uI znwVNlImqaM!;2w)TtJFO^}Jjj8?AD?fY=vcY-9K!2)F34-OZaf%_U4A>oH1uOR%`l zo&w|Tp}7lPAGBT*UbnqgTtVuc`XNn%&-_0RCxn{)=@W`GR;l2>L#H~>2`x|s@|=_Z zKYJ>A*Zy};1#C58YU|%Nh1T$HQqVtBd?u5b{BCUKh%DQ0)>`)b#^9Le#BWS>4JVI9 zLvjeTAJ}lY^}j0e+9Pr*syw^olb4+h50AD%54r%rEmMRLBLx`osi;lMC5|aJeDIj| zW1A;N4JWZteoQ-8LjpP&h{1e>w!x$~a;ygj+YSMv@O%IMJyXO>Ne>YyKvUPCm6#S& z$v=PE@5Fhsp*)>u3j)bkb&sIUcE4N?1C!Hv@EQgHA-%L+VG90G;GJ8*cPkWX6|A_A zm6mmUlIETIJt+v%a{s_!%osw;4e%a-k(oeB>gJP2z! ztg~xN4ue`uB}4_rY^}+VON;f}h?1YfIJHIo-|DOd%k!k9XW&G*YzacnNoXu3X2*vq z^j!CXBqS~qYG@$*MZ<6Mo;;-#D`c+{PwOKX7py-BZcNB~3wD>Nn3zClKD~&E`MjeD zZAZWhkv`skvN8~V{sK}v;FSi@M3MCwG>@j^0;#vv!x*tMH=?-$r0Prz(e9j}$Nk@{ zEH$GbhHeOsqzk}x@Y`eg8rovc<%`?CO;@?T&S_|omXz-ye_^?Pl`za+3a)8BJo<*( z(CXd!(c(vUVys-oy)FSSU*zOg?NvK_BAM{gkQZh&T z%x0iZ`6f09-pt||z$?{a62P8pKcb<7Z;sNh4BLP7HI zWnOr>RUhBXs*%$$#a>$GFg@9uq7P@8W0jiE2&7ttQSu#Sf~pw3gNc0Au5(`JoG7YD z=jIj``BQ<)=rmNPD{(V8%h6S$P1jRWjl`G*4t)vL{Ey7o2{9brEXxTJ-@^)c_^O() zDkU4{ICC8>WEq;1d0}77(0Yn)DA!nN!vC;1+3UbSuAjY2<{Kt)R8&Zt_y*p2vUa<| z*6z$1M9yM?pAepm1#=b2=%SydsLDeA+T zY<~-`+9pNQ_>BP9Jw_dQ2n9wF?@$wYNNK3chOw>b;^aLoZk5Knz^>J6$!%8_y1}}fuz*&CU zb%EM+m=OMZEdtNFjY-A>8%@^7iqc7+b3$wgpMz&OBS1J9aL%cidLyV-Ay6%iu;oKY z!n`;X$G$2wx>4fpuZ}pCtjO+mn6ksTwT7xGFFXzP_)N)9Pegha5efZ#t6ta$MVUO! zA*^(%$7bK~E93XV6JOKC=#JV-)lv*^uW(jF3OpiAjc$GkcgcuV23sCVJX< zROnFiJhXNVhugjE6I`f|XGF);^2yfL5YXG;)@rC>++jCKF~N9VC2u|tOd?4&c*Ufg z1znmCtxZ-Y?&3Xy_e!G_%qOP81brwK>HYi}$P)i)qi6evo(bUhezc3Vc!Nms+z|Y{3d?$w3a@4*RFus19^nPdWG zWyc4Ob<|}zB<{MSF*zo&V6|WzJk%7sx+FK4fx8PC)?q+D2A-a=*y2|v?3GaZFj!)A z$W~n@{$Z$H3CC7XU{I2%kZ}>2Tx73a{R;<}L@ONm5=-9@6;vH3^1}eDE#7YTPd~1~ zb~wybduJRReGLL<8xBLA=F3Y}v=LI<+=x1Y*$&>ZH9pWeWIy?H#PV+Pt@>8Aidgj3Ep3hYugLlT9JxA~PKg^!{%|gLk|w zXR-ECFX#1W^=T*pcb5jgj?O>7a;fE_ps&^5kD6n>_4EI6@VM!J7_vYA1cj*?>?zMs z(+3%Cu?g_$z(Fdf+^#zyks|?htjf*l<36aM;8Zv(4ep~k$Epj$Gi)@K`}Y4s={2YS zaZb3c{zCo6PjE^BIpNtj-Wu0bRC|DbV-to71psu*pDBI>r65^*t3p4 ziuYEJXI9tNke604sIgNq+d+XzeL_n4*h{$^(B{PoO_?Yl8v z>e^vurWI@IaMnXxm-#j;!FjM#DJF_gf==FuTwZ5x#bVh0?y536VY@8>)(EN+7i5)C zQbKlJX!`_$uVeyT(3@~FjS_PO^C_H^^KERDgA>`4$R_)Q@&uACFD{Z>8h6H%DL_Gh zM&8KnU&e#KbsdA3ZAvX5|Eo5LRTC|PJ^snf^iSZ8S^mtG%*M${ge}80(0K>w?ZP5= z=V%K2`_oU39ePjUKj0`h8OC?^@pk`>F#%5i^8iS#ZpQS<;;jiv&+F>iphJ|N0qn zPm&M`0p?yx7El&Ur>D0n+0X$Sxt+O8%uRPZtblnAwTXN7bmvkParK^uDu7C_OyMOR#Wt444RZF%KgumY2SNNkhoGSak-iTv zabmbZXu~0cFhcxkp;*%J$h4xQy`@`{SixSCPnwY=BCewfT z|C!T3sPQ{l2=Io?PZ1YW;tX@Z2LRs(E>Eeuh(B8e=z#It=K*Tl22}($-o!qGcUrwb zn#)+PJ<&5vk8Ql0IbAk1QEU&eyr&;f8O}+bGKI#s=Vz74L?&kA7}j%z#IQkV01q-x zi8_U^wMY4HGwb-p#C3c=`u2Y^ zAD~DCteY4~{8+StNDwr*OpLEhk$zq@X0Go8+*x&v5&JH8ON){KgEP;|A@yu?LlmG% z_OZi=o&vlSuO03mqo~?96M3(ZM+TBr-&cS(q`8{ynK%O9snS!+cSke`#a^h+&i_PD zaty8)P)$^DhN7xF^9AE*eW}QD{m@daONIGDfmw_!OC zhytqZ0L3`x{bjyr;EHGm@$5B(S(Eh<*zC?jUQa0DOQ6cTRudIcen|C6Rvp7N}kg29=R6f+YdS(ACMTB zc?Q&v)7nn$6uxDS5sx`=fWjwrS-1j`wGw(gmwxIMGF`7Id`MVl`-&d_+Y=}bJH&^` zIw=g-9X5n-cq<2{e}qQ?{~+!V^n#akR-sDA40O!5j$r}XS#^Ks*_qqFUak~{S!O}s zGFuS#<$F)ch~ZUL+g{BO#6bCbkK)#z6K-j2gC}xNyVKq_=DUUv(_zoXDw<5 zqWv^DKe9vCCbW$&EEnAVvD}+!o0>x50sUe=sTg<{#{2PQti&-2jKb_h$SlT3)~=}K zA=-X8dn?yrk(+g_JZ|ZjW9mE{qu;rxu;sni4*!kI)IJd_2L)l&zMU^PJp2jA4fxH5 zY2k=rur+{vh%!vUAJUjn&l`Cq{myp-(HV~mD=U?GIY@_`&&S6cOx-&l(-;p0Hig`NNS=Ia>BYp>TIea zR*8i@{2!c_yOAvIBX#uXQ9<}Y9ZUvNB09hP<<(_$$X)5~7fw?H%D2UYgBV_&q}Sx~ z$kSU|A7><9v-IHFLVjDI3h=uE`b>HYU>Jk*{1#>A#hcfwtX`teKyHD{yO_}tG2Y=0L}1?hnkvK{ zehVd3F$odu|FphB8y6bIIg%!DTnSvaFIZHAu^#~LINp_DkAp!9NuZag?%+u^7028VkABGiSMD9 zk3NOfg3c2x4ERHj`C28!fe8d0s`v%{V@mOYSH=zi^S1o%%|LehA0XFl4%Y4m0>ENH zKR|RLWIWSTd)d8C^J9evT~+l9QKv!zkIrBJvqkEy@FxCm(B}@7KZYC`@IxS^a|5|o zp-+rf*b~eh{E~#vUX_t65?RgUuYKZ^e1l=3rO9^IMzER^0USe%8aD3LEbt39FgKQ} z9rZ!{bU(qLQ%t(rf}d&voLmh?|ahdJEBPb zRp(UHlVeVANO4XrAe%UMmv)e^=J;a^JbdyWX-qd z(i;u(Ly0lZnziEpqNN7z^h~x;hPtJ zS!F5#oB#l&Q3yR|V10S=uKqqC^n!A%tQivm#SchLP) z(jfW4l;fxS$ThV!0J4pRB|7qX`+cEJ@`o__5jGJQg~^r1)BfAMGy8d6w=yELc|YCOxWYowTOS839d z;hza5FX0LKKw&xI(6r{8M9Z z&*eGN|6&H@{1Zj_#Pe+^W+*JfXbgmJ8}QUeOU@)6Ci6HD_nUR0hjMPqqI!8saHPt2 z>A)Q?U7_gtEx7g6M;Lni040&1y0guJYcbh+eG@x=%_d*RVbZq02kY`>j9~tc7q$Qm z@%`6dk6_EcV|!cqL_`n6Di0LHu)EfHrP_K9!jb2`kMfZLQBtuKoJ^zS`mdhEL4%{& zv+ri+T{11gNCR_tET%vR(ZHTp7&bmH9{O)ojVli3xpfVqL+LT^{EMj0D3U z3H-Ueo+@~cV1KLK4r~MPv1#~Ng^DyKHenp)u@>jy{UVosxz^{wLS%h!jCds5@D0jA8lvOM(Hm5mMps9gk)I|NR5mvNEzVGfpxpl*kBCnIXw;kX0m+Y(ls+NTneznP)~; zq0A^6N+D8KG>k|HxnD=$>-ycl`|-H%`=8(SxcZ~gd7Q^_eD?eGdcG)9g<98>yk(TO zR+H894LCoIJ)a)&zE~8;o{2wNoE9r-iJIvl@yh)x5WO`{7pO?6A7v;2O?4 zyGy*@y&|)?efZnp=BpMClD;SAWNiy+U#Bh8Wc4gF)MELl%7brX4r%D(J5QSTl>%!C z$68e`l$(XyC=D$_Th&p`>PTHnXc^^(=((#;FftM1p@KE1J%Ma;5;7Y+&X697Yf=sp zKg529hEHg4SCyt$?-XCJLVKj3?nE1V-SF8NQH-o_P)ubkF{95?+SKSAFnJU;&W5iC zt0i8xu*PosF&{|x=X1W{N^wxlhEZ98E~NwBkvvo@8<--=-_^L-{9hWk?)Hu8u;zyebj>ThjKO-M=zo zSC1FK7QRlpMuPanC*=I|d<2&UmH$)!>oqLQ^DZBPmi)zD_ZZ+ZeiS^zp`zKyB17ha zsl?FmfMxDTm80(4h`PiP#`6~DpN4T$Tw6JCR}cEw>?ejVZZ66(EZdpfO^mfYyYe$6 zzoj!AKjQF&O8XI@!$@`ET$iNQ&WiV%aqRIUjk;ik)PgCzVbx~@{^u7`#b zAG57KEBqxt zKMj4%T??n3C#K0S%n}nF*8iT)A=94}6iPac>*jTQ`{s&!Qf0*@Wr_yCP;={FKKRwVg{?-1s3-&o#bpKv)&Lfm5ad)}#?{D0seh743xNllq+WLgV-j zqz_eVl>@@iHF(v!VFUX0wZ_^Re4J8t^vd#u2>yM-(VT1%m^vQzNnqWJtv_iTIRlg? z^q{gg0|%MG^!pgWxDJBEe~NB%VI;?k6yo8OkYt4Y28(GBjfr$?-SBFNYJxyQX{Fl% zAmD9t`{&i5_K@xFJnDW>JHtTh@{iu-o}=Eq>Qd2+U7yv$I6B*|OjyXO1hilb`8Y#8 zI`uk#%D>rP1s!-opS$o=u6nIwGFH!cNe{fj>O}V{Wi&b0W1|$uHkFuae)sS|#paq{ zk0gokyvE+4t6bya#uqQ`#zH-*S>)j(p{Mr>&crPf1>fuHR%ya!NszG#6A#(-IH+E}!KO-O}2D0c2PWgOuR=sln%zII zH+D8Jx@?|7z2U-FjD{eJ(bd?t+ttaxPvuDnn)`V+&9cFaapU*aM{^lXb-X(;MV6}& zuSQNrX!-B2@)UT#f#sIjH&$vIZTd=09BH)S?>PkDonj)*c?y-aJwi-X;FUOAf~bjM zagiXS63^gYW)%aS&947xG>&TtWA^Z%^@ew2f_Wh=E?J7FiR1WQf4^p9-|UlsZ(8Y- zf13Uq0XngIv#k6wMdft0qB|#Zl|lnu32dUp9(6_s^M7kf&f~xCpn!2y!-bF{bgCZJ z3S6O?vcnPK{6xoB<(j=)*pA5Gw7Y(;YoCbAxE$8ZYz#WIg#OqTaes&TTnkV89(Mf5|j5Pq@hZg<>* z^*>egSaa-Gv*P_XFlqAOCwmwphtmXm&h!xKr;;BlyhA;5%AH6n>Tt9b1^8 z&T!3_och?I;DNTqMY5bevvByz{K#aW^T6bm%gwqHcw|d!i6`|aIGTK8>~4ijv`*>u z*^dzs)Z3Ou-GNIIhGA1eca)atuErdGu86i7gi@T}N2*POQvoe8%u~WwT(WS~>PpaV ziO`@U%7&+sqB-kc)sCwCcyAII{7i-C^404O`WWHJ8;+^vQ@(eerZ^AVs79GElLihm zHk}V|b(ODB=MTM>V+(bY7^CK zOkfFVI4UdR8ddYBbAAFTZhqFCy?_4+HW|NjG(-oDo#u&yjEUpCbV}=2OeG~e_DuX* zcHk$yzIR2%#8>^y9v#vTP$}fS#eSob1p~eNnopyB4fj>{G zG1oJXY~omd`uZ`)!v$0w8rDTm=S)|%$e7#iUXR0@>#d5BPh1zT_$RN{>AFJTOtm&O zUBR|&nQi;ZhSh15PX|+^oR2Rm_q|qUZ^Xgb*KB`3x%fEywCb?1E(Uy5;8Vs{_liN= z%j;ql$Ha~6rk;})2#so6pBINpT;E^tI#Vt0;B%3Ya4P3k4~VaO_CCuuB-Y&BiY-cC z|Kxz&hY$N2;Z(l)ljdFM7w+fwl$avJ_zs7x44bV#O}5>%j&2l)LG?T?V8Dywv&T&_ z)%p9xiTMwr0#)Y6gPyLvVH2Z*Pds(Y_QyHaL)s=#C{LA5PJU4))WUN@!eI`2EBsu~ z_r3r%GDJByM3xA?RZlDYfvzQW{l#U+B8~GAx7ZI!KO)BE6dAadD*7{8HCZ#kPVeBwf zl*5tYU_2fK{7;10G9mI@wn=o91+AHB>bss#2^R}<)m;-Ny`|3i1!q2VO>8B-SbU^^ z`K_@LZEZr+g}3ry&HJ6yrrai@zV7w3ShsNv=J>keP1!M+E&sloNK;IsIZG!MU3d22 z*4DSZ*FB&D4i?2g|KZZ&UgN)>N#@R3_uKTZxF3?#W zDf^fvxgXwn91n=sjrxUY7g*y@dRpaGBow=ax+*!MdR&|9S!1?bwqCU;W14;brCJ{c zTK1HUybE;eijR13t@GCI;gxpv((MezbNowa0NuV6{+4?Q!Z-I(q@L>NyT;Us^-$o%du1$r2*T^2gMRs zPa9gE;XGYgs-^qGkWkt5ACA9D|mtn1RAT{59SlY&uV%k|rpS~&Cik$ur)ZWeVQih05qaZG=# z3{L=kQlif(3J2Za#kVtkx85YhZEqbnC^WNXs>Z4r-d>o&4hs zOXjvm8t!gBu%cSNE8^?BRg!A0jE47kvkf=xO5($ovLm8j*yxo$%dR9Nt$pDkpXs}A z-Asl&z@DP^_KR&;5(~4-LnEeHg92yKXh@Pi73pF~z8wltbKtf;l9_Fy#FM!8_zSavM}`gh%{>{OWOqt+q3PF zeuqAjN}_PDkgcV&BdrvLop&AzmoIe5H0?|37CY9&ut0hxsXIbCG@AIm8@!@oY@>|y zPc}~;1C3&UF?R&(4$Bt~j>&6_-|+m(PnF%YfB5jLOxkj^_DD!*`nV+(htmemgbLuK z{p15jv~&^4WiFv%2X8`7?aD~;UULsUyh88BC1Hm9odzZKi6L$?7pO`_+&-x+89VL8 zGz{mvC~Ukbf)^~z|59^#I6INvv-YV8-`}IDgFl+NVUnp|ShBRIY#g*Kp+POS%Jz^> ze2K@2u*2Iu);BDvn#=RP43x{Tp=8*|VRNa2XGD=Q;>I6vjt$X=QTdl}ZsBP?)sg#g zJiAP*bC)~CL_-n#*gZ^BKlO0dch}Qo_{hI+s9Y2ML&1hdwA^x?uUDz9@0gzWUv9kh z6t5X}j~X7?O^!GH%`b)%$COqr>>B_-i4kcIERN~Dh5KCW?o}O8dt+DiEb;apekP1!s>94How#I zX1iR&!ymL)RDd#Rj`TsPMr}iYzX)Z$2450lST%f zCi3wJwacnp5qJK2Y=3W_b2O(>wN>Pjxh-dkttdv-j zS=8~c)wCfs!&8>&b===PStE6~dys=|yWWAI#wAtq)HGtdN38VJrL6|trz(ge1>umT z2ip^_QD=np>@4mxW1xKE($T-??)h*>YUE3m>1m#ACwoY%UPIc@K(delf$RaU31>u9I|mgRaVu`+@s7&(=G3nl*5ryEvJ1R*|ec{ z2TZaaz2#$zI`f8Ux@6hcWid>Zw#GX`1;;<|5lnUEdn|92%KY{3C^UQxZfNHUrsxGv z!OhA|UL_JmolFVVbrd4OpiW{HEJ)D{M>8I{KHm}56S*mEm+voE6`Zt_4encxVO$ZF zL2g6kDw%r;BX#SI!#T@62RBgEUAjJN4qIKb>ODY(+?V0Eozl`(hgJ6sbh1-k=D2R0 z`0#rA{7tt5@>?KV#1iTFFH7c)lsjE{?fbH#i{9BtUfbvD&s0i@iGGVgdhKVWIfHfu zg~q@3-j`M1f7`12t(Xmkr#S6lr+a-*ANPW0M}!@Qw;_ASyhhLBnb-L*r91i2$MZ#a z{7}g2#bnn*;Y5wKYpuwc&I)Nq?EKKNOdS;J^9rH^7Cqycrwz&Ra>29s^Sr?!zLa|B zl1%Gj4n;lzuMh)yk7KO6yh99UN2JHv+GQ!~s#P|YR7;&4YwYfDYDHhZ%c7%+in=5k z$dg+KXb#lPez$DcNxhiJWtv?zs-^UY zk%>}nz=sN#hfFhFb^EA(R}OX@DtSt$v5~cHq@yRCnw*V|-=l}TnNKS3lv!__nZmD0Rw#v%x2+iX{ zhv|`UF0(ba^wG_GqNda0T#?y#$lz)RZ9(z)UXP8EkD8(*H%eltmQ~mUzlU0GPIk#$ z*6f?yj|DdMI}frZQ?jkN^t~gjs1N$)EG1VSttz)Yv*v9ZpHAvU@8hZ7 z#>Pa$&xMWGsLFPQ$_G|S#o*^6)AVSRQU|9+y=jZ}x=-^N72ja2n91c~2`u29vV8Zz zEl|$81r~@cX-;Qn7P854JiJoSyWtF%sMn%Go;6D@4=;`FdfjYi&VYSOzmE`JoT1^l zc`4}%kl>28IvQH1rH>9l&?`MCTMBw@Oe8b9@aRPhhVfb*H&HHoHkj)No!L; zx^@QrHtkA0!Xxxdma^$~aXZ`I=IAd!?T&zy{oK-EN=|*hb7}Wtft^bk)7o~vD+)HY z)bJ-BqVNi|2~2+XlA2- zdXDg3?D-jFZuR%kg5o#a2dEWxjyEc)@{y+u)>bl}GBHRhw#z#|rzE>cqPkdEI)RtE z;|v$Cd-}yEy^cN^ncr;C12I#Qy)LyOw8(3qM~$;!gx)*8xjnvKrOBZ$nteucKDe&T9&+Gg_U` zK5VzS6j;)2vm==5nU})du&f%#_7V#})~F|udqMZSd=!z<#6 z2WvqEjb(WDD;>FeZ{D`Na^+G^I9F#^^4@W`2gki53FQ*{_7es1=gd7<6ewQl&gGMm z`sI5WpQKZsAJon<&`~Ua8sUTE?mP=QbTNw~rFDlY=skn$?mV{M%9EdBnXTh3R9j>% zH;`?qlV#N^Ik#cjZI828(b_#mKJRl656-FA9bOZcYvJ*1soW9EC!gq7o$Y2>dRXpj z?EDO>!jN95z-G#NyO_(Rj=qnts7J%4mkal(994h$HQ*YA1cTjd8X64e=&Dg z)=Ru3l+Gk7o5B+e40%%C>76=XBzT)Q>N>Wa#VoDmdk;$=a4)locJeeBzoVsp*_y3N z<-jWc{`_+tofWz&6}rmj3Vw@Qrfb`HmFJ}a;H%pyh(qkgmoD$V* z@2_XLf8>QLuU}QUop|hl0-~mt$Itr;?3P#s3&f06RQPjhhmW6bMHW2Uhkho?LHK%w*wvY7jzzd+H^!z2^I2& zv01f*`ec0iUAgbI+^bD=c6$%5cBY$BVv?Aax_nOU*xmfSS_#%-Il033o8fJ6MB~z< z=D!my&*+SU?=Y9A;>ieIO~KuC2Uq2`OTCPX`9j`TcZb|(?ui*`eJh0GV_SLWPrFy< zPO5{EA)LG!sf*RvTyMF$H&Odu!thq>az~b?*mkOB!b)gnmLJvu$$`A3jX*5~Pq8{tbRO zZpebkAJ8UF)E_US%5V8wf@b^2mK* zX0+?ewI$X_94O9CE%ust5J`rTfGZ|&NgB5H^03g8>9|098Cm!#{xnTqhPS(MDY;0`G7u^mYeiJAXRZm0>Oluwj6N@Gr^>qn9 zb~uE*6agkM)X@oFLfPj8N#25BvV8-sv%9-&%C7Z5(27)m@GH4-W9hqtu%v%2(v5S=4|I)vhV|_ za}-FI&ff={yYIPikQO1ghitf`OzBZInkith>S49|j)W62d3jdEGBg38J{&5<1KPL{dxVL3>gyq1f*IfAmh(ZGn3!!S%dc{8h?i8*y`#JpPpYHmk#-30nPpS z|A^l6*~HQ<@NLMgN{SSIek~3=7CgPh78_2G<7o*+W?+4WBatP-$;tWgo9h<(oQR_Y zjNFI{0V!?A#lKJABF%1SCs0GpbPtR@i=V3w`W~vT)Ia&DdGURX;MY)AaVTD+$Hv@a zw?=9)dlEuAx6MiWb;t9Tk&$cslc#0|mwg;-K0HgnKUICa{Im0#_O=&^wdDP*Ls0^6 zmdpgKAP_+Vd<{~ud;=^rY5&iEhfA*em+$z;$@Bk<6_kO~ew|&V&uW+kNDL>aFQ)=7 z*uMf!`gq%hN~8b@UX*efd78@pf>oSMoDqW{9zii{Ra{pAY(7wIhQJ1(Yqo>@0f5oq z?bvmC(8zCk$kL_qe={!n#)g3UssfieEaP=AUT^^3DnmBWpT?ki0N?jdyvb`ygrJ;@kpWTC{1-8iuIbK~CYU$nfDNX+e`G*H9=FZZZuA^0GBM&a!ctY0ai^ zM&6oytcI5aPT{Pakdk4bb8f)2^ zm~;TQ9SN?5_XZ6x6C}Lp?Wl)`133mK$hF$`yLZ4zA~=PG*$5H?u}At+qt;|+-WnJV zMvlLzhQ1#H*PvU#I0374|42f9sjUJ@c7sJ047T>*_tB(%7Ngacz60U6Chr^{Y+?rF zhKv@h0(b^edG_$R2XcAW9SF0NNTdPhCzeBjK$rapb)G3DH5fd1+HjA_^5y$*U4mG= zO|!@{4_`=Oa>pCwsE|!q1))nwLE;S`56yq$fg92I7CfSbNWK#7oqw^f(~LvYhYO++ zJqDyX9n7NOQ9liSzly|1VFDyERn|9$Myq{iA$K6r?08ZnJCsC+1396E*{=z=g0U}G zm%JZfYPbIAx&62c)E;OJGx3Z_gC@cVOM+&Oxbe4bfMJ*&FNX)CJ|orU>$kzGTD+dN zl@I8Pq^1HogLx!f4UCVEH(k81u12%X_|J@UIQ7CFU|FEVUAU-{8JEGb5wUWkr56=;UPl@>ajUZ3;-CJz1_s zq{p9S1k!}i!bun`=B12Nll|bVW)(ss z9(gsEmfV=ql#bzRh!PKs=WY1r4l#!9i%E%zl15@XE5J`ur z5b){`-wykHj^-rO+-wSMBapozCN5q-umE0aAfhY#X0YY}t?K>2O;OFjm*Rm%0GB`? z0@mx%oBV?f_z>t+(E&*ODEUSq>s;u_F%#{>I%#@k=BD}$$Ud)w@Cb{|(i!#jZpnJz z5238j2RRswi+kbf^NZ3xA#)}J6Qasu1It;aOlsYPmx6_D?*^$p95-?|A=Pu5dERk0t?9n72vjD^x=y^paGjkm(l)j6;DO3MYcV z%B8W&cZhTC+PIG&o&QWbCq5;f*&h6C0O^NhMi8DJ*%mN&Jr9C>;A92_D7c*6kt5P% zKp|#xM&#gZ!!l_KpcfaokwEK;bu1!-C}{{#|G^(17wYigfbXY{4!blYx>5)5-SLlg zG`Vj)h&oJOAk>$@sE7y8K*q`;p>GPH-YqjD+*}T8aA)^x@nR4yNXE2)cF#zVcSn#I z8TkLumxESFw>v~U*ifY|(p(n|&!$SNtd zz-uM^{Oiz=P4nZd&Fw!BSQnvKRo0uF^aRvQvWzO0HdZ z9o|eh+1n>#G&wW#Ox^k>D=Vw#mqBhWQvED34c(dx$Z{mgfQ%0Um*`;gIbPfbHViOr zA&M7E9bEvH{52esMweiIGq@?pJ+!uPmY$6v;{b4I1m}AK=k1!lEMv@G5 zLASq+SzA_7@iB~2D*j{UUOlDsND9&#wn^Qe1@+lKY?A+B d*!HIt8poWup%j)^>k0g4Y+$N?SI;@}e*wn