From 37413496322651a5a4f6ed60d212737e39913302 Mon Sep 17 00:00:00 2001 From: ModelHub XC Date: Thu, 7 May 2026 10:57:13 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=EF=BC=8C=E7=94=B1ModelHub=20XC=E7=A4=BE=E5=8C=BA=E6=8F=90?= =?UTF-8?q?=E4=BE=9B=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model: pmahdavi/Llama-3.1-8B-coding Source: Original Platform --- .gitattributes | 36 + README.md | 63 + all_results.json | 8 + config.json | 35 + export/exp_avg_sq.safetensors | 3 + export/fisher_diag_s1024_bs128.safetensors | 3 + generation_config.json | 9 + latest | 1 + model-00001-of-00004.safetensors | 3 + model-00002-of-00004.safetensors | 3 + model-00003-of-00004.safetensors | 3 + model-00004-of-00004.safetensors | 3 + model.safetensors.index.json | 298 +++ .../zero_pp_rank_0_mp_rank_00_model_states.pt | 3 + .../zero_pp_rank_1_mp_rank_00_model_states.pt | 3 + rng_state_0.pth | 3 + rng_state_1.pth | 3 + scheduler.pt | 3 + special_tokens_map.json | 17 + tokenizer.json | 3 + tokenizer_config.json | 2066 +++++++++++++++++ train_results.json | 8 + trainer_log.jsonl | 112 + trainer_state.json | 811 +++++++ training_args.bin | 3 + training_config.yaml | 27 + training_loss.png | Bin 0 -> 48832 bytes zero_to_fp32.py | 760 ++++++ 28 files changed, 4290 insertions(+) create mode 100644 .gitattributes create mode 100644 README.md create mode 100644 all_results.json create mode 100644 config.json create mode 100644 export/exp_avg_sq.safetensors create mode 100644 export/fisher_diag_s1024_bs128.safetensors create mode 100644 generation_config.json create mode 100644 latest create mode 100644 model-00001-of-00004.safetensors create mode 100644 model-00002-of-00004.safetensors create mode 100644 model-00003-of-00004.safetensors create mode 100644 model-00004-of-00004.safetensors create mode 100644 model.safetensors.index.json create mode 100644 model_states/zero_pp_rank_0_mp_rank_00_model_states.pt create mode 100644 model_states/zero_pp_rank_1_mp_rank_00_model_states.pt create mode 100644 rng_state_0.pth create mode 100644 rng_state_1.pth create mode 100644 scheduler.pt create mode 100644 special_tokens_map.json create mode 100644 tokenizer.json create mode 100644 tokenizer_config.json create mode 100644 train_results.json create mode 100644 trainer_log.jsonl create mode 100644 trainer_state.json create mode 100644 training_args.bin create mode 100644 training_config.yaml create mode 100644 training_loss.png create mode 100644 zero_to_fp32.py diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..52373fe --- /dev/null +++ b/.gitattributes @@ -0,0 +1,36 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..77bdf07 --- /dev/null +++ b/README.md @@ -0,0 +1,63 @@ +--- +library_name: transformers +license: other +base_model: meta-llama/Llama-3.1-8B +tags: +- llama-factory +- full +- generated_from_trainer +datasets: +- tulu3_category_loader +model-index: +- name: Llama-3.1-8B_tulu3_mixture_coding_full_ebs128_lr5e-06 + results: [] +--- + + + +# Llama-3.1-8B_tulu3_mixture_coding_full_ebs128_lr5e-06 + +This model is a fine-tuned version of [meta-llama/Llama-3.1-8B](https://huggingface.co/meta-llama/Llama-3.1-8B) on the tulu3_mixture_coding dataset. This checkpoint was released alongside https://arxiv.org/abs/2509.11167. + +## Model description + +More information needed + +## Intended uses & limitations + +More information needed + +## Training and evaluation data + +More information needed + +## Training procedure + +### Training hyperparameters + +The following hyperparameters were used during training: +- learning_rate: 5e-06 +- train_batch_size: 2 +- eval_batch_size: 8 +- seed: 42 +- distributed_type: multi-GPU +- num_devices: 2 +- gradient_accumulation_steps: 32 +- total_train_batch_size: 128 +- total_eval_batch_size: 16 +- optimizer: Use adamw_torch with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments +- lr_scheduler_type: cosine +- lr_scheduler_warmup_ratio: 0.03 +- num_epochs: 1.0 + +### Training results + + + +### Framework versions + +- Transformers 4.51.1 +- Pytorch 2.6.0+cu124 +- Datasets 3.4.1 +- Tokenizers 0.21.0 diff --git a/all_results.json b/all_results.json new file mode 100644 index 0000000..b53e79f --- /dev/null +++ b/all_results.json @@ -0,0 +1,8 @@ +{ + "epoch": 0.9995220557226798, + "total_flos": 145966987345920.0, + "train_loss": 0.845390060482317, + "train_runtime": 63839.4451, + "train_samples_per_second": 2.229, + "train_steps_per_second": 0.017 +} \ No newline at end of file diff --git a/config.json b/config.json new file mode 100644 index 0000000..473ed92 --- /dev/null +++ b/config.json @@ -0,0 +1,35 @@ +{ + "architectures": [ + "LlamaForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 128000, + "eos_token_id": 128001, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 4096, + "initializer_range": 0.02, + "intermediate_size": 14336, + "max_position_embeddings": 131072, + "mlp_bias": false, + "model_type": "llama", + "num_attention_heads": 32, + "num_hidden_layers": 32, + "num_key_value_heads": 8, + "pretraining_tp": 1, + "rms_norm_eps": 1e-05, + "rope_scaling": { + "factor": 8.0, + "high_freq_factor": 4.0, + "low_freq_factor": 1.0, + "original_max_position_embeddings": 8192, + "rope_type": "llama3" + }, + "rope_theta": 500000.0, + "tie_word_embeddings": false, + "torch_dtype": "bfloat16", + "transformers_version": "4.51.1", + "use_cache": false, + "vocab_size": 128256 +} diff --git a/export/exp_avg_sq.safetensors b/export/exp_avg_sq.safetensors new file mode 100644 index 0000000..69d360f --- /dev/null +++ b/export/exp_avg_sq.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fb0af75bc627783ba0187e3f8b2264e4ba60696e0ded25993c9a3bec8bd7c88c +size 32121079000 diff --git a/export/fisher_diag_s1024_bs128.safetensors b/export/fisher_diag_s1024_bs128.safetensors new file mode 100644 index 0000000..951fba1 --- /dev/null +++ b/export/fisher_diag_s1024_bs128.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:79ad01a1a1cebb118a5b35a93ffb1692681771b4973b8fcbc4522675ec85145b +size 32121079000 diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..ba91d8f --- /dev/null +++ b/generation_config.json @@ -0,0 +1,9 @@ +{ + "_from_model_config": true, + "bos_token_id": 128000, + "do_sample": true, + "eos_token_id": 128001, + "temperature": 0.6, + "top_p": 0.9, + "transformers_version": "4.51.1" +} diff --git a/latest b/latest new file mode 100644 index 0000000..88367b1 --- /dev/null +++ b/latest @@ -0,0 +1 @@ +global_step1111 \ No newline at end of file diff --git a/model-00001-of-00004.safetensors b/model-00001-of-00004.safetensors new file mode 100644 index 0000000..ec844ed --- /dev/null +++ b/model-00001-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:01aabe91e0e8bd9d90749107831611e150ac3582fd141e612323de374e9bb9d8 +size 4976698672 diff --git a/model-00002-of-00004.safetensors b/model-00002-of-00004.safetensors new file mode 100644 index 0000000..bc5584e --- /dev/null +++ b/model-00002-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b2e147cf033413e63247a438c0b23632c7bafaad032c717431b7b77c1b24972a +size 4999802720 diff --git a/model-00003-of-00004.safetensors b/model-00003-of-00004.safetensors new file mode 100644 index 0000000..5d6b9f0 --- /dev/null +++ b/model-00003-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fc4f96a5d98f58401c550db8dc8495f6f9606541520d57070c7be3e650b42d82 +size 4915916176 diff --git a/model-00004-of-00004.safetensors b/model-00004-of-00004.safetensors new file mode 100644 index 0000000..cf8149f --- /dev/null +++ b/model-00004-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7bd0a4746e3a0712c57bdfdf184d65fe3542e4e1a8600a2285269f632df28d4 +size 1168138808 diff --git a/model.safetensors.index.json b/model.safetensors.index.json new file mode 100644 index 0000000..0fd8120 --- /dev/null +++ b/model.safetensors.index.json @@ -0,0 +1,298 @@ +{ + "metadata": { + "total_size": 16060522496 + }, + "weight_map": { + "lm_head.weight": "model-00004-of-00004.safetensors", + "model.embed_tokens.weight": "model-00001-of-00004.safetensors", + "model.layers.0.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.10.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.10.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.11.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.12.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.13.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.14.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.15.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.16.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.17.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.18.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.19.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.2.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.20.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.20.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.20.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.20.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.21.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.21.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.21.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.22.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.22.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.23.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.24.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.25.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.26.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.27.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.28.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.29.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.3.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.30.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.30.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.30.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.input_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.31.mlp.down_proj.weight": "model-00004-of-00004.safetensors", + "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.post_attention_layernorm.weight": "model-00004-of-00004.safetensors", + "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.4.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.4.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.5.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.6.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.7.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.8.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.8.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.8.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.9.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.9.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.9.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.norm.weight": "model-00004-of-00004.safetensors" + } +} diff --git a/model_states/zero_pp_rank_0_mp_rank_00_model_states.pt b/model_states/zero_pp_rank_0_mp_rank_00_model_states.pt new file mode 100644 index 0000000..69f897d --- /dev/null +++ b/model_states/zero_pp_rank_0_mp_rank_00_model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ce1cbccd213c73f5f14cd8ef82d784b903a600bb3fcefec401a7b6839aae1be2 +size 149093 diff --git a/model_states/zero_pp_rank_1_mp_rank_00_model_states.pt b/model_states/zero_pp_rank_1_mp_rank_00_model_states.pt new file mode 100644 index 0000000..73f2989 --- /dev/null +++ b/model_states/zero_pp_rank_1_mp_rank_00_model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5f77c8df7fbe5a5055801516cc593d0504d6a0928671f8cfb7ae449d08be203b +size 149093 diff --git a/rng_state_0.pth b/rng_state_0.pth new file mode 100644 index 0000000..b24ba52 --- /dev/null +++ b/rng_state_0.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c8d6a959372d5e0c2ea025dd26c9d0ad2046fce19352056cae8074dcbd0a6fd4 +size 14512 diff --git a/rng_state_1.pth b/rng_state_1.pth new file mode 100644 index 0000000..9350a82 --- /dev/null +++ b/rng_state_1.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0f68a37892a1b445d21bb35cc10bf7a058a6f9ec8c363f5ed156ff4f49d90fb6 +size 14512 diff --git a/scheduler.pt b/scheduler.pt new file mode 100644 index 0000000..683c907 --- /dev/null +++ b/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:36dd5f953aa6491cbd5bcc3e1bb0bb52769f592a7ef677e16c6afe767515245d +size 1064 diff --git a/special_tokens_map.json b/special_tokens_map.json new file mode 100644 index 0000000..04829af --- /dev/null +++ b/special_tokens_map.json @@ -0,0 +1,17 @@ +{ + "bos_token": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": "<|end_of_text|>" +} diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..1c1d8d5 --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6b9e4e7fb171f92fd137b777cc2714bf87d11576700a1dcd7a399e7bbe39537b +size 17209920 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..02d4abd --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,2066 @@ +{ + "added_tokens_decoder": { + "128000": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128001": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128002": { + "content": "<|reserved_special_token_0|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128003": { + "content": "<|reserved_special_token_1|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128004": { + "content": "<|finetune_right_pad_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128005": { + "content": "<|reserved_special_token_2|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128006": { + "content": "<|start_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128007": { + "content": "<|end_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128008": { + "content": "<|eom_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128009": { + "content": "<|eot_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128010": { + "content": "<|python_tag|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128011": { + "content": "<|reserved_special_token_3|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128012": { + "content": "<|reserved_special_token_4|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128013": { + "content": "<|reserved_special_token_5|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128014": { + "content": "<|reserved_special_token_6|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128015": { + "content": "<|reserved_special_token_7|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128016": { + "content": "<|reserved_special_token_8|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128017": { + "content": "<|reserved_special_token_9|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128018": { + "content": "<|reserved_special_token_10|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128019": { + "content": "<|reserved_special_token_11|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128020": { + "content": "<|reserved_special_token_12|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128021": { + "content": "<|reserved_special_token_13|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128022": { + "content": "<|reserved_special_token_14|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128023": { + "content": "<|reserved_special_token_15|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128024": { + "content": "<|reserved_special_token_16|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128025": { + "content": "<|reserved_special_token_17|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128026": { + "content": "<|reserved_special_token_18|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128027": { + "content": "<|reserved_special_token_19|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128028": { + "content": "<|reserved_special_token_20|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128029": { + "content": "<|reserved_special_token_21|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128030": { + "content": "<|reserved_special_token_22|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128031": { + "content": "<|reserved_special_token_23|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128032": { + "content": "<|reserved_special_token_24|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128033": { + "content": "<|reserved_special_token_25|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128034": { + "content": "<|reserved_special_token_26|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128035": { + "content": "<|reserved_special_token_27|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128036": { + "content": "<|reserved_special_token_28|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128037": { + "content": "<|reserved_special_token_29|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128038": { + "content": "<|reserved_special_token_30|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128039": { + "content": "<|reserved_special_token_31|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128040": { + "content": "<|reserved_special_token_32|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128041": { + "content": "<|reserved_special_token_33|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128042": { + "content": "<|reserved_special_token_34|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128043": { + "content": "<|reserved_special_token_35|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128044": { + "content": "<|reserved_special_token_36|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128045": { + "content": "<|reserved_special_token_37|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128046": { + "content": "<|reserved_special_token_38|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128047": { + "content": "<|reserved_special_token_39|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128048": { + "content": "<|reserved_special_token_40|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128049": { + "content": "<|reserved_special_token_41|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128050": { + "content": "<|reserved_special_token_42|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128051": { + "content": "<|reserved_special_token_43|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128052": { + "content": "<|reserved_special_token_44|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128053": { + "content": "<|reserved_special_token_45|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128054": { + "content": "<|reserved_special_token_46|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128055": { + "content": "<|reserved_special_token_47|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128056": { + "content": "<|reserved_special_token_48|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128057": { + "content": "<|reserved_special_token_49|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128058": { + "content": "<|reserved_special_token_50|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128059": { + "content": "<|reserved_special_token_51|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128060": { + "content": "<|reserved_special_token_52|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128061": { + "content": "<|reserved_special_token_53|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128062": { + "content": "<|reserved_special_token_54|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128063": { + "content": "<|reserved_special_token_55|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128064": { + "content": "<|reserved_special_token_56|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128065": { + "content": "<|reserved_special_token_57|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128066": { + "content": "<|reserved_special_token_58|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128067": { + "content": "<|reserved_special_token_59|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128068": { + "content": "<|reserved_special_token_60|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128069": { + "content": "<|reserved_special_token_61|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128070": { + "content": "<|reserved_special_token_62|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128071": { + "content": "<|reserved_special_token_63|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128072": { + "content": "<|reserved_special_token_64|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128073": { + "content": "<|reserved_special_token_65|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128074": { + "content": "<|reserved_special_token_66|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128075": { + "content": "<|reserved_special_token_67|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128076": { + "content": "<|reserved_special_token_68|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128077": { + "content": "<|reserved_special_token_69|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128078": { + "content": "<|reserved_special_token_70|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128079": { + "content": "<|reserved_special_token_71|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128080": { + "content": "<|reserved_special_token_72|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128081": { + "content": "<|reserved_special_token_73|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128082": { + "content": "<|reserved_special_token_74|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128083": { + "content": "<|reserved_special_token_75|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128084": { + "content": "<|reserved_special_token_76|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128085": { + "content": "<|reserved_special_token_77|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128086": { + "content": "<|reserved_special_token_78|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128087": { + "content": "<|reserved_special_token_79|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128088": { + "content": "<|reserved_special_token_80|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128089": { + "content": "<|reserved_special_token_81|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128090": { + "content": "<|reserved_special_token_82|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128091": { + "content": "<|reserved_special_token_83|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128092": { + "content": "<|reserved_special_token_84|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128093": { + "content": "<|reserved_special_token_85|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128094": { + "content": "<|reserved_special_token_86|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128095": { + "content": "<|reserved_special_token_87|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128096": { + "content": "<|reserved_special_token_88|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128097": { + "content": "<|reserved_special_token_89|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128098": { + "content": "<|reserved_special_token_90|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128099": { + "content": "<|reserved_special_token_91|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128100": { + "content": "<|reserved_special_token_92|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128101": { + "content": "<|reserved_special_token_93|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128102": { + "content": "<|reserved_special_token_94|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128103": { + "content": "<|reserved_special_token_95|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128104": { + "content": "<|reserved_special_token_96|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128105": { + "content": "<|reserved_special_token_97|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128106": { + "content": "<|reserved_special_token_98|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128107": { + "content": "<|reserved_special_token_99|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128108": { + "content": "<|reserved_special_token_100|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128109": { + "content": "<|reserved_special_token_101|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128110": { + "content": "<|reserved_special_token_102|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128111": { + "content": "<|reserved_special_token_103|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128112": { + "content": "<|reserved_special_token_104|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128113": { + "content": "<|reserved_special_token_105|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128114": { + "content": "<|reserved_special_token_106|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128115": { + "content": "<|reserved_special_token_107|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128116": { + "content": "<|reserved_special_token_108|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128117": { + "content": "<|reserved_special_token_109|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128118": { + "content": "<|reserved_special_token_110|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128119": { + "content": "<|reserved_special_token_111|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128120": { + "content": "<|reserved_special_token_112|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128121": { + "content": "<|reserved_special_token_113|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128122": { + "content": "<|reserved_special_token_114|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128123": { + "content": "<|reserved_special_token_115|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128124": { + "content": "<|reserved_special_token_116|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128125": { + "content": "<|reserved_special_token_117|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128126": { + "content": "<|reserved_special_token_118|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128127": { + "content": "<|reserved_special_token_119|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128128": { + "content": "<|reserved_special_token_120|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128129": { + "content": "<|reserved_special_token_121|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128130": { + "content": "<|reserved_special_token_122|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128131": { + "content": "<|reserved_special_token_123|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128132": { + "content": "<|reserved_special_token_124|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128133": { + "content": "<|reserved_special_token_125|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128134": { + "content": "<|reserved_special_token_126|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128135": { + "content": "<|reserved_special_token_127|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128136": { + "content": "<|reserved_special_token_128|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128137": { + "content": "<|reserved_special_token_129|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128138": { + "content": "<|reserved_special_token_130|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128139": { + "content": "<|reserved_special_token_131|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128140": { + "content": "<|reserved_special_token_132|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128141": { + "content": "<|reserved_special_token_133|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128142": { + "content": "<|reserved_special_token_134|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128143": { + "content": "<|reserved_special_token_135|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128144": { + "content": "<|reserved_special_token_136|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128145": { + "content": "<|reserved_special_token_137|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128146": { + "content": "<|reserved_special_token_138|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128147": { + "content": "<|reserved_special_token_139|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128148": { + "content": "<|reserved_special_token_140|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128149": { + "content": "<|reserved_special_token_141|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128150": { + "content": "<|reserved_special_token_142|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128151": { + "content": "<|reserved_special_token_143|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128152": { + "content": "<|reserved_special_token_144|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128153": { + "content": "<|reserved_special_token_145|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128154": { + "content": "<|reserved_special_token_146|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128155": { + "content": "<|reserved_special_token_147|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128156": { + "content": "<|reserved_special_token_148|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128157": { + "content": "<|reserved_special_token_149|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128158": { + "content": "<|reserved_special_token_150|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128159": { + "content": "<|reserved_special_token_151|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128160": { + "content": "<|reserved_special_token_152|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128161": { + "content": "<|reserved_special_token_153|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128162": { + "content": "<|reserved_special_token_154|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128163": { + "content": "<|reserved_special_token_155|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128164": { + "content": "<|reserved_special_token_156|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128165": { + "content": "<|reserved_special_token_157|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128166": { + "content": "<|reserved_special_token_158|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128167": { + "content": "<|reserved_special_token_159|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128168": { + "content": "<|reserved_special_token_160|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128169": { + "content": "<|reserved_special_token_161|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128170": { + "content": "<|reserved_special_token_162|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128171": { + "content": "<|reserved_special_token_163|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128172": { + "content": "<|reserved_special_token_164|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128173": { + "content": "<|reserved_special_token_165|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128174": { + "content": "<|reserved_special_token_166|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128175": { + "content": "<|reserved_special_token_167|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128176": { + "content": "<|reserved_special_token_168|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128177": { + "content": "<|reserved_special_token_169|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128178": { + "content": "<|reserved_special_token_170|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128179": { + "content": "<|reserved_special_token_171|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128180": { + "content": "<|reserved_special_token_172|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128181": { + "content": "<|reserved_special_token_173|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128182": { + "content": "<|reserved_special_token_174|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128183": { + "content": "<|reserved_special_token_175|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128184": { + "content": "<|reserved_special_token_176|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128185": { + "content": "<|reserved_special_token_177|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128186": { + "content": "<|reserved_special_token_178|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128187": { + "content": "<|reserved_special_token_179|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128188": { + "content": "<|reserved_special_token_180|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128189": { + "content": "<|reserved_special_token_181|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128190": { + "content": "<|reserved_special_token_182|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128191": { + "content": "<|reserved_special_token_183|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128192": { + "content": "<|reserved_special_token_184|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128193": { + "content": "<|reserved_special_token_185|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128194": { + "content": "<|reserved_special_token_186|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128195": { + "content": "<|reserved_special_token_187|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128196": { + "content": "<|reserved_special_token_188|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128197": { + "content": "<|reserved_special_token_189|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128198": { + "content": "<|reserved_special_token_190|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128199": { + "content": "<|reserved_special_token_191|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128200": { + "content": "<|reserved_special_token_192|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128201": { + "content": "<|reserved_special_token_193|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128202": { + "content": "<|reserved_special_token_194|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128203": { + "content": "<|reserved_special_token_195|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128204": { + "content": "<|reserved_special_token_196|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128205": { + "content": "<|reserved_special_token_197|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128206": { + "content": "<|reserved_special_token_198|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128207": { + "content": "<|reserved_special_token_199|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128208": { + "content": "<|reserved_special_token_200|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128209": { + "content": "<|reserved_special_token_201|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128210": { + "content": "<|reserved_special_token_202|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128211": { + "content": "<|reserved_special_token_203|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128212": { + "content": "<|reserved_special_token_204|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128213": { + "content": "<|reserved_special_token_205|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128214": { + "content": "<|reserved_special_token_206|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128215": { + "content": "<|reserved_special_token_207|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128216": { + "content": "<|reserved_special_token_208|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128217": { + "content": "<|reserved_special_token_209|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128218": { + "content": "<|reserved_special_token_210|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128219": { + "content": "<|reserved_special_token_211|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128220": { + "content": "<|reserved_special_token_212|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128221": { + "content": "<|reserved_special_token_213|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128222": { + "content": "<|reserved_special_token_214|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128223": { + "content": "<|reserved_special_token_215|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128224": { + "content": "<|reserved_special_token_216|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128225": { + "content": "<|reserved_special_token_217|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128226": { + "content": "<|reserved_special_token_218|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128227": { + "content": "<|reserved_special_token_219|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128228": { + "content": "<|reserved_special_token_220|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128229": { + "content": "<|reserved_special_token_221|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128230": { + "content": "<|reserved_special_token_222|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128231": { + "content": "<|reserved_special_token_223|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128232": { + "content": "<|reserved_special_token_224|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128233": { + "content": "<|reserved_special_token_225|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128234": { + "content": "<|reserved_special_token_226|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128235": { + "content": "<|reserved_special_token_227|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128236": { + "content": "<|reserved_special_token_228|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128237": { + "content": "<|reserved_special_token_229|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128238": { + "content": "<|reserved_special_token_230|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128239": { + "content": "<|reserved_special_token_231|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128240": { + "content": "<|reserved_special_token_232|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128241": { + "content": "<|reserved_special_token_233|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128242": { + "content": "<|reserved_special_token_234|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128243": { + "content": "<|reserved_special_token_235|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128244": { + "content": "<|reserved_special_token_236|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128245": { + "content": "<|reserved_special_token_237|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128246": { + "content": "<|reserved_special_token_238|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128247": { + "content": "<|reserved_special_token_239|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128248": { + "content": "<|reserved_special_token_240|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128249": { + "content": "<|reserved_special_token_241|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128250": { + "content": "<|reserved_special_token_242|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128251": { + "content": "<|reserved_special_token_243|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128252": { + "content": "<|reserved_special_token_244|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128253": { + "content": "<|reserved_special_token_245|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128254": { + "content": "<|reserved_special_token_246|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128255": { + "content": "<|reserved_special_token_247|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + } + }, + "bos_token": "<|begin_of_text|>", + "chat_template": "{{ '<|begin_of_text|>' }}{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% set system_message = messages[0]['content'] %}{% else %}{% set loop_messages = messages %}{% endif %}{% if system_message is defined %}{{ '<|system|>\n' + system_message + '\n' }}{% endif %}{% for message in loop_messages %}{% set content = message['content'] %}{% if message['role'] == 'user' %}{{ '<|user|>\n' + content + '\n' }}{% elif message['role'] == 'assistant' %}{{ '<|assistant|>\n' + content + '<|end_of_text|>' + '\n' }}{% endif %}{% endfor %}", + "clean_up_tokenization_spaces": true, + "eos_token": "<|end_of_text|>", + "extra_special_tokens": {}, + "model_input_names": [ + "input_ids", + "attention_mask" + ], + "model_max_length": 131072, + "pad_token": "<|end_of_text|>", + "padding_side": "right", + "split_special_tokens": false, + "tokenizer_class": "PreTrainedTokenizer" +} diff --git a/train_results.json b/train_results.json new file mode 100644 index 0000000..b53e79f --- /dev/null +++ b/train_results.json @@ -0,0 +1,8 @@ +{ + "epoch": 0.9995220557226798, + "total_flos": 145966987345920.0, + "train_loss": 0.845390060482317, + "train_runtime": 63839.4451, + "train_samples_per_second": 2.229, + "train_steps_per_second": 0.017 +} \ No newline at end of file diff --git a/trainer_log.jsonl b/trainer_log.jsonl new file mode 100644 index 0000000..f3a4d55 --- /dev/null +++ b/trainer_log.jsonl @@ -0,0 +1,112 @@ +{"current_steps": 10, "total_steps": 1111, "loss": 1.1112, "lr": 1.323529411764706e-06, "epoch": 0.00899659816132025, "percentage": 0.9, "elapsed_time": "0:09:34", "remaining_time": "17:35:06"} +{"current_steps": 20, "total_steps": 1111, "loss": 0.9362, "lr": 2.7941176470588237e-06, "epoch": 0.0179931963226405, "percentage": 1.8, "elapsed_time": "0:19:07", "remaining_time": "17:23:32"} +{"current_steps": 30, "total_steps": 1111, "loss": 0.8806, "lr": 4.264705882352942e-06, "epoch": 0.02698979448396075, "percentage": 2.7, "elapsed_time": "0:28:40", "remaining_time": "17:13:04"} +{"current_steps": 40, "total_steps": 1111, "loss": 0.8947, "lr": 4.9997341046993195e-06, "epoch": 0.035986392645281, "percentage": 3.6, "elapsed_time": "0:38:10", "remaining_time": "17:02:02"} +{"current_steps": 50, "total_steps": 1111, "loss": 0.8764, "lr": 4.997607281643338e-06, "epoch": 0.04498299080660125, "percentage": 4.5, "elapsed_time": "0:47:51", "remaining_time": "16:55:29"} +{"current_steps": 60, "total_steps": 1111, "loss": 0.8758, "lr": 4.993355445074358e-06, "epoch": 0.0539795889679215, "percentage": 5.4, "elapsed_time": "0:57:26", "remaining_time": "16:46:14"} +{"current_steps": 70, "total_steps": 1111, "loss": 0.8839, "lr": 4.986982212538754e-06, "epoch": 0.06297618712924176, "percentage": 6.3, "elapsed_time": "1:07:01", "remaining_time": "16:36:41"} +{"current_steps": 80, "total_steps": 1111, "loss": 0.8741, "lr": 4.978493006508408e-06, "epoch": 0.071972785290562, "percentage": 7.2, "elapsed_time": "1:16:24", "remaining_time": "16:24:40"} +{"current_steps": 90, "total_steps": 1111, "loss": 0.8859, "lr": 4.967895049767168e-06, "epoch": 0.08096938345188226, "percentage": 8.1, "elapsed_time": "1:26:02", "remaining_time": "16:16:00"} +{"current_steps": 100, "total_steps": 1111, "loss": 0.8506, "lr": 4.9551973592655565e-06, "epoch": 0.0899659816132025, "percentage": 9.0, "elapsed_time": "1:35:30", "remaining_time": "16:05:31"} +{"current_steps": 110, "total_steps": 1111, "loss": 0.9021, "lr": 4.940410738448974e-06, "epoch": 0.09896257977452276, "percentage": 9.9, "elapsed_time": "1:44:58", "remaining_time": "15:55:17"} +{"current_steps": 120, "total_steps": 1111, "loss": 0.8752, "lr": 4.923547768065916e-06, "epoch": 0.107959177935843, "percentage": 10.8, "elapsed_time": "1:54:30", "remaining_time": "15:45:38"} +{"current_steps": 130, "total_steps": 1111, "loss": 0.8824, "lr": 4.904622795464018e-06, "epoch": 0.11695577609716326, "percentage": 11.7, "elapsed_time": "2:03:56", "remaining_time": "15:35:18"} +{"current_steps": 140, "total_steps": 1111, "loss": 0.8519, "lr": 4.883651922383059e-06, "epoch": 0.12595237425848352, "percentage": 12.6, "elapsed_time": "2:13:35", "remaining_time": "15:26:33"} +{"current_steps": 150, "total_steps": 1111, "loss": 0.8565, "lr": 4.860652991255274e-06, "epoch": 0.13494897241980378, "percentage": 13.5, "elapsed_time": "2:23:06", "remaining_time": "15:16:51"} +{"current_steps": 160, "total_steps": 1111, "loss": 0.8884, "lr": 4.835645570024666e-06, "epoch": 0.143945570581124, "percentage": 14.4, "elapsed_time": "2:32:26", "remaining_time": "15:06:07"} +{"current_steps": 170, "total_steps": 1111, "loss": 0.8394, "lr": 4.808650935498216e-06, "epoch": 0.15294216874244426, "percentage": 15.3, "elapsed_time": "2:41:57", "remaining_time": "14:56:27"} +{"current_steps": 180, "total_steps": 1111, "loss": 0.8542, "lr": 4.779692055243149e-06, "epoch": 0.16193876690376452, "percentage": 16.2, "elapsed_time": "2:51:30", "remaining_time": "14:47:02"} +{"current_steps": 190, "total_steps": 1111, "loss": 0.8633, "lr": 4.748793568045682e-06, "epoch": 0.17093536506508478, "percentage": 17.1, "elapsed_time": "3:00:52", "remaining_time": "14:36:45"} +{"current_steps": 200, "total_steps": 1111, "loss": 0.8532, "lr": 4.715981762947854e-06, "epoch": 0.179931963226405, "percentage": 18.0, "elapsed_time": "3:10:27", "remaining_time": "14:27:31"} +{"current_steps": 210, "total_steps": 1111, "loss": 0.8544, "lr": 4.681284556880294e-06, "epoch": 0.18892856138772526, "percentage": 18.9, "elapsed_time": "3:19:50", "remaining_time": "14:17:24"} +{"current_steps": 220, "total_steps": 1111, "loss": 0.8463, "lr": 4.6447314709099436e-06, "epoch": 0.19792515954904552, "percentage": 19.8, "elapsed_time": "3:29:20", "remaining_time": "14:07:48"} +{"current_steps": 230, "total_steps": 1111, "loss": 0.8652, "lr": 4.606353605122954e-06, "epoch": 0.20692175771036578, "percentage": 20.7, "elapsed_time": "3:38:52", "remaining_time": "13:58:21"} +{"current_steps": 240, "total_steps": 1111, "loss": 0.8737, "lr": 4.566183612164116e-06, "epoch": 0.215918355871686, "percentage": 21.6, "elapsed_time": "3:48:22", "remaining_time": "13:48:47"} +{"current_steps": 250, "total_steps": 1111, "loss": 0.8448, "lr": 4.52425566945535e-06, "epoch": 0.22491495403300626, "percentage": 22.5, "elapsed_time": "3:57:54", "remaining_time": "13:39:20"} +{"current_steps": 260, "total_steps": 1111, "loss": 0.8531, "lr": 4.480605450116879e-06, "epoch": 0.23391155219432652, "percentage": 23.4, "elapsed_time": "4:07:39", "remaining_time": "13:30:35"} +{"current_steps": 270, "total_steps": 1111, "loss": 0.8569, "lr": 4.435270092615835e-06, "epoch": 0.24290815035564678, "percentage": 24.3, "elapsed_time": "4:17:12", "remaining_time": "13:21:09"} +{"current_steps": 280, "total_steps": 1111, "loss": 0.8559, "lr": 4.388288169168121e-06, "epoch": 0.25190474851696704, "percentage": 25.2, "elapsed_time": "4:26:43", "remaining_time": "13:11:34"} +{"current_steps": 290, "total_steps": 1111, "loss": 0.8487, "lr": 4.339699652920407e-06, "epoch": 0.2609013466782873, "percentage": 26.1, "elapsed_time": "4:36:12", "remaining_time": "13:01:56"} +{"current_steps": 300, "total_steps": 1111, "loss": 0.8427, "lr": 4.28954588394019e-06, "epoch": 0.26989794483960755, "percentage": 27.0, "elapsed_time": "4:45:47", "remaining_time": "12:52:36"} +{"current_steps": 310, "total_steps": 1111, "loss": 0.8645, "lr": 4.237869534042848e-06, "epoch": 0.27889454300092775, "percentage": 27.9, "elapsed_time": "4:55:26", "remaining_time": "12:43:22"} +{"current_steps": 320, "total_steps": 1111, "loss": 0.8739, "lr": 4.184714570485619e-06, "epoch": 0.287891141162248, "percentage": 28.8, "elapsed_time": "5:04:52", "remaining_time": "12:33:35"} +{"current_steps": 330, "total_steps": 1111, "loss": 0.8388, "lr": 4.130126218559396e-06, "epoch": 0.29688773932356827, "percentage": 29.7, "elapsed_time": "5:14:23", "remaining_time": "12:24:03"} +{"current_steps": 340, "total_steps": 1111, "loss": 0.8344, "lr": 4.074150923110149e-06, "epoch": 0.3058843374848885, "percentage": 30.6, "elapsed_time": "5:23:57", "remaining_time": "12:14:37"} +{"current_steps": 350, "total_steps": 1111, "loss": 0.8489, "lr": 4.0168363090227425e-06, "epoch": 0.3148809356462088, "percentage": 31.5, "elapsed_time": "5:33:35", "remaining_time": "12:05:19"} +{"current_steps": 360, "total_steps": 1111, "loss": 0.8827, "lr": 3.958231140700742e-06, "epoch": 0.32387753380752904, "percentage": 32.4, "elapsed_time": "5:43:11", "remaining_time": "11:55:55"} +{"current_steps": 370, "total_steps": 1111, "loss": 0.8257, "lr": 3.898385280576696e-06, "epoch": 0.3328741319688493, "percentage": 33.3, "elapsed_time": "5:52:40", "remaining_time": "11:46:17"} +{"current_steps": 380, "total_steps": 1111, "loss": 0.8586, "lr": 3.8373496466881986e-06, "epoch": 0.34187073013016955, "percentage": 34.2, "elapsed_time": "6:02:11", "remaining_time": "11:36:44"} +{"current_steps": 390, "total_steps": 1111, "loss": 0.8633, "lr": 3.775176169355816e-06, "epoch": 0.35086732829148976, "percentage": 35.1, "elapsed_time": "6:11:39", "remaining_time": "11:27:05"} +{"current_steps": 400, "total_steps": 1111, "loss": 0.8385, "lr": 3.7119177469997506e-06, "epoch": 0.35986392645281, "percentage": 36.0, "elapsed_time": "6:21:02", "remaining_time": "11:17:17"} +{"current_steps": 410, "total_steps": 1111, "loss": 0.8167, "lr": 3.647628201132818e-06, "epoch": 0.36886052461413027, "percentage": 36.9, "elapsed_time": "6:30:52", "remaining_time": "11:08:18"} +{"current_steps": 420, "total_steps": 1111, "loss": 0.8331, "lr": 3.582362230568044e-06, "epoch": 0.3778571227754505, "percentage": 37.8, "elapsed_time": "6:40:24", "remaining_time": "10:58:46"} +{"current_steps": 430, "total_steps": 1111, "loss": 0.8337, "lr": 3.5161753648798367e-06, "epoch": 0.3868537209367708, "percentage": 38.7, "elapsed_time": "6:50:05", "remaining_time": "10:49:27"} +{"current_steps": 440, "total_steps": 1111, "loss": 0.8371, "lr": 3.449123917158331e-06, "epoch": 0.39585031909809104, "percentage": 39.6, "elapsed_time": "6:59:37", "remaining_time": "10:39:55"} +{"current_steps": 450, "total_steps": 1111, "loss": 0.8368, "lr": 3.3812649360970988e-06, "epoch": 0.4048469172594113, "percentage": 40.5, "elapsed_time": "7:09:12", "remaining_time": "10:30:27"} +{"current_steps": 460, "total_steps": 1111, "loss": 0.8322, "lr": 3.3126561574549975e-06, "epoch": 0.41384351542073156, "percentage": 41.4, "elapsed_time": "7:18:48", "remaining_time": "10:21:00"} +{"current_steps": 470, "total_steps": 1111, "loss": 0.8295, "lr": 3.2433559549334475e-06, "epoch": 0.4228401135820518, "percentage": 42.3, "elapsed_time": "7:28:15", "remaining_time": "10:11:20"} +{"current_steps": 480, "total_steps": 1111, "loss": 0.8343, "lr": 3.173423290510937e-06, "epoch": 0.431836711743372, "percentage": 43.2, "elapsed_time": "7:37:47", "remaining_time": "10:01:48"} +{"current_steps": 490, "total_steps": 1111, "loss": 0.8548, "lr": 3.102917664277007e-06, "epoch": 0.44083330990469227, "percentage": 44.1, "elapsed_time": "7:47:22", "remaining_time": "9:52:19"} +{"current_steps": 500, "total_steps": 1111, "loss": 0.8558, "lr": 3.0318990638084055e-06, "epoch": 0.44982990806601253, "percentage": 45.0, "elapsed_time": "7:56:45", "remaining_time": "9:42:35"} +{"current_steps": 510, "total_steps": 1111, "loss": 0.8441, "lr": 2.9604279131304685e-06, "epoch": 0.4588265062273328, "percentage": 45.9, "elapsed_time": "8:06:05", "remaining_time": "9:32:49"} +{"current_steps": 520, "total_steps": 1111, "loss": 0.8346, "lr": 2.8885650213071746e-06, "epoch": 0.46782310438865304, "percentage": 46.8, "elapsed_time": "8:15:32", "remaining_time": "9:23:11"} +{"current_steps": 530, "total_steps": 1111, "loss": 0.8363, "lr": 2.8163715307035897e-06, "epoch": 0.4768197025499733, "percentage": 47.7, "elapsed_time": "8:25:07", "remaining_time": "9:13:44"} +{"current_steps": 540, "total_steps": 1111, "loss": 0.8637, "lr": 2.743908864964741e-06, "epoch": 0.48581630071129356, "percentage": 48.6, "elapsed_time": "8:34:33", "remaining_time": "9:04:05"} +{"current_steps": 550, "total_steps": 1111, "loss": 0.8439, "lr": 2.6712386767551663e-06, "epoch": 0.4948128988726138, "percentage": 49.5, "elapsed_time": "8:44:05", "remaining_time": "8:54:34"} +{"current_steps": 560, "total_steps": 1111, "loss": 0.8485, "lr": 2.5984227953036124e-06, "epoch": 0.5038094970339341, "percentage": 50.41, "elapsed_time": "8:53:31", "remaining_time": "8:44:56"} +{"current_steps": 570, "total_steps": 1111, "loss": 0.8279, "lr": 2.52552317379751e-06, "epoch": 0.5128060951952543, "percentage": 51.31, "elapsed_time": "9:03:08", "remaining_time": "8:35:30"} +{"current_steps": 580, "total_steps": 1111, "loss": 0.8744, "lr": 2.452601836671977e-06, "epoch": 0.5218026933565746, "percentage": 52.21, "elapsed_time": "9:12:39", "remaining_time": "8:25:57"} +{"current_steps": 590, "total_steps": 1111, "loss": 0.817, "lr": 2.3797208268382096e-06, "epoch": 0.5307992915178948, "percentage": 53.11, "elapsed_time": "9:22:10", "remaining_time": "8:16:26"} +{"current_steps": 600, "total_steps": 1111, "loss": 0.8454, "lr": 2.3069421528961493e-06, "epoch": 0.5397958896792151, "percentage": 54.01, "elapsed_time": "9:31:40", "remaining_time": "8:06:52"} +{"current_steps": 610, "total_steps": 1111, "loss": 0.821, "lr": 2.2343277363763437e-06, "epoch": 0.5487924878405352, "percentage": 54.91, "elapsed_time": "9:42:43", "remaining_time": "7:58:35"} +{"current_steps": 620, "total_steps": 1111, "loss": 0.8444, "lr": 2.1619393590558857e-06, "epoch": 0.5577890860018555, "percentage": 55.81, "elapsed_time": "9:52:14", "remaining_time": "7:49:01"} +{"current_steps": 630, "total_steps": 1111, "loss": 0.8296, "lr": 2.0898386103932634e-06, "epoch": 0.5667856841631758, "percentage": 56.71, "elapsed_time": "10:01:44", "remaining_time": "7:39:25"} +{"current_steps": 640, "total_steps": 1111, "loss": 0.8425, "lr": 2.018086835126831e-06, "epoch": 0.575782282324496, "percentage": 57.61, "elapsed_time": "10:11:16", "remaining_time": "7:29:51"} +{"current_steps": 650, "total_steps": 1111, "loss": 0.8356, "lr": 1.9467450810814984e-06, "epoch": 0.5847788804858163, "percentage": 58.51, "elapsed_time": "10:20:53", "remaining_time": "7:20:21"} +{"current_steps": 660, "total_steps": 1111, "loss": 0.8123, "lr": 1.8758740472280372e-06, "epoch": 0.5937754786471365, "percentage": 59.41, "elapsed_time": "10:30:33", "remaining_time": "7:10:52"} +{"current_steps": 670, "total_steps": 1111, "loss": 0.8124, "lr": 1.8055340320392002e-06, "epoch": 0.6027720768084568, "percentage": 60.31, "elapsed_time": "10:40:13", "remaining_time": "7:01:24"} +{"current_steps": 680, "total_steps": 1111, "loss": 0.8469, "lr": 1.7357848821865914e-06, "epoch": 0.611768674969777, "percentage": 61.21, "elapsed_time": "10:49:55", "remaining_time": "6:51:56"} +{"current_steps": 690, "total_steps": 1111, "loss": 0.8171, "lr": 1.6666859416219384e-06, "epoch": 0.6207652731310973, "percentage": 62.11, "elapsed_time": "10:59:27", "remaining_time": "6:42:21"} +{"current_steps": 700, "total_steps": 1111, "loss": 0.8105, "lr": 1.5982960010860882e-06, "epoch": 0.6297618712924176, "percentage": 63.01, "elapsed_time": "11:09:01", "remaining_time": "6:32:48"} +{"current_steps": 710, "total_steps": 1111, "loss": 0.8342, "lr": 1.530673248088687e-06, "epoch": 0.6387584694537378, "percentage": 63.91, "elapsed_time": "11:18:33", "remaining_time": "6:23:14"} +{"current_steps": 720, "total_steps": 1111, "loss": 0.8715, "lr": 1.463875217401099e-06, "epoch": 0.6477550676150581, "percentage": 64.81, "elapsed_time": "11:28:05", "remaining_time": "6:13:40"} +{"current_steps": 730, "total_steps": 1111, "loss": 0.8355, "lr": 1.397958742104687e-06, "epoch": 0.6567516657763783, "percentage": 65.71, "elapsed_time": "11:37:35", "remaining_time": "6:04:04"} +{"current_steps": 740, "total_steps": 1111, "loss": 0.8148, "lr": 1.3329799052361094e-06, "epoch": 0.6657482639376986, "percentage": 66.61, "elapsed_time": "11:47:05", "remaining_time": "5:54:29"} +{"current_steps": 750, "total_steps": 1111, "loss": 0.8229, "lr": 1.2689939920707667e-06, "epoch": 0.6747448620990188, "percentage": 67.51, "elapsed_time": "11:56:45", "remaining_time": "5:45:00"} +{"current_steps": 760, "total_steps": 1111, "loss": 0.8409, "lr": 1.2060554430849951e-06, "epoch": 0.6837414602603391, "percentage": 68.41, "elapsed_time": "12:06:22", "remaining_time": "5:35:28"} +{"current_steps": 770, "total_steps": 1111, "loss": 0.8258, "lr": 1.1442178076370426e-06, "epoch": 0.6927380584216594, "percentage": 69.31, "elapsed_time": "12:15:57", "remaining_time": "5:25:55"} +{"current_steps": 780, "total_steps": 1111, "loss": 0.8162, "lr": 1.0835336984062122e-06, "epoch": 0.7017346565829795, "percentage": 70.21, "elapsed_time": "12:25:41", "remaining_time": "5:16:26"} +{"current_steps": 790, "total_steps": 1111, "loss": 0.808, "lr": 1.0240547466289646e-06, "epoch": 0.7107312547442998, "percentage": 71.11, "elapsed_time": "12:35:10", "remaining_time": "5:06:51"} +{"current_steps": 800, "total_steps": 1111, "loss": 0.8314, "lr": 9.65831558170037e-07, "epoch": 0.71972785290562, "percentage": 72.01, "elapsed_time": "12:44:58", "remaining_time": "4:57:23"} +{"current_steps": 810, "total_steps": 1111, "loss": 0.8437, "lr": 9.089136704659823e-07, "epoch": 0.7287244510669403, "percentage": 72.91, "elapsed_time": "12:54:27", "remaining_time": "4:47:47"} +{"current_steps": 820, "total_steps": 1111, "loss": 0.8052, "lr": 8.533495103777367e-07, "epoch": 0.7377210492282605, "percentage": 73.81, "elapsed_time": "13:04:01", "remaining_time": "4:38:13"} +{"current_steps": 830, "total_steps": 1111, "loss": 0.7951, "lr": 7.991863529880936e-07, "epoch": 0.7467176473895808, "percentage": 74.71, "elapsed_time": "13:13:30", "remaining_time": "4:28:38"} +{"current_steps": 840, "total_steps": 1111, "loss": 0.8461, "lr": 7.464702813791308e-07, "epoch": 0.755714245550901, "percentage": 75.61, "elapsed_time": "13:23:03", "remaining_time": "4:19:04"} +{"current_steps": 850, "total_steps": 1111, "loss": 0.8045, "lr": 6.952461474238182e-07, "epoch": 0.7647108437122213, "percentage": 76.51, "elapsed_time": "13:32:40", "remaining_time": "4:09:32"} +{"current_steps": 860, "total_steps": 1111, "loss": 0.8562, "lr": 6.455575336251549e-07, "epoch": 0.7737074418735416, "percentage": 77.41, "elapsed_time": "13:42:03", "remaining_time": "3:59:55"} +{"current_steps": 870, "total_steps": 1111, "loss": 0.8345, "lr": 5.974467160353226e-07, "epoch": 0.7827040400348618, "percentage": 78.31, "elapsed_time": "13:51:29", "remaining_time": "3:50:19"} +{"current_steps": 880, "total_steps": 1111, "loss": 0.8344, "lr": 5.509546282863806e-07, "epoch": 0.7917006381961821, "percentage": 79.21, "elapsed_time": "14:00:56", "remaining_time": "3:40:44"} +{"current_steps": 890, "total_steps": 1111, "loss": 0.8256, "lr": 5.061208267631315e-07, "epoch": 0.8006972363575023, "percentage": 80.11, "elapsed_time": "14:10:26", "remaining_time": "3:31:10"} +{"current_steps": 900, "total_steps": 1111, "loss": 0.8427, "lr": 4.629834569477626e-07, "epoch": 0.8096938345188226, "percentage": 81.01, "elapsed_time": "14:19:56", "remaining_time": "3:21:36"} +{"current_steps": 910, "total_steps": 1111, "loss": 0.8195, "lr": 4.21579220964923e-07, "epoch": 0.8186904326801429, "percentage": 81.91, "elapsed_time": "14:29:37", "remaining_time": "3:12:04"} +{"current_steps": 920, "total_steps": 1111, "loss": 0.8277, "lr": 3.8194334635482777e-07, "epoch": 0.8276870308414631, "percentage": 82.81, "elapsed_time": "14:39:09", "remaining_time": "3:02:31"} +{"current_steps": 930, "total_steps": 1111, "loss": 0.811, "lr": 3.4410955610097745e-07, "epoch": 0.8366836290027834, "percentage": 83.71, "elapsed_time": "14:48:59", "remaining_time": "2:53:01"} +{"current_steps": 940, "total_steps": 1111, "loss": 0.8187, "lr": 3.0811003993797327e-07, "epoch": 0.8456802271641036, "percentage": 84.61, "elapsed_time": "14:58:35", "remaining_time": "2:43:28"} +{"current_steps": 950, "total_steps": 1111, "loss": 0.8352, "lr": 2.7397542696386145e-07, "epoch": 0.8546768253254238, "percentage": 85.51, "elapsed_time": "15:08:06", "remaining_time": "2:33:53"} +{"current_steps": 960, "total_steps": 1111, "loss": 0.8076, "lr": 2.4173475958028855e-07, "epoch": 0.863673423486744, "percentage": 86.41, "elapsed_time": "15:17:39", "remaining_time": "2:24:20"} +{"current_steps": 970, "total_steps": 1111, "loss": 0.8357, "lr": 2.1141546878265696e-07, "epoch": 0.8726700216480643, "percentage": 87.31, "elapsed_time": "15:27:13", "remaining_time": "2:14:46"} +{"current_steps": 980, "total_steps": 1111, "loss": 0.8375, "lr": 1.8304335082129032e-07, "epoch": 0.8816666198093845, "percentage": 88.21, "elapsed_time": "15:36:42", "remaining_time": "2:05:12"} +{"current_steps": 990, "total_steps": 1111, "loss": 0.8482, "lr": 1.566425452534784e-07, "epoch": 0.8906632179707048, "percentage": 89.11, "elapsed_time": "15:46:15", "remaining_time": "1:55:39"} +{"current_steps": 1000, "total_steps": 1111, "loss": 0.8334, "lr": 1.3223551440506244e-07, "epoch": 0.8996598161320251, "percentage": 90.01, "elapsed_time": "15:55:56", "remaining_time": "1:46:06"} +{"current_steps": 1010, "total_steps": 1111, "loss": 0.8218, "lr": 1.0984302425904869e-07, "epoch": 0.9086564142933453, "percentage": 90.91, "elapsed_time": "16:05:38", "remaining_time": "1:36:33"} +{"current_steps": 1020, "total_steps": 1111, "loss": 0.8242, "lr": 8.94841267874974e-08, "epoch": 0.9176530124546656, "percentage": 91.81, "elapsed_time": "16:15:10", "remaining_time": "1:27:00"} +{"current_steps": 1030, "total_steps": 1111, "loss": 0.8276, "lr": 7.117614374173353e-08, "epoch": 0.9266496106159858, "percentage": 92.71, "elapsed_time": "16:24:46", "remaining_time": "1:17:26"} +{"current_steps": 1040, "total_steps": 1111, "loss": 0.8399, "lr": 5.493465191465458e-08, "epoch": 0.9356462087773061, "percentage": 93.61, "elapsed_time": "16:34:25", "remaining_time": "1:07:53"} +{"current_steps": 1050, "total_steps": 1111, "loss": 0.8231, "lr": 4.0773469887692154e-08, "epoch": 0.9446428069386263, "percentage": 94.51, "elapsed_time": "16:44:05", "remaining_time": "0:58:20"} +{"current_steps": 1060, "total_steps": 1111, "loss": 0.8089, "lr": 2.8704646273687298e-08, "epoch": 0.9536394050999466, "percentage": 95.41, "elapsed_time": "16:53:30", "remaining_time": "0:48:45"} +{"current_steps": 1070, "total_steps": 1111, "loss": 0.8546, "lr": 1.873844946569614e-08, "epoch": 0.9626360032612669, "percentage": 96.31, "elapsed_time": "17:02:54", "remaining_time": "0:39:11"} +{"current_steps": 1080, "total_steps": 1111, "loss": 0.8213, "lr": 1.0883358900435626e-08, "epoch": 0.9716326014225871, "percentage": 97.21, "elapsed_time": "17:12:27", "remaining_time": "0:29:38"} +{"current_steps": 1090, "total_steps": 1111, "loss": 0.8261, "lr": 5.146057843814223e-09, "epoch": 0.9806291995839074, "percentage": 98.11, "elapsed_time": "17:22:11", "remaining_time": "0:20:04"} +{"current_steps": 1100, "total_steps": 1111, "loss": 0.8431, "lr": 1.531427704675459e-09, "epoch": 0.9896257977452276, "percentage": 99.01, "elapsed_time": "17:31:46", "remaining_time": "0:10:31"} +{"current_steps": 1110, "total_steps": 1111, "loss": 0.8082, "lr": 4.25438816009649e-11, "epoch": 0.9986223959065479, "percentage": 99.91, "elapsed_time": "17:41:19", "remaining_time": "0:00:57"} +{"current_steps": 1111, "total_steps": 1111, "epoch": 0.9995220557226798, "percentage": 100.0, "elapsed_time": "17:43:58", "remaining_time": "0:00:00"} diff --git a/trainer_state.json b/trainer_state.json new file mode 100644 index 0000000..cbdc01b --- /dev/null +++ b/trainer_state.json @@ -0,0 +1,811 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.9995220557226798, + "eval_steps": 500, + "global_step": 1111, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.00899659816132025, + "grad_norm": 9.407082207126406, + "learning_rate": 1.323529411764706e-06, + "loss": 1.1112, + "step": 10 + }, + { + "epoch": 0.0179931963226405, + "grad_norm": 1.6078560053885997, + "learning_rate": 2.7941176470588237e-06, + "loss": 0.9362, + "step": 20 + }, + { + "epoch": 0.02698979448396075, + "grad_norm": 1.624171273563677, + "learning_rate": 4.264705882352942e-06, + "loss": 0.8806, + "step": 30 + }, + { + "epoch": 0.035986392645281, + "grad_norm": 1.4554360159842772, + "learning_rate": 4.9997341046993195e-06, + "loss": 0.8947, + "step": 40 + }, + { + "epoch": 0.04498299080660125, + "grad_norm": 1.4128774633791414, + "learning_rate": 4.997607281643338e-06, + "loss": 0.8764, + "step": 50 + }, + { + "epoch": 0.0539795889679215, + "grad_norm": 1.4582318570520234, + "learning_rate": 4.993355445074358e-06, + "loss": 0.8758, + "step": 60 + }, + { + "epoch": 0.06297618712924176, + "grad_norm": 1.241568669359911, + "learning_rate": 4.986982212538754e-06, + "loss": 0.8839, + "step": 70 + }, + { + "epoch": 0.071972785290562, + "grad_norm": 1.4053494360834542, + "learning_rate": 4.978493006508408e-06, + "loss": 0.8741, + "step": 80 + }, + { + "epoch": 0.08096938345188226, + "grad_norm": 1.4737910013351594, + "learning_rate": 4.967895049767168e-06, + "loss": 0.8859, + "step": 90 + }, + { + "epoch": 0.0899659816132025, + "grad_norm": 1.3138356543765093, + "learning_rate": 4.9551973592655565e-06, + "loss": 0.8506, + "step": 100 + }, + { + "epoch": 0.09896257977452276, + "grad_norm": 1.3758428201688702, + "learning_rate": 4.940410738448974e-06, + "loss": 0.9021, + "step": 110 + }, + { + "epoch": 0.107959177935843, + "grad_norm": 1.4334286224262116, + "learning_rate": 4.923547768065916e-06, + "loss": 0.8752, + "step": 120 + }, + { + "epoch": 0.11695577609716326, + "grad_norm": 1.3787955824383673, + "learning_rate": 4.904622795464018e-06, + "loss": 0.8824, + "step": 130 + }, + { + "epoch": 0.12595237425848352, + "grad_norm": 1.3627983383165039, + "learning_rate": 4.883651922383059e-06, + "loss": 0.8519, + "step": 140 + }, + { + "epoch": 0.13494897241980378, + "grad_norm": 1.3424633268190644, + "learning_rate": 4.860652991255274e-06, + "loss": 0.8565, + "step": 150 + }, + { + "epoch": 0.143945570581124, + "grad_norm": 1.4227597124829214, + "learning_rate": 4.835645570024666e-06, + "loss": 0.8884, + "step": 160 + }, + { + "epoch": 0.15294216874244426, + "grad_norm": 1.3745439024877604, + "learning_rate": 4.808650935498216e-06, + "loss": 0.8394, + "step": 170 + }, + { + "epoch": 0.16193876690376452, + "grad_norm": 1.1913982454389247, + "learning_rate": 4.779692055243149e-06, + "loss": 0.8542, + "step": 180 + }, + { + "epoch": 0.17093536506508478, + "grad_norm": 1.3183554482150048, + "learning_rate": 4.748793568045682e-06, + "loss": 0.8633, + "step": 190 + }, + { + "epoch": 0.179931963226405, + "grad_norm": 1.4397694519607327, + "learning_rate": 4.715981762947854e-06, + "loss": 0.8532, + "step": 200 + }, + { + "epoch": 0.18892856138772526, + "grad_norm": 1.2038269967391835, + "learning_rate": 4.681284556880294e-06, + "loss": 0.8544, + "step": 210 + }, + { + "epoch": 0.19792515954904552, + "grad_norm": 1.2701132880820947, + "learning_rate": 4.6447314709099436e-06, + "loss": 0.8463, + "step": 220 + }, + { + "epoch": 0.20692175771036578, + "grad_norm": 1.3356581457961003, + "learning_rate": 4.606353605122954e-06, + "loss": 0.8652, + "step": 230 + }, + { + "epoch": 0.215918355871686, + "grad_norm": 1.3140257601689191, + "learning_rate": 4.566183612164116e-06, + "loss": 0.8737, + "step": 240 + }, + { + "epoch": 0.22491495403300626, + "grad_norm": 1.2150684493886468, + "learning_rate": 4.52425566945535e-06, + "loss": 0.8448, + "step": 250 + }, + { + "epoch": 0.23391155219432652, + "grad_norm": 1.3356045800197514, + "learning_rate": 4.480605450116879e-06, + "loss": 0.8531, + "step": 260 + }, + { + "epoch": 0.24290815035564678, + "grad_norm": 1.3621827029728837, + "learning_rate": 4.435270092615835e-06, + "loss": 0.8569, + "step": 270 + }, + { + "epoch": 0.25190474851696704, + "grad_norm": 1.2958362488368202, + "learning_rate": 4.388288169168121e-06, + "loss": 0.8559, + "step": 280 + }, + { + "epoch": 0.2609013466782873, + "grad_norm": 1.315904667574477, + "learning_rate": 4.339699652920407e-06, + "loss": 0.8487, + "step": 290 + }, + { + "epoch": 0.26989794483960755, + "grad_norm": 1.192500405781859, + "learning_rate": 4.28954588394019e-06, + "loss": 0.8427, + "step": 300 + }, + { + "epoch": 0.27889454300092775, + "grad_norm": 1.3463647117462414, + "learning_rate": 4.237869534042848e-06, + "loss": 0.8645, + "step": 310 + }, + { + "epoch": 0.287891141162248, + "grad_norm": 1.302446463869839, + "learning_rate": 4.184714570485619e-06, + "loss": 0.8739, + "step": 320 + }, + { + "epoch": 0.29688773932356827, + "grad_norm": 1.2998957392856052, + "learning_rate": 4.130126218559396e-06, + "loss": 0.8388, + "step": 330 + }, + { + "epoch": 0.3058843374848885, + "grad_norm": 1.1895515530331358, + "learning_rate": 4.074150923110149e-06, + "loss": 0.8344, + "step": 340 + }, + { + "epoch": 0.3148809356462088, + "grad_norm": 1.2291112546963492, + "learning_rate": 4.0168363090227425e-06, + "loss": 0.8489, + "step": 350 + }, + { + "epoch": 0.32387753380752904, + "grad_norm": 1.2187350361306888, + "learning_rate": 3.958231140700742e-06, + "loss": 0.8827, + "step": 360 + }, + { + "epoch": 0.3328741319688493, + "grad_norm": 1.2251085843679796, + "learning_rate": 3.898385280576696e-06, + "loss": 0.8257, + "step": 370 + }, + { + "epoch": 0.34187073013016955, + "grad_norm": 1.2708814495416274, + "learning_rate": 3.8373496466881986e-06, + "loss": 0.8586, + "step": 380 + }, + { + "epoch": 0.35086732829148976, + "grad_norm": 1.3212728062854284, + "learning_rate": 3.775176169355816e-06, + "loss": 0.8633, + "step": 390 + }, + { + "epoch": 0.35986392645281, + "grad_norm": 1.3153556245818046, + "learning_rate": 3.7119177469997506e-06, + "loss": 0.8385, + "step": 400 + }, + { + "epoch": 0.36886052461413027, + "grad_norm": 1.2072741564966885, + "learning_rate": 3.647628201132818e-06, + "loss": 0.8167, + "step": 410 + }, + { + "epoch": 0.3778571227754505, + "grad_norm": 1.2379563543143086, + "learning_rate": 3.582362230568044e-06, + "loss": 0.8331, + "step": 420 + }, + { + "epoch": 0.3868537209367708, + "grad_norm": 1.2879168901856428, + "learning_rate": 3.5161753648798367e-06, + "loss": 0.8337, + "step": 430 + }, + { + "epoch": 0.39585031909809104, + "grad_norm": 1.29505065565257, + "learning_rate": 3.449123917158331e-06, + "loss": 0.8371, + "step": 440 + }, + { + "epoch": 0.4048469172594113, + "grad_norm": 1.2295083553559263, + "learning_rate": 3.3812649360970988e-06, + "loss": 0.8368, + "step": 450 + }, + { + "epoch": 0.41384351542073156, + "grad_norm": 1.3554176223589234, + "learning_rate": 3.3126561574549975e-06, + "loss": 0.8322, + "step": 460 + }, + { + "epoch": 0.4228401135820518, + "grad_norm": 1.3079414538272678, + "learning_rate": 3.2433559549334475e-06, + "loss": 0.8295, + "step": 470 + }, + { + "epoch": 0.431836711743372, + "grad_norm": 1.2635231121274326, + "learning_rate": 3.173423290510937e-06, + "loss": 0.8343, + "step": 480 + }, + { + "epoch": 0.44083330990469227, + "grad_norm": 1.3150644499613324, + "learning_rate": 3.102917664277007e-06, + "loss": 0.8548, + "step": 490 + }, + { + "epoch": 0.44982990806601253, + "grad_norm": 1.366414847286227, + "learning_rate": 3.0318990638084055e-06, + "loss": 0.8558, + "step": 500 + }, + { + "epoch": 0.4588265062273328, + "grad_norm": 1.2798875567653052, + "learning_rate": 2.9604279131304685e-06, + "loss": 0.8441, + "step": 510 + }, + { + "epoch": 0.46782310438865304, + "grad_norm": 1.2205525459157436, + "learning_rate": 2.8885650213071746e-06, + "loss": 0.8346, + "step": 520 + }, + { + "epoch": 0.4768197025499733, + "grad_norm": 1.3131391162470558, + "learning_rate": 2.8163715307035897e-06, + "loss": 0.8363, + "step": 530 + }, + { + "epoch": 0.48581630071129356, + "grad_norm": 1.2817420417534302, + "learning_rate": 2.743908864964741e-06, + "loss": 0.8637, + "step": 540 + }, + { + "epoch": 0.4948128988726138, + "grad_norm": 1.4117777334581971, + "learning_rate": 2.6712386767551663e-06, + "loss": 0.8439, + "step": 550 + }, + { + "epoch": 0.5038094970339341, + "grad_norm": 1.3316023582639032, + "learning_rate": 2.5984227953036124e-06, + "loss": 0.8485, + "step": 560 + }, + { + "epoch": 0.5128060951952543, + "grad_norm": 1.220300819321901, + "learning_rate": 2.52552317379751e-06, + "loss": 0.8279, + "step": 570 + }, + { + "epoch": 0.5218026933565746, + "grad_norm": 1.2405242839005477, + "learning_rate": 2.452601836671977e-06, + "loss": 0.8744, + "step": 580 + }, + { + "epoch": 0.5307992915178948, + "grad_norm": 1.160142751072927, + "learning_rate": 2.3797208268382096e-06, + "loss": 0.817, + "step": 590 + }, + { + "epoch": 0.5397958896792151, + "grad_norm": 1.2558513559088742, + "learning_rate": 2.3069421528961493e-06, + "loss": 0.8454, + "step": 600 + }, + { + "epoch": 0.5487924878405352, + "grad_norm": 1.2495369982031077, + "learning_rate": 2.2343277363763437e-06, + "loss": 0.821, + "step": 610 + }, + { + "epoch": 0.5577890860018555, + "grad_norm": 1.2645404439096872, + "learning_rate": 2.1619393590558857e-06, + "loss": 0.8444, + "step": 620 + }, + { + "epoch": 0.5667856841631758, + "grad_norm": 1.2701388082173177, + "learning_rate": 2.0898386103932634e-06, + "loss": 0.8296, + "step": 630 + }, + { + "epoch": 0.575782282324496, + "grad_norm": 1.2620244980098876, + "learning_rate": 2.018086835126831e-06, + "loss": 0.8425, + "step": 640 + }, + { + "epoch": 0.5847788804858163, + "grad_norm": 1.1594661341735868, + "learning_rate": 1.9467450810814984e-06, + "loss": 0.8356, + "step": 650 + }, + { + "epoch": 0.5937754786471365, + "grad_norm": 1.2189222560662132, + "learning_rate": 1.8758740472280372e-06, + "loss": 0.8123, + "step": 660 + }, + { + "epoch": 0.6027720768084568, + "grad_norm": 1.2444702547045237, + "learning_rate": 1.8055340320392002e-06, + "loss": 0.8124, + "step": 670 + }, + { + "epoch": 0.611768674969777, + "grad_norm": 1.2315293692576161, + "learning_rate": 1.7357848821865914e-06, + "loss": 0.8469, + "step": 680 + }, + { + "epoch": 0.6207652731310973, + "grad_norm": 1.2995895579660914, + "learning_rate": 1.6666859416219384e-06, + "loss": 0.8171, + "step": 690 + }, + { + "epoch": 0.6297618712924176, + "grad_norm": 1.2091823608533048, + "learning_rate": 1.5982960010860882e-06, + "loss": 0.8105, + "step": 700 + }, + { + "epoch": 0.6387584694537378, + "grad_norm": 1.2705740309640616, + "learning_rate": 1.530673248088687e-06, + "loss": 0.8342, + "step": 710 + }, + { + "epoch": 0.6477550676150581, + "grad_norm": 1.2981521802148184, + "learning_rate": 1.463875217401099e-06, + "loss": 0.8715, + "step": 720 + }, + { + "epoch": 0.6567516657763783, + "grad_norm": 1.1925093116586027, + "learning_rate": 1.397958742104687e-06, + "loss": 0.8355, + "step": 730 + }, + { + "epoch": 0.6657482639376986, + "grad_norm": 1.1985043813628813, + "learning_rate": 1.3329799052361094e-06, + "loss": 0.8148, + "step": 740 + }, + { + "epoch": 0.6747448620990188, + "grad_norm": 1.1010299588672359, + "learning_rate": 1.2689939920707667e-06, + "loss": 0.8229, + "step": 750 + }, + { + "epoch": 0.6837414602603391, + "grad_norm": 1.2430718303897152, + "learning_rate": 1.2060554430849951e-06, + "loss": 0.8409, + "step": 760 + }, + { + "epoch": 0.6927380584216594, + "grad_norm": 1.269761061806062, + "learning_rate": 1.1442178076370426e-06, + "loss": 0.8258, + "step": 770 + }, + { + "epoch": 0.7017346565829795, + "grad_norm": 1.157246567841852, + "learning_rate": 1.0835336984062122e-06, + "loss": 0.8162, + "step": 780 + }, + { + "epoch": 0.7107312547442998, + "grad_norm": 1.2172574409120116, + "learning_rate": 1.0240547466289646e-06, + "loss": 0.808, + "step": 790 + }, + { + "epoch": 0.71972785290562, + "grad_norm": 1.2870129001272486, + "learning_rate": 9.65831558170037e-07, + "loss": 0.8314, + "step": 800 + }, + { + "epoch": 0.7287244510669403, + "grad_norm": 1.2970054704276492, + "learning_rate": 9.089136704659823e-07, + "loss": 0.8437, + "step": 810 + }, + { + "epoch": 0.7377210492282605, + "grad_norm": 1.1999856118846624, + "learning_rate": 8.533495103777367e-07, + "loss": 0.8052, + "step": 820 + }, + { + "epoch": 0.7467176473895808, + "grad_norm": 1.233946813798178, + "learning_rate": 7.991863529880936e-07, + "loss": 0.7951, + "step": 830 + }, + { + "epoch": 0.755714245550901, + "grad_norm": 1.2668438696715465, + "learning_rate": 7.464702813791308e-07, + "loss": 0.8461, + "step": 840 + }, + { + "epoch": 0.7647108437122213, + "grad_norm": 1.2263597703226994, + "learning_rate": 6.952461474238182e-07, + "loss": 0.8045, + "step": 850 + }, + { + "epoch": 0.7737074418735416, + "grad_norm": 1.2325614523967854, + "learning_rate": 6.455575336251549e-07, + "loss": 0.8562, + "step": 860 + }, + { + "epoch": 0.7827040400348618, + "grad_norm": 1.243498384623771, + "learning_rate": 5.974467160353226e-07, + "loss": 0.8345, + "step": 870 + }, + { + "epoch": 0.7917006381961821, + "grad_norm": 1.158311860753632, + "learning_rate": 5.509546282863806e-07, + "loss": 0.8344, + "step": 880 + }, + { + "epoch": 0.8006972363575023, + "grad_norm": 1.2163583553306607, + "learning_rate": 5.061208267631315e-07, + "loss": 0.8256, + "step": 890 + }, + { + "epoch": 0.8096938345188226, + "grad_norm": 1.2102463242463388, + "learning_rate": 4.629834569477626e-07, + "loss": 0.8427, + "step": 900 + }, + { + "epoch": 0.8186904326801429, + "grad_norm": 1.1969681545587425, + "learning_rate": 4.21579220964923e-07, + "loss": 0.8195, + "step": 910 + }, + { + "epoch": 0.8276870308414631, + "grad_norm": 1.2573696493635405, + "learning_rate": 3.8194334635482777e-07, + "loss": 0.8277, + "step": 920 + }, + { + "epoch": 0.8366836290027834, + "grad_norm": 1.2979421497920385, + "learning_rate": 3.4410955610097745e-07, + "loss": 0.811, + "step": 930 + }, + { + "epoch": 0.8456802271641036, + "grad_norm": 1.1946033424208522, + "learning_rate": 3.0811003993797327e-07, + "loss": 0.8187, + "step": 940 + }, + { + "epoch": 0.8546768253254238, + "grad_norm": 1.1959972915578303, + "learning_rate": 2.7397542696386145e-07, + "loss": 0.8352, + "step": 950 + }, + { + "epoch": 0.863673423486744, + "grad_norm": 1.2612864041438945, + "learning_rate": 2.4173475958028855e-07, + "loss": 0.8076, + "step": 960 + }, + { + "epoch": 0.8726700216480643, + "grad_norm": 1.2393073900072233, + "learning_rate": 2.1141546878265696e-07, + "loss": 0.8357, + "step": 970 + }, + { + "epoch": 0.8816666198093845, + "grad_norm": 1.3961098525762, + "learning_rate": 1.8304335082129032e-07, + "loss": 0.8375, + "step": 980 + }, + { + "epoch": 0.8906632179707048, + "grad_norm": 1.2304674879812914, + "learning_rate": 1.566425452534784e-07, + "loss": 0.8482, + "step": 990 + }, + { + "epoch": 0.8996598161320251, + "grad_norm": 1.1275632452975384, + "learning_rate": 1.3223551440506244e-07, + "loss": 0.8334, + "step": 1000 + }, + { + "epoch": 0.9086564142933453, + "grad_norm": 1.1697390212239938, + "learning_rate": 1.0984302425904869e-07, + "loss": 0.8218, + "step": 1010 + }, + { + "epoch": 0.9176530124546656, + "grad_norm": 1.1549729040350385, + "learning_rate": 8.94841267874974e-08, + "loss": 0.8242, + "step": 1020 + }, + { + "epoch": 0.9266496106159858, + "grad_norm": 1.2550852449415948, + "learning_rate": 7.117614374173353e-08, + "loss": 0.8276, + "step": 1030 + }, + { + "epoch": 0.9356462087773061, + "grad_norm": 1.1789618223129013, + "learning_rate": 5.493465191465458e-08, + "loss": 0.8399, + "step": 1040 + }, + { + "epoch": 0.9446428069386263, + "grad_norm": 1.308555979964644, + "learning_rate": 4.0773469887692154e-08, + "loss": 0.8231, + "step": 1050 + }, + { + "epoch": 0.9536394050999466, + "grad_norm": 1.1941396262140982, + "learning_rate": 2.8704646273687298e-08, + "loss": 0.8089, + "step": 1060 + }, + { + "epoch": 0.9626360032612669, + "grad_norm": 1.2336678582691885, + "learning_rate": 1.873844946569614e-08, + "loss": 0.8546, + "step": 1070 + }, + { + "epoch": 0.9716326014225871, + "grad_norm": 1.2123027368688872, + "learning_rate": 1.0883358900435626e-08, + "loss": 0.8213, + "step": 1080 + }, + { + "epoch": 0.9806291995839074, + "grad_norm": 1.101205104982958, + "learning_rate": 5.146057843814223e-09, + "loss": 0.8261, + "step": 1090 + }, + { + "epoch": 0.9896257977452276, + "grad_norm": 1.138150004760349, + "learning_rate": 1.531427704675459e-09, + "loss": 0.8431, + "step": 1100 + }, + { + "epoch": 0.9986223959065479, + "grad_norm": 1.1694006998873614, + "learning_rate": 4.25438816009649e-11, + "loss": 0.8082, + "step": 1110 + } + ], + "logging_steps": 10, + "max_steps": 1111, + "num_input_tokens_seen": 0, + "num_train_epochs": 1, + "save_steps": 600, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 145966987345920.0, + "train_batch_size": 2, + "trial_name": null, + "trial_params": null +} diff --git a/training_args.bin b/training_args.bin new file mode 100644 index 0000000..db2eee2 --- /dev/null +++ b/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9bb25bd9bed986bab5e24d067e0cc007f63f1029d8b2d36e7cd337b9ded77ec3 +size 7736 diff --git a/training_config.yaml b/training_config.yaml new file mode 100644 index 0000000..7949996 --- /dev/null +++ b/training_config.yaml @@ -0,0 +1,27 @@ +bf16: true +cutoff_len: 4096 +dataset: tulu3_mixture_coding +ddp_timeout: 180000000 +deepspeed: examples/deepspeed/ds_z3_config.json +do_train: true +eval_strategy: 'no' +finetuning_type: full +gradient_accumulation_steps: 32 +learning_rate: 5.0e-06 +logging_steps: 10 +lr_scheduler_type: cosine +model_name_or_path: meta-llama/Llama-3.1-8B +num_train_epochs: 1.0 +output_dir: /scratch/pxm5426/runs/lora-exploration/llama-factory/Llama-3.1-8B_tulu3_mixture_coding_full_ebs128_lr5e-06 +overwrite_cache: true +overwrite_output_dir: true +per_device_train_batch_size: 2 +plot_loss: true +preprocessing_num_workers: 16 +report_to: wandb +save_steps: 600 +save_strategy: steps +stage: sft +template: tulu_v3 +trust_remote_code: true +warmup_ratio: 0.03 diff --git a/training_loss.png b/training_loss.png new file mode 100644 index 0000000000000000000000000000000000000000..6e59c6bdd305ce29e60f3b015ff0668ce0dd07e3 GIT binary patch literal 48832 zcmeFZg;!Nu^gp@}EiKZG0jNkRoq{3;sEBkUsdV>2QjxF-krEY=MhQXSAf=KD2+|>f zG$JMS=ED2^z2AG|y)oV&@b0*GT;zE6-fOM7=KRdh=d<>mE1KtMDA_4745K-(entz! zi0~MO8=)YB-$;BJ_yPZubw8`;e$C0+-OJq73ezxmcfRH1e#_2+*VD??&Cbd3q_CK% zu$U09t-HIko2-b4!~gml!cML>B5qDQ6L6J%&g%MZ7)ED~{=>ae%Cf_-YN7LIPHTHV zogMV@zt+7;zS!QXsaxS?cahC1KKd5@+e@TOjN0MPq_5PBozZ4w>PkzG6cjutjbmcM zMMd)ZEhre+{z^TcEV;R}qrTwqfZxHOkKZnBZKgZH)z59=lg5+HA2O5j5$5{wq9fmzaI@4F$NV1zRO6@taW5m|Jz&N#RVBDr&fG*rNVl9D?D;8-j+8E+Sw2wA;P)3x*82WBGc8=i#-33 z|Fm#|<;L%c_4&$$k-53S^vT|uAf+MSnGC>lK#cxoQ-uiA)Lbn#y`CC9#l)h}xlOu3_G(tA5vpbDvvMr^q{?36f6&^ReeSG+& zrCCGB=^E35c77F0b~nW;?L=Ny+^FwvwPier^_Dttu!h&Bcn$rUUhT5|{o_;j?bm_Z ze`N-Oz871mU%p(nS$RE2XLP2ZrV-{PDl65O^1=&-gzl&Cn zidl#dur?U+t20Zrpl5vc`-US|969QOLy;5MxJ@bZ`Txm!m#6w&E?RDi1EKGGa9aKFI|6p-2W&(ehi*7 zdsS-q%Id;!Y}s%iM`zaMh?<(9%%_>cFeA5J#v9*NuxUJd_KXNk!qk_{{VXix!OHmU z&9(83$+V^n)riIMSfy{i_j*i&xJE}ur-rKJN*(%jCMAUoOR3uxH-#GSv4yM+2l22+ zlimE5TiaV?xdekBOZA&C>f8NplXiAts0x=@lb#FFL#ObE9-?*ePv>R1fT1UfN>4sS zb4Xn=Ft}^i`I1juoMB>OVtu8@)TPJR%Y5SVWACk{rsUicMMXug;Gdp=`6iyA`tNUS zmTGpl3r0D9S{x*7sj-Qg;SK$zyPuzn<@TP+h)2D{K=n_oow)g zi@$%n+HsKZKiqfp=uw?o$~w_R1wTpH5#(>3hPlO)s1^KH>*oh6Hv*0wI~JFi7+GI` zMq68Zu|MkWUHbj|gLk$!>ugUEVJ|PI(em){JkQFqob4@ITv|GC+pgm7X-2=VHSP3Pia57f)l#Mn;_Q zASQFHsu8pe9T}<{L2iS@9Fn&PehXC*Q&YAdZ-`U#D01o+n3VX=K0dDN4B6tKygVn2 z^UIen-Xp=}>DtHgR{M)A$=@2BP>W{2KEAvG(~YL?$B;3@M zeJzi>%xl4fB24A|@6X4#O3TZo?{Ud7Lr_U6t>#!Z#q^fDWL|TyyE`Km@Yj83p|1F1 zq-DYJX~;~h684N+TU#UJ;~}GU5xSKdF2;eIt8;ZvW!&f)7>JyloGvAb;x;xmx>IM8 zPTIuXzkhW+fz7Xa<0r>J;N~0!4NaK#v9mCPbAew*Orne{Ju_f3;+iL>zP_MRns*&< zj7C50vbm4d@5^aQQw(5-L~#x>4d260^xWJrXvc(wg_V@LOm$==N;0S#%W~|}6 zhPlmtt78pOiKpxsR8&-W6PmwN2l%xlOVQ!qO^&=sFd@R~o0^sj4ketIV5Rn+O!10~ zi^H&fD8Gx2hy{yx#1u+*VScRJ$=@8QEFPVbk<^y1H|sgccBZ#u4TZ1BP2MzM&L`o7o9 zs!MjNz5{`#0wtrrpCg2f<^vQEr|j(kn>Qy1e-BizywJ*~XJ@xxS!>>omOVhYc$k5N zgrw`}xM}BPL}pQI+r5W5c04u}O<^j078VxM@|sg8BtkWv>nmXuHN$?kCPy5(7>{A8fq(4;A79ATe?z8{5vP>; z#kT&{)l5*rJw0?|r-%Ysp>O71qoe?lL9HdK`mF60yGSog-Pb zz1r{n>qP=!65rL?J5cP~9m`fA5Q{wL3}y!_%i4(b++1DhIXPjqAjqbMYm^kX|6F^& zIbSI#By`5xTN*tPq-|Zf#&3CsVxF^lISynYxKE!x!LsrK@W}K%x3f^Qs|Xmu#DD25 zJTrR0y(gOL;KP2>wfy*T!?wgge!1l&1s&ZvYioW;>qvDd2qag2E3|9w8yNrci|5^XV4dv0RCMGcgY5oj=4W#|oMWK9# zCrdev!A{O!k=X@IgEkx*>&Eh=Dlh#a4rQv2tjj#^QysX4_sg^8?_DP-=WBKYRXT;xSjE2-u>I@qG96#+yIC-ros~+22^qYlc*w8u0p^aE zK5??Z0iHBFJ3CR#lI%lWoy+`yjQ_?&g0$B>CoQ+U-uUqDd!OYF*!JkRdWB}VvGMWy zu*(S>YqNlD#Gpz=Hdx5-tdB=VMb(wu>{*1F9s_h#r?55<(wrbNo|P8(0ZMyIP^oj$ zbI4#85Xt2W)tieSs6<8}qwBr3-7joZjvo%(Y<*poAZ+~M$ESy4H@^wVul8En4p%Gq z`};5UTE>n-!9zQ&X5s#!bIu#Tt={kao#{4dmzysikB^}1TB#U@{R!KUcqnK$;MVtI zQn-mQ5ixNQ27ruH!IzOI;Lm;wOUo}WHIfZ0Xxq>_25wBSAixAu9io<5 z3pE`|&eDKe`}O{sw{ICqNlCGmeN6dOzv0TBf1i46Z`odgg2}|hbmiFN-nVuKU>qWf zgx5KLt7C74g@=#98WW&0YzX^4{1gh4T-VdX0*F-xTeb<+%C2usmReeB;vrTpslWe* zl8Q>qzDrPkkwA>_5j`_A8GNa7zJkhcZN35ET?{Rk>_|(}$@TT|SQnVJEeJbJo|@A3 zgXw<*h){av;NWOBAR-W%#d0frcB*t4Cp-S|Q}4b5{Cab8Uh_sey!3^YUJKTM^KF>1 z%})YpdUGco`c5%pmCmCofrc(FV#gDj^~243q3Kxa7cqJNk;rVaJ4wGTpI>#Fsq#ry`tceUFIwUXch`b;o4#4E%?}FI ziOzC3y{~mTe(ab9t`PQfZfq>wW)S?1S=brhAqtAIUZ&$ZdR&JjBp#Cz<_F%r>01t3 z>DJ9jIVmX_pCajCP%@_)PHi4R&Guo}4^XfWRtlv&S0`tpOvJe25h)>6#w{G~t>7|7 zLWSRk^l>wwI?te_xT0-9ro42i9kag9p;&tnP%9ZiNmX{Qp1H<@BTw-ddBUr^-`HA!)~2Ff2Z4rjE4Qj%Fd6S zRFtFw`?ofhb4ogg+%9QqS{s%*I+R5Foq*1v-KihC9xD9f<_Vo1l$pABN@r6hVbj8# z!*k@X${7x<6b8gfZ?eUf~-ly*RQ#&GCsc#Q{hu&++KFCvY7hc zCnZ3T)oe-5%f5L2TyM@$OY6hu8{(^vbxn%;b^s6;4-F07(${CJ(a6(zBc9YUUE=m< z>%@1z)|9IG0l#qw?#VJXYBEV#*(X-4;Q_nbYa4R|3bntttwmXL6&K34KYaMm3%I4; zJejK`dh=Iz4vXL4KXzF>?ZraLX=$c&3vF!g-o2Y$9IeZ1l4|q)bvs8tPp~qf z8j75UtU$Bu&Q{^J(YW+-nS&EgOy7<`EXmSDWn6X#}MdM!&6)s7xiPK@XNYNa69bC24DSb8|Bjp!MSFYUA_s57+0Qjf8ejN$c9RytXQbeo0i5Sfv~} zp*fRzD(yl99ftR47)x$j5fn~9lVm*pi@}4E(hm|6B92~r$*ZV1#L1>RR^dlQMO9?q zeeC`E*dEdwZiU*uV8E62QaBz8X|bk5u?@6Kaq5S+YRGBWNwBSzF>!x*WK|!ZcXi`W z75tcg{rV;0ImZR*(Z|QfdpbKSxAF2lwv%Lc?%e6NF2b#*Wm-u^804v)K8^dkwm=te zmE-a2yDS7vzR|ne@GR+th4hdn1qB8126>IJYDIo)j)|A}F}yM_;fch}D>^#iS2NW{ zm)n(^pq$QG7aTlvNX+~L$f=~tF+BCR0@K)~@g~D)nQcIWDeS*T?^T$1HMa+5T!}QP@(s4qmj^Vp9t5*~Nx2k+sUU$yO-Rc!Z$U|v&>tw3;*gmMtue$9g(-u!a+oA%oCJ9!)!0i zZvc=9^OnNiXmvSY_*{(t>USG}@OuTlk;7>OaA)J7*RXLxX}8+3#}*JcI{krAIS{&n zaM-HKgXS053OP(#p{q8j{h;64zFHXdTp*#$p}(}R6sWO`_wNVnbK}k&tex)8xgQ@N z{`ISt|8lF8r_$C9as3k9TgosZXzsdKbp;s(g@yB%_K}m3p<9)^|6*_)dcTh_U@Qui z>;0dv7bEZ7$zEED@3NO@jNyuc3jyoEqjS_`K15T9*^s7%iGrkjcW2XO`KLxsQsY3b zLBB9FaB0w4X7&GwoQvY1r^g`TVab|c4;8{bftK-*j0_t94*=J(gX! z(XXei9XgrnM+?{*nnj9z`$i*ToF~8ha_=58j5oEIbNMMT<|rC(T;PHB2nES)`@uNi zh!)q@3RVM)xftq_VB3A|>@0$8hY=v+LIwtmAU&(*{VkI zq})Dd4!^W*7{S?|d}Gvm0y6-U0zj={S|0jaoq~cn>AAu}QFv$~*cd$3>tj@?i}o6D zIT!L+^4Rg?Z)9sG6kNyZ?*JFq1bM+3lC?RkdvD227I#_Wk|lktktC$7JOU;zKfTJ>Va}jKN!|pekYv{xC{QxU#9-qJHpi(;Br z%lKmJJn@;Sl1Dp!9GCKaz9l)N@-!n86Q8UsJA#hxvRtTt*x_#@jc90SC=cA0g;bVj zLXx5x;Wz(|9B$kQ5aP8lfk03OZex9OQ}^agTU)`PS;#!FF*qwd=ZfWejgLM4JsvM4 z^O=z~TCK9DF=?j6d&%Nbnj%-w))G0&fAX8NZxFNwl<<-KFr=UcXf`?Ce$1+xw1vC} zup}K)N5T8wlne(BjLywPx3pY>@29j60To=`S9S;*-{IZi-R-oxCB;%padb^g8K8%f zNhg!zgzqUyo1t??9$2$WW?Q-E9D6t&PYf_gKu3{?iHSkAfMR!C5&8oA_9Y!f4t|X% zp@?_{)tQo9a# zVm~*xNnUO%G?_Wu(cK9rdj_=vl$xt|cl^+>AptjC6sZdB=(qwb3~F*9%Es4bpoP6o zkPq6~Vi!1!y1GOuCvJEsx|Syr%}H5Cz&Zb=aH`ps++FDH)iVML9(p15^XF?5Oatvo zg3Qc#$!IxkCGO|(*G14ZK~8j8UmW$d`f;M$z1^;*$rUQ4B3z0pLBzCl^F;~&N&ruH zXq4&Qs3??CvW^pVtr+xfZb~Fw+M{D4KIj-T=&$gxOWsNx40lOlxd>kjPe@3(&cE>I zS10PPtXmR;9|~NyfSM0H+ocpI_LbVRRYrcW)vg*D-P25wz25HybedkdGappFT5<+p z`T#aG0P|32UPrdWQGEnRUT9rLVZl3ox$D$g>gdFGZ_IQyFwoPRd8u^)cadAMJpKA& zf-rh2lb~&n+2I-V< zjeR9I^Q@;FhpPweYkvIrVc(A}hWy^%+IsrBv(%!25)}+Y-7}vs2-SaMJqld=tJ2}_ z9NpUD+-{yFz*9Ks)Rmq`J;ct6qL6=r6B1a2jVrY0yjVm`LX~-6EtBtSE=Fqs{g7T( z6Yl4SF9vzc$&wDme;WgHPD(j2v9LgG=JMX0E6cl}@IBD;$dM!0E2|{}jykVOjO{44 zrz#j)%5hc20LLtvbg3MgcUku-f>V2I>sf%v*A@f_{=`qktth|c>W9!Nu46U3J94NE z%;Cs9sjB#NV%EcnqGszil|h+fG)$U-aTAaRhI4DsMC!G6eYvnAzCapqW#0 z2Ey$(kC1N_HzM?I+&COBYE3eCx+4{y&p!3In%&5$n`#FQ#7s1=0KzJEDLBlcA%fQ`*8qbR7f^ zAHTujXk4b?^6|ip* zQ#yJY=Xas~bsqgdn&rqp;MefGpS8KTyYUT(P>X}y19WD>JrAMq!FK#OACWfUcj&|k z8j-3$4D(fMCWuR1pUR9awCm*ehsEyZPI&y-N}^foh1Mtx=PcwCPBL;09|df2yv+W1 z5-%gA3(UScU)J>p#qZz0r}znuWO`NA12jeH=>$i9TVNb8GY^GF z7mf&TFY`C<7j(f79zM+bOuz~+xYa9Q~Bj&KE*p(FDgT~3qh<+qTxSd{|hYqlR=R;-2YH3nLvTA)s$ z&8q4i=f9Ll*u{Y$N`)T0^RiMjfUZnX;E=5745%c>%3->*K^)TL7Eo)5Fhyksf_)U) z&&gpUj};&uEr3Bxly*4?>54q077wHltx5(xG=U=6vR*nw#kVIfn4%gGJ0xn^NR2=m zY^B?5n!+|z*5awodax%g+XuCUmi48A9TnU)%9=il(imb;6=y&Lu zn8Kin&JMgg086qC-AZk-5Vy~d{jeI5+iA|dd0O_S69--J3 z`mWgj`1$i2>+@49DeYXzkB5ebAC!};Y-@6>Z*PwkIsT$& zW<1;5z~+`5e#}h>kpASULif9QN9z$!GcJXfQc9 zUQej8++{rO(W6k%_C{f$&p-KeXHL=n{!f#4TilNeceH4{!MDKb{TYyfxm|#I~!TPlmzhrbUO-wO<}-i zk3(CGgg?dIEjLiR%z!NF<_40L8^93ua})?FhYvs>Mv!xRV}KDgjim@nlMyNzBh6`| z)FGjvB)5j%fB&^)ebT0_9v&+eh|Fi8v-DQ^Ne%m1*umOf$<{`>0`%6nhYv$|0@ra+ z$Lc{onK9OUks-e};0A2=uTdwGZ)Dl+b z2a}`k-MbML6{R09kx&m*L%wG!pK5p^1o-Px1>n@UPFhp%F7O0x-RFX_azJBo_sl^_ zY3aliQF9`{*%PR!b4=l88HmR9I=*L<>~K=TW_Q zPq4sfT)u1r=%)8e=DFqm^0gkA%wBMF6l-Na(ztXf*VDl`K!KW!c(dhXdm+>fpf7bM zSk1D2pp5$+B#v)QO%W@l(v^Z2+ab2e?4!mEk5`=aKZg0H+6OR z9~QZ|xXhCC_!B^Ss^}YzpBt{2SoZt-D{C&b|F_cyREF8meFyY{f`W81t`x!mZoDZj zek$BEI_29n0Rr!=el?AF(mf#etF9kC8gKR8Yse=e`tDr|Qlj8)iH@0-Sty*XjZIB& zEQ%7N@7ytm&ZgP^6njpST!m)5;?|-nv*bjn-Z!ITYQR&1-s-%*#kGCo#*N;9e6Q?zU-s)gxgLnlXs);bBVSMAi7zoFQ#*LS5H{s3 zXueI;?iB{@R@pDxW4E^aSBLx-tg7C>@5SRWPuPQ73N1+tmjOfw0=uFIb;9OUoR>#_#yh?tcTgX6WF(sD zN`;gIPVDQhcq!mQ2dW0jqo7#2{A^*XJUKLTt?7Qb$O|p~8QJOlV*%H7t*op{VkW5f z?Q_waesRRMr6-w4S%qTh3iSVp0K~z1w1kwZ{K|P(kB91qpe3HnJ*L%bX>%jNusAQx z_OgbAoZ;<8|Ejqd++nb!MQ+Lj?Z}&keA{nh|FR;atE9mUXJ zqp&l#WdO4py^1=OswL(W40H*!vi<=93Kpv(==Uw-KQCOoxSx$p-=TFi{x%W8Fz)f= z?x`q=17LD{?gEj|D=XXA-RiuL621=h1Zv{0eh$l zA5aQ>_tgb&{(#gKrkSN07^wL5>(>s*Sj>jA@A&r&dv-;@MZo5T=N3?kPo_2pXRQpUqVlpNqu!$wZN`*Y8oc<|1|xbs0W`-81$Omzr>J10}+J@ zWg(}8_68ODmL9>TRG*(ovI^F~v(QI1M0Xt=JNC~p*vi>l49F9lCBd<=ms7YQxA&^& z9n3#tOmLiv2$5POd?sFZU6X(T!$K#2wJ-4ovT;8Rs6%^TGuDB{8EhrY!bXwE`%qn74S<}f zdh7SaoYE?=Z`cCK_Wu3*4gmA-ml`=BSw{QFsB>M#ro!8-kGY8EQEcWjS5zDgrT;p{ zdXGP0tE(Fj&^>^KOaMI_Al-vH$Z=ReAoJC$$l*yq(+vnQP_YQ7!{^;GOSR-YBhWh# zz`hg$jQ(k8W1u>NISG6XYS1$|gHuC(rR$pa?@t0QzrG&XS%YWPdt*)0mZlU7J=R!G zK{b+8jV346+37rRl^LGq)fhg<$4b}{qU)^?S>`^Dm?8I1?*vxvhO18*nW*LIj3ZSM zI5SaD3o(MOX+Ri|z2iviL>?{-S*p0a21Xa9!IOdD20~;Q0di6}u;|}>ccAQzBLUg0 zZ8ZQm)ZTY)0)1spbeK0NR~rH2V`GaT^jQp6czA30BRiVK z`(K;SKrDTR%$eRZBW6ax0+aQG)F6|fwonn9?aoQ4vjl$tfZ%7B(-gNP0Vshu|KY}K znL~C-8aG(UfPnni+&o@}+=>XTLj4UdE*6Jo8=N8C8$J&jRquy!Q2xHJ!pEYjpV9RF z(hM1~T<%AZwHAv(;k518L3r_Pxb zgya!+?Gj_8Bn3#ne&-N|REDkfCE1dOZ(yz3&%qG|C>nWmZ2L=3I1ZG%Tx>c_6Dy8) zj5KFGv)Sw-d@<~r)Z=_dT1XgD<3P+IHv?*gfk^%W`YCcx2!dk`*(ITWlJR(o>;qBE zCsZK!B0rZ^b9_d5IV*KtSEU0`>Suj@Wq~w{7^?EKgtc7<6=OO;{;RYg=B`Ra7D*{8 zav?B$FN1i#hsGa>5^uP}Rq*YB=wJ>y4+RB<^IulP6oSW7)MHv32sR}6|LaY1y+CZ9 z5tiTGT1GxIFj)a18zMF2P3%;)qEi)no}Ig{SZHQi%^QRj(l8a=3$5?zA%;$jub0|4 z01;&fY>M2yhvMDDyg(ha-=~Jzyr`YVNT zKYxB$$h3MOd%MKo1X!XL|IQQ~I(qceydtn=P0%DQ0Zjq?^Z}G;8e@54p~H~8(K!>w z;k%#ZMrxIZY-YVjG%W1bTl+|ekaq{US@1YwXQ-E;%P4_;AtWbRjXeoQ#tB9h)2Xj) z00qGbdlWNUneO&n(}y3`VmR+4nE39)7Te?&%m8)lxeUR9Xz29k1U+TTezP298CugU zYF@t3xJEFtSxP)ie9OswA9stIoR*Uix&KgZ=@9{eaY8ULEl*~*eTG(cZXkO|x;QE8 z0lg&Lb1l)mqN#jB5^MSN>9n3+Vs|Mp^T6^+`?rAkCjE-wdRk&)qM98?4=E=TZwMae zdfnj1>r>IL!h7{W9=@3y`_64rC0qNq5)f9=uH3@Jg3Ofb9>JHtNTYfcdMb{Y0}GFc zI0Sr^y`iI{F!)cn6hSotnx=g^ON`<;vGRfDXBQt4Vb0-=I8yV&Is~)lxsZRj7QSQ5PY%6Ij&t5scPl6ch%h z#RhhOudG9C6ext2f2O`{EkNMJLO-?NG;rg{g9i^5;Z7q^4bA}|PoBRcL-4@85$aGJ zV6or}dElzx$gEplZiU7-_#f--v^bs*aRjiGCU7AB!y$1z6ymcQ$G_$2zj@gwX`@q~ z4CDvD=2P$E8aEk@SPJi3@iFcD!uNqL%juZQwDmpJFn~#*R5hG?&JbciC z%rLOcNL}A?B6l`OEg-!OPgFpQ0RC-c+J;JluQekevL4#mGw05Y!xR_P{}jrXBot`i0G&8zMuCGfTb7UC5Y zqyXvW)E{4kUAa7+DDWZdho%yGSOlJnZuSW0G!?)^fxF{2FwMX%;UPhjRXjNH@SHoY z>=aRy4>y1O*-B@(6NKh>RF<}l&0Lq<8E%pH5_1=4n>Es*TK@kEw9@)^G0s6lM@P;}8D3^E^Q^|GXgnd{k*HSezkSPOOR(FLoJZzMPLnU%OCS(2*DY#6E4 zm{Wc_5o4S|~-?bibkg4HiGhvwgE#4H% za}G!j)SJ{dHX3X@{{8a{33p&%;G<8J`wKD@V7t4pXNWOS!{Nq=xCK26`zn0HLS8XbjGN0Sel&f%%byMn~w&0L5`F7W631Ye)N=K z7%Cs9dFjmnpn!OY?63DzQ)8fL?6`nbs0_1O2qB6J2N1%*qm2gb5{T(|VNO6)@GQ7L zck^a;(fn{~Xa#;SZbC}=;l9QAyYyVVnhV>CTQ4xE@${5Q^XkA@qAh>-VM1@J$@P4^@ad)ZQ95Lz;7?Bx_2X&eV2SGgMnKVsa8{FDlKRTiS&r;XkPIv$Q{oXPs9z-lEfzA{#1&fOz}5JD@d2EJr4 zEkJB*3F4$KE>gVXg^wr5b=h#GG!xcD39{FJJb^^NJOP|D>0=EFBIB9BJ9k;{m93Q zgdmvZa$xgQ6n9ZW0=m;!7>kKAa*G6Rd2`CRl0eEN!jR8p9qj#u?!UASA2~7#%z7hq z4<-eZ7g}tY7f;x>=${FC+7fDT^NeejD9MI;#utt+ORbEFdLxTB>-J9zy6Caj=K?ssMkHs69XW_I zh|q2$Y0GegufMxKz+eTmODaM?4z$MogamSe5q%Ozh=%+DIQt+Xa?Zqr8$QU%6S>raZf8W+AsIFZ`_wQ_ z;{Yl?!EjW97~>&%@Ds{rT9Cte4@RuwXf;*ujjAeRs}qFd+SCvq>?C!fpzSGxN8pcd zRZg-hJFxRFTTD;v4Z-n*Su(YmTt2gz$kv|^uzMK+eueM{mmkHMl6S~r z_h^2HpE-s`TFUp z=DpoT!MB1|!D9V9Z#OZa^znhy!kuUKxPmTXG~;GAC(JZ0tjHEsN?#vMhCVPf&iXcF zj0TPQlCnn{lj@|{h0da7?1F{G@r>IjCG?Q0G5WUxZ8Caxo`CwnOr->v)nAUyzxw-({mD9FpG}2A~ zx7G}roPO5U0+xL0OAaW`q!y1p zp+J|T7lRd|2 ziamJ7w7x{|EF1D)RHU4{fz3=q7X&~xw7V3(vZvSf!$Behm;h&tLuEMQ>+2ic)NNk^ z5|55jbVHIXp0imZIOP-TUQc3mGahHAsYwfPGPe{?(g2^6@$7^A)n`t_d#nxUU5j8j z=LJO@PN=BZ*=5&_3xh$0GVJy)u-bNSCkdo@<+(wEgK7Zp0jeu6C3y3$4B9#zgq2cX z?hqnY-Wx4CnFO8|I5=|;7#ZZy*rVkW|Mw>Ao8sx~Te>%;3e^Rbv(z1pM;@r%Jb+gG zNrW}q|E1T0;|Oq`XHKyZ5IMBUb>I@<;@172-m6H@3QL(LlO)Gy*>diMjQtlOVgjhM z=Fq_C^dlV(9gOt91TJ1g!axRUbWFWQLez|PB;DcN!2}Vj85-9I!K4T7O;A+F06AkV zRwiKg{&cx7QO-*kY!jascIyvZ=9Q3OLQV)!!y^IPaw6gyn&I|^W#E*?ApGh;5J%ty z9fbn)1XpwXI;r!pqJ`wAIDWENyMsd``nwPc4{zyML1H&(#2~!x?XH1W1;aXDYSaPq z1^lUlM2Hl`3Oc|8*GF91K`>^4x?Fq==LSO2rBrz~SV3^3)Iw0WfW;v+H1yQqft)bB zxtWE2Rt0|i_WRM}Oc~lY2@?ZaftU8?Ch|lq1qSX7gK^oi(j0&1SQ$8wQr84ZT|VEF zZ?DSD@oEAGm4or86KqLWO^??nC13c*{*J|=)zjjCjPY!NyZ&iBPm?&!T!`q zu;{@1oTbO}-$tAHxiw^ev6r+Dm9g}j_n_m_(mZ;~*YZ%wUG!;c41A-}11HCUsU5pP zv3j3iX5pe(xfGwhmoIG~Uq&w(6Et-^%vs5dpB<|VK6!yX`#QSWk~>_usG01Xy9*We+EOd{3 z$4pZyZeSq+&m(^)gjfZi?4kkwIFw!vGMDAbFCwzn-$-Bj?xo}+l!SD8R63pt`(x^C zdznld-FCbYldWXr6u4u`kpDc`@L#6Jqwzfs7W*cK%`)U+br_)8IMde-q5qQBL#S1a ziM}mFWSCMC>s*~b4r_8`lyFD5oPKO{@9sxQT7R)|V3@RTRXN{(B?w(sr>6veVhJ}w z^KVfsqS39o+?YkNKX++Ju&2#hDw$G}H~zOvm3<}UJRo>3vVhU|(8UQebPtO#^eqou zPYE@(SsVB2SY_~kxaBp}q{ecUjz-xwqMy#=SnQWGXvISHpe96%m32T7s~Q6|6Z?Xt zPCG3)6s?I}JbL2WzWJmrzZid!C+Xp#_t&EO%wBA_w`6807H~C;M%&yXz^Pb`zT)1C zL}~PU^NAeG_hIw>CNA1^Ky6X*eM1*(IpK*TzP^IRUi}A}E}&I61ki-%&TC;Kyd{#L z{QQ>Y%YT~)3Q%GpP8?4uBGwnSfu#%WT^`{SOCz>KLpEXxOyPKCaO}0ci_yanzmeSj zTR?g%+x{&5f6@jn-EjOHQGUCYb9_0|>9zrFn8qRd|7{w#w4BRWu|_f2f&+`+j_jsS~9pE1|(*v+o`R<+Sbv87lnI651@H7_~hIHP##OVkAEHELwne zH3*%`Zdgj1Tb8t#$c>yh-Pi9X#`kKP8w!tgeZosaC7rlf#+1x04BO1#WBw@YKJAT) zWid~aXmoKL6Zto$w>QuPy?!ymehkZHC-1$|v{!;BGtdKc37KiO_hKI#R2U!Gxrk-G zHK)CR9wHXXI>!O+YFVLvQGMlX^~ipM~md5oG ziXm@i$UK1iqd~MmN320f+*1<7z$K~zMjJTBinItI8ejG)f_df~=pHx3+5WE1MS-7D z%75GmS0`$=-mT0&pEz*KjGY`M?YEb>_hYZS?E&SR!x?_fxip|-;mCs-SPKwQwzF2V zizo|Z$f#)tZ;zNw8)r+J5)bO|K`R9Y9nfx%-sMb_P?~afiim%}=>$ieuik0taPviw zp3Q*~r(N@zc&uA!)yxcM2;zrVq2Er2ss^4V3=c+uXD}|rYhoA}9KllR4KB4sXh3AU zng54K-lwEg^H(XGeart8vG1X%qVxL~AKu~kk$Bdg%eT6cONffo*FpNj7M$ZYg4wD+ zaH?Uuf6(U)Sz$id{qg9`jKf7u%|_6Y;4K{~v{2gK^qm8e0NgoSiu1Qc$Of_IS{I_K zBU9`zzylF^cs|-{!I_4EY6QPYo2em*s*%3H&DD!Hc z(;X@-nuZhOKYKhqE-K?ItMjz=`4H}8lnmzc;Nih1Q3sB`+){M<5B@-mfm!I!?DtN4 z9&qJy$$2hr_8xkC0o+bDE1kXTVt5+)>*o$LnweS2B<>1dl@f(m|N6lD|Ead$`SZse z``I!8hoU#O)8G)o^OrBjCK5~=K~i6W16G!G;WS92-UvXgX+ia0N+gt-F?3!A_=6DO z{S{eB@Sq!@vwr|1E*m87`>esYjg#(4TXRxKp}m zvJp35t%Y5)OhHS`=ZLC@F57NxuX?S1Y%jL1epXOx#+m=xcfm=72qOI;?X>PAh1}b? z4{K1tk-gnVgv~xfO)1~=vrtJ|B|K4#Zy>AZO>thsyZZuA%}b zFI>^ezxj3<-LYN)a`w;A=(o&;FVur%AjuQziR6T@ATfl(ZXQyin?IlJweeIN_XG-z zz>`(XzIFyZtBVX=)5x+1_hVr21SM9uV2}HKu_W4ekM}PJ7T7Vy?l!DjB;Bf-)>|UA z15qckj#8n#a>@oW>bsY$#2-hC@&9naDW|<(2@=<_B{Sl1#81^lYNJ&=#?$udmN1&H z@n*uQ(=VP*n&bv=$-|u~o9X(JWgZcsM zvF8W*=4Tmcv0fLxvQKRn_inlO<nHsx)msJDIlWb{(Z~$>XnW~_f zvqS;4R8({FpMkyW8lmgvQ}H&IWTG+M=I56lP#k|7GUv|J;TaZlekHvUUn~ zMlim%@U!Dc_vTlluVi}{;Dif2y-wBA2~8&hIG+rpK0r}dK<^yy=FZ%`F+%3|Lmc&_QCAKS9DPx9V=p&z*;_j zR0jDL+q0k}&odHO(eVv1uXZM8IqI-V36c_H)#73%Y~Q)D#a}VUKVGvl>f=TO>$(j0 z2No9mg>X6#_*--q9m9ZyXJUH$I`j>RKQ=WMj+K|4J5|xJu9|mpZ2M8vKY>|gI^!xa z0SObx*a(^xIwE|JTY()zngcjsC4Ocn!`bCT4;M0e(b)6;oB5h%tY+g&mpCi0&LUkM z;vxnH0J5nJK!FG_Wo>~dNi*Op{_JtQj7_^CGBJUT8)MB}#K`MlK$kJ>wfFX^IjqyJG)KTVS%Bkqz zJ6a89d(f|xLG=e11#%rIQWhocV1gkHH_r_O31epGa5BEq1Lwxg#YjhxXD(FFb91?g z0&FZ-jRphmqoc_397ZUp*|}R}*<^uEa=?Ai5%8?oH~$$xL567pMdm*EYk`vZG4b8> z{umfsn!xQ_1n*#asuaYHjGsuu0#-n8t|$ANzzrLlWu^G<7rZ`@;`0qC+wPkb;ilO0 zVCR3JBhN0BLQilM7V44uVIUvLF{C!5R}bN^;NW0?>kc-soYh*!DulD0v`zaF&}Ugiwx}AW)l%sJ(T=Y=fHLJJQiuM#OQ{= z(E--tcYUj%3YUvrM(?y31MyDP!r~xn_-P0+apAC#12MjQ!P zKkb-f_e%#z6&l3Dd(lJv+L{Y z>2P3ywtakIAr`!Uqi{--b}b1A9XQai`|$q#(Lc-OOhi)pRK5FyME1?k*|p6FRyQxL z>`@VZwy^Ku8|Xz1=oB7Y)B>zw=;Z|+5XPWlwPsuXL!$@eJZDGBk}mPX0X||tsX0qU zkkdf!6J@2|dy^ClKyIK6&hP;*rh4W~Xk7sKzaW9+20po686RBqXFZ(+_{~+LqlDDs z8~h~axOCu_G@P&6&%+Z7YRtD^<4C+ms^hnO<0we%Bk28{2eTvY{vxfV=|3Z7sOb%*B(?J3f{-(OK$sP)M9X>_kVa4)9;X zUlDX~J~B<>I0S`0@Amt}MF$rQf7Vc>V(^|7+8Ww+U1E4R>dpfv_98(U4>Y`Tg&w`i zf|^y7@M|2sdmT4)8Vw$@QXKIH&`9OiWjoHY|`Ch(&uZ zUt3scU-*Azwg&zla5B67nWAIr=jrZ~Wo5PPP2$dqhGD~(t;bTi+gktZ=>VPfT_t#8 zc#j6W{ov2S@UZVB6!0h<6pwo%P7Hfyrc3)hcZFvhnEh4$W9e+g~&E^gdOSK?8{xPcLm&W zZr0S1s?SxzgL@T(0$&gO3lGKvXJl)^EcF%DOA?~B zG?EYZ?}T}ewyk3L3Auu2iQg#Dcm^~xV{m`K)e5eQV0f#_7`&Pa!^2xL(%}$5v`?RP zgeg_;frHA}c;z|s?x9mgA9Y6Gr06a+_QvEpGQhr75jy3KNrOj-J*dWQQgdfFK}i2= z2Xcl9n_bqo0tC1C`QvKhPV4?$xu-Fc3%2AucA0@a2*;~)omw~E0Bu-vuF1LCFwVe7 zg-h8GP+Gc2p_CR?SFgJ`)wRE+;C)3S)yl3VR&J)|v16UDh7Hw#GBuS<3dL2tY*&m=aZQNCjSoZitx*>vcaybu4FaIy~k^VS7)jSH(!^H_ zpp5v+lL!WMv>01>ZRcZsg45bCzW6q!D#FqEFfS1!;`mffj0j-LhYsNu@)rig)pE{* z-K}Wi3=YG*lCWUY3hoRP=XhUzCJL;QqM48liP~c{-^#W^I zTZ#Kw#U8M~218R-KdXYJD(nkA#6Y?DBM#h{la=|S7aX(yyMhq9LQ%)E4bJfYVC*f! zs_NQz@44viSd@fBe$@%R7b`j3kbkcQbaaRhpi*$M&9kOq9)2{w?zv!*M8#RyxZxI!F>ct#zlJZ!k7 z+bR)}NXPy^pV~#=lXP`DxCuc6d`Wr^x5WxGdjgJdIxXUU=w^Lm#3D3m3+>cT}R1p5XJ7#_UHahXV_$; z%cXk4|K2IT5`zaKUHdH>617kC00S@lj#*@ui(X4q^ZEhOTv>{dEl6tv%udVwISIMR_R+ZY#gLoXgh@PtBCLpM%P86Kc- zvl`suX4{8=+KZ&0|Icr@0{2wjxDzfnVjW_P35k5@SE{Y*{O`6(rv(=n@hPno*EI11 zNYp54@jzqmBJjP(e}Y{6GtdwqouJ?das`Hzl53KRcaZyM7p9-y54-&*HpD**L4V*a zm~&?P*YkfUFlBmr{rQX5bGw-Tv;Kae=^GR1#ArGW5@KLBvm3bwisE4tiQF3P z&X+7}YU=mnO8#G#1S7tbm+E3}^$; zJ{ah+5rEnX&IA^Mz+}vZXzMVN1#~6=m<&%xkTFg0Vf~qRxTY|H^xngUgd6dY3U)&gEg^`)?oA#q-3xbK#&L4JOHL z4@X<;^|$Go)ee0MuG1Alk>1l3^d$3dgDlP*Ju2dxm$CCiL45n+}{z74X7 z8~+5ZfVG>>nHf_1PnY!nPhPq-A&~dKbJ6ss5?n_*RO3g6RG|td#IXyCu9ysF;{fuD|2j)#?IU+`T~AG`nioG zwg&Et3ebZr0?jkPd9y*G3?%j|cin)`r%BA8Z+7?5Kz~AdIsv$4=tcmr*kA&kkW87+ zP43mVTtOM$ocEFA9FDhV#NjJ01qsaNarcElV~o+jMQ9zM1JDBFGpIuC9_dbneXji`sztwhle zXdMk6VncGJ!XKOQIi9*w%88Zz#@INvqekVCdSZ4#OUxR50}F-!&EZzTfbAFoP>Ml? zu~{|Zo0~A37>m?2hCoh=^e+x&T-es5{UhWrtTjNXei>-i{aN=3pEEglRI35#AFK__ zuE7BXDPx#sg&=Nis8|UmVM5DgK0fF;1kh8^cpq{^ia-5{o@H=x!&7uP8o-yg=l|{V*4up4F#)vxuQOZAh_72%Kp5Yyr zSIVerZ6$clPR@Or^hG&~yW&^CX50*wh`Zn2Xers4L<|G3%o!|f04Wu;>u_4{GHMon!8^bV{9Z~LH6G%#eS!n2x1Co)lzig0Rq zcYXR#v`M*o)^f%{jdOp5-V9va@Ykiq1BZ?WI9hm+-@R193<&Y%Q>VS7h2{V#+h%n9 zDnwh;#&L`D&lLLbWI;P_CdP_qF@VlGWv@rkg4up7l2dcf@iPUw9%@8))aT(~&8RwR z6UH;#^x>Zt`ZF$H@KE2}Ogr{=q-pQ~qMcf^mKgF{KN}@X7-qJHjDV|B4Qy{_^T!fay>4TnmS z3OVG6l))s%eOx!7pt2QIhah(9k(lx?yY|k}0x03g;}mPPC4+N!dwz6^{NiEgj7J_c z`pMZ?a=<|UI?%=?GzAVxHx|FnB#!_MTAIuUh2%TIWK!%@>OL6UMWO~6TaXrj~=~#fZNYXhKP>>N)o4GI=O6n z4uojXoqN4YMXH{f-;-c$KGC0VVQ%%G|JFV~TrtPnrvOtT04y!;7d&|9D|mb zu3`~D89CVl)bdqoT~zgqX!~;oT-~b zFTUtm*{)068?I|KMu*T6lIIiH-EFzzr03Q9D zBGx^VoGa#55do$iy|mvaHt=hT_Oi&fLw3S8elHb3;-N{}sfO$Db^=Ku^*pPcKP0TJ z8zol7a{Sjf4Tp%4M;HeQZvf_|T_&X9faowWUhAR{n?i;fu`H|4yJBJB#F0K~A3M!C zN364R|oiz#YEW`H|z>K zb&uY)G2WJZog1hyIrP+}(1T8nPwx0F!9`_QSt@=>$=F(4;*%MF_N5h>y-vSMdt;ZY zLnYbcSVDAz9i$cT64Q-XOi@Vt8IGZOj zU0b~52Yp^WO7VoVT<~a~L%o`%u~$9VR$V0lM$#>`5-Jz+V6by!HjUWo$}~DW%k$-B z@2lbgofZs8ZQ!J4*DTX1TN8#{Tb&x8Vwm^QAp_lb1sDqqd8?~W^Lp6`_)iL1Gz`5L zT1W%{r&W$POEvFFXW1d;P&qOPnjxT}M& zUwyO=Wge);@%#Hn|4v{P!(}=3_d;TEF5m>QHj^jXmm^^O2EK}$Dj!X-B}C~kvI>i}g-LeCVS18{xd!E!vH{4MZwI6Gy$Qh} zwyavsi0v}^poYxO%8Rt1^VN=95nBE6=6bRBESS*QO7?`9PYHK}zEBizh_kI5j3wn| z9uabFr795hO4G>R?pN4)KmX*a@rOAewhp}yMA?yw7%iqQyi^tp=RyAx%h@$QBqM@!nQmPY-9-dBs)f+tSp>#VTu6S|F+^e z-|*?w5!o<;8hTsA6BSwd3fVCU9iIt4R?dMc4vbnwhA$)x^>7+bzN%6x%mW(zld>Og z`#@R)s-ek%r^Ey@+UA5xnk!u|53HdK1ZHY_dH4u2xLhODAZh2ehgs#x3OD82!hu=E z?}Hy?v_HeZ6q>YDBJ}Zf!>+5@PapzDK4JN8NYD^U*tp|OT<^RC@q0YH6%u(;;1w&gS0G|(UKW!_W`rX~l?^x+`#ekBPBuZ|Wfh{?gOsElD=ObPvAjBB~*Vq1A*%=#7ttk{GA z0R4b_B3~@JqNI#pl|((pFf95+HVu!pb;6#~!@d*yo4L->PllER5NColOwW9SKRq*m zA-{}3=~;J0Nm7f6UoVVNIIo2vhtU82r|AS59v2(O zxd>MJiDuR*VMVG!E#osKA(VzySHT}$KGVr1M%K_rnUo1>L8_`kU9G`_8 zAI#;mr|8e=ick#4GJG^NSgXv9E_FSSs?i4 zm|+)5V$z#X_&i_e%^9FzLodIQ|Ke9nu?vLcD&nYg|5kTaGW4A9J>{5U`r6*}UI*kk zk3;9443HLV7p)Eaa-ANYCCYADZfB;ja zS?8Zde){2zeI%@p-TClWwZRQFwYFWK{eXjJe}|M1n@B0CCs(+}6O;Kde+Pe`vq6U5 zO3B9on;(wT`|Edu@5}#+`snV#z(%FdUGd`Ku>Aa3w^m8!I4*eg0*1!*?9g8<<9T4D zm~$;hidzz1W-)eu70;>py>{@s4ySY3`?)94Sutb%vo`Kl5#uPNs>{=ibu9cJR$JS? zCC-X?ZWidp<^~HJ$131b&lNMljMuHh`FxqOHl%@!S&ZOL>dD|=e>YVq&FD-# z&BD9`#XF3az6*9UcxVIlCpp84%3g+zpgQ&x_AkF-j!ASgfaLbKNhPz%=u<;U5+A28 zY6EWIs~S~YFscdvHO6cQs&Z9+F5E#3@qxeC6CYIdEd!j?g~NSgL~Hg=onjZ-Yi%O7 zzcjP5cKZWH)cy{YUg<~&HuniJ{MGPXiY( z^A)zQ`WSC7GYB5+)V|I8NB8EBdIA_9^Ope$QbE{~6 zmb1m91Owv4Y#;}?&-@N{r>5vbPO%BZNs~@U>A>v(Q&Xa_rPVShMF2=Q;G zhcu@`{MT-!ip_37$}~;*eQTTx#eZ*2HEbjTbK0>d|r9p)%93lFo7cVA}D*Nr$%&4>FUk16JlE&_P!n<&8xsQBcn3@KPS zMz>Qp#3TyqHsK15v;mcEPQ7UynZ@Qd%$*(iiOup06J~*EIGN5%1$3nf}fQ+$3&4?+ymKh52n9W#fhf~nErHMX>O8j@pjCVCDhdB z+@mV?>eqfAUp&E5u16S>(6j8_;fFKxBrnPAGC)uOj#Du9T+6h~LjwAltdjT!TOYeF zV-_w~c4>I$DGN=y&>;J|Z+>h13wm&yLJGr>kkYwIU48J@r}d9M5`{@NI3;DXvLv!) zk8Il`qphBD3xlskWJhW0WTEl}rF%4qNHsct6UUl=6pa!hZX-8+Us`Z)w9bsDfk;V0Z^f!8SpZzu1M%Ny{#$A9K`ZRC<0U zOzn!%uO9mDE-H|VZ?KSY8vLM)2_+s?|3ZfXECTF743DX-&T9v3aP5^$P1@0IBOA*~ zzW7yY_x-%zWDuLv&J!V-(;@;%zuRyi46{u+Xb!tr(@kg*)02ZO`SlY@!h2LzU?FWIqXi93rpf{d2* zzMbz{oV+vLA@fj1m=wwn%ragDuo0@1(XJV{(yL6oDLGwtKCQ*h<~ zT`CK|AD@3@4k{Gg#{t%X7i$uho+++d(Hv6OdjjtVtJ*Lg>VD-ZgrLxihvQ&Wf>R?L z=+>`ozs>09Cf`!7(M<{~7}P*IdpGLcv8zp#t3P8By{_ZBy>NvgH)xM}Qi%U>R|ur$ zBfN^r+SplMcYpU-W3&V4?KUt51AGi{6zTznhYs^Mz`{fx3~g9b2LIOjQm{vRT>nG?>+y(^9z$Xz;jxsdpA;X~0;LnJ6O4u2LAK zoh&0h^EHZvIXc@Qd8U(VuUtHMaGWmA^mY2^R=NcS^P~zx<6r%M3=h%P9B6Sjn!rOd z>43UJ8{(tcx02Hxpuh*yuZt{T+9J~P?LgJEU2B>iXaU_q-8#&-s0v-{%WaYq2F=%t zhB{tl*UhZ7-+QyLu+3Oqi?~-z@?Ncnj0*&Fij$_ ze`|sRY7_-N5!=6v(bQ^0XCZob0v>6XIp(?a1>h|10- z_2Aah5Ww1|xVXGlm4!CeSJei*{7y%g2Cru%fn3)#M<~rO0Fi$FH-zc-KSrEDl=%KO zqYa7c=u@uZB)CADQNUzO`ZGcg`8uZ-8IyU<*&okLv${9vF=B;KC>v4NNlh(QT7dXm zS~*VRRq*Uze&v~OeL=f8=SlKXm`7+j{>|iRxxPlWIOTgYofS{$^EkuBL5QhA`XC04 z8%9^ z0~DaAt;xYVY}KW+n>;{Tbg=Ld!6Q&TCCH0fS0X#YRe59s!OayRySOZkA=@`2wuR@D zeD(v^ZhO>(!+UxTF{<2JU?$jM!lhIaQs$uUq~O2IOZ`#7wli;_Jt22u%hGCCh{vJv zgZ`70^fqZjO^iyAQty8FIc-tvVg6gVmI;q}mAl7Q*LQhz7E=Ipt-!n|TPuNFnU=8>zi#n_~GDORma+`vvWYE}fpo9Nl z+e^+%d%wqZMwMKiIfArLUYIXKCc!=lpq#(%VYdI@U`(_@2imF(484I>9XN;z1^gR~ zz0kf9wBUqFw8a7X*Z|-|VmX#A4-8srUTHvRjvCmB+)z?denUK-!1RngRJ;jgsU|6F z=vLF*){*ITXHzXyCIuxb3^QNd%V=9K4ZB%eZzPVk_&Ek%U!5Rf!NisuYu{C?D%h0m zk&po=5-t4(Ywr6NZy%kwIjoz4_S57$uW>q!F_Z5OU+&Z-X0Gpcvq1LVO0cMF@d|9TE_I>u7mB>G_mS^i%%D)JB@A)uRdvy3WCN4~v63Y#?OO6d* zYb>+Xl+FDl8}vfueFXhMoZCCD&G~nWDvb6*FZolBot}N~RS) zF=koBuUdLAU!=jLYfG041dKcd)%`L!TL98EZO~o5*)(>jxG4yXQpl5m>5t zZZc3T;bAan{R19LmyZ4Fl;Un@y6b-_*4AZD@K=b`%-znxs`Ps~+nLks;FpHFugw1hv$X2{R;4dp+AA-WJ-)U>F; zkIt`w9bEA>Cht3lsThD1KnmjIvvX$86(cZ_vY&ReSkPP6Zf~Go~;_mVDm(%>j z58EIWRKdOn>{ngE;a9-u>Au$B*V6by>)7ZktoX**n=e$`+=_NKXKtvv?m}nNGD0|l zR<+hJa4hYpw$!xdroZ0s_VQwV$V3on%BwPdegyzO&RvV{KWZr?F8o%WF(YDU}`OK{tj@{&JbZk!Ej2QUo!C|a~d89GS>l7C=t>4F3T7e5vNEc07||B8r-c>tVI0a!%} zuo`gqRnzXvw_K*DT#xg@lwE>V+xMTO{@tD$UGO}gip&C$%o_D{XAuI zX}u^)I(9!^B^6;vyrvYl1SD?{dCA{w!&qeHJM1!H)w~$P0*S-X}mT7FP!s@hf867{jAKb1<}aRT7_bc`DZp#@Bg*1Pc(~ z|F4{Par6?P+em=20^s|%g@h>4=M#VYDA& zoljr0mLTOCGJN56tCLZ7=c)369Om@edV`s`$mCU|29Ta77e`Y|7{h@k#UKP^~B+=Z~C z+>Qc8(hEOVjCbq4S8iOCJ0!NtezgRF9|k(J@~hC&sc(OKtQ6wxn#`J$vA@KDw)9b< zqoZ`<@F+pu$2s>eY!%Q~%<`>=`y7 z-cL3n#x8J*E^FfYMt^R}4VplJH^8=O#G&N!?6rrnOxIZRU*gWtVHT8%gMA+$NG`Z?B_?Nkuem#Q zj-N#LvS$U57dx?^*6P;M>eCIwMX$wPlj`tUn(API{zR^eAB)AG0|oDU=*syp)9_+_ z8SF z`dn*xH4f#QK`*g@TsXKH|9)WQD%LaKVS~vmy`*m9+sp+h@{Lmy7T^TiTx}(X;AFrm z7CaWCt1}zZ0p1=khA5Ig3*_Ih1O*>_W~y<-4*Xe=Q5w3dGq6L|j1pSqkd`9@)Yxaw z0vtZj^9;~?u%k4-tRUhe8)ulcPeZg+a;YZ2!P-B2%K|z#Zx0)>Q5=+L!@l3Nm7l#^ z+`kUFhEcg!;yuT<|5)O4E(er1>U zu%)4A4$N+h!2?hhPYuh=c6Nt*)YR`hYzt_ffcI;K{WRDW@o&N+4n=8G zpFN=g9&QR}1~4)_6kyWjz|e>CRgV$22k0@9o<;Ql*ISa$Gzx4K?Ivsrg;OZ0B%GId z#!|;8tn=Tb366R_<$wnQxKTrI38)Ia5S1J2*N!ePs3mS+%VdSP%F_L7cjiP|r#=+xt-i*3v@`{-XBS!Iy(B(TZF8J=eT5?p|t8^{!e}9k*OD|aRh54`Sy^3 zy(v8eSi+nlgxnn@8PhU$$k(eYVCvM=rAi-V({RVRVQ@|my)!xmkG%MEpLA&kW5}&0 z^SD%422@E6+?O%4_LFGWX8pW7LQ;2P|MUjV+YX=y{ z>foej^l{fouzP;vl6cS}1z1I(x2fVtmM!G|AiKCMss#|ZD0L1RD^s029~kH83G2-7 zPrs|i>&1!|m#hOf5>3y85g}M*@9d8`?@cQ@Amlk{kn6woJYx)4MEVK4BtvJ3kxow* zU4?xUzm?ye_OfMw29y*KL`bXQK96T_;}`8ZYrlvi;mc*8`b`34ZKw=2on5^R3TwmJ zzt%6g1uD)DT{V8k_`Vh6r&Z*BKxD;tNlVT0TImi73!_y-$Gi|{99C0qrI!eLQ##rw z>AAJ_kQV4tDn7Emq|njH*=(a~@J*Fhmj@(+`bNo{i)U*^zb$;UYUm?QaM{}Kn-fnf z-zjMfH~mGI0|}3GCq{;W=F1qphSqbUD~Sfphpe>LFJ2=+PaU{%RiwjA6SpAopA;~F zj6Y4*+h4a7o7bE(-1dX|Z9X*Etlk&lehTS#WZSUX|2BPJ&%%6lF(d~}zCytyHy##i z%8vtIHvT|Q>!B}}m8SqY{n$2B>?Qy&nur>>5ERY&(-^*~Qr)#{>16*NZe#LFvs74N zMgJAlTTbhABtwn|@9*B&0kdf_YKW%L%MYPdFtu=Kw3yMOzb2PC0&SNHp_1#fGhY$^ zTi-Z))i7dLXR`RgFN#94c92tqJuz~<4{kCwPh{AyRdi3WY+Y^}h+N&qCjgUrju< zx}av<#eMWw}A$|dg(37F{0#*0qT?J3X0fI>-2LnSU%oFUyN&u z^YBlS6gZ(b-+fy>Bdbi~2r}^%VD_ak{RvRsHfs5KiW1~Jo3Km{9$AUv%isHzv}oL{o? zciSmo(43qeuC)=b)!`61Q>t(r{*&QXOvK^^6U7ER(2v99beCY>v~rE;Yy6E$H$(S9 zR~hvlpPTT%1NK)mBziHJJcWul6wDRz6b6wqB;^pU+p5BmcDB`!=L92 z8b`$){XKr7GCqD_C#CR4AEp3ZA$pDofv*wfW_G*spzs>48hiE-`}aR9Ss?eds@cXH ztnxGDtHXGdMoB5p#nXE;%iv=&Jt;>psV{oMRx?s~lFqb(5uYl@FN3)=WF#=ihg7kK zcr8X7n{NQX^B`$7p-Dt@mKcaembXsWuG|DE#npJ>I1!|^ABOHw5?5e?J2JBHz$(Uk00txr9Tr#f=GLpJi z*n5UXzQA@O!VK6&1{BslO3W79U}qhcSMfwKiV_TJ=X{$?kE|{^!qpp?k zGf@Lxz@PU|5D4^qo5{jPkWrbC+#T1euyv)zmtIVK4Y#&<<)`h!7AzSo3FzJoo`6hg zyM{wl^z`ju^G1sOcuu&!uCU1TqzPNFvBUj1^%~uUyzFXw|6?%m1;vd4$#V>F*v2Y4 zv>mQ>*1N2ZXvD5K-Td?Q6&__U>1H9Zz4JS)GDEL0(rLX z_lddyBxd6RRU1kA^?W;ju~6FA?g5O`$wIRRqulq5wKvx;UPXd=v;At!5XbGWX4WB* zVPCre*m+>qW7(>M2S55V;I*O9fn%+IR-0u?lz;-TvF#-2F9~wH)Fu$5=^L7#8C-)> z?p7IX5n8?pa@Z`{8O*lK|IABW1#_{iWfnqY4&=pLOCeut{huD743{=M6bLlEd&h^s zL#()U-ikW@-!#Gg?&G_>jkU$Gb!YalaQ|vTwC)Ws1wV!-+!xxe-32h3*;R){r*n%R zL#g^!<|n;?&bCOKd9STA{eit|kF9pW!lB?6gk$s+8hysaiiG1~ZSa$7i#Tn5u&#&; zy=i(L0acjVwr_l!-A{>fv(4YPZxc;7GVK>BcVYx%4C_x8F&!`JvxsQFs56yi>oV|N zrUR@2AbFFGElzG}`h2KjUXLNK8=Om3ZLMeK*NNcy0Pc)pmDRWUPz5ZY=#K;ddCRxI zJr}#?&VdPpAs4gN0|YG#Zs9n&krqNm`j&FaZsLIF1k>DOb-b!{6yTeI)n2ZxBNS^c zYAfJx^U*%>z8HNAxj2^xnO@34AY-^C8m<7oMi2@wkdC6uf5$8${zx z&%cX-A@!VvZ@1||G158M_P%$q7<S7kcQsxQJONN{9t!}J-HKq! z8&Ddu%&n_W5?bCO!X~Piw59>~%!AWz?N2Tt%nEOHZvTfu9mc_|E&4PCmdE1v*Fl{P-W13RyT$NXJ9VO96e~2k&sswO1@RMZ$a%r0%w3}Ne66fev^Qo)nq1c{M7 zr$-H{yqs%72@2uzfGJr>IRa!m0ajw8g~qPyQ?_8H0yWs;=Y>_jv}kf z4S&>hd7BR?0-udyxlN^Y9WB5JbU20Vn28y& zAh(65zTRq3!b>yKN}41ybV&pS#iCv42y(UW<^YzhL6jX#dUXzs4N5>*?>LKg|B{>} z%`si(#T#Oy#zktfu`zIbe4>Dx4qzr$-KBq0c+d!rsYFXkhOgCVM;f*6&|izCg#pRC z)sF_oX7SqFAk1-ceY4!mlLB&KdF=+0$d%S}dwc%k=zO{MeJA(~2i%AKfz@8n&Wtx7 zZN8uyAbKK&(x#pW1vbDHU?B=MB*41q{J>QrKmk+PoT3*G)7PbDxxt6y5o%r?XTMfa)(0Qz%^kfl8l=0vAHxX1d zlQh~1bAOEwPEL7>sS^yMz*+4MJBJ1)rK3C8CKh7`2c4Ai%dm!}aaUsep8=U^L;f3Q z*KY2Qo^jZ|#n%wT|03&9`F+Sz>S(ZicX!FV}7g%t(mH}qWC{4I|i`Ia6~SA-dK58$9CUh$kVJG*Nd-=CE( zc?i`PFU?{CKbtl>5AfL%I`a7)b?rmRK|-Gz~t18_`ZG2)(>SGr#37acHl)a z1CPsw(6E4t=lXs_F#R+6^AA6sSb1A1yKxW*UQQb&Z;B7m+w=woPNTf@Ni$kqI^o5z zM3q#nZyO3WEv{vSj!&Nr9lga>Y&7Q`rc9Rn^I7|F>lmxE7S$6F@_fQ?9R#bNiqb+4 zi1UIIH}aJw@7hKmyiZG|qpZ8q{jP8$Q>qpz`_GDi#w;O^z4S+6rinB)d(Q9eDy)qy!#%J0&C1_a`9GJSo|hq&;A4Y4 zFEK2?)bBYnqxR227MQMYMLm!YzlG}{F@`*9--{}3@7yC)?I@_fRtR5;efHWfAzGG< z=y}Q=njoc7DBL-haGIbzvKEDV3mi6L+NprqZe-jZ@yP&A9uz*!|Na+7s78C(;y2+n zUNNn2v1`O6dQ*~G(<{l!4>bFQm8F?eIe*<-fMw6)+S)1Kb6<^sS4Wm2K?wP<8J}77 z%lYOlA2XzklQ`GEC$&GahwuSV&pv|_D?)=x==xd)SUM9p!C^KBU z$a+w|Kdh>~rr}{5$91r)$LoA=xQ{pQLTq-cy-cXI0xrD#($Y!oa02OZ0e)cRB1^OP zF7C+MaH*|p)p(F-t9Z=g)9Ka12b8{4(h+~^qN#*irHtcz+N7hH@{JKKVwh27{d-QwUVMSX;nYSg5QfYn$APDk<4ta@-dW{gX7j-bKmG+?#MD@-wZv zU&LO3OOoB*yM#Xho5uYgLyz)WOXm8QhOmS;UQEvSJ?phyJl6CQF*m4)@2lN9y1Q{K zKD_SNL>LXDyT|q7pS)vHNtjwkZr3`dwT8FxnCGy44Y&s7jn``LM_(2 zsQg)e!Qq)`vWPsd*`LHQo>aKj8`#2VIZylWw?pj%|J&E+vjXp(+*W_zG|^5!s#uNO z>60&A9{;(Ie5}tsQY&$=`G(R`zvRc`nq{S2%(W6_4dANBRy|-yTj|8(tXq`SyerY@ z4}0(E3I3g_kAcz7=kM6>V}-nBOOH$R1Lp0^&|;V{`6CyzKT7ItU_HetG=xK3ZPt-5 zpWEFWX;3I3Locw#@ho&t5SD$c7_NbXAbWa_G!!;JFVZpIQMI_`(C#8D5$Yzc96h8a z4|WT7+C6weS%`ct)dn8-i`_E&o>Sf70%tF)i_6Qdzc04+2(H@vH!F}je9lcpas*fl zGAl$s+eoJ@`+gbx@?{@ol~s)Uv+dX~{3|};qe7#34Dsuoihp{9^okfuw>O<;Ueyw( znCJ-b<)6jEes=!nA?_*&p$I-G78&4Y490oqKmOZd<;}+Zh`7;1Q(`D9HRRvq-u^je zs`vz>uG=7za3MC<8;6Xnh_UBOu$8xR-qtOC@3LY1-q4-fm|F=?aH_>yI0vEwyvMT* z)SPU()s|H|SP0WFCTg(fz4%)+G0BDAjR!wYfJ5HC7?#-7Io$_#X{--qjFT{)OXRRM ze*G;U5RRx^?OWqL@30fPHg&@YoOP$SU@YyURoSt*S^DKm?c5;9yjD_Zba9{!B7FU? zn!OiWeJN`nm}7Ufc^Q0rA2U=s&6`1;T`LD`a$b_L-@TXZN{Lk*B()~zBklYLZ@elD z{`w>ZcU?t_r%KO7JKasM{B@~3BORdUw=}bM)>3gGR>obqdfkvKFvZ0@6A{=;AapD? z`F+;9gvDCpFDsiMX8oET;r0HX-1F8Q&YlQQre0%RO96XJVS93r$BxVz+}(9<+f5r@ zj0S_Ye)uC1iRDA=it`aBGb9gsMf0_OuvUkriwjcr**n>J z{B`}@KxvyUX_IA=uY+9q$@ug0ul?-}-2!{J)xRlqma{`_Qs--fm--DFR*KweU?1K$ znam<$VbQ~gf@Qzp&?JEPT`vt|4e}xi#XZ^RYSixCzCEp`LKNIh{>0wq1*S!g9SbVG z?2TH(7cYBueog<#@J(^J7gx2Ue?`MOrd$d?)EgSfZZ&p0;Kiv3v68ccajbG-JjR@N zzG%q_Axrfu^3hmjgHV%bde8dM@a~)+SK#@zwRrQ;JR0d)+aa|M^pbq>2(MY6fF$i6 z{k(7!oRDK()nuRKF~q!E)H`m>@ZqM4;#HCH!NA)>OWCbZgNwiV3iqd(Aa7w#7mwBL zO_8s$qC&s)>*wFT8RW%z`WcV9==F$~s)KONSg8K=b--hMIQ}!Rd;$m&%6p^p_iCP-$ci#is66^@n~LlioK1Xx2%r&of%QMQ+8Ka=ua zF#7eHG{hkC*3LVe z>iGZrALJ-o*0B{LRQBGAND74zvUgTyIA+O~6`7gICPnrvWRxwljO>{aey`K_{@vGo z-T&ObKW^7?xi05$-sk;(y6xH4;=OJjxpTa%qlE-;CRA%tRS?E$7 zeJV5U*z0;`oQw~s%k(#*c=Ouw+cr_k-+eK2&f&U#>%1cqGU)HB_e4g@8fuvE>Abp` znV1<Jar3mTPuWk+FRtWmWRwsYkIK^;Vg-%+!>sS!(1m>-m+x5JZ!0JAyrB$t|#feD@Xl0LzcIi|QJmM-`%kLRbF1NOCaNxi+Q z=rCx&x{%11wve6s{?IpVl(PCzqNZHDD7ATnCnB~XAF=rGyC|kI$(p)Qm95tzl*M6o z;jj(#zIIfN^X?%7K4f0;xQv`?K~4v_(vf&)0@dE~M&wi*)EEYD+g}}Z!OV*HcgLm4 zvYRmo57fDP{nZ#pg)R^3Yikla`U#0JXb}lI4olWLW#lr+dizM8EGK^JU)!1ac#Q|_ zeDsNt;Yx`^hcG2?lE!PSHO|zf4jLo#kv;8Q)#{;3Nv*#U?mgA9RE*y3c6{;Y_ns%j zNL75n*)j}O(fQMb1fGrQiGB}DWooxnYcL>!w|cMhf7jfRiTjn zyf@|EG!^J4&EoPUvS*`YpA&phHr?EL`l;-f5F@kfrJwZPR05%Xr_X%^##0UNJ;8q! zBYXGvch!t<&pxvaY)p|I^VwY_{ZGo@8c~jhvF~}mOZHeRYWbze*NHMa`lCrP>R4-u zx=Xz|MGFcAQ#ctTxR*Zpr5hH3YVdWG1ZfG2e}>~-*@O!1A9DjhTKXEgW4Xe$NY8jkviHsP?oOwn5`qbjD$J zeQnpYp!~aRxqEm(ylu;g=9BOE_auGxrSF8{UHr3ZE_rrUUXYHXQj`x}T;6@XB;Ahg z8(3>SH$2ySDdeIn=^Bt9PY;K7aMOSY+U6 zhLU|P!);PB0TOyMhALgpV-Gvo!SJ^hmQlZr`A6Ij{z9%W;+#cy!Wt-iJ8(kiz_@>j zQ>%g(2M6#x7-DIwez&KvZ6rI+S`t`k3^PM`3f_{PtEuM1a!tU74sWDTf`Ndq(bHXjk`TF>Ow& z`ZEE|AN$2k%ilO$2JKvf;p66V^qEF%!Z+KB3X-jfIqAOP z731?P;O7<0oEQmSss~#^`>X8o=(k{5`%ldZiN7GxV;3c$%z}EvhFRQQbo0dp2 z-nVevPI~@x=WvFMZ{zjTTFQ^stV2RqPBt69YRcym>JPGJ+;DP{V2+$bYaQe@*` zMgsimu?;TMWmg7|mKccwl9ym5bZxJE)OT=#tc8(m-{)CtlVKF5Ty03->a&*B_2j*m z9JMF2-lAjJ*`Sqq7ftT`nNoJ7P(KwM^xZ9k?__&sLpmt(#jg`L90>j->YgL+e2gc_ z=7qe_^eQ+*lg>HhJtwB_^*CrRnQ}ZzH)1URfiQk=rr0g3Cobjv6uYMvCSS@I#`Td$L^?u!qC#ekwS%jwlxCK>Kb|JaELwQ#l`S9KR*gg4zX}&R& zXkSMgCl89arE8R9CMr+sW{+-4rJrjZI#bi@y)uTfP+PoQaDOuia=TOhq4QcB1ZSn0 z+|fdt$x01V$n(U441y(rb2FiNs@5rU)J>^>hL6--@QYwUe^_^AjW>Xtso~VQ~hVY zZuVnE-#NaRLz`P{#ZHy17hH)Aey%$m;XTq)m{lp%45~7skGIDtUEYzqcR48FUFSJT z%+oNAvYsg2y4AI0OJV~9~z2WF_d$*7CpbVr7Ii8CC@W`bF#hR4az;#ves{6>q%XcPvt<>Qz( z7F;^u%|K5m5h1&pC_b(fjqMOM9E#9|0fR}5tM-)ye#c`DUq260^zh)ut2wcxHhZmC z_QoFt-$QtmA~+MEkimmm_o~WL{2`VWnaHw8Oj!1`_+?7Ztjy&VBlUI5Q-w{bXI-r((a-IT9}&~xb;!F-q~0t(ugk6z z!Ho)^wbkC5xbZmW3!G4(<5S1jlDgD10vC`(Q7-!huVeYA!}qU9sawdD@@JE`t}Stg z(_N&KLk8UZ9zK)&eb7LZ`SL4{CdYUHzy;9#u{rhjejhh9jsBM{S? z7P8B9_3|=(6vD^xvlEvN6hY4`cnM$Sq}F+u;2(?bb>k=;$ReH)5-#aQ)oflArD*-N zJc*m26`P3S=*?`KfjiV5T7mDpdIl$G#mJARRjZ17iSdEC4$-TViI;I4v0ccQ*PE0j zw90!0N27=Yb4|6s^rO*+*iS;E6;88mB`vVvA&TZiXmY(1M9vnD59-@YiFCV%zU&sU zZ^GR&7qWF!@uRxRiI39~qQg;OkKYWs3DX>nHI-lkwWyGY;;DhutSQ3Z)%S zsseC=!sqWwhVi@)q|hgCW5c_t--!3vsxAz4O)81@PzMo+y*`xjy`TS=^I^nvICpEiS?!QLWh%qhknu-^MUS=wThQ@ z=BL*)m-*24A%{F&8S;zgDC`+=Z8yD_$pn^;#$nIJE05Wyj-9CztLg><{DiX zJq*u!91?ON#y0X2GVA$@(Ht_!PchOtm+<{P*TPB?yFTtCDfhQwNvMl7nH~zbA)Tyu z@-%~X3K{-hF_s(IE4Fdldv#2CF1|6X*pJ1FWgHCT-93`ul0n|#9W~t4QgC1iWg(Q1 zHXJ>yQKDGtXD46Iy$wFdPW!nswRkc#xJ?L8kPC}EbTKIWsH7B8vh1vGDoC9)1Lt)E z2CTlxDReY`7H*~C_|5y!$$}8P`P}cW9)6z&g$iHA0$j1U!I5L;7f?Gd%5%@&_!Q)iz=i;|iqbX*#Xa0V~ZkA6+gENh#ZooGhkbccr zm2%H5vu;@Vh=_>tVc-ZJd90Lml-dk(KLzRd{qd=j5Hi`n6Z@!>XE6! zqNlm2dh$h9)|RaBo8x(f%Kf|s!@qOAmaiqoaVJG@0zSn z)&Y+pvA%s<>PuG_whd-G@KSnc_JELj z(*%mW0_T0#mu&0^BlG?GAok{HG9)D>h2rcP<1*dmdWSOIyGBOWj=UY}cD1E;ziVL? zSAnyk6v-GW?lKQkyQ-9? zLQwSLS#2RMI(i*=kQBrR5*`(!#K8*wAt529C89z?jW$(ZPSQZGcXn_5aYu>oE)OE7 zqH<0`Qql=l$P~IuX?I>D8U0on{p`LLm%t7jrFt_vUk9u&uOLW#03J(Tj9HMui;%xj z${>{rq({XYH-dMnjQO6wctIE!AOHD;Iq%!KG%`PH!eB?5AbvOb|EveY=zS96&M1Gj zAXkPh2;Lt1GtaR4v~dXuCIBY)SjtK48yyWb?@GWy);`RUVlnN2JVh|j6+8W)>0X)3 zg}tFxS}oo8U}JwI){eflbDLO4B!vXn(u<3Wn0_m@omCK(#BnU{3GJqe5c-oCeRJnP zTJ0a{^yWLI>}5(Hq99^V*PH)vlI)*BJ_Y;;d92w#FQ9N%MF5)_U|kE{{{r~M5@S6c z{7NO{--Zv0X8#vj!1!X}K5gtX|D%5sF3>N^_nN;QK}1rnG`VBnBU(2!eSCJoLm&&; zxHz)vhRFH)`Ub53d$(ir>hYL_goMTMn`a|AT=uxi|gBgKzKmSqX*K$3Kb~;+L^vnhX5z zA2HXnE@Ec_J30S>SpGj|i4hmevIlm8eBq!8ASFhsY<038TX*VKSdbKd+EN8x+C@o8 z2GI#eJZ>JImSU&cA6P<+fS}+bKJ>=7^b;x)5|T&5_jGhjn*#9rmUn@d+s+mxh1)PC zn{>_mrYEq!o17S#pKi!6*;YoDiGv8XL+z@FnT3Ty-nY!`>@z^Ch+(!2|7VQAt#yUF zRlB;JW?Mr`7p&~<+hRFx84J+^%ffN^p?lMeBk|rXl_WV<4D8aZ45m* zGKnO{50&Lz(SL6Y){33(4yXP7xu_QAsNrNL3D>i-`%PpSgEleg=}`-5-VPZ?^eShR z&f5TQR;@_80(1z%n54bpjt_?$EH1qa6&4a=KO71P^4y*;10+cblelvblb9psq5F>~ z?=mtl;*6wA$=+)pp8*iP0(SR%%U));!TUO8uLLbsqmGV!MVw}@Ya1GV-CLjP6|oy> z?6wd*`myet_&hPr5y&YfH*Vap8^~5!@h4!8u2)|ksU8~A*4EYS#5StG2a4V3=XsD_1&af`o+`^(9JY!s{xJ$_KcKWFqD$xIU>5gA)apsd1Yhd8nfp3GR z?M}BXhsNFo;JeTctgi2YS}ziUIE8A;C8x9>J_7U zgTr+>)<(e1j{?{ByoZjWj;SddUSs{iJVQI$bai!AEl0D=?_@iDKp3mvjMwP4GSY;m zJQ?`unKn2)d~5jA-k5E-m5@MGhv0{6tW7I!k_^&bLgOIseKbSn$Aymp!|~kdR~`k8 z={GXW#Hd$;Hs-+8`?OQFx967IZ}o37vkJqzi6U&SI#7kblKKBTZH z(D3Ru`W|!{h4`%IrZ|`YUg%MHzeM{*0VI4lxekH|ue7xEO79LpY`$R0Dvc%2iABb6 z)mcl1!$4pm>(eKLL;Bur2#(MsFL~a>Qc*`)nE+vBW5YvA-PaBGpVpTO17eDokdQE2 zc%sad0APL)__A&1dr}BedP1O`o^M5p4IWnJcpe=je$4Dkb}86>yQ>MH{ZabQ4mDS2zlc|$5HRnnbwGE5}{ ztPMIUaRGJ-urDbOa^P0C{5{yl>60*g?ol)fV|4@#Z^a_*Tf^AYBR)Rf%GQ`TIqeO z5d>bCGpX%vK?H-HTX;2AP#i+*z?rEZ8=FynfAQAq{xD;~HdtAH0#e3CEWil_-1U?d zQ0MG%IHO1HLKHF=HLOSb`jEoG8>zIWR0>C5@cCg=)x5l$RaYmy za@Z$asaI5P%PW&W4A}6n_I4$#S~`4KcuYOrCOP@LAgfq#ui2!mPokX_$d7;wAUhg^ zcG$ZQAChEvGkF~e+E?lIkBp%AK1{7E+uGU=yqg1;8qmS)P28~GcI~dFW&j|Tn6Mcs z5TEccLz@b%wmplzo?96RpFd1>?(LKi@JUOyu#nW$)Lc9~x297`n8Nud+^2kW8+@hf z0rADGPYD@HplS}Ge#@6Hs^G$J*<$I~z+!!N+*a*0r-Y7ZXJ(mCd<&iUvl}xlSfPlW z!pcf2*gY{PsY7LtA<6vRAF>G0qJOkEDq7kWo2rnaH$4_DUsEMphRW*dn1FyDUwi;9 zq7=j^$fm>PJ*Yz$TqT;eXINwjgK=SmNYPyI^WkK$%|~4cS2hlw4hUmK0+RSmt>9wD z%SG-uvudz2>^`45J#h!`!S4q7nBdFLRNB_Bx_!4(rb>a6_8y2p+B!O%$IcR|1HykM zT*!F1xQvbbj)li7EXjCycudXB%?IX8j$&<#KZam^xep{gyzoT9SYqe7B3z5`A$}UH zYgWyPEdsBZD&`bW_h-^utz@_Y=1r$&9(Y~m5F329K69@0Wh&bBXWb|-63bP4)$F!5 z@%g>|csp>&d{nBhg;qlLmXMgpS>^wr+JQ+hrDvtO;kd|^q{#G+TW#vvFu9|eDADLU!FR#;d__>lQ26ArEnQrP=KtFIfbfyYMD79>DZH)i~OH%h*4DrRt zvgjeuq=hWzXRNa2e<;lVKaxL{Rgj!$B@vy7UL0o)ik82DJ(&3K>@k)~=5UcME-nsR z?LjZ|uV24Be#R(cRVt=Ixd^AjXWFGCH)}aAz33y<1kLwREjg5_E8TtW43tN$ZV+KYV zJUTl&G2^T41zLH{X*wmtw@x2wYongE_$MbPDikvo@hL1Fgs=VdBFyNM4JjEgjGm51`&}cr6!Q{yF-9z77n;oLn4@4rlkBX*VG9OC*vMn4#3EaOF3o2<#4C01 z^CkGCR-iya?E|cq zK~j$euU_R@ShF^W%cbCtu$(k8_qEqMpA1WAA#~dKO(PMF+pLParm5$nB2TyeE6Xsxh;SIQ4>`Gf(UuGmFfL#;>1+y`@N)O11oW9)_ zZE~n)yLOEXh#VwcXJN>+kZiOM>KKtkbp!$waqdsUsn4t-7o7M#Hw`blK6M5T