From 4dcc3ca28079fadcba5107f88b1f5ffa6dd7be4c Mon Sep 17 00:00:00 2001 From: ModelHub XC Date: Sat, 11 Jul 2026 02:58:07 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=EF=BC=8C=E7=94=B1ModelHub=20XC=E7=A4=BE=E5=8C=BA=E6=8F=90?= =?UTF-8?q?=E4=BE=9B=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model: bmz123/aligner-7b Source: Original Platform --- .gitattributes | 98 ++++ README.md | 61 +++ added_tokens.json | 24 + all_results.json | 8 + chat_template.jinja | 54 +++ config.json | 58 +++ configuration.json | 1 + generation_config.json | 13 + merges.txt | 3 + model-00001-of-00004.safetensors | 3 + model-00002-of-00004.safetensors | 3 + model-00003-of-00004.safetensors | 3 + model-00004-of-00004.safetensors | 3 + model.safetensors.index.json | 347 ++++++++++++++ special_tokens_map.json | 31 ++ tokenizer.json | 3 + tokenizer_config.json | 208 +++++++++ train_results.json | 8 + trainer_log.jsonl | 104 +++++ trainer_state.json | 764 +++++++++++++++++++++++++++++++ training_args.bin | 3 + training_loss.png | Bin 0 -> 34514 bytes vocab.json | 3 + 23 files changed, 1803 insertions(+) create mode 100644 .gitattributes create mode 100644 README.md create mode 100644 added_tokens.json create mode 100644 all_results.json create mode 100644 chat_template.jinja create mode 100644 config.json create mode 100644 configuration.json create mode 100644 generation_config.json create mode 100644 merges.txt create mode 100644 model-00001-of-00004.safetensors create mode 100644 model-00002-of-00004.safetensors create mode 100644 model-00003-of-00004.safetensors create mode 100644 model-00004-of-00004.safetensors create mode 100644 model.safetensors.index.json create mode 100644 special_tokens_map.json create mode 100644 tokenizer.json create mode 100644 tokenizer_config.json create mode 100644 train_results.json create mode 100644 trainer_log.jsonl create mode 100644 trainer_state.json create mode 100644 training_args.bin create mode 100644 training_loss.png create mode 100644 vocab.json diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..79c7073 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,98 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text + + +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text + +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text + + +*.rar filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zstandard filter=lfs diff=lfs merge=lfs -text +*.tfevents* filter=lfs diff=lfs merge=lfs -text +*.db* filter=lfs diff=lfs merge=lfs -text +*.ark* filter=lfs diff=lfs merge=lfs -text +**/*ckpt*data* filter=lfs diff=lfs merge=lfs -text +**/*ckpt*.meta filter=lfs diff=lfs merge=lfs -text +**/*ckpt*.index filter=lfs diff=lfs merge=lfs -text + +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.gguf* filter=lfs diff=lfs merge=lfs -text +*.ggml filter=lfs diff=lfs merge=lfs -text +*.llamafile* filter=lfs diff=lfs merge=lfs -text +*.pt2 filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text + +checkpoint-1000/training_args.bin filter=lfs diff=lfs merge=lfs -text +checkpoint-1000/global_step1000/zero_pp_rank_3_mp_rank_00_model_states.pt filter=lfs diff=lfs merge=lfs -text +checkpoint-1000/tokenizer.json filter=lfs diff=lfs merge=lfs -text +checkpoint-1000/scheduler.pt filter=lfs diff=lfs merge=lfs -text +checkpoint-1000/global_step1000/zero_pp_rank_2_mp_rank_00_model_states.pt filter=lfs diff=lfs merge=lfs -text +model-00003-of-00004.safetensors filter=lfs diff=lfs merge=lfs -text +checkpoint-1000/rng_state_0.pth filter=lfs diff=lfs merge=lfs -text +checkpoint-1000/model-00004-of-00004.safetensors filter=lfs diff=lfs merge=lfs -text +checkpoint-1000/merges.txt filter=lfs diff=lfs merge=lfs -text +checkpoint-1000/global_step1000/zero_pp_rank_1_mp_rank_00_model_states.pt filter=lfs diff=lfs merge=lfs -text +checkpoint-1000/rng_state_1.pth filter=lfs diff=lfs merge=lfs -text +checkpoint-1000/rng_state_2.pth filter=lfs diff=lfs merge=lfs -text +checkpoint-1000/rng_state_3.pth filter=lfs diff=lfs merge=lfs -text +checkpoint-1000/global_step1000/zero_pp_rank_0_mp_rank_00_model_states.pt filter=lfs diff=lfs merge=lfs -text +checkpoint-1000/model-00002-of-00004.safetensors filter=lfs diff=lfs merge=lfs -text +checkpoint-1000/model-00001-of-00004.safetensors filter=lfs diff=lfs merge=lfs -text +checkpoint-1000/model-00003-of-00004.safetensors filter=lfs diff=lfs merge=lfs -text +checkpoint-1000/global_step1000/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt filter=lfs diff=lfs merge=lfs -text +checkpoint-1036/global_step1036/zero_pp_rank_0_mp_rank_00_model_states.pt filter=lfs diff=lfs merge=lfs -text +checkpoint-1036/merges.txt filter=lfs diff=lfs merge=lfs -text +checkpoint-1000/vocab.json filter=lfs diff=lfs merge=lfs -text +checkpoint-1036/rng_state_2.pth filter=lfs diff=lfs merge=lfs -text +checkpoint-1000/global_step1000/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt filter=lfs diff=lfs merge=lfs -text +checkpoint-1000/global_step1000/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt filter=lfs diff=lfs merge=lfs -text +checkpoint-1036/global_step1036/zero_pp_rank_3_mp_rank_00_model_states.pt filter=lfs diff=lfs merge=lfs -text +checkpoint-1036/global_step1036/zero_pp_rank_2_mp_rank_00_model_states.pt filter=lfs diff=lfs merge=lfs -text +checkpoint-1036/rng_state_3.pth filter=lfs diff=lfs merge=lfs -text +checkpoint-1036/rng_state_1.pth filter=lfs diff=lfs merge=lfs -text +checkpoint-1036/global_step1036/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt filter=lfs diff=lfs merge=lfs -text +checkpoint-1000/global_step1000/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt filter=lfs diff=lfs merge=lfs -text +checkpoint-1036/tokenizer.json filter=lfs diff=lfs merge=lfs -text +checkpoint-1036/scheduler.pt filter=lfs diff=lfs merge=lfs -text +checkpoint-1036/training_args.bin filter=lfs diff=lfs merge=lfs -text +checkpoint-1036/global_step1036/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt filter=lfs diff=lfs merge=lfs -text +checkpoint-1036/vocab.json filter=lfs diff=lfs merge=lfs -text +merges.txt filter=lfs diff=lfs merge=lfs -text +checkpoint-1036/global_step1036/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt filter=lfs diff=lfs merge=lfs -text +checkpoint-1036/model-00004-of-00004.safetensors filter=lfs diff=lfs merge=lfs -text +model-00004-of-00004.safetensors filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text +checkpoint-1036/global_step1036/zero_pp_rank_1_mp_rank_00_model_states.pt filter=lfs diff=lfs merge=lfs -text +checkpoint-1036/global_step1036/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt filter=lfs diff=lfs merge=lfs -text +model-00002-of-00004.safetensors filter=lfs diff=lfs merge=lfs -text +checkpoint-1036/model-00002-of-00004.safetensors filter=lfs diff=lfs merge=lfs -text +checkpoint-1036/model-00003-of-00004.safetensors filter=lfs diff=lfs merge=lfs -text +vocab.json filter=lfs diff=lfs merge=lfs -text +training_args.bin filter=lfs diff=lfs merge=lfs -text +checkpoint-1036/model-00001-of-00004.safetensors filter=lfs diff=lfs merge=lfs -text +checkpoint-1036/rng_state_0.pth filter=lfs diff=lfs merge=lfs -text +model-00001-of-00004.safetensors filter=lfs diff=lfs merge=lfs -text \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..69a595a --- /dev/null +++ b/README.md @@ -0,0 +1,61 @@ +--- +library_name: transformers +license: other +tags: +- llama-factory +- full +- generated_from_trainer +model-index: +- name: qwen2.5-7b-thought-aligner-sharegpt-ds3 + results: [] +base_model: + - Qwen/Qwen2.5-7B-Instruct +--- + + +# qwen2.5-7b-thought-aligner-sharegpt-ds3 + +This model is a fine-tuned version of [Qwen/Qwen2.5-7B-Instruct](https://huggingface.co/Qwen/Qwen2.5-7B-Instruct) on the thought_aligner dataset. + +## Model description + +More information needed + +## Intended uses & limitations + +More information needed + +## Training and evaluation data + +More information needed + +## Training procedure + +### Training hyperparameters + +The following hyperparameters were used during training: +- learning_rate: 1e-05 +- train_batch_size: 4 +- eval_batch_size: 8 +- seed: 42 +- distributed_type: multi-GPU +- num_devices: 4 +- gradient_accumulation_steps: 8 +- total_train_batch_size: 128 +- total_eval_batch_size: 32 +- optimizer: Use adamw_torch_fused with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments +- lr_scheduler_type: cosine +- lr_scheduler_warmup_steps: 100 +- num_epochs: 7.0 + +### Training results + + + +### Framework versions + +- Transformers 4.57.1 +- Pytorch 2.9.0+cu128 +- Datasets 4.0.0 +- Tokenizers 0.22.1 diff --git a/added_tokens.json b/added_tokens.json new file mode 100644 index 0000000..482ced4 --- /dev/null +++ b/added_tokens.json @@ -0,0 +1,24 @@ +{ + "": 151658, + "": 151657, + "<|box_end|>": 151649, + "<|box_start|>": 151648, + "<|endoftext|>": 151643, + "<|file_sep|>": 151664, + "<|fim_middle|>": 151660, + "<|fim_pad|>": 151662, + "<|fim_prefix|>": 151659, + "<|fim_suffix|>": 151661, + "<|im_end|>": 151645, + "<|im_start|>": 151644, + "<|image_pad|>": 151655, + "<|object_ref_end|>": 151647, + "<|object_ref_start|>": 151646, + "<|quad_end|>": 151651, + "<|quad_start|>": 151650, + "<|repo_name|>": 151663, + "<|video_pad|>": 151656, + "<|vision_end|>": 151653, + "<|vision_pad|>": 151654, + "<|vision_start|>": 151652 +} diff --git a/all_results.json b/all_results.json new file mode 100644 index 0000000..82dd8fa --- /dev/null +++ b/all_results.json @@ -0,0 +1,8 @@ +{ + "epoch": 7.0, + "total_flos": 273471074074624.0, + "train_loss": 0.24805861423295494, + "train_runtime": 7529.4023, + "train_samples_per_second": 17.517, + "train_steps_per_second": 0.138 +} \ No newline at end of file diff --git a/chat_template.jinja b/chat_template.jinja new file mode 100644 index 0000000..bdf7919 --- /dev/null +++ b/chat_template.jinja @@ -0,0 +1,54 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0]['role'] == 'system' %} + {{- messages[0]['content'] }} + {%- else %} + {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }} + {%- endif %} + {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0]['role'] == 'system' %} + {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }} + {%- else %} + {{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {{- '<|im_start|>' + message.role }} + {%- if message.content %} + {{- '\n' + message.content }} + {%- endif %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {{- tool_call.arguments | tojson }} + {{- '}\n' }} + {%- endfor %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} +{%- endif %} diff --git a/config.json b/config.json new file mode 100644 index 0000000..6fccab2 --- /dev/null +++ b/config.json @@ -0,0 +1,58 @@ +{ + "architectures": [ + "Qwen2ForCausalLM" + ], + "attention_dropout": 0.0, + "dtype": "bfloat16", + "eos_token_id": 151645, + "hidden_act": "silu", + "hidden_size": 3584, + "initializer_range": 0.02, + "intermediate_size": 18944, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 32768, + "max_window_layers": 28, + "model_type": "qwen2", + "num_attention_heads": 28, + "num_hidden_layers": 28, + "num_key_value_heads": 4, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_scaling": null, + "rope_theta": 1000000.0, + "sliding_window": null, + "tie_word_embeddings": false, + "transformers_version": "4.57.1", + "use_cache": false, + "use_sliding_window": false, + "vocab_size": 152064 +} diff --git a/configuration.json b/configuration.json new file mode 100644 index 0000000..fe4ae63 --- /dev/null +++ b/configuration.json @@ -0,0 +1 @@ +{"framework":"Pytorch","task":"text-error-correction"} \ No newline at end of file diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..45e2560 --- /dev/null +++ b/generation_config.json @@ -0,0 +1,13 @@ +{ + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "repetition_penalty": 1.05, + "temperature": 0.7, + "top_k": 20, + "top_p": 0.8, + "transformers_version": "4.57.1" +} diff --git a/merges.txt b/merges.txt new file mode 100644 index 0000000..80c1a19 --- /dev/null +++ b/merges.txt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8831e4f1a044471340f7c0a83d7bd71306a5b867e95fd870f74d0c5308a904d5 +size 1671853 diff --git a/model-00001-of-00004.safetensors b/model-00001-of-00004.safetensors new file mode 100644 index 0000000..f875a36 --- /dev/null +++ b/model-00001-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:60efc0f2ad8d57aeeecbec1cff3895551b47930c2a95c3c20fdbe36b1c624063 +size 4877660776 diff --git a/model-00002-of-00004.safetensors b/model-00002-of-00004.safetensors new file mode 100644 index 0000000..c18ae3b --- /dev/null +++ b/model-00002-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:58142e0c445d645180977157720b3bfdc25755806f31a53995c6b723e2a1ac54 +size 4932751008 diff --git a/model-00003-of-00004.safetensors b/model-00003-of-00004.safetensors new file mode 100644 index 0000000..099680e --- /dev/null +++ b/model-00003-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c670eba05f5037eef17ab1064ddebeccef18f99b4ec80840ed6e07fa5d55aa9a +size 4330865200 diff --git a/model-00004-of-00004.safetensors b/model-00004-of-00004.safetensors new file mode 100644 index 0000000..b4c7da0 --- /dev/null +++ b/model-00004-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:36c08a7b3483b01469260d365a7c03c164a5017a8defa9c516f026a8b271d8fa +size 1089994880 diff --git a/model.safetensors.index.json b/model.safetensors.index.json new file mode 100644 index 0000000..098a4a8 --- /dev/null +++ b/model.safetensors.index.json @@ -0,0 +1,347 @@ +{ + "metadata": { + "total_parameters": 333312, + "total_size": 15231233024 + }, + "weight_map": { + "lm_head.weight": "model-00004-of-00004.safetensors", + "model.embed_tokens.weight": "model-00001-of-00004.safetensors", + "model.layers.0.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.0.self_attn.k_proj.bias": "model-00001-of-00004.safetensors", + "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.self_attn.q_proj.bias": "model-00001-of-00004.safetensors", + "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.0.self_attn.v_proj.bias": "model-00001-of-00004.safetensors", + "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.1.self_attn.k_proj.bias": "model-00001-of-00004.safetensors", + "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.self_attn.q_proj.bias": "model-00001-of-00004.safetensors", + "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.1.self_attn.v_proj.bias": "model-00001-of-00004.safetensors", + "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.10.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.10.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.10.self_attn.k_proj.bias": "model-00002-of-00004.safetensors", + "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.self_attn.q_proj.bias": "model-00002-of-00004.safetensors", + "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.10.self_attn.v_proj.bias": "model-00002-of-00004.safetensors", + "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.11.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.11.self_attn.k_proj.bias": "model-00002-of-00004.safetensors", + "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.self_attn.q_proj.bias": "model-00002-of-00004.safetensors", + "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.11.self_attn.v_proj.bias": "model-00002-of-00004.safetensors", + "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.12.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.12.self_attn.k_proj.bias": "model-00002-of-00004.safetensors", + "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.self_attn.q_proj.bias": "model-00002-of-00004.safetensors", + "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.12.self_attn.v_proj.bias": "model-00002-of-00004.safetensors", + "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.13.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.13.self_attn.k_proj.bias": "model-00002-of-00004.safetensors", + "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.self_attn.q_proj.bias": "model-00002-of-00004.safetensors", + "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.13.self_attn.v_proj.bias": "model-00002-of-00004.safetensors", + "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.14.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.14.self_attn.k_proj.bias": "model-00002-of-00004.safetensors", + "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.self_attn.q_proj.bias": "model-00002-of-00004.safetensors", + "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.14.self_attn.v_proj.bias": "model-00002-of-00004.safetensors", + "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.15.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.15.self_attn.k_proj.bias": "model-00002-of-00004.safetensors", + "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.self_attn.q_proj.bias": "model-00002-of-00004.safetensors", + "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.15.self_attn.v_proj.bias": "model-00002-of-00004.safetensors", + "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.16.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.16.self_attn.k_proj.bias": "model-00002-of-00004.safetensors", + "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.self_attn.q_proj.bias": "model-00002-of-00004.safetensors", + "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.16.self_attn.v_proj.bias": "model-00002-of-00004.safetensors", + "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.17.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.17.self_attn.k_proj.bias": "model-00002-of-00004.safetensors", + "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.self_attn.q_proj.bias": "model-00002-of-00004.safetensors", + "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.17.self_attn.v_proj.bias": "model-00002-of-00004.safetensors", + "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.18.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.18.self_attn.k_proj.bias": "model-00002-of-00004.safetensors", + "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.self_attn.q_proj.bias": "model-00002-of-00004.safetensors", + "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.18.self_attn.v_proj.bias": "model-00002-of-00004.safetensors", + "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.19.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.19.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.19.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.19.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.19.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.19.self_attn.k_proj.bias": "model-00003-of-00004.safetensors", + "model.layers.19.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.19.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.19.self_attn.q_proj.bias": "model-00003-of-00004.safetensors", + "model.layers.19.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.19.self_attn.v_proj.bias": "model-00003-of-00004.safetensors", + "model.layers.19.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.2.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.2.self_attn.k_proj.bias": "model-00001-of-00004.safetensors", + "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.self_attn.q_proj.bias": "model-00001-of-00004.safetensors", + "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.2.self_attn.v_proj.bias": "model-00001-of-00004.safetensors", + "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.20.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.20.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.20.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.20.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.20.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.20.self_attn.k_proj.bias": "model-00003-of-00004.safetensors", + "model.layers.20.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.20.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.20.self_attn.q_proj.bias": "model-00003-of-00004.safetensors", + "model.layers.20.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.20.self_attn.v_proj.bias": "model-00003-of-00004.safetensors", + "model.layers.20.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.21.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.21.self_attn.k_proj.bias": "model-00003-of-00004.safetensors", + "model.layers.21.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.self_attn.q_proj.bias": "model-00003-of-00004.safetensors", + "model.layers.21.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.21.self_attn.v_proj.bias": "model-00003-of-00004.safetensors", + "model.layers.21.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.22.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.22.self_attn.k_proj.bias": "model-00003-of-00004.safetensors", + "model.layers.22.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.self_attn.q_proj.bias": "model-00003-of-00004.safetensors", + "model.layers.22.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.22.self_attn.v_proj.bias": "model-00003-of-00004.safetensors", + "model.layers.22.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.23.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.23.self_attn.k_proj.bias": "model-00003-of-00004.safetensors", + "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.self_attn.q_proj.bias": "model-00003-of-00004.safetensors", + "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.23.self_attn.v_proj.bias": "model-00003-of-00004.safetensors", + "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.24.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.24.self_attn.k_proj.bias": "model-00003-of-00004.safetensors", + "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.self_attn.q_proj.bias": "model-00003-of-00004.safetensors", + "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.24.self_attn.v_proj.bias": "model-00003-of-00004.safetensors", + "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.25.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.25.self_attn.k_proj.bias": "model-00003-of-00004.safetensors", + "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.self_attn.q_proj.bias": "model-00003-of-00004.safetensors", + "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.25.self_attn.v_proj.bias": "model-00003-of-00004.safetensors", + "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.26.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.26.self_attn.k_proj.bias": "model-00003-of-00004.safetensors", + "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.self_attn.q_proj.bias": "model-00003-of-00004.safetensors", + "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.26.self_attn.v_proj.bias": "model-00003-of-00004.safetensors", + "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.input_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.27.mlp.down_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.mlp.up_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00004.safetensors", + "model.layers.27.self_attn.k_proj.bias": "model-00003-of-00004.safetensors", + "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.self_attn.q_proj.bias": "model-00003-of-00004.safetensors", + "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.27.self_attn.v_proj.bias": "model-00003-of-00004.safetensors", + "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00004.safetensors", + "model.layers.3.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.3.self_attn.k_proj.bias": "model-00001-of-00004.safetensors", + "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.self_attn.q_proj.bias": "model-00001-of-00004.safetensors", + "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.3.self_attn.v_proj.bias": "model-00001-of-00004.safetensors", + "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.4.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.4.self_attn.k_proj.bias": "model-00001-of-00004.safetensors", + "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.self_attn.q_proj.bias": "model-00001-of-00004.safetensors", + "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.4.self_attn.v_proj.bias": "model-00001-of-00004.safetensors", + "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.5.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.5.self_attn.k_proj.bias": "model-00001-of-00004.safetensors", + "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.self_attn.q_proj.bias": "model-00001-of-00004.safetensors", + "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.5.self_attn.v_proj.bias": "model-00001-of-00004.safetensors", + "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.6.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.6.self_attn.k_proj.bias": "model-00001-of-00004.safetensors", + "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.self_attn.q_proj.bias": "model-00001-of-00004.safetensors", + "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.6.self_attn.v_proj.bias": "model-00001-of-00004.safetensors", + "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.input_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.7.mlp.down_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.mlp.up_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00004.safetensors", + "model.layers.7.self_attn.k_proj.bias": "model-00001-of-00004.safetensors", + "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.self_attn.q_proj.bias": "model-00001-of-00004.safetensors", + "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.7.self_attn.v_proj.bias": "model-00001-of-00004.safetensors", + "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.8.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.8.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.8.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.8.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.8.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.8.self_attn.k_proj.bias": "model-00001-of-00004.safetensors", + "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.8.self_attn.q_proj.bias": "model-00001-of-00004.safetensors", + "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.8.self_attn.v_proj.bias": "model-00001-of-00004.safetensors", + "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00004.safetensors", + "model.layers.9.input_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.9.mlp.down_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.mlp.gate_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.mlp.up_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00004.safetensors", + "model.layers.9.self_attn.k_proj.bias": "model-00002-of-00004.safetensors", + "model.layers.9.self_attn.k_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.self_attn.q_proj.bias": "model-00002-of-00004.safetensors", + "model.layers.9.self_attn.q_proj.weight": "model-00002-of-00004.safetensors", + "model.layers.9.self_attn.v_proj.bias": "model-00002-of-00004.safetensors", + "model.layers.9.self_attn.v_proj.weight": "model-00002-of-00004.safetensors", + "model.norm.weight": "model-00003-of-00004.safetensors" + } +} diff --git a/special_tokens_map.json b/special_tokens_map.json new file mode 100644 index 0000000..ac23c0a --- /dev/null +++ b/special_tokens_map.json @@ -0,0 +1,31 @@ +{ + "additional_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "eos_token": { + "content": "<|im_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": { + "content": "<|endoftext|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + } +} diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..51ebb3b --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9c5ae00e602b8860cbd784ba82a8aa14e8feecec692e7076590d014d7b7fdafa +size 11421896 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..fb44f02 --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,208 @@ +{ + "add_bos_token": false, + "add_prefix_space": false, + "added_tokens_decoder": { + "151643": { + "content": "<|endoftext|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151644": { + "content": "<|im_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151645": { + "content": "<|im_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151646": { + "content": "<|object_ref_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151647": { + "content": "<|object_ref_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151648": { + "content": "<|box_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151649": { + "content": "<|box_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151650": { + "content": "<|quad_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151651": { + "content": "<|quad_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151652": { + "content": "<|vision_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151653": { + "content": "<|vision_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151654": { + "content": "<|vision_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151655": { + "content": "<|image_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151656": { + "content": "<|video_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151657": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151658": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151659": { + "content": "<|fim_prefix|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151660": { + "content": "<|fim_middle|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151661": { + "content": "<|fim_suffix|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151662": { + "content": "<|fim_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151663": { + "content": "<|repo_name|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151664": { + "content": "<|file_sep|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + } + }, + "additional_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": {}, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "padding_side": "right", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/train_results.json b/train_results.json new file mode 100644 index 0000000..82dd8fa --- /dev/null +++ b/train_results.json @@ -0,0 +1,8 @@ +{ + "epoch": 7.0, + "total_flos": 273471074074624.0, + "train_loss": 0.24805861423295494, + "train_runtime": 7529.4023, + "train_samples_per_second": 17.517, + "train_steps_per_second": 0.138 +} \ No newline at end of file diff --git a/trainer_log.jsonl b/trainer_log.jsonl new file mode 100644 index 0000000..936dabf --- /dev/null +++ b/trainer_log.jsonl @@ -0,0 +1,104 @@ +{"current_steps": 10, "total_steps": 1036, "loss": 2.5672, "lr": 9.000000000000001e-07, "epoch": 0.06791171477079797, "percentage": 0.97, "elapsed_time": "0:01:10", "remaining_time": "2:00:23"} +{"current_steps": 20, "total_steps": 1036, "loss": 1.6271, "lr": 1.9000000000000002e-06, "epoch": 0.13582342954159593, "percentage": 1.93, "elapsed_time": "0:02:32", "remaining_time": "2:08:42"} +{"current_steps": 30, "total_steps": 1036, "loss": 1.0035, "lr": 2.9e-06, "epoch": 0.2037351443123939, "percentage": 2.9, "elapsed_time": "0:03:42", "remaining_time": "2:04:31"} +{"current_steps": 40, "total_steps": 1036, "loss": 0.7978, "lr": 3.900000000000001e-06, "epoch": 0.27164685908319186, "percentage": 3.86, "elapsed_time": "0:04:51", "remaining_time": "2:01:06"} +{"current_steps": 50, "total_steps": 1036, "loss": 0.7342, "lr": 4.9000000000000005e-06, "epoch": 0.3395585738539898, "percentage": 4.83, "elapsed_time": "0:06:01", "remaining_time": "1:58:45"} +{"current_steps": 60, "total_steps": 1036, "loss": 0.6387, "lr": 5.9e-06, "epoch": 0.4074702886247878, "percentage": 5.79, "elapsed_time": "0:07:15", "remaining_time": "1:57:57"} +{"current_steps": 70, "total_steps": 1036, "loss": 0.6566, "lr": 6.9e-06, "epoch": 0.47538200339558573, "percentage": 6.76, "elapsed_time": "0:08:30", "remaining_time": "1:57:22"} +{"current_steps": 80, "total_steps": 1036, "loss": 0.6369, "lr": 7.9e-06, "epoch": 0.5432937181663837, "percentage": 7.72, "elapsed_time": "0:09:44", "remaining_time": "1:56:20"} +{"current_steps": 90, "total_steps": 1036, "loss": 0.5945, "lr": 8.900000000000001e-06, "epoch": 0.6112054329371817, "percentage": 8.69, "elapsed_time": "0:10:52", "remaining_time": "1:54:22"} +{"current_steps": 100, "total_steps": 1036, "loss": 0.598, "lr": 9.9e-06, "epoch": 0.6791171477079796, "percentage": 9.65, "elapsed_time": "0:12:00", "remaining_time": "1:52:23"} +{"current_steps": 110, "total_steps": 1036, "loss": 0.569, "lr": 9.997718922447669e-06, "epoch": 0.7470288624787776, "percentage": 10.62, "elapsed_time": "0:13:14", "remaining_time": "1:51:27"} +{"current_steps": 120, "total_steps": 1036, "loss": 0.5606, "lr": 9.989836388165012e-06, "epoch": 0.8149405772495756, "percentage": 11.58, "elapsed_time": "0:14:21", "remaining_time": "1:49:34"} +{"current_steps": 130, "total_steps": 1036, "loss": 0.5565, "lr": 9.976333112771488e-06, "epoch": 0.8828522920203735, "percentage": 12.55, "elapsed_time": "0:15:26", "remaining_time": "1:47:35"} +{"current_steps": 140, "total_steps": 1036, "loss": 0.5414, "lr": 9.957224306869053e-06, "epoch": 0.9507640067911715, "percentage": 13.51, "elapsed_time": "0:16:39", "remaining_time": "1:46:33"} +{"current_steps": 150, "total_steps": 1036, "loss": 0.5142, "lr": 9.932531495342005e-06, "epoch": 1.0135823429541595, "percentage": 14.48, "elapsed_time": "0:17:44", "remaining_time": "1:44:47"} +{"current_steps": 160, "total_steps": 1036, "loss": 0.4346, "lr": 9.90228249311053e-06, "epoch": 1.0814940577249577, "percentage": 15.44, "elapsed_time": "0:18:54", "remaining_time": "1:43:33"} +{"current_steps": 170, "total_steps": 1036, "loss": 0.4156, "lr": 9.86651137379893e-06, "epoch": 1.1494057724957556, "percentage": 16.41, "elapsed_time": "0:20:05", "remaining_time": "1:42:18"} +{"current_steps": 180, "total_steps": 1036, "loss": 0.4199, "lr": 9.82525843135379e-06, "epoch": 1.2173174872665535, "percentage": 17.37, "elapsed_time": "0:21:20", "remaining_time": "1:41:30"} +{"current_steps": 190, "total_steps": 1036, "loss": 0.4088, "lr": 9.778570134655351e-06, "epoch": 1.2852292020373515, "percentage": 18.34, "elapsed_time": "0:22:31", "remaining_time": "1:40:17"} +{"current_steps": 200, "total_steps": 1036, "loss": 0.4198, "lr": 9.726499075173201e-06, "epoch": 1.3531409168081494, "percentage": 19.31, "elapsed_time": "0:23:42", "remaining_time": "1:39:04"} +{"current_steps": 210, "total_steps": 1036, "loss": 0.4203, "lr": 9.669103907725243e-06, "epoch": 1.4210526315789473, "percentage": 20.27, "elapsed_time": "0:24:53", "remaining_time": "1:37:55"} +{"current_steps": 220, "total_steps": 1036, "loss": 0.4288, "lr": 9.606449284406685e-06, "epoch": 1.4889643463497453, "percentage": 21.24, "elapsed_time": "0:26:00", "remaining_time": "1:36:28"} +{"current_steps": 230, "total_steps": 1036, "loss": 0.403, "lr": 9.538605781763464e-06, "epoch": 1.5568760611205432, "percentage": 22.2, "elapsed_time": "0:27:11", "remaining_time": "1:35:18"} +{"current_steps": 240, "total_steps": 1036, "loss": 0.4014, "lr": 9.465649821292148e-06, "epoch": 1.6247877758913414, "percentage": 23.17, "elapsed_time": "0:28:21", "remaining_time": "1:34:02"} +{"current_steps": 250, "total_steps": 1036, "loss": 0.4104, "lr": 9.387663583355862e-06, "epoch": 1.692699490662139, "percentage": 24.13, "elapsed_time": "0:29:26", "remaining_time": "1:32:35"} +{"current_steps": 260, "total_steps": 1036, "loss": 0.4106, "lr": 9.3047349146132e-06, "epoch": 1.7606112054329373, "percentage": 25.1, "elapsed_time": "0:30:38", "remaining_time": "1:31:27"} +{"current_steps": 270, "total_steps": 1036, "loss": 0.4055, "lr": 9.21695722906443e-06, "epoch": 1.8285229202037352, "percentage": 26.06, "elapsed_time": "0:31:50", "remaining_time": "1:30:20"} +{"current_steps": 280, "total_steps": 1036, "loss": 0.4238, "lr": 9.124429402826398e-06, "epoch": 1.8964346349745331, "percentage": 27.03, "elapsed_time": "0:32:59", "remaining_time": "1:29:04"} +{"current_steps": 290, "total_steps": 1036, "loss": 0.3975, "lr": 9.02725566275473e-06, "epoch": 1.964346349745331, "percentage": 27.99, "elapsed_time": "0:34:13", "remaining_time": "1:28:02"} +{"current_steps": 300, "total_steps": 1036, "loss": 0.347, "lr": 8.92554546903875e-06, "epoch": 2.027164685908319, "percentage": 28.96, "elapsed_time": "0:35:22", "remaining_time": "1:26:47"} +{"current_steps": 310, "total_steps": 1036, "loss": 0.2279, "lr": 8.819413391901353e-06, "epoch": 2.095076400679117, "percentage": 29.92, "elapsed_time": "0:36:32", "remaining_time": "1:25:35"} +{"current_steps": 320, "total_steps": 1036, "loss": 0.2456, "lr": 8.708978982542766e-06, "epoch": 2.1629881154499153, "percentage": 30.89, "elapsed_time": "0:37:42", "remaining_time": "1:24:22"} +{"current_steps": 330, "total_steps": 1036, "loss": 0.2394, "lr": 8.59436663847354e-06, "epoch": 2.230899830220713, "percentage": 31.85, "elapsed_time": "0:38:54", "remaining_time": "1:23:13"} +{"current_steps": 340, "total_steps": 1036, "loss": 0.2537, "lr": 8.475705463388467e-06, "epoch": 2.298811544991511, "percentage": 32.82, "elapsed_time": "0:40:08", "remaining_time": "1:22:10"} +{"current_steps": 350, "total_steps": 1036, "loss": 0.2555, "lr": 8.35312912173928e-06, "epoch": 2.366723259762309, "percentage": 33.78, "elapsed_time": "0:41:14", "remaining_time": "1:20:49"} +{"current_steps": 360, "total_steps": 1036, "loss": 0.2424, "lr": 8.226775688169943e-06, "epoch": 2.434634974533107, "percentage": 34.75, "elapsed_time": "0:42:25", "remaining_time": "1:19:40"} +{"current_steps": 370, "total_steps": 1036, "loss": 0.2458, "lr": 8.096787491984128e-06, "epoch": 2.502546689303905, "percentage": 35.71, "elapsed_time": "0:43:45", "remaining_time": "1:18:46"} +{"current_steps": 380, "total_steps": 1036, "loss": 0.2545, "lr": 7.963310956820085e-06, "epoch": 2.570458404074703, "percentage": 36.68, "elapsed_time": "0:44:54", "remaining_time": "1:17:31"} +{"current_steps": 390, "total_steps": 1036, "loss": 0.2544, "lr": 7.826496435713497e-06, "epoch": 2.6383701188455007, "percentage": 37.64, "elapsed_time": "0:46:04", "remaining_time": "1:16:19"} +{"current_steps": 400, "total_steps": 1036, "loss": 0.2587, "lr": 7.686498041734121e-06, "epoch": 2.706281833616299, "percentage": 38.61, "elapsed_time": "0:47:18", "remaining_time": "1:15:12"} +{"current_steps": 410, "total_steps": 1036, "loss": 0.2537, "lr": 7.543473474386962e-06, "epoch": 2.774193548387097, "percentage": 39.58, "elapsed_time": "0:48:25", "remaining_time": "1:13:56"} +{"current_steps": 420, "total_steps": 1036, "loss": 0.2549, "lr": 7.397583841973584e-06, "epoch": 2.8421052631578947, "percentage": 40.54, "elapsed_time": "0:49:36", "remaining_time": "1:12:45"} +{"current_steps": 430, "total_steps": 1036, "loss": 0.2576, "lr": 7.248993480113585e-06, "epoch": 2.910016977928693, "percentage": 41.51, "elapsed_time": "0:50:43", "remaining_time": "1:11:29"} +{"current_steps": 440, "total_steps": 1036, "loss": 0.2597, "lr": 7.09786976663073e-06, "epoch": 2.9779286926994906, "percentage": 42.47, "elapsed_time": "0:51:54", "remaining_time": "1:10:18"} +{"current_steps": 450, "total_steps": 1036, "loss": 0.1914, "lr": 6.944382933012207e-06, "epoch": 3.0407470288624787, "percentage": 43.44, "elapsed_time": "0:53:00", "remaining_time": "1:09:01"} +{"current_steps": 460, "total_steps": 1036, "loss": 0.1304, "lr": 6.788705872653429e-06, "epoch": 3.108658743633277, "percentage": 44.4, "elapsed_time": "0:54:09", "remaining_time": "1:07:49"} +{"current_steps": 470, "total_steps": 1036, "loss": 0.1348, "lr": 6.631013946104348e-06, "epoch": 3.1765704584040746, "percentage": 45.37, "elapsed_time": "0:55:21", "remaining_time": "1:06:40"} +{"current_steps": 480, "total_steps": 1036, "loss": 0.1306, "lr": 6.4714847835366625e-06, "epoch": 3.2444821731748728, "percentage": 46.33, "elapsed_time": "0:56:33", "remaining_time": "1:05:30"} +{"current_steps": 490, "total_steps": 1036, "loss": 0.1341, "lr": 6.310298084654467e-06, "epoch": 3.3123938879456705, "percentage": 47.3, "elapsed_time": "0:57:46", "remaining_time": "1:04:22"} +{"current_steps": 500, "total_steps": 1036, "loss": 0.138, "lr": 6.147635416273679e-06, "epoch": 3.3803056027164686, "percentage": 48.26, "elapsed_time": "0:58:58", "remaining_time": "1:03:13"} +{"current_steps": 510, "total_steps": 1036, "loss": 0.1367, "lr": 5.983680007798277e-06, "epoch": 3.4482173174872663, "percentage": 49.23, "elapsed_time": "1:00:07", "remaining_time": "1:02:00"} +{"current_steps": 520, "total_steps": 1036, "loss": 0.1341, "lr": 5.818616544823755e-06, "epoch": 3.5161290322580645, "percentage": 50.19, "elapsed_time": "1:01:19", "remaining_time": "1:00:50"} +{"current_steps": 530, "total_steps": 1036, "loss": 0.1417, "lr": 5.65263096110026e-06, "epoch": 3.5840407470288627, "percentage": 51.16, "elapsed_time": "1:02:31", "remaining_time": "0:59:41"} +{"current_steps": 540, "total_steps": 1036, "loss": 0.1432, "lr": 5.485910229089739e-06, "epoch": 3.6519524617996604, "percentage": 52.12, "elapsed_time": "1:03:36", "remaining_time": "0:58:25"} +{"current_steps": 550, "total_steps": 1036, "loss": 0.1378, "lr": 5.3186421493530845e-06, "epoch": 3.7198641765704585, "percentage": 53.09, "elapsed_time": "1:04:46", "remaining_time": "0:57:14"} +{"current_steps": 560, "total_steps": 1036, "loss": 0.1396, "lr": 5.151015139004445e-06, "epoch": 3.7877758913412563, "percentage": 54.05, "elapsed_time": "1:06:02", "remaining_time": "0:56:08"} +{"current_steps": 570, "total_steps": 1036, "loss": 0.1443, "lr": 4.983218019471027e-06, "epoch": 3.8556876061120544, "percentage": 55.02, "elapsed_time": "1:07:11", "remaining_time": "0:54:55"} +{"current_steps": 580, "total_steps": 1036, "loss": 0.1328, "lr": 4.815439803797483e-06, "epoch": 3.923599320882852, "percentage": 55.98, "elapsed_time": "1:08:27", "remaining_time": "0:53:49"} +{"current_steps": 590, "total_steps": 1036, "loss": 0.1334, "lr": 4.647869483734441e-06, "epoch": 3.9915110356536503, "percentage": 56.95, "elapsed_time": "1:09:36", "remaining_time": "0:52:37"} +{"current_steps": 600, "total_steps": 1036, "loss": 0.0788, "lr": 4.4806958168510235e-06, "epoch": 4.054329371816638, "percentage": 57.92, "elapsed_time": "1:10:40", "remaining_time": "0:51:21"} +{"current_steps": 610, "total_steps": 1036, "loss": 0.062, "lr": 4.314107113911151e-06, "epoch": 4.122241086587437, "percentage": 58.88, "elapsed_time": "1:11:55", "remaining_time": "0:50:13"} +{"current_steps": 620, "total_steps": 1036, "loss": 0.0649, "lr": 4.148291026753159e-06, "epoch": 4.190152801358234, "percentage": 59.85, "elapsed_time": "1:13:02", "remaining_time": "0:49:00"} +{"current_steps": 630, "total_steps": 1036, "loss": 0.0662, "lr": 3.983434336911636e-06, "epoch": 4.258064516129032, "percentage": 60.81, "elapsed_time": "1:14:13", "remaining_time": "0:47:49"} +{"current_steps": 640, "total_steps": 1036, "loss": 0.0633, "lr": 3.819722745219617e-06, "epoch": 4.325976230899831, "percentage": 61.78, "elapsed_time": "1:15:25", "remaining_time": "0:46:39"} +{"current_steps": 650, "total_steps": 1036, "loss": 0.0662, "lr": 3.657340662628116e-06, "epoch": 4.393887945670628, "percentage": 62.74, "elapsed_time": "1:16:37", "remaining_time": "0:45:30"} +{"current_steps": 660, "total_steps": 1036, "loss": 0.0625, "lr": 3.4964710024786354e-06, "epoch": 4.461799660441426, "percentage": 63.71, "elapsed_time": "1:17:46", "remaining_time": "0:44:18"} +{"current_steps": 670, "total_steps": 1036, "loss": 0.0628, "lr": 3.337294974462623e-06, "epoch": 4.529711375212224, "percentage": 64.67, "elapsed_time": "1:19:00", "remaining_time": "0:43:09"} +{"current_steps": 680, "total_steps": 1036, "loss": 0.0655, "lr": 3.1799918805000097e-06, "epoch": 4.597623089983022, "percentage": 65.64, "elapsed_time": "1:20:07", "remaining_time": "0:41:56"} +{"current_steps": 690, "total_steps": 1036, "loss": 0.0616, "lr": 3.024738912766717e-06, "epoch": 4.66553480475382, "percentage": 66.6, "elapsed_time": "1:21:26", "remaining_time": "0:40:50"} +{"current_steps": 700, "total_steps": 1036, "loss": 0.0614, "lr": 2.87171095409866e-06, "epoch": 4.733446519524618, "percentage": 67.57, "elapsed_time": "1:22:35", "remaining_time": "0:39:38"} +{"current_steps": 710, "total_steps": 1036, "loss": 0.0635, "lr": 2.7210803809970853e-06, "epoch": 4.8013582342954155, "percentage": 68.53, "elapsed_time": "1:23:47", "remaining_time": "0:38:28"} +{"current_steps": 720, "total_steps": 1036, "loss": 0.0614, "lr": 2.5730168694571457e-06, "epoch": 4.869269949066214, "percentage": 69.5, "elapsed_time": "1:24:56", "remaining_time": "0:37:16"} +{"current_steps": 730, "total_steps": 1036, "loss": 0.0598, "lr": 2.4276872038384037e-06, "epoch": 4.937181663837012, "percentage": 70.46, "elapsed_time": "1:26:07", "remaining_time": "0:36:06"} +{"current_steps": 740, "total_steps": 1036, "loss": 0.0595, "lr": 2.2852550889926067e-06, "epoch": 5.0, "percentage": 71.43, "elapsed_time": "1:27:11", "remaining_time": "0:34:52"} +{"current_steps": 750, "total_steps": 1036, "loss": 0.024, "lr": 2.1458809658603288e-06, "epoch": 5.067911714770798, "percentage": 72.39, "elapsed_time": "1:28:21", "remaining_time": "0:33:41"} +{"current_steps": 760, "total_steps": 1036, "loss": 0.022, "lr": 2.009721830744201e-06, "epoch": 5.135823429541596, "percentage": 73.36, "elapsed_time": "1:29:26", "remaining_time": "0:32:28"} +{"current_steps": 770, "total_steps": 1036, "loss": 0.0214, "lr": 1.87693105846232e-06, "epoch": 5.203735144312394, "percentage": 74.32, "elapsed_time": "1:30:38", "remaining_time": "0:31:18"} +{"current_steps": 780, "total_steps": 1036, "loss": 0.0224, "lr": 1.7476582295810314e-06, "epoch": 5.271646859083192, "percentage": 75.29, "elapsed_time": "1:31:49", "remaining_time": "0:30:08"} +{"current_steps": 790, "total_steps": 1036, "loss": 0.0212, "lr": 1.6220489619216988e-06, "epoch": 5.3395585738539895, "percentage": 76.25, "elapsed_time": "1:32:54", "remaining_time": "0:28:55"} +{"current_steps": 800, "total_steps": 1036, "loss": 0.0213, "lr": 1.5002447465312675e-06, "epoch": 5.407470288624788, "percentage": 77.22, "elapsed_time": "1:34:09", "remaining_time": "0:27:46"} +{"current_steps": 810, "total_steps": 1036, "loss": 0.0201, "lr": 1.3823827883013723e-06, "epoch": 5.475382003395586, "percentage": 78.19, "elapsed_time": "1:35:21", "remaining_time": "0:26:36"} +{"current_steps": 820, "total_steps": 1036, "loss": 0.0214, "lr": 1.2685958514155456e-06, "epoch": 5.5432937181663835, "percentage": 79.15, "elapsed_time": "1:36:32", "remaining_time": "0:25:25"} +{"current_steps": 830, "total_steps": 1036, "loss": 0.0213, "lr": 1.159012109798598e-06, "epoch": 5.611205432937181, "percentage": 80.12, "elapsed_time": "1:37:39", "remaining_time": "0:24:14"} +{"current_steps": 840, "total_steps": 1036, "loss": 0.0191, "lr": 1.0537550027366444e-06, "epoch": 5.67911714770798, "percentage": 81.08, "elapsed_time": "1:38:53", "remaining_time": "0:23:04"} +{"current_steps": 850, "total_steps": 1036, "loss": 0.0206, "lr": 9.529430958304092e-07, "epoch": 5.7470288624787775, "percentage": 82.05, "elapsed_time": "1:40:00", "remaining_time": "0:21:52"} +{"current_steps": 860, "total_steps": 1036, "loss": 0.0192, "lr": 8.566899474384299e-07, "epoch": 5.814940577249575, "percentage": 83.01, "elapsed_time": "1:41:14", "remaining_time": "0:20:43"} +{"current_steps": 870, "total_steps": 1036, "loss": 0.0213, "lr": 7.651039807606137e-07, "epoch": 5.882852292020374, "percentage": 83.98, "elapsed_time": "1:42:28", "remaining_time": "0:19:33"} +{"current_steps": 880, "total_steps": 1036, "loss": 0.0189, "lr": 6.782883617062286e-07, "epoch": 5.950764006791172, "percentage": 84.94, "elapsed_time": "1:43:42", "remaining_time": "0:18:22"} +{"current_steps": 890, "total_steps": 1036, "loss": 0.0157, "lr": 5.963408826839079e-07, "epoch": 6.01358234295416, "percentage": 85.91, "elapsed_time": "1:44:54", "remaining_time": "0:17:12"} +{"current_steps": 900, "total_steps": 1036, "loss": 0.0064, "lr": 5.19353852444569e-07, "epoch": 6.081494057724957, "percentage": 86.87, "elapsed_time": "1:46:06", "remaining_time": "0:16:02"} +{"current_steps": 910, "total_steps": 1036, "loss": 0.0063, "lr": 4.4741399210133154e-07, "epoch": 6.149405772495755, "percentage": 87.84, "elapsed_time": "1:47:20", "remaining_time": "0:14:51"} +{"current_steps": 920, "total_steps": 1036, "loss": 0.0054, "lr": 3.8060233744356634e-07, "epoch": 6.217317487266554, "percentage": 88.8, "elapsed_time": "1:48:33", "remaining_time": "0:13:41"} +{"current_steps": 930, "total_steps": 1036, "loss": 0.0059, "lr": 3.189941476551034e-07, "epoch": 6.2852292020373515, "percentage": 89.77, "elapsed_time": "1:49:44", "remaining_time": "0:12:30"} +{"current_steps": 940, "total_steps": 1036, "loss": 0.0055, "lr": 2.6265882053943204e-07, "epoch": 6.353140916808149, "percentage": 90.73, "elapsed_time": "1:50:54", "remaining_time": "0:11:19"} +{"current_steps": 950, "total_steps": 1036, "loss": 0.0055, "lr": 2.1165981434738025e-07, "epoch": 6.421052631578947, "percentage": 91.7, "elapsed_time": "1:52:01", "remaining_time": "0:10:08"} +{"current_steps": 960, "total_steps": 1036, "loss": 0.0049, "lr": 1.6605457629533417e-07, "epoch": 6.4889643463497455, "percentage": 92.66, "elapsed_time": "1:53:12", "remaining_time": "0:08:57"} +{"current_steps": 970, "total_steps": 1036, "loss": 0.0056, "lr": 1.2589447785450903e-07, "epoch": 6.556876061120543, "percentage": 93.63, "elapsed_time": "1:54:28", "remaining_time": "0:07:47"} +{"current_steps": 980, "total_steps": 1036, "loss": 0.0053, "lr": 9.122475688417954e-08, "epoch": 6.624787775891341, "percentage": 94.59, "elapsed_time": "1:55:34", "remaining_time": "0:06:36"} +{"current_steps": 990, "total_steps": 1036, "loss": 0.0054, "lr": 6.208446667403167e-08, "epoch": 6.6926994906621395, "percentage": 95.56, "elapsed_time": "1:56:44", "remaining_time": "0:05:25"} +{"current_steps": 1000, "total_steps": 1036, "loss": 0.0051, "lr": 3.8506431953060365e-08, "epoch": 6.760611205432937, "percentage": 96.53, "elapsed_time": "1:57:50", "remaining_time": "0:04:14"} +{"current_steps": 1010, "total_steps": 1036, "loss": 0.0052, "lr": 2.0517211914545254e-08, "epoch": 6.828522920203735, "percentage": 97.49, "elapsed_time": "2:00:08", "remaining_time": "0:03:05"} +{"current_steps": 1020, "total_steps": 1036, "loss": 0.0052, "lr": 8.137070298769312e-09, "epoch": 6.896434634974533, "percentage": 98.46, "elapsed_time": "2:01:19", "remaining_time": "0:01:54"} +{"current_steps": 1030, "total_steps": 1036, "loss": 0.0053, "lr": 1.3799525671720492e-09, "epoch": 6.964346349745331, "percentage": 99.42, "elapsed_time": "2:02:26", "remaining_time": "0:00:42"} +{"current_steps": 1036, "total_steps": 1036, "epoch": 7.0, "percentage": 100.0, "elapsed_time": "2:05:29", "remaining_time": "0:00:00"} diff --git a/trainer_state.json b/trainer_state.json new file mode 100644 index 0000000..32e6d28 --- /dev/null +++ b/trainer_state.json @@ -0,0 +1,764 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 7.0, + "eval_steps": 500, + "global_step": 1036, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.06791171477079797, + "grad_norm": 39.06894209264497, + "learning_rate": 9.000000000000001e-07, + "loss": 2.5672, + "step": 10 + }, + { + "epoch": 0.13582342954159593, + "grad_norm": 9.408641592679151, + "learning_rate": 1.9000000000000002e-06, + "loss": 1.6271, + "step": 20 + }, + { + "epoch": 0.2037351443123939, + "grad_norm": 3.5736160912411656, + "learning_rate": 2.9e-06, + "loss": 1.0035, + "step": 30 + }, + { + "epoch": 0.27164685908319186, + "grad_norm": 3.159979908410021, + "learning_rate": 3.900000000000001e-06, + "loss": 0.7978, + "step": 40 + }, + { + "epoch": 0.3395585738539898, + "grad_norm": 3.23976253664356, + "learning_rate": 4.9000000000000005e-06, + "loss": 0.7342, + "step": 50 + }, + { + "epoch": 0.4074702886247878, + "grad_norm": 2.5239516324886613, + "learning_rate": 5.9e-06, + "loss": 0.6387, + "step": 60 + }, + { + "epoch": 0.47538200339558573, + "grad_norm": 2.864395183943674, + "learning_rate": 6.9e-06, + "loss": 0.6566, + "step": 70 + }, + { + "epoch": 0.5432937181663837, + "grad_norm": 2.221989312084371, + "learning_rate": 7.9e-06, + "loss": 0.6369, + "step": 80 + }, + { + "epoch": 0.6112054329371817, + "grad_norm": 2.622798646579124, + "learning_rate": 8.900000000000001e-06, + "loss": 0.5945, + "step": 90 + }, + { + "epoch": 0.6791171477079796, + "grad_norm": 2.727376011862893, + "learning_rate": 9.9e-06, + "loss": 0.598, + "step": 100 + }, + { + "epoch": 0.7470288624787776, + "grad_norm": 2.588442039618945, + "learning_rate": 9.997718922447669e-06, + "loss": 0.569, + "step": 110 + }, + { + "epoch": 0.8149405772495756, + "grad_norm": 2.3467194997795335, + "learning_rate": 9.989836388165012e-06, + "loss": 0.5606, + "step": 120 + }, + { + "epoch": 0.8828522920203735, + "grad_norm": 2.3847506796962556, + "learning_rate": 9.976333112771488e-06, + "loss": 0.5565, + "step": 130 + }, + { + "epoch": 0.9507640067911715, + "grad_norm": 2.1204168574702957, + "learning_rate": 9.957224306869053e-06, + "loss": 0.5414, + "step": 140 + }, + { + "epoch": 1.0135823429541595, + "grad_norm": 2.0962709577545704, + "learning_rate": 9.932531495342005e-06, + "loss": 0.5142, + "step": 150 + }, + { + "epoch": 1.0814940577249577, + "grad_norm": 2.420185175502751, + "learning_rate": 9.90228249311053e-06, + "loss": 0.4346, + "step": 160 + }, + { + "epoch": 1.1494057724957556, + "grad_norm": 2.175482934612183, + "learning_rate": 9.86651137379893e-06, + "loss": 0.4156, + "step": 170 + }, + { + "epoch": 1.2173174872665535, + "grad_norm": 2.015548954416845, + "learning_rate": 9.82525843135379e-06, + "loss": 0.4199, + "step": 180 + }, + { + "epoch": 1.2852292020373515, + "grad_norm": 2.2212942924087415, + "learning_rate": 9.778570134655351e-06, + "loss": 0.4088, + "step": 190 + }, + { + "epoch": 1.3531409168081494, + "grad_norm": 2.2774181484385014, + "learning_rate": 9.726499075173201e-06, + "loss": 0.4198, + "step": 200 + }, + { + "epoch": 1.4210526315789473, + "grad_norm": 1.951548193520802, + "learning_rate": 9.669103907725243e-06, + "loss": 0.4203, + "step": 210 + }, + { + "epoch": 1.4889643463497453, + "grad_norm": 2.0166680690881154, + "learning_rate": 9.606449284406685e-06, + "loss": 0.4288, + "step": 220 + }, + { + "epoch": 1.5568760611205432, + "grad_norm": 2.1215872162254445, + "learning_rate": 9.538605781763464e-06, + "loss": 0.403, + "step": 230 + }, + { + "epoch": 1.6247877758913414, + "grad_norm": 2.21540975350655, + "learning_rate": 9.465649821292148e-06, + "loss": 0.4014, + "step": 240 + }, + { + "epoch": 1.692699490662139, + "grad_norm": 2.2219332688595976, + "learning_rate": 9.387663583355862e-06, + "loss": 0.4104, + "step": 250 + }, + { + "epoch": 1.7606112054329373, + "grad_norm": 2.139070251547491, + "learning_rate": 9.3047349146132e-06, + "loss": 0.4106, + "step": 260 + }, + { + "epoch": 1.8285229202037352, + "grad_norm": 2.114857623710178, + "learning_rate": 9.21695722906443e-06, + "loss": 0.4055, + "step": 270 + }, + { + "epoch": 1.8964346349745331, + "grad_norm": 2.0517966830796586, + "learning_rate": 9.124429402826398e-06, + "loss": 0.4238, + "step": 280 + }, + { + "epoch": 1.964346349745331, + "grad_norm": 1.8301929541835302, + "learning_rate": 9.02725566275473e-06, + "loss": 0.3975, + "step": 290 + }, + { + "epoch": 2.027164685908319, + "grad_norm": 1.61769307419241, + "learning_rate": 8.92554546903875e-06, + "loss": 0.347, + "step": 300 + }, + { + "epoch": 2.095076400679117, + "grad_norm": 1.7835414384395631, + "learning_rate": 8.819413391901353e-06, + "loss": 0.2279, + "step": 310 + }, + { + "epoch": 2.1629881154499153, + "grad_norm": 1.7449573933209457, + "learning_rate": 8.708978982542766e-06, + "loss": 0.2456, + "step": 320 + }, + { + "epoch": 2.230899830220713, + "grad_norm": 1.8762842446890258, + "learning_rate": 8.59436663847354e-06, + "loss": 0.2394, + "step": 330 + }, + { + "epoch": 2.298811544991511, + "grad_norm": 1.7977301694269836, + "learning_rate": 8.475705463388467e-06, + "loss": 0.2537, + "step": 340 + }, + { + "epoch": 2.366723259762309, + "grad_norm": 1.693865524446407, + "learning_rate": 8.35312912173928e-06, + "loss": 0.2555, + "step": 350 + }, + { + "epoch": 2.434634974533107, + "grad_norm": 1.9696999644652728, + "learning_rate": 8.226775688169943e-06, + "loss": 0.2424, + "step": 360 + }, + { + "epoch": 2.502546689303905, + "grad_norm": 1.7928317986379525, + "learning_rate": 8.096787491984128e-06, + "loss": 0.2458, + "step": 370 + }, + { + "epoch": 2.570458404074703, + "grad_norm": 1.6035422026428618, + "learning_rate": 7.963310956820085e-06, + "loss": 0.2545, + "step": 380 + }, + { + "epoch": 2.6383701188455007, + "grad_norm": 1.7086388697991841, + "learning_rate": 7.826496435713497e-06, + "loss": 0.2544, + "step": 390 + }, + { + "epoch": 2.706281833616299, + "grad_norm": 1.7784954050779234, + "learning_rate": 7.686498041734121e-06, + "loss": 0.2587, + "step": 400 + }, + { + "epoch": 2.774193548387097, + "grad_norm": 1.8489681135255784, + "learning_rate": 7.543473474386962e-06, + "loss": 0.2537, + "step": 410 + }, + { + "epoch": 2.8421052631578947, + "grad_norm": 1.6154424089494572, + "learning_rate": 7.397583841973584e-06, + "loss": 0.2549, + "step": 420 + }, + { + "epoch": 2.910016977928693, + "grad_norm": 1.6945683455240643, + "learning_rate": 7.248993480113585e-06, + "loss": 0.2576, + "step": 430 + }, + { + "epoch": 2.9779286926994906, + "grad_norm": 1.7774529548842273, + "learning_rate": 7.09786976663073e-06, + "loss": 0.2597, + "step": 440 + }, + { + "epoch": 3.0407470288624787, + "grad_norm": 1.3291164143277696, + "learning_rate": 6.944382933012207e-06, + "loss": 0.1914, + "step": 450 + }, + { + "epoch": 3.108658743633277, + "grad_norm": 1.4399162979563198, + "learning_rate": 6.788705872653429e-06, + "loss": 0.1304, + "step": 460 + }, + { + "epoch": 3.1765704584040746, + "grad_norm": 1.4090235140542846, + "learning_rate": 6.631013946104348e-06, + "loss": 0.1348, + "step": 470 + }, + { + "epoch": 3.2444821731748728, + "grad_norm": 1.5024124740765719, + "learning_rate": 6.4714847835366625e-06, + "loss": 0.1306, + "step": 480 + }, + { + "epoch": 3.3123938879456705, + "grad_norm": 1.4037148184869794, + "learning_rate": 6.310298084654467e-06, + "loss": 0.1341, + "step": 490 + }, + { + "epoch": 3.3803056027164686, + "grad_norm": 1.5382989963970903, + "learning_rate": 6.147635416273679e-06, + "loss": 0.138, + "step": 500 + }, + { + "epoch": 3.4482173174872663, + "grad_norm": 1.4075458945664348, + "learning_rate": 5.983680007798277e-06, + "loss": 0.1367, + "step": 510 + }, + { + "epoch": 3.5161290322580645, + "grad_norm": 1.379295875695038, + "learning_rate": 5.818616544823755e-06, + "loss": 0.1341, + "step": 520 + }, + { + "epoch": 3.5840407470288627, + "grad_norm": 1.6516703780226065, + "learning_rate": 5.65263096110026e-06, + "loss": 0.1417, + "step": 530 + }, + { + "epoch": 3.6519524617996604, + "grad_norm": 1.4236024491217225, + "learning_rate": 5.485910229089739e-06, + "loss": 0.1432, + "step": 540 + }, + { + "epoch": 3.7198641765704585, + "grad_norm": 1.5628385416315445, + "learning_rate": 5.3186421493530845e-06, + "loss": 0.1378, + "step": 550 + }, + { + "epoch": 3.7877758913412563, + "grad_norm": 1.5399714943796112, + "learning_rate": 5.151015139004445e-06, + "loss": 0.1396, + "step": 560 + }, + { + "epoch": 3.8556876061120544, + "grad_norm": 1.5044051222954915, + "learning_rate": 4.983218019471027e-06, + "loss": 0.1443, + "step": 570 + }, + { + "epoch": 3.923599320882852, + "grad_norm": 1.2731582701152808, + "learning_rate": 4.815439803797483e-06, + "loss": 0.1328, + "step": 580 + }, + { + "epoch": 3.9915110356536503, + "grad_norm": 1.2890321476898117, + "learning_rate": 4.647869483734441e-06, + "loss": 0.1334, + "step": 590 + }, + { + "epoch": 4.054329371816638, + "grad_norm": 1.3973597889025486, + "learning_rate": 4.4806958168510235e-06, + "loss": 0.0788, + "step": 600 + }, + { + "epoch": 4.122241086587437, + "grad_norm": 1.1936189947404265, + "learning_rate": 4.314107113911151e-06, + "loss": 0.062, + "step": 610 + }, + { + "epoch": 4.190152801358234, + "grad_norm": 1.091238397517897, + "learning_rate": 4.148291026753159e-06, + "loss": 0.0649, + "step": 620 + }, + { + "epoch": 4.258064516129032, + "grad_norm": 1.1958745470057501, + "learning_rate": 3.983434336911636e-06, + "loss": 0.0662, + "step": 630 + }, + { + "epoch": 4.325976230899831, + "grad_norm": 1.1212711189708877, + "learning_rate": 3.819722745219617e-06, + "loss": 0.0633, + "step": 640 + }, + { + "epoch": 4.393887945670628, + "grad_norm": 1.3264229165705554, + "learning_rate": 3.657340662628116e-06, + "loss": 0.0662, + "step": 650 + }, + { + "epoch": 4.461799660441426, + "grad_norm": 1.1770811940347883, + "learning_rate": 3.4964710024786354e-06, + "loss": 0.0625, + "step": 660 + }, + { + "epoch": 4.529711375212224, + "grad_norm": 1.2943958821442108, + "learning_rate": 3.337294974462623e-06, + "loss": 0.0628, + "step": 670 + }, + { + "epoch": 4.597623089983022, + "grad_norm": 1.345364224404323, + "learning_rate": 3.1799918805000097e-06, + "loss": 0.0655, + "step": 680 + }, + { + "epoch": 4.66553480475382, + "grad_norm": 1.0977379456012588, + "learning_rate": 3.024738912766717e-06, + "loss": 0.0616, + "step": 690 + }, + { + "epoch": 4.733446519524618, + "grad_norm": 1.1868517773950296, + "learning_rate": 2.87171095409866e-06, + "loss": 0.0614, + "step": 700 + }, + { + "epoch": 4.8013582342954155, + "grad_norm": 1.110887449804357, + "learning_rate": 2.7210803809970853e-06, + "loss": 0.0635, + "step": 710 + }, + { + "epoch": 4.869269949066214, + "grad_norm": 1.2822194078234743, + "learning_rate": 2.5730168694571457e-06, + "loss": 0.0614, + "step": 720 + }, + { + "epoch": 4.937181663837012, + "grad_norm": 1.0805554157065451, + "learning_rate": 2.4276872038384037e-06, + "loss": 0.0598, + "step": 730 + }, + { + "epoch": 5.0, + "grad_norm": 2.320950039949113, + "learning_rate": 2.2852550889926067e-06, + "loss": 0.0595, + "step": 740 + }, + { + "epoch": 5.067911714770798, + "grad_norm": 0.7541720353663811, + "learning_rate": 2.1458809658603288e-06, + "loss": 0.024, + "step": 750 + }, + { + "epoch": 5.135823429541596, + "grad_norm": 0.9116539389923982, + "learning_rate": 2.009721830744201e-06, + "loss": 0.022, + "step": 760 + }, + { + "epoch": 5.203735144312394, + "grad_norm": 0.8703094574463964, + "learning_rate": 1.87693105846232e-06, + "loss": 0.0214, + "step": 770 + }, + { + "epoch": 5.271646859083192, + "grad_norm": 0.8382504818211445, + "learning_rate": 1.7476582295810314e-06, + "loss": 0.0224, + "step": 780 + }, + { + "epoch": 5.3395585738539895, + "grad_norm": 0.8489442445106106, + "learning_rate": 1.6220489619216988e-06, + "loss": 0.0212, + "step": 790 + }, + { + "epoch": 5.407470288624788, + "grad_norm": 0.8903040566842944, + "learning_rate": 1.5002447465312675e-06, + "loss": 0.0213, + "step": 800 + }, + { + "epoch": 5.475382003395586, + "grad_norm": 0.9430023584705125, + "learning_rate": 1.3823827883013723e-06, + "loss": 0.0201, + "step": 810 + }, + { + "epoch": 5.5432937181663835, + "grad_norm": 0.877235765811111, + "learning_rate": 1.2685958514155456e-06, + "loss": 0.0214, + "step": 820 + }, + { + "epoch": 5.611205432937181, + "grad_norm": 1.0592040952393595, + "learning_rate": 1.159012109798598e-06, + "loss": 0.0213, + "step": 830 + }, + { + "epoch": 5.67911714770798, + "grad_norm": 0.9416315998080429, + "learning_rate": 1.0537550027366444e-06, + "loss": 0.0191, + "step": 840 + }, + { + "epoch": 5.7470288624787775, + "grad_norm": 0.8017668057640382, + "learning_rate": 9.529430958304092e-07, + "loss": 0.0206, + "step": 850 + }, + { + "epoch": 5.814940577249575, + "grad_norm": 0.8439085846162264, + "learning_rate": 8.566899474384299e-07, + "loss": 0.0192, + "step": 860 + }, + { + "epoch": 5.882852292020374, + "grad_norm": 0.9517820142849852, + "learning_rate": 7.651039807606137e-07, + "loss": 0.0213, + "step": 870 + }, + { + "epoch": 5.950764006791172, + "grad_norm": 0.9056469249009282, + "learning_rate": 6.782883617062286e-07, + "loss": 0.0189, + "step": 880 + }, + { + "epoch": 6.01358234295416, + "grad_norm": 0.4576722465983699, + "learning_rate": 5.963408826839079e-07, + "loss": 0.0157, + "step": 890 + }, + { + "epoch": 6.081494057724957, + "grad_norm": 0.28634320188291257, + "learning_rate": 5.19353852444569e-07, + "loss": 0.0064, + "step": 900 + }, + { + "epoch": 6.149405772495755, + "grad_norm": 0.4684376300431715, + "learning_rate": 4.4741399210133154e-07, + "loss": 0.0063, + "step": 910 + }, + { + "epoch": 6.217317487266554, + "grad_norm": 0.30013129798891736, + "learning_rate": 3.8060233744356634e-07, + "loss": 0.0054, + "step": 920 + }, + { + "epoch": 6.2852292020373515, + "grad_norm": 0.5743674767243917, + "learning_rate": 3.189941476551034e-07, + "loss": 0.0059, + "step": 930 + }, + { + "epoch": 6.353140916808149, + "grad_norm": 0.42277606258594225, + "learning_rate": 2.6265882053943204e-07, + "loss": 0.0055, + "step": 940 + }, + { + "epoch": 6.421052631578947, + "grad_norm": 0.45134401222170456, + "learning_rate": 2.1165981434738025e-07, + "loss": 0.0055, + "step": 950 + }, + { + "epoch": 6.4889643463497455, + "grad_norm": 0.37311468378603657, + "learning_rate": 1.6605457629533417e-07, + "loss": 0.0049, + "step": 960 + }, + { + "epoch": 6.556876061120543, + "grad_norm": 0.4991066248356068, + "learning_rate": 1.2589447785450903e-07, + "loss": 0.0056, + "step": 970 + }, + { + "epoch": 6.624787775891341, + "grad_norm": 0.42890580429476455, + "learning_rate": 9.122475688417954e-08, + "loss": 0.0053, + "step": 980 + }, + { + "epoch": 6.6926994906621395, + "grad_norm": 0.44354296838652474, + "learning_rate": 6.208446667403167e-08, + "loss": 0.0054, + "step": 990 + }, + { + "epoch": 6.760611205432937, + "grad_norm": 0.32840772053670925, + "learning_rate": 3.8506431953060365e-08, + "loss": 0.0051, + "step": 1000 + }, + { + "epoch": 6.828522920203735, + "grad_norm": 0.6268189928914281, + "learning_rate": 2.0517211914545254e-08, + "loss": 0.0052, + "step": 1010 + }, + { + "epoch": 6.896434634974533, + "grad_norm": 0.2742284364295698, + "learning_rate": 8.137070298769312e-09, + "loss": 0.0052, + "step": 1020 + }, + { + "epoch": 6.964346349745331, + "grad_norm": 0.4536117363233092, + "learning_rate": 1.3799525671720492e-09, + "loss": 0.0053, + "step": 1030 + }, + { + "epoch": 7.0, + "step": 1036, + "total_flos": 273471074074624.0, + "train_loss": 0.24805861423295494, + "train_runtime": 7529.4023, + "train_samples_per_second": 17.517, + "train_steps_per_second": 0.138 + } + ], + "logging_steps": 10, + "max_steps": 1036, + "num_input_tokens_seen": 0, + "num_train_epochs": 7, + "save_steps": 1000, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 273471074074624.0, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/training_args.bin b/training_args.bin new file mode 100644 index 0000000..f71e484 --- /dev/null +++ b/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5616160e7eee8759f49236b6fb869b421a3a17589e48667ff83ab8ea3947a195 +size 8209 diff --git a/training_loss.png b/training_loss.png new file mode 100644 index 0000000000000000000000000000000000000000..522af073138ff6d2cb6c6447e94ec211ed342b38 GIT binary patch literal 34514 zcmdqJ_dl0y_&P?8+)LGvj3|70TW-vMGCg zj??}Aj32&#!S{YVdfa%uuIoC-F`mcsc%E07rp9GTa%OT2!zfj+DBr{|e18nX8zm)% zzle7XPQw4BTrcUm-g2;T^)PWZ$J9++9c>+4ZSR|Mx|=(@+;^}S6%-K`JTJg$lv7D;%MXd*K7KS{$Z%yr! zEVq5J?|<+0^CtO=$OvxTl1H2=p{Gw%HM!{@Q&zswF~>fN`+li0c@!5On95za@wCb# zpd>;t#p|A2jr`A`$0hAQ->bfE+gba1v>u(g>l1w z*+yzuSZHWyB?Y|#`dz4=@G1P-(khI{1HZ1G`~UyPIH{j+Rp}$xq}MA-76wX%w$~T# zW~^xG*p0iBoYHuM^T#8gp`pQ4pXWbVG1Uptde{G=m`C-&-dfL`7<;x?Ud_G~k7`}; zk>j)8BWBG}!O6)+iA$xqs?USS=y}V;9`pP4bbq`{0@vFQNyRlz+AeA?CN;492(wvv zP3YMA@+`X5vxJ1XSxz-Gw54ty+J065Qe@8lU+y#^o7ecW6h za1gN{EVEv)?99BuDDxn{zVPkaw>I|285kbJ>{IJj&0AlZ3|D$G3L7h|$z2gYdi*%d zdlEZDgH!09H7G_N8u*$BA)caVyII)uE>SOEY zbYbWB*q@hz22U;q5?_4IqZ8t6U|+pu@%G(2Sn`|2n#(>dLW2<7lLDygEn{#=Nprjyg{21?qki%H4J72^aAmVaHB z?;8<1eHxikJGd5%HYg-yqWi8xa-SrEq4D=oZ-W)x`&FSm3!+0p9I$)o#tGFi%KG2nW+QZh6W@)!G!r7Vhtl5fT%d zev23B{p=wG9zR&_f`|F-u8_db`+GYJ17~&fvFAJ{o6BQV%~nGd;)_)qh9Tzi`7LQtRI@=u-}1C$Xb7i-Q3e%yl(k2 zVruq?BFnb4l$6nIb8W*LH)tt{m5O`vu0DJy)tze)K2{&1^IBKG%n|Rz<)|Qy45ceq zNRxcG@4tBY($FpW%$YN!M~)C+m9T>AbDgo`Y|}py>+?`U&~2_e^;(=D#0Rvp)!VcDUdhd01q>@0 z7y63>^**W$l-MowXpQ#wOlE6sJoJ8-z3rtv-GArT+)9bvfa8E&DHkUvruUJ-XZv66 z@9zn0-374&FmR`Xy^Ry6Pm{`et$#STPk%~dbn)|&%DsC$V$77LrlyV~J{w#@LV0x+ zI{Y5k6?OH-nCo(3FVEhsYi=gt;^JZyF;xyOEFG%v&NGTWmF!IiK?nA@pyps>#4lYZ z&kzhg7(D+m1=GzxZi_ueewPv??7+?2a}A^g^gfY7uy!jm9Qe^lEx${{;kPR?K0ZDQ z+a%|+Jvv$&xW0zA9h?E*U+H{=%>}P@fyb0A#70I&$eLeSCS^Ge<=(##XSpMg)SXz% zZ;gdeb3{!vM^E@2>~P5KZ<_6VZ*VUl@)x)19*r~dt%ZC*!^~`bpeVoh?-m7{)KO}g zWsHKD;CEYcH0=o$<1f!oQl0P0mRak(1+G+QlUMcl#fzift6xsV3!C8k?r%-o*h~9t zySCgrhc>RIwY7d{sV16){Q(9$P?RTX(=DJDFN|09aIH?XO(qa7reKkvxOnj*Rn3R4 z*jKN@!2Qz8${1m(mBCIR+(@Hg)a0)r~C9MsQyTl)?unoRSId((GdcdmO0q{tH~F;2~n}V z)WPV3Q4RZ0gf%rf*|xx{no)WqL5`H-7gL2A58 z8X8oF5dC)HFY(eIg5whtD8lxa`b<8%ygcMFGM41ESXN#{vYhO>J2 zjF!4|j2a>SMb-wAm@SXi_EveFC#GQ3e!z@e>Nv$u4nJEcAXuVYK>ER&f_Kejt1Xj*z*+3G^m-wkM-l-(x$Xa;ldW*7(@0p z=KJWu%j+LgmW@4T!7!Hd_Au)D_v#7rbLN(o6Ehv@RO-u1m4B%WJ*SREuwI~xmRTh& zEGk+-kFrfL6qL4txqjul+D!J%gF@o~ckcz{5fQ!WPPZW~zKyQh%T zrI^!Y=q8(Re)vKCezT&fDK|vy#5w!wM0r0s0fr;&4@Lv7j&NLX=0V{OqV`y_-+rxS zvd`l;Z&u$23JD6Eay*B56bJr-HPwkMeVhQV(~O9t$f zw6u;6KbWA5|87ya^*e#8r+pPI8~~#6FQ<}&zysxW=0yj6HioWTy9V*;N^$O&*sO)M zvdY(n(m5`&hD8>X#FWfI4Gqeb>-{!Qzub!#wWP8ypJK2bETc8}?C#b(9ZGxR>C>nF z5IlRhE(o<53mcE(0@~-+9?WKC&@eHDxlI2s0rzS7{{51Ig1^s@O{_FF-&;U%x5Ty< z9vl)M&j`T6xckGM89;bc9CG?&8-HMQ%DTFYhCXZB5dZKn-_0?S2*6(iBvcJvOI0C& zn7+LhKLXQifQ&W2(T0%C#;})Af2l(>EH&4uQzU40n8mHN{#fqq?=0QW)bwAOXioq1 z>1eX=_8Bw>R!K(;5clRr%|Ybf={B>eud%W2t(j`7s$|=1b0METiK)Gjr0wYF*o0i3 zZ}I*0oS-Eap`c+UY0Q@(6cZbd8wKeLZdBT{)LjF3L4xlu2{9me10#K<#!JN}r8QOs z0}V;0Bo4Id$xSSqdU==iWpiU=n21?ZuV0d=WfaD_y|E-Dz!2`U1!3XDq;ol}Q?bz8 zuZ-v!__sG*Y|}%Ir9qfi%m@zC*FH?xO218BpI@$hpWMH8C93IK z2x>&w3T)#*94u`Igflz^K-$gKl@Mu6CtIGaCrf)Q3IGb?!a~7k;hDN+j;R>u`oh4e zAR6wICoi>3Ij{VXUXpBZ8pYwIYMy3&x$)=MWsGLT@1V-(m0{lTXV0FELzb(aotAPM z4}z3Bo*KelVZrqF{re|)1jM0tiY&qaF({#^p9aWFkvAsnW}ezI8T;-2+r} zL$F+m&v~A`c_R>1w!3p{U%IsX++#qfN&xAcAUN(6g2j;rD{|#lMq)I4v0r^zxq^C2 z9g;Clxr1Goox&W0m>lP-9fny?zcA{+7{?4D=I!>lypp~U-ETwDhQ2^ zK3+QfFvsWBf((Z3>akR?t-&2l5e=ia@0p;4PlldFr)3>=Y!T-iZ=Px)nECi(80wI1rKDzG-=RG7(W#9dwNct=(Nf7Hbf*Xzl z?96pnTdV`Xos)h&S!Utf6>IBR z@lv+)_HUF9aE4V6@$w})@Ntj`omR##$!yQ0Yn~R=`7O7MBY-@fecM|{ufp|Ah5N#6 zJL?z#uV!{H3P5I(-kZXhP0eSRf+>^{+Kc?c!ZhGB^;>_Zb9}su`XY|;-!_J-$*TWT zmX9Yy*~OXzU)%mqWLsNXJu6>6s6?|@_rF-BisU#h)1Exu#0 zG}iLX{I~S=BLD~C0N(w|xSr=Mo6r==Vz#w9^~(Lk&2i*Ud?db+9P-0smUp-Mf28-LE7*J&yK>@`JkJ_b9Ca>d1_9BOoI=I z#v?MyQX-+ix{D8CB#6K3Q(z-`g)d9|4)y`(zS^5+u`k1uN?gnyv2p8Me`xC>+w;T5 zAbMkQC>+IQh!f3?;l~%IeF2oCP)^Arp*7R*H05!J>F)Pmt5dBv^!2^F+;VO$PPHOO zhwx%GTzLW3h5u~xWey)0)ZqT#ud)14`-k61SGlko)jH+uI@kSuU)mR2h1=ZioRI{mce(la$N>&U`@Dkc#w=R)cfO(e*9(%8 zjQl#e4S)v1&X0K0r>3T!77(ahWHe|*#(qLhuKMTHwU+|52rpH6ZAkrN%CryJh6R`2 zwf$T+IXT(+@a|o@zDiG*khP=ajoLf)2I|(gdm+^0Z)C>M!<+d z*6Ql&_y%<~-be=Q?QH1u&Bw;ZYEMmPYV<8a>W6?xMP>{`PD@MU6N}TiJ&bTM1cb23 z$$M{PJn5eEaGg2xC5cfE9JUt>_Tj%7lj}inH#{!o;5LU_a$Cgy@$vEGHP;Y=_MvVS zP5556RK0^Lwp}1KsK`{+2}wxI2Fsk{<$R=mB*}&Y%$S^-N`3pbp*rg7J4HuF$6&kZ z?+GNBw!{Jcv6=%&?-zMv`gZOBt@9=Intyb3v=aR-Z82u`;J+tNpKcj@`26+jCo?lv zn>#zwA9VIT%6JP43pH=vyg4>V!e{7Zo4YHUBk&_z|D&z7muyLSlFf8+ZZnWnm_PZ@ znXj#_tuAF$FKU)vym+BCW5WQ>wz>HLmH!+*|7fA|3V~o{GSi;=05UEA8_}*a8^GPD zB}h=Z{Qg!8sh#z8693$s`&3oO`UQvbsh3yf(={>(G~Jj6UTVLzz3o0^)saTzyEPSe zHCo%y-MvJ~NLOB79*TG_aq(lweH^3oJ-_FuUdYkn`v-vpbKF@dO@{&>Kru;OeZ3R> zY9xBwqn0pVZL8*qEyNC##nJ#3Z!Y;AghP5e9eWk!T$B9zt``TbSu!^+`9)XsclYU) z+DAgchESevi7gX@fUf_+{zlD%wJx27U%CipqwNKho^RSnj*!FysOIw>$Leg>mz-Z_ zfBjTpZV-3kK7=#-sH*HW~gKn_x7N!)RN*Dxh}sK1R<4MM5LgKh$UG*;Xs*ucDUQ#Mq?WKe!I@w;il*B!Y*R z*M+(dR9k+F4=6Be9*63;`N=VUVJPIh4t}og9fJsTU3Tp#&|-lA(2&u~h3x|>*7-A9 z#J?0-;)Z~)vWm)W-5T#mYo7`r3)uF{JQs?4tYx&-+*-G;4t@5x4EYBp*G$;XX3!St zRr`ntVw!HXcX_>-5Li!EYkO{j-BDX-=W~c7+i?c;3iW5mj|uU@k#P;Bvc`l4DUI5>E{hR0@B&wK~! zf9?1UK|w*P9GyHB6O+{X@0o?Gz&?sFpSO>QWH~?HnRRQOHxI}`v(?Ejn`=ErCK3r9 zpFJEx+IWH#yuEj|>3ioy%)cH5`y@&BUL{kBWY(VB>wDd3D{kE;e^9C9;%crQ02y2f zR$&-8ot(v;hA?_6@l#*Ee96}pSnq$wBx)h~hPveSw~9@I6KE*6KPq0l7@%}^G`6O` zq|IZhrIFu;w7$MRwZn>!ZAGuIa5j6k=(e_mo2RUMQ}M6vf7!qOCd7R(JD1~K_u@^Dpu4GxRgVi9y7p?kyP2w=(6}P|MvaYaTzmquvi$Wk$-XZ zBm$`+H5L3U9nt_d5GmanCzQ8rXm6ie@1bYt_Ekbul>W2FQm*hhXCMfKU#Z25+q|sb zhwJ`cjqU=(|KUy%F|h2Vr~JXkQ0@Buda`};lX?qa^`5p)A=aI*b3L^1nM=<^AhjG0 zsIpKX8Ogw@U1R5_rl!VF{Xcc9k75r|ZLkaB2u9qvaRWodNeY0AbsckY?~>QN16MY( zd*5g07sTeBy>MMk%~kT{!p2t#{33~8sub)pjDW(RUJ0&Wjh5eMe9qHn*8ay8(ACHG zo_{q5dwKP30BRubH3P0-+kV4)>QqmeyoskaAauTjib{6riT+pd@u84%P=Wv9lYDP+ z(ox!D$EKFc?Y?LO_6oMSxU;?hnKk#}*^f{31@o3aZvjQ2^ZB<2Dz5*yKG+!YFcD+s zuXNaX0{{uCOE$#Zhpg(>_{vhTN^1V8hE0rs+PlwhxY|b=$iCm2CYoKIztS&LhZGi_ z^ha9{XqN5$5k5h|Ch)+8d>^;3<(Zk8KlcNUFbF7UXhb#3hJCfM#_W#&1+!&>M{>YM zE|yJ1h$zo-?1Wl?>PsQAG%hqPiDhOjPDC z4weIg>vQYkt=t?B-yAvE4*?3_WV|7al!~eeNQ4l;mu3*c)~gK(h$$2RS3vR)2n{7= zz2Hm;i%tZbA-Jq((dz)o2^CXfU=ReovR-Pi1d)@ zP)xs;Mc$?S@2y}FVuQXr z3kVyn&-Vq^9PEFt(|5o1Uac1R$N<1*Y>@LV0qgVR$rDE?DR-e`az#}&E1%Ms?Wxnp zLckGc#F!(M7KpJyK*%V~B0>!oAqCtDs#B9#zw-6jt|PEwfjp|~ye8%PQNRKsR1Hi> zo>KEBG7yLoO%VMSepv(XnE3bSm+mJkT6V8_5yUf!^X*&;XBY$1xU8s%kMb~#tz&HO zu0z!^U^dp>nj=|4F1Y;eou2Xrj?!s$QuW_l4>ec}jI&0uwW5Z;Tjpr4@O|9^QzBo0^=hxhzHqHyX7?n7J#Ge;;1MKuW5CV806Ylx zsfHc{`YPJ*Q%Mi%l8j3L#sa1A(cN~c>)&hrxE?c0eQEP^Ox0VzStyvrNCB8OmN`wd zXWlsRM%shQrYAR8BcmR&?wyIfT!ypgP?Gn04`PuxTv2$e=dWYq+X6@Uze8~ z_SrBP1iBAxDKGYFSfYE6n!5;S2T+er%nTD^TcvJ@YxFcJeA82LhVbP<01#pSZ zUGf@3H)W%x+_RvPzb0ngc?G@@@`nV|y6~>)U~l9A6>_P!b1uP*8UVr1D{Z#sSTn~% z+?4#_G383As3lAJEw;WR=XJ>#6rl7@PELv+Z->J$o&Zz~2f7dKyFrm!Zp2+uE+B=@ zES68Rqw*gtdIr%8f6k$114_ZPbadRE&>?}kEdHGN6DSPm6?HT2pq~NG1}<-G9Ivjf zP6Im_huF9zZbqjV3ZNxCfPjP?yf(GV8ufhyN^JXX-uLXh#~jb!`YC(q%Q~B-rR6{z zQ=|%D9*0l>%c)v?muzi6#7ntlpFVrmt+(4&A{@9^CFM(($T&DSI-m-3g3NKXwIU@& z0r+EW=o{&0YVpxBGxJq=ED6f(E-QunvDUQLi<*70AiFVm0y;SQ_wVReKDZ$RG?QY+ zuQ1>=R@#-Q`!W>DRtF)jJoO=uSRMA=vBOiST>Z))%Z@@*nq^y3T~Ch=0MI8;VTD14 z4u@FK*8|1i^>??(pi$gp@`QB2<3?A`?$Yyx9f5-jzT1CJ9j+Dw1MgAtU`Q6nJciuj zphnhtSD<4K|HB+q;HtXC=l=%gQQ%04FNsL)NnIE?&5B z!4xVoZZR<}s7u1Y0G@#H-`-)rU9As<_i1p&AeD65@2|z_VL6Whq35dWu)n*N2HXfT zPxpoXLjn%*)#z{QZeNXB0a|mJnwbSd4-2Y{Y+<+OU+&QXrhWn4HE0HC+gy$B?9>tg z#vCl|l9g5F0|<8uHEks>X;M>f5);p&Sc-fM+Rpr1e6c`+(?I_+yW8WBJrCEQu`*uQ znefI((2+xc7ziPEH%YBF=#&HZDhxS7W@|F0tDpDeNg`lbsl+Xnd(PSR3bRVNX8x?c zn-tpfW4oik>~c_WaNmwhf37@-_wvQayQDMQp6VR9$Jqe;6#b&0T49Io>}utmFIgJ; z6|Q86FWbvHtL&EQP2z>?XDtDOC%Wp#a{nK1mGlC|8YuOM%t5lx zIuk_jAn2V&u*))o@q5g?xvs3mQaa?$0tMg!*NGDZ*2T=wAxC&6=gq%?p%s_{)a+N( z)EXd51kTUff)|-WCl~7Uhs7j`Pygmqb0Qf`oC(MOx9WwJ@a7SQaH8q9SCB`1mO7fP z1a=CDQ!`5|m#Xos2vYLonHT6f0?Q869gJV;-OYmTmU_6QFe8XPB4JfeaB zMo*nAhPEVhmteGD&<1Q&U9=GVpfm@Xy3SY_)BS7Gwx|J9W$&S97EeB&Jjc|pLSlStm1x&Tm>{tKa=8?8j+CdZ!iCo& zTB8hlAlc1%Cu4jO$|h=kyaB~Li-P$DGC90?*S5@=c9`sf3>l9QXe zPR@}KwPM|uh7Hnnb3Z{-x&aFBsW>AQh}Ed&+1c5tw-7UH7P1B+7J^gWgHgLCcT97QrtTfOAS|SJH z>2ZAs^+d;Ij!RBXBJ4o@ST9XOL!j={{a?$#*iOI@QTBQk6GNc-JXg8|@;oS>(C$fk zuJWK(E@&raDmE_v=-{Not2Bj0GiyEqTeWixKW4XMh~z*wCwCr zFf!C_kIfbJSk#BAfULZ{yvOIw?yfgldT8wcwO_H}^6ta#%@0z~gTL+4c~?j=;mKW{ z-@Wd4IRc@0110kqFBQK6t{Mbb$;!tzH2#DBdZ$t z?eyyEDyHwXzFJqO2%E?S$qR`HphfH2wYK(9cJ=RHHp3sBad&ct6M{CmJFf><(w!(t zs+@UY1}R|UDhOOaf8n&d>4cbngzYtwHOAehtL%#1-0kvXo0$X-OC%w)dU=z z-x8&dBPT}=15Iz^u54|XUU81niBmR?>Nqrn{rw#W)N&FL>Eih60Ipf#W)s0Ip8eI@ zp~{fD%FyB9g1kIOK|z6=VX(fYbwl#b&(x=?p%m@L|4IjFXEW+b;@k-6t*w_-WNLC-uzURFC4~+zcO!SXdv~OGh4A9WwD;YfN-8FSJCZnuY(nC z7SRxHo_0!Y_h&m3=XmQEO$J9Yq+8|^;toQAARM}tRR=dBZU7vK4~O*GHolj60UB-^`t8N-s1-6^}JUF7ixl}**|LEZqOnWUmL;VB z)JTJG3UY}WMT^E=ZI@J@tDAwlw)_$gCCry9Hib~+vbi-8E#CyPl=-)z*4MdWj0;KQ zgZ-1Q21wkHnwL%p@v(Gyg|h68k7A*`qmNr|AF-0S8#BOSijRlw{Ftmbjby&or{l4T zGk5W=3Ux7z?JN6{(gWM197Xsfy8zf)Q|LOnRgXY|;T9Io|FJQ57Jc1|()i9__vT&< z!v(l_3corl7#DE(NC*a{u8}Z$t1RgU4BtZar+iU#%K$FpzpbvbB z0qhzl^hO=3w|D^)zgK(p2+$y?7L9+rxj?le?Kp}drNz!ZLfJb4#iT8#S4lUz}rJgP`~ltoR@ zv+G_`55O^S6!L@Zo*Ku=7TOPYOQ|66mM)n==^^R2FC$^w>mMz*O`2D=&TL<^=MF+% z6X1?f=W+#lAuLe;5n-?fCMf5C91Lc$44}#soD|h+d+|tNVgZ~#Qm^f7EHy*bk17x# z#qqIxpB;A$2cn**Nl8(l#OW^u9nD#&vBN;?F%Dt39uT+)1jd~m#9shn67k#jo*${< zIH3}Ovnw647})?iZyDGOv$`NMqq9;T zBmnR?oPvTs#;TCrLH8Qm?S;?&0Mxu(GBT{tiouM1wzU-$Fhs~8`DDDTmjuFO2yH{r zkNHERWgI9W90KFCv;jcNn?Oo>01A-#=2>W!+E=cqe0y_&9_$nXc{5a(kUE;lS5OlW zI8+SLhQMunOP1#VH-&Ed8GsBlY;1NTxj>Hq6TyE-#6Z&+N$!BpGxs|HzNWs=y!B~9 z!rl8!D|c`@&CCktPu8SSb7F&<5_FqZDtK2TEp&n|DJ$C}dTOR8j|0hukj@VSVv0W)BHeIH8!E(q*d5>(faL7bp5jJm5_ zQS7o0M*$o4M3e$UwLVFWL7#KJSq_{5@PytcD?H#mc5L>x*C1`bm8}M$*M)h}GiUz1 zZhn_79vYUylJ=DjH-k|s?sb-FW;Un(-MSKS#98}+vNGlK=$lcnO|E?fIoAZ^kYuU1 z(EP~l54r&WYgv)b8*GCZ1C<}u+_mRgf6j%!^Rn}KRC`Y6hvgyXc1;5X-}S&{R=FkM{WS5TG0l zda4oIh#W98v&Ha88EBbdpAfbL_6%r<-bzo=?SFsFM(hDzqkX~@-rWQ-hER`khDIWk z5x_zO8&>fx4R=D?@^ZETN#Q}~jO1jfI?n#8;rjP@fhYqpyX%B~eiEg=1Q7j7k>Ifs z$h+Qhms4N`&_GnYXK9%ZrEpJi4N_V%M-rPgMbrZ3MK~8R&ktr&L*P&a9)|E02;F6O zLAr>x3o)_tL!XaBZ$J!#%9s^|C8&3KPE_<+oK6*9_*uPAI09-Jf55wDV|Brx>k=z* z7>NRDS>MjCI>^G{a6mokFd)4F;B80705x&wz^Q*(5ciMj`df19kt2^VV27W7Tp?l; zAh9qorK1S-qR0TGw?WVTC%v;Tg^Z#=!8-;JwgEVwK;OOXF}PoTo+uPVs6X5$wceGh>(~+ zI!FNIMH93x~Gs) zdK({Guad&>^vtzCb8@HIl=WY^v1-doUc>V3aN&xpL?vFVtMXZH!dTm{igik7{P;%1GmZVNm zzdWEob68-zvO#XkY8CfCMvp7$zG6qxuu$uZf^jDxyFvMaeW2&QX71-7V@ao& z0y(Z$bY8>y1&cW;|3! zoYh7i7dzz_8--8o&+yDw^NV@QS&4C54KmZf;x$G#+8gy;s&o)8XxQ1Qpp>0;%%pX|Quh3QOzrb>b z%w#DGVVoqAG(nK-t^Lu`eBtSBa?>&(U?jy!(Y!6(!bhi#`05GU zBMp*{x7YTrGPal#fH`c&J%u1f&e&@)v_d{52 zFR#queFBeAmNscNs6Ap22Z#>BnRCNp>`uOEXB?=(1&I`=MmK15c7Xf{h!J5=3K--tx6{)PDO~# ztG2qCYQZfh|DajC?gaBbtlpA7v04Vh-<2sb7$I|O%s2p@oT|9KH+1)oy^sLys4KH>nZD~|}Ow&%Pxjd_`>|?+?p|}YUq+97z z*QprQ(>6pZl&kWtPm0$G?ol4j>^&45b3J*@AXi&~1Y`{PRG2vuX{Dg6mpy0!QJzuU z`Y6=mCQty9K(7iN(Qt$|!^4Q(e%tRj@hrS~-P;5YxN~ScPpViVP@&AOMyDuNmYN1K zCqxE-01cS`rK>&V(9O94tPY4qd*g=Sn9fXZ!HMeK)mCe^Q&6el;o$*@3Dn5QiRsPz zWZZLSMxBAeo+C`*V|M?*+Cx9eANqW?)YKlA@`8jj8dSgl!e%G-7Z#b7DMw>bfs57F z)@tV#K{Fo&J|iZ5fI3lUKcngPt&-;2u6sPfxc{QCUf?wh)HdjZ4NC2RKecn?o;^cd z)(~@8UZ}Y;omr|3qoE=Xi7V=R?SEEW9a|u{=sbsFHTSA3r63-wysr!uyG$rRyPmCT zM1Vl^WQSiuqU1-k)5+j@DI1Jsz#4OD$L?++dtqSK!k09uBNVt(NC=&}OKNK0w0j~W zB9tvG-q#m`gaFB0%D+Re^I2S+rC;I&m-iau)y#cvGE`KU-sLT&lW|-by`e9{8-I`< zI))bTF2OYD`~@V|AP^dJ%E|4&9^g57(qwJ6i>ej%e*wYrmEOqdf|fazkmW7?=7~{E z;p5YEyAw9vUOy!d&D)?Gj0t);6HvG{03aZQJ_+bsIiV0ddGh4C8z|!r{#BN$EvogLUocut-Gl!At z&?ip{Yv8Pi<&G49*(=wtmmUmOd3hk|#3tBm2Xv)T6B*Hq+9XkBj}$HTo>=PTpJ&PLfCU0&nyPi(cuqRkmnQaVZfA`3Ol zkbAXta#~|7g>wsVqcJH;2r-<5(}#h%6}V^sR_bl<^ z;(8ml#a;}t9>y-UV&>fvqS94l?YV8Qq14Vduux%omH1i zx}zEQUxJ3z_525X)caY}8ZdSAv-& zMM5TyflTa->*uvwU?##Dr7QSmg&pokB_uj(&YZw6DR^`K&l6tj+2EZ&W%R*KA1%Qv z=)_=`*dWukSG|I#Ij<^y+(6+cDU2+IBr~TRrg{0VCWec8ajiY!(A1xuhYi7-9rwgZ zkd670uH8!Sw5lEcG^n=}4HJ-|hNo%9spCMz&b zsG3%uj73lP>?_qe?t=xi?qM5mr_!!CWXJfxvOb%Jt;CzHAQqO;4UgyKh!?<@zA4xR z(l)fT!g&8M=d8tTtpA^t#YdvSuZJdK7y^SXlc7{wNFvfV-1uD*!1>>UpqKLy3m$xp zb(`#mye&NS2=*a*&{Sz?0WJC zFCxu_XUjy4>!)tF%B}abK-Og^0!vUR<;&pcxxhli*`O1QUq$x$rpv7N0lKd*y6;`e zltwaj+__gla%Vpj``bhppjh?nJX+Z|cfgf>zP=9V;V}De4mfyP?+(^IRW$nbPa+NR zD?--qrxhL=hTV#~2)A4eJX~Dr6qj|5$sEPz!av5{1q@)DnY93+@s6$&wtJ?#?v#Ae zASsGEv^-`Rn0ksgWr|~+0!_-UtMG9XVJ9g@h-r@_;9mJ%7`}f8nF}7i!si(*Xgfy#m9(G8F ziQj{r3A$<^pNZF+(OM zrZsv;N)*4o_NDvV0r23DzNbh!-6Ms<*xr3%GGmas11;eX2MoHcK12Wfkl-f=yl^w9 z_*Swy4r~8>iBZd$>823Fi3>UThEb~U>*YUgb`5ia?z89=p7$4RF)|!gWkctEK`pj^Z zboJi)UDu^yHjqC##SWlO7&N5TdtfnU&l(P#9^H;7Y3EhUxEO-k9N8!CthpafLNHGq z9eW4 zdjU9JVXqs%e;dzUqdK&b&a8?p4$S{eMr{PXpn!mDW)&O>6SYi(=0 z3@jgD%-MJHqtExIpBb4d9(0e9_S}EtE0gfz$fcQ`F{u_FImOU>xP+KacV z{XVCInaoh?iyY^~#K_hzg~VgJg!%szfX+>TrJ6oid~Y!=q-c( zp7u$ex>uYmg$EB}qoYJd*>LoAn6Vqm6y02sx%2;A2?uq9fy>8?A-u$2 zaN$J<+k+lIM$J1XC)dSy5cYE}j}G7Jrl?$eAW=wu)qwd33SSYe^f-`srod?u%pcCz zOu*@$COFmd7%DLnIJYYp^`q+az2|!exvZr4*!TLq_ADpPZwvvAUXqN!I<&-;$>9C_ z`?nr0#l!e-f51WSnj8bII*6N`%&qJ%bH&xZB&cu}R7ju=E5JT{$K~ceLpXZY#S{Ml zoE98^cA32v=?0-kgiaA-{y^WNcM;@L{-6ZTqJ535XMrs(<#vmsOcBjao3jQ&FxSUc z2^~RU58X;dcXtWY`Ui;&hJy~auyyC@1>1;bwwVopl?)k_ z{=ZGhT*IjHe+p90;gMzK+Kg%V-m_EjIz=nW671VG3+?xh_U=3TS-o)934uLw)y^&R z`^*y8g9*axAVB6&cp&5QTg`R8mlo6y^3ZM4XME-599NFnRh&O^$m4t2JO9i& zf(c|Y>nC9v@U8`r8({v>02~ErRwIbMoj@o8tYBYbM>$7u!Z#wT6LG4H!+laGzX?hA zQ2^60Nd{LY$@wsYJ`)cUfcG9i8&Vpv+y)iM}i4 znv;jmok7pd*5U)T3IT>r55d7l40{w9xF8|^p~V^xZK$~43pol&nSfq zT`nDW${(G@o%|BB0`HPA0r4{9xqGD8DAd((?jKpz<*a;lf(bb?PAo#>33ylGUy7>~ z7RN6m5Ueu+_6($_Kge{^VQY{*q(X}s^RGGBM`EIc|AzdUB)<}#+aakNzI&72cck@H zAbPlHa1H^IguE2EbJJuCb>ALY4JO;sgns#MjZK{+qaN;LToROhvQ+whe3-?IOZ8{* zd1O@&!HZ}RBhLUzX*e^AVeoE;aOh$y3QoUq*16g8LEL}w9C=wKER%>?qZ6w}Tl3;4 zR-PuJ&q$fTogLJg;2#qcbJ%x<&g7Bff(>w>Ze=u(It_w4;`VduU8EQLA&Um+9XkxI z-Zim-h=Cf61z&;_5os`0c;^HKYPf(X^2FJ*l+Y=Smvl-)mtGC_lE52?(6GkGZ8toh z$FH&pM7-10av3~*(0L=jB_uJj7o;E~tA$p0f$!c!@ctz5ewFYYfM^Q9Wx86klBVA= z-d`8f2|;Te>*$G}c{>M=Lu;CNnwW5b3Rg+JdH&DY~9-5}a^9#SP8 zOL9U3;E%gbKo9RGKy7q5sj+_F7_}YXB*bUQa~p487$x0pvno*#bZN_SRyox;mP2{U zNm7=BMMFPdo-Yd4S-AV=m#~>U7sOLg5yj z;2Q=k69*EBO&{F89KKm;F!UN74y;jVWv~ErHcz9B(44n>}B3q`s-U?%})n3ek6H>DY&LkKX$VY8gWl-jGMbYo)c05 z`Yf=XM=TWhsjpo&%CT0j%Imi<+^W*6D?hR&hp17gl?SBQB0AM6IC_N*2gr9nA_Lpc z$}y@G;pQnbKrRMZvInwc0Lqf#mOvijHWNPnmHIL8^2xW|n}_~5IRq65=XD#n+K?=D zhVD3F)t5)FY-0YqTIcZUHUf!wnRGJ7{2zZd=lx$|{s(r~>-*lA9={AWHd%#kpba4E zlsFy%QVF;^FyN=(84VZzKm?O_J_ggfG8UkYlW3!YzQLwj&z-l10?tliaPcrIUklp^ zX1LP*TdzeNQ5kt@B7B!n&cLUxYYhC|^CVgudXj(_=eKU#oE;sPoqqa8d**OqDp3i0 z?;57gt_vm7`3A!AZVB3a($N_0=Kq;GaM5V$#MB@02nBoa1^bet0;g9Gm+hGc`tWNH zMn3kc*#Bd?Ygv(*7-rl*a#?2G1wEJ>jhZWF{$QgafK#H4(uyor|MB-P#|!o!l;qTd zXRa~>Ab6@M7=6JhN~O0{GSE35+)mk=8Yp9tL2oEVum+bKxeyW&;mNJGw>!O^!}Xl4p9IkrDT6p zm?j@-546GXI5l(XZk6(2n8gRlD7wHyr4y~VHzhBg03V-X2o;+!2p+6I`a%AHWB7zf zqK}X%B+oIFRpCr*BJ2I3<$uT?PoJTR#P@mn%;<^3`DD<$!mdyzp*K#&qO%%sO)Y z1?d>@yvugu56#N@DYF{BE*x=$!@#CX!&TUaUzMnN2_z!C{KW$_)FpWbpzfOh?i9)R zoc_(+*ai7*Bea7bp|`pm8|#UOS*jMwk>~arJEJ~+AKqE682@7-q6N}tn-Bd;ToP;r zqz9)^@t~0?g@+EPN5Btb@a`5O>}^KIK+Hu9b?Kqa9vsWBgCza&B4)&>`5zsRPmF<* zY@gZYtg(^}u|4}@h!>q1hAjz6lXJ=VWwXGZdvKEt8|2*usV$UwPvhdIpZ^}EY%X$= z$RG@Jx@g7p$7re9l-K}Y zf`tlizyIIPH8yev_SY>KVuwu&fD9Q=7ne<&qmpluqDS1z?Hj~lCzYVH6DayE{eb&(>pnR6OA~*Fs0Ris&*?Jg0V}wP3 z$4hpnLpNfN2JYbRV?xd6JeB9(io=TWQ(Kz4tP&k!#>ozdo05s|P2}hnuR^K)Ak$Va z{``RqMNHrLI2e7X%yEcs)0E==5;5DSz~dVVzTtoCdozBB-DDWx!wmjw*tJMX51xGW zl-t3_9c+lOO~*zNDpCkf6&etp-rN)n%+sU7S~ebet}a%IFo3-)I24Ws6YZFc(JR73 ztRu*b35;`yY-jc`;2od-sdw-P)q!1x&{)R8{ah-<;!OzS*JB?ZGBXpHml#L6X~Z;QV7qIb1t4FyjYIxB^7^&SP7s%4;tv0x5>La}nS zx}A+d3}UHUb76tw#)bdZLL?rOz@#F@@dPB|O@uw$b{{}RvEq6_h~eh^SnFP0YZOEo zzC{!bMT?y26WE z@_DuIwcmwUWFO&579w?O9*N5qsV=2bnjv~OB)Si9qXv@6aTBT_m0bX z|NqCY_C$N2bwyICq>PUCqEexyp-n2O2&J@_h8Ar~LQ6|#dnwVNDWK)K4CKA zvSSplo)U=DA)pS=p-I+`IIC^R;lKKh+!+j#4^MPo>>2Yt zz!kTa&YFg@ck!eA^;LFWzwY_FDBN={-jOiooaKBpD>tW&tU0=4Y>Id!z<^P zuh&rJpdAX@IM_dWGfsEy-Ki)p&ca8cTZ%cj7jH1_V>{1}=Sh!SoT%JFubCVSc$`;- z3Aru_o6C(;y|%eKt*SeKnpDCTe6-eal5+iIpIAF1y`5eEr=ZEh}aZ$R%Na89LtaAmmd9o3#xHn zP0*`)anFrrw=B<{&C3T*NEWi3ThT*HnHflK{^C0>_&YaI#?_a9F>$9&KUyfjw=@{} zPu9z%bF-9Se!i)@m0^Ov52%C#+H)!^1(+%b81h{lKXGh>ynT-F>oKv?_0;u@d^B6< z>Ysi&{M^qX3LQh!_vGUc#`7*O6--YNqZ?4j*jS^2oIkQxM&5ft`2Lf#6B9rLL{9+A zz<70gG#|xF_AJ57%vHm zrO(OVSA&fPyfA06l|jO4>Ej>`(M!t^seT!Q%Ps8){IortC|^E zJgDVa(b(^R5V(Z_$M3yZ!q>BL`H&;$=A=Er&p+ohBf;^O$#0k(XPtU3LHkkoEn|*u z2%4h;U;E83j#oTig;wuYLrSf-&13YNS(j`+yEAw-)58k!)l91y*$_qX*)QMi85KU% zoTzJ%-4QxF`SK}e-fm2i#UdXJ#|7^9u@ypAurmjte06}Sk2d2K8VY(xCYElYx@)!4 zBs73)p~ndW+&i%HK@$q!y-zL5F}UK}+AD?_ez1n*Da72k+ul zKqiVrm3x?wDN`s%FWEFjXx7Gx`_zhU(ste7lsS&O|FU`#bkhZU=VLiz zIbY3~9CONJ+A5BvVS2K>g5?Xx`SjWOTvHW39pH@QkP{NTBdx?FC#M{oFUUX%mi5*+ zK9{iZrkF%ebqHR#f|{{~KqW(pd*c)1Txu=L!cV0Qa>=B7q~QjW+cqzZn;2CeIUo6Ct6K$giCR+pp-nk$ zZQT4*ohKBRH6mP#7arIawj(qqdYJP+hv=DGyFT5UT1;7^z?QdZPj0Hgn`c{jkRsX$ zWj+#m(0&)i@9g(yhvm<>H0G>edQoW2)pNr|fLjA?poZ_q=(3ndhtT!mMHZk&FkFuW z-r3?7=o^Q{wkJ0mO_%@QC{4z3Ko&A;=tHXkuzysRJ(=IQ8nX`y`eWaA?o4&r1 zZeYjT*xv2lCTpFCa?ZwjA3iqHxcDRzjb@7|%&Ye1Hm(XLEEtkl5AHp&gs_iDWro_v z*~P`|Ry;&P{rzwMJd>Geb(%9pcdcH%`RKOj89EBVbhXCz7^Msd=j|x2v z5D|#FNtu8d5GM$NXs6wU zs&MhvF3@M75^W#U+SWGkc6{b&d{}i)(JKu6w%swELYng8wfU&!g5jaKiGKw&DOlyv z-4^F5ciphEZQVFnM&{7EY67)`5INCiV?v{kh!cVUU5u%fvab{dBU5GQjQ2UUEIDr| zk>_`_T5Y6){k;%-o_YK1JKO1*zI6|1C>zX$@2&_Ej%@98(?Q_z!=(r;U(O2J|9!)| zs=G5XGNR++N(NLRvOsDu2t7#+{=u#kStI)uO$j#8VYHF#yQ91^4J|A*Phtk&xzbV6 z@2qC%{!z(apj#CyE{+jpuEwfhaGNk4v#j;64$NqEY@FyXPC~6*8VE5+!eRjnv5PgO zNFzU0T(*Uu;a7z|*^q5LDaxG#g%yFM5+Es& zYCzL;1iUwOj2^*(XQBwG_$v~*7`sKnOoPyk7bwAmiwa>JQAkD2wu_&>IU#P6!RPX* z>7uR7zIgs%hn(dO?`~dYo;Mry@>hNLC4hTDHSrxumD&ESh1#EB zrtq})i3lX0Jb zOm@WLjS=`)Y`KDT`2TKvE$l6R_D)AQCO)PGB#)vqw&mRFJ? zubHHaOqxGl(?mlW7U7KT|n&A*Xl?vwI<6QJt37{LA7L-oPVCcN1P4JjS#Iy!f- zRK53*tz^sohdN|mnY-8;vuUTxG>m7Z4;5)bG$lZenj*pi$ODb87 zTsF6FTK8c6RXy%2ydRXK)1$A9j{eMIu`ivW_o?Fw6a3!w`i1Fr-wEzp?9ptq5gjr> z2IOr!6#C3Lq+T~<$bQ{&CUO^e!i0WWEGj}289~$9UB>yr`m)UhRIP&d23(>vJw?3CpDn~O2@W@kM?spE zBA44@AEGId=N=%R8UD?D=$Z3%(PH(}akx4BZyc^$y7MxW)a$cmuq(&YDvPZCp~PxQ z?=@m0%xX}&<9?mnmQ$Yw-X07MJcAU3(av|8>6Nt%Eve#8bS6-XWk#p@MB?}vt2M{l zG^(Gep$F0UebcY~-E^^Q?5wDCJ&&r?mH;zWka2n>$4CMNoYTeevrVR+^4d4+msd1e zp@wYGTYEzy$fA%_S}C()%hynqUiLnzywuLz#O)^StKY=)Att=^Z?nF_t1!&LO)#_V zGLo%lU&SKGVKaKuyOn+?jjGb~1FKJck_fo9u&VBv*S%)}&o1i-NX!|luaOAV)~*Q( z`~4WzB&rzR`W47$u8U@!2{~Q#`sYYE2f;Kmb&pxxN+`2R*p|b{sk}tdC1$mK>7KqU z8FbI6^)2C{yTm_9u)WZ{<|HyY#Z3+`W7y_1@2Re9?vX6_Ojf6C&e|PkY?0H@ zq%v+V+iCgzI(ze@pXtn8NylFP=Phy@LLw>Imu0WO2HX%%1q@Qn%aQe$JijqzuA_Zo zWUGCz)$z*}XMthWqAsRG=o>fjyC(~AHKJF2{I(tKZAF%=k3ie=qkUGNK^b5*7A<1K zVLF$(h~gY2@~qH{o=AOp(MA$~E~Vm%4K=+_k=}UDJ>lXdv6V8K+Iz!3ACd^JZsLxf zxEGdZqr+iW7hf*}F!$4nGqs>e7|u=DbDs z_Zr>DvO-U+4ySG$^4rm9pPiRLjH)A=@B&8?a$XN9-j75U!fq^F>4bA9d(dK4d8Ku=s&*?2m- zeb1kBZ5KqdOlCN6DaFfT&ET?oo71dA$Cx&hfAEsHe%YpSkexQTOh3VMhgMJR9%g^V zWjdYALG2SGy+;A#X!&!Wx+;NakmXz6Q+hx?^H#5YTsD=CoPB2go*|zPv_7Zem%e9m zJM#uhDJwHF*E)b1{N%xos)U&639kPnfE-&oL#q79TZ*vzCc{K6OvCLSYESh4)~%7V zM|w21?N+#B*kcDFyE7|=9O-&7n^taR%nLN6H2v()KAaeBbL!7?&mPmv6P8};q6t#t zv2lUZuI5Ua>&US&@F8VM)74Z1$=g&h`d|(_UMsz~JrY(?=1B{#XnGR3x|y^MhZ3is z_}}}>Cq~aVrOFi65xQ`&iSu0ppQ6Qf^_qt(>jA(@pWEr7lgv#qpQh{8bb7@r*bh z<&@=_5E@9?uHGFk&;GuTneJa*^a4@hye?u}SAF9@=xM$3Ogc z&CI>-6l)LE2-KCEvtog9z^v28WAnLqbZEg^ele^1KwhFSrc+Own^TFWgreDgn>LUrG&babPsWO;3TCExUx*bQkV#%owUTf8hh$#V0Cfs83@m&!Kn z!P9@Q=HTPTBQ{bB)WSW?FSG(}g^X%9Skv>p-x7uT=!V)Q*^NU?i|?MYA3K)d5HHGU z%m4R=UPrA@Hqgv)*WTJ&CybbOr`_k<`L-s6F@@pdv5TWs{T=fQp&C}Jh4tM+k&pxG zIC$)e_N&If@9ks28Y(8d`1)7{BaeBS=70TYxQ`93UVHgH_SY;Cd{BOY+S)7GbJ=Ql zXZku?e%3k3hgmDnvQb+7H=q*6{w1l9F1nrT$a01sO8OY$xc0N6THhBNyvK4;DP0C| zj-D0kB$YB8a7cOUv=tBdm}y9KJ{nIe6@PiizUjyG{+a35kLH(aoY9QDqr9<6YrD6Z zr=GT{@ZXm@tCd=A|9KCJ4LrlmeMfjxmm{T={uCwMmj#Ri<#J^#&RFVcnU$DUK)PIJ z#5XbZ;%g$sEv~vrjo~E$D!9xf+uW#IA8a?rYf3n{-hER%@28lt&hn@vUDc-Srn)M> z`LUekOpZr(tz@i@{ZOQuqHVgNCTPR2s?k(r>41?9K&m~EdE-}OI6uI?zJrv4} zStsX3Gcl85ZKDd#@q2D<6!qfdW{c9`U9uW$(=;Tux3|k_#Eklwk6+%|;999}r&IYR zKUu6tk6}oPoq@vABUSo+ez$b4S*KXI(%J`dN_n|`;mMU&8WDz}HWC_QDL6{5`MtcI z_oh0E^<}FT?~H11BvC@@o(M`f+bWN64iu|lWUT+5?ydZp9SQ;$%^VRP*(dv$L1z2< zij(To@F&R3iJ@Wn#MmyhOOM9x3@tV|3S~QO(YNV%)}CCQi)t6>6nT?HVs%7S>Uq{w zv3kqiXmJoxV42z3fpkSP+z5SDMljI#2VP^v#%=$zV#=+bcJJEaTpqPUYGLQEUw+)l zOu_r;z5lN6&7YBy=;1giZ2P;bUib#igTw5(kMrkUxO!xUmrY-w6HTG!`O zF|8{3dbc$PS2J$-pdW|K5&3{;ugC2vdq#95NQg-oT)&vt;n8^M6`N(v_{y^)_q1bX zKfHWHRzbZlJur^yX*WeBUu7Fbwy*e8uI~)ob6AmE(9gdm>e~fdaiJeda~9GT&%0jp zpdG|EXvTKD038cdaxakYmv8w>Wgk;kWW<}F@5c-%X5O(_d2Oa|eL7#XU(x(oJy+_| z@AHA#Y&H6A`R#A^S&QYeB4HMQfWDmdDA}4EgN9(Dwa1a;^{DOq71=b%062D;Mq*i! z5fFUuqW%Pc_AQ_^kjg$c(E!8NBROm8uu~(2ba=@!`SSJrxsHf};WH-5!Dv$80N0n* z9pHOy@pff|_A9?X=MtT*#T-()a+}rTgW{+^H}JT z?Rj~AH-E)vL)oR-51Jc4`XQZ{w%xRAzmWG+g{Eces6TqCO(L@nbv;TWCo{FrCH%cK zzk7qU*W*6vISyBkGaTWo(i!;_L6X|j=pX?hAMAu5Y1X^OHs+mlQ(Q8qlVY;YG3;72 z>nDvPxr}IBa0uqkW-erFzl;8R$Xgijm(ROybmo$O6)AaR#m{Uzx0aauv@1I$udRsy zGH@#8h=opa@EeMVB1=CNRoxEFlI@Q9q9QHy2e4Ls>NKfqUi>G|0)n#|kLjGnskTNd zaNB)-H}~%{anrsR;h25fB>EBmDKFI<^y7yOyj9D{4->bVZy)CZQhlx^e7K9rmiOUe zuWK<6%_bi|04eNGz|ihW!zS{OPLN_U>eqg58*>QyhuE+*k+XfT-+%a&H=5pi7{0MS zK|ibahK>MvX@PM2a%CvM&~(M$!^}CMy0bARp9i6dhypce}iD5|nStN;x5kSoN}S6O5|dtJTov zOVLr6O+oH#F@0!FxH_gD2<`ybHQ8Tp@ZglsU@+vPK}4pAtVV1{P0~}uBfm#RK9d1k z1QOg+OZM?h#3Sd;EAOMvSS5yC%Db^{Fk4smkb={G{iWtY9WAG{GI$x!T|4zj58wOA zRrr+WtXPTAsv^ceTiAtfRqydq<#b1(Eb^RfM8-QMm$9nRxHR~8i_h)W$^wZE)@Qk z24JBHpa+=b)#* zlyu2w@1HVO+}Ytm625_*%GY8B@_bX(taK(g?(Y{#;r~+*Q>purGRzE~bU%MLE0Zz* zK>Bmv={03T(Y5;zSFBmb1R4#AEVz$nuC91prdzyi(&-%;FYLLAo&2kqNoPyqa^XLh zPxU$5JpPgV0g|%}b1=kL&WHWuQ5_!j=%aVbf)ofcFQjg;%$wz5XichY?h+l3#AWE; zMlTtl?z$Y3{gFGF>F@HV?ys<jO*C+`Zw&1Y=GNl1~nEU;hp3`U}k*&S=!z7BH<6t7pdDoz%^Zx=7GrquMK zpLwpTu4K_dtj{jq#E!IX(0B=;7w%{KpZjqA;APnt$a1kXZQ56KNkNVySUvF=%mXz8Q;_Hh}e)yLn&R50SiRKaZcDzXWNvnENQYbU(yEpxY zewP0_{H}b#plRbMv@B5Tf_Ln?d}5Nbn^E>*mRFw1=@MF%$`5llO&;}I;Fv1b z1?6bIGF7FZg*N4}a&`P3<}0rhuAK3~`MUB9gsk#UX|2oYmC46=78zNjV!Y|9_4&A zsun>HS^d`hy*Wj4*q}QeM-copm)1|?$Z5lYnCj9L_s)A<+uW%vUy67kRS;^v`YuM5 zE4O#wSZdiJ(=uC0BYml8-y=paA})YXRiR;}$ofIUFmV}fQSQ#ixx6{LjsI%N{jCVK z=N>p;CxuLc3txjrg?fIXR(`5h={40RBiSm|Jng;n3aLBXY~8L6d}E<;^-v1c-;(Ns zQZ!8@%fdsxdDl_v^3}RNz`F<=9|+VY*Wc@*k@W2L$)sndZ-Y59oJ(h``sZl|tTe(p ztTr4!Q}cjyrxr@Zr2@Mu8l5%JgPCj}o6^j_C)-qrsfn#rV}#~UrO%~i9h`!>8b^Yc zHTe$QeXo!Zzg>Tr>ig+1UWdeF^R4Vil{tcVUmo%ijBAU(t^`Agjjo zZ~fF_o!g{1YapU&|3FY+KkKAHLGXE9hoIrS^SU&pyWVlq1#M;cvvF`HeH`PpPEkjf z(knBRee+eTR*ewtHh`v!1Gk<7(0}gbt|Xe`)6UADlu}m>>2Vn((`V4P zubLEat}0Imzs^ik)vBDUEtV#lvI;Qv+Kr_@giIvP*y%BDuk1SJdYu{lsC@>ljhhs< zS0;FvTYk5fa#q0q^UoLjTA=$XE#1UJQRC3HAw9b^fd>fuJ00cou9UnlD7Cxz%sKFk zKuP|!*iZY?-TkuW%{oRc%s;-lY~EmeP^fm=eX+heve%sZZGPo|k=yiQTC?7PY4@U? zHt!BibOciU78#e{-{5}|%u&*Pw>$ZDH5pMsik+VLAwe(&Y5{v4enNA~9%hZ{;L#A( z8p10T*ytfZu*J}Uyfwm&IkZ`5pxK*83hd~kY34n1&bgp#$rK_WbzbNaZJ^R%vAfi* zmH4$VSW(I`@T#iM6_cPoeL?4BcD`8kkZ0k;j_wbfJY%XQ6wph<7#(|;lg%J!wZY1WPnEVnHf&J0Ks z5qt*j)jILA%b(u35jfxt&!=8c@qPg)IE4Zc`Ife}7exOI|842;6TVOUP%nd&e@x(G z$P|YY3rWBbO9s||<)`ONavypvpqNQ-$Tu17_=Lh0%aLd>V5Rz`o+G%!v0{)tb6tXX z4$b7pG`Ycf7Ut+m&i*t#bAx23(!9siT)E=f4e2t-@|I3Yk8y>;C9k zsl+b3jb$&#&dV)6=ZiYXb8Fr8iG<;+22OYGj?U%zTCZ=$%(0M0QGlJ8r2HD0NThTS zWhG!Bs~;Y`?)mf-^!EOdS+Utm6!3JLF>ie&>VMp7#rrR>xim%%Cm=-c7QTD>V5NhWbv@Pw# z)ZPmMH~Kd^$jt3$TwI*g=?#!TB^HL&%RsZa!iXUCJK>7OnxxQSrR8efJs+NSPwG=R2d|r{i%XzPY2doj?>xcH*HkD|cW?esrEE6hbUr*M z8=m%@eSR)krv4le*{TDrt3Pi*{|X95=hY zW7n?a=5G`z=G~JTT!W9hNrGVid+r5w84YOv5pP!lUYC@TYJ&VUg;>Gp=Ih22Iao>q zEH>i_#R>|Rgabhw1VIvi0kZhfvAd!Ke|4KNg#s~}+V-xlw&Jq7Q*m+ZgcbryTh+2p zFk%ZcJbJ6C3Y@qvFx4SSjZAX2pSbWBak=fH!6aFG$=&TZ2K>zQldplR8{|@eanc-AO>TQ^ z+1Azi!B_Td*~f~woX)5BEdD69YDs+Tdgetsi5Z5GaC1?9avtovSi88$IF~6K8M|@E zugW5wL$`(t18S>ZcenlXXfMu-D%bYc2Kfk4mn9-y(4Q{WI#B+DJM z<&25He*V8YC-*EN>=^ir5Pkx}5E1FJh%jj@*U@WgXnmBgzJURp(~d*hyP>;# z8D^oI$XhqC-N2wZQA{hh-;Kx|uHAj9!C9eyTyItYDz30&NLM_N94Od%^C)D+qlji5 ztTKs}E9f#6hsTY+e}@|?hRV*bPP{;oAmOTr>`BOml#$cps3PILWP-XzaVSorU10o>~ z$p=XQ$E6`?Bm;L28CEv7>B+5d9i(pDXkN0n)(S%9yNL%aF~6Pvyh9>|Sl>ZWxSp6R zT0$(8XkUW~>D#L3;|@Oue%y7=O~YPczl51P_p}YZ$Mpg+USz)m!HNO|Edp~~T|zTi4~~~bz^WU- z9=wkR@v8#jhJ-~$49|!F`79RMzk?2f7SHjut=Law_+)J)DbY~5+EQc3?~;+0M#4Y~ z)SyJk#S>v;qBlMZSakp3-A};$+m=Ak!V$t|^XDJztBOuY;Df~SVF%N5Yor0k<$%Bl zKfKMdvZNuWOUz5ACt+rwO(7<0yqcDB`xG(^AviHKm;uKNC{@7fZ}2^nmlIUIsbXRp zB{sKq0d{cr9HlbOeLRAHF9%2|B_9G~Td4?ct$2^@?at!hSOmmR0d{vpdX_dA;)YRy zIiOHkZ%bZSD9PLL%7L>@4KjgNWdG^V}%; z8srm4KCJnm3ej3Ih!{gAgn&BcT@*6diMa)o>1+iu#ydJY=_oK;qN5ZHF)DztA*q zgOUf7sOG1ev37}-F-7L(ZBq-ETSOM+-P09t&a3N!GKx3WF|pEmba{3us|C}@16Zt4 zsvb{y9i|T^ZWAj0eL6e+gdzP6iUbk;CmtIVA_@XrMMy}<(n%oU@Sn|fMlLRgPMi*g z4vG_)O}OiN5uYvFst*JEKs+i_))==L$km2Jipw507f^nN)plHMfRuFohp`pH!tZA4 zH1pKGyvhgD<{2=|MobvkU!n#Eg-r7ujOob?teDz3Z_In027fQZ1MXKbvj zKKehNMYD+d)e!j-B)UV_4x}Iunm~N{jJ* z?qYCC17p$KasZeih}zQJv{ALVqY6zJs9EWq^{jPgA|cia%t-yUpaLe$#mk`v-{+pT)emnD4B-Q&}-1neRJeC>!y8&`pWFo2y_OK zt-78#>Ogg_VIk=3EEIn7^&$#m6GYS!po>qzBif3!DRgfWHbW>!9+rGQJZuARxX!vR ze4%}XWe<#N{lD&&Do=HD&=>pwt_&c#Y)uxOJeK?CD0y+zEx(QTDXmgEn|h}Q)O5wdIs_G^FJB% zgKVV^mSZcvxdm}R2loJorpo$5Sr?oef@uh)gotobWn|)Zr`qUz-PxdtC*p*AtC?pK z4WV2;2v=&}&)Q@RW6J!+7#xoHBT@hK=0KdW2XdGi|*3*z5O{)Pi3gpXqo+M+>X7Sn#3>H=#Z z?AjbqAx_+Ud?q(tEvODspV^FUEYMmZdWSY3)mviE>P13EhkkbFhaD5ZDXQ+Le`#%*zEa15C?w$`A`NUU(_#8kb zGX@&ugM)gXq>?$tE@_kf=K|wtA~uhlmz|q{=&5qefAb*1C;XuBi?sOv`P=Qq@k0Ob zLB#5?uK>#7Q~#9{?i~fslPo71C@Dxh-4Fwb zWe+UZVsJ;lFL)?L9jcjSgz+Eplo`;Gn(gIpzNX{i><aRbC+a5_@b_-` zBEa~u1tJyM-~7P4iB$^50XxQgu;X(Pdt@N!W(O7bM;=} zyC*l6LEH+-Ck+LXm}=zw;H|P-jg52X!3vi2=;g#%cN9tit%w*&XU|4Lsg??FNQ5n2 z>UKy8hkU)CnL*OlZ(!96`DM2B(+c8x?Fn>bWpyW5OZC9odq6U2=f=FV|M5w?18 zZ!@%^hZavmz{j0Dt4F_8GKnU}tMPGSzqVoD{RYUA^b{EIlh+O^^BQ0)4_vj4C8h;< zhmRvikq`Ld#lyps3$i#=T+eI8+t;7`Nu=4~6sJ`-=!A879F}IbpEY8li?;s~A=l7; zb%2=1BOc4_tr_ai%W$cg#GUow*3F^O8f7leO%NMKFFe3al4j+zqacb KX