From 118553032af1034d51088911ab7109644dd47f0f Mon Sep 17 00:00:00 2001 From: ModelHub XC Date: Mon, 7 Sep 2026 18:08:16 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=EF=BC=8C=E7=94=B1ModelHub=20XC=E7=A4=BE=E5=8C=BA=E6=8F=90?= =?UTF-8?q?=E4=BE=9B=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model: israel/AfriqueQwen-14B-multiturn_2 Source: Original Platform --- .gitattributes | 37 + README.md | 61 ++ all_results.json | 8 + chat_template.jinja | 85 ++ checkpoint-34265/chat_template.jinja | 85 ++ checkpoint-34265/config.json | 75 ++ checkpoint-34265/generation_config.json | 9 + ..._zero_pp_rank_0_mp_rank_00_optim_states.pt | 3 + ..._zero_pp_rank_1_mp_rank_00_optim_states.pt | 3 + ..._zero_pp_rank_2_mp_rank_00_optim_states.pt | 3 + ..._zero_pp_rank_3_mp_rank_00_optim_states.pt | 3 + .../zero_pp_rank_0_mp_rank_00_model_states.pt | 3 + .../zero_pp_rank_1_mp_rank_00_model_states.pt | 3 + .../zero_pp_rank_2_mp_rank_00_model_states.pt | 3 + .../zero_pp_rank_3_mp_rank_00_model_states.pt | 3 + checkpoint-34265/latest | 1 + checkpoint-34265/model.safetensors | 3 + checkpoint-34265/rng_state_0.pth | 3 + checkpoint-34265/rng_state_1.pth | 3 + checkpoint-34265/rng_state_2.pth | 3 + checkpoint-34265/rng_state_3.pth | 3 + checkpoint-34265/scheduler.pt | 3 + checkpoint-34265/tokenizer.json | 3 + checkpoint-34265/tokenizer_config.json | 15 + checkpoint-34265/trainer_state.json | 272 +++++++ checkpoint-34265/training_args.bin | 3 + checkpoint-34265/zero_to_fp32.py | 760 ++++++++++++++++++ config.json | 75 ++ generation_config.json | 9 + model.safetensors | 3 + tokenizer.json | 3 + tokenizer_config.json | 15 + train_results.json | 8 + trainer_log.jsonl | 35 + trainer_state.json | 281 +++++++ training_args.bin | 3 + training_loss.png | Bin 0 -> 35158 bytes 37 files changed, 1888 insertions(+) create mode 100644 .gitattributes create mode 100644 README.md create mode 100644 all_results.json create mode 100644 chat_template.jinja create mode 100644 checkpoint-34265/chat_template.jinja create mode 100644 checkpoint-34265/config.json create mode 100644 checkpoint-34265/generation_config.json create mode 100644 checkpoint-34265/global_step34265/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt create mode 100644 checkpoint-34265/global_step34265/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt create mode 100644 checkpoint-34265/global_step34265/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt create mode 100644 checkpoint-34265/global_step34265/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt create mode 100644 checkpoint-34265/global_step34265/zero_pp_rank_0_mp_rank_00_model_states.pt create mode 100644 checkpoint-34265/global_step34265/zero_pp_rank_1_mp_rank_00_model_states.pt create mode 100644 checkpoint-34265/global_step34265/zero_pp_rank_2_mp_rank_00_model_states.pt create mode 100644 checkpoint-34265/global_step34265/zero_pp_rank_3_mp_rank_00_model_states.pt create mode 100644 checkpoint-34265/latest create mode 100644 checkpoint-34265/model.safetensors create mode 100644 checkpoint-34265/rng_state_0.pth create mode 100644 checkpoint-34265/rng_state_1.pth create mode 100644 checkpoint-34265/rng_state_2.pth create mode 100644 checkpoint-34265/rng_state_3.pth create mode 100644 checkpoint-34265/scheduler.pt create mode 100644 checkpoint-34265/tokenizer.json create mode 100644 checkpoint-34265/tokenizer_config.json create mode 100644 checkpoint-34265/trainer_state.json create mode 100644 checkpoint-34265/training_args.bin create mode 100644 checkpoint-34265/zero_to_fp32.py create mode 100644 config.json create mode 100644 generation_config.json create mode 100644 model.safetensors create mode 100644 tokenizer.json create mode 100644 tokenizer_config.json create mode 100644 train_results.json create mode 100644 trainer_log.jsonl create mode 100644 trainer_state.json create mode 100644 training_args.bin create mode 100644 training_loss.png diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..dd2bd45 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,37 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +checkpoint-34265/tokenizer.json filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..d71f12a --- /dev/null +++ b/README.md @@ -0,0 +1,61 @@ +--- +library_name: transformers +license: other +base_model: McGill-NLP/AfriqueQwen-14B +tags: +- llama-factory +- full +- generated_from_trainer +model-index: +- name: AfriqueQwen-14B-multiturn_2 + results: [] +--- + + + +# AfriqueQwen-14B-multiturn_2 + +This model is a fine-tuned version of [McGill-NLP/AfriqueQwen-14B](https://huggingface.co/McGill-NLP/AfriqueQwen-14B) on the afri_multiturn_2 dataset. + +## Model description + +More information needed + +## Intended uses & limitations + +More information needed + +## Training and evaluation data + +More information needed + +## Training procedure + +### Training hyperparameters + +The following hyperparameters were used during training: +- learning_rate: 1e-05 +- train_batch_size: 1 +- eval_batch_size: 8 +- seed: 42 +- distributed_type: multi-GPU +- num_devices: 4 +- gradient_accumulation_steps: 2 +- total_train_batch_size: 8 +- total_eval_batch_size: 32 +- optimizer: Use OptimizerNames.ADAMW_TORCH_FUSED with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments +- lr_scheduler_type: cosine +- lr_scheduler_warmup_steps: 0.1 +- num_epochs: 5.0 + +### Training results + + + +### Framework versions + +- Transformers 5.2.0 +- Pytorch 2.10.0+cu128 +- Datasets 4.0.0 +- Tokenizers 0.22.2 diff --git a/all_results.json b/all_results.json new file mode 100644 index 0000000..bf159f9 --- /dev/null +++ b/all_results.json @@ -0,0 +1,8 @@ +{ + "epoch": 5.0, + "total_flos": 879165899800576.0, + "train_loss": 0.4208107623917577, + "train_runtime": 110380.4687, + "train_samples_per_second": 2.483, + "train_steps_per_second": 0.31 +} \ No newline at end of file diff --git a/chat_template.jinja b/chat_template.jinja new file mode 100644 index 0000000..699ff8d --- /dev/null +++ b/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/checkpoint-34265/chat_template.jinja b/checkpoint-34265/chat_template.jinja new file mode 100644 index 0000000..699ff8d --- /dev/null +++ b/checkpoint-34265/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/checkpoint-34265/config.json b/checkpoint-34265/config.json new file mode 100644 index 0000000..7026c3c --- /dev/null +++ b/checkpoint-34265/config.json @@ -0,0 +1,75 @@ +{ + "architectures": [ + "Qwen3ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": null, + "dtype": "bfloat16", + "eos_token_id": 151645, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 5120, + "initializer_range": 0.02, + "intermediate_size": 17408, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 32768, + "max_window_layers": 40, + "model_type": "qwen3", + "num_attention_heads": 40, + "num_hidden_layers": 40, + "num_key_value_heads": 8, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": false, + "transformers_version": "5.2.0", + "use_cache": false, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/checkpoint-34265/generation_config.json b/checkpoint-34265/generation_config.json new file mode 100644 index 0000000..39331ca --- /dev/null +++ b/checkpoint-34265/generation_config.json @@ -0,0 +1,9 @@ +{ + "_from_model_config": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "transformers_version": "5.2.0" +} diff --git a/checkpoint-34265/global_step34265/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt b/checkpoint-34265/global_step34265/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt new file mode 100644 index 0000000..7bc178d --- /dev/null +++ b/checkpoint-34265/global_step34265/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4cc8642b3caa978720283df76bb74f70a56dfbaedf89307b61e606ff0e8ca0d1 +size 44304931131 diff --git a/checkpoint-34265/global_step34265/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt b/checkpoint-34265/global_step34265/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt new file mode 100644 index 0000000..0190645 --- /dev/null +++ b/checkpoint-34265/global_step34265/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8119100b6843d88c0c79443e69d03c8cc589fc549184c7f1ed148c987861d9a0 +size 44304931131 diff --git a/checkpoint-34265/global_step34265/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt b/checkpoint-34265/global_step34265/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt new file mode 100644 index 0000000..29ecf36 --- /dev/null +++ b/checkpoint-34265/global_step34265/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:db06408172d3f9046d576298d4cb50f8e250abc3edbfb31b796ebd8f235649d0 +size 44304931131 diff --git a/checkpoint-34265/global_step34265/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt b/checkpoint-34265/global_step34265/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt new file mode 100644 index 0000000..a37f15a --- /dev/null +++ b/checkpoint-34265/global_step34265/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e25b45ff37ba71b5bc526f2e4ae3ccd445ef0272f56041d8da776dd41779c260 +size 44304931131 diff --git a/checkpoint-34265/global_step34265/zero_pp_rank_0_mp_rank_00_model_states.pt b/checkpoint-34265/global_step34265/zero_pp_rank_0_mp_rank_00_model_states.pt new file mode 100644 index 0000000..103e8e6 --- /dev/null +++ b/checkpoint-34265/global_step34265/zero_pp_rank_0_mp_rank_00_model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e1a3c69cfa37a40b0012e147f818a20bba4e813563dcb10517df264e6ac3fd6d +size 225000 diff --git a/checkpoint-34265/global_step34265/zero_pp_rank_1_mp_rank_00_model_states.pt b/checkpoint-34265/global_step34265/zero_pp_rank_1_mp_rank_00_model_states.pt new file mode 100644 index 0000000..1e66410 --- /dev/null +++ b/checkpoint-34265/global_step34265/zero_pp_rank_1_mp_rank_00_model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9cffa6c83b5cded0b3b59efa268b1d445f3829531df00a4cdb7dea63ae97a23f +size 225000 diff --git a/checkpoint-34265/global_step34265/zero_pp_rank_2_mp_rank_00_model_states.pt b/checkpoint-34265/global_step34265/zero_pp_rank_2_mp_rank_00_model_states.pt new file mode 100644 index 0000000..edacd27 --- /dev/null +++ b/checkpoint-34265/global_step34265/zero_pp_rank_2_mp_rank_00_model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0552dc1cd3d4a9a9a2a347b48ee4bbc750221f4c507ef4097a4435b8648e4c2a +size 225000 diff --git a/checkpoint-34265/global_step34265/zero_pp_rank_3_mp_rank_00_model_states.pt b/checkpoint-34265/global_step34265/zero_pp_rank_3_mp_rank_00_model_states.pt new file mode 100644 index 0000000..481fe71 --- /dev/null +++ b/checkpoint-34265/global_step34265/zero_pp_rank_3_mp_rank_00_model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4ac7747d1566dd8677b2c5c41dfa9db4108c7f06a37b76ea63f8e7e651f53763 +size 225000 diff --git a/checkpoint-34265/latest b/checkpoint-34265/latest new file mode 100644 index 0000000..d925976 --- /dev/null +++ b/checkpoint-34265/latest @@ -0,0 +1 @@ +global_step34265 \ No newline at end of file diff --git a/checkpoint-34265/model.safetensors b/checkpoint-34265/model.safetensors new file mode 100644 index 0000000..751861e --- /dev/null +++ b/checkpoint-34265/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7c3523021a3a7fc9c90ee0106c17c8aa38a89e366e10c3523faa9987b8deef74 +size 29536666272 diff --git a/checkpoint-34265/rng_state_0.pth b/checkpoint-34265/rng_state_0.pth new file mode 100644 index 0000000..16cd5c1 --- /dev/null +++ b/checkpoint-34265/rng_state_0.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:81d5f83aeb4b3f559bd28377336d47659b320e7f6ef2e5a723d284716278a151 +size 15429 diff --git a/checkpoint-34265/rng_state_1.pth b/checkpoint-34265/rng_state_1.pth new file mode 100644 index 0000000..726c9ed --- /dev/null +++ b/checkpoint-34265/rng_state_1.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2626437dcb133ffcf003ac89603f8cce07459b93a98d760cd9419e0d6a994067 +size 15429 diff --git a/checkpoint-34265/rng_state_2.pth b/checkpoint-34265/rng_state_2.pth new file mode 100644 index 0000000..a48d05f --- /dev/null +++ b/checkpoint-34265/rng_state_2.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ae777e24d50cb7159634e1245f0697ba0fc64d5b26d535f2c80e411371a90b1c +size 15429 diff --git a/checkpoint-34265/rng_state_3.pth b/checkpoint-34265/rng_state_3.pth new file mode 100644 index 0000000..0ad0950 --- /dev/null +++ b/checkpoint-34265/rng_state_3.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:afc5a67564eebcfc961e8f1406a7418cc73497c2935a39af0232ef59f8153a6a +size 15429 diff --git a/checkpoint-34265/scheduler.pt b/checkpoint-34265/scheduler.pt new file mode 100644 index 0000000..9722ba4 --- /dev/null +++ b/checkpoint-34265/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c94495b2fdaa190f2f28183e73d97cd217405437055caeb77964e4120313faee +size 1465 diff --git a/checkpoint-34265/tokenizer.json b/checkpoint-34265/tokenizer.json new file mode 100644 index 0000000..c7afbed --- /dev/null +++ b/checkpoint-34265/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506 +size 11422650 diff --git a/checkpoint-34265/tokenizer_config.json b/checkpoint-34265/tokenizer_config.json new file mode 100644 index 0000000..ea4d101 --- /dev/null +++ b/checkpoint-34265/tokenizer_config.json @@ -0,0 +1,15 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "padding_side": "right", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/checkpoint-34265/trainer_state.json b/checkpoint-34265/trainer_state.json new file mode 100644 index 0000000..d45ac3d --- /dev/null +++ b/checkpoint-34265/trainer_state.json @@ -0,0 +1,272 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 5.0, + "eval_steps": 500, + "global_step": 34265, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.1459321415541773, + "grad_norm": 2.5889114643259257, + "learning_rate": 2.915086081120514e-06, + "loss": 1.120510009765625, + "step": 1000 + }, + { + "epoch": 0.2918642831083546, + "grad_norm": 2.2954157454578574, + "learning_rate": 5.833090166326233e-06, + "loss": 0.919832763671875, + "step": 2000 + }, + { + "epoch": 0.43779642466253194, + "grad_norm": 1.9740216723771231, + "learning_rate": 8.751094251531953e-06, + "loss": 0.866156005859375, + "step": 3000 + }, + { + "epoch": 0.5837285662167092, + "grad_norm": 1.6025712306944424, + "learning_rate": 9.991513345767592e-06, + "loss": 0.8476719970703125, + "step": 4000 + }, + { + "epoch": 0.7296607077708865, + "grad_norm": 1.6016651354512892, + "learning_rate": 9.936020028278053e-06, + "loss": 0.8244376831054687, + "step": 5000 + }, + { + "epoch": 0.8755928493250639, + "grad_norm": 1.6615443914575065, + "learning_rate": 9.829343371836088e-06, + "loss": 0.79805810546875, + "step": 6000 + }, + { + "epoch": 1.021452024808464, + "grad_norm": 1.739839844907049, + "learning_rate": 9.672589544454328e-06, + "loss": 0.761253662109375, + "step": 7000 + }, + { + "epoch": 1.1673841663626414, + "grad_norm": 1.8214814374223849, + "learning_rate": 9.46738398205746e-06, + "loss": 0.614188232421875, + "step": 8000 + }, + { + "epoch": 1.3133163079168186, + "grad_norm": 1.5403816299740094, + "learning_rate": 9.215854533761766e-06, + "loss": 0.6142713623046875, + "step": 9000 + }, + { + "epoch": 1.459248449470996, + "grad_norm": 1.8486916036138623, + "learning_rate": 8.920609397454381e-06, + "loss": 0.61425341796875, + "step": 10000 + }, + { + "epoch": 1.6051805910251733, + "grad_norm": 1.9343055796796615, + "learning_rate": 8.584710074466158e-06, + "loss": 0.613021240234375, + "step": 11000 + }, + { + "epoch": 1.7511127325793505, + "grad_norm": 1.3591298836511538, + "learning_rate": 8.211639623780629e-06, + "loss": 0.6173475341796875, + "step": 12000 + }, + { + "epoch": 1.897044874133528, + "grad_norm": 1.4881517892702774, + "learning_rate": 7.805266544962458e-06, + "loss": 0.6191266479492188, + "step": 13000 + }, + { + "epoch": 2.042904049616928, + "grad_norm": 1.7095977461968044, + "learning_rate": 7.3698046643160645e-06, + "loss": 0.5406383056640625, + "step": 14000 + }, + { + "epoch": 2.1888361911711054, + "grad_norm": 1.3383769030634658, + "learning_rate": 6.909769440229038e-06, + "loss": 0.37209414672851565, + "step": 15000 + }, + { + "epoch": 2.334768332725283, + "grad_norm": 1.5984154081302522, + "learning_rate": 6.4299311407857035e-06, + "loss": 0.37489187622070314, + "step": 16000 + }, + { + "epoch": 2.48070047427946, + "grad_norm": 1.5178018437242096, + "learning_rate": 5.935265379168761e-06, + "loss": 0.3743317260742188, + "step": 17000 + }, + { + "epoch": 2.6266326158336373, + "grad_norm": 2.243985181618142, + "learning_rate": 5.430901519764892e-06, + "loss": 0.3719102172851563, + "step": 18000 + }, + { + "epoch": 2.7725647573878147, + "grad_norm": 1.817398251882934, + "learning_rate": 4.9220694899697185e-06, + "loss": 0.3770401611328125, + "step": 19000 + }, + { + "epoch": 2.918496898941992, + "grad_norm": 1.400063403289345, + "learning_rate": 4.414045549219315e-06, + "loss": 0.37033236694335936, + "step": 20000 + }, + { + "epoch": 3.064356074425392, + "grad_norm": 1.9568348297606881, + "learning_rate": 3.912097577588397e-06, + "loss": 0.28135635375976564, + "step": 21000 + }, + { + "epoch": 3.2102882159795696, + "grad_norm": 1.6677571993666531, + "learning_rate": 3.4214304512770823e-06, + "loss": 0.1744761199951172, + "step": 22000 + }, + { + "epoch": 3.356220357533747, + "grad_norm": 1.6030223311676182, + "learning_rate": 2.9471320714071095e-06, + "loss": 0.17265531921386718, + "step": 23000 + }, + { + "epoch": 3.502152499087924, + "grad_norm": 2.0719250695035543, + "learning_rate": 2.4941206057740675e-06, + "loss": 0.1734700469970703, + "step": 24000 + }, + { + "epoch": 3.6480846406421015, + "grad_norm": 1.715926337353297, + "learning_rate": 2.06709349062457e-06, + "loss": 0.17093397521972656, + "step": 25000 + }, + { + "epoch": 3.7940167821962785, + "grad_norm": 1.6180812645074687, + "learning_rate": 1.6704787212769829e-06, + "loss": 0.16509759521484374, + "step": 26000 + }, + { + "epoch": 3.939948923750456, + "grad_norm": 2.048625803274215, + "learning_rate": 1.3083889366705216e-06, + "loss": 0.16218829345703126, + "step": 27000 + }, + { + "epoch": 4.085808099233856, + "grad_norm": 0.9085706761456963, + "learning_rate": 9.845787739562829e-07, + "loss": 0.10715762329101562, + "step": 28000 + }, + { + "epoch": 4.231740240788033, + "grad_norm": 1.2437558088436274, + "learning_rate": 7.024059353355333e-07, + "loss": 0.0650710220336914, + "step": 29000 + }, + { + "epoch": 4.377672382342211, + "grad_norm": 1.0557746493951432, + "learning_rate": 4.64796370857008e-07, + "loss": 0.06532522583007813, + "step": 30000 + }, + { + "epoch": 4.523604523896388, + "grad_norm": 1.6388458174836673, + "learning_rate": 2.7421393820510846e-07, + "loss": 0.06470259857177735, + "step": 31000 + }, + { + "epoch": 4.669536665450566, + "grad_norm": 1.2105021817675277, + "learning_rate": 1.326348540874095e-07, + "loss": 0.06277722549438476, + "step": 32000 + }, + { + "epoch": 4.815468807004743, + "grad_norm": 1.0929519804389642, + "learning_rate": 4.152720214406214e-08, + "loss": 0.06431336212158204, + "step": 33000 + }, + { + "epoch": 4.96140094855892, + "grad_norm": 1.5036224042092285, + "learning_rate": 1.8357098688476238e-09, + "loss": 0.06442949676513672, + "step": 34000 + } + ], + "logging_steps": 1000, + "max_steps": 34265, + "num_input_tokens_seen": 0, + "num_train_epochs": 5, + "save_steps": 50000, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 879165899800576.0, + "train_batch_size": 1, + "trial_name": null, + "trial_params": null +} diff --git a/checkpoint-34265/training_args.bin b/checkpoint-34265/training_args.bin new file mode 100644 index 0000000..72e46a6 --- /dev/null +++ b/checkpoint-34265/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fb0a977c0c6f36154f1a9b9fa166cf35a0df584dc1bc5a147b821c8d5bde01ab +size 7377 diff --git a/checkpoint-34265/zero_to_fp32.py b/checkpoint-34265/zero_to_fp32.py new file mode 100644 index 0000000..5995d6e --- /dev/null +++ b/checkpoint-34265/zero_to_fp32.py @@ -0,0 +1,760 @@ +#!/usr/bin/env python + +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team + +# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets +# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in +# the future. Once extracted, the weights don't require DeepSpeed and can be used in any +# application. +# +# example: +# python zero_to_fp32.py . output_dir/ +# or +# python zero_to_fp32.py . output_dir/ --safe_serialization + +import argparse +import torch +import glob +import math +import os +import re +import gc +import json +import numpy as np +from tqdm import tqdm +from collections import OrderedDict +from dataclasses import dataclass + +# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with +# DeepSpeed data structures it has to be available in the current python environment. +from deepspeed.utils import logger +from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS, + FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES, + FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS) + + +@dataclass +class zero_model_state: + buffers: dict() + param_shapes: dict() + shared_params: list + ds_version: int + frozen_param_shapes: dict() + frozen_param_fragments: dict() + + +debug = 0 + +# load to cpu +device = torch.device('cpu') + + +def atoi(text): + return int(text) if text.isdigit() else text + + +def natural_keys(text): + ''' + alist.sort(key=natural_keys) sorts in human order + http://nedbatchelder.com/blog/200712/human_sorting.html + (See Toothy's implementation in the comments) + ''' + return [atoi(c) for c in re.split(r'(\d+)', text)] + + +def get_model_state_file(checkpoint_dir, zero_stage): + if not os.path.isdir(checkpoint_dir): + raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist") + + # there should be only one file + if zero_stage <= 2: + file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt") + elif zero_stage == 3: + file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt") + + if not os.path.exists(file): + raise FileNotFoundError(f"can't find model states file at '{file}'") + + return file + + +def get_checkpoint_files(checkpoint_dir, glob_pattern): + # XXX: need to test that this simple glob rule works for multi-node setup too + ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys) + + if len(ckpt_files) == 0: + raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'") + + return ckpt_files + + +def get_optim_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt") + + +def get_model_state_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_model_states.pt") + + +def parse_model_states(files): + zero_model_states = [] + for file in files: + state_dict = torch.load(file, map_location=device, weights_only=False) + + if BUFFER_NAMES not in state_dict: + raise ValueError(f"{file} is not a model state checkpoint") + buffer_names = state_dict[BUFFER_NAMES] + if debug: + print("Found buffers:", buffer_names) + + # recover just the buffers while restoring them to fp32 if they were saved in fp16 + buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names} + param_shapes = state_dict[PARAM_SHAPES] + + # collect parameters that are included in param_shapes + param_names = [] + for s in param_shapes: + for name in s.keys(): + param_names.append(name) + + # update with frozen parameters + frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None) + if frozen_param_shapes is not None: + if debug: + print(f"Found frozen_param_shapes: {frozen_param_shapes}") + param_names += list(frozen_param_shapes.keys()) + + # handle shared params + shared_params = [[k, v] for k, v in state_dict["shared_params"].items()] + + ds_version = state_dict.get(DS_VERSION, None) + + frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None) + + z_model_state = zero_model_state(buffers=buffers, + param_shapes=param_shapes, + shared_params=shared_params, + ds_version=ds_version, + frozen_param_shapes=frozen_param_shapes, + frozen_param_fragments=frozen_param_fragments) + zero_model_states.append(z_model_state) + + return zero_model_states + + +def parse_optim_states(files, ds_checkpoint_dir): + total_files = len(files) + state_dicts = [] + for f in tqdm(files, desc='Loading checkpoint shards'): + state_dict = torch.load(f, map_location=device, mmap=True, weights_only=False) + # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights + # and also handle the case where it was already removed by another helper script + state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None) + state_dicts.append(state_dict) + + if ZERO_STAGE not in state_dicts[0][OPTIMIZER_STATE_DICT]: + raise ValueError(f"{files[0]} is not a zero checkpoint") + zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE] + world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT] + + # For ZeRO-2 each param group can have different partition_count as data parallelism for expert + # parameters can be different from data parallelism for non-expert parameters. So we can just + # use the max of the partition_count to get the dp world_size. + + if type(world_size) is list: + world_size = max(world_size) + + if world_size != total_files: + raise ValueError( + f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. " + "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes." + ) + + # the groups are named differently in each stage + if zero_stage <= 2: + fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS + elif zero_stage == 3: + fp32_groups_key = FP32_FLAT_GROUPS + else: + raise ValueError(f"unknown zero stage {zero_stage}") + + fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))] + return zero_stage, world_size, fp32_flat_groups + + +def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters): + """ + Returns fp32 state_dict reconstructed from ds checkpoint + + Args: + - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are) + + """ + print(f"Processing zero checkpoint '{ds_checkpoint_dir}'") + + optim_files = get_optim_files(ds_checkpoint_dir) + zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir) + print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}") + + model_files = get_model_state_files(ds_checkpoint_dir) + + zero_model_states = parse_model_states(model_files) + print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}') + + if zero_stage <= 2: + return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + elif zero_stage == 3: + return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + + +def _zero2_merge_frozen_params(state_dict, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + frozen_param_fragments = zero_model_states[0].frozen_param_fragments + + if debug: + num_elem = sum(s.numel() for s in frozen_param_shapes.values()) + print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in frozen_param_fragments.values()]) + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + state_dict[name] = frozen_param_fragments[name] + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +def _has_callable(obj, fn): + attr = getattr(obj, fn, None) + return callable(attr) + + +def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + + # Reconstruction protocol: + # + # XXX: document this + + if debug: + for i in range(world_size): + for j in range(len(fp32_flat_groups[0])): + print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}") + + # XXX: memory usage doubles here (zero2) + num_param_groups = len(fp32_flat_groups[0]) + merged_single_partition_of_fp32_groups = [] + for i in range(num_param_groups): + merged_partitions = [sd[i] for sd in fp32_flat_groups] + full_single_fp32_vector = torch.cat(merged_partitions, 0) + merged_single_partition_of_fp32_groups.append(full_single_fp32_vector) + avail_numel = sum( + [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups]) + + if debug: + wanted_params = sum([len(shapes) for shapes in param_shapes]) + wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes]) + # not asserting if there is a mismatch due to possible padding + print(f"Have {avail_numel} numels to process.") + print(f"Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + total_numel = 0 + total_params = 0 + for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups): + offset = 0 + avail_numel = full_single_fp32_vector.numel() + for name, shape in shapes.items(): + + unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape) + total_numel += unpartitioned_numel + total_params += 1 + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape) + offset += unpartitioned_numel + + # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and + # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex + # paddings performed in the code it's almost impossible to predict the exact numbers w/o the + # live optimizer object, so we are checking that the numbers are within the right range + align_to = 2 * world_size + + def zero2_align(x): + return align_to * math.ceil(x / align_to) + + if debug: + print(f"original offset={offset}, avail_numel={avail_numel}") + + offset = zero2_align(offset) + avail_numel = zero2_align(avail_numel) + + if debug: + print(f"aligned offset={offset}, avail_numel={avail_numel}") + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero2_merge_frozen_params(state_dict, zero_model_states) + + _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def zero3_partitioned_param_info(unpartitioned_numel, world_size): + remainder = unpartitioned_numel % world_size + padding_numel = (world_size - remainder) if remainder else 0 + partitioned_numel = math.ceil(unpartitioned_numel / world_size) + return partitioned_numel, padding_numel + + +def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + if debug: + for i in range(world_size): + num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values()) + print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in zero_model_states[0].frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states) + state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape) + + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +class GatheredTensor: + """ + A pseudo tensor that collects partitioned weights. + It is more memory efficient when there are multiple groups. + """ + + def __init__(self, flat_groups, flat_groups_offset, offset, partitioned_numel, shape): + self.flat_groups = flat_groups + self.flat_groups_offset = flat_groups_offset + self.offset = offset + self.partitioned_numel = partitioned_numel + self.shape = shape + self.dtype = self.flat_groups[0][0].dtype + + def contiguous(self): + """ + Merge partitioned weights from flat_groups into a single tensor. + """ + end_idx = self.offset + self.partitioned_numel + world_size = len(self.flat_groups) + pad_flat_param_chunks = [] + + for rank_i in range(world_size): + # for each rank, we need to collect weights from related group/groups + flat_groups_at_rank_i = self.flat_groups[rank_i] + start_group_id = None + end_group_id = None + for group_id in range(len(self.flat_groups_offset)): + if self.flat_groups_offset[group_id] <= self.offset < self.flat_groups_offset[group_id + 1]: + start_group_id = group_id + if self.flat_groups_offset[group_id] < end_idx <= self.flat_groups_offset[group_id + 1]: + end_group_id = group_id + break + # collect weights from related group/groups + for group_id in range(start_group_id, end_group_id + 1): + flat_tensor = flat_groups_at_rank_i[group_id] + start_offset = self.offset - self.flat_groups_offset[group_id] + end_offset = min(end_idx, self.flat_groups_offset[group_id + 1]) - self.flat_groups_offset[group_id] + pad_flat_param_chunks.append(flat_tensor[start_offset:end_offset]) + + # collect weights from all ranks + pad_flat_param = torch.cat(pad_flat_param_chunks, dim=0) + param = pad_flat_param[:self.shape.numel()].view(self.shape).contiguous() + return param + + +def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + avail_numel = sum([flat_group.numel() for flat_group in fp32_flat_groups[0]]) * world_size + + # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each + # param, re-consolidating each param, while dealing with padding if any + + # merge list of dicts, preserving order + param_shapes = {k: v for d in param_shapes for k, v in d.items()} + + if debug: + for i in range(world_size): + print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}") + + wanted_params = len(param_shapes) + wanted_numel = sum(shape.numel() for shape in param_shapes.values()) + # not asserting if there is a mismatch due to possible padding + avail_numel = fp32_flat_groups[0].numel() * world_size + print(f"Trainable params: Have {avail_numel} numels to process.") + print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + offset = 0 + total_numel = 0 + total_params = 0 + flat_groups_offset = [0] + list(np.cumsum([flat_tensor.numel() for flat_tensor in fp32_flat_groups[0]])) + for name, shape in tqdm(param_shapes.items(), desc='Gathering sharded weights'): + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + total_params += 1 + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + # memory efficient tensor + tensor = GatheredTensor(fp32_flat_groups, flat_groups_offset, offset, partitioned_numel, shape) + state_dict[name] = tensor + offset += partitioned_numel + + offset *= world_size + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero3_merge_frozen_params(state_dict, world_size, zero_model_states) + + _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def to_torch_tensor(state_dict, return_empty_tensor=False): + """ + Convert state_dict of GatheredTensor to torch tensor + """ + torch_state_dict = {} + converted_tensors = {} + for name, tensor in state_dict.items(): + tensor_id = id(tensor) + if tensor_id in converted_tensors: # shared tensors + shared_tensor = torch_state_dict[converted_tensors[tensor_id]] + torch_state_dict[name] = shared_tensor + else: + converted_tensors[tensor_id] = name + if return_empty_tensor: + torch_state_dict[name] = torch.empty(tensor.shape, dtype=tensor.dtype) + else: + torch_state_dict[name] = tensor.contiguous() + return torch_state_dict + + +def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, + tag=None, + exclude_frozen_parameters=False, + lazy_mode=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with + ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example + via a model hub. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + - ``lazy_mode``: get state_dict in lazy mode. It returns a dict of pesduo tensor instead of torch tensor, which is more memory efficient. + Convert the pesduo tensor to torch tensor by ``.contiguous()`` + + Returns: + - pytorch ``state_dict`` + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + # do the training and checkpoint saving + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu + model = model.cpu() # move to cpu + model.load_state_dict(state_dict) + # submit to model hub or save the model to share with others + + In this example the ``model`` will no longer be usable in the deepspeed context of the same + application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead. + + Note: the above usage may not work if your application doesn't have sufficient free CPU memory. + You may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with + the checkpoint. Or you can load state_dict in lazy mode :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, lazy_mode=True) # not on cpu + for name, lazy_tensor in state_dict.item(): + tensor = lazy_tensor.contiguous() # to cpu + print(name, tensor) + # del tensor to release memory if it no longer in use + """ + if tag is None: + latest_path = os.path.join(checkpoint_dir, 'latest') + if os.path.isfile(latest_path): + with open(latest_path, 'r') as fd: + tag = fd.read().strip() + else: + raise ValueError(f"Unable to find 'latest' file at {latest_path}") + + ds_checkpoint_dir = os.path.join(checkpoint_dir, tag) + + if not os.path.isdir(ds_checkpoint_dir): + raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist") + + state_dict = _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters) + if lazy_mode: + return state_dict + else: + return to_torch_tensor(state_dict) + + +def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, + output_dir, + max_shard_size="5GB", + safe_serialization=False, + tag=None, + exclude_frozen_parameters=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be + loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``output_dir``: directory to the pytorch fp32 state_dict output files + - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB + - ``safe_serialization``: whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`). + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + """ + + # Dependency pre-check + if safe_serialization: + try: + from safetensors.torch import save_file + except ImportError: + print('If you want to use `safe_serialization`, please `pip install safetensors`') + raise + if max_shard_size is not None: + try: + from huggingface_hub import split_torch_state_dict_into_shards + except ImportError: + print('If you want to use `max_shard_size`, please `pip install huggingface_hub`') + raise + + # Convert zero checkpoint to state_dict + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, + tag, + exclude_frozen_parameters, + lazy_mode=True) + + # Shard the model if it is too big. + weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin" + if max_shard_size is not None: + filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors") + # an memory-efficient approach for sharding + empty_state_dict = to_torch_tensor(state_dict, return_empty_tensor=True) + state_dict_split = split_torch_state_dict_into_shards(empty_state_dict, + filename_pattern=filename_pattern, + max_shard_size=max_shard_size) + else: + from collections import namedtuple + StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"]) + state_dict_split = StateDictSplit(is_sharded=False, + filename_to_tensors={weights_name: list(state_dict.keys())}) + + # Save the model by shard + os.makedirs(output_dir, exist_ok=True) + filename_to_tensors = state_dict_split.filename_to_tensors.items() + for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"): + shard_state_dict = {tensor_name: state_dict[tensor_name] for tensor_name in tensors} + shard_state_dict = to_torch_tensor(shard_state_dict) + output_path = os.path.join(output_dir, shard_file) + if safe_serialization: + save_file(shard_state_dict, output_path, metadata={"format": "pt"}) + else: + torch.save(shard_state_dict, output_path) + # release the memory of current shard + for tensor_name in list(shard_state_dict.keys()): + del state_dict[tensor_name] + del shard_state_dict[tensor_name] + del shard_state_dict + gc.collect() + + # Save index if sharded + if state_dict_split.is_sharded: + index = { + "metadata": state_dict_split.metadata, + "weight_map": state_dict_split.tensor_to_filename, + } + save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json" + save_index_file = os.path.join(output_dir, save_index_file) + with open(save_index_file, "w", encoding="utf-8") as f: + content = json.dumps(index, indent=2, sort_keys=True) + "\n" + f.write(content) + + +def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None): + """ + 1. Put the provided model to cpu + 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` + 3. Load it into the provided model + + Args: + - ``model``: the model object to update + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + + Returns: + - ``model`: modified model + + Make sure you have plenty of CPU memory available before you call this function. If you don't + have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it + conveniently placed for you in the checkpoint folder. + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint + model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir) + # submit to model hub or save the model to share with others + + Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context + of the same application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + """ + logger.info("Extracting fp32 weights") + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag) + + logger.info("Overwriting model with fp32 weights") + model = model.cpu() + model.load_state_dict(state_dict, strict=False) + + return model + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("checkpoint_dir", + type=str, + help="path to the desired checkpoint folder, e.g., path/checkpoint-12") + parser.add_argument("output_dir", + type=str, + help="directory to the pytorch fp32 state_dict output files" + "(e.g. path/checkpoint-12-output/)") + parser.add_argument( + "--max_shard_size", + type=str, + default="5GB", + help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size" + "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`" + "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances" + "without CPU OOM issues.") + parser.add_argument( + "--safe_serialization", + default=False, + action='store_true', + help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).") + parser.add_argument("-t", + "--tag", + type=str, + default=None, + help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1") + parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters") + parser.add_argument("-d", "--debug", action='store_true', help="enable debug") + args = parser.parse_args() + + debug = args.debug + + convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir, + args.output_dir, + max_shard_size=args.max_shard_size, + safe_serialization=args.safe_serialization, + tag=args.tag, + exclude_frozen_parameters=args.exclude_frozen_parameters) diff --git a/config.json b/config.json new file mode 100644 index 0000000..7026c3c --- /dev/null +++ b/config.json @@ -0,0 +1,75 @@ +{ + "architectures": [ + "Qwen3ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": null, + "dtype": "bfloat16", + "eos_token_id": 151645, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 5120, + "initializer_range": 0.02, + "intermediate_size": 17408, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 32768, + "max_window_layers": 40, + "model_type": "qwen3", + "num_attention_heads": 40, + "num_hidden_layers": 40, + "num_key_value_heads": 8, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": false, + "transformers_version": "5.2.0", + "use_cache": false, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..39331ca --- /dev/null +++ b/generation_config.json @@ -0,0 +1,9 @@ +{ + "_from_model_config": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "transformers_version": "5.2.0" +} diff --git a/model.safetensors b/model.safetensors new file mode 100644 index 0000000..751861e --- /dev/null +++ b/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7c3523021a3a7fc9c90ee0106c17c8aa38a89e366e10c3523faa9987b8deef74 +size 29536666272 diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..c7afbed --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506 +size 11422650 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..ea4d101 --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,15 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "padding_side": "right", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/train_results.json b/train_results.json new file mode 100644 index 0000000..bf159f9 --- /dev/null +++ b/train_results.json @@ -0,0 +1,8 @@ +{ + "epoch": 5.0, + "total_flos": 879165899800576.0, + "train_loss": 0.4208107623917577, + "train_runtime": 110380.4687, + "train_samples_per_second": 2.483, + "train_steps_per_second": 0.31 +} \ No newline at end of file diff --git a/trainer_log.jsonl b/trainer_log.jsonl new file mode 100644 index 0000000..397afa6 --- /dev/null +++ b/trainer_log.jsonl @@ -0,0 +1,35 @@ +{"current_steps": 1000, "total_steps": 34265, "loss": 1.120510009765625, "lr": 2.915086081120514e-06, "epoch": 0.1459321415541773, "percentage": 2.92, "elapsed_time": "0:53:35", "remaining_time": "1 day, 5:43:00"} +{"current_steps": 2000, "total_steps": 34265, "loss": 0.919832763671875, "lr": 5.833090166326233e-06, "epoch": 0.2918642831083546, "percentage": 5.84, "elapsed_time": "1:47:04", "remaining_time": "1 day, 4:47:29"} +{"current_steps": 3000, "total_steps": 34265, "loss": 0.866156005859375, "lr": 8.751094251531953e-06, "epoch": 0.43779642466253194, "percentage": 8.76, "elapsed_time": "2:40:42", "remaining_time": "1 day, 3:54:52"} +{"current_steps": 4000, "total_steps": 34265, "loss": 0.8476719970703125, "lr": 9.991513345767592e-06, "epoch": 0.5837285662167092, "percentage": 11.67, "elapsed_time": "3:34:16", "remaining_time": "1 day, 3:01:15"} +{"current_steps": 5000, "total_steps": 34265, "loss": 0.8244376831054687, "lr": 9.936020028278053e-06, "epoch": 0.7296607077708865, "percentage": 14.59, "elapsed_time": "4:27:42", "remaining_time": "1 day, 2:06:52"} +{"current_steps": 6000, "total_steps": 34265, "loss": 0.79805810546875, "lr": 9.829343371836088e-06, "epoch": 0.8755928493250639, "percentage": 17.51, "elapsed_time": "5:21:10", "remaining_time": "1 day, 1:13:00"} +{"current_steps": 7000, "total_steps": 34265, "loss": 0.761253662109375, "lr": 9.672589544454328e-06, "epoch": 1.021452024808464, "percentage": 20.43, "elapsed_time": "6:14:32", "remaining_time": "1 day, 0:18:49"} +{"current_steps": 8000, "total_steps": 34265, "loss": 0.614188232421875, "lr": 9.46738398205746e-06, "epoch": 1.1673841663626414, "percentage": 23.35, "elapsed_time": "7:08:07", "remaining_time": "23:25:35"} +{"current_steps": 9000, "total_steps": 34265, "loss": 0.6142713623046875, "lr": 9.215854533761766e-06, "epoch": 1.3133163079168186, "percentage": 26.27, "elapsed_time": "8:01:40", "remaining_time": "22:32:11"} +{"current_steps": 10000, "total_steps": 34265, "loss": 0.61425341796875, "lr": 8.920609397454381e-06, "epoch": 1.459248449470996, "percentage": 29.18, "elapsed_time": "8:55:12", "remaining_time": "21:38:40"} +{"current_steps": 11000, "total_steps": 34265, "loss": 0.613021240234375, "lr": 8.584710074466158e-06, "epoch": 1.6051805910251733, "percentage": 32.1, "elapsed_time": "9:48:50", "remaining_time": "20:45:23"} +{"current_steps": 12000, "total_steps": 34265, "loss": 0.6173475341796875, "lr": 8.211639623780629e-06, "epoch": 1.7511127325793505, "percentage": 35.02, "elapsed_time": "10:42:19", "remaining_time": "19:51:47"} +{"current_steps": 13000, "total_steps": 34265, "loss": 0.6191266479492188, "lr": 7.805266544962458e-06, "epoch": 1.897044874133528, "percentage": 37.94, "elapsed_time": "11:35:56", "remaining_time": "18:58:23"} +{"current_steps": 14000, "total_steps": 34265, "loss": 0.5406383056640625, "lr": 7.3698046643160645e-06, "epoch": 2.042904049616928, "percentage": 40.86, "elapsed_time": "12:29:22", "remaining_time": "18:04:42"} +{"current_steps": 15000, "total_steps": 34265, "loss": 0.37209414672851565, "lr": 6.909769440229038e-06, "epoch": 2.1888361911711054, "percentage": 43.78, "elapsed_time": "13:22:53", "remaining_time": "17:11:11"} +{"current_steps": 16000, "total_steps": 34265, "loss": 0.37489187622070314, "lr": 6.4299311407857035e-06, "epoch": 2.334768332725283, "percentage": 46.69, "elapsed_time": "14:16:23", "remaining_time": "16:17:37"} +{"current_steps": 17000, "total_steps": 34265, "loss": 0.3743317260742188, "lr": 5.935265379168761e-06, "epoch": 2.48070047427946, "percentage": 49.61, "elapsed_time": "15:09:53", "remaining_time": "15:24:04"} +{"current_steps": 18000, "total_steps": 34265, "loss": 0.3719102172851563, "lr": 5.430901519764892e-06, "epoch": 2.6266326158336373, "percentage": 52.53, "elapsed_time": "16:03:25", "remaining_time": "14:30:34"} +{"current_steps": 19000, "total_steps": 34265, "loss": 0.3770401611328125, "lr": 4.9220694899697185e-06, "epoch": 2.7725647573878147, "percentage": 55.45, "elapsed_time": "16:56:56", "remaining_time": "13:37:01"} +{"current_steps": 20000, "total_steps": 34265, "loss": 0.37033236694335936, "lr": 4.414045549219315e-06, "epoch": 2.918496898941992, "percentage": 58.37, "elapsed_time": "17:50:24", "remaining_time": "12:43:28"} +{"current_steps": 21000, "total_steps": 34265, "loss": 0.28135635375976564, "lr": 3.912097577588397e-06, "epoch": 3.064356074425392, "percentage": 61.29, "elapsed_time": "18:43:52", "remaining_time": "11:49:55"} +{"current_steps": 22000, "total_steps": 34265, "loss": 0.1744761199951172, "lr": 3.4214304512770823e-06, "epoch": 3.2102882159795696, "percentage": 64.21, "elapsed_time": "19:37:24", "remaining_time": "10:56:24"} +{"current_steps": 23000, "total_steps": 34265, "loss": 0.17265531921386718, "lr": 2.9471320714071095e-06, "epoch": 3.356220357533747, "percentage": 67.12, "elapsed_time": "20:30:57", "remaining_time": "10:02:54"} +{"current_steps": 24000, "total_steps": 34265, "loss": 0.1734700469970703, "lr": 2.4941206057740675e-06, "epoch": 3.502152499087924, "percentage": 70.04, "elapsed_time": "21:24:33", "remaining_time": "9:09:24"} +{"current_steps": 25000, "total_steps": 34265, "loss": 0.17093397521972656, "lr": 2.06709349062457e-06, "epoch": 3.6480846406421015, "percentage": 72.96, "elapsed_time": "22:18:15", "remaining_time": "8:15:57"} +{"current_steps": 26000, "total_steps": 34265, "loss": 0.16509759521484374, "lr": 1.6704787212769829e-06, "epoch": 3.7940167821962785, "percentage": 75.88, "elapsed_time": "23:11:43", "remaining_time": "7:22:24"} +{"current_steps": 27000, "total_steps": 34265, "loss": 0.16218829345703126, "lr": 1.3083889366705216e-06, "epoch": 3.939948923750456, "percentage": 78.8, "elapsed_time": "1 day, 0:05:09", "remaining_time": "6:28:51"} +{"current_steps": 28000, "total_steps": 34265, "loss": 0.10715762329101562, "lr": 9.845787739562829e-07, "epoch": 4.085808099233856, "percentage": 81.72, "elapsed_time": "1 day, 0:58:42", "remaining_time": "5:35:20"} +{"current_steps": 29000, "total_steps": 34265, "loss": 0.0650710220336914, "lr": 7.024059353355333e-07, "epoch": 4.231740240788033, "percentage": 84.63, "elapsed_time": "1 day, 1:52:17", "remaining_time": "4:41:49"} +{"current_steps": 30000, "total_steps": 34265, "loss": 0.06532522583007813, "lr": 4.64796370857008e-07, "epoch": 4.377672382342211, "percentage": 87.55, "elapsed_time": "1 day, 2:45:43", "remaining_time": "3:48:16"} +{"current_steps": 31000, "total_steps": 34265, "loss": 0.06470259857177735, "lr": 2.7421393820510846e-07, "epoch": 4.523604523896388, "percentage": 90.47, "elapsed_time": "1 day, 3:39:30", "remaining_time": "2:54:46"} +{"current_steps": 32000, "total_steps": 34265, "loss": 0.06277722549438476, "lr": 1.326348540874095e-07, "epoch": 4.669536665450566, "percentage": 93.39, "elapsed_time": "1 day, 4:33:05", "remaining_time": "2:01:15"} +{"current_steps": 33000, "total_steps": 34265, "loss": 0.06431336212158204, "lr": 4.152720214406214e-08, "epoch": 4.815468807004743, "percentage": 96.31, "elapsed_time": "1 day, 5:26:35", "remaining_time": "1:07:43"} +{"current_steps": 34000, "total_steps": 34265, "loss": 0.06442949676513672, "lr": 1.8357098688476238e-09, "epoch": 4.96140094855892, "percentage": 99.23, "elapsed_time": "1 day, 6:20:07", "remaining_time": "0:14:11"} +{"current_steps": 34265, "total_steps": 34265, "epoch": 5.0, "percentage": 100.0, "elapsed_time": "1 day, 6:39:40", "remaining_time": "0:00:00"} diff --git a/trainer_state.json b/trainer_state.json new file mode 100644 index 0000000..a24fce8 --- /dev/null +++ b/trainer_state.json @@ -0,0 +1,281 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 5.0, + "eval_steps": 500, + "global_step": 34265, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.1459321415541773, + "grad_norm": 2.5889114643259257, + "learning_rate": 2.915086081120514e-06, + "loss": 1.120510009765625, + "step": 1000 + }, + { + "epoch": 0.2918642831083546, + "grad_norm": 2.2954157454578574, + "learning_rate": 5.833090166326233e-06, + "loss": 0.919832763671875, + "step": 2000 + }, + { + "epoch": 0.43779642466253194, + "grad_norm": 1.9740216723771231, + "learning_rate": 8.751094251531953e-06, + "loss": 0.866156005859375, + "step": 3000 + }, + { + "epoch": 0.5837285662167092, + "grad_norm": 1.6025712306944424, + "learning_rate": 9.991513345767592e-06, + "loss": 0.8476719970703125, + "step": 4000 + }, + { + "epoch": 0.7296607077708865, + "grad_norm": 1.6016651354512892, + "learning_rate": 9.936020028278053e-06, + "loss": 0.8244376831054687, + "step": 5000 + }, + { + "epoch": 0.8755928493250639, + "grad_norm": 1.6615443914575065, + "learning_rate": 9.829343371836088e-06, + "loss": 0.79805810546875, + "step": 6000 + }, + { + "epoch": 1.021452024808464, + "grad_norm": 1.739839844907049, + "learning_rate": 9.672589544454328e-06, + "loss": 0.761253662109375, + "step": 7000 + }, + { + "epoch": 1.1673841663626414, + "grad_norm": 1.8214814374223849, + "learning_rate": 9.46738398205746e-06, + "loss": 0.614188232421875, + "step": 8000 + }, + { + "epoch": 1.3133163079168186, + "grad_norm": 1.5403816299740094, + "learning_rate": 9.215854533761766e-06, + "loss": 0.6142713623046875, + "step": 9000 + }, + { + "epoch": 1.459248449470996, + "grad_norm": 1.8486916036138623, + "learning_rate": 8.920609397454381e-06, + "loss": 0.61425341796875, + "step": 10000 + }, + { + "epoch": 1.6051805910251733, + "grad_norm": 1.9343055796796615, + "learning_rate": 8.584710074466158e-06, + "loss": 0.613021240234375, + "step": 11000 + }, + { + "epoch": 1.7511127325793505, + "grad_norm": 1.3591298836511538, + "learning_rate": 8.211639623780629e-06, + "loss": 0.6173475341796875, + "step": 12000 + }, + { + "epoch": 1.897044874133528, + "grad_norm": 1.4881517892702774, + "learning_rate": 7.805266544962458e-06, + "loss": 0.6191266479492188, + "step": 13000 + }, + { + "epoch": 2.042904049616928, + "grad_norm": 1.7095977461968044, + "learning_rate": 7.3698046643160645e-06, + "loss": 0.5406383056640625, + "step": 14000 + }, + { + "epoch": 2.1888361911711054, + "grad_norm": 1.3383769030634658, + "learning_rate": 6.909769440229038e-06, + "loss": 0.37209414672851565, + "step": 15000 + }, + { + "epoch": 2.334768332725283, + "grad_norm": 1.5984154081302522, + "learning_rate": 6.4299311407857035e-06, + "loss": 0.37489187622070314, + "step": 16000 + }, + { + "epoch": 2.48070047427946, + "grad_norm": 1.5178018437242096, + "learning_rate": 5.935265379168761e-06, + "loss": 0.3743317260742188, + "step": 17000 + }, + { + "epoch": 2.6266326158336373, + "grad_norm": 2.243985181618142, + "learning_rate": 5.430901519764892e-06, + "loss": 0.3719102172851563, + "step": 18000 + }, + { + "epoch": 2.7725647573878147, + "grad_norm": 1.817398251882934, + "learning_rate": 4.9220694899697185e-06, + "loss": 0.3770401611328125, + "step": 19000 + }, + { + "epoch": 2.918496898941992, + "grad_norm": 1.400063403289345, + "learning_rate": 4.414045549219315e-06, + "loss": 0.37033236694335936, + "step": 20000 + }, + { + "epoch": 3.064356074425392, + "grad_norm": 1.9568348297606881, + "learning_rate": 3.912097577588397e-06, + "loss": 0.28135635375976564, + "step": 21000 + }, + { + "epoch": 3.2102882159795696, + "grad_norm": 1.6677571993666531, + "learning_rate": 3.4214304512770823e-06, + "loss": 0.1744761199951172, + "step": 22000 + }, + { + "epoch": 3.356220357533747, + "grad_norm": 1.6030223311676182, + "learning_rate": 2.9471320714071095e-06, + "loss": 0.17265531921386718, + "step": 23000 + }, + { + "epoch": 3.502152499087924, + "grad_norm": 2.0719250695035543, + "learning_rate": 2.4941206057740675e-06, + "loss": 0.1734700469970703, + "step": 24000 + }, + { + "epoch": 3.6480846406421015, + "grad_norm": 1.715926337353297, + "learning_rate": 2.06709349062457e-06, + "loss": 0.17093397521972656, + "step": 25000 + }, + { + "epoch": 3.7940167821962785, + "grad_norm": 1.6180812645074687, + "learning_rate": 1.6704787212769829e-06, + "loss": 0.16509759521484374, + "step": 26000 + }, + { + "epoch": 3.939948923750456, + "grad_norm": 2.048625803274215, + "learning_rate": 1.3083889366705216e-06, + "loss": 0.16218829345703126, + "step": 27000 + }, + { + "epoch": 4.085808099233856, + "grad_norm": 0.9085706761456963, + "learning_rate": 9.845787739562829e-07, + "loss": 0.10715762329101562, + "step": 28000 + }, + { + "epoch": 4.231740240788033, + "grad_norm": 1.2437558088436274, + "learning_rate": 7.024059353355333e-07, + "loss": 0.0650710220336914, + "step": 29000 + }, + { + "epoch": 4.377672382342211, + "grad_norm": 1.0557746493951432, + "learning_rate": 4.64796370857008e-07, + "loss": 0.06532522583007813, + "step": 30000 + }, + { + "epoch": 4.523604523896388, + "grad_norm": 1.6388458174836673, + "learning_rate": 2.7421393820510846e-07, + "loss": 0.06470259857177735, + "step": 31000 + }, + { + "epoch": 4.669536665450566, + "grad_norm": 1.2105021817675277, + "learning_rate": 1.326348540874095e-07, + "loss": 0.06277722549438476, + "step": 32000 + }, + { + "epoch": 4.815468807004743, + "grad_norm": 1.0929519804389642, + "learning_rate": 4.152720214406214e-08, + "loss": 0.06431336212158204, + "step": 33000 + }, + { + "epoch": 4.96140094855892, + "grad_norm": 1.5036224042092285, + "learning_rate": 1.8357098688476238e-09, + "loss": 0.06442949676513672, + "step": 34000 + }, + { + "epoch": 5.0, + "step": 34265, + "total_flos": 879165899800576.0, + "train_loss": 0.4208107623917577, + "train_runtime": 110380.4687, + "train_samples_per_second": 2.483, + "train_steps_per_second": 0.31 + } + ], + "logging_steps": 1000, + "max_steps": 34265, + "num_input_tokens_seen": 0, + "num_train_epochs": 5, + "save_steps": 50000, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 879165899800576.0, + "train_batch_size": 1, + "trial_name": null, + "trial_params": null +} diff --git a/training_args.bin b/training_args.bin new file mode 100644 index 0000000..72e46a6 --- /dev/null +++ b/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fb0a977c0c6f36154f1a9b9fa166cf35a0df584dc1bc5a147b821c8d5bde01ab +size 7377 diff --git a/training_loss.png b/training_loss.png new file mode 100644 index 0000000000000000000000000000000000000000..2f150f5e22cbde93d35e175d5e2c622aeabfe486 GIT binary patch literal 35158 zcmd?RhdY;X`#*e{A$ya($xg^#kx?Xs>`{_EB0GC4GD?vova@B6P!uw=x3Xo=e&==H z-|z2vj^|%^I*yJ?*XLUAbG*j+{zU2AQX?XuCqNK{Nd1P2E`neNAqd6{9xnXkYX8In z{2}FW-O%HX^L-C5OE+8Orlp6AgR_UjLo1fYwr=hZot>@-iU(1rWUHz6gZzv1prj@}bB zJC2GR0mmfq+D7KTtg_eUfBaGZG5J742@it}e!0<$`d&~_fnQZ$u%AUnMs|G_#^6Bz zLsScif`7jzq)cl0zHW+Y; z8Su2VwQnSbng^Z@O|cFc*Vfh^HvK!1$Hu`?OyrQ+8@y9(64;$0@QefhUX{Zqb-#sr z4)xYYB@G5yo~y@R)I&sC2xC{VY2DV?qY~CDR|clCbgsUVbY+WYln$?+4!AV!w?icL zc+GC}nf;5eHo;9z@>9N>4df-))zp*|Irt4r-wrx1_q+^Q>`t}*_Ja3tDcxB(;Pmhn zakc7eNmtDIjyUDBaYgez=Z3J;(^LC^3qvKM(=*1T&Obf%!%(Q-{i(T6{N?`R%_#vSa=tU+o`AgX zej&mFYjOQxV^VXn%6<8VZKDBwc0UgR0YPZLU7>Ez(4XbZKx)2Z4S4lYsb)Uven*R` zbBn3w)GaM7N<{`S|6L*jV?)EVt?g~H5L_a1g0Oc=nVtS8`}ko>tYY+Wv{D{+Q6m_U z(i=6(emmbK=F*)T$u3g!lUjGgk|FI42UGX3Hs9GJk-y3x+zV(NI@tR@T`7OIn}o24 zJ5QPHJT*|fbt@iW`RwtB4Sffnytl2kXF^B_UTSe2@BU%e;!0{zet(H)E!jKKAAa_G z&&N*e6@hcML4os$H(&BfRki1RUS3`V5fTzQ!L?EjJl(#~+S=MzW~*7SI2~{zfQgB@ z1q(F?L2%u~gaO`2(XjfhTdFbC0^ej+EiGB~_^50|jdJ9$uCBL+U8I!{kf9fMh;$sU zYMGv%{yUbWEfuWVp<0|R1%LD1WM*DElVe<2k?(y=*hS|ok5}sD393usnR`Z9O z>EBFB)RVYqC7fO_E_X8d68bbSJsK`lw6wG=I-WF8)o*-snO#VT1eSfMNB^Xt#-yeIFcFmm&VPRuy)fLb+v1L&w zhiipHG!hHks|qzP>Nwt8#Xy+-wlF?>uHu@Tn?ta%@bM88+$|AwjZGiV;Wuw&NaV0w z{_!SF#2WYX>}+;vDZFm8aXhhxvaXiM!gaiWIq;Osx#>(?NlEE#zg=HbVBl-{z{d7Z z%e|Qi;Y62OzkI>z@9!UN^!H63qcW@aCOeq)V)pa%n^{?j7Bu~AG5Oh(bNJ+Rr?($o ze$D64t>fdD7^NS#A4~^2Kau6;zMw6;M+uJzsYOOg8U!JOb%BJ5Um{#Ng4l9vrlrrm z(#6Y5?ce#?_GZ&RW3#~Z$`Mj(YFq?;b`EvE=UaN>Q<|xQmMeoF?Zzs2O-xK?R##(A zHtKrw?H|I&pk!P&DCGI{uj%~wB0TK+SY?Xs-C~m#WfIxY=4OSwA^Qr4U*zWgd-^%n zm#ds6r3e|ND%$_T)tC;Ze1$gZcW!z6_!v9h^OOD3(P0G-WH1uA{pj-D;`VN#rdW8# zZ9_v;zkdC)t->&=_ZDGeXLo+mKxO_<;%;|xWv{I7z6Df=t=-+wf`ZFF2ODv)y2eF2 zkj?D!^2{+bLRA{)NONy52?U>#lat_7;6Hz)xj$QtRh>NG-EB147v_y*Oic08oAne> zW)(Z*8E5C_f`^Ol-3#zS19o+|D68L&l!_`0R>#QEeAuh+y?T<r>xI}9ZE)|>pn^St|?u=^Ba8Lp8(=*{>7*$|WCQosuc z+?r32w;HQ>xV5uW?x*N~{3n-I&QG!G#kjK*P^&Hf~u<} z?)PTI)znD({r$?FChkaYP-LvXQMFmWLu?~Tm*Kx}WMgYP3pJ{3^po|c2fa+)Qc${@ zD;^H?l|RsyIolnGjEQNBBx9dhUw1shEQo#op2K5xnCR%}C{4~^%Fo~blhqdjLS{Jz zNV6(GBLSl_Z0L8LaK;$!Oh59lurRb_?63U_5wY$-gi|F8%^K(jKHOA>^?#O<5)Roh zGx+fir&F_plK#xSlj+mOXbtU z`Ghn{*LOoA@EUNOot@j;+YNu%rV5(!2U?(k(<}e49X|5>%^Sx*p|_upA}k^zWD&%S zt(TnZEj=e7x@S(0b_}Z==%8e(#xq2rKOqg%TJ+Uy)0*u644gvidG zx!)as=m}Z6AGg-ZdzEqrl8yKpJpSm)9RJS#JzR7t!)JvT(uk}rf+Qj#fka7Jxm)I6 z^LuLa1oiIZo=`VlKt9vj|a91ca zB4Qr>VdLT4#R~f&0y8tS(JIGF;Uvt&2ox-7Snha+Yr$)44tABsXIhWOs6J;UGP4q-nBM<62){dBBpS zU$pj~ZF=O3&)!PdpMl)@@?N>N7@@|IxNw zykv2C`QBOK$m;b1gCdotfV0As>4_AlUuCwKLA1gaLHgtIxtRgRU5jqdU%q^3G+gK9 z+z-FId($PHsS%Z(rSC;!xB-xum_d?{*V0H^Cm#2WjNJ8D8C0jFqB0u3*XUP^SXB0w zO)wnX(U#>zBG<>O@1-m*EC5)>gL35bkmT`*saI5rz`ZHA>5Pnwx$RD7A&KzxI9d^< zJUu?w*;btHGDtQ85)zVrSoiL-(NROdKmX287B|Y-wx>?(sOOr*y#TpX+WsmuDi78D zNcrc_pA=iEH@Cc>Jc){6^64sb_`C3xO;XbE{f7_6>u7bRflH;#>vM>+h8lkM_~_#5 z%1gEkWkAKwE^krl4>PyT4=xp7Vy1&ds!hU>cWcU=ZnYVl+|tsbghw#w=Dzis2Z@xJ@@})A++QD$$;->T*I`UAAyIv=3d3*z z*MsXSDxp05{FF*B&ZVhZPv01fS+_@$r8ss|P*Zauk*0NzjZ)6`M$9pT;3bejZ7Ab6 zudurv_1Ie?u!Jfd$_L)XifnPFQL$7^Y7wzZMSJM7V% zLG_+pUpLJSRxl35EussjUh4(4^6i?qq-cvb#B2AIcw) z6q_{|&(2?pQ$0O9n5Gr6!iH6~Gh$+3h=q)`fV$cB<45uCN&by`iSy!^=xDTNtH#l{ zba!{J9Kbc~AgnFFzq*=Y3&mGL^vV?x_!+YF*jWDG=@o!XxUAT^ckT!zKyiV`Guk(T zqPnyB!FjsTz?Gg5x6rtfAX_b=%H#nM4rK)e1Ugt9L1tzY1-J2fwb;qQrkb0ZD3kA+ zNWrSx&-dy8EDE^kATR0KKVLKLyjK!XBQupHTiP|eD zFrTYQkljxkLLp6s->rgffbZLXax>@PJ(cK4*VJu%zIaL8p&6cWYuvfXa=6gQuqi;c zBbE+}RV6I`*SqT%J=VvD{<^zfeFAj^Kv4*JcfpD5_!ul1VXcXoRc&2eoxZOIAmHD> zfAcj>UvpnVi>vqY8?V>-0`8{Gd=}ig2HD${k4EN-s;63kE@ch;;qSgYc7M4;NF7OO z=Vxcn{x(j{{XCBTYMHTimd=ZDreb~wS-yU1g2NJx#h*EvD78Y1v%^FU8DLkt!8|fx zppme;Z+#CoQXGL4$O4lh%gV|^+s!jpWns%!?1J_Ulv6-2P=jWz8P{SX%F4=`rNSNg zSC3;VI2aSH(Lhn#0Cf~PB^ntSr7QzqxL$t$d#U{ppH_wpT~O`u2;e+e*Yf&=o*pfJ zvwB*#%G27b{cxxKEA&9{5$UG>K2Y+ zf3B;$o7dJEm*MwU9fdgu6Ykn!!zB3lpH}pQp+FrTAI|}}`~nzD#9>r;VQGm?LSm#W z-bAq4M@S;@0u}eM#N7iLp^~lxV@HGGWfl&O_w8Pd0K?*$WPQdvjVtY`%mR*V;r-N( zr4#53iC}oJrSe5<-z z1UN&|eTi_m*feHubz~k;VFxtMr+3Z$sY;s8r4Wn3yxS;J4met*L37&s-`T4OYDVQGF;EDdEpEf%we(y)cYIIN$qlO{m-Uve5)ky)1_4@ zGjoCZFary@?0E^mIZv|2AoQ+@OB-+@GC;??S!<)8*v06HsvO6|af#?zSXhvouf!C4 zGi0~nMJVJBSVFlh~dN-eRtw9Lf=7u5-Xai9iU zBj$ld`nCM`stTo)>m0u{SoCJfJ9{Qc&=LY_F6W?4o?HeDqrorle_R1e-3;Og6uXY! z{n=R~t7u=VKieNqyZR`_HZp2v#zO0j6cqrV>rnBeJbnkET%`xQZ+e=!xVYH1gC*&! zQ(d{@lWi#3^c38;+9Q|)?0$VHEiD~vu~SQ6ii5jhM5`_)78U^5I9R*s6@PDU?`AtO z13SBjzg9r98I3FKEcMIMQ&S^A8TilLpaO+eCJNdi-|l8%m$F3%9st-Y=VQFg$EW1r zz&}>y_{hk>#MpnQwyyS%vW*P~6i?$P%MV9OEe$K|$e^-X4yj~~Lc1lSp^2coa^En8 z>h6#u1T_)8SgBYx8jl`Qj8NO9IXF49?v|KwS`pyk=51LXz8!!60?&qq z22GO?kQqjbO$)Zp3&ugLZ~ zPyk-s`*!=)-#G!f{qKSOf#*ldIVsd)&QlCU_o_Sc^$XC-iGAV1Jx(&WS_UXRBmw6~ z4;bVF>f84vh;j6ukDkK|A#eB*2&uiWxHt=sh7zrNbFR=b=AaTZE*=%ff(Y0OF>{ed zFx0H18blAjCsDonuH==DiXjS#yUuyx(Er9Pr>`#rQA+}zgZr$^8O zZzqoh=+Qvy9WnDI(w5oA24N%OcD5?3pdfK~s*swE&4OdHPvchW#Y0feguS;cP=QNM zPR{U?6^@m)_4ZD${JpUxA_hrPMMcHmQw`FPb*?Leyglg>Bp?@rMMPWxFa#wo7?f8U zXpLcYb<*Na6Oqu|jFL-}m(Y$5*a4+kXxiS&bO6AJr4}$|k(Ffx zb*a50?$Yy&4703JxNo?{kK}ji{UJNH_Vxrk9YmTZmL7dg^B% zc@_fUI=~iTKy&S&0|5czB!2~@x*}zfqw#Wn46zSRrOVhzF$PDwFpNMupQPeG<2}s)`Es+}vC#z*~UD7K`7K zImcYlU_&)Ha&q!Q@&>a8AItuC*U{9Y7gdLV1PK}yDA5856^{A*{5&!tp%Wy(z6M_} zc>5*amdi)~ogc_ETyw(?sukOVYU{mPRE?wsMmUaDC;~UssOZEB%FXa0c>44yKwg!o zksledF>p<4eb<;Ld#R-#Hvg+Xt; z?%`1}TI-$*Adej00pG#v7vfYCVHasoh5B-ReSN5EHVP1}pjjKN@*7v&*j*XY1i(X3 zX)FLL-xs*q>jnlXq^&Yqt?1v!s~jQ9BV63wDImP5%>A~n*NWg@ttGjsj z?p;*#DwySQl3}Kz!~Ib29YzXgQkH?j)7sms4F|J)5+4~yRM`2?Duq2lOZt6Fl(UN8ITke^ef!Fyy`rZ1R~&Fh)%l)1T=!y zLxrM7(z)Tb<_W)|$-}YlZIAo_N`Pps1mb7^Cu_A&aJ#w@r!_9K!6>EIDHviC7S=(< z3oGCPIU75DS;|%mUC&e8L3-r^vcrjD9=2lWCQ$&JBj3Jd>3(38-Ty)=Hi=XTWvsO( zf`FC^!w(Pl$DMJ4M0;nUQeM~6iiO*xq>ocoQB{owNj&Q8FE*9l zhGwsCM@!54?YvQNeW%9eD>*7~0Z%R}76OL@Mf|hN%iJcWrs#c02A=zcHI;-32g;|4 zYu$OcId2f%d)p}CvlM4;c4BBAPSJW4-&Y$^(EFJ|Wou95)LdgE*1x^YG9rI|nBAbv zduKbAM51@g@*Mx{p7T0-S)P<^fb z{LHbNs=r3eAgQzT!c1l8YX)bK%KqaW$pnczAN5jPxjs5s;Ij<*%+Ai@5fOcPkm-N_ zp@TdymwND`w9X?V9cGqXnVeC9=G;}Uv-CgqqtipcH3suCQ$6G3<3Gz^uHbS7q2}eS za4ba%036rosxS&MlfH%mMg?(gz6zIv3j$}HLWu>Ec5h|qvPg47xdo;n4nUZO^=tRQ z;_PZf3)TIvu3TD3!CP;iB6z&|($0oHmKM6ZQ@MD1OD=&A3X&Ke$OU$NZ!xHa%p9o3p}QwQkT-+X0UWjc za!GygyLazi$uR1vYax%a%!y4~Y8OoW%C;G_xGXna;6+5Y*p+o>mu2{FTssB-2Zag5 zOfr#BJy^N8a8dou6ODZ|Iwd_KPyz^aHg*nnCh@3IFAJ7Oiph9?G|E}lWT0{rm>Bc`5!1^R1PW1Jf z6#tkHnf7#l`;;X_zf71(_-)4ZJFB|=KwP0R7wr3E^7LOeOyo$@TC!h*lUf5w^@fEZ zqoBY7>%P#WmYg}@&@!%FsXBr)s8X-gAVje!7z2r^GHtFb#FSQpSI$7`_R^Fo8Dw(& z|9YIpDO4bpO*i`60iFrlpZ)VE3Jf{TPBH`(K1o2BIaam5z|@DP6Ao51J~ToVkb_*I z67sDl8-E)yjWgJ*up|E+-^xZYWght9G!E(qZ%u8jCHVNLNjO|$o&d;d4gyFu zAV7{wJHur5Oa>T#nzWZ-z{$GA>uc_~+Hyx6qN1YtVuVI;1u|yl|CmQ${T_lzu?0Tg z#6Fl&!We})ev|rODTb7!VisTXz)>R?UB5(0Mm9u`8b}+p%S^G%@=T0U9?eHP%V1)nu>i*Q8>m052m0pXRNcp2iNGoXj3S15w~yDyEP=9lFDBo0@(t9x z!oiB&NmqLl%taLQ8W>Pl7&}&OjN6ePJ`BXv!aO{%q7RUFqhf5U-yX0wy`~^WcKa$X zKfel4RH)MVFLosfpo~H5wFZFi4bXX=@mBK0ir2lQgU!#uuuv~dxLUgPagemWtAv6Y z*Y?f7HGj4qS!1AsFo;HNRRF@0zWe;>90jBtS}8$c{N!b)*OrD1TLo+^Ve7zv##kLI zGV?}WJG->YuD28xKoRT%G?Cs&c`CuIf$Q3+2nyA|r%P0$pOqxk5?ER-#!Ym4o>I`p z-av1)IE8KUgQxkfw6#d3HoI>G1dOK%DDC>%R&GrE7=>T_N>Y*g3#(bTyS67-cJNls z9U6mPFuajfW@lr2Clw2|^TQBNmTs4Y*vq75vh|hhn~~gZp16J$Pup5a$;kTGEinzd zsZudNEdF6lQup4wwS#09>~qd63_19Q1MF7uq=b0-(R-5!x$*t#-J{2Z2Q7W#EyDOS zMV{O(xBbwZEQ4oulN-@Q26lE+Y=sEzag&w8O=*6XZtJ-jrISOyOpb;dSzf(=3n{-y z*hkmX))abNW962Bhcv&IzoqCb{7XO%T^V2WO!PSn8}vJNvJxqnx7I1}<-Mq^?(A%R zZ+dM5P0&6U@R23x@I#H$;H$QEcNdHutn$O>Fakz2F$UcJX(F;>8ZuYFZHs` zhcCdqQP(ig2bcsPW|ni<)XBp{k8j)k!hTcM42b9s%79&UF?Q43i` zx^;wWhjaY?UJ4I8TAJ3B%zSOVGYY4YGbk~O1NXNLOnx9)01XkSk^K6_?n^z278Wc3 zk+(W7IYUpjfSv)Ig^?MM$k3()7+2!w&!72!w4}JXf;@w&3kXE<`{=Z~c=^kGite$?6WTH1 zWvkDud*-fDvXVWwCRJiFcP{PD7H#u^Hcdn#vxPAE{APs`O90+?3z%gBCx@UQxa_Uy zJB(KmG5hb5q0TXCj!#WZp-urxXHeU+#_vcJVSxb-ATglCbb$G~HJ~kT36%s4=2wep zXk(H)-BL!qJ=FZQva&+8)7R1uil2_vc?lt?%8i0Skj$-rJqFJVpg1PlJ*X}lSRDOb zGz61uaqr>42TMpM!&)~^5Kp7Q7a#%NQnuy`+8|2A&$5p}<&4d8|phB|;@u>l+b~Hq2j*HQ;OQ+u1?;S>7*0-0zCF+`pQ2La z%1{9wAt6eYQMNM=if0AGWiW4`khFr)jH+qimVph6hf1hqy+GGkQoAt_pr*lETy5Wy zFQdg9WCS)jIVQlTAg?brSUvW#9T!$elQW8O^Real00Atnh$Vf1?ZDfSz>(`}EHmF1cQmb*IJc8;X07c)$pq)@|h4la)vC2M@8R)84TROG|FtMc|dKE|5ecZP?gEdEF*`~1rq$( zR~vMkr@~kOA|e7boOfQEQz`3@HzrUZgam2WHfqzwb!afw0o#X{bOWZhAPu@>OC)Or z_F;I(U`5gJ_y`^zUQkTTMKr>Ix0-NTK7FrPZajDd6Y*f>L8sS zJWXs~Y3r?`V#97+L=DPA8~9W(jYCm?`nMaMMiBAdni(IU2&5H12D0%{GTVm7(};O!N(cAA4n zf;s>XGM?bxOc(zGQ!u5@({f^VeF)N*qe+7zX0+CUJRZ~Z@B9*on|~b}?uuFq&G40a z3ca&VEq7Vdpl&@g=@dm6S2tOq*70dMYtdg2y{0x5OhA+te7xd4Iy=zd-HRx4xq z@1QO3J)(}ZbX;xW2%;HgSQK+rQOTJJqK%Je8n_2%5{9k0PDhu8Ka@U z%x%rKp`>yv@ZUk;;i}My7N*(lsban^4uuE8thuFmO5^oD4=#Ft=5d{~7f;!UESklG zG>o6){q}cfXPp6Gzj|)^&-a18=SGuhn;$SirZoA8Axr~9Cs1DfK;lJu26_PGW1&38 z%4+QjiOuycTk`rzy&K&_>;iX5NNyg{R5eA=3lTZSFB>p_>AciyQe$cgiV%<>zsX_f z2WOt_VGC|Rcn8bTE=D=)vTql3`Uq>_*S_K^DR@z1jnb?&z%HRo49;695PB&fo| z?ipzRUR!t7O72%Ui?MbVu-;g{GpWzoqCVJEe-A2DujZ5$7HyHBd`I<1g2tu{nQ zMZSJxBKY8^!mTz-si%4dK*~VRDoo+z|U? zx+h7qiH`)7sP1F1`)LKu>LbzNGcY3lW2>SAp+HwEWLrzb2^yshxXc7I5{=P_aEZvnp_)GoqN&ylCR)tZbA_50CgHT-x_g>*R zP7M^2wGrx7Wzjd10$4;o{_VC>EMZ)gc2^#;B<}nH_HQCt-Wg3o{B?72tm)G zUkQ!bU=KXp8Xm}_VTRAaB>(Q`k^&h}>wr48^H^Vz3qvXd|C+u-(+J$l($pk9BJ8~L z#IK3F@wEVYGsPg8MSO)MEXaZ)cKeNzNJF4C(tNpzv2$sFf#Vg_PBQ4q^QZh*f+qr` zn=i@Az7^T{Ly8HwcYBj$ZgjEG{+AF6`v=X6C~7*+kwfoixi z(R1n%(8Y4GWYZ^!k;IXAGsmiy=Hk-m{@v;txu9ccQ&@F|R>?DA3=bq_*22kM^;sGA z6Z_J&mRnmomrErC8GBDX-j8X}p{EQ;L!=P->UtA@;V{yiG4_bc&`U?Z;eE?|M$Ql890+d_KP;tYy^3K*B##9&2@^X3-2x|;8`4# zM7RjcWp~8Gtdz>t2>iDpTLm3-ir?7O$*1ZIIFGMh4EkZw`mbm-0ig8N$1*6#qXTn1 zw8ZzjCwA>~CBe33k_xrek<3Y z8Z<*Z(My|fvFa3HAZv|j1R?!;^PPicnF2OC-7LT3xU|mBSsKUPKmPaZ`fb>KPd%cy zo*J_%`jyRm5ew^*)sFZX4CV&=e) zy!pNDWjjMU!(G87;pGj3w5bv@WG0*Ffyl`L#cT0+h}sk@%-C9B)pRwZt5>gjp+1~$!)pbQFy-sD6_oG1kO~k z21CQ^uoLQyj>TBN^bT}4PmVR_@8$`o-<0I3uM!5Leu>KljOz#$X8a`DYcO#Uxk_@t zKWbYZ&x_u?IK+{lPJv>;G6-J@ak}Cui8HGab?RR%>6s{Lon(DdnXY`RJ9K>NG zezok>6pdpsyD|;mty!Ui+}A?hMC#{06C3v;!00IthqMUCv6gPu=j<%L_>gx0h?t=% z(RRlWE$CDf3f%7QxzF19#~;JHt??8N@UNGK6>{Y)N8s_p%t=0(f_VO10CHPN;xWOm zCvrnhS3Je=cblCP4iqv!oC6>CwjZ*r|N8hTJo(jlc$%^mO1I(tH%ie@KJ$I@u;SKQ z9mhjTJ=FL+geJRRD#L~CN#Vkt*16AOPuUz~(}NcnPV_Ex+mdI*J&X>UmukMr*|`z( zkXFu_dmINOm?2WGpj$mRixhf00-mWW=vEP5(&n0n)No2mqL3OcmnQ!K%WrZ>Jmx7hm!#~; zCg%$j&3rC{7rSi8YZ8ruu0&nC@#CoEN_av#`Atk&nEAx5)hbixUrpeid&x|~uI;Si zFGRziK^m5_C*$bm01u}X@AI#ikSj=khLA=_HG}CvlADGei3G8S+B`y1lech{sjc=s zgUsLW80{aHTCVeUs%4K^xvb96%L{@I$n~HmFf{_-f=Y4#N6*lvJ?w?aYt3)h8>dWU z`ICae&o}H$r^`DiS_)6uv`%JOsxmck5Sd?1A{lGbw4#@iRlvEKclmJQWt4^(JiTJ3 zz?Iaz7>oH@{hQ*qy%ozaEu{e#5;2ltrJTIlczWDy{LRxR^yV9pI0&IVT_}lKWmx)H z)f+vu8M}9!9DGO5Qf{vB^;(!DLuS;Vi7eti2{6c7tRvz;>4g^1<|%rTFGUOkN3%2 zuu^|+|CYG${8d3nLOQ|4&g-mFt+}db<&O_LaE7d5K> z@^yM|*3p3+)%Yh@%aH`Xa^}tw8)ep{t))l`|^-KR`n_yQ}rtTPTn6Ux2xr|;V6q^1O9HL4=cKfCGzx$Ym zob^H93}79i3VunZz6hB!T5R<*?dNKzV4}CJWYWZYdX(NfAJWu|iGaRib)H7NRs1wG zDKc_JZmQ`eVt_C7_V3~fp`_0>3fGTENRM)-7=er0DyeoJ)sVDD&onpr_AcMf+kPTH zlkLU!c#zg_7j?3*z5AT-KM$WbC;E3+((f9@i0DO(G+)AM=b12wf+3&RkSE@Ut7kWE z2n7KM(%y7Vz!1PJ*rSD_SiOBtRGLNkE3`VDz&{=Lt$8_OXG#eL0W@qW@i!Vu28y>U z8FP)EQ8&p>VWCY;+Zl#r);gUdO7zChQJ{rEp-}b_s$+Nsf$F%9HjfQ?VwvaR&gfN! zTV0`9-o>?ie_ihYJ$SRN;K37$OquwS*Gef@u7Q}^MFyKN)04h(R1_{pu{s9do*u{H z!gyW(D$cXBcvj=SVkLtzZ|dycY!T_kYBawj@XrYh9wi=Wlmpk3hbFWyfzNAD6d%Zo+ zr#rkh-j*&`kmk8{n&)X>rUjr@&nV!}z^IcjF!7!Y*-J2=q5||0?12!N3e_EfDfZ3n z){8KSaV?ToI2@N92!xv!g@e-k?W<^>UHlq=pnG>JDMYfQD%x7!asH z19Q6}TU#!KFwg_ys6uT{xZV{?H@Y#2t&X<_idzRdZ5HH`ub)N9CvdY}Ti(~)fE}5d z%IR2$0dsw9(%tfc%Qh(c?q(%dfW0_a7l0+CC=G8 zkPJcXB_X5}ecv+=d<}*O?#br)JgLohQHxRVc=#KuQYpMJn9V`vW$Yx)Pe79OMgCRJ zGjY=6*vfboKbUiypC`aV|GK7vnfB@xL3V#+v%>236gv8smS(sj4z@)LYT&`p83LejesoJHW^i_S!H>)5_t#02({Y z7_BH??VVPRdnyo2y718HQq7qO-w%!@Js;-NYpQvx&ugreM&G(e1+HX3kIf@~+DY$cPcFeSQ}G=w;Zbizt3W`eEBm)APkb z-L9?ExA)VICl&dp9+O#1NhrKEunx(C^s*;WFKyhZvVHKNZD4=`hQJ+2>@Iw)l|l;u z7?dV@qPtGy2(yHK?KnQGOK-3#0OF{AM&<2|21f@{C^)s^o5=)Sw1rS7@=|=O%M-08s0h3*%P{<#c7$lhJXCrONpw}ds9f(@w_PA|1 zMLRYoBe^>vuRYG9i=y@r7!7&C7e)xrkbOHy8kw2N(ON4|Aemjpy&9$~XgU6il9>1= z41T}>#5L@fT1p-B$27%sQJ zK6_~%YJIoto5@JGzAYWYF?1~!^WK)MsK~Bu7y*e`SS?mf4R1t(F2Ahapp$8QL<`OQ z+3W1KM8{86cxb*CqbDTT>Qi7%(s?VvDywb2oi|7>s3w-0>}pi^fGSG(6ToDi_{MnjE3)6cpLRr#^|#dNhZ+QmL(E zT6ElY?z3)TXDwe)NtEsfkb!KHI*$dgt<3`S#;q*TbtI-XpYqV&N=@ak7P=({m*9BB z>(+O#2p|T#?!z0qgSY=vAu*`ulx^V;pWJjwNge%Vrr7YzI)dC{+xbtx?RL^6YPZ|-dEkQ(3M%TOpsRaS8EY;T?4fA8b) z-UNKYPfLRnrqx;!Q8G8Vms=o>lI(O#^q}*-$3b6zNzuk8mvpQTwKH!{4d5cBcH`Im zD%q=y=h4NWf(I6>H_>;||7t0W&$0YMoAI|Rcm zCXfFi@+;?UU0zi^g;hF;7Aqm7VKxQHT1_YsFOJG%Ofh=-=-+`UAl=v zAIYS&JEp8C4UvQk>IGNOciAdr{D>?(9cLcU7EIdudbZxjJ8|h+7_Qc-y>qe#6?(xa zxF8h_T_6ehp!5X+^-jcR?B{7_jQSOpR^i~BoTvfMdJlcpTq(Hhj+?&a7v+Au!F<%? zu}(dY86B+W+f0SG#sz7UduM`JnDg#kfpVY01GkpwhlCtmY=%HUWtq;Fko)Hz*eFC}1WzxI4?Xu^ue&g^2i;xCIHPgwV49;3 zt4VH(@y@Y|Qesdhu}Q~_cD1FsiBBanplQ^PPrQ@(l>^)Nm2}A9;GsVhILQ2P{REHe zk<}N;e_j-}?{})NJNKyckb_$F*;EVp`cQw_MYx)raW+Igb1lj@TZsj`)S??FHm!-` zL6W2|9;D<2NJ-;8neX@(Acqj_cF@bFM5soQY&?oV3Fo+eRMJS9>+-Vn)=<1*nihj# zIILj|tf2r+GJ~8GAJRO;Kdv<|o~JK-?vqb*EVdUhWzl-+k+#+8U0;{@93a0o%5Wzi zhpKO1-HOAkpyy8?#CMLf!->-k2kFqIvvXW{yP#n-KrjsJ2H^fUYgqXrZ-Zx#jKai= zqLVrB=Rb$eCj5~l8xSQ;#70^1w&sCVANTn7{5duy^^ilQ7!Wrh_0pQz|5q=9z4ZIU z5l$hbm`xx*LOVK^AVE79MXi&k9zRotN)Tx344uzq*r0W`@D;&PfYhy<<7j}zm z>3ylKtoC7TfuW<8%B`3KXBa8OM8N2o{gA@0{Xj+cF{@I&={los*#7{{A4L8b2(ZoE z;zbK*6RG_#AFRzyq<;S09lewq-{ATO5kwG^7j9x(uGjMtF4G^@ynC8(k^b;iW`uX0 zQnpPL3`?+E1N+|+!s{%P7i0v?h?XW$+xCfbjqzAf_3OmyJO0d5O}a?!UbQEg#|lMb zi7eruabz`J!mYv!j){h|4EfgwmSaEbdY6%U@dwx9n1^xAD=u|%wzEmHKb!F<*_RN+ z1DCz{J~f-1Ec5oCqZ>DR)%?Z+d)+;7Ic!&-y(nsHtRd`fe1-kldbWlKP|Ng>hClz%hcJ**VhBiFg)7vF*O!DK>R|tK0k+kJUo>6_g{Dj48aA z99fOk7p1~ZW{My0iR5^(^D{Ar-I|?(^$}jI3-R`yRHvLAJV^tRMO>GP6dQiRgl{T< zox429Ozm<;+uWBJlxzX-B7yy~uEMVW;KH+A{XyoAb!XLuJ^(}n_=4JL` zS#14v@{jQoITW}sMAd^$xD>gPScC!s&H`(DCNi{L;k#B6Sy;|B*NVBlaD3SL(t49M zSg$Z^vo@VKX~E&j8u7%5*n|(MfT0Zt@8q&~_0!EP7kAW)toJYa2-yG^2s+p_*1=4B z_AChIKrs-Q?YxU_2=HEbu7(aef+L-)d-6&?P!9bT3=S@?cVK=V!kh>hBctik@ynMl zz5nE-mcq;|7~1Uo{Dhf)fAPZ!>2#B;RXo$ujWNu*ol`>o z#S*A>#BsXB$x66kTER+OAtr*Rfljv$ztLZE0j-$hJ3KEMV=V=3vWbU&}x~*VVBL2oS*@u4Zt`<6vJGg1}Zi0s_7%;zvXI`Y6jm zH|~RhKMzv?idtHP=wLulP~sJ7m^Ge&UEVMk&Klg}Rb;F1pQ4d=zL3=A^= zd2%djp1Nh8c!%*Qo-{~3t-!kIU&N4THmUH-b6-Kl;MF+wDWb0#g|BjhA6;4ryG(~u z^U5d^^mG)LrGA{;{#nvl3LYk4P%&bYZ_O97$CU0}T+)TGxL58=wFR>Wu(jY}1&mwC zpKf6R3z~)Da@6w&<_h0+C2y8>hJLLCI&1?25D3DRgb&l&FsRH8`+kI87zWO^hN0Uh z(X;R*S9`Ttk%?YQnsX)hpU{hZNQbAM$P(I2;!CkLw#ofg;uR{U;LN}4lS#|Gz?R?O z{jV6OAqf5E7#Lk)!5|U=Pb@|2tBsid)oatJ|!~J>C1+GGraf|-MUK+FdacPr3Sk@VL%DaTaEbYZ06)p)44C7_R9oD<=CP9E+ zqXeFr{AB|l9iSHAd>R^FBYm?fdiHn|o_3m&!i5|Ak~A1ntqMcU^UM6WRIcs}RK~`5Lpc>N>&Isz zep{xRPc&H5QY+l5!avtPP`kx}z3r|)ZO*>j*i1Lqy)_=gdIayH28M_C=PG$^KH63X zqFMOz>8%LCbI{Z0PW&rg*Iq|?>?S3vdK03c=(>@|zw zDsj(kpgbL<+ep>*+)7u-_xW~mKW8*uU!2+AkYcaKUU@AsF`1ZmYQcoIk6)(=eqX8g zw?o7Q08%YEsHN-w{J&rf(S(JRZ=s=~Ly(x%6A%*0oa^98`oP_03bWVZo1lcJCYn@S zIcDP0)?wl)n+f-(QPndo-iHE(Sz!VfzJ)0j|4{g;NX?x*$W^N*+^4%8#33{wg>q|4D)z)}>`DKJk z>g#fD^k|$X&>c1$Sg2Yd5p;%sqll+hRr?(=xeaS}TysWcdf<~PYKF$F_d*0R{__0i z`Yk3BVt3b~Nxy?no`MKs{`W*N0Y0hgM_QBN?o(mLc~C6xY;TC4C=Ln{ma0G;G)wO6 zia3L#hED!(E3~Ylb>GU_y&f3#BEXibBX&frI{M&M{txvEZbpwPJcfxQm`#QIqy9n6 z#fp4BC!P@+YAbJM#0wc-z?pb*D5|Fa470S#1D**kGBoNm=@<`aw!qQ|f=qz8Z$NGb z0uenl7_Q$Y!*hf-OdP+FxT~dYqt;31L%9>*T?B#vGe}7(I1HLE+Ux5$x_RxP%3a?@ zf{n!q1!eLPuxp9Q0kQQ_)6>)8!`Y=XuVx)JfD`emOSgF@`VO$neW^`}O+4uIauv{# zEAN(TmRUuor>FC@o;r1^RVX$s+&M)uSeRhW#mYlQ`<^`COrg6OcGM?e1&;sY^zQ+sTd%@fGy;E@DTjyQxBS}PM>BR1xU>IU(fEG$3ITtufz zS1)+E#Xp7z?^xej^FZ-?aKBT1Kr$^4chBxZ{$-P(HznKK5e+Sl1wAS^r#*y3#ZE+6 zUYr{{t>DNbe{Z+^HpQnr6Q84ZH|OY9-H0`KGkTaI^DjPQEkq4RIy?KJ^7?=sSnVLls z?d-$08QOJ6lQhOp0T5w3$<35NX}mk#o(v>OMIF*DQ*m_KsfrH;YxdY?u6F#=nZGsR zz;EgWYWIayipM0H>AU>8AOMo9c4x;Zb!|Hkh3Anq)qh_YYbN_i)}19a`7(HOOMJ+f0wsyuY7 zR_&|n8d_>D)rBS^+*t^suDnfPlc0B>N+0VP&=VxwP!SNvm2tU_YgQ_c5X|boVx7 z1D;molj`x6(-}Q~nPQ!{a_CFs9=-Al5;V5oYC?*=n|nmCI*$>bPUl3NEtSiJw5TU4 zgyze0ceoHy54!l@Gq zz^m>oy_t>27I#m;e&F34b6kI#ZAQ=d22Fy+XKlS}HIe}HJY|-KaG{%{gh};6sn**Z zK2LC8E)pV?VQX{ZHx5lVyXSUdH$MRw40leCGUOS8 zx5ua378kaLaBbm}gisA&o{o}zE@OM{fi3Q(w|5B71sM77hF@Z;4xWDe0DD6suU^P%D`6^e);N`D!7mNnt-oy6UX+s*r&DIV}`{msZSx`&-3 z!rZIgZQ_-r**3Kxn!fvK?pH2Q-oU*PyUUD|_eP7gK~Kx*WKr<-3w5$&hzG>xZ~n<< zy#ClZd+dO#$?MYUhJ1t9_i&V?8j3d5dzdiIoYB?tqY?h|L-V3P-*WEGK=ON^(fgRX zYL36~smd)m?V#_I!}n%oN^nY`JsFUvzzlVi2>ex57eu@8C8ci*%Vgcd($yjfOXyMj zmzDYJb#NcZz@5}9jBRLs*2hsIlO05LfOB=I`X2_$?rY|q(kaJ%Nx%&~SKv;x+@_UL z**IBf^m)bQ(hRVH2A(8^xl3teJ^h1w#kll}$YmkMPjme^4}tF4JDEJb;$zgdYVNER zUz#k`9C{>Uu`4XSZvI_Ytb|B;N6@y68&Ci2e|qGAq6W(aE5;iwQj<-+2|nMi(zQI@ zm>22kv&0EnR6=El^ zSJ|~?(uIxb;0jExTz~p=V5r>CkRzQz@LB5jnKLivPFlM7A|K81k+(uzzs<4IlN0Ku zilj=zxo2hTrkL_WBEk!ZZSB<=;Bk^PzN+G$Ecx3v&73wFiYi?_*>rHjjri7*K}Vj@ zfcqKMcy$pFDjg-j$O9AS&RSJC7ckbuNfrU3JRNX787u$Q%#8;nHSSPiP2d``@4E zs~XySpiw6_Je`hwj;mKzah8Z}Q#5SzzpmcT-y=XH>`ZzdpWpAfKIVV5)bXwE$V9$| zHZ!-0OA}_KU>L_d5x3maOU+X@uwI5EbQ9-2J7_H;xqaZLmwJ$vv;E}pHw89g#(XLWS=ha z-YUOp_uN8@S=-wH;g$QRDGFWdCU5ozJjTghQ97pHQ;T2h%<4!z0!1iFTu2TsRr~)m zcd9~jPAgvG(yC22kJ3rpzlO|E`jWlm^EqQr{CRJx|Z+4dqcVR)m>d6pkgmQv1Xnn8ap_!wrt6 zKhj2Wuw7|(vibY91KTuM3Mc$;mb8Rqt2uNIS3gVrv9P0{PcUZg-nDrm>fgEUyS={*Cnj>nX(i0TrmFm z-kq3-nz5#bizFDJtv(DuDE=<%2^Z3R`mM;W`Au?>bO{TO?JxPGCByOf*brJU=TzPK zx38{XqnyUS9W`bWU+Bob;Z}Hd4~`;2Vl+m(cN4o$JG<*$({**4II=!?NuZ9JKEk;xN3{FreZR>fK@Fvv&N+!3?F?|_NUmxAfAhFG#Zm-;Ibqt#uDtBUI%wR z{>plMtJhjqOc?h~YdPO93UX3TX#AD$v_0)I9)*|tY!;OeQ#%z-Uvd|YV{lORA(8-8 zlYITI&^7@c3H>kz-Y$CWuXVz8jMsKF*c`bMpLOAh1TNm|a?TK%ncrQ9VgLfuP~bVf zS$=g%VE0E$pk3m_0ASbj2g%X6^E4FqIajCmb*x_meX`Exth~bDt10T8D%qoBx@@Zs z?fBT(GI-8Fd6EACGFf;q%9JibE5rr;5NRhT!#0=-(z;OxHzF>6eoF_nGAs77E83b| zX3E&BfMI`UT>fHa_KaT+-yVy#jQ82<>yk&mT}u>Ny;$vJ%=FX7r~&++#zTQitmA>3 ziCJgD@2E(#&FY0Q9%AaayBdI}P}ok&Ook6S>5}6wC0qB({i}Yhhl!wJ!TSQs%|dEk zZO%Q!MK(QMC;;V^Y?t`mO&EOCYLFh_{Ch@~=3yG(=!V|c4JG$yDT?3MPpBq60!K@f zDZvfQX)V%B~;zMuc?gM;+i&p6!frv0*APmEiI10R9L4pa`9e2T#UI~?K} zVtm@t+S&luCnhRY-k$bTcX3SRsWGjr&g)>7n7{S+^`+5__@Lt`7;MFkCQ5Ajex|yY z{e!PoxheDgR$|wD53zDv_4SRnM~Cd@0}TdyZV3^Fc2XN{fw~`iJpDuPLoa>fIXX5T z(PFaIh!&bRC+O@4x-4vU9UR2q6&41kE^F1$#^z>i$-{zzH+%CcR@*sW76#1ukfV&Q z09MahSrn%tcJG|aqrG8RPV|dYc)p?;Pq@`ji<{%FLa6zASkbEjXb zP<4$(_NnRxM>0N1X`8;CT>PR5a|P3%49;6B5|d+Yom@TN*M$}zzN8dV|8@%+Nhw^p zzL|IX$__!J4#8;r?Qe3D0gnU~?|6=5X8VjMbF9b?$-&>A%${lV**54>R4vMHw~%&3 z`&WujA*ZO;>r}nDqoSzoF`c4*JA>)Szt3Ot@7?RDV;ShME534Bsd-k=YhC6-J!Ty3 ztfF}Yvgbo*CAOW_Up`lnJK6i!*zWPyUBEf(m;EfeX!yvAUF?B6IW@i}O+o1mAfFu_ ze3K;WAGZIz5iJA&yq-_u1I!%-Ey`$JWmJGvK^yD1k{9QWSZgOu*Bdkzn+wNQm`gjF zN8HE^V_9Bxv7yy0;@G)u=ikTJ=$h}~UU6xHvf!(2ipZIsd$vQTOiz8|N-4)Q`P9Yr zO8Z;Z=J1X3Z0(wjuN>l0on?Q{r<~Z%Op_WC6jaI5^-#^@tcr;2a|R>H>n)CQM`RUP zc=>mCuNfWl%VG|l$c~bv0fTj%?eAkSD+X@IH<%R$RatSM-f@*-u<7!w$mQ*jI+A_A zXT7X~Sk-l=o{PFIGV&r-B}_dGx>%k7@Wfup*LN=Jzq?qM99)rq@GM}TO&{J~Ac)&= zP2UAV%THOqELrmAQ+q+U2ny2P{A6*!F{Ix~VV%FHWueoAWJrlb2xC@B<5*s@w&gRr zt%8>%M|fAP3t_~})L`%rQGRQ1>-N)I2WUm9rK+%UMVR&y?A^Vk~6Jo^4x8YL0Y zn^wA6iPpW3xLXY|?{xnfrcqBd7oH7j8+&XfP|o zKY{YXhdP|A{=o*jO>iFgby#g+(f9@gK(DVHp7Z>pORDHIVEBPCcJssR{h-oZ-^FPMO{)V`tn>_Sm8Xr6=M z#EiNfHIwH~TYA}r$hWlD4B9G>Jh_%xB|vELRBG+7fQaB*5ldCBSl$ItU#_1qx!xk> zF8Awoc98ddW<8@;O=xyE2dYMjiRYh>NGEmFP1T+861$`aKAupK?-R=fMZxW|69YCi z<3p`Y=Qpf8iDIy8-_l*lkzZp`k^2Go(ALa&|JE~f+Z26VQY&TVfQuxlTo&P0K^GtW z*F>Wqw$cN+CwkDSuasuff;A%&0OD|hd6QGPRox;d>fStD(})OJGXYM zW~mGT^9AcfEicRv9&Wz7rLT(x?cv6|83inoOEqM;P>7YNenQoqB?)IfmY<72;9bxHHO^&K&$t?lKHS{bO) z^vO2>tis@M00 zS6q%dZz$JbyDt7;x}ipi*cKDA@ObWxTl@-MzjKjr+OOY+ZriWxiqu?r#SHTb()7P| z#6K>7^DE6;I*upE6c->6Nky1<{oi6@=z7V9U0ja;&QaHn29DkA3x znMWgak!lwjZ<`lwZiyy6-alb#WCz4S%g63XEkuuI$5M414==sxfmm}o?`T?F+^(e0 zY@S&Vz#aItb0wPc?$cIEdK(v48uVQ3S|!ch!?QHFN58TSF%JC5502Wx`of_r>mc{7 zt^q7VdK-G9Tlvc~O?ikQ3K!@8IAh0B|1v+|SnSoGvoQ7G{xp1k*YRJg%G{OV4v{Cl zZ;8u<5jM_B6vvuQxoc!QX9orO_EuyHe+fm)!{(Vx<5^bkVIzmf#{i@LA*J~Uy}+`? zZvI<20!M!^O~10kXXwNP2N2j-?!FRIposm=|6Z%pK?$MbjHYGGs>{Cj>3I1VI=D~u zA6!36x6yH>z~qaK&6@)HIX=hiDo$QlZt{#Qgd-A>VIeA+HfOua4i(AFe|NHYs8qmk zPk_PL=2=rUK1yy~?}290^fH2Pa(e4^En89Q8Wfqn+CLTUTkWt=); zVEWi{+UnMFmfJl_W=+>m%~`Q@gCuZj5tvI=yO6#sZ_!w==Bsnt>K;ZUxzMc!_1@O4 zv1($E_~@_;Fynn?YhuD(*DU?6eKza?*rxonyVNcu-aZq)&V2i8_BGMYtbv?chF(lh zbxbdi4V{Bu+jQ5`O%S88V!tQHf`v8Zg|i$#SmT#IbDe+o{Kq4JdRhQpo_};?oA+7Y z|7%8Z`gEv(ni6*sL1^O=pvu_U)l4rNQ)LxwseR?}!v1OkD@9(SWi3CXi3=L&*0vC# z4~rJPyR@(xOV&SU_vDX_vkiG=ld=0UKZpsWGT7T|i$dc|0$R=N_y)+N5|9b^z$p_$&mj<6ih7xoxUJ^z+sfmgUP5B=bBt5~w2d zphyP{e)}Gi13QR769yO1^va!CK)}cVY;fyfEmH~`M$~f-19ia&orsJUUnh3AddPWT zPT(`!z7^Q(oBt@)f3;kg{5_eBYVjhF|G$@Q9#>@hcT6lin>#(MV)IP4 z3n~j2H2x}gIo^30C6DeW$n`{Y4{`mCzAw>w;^gAo`nMUS%HPmxZu@VPB%gOXcU)ni znjbqTN#m=&M&pF!6{XD-JZy2d)H!h5Ikf;jm5ZYsV)69ojKus=yXc@fqnX3YvH$1} zucklc7ZRyYvwlKb-6RhZ9E zA$MWv&&M=#KBuh-QC&31<=B4t*{TcU3f{|!3gOrZyLm0KW4B7YsV>niTlior2%kms zRyr7D!itOK%P`z?^{f463*uX@eq7QjZ%J)$Z?CBi0u&zXzTE-}2Tc$&L2(8^@K#L% znw^TOvrItD&sV(KMrd34oHlw`<9B&CI()o*;2!I2{ebz7@wc)RnvJ3c@zo8Id8Ysj z6y$mL`z|BdC6J?|UGvn&4P9j7V(Oq8H-0G)z2cv9t;WorV0%KQCT1(jWO(1CXIO@CoXX=`g|JYc8c?sAbJr<6QTr zoNjFpq7mwmzEbMcDF&+%Lj8CIo_8#j`8y(UFRSwiByFeSpZ3h!q^-8|!AM9HpN)I~9K<3LQZ#*UW!5tWA8l*21 zV0fTBtzETuBbrBAfKMy(dsv=Y0=3h0p@{WyVj?e5TJJP6(gC5MMd(FbfA^ejP}O2;kzh8w$R<eODXA+bJ+Z5R7_UN}+FZ{9Y zLEc1mWRkribHJZ(YXc^``A5dtVkPVXQslR7E|JE3E%!}+a_xVZo?yhbI`MSyUjjQ0 z2t&zyX4YnrJ;yZdO@3vk#MAGZy=y3U!OCOd*KDR+K#_S1cw(VOk{}v$xfQEjgt-X= zh2zxJ(~M`EpyCa4S^qKTf8W|UMJ(%MZR6f!86qpj+2#B7zT8jA=kdPB6f*7SHjpv4 ze+mGR2Y888$A%Xx-7uulv@%MevOR15Ex+U9HhJfu$UM$IDsv>G)5*7}_RzC{ndjIx zz?~;d$e5@iuJSC#rq=n{+^r*Ff`5Pg4L2lNqzc9n1pEL>%_OPZtwHTEP~t5lQd z6DZXX-y4>(v)*UHay|MNhB41?Q54Fy$)-tjti^T0gZmhugL6N<3z7_lIscqa(KbN} zt)%5odS1`4D^H$zY>}X9KUl%R^kd0qU zcX=jPszb2wPzoIBtQZatU+VX%I7W|>8Jvpx*gf8MW4}z*ba$U#ItzzC`60RHL_r=!f`-dr5x-8_Ez zuNm{6IcHAnluD7QITEB6(Mf_I8a_<@?hX0?x~ewbzVT4>40O>C{oMoud~B^Nq&~dy z)nF&Qm4hqsfqsDB4|aJx&YOOU(-(|riyV0e8oNHTxOnd|Wm^2H*Dv%j5bGgm78By=VPJd#(*kzD8W zRRM+y@fYveDYDhzB+243b~An$e1zlzo}(M>?{Y7Fdb5sAeakdl4FQos#cq}Rsn7qW z*5sdiU!8>Pig<))J_w_bSqAMy^4p?V!lH-zz%1BC@`nE5KzIe+8v9sEv{`SJIiSOS z`QWh87-MX!1QEoS+J1dB>8n=yb$rWx&RYVa%!?5oZ{eFlj2H_FWMN|NAEvFN<8^vN zK6Q1^9ZG)B>TapJ>KA`N&22^_&!LNu&C|kCn17ym-0-i4cZbx6E`fjk8T!xTaZ_-0 zZT!Wolz&80Xcn$6tABp*to0v*)-HSCmvesJLfAkk>fBf7ZEw_tz-ouy$uh%UZS}vU zsv?-~aa^O1ZWKH6qUYIxAkTSq*Nx2dXrR&%`#$2mv!!)gN(qkKn=%9kZhYlNePe1d zZu~!5T}a4XfwbB~hAA;Nem!>AIH;C1!e+Z(MP8De|JGtgye}cer6@e%Rd2hS#jC~K z@*ni1{b#S>HaQNr(EAy?b~q$+u%w0b?_S`}DfBy@!9{!Zxy0eYR#C z=&bdnek-wRYHwFY9^{9^DhmI5B{ETuoP<}5TA7QEC|tK#9!T&10Mb@Y5H(SbqnjGv zHSKO|@!e%b(79){TzFQ?&ac*YZTimwuTzdS3peh3e%h!+$+5$MuV(UUB)F)dm(!(F zraVfS22HgqU+#6B|91cD?f5MH!`nfy3aZ&n$F-L|JN9zWf40x?r|2bq-kKTN%FlpW zI;QCy)09_X++oOq-+0LJ{Y^KnAKfgy_IApNgYTCw6s2(7FZ#xu)~1pl>HBX1G9C{x zBgtTRACksn8vSXvk+8j)R}5&~wv9rU{N}VO^VL1;hFd_BF0bYUezSMG4mfolC4%e3jX4)-z47aH0EjIZE1>3X)?# z+tKlUad_^-N4BjVTLhbn5;J+99^ru<3|Xn)ykr1m*g9jkapqK@_ZIk^etEx>Ve)^v zuP~qF(z3I)VQgPFMq9B^dEtcBaRYlBXJNM6)BD4QBuclcd_S|l;iWD!7Uz3%hCY>2 z#eWO#%9eN8H2>GN@hy|f9yYqE!u%P+geH_WmvaL263Cj}eU~LJ)hsAK?c&ZhxyG(G zT8X24S5l6xtUnMeF?CetVp}?!RlN4v(6Yj?s@|^R*FWq2C|^f4g0-<-My72UY=Xa+ z34M-gMU#YyGkPf0*3Eev;>t*(X(eDo-!bBF7a2=u{RRSs|uw8}s?R7%+3uCD5MH@%?0< z6g^sOSf|Y!y6-W0wbK{gCU}+D-7F6FhoT7*M4Sy3F*@H8%%o{D!o#c3Gdrfxs*on} z$c{(;ijn*kD(lJVg<~_|48VQKRhD}7AT|GNt5^uUShRM;fj*D?w~6TR$^5fs!!vzJwfoY)+hn)hMh$&r#PnH7%wS@dQ5Tn%{K0;(Y;DPB<= zwX_I#(MHp$f9_@6%XI3+F-2zHG$VO|;=h6F2C@oQGOvbI3LFsKY$5o5PO8)>-q~f3 z-7x}EhlgXcGG#f{bCvbvOTm=SVaHV<5{ye&+E#T&9GBi1w%kiw18lmyicm=W}|4=)zYGc4OO&w!C4;-!y%Q@rWyO_u?B(X`Ucz9y_WIR2;pjZ z@?;09UXcWqV<%vHZXRqTsl9XO&Wo?RoXh6z`h&)@B(5z#v2D$3AB}>kfW*dA4uubA z1`E~(K2G2K1u8}{PX_|WK?lpWyiVY@c|OTWg}<7A*uM3XORr!0g<)H+#l-s<<1I|t z)PjjUiVA-+*(Lv9!}$6-41C2Asv!`~ND?FvUD4sIw3v$_q(7;ckP^Y%jm6;58B>SP{}dfwu{kqdjar@B8?FfLER% zksTw=4^hX2h+tmKD!L?DhWi-Y1qcQKJ;AXuW3Y@#vSpz;D{{+7pcL%8%qN7@{dfuP zqnEV<0Z&V@#-;PfTtqMeKZqM7=8rhm_r7=YeX6zw3~m{ZYBmAI7U$Las%rqY=pt&9 zB#6R39Hb7~W9{jsR?W$oalMo3rg5iU!1s%%`^n)L^`m8%v(gh7SXW$!x+FO`5@ls1 zDBCZmAN+H0&4&!mV-J*08t#pTJ8AabT+^Za;u;<#&v7BNvjk@2Jm|vz;3#|w-p2;3 z`zNKL066N4p74P~fuBp8Nv0mO<_T|V18c`$WJe(g^6G~(7aoZq9%?PL03sZ#RV4@q zpE&p*eRk!DH$gL%pLK2|fey>g*MfuIOvHE*F$v>h*Nc2;F z_j@FQ;A|sJR3{#YpV$NsR?GrXgdfXJU`TcX;xeB{hK2)7%Wfk3WQZg^J-n8bv=+*M z$52l^hV>%};u5j3v1z=i8znpOxvz3RvJa;)?hzn%L_$n%iA*OCSmz0BU#2YcEdciF zCS<%n`5EllQ*PB9*RGHJ0TNlU_1rvSNasor+Zhh0^uqXkivDS≺UjSqSW=Bs>hGdmL8*R{8K|w8(FgVNkh6J7vf$sk%i~Os5^86 zT);$;TtSi&%jZuj3h3f7YXsxt%PO;35|rTthl$+cBS(*tA4WLiy4Jf;HZW5TAgX9F zovKH&l8%GJ%_AAO-Ask%>rbm@aA^@6IAf%Iyl{Sx%qD^bcFnuyjgE#iJ1RuLbesfK zTPKq@MLx#4xxrR} z@zGOaA-^R5jwI|!u8hvgT#g)5Vl+GbyP{|=fa%eRtlK3l6^uYU{x8^`OLI%!bFd%w zA{PL05?pXD)+MP+fHUcc`LN$N$fdO5vn%`UEJlYIWnwaHm}qQl6hW8)rZ|lAhczO* zbH9vjglR!UM1*;%zn|YtJX+@8-nX{0!nXWg!Da|qS{xOCr8eI+MpVM}eYvvaJ%|Pa z!6zjJ`z_i#fyoiST!2*$FfZSUSfz)f<733?ohTCM>b&q&&a0oEQIY=f&inl%WXX{v zU829p%yjd8X7W6u)2R^_xN!)x!ULwdFzYe%OKS8|x8CmkZ`3IL#3f@4r!$*qiBGL@ zriRO%oV-26P8kmR#LNUV?gqG3UIqheXl`m4=uwDa_Ida#b}0d|#9!$t$Uc(&TqA9( zy2wH=B_#!y(_QSkm1iPj0Niv3^%B){VFP6pI5=IA`(U{6;)Nkcoug+S>ldp#5)f) zo~o`1nmH(VAH6^s7evV-o)6(SxOi^Z%fr>M^tI$s$L(+huKnN(5l83k-JmM#Il#}D z1g)$V&-W$LBNCa68~R2q(JHIp6wIw-4qOA3UjXUoc-eMxNi8iqt-~ybuPC`h!i|~S zJ%nd85qJ=D)<5is@{x=fgb{6hS7yH+fetkP65FixbAcBm+&x;1O}d1zz$+eqK~Gc* zaEe6a1c1}R*OgP%%NQ7v{yqFwh4;?J0l%(i1I%Ob^SO| z0HPCeksGj)(QN(ti)0;>j64wJ5y#tjx6d52X-?e~akaZK3N&?}R6TB;y16g6BgaIX zNNxZOW1ZNbvG9HWooX_C*&g3dG?>HYkAiR z@bf!hBV7G+^Radj4UPWkezINQ*|L21UHRl2s?hv|^z^shviz=pKRIGzw1|x|FFkhb z*kSnint$8AdGm4vUzZd>f%UfzVeB<;$gKrm@J3Ef%iVVDnH`9X@IhAL>)fcZ@#zn( zcwn^M-Q}rfwKa%ezpBIz`&m}3Etx!K-S>ZAdUgBTjp=rS*W;fTtL!2iq?iRt1&(lQUyD|xh1!-A)0DRK^H!ok!blGV7HFL zN@Lkfrp~mwDW15%#fLt_(aMd;6VyFL4>n7fYP%ANO0QrlA`QhjDcze8fGz zkDCxHBgh0B0e^2GJJzuISVK<{yEUvWFAx{%&g(nqkPWSaY^FIpi{#9;O5N`s{IFNZ zBpJRcwza(^tZoG70<&qvI0k~Az0w7zykJ9ufB~<`l@X{B_ypDH)bg9 z*fnllU0w4DByU+Fk=K|+1|b`$6cN(a534<0@XwUg6v@FxmJ@FqJZHqP#$SZ7GV+O5 zk`qy{K0;M}(DDjFL@9LiB+rW%=VBbcR6KPc-qOaolW01p+SR@#hJS$CmKSc%-1yAc zG4~Z%mlrkldt17m(SZH#V{9!KYinx}C?h5-Yebxw*4FG@{V^cXOO85lu=f7^rB|U# zb2T8~;dobm3as_Hoy)ACtJ&F>uJY2kKv`61EPdPe4sG)DxSGLl-!}gK`OVE>@xRA3 zK7^~@>q9h_T1bvg(gAY%|0*DiT@W{mbg+=>1`!a`|EmH~x!9F&k#%O`HAXxbsWLdR z1%X)U@Hmo>ObApo5L01uqbTUfR&8#f!T^PCa=j?-Lhvw(=djbl|l3G8o zycD74&kt$YLgHnmkY*~s>ebNDO=O=Y8QZH=N04(f9)mv}NfwKM*t($%{?1$p`T+dnBQDSiL(<91ou7E9R` z2K%IV_gPw|u{G?~*AJ6;;#}r@_Uu8gs)Nzd(O1&aI^VoWJ$mfe?C>(y6;z;H&0eae z`u1(fUkm4aw+YYA&K^H=#tgOC_QIIQ4i73S_8vc;E5s3oFB`d(Ynhp6e0_g?w5fWx zIMvtNn`b1Mb@l4ijbD~g=TF7S0wH6gW9JXmvV$Uo){e?UI$1 z)xFBfU*+CONlCV6ab=#kvfFm02S#Ro;Y(3PMY{ZLVMfMkj7W7&&AB`mSJ&6CUnl3~ z<&B(H4fH!IyW-``mkGFTiH{>AIviozWo2cyy{y!{oOvCt!bvgj;zfpxYu9Arb~<5U zM?R{vlhfXV2Q!AID%zhvKNA>O9H3FWxOmRY)HLVL9sRhSI_NMeuxV#yWj&~_Uf8Of z(t6y-=Kyw*-nVaGzIv6Yr>8gfm$PmS8uUSK&p)1)5#l)E>uZ^7YUo;Sy=&L5PhY<# zcTXhQubTF&e(I>8s;aur-@mGUu%xNEnU9G+(fE$p{{7;$@Bbbb8~^#o%+|K}{(Yq* z|5!3y*s*)ho|Q!=)Le}0H~e?8hMt;#&&I*=X==&@i^Rnm7HVGM2QLK`wKhdhQS$t_ zM*O7#8V0J)ktyiEzC$ZQQ9)0*C>n|~3ZEYiA&dwt;k SYf+1DQ3krkIz?J`=l>7a^)Ma) literal 0 HcmV?d00001