From 739a4afc6f0e0880fcf0a5199af3cc96fb04179a Mon Sep 17 00:00:00 2001 From: ModelHub XC Date: Wed, 13 May 2026 06:26:36 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=EF=BC=8C=E7=94=B1ModelHub=20XC=E7=A4=BE=E5=8C=BA=E6=8F=90?= =?UTF-8?q?=E4=BE=9B=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model: israel/AfriqueQwen-14B-multiturn_1 Source: Original Platform --- .gitattributes | 36 + README.md | 61 ++ all_results.json | 8 + chat_template.jinja | 85 ++ checkpoint-34265/chat_template.jinja | 85 ++ checkpoint-34265/config.json | 75 ++ checkpoint-34265/generation_config.json | 9 + ..._zero_pp_rank_0_mp_rank_00_optim_states.pt | 3 + ..._zero_pp_rank_1_mp_rank_00_optim_states.pt | 3 + ..._zero_pp_rank_2_mp_rank_00_optim_states.pt | 3 + ..._zero_pp_rank_3_mp_rank_00_optim_states.pt | 3 + .../zero_pp_rank_0_mp_rank_00_model_states.pt | 3 + .../zero_pp_rank_1_mp_rank_00_model_states.pt | 3 + .../zero_pp_rank_2_mp_rank_00_model_states.pt | 3 + .../zero_pp_rank_3_mp_rank_00_model_states.pt | 3 + checkpoint-34265/latest | 1 + checkpoint-34265/model.safetensors | 3 + checkpoint-34265/rng_state_0.pth | 3 + checkpoint-34265/rng_state_1.pth | 3 + checkpoint-34265/rng_state_2.pth | 3 + checkpoint-34265/rng_state_3.pth | 3 + checkpoint-34265/scheduler.pt | 3 + checkpoint-34265/tokenizer.json | 3 + checkpoint-34265/tokenizer_config.json | 15 + checkpoint-34265/trainer_state.json | 272 +++++++ checkpoint-34265/training_args.bin | 3 + checkpoint-34265/zero_to_fp32.py | 760 ++++++++++++++++++ config.json | 75 ++ generation_config.json | 9 + model.safetensors | 3 + tokenizer.json | 3 + tokenizer_config.json | 15 + train_results.json | 8 + trainer_log.jsonl | 35 + trainer_state.json | 281 +++++++ training_args.bin | 3 + training_loss.png | Bin 0 -> 35297 bytes 37 files changed, 1887 insertions(+) create mode 100644 .gitattributes create mode 100644 README.md create mode 100644 all_results.json create mode 100644 chat_template.jinja create mode 100644 checkpoint-34265/chat_template.jinja create mode 100644 checkpoint-34265/config.json create mode 100644 checkpoint-34265/generation_config.json create mode 100644 checkpoint-34265/global_step34265/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt create mode 100644 checkpoint-34265/global_step34265/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt create mode 100644 checkpoint-34265/global_step34265/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt create mode 100644 checkpoint-34265/global_step34265/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt create mode 100644 checkpoint-34265/global_step34265/zero_pp_rank_0_mp_rank_00_model_states.pt create mode 100644 checkpoint-34265/global_step34265/zero_pp_rank_1_mp_rank_00_model_states.pt create mode 100644 checkpoint-34265/global_step34265/zero_pp_rank_2_mp_rank_00_model_states.pt create mode 100644 checkpoint-34265/global_step34265/zero_pp_rank_3_mp_rank_00_model_states.pt create mode 100644 checkpoint-34265/latest create mode 100644 checkpoint-34265/model.safetensors create mode 100644 checkpoint-34265/rng_state_0.pth create mode 100644 checkpoint-34265/rng_state_1.pth create mode 100644 checkpoint-34265/rng_state_2.pth create mode 100644 checkpoint-34265/rng_state_3.pth create mode 100644 checkpoint-34265/scheduler.pt create mode 100644 checkpoint-34265/tokenizer.json create mode 100644 checkpoint-34265/tokenizer_config.json create mode 100644 checkpoint-34265/trainer_state.json create mode 100644 checkpoint-34265/training_args.bin create mode 100644 checkpoint-34265/zero_to_fp32.py create mode 100644 config.json create mode 100644 generation_config.json create mode 100644 model.safetensors create mode 100644 tokenizer.json create mode 100644 tokenizer_config.json create mode 100644 train_results.json create mode 100644 trainer_log.jsonl create mode 100644 trainer_state.json create mode 100644 training_args.bin create mode 100644 training_loss.png diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..52373fe --- /dev/null +++ b/.gitattributes @@ -0,0 +1,36 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..1774ab1 --- /dev/null +++ b/README.md @@ -0,0 +1,61 @@ +--- +library_name: transformers +license: other +base_model: McGill-NLP/AfriqueQwen-14B +tags: +- llama-factory +- full +- generated_from_trainer +model-index: +- name: AfriqueQwen-14B-multiturn_1 + results: [] +--- + + + +# AfriqueQwen-14B-multiturn_1 + +This model is a fine-tuned version of [McGill-NLP/AfriqueQwen-14B](https://huggingface.co/McGill-NLP/AfriqueQwen-14B) on the afri_multiturn_1 dataset. + +## Model description + +More information needed + +## Intended uses & limitations + +More information needed + +## Training and evaluation data + +More information needed + +## Training procedure + +### Training hyperparameters + +The following hyperparameters were used during training: +- learning_rate: 1e-05 +- train_batch_size: 1 +- eval_batch_size: 8 +- seed: 42 +- distributed_type: multi-GPU +- num_devices: 4 +- gradient_accumulation_steps: 2 +- total_train_batch_size: 8 +- total_eval_batch_size: 32 +- optimizer: Use OptimizerNames.ADAMW_TORCH_FUSED with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments +- lr_scheduler_type: cosine +- lr_scheduler_warmup_steps: 0.1 +- num_epochs: 5.0 + +### Training results + + + +### Framework versions + +- Transformers 5.2.0 +- Pytorch 2.10.0+cu128 +- Datasets 4.0.0 +- Tokenizers 0.22.2 diff --git a/all_results.json b/all_results.json new file mode 100644 index 0000000..b4b04cb --- /dev/null +++ b/all_results.json @@ -0,0 +1,8 @@ +{ + "epoch": 5.0, + "total_flos": 481722301939712.0, + "train_loss": 0.401988563902201, + "train_runtime": 92209.7944, + "train_samples_per_second": 2.973, + "train_steps_per_second": 0.372 +} \ No newline at end of file diff --git a/chat_template.jinja b/chat_template.jinja new file mode 100644 index 0000000..699ff8d --- /dev/null +++ b/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/checkpoint-34265/chat_template.jinja b/checkpoint-34265/chat_template.jinja new file mode 100644 index 0000000..699ff8d --- /dev/null +++ b/checkpoint-34265/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/checkpoint-34265/config.json b/checkpoint-34265/config.json new file mode 100644 index 0000000..7026c3c --- /dev/null +++ b/checkpoint-34265/config.json @@ -0,0 +1,75 @@ +{ + "architectures": [ + "Qwen3ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": null, + "dtype": "bfloat16", + "eos_token_id": 151645, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 5120, + "initializer_range": 0.02, + "intermediate_size": 17408, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 32768, + "max_window_layers": 40, + "model_type": "qwen3", + "num_attention_heads": 40, + "num_hidden_layers": 40, + "num_key_value_heads": 8, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": false, + "transformers_version": "5.2.0", + "use_cache": false, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/checkpoint-34265/generation_config.json b/checkpoint-34265/generation_config.json new file mode 100644 index 0000000..39331ca --- /dev/null +++ b/checkpoint-34265/generation_config.json @@ -0,0 +1,9 @@ +{ + "_from_model_config": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "transformers_version": "5.2.0" +} diff --git a/checkpoint-34265/global_step34265/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt b/checkpoint-34265/global_step34265/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt new file mode 100644 index 0000000..61f4d2d --- /dev/null +++ b/checkpoint-34265/global_step34265/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0bd50d884195d5a02ea5f502a46785abfabd5b1cf47691bc32ccc118d9f4a97e +size 44304931131 diff --git a/checkpoint-34265/global_step34265/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt b/checkpoint-34265/global_step34265/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt new file mode 100644 index 0000000..c53be0f --- /dev/null +++ b/checkpoint-34265/global_step34265/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4ccd58cb4194e2cf5cf6e1f0a50cbb3e468a2684f9038c240b2a1b3f03ce6e89 +size 44304931131 diff --git a/checkpoint-34265/global_step34265/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt b/checkpoint-34265/global_step34265/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt new file mode 100644 index 0000000..cd8ce98 --- /dev/null +++ b/checkpoint-34265/global_step34265/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:53b3d5120bd406c8d013b1b00bfbdb667f1ab91aab4a38e481cb6c4dacb00f24 +size 44304931131 diff --git a/checkpoint-34265/global_step34265/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt b/checkpoint-34265/global_step34265/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt new file mode 100644 index 0000000..be198a8 --- /dev/null +++ b/checkpoint-34265/global_step34265/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:22ae7249ce9ebb4f522b9ae323e6e49485156009bdf4a90e9b05200f536fbc5a +size 44304931131 diff --git a/checkpoint-34265/global_step34265/zero_pp_rank_0_mp_rank_00_model_states.pt b/checkpoint-34265/global_step34265/zero_pp_rank_0_mp_rank_00_model_states.pt new file mode 100644 index 0000000..103e8e6 --- /dev/null +++ b/checkpoint-34265/global_step34265/zero_pp_rank_0_mp_rank_00_model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e1a3c69cfa37a40b0012e147f818a20bba4e813563dcb10517df264e6ac3fd6d +size 225000 diff --git a/checkpoint-34265/global_step34265/zero_pp_rank_1_mp_rank_00_model_states.pt b/checkpoint-34265/global_step34265/zero_pp_rank_1_mp_rank_00_model_states.pt new file mode 100644 index 0000000..1e66410 --- /dev/null +++ b/checkpoint-34265/global_step34265/zero_pp_rank_1_mp_rank_00_model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9cffa6c83b5cded0b3b59efa268b1d445f3829531df00a4cdb7dea63ae97a23f +size 225000 diff --git a/checkpoint-34265/global_step34265/zero_pp_rank_2_mp_rank_00_model_states.pt b/checkpoint-34265/global_step34265/zero_pp_rank_2_mp_rank_00_model_states.pt new file mode 100644 index 0000000..edacd27 --- /dev/null +++ b/checkpoint-34265/global_step34265/zero_pp_rank_2_mp_rank_00_model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0552dc1cd3d4a9a9a2a347b48ee4bbc750221f4c507ef4097a4435b8648e4c2a +size 225000 diff --git a/checkpoint-34265/global_step34265/zero_pp_rank_3_mp_rank_00_model_states.pt b/checkpoint-34265/global_step34265/zero_pp_rank_3_mp_rank_00_model_states.pt new file mode 100644 index 0000000..481fe71 --- /dev/null +++ b/checkpoint-34265/global_step34265/zero_pp_rank_3_mp_rank_00_model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4ac7747d1566dd8677b2c5c41dfa9db4108c7f06a37b76ea63f8e7e651f53763 +size 225000 diff --git a/checkpoint-34265/latest b/checkpoint-34265/latest new file mode 100644 index 0000000..d925976 --- /dev/null +++ b/checkpoint-34265/latest @@ -0,0 +1 @@ +global_step34265 \ No newline at end of file diff --git a/checkpoint-34265/model.safetensors b/checkpoint-34265/model.safetensors new file mode 100644 index 0000000..17146cb --- /dev/null +++ b/checkpoint-34265/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1ca62e96fb25ec28dd201b6314f4ce7ee390c52f386feb103ec06cd666f0fc22 +size 29536666272 diff --git a/checkpoint-34265/rng_state_0.pth b/checkpoint-34265/rng_state_0.pth new file mode 100644 index 0000000..16cd5c1 --- /dev/null +++ b/checkpoint-34265/rng_state_0.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:81d5f83aeb4b3f559bd28377336d47659b320e7f6ef2e5a723d284716278a151 +size 15429 diff --git a/checkpoint-34265/rng_state_1.pth b/checkpoint-34265/rng_state_1.pth new file mode 100644 index 0000000..726c9ed --- /dev/null +++ b/checkpoint-34265/rng_state_1.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2626437dcb133ffcf003ac89603f8cce07459b93a98d760cd9419e0d6a994067 +size 15429 diff --git a/checkpoint-34265/rng_state_2.pth b/checkpoint-34265/rng_state_2.pth new file mode 100644 index 0000000..a48d05f --- /dev/null +++ b/checkpoint-34265/rng_state_2.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ae777e24d50cb7159634e1245f0697ba0fc64d5b26d535f2c80e411371a90b1c +size 15429 diff --git a/checkpoint-34265/rng_state_3.pth b/checkpoint-34265/rng_state_3.pth new file mode 100644 index 0000000..0ad0950 --- /dev/null +++ b/checkpoint-34265/rng_state_3.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:afc5a67564eebcfc961e8f1406a7418cc73497c2935a39af0232ef59f8153a6a +size 15429 diff --git a/checkpoint-34265/scheduler.pt b/checkpoint-34265/scheduler.pt new file mode 100644 index 0000000..9722ba4 --- /dev/null +++ b/checkpoint-34265/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c94495b2fdaa190f2f28183e73d97cd217405437055caeb77964e4120313faee +size 1465 diff --git a/checkpoint-34265/tokenizer.json b/checkpoint-34265/tokenizer.json new file mode 100644 index 0000000..c7afbed --- /dev/null +++ b/checkpoint-34265/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506 +size 11422650 diff --git a/checkpoint-34265/tokenizer_config.json b/checkpoint-34265/tokenizer_config.json new file mode 100644 index 0000000..ea4d101 --- /dev/null +++ b/checkpoint-34265/tokenizer_config.json @@ -0,0 +1,15 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "padding_side": "right", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/checkpoint-34265/trainer_state.json b/checkpoint-34265/trainer_state.json new file mode 100644 index 0000000..eca68de --- /dev/null +++ b/checkpoint-34265/trainer_state.json @@ -0,0 +1,272 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 5.0, + "eval_steps": 500, + "global_step": 34265, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.1459321415541773, + "grad_norm": 3.8735740724697223, + "learning_rate": 2.915086081120514e-06, + "loss": 1.154789306640625, + "step": 1000 + }, + { + "epoch": 0.2918642831083546, + "grad_norm": 3.513073777315236, + "learning_rate": 5.833090166326233e-06, + "loss": 0.9386199340820313, + "step": 2000 + }, + { + "epoch": 0.43779642466253194, + "grad_norm": 2.667804789171682, + "learning_rate": 8.751094251531953e-06, + "loss": 0.8885218505859375, + "step": 3000 + }, + { + "epoch": 0.5837285662167092, + "grad_norm": 1.8311641416860456, + "learning_rate": 9.991513345767592e-06, + "loss": 0.8694163208007812, + "step": 4000 + }, + { + "epoch": 0.7296607077708865, + "grad_norm": 2.250397237666049, + "learning_rate": 9.936020028278053e-06, + "loss": 0.85284765625, + "step": 5000 + }, + { + "epoch": 0.8755928493250639, + "grad_norm": 2.6756387164181197, + "learning_rate": 9.829343371836088e-06, + "loss": 0.821312744140625, + "step": 6000 + }, + { + "epoch": 1.021452024808464, + "grad_norm": 2.6729990910245593, + "learning_rate": 9.672589544454328e-06, + "loss": 0.7774617919921875, + "step": 7000 + }, + { + "epoch": 1.1673841663626414, + "grad_norm": 2.642277321184833, + "learning_rate": 9.46738398205746e-06, + "loss": 0.608907958984375, + "step": 8000 + }, + { + "epoch": 1.3133163079168186, + "grad_norm": 2.1544218289818913, + "learning_rate": 9.215854533761766e-06, + "loss": 0.6088528442382812, + "step": 9000 + }, + { + "epoch": 1.459248449470996, + "grad_norm": 2.3990917619771497, + "learning_rate": 8.920609397454381e-06, + "loss": 0.6051533203125, + "step": 10000 + }, + { + "epoch": 1.6051805910251733, + "grad_norm": 2.5614175915079387, + "learning_rate": 8.584710074466158e-06, + "loss": 0.6031138916015625, + "step": 11000 + }, + { + "epoch": 1.7511127325793505, + "grad_norm": 1.5472599757402412, + "learning_rate": 8.211639623780629e-06, + "loss": 0.6085886840820313, + "step": 12000 + }, + { + "epoch": 1.897044874133528, + "grad_norm": 2.384688518854998, + "learning_rate": 7.805266544962458e-06, + "loss": 0.6080531616210938, + "step": 13000 + }, + { + "epoch": 2.042904049616928, + "grad_norm": 2.0076063347622974, + "learning_rate": 7.3698046643160645e-06, + "loss": 0.520530029296875, + "step": 14000 + }, + { + "epoch": 2.1888361911711054, + "grad_norm": 1.529742164922155, + "learning_rate": 6.909769440229038e-06, + "loss": 0.33792144775390626, + "step": 15000 + }, + { + "epoch": 2.334768332725283, + "grad_norm": 2.116689548920815, + "learning_rate": 6.4299311407857035e-06, + "loss": 0.3347291259765625, + "step": 16000 + }, + { + "epoch": 2.48070047427946, + "grad_norm": 1.8084278631560584, + "learning_rate": 5.935265379168761e-06, + "loss": 0.3385657958984375, + "step": 17000 + }, + { + "epoch": 2.6266326158336373, + "grad_norm": 3.29945063597593, + "learning_rate": 5.430901519764892e-06, + "loss": 0.33349261474609376, + "step": 18000 + }, + { + "epoch": 2.7725647573878147, + "grad_norm": 1.9290178050582503, + "learning_rate": 4.9220694899697185e-06, + "loss": 0.3383995361328125, + "step": 19000 + }, + { + "epoch": 2.918496898941992, + "grad_norm": 1.8157318709036212, + "learning_rate": 4.414045549219315e-06, + "loss": 0.3316441650390625, + "step": 20000 + }, + { + "epoch": 3.064356074425392, + "grad_norm": 3.0277622115117175, + "learning_rate": 3.912097577588397e-06, + "loss": 0.2421361541748047, + "step": 21000 + }, + { + "epoch": 3.2102882159795696, + "grad_norm": 2.0816758739547914, + "learning_rate": 3.4214304512770823e-06, + "loss": 0.1313203887939453, + "step": 22000 + }, + { + "epoch": 3.356220357533747, + "grad_norm": 1.739691217957409, + "learning_rate": 2.9471320714071095e-06, + "loss": 0.12995486450195312, + "step": 23000 + }, + { + "epoch": 3.502152499087924, + "grad_norm": 3.1257187709887875, + "learning_rate": 2.4941206057740675e-06, + "loss": 0.1304095458984375, + "step": 24000 + }, + { + "epoch": 3.6480846406421015, + "grad_norm": 2.2140995652592785, + "learning_rate": 2.06709349062457e-06, + "loss": 0.12645170593261718, + "step": 25000 + }, + { + "epoch": 3.7940167821962785, + "grad_norm": 1.7680132762507759, + "learning_rate": 1.6704787212769829e-06, + "loss": 0.12362543487548829, + "step": 26000 + }, + { + "epoch": 3.939948923750456, + "grad_norm": 3.186667247027872, + "learning_rate": 1.3083889366705216e-06, + "loss": 0.11934644317626954, + "step": 27000 + }, + { + "epoch": 4.085808099233856, + "grad_norm": 0.9381402976600279, + "learning_rate": 9.845787739562829e-07, + "loss": 0.07250856018066407, + "step": 28000 + }, + { + "epoch": 4.231740240788033, + "grad_norm": 1.5081719675375433, + "learning_rate": 7.024059353355333e-07, + "loss": 0.03657876205444336, + "step": 29000 + }, + { + "epoch": 4.377672382342211, + "grad_norm": 1.3793894154463466, + "learning_rate": 4.64796370857008e-07, + "loss": 0.035053966522216796, + "step": 30000 + }, + { + "epoch": 4.523604523896388, + "grad_norm": 1.9377667427717677, + "learning_rate": 2.7421393820510846e-07, + "loss": 0.0349309196472168, + "step": 31000 + }, + { + "epoch": 4.669536665450566, + "grad_norm": 1.109997862535006, + "learning_rate": 1.326348540874095e-07, + "loss": 0.03447756195068359, + "step": 32000 + }, + { + "epoch": 4.815468807004743, + "grad_norm": 1.152236730081828, + "learning_rate": 4.152720214406214e-08, + "loss": 0.03444968414306641, + "step": 33000 + }, + { + "epoch": 4.96140094855892, + "grad_norm": 1.6457557752833996, + "learning_rate": 1.8357098688476238e-09, + "loss": 0.034835365295410155, + "step": 34000 + } + ], + "logging_steps": 1000, + "max_steps": 34265, + "num_input_tokens_seen": 0, + "num_train_epochs": 5, + "save_steps": 50000, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 481722301939712.0, + "train_batch_size": 1, + "trial_name": null, + "trial_params": null +} diff --git a/checkpoint-34265/training_args.bin b/checkpoint-34265/training_args.bin new file mode 100644 index 0000000..ddad9ab --- /dev/null +++ b/checkpoint-34265/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aeba44e7611f40dbc5cb19262d1f7cf185351624bc850381dceb5317d44a57f4 +size 7377 diff --git a/checkpoint-34265/zero_to_fp32.py b/checkpoint-34265/zero_to_fp32.py new file mode 100644 index 0000000..5995d6e --- /dev/null +++ b/checkpoint-34265/zero_to_fp32.py @@ -0,0 +1,760 @@ +#!/usr/bin/env python + +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team + +# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets +# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in +# the future. Once extracted, the weights don't require DeepSpeed and can be used in any +# application. +# +# example: +# python zero_to_fp32.py . output_dir/ +# or +# python zero_to_fp32.py . output_dir/ --safe_serialization + +import argparse +import torch +import glob +import math +import os +import re +import gc +import json +import numpy as np +from tqdm import tqdm +from collections import OrderedDict +from dataclasses import dataclass + +# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with +# DeepSpeed data structures it has to be available in the current python environment. +from deepspeed.utils import logger +from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS, + FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES, + FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS) + + +@dataclass +class zero_model_state: + buffers: dict() + param_shapes: dict() + shared_params: list + ds_version: int + frozen_param_shapes: dict() + frozen_param_fragments: dict() + + +debug = 0 + +# load to cpu +device = torch.device('cpu') + + +def atoi(text): + return int(text) if text.isdigit() else text + + +def natural_keys(text): + ''' + alist.sort(key=natural_keys) sorts in human order + http://nedbatchelder.com/blog/200712/human_sorting.html + (See Toothy's implementation in the comments) + ''' + return [atoi(c) for c in re.split(r'(\d+)', text)] + + +def get_model_state_file(checkpoint_dir, zero_stage): + if not os.path.isdir(checkpoint_dir): + raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist") + + # there should be only one file + if zero_stage <= 2: + file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt") + elif zero_stage == 3: + file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt") + + if not os.path.exists(file): + raise FileNotFoundError(f"can't find model states file at '{file}'") + + return file + + +def get_checkpoint_files(checkpoint_dir, glob_pattern): + # XXX: need to test that this simple glob rule works for multi-node setup too + ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys) + + if len(ckpt_files) == 0: + raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'") + + return ckpt_files + + +def get_optim_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt") + + +def get_model_state_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_model_states.pt") + + +def parse_model_states(files): + zero_model_states = [] + for file in files: + state_dict = torch.load(file, map_location=device, weights_only=False) + + if BUFFER_NAMES not in state_dict: + raise ValueError(f"{file} is not a model state checkpoint") + buffer_names = state_dict[BUFFER_NAMES] + if debug: + print("Found buffers:", buffer_names) + + # recover just the buffers while restoring them to fp32 if they were saved in fp16 + buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names} + param_shapes = state_dict[PARAM_SHAPES] + + # collect parameters that are included in param_shapes + param_names = [] + for s in param_shapes: + for name in s.keys(): + param_names.append(name) + + # update with frozen parameters + frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None) + if frozen_param_shapes is not None: + if debug: + print(f"Found frozen_param_shapes: {frozen_param_shapes}") + param_names += list(frozen_param_shapes.keys()) + + # handle shared params + shared_params = [[k, v] for k, v in state_dict["shared_params"].items()] + + ds_version = state_dict.get(DS_VERSION, None) + + frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None) + + z_model_state = zero_model_state(buffers=buffers, + param_shapes=param_shapes, + shared_params=shared_params, + ds_version=ds_version, + frozen_param_shapes=frozen_param_shapes, + frozen_param_fragments=frozen_param_fragments) + zero_model_states.append(z_model_state) + + return zero_model_states + + +def parse_optim_states(files, ds_checkpoint_dir): + total_files = len(files) + state_dicts = [] + for f in tqdm(files, desc='Loading checkpoint shards'): + state_dict = torch.load(f, map_location=device, mmap=True, weights_only=False) + # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights + # and also handle the case where it was already removed by another helper script + state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None) + state_dicts.append(state_dict) + + if ZERO_STAGE not in state_dicts[0][OPTIMIZER_STATE_DICT]: + raise ValueError(f"{files[0]} is not a zero checkpoint") + zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE] + world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT] + + # For ZeRO-2 each param group can have different partition_count as data parallelism for expert + # parameters can be different from data parallelism for non-expert parameters. So we can just + # use the max of the partition_count to get the dp world_size. + + if type(world_size) is list: + world_size = max(world_size) + + if world_size != total_files: + raise ValueError( + f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. " + "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes." + ) + + # the groups are named differently in each stage + if zero_stage <= 2: + fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS + elif zero_stage == 3: + fp32_groups_key = FP32_FLAT_GROUPS + else: + raise ValueError(f"unknown zero stage {zero_stage}") + + fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))] + return zero_stage, world_size, fp32_flat_groups + + +def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters): + """ + Returns fp32 state_dict reconstructed from ds checkpoint + + Args: + - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are) + + """ + print(f"Processing zero checkpoint '{ds_checkpoint_dir}'") + + optim_files = get_optim_files(ds_checkpoint_dir) + zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir) + print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}") + + model_files = get_model_state_files(ds_checkpoint_dir) + + zero_model_states = parse_model_states(model_files) + print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}') + + if zero_stage <= 2: + return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + elif zero_stage == 3: + return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + + +def _zero2_merge_frozen_params(state_dict, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + frozen_param_fragments = zero_model_states[0].frozen_param_fragments + + if debug: + num_elem = sum(s.numel() for s in frozen_param_shapes.values()) + print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in frozen_param_fragments.values()]) + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + state_dict[name] = frozen_param_fragments[name] + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +def _has_callable(obj, fn): + attr = getattr(obj, fn, None) + return callable(attr) + + +def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + + # Reconstruction protocol: + # + # XXX: document this + + if debug: + for i in range(world_size): + for j in range(len(fp32_flat_groups[0])): + print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}") + + # XXX: memory usage doubles here (zero2) + num_param_groups = len(fp32_flat_groups[0]) + merged_single_partition_of_fp32_groups = [] + for i in range(num_param_groups): + merged_partitions = [sd[i] for sd in fp32_flat_groups] + full_single_fp32_vector = torch.cat(merged_partitions, 0) + merged_single_partition_of_fp32_groups.append(full_single_fp32_vector) + avail_numel = sum( + [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups]) + + if debug: + wanted_params = sum([len(shapes) for shapes in param_shapes]) + wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes]) + # not asserting if there is a mismatch due to possible padding + print(f"Have {avail_numel} numels to process.") + print(f"Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + total_numel = 0 + total_params = 0 + for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups): + offset = 0 + avail_numel = full_single_fp32_vector.numel() + for name, shape in shapes.items(): + + unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape) + total_numel += unpartitioned_numel + total_params += 1 + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape) + offset += unpartitioned_numel + + # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and + # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex + # paddings performed in the code it's almost impossible to predict the exact numbers w/o the + # live optimizer object, so we are checking that the numbers are within the right range + align_to = 2 * world_size + + def zero2_align(x): + return align_to * math.ceil(x / align_to) + + if debug: + print(f"original offset={offset}, avail_numel={avail_numel}") + + offset = zero2_align(offset) + avail_numel = zero2_align(avail_numel) + + if debug: + print(f"aligned offset={offset}, avail_numel={avail_numel}") + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero2_merge_frozen_params(state_dict, zero_model_states) + + _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def zero3_partitioned_param_info(unpartitioned_numel, world_size): + remainder = unpartitioned_numel % world_size + padding_numel = (world_size - remainder) if remainder else 0 + partitioned_numel = math.ceil(unpartitioned_numel / world_size) + return partitioned_numel, padding_numel + + +def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + if debug: + for i in range(world_size): + num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values()) + print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in zero_model_states[0].frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states) + state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape) + + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +class GatheredTensor: + """ + A pseudo tensor that collects partitioned weights. + It is more memory efficient when there are multiple groups. + """ + + def __init__(self, flat_groups, flat_groups_offset, offset, partitioned_numel, shape): + self.flat_groups = flat_groups + self.flat_groups_offset = flat_groups_offset + self.offset = offset + self.partitioned_numel = partitioned_numel + self.shape = shape + self.dtype = self.flat_groups[0][0].dtype + + def contiguous(self): + """ + Merge partitioned weights from flat_groups into a single tensor. + """ + end_idx = self.offset + self.partitioned_numel + world_size = len(self.flat_groups) + pad_flat_param_chunks = [] + + for rank_i in range(world_size): + # for each rank, we need to collect weights from related group/groups + flat_groups_at_rank_i = self.flat_groups[rank_i] + start_group_id = None + end_group_id = None + for group_id in range(len(self.flat_groups_offset)): + if self.flat_groups_offset[group_id] <= self.offset < self.flat_groups_offset[group_id + 1]: + start_group_id = group_id + if self.flat_groups_offset[group_id] < end_idx <= self.flat_groups_offset[group_id + 1]: + end_group_id = group_id + break + # collect weights from related group/groups + for group_id in range(start_group_id, end_group_id + 1): + flat_tensor = flat_groups_at_rank_i[group_id] + start_offset = self.offset - self.flat_groups_offset[group_id] + end_offset = min(end_idx, self.flat_groups_offset[group_id + 1]) - self.flat_groups_offset[group_id] + pad_flat_param_chunks.append(flat_tensor[start_offset:end_offset]) + + # collect weights from all ranks + pad_flat_param = torch.cat(pad_flat_param_chunks, dim=0) + param = pad_flat_param[:self.shape.numel()].view(self.shape).contiguous() + return param + + +def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + avail_numel = sum([flat_group.numel() for flat_group in fp32_flat_groups[0]]) * world_size + + # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each + # param, re-consolidating each param, while dealing with padding if any + + # merge list of dicts, preserving order + param_shapes = {k: v for d in param_shapes for k, v in d.items()} + + if debug: + for i in range(world_size): + print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}") + + wanted_params = len(param_shapes) + wanted_numel = sum(shape.numel() for shape in param_shapes.values()) + # not asserting if there is a mismatch due to possible padding + avail_numel = fp32_flat_groups[0].numel() * world_size + print(f"Trainable params: Have {avail_numel} numels to process.") + print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + offset = 0 + total_numel = 0 + total_params = 0 + flat_groups_offset = [0] + list(np.cumsum([flat_tensor.numel() for flat_tensor in fp32_flat_groups[0]])) + for name, shape in tqdm(param_shapes.items(), desc='Gathering sharded weights'): + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + total_params += 1 + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + # memory efficient tensor + tensor = GatheredTensor(fp32_flat_groups, flat_groups_offset, offset, partitioned_numel, shape) + state_dict[name] = tensor + offset += partitioned_numel + + offset *= world_size + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero3_merge_frozen_params(state_dict, world_size, zero_model_states) + + _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def to_torch_tensor(state_dict, return_empty_tensor=False): + """ + Convert state_dict of GatheredTensor to torch tensor + """ + torch_state_dict = {} + converted_tensors = {} + for name, tensor in state_dict.items(): + tensor_id = id(tensor) + if tensor_id in converted_tensors: # shared tensors + shared_tensor = torch_state_dict[converted_tensors[tensor_id]] + torch_state_dict[name] = shared_tensor + else: + converted_tensors[tensor_id] = name + if return_empty_tensor: + torch_state_dict[name] = torch.empty(tensor.shape, dtype=tensor.dtype) + else: + torch_state_dict[name] = tensor.contiguous() + return torch_state_dict + + +def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, + tag=None, + exclude_frozen_parameters=False, + lazy_mode=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with + ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example + via a model hub. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + - ``lazy_mode``: get state_dict in lazy mode. It returns a dict of pesduo tensor instead of torch tensor, which is more memory efficient. + Convert the pesduo tensor to torch tensor by ``.contiguous()`` + + Returns: + - pytorch ``state_dict`` + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + # do the training and checkpoint saving + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu + model = model.cpu() # move to cpu + model.load_state_dict(state_dict) + # submit to model hub or save the model to share with others + + In this example the ``model`` will no longer be usable in the deepspeed context of the same + application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead. + + Note: the above usage may not work if your application doesn't have sufficient free CPU memory. + You may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with + the checkpoint. Or you can load state_dict in lazy mode :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, lazy_mode=True) # not on cpu + for name, lazy_tensor in state_dict.item(): + tensor = lazy_tensor.contiguous() # to cpu + print(name, tensor) + # del tensor to release memory if it no longer in use + """ + if tag is None: + latest_path = os.path.join(checkpoint_dir, 'latest') + if os.path.isfile(latest_path): + with open(latest_path, 'r') as fd: + tag = fd.read().strip() + else: + raise ValueError(f"Unable to find 'latest' file at {latest_path}") + + ds_checkpoint_dir = os.path.join(checkpoint_dir, tag) + + if not os.path.isdir(ds_checkpoint_dir): + raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist") + + state_dict = _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters) + if lazy_mode: + return state_dict + else: + return to_torch_tensor(state_dict) + + +def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, + output_dir, + max_shard_size="5GB", + safe_serialization=False, + tag=None, + exclude_frozen_parameters=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be + loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``output_dir``: directory to the pytorch fp32 state_dict output files + - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB + - ``safe_serialization``: whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`). + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + """ + + # Dependency pre-check + if safe_serialization: + try: + from safetensors.torch import save_file + except ImportError: + print('If you want to use `safe_serialization`, please `pip install safetensors`') + raise + if max_shard_size is not None: + try: + from huggingface_hub import split_torch_state_dict_into_shards + except ImportError: + print('If you want to use `max_shard_size`, please `pip install huggingface_hub`') + raise + + # Convert zero checkpoint to state_dict + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, + tag, + exclude_frozen_parameters, + lazy_mode=True) + + # Shard the model if it is too big. + weights_name = "model.safetensors" if safe_serialization else "pytorch_model.bin" + if max_shard_size is not None: + filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors") + # an memory-efficient approach for sharding + empty_state_dict = to_torch_tensor(state_dict, return_empty_tensor=True) + state_dict_split = split_torch_state_dict_into_shards(empty_state_dict, + filename_pattern=filename_pattern, + max_shard_size=max_shard_size) + else: + from collections import namedtuple + StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"]) + state_dict_split = StateDictSplit(is_sharded=False, + filename_to_tensors={weights_name: list(state_dict.keys())}) + + # Save the model by shard + os.makedirs(output_dir, exist_ok=True) + filename_to_tensors = state_dict_split.filename_to_tensors.items() + for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"): + shard_state_dict = {tensor_name: state_dict[tensor_name] for tensor_name in tensors} + shard_state_dict = to_torch_tensor(shard_state_dict) + output_path = os.path.join(output_dir, shard_file) + if safe_serialization: + save_file(shard_state_dict, output_path, metadata={"format": "pt"}) + else: + torch.save(shard_state_dict, output_path) + # release the memory of current shard + for tensor_name in list(shard_state_dict.keys()): + del state_dict[tensor_name] + del shard_state_dict[tensor_name] + del shard_state_dict + gc.collect() + + # Save index if sharded + if state_dict_split.is_sharded: + index = { + "metadata": state_dict_split.metadata, + "weight_map": state_dict_split.tensor_to_filename, + } + save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json" + save_index_file = os.path.join(output_dir, save_index_file) + with open(save_index_file, "w", encoding="utf-8") as f: + content = json.dumps(index, indent=2, sort_keys=True) + "\n" + f.write(content) + + +def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None): + """ + 1. Put the provided model to cpu + 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` + 3. Load it into the provided model + + Args: + - ``model``: the model object to update + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + + Returns: + - ``model`: modified model + + Make sure you have plenty of CPU memory available before you call this function. If you don't + have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it + conveniently placed for you in the checkpoint folder. + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint + model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir) + # submit to model hub or save the model to share with others + + Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context + of the same application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + """ + logger.info("Extracting fp32 weights") + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag) + + logger.info("Overwriting model with fp32 weights") + model = model.cpu() + model.load_state_dict(state_dict, strict=False) + + return model + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("checkpoint_dir", + type=str, + help="path to the desired checkpoint folder, e.g., path/checkpoint-12") + parser.add_argument("output_dir", + type=str, + help="directory to the pytorch fp32 state_dict output files" + "(e.g. path/checkpoint-12-output/)") + parser.add_argument( + "--max_shard_size", + type=str, + default="5GB", + help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size" + "lower than this size. If expressed as a string, needs to be digits followed by a unit (like `5MB`" + "We default it to 5GB in order for models to be able to run easily on free-tier google colab instances" + "without CPU OOM issues.") + parser.add_argument( + "--safe_serialization", + default=False, + action='store_true', + help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).") + parser.add_argument("-t", + "--tag", + type=str, + default=None, + help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1") + parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters") + parser.add_argument("-d", "--debug", action='store_true', help="enable debug") + args = parser.parse_args() + + debug = args.debug + + convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir, + args.output_dir, + max_shard_size=args.max_shard_size, + safe_serialization=args.safe_serialization, + tag=args.tag, + exclude_frozen_parameters=args.exclude_frozen_parameters) diff --git a/config.json b/config.json new file mode 100644 index 0000000..7026c3c --- /dev/null +++ b/config.json @@ -0,0 +1,75 @@ +{ + "architectures": [ + "Qwen3ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": null, + "dtype": "bfloat16", + "eos_token_id": 151645, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 5120, + "initializer_range": 0.02, + "intermediate_size": 17408, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 32768, + "max_window_layers": 40, + "model_type": "qwen3", + "num_attention_heads": 40, + "num_hidden_layers": 40, + "num_key_value_heads": 8, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": false, + "transformers_version": "5.2.0", + "use_cache": false, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..39331ca --- /dev/null +++ b/generation_config.json @@ -0,0 +1,9 @@ +{ + "_from_model_config": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "transformers_version": "5.2.0" +} diff --git a/model.safetensors b/model.safetensors new file mode 100644 index 0000000..17146cb --- /dev/null +++ b/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1ca62e96fb25ec28dd201b6314f4ce7ee390c52f386feb103ec06cd666f0fc22 +size 29536666272 diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..c7afbed --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506 +size 11422650 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..ea4d101 --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,15 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "padding_side": "right", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/train_results.json b/train_results.json new file mode 100644 index 0000000..b4b04cb --- /dev/null +++ b/train_results.json @@ -0,0 +1,8 @@ +{ + "epoch": 5.0, + "total_flos": 481722301939712.0, + "train_loss": 0.401988563902201, + "train_runtime": 92209.7944, + "train_samples_per_second": 2.973, + "train_steps_per_second": 0.372 +} \ No newline at end of file diff --git a/trainer_log.jsonl b/trainer_log.jsonl new file mode 100644 index 0000000..0bd3754 --- /dev/null +++ b/trainer_log.jsonl @@ -0,0 +1,35 @@ +{"current_steps": 1000, "total_steps": 34265, "loss": 1.154789306640625, "lr": 2.915086081120514e-06, "epoch": 0.1459321415541773, "percentage": 2.92, "elapsed_time": "0:44:56", "remaining_time": "1 day, 0:55:14"} +{"current_steps": 2000, "total_steps": 34265, "loss": 0.9386199340820313, "lr": 5.833090166326233e-06, "epoch": 0.2918642831083546, "percentage": 5.84, "elapsed_time": "1:30:05", "remaining_time": "1 day, 0:13:21"} +{"current_steps": 3000, "total_steps": 34265, "loss": 0.8885218505859375, "lr": 8.751094251531953e-06, "epoch": 0.43779642466253194, "percentage": 8.76, "elapsed_time": "2:14:43", "remaining_time": "23:24:07"} +{"current_steps": 4000, "total_steps": 34265, "loss": 0.8694163208007812, "lr": 9.991513345767592e-06, "epoch": 0.5837285662167092, "percentage": 11.67, "elapsed_time": "2:59:17", "remaining_time": "22:36:31"} +{"current_steps": 5000, "total_steps": 34265, "loss": 0.85284765625, "lr": 9.936020028278053e-06, "epoch": 0.7296607077708865, "percentage": 14.59, "elapsed_time": "3:44:00", "remaining_time": "21:51:05"} +{"current_steps": 6000, "total_steps": 34265, "loss": 0.821312744140625, "lr": 9.829343371836088e-06, "epoch": 0.8755928493250639, "percentage": 17.51, "elapsed_time": "4:28:31", "remaining_time": "21:04:59"} +{"current_steps": 7000, "total_steps": 34265, "loss": 0.7774617919921875, "lr": 9.672589544454328e-06, "epoch": 1.021452024808464, "percentage": 20.43, "elapsed_time": "5:13:07", "remaining_time": "20:19:38"} +{"current_steps": 8000, "total_steps": 34265, "loss": 0.608907958984375, "lr": 9.46738398205746e-06, "epoch": 1.1673841663626414, "percentage": 23.35, "elapsed_time": "5:57:43", "remaining_time": "19:34:26"} +{"current_steps": 9000, "total_steps": 34265, "loss": 0.6088528442382812, "lr": 9.215854533761766e-06, "epoch": 1.3133163079168186, "percentage": 26.27, "elapsed_time": "6:42:28", "remaining_time": "18:49:49"} +{"current_steps": 10000, "total_steps": 34265, "loss": 0.6051533203125, "lr": 8.920609397454381e-06, "epoch": 1.459248449470996, "percentage": 29.18, "elapsed_time": "7:27:22", "remaining_time": "18:05:33"} +{"current_steps": 11000, "total_steps": 34265, "loss": 0.6031138916015625, "lr": 8.584710074466158e-06, "epoch": 1.6051805910251733, "percentage": 32.1, "elapsed_time": "8:12:16", "remaining_time": "17:21:10"} +{"current_steps": 12000, "total_steps": 34265, "loss": 0.6085886840820313, "lr": 8.211639623780629e-06, "epoch": 1.7511127325793505, "percentage": 35.02, "elapsed_time": "8:57:00", "remaining_time": "16:36:21"} +{"current_steps": 13000, "total_steps": 34265, "loss": 0.6080531616210938, "lr": 7.805266544962458e-06, "epoch": 1.897044874133528, "percentage": 37.94, "elapsed_time": "9:41:39", "remaining_time": "15:51:27"} +{"current_steps": 14000, "total_steps": 34265, "loss": 0.520530029296875, "lr": 7.3698046643160645e-06, "epoch": 2.042904049616928, "percentage": 40.86, "elapsed_time": "10:26:21", "remaining_time": "15:06:39"} +{"current_steps": 15000, "total_steps": 34265, "loss": 0.33792144775390626, "lr": 6.909769440229038e-06, "epoch": 2.1888361911711054, "percentage": 43.78, "elapsed_time": "11:10:38", "remaining_time": "14:21:19"} +{"current_steps": 16000, "total_steps": 34265, "loss": 0.3347291259765625, "lr": 6.4299311407857035e-06, "epoch": 2.334768332725283, "percentage": 46.69, "elapsed_time": "11:55:04", "remaining_time": "13:36:17"} +{"current_steps": 17000, "total_steps": 34265, "loss": 0.3385657958984375, "lr": 5.935265379168761e-06, "epoch": 2.48070047427946, "percentage": 49.61, "elapsed_time": "12:39:52", "remaining_time": "12:51:43"} +{"current_steps": 18000, "total_steps": 34265, "loss": 0.33349261474609376, "lr": 5.430901519764892e-06, "epoch": 2.6266326158336373, "percentage": 52.53, "elapsed_time": "13:24:42", "remaining_time": "12:07:08"} +{"current_steps": 19000, "total_steps": 34265, "loss": 0.3383995361328125, "lr": 4.9220694899697185e-06, "epoch": 2.7725647573878147, "percentage": 55.45, "elapsed_time": "14:09:29", "remaining_time": "11:22:29"} +{"current_steps": 20000, "total_steps": 34265, "loss": 0.3316441650390625, "lr": 4.414045549219315e-06, "epoch": 2.918496898941992, "percentage": 58.37, "elapsed_time": "14:53:49", "remaining_time": "10:37:31"} +{"current_steps": 21000, "total_steps": 34265, "loss": 0.2421361541748047, "lr": 3.912097577588397e-06, "epoch": 3.064356074425392, "percentage": 61.29, "elapsed_time": "15:38:36", "remaining_time": "9:52:53"} +{"current_steps": 22000, "total_steps": 34265, "loss": 0.1313203887939453, "lr": 3.4214304512770823e-06, "epoch": 3.2102882159795696, "percentage": 64.21, "elapsed_time": "16:23:17", "remaining_time": "9:08:11"} +{"current_steps": 23000, "total_steps": 34265, "loss": 0.12995486450195312, "lr": 2.9471320714071095e-06, "epoch": 3.356220357533747, "percentage": 67.12, "elapsed_time": "17:08:01", "remaining_time": "8:23:30"} +{"current_steps": 24000, "total_steps": 34265, "loss": 0.1304095458984375, "lr": 2.4941206057740675e-06, "epoch": 3.502152499087924, "percentage": 70.04, "elapsed_time": "17:52:36", "remaining_time": "7:38:45"} +{"current_steps": 25000, "total_steps": 34265, "loss": 0.12645170593261718, "lr": 2.06709349062457e-06, "epoch": 3.6480846406421015, "percentage": 72.96, "elapsed_time": "18:37:12", "remaining_time": "6:54:02"} +{"current_steps": 26000, "total_steps": 34265, "loss": 0.12362543487548829, "lr": 1.6704787212769829e-06, "epoch": 3.7940167821962785, "percentage": 75.88, "elapsed_time": "19:21:44", "remaining_time": "6:09:17"} +{"current_steps": 27000, "total_steps": 34265, "loss": 0.11934644317626954, "lr": 1.3083889366705216e-06, "epoch": 3.939948923750456, "percentage": 78.8, "elapsed_time": "20:06:33", "remaining_time": "5:24:39"} +{"current_steps": 28000, "total_steps": 34265, "loss": 0.07250856018066407, "lr": 9.845787739562829e-07, "epoch": 4.085808099233856, "percentage": 81.72, "elapsed_time": "20:51:17", "remaining_time": "4:39:58"} +{"current_steps": 29000, "total_steps": 34265, "loss": 0.03657876205444336, "lr": 7.024059353355333e-07, "epoch": 4.231740240788033, "percentage": 84.63, "elapsed_time": "21:35:41", "remaining_time": "3:55:14"} +{"current_steps": 30000, "total_steps": 34265, "loss": 0.035053966522216796, "lr": 4.64796370857008e-07, "epoch": 4.377672382342211, "percentage": 87.55, "elapsed_time": "22:20:07", "remaining_time": "3:10:31"} +{"current_steps": 31000, "total_steps": 34265, "loss": 0.0349309196472168, "lr": 2.7421393820510846e-07, "epoch": 4.523604523896388, "percentage": 90.47, "elapsed_time": "23:04:46", "remaining_time": "2:25:50"} +{"current_steps": 32000, "total_steps": 34265, "loss": 0.03447756195068359, "lr": 1.326348540874095e-07, "epoch": 4.669536665450566, "percentage": 93.39, "elapsed_time": "23:49:25", "remaining_time": "1:41:10"} +{"current_steps": 33000, "total_steps": 34265, "loss": 0.03444968414306641, "lr": 4.152720214406214e-08, "epoch": 4.815468807004743, "percentage": 96.31, "elapsed_time": "1 day, 0:34:23", "remaining_time": "0:56:31"} +{"current_steps": 34000, "total_steps": 34265, "loss": 0.034835365295410155, "lr": 1.8357098688476238e-09, "epoch": 4.96140094855892, "percentage": 99.23, "elapsed_time": "1 day, 1:19:16", "remaining_time": "0:11:50"} +{"current_steps": 34265, "total_steps": 34265, "epoch": 5.0, "percentage": 100.0, "elapsed_time": "1 day, 1:36:49", "remaining_time": "0:00:00"} diff --git a/trainer_state.json b/trainer_state.json new file mode 100644 index 0000000..f04115f --- /dev/null +++ b/trainer_state.json @@ -0,0 +1,281 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 5.0, + "eval_steps": 500, + "global_step": 34265, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.1459321415541773, + "grad_norm": 3.8735740724697223, + "learning_rate": 2.915086081120514e-06, + "loss": 1.154789306640625, + "step": 1000 + }, + { + "epoch": 0.2918642831083546, + "grad_norm": 3.513073777315236, + "learning_rate": 5.833090166326233e-06, + "loss": 0.9386199340820313, + "step": 2000 + }, + { + "epoch": 0.43779642466253194, + "grad_norm": 2.667804789171682, + "learning_rate": 8.751094251531953e-06, + "loss": 0.8885218505859375, + "step": 3000 + }, + { + "epoch": 0.5837285662167092, + "grad_norm": 1.8311641416860456, + "learning_rate": 9.991513345767592e-06, + "loss": 0.8694163208007812, + "step": 4000 + }, + { + "epoch": 0.7296607077708865, + "grad_norm": 2.250397237666049, + "learning_rate": 9.936020028278053e-06, + "loss": 0.85284765625, + "step": 5000 + }, + { + "epoch": 0.8755928493250639, + "grad_norm": 2.6756387164181197, + "learning_rate": 9.829343371836088e-06, + "loss": 0.821312744140625, + "step": 6000 + }, + { + "epoch": 1.021452024808464, + "grad_norm": 2.6729990910245593, + "learning_rate": 9.672589544454328e-06, + "loss": 0.7774617919921875, + "step": 7000 + }, + { + "epoch": 1.1673841663626414, + "grad_norm": 2.642277321184833, + "learning_rate": 9.46738398205746e-06, + "loss": 0.608907958984375, + "step": 8000 + }, + { + "epoch": 1.3133163079168186, + "grad_norm": 2.1544218289818913, + "learning_rate": 9.215854533761766e-06, + "loss": 0.6088528442382812, + "step": 9000 + }, + { + "epoch": 1.459248449470996, + "grad_norm": 2.3990917619771497, + "learning_rate": 8.920609397454381e-06, + "loss": 0.6051533203125, + "step": 10000 + }, + { + "epoch": 1.6051805910251733, + "grad_norm": 2.5614175915079387, + "learning_rate": 8.584710074466158e-06, + "loss": 0.6031138916015625, + "step": 11000 + }, + { + "epoch": 1.7511127325793505, + "grad_norm": 1.5472599757402412, + "learning_rate": 8.211639623780629e-06, + "loss": 0.6085886840820313, + "step": 12000 + }, + { + "epoch": 1.897044874133528, + "grad_norm": 2.384688518854998, + "learning_rate": 7.805266544962458e-06, + "loss": 0.6080531616210938, + "step": 13000 + }, + { + "epoch": 2.042904049616928, + "grad_norm": 2.0076063347622974, + "learning_rate": 7.3698046643160645e-06, + "loss": 0.520530029296875, + "step": 14000 + }, + { + "epoch": 2.1888361911711054, + "grad_norm": 1.529742164922155, + "learning_rate": 6.909769440229038e-06, + "loss": 0.33792144775390626, + "step": 15000 + }, + { + "epoch": 2.334768332725283, + "grad_norm": 2.116689548920815, + "learning_rate": 6.4299311407857035e-06, + "loss": 0.3347291259765625, + "step": 16000 + }, + { + "epoch": 2.48070047427946, + "grad_norm": 1.8084278631560584, + "learning_rate": 5.935265379168761e-06, + "loss": 0.3385657958984375, + "step": 17000 + }, + { + "epoch": 2.6266326158336373, + "grad_norm": 3.29945063597593, + "learning_rate": 5.430901519764892e-06, + "loss": 0.33349261474609376, + "step": 18000 + }, + { + "epoch": 2.7725647573878147, + "grad_norm": 1.9290178050582503, + "learning_rate": 4.9220694899697185e-06, + "loss": 0.3383995361328125, + "step": 19000 + }, + { + "epoch": 2.918496898941992, + "grad_norm": 1.8157318709036212, + "learning_rate": 4.414045549219315e-06, + "loss": 0.3316441650390625, + "step": 20000 + }, + { + "epoch": 3.064356074425392, + "grad_norm": 3.0277622115117175, + "learning_rate": 3.912097577588397e-06, + "loss": 0.2421361541748047, + "step": 21000 + }, + { + "epoch": 3.2102882159795696, + "grad_norm": 2.0816758739547914, + "learning_rate": 3.4214304512770823e-06, + "loss": 0.1313203887939453, + "step": 22000 + }, + { + "epoch": 3.356220357533747, + "grad_norm": 1.739691217957409, + "learning_rate": 2.9471320714071095e-06, + "loss": 0.12995486450195312, + "step": 23000 + }, + { + "epoch": 3.502152499087924, + "grad_norm": 3.1257187709887875, + "learning_rate": 2.4941206057740675e-06, + "loss": 0.1304095458984375, + "step": 24000 + }, + { + "epoch": 3.6480846406421015, + "grad_norm": 2.2140995652592785, + "learning_rate": 2.06709349062457e-06, + "loss": 0.12645170593261718, + "step": 25000 + }, + { + "epoch": 3.7940167821962785, + "grad_norm": 1.7680132762507759, + "learning_rate": 1.6704787212769829e-06, + "loss": 0.12362543487548829, + "step": 26000 + }, + { + "epoch": 3.939948923750456, + "grad_norm": 3.186667247027872, + "learning_rate": 1.3083889366705216e-06, + "loss": 0.11934644317626954, + "step": 27000 + }, + { + "epoch": 4.085808099233856, + "grad_norm": 0.9381402976600279, + "learning_rate": 9.845787739562829e-07, + "loss": 0.07250856018066407, + "step": 28000 + }, + { + "epoch": 4.231740240788033, + "grad_norm": 1.5081719675375433, + "learning_rate": 7.024059353355333e-07, + "loss": 0.03657876205444336, + "step": 29000 + }, + { + "epoch": 4.377672382342211, + "grad_norm": 1.3793894154463466, + "learning_rate": 4.64796370857008e-07, + "loss": 0.035053966522216796, + "step": 30000 + }, + { + "epoch": 4.523604523896388, + "grad_norm": 1.9377667427717677, + "learning_rate": 2.7421393820510846e-07, + "loss": 0.0349309196472168, + "step": 31000 + }, + { + "epoch": 4.669536665450566, + "grad_norm": 1.109997862535006, + "learning_rate": 1.326348540874095e-07, + "loss": 0.03447756195068359, + "step": 32000 + }, + { + "epoch": 4.815468807004743, + "grad_norm": 1.152236730081828, + "learning_rate": 4.152720214406214e-08, + "loss": 0.03444968414306641, + "step": 33000 + }, + { + "epoch": 4.96140094855892, + "grad_norm": 1.6457557752833996, + "learning_rate": 1.8357098688476238e-09, + "loss": 0.034835365295410155, + "step": 34000 + }, + { + "epoch": 5.0, + "step": 34265, + "total_flos": 481722301939712.0, + "train_loss": 0.401988563902201, + "train_runtime": 92209.7944, + "train_samples_per_second": 2.973, + "train_steps_per_second": 0.372 + } + ], + "logging_steps": 1000, + "max_steps": 34265, + "num_input_tokens_seen": 0, + "num_train_epochs": 5, + "save_steps": 50000, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 481722301939712.0, + "train_batch_size": 1, + "trial_name": null, + "trial_params": null +} diff --git a/training_args.bin b/training_args.bin new file mode 100644 index 0000000..ddad9ab --- /dev/null +++ b/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aeba44e7611f40dbc5cb19262d1f7cf185351624bc850381dceb5317d44a57f4 +size 7377 diff --git a/training_loss.png b/training_loss.png new file mode 100644 index 0000000000000000000000000000000000000000..3191ca2f7cb4e60e76643aba7a8477f96c4c646f GIT binary patch literal 35297 zcmd?Rg;!8r&^LPMP6d%}6eLBu1OY)5MH-|_X{1|1xFvC9+(oHdM!@n)8|9|vjE#3R~$vM?hos=gJHh$7I`y7^zdU6+XJCEUB;y1eW z@BA!lq+;$IBSVjgDyGqCPuTl@K{xV3q%O$d}jaPKCZYP{2 zvJ*HqZ7YR-9^Lxr=csplIhFUrl96F&dbZDrR# z-#`6e8M%@(gzxC+=zBC5_Hop0uqZsgXuzeCIMMg%69KP5O_$_mQ}CNNZ=U$$jg$#xj9Cx7X{U2+dX}1+N*h;H zNXjS`UhpuFRap3<>U+vWc0EaU?il9R+e6As1PH5yM1#^nMoJ2otHe(R`_~y6{5!k5 zMdI0!1jNJ_7kc7qKUWlSb8#sn zWi1I25s%v=6=K}r-u`|n+~6!#L7luCwxjsTsvM`Mr(HZ{O{^_v2L{`JipmHB0s^wM zi#|HGo>{Gv^Y97?-1qeK49CF2#SfK#kTuhz5JpN6D$gQJ8$%=I=~S_gbV(mhv7w{4 zy$~lG8A^01Qqa5`ADMrK$C&@jy7wRwV9NCFTWG(^|ID;ZX|qtvUbcjQ%l%- zfIdr;ThYcQ-{wbEfyt`}K39vAJrmt7$0#kkO4ta$B6Q5rF4{cu+FKm@w;CKe4Pi&CBo)ii;zYIX^O&+G@iJkBGQ=|9-q)T36StD!Yjf zQ|61me%*Ypag#Gi>yJlfuA-viT!J#=%^anea@#RJPamJxqta)4!^;zuTFMfG-gNT%rG*!k=g3qw-e#uGRpHAgA zk$H&UHX{Be1i>dI6_KXC>TSWSLYb?c5&NqzB}c1B`+m{Lc-7+zh{<%5*Yaoyos_in z-0W=N;NbmYqvnQjK?lc_53Va35P&e*X1~9rZ56Su z`+u~g4#v&5_V+{eOO%SVOSaAqXLKZLSm{LU+Gg8>qj>-QUCP*Q!y+^3jb|BrQf)O! z{YZUiXlT6LGh|Ysne?Kt^305|?obMumQPjNh3*9k&P1BL3|0bFG zl54)gOPI6;V!$PwTwS|iyPOjjpOh4=7)71N5rEA8`W5Qf@)uW6PwxnR zf*@S?-UPC%9KloF{S`|uj;WG3SYfVQxsFFlA6{NAP-z)({+G-9V2yx_i>t1#&SY!0 z9nw#gUK+s#&bEn(%Q|I7AGhW@=hoJUA%{u*JKYtaAHO#W)uxk; zMyl!D+Ye6EL$gnhV_@kMUlziabYGLpyO)kmu894QRDXBdu~Nm?BKDa>HsrLlk>)+I zY4*lHtOr8~E_^QGA#5qzeEiFz?PWgN87GA{|RTOMW(QS|u zw)*&6XNT2gLsL>NA*nxaou8jL7VB3N4(F+drKF^+VEuuh=qWet6mVbDg{5DtTY*U? zbHsdpv=B$>*j4NEryy^3V2GFEPTUN?gUjd=N#@ac*|>hP*-DnC_IxQLfN^DI zvzejQX11c|!D-yMDVI^-0x*_FHuB_kuodvC;^p~3JaRS+SxrPnSq zidh-X%dzZxd0R!L{jIc*)4|$=ndq!Rx{M#Kw9f$=`bNInidT|VeRumsmwz^jLQ;u< z$&aQH#B^|Q@I08X7KFt@Oe>uK(=OK-%FRsdXS@}Tm6KpJC4cGJsL)$pIFE?9zy(p= z`JEt#wE{!V*Ha#Ey1YS+Z1B!Xg-R6Yp0Jt2Qmm^a;&fSFz;SyZLO$Lz?mBqkk>C~>YTB# zaEaT(DcA&D7H>n=r>(T=mxpyM;56GNDM3BrGU|_ecOFa@ypI|*5rIkT#}08PY1VHk zkgkoNjd%Nmu3uMlaw^9sBowN~YdoCxQG~4gd8$^sh-x_ZE|2K*NfIU*!+cF{3Mwi~ zefRYrH|FN&S-toFSn5+VFz_PbaWcpGQ-0d)(~|+C@~30JH(*!=cwbBV)ZFJzX1#G^ zr0SZitZ`j^eMAqV&zCB@`L2kLnHghtF0R+>yIEn3l=>y3u1%3$oFU{heuwFhjjm8H zK{8X6lM7stm~8T@M3`X-SZVVK2$bvkRAk$hz*72nU{F<>Z=ZZG?*gQ_Y=|YRv0pF< zk&he|$NsIiI@WFu+XzQ;-hBl?AHQ-Y>CGE@3VQl$8}&45z zsU&?lU|?e6uhK7>*x6p_0YFz$J1xXBnWcb7K})M|Rm|b+;=)}?G7IUFKPWg@(a7j6 zdG=yP0eNV~V~*zMWK0e}YNVb$dxjY}UhA0gc)~c~j03^P!^3m_yJI=L1EncfSF&kR zb}TZQN!AD(7neG!HrpO?{P%^sh-7yCgx8))am+T3Q98t^Oq9px@TbS66XM>lUoZLl z`_G@BZ1UG^M!piUr=3Jb&3v)#ls8&VHYg{Rf9gL9Sm-8smzS4g{q-%CcQgT_F0~yp zv5*e=u){=PcIQ{$FNYJTh_6PvE{^ybHnUjN0f$m zyvgf23_z?>3~k!Wlamv3SfS&U)?|<%xX)U+DK|@GSA&8u-@SXs%+8Jtm$H)Ggo)3} z%tW9r-r$AExc=?iH-RVLuOOl8Rinjv!C4An^7zjd7cGW)SXnWlxa2}yiezqCL)~Jk z6x;Wf)-E-SfQzJ+R<`(^boBJ#0fZ^r`g>W{-?u(4my}5+8X&0JTd9e)hBpGHm{t1r zp>b|S0N@wqKSt49w<6Zi&`?{tA}L9CcD&zN`l!)Fm^L9TjR0|ScE0*FKyfwd^QE&= z>XF}T6P0+uxFqr?2kU6gL_({B64}#A{{qoq=H(^HQH~QissHrLB_uS|6pr~AE9tdk zQMu`Ju=e;^x`gi%pJ5w>T`an|C(;H#AV!%}`l5I}+}%ZN)eAHV1_3Zg{PQnxMZ-bW zR`?u}?`*oqbUVZ^)CWg>qahK=TL0Ua0vuMN4paF%HHs=0w{M5*i36c<@TvP|*y3BS zYj9<@Rhc`5-&kpNv_#N$^qSqz2C7F5t~YPr#y>vXi~~-Q>2tV|YcF%Ur2wJJh=GBD zPeSrB^JcJs!<1NM*M>WggWgY{ZbEKUjc&IQX6556)#VD`(PW>Vnlk*7E_n%(U;9Xr zPQODUI+OouZO2IaUyJs9PUaDGp1*~J#sxGtH4WA}8V{e`zH_G&7Js?TP&TExs+wA{ zlN}BY&iDsQqTd7Qwl#5Z9^2Kc1n_`@woE@D7)~LB17arQ-n(7On$8(nS__#Xl~;#5 z+45Yi&t-dF5xH_8cBTz?{z%#Q&BDq`K9POcyc3p(eA$>e&|!Rhd^s~SRuri~biL*X zgqaNq3k%azQJP>F%) z6ck!2?wQP+Sa9m=7A6rp8sH1t0J(vjowq?@N=Qx)EiS(H@N&JO!-0{*KVojxJ;#74 z2)r$i18kwJbHm4LgL`Gdx@S`#(7)iy(zOJ6e}5>B7peJx_z|LVpOkjt&()v|?t z(HU$cnqHg|pix(Y+iJPju9ffJkTN=Y3-?i@`wbGl zvq1$inATV8j^FbictT=Ic=Lw%(xpqMkdw z23glkvT<_qYx04QchNw)oJ?m#hwo!RPLV}g3SF=$ZV919- z4<`KH$_BKI?*Jf4W4M5TI)SFUXxbZu66vHUR&U&Fi2?-n`g!Ws;b9Me7eTi_n(UmM z*#Ml-LBDesX>M+wTVB?ELM)WF7Ar~j;?m}yks?8mUQiH~YrnIzqe>s`j!qJw-3oZH zqvK-$pjd!r<0noauRUuw5hx0+A08L68xI7;{c2>*%G%l-5+KcuCl?t#XP=p!yb(0V zeaUZZl!aZ+8tH!kl96x`1J9K!a;~nzKdNo5^|dt({7&i{n^zQ|Twq)rEll=!`<#>^ z5+z@d^aO3Q?^^(y1tp*z5a->#NXC(M0F5McBEmOvP@WHZN`>!V7j|~`Oq64p?QCxs z41O(Tg}cT=!WceGE)k50iNU!)eskU(jsQ?2AOjkKi?s*L9b!_r4+_MaiTdIJS#01F zcYJT@C;aA5{@cV?RNUC;Q(=2QZ+H%JE7i2@1Xm446^r76xRwqWf=x(BXd*!Kh4D3JsOWcqP-)Vtt?7n*%8*Vf{Lj2AUxkN+V51a$c_>G@ zzM7unD*tr0`ONvHvlL^0Ju~~8z;g$4@6r_;)>HE6l!mmnw(3!`r1UxeJ3BOoMJooA zU$O4UK$y>3VLw$@p$D+H!tu z3mk5E2Py;gNi(-MpMF(m1ud@H7=ouKc7n9Vf zC9Mkz3R0!KgbW`&67)NEt#m`P1kkN=zZ3U|n9IPfVJsk1$Ot4~1*sK7M@Od#P*?7* zvA!DBNVAUzn)i&7?;=nGqu^Poe7t;oasX26);cgieCTiZpC7jV!w9^p2YDnx&#+cR zGQY4phHhc1{;Ac{>P)~tUXdns74BWHN7@tc|F0hf%gqTWRcBErJ`o)S%@HUJ(gu=Q zP#B{(J}7q}i19wn{0cyC0XRT=AO@Cz)fXCAGG^^TIQRqv1yuI3Fk#c43nWte-ybed z*77UG(np}2?oq?V7&_6G9R05;M&1Oir#nO_N4k97s;#pVtM4VR0t7q25c4PMqpbk} zKt!^j1cKD@5mu;Rz_~9mqZADvA0HGe1%L)9N6my>>E!O-Jz8Sm3?#CWwkEM@cR&&? zTqyeiy-_~v{_JbVmYe?m=L|8;S_z&Qs4#K_LIUWqLmw>ftj)AC8Pqz2=H&qoqlX2F z5)sHPYH!3UbPXC0X9AceYaNw!b!ni&g~0L!NKU03$M_NMneghBe%5$)PL4B5fgd*U z8n@6xaD!}R5U~wQj~J!X&q?T1i^U7Cz1N8FnD)X`m-)*M{3Hrc6pMreEfndiF9)lh zj1YSKbh~8r^=-`e@0uV%nt)DY6_Aevs&RJK&iF?`Ok{&P<_VPpm2M21y{X`_!itKD zu0AFpAZQ0+E)aq~2g*;Xm@_M2Wr_O09K$vFG#}6;}vX--pec?6S9ejP=m502bl#X;FLS#E9<|LnSWcG zt^Y8u6yKG0p2wc{T4F##Wjmn;WrLLP@y7y28<1iwV`F1tCYh%32BJ%Rv!GXEB8!WQ zvp~?>KUnsWQ&O5GJVuy&)&zZbK5+rXXzPl&kdT@h4uUByNN(JTOQatu%d9;^P|qgt!| zFOnv0brA7~Lk1y$Vn4UI$PVeB3-Cvgi#cVslT9qJF%ZYDK?YF-DeBAfoVtBqO$(aT zop~zxa7fYN%1nNmo?c!A*4EZQ`dOeR>-=%`LMsYMSp^hv%DC6@@yt-sIWaLYLo^C5 z0@cYjZuK`C%2uR`wLd@oD_=D7iex_tj=D$~>baPptm^j_G=+|#p$AhyndQSjggvW2 z({FV9x!!5(3t}Rxn3!%pEGH01D}Y;2hMk5-MpDkr&B>Q&6G0$s16qs%eD|X6E7a8p z5MELTZo=5nh`YQSudz3%sJr)8>MDo>Y@pApwe|s7?=pQ8LJnnA0SF`zW;2nl9(CC> zcI1UOX|3Z7A1f`$hEnWmS|M_ zuXLs~yR!{0Gz36#yX%5Fl!*|+k6NQGj{lmjt$S=kcv8mD-k{`8R;_ZJF+!t>iWXNX z{F`jqont>ks8u$QJV83J8cS(6AqU7!X=G%CjtAIW;k~`RgaJlG503sCeiOLM(HysP z@$6pmUSV(fBMb3Fa=SSdx40pj5U3PN(KOFKMqM(Hq%X|QzFDeMqHno=UsX$Zp>zGZ zuWkprhCv3MoeN{?hQFI^FI-4ipJRj43m_v?yQrV>@Ep{>+js9aFBHt?yRQHaE2uQS zZ>szp(A2{F6eD>wFM`Vh<_c@!f-ctW>S>_*hff;G2ZF(U%7%0yaQEi)MD0qV(7vk8)kfBqDa7* zk{MW72Qut%?yT8UjvEgv_;Z!VrctFcqvO)XhQpr;H8-paaxhJ-lIGX3ZeF^3^03TE z`gWq`hFmv<$VYHdut4CgYiP)Ze;%!twO+e$4dOL{YC?^!ZEel23g}9l^nq?DZphDS z+e@hKfL1=Ir%yX!kqWpj-v`ALVUTdm9~6vHVgo9Pd+Ab&ByD^uJj*q8{AXXjsC_Te zDH}%)4>~xRFzzIVKz&RWbMe(fJ(O2}Gg(Uw>s@-D)<9iNaQg^3u|LsV-7Ximyf9jc z4P+gjk(g#1?-Ed{C0{irqgUkzSO)`=dr~KDhUxim=+-)$a_qHgHGlt>iFdmoq9KG! zBNnLnsaZSzbeH$UcgycJq`b?3(m27Az+qy_(CF(2ZU&mo-u!|w3{PRynhb_vg&7WG zz^OB;5yP^ve3>roBl2x{_-N@b`Sb1(y#@1c!&T6=WgaeSvoy(hFX#dc{vp>SZdT9$>e{8Avb3}18j8xL>08I z%`YrqAYj#i)UWMD2EHwWJpS_jvCvZE`vwKR6ost&<8E}VJL6vj1W<(?vQ8u~ldQkg zOTI^u5UwClcMI#7LjJ1&n61&L15yxfl|C;#{scitcz*1+71uidGYJuYlxV%%D+Xe%Krs z5gxwgCLv(Z4f2quJ4|Ev8u5RTDY@NE=a>n~SMrZskIiY+(J5xc)8eneU}t6hZ~?_> zZ@vE%sM=QoO}mo(nKm-KIF!JImm|bPT^J^gS{|Q<7*|YEvU6ghUSrbR)ARd%O^ds} z*oH**5^f9>8S5RJJ!Sfm7vb3V0Kb9L6CWKN$>j`X0)7y4T_z9gsBq>`=|EL$fLV}1 zLG{QR&(Q(?f}*1SeiA?vs4xqncN};^uRl+n!IKsXoR!E7dL>?{0yx7zV^TYSY)vI+t{N_1Mm zOReugtqZgOpavM=&yO#zK6C>8ci0=s7>fK*I~Y}DApb3ZGg7kW^X?TGg4KgnEi6YT zzN~y4mJ0z~rE}8<`y~vN6bX+8>7sSrjv}>Ur9V8xy0S=Bl=5j{pt~cKr~>RiVtO%h zq}+4c3{`bN_GFOo*dRtcS4(Q0zDggyXvjdVyGrY?-F}BZ@sL#602vXe6C2yW_Co*_ z0(ebdM8x{r(a zZY_x~QaL%Ova%klYieEuLNW*-d{_v;9TfOk7^YlHpatA+On_Hz-@YxE!uLq6Z_!0u zUO}Nx`n;q%B9XoQ>sPXfh=^QEpdhyZSb0MZV^zf<6^4;A$OAOzsUrKIQvR@q0s%)3H9Oy%)&B$;ft-dWT*t@@ z2VPq~iSw=#SSsA>Nrg-o+Vnqns)V{@IN2^S%*rI|I*3Dr32yvMLH%B zv0_hu6Qkf7BxV$Kcbs^E{UzYBVOVV3%82SJ5HEm|95f%(e1PMa0hGck$-(%y9L*CI z8`N&Z9xC4mhNbyyr8r0}QJ}6w7MWcNt_WBA^1($Da;HHPSSNMxd%Kq{E)y_Fcyx3Z zXvH`l9*2aR=0G|?QV|CF69#pp18hiG0E0Hd9`s|N1fu2B?09d*W~N2@V0|)d%*baQ zy`ci7ywjTm$rlL;7$w?KdwZ@>%(Pcq_Ho(#rCo635rctl^OJh^klL;G+676SRcFXN zdSIvlUjQZD@cp|UZOC9%pdCRK>6&zGwO@BCd{>-de|Nfdm2eap)gY`Vb|4Lnv18ht z#&<~zDY3p#!OuPMIROe*NuBRsA!>f3XcS&Aeo8`nI^e>Bz{PCClx{@BVEmqYhlz+i zzDPqiN{(!8Jg=mu2Qj}e6d1*~_jiWspIW4;DM<+$b4yDogi7;2wJ+hMvN z`zas@)0rD%y#y)gU8q|g&(TLId?3$~lqrgDvA80bWkCL?QJh(ZjcjJ4#bSL6GID;b z8K!R3Ckx6c#~LiPPoKk-uJ%M>5iaQ&)?FuzNb?#cj)SBAT!B3HW{EfM8HR~&qqoha z+2{1>7#R+8vx8`X^io4?#re5zAj}*^0vIpY3Cc+M*3zqY?q3_|vDW7mL5uC{RAkhJ zQPdV0c5xM$qCbECG#(GPYA5&svS}}6+yCFQkixU{i5F4b=dlnVoK=b)s5*WBtUv3; zopCP~f<``eJwbU^*6eS0Qw1@Yq>Y*;}6+t7=8<2C(mQ_my%e9Gmy4 z!GT4p^ zG%cVkA>j3*T}*Z-popNP5bYp%_w#fv8rNe2-GCJgsqAXdLu63Rp~FcuR^pV=(W zg?D}Nz4qf{dsakvA-E9q>C@c&EZoyCmOxZOS0`HU-bnjU0|r68LA8@3V1Ulxr=c!s zhVKpm^nwIKQ&JoF!eI2a!JBX@gn^N#0Uepo=o)?E8el9W5DuM(htLJ*2m1guqODfXJ3ZErROyy~-c1 z1^Sh&cCqd=(2-{WZg+t;qIM$VG>dTFdlL?x`2u*f%foqOK$+7ePgpJ>a+JyS^Sx^S z7cb5I`Jq+xvMqNBybekuAfQQF}3GhGBmhLcI6%l19rU#yWRny5(0+fPjUnv?#^~Hix!}p?Vnf zIM_*OLxCHFRB}zX90Ah`ntHmy+Ct@Jv+Y}oiZ}UEun^FH2%uqsiboqw-9WVD?d-0B zh-&?VPfCg&Ooeuz&E_zGN6gaF3?MfMlmskig_S=Z=6v`5eX?h)!@r5?&xwbdKO%1g zx`kL@Mc4m*I@u812DaL^K(B!GT2~vWU}!&*0(API3Lo0e0~({C!&IHA{!F13&%+%& z1bQ`WhI3<9d_a>_S62tk3KJQ4D;))7J1tQbS)Oi|1mr)EAsZJRO$&jY7SCvz|FSD679oNxuw}7e&v_gu6D>s z?4t)R_4KGy1%vpXTL29Tp1+cHQ1=f3h%3y)uc8sr7p-JIf>jpyAvQfZ83#lXC?csG z5y&irdcx5fg)Oogp6wo3P`i)K_pKsJa>L%3OdjDcmBC1#7Ha)|>sHOJM0T;li`rf) ziJB|AjUJn^AWU57iDd{@ecu5_#m2HMU}+Xe(5*f5sjY zq`r`_9KuBG#Xr%#L&;NW|c}O;K3R;YWn`YLL>41p?(G3nODvf`}O{ zeH8WX9Xk}v`Pc(dXnBFKN3Ahv`RRacwf+5}I@;y-tgMV5nGL;U)CSE|fzT*2fBJXV z6M(D}upuc>w4pvRF^P$Zq0j?`Xx`r5p|L-!9zTp`kferC?}W*tfQ1!|M`pG$Ri7&o zfVRPbU?Kn+0>l7{-KDRhtIyPud$7x6@Aq0dJ-t#a&tJg6jCe{E30m9C8Jq3a8ZSz< z-?}C88HNgcln(Ii2H_4U#``bVpq&gqGN7;Q{CJiBU2bkRP$w_~@CKn_>`R91jHVT+ zA*td|Ip6whh*v!x#)bBW4p7}DbM^$P zJXoxE9n?2^6up6kgMpypB8nD()X>8+=>)?CZGW5|@_8@6hGU5HazoKoK!PeELeEC; zE7dyIl$f_Z$b}K$)g9V zB%b>a8-8{YsD0@daLYgp77k5%P0LWqEAYKEFSlo)`3Y5!2s4F*<<2k8%l74`1ji%n+B`lG#fz1fde zg1f6eaFvFu8k6=tu@{cgRPRuGpWdSl5f+&NPtGO+nJHIB$A8Ya&9A)MdqNi;QBZ)F z4Lu$#f`SxsbAi3d>4`BL?AZ=_Syy2Bsjx|>Z*e7Ry|F$}w&~Sz!Vdl5q@5dHk3cM3q z^Pt!AXzSdaMkIhtBt5pLJ84WXKPI3bSM`=A-t9ze=!ylVnwgo|hB9mDLn;2tgLk1# zqPLP#`kCX%hqpYrdk)e3|6;pVSJks@+0^T1=(V4R_fErGaFSSvjuX5i<)$SPxUL@O z+nl8_d2f?0h*M_Z_Q1Ef64`<8w>(k2)+dNj9Q>V7{LPKf zVNHK6JHAA4+3WwD?*1C-?&;|q8ymArgA@pDAn~tX8_bJ9_63Dfz(+4io-er3d~Q!9 zc^%)SXaegs+jt+r*tcDQ^ONA)UoCx5wa&o0mS8qLRnmhGdU95%$jMDcJf$3DaPPHr z_Ov0h*$?K!dK%g$6=MLXdd`P+_VVOU4GN*NSvzizxxBO#1`b@MB{&jJZf>E$!3q(R zU*1a7p$!N+hK6rAa{-Kd`-XS)6qdYaXK0rQCl<$mttCF8F8 zr%PKSI>x3Wg<5EPVhP1L!;r+4taz0{I&2}A&0@CRl7>*vxugf{%qneUC;SUV3=c-g zInZn)B9eqK%J|ZN6e$P!7D1qI{ucZNb^o!DQsY(!Ra^{aF)9lgv8~WHzUDzgp2jOzuO9~AT`G4{zPKnv5XDHK*h<~<%MqOzyzkFX zYO5#INHUuX{{#-c_Y%y$tmIlu7W}~E$5&-%;RB`WjSLv|;xI_mI^Qpd@#Z1k!4*Fa zcvL85>RNfdMR%2qo>9c{N{94z6+fAjg_qVNgJdwEQWB_i6{wS_OQ8q3^?SqTW!>(# z8gcdC{kmv!dWa`(fDc5K#01j@ODdg+2}{RKsx1)^$C1ChHMWoV+!K91Ya$lN{*bTj z10PGZJ#oOZ`O#YWsxlN^_vD;`BANLNCN3o35$bxsns{p+ml|>yjD|>pOp+^*U3eh7 z`1Ox`sV>a>nC%=|8G7m!`QN!CTIdz^S;C@}Ffy%vU`!VpnD0)=Pfs|#yzvai#0>;M zx@}|0wpVwNXC3eZnjeJ7*3Y*#QY~xbLObJvAy6br;lbG=y)n7Q?kmSn#%xlD@GE=3 zMLQ36RcN+HWDM_ps_Cr&bus@5?9B+zj%B`ahJU23Fgem$meklWrWMoMb#1EAg#SR1 z!(i@nuHXZQGdkoS=p){JkXSKL7ltXB&f z>+fmv;pbGyD{FZ?_jzHpu>CVv<2*=YnWBVEA2u#dADy{Y-@6O?&8?9z7!?BBMT{7Y z=XByY<9>plvYVgYP!*S<8)4+XC5(k!`*|st;0`HE^Q*i(v@#c(W7;Sf3ajop{Kg73 zl$TeML7&w1jf_g7K#4kjix>OFSZ?uyTb$-cIzAN1FDxr)(BkO}V;c=c2n4a`YMApH zPo1y)e*V!s+~!xamb}Mt6HYhNwmL%>F9w z-cd326sW?8#`N3`u-@8fNTOcPl!y2KD2P#Ide`^SU_%QJGJF3ENmuTot`u&yd@LE0 zO_UB^ZJ-*azc@`;SgT~U7>kZNFD1CO@k-@VA|n zU?7$JcFA6Tcm%tl63|;i#z{yF%5IkTYz+!3ekU>B1ho*K0}XEwcR_Dgb~8JK2m7!}NCU3b7)pRM`eBBf4*Z zm_la$H9wyt5;H>Y_Fe#^iVe_7^XtEDdO331Zy6;Axb#buOW9RY$JuCFrJOG{J^Ef3 zA0O(owD~cEtL%U8)qy=YAeO(lJp5%+*mA0r=w-UlhI@LjRJo}VnJdA8oOTQ2!W_{Wl*>=V|wn&o4y7*(foGuy*WNl=3b?e;N+XmLt&f6dFwb5aVOzpfk` zx%n)&g%4EGAXU@)QLHd?4MDfR58$0ThgCuq7bF0U{xs_WxI1Z9bh!B$z$(sx6UO_c zi?i}(oY_eqvReQ8>awJeul)+imEjwd;O3?5+21o%GxVmhF140>A3v9a_XEumye@W- zUU?6k{_#P=<}S}8x}7J=%E>`@iBc#b;bf-D{_Oy_@Q#aoU>KXQs#i>Re0o{WFi)&0 zfPSRy`rO?u*H+GW_5!82JHQt|zTKF4Z;C#QH^#@jB`^~>;kB}t|0Jl$NsHn^O3!-x z8f)>xE0xd0O3x1eNkx5Z%)vkoYs$jNHXIEWK5;=-C-rF#n297Xp&CEIc~Nwo86E~^ zw=UtKiGHa@@8Do3#Z9BwE$4!?ShiOB=#RFo=|R(1$Z3~j9T^EiQz{#jVinOxnckd1 z;Em>R_A;vI86e?MBqhce{tOtd6SuvH>doaT7Yv0*CXJjNErdA#xniezc?&yvv?UUx z4|R>b*N?jiR|z|GM<+f=ci!04^hOK6^!Gs!5Tew&HFBWu zD6t8uU>GVY@TgvU9Ug9XVRHszB;?Zli9KAl(y3lWR~M1_>pN=eI8;Ie473ee2KJAh z)Q~0xNCgE87R2|lHwhL~G1l{x>mNYs`G0RdANhkwiOZU8-OA@?KnbqLjh&F-!avo| z{(7T)B%)fm3=`pWCF-zM&>z*cM$M!~&wo~!x*Sb`++cGO5$K6AdSKz2n}#I}+}yz5 zb1ei`RLw?+im=FFUYckN{V>JA8yV;vaA-dfHK|{r4@W?TgeayN{K3R{Pjz`uX)Q*r zy`|S;ag{`C`mjod<>-l>az2$1nsM-`HxzumjczEa!`gsth>2P@3@?^vzLAxzHp(Xk zroNy*^%82)PprOae_HY9Q|Xrbg9SrhPNTDR%^SDW;2cRfXQbDH@`s31r!=qMT=oNI zW$U(?B}z!C+r=Bzxl;!(&n6V$H^KXZC@q+QC=;d+bKTT9CHVHPQY72mEeUl{x@-rUbmgm8$&!Y{c zMKV;{3W`<)Xg8ZOz73zflDnEx&LwpDBuArA&Q3T8*-1}4X0_D}RwBBO!__JNf zu6E+D1EZUQc0~DZ!t!AJGewRFIPQ9t4@&|7x*%5=Dy%B*ExavX+x_5Q$Gjs;-%hs@ zdvy%E3aXyhkW?XoNjk&MQhX&ECtFS$a}|t`z}0IE1D{pHmR^eKzP=8x`IFVigJB4ST@ zv6T7pveoV>exAy?{Fj4{oyMeaJ z!A`OiU4LlGM&&sWMbLhwz0DaWa4Pv~N8^i&>N9v6up6+H@dB zj_f9JmiM&i?7;NMFZ~dKjW#I4t`vMSGEv^7PZscJ&-vNOovB=C{}OXq48J{0!^AtIL%)sxUrTrsro!DYReF`>oBC6|} z(D(wn_!Ahb-7$$wkLyfVEGXxPcPq|(DG5KTU~X=XBke}_ZugIxewwx+QbgZK65a?O zMjd7lLkT6@X0u5C4LyUL7gt>0YU$5G z=t>3NC-*@`ACE%+ibdcW{ACn?8~^M~O0$mNO7N7*M(M}rgGD*>gUU*ogoR?|c-KTFPv^%lm?Ox4* zxW&TO`W>SsN@}LHoO-KvLC2YerLx-`*vi;JJUH&&`>nvgO8Gj1$ejBq)GL`KUj(Q!ebTXUk<(D_a(AkM>tgKJ*@%a~z`(zKni_w$(f{0D{G zyh}rZkTo8m5B)YuiGZt`UI_!xUy6YdK)2N}1&ZaB@Vf7YYUkVduE#qUl+hA}-VrCD z1Pftq3A*lACt0xd^XQ(_10E*ykhrw>A7(7p)7ngj+bFnNaw^kE) zj@&(5?}nmA{{uDT6W`GjB5UkVy(4VOd1YJfz!qk!S$Hv!%`-3R$pOyqw|!w$Uv^=~ zuLS{%2C^sj(Hm>#iF_?{^u{-GdE~mk`19TEwaZZ6-2ktDpi90?`dd4ZeXhjEV6Ie& zv3K;cfb#^)JB#F*G?Yu%m|@;_?*z+lQL8~AOoOkpqaq7XT30e%ed_5j1;CCQ6r3<7 zG3MxTeR?FJYVx2tJQl}Eo}T`x66)CF@jOcDnS!9`*5`(Y|8rkT^u9uA*i2&`1I)=Q zB)%x0x<)fh0Z#dda;eEs>7D?f{4E9D1VV>La-RT^G$Y4={s8Q?0^`kS;OMVgi z;9j73@UsJ6HL}%qE^rXsW&xVb)9v-T(q}Yu+T+)c(x`esXi;(tMfs)y!0^;9R*5}*C+4`<(ZEfFLw{~J>JfQ12Mu>5D_u>QE2?0i z0MuUe_N&OK^fzq1gz*)cEi_#iG(Z zj2*(lx5YOO+`EG+LZyR6-$wU@hn_fzb2NTFK@rAPnAi=Q7cO;@+!&hbyv)d+-Fyra z(<-Jq7uA5#*S2{6LlYh!Ja}E+kMO#Qo4l1lnkfz}M&klGToNL#3RZ6!6W&?K8d!}5Y>eS91MioF0SKflgwqq1n#PZ4#;+LZ~&m!^N=vKFD%4L6_FU5j}lO-eyAl#Ygj3QJ1+Hg$w0!($HE4gh(Oiy4_96|okwF0M zb)B;)`t+9WSP=a*8DWXnMm`T#`O+I!fh+l>&gO;gk&|+hP(BWbVym%_KKc236!tMEE)q~a3Ut`61xOJC=zOEz+ts{#2qg4~R+m;r)-!io(ze6`Q%G7AR-`*Ge z3gwaG-|3s`j+x%ETd5eaW`NLCy(rRw@`S& zUuwz<%VR`=@=eO0SW62Fu#;KA%>})7lr*f+q%RGd-ST(NopMQeU&CUC*fikPAa`|- zN#?}Ad5?u)dy36Q7{yRbci^4t?^-08S(8Al|<+W zEQq9R|Ce4Do22v2Y4w`B5P{8PTmK7kcVgFIeU@6Ia!`)R|CiUruM5!PjM6s;VFt_n z({9;v(Ke8LN2v1kY8xQIak{kV=7djmFW%VK80Eakx>L^O9paho6Mjkms9v74sQD8w zMFPkFeS?=EeUPWTiUPoG^T^nC-D#t@y2|jOP#u@h`*Hdn#(UhZ_)ZGIXM}g3DqUc@ z6mle{kYmBiLNP%$-I|7I6Yw)nx(Kpd2s3kN>rsr^df75cn)f+`92Ih;MK5$&yh4ONY2?co<-*pBsa`PeN$ z&7g)qSb6gnvm9Y{-FzEs^Lr_X+P{>BVYGZeik6F&n`m>Uni<0mzHULyXBdvQ1fcu= z0Cq#abtLq{V-bd8Li2C6m8a!Aj)l)CQ)}d!FV=>`@0eqU8rh~ua!eJyzjdkcp`hW^ zrFvc$jQ$FJWYfiV(j~Z2Js$E!-fp|wViXT3rdu&WtFfS6K>?f~WES>IbOLRNLpxgl zZ!1G$;Ai9+Z*4CTWU{MlIS;{L25+z!<^Gb#@3LM@OKdkL?Vi_g5n0|!ApdGv!H2oj?d47+kw*KN_QFNB0a@TC#x zr-d)w=vw`)suEUvEkbgBehxbX;BzlmfmnE2JBHW=RWIR`^0KQ0wT@zdhT4rTHHRl2 zkgP+eFtI?ZxVZbT1z&`T6NrAX5tb+PGoqH1GFS>ua~(MF)dYOR>3HukHV%%Kr!4dk zcEN@(bpJiuwQI!amm+|jp`Djt;9cY?8(u*hBHLi6F7!exL4OXqF&d0UtzpTWhTa}6 z5*17#mwjEo#mI?~ zy7PfHfmZW~vg^N#Wn=bx%2+r#@S>OqLgR{Zf5ZWTy3z9|U9K?rUS2Tx_vq&^(9YbK zanOkXyW(|Q{rxKJaY<;g$;imy11n}Q zcIeoc@0*bK@(3^+JKkz(*(zG`8P+k6VYjJ$djHdGzDAqM`@kH1QtN8WSnm3(UKg{s zN@yIrHIDq|Tyn_CMN=+^wpEPN3D7)~>2IDZb8Bk>wf*nPm2XdpX3IuQw(IV>xv$*< zhrKCaEQq!ZxJWy+H%fc&A@_CJ>Y#xZ<=S9{2Erb``jgxD?mcfe2`nr7J8lo%GGfq! z88L}&UWa|QqTR6!QP76T3SWSL24BUSH!*(kyZ z!wbH?oaZrj!eDWLTVQ%twSnqy*@+*v+s00|R(cO;#-FEyMeql+h$|5NmX6HPEEduR zUnJUwHE9Y@G~VJTiRu;54)ML$8MX%a`uf7gW>(n$5%T=`JoFu*U#d~Pi!2SKM?uR= zrP4ghNp}1tnFBWsgEO6SdHs>6ikfEB8kKIq@=@(4sOe3)`x6hT{XRkPG(xWS#m!G_ z<)#ktztjHu9w2aa<|XKG6aKFY294$C#Fv9mk`BY#1`3H@mLKfLfleqk7M6B9FL`-P z*yW-+)jc!A1k-WDXKE7sVo1^ONu}hvelU&`{-4Ia10L)3|NEw4m5hwa$cRXYL`dRB zNl8gU*(xJKLWD9CQrROTLRJb5dqgR6A}g{=l#Eh}^1LtScTUgv|DWeMuX9eXxbN?M z-QVvuKI1)-lTUkskZ^dTO4_6SfC0zdWTKFg)S*>o07<-!nDEznOh-de>Gh=Y zzH4fY76O`Yg`Tuo2eTp1vxL(5=D-lhUv5taO zlv%)#wm{s7apDAMyRq_)e1!7{<_}*(1b+j7K03VW7M7N0e*HXq=m+X%gbzgE`rRcC zp}^%6VI0XbjogCbZO>bl0pEdW$V6LGrGH>(z&&Ve+)ZG{|G_rhMKfJ_ezNvtdk#lhT3RkIA8W6slLyAv?EP#eO0HWC4 z0*h;f(%8^UxOh*~4_oiZ(BAxIOMP$81yCpk{0}c~k~d^RRA+3sg9g-7QKD-AYD#Rh zho|Qt4(kT+)bhC3utsT$Tz~n^W^;en$2E-|F>kDdw!U1_tFEkP#xm+^ zA5eg}xOzL5qM{;g-iV0cT&b6@SWky*V|QsjCRNVqr1@5a;(ItKzu(g{p8%24WbnZ} zGFJ4-?t=`3IiyBUFE9x>l-cy~Q{X*g$?%D)*fx1PnwEwW(Ta0ghC;gNefc(<)yXSK z9hISCMg`z*N>$9#lDh!zdTQ%Q)LUOdi2#hUepFlUxw%vI0|9MtSI9#Iv&~HSaIK$k z2&2P}>%IWflK9wAsD-WRQ`uGjski+8nqsxdN4*E$<(4?zpi%?e7fKGU><3y?Tz~(y z%PrEX)MDKp%I1>fJ#`tw_V=?!7>|S%8O9yoSN2k}YURR{4D+eadt-ML29VDd^gUF4 zm|@}K`kyV~UqT*4M+dFf)pBK;62JQ#WwOc6ziKfv@18I)byv81dnfBrKE#44?DVW% zC_2BPP_~F|-`&(2Wtgce%YQ$A;xcgDZ^z9ekWktY@yPZXK*t40%zaBX3o|@$7UFt(^v8zihqp7NjAkDTZZeIS`2NIjn<}^u zRZFi1=S05V6|!De6la8t9Su{Ehf$II+R=y0I75-6Ga|cAvu)4xK7c>)oB^3xEqH+jjtIvJTb|J$camd&5m-752i%xtSY zvC4$kD;4lN(wVIgS$Uc*v7L_6|EmtnSCC$OYTB<&9IC;yFJX zu;$HA4)=yp0Gj>H6&VsI#|h>mh<_Mk^T_-AP7HAz6P;6P3pS)jP;_>=&hOEUMPFv* zIQUp2d&w-8YwpM>6xyxPO~Xm>cFwL^vk$UZ<6;A1UMKsraP&muTv)tmCe}SPoKxtV zp+1y)b~76<+Kx|^?t;xdo^ce^%IC=;g@LWN=u-Pxf6m;f2H z#Nd8>h{yf&EPsOuq)9*du$j&-Y+1aRaYghul}^$b0`cqRF=|Ee%t>YR4{heMT@TvR99dHm5ObM*lgKHV zdTpE%>OcFS|I%C!Ers)QaLxNyQ|*iQiQE9rq4W26%?5ezFOiVBe{6?x_YQi>T*G8K}+Z!2I&@g;6eEpFVS@|qjGpIrd7RBb2 zERjgN(IoAoqlNOdlcWlJ!Tgh5X(7Slm#`d;>YT-=4Bl?fho2q~ds&$w<+@ym@5kr$ zoD+}g8c>6J_xBZ!;R-k+TKH@u&i8a&s@^#H`N_T5=~sGI!yNkR_tCctAwwZEi2cqf zZF#LI+M8jv5v3Cj%Q8%AGG9$SIQRQ_VrWZN^e=YSuC?ULpHY#NA6eF%<*8I9i27+{ z{w~8At0Qf%Y+2Ba9$@-rG2XhV-E9SsdDY~`H+Rf5k1Uha1-Nn6Qd5^Cjb*e#Hu*of zqzaTm%4owmaK_eUvQ;f(;#?1+AY@zsNLO&5Gz%y(MvnufN~kN^VRxjR1peuwtu39q z(kfAoXqE0|;QG++p}nBO5=vLu7xADi(RL>%%G6e}4E;ni$mP8b(Xs<|Q3KdKAhi7I z>gr@lL!mf=cq#(QK1Lf43g(2k`IcU95++r1iMW?*a3&{~#fJ-+ey;xJV(lZs!V2Kw z9?H%i!|YJPVt_u=M#Rk_g!UgE9zKh_Zg1}sGifuRJBgl3YnGl6ljdHkTq|>-Fv`?s zwO5i!=kn_tTIEMySyZ~b_yTP&eAS`q$Drqvg=}}Yk(Fp-L_f8X!JkBFDC6|^*T#<3 zNP$7jxKE9xx`rC}(7gWoz>d^Q= z-qa8E@qv^Zmf!2GR{U^Y75=0zYswP2@3w7`6axbr8xcN_VOCs&8%%PJzSDnNcd9Ej z_gM9yK2REfa@`nU9Ct5xp4&R@Rx|TeS-H@O|M={Tsma4?Wq3Son2r<7!5*GNRlQ@hyaQ?CAY^IW+W6 z;4br@cPVwk4bulzgw(z#@gv4X6OU&b=#uZMx)_0w#v835(1-H(4FZZgq9Q0uaPZ!Su+H_r5 z;@6w{K&yYsqe&5$LO#Dlq^rQ05a@Y&Es%JY<4zQcW_nfN^^U2h(M-Yno0n678$%0~ z{mQ4+l#E0l9kzDsGM}E^>kvphzP#MOL{q8iFTg7TOYQ`1ZuwRd^u1IA1u}*;KisW* z#lFtDi@xS7ukpJTlC>XPR*p$2`m`L`e=3_*zOOGBc&{*t_bR8Uo;ua`wbya6t=$?CgBmI>n zzOmQ6FKmCgEUYUt5}^T$0FAm;5CaI?HdB1{sqiLEZL0OapKD;>|M*y`|8r&^s+lIh zyD&GrsX{bKcy?TDMbBTKSIj2e8gd0>BX-t#B9S}(r!fEpGW*e$_J%XVD*tW1!tHt8 z_wJl}*y*9dpd=-1;b_HJG(D4f_S(|>c?I>o@^*AR+g=v2GekyB{im?ZdN|_IOXk8f zS5KjElM25iu4{!uUV><`#^;?o@T^s(YGu-;H>_)i{lZuF0{UzuW+XUQHz&V{I|3hn zOk1hynRa92#dg&V+yi3ct-gBa^_?EHB4+z-*uk3zt)v;`a#@k&lKeS!=Lok^d0`Z zU-y_Jo%?OfJSPRb$8>)^vPP6-HSca&{;PPcgqOl<-1b~{;mEK*H(CYrX)OInN~xcl z?_q$*8$i%daFrEP#;nWrA{`nD?j9QNggGN6YMLVI=j-B`VuQ!F#pzsZdEw#5E}N$$ zRVcSTo8tp!I{HNvU%V?*a_8k^T-T&WgM{4Ax!WnXWu!)>3#UB_C$Z^T70I;(1W@lU zG5lc#(AnTp9@sQ|K+I~Pd^o_gDNUX`5=gXyD&P6Ef@PY+qmI99ef1~^ixc%3(W90( zy`0cAd_dp1ku#|;7-dmsS5I81v3Ev3UV=-9Y?H*MJsWjc zq5>FF4@#<6^aal+FeF#7xy)p~TYclv{*0?}8v->J@Ko<=*yoqu zDO0TGCMzbPLw`(88x5HLTm#iJU&_6|B9E&xo#zc{DLqyPL|)l>({$Au z*!_;w&o8W`wNCstJc6pg)1*rWnF)`@dfev zMcC}8lvd;sGit(5Bd?vQvcQ-g_AH=&^Z!pT2Cc+{$Ngwxt`z}lw9+9FqP z8KK)0OU{K$n@_Oia)`6G3n14Dwtc9&N(&D$<<+_~0@Y>b?gNBTsq=`*JKjU6pek6@ zNBPP-I!R+O#5S7{C$qLYBlmkfbJ>)#n^Dyjr*+rSAXI5+~?kSs1+8BAl z_SFvd#z?RTAW&TVWCceqx%u9$EVK$|i$9h4stzbzELi2hK$*4rvMez|%KNyvV9`GE zN_*9w1ec`^x2%NH6OH~(uWO8FRZmD;yw4e0%#o>rIT~b`T@>9Da$}E`R%4uEm7CEC zb9bb!L;ds&T6^yocu)Cl49O}X8IP4vcdRXq%hhD#3!75_@t#=WxRMfExq}TA)^;mY zxSBDdSb@-9Ky1R>wXZGacr*p_0b~;4hir+ismSsX}M^ z*1Hel_h0mUFA0pfw-y-NS*xaicyRn}85Unv_d@UQ>d$o!ZY~zend%8 zoW;SgdM#@4qPoz!Lv5Us&%#WsVq}$eN1`oCr~&>IvZr8;+oEJwB(r!$&5v3B1$Wk} zQSf0LeV-AZqt(o%Sk8LgNAaSZR(_k=4XV}O_wfK`6-8D5Am(6D+QPYM(~{sZFq$CYTwVH z6(mDZb80pdH2(Yb&^SOzXbDA9M1OvZQ)>yj{C*>I^2;b*Q|E-=&tG^{BNsx|pCi+~ zNDuYhtHSHI$nuYFf46Fd_a?v-${pFuc9h8pGt8PVSS#PVcIXLf7vthrL}OpoyR^nH zRZ7IBuT=#|V-_Tzs{(XtCKis@vhU>4%n!e$!QKuy1!jvO``em^s@A7n=Sl^VOw{bcLT-Or!jd9pjC@!J0bRtj35+1C%Pq?7Y) z;%x*<@C5M*QOaVcVf5d$t_g3Q5*u6Ld?Lax6tPm(GtoyiO-IA-b;wTkcg(TIc#iDZ zzs?%X*LZuwCgzw>g7tc*-b zyDj=J%twxNh}Y{<^$$%mt}yhO7hj>7;l?>?8_!Bg~O0_wn)r*#yva+(RxQ?LUotGQmp$@k+ zWm>)FhHzsOgI%Vp`XSR3sE1h&%g>#W_@){4eJ?fI^1pzVs#K`W5mg{Up^v8aN$%Xq z4`!H+6kuX0S+cwBZMOj_)EC+T@Zw=zH;tgc)4Q9*s?U|i> z`!^jQ2UqY}K8FyL};+AJ@~IAioN{WtDA?eODk6 z7=6?YO&&1V{^WRB%LumVrV0y4_eAnuy23%I*}%HjV=` zo;kx%9X@h?sQTsdtHNfzckg^qauij0>lrz|B*c~#wGpkbwE-Sq-Y-H(A?ry*ydYNn zv~7zpnk^j#rt(py=w!Jn@9V9pth2)}~NcY;c$ra}#qXWIMI z>3RmfpW5+K>?QBRWy`%D&CvezE?e?3!Zfd>p@r%?VsFEvYS*uTK#R*kDS_aENi`lN zdfTstYV`IB9=tB{$+hkGYqY9)*F74TNzHqyl429FPk$|xxUz`un|S6^lzFpm<*uE` z17k4oeqNA6OK;f7=Tk(mLd)`%G^k+DO+0vxoOTjn6|^(^I|qgC*%eK{x8bH5!1?V^ zD|`9!RjWnA{>H{yx`bx)@z$DC@*5Z)JkFNaJj|)L#bVI|6qIYNOK4lWz2TD z*Rl2w9W|x%D|5OC?|ibr1=t)5qxYk?YJ$UOw&b|@X6Fg&?fW_$^CkU`X?6z<3D}xT6u1II*w9W@yA%l=Gd#?Y0g3#f#MshhY7G^@A zIww=kj_m4KcLgwihE2@%=5s%&+^0+SxgLDFCD(h$6>tW|*PayEbM5eua8K1gnIrXV z7r;NDfdPk5OTbK7rV&@aG4PT*cKvu{RbJL$-c7TbIIXPI+o&szXAX2Hu!J=L!TMaJXu-)|UIEt3drCmc&_!2^ORA^6)JBe#JN)NUb zU#an*w2(n^PLou9(jYUgfay_?I{}!Y2Dsexh=~8U@P9K5`&|cZR7w)}_?j$VF>!$3 zHsDRSK<{1P%#dK;l^0)gUg$ara86e?c|6W)Evewl`)JpSG{Ir*$+*?sYyJ#|!zZ~c z928107Xsg3DEZ%*oZxjS|YY_UavU2ZC;Y^tIK?JroB=Cq=kl z=c@!I-lh5XVOs4CH`!nRV^+d0h$dscMCCGmOTtDB+7a~%~}HG>#JU0;MCR6{`n8$jWu#}>$)q!k0V2AFR?C9_pRP&wtB(4 zecjS4v;iOeeWz^JTw-BSi{)F_q+?36hgNv^gCDD|3HQz$Jt(U(qas!G@LV#@c%@#^ zUf*B1`o@Z!VYVC2Uxp8O?=AX}vQ<}j({z>+ma+Am<>ZUzy}Mh`6uCcnuKxM?c)_xO zh!-l7rKpkpXSGn4Dh=oM8z6fX@zb_%5z3Qk@&!x=SqR zlJ@DJl}1Nnr5~NVK>_`T)~ixjKY~i-Hz4#<-Mh` zbq$4re#ex_g3Z~}JR0Zg|J@?=<=2GK;vP7kT|PV-7GUc5TKYZbe5tai>8kXNI<)8I zzihK}j7^M+C^$zQXz-CtV0(5NB}kKnO!}2y3tt8a0eSWb{c-ElhNHk)RXQDV1v1=+oCT0JDnT9Z8-662!sSoM6~O^+KwF? zJRp7KmGhQ`iBRbCF|N?0L5K+un{>J(yF%93=TG7#!GGSa0${s&!$aW@VF5$J-)sku zjY&(#94eY>4!qatEJ8zhS=CG9*xSQMs!uXj;!D8m3z7r%Q9KCw#ve9Ga-$XZ-qDLZ zA4dJ_hZN00`OiYD2a&&W$yaHS1^1?a8c1A>v1Q zSsBG&^1N)F?l;?|Nc7N*pNIXzW@6()x72DfJMD1OtaTRLV zw)zTRK#-Q;d6;OFIu3Tytd-UId6`j3G!4|8qnTTBmBa78a+b$G5jJ zBzU8^7!#c&hltPS`C~6rQ_lyKzUWw2=-#ox>&H1^93YLtglxz6^6s&zClW1A`2Mm)2x_A7$!wQN! ztEjMRp?z-OqZ$g1SX7^ul?A5(-{t!F^dph+U^(n&1JYOi30zZf_S<}6eUl*KaU(MZo$gM*hQ<)_~`7eP841SI$x=5My1!mV);CM*VUB($9dTw8?fRXq*$QXncmsQ6YEayvw;T2=G+Tx8oQ9C2 zDA4xpe0v?9{JBdUXXQ3?%?VxycIV|r>kq7h%lJFu9SUNi3Z!)C(+^FxWUNjsyT<%$ z{IEY|-b4=Q3V#;I6I;KX(X@0tfuN?gL1X2foaF9j-GAzc!Qw)ICO?)E5rRy`wWY51 zAHqRo7ww`L@dM!)l)L#K6W|62P0s81-r7sS5Gqnr<>10(K?upFE>i`$F&N ziR0zJ#mC5LDRY@zuv?kE_d6J$61(w!(AxH9T zLt_f$`<=C!)7TdFG7DWVBEaykZ!R)UXbD{Je`jK7`5{fOO5@Pi;>m?(+hry%qTd!B z&-1^hN#h|WdiBqx^*KV;cyzMI;gAwS3j|wJ07HcEF3HE^XI+juxPOWVWET+;o9vg! z8*xFv10l5(*plvav~?OMcTT*u&S)?9z2#dAlB*mCf33^mNy4)E<;Z^lBanJ#S?dSbP2D}b_DN~O18P z$F_N1V$B;nWhb&xsR7lP;KpY_qLa3hN>yInjHacz%t>j^im#gHCAlBA+)Lk|ib_Qtj76q0_(V~Zfbvl01WGD7EP&8MW4n{O zG9`nMM~V71BRzeg8<}y`C>wT$6c{B!C6|qj?Rm7Quy9pNOF^9jV6x(e`6z!_rjq$% z)#R=pjYV%>Jc=@;k&S~%?P_VM*f7|Y2X_ntXi2Ir)cf8AmrKESY)>^5@rBTrFq zF%d^yBnZGjR7PeM2&FU}!cA}#q`;;NWvM*g@WY*u@sWd@5-sKY65NUQT@W3I@Ke$9 zLPiv~{uV*!g37u@Tw0}!;aE>OD5q2Z{Yq%;J~$Rhed1(@9}K)L{R zPn^jJIQ*+;`O!F36N%(61_(96YDnL1p8EC-Q$>=%BY+FO1iw;1m7l+65+IsQwZLUV z4V~~*091bQ4C;IqK%)>nD?A$_N1z5n z4lDGKS>Rqk7!rf<{J2PhW*iK)QuZfMmyn_#>&F7 z9;-$KQ(A&}-V#HtSDxj^rHF+hv}Ef64Xs0s3nr#9kdH6Peh%qSf5^&XugZjG^Oaf9 z`~yrRxu8v0^x^#Gr+95WF;|9A<0eWl)b``@z#I4;0HX#-wk6EMJ)jN`=`7;Js$yd!-1#90d+?ZZ|_=+R7~n)_w`Ed;Z$~10}tBSh&=|)wK>qEOy{hiY@fp$?_nZ`iA_y zd-t+yXB?czG|RLm&ViJ=-@oPpSOOR>!N?LDXQmAFoL7T2CkW3+dV#u7sfvw@me0hx z52V6i+Y9oCUC!57Ze1I2^X5&$WYB5C_5c+;PI8xE!3f$r?5JUVmMH_T-P>1%_rNUX zFs=o*%?8x$_hI=9K*e)IyqulQ&kMtikrvq7Y^LC(7Xnw)BBQsK);FRlAo`bQoVIKt zuV!OsUq*qt|9$~~YfRCPBQM8CsKGS}^AP4fHny$g!hT%09<-~bk3*ShUj&vGgvHlr zRmrLTmx>E{+(%~~x=#tBd>sU#dUGr;=$;G0ed&Nob!`77LNXvK}??^4!0<}N=%F#?BX)e-+w@TBH#A%pp@7*p@-tsmV{reH!7T9vQw^%GCen7y4!3Jv$ynM-|gYA{WPcH-u%3q$#5^W9Q zIZNT3<9&E$NHcGqM|?DqQUzQ;k#~n%%>ev)NY8<&<6`fmEi89&hJ>{1csGZKhli7! z+Y?B}zQmc!v-83m$%2T#n(54ZIs&0bbxV7lf>X-q0$^eqXvWYkS#sC5J;x{^HgZP! z>q~VQ6GB>o{1V$Gb{yufA3ItuN4A%N@G*ce2bPu*#p>B|+Df9tViVWFe1QBk1et>% zxC{H2&cDoGKfp5_z_P~)+JJdQ?AhRc_p00c=)O}U#H-ltoP!}FKTQW^u}Lv)QC@;r zqkm-NL`IfvIQDgzd|n0^ALzMVw_gdsAvX7U-b616LtGBs&(D>?k}OtyDsj0o_h$Lc zoBUW2FJ6C~@x>n4GYH>e z0|OJ&63V@gbI)4_MgVJs1ix-_dxe+_H zjgigPwFGu|Z`;ZeLHK;YVhFZ%TKFw?q^|QjF}HCB?wcqF98bYx`c~IIA87aE$b7 zOOu@YHTRthY#wqpvGw)!ZE)-I83^F2L%XK_;xZ=N7t+#*oCe{OK>nlkorO|(9Xan{ zAA;?;wu<=e&V1|$1!OcBJ7H;^xtXHthxL#oUXKQr5bQ`Swtd=TWE4-%DTt&5lLG+( zVgQ(665Bq7sHL)?E5lCqc$fFj?>;LOwN8qFB@OYW48`C=v2)#p6LTxv(TM?7WK7J7 z>m!!fhR}k$jF2UErE%o->z3k{+1P}px9j(954mz>H3ZtD$#)G759fsE4;OJQf{~ZZ zcx`m^Q-u4%(4iKCkV!pUu~3AprVjjgnXg3wj$kGM z#-NKHXqx$Bb0;QRu%IGJ%{Ft`hgp#ZaW*$M*R;1ER)$}tpsehx+~<`Gzc<0wjm#A? zI(Nhtq4hsNB!XZ-A z(_>6_s2lSP<~|)-M@MS=@>jxJh^5APCX8e=Ru1+{NLphBu}7q}i!n0!xmzG0C&+@5 z2r&Nt@oP7S8h-{Dw?8PZT`yP!rvJYe=nD8hB>#!DzZvLsV4m&(YFKy=CNv#oy`&^F z6v45M)}iI>1j86X`1IVbOpC82GT&Gd*cR%ckq@(++%L;Rg2i{4k5dp|bYs>NcXluI zQ;8@1)wsA2kf<%8ER9oI1BVc?PbcT2D%#nItB`daA^zk*854@y@)g{^STw2`2WcqS zvz8+k#%v}o_5|xq+*#qgZS-aQcjd7;hJej0h7^7)wYBLq2n)!Kb{g)!Zm})?nqr1c zMy7;Sy1lh1Q@}>042h7HOzUzb3>CuSZJyak+z<>XN(nx6#mDltYr*I8Q-jTOzdr3B z86EYOziry$Lg|m7_oZm})z=3k-@F;6Bl1omTv%*vnD(`c)hW|I%ErdV`aUA2o%#7Q zCm>+i(dZ3W$3_ROtuvHc4jnlXBP_;3fAY*3z0?D{U%Yq`pP1M-Ia!#QnW=B^}8Ugk_(GER1IXQpT)?Nya zh}ijcWq{g5%lP>Cw6`g&UNp*AQM@?VRVf<^%F0Z7dU~{Pzv}Mp{_D+~L`^NNs24Uq z#uIZD_{sgEqUol`@$vB%{9KgF;S~BR`er39Rt6N-Bl8MzTCnuMknz{>K(u&H;ZSCz*$;ruW9UZsF#%#}?J7;(9+!0Yx(P?E} zb#-;cojZLd=4NNJ{QPKmd3a8`xEvZ98d}T4GrX{qvXhT|H7OqEO;OsX+}sRGOG}-3 zv}9#vwT0O(!+BI$_pYH%#f0676Iw}0Nr;Enu34ie6W7+;nnDb#{v-%k&1~l%4AF~3>{}bt5d2LGAk=h5h`_ecFulIr&R5<)TN