From 76bef74feca5109b7dd85d729ab19e6fb86e4e6a Mon Sep 17 00:00:00 2001 From: ModelHub XC Date: Sat, 18 Jul 2026 09:15:05 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=EF=BC=8C=E7=94=B1ModelHub=20XC=E7=A4=BE=E5=8C=BA=E6=8F=90?= =?UTF-8?q?=E4=BE=9B=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model: OpenBMB/BitCPM-CANN-1B-unquantized Source: Original Platform --- .gitattributes | 52 ++ README.md | 126 +++ config.json | 169 ++++ configuration.json | 1 + configuration_llama.py | 206 +++++ example/README.md | 103 +++ example/ds_config.json | 29 + example/ds_config_z2.json | 22 + example/gpu_pretrain.csv | 51 ++ example/gpu_pretrain_loss.png | Bin 0 -> 49881 bytes example/gpu_sft.csv | 51 ++ example/gpu_sft_loss.png | Bin 0 -> 69683 bytes example/npu_pretrain.csv | 51 ++ example/npu_pretrain_loss.png | Bin 0 -> 47645 bytes example/npu_sft.csv | 51 ++ example/npu_sft_loss.png | Bin 0 -> 68887 bytes example/requirements.txt | 8 + example/run.sh | 38 + example/run_sft.sh | 40 + example/train.py | 203 +++++ example/train_sft.py | 424 +++++++++ generation_config.json | 8 + modeling_llama.py | 1598 +++++++++++++++++++++++++++++++++ pytorch_model.bin | 3 + qat-convert.py | 176 ++++ special_tokens_map.json | 81 ++ tokenizer.json | 3 + tokenizer.model | 3 + tokenizer_config.json | 116 +++ 29 files changed, 3613 insertions(+) create mode 100644 .gitattributes create mode 100644 README.md create mode 100644 config.json create mode 100644 configuration.json create mode 100644 configuration_llama.py create mode 100644 example/README.md create mode 100644 example/ds_config.json create mode 100644 example/ds_config_z2.json create mode 100644 example/gpu_pretrain.csv create mode 100644 example/gpu_pretrain_loss.png create mode 100644 example/gpu_sft.csv create mode 100644 example/gpu_sft_loss.png create mode 100644 example/npu_pretrain.csv create mode 100644 example/npu_pretrain_loss.png create mode 100644 example/npu_sft.csv create mode 100644 example/npu_sft_loss.png create mode 100644 example/requirements.txt create mode 100644 example/run.sh create mode 100644 example/run_sft.sh create mode 100644 example/train.py create mode 100644 example/train_sft.py create mode 100644 generation_config.json create mode 100644 modeling_llama.py create mode 100644 pytorch_model.bin create mode 100644 qat-convert.py create mode 100644 special_tokens_map.json create mode 100644 tokenizer.json create mode 100644 tokenizer.model create mode 100644 tokenizer_config.json diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..2c4d451 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,52 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text + + +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text + +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zstandard filter=lfs diff=lfs merge=lfs -text +*.tfevents* filter=lfs diff=lfs merge=lfs -text +*.db* filter=lfs diff=lfs merge=lfs -text +*.ark* filter=lfs diff=lfs merge=lfs -text +**/*ckpt*data* filter=lfs diff=lfs merge=lfs -text +**/*ckpt*.meta filter=lfs diff=lfs merge=lfs -text +**/*ckpt*.index filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.gguf* filter=lfs diff=lfs merge=lfs -text +*.ggml filter=lfs diff=lfs merge=lfs -text +*.llamafile* filter=lfs diff=lfs merge=lfs -text +*.pt2 filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text + +pytorch_model.bin filter=lfs diff=lfs merge=lfs -text + +tokenizer.json filter=lfs diff=lfs merge=lfs -text +tokenizer.model filter=lfs diff=lfs merge=lfs -text \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..911f4ae --- /dev/null +++ b/README.md @@ -0,0 +1,126 @@ +--- +license: apache-2.0 +language: +- zh +- en +pipeline_tag: text-generation +library_name: transformers +--- +
+ +
+ +

+GitHub Repo | +Technical Report +

+

+๐Ÿ‘‹ Join us on Discord and WeChat +

+ +## Overview + +BitCPM-CANN-1B-unquantized is the **unquantized QAT (Quantization-Aware Training) checkpoint** of BitCPM-CANN-1B, designed for **continued pre-training and fine-tuning**. It preserves full-precision latent weights with ternary fake quantizers (weights โ†’ {-1, 0, 1} with group-wise scaling, trained via STE) defined in `modeling.py`, enabling the model to keep learning under quantization constraints. For technical details, see our [Technical Report](https://github.com/OpenBMB/MiniCPM/blob/main/docs/BitCPM_CANN.pdf). + +> โš ๏ธ **This model is NOT for direct inference.** For inference, use the pseudo-quantized version: [openbmb/BitCPM-CANN-1B](https://huggingface.co/openbmb/BitCPM-CANN-1B). + +## Continued Pre-training & Fine-tuning + +The **only requirement** is that the forward pass must go through the bundled `modeling.py` (which contains the ternary fake quantizer). Load with `trust_remote_code=True` and do NOT replace or bypass the model's forward logic. + +### Option 1: DeepSpeed (Recommended) + +We provide ready-to-use training scripts in the [example](https://huggingface.co/openbmb/BitCPM-CANN-1B-unquantized/tree/main/example) directory (using the 1B model as an example): + +- **Continued pre-training**: `example/run.sh` + `example/train.py` +- **SFT (Supervised Fine-tuning)**: `example/run_sft.sh` + `example/train_sft.py` + +Quick start: + +```bash +# Continued pre-training +cd example && bash run.sh + +# Supervised fine-tuning +cd example && bash run_sft.sh +``` + +### Option 2: HuggingFace-compatible Frameworks + +Any framework that supports HuggingFace model loading with custom code can be used, such as **LLaMA Factory**, **HuggingFace Trainer**, etc. The key is to ensure `trust_remote_code=True`: + +```python +from transformers import AutoModelForCausalLM, AutoTokenizer + +path = 'openbmb/BitCPM-CANN-1B-unquantized' +tokenizer = AutoTokenizer.from_pretrained(path, trust_remote_code=True) +model = AutoModelForCausalLM.from_pretrained( + path, + torch_dtype=torch.bfloat16, + trust_remote_code=True +) + +# Use with your preferred framework (LLaMA Factory, HF Trainer, etc.) +# The ternary fake quantizer in modeling.py is applied automatically during forward pass. +``` + +## Post-Training Conversion + +After training, use `qat-convert.py` to fuse the fake quantizer and produce inference-ready pseudo-quantized weights: + +```bash +python qat-convert.py \ + --input_bin \ + --output \ + --quant_type ternary \ + --group_size -1 +``` + +The converted model can be loaded for inference in the same way as [openbmb/BitCPM-CANN-1B](https://huggingface.co/openbmb/BitCPM-CANN-1B)โ€”no special quantization libraries required. + +## Workflow + +``` +โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” +โ”‚ BitCPM-CANN-1B-unquantized โ”‚ โ† This model (QAT checkpoint + fake quantizer in modeling.py) +โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ฌโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ + โ”‚ + โ–ผ Train (DeepSpeed / LLaMA Factory / HF Trainer / ...) +โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” +โ”‚ Fine-tuned checkpoint โ”‚ โ† Still contains un-fused QAT parameters +โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ฌโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ + โ”‚ + โ–ผ python qat-convert.py --quant_type ternary --group_size -1 +โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” +โ”‚ Pseudo-quantized model โ”‚ โ† Ready for inference (same format as BitCPM-CANN-1B) +โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ +``` + +## BitCPM-CANN Model Family + +| Model | HuggingFace (Inference) | HuggingFace (Fine-tuning) | +|-------|-------------------------|---------------------------| +| BitCPM-CANN-0.5B | [openbmb/BitCPM-CANN-0.5B](https://huggingface.co/openbmb/BitCPM-CANN-0.5B) | [openbmb/BitCPM-CANN-0.5B-unquantized](https://huggingface.co/openbmb/BitCPM-CANN-0.5B-unquantized) | +| BitCPM-CANN-1B | [openbmb/BitCPM-CANN-1B](https://huggingface.co/openbmb/BitCPM-CANN-1B) | [openbmb/BitCPM-CANN-1B-unquantized](https://huggingface.co/openbmb/BitCPM-CANN-1B-unquantized) | +| BitCPM-CANN-3B | [openbmb/BitCPM-CANN-3B](https://huggingface.co/openbmb/BitCPM-CANN-3B) | [openbmb/BitCPM-CANN-3B-unquantized](https://huggingface.co/openbmb/BitCPM-CANN-3B-unquantized) | +| BitCPM-CANN-8B | [openbmb/BitCPM-CANN-8B](https://huggingface.co/openbmb/BitCPM-CANN-8B) | [openbmb/BitCPM-CANN-8B-unquantized](https://huggingface.co/openbmb/BitCPM-CANN-8B-unquantized) | + +## Statement +- As a language model, BitCPM-CANN generates content by learning from a vast amount of text. +- However, it does not possess the ability to comprehend or express personal opinions or value judgments. +- Any content generated by BitCPM-CANN does not represent the viewpoints or positions of the model developers. +- Therefore, when using content generated by BitCPM-CANN, users should take full responsibility for evaluating and verifying it on their own. + +## LICENSE +- This repository and BitCPM-CANN models are released under the [Apache-2.0](https://github.com/OpenBMB/MiniCPM/blob/main/LICENSE) License. + +## Citation +- Please cite our technical report if you find our work valuable. + +```bibtex +@article{bitcpmcann, + title={{BitCPM-CANN}: Native 1.58-Bit Large Language Model Training on Ascend NPU}, + author={BitCPM Team}, + year={2026} +} +``` diff --git a/config.json b/config.json new file mode 100644 index 0000000..7bda3ef --- /dev/null +++ b/config.json @@ -0,0 +1,169 @@ +{ + "_name_or_path": "openbmb/CPM-2B", + "architectures": [ + "LlamaForCausalLM" + ], + "auto_map": { + "AutoConfig": "configuration_llama.LlamaConfig", + "AutoModel": "modeling_llama.LlamaForCausalLM", + "AutoModelForCausalLM": "modeling_llama.LlamaForCausalLM" + }, + "bos_token_id": 1, + "eos_token_id": [ + 2, + 73440 + ], + "pad_token_id": 2, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.1, + "intermediate_size": 6144, + "head_dim": 128, + "max_position_embeddings": 32768, + "model_type": "llama", + "num_attention_heads": 16, + "num_hidden_layers": 28, + "num_key_value_heads": 2, + "rms_norm_eps": 1e-06, + "rope_scaling": { + "factor": 1.0, + "rope_type": "longrope", + "long_factor": [ + 0.9977997200264581, + 1.014658295992452, + 1.0349680404997148, + 1.059429246056193, + 1.0888815016813513, + 1.1243301355211495, + 1.166977103606075, + 1.2182568066927284, + 1.2798772354275727, + 1.3538666751582975, + 1.4426259039919596, + 1.5489853358570191, + 1.6762658237220625, + 1.8283407612492941, + 2.0096956085876183, + 2.225478927469756, + 2.481536379650452, + 2.784415934557119, + 3.1413289096347365, + 3.560047844772632, + 4.048719380066383, + 4.615569542115128, + 5.2684819496549835, + 6.014438591970396, + 6.858830049237097, + 7.804668263503327, + 8.851768731513417, + 9.99600492938444, + 11.228766118181639, + 12.536757560834843, + 13.902257701387796, + 15.303885189125953, + 16.717837610115794, + 18.119465097853947, + 19.484965238406907, + 20.792956681060105, + 22.02571786985731, + 23.16995406772833, + 24.217054535738416, + 25.16289275000465, + 26.007284207271347, + 26.753240849586767, + 27.40615325712662, + 27.973003419175363, + 28.461674954469114, + 28.880393889607006, + 29.237306864684626, + 29.540186419591297, + 29.79624387177199, + 30.01202719065413, + 30.193382037992453, + 30.34545697551969, + 30.47273746338473, + 30.579096895249787, + 30.66785612408345, + 30.741845563814174, + 30.80346599254902, + 30.85474569563567, + 30.897392663720595, + 30.932841297560394, + 30.962293553185553, + 30.986754758742034, + 31.007064503249293, + 31.02392307921529 + ], + "short_factor": [ + 0.9977997200264581, + 1.014658295992452, + 1.0349680404997148, + 1.059429246056193, + 1.0888815016813513, + 1.1243301355211495, + 1.166977103606075, + 1.2182568066927284, + 1.2798772354275727, + 1.3538666751582975, + 1.4426259039919596, + 1.5489853358570191, + 1.6762658237220625, + 1.8283407612492941, + 2.0096956085876183, + 2.225478927469756, + 2.481536379650452, + 2.784415934557119, + 3.1413289096347365, + 3.560047844772632, + 4.048719380066383, + 4.615569542115128, + 5.2684819496549835, + 6.014438591970396, + 6.858830049237097, + 7.804668263503327, + 8.851768731513417, + 9.99600492938444, + 11.228766118181639, + 12.536757560834843, + 13.902257701387796, + 15.303885189125953, + 16.717837610115794, + 18.119465097853947, + 19.484965238406907, + 20.792956681060105, + 22.02571786985731, + 23.16995406772833, + 24.217054535738416, + 25.16289275000465, + 26.007284207271347, + 26.753240849586767, + 27.40615325712662, + 27.973003419175363, + 28.461674954469114, + 28.880393889607006, + 29.237306864684626, + 29.540186419591297, + 29.79624387177199, + 30.01202719065413, + 30.193382037992453, + 30.34545697551969, + 30.47273746338473, + 30.579096895249787, + 30.66785612408345, + 30.741845563814174, + 30.80346599254902, + 30.85474569563567, + 30.897392663720595, + 30.932841297560394, + 30.962293553185553, + 30.986754758742034, + 31.007064503249293, + 31.02392307921529 + ], + "original_max_position_embeddings": 32768 + }, + "torch_dtype": "bfloat16", + "transformers_version": "4.46.3", + "use_cache": true, + "vocab_size": 73448 +} diff --git a/configuration.json b/configuration.json new file mode 100644 index 0000000..bbeeda1 --- /dev/null +++ b/configuration.json @@ -0,0 +1 @@ +{"framework": "pytorch", "task": "text-generation", "allow_remote": true} \ No newline at end of file diff --git a/configuration_llama.py b/configuration_llama.py new file mode 100644 index 0000000..3f1b8df --- /dev/null +++ b/configuration_llama.py @@ -0,0 +1,206 @@ +# coding=utf-8 +# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved. +# +# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX +# and OPT implementations in this library. It has been modified from its +# original forms to accommodate minor architectural differences compared +# to GPT-NeoX and OPT used by the Meta AI team that trained the model. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""LLaMA model configuration""" + +from transformers.configuration_utils import PretrainedConfig +from transformers.modeling_rope_utils import rope_config_validation + + +class LlamaConfig(PretrainedConfig): + r""" + This is the configuration class to store the configuration of a [`LlamaModel`]. It is used to instantiate an LLaMA + model according to the specified arguments, defining the model architecture. Instantiating a configuration with the + defaults will yield a similar configuration to that of the LLaMA-7B. + + Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the + documentation from [`PretrainedConfig`] for more information. + + + Args: + vocab_size (`int`, *optional*, defaults to 32000): + Vocabulary size of the LLaMA model. Defines the number of different tokens that can be represented by the + `inputs_ids` passed when calling [`LlamaModel`] + hidden_size (`int`, *optional*, defaults to 4096): + Dimension of the hidden representations. + intermediate_size (`int`, *optional*, defaults to 11008): + Dimension of the MLP representations. + num_hidden_layers (`int`, *optional*, defaults to 32): + Number of hidden layers in the Transformer decoder. + num_attention_heads (`int`, *optional*, defaults to 32): + Number of attention heads for each attention layer in the Transformer decoder. + num_key_value_heads (`int`, *optional*): + This is the number of key_value heads that should be used to implement Grouped Query Attention. If + `num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if + `num_key_value_heads=1` the model will use Multi Query Attention (MQA) otherwise GQA is used. When + converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed + by meanpooling all the original heads within that group. For more details checkout [this + paper](https://arxiv.org/pdf/2305.13245.pdf). If it is not specified, will default to + `num_attention_heads`. + hidden_act (`str` or `function`, *optional*, defaults to `"silu"`): + The non-linear activation function (function or string) in the decoder. + max_position_embeddings (`int`, *optional*, defaults to 2048): + The maximum sequence length that this model might ever be used with. Llama 1 supports up to 2048 tokens, + Llama 2 up to 4096, CodeLlama up to 16384. + initializer_range (`float`, *optional*, defaults to 0.02): + The standard deviation of the truncated_normal_initializer for initializing all weight matrices. + rms_norm_eps (`float`, *optional*, defaults to 1e-06): + The epsilon used by the rms normalization layers. + use_cache (`bool`, *optional*, defaults to `True`): + Whether or not the model should return the last key/values attentions (not used by all models). Only + relevant if `config.is_decoder=True`. + pad_token_id (`int`, *optional*): + Padding token id. + bos_token_id (`int`, *optional*, defaults to 1): + Beginning of stream token id. + eos_token_id (`int`, *optional*, defaults to 2): + End of stream token id. + pretraining_tp (`int`, *optional*, defaults to 1): + Experimental feature. Tensor parallelism rank used during pretraining. Please refer to [this + document](https://huggingface.co/docs/transformers/main/perf_train_gpu_many#tensor-parallelism) to + understand more about it. This value is necessary to ensure exact reproducibility of the pretraining + results. Please refer to [this issue](https://github.com/pytorch/pytorch/issues/76232). + tie_word_embeddings (`bool`, *optional*, defaults to `False`): + Whether to tie weight embeddings + rope_theta (`float`, *optional*, defaults to 10000.0): + The base period of the RoPE embeddings. + rope_scaling (`Dict`, *optional*): + Dictionary containing the scaling configuration for the RoPE embeddings. NOTE: if you apply new rope type + and you expect the model to work on longer `max_position_embeddings`, we recommend you to update this value + accordingly. + Expected contents: + `rope_type` (`str`): + The sub-variant of RoPE to use. Can be one of ['default', 'linear', 'dynamic', 'yarn', 'longrope', + 'llama3'], with 'default' being the original RoPE implementation. + `factor` (`float`, *optional*): + Used with all rope types except 'default'. The scaling factor to apply to the RoPE embeddings. In + most scaling types, a `factor` of x will enable the model to handle sequences of length x * + original maximum pre-trained length. + `original_max_position_embeddings` (`int`, *optional*): + Used with 'dynamic', 'longrope' and 'llama3'. The original max position embeddings used during + pretraining. + `attention_factor` (`float`, *optional*): + Used with 'yarn' and 'longrope'. The scaling factor to be applied on the attention + computation. If unspecified, it defaults to value recommended by the implementation, using the + `factor` field to infer the suggested value. + `beta_fast` (`float`, *optional*): + Only used with 'yarn'. Parameter to set the boundary for extrapolation (only) in the linear + ramp function. If unspecified, it defaults to 32. + `beta_slow` (`float`, *optional*): + Only used with 'yarn'. Parameter to set the boundary for interpolation (only) in the linear + ramp function. If unspecified, it defaults to 1. + `short_factor` (`List[float]`, *optional*): + Only used with 'longrope'. The scaling factor to be applied to short contexts (< + `original_max_position_embeddings`). Must be a list of numbers with the same length as the hidden + size divided by the number of attention heads divided by 2 + `long_factor` (`List[float]`, *optional*): + Only used with 'longrope'. The scaling factor to be applied to long contexts (< + `original_max_position_embeddings`). Must be a list of numbers with the same length as the hidden + size divided by the number of attention heads divided by 2 + `low_freq_factor` (`float`, *optional*): + Only used with 'llama3'. Scaling factor applied to low frequency components of the RoPE + `high_freq_factor` (`float`, *optional*): + Only used with 'llama3'. Scaling factor applied to high frequency components of the RoPE + attention_bias (`bool`, *optional*, defaults to `False`): + Whether to use a bias in the query, key, value and output projection layers during self-attention. + attention_dropout (`float`, *optional*, defaults to 0.0): + The dropout ratio for the attention probabilities. + mlp_bias (`bool`, *optional*, defaults to `False`): + Whether to use a bias in up_proj, down_proj and gate_proj layers in the MLP layers. + head_dim (`int`, *optional*): + The attention head dimension. If None, it will default to hidden_size // num_heads + + ```python + >>> from transformers import LlamaModel, LlamaConfig + + >>> # Initializing a LLaMA llama-7b style configuration + >>> configuration = LlamaConfig() + + >>> # Initializing a model from the llama-7b style configuration + >>> model = LlamaModel(configuration) + + >>> # Accessing the model configuration + >>> configuration = model.config + ```""" + + model_type = "llama" + keys_to_ignore_at_inference = ["past_key_values"] + + def __init__( + self, + vocab_size=32000, + hidden_size=4096, + intermediate_size=11008, + num_hidden_layers=32, + num_attention_heads=32, + num_key_value_heads=None, + hidden_act="silu", + max_position_embeddings=2048, + initializer_range=0.02, + rms_norm_eps=1e-6, + use_cache=True, + pad_token_id=None, + bos_token_id=1, + eos_token_id=2, + pretraining_tp=1, + tie_word_embeddings=False, + rope_theta=10000.0, + rope_scaling=None, + attention_bias=False, + attention_dropout=0.0, + mlp_bias=False, + head_dim=None, + **kwargs, + ): + self.vocab_size = vocab_size + self.max_position_embeddings = max_position_embeddings + self.hidden_size = hidden_size + self.intermediate_size = intermediate_size + self.num_hidden_layers = num_hidden_layers + self.num_attention_heads = num_attention_heads + + # for backward compatibility + if num_key_value_heads is None: + num_key_value_heads = num_attention_heads + + self.num_key_value_heads = num_key_value_heads + self.hidden_act = hidden_act + self.initializer_range = initializer_range + self.rms_norm_eps = rms_norm_eps + self.pretraining_tp = pretraining_tp + self.use_cache = use_cache + self.rope_theta = rope_theta + self.rope_scaling = rope_scaling + self.attention_bias = attention_bias + self.attention_dropout = attention_dropout + self.mlp_bias = mlp_bias + self.head_dim = head_dim if head_dim is not None else self.hidden_size // self.num_attention_heads + # Validate the correctness of rotary position embeddings parameters + # BC: if there is a 'type' field, copy it it to 'rope_type'. + if self.rope_scaling is not None and "type" in self.rope_scaling: + self.rope_scaling["rope_type"] = self.rope_scaling["type"] + rope_config_validation(self) + + super().__init__( + pad_token_id=pad_token_id, + bos_token_id=bos_token_id, + eos_token_id=eos_token_id, + tie_word_embeddings=tie_word_embeddings, + **kwargs, + ) diff --git a/example/README.md b/example/README.md new file mode 100644 index 0000000..c22ab85 --- /dev/null +++ b/example/README.md @@ -0,0 +1,103 @@ +# BitCPM Training Example + +This project provides scripts for continue pretraining (CPT) and supervised fine-tuning (SFT) of **BitCPM-CANN-1B-unquantized**. + +## File Description + +CPT and SFT each have a pair of scripts (training script + launch script) and share DeepSpeed configuration files: + +| File | Description | +| --- | --- | +| `train.py` | Continue pretrain script based on HuggingFace Trainer + DeepSpeed | +| `run.sh` | Launch script for CPT with hyperparameter configuration | +| `train_sft.py` | Supervised fine-tuning script based on HuggingFace Trainer + DeepSpeed | +| `run_sft.sh` | Launch script for SFT with hyperparameter configuration | +| `ds_config.json` | DeepSpeed ZeRO-3 configuration (with CPU offload) | +| `ds_config_z2.json` | DeepSpeed ZeRO-2 configuration (used by default) | +| `requirements.txt` | Python dependency list | + +## Environment Setup + +### Docker Image + +Use the following Huawei NPU image: + +``` +swr.cn-south-1.myhuaweicloud.com/ascendhub/mindspeed-llm:openeuler22.03-mindspeed-llm-2.3.0-a3-arm +``` + +Other Huawei NPU images may also work but have not been fully tested. For GPU environments, you can skip the Docker image and just install `requirements.txt` directly. + +### Install Dependencies + +After entering the container, install the Python dependencies: + +```bash +pip install -r requirements.txt +``` + +## Continue Pretrain (CPT) + +### Dataset + +The test dataset used is [C4-Pro](https://huggingface.co/datasets/gair-prox/c4-pro), stored in parquet format after downloading. + +### Usage + +Modify the path configuration in `run.sh`: + +```bash +MODEL_PATH="/path/to/BitCPM-CANN-1B-unquantized/" +DATA_PATH="/path/to/c4-pro/data/your_file.parquet" +``` + +Then start training: + +```bash +bash run.sh +``` + +## Supervised Fine-Tuning (SFT) + +### Dataset + +The test dataset used is [UltraChat 200k](https://huggingface.co/datasets/HuggingFaceH4/ultrachat_200k), stored in parquet format after downloading. + +### Usage + +Modify the path configuration in `run_sft.sh`: + +```bash +MODEL_PATH="/path/to/BitCPM-CANN-1B-unquantized/" +DATA_PATH="/path/to/ultrachat_200k/data/your_file.parquet" +``` + +Then start training: + +```bash +bash run_sft.sh +``` + +## Training Results Reference + +> **Note:** BitCPM has its own training dataset and data mixture. It is expected that the loss continues to decrease when training on open-source datasets. + +Below are the loss curves from smoke tests on GPU and NPU for both CPT and SFT tasks. The results are highly consistent across GPU and NPU, indicating that users can continue pre-training or fine-tuning on various compute devices: + +| | GPU | NPU | +| --- | --- | --- | +| **CPT** | ![GPU Pretrain Loss](gpu_pretrain_loss.png) | ![NPU Pretrain Loss](npu_pretrain_loss.png) | +| **SFT** | ![GPU SFT Loss](gpu_sft_loss.png) | ![NPU SFT Loss](npu_sft_loss.png) | + +Training log CSV files (corresponding to the loss curves above): + +| CSV File | Corresponding Loss Curve | +| --- | --- | +| [gpu_pretrain.csv](gpu_pretrain.csv) | GPU CPT | +| [npu_pretrain.csv](npu_pretrain.csv) | NPU CPT | +| [gpu_sft.csv](gpu_sft.csv) | GPU SFT | +| [npu_sft.csv](npu_sft.csv) | NPU SFT | + +--- + +These scripts provide a convenient, ready-to-use toolkit for QAT-aware continued pre-training and fine-tuning of BitCPM-CANN models, so you can quickly adapt the model to your own data and tasks while preserving ternary quantization constraints. diff --git a/example/ds_config.json b/example/ds_config.json new file mode 100644 index 0000000..d827f01 --- /dev/null +++ b/example/ds_config.json @@ -0,0 +1,29 @@ +{ + "bf16": { + "enabled": true + }, + "zero_optimization": { + "stage": 3, + "offload_optimizer": { + "device": "cpu", + "pin_memory": true + }, + "offload_param": { + "device": "none" + }, + "overlap_comm": true, + "contiguous_gradients": true, + "sub_group_size": 1e9, + "reduce_bucket_size": 2e8, + "stage3_prefetch_bucket_size": 2e8, + "stage3_param_persistence_threshold": 1e5, + "stage3_max_live_parameters": 2e9, + "stage3_max_reuse_distance": 2e9, + "stage3_gather_16bit_weights_on_model_save": true + }, + "gradient_accumulation_steps": "auto", + "gradient_clipping": "auto", + "train_batch_size": "auto", + "train_micro_batch_size_per_gpu": "auto", + "wall_clock_breakdown": false +} diff --git a/example/ds_config_z2.json b/example/ds_config_z2.json new file mode 100644 index 0000000..3f005fc --- /dev/null +++ b/example/ds_config_z2.json @@ -0,0 +1,22 @@ +{ + "bf16": { + "enabled": true + }, + "zero_optimization": { + "stage": 2, + "offload_optimizer": { + "device": "none" + }, + "allgather_partitions": true, + "allgather_bucket_size": 2e8, + "overlap_comm": true, + "reduce_scatter": true, + "reduce_bucket_size": 2e8, + "contiguous_gradients": true + }, + "gradient_accumulation_steps": "auto", + "gradient_clipping": "auto", + "train_batch_size": "auto", + "train_micro_batch_size_per_gpu": "auto", + "wall_clock_breakdown": false +} diff --git a/example/gpu_pretrain.csv b/example/gpu_pretrain.csv new file mode 100644 index 0000000..7ef7e56 --- /dev/null +++ b/example/gpu_pretrain.csv @@ -0,0 +1,51 @@ +step,train/loss,train/grad_norm,train/learning_rate,train/epoch,train/train_runtime,train/train_samples_per_second,train/train_steps_per_second,train/total_flos,train/train_loss +2,2.7920000553131104,0.03527498617768288,7.999999979801942e-06,0.010457516647875309,,,,, +4,2.8011999130249023,0.03495891019701958,1.5999999959603883e-05,0.020915033295750618,,,,, +6,2.7964000701904297,0.03271934762597084,2.4000000848900527e-05,0.0313725508749485,,,,, +8,2.763700008392334,0.024968057870864868,3.199999991920777e-05,0.041830066591501236,,,,, +10,3.281599998474121,0.31758183240890503,3.9999998989515007e-05,0.05228758230805397,,,,, +12,2.941200017929077,0.044055406004190445,3.995128281530924e-05,0.062745101749897,,,,, +14,2.851799964904785,0.03649706766009331,3.9805359847377986e-05,0.07320261746644974,,,,, +16,2.7869999408721924,0.022624235600233078,3.9562950405525044e-05,0.08366013318300247,,,,, +18,2.7825000286102295,0.021830420941114426,3.922523319488391e-05,0.0941176488995552,,,,, +20,2.7857000827789307,0.01685911975800991,3.87938525818754e-05,0.10457516461610794,,,,, +22,2.7571001052856445,0.01572061888873577,3.827090768027119e-05,0.11503268033266068,,,,, +24,2.762399911880493,0.016891509294509888,3.7658952351193875e-05,0.125490203499794,,,,, +26,2.7411000728607178,0.015683824196457863,3.6960962461307645e-05,0.13594771921634674,,,,, +28,2.733099937438965,0.012847283855080605,3.6180339520797133e-05,0.14640523493289948,,,,, +30,2.723400115966797,0.015209181234240532,3.532088885549456e-05,0.1568627506494522,,,,, +32,2.7342000007629395,0.01241038367152214,3.4386797779006884e-05,0.16732026636600494,,,,, +34,2.7321999073028564,0.012879018671810627,3.338261376484297e-05,0.17777778208255768,,,,, +36,2.7314000129699707,0.013242729939520359,3.231322989449836e-05,0.1882352977991104,,,,, +38,2.7065999507904053,0.01113435160368681,3.118385939160362e-05,0.19869281351566315,,,,, +40,2.6958999633789062,0.012413726188242435,2.9999999242136255e-05,0.20915032923221588,,,,, +42,2.7516000270843506,0.011661508120596409,2.8767422918463126e-05,0.21960784494876862,,,,, +44,2.713099956512451,0.012248368933796883,2.749213126662653e-05,0.23006536066532135,,,,, +46,2.7102999687194824,0.011450185440480709,2.6180339773418382e-05,0.24052287638187408,,,,, +48,2.7021000385284424,0.011155751533806324,2.483843854861334e-05,0.250980406999588,,,,, +50,2.680500030517578,0.010021247901022434,2.3472963221138343e-05,0.26143792271614075,,,,, +52,2.699199914932251,0.010751751251518726,2.2090569473220967e-05,0.2718954384326935,,,,, +54,2.694200038909912,0.010503941215574741,2.0697989384643734e-05,0.2823529541492462,,,,, +56,2.7091000080108643,0.010059370659291744,1.9302009604871273e-05,0.29281046986579895,,,,, +58,2.699399948120117,0.012161476537585258,1.7909431335283443e-05,0.3032679855823517,,,,, +60,2.7216999530792236,0.010671027936041355,1.6527035768376663e-05,0.3137255012989044,,,,, +62,2.7158000469207764,0.010463157668709755,1.516156225989107e-05,0.32418301701545715,,,,, +64,2.7214999198913574,0.010665320791304111,1.3819660125591327e-05,0.3346405327320099,,,,, +66,2.7116000652313232,0.01046629250049591,1.2507867722888477e-05,0.3450980484485626,,,,, +68,2.6923000812530518,0.010609752498567104,1.1232576980546582e-05,0.35555556416511536,,,,, +70,2.6830999851226807,0.009290814399719238,9.999999747378752e-06,0.3660130798816681,,,,, +72,2.7093000411987305,0.010727670043706894,8.816142326395493e-06,0.3764705955982208,,,,, +74,2.698699951171875,0.0109737953171134,7.686770914006047e-06,0.38692811131477356,,,,, +76,2.712599992752075,0.010320967063307762,6.61738795315614e-06,0.3973856270313263,,,,, +78,2.6993000507354736,0.009841523133218288,5.613203938992228e-06,0.40784314274787903,,,,, +80,2.6861000061035156,0.010179675184190273,4.6791110435151495e-06,0.41830065846443176,,,,, +82,2.6828999519348145,0.009790077805519104,3.819659923465224e-06,0.4287581741809845,,,,, +84,2.699199914932251,0.010508442297577858,3.03903811982309e-06,0.43921568989753723,,,,, +86,2.6988000869750977,0.009589221328496933,2.3410482299368596e-06,0.44967320561408997,,,,, +88,2.688499927520752,0.010065913200378418,1.7290908544964623e-06,0.4601307213306427,,,,, +90,2.6928999423980713,0.010363687761127949,1.206147544507985e-06,0.47058823704719543,,,,, +92,2.714200019836426,0.010142815299332142,7.74766078848188e-07,0.48104575276374817,,,,, +94,2.672300100326538,0.009833029471337795,4.370479871340649e-07,0.4915032684803009,,,,, +96,2.7018001079559326,0.009937037713825703,1.9463863054625108e-07,0.501960813999176,,,,, +98,2.7121999263763428,0.009417451918125153,4.8718995060426096e-08,0.5124183297157288,,,,, +100,2.7028000354766846,0.009256146848201752,0.0,0.5228758454322815,365.8839111328125,139.93499755859375,0.27300000190734863,4.629706395531346e+17,2.7395541667938232 diff --git a/example/gpu_pretrain_loss.png b/example/gpu_pretrain_loss.png new file mode 100644 index 0000000000000000000000000000000000000000..764acfabc4edc960e89510814dd1ab9510cf0d47 GIT binary patch literal 49881 zcmcG$2UJvDvn`4_iy|0Mz|e{WQ4mm?926QPNX~o+N|M|l8AJs^NsWl0c)fh5umj*EI1e)_3st4DD{y$Q$Bq?povTn%z3-c-zk2 z%-ZTQ7xyJD0nU@Ac)X20_Toj$f4{+HZD(@v-1TNHxXLaY87+Gnnj>`R|C>x=n&4jV zui>P`Rh?dqcHeUfn_XF(Y;(UuN63D^_srekfEv|Jw)LqW_2l#nE|`Bb$oMAPsyMZeEX|Ic-rn(}Wzw344?}2ubWv#lEa!h9!B3C;g_a4PE{ z4Nc$Y!xJz>!iTyj!P)O7F=job&f0mVA-nhNarp6eQ`qz8sW%g@n|Hm*!8!~lT6Gtj zeF{E>oo!KR%{8gdu6F(NZpUncB(0jO{>$3r^mNXA^H$lFrN3=2xQ*}&1L0}5{Z)p4 zk6p2PrSCeHA3K>lc~ zXCi!wZhCM@&Uuq{CJ&)Y4_LNs3V60l?`O@y;@GvO7>*I3+VLWGzVKUw-e zd7`GPn{L@zKz?am-uUHW1t)H}H7%Ukb(C8V-+0YIq@udN@&1+_c$mNpt=#JqSv+R3C}c0+Ld%WV5?xfv)%W4TRHH%2mbv0P&|@Z zl4st^zOvN27C4lXVAfmiUi{PN_{?Y4m=>4b?vi^JbEDsJEZEktW^xM*`zUX8Xy_r| z?;`1V+H)j7e*73IcrTOL?T`4r6IZhp20eb< z-;wvvyUiCgv&GOXL<_s-+5D84o#++)H5Vkh>J(LXNKPb1@rWM&X8T*R_a3f>!d{Ow z^!T-reA?x1MZ0(J&U5{1RW@BGh#g7KCDhlY(kMdky&@Q+e0bk`G`2BmkaBs@ZGy7OX zbZLeDcpbi=owzVPXfZR?I2+v@EB;{Sr!Py*9T<)t{KRphM{mMjs+l9JG``4w(8q0@ zM2v3w^*Pj`hK=~iJbzKR?e%RrJ=>2Ai&I}N@>%x??PC^tQ!`D^z(6h1}X z(BLy`l5#B36A%y>ZYBn0R;;>|{q1ozbhT%?Z1F8U_})A0HyXF^uWvJodXyI5o5;K# zz{n;jsDs=^rQacNH}~0tj$}H!rA#yR4wCrqWx0a8lC&zh{pB#O>6L&zuT|Gohw&~g zGMQ}Nl<4QP_gFk@jGJ;&Qj&v*!RFRkt6OK!o^=7Q;{d-)lMZ4w|CX-xvCa0a*QVl? zxlTTlx>x2whb#k*U-@>qUg*#qV+_A#WN&|J4{bB=Y}Ik=q2=j%4OsDCHdl?0`aI{< zqMK?TY0GMRKr51Rx$deY>^hA*caoc$n_DyW1d%s#nQ4^%u&R4?)s?Nwt`&UGwp#nH zH?|&~sH~ ztUc@|YGmo3`WqqYS>Eedu{>~5((3PQn}LP^F_1~nrz=YYJk$3A%g*5Hve|DsBMzhe z^^^1cK`*tmv`*C@noo0In*05XP31lF#7I`f{-WNm5tnOS$BU}-v@_J0UB^1|`LOy9 z70gaQ?)w|v?I|s)>b88lw_7^*IjJP=;^mIS-kV!X1Er(>+i^DZhhvo>n0jMZ&_9vv2EdB zl1M5<*t;Rp%r~!Jzy8$Z*|Nv{&mCi3MVC#5goTCgwkldWmFerbkdumG2Jk}-ag1ZR zmYw{1Ht)8%OnqjxaL5!KerJV?byzjCpyS;Dj-9v2gQ9EB`R+B(FB)XRGKlaW z!Umc}E`Ge|5eHV}n|^rD?W(8877ej!^W=>R{rTt@Y%!e%WHqvRt?%q zej@O4L8m~~!=oI6{DbHUn5LF8*L!FI`LJye2rxMHG&Kl^Z4g(~l#~wSbwm8-b)GWT zv-^COkeHsH?q6b#X8e~(+<0*-ST}%TeeHh zEUOi*tu7(|^Uy65wlX-ahyv^0%=q}z6@w6U6ZUYu5tDn)dGpSlJDkakzn)wy-zlKq zJYu)Hyx_3}K2cNf-uR4;wyLsnd~Usn4p=j72f7~wFP^=U3kwV4_d1thQ#tgy|5cmu z6kJt!yfinKrs?Qh3IV(3r_FqT!*C0!qwTfL&-ebKCdS4oDntc#Hnv6>eri`1xRRRB zpTVSjx~vKzR4`3#-wj}+=>x53Ixh5_nqYVbcD2m1G5!&X?+}fBN?pn5!yKdP>u&pJ z50YaZ?Vx3DvB)YrJmmy?Rs}+_n!0*2%;U(9cMsI+#sCyGLg@6aic%nA@k6mspYCPC z;$`4b!oLipPHEBv-%WvNr@C|YryQ1X=hnI7*xxlASuHr$DAj>1HkITS3w;QT-@vdA zzjjGBsPHfcoKP0XBLB0ed3f=1YbCh3S=cb!1Fpb?sU+WM1dot+_d5^6;lsl@)y#W_ zCvHV114zpZW?}zu3_vD;Q!od1i25J0CYmNimPT{^MOk_uV!OKY1Fe#h9^ANT+@?v;@-T`1Yl57zPdOPTp#1`O^Z+DTejhcJhufq%kE;eFX0#4V6k{1 z^uh!yG&S~^+>gusc&pJUNI3cZdkM?-Y;mw|4RLYtPa^W3YfHU=hR)%_?n&ro|8pPt z$KdIOnP$ZY|B0)y-rk$tA(EmX zEjcHsTV!{;q1~ZPj|h9)qPy5Z`?awjzg1TXcuO?bEs>UM!1>Ja zEi@Pcpyt!Y27Um@g+@inT>oYkT)dW2F8FgX=-0qN1biCr+~>y{xn;+GUY&xwS_3(~ zZqv1~My$$;L_l8`e{rMG@onTyvRP!sQul-e^HN4(&+eiX`77P@c1st_=ifs-D0>1? zu@>UeHl{+ILM!pk{Y+TiUnho}20!thX>KM-`tDEuGBDLyP+Hpc_3PKY9dlr)d$Cq| zE?5BE?&6(!7HxH$7cbH}clk2AoHt#9O{TuY{7vf3UxCY)bNh;gCMyrsa_JSn;D32x z)B!+-$+!8Hp{ArK&hvkMvJZy6e0e<2;7yN&epcIPQ<7Z69C%lQhJv5v8mYrk_#1E9 znEfo4UW9RI_y|?UUwh(RkYTOJiu(kZXy*q;M3cjtg5G-gj%h((D(%oIM8^RxBd#$2?1aCJiAr=zE$bubv^gl*2XBM z%`dr)t~Y2xE{4)Q->%i?I#!JpfF&T;D_@i~iE($#A6y!^=t;U$8@Vmu?{F(e%I>oV zCyMb)V})O=3YP)|PV7gZ4(w(6c+kMI`e0I214y<0{ zXKF*>-z?7+j$chxV!pI2b2?b{FaTa;fc2S;1q3|IMjB+ShA{?UMxRn6A(|nywYNwy zCCSXdeKw5Q`Bw;IeHxdYy}d)T{FPRxrHIQW=2O2uqqLXbvLg||d1_*!lt@fg%?fhF zkjZ_!SM0%g_=(5#%igU4EKAY5pUJd2@p>GdgHb6ERR#tI80rx?AB?&etsuIBhulLF z3|A$bOK;gD^WowWPm6N5j&+%u{qL`hUYoX{pyB|a+p;5Xvh$qoBW@bZAxQVd9@z-{ zNVt1FAwvFwlC{W7h7z9!jgAr2SR*IQQj+rk;_X@zPXY(`T9Z?p#nhssi@(daOR4N( z;9BhS^K=fYaT4DUpk9<*`Ca>xW3%`B_wN}WN%2{}(N%$ziWKcW_dS-jUhUZBJ2%4W zefPwDn04A?0uDL9qnrx>xmrT(Kh4SV#ZxsLD~npK*Byi(4#&fGJr&OGJHqgM{jvGY zTVx7*Tr8B<((6UnbYnbMoj)2u{tqsE_?e1ABV{cTq<2J3ASWHOI>TDgqkrw%HJ8KGZGFvn6 zhdDw6!$`@+_Nq)b}bl<1qYWkv>X{?7Qa zc%-!1CN;1K<>uyou(HnmE>V7kZ-vG+y+R?S|J(!3`if+hnllV1M>`8Io~|a5;OR*a zGqL?oFXI7gSPayJE&lmHYr$|8*LvlIV)9?&x!Pu0W}%d31;LEE`=P2U^{pqxHbQKYD*&335a6e2o2=RdHt zWawNGUK~nLv!bQ>>;=@0k|AvIQ)?Pa;CNA{WyIsVwmc}xaO$~P??6l@`-x#YNNRze?wS&!-X%d$KGo3Q-%phllHKR67YV1C#S@ z=xW^6sJh^OKUFRDFwELo-mu+a3|Lcg&MdR%BcgG1Pg9`*SiXSC;+@jRqW(=cgaXKi)Tw$kv#$^WC&6k zR#sN+f7`|!N6MGRn~53sVDzAAG@$JzD<3G<q}1=`BJajKPB3f zqGZjxVC`3^;(M+jpncweOFCHlf?E>sj)s<2#J5bnv^E1zVdJDONUrFnuF?c^qKEgO z#BsS@hq-i93|6PYcePfZd1b?r*b8G0mK(y5A+w?t5V8C9$))z=t(=FJL(A{?T--Xl zXV)Oi@ZRAy+p1$GTP^PaEk#Mp>yOB$^h?RT*?pftAiSuJ%Y3ia06|{Jbac92l-Bv> zj8dJ@^w-!N6gu{t@f_%S?Cg+C`}N5u$>GX&R%W* z-?J_&t07E^OoEPkzkhWA$6GwtVUZ7x!k;^+Iw9P^L%?`qFZd^e<-Q}c(e*G*S%x2c z9=Tb)f4Hl7DE54&5AO#=o-DzDmt1cgf z&F{@A@BB~^qyz3Nvuy~Z5rKv!KfPSH)48p}$xYNpdpCqF-iqHXzWbS)#tM5gfs?yL zo%I(lU#3}P7G1Y4o%BI`t9Qcd*RQkJY;D|b|B_~LPXrh|{I_Dwy`2U>@-4$`mp>ksHPMXI>@D)|SO1l;+D!~Gp^4s(|o_vgV9P1zfK4g&FciNpbebMPxh%_F! zxov6^rAy-_C{aUnkU*3I@XlvPOV=jWRu!YL4yk{2)78_?DMqKOCt`=ta=;n^vLeOC#@6&h>Z4MzI-527^>Y1uz!9LDfIU-!LAC-HMY%9tHu_xr zzBDhO%ZtC}!!-0st`lV%kalJrU7G8({=okzXh%tRW^NspBc-|q0y*#9_Di#HVc?7m zCkhTL6k2uj8ou8u44BKKAM&Ucc)~Q~&LA@13_T~W3PdlM6{c3XO=e-t+B0o!^=}0* z2cf<5EhH5_hhbGd$S1(ngc%+M_bis*R>%FboCSFaVSxT93%OPwg&ejXB>BC28XIN5 zzOCVfTpbXnVo^~MVp{m`w#UK*xWsTAE1U0o63KI0Hop@=2*`+&6Ngqfh3tMk?t!)O zzjXf08A&uWprLANYVu%%1ee9ZZ(9DEP{5TQrNO$W*~&wT4@cd9PA{OVJscC@fhiJ(Y^N9!l?SW$ zItHiJN{kJGICL2%5l{2S0IV26ymuC`<=JX%U~{{V$*K(?#t@i$31)lXbs^i*Xm-g$7ODmd__GB znH~5VL>xSbwlC7HsjsJ9ipx9o31TU1XAC7YrV4M+GX?vVkJrv@3VX+=xv;a8sDEUT2e4DYdO#id%qjh z+{q<3D~h(B2@;A2_|ZC~2_~@!fdLN|^v!&K$Uphovw++MfAQoSUqTB5M)g|r{o0{a z*nchnHxcS?n}2io$+d~n=|F_;w0pU28)C&7xo=5j%;ByBqG^RR!C@dwLw#X&hpeJv z8_JR$*}CoPg^$b?nVNPMWQYL2!d_7a+y;8S@i+Z}FKRiyU+Xt9v|QVt-8oi_7z$s} z-RpLUylfJ9=j)3&sIMp>wg~pOs;X)tSUjI)hbnq- zAiD98+#pavTe7zftR5i>NFOZ9pcJsUIFgkQ#US7S5a9p*{X6W}uGpL3(&2iJ zS~tR5wqa2U?=*Q!cx9$JQrNYiz^a>Zs_wqua9hP{K6)D9K+M2dfz5~W@bI)G$pP8l zcny)LP;4U#EU>E#$V}mQvq-krJkuvg7h^7MhncMIS`(RIQCdsH2d`_gptqu z8RVuPm~$aL&<1z2IvM0?3T2{>Qs-RAQt4Yux&s*bWDySmE06(fp*rXByRiIVdPyid zd9Zz7>a23D4;S!`@IvbdzElxbt`(C0JfrIU3`dT98JT4V>qa~os>_+ToYw=|_|$~h zJupQJ2=b8sKSvOSfkWLCvO3_r3W4A=&eskDOMr2t03nM7YsHh3!kg-MiOF92Dplmk zxnAN_-u`7|{a^pTltR&^A79leWu+m3x5I*81-j|L1Zn-2s_kHUFpY`Gc3ceDqv+m( zr}^brV&tNb)y9$4cD&hsvtQuI=A8i+TXt4CY}(piOyiyO-r=WXi36+vc1$iwW%E__ zpHMHVc@dSHovlQ826eLB|M^#qCHE$<6LLu#)m^UJ62!l~o(8bmxe8b^CLfpq$S#2k zN8BnDr&1s|wl2N^CL+En;5ZS;q%^fO_1B$alyV4|BB~pLFT3e0iug zL#TlulX#^-gy@w3wl(%WR~q<>Fiig4@5+$zsjRK8Sh#&f8CFV)43t=EYHQUX0#a6; zo?eDf>@boHStA<5Y~)goGT-yIO_LuTHa)D$U!MnCJ(_pQKZ{co?}YG#DfCwdwLwkL z0bsXfPpQ`QV7&&=s?RUzB#XE&n$|}P7cUN{SvY*%!Q!SQx;me*pG71c!Z^hN6xkDh ze~m~0X2cv^O7XyRo%k95^(P`|)TmHhMI1ww5&S-f(&SqkBXwTlEcB3O==J@{~&A;w2`*{8~QlbY1OrY{>A=}^Zv$Fk703kgb1#Oot0G@SnWoD z@+uQX(Eu(MfuBWFb-RYXHcIfy=F8hC_m^Q}W5ME-)zzP04?Mw!(|%)PC>8j<+}#y{ zzWC~ORfP=0zSDR@V2|_oeD9&z7nC2;kYdFjDnJlw1C;;<2Y&PEC+@Z^gRFrs;nFi3 z-#L9&g7`hxWCZG=-{2v=qji7=suNkdi}VLqu4oul2hO^RT)e30?CiYw;{ok*6{;<) zKvLNN*x5U90kVmaF8lhh`oZS6_qT97A79sW{5TI@=D@BU`%g4Lag)!sPb%N4TN}dr zYwZk_4MW`*gM&W-?fHY(UYV<{uMfvSHrQ_M28;+WKi-ILLfg%}yEy+wa|?`)y|_Nc z=KZckVDoS1BuY8%pgiaSZnlG1ROwX|o_z+PHVHBc_>BWQ2ZsvmY(AG6Q$44jk5KhH z46@XIqt#IpQ57>Y6qb{~4VnSd0rUcp{Ebk!Nt3%+rrM@=FY3g8X?lZWlskyo@!V+U z?HBg*$-FnwLr8=hRE@r>PR+Nov%BS}aIz8h{(s+N!=YN)Tu9pkum4B&i`Uf-+fT$a zQMT4UZ9zjtk20MUDbMEh7^>WTrcG|)zsF|G+YYLDE;S~#KI(B&r0iC&$0yOp3;Hp= z{YkEFh6NOBI9YT6yUA8{n16-&NZ8B&e6w%T^TxUoM~2~{g4l_$t*TgtL+d&m<*jjt z4zP+etD1==sc>J(-hzgnxKo1lq(Y6*J0s4l^*45-H#UcV!>vnoyjsMEwpQYPG-3Us zDcT+C8!57^>k1yRR5VD8YLd#$BIC%d6m5{#FVF*3;@~@3Y)rP*hCJw2G*U;=0OHuA zFr@Zt>tW&K0?0UQ=~A0;efu5Jcf4A}&@XXsWQkMlb(9f#nW7scK7%qKyu@~cz^D)7 zmUL)cHne^P3F_s-(B=N)54!RuwyHxEqdmEMXO!2~T{`4n@lBNZ4PHiX#1SbG>g4)w z71!^=j_&f1dP~Y57~P~pNJarK60lj1Lv6woHs)+InzQ$6*f7^5JKA^amn)(?S!NS1 zK5P9A%A@`hd8b{L)vtVgDk;-+{|m=L>iIkDjyC~q!b`>qt(>ab04Aq&KYC+*nCD6H zF`<=Aqv#!)&p|MP2g>a0x+*rvdLBO*=7_q*3z{c*#Xp2MwopGMt$)fMMO9&a-uT-T zL%F<+BjtXb&7ZGPO@}goXHKa)`bQG{C>K0%7JcVg@e0fFP4&&q>$5kCW^bpie%t8} zqK@&@S^SP3`#`%)seH2M5Wez4QQ3w`z|||o|4C9wyMBMqXrg4MW-U_4(Rh7AQ+@JH z6WhFUB6X@FC>PNrw^W*aGMw%8p-z|ab?PO2^vp+R3~otNSAFB_&#W%Dw3933bDpdV z$%0EMBUB?l8{3(V_y75j@+^-p!D{W7P{ZENrOvMe`dSK3k(JSpVsV6HV5mFdT(K|= z^b_SBl)pKmma8@ODH~Vj4$Q~XqO#;Jk+qz?FXPfld05O|(sPY?2NMuP^53r?7T8Oi4y%jCx+kV^4_oT=h^HB#d^^aos#B>==Kb{tEEA_b~D#&|4}L{?^0x`EL` zwK?OX*ZHKkuy1>$pf-cYN>NP^mvkekd495=&vouDs_gkz3@a#Y#!eiaMSo%kT-he^*rqY>7 zc8-7dE*pU8msZ6%Vw7Vf6efF}K*}}?`KlZN)G|q}X&P-qPN)!q>hH&`-iSjHHuGrV z_q&V0?&#dy236Y?4v8$9eb>ww!W6z!7lJ;7Cr(ApWp-F?auB#u#<6CA6zt)(hy@@d z!uY;f+nL&=rs8+s#pvIldPsu#fw*RnEn#pG5fQ_U3DQ!5TH4y|xSCI&9)Fr_uu)Yb z)4wUrY` z{Nmf#)W+7WcDf{vo8c@G%^^ zSFxlQ@OK1?@1G{&;n@M&JbLu#V1$7@$hkg_>i)S`Iq*#N7*|mceOh+Iy_tG0oa}U` zhy6Q|{PtX-2f1%1RMoyM%>(Jzc+qn~4u^D* zq@N#r+m>FKYSwJ#h`Jm0WsqlVx7{bE^1^kT?8iR99#s{{lmGKv<>CN+GQ^J zhdDKs1|M1h@q-1jN)7UU#Hb<3UB(=x^fzh5pafDhhe!?g%KlgTBHB!91_b-Ad^<-Z z*+r`TuFwknfc=%~)p z*g61!ym!7HMFn~gwKhS;>3oT>l-6#gM{hk82+t1wv7~RgQThrLDaaN43Z04z#Xv;@ zCF_L5M|+NZb6FF1of`snq76_nAMiU!US(!x);h6s>&{a9C`?l4vSjk8R|DxW)k{2t z>;!{j4yvB`8`)9eS%FC74E-`~NZp$TV?3F21b@W+B;8C3eL(%eGW-C^X+dr%N8q*j z@s@Ey14vDjT|jcK0^&5_sCa?JyegY3C3!%TY%F89oinJ`aZ&Uf)Z^fVNqDw1>P~%h z20R}k@OHUaHO60KsI9QKw_k~TDyNL3yi-#%?{=`v2ZTR+hGcYY?;q^}g|H&3Y5;{0 z^!tn07P$5^!|=TyDqgc?6PqXHKT=$*%waa$zr_H~P5UbSBCp!UYZqE&K-!{}p$@dF z{I+e|2#N3~VhT;obRmVrkb!RU=vNe7$%7pT;vnodvQCHZ%_$8kr!98&L%f3Q(5jz-<=)dA|qwHh|1&Rn<_Q&zheG z9wW7@03>$BoM@*bkZV9YV0j8vSxb>ftI@fF=*s?kKF>M4}9|(1EbU~g@&khzeeO|K-*@$Ag z=qkI@@v^iUS_O zujTx?mo@FV;Xz*m_`j9-^^nkWLn0MD@uBFh+_!&!V3TEB;aRGM3c@X~BS`*}BWm+; zzau3ZNbV7gK13-@!ORe_2@8uOPM?K3ht6mY=@3i^<-GnvmR0(XQw|4X?mA4sqG#KM zV8WyQuM&SmE6}-9u?OXXv9heL^Ontb31@x`b8hUCgJ=xqDr5+MN|$XnkOIP)Ap~tz z$Y34C{@dG4=}n#j~f;3bYgxH&O;a$}F!% zZ!Fm_l;X&^yJ8tiFfq74>nMCb81HI-xPevCHNDIt#lsYiwW?yZ_DX)=zs5~XnO-9(^>#qGBb4k!hEH*)BK^Xp z?bO%rKnr)+a$ikBd6ojfST`oh8z$;L-8~ykw1wU__)#6NF}jnO857t>c-uw}aTrFw zcdI&9Af3SX=NtYMFE~6d$7}z2i!cQ~k{w})ns(?@f6|&51Abg^!^Zs3UEem4lEW14 z3;%1>lJ=XMQ5h8 zmO4;HMUVk1Iw&~Kjdf~_7+FC(&&7)6*H%RXhoeCJZ+GvqCsxBLO9-S_*uK_Hg`q2FI4MY}s z&eL~lV?+(0R)<;-pw^Ip0L^e(R%R==y`3E%1V1By3tFI9>hNLT6)CehncE>l$bvRv z^N@0K6FK>%)oy{FLTiIEQ1&1_JQb@$Sx}!3`+MbY@5Emxi1xG5tc=kzs@U5C%PdsV zhE9({2+s$T5)(0jm&GON17wM}s7Vf;?t`6C@VhfOK+vdM(I<4Am-npIS_9i@$TByk zN2i7}YtOI$8#?!uiH}cmW&gkC?Q?QKf{1E9P)+nOLa-Kut;tX^VpYh+rUE^7sv0z= zDW#=CP*ZGH+OT>f*=sy}0$c zU6=w;n5|HLB5Bg>! z?6&dJWyvKWN4>&wB)rWQdI8~8DJJ_+0a-5+C_@cvt8gJmTceYUw?L4E$})b3SW|&Vg^Ed7t2q*Vt12m- z#KE7f(7W&~B7$6Ky#ZXstD~TSh^u_Q&%gTYS?LMpIY`<~Yo4(&^w*0lrDv8-)jY1? z!c>4WQ+8*M+F(2$4hTCJq=>7p5fv+`G6#)ESck)<^25*<)E^ag8Hj z67*#P;*G9_q~qg5uA;wBBCv9k{xi76>wUh|0J{tEJ@G?x_|(PmC+J%=m4H(YT!R2Y ze*o(&Pkvl7UQJ z^IzZ=u!hu4Kh~x&h1DS1joZ|oKjM#@Qh``|^1h}Y!L5M^OH^UAofC?4MQARJ7*7Pq%q`g?4s34beE?4wBIxmt z!3MHHy?^nsT^!r0#EunwQUfgMu(5+E7dxOi08Hw^bo?!6ruAE(`908Hk0Uglft+V- zXUf0xbrj}nH@Jms!{ZLhf2OchFztu#?M(odm8kYw`}Ftb&ubgLkMbDeq)yST{>GxL zY^sXHPs}XnHeQQpiDDsGUL~wR-qhentq)MfdlHO!1L>gTcXzzrzdi*aE_$g7;6D2$ zxjviF39WBa!Ujod{F+IXeJanCpqiQ5nM^^v-two*PYgAXulfpqDSQ>lPkN|MG+KNcqzQ?S@odk;izbEF;B=;gGMRA zLxhTR$Yj-DJhvlN1=r_r3tYqS{>m#1+nMdcQ9>dljMNWGWmQZF27gd}ct5jTkT|%^ z%j?^$4;;_w^vZsg<@O7NZpH;!BBw9ClWOET^G}DeJHxY^!W0@)>>C+1jA)3JXTuLwJu-KFVzIa zW46N8pIa~J73Y1cUO`H|7?4gQDGG>iGJ&Wwht82=P$xQo1~#SiRTJI(6E$E!PW%f* z&%+&%Xa_M}(8OT|n)pa)TSR?8&?huYZk9&PFQ^?k&Bz0~RTevYSI+?%uPCx01zoxg zAhOB_y*ji~;gGs{ykw&MeA#JCnx;DT_J-F{VmpNG#y87j$MTw?)V)%L8nHmorbR&A z*8s_itE%0G;(ReUq$uD5ZL0qy64gVaCIKT1?{$>ivvwy2(bIjAC60Rio1aVlcYZn- zG(r9YPM{>QZ5-O7sq<32*KRGb0-xV zXZboz;2Z!#BB)zAu3WkDi=PBYWv;#_df(V408^=|sj1n&(5i9s=FP8ltqr51#?6!A&wsM182S5R*h8K;0<*`w;R{o_U7bgd_dDo@9 zO4IX`R@VC1PnQ*zWw!MX^XC6h0U!chdjNBd)INl&jce&|Sd+14y|6BPS2>ya5x&5w`C?p_$;0bbe6llO^awFKX-{w6c>x zIuAYFAYbS5Z`Tc31~hrAs8oUA|0Tf@yS}YC+2fkkH1y6k^YVEtJ9UGC7_}bEd^}=+ zeh5XeznybXUJS=T@{&Iv@4K_Ryn4HBSN8SpJXV`is5-UW^$a`72I{R*KJrP4C{-h3s1${tmzmQ*T zLH!V@%y>I8dGCh?NXkyAHqNXrP57pJr3U<;R0XaRB6qHGk@*Fg-rcr|dE%aYxS;?mP&69%|Dxma+wE+>Vf!pLG zlq0)?AQ(yOAVm`b8P$7hK@e_36GxO|hZz>U@rYYFl(0sS^cm`b=#U7gKp?sXsk%{< z{ho`ZN^tfC(%hT7kD|6XN*Tg+TmWr;q&MHi`YIOq19hk&1a~uk zak?1}7C9DVMsrYOcvmKm|20Ko{|2eI6LN?dcp^o}XTM!2?4q0L0!?lbxEoEB$Rh3O z5Y!$Uq3Mw+z>J3bNNN+?pK-g8=L3K*&7E4%6QEnw;J^~du5t(Vip(_sT~9ymCq0Wt1E6V{fR~5tt#N zptfs)^EaS9Sjvc9?8c*1ynn%6D!IMcUv>z>AqiSKin_yM-VP#y3b~AiIZ&AM>*!h~ z?vPRyVvpJ^$?8H)SjAzefCOrI<;%0fiSc$S^3CxqP848zKmi^J;!bVEWocOTCQ!a+!JehS2yj{h zg;8GyD5uKAN$x-$IgKxNA5fW0!OQ02_CNVGr54z@&Q%|EFs7MeqL3$62beWAC*Pyy8u~TwIF^d7@d>?9HtBb z(IU`e#KytFahSN{);Z=dg{GW4ePQA%H=Xa+?LqS)cJ=1ID0C`<=<-imqhXR4VlXKYOTvgcE#iy5LiI97fx3LIY;$HxM;8H7P@j zAJWUBE=wdRhcD>a*Ybh-+n%x#%P!c3n89Fb{qBIl&0SGQP61!FXwNo8{4L^>9mq}B zKy(cM1IO728#^B48D{VW5DK)xktKzuH?}<{&8z)iDoj0K708dnvV$nY8)?w3+Q9hB zpr{I92zJ$zDpju+{xAmx52(7BO1iKHGeEx(pHpu>_yeJh9gc#5Fdl@y%I>h_E1Z*x zuh}SKy@PNMI%hb%sMY%XXI}XtU9RtA+>7}Az_nDb*t*DVk}H``OeJ>q=a$x)$olFj7hL{IDL~LjX~Eur<_C8e z6e>;U!x1>>fE3i9>wE)nEj;i7N=PivE=V3}q?d=>*UiedZb-o`7!%;_H2Ds@%NB|)FCv;e_4ArT~`YF!zb09`x&{20~gLRuLZNJue=I0 z8|B6Sc@s-YDfM5kc|Cp&TV@Mmi`o0ge5s!nVQt()2Dxne$WY$i< zdq9y5ZBREtK!6;in}EtXam|S8RQnLA7fjeXLf#N5QGUoKpBJqF^*FS+V#_E;hTm-{ z-{^2sB8PP{zR8@}E@O@;(42AduRoq&p+5+HUFaGH&m%9=wHZDl+fI<_JEtkkq)q8# zH%@d|)M?wJCrKYmn%HcdyW{LslFe!^-4dh(U>Xdt0k`_}P;sQlO*ukZwy@f9U4ktG zgu&@^Kr({iVHVdTLLVG0_c+)|1G|l%ESdD8g=TSMnA0`G}^%km8df8>sA04YqXd>x7k1L2+9gM1`6GWei zoj~)qPDk0M;Cc3*N$J?A0XU(dp3H_?YEpF^(1DF@W~|gQdk!Lz#ZqL6c%9L~Q#i@Z zOW3f;4yWMt^56j-;5xQGgZi962BDQMMwU3I<qWA>Bf(l^3v9^I!X&m|j0v!qyu3ty6Z_krB&O%Cc@OII>nRDFhBrS$a!uynXBb z#@qC&0S=C9#FZN zG7s8l9y*(-<9DTuA7lDothyUoR3%_(T0SwyI6rLS?o>(xVoj7jyOQF+C?(phTh*}N z>7X=hY(Qk9TIUd$+3;uH+E_HIR6K#oB|jFHV2zE|75RKTxEG_G6q*R13!hEqKLRt2 ztSBCAJ%#v|ot15pJ!*>4cW5LpluD@%;rtpP9w4M%4EYpl0!)Ec4b&3@J$H535zexO z-!C2XW%I0iMG#yWhVDMpruD|w2msp+)Jx+gV(`#l7vbhVKxV=XAP@P$zhM}G7kUmo zeh4Q-3`4_hiW79Hn;?b<1>qTTGoSk+p3pW3eYv7cp6;G6@`)5_hS&?D23PTgh) zFF%|JZeAN2g}y<@Pd=P92LVPA<&3}*fUq-F6o59LNHX9C=;})bzKx6yV1O2D=$ktY zM+nM7^OhR)X!1f=1v-DwzmG$b2h=)tnwXboC%s?2EmtTBdeI?4t(QDYgyt8gKmTOt zm8jqAvh@X2K>pix(PQ$7e>NaU{nSaq8-VvWpknx}ncM9Du>@u80f7Rj z^QH;T^+|-%RqWu!H$OJ>!D$oF)P&k1%|*=;pVAe2<||$Ih9FRN>1nJv(V1#o=fM<6 zSm^9!@KjJ#%xV;Y){k%u9EIo8{kz>B+UN9y@Aw*PKnK996u@$g7&kEn&jbkp_R}s_2EC; zrT(~-K}Sfd^-+l$4eSNoH%bUXVaPqM6B&~paJ=1Xj2WExg&JU8R^fQ3W@t#m;GnDI z@!K0A=`}*DU&u{#S8K+C%~G8sJ>TY;TGyo4`BD&}|wG^4g0y#CQSaFAX719}dX;(HzNznOLgk15))+=uC{D2m z3w`!JOaThwD0$9z7*R*!1#~8dIN8Mv8l=!+RA7Ce zovo?yf6pQI`uya)#{RQ*aIg(y$8)M#4at3OK-xg;;4Aw)XxN+`@76K!ShyYFD!^oo z5N@ecj#c@>9oV7@ke(?+0}Se-1Fdr!KwLs1Y#QmsvnTzKh@W-1dn9nz-Y7>V?l1#t z!^q?GXFB>CkZ!^02nN6ot3dPk4Di9IHOxUoA5y0o%E2`D)z7~$jjTQNVfzj{n2-Be zs8vdKo?VaV=pXhhTiH7eXnT8I#gP;vfGR-e56l2xo&r^RI7Q)lOKLEjX*mllx*WD+ zZPz}H*J{W~#}%!%Doi`N^z$A^KNm}tjcE7`CuF?-!qxor*kx#*PXi~V&bu1u19^}E zbC5KmY!%=zp_Hf{eivaR7&Do-bR{es(JdP)zi{{-2(HahMu~ci?f|I-jBlH1FNhbw zy2+qs-03ootPcF%N)-^5DX3j*ZLzHalG}uAxrqiFj1EQbPWthfltFcxQcAKjcyv4) zyF>ycczT{IvxbH4a2f)VpZZsWScDMB&q9lIr>Lp6rmjvMs_hsvkY?Hy>7uSyXtG}; zf%fB|+gX|G@4^)PSg;QJ0Uz`zWMMbL(kUp3L++xn>F@fOEX0vr+KZ0bfu`ILbiS7h-+Afbotb$5dLi?EO#>p^aUljH(b1V4r@shx=p|6q)e-?ln*06TWIqwVu0g3&7anO9|% zHe+lwA*Tta+yJQ_6-EH8i;V_H$U{hMBFP<+tZiNyT>PM5{pt<{8OrcMcop8Jj!k04 z>0mxw@*clMtuH^OhhH)aiKuYJtZDjO5TGof*buKZv9LzO1l^IQ1q#bbv}ebE|OKDQ&i}>0?~6;X?WLuQv#`pP1S(tEU0;lQg})cMOP-2HGKvLdUiuTCI!J? zi57sXx}+WEyBY&Kp4ylbTTU45=ML6H-h50t&6j}t!8Gc~%%2}c-FYr3te*mcC1O#4 zsX3n%_uFifd(_6wplBJ(3rx@cFfPG3TW;H4F)( zp`)@|$iuLIwU;3tJDiGnaQQhlM)NDLc?@WK|O3q4w_EmEs6vqD&Fg z5S3_2%4E!=@%0rE(?eGeW)D*!@FoU$4(0U+lg2)lUtRCE2}VsjE;327M5|iwCG5>H z3O-KZKv!2L0S|SQ@dkC24*>?v2Fs%BJfkAhQ)n?0DG)}m-IBQ8trHtaxg=D#t!g>V z85`O*>g#%x3jp(Q7$ApNNjsIIP&z`niZ%ruBvG%iimq`U@?wS0`jltM>j#iYv4VVn zo-LA;Jb1S%)`5E0lbhie*r|x*n#~f#8Tyos)jUau-e;fP8k8gfTeq}w+;;<>bwHJK zI0X}nAbF<>2EW_%TzLR>jkVDlrvP#I-0;bYP<)*9S2olDDf?i40%e6(0Td_Z z(eH}l^oe_CN%b`;AkT#lq1~zp62e3{O998ocoaMKr-t`II6Y;R4(x_ik_Ou&HPw?} z04V9CjnK{wqTV|ajq5zvmm0FK+=y#MTC3t6zxJK7`cAQDv7InsOxt?OxXyf};I{?6 z<w|HYzzeLUg>=#NTl>{{T;XyR3l8i(?by6P@e7j zZVfE1$yx6%c9ntI5tmz-Lxt;SyF7D|6YPvvW*gF!)`e#m31P*?tOJv>g zYhU@BX)e!E*<${$s>3({S|jkEyN%x|f&?{F(Uo;1tn>&~wUO*F&kYWPuRMRopiA}g10~d1Kv3ya z82$RTacW!p*n3%A|J^qc=;8c@m1TEC;6zZw7v3$tp8wj# z{tS{!FH}%u3;nih;J{#1|M%mvAdZFp=NA;FqG{)ek)_%H-H(?asA9_c3I(jFK*Ndg zC8&+0i!EsE{CBySpfXGLAyj#=sla_x+S(`Jo=`pbx*+twZ9RduB`cmTFY4`{corRd zS@i!A!zEfuL28f-0P!E>SdT(nB7nvQ&=2N-EEhz(eFEs%$Es2efe&ckLE;FJ%z-v7 zSyffg^t1I_>t+iMxVoL8C=n$}I>p|^bP$c}RzT_hmL?CL4n*k;*p^Q~Z?@UqOcj(j z0&xU0;JAWX^FS-sD4=c50_kT}l5^mq!9nNE35XP-bj8*{%2z1C2l?v2p@e8lpd8r8 zO-1rgaT#Li@_5L^-#zOGU&qA!-yMQDO+P;4|G)UKjC&1Y(}ao|fEzqjsPh%@^rk^w zcOY{L`pn9BNkbt@N6_&ud^k!zSm`FG?I#Ex$Z^euzx2bN^s4Fq^Z}+6*eF!;3Q){{ z=LCLy_%Xpo$Ga1fhTdB4w!xI8)nzp9wHC21A@m zAYZ^CSn#0hB?t~vhc=ydiy4VVzW?9mKyOWCVj^P=-*x6KV5xzJLkAaEDyU;w0_ z7q);`@k^2;cOn$SP;~8Lg$e$W?&iz5tr*LzF^b;Yi!*R>uab~BcfvJwRsy;L)LFmt z#%#;9I?G~cz53TxE2>ex#VO{|@ety=fK6qBst3T00|!9?pormt5)K~y1c2NXTn~uP z00D>WhxO?vs|0^f#>6Y;w3f%3mQ}6XE6+Cev2sRuI#TcDYdJB`87n;slnng`l*A6Ys@-xn9RPh!gh86Y=`A;p^w(%K=zbO25 zMMHVaoX#ux8Oc`x4oYR+ET#IP@C2-}T!J68LJ8P}Q*NE9v}z$C>65?t1sHpHaw4Rt z%MhM|D{q&=6qF7t!&-1+3N%t{a>EeR1Y}|BnFaP$;{)MB!S>eYtul!hF8@EvCelA| zKy*fs005$vQ7{uA3li3`j!LjKc0fG_b)SOG#z1WXVnd+L#SW-0h}UZXVh#ocMIgKs zzQFtKC&VOAmS?-2hew*nBWphtx%3b$YZ)%)nXhQxzh;rcSlEBa;&Rr@ABO zzJcP4jkgA*CQVGq_&Z0N>FJg?r*@f4yPp~c(kOr*cE}c4X&h~7{DCLO(CSnvNNRFH z3}|oY9%Maf3Av$J79a`_!DrUY1^I_t`g2115*%DO)pn>26XrZhJmf;`69;lv?17W_I`%kb|rEmxqYiQhj z4*29i>6#fdz5&$E)U79?ch_aqPizbl)6c3_=4?IaG>Kx`F&u`~?QYE%JEXh#>165Y z36T)1*;$`=+ZHhKs-l(YhssDex<+ZesAbp>P`jKRH*h&lwNu%0TIZolG~Em0O?WY8 zQ{~Z}IK#i$BSoq)7s&a~|B%xV_k{`!{$HqpA4qs>Wa@#w>MW&oi_UU_tCKN0prv*P z>x;9mm{(yOcJ@{^9&;`ER3>3L4%Vuen8pU9UP@8-#%|^0fu1}CO3iQut~7mJIGhQO zsj_$X{Fn&Z66Jdhcrb&vv`0&OZzVuv1l{n<({#xVK#BxS9x(w)E~w5O@bW{`Ke)*X zT+h=e*&#w7hz>G45m1H$PHQtUInaEu#eYa~J|h^DbUB!wok|r*o)~5{3XX|ruupXz zJ#ESqyePYlwBD9}*Mbi`Sc-i|U1aJ+lT8^+Qf3W*@k4am2l+&J)7KuHEiY5{fJiHQyx@!PUa7Q3r&4(_%VewuHo zH8RU9+891_Yeu&WM5HBp=&fpQRHDosvF8_P6fhM?21V~3ZfXV25_#(GMo)g)RNx_W5Yjs`kFQNBCLD{zw7f~hw))rl z^}84VL!beI{uWbK$pv^!5a}0GQ^u~_ z8gMDew+UML;K|zKC4uVIx&9j9kMIh^Zz}K3Yf`Y>OM26!9?}b?c`|%9vHO@K-HC8! zvjtCnKh+W>madVL(Xp;ZsMy{#!2g}tX$(%vaHaJ!pt-N)-ZhI?WG;XVrl#|16gNpt zN@Yf<2H0^<7++CK40}RKLsC*kvZq3}Ud|Q}oP5I7%a!#_o3TQ=K+vznlU}DdiG`R@ zF-iiZYgKn)ie96=^rE71RfWt zJo%@wABk-EpmylMPR8Fz-|T#qtMWuAbz2*(nUo?)NC`3({2dO`F*H)_ZI64)Ht$fU zPLJd&c**Z4Yp-gW99jaMn?FO|OLf}ErHVP4=_?HUcDrAH6mG5YB-FmUO{to9D~2Yl z)YXpU-D70rZlK?$hBWRkri;oTR0P~M$HTT*$nz$pxJ%Xpn8&rMlLdq#ydtvlhCVSH z10lPalp+moAw|(wlW~5V=h7?`{T!}|B?xSjDPo{5;X{`Ea3!hd@&PUsvYL4{9 zf7#9v{1jZG+=Pb{FulNz0zeUqXPG$JmHl+LS&JzB9(^k-jSnIU#EvOd9GJa%FX-tw z%XIe#Uz)bbydNnSf?=m_(G8QY!pg5tdIr3O(p_vdR>$jGU(a3&BlxQsgyOK8g`sPM za1VB6j=&UZ8JS^}{g|ws%9optS~1(lJ$c;@eprMMW@bX6X$8!cNNI`5hnnu9`vy_( zM~?VDdd@pnp1oIiUE^68IX;mZB5+gXTut7Rb$z)z>nhsDWO5YUO%@EER`vQeE1#=U z{Cqj|Nw4gu+$rz3XIfKT{kTsCeu5H76n`ncYmKU;=ZW|>WORwTX(sV0^3&6HrdQoX z?&-JhN50qbJYU)wAl?wi>K=WO`y>EKP?S(~^WC^Pi)c+5N1r%V}X;MI}l4Z7@s>= z-9@Hk(wjLv!+&Y|#y39FjWQrds3pQZ+S;%n=6PLSt1G4mIHMwZUvcFsaHrD+>OjjK zVd}XvyWX;?;Mdp>8P0{4`;rSFejO&0ov8xJYsuVrQDnRO`tgRc>9~s0hyt>!ihOQ5 zhnHK{&~%8<#76mlz0DL<;J4#bJ$A^lj!!{B(G1?x9!BhNGE$2y&ziSEV< zhf?B5d$wId3W)3fGS*yqWWpPMs%OLdWC{-D_@ToLKmqxVfFCmqfmDfc8P@XMbVPku zlA~`w7ZVMkap5P|@9V<}X5&>(_WFnHcfqfDt3b%RN{~=}y)0K}ty9enBX@$NycVkAm{v21(p578(WQW=bGK5gI830C}7F|%|z znH>CkM9>}TLobSGAh9(BnK`cVcVz3X5|SFHVhE+3Q@=^vfV#~DzUspgR>;VM3t(rP zm%o7^RZ&&&ExlpM_LPxmWX_oNIpj=%SlMzhVn*#}e9( z?vT0Pt+PXX-|h|>Z+PkgzeI7StWfZ-g7xh3UAh9vsp1Pt4v`{tlCa*>J7!4@^AN*w$Bpct0RUY$M zSIfhb7sz zkfH{@jVho3H{(#4;uK;)vuV!HDRx%&?5~RBr$sroxGnotz#`jCw4HBAUl_p}diH}%vCc$K1pbQ#)~RX6G# z>Jq+CbuE0=rvDR(1<`0z!{QCS#B4Y7DM9l744TSr?4RK}eE-yYdsKR4r2ScXB*uz;`UR- zlrFlORb#v7?Uu!&;_oqA*)W66tuU*0cd5BYczXpNxI2Y_vTSl{aN?{WmE(ol4bq7- zW7!`?GEC-}i8p<1hwP+{a$YE2_ZA)(A_(wn?E7lcs~2u?>PlPFXFuXJ+=cYBmu(ly z_^VSnZU=hHVzw4sAiSMuCZ&ZMNS5ZU%xCW<)=Rq=uChkpeeZ{TM7Okj3O)CqlC z`hbEG!OY=ua_CawF+s>LtE#v$oGlmw+?({p-Q4QR1St^H(Dpp_nAFV0i%D>C_f;!k zvNiFX9DI|h*xkhv8F4#Y5Y#x&h!%|Q_MglxKG%hTq%HdD$f*Urp9XZ3t$ ze!a#{hqER=;}8bcAu0l{6G;FV&HT4b`Sj9-W*y|oR8|KY%VhwOdcy6+O4&QRH#)Pc z$p^eQ2%FlzoC4U?^>>r=nE1SL)Fiz89VkxD^$Q4$er zHQDAK;E{N>LY>Kv=s9~*)lj)lJn!$Qmh@+ajys=hqJ}zp^i4OTsz}>0{`^4=zvZkV0APmf%kGD+^)<#vbJfMr_A^zX*Zt`2{76p4Fp!D z`Q8?2dXfgzn4NcqeL-|$L2Y2!`orUyix3Mqm z?l@O3x3O2Nmmbu340hy~k1~{HBN&DG>k2#5@A8g2aWifr$v116IA(bV^4q0_d?_=~ zC3?N)-J?P!dVf(m7Zu<9Kx_-p4^IH#%EjHKa{m8VYqC#hs9U->nQNxM!EnJ z*qy%X5^ypv;|>Zqk=`YyPd-kZAI(3mA>F+@naBH>6a2mVYXzS&(~Vh?8Z)Ey_Rl_! zo9twb`)Vp%3U$RE4Kn@AHYP`?p>8@#^(vT~OO_n7+?^>*QuIRkSd*h<4KuiOu}9d5 z?>*_Pp$OJ}nFd$Dx3fW%F{KPd1n8X2sN><4QY!d?LJaCs$_#H~{`pv?;^)ev+3Oz1 zRiMD=G8wDpfFt4M2lxt*o;-zu5}rQ@9%-A6G<(vz>Obkh55K;CUO+1~yD>NtOnPc& zGFWgqWp7E4@B;QoS!wTdt4f&3y{e0tW2Q(gWDafj4wZX^yv7nw5xCVe3_?{ZWz&Ee z5!41(ts6g4+(JCg=nlQIMO;lx3>I$fDq5}*!0S_i@wPwzV5d7n5Sd9LUZ ztoZjd=8Z?JkfUUQBy$A`>nV7+?a3Qi=TN8h6Jj8rUIE>O#MK!UkY*eHX`y*zRHar@ z;b&P|Y91g)Vh*mnp(yGJ`(dw6#`zq^ z7M7J#(nN{!k~N*d$wZ6GzaP`5;HreY04Dwke=C z%&HdV9L3(LuM8RRnVXVi7BGW%XzK4-b`L4FS@$`2-dCwRE&+|6cKy!!XY*&M%AMjW6CHfNi&>0KsI`nJ0DJWKo1 z%v@bzh4!=e8Nt_EMUHM9S&&|uz#NFGO}vD(v=X5pak$H(`40As>9c7()lEYambBMroGqPbMe4`gBb^R7+^LJkIad>X%T=d1;r4#BAZF z6F@PH9lrqwQOS=BT&Ewxz~yepZWSi|Q*c$36C6f0ze8Q!P+qTa)I5RA^}-N%sNHt# zLHlSFsGwicq+X(*!A?TJ{>7C|30%vPf66vrDg!9PT%s_u(F2eL>I~Q`t6u&gGrxOa zIv0}memMX2EuI&sB-`9oBn>t;8pQ;LBJV2ZB&6spqM;@d5TvLHf7mTVG+M?uBa`_S1E7tHzH;`?8xXa!;|u z9LWwhua*TfvM=XIto9rV)-2DnK`q+&x#LFo9!EOt#}hK7CtCm3xKv7Xsp44UF~&E? z_Kg_?dtJkcQ(A_mduQD|=N60{_AMi661z(ptGIbUtpx2E4`*R1^-ih&)vPTZ$pi0r z^o=)N!;g4VoVUKkmJKr7XICf529}cya(B@kf;0)JrKO#kQorg8Mn*vY!KG!*M^M&S zwyAyN5@*3{yoFuTb2?7eZ7&7#6@wi~EcZB&UP7dQNI!nPP!fXV2x1BEZtkWur$=|c zJdH7M*{m(CRPy#Ued*!sLlod|bN4xO&72ugS(DsDeY09)bF)2jl9h;?MfbV7-2bD0 zFS~oiYz`l#;zL2wZF`t|Abj5Pcw^-3Is!Ee zuDD<~|IP9yw*=0`WdbxBF;;AR&1_oLbls&+Cfq-V_B;1dw4ueRlEOaBLA-5M&cZA% zJ_D5OtHRjFGZw3q>@DRc|F|`GVY%1NrQI>uo6UMJAoZ+9@T+eR`kV0)C7FTOxl1HA z4`vQNIAmrAj5z9nj{`O9UTVa6Dr~{a4!T@IRSlnC-W*Jv!_Y_HU1vXnWabQ)j)@{`eJ+obSUtX;_ag(j^4?=Q4j{i*Y$!C@cRx< zr?XfE+?;Rh+#Ax|lZWIlq*IVgjaKih_WN2=>81s|Jo|U{lUyuAi3ErMRaf>s@KW+PuXJj()M9i947PFs=$jk+bQNMbXO7u&`0tNr^n9fYL z+X^0S+R5lWwyi4(Ckm0}v1MA#ceS6qd^DMuUhYZ$Qfe+$gE>*&wv)JY;}ItB!t@k= zqhnrEQYB{%b&J0Zw#aO63#8mHP!ivGmi-}9mNKbU_$zNoqt<#}`IotmOi`W_Ka;by zpKDgIP+DThzsjELoj(u@_eqjc*qHp_yV9e?z-`&Yb=HkQ8G59`^p)1cT~Mw{R3j$6k|+-=Yeal1W=C3NypmC|`-!;L9|s zF+Nyf+!J$CG4OU(ni#L5`yGjHIAEqO;oa^%ewU|sV5Q86mU4^JkdiJJMZNeyA-}gW z7S4h(8^TUso?P}SuX8{p4+m_%b$2{Ma`3c=>8<8df@6I*&xtQqSaH_04J*_4%2F4o zj#G?bQ)7CwnOk-oYV94KafR!(coiISOL0s@KBGsuJ}(Q>YXb-A4o0j{%_YnFW&6pp zTD%PBCOw7mk=0~Wil}G5JJ{n@rd6R&iF=^B3aKkl88!Wmc&I(}t&@#~DJ$b*GmCJz zd?K{%6TKIT<7W2n@=u2o$}ST@@W2yxInlOTO6W? zbD|=!d`j{im+)+FvIx2Q+*&m1MtcjtC&-y4C?T5BPk%=Zy!mi1uiJh7*Wjzd%UBc) z?AS1Mk;S9;y-`jTp;%?v&TEoddgbWQW&Y%&S|Qln%!c0!bsSm<3)vE%e^!GIHute! zWkHMhvFK-3iVDR-mI_5lhd!ce^Iw%FUwd726 z5hVwGClpVgh7l0Yi3DRfBk%9BPDaNWEF5B@GLU3^yt-7JCe|?tcf|RM5mG}G za<|EywF@ijFGz+o#=Tg#w@^TT`AW2zq*vkjpla}1Q_^UlS9V`7#ZPsya0;J?Vvn%1 zlWe7O!dbZGfMd$iqtCky_KHbXER9--Trca38rGv+ z7+0Y6gvKAbI5I9ZJDYvX57WbO(s$>-<=*FgnrI^oJ?o(FxtFRTij3nF2Q4~mx2^+0 zCuSix)=Y#C_@|B93BDOVHqL1EIWwJ^675>_<@8RL{P5ueY#=hkR9_ue`T9JVc*V?6 zdx-mfH1;@r5_4xhPU%JU#x3$vU+u73AU8J}PoJta$H(#$7bfu7_~(s8 zBmjoTE5B;%%n zaQ+4B>uH6vGo^diF3sPyWRwTFmAxNwXIaab;azhr3dX&y?YsZki6qQOTU}P_HHLR4 zCsEN}rF%IguV{0z>q*x7dOUGxYSE!-?L7(Xd8c}jdpJuSe-&K`T*)8gKldu(V(nXd z^s~4A)+!{Zbo*`%{)#qT3CX=V>iM8ZaAk(LF|R&NU`3d3Bd7l#C(oE0mRE`B&*xNq zzmc#TD|VD|j^IhX{VSMYO{=wVMZtzo!oFx)GwSzS>D*ZLCxePHROS9jt`L;@#T{L+(7w}xv(Dv+<96x<8 z=2lMr0vjvSbGnEl=kR);e@qL~ypEm~oGOmgIiwjM|Pe`avY8QOFE)!nyCZ8zJP5 zf{#5|^V?BW$9w~MTgmL+VfPck1sff?Y*E22^N@E_r=FOrlHw@aUNvgjZ)$_l2T$>K zboXCnmKa>S=G^m4qQ99wB@K>>eU>M(pCL{|_jeAS;UB?VMhJd_x0H#DI)7@OgNAY; zI7xMeO!3(DOIvdJ!s01vC*=$wN|v|v?x+)uYVakK_y&O^m0G*mbWnh!oX|t0SKD+4 z4$O?YLQ!ucP1^oEcztH@HZ+E+jG}f-DA42So)r3}oVruOi8>*6ZdXN)(E}v=Q=I6T zJRyA&ls&CPt8+q)j}OgB*0o-5)sj>hw*fiI6|nR!i+n6P?uUL&uk>|JDoV7TVL3s_ z@cO4YZ3(A+0hKuKK>b1>Vo9sjSxEiDZAk2qHz0k`1?({s_3k_&FSx#dR^^d=4dSAr zA}2tAV+}}e=0}7V!ML*!FzhMPY#OwtEJ0H@Hx}78Kdg{HCs2?#AV%3!i`r+wtVu?H zPLST-@+Ko#d~2K)wm%F%dk^OD+=^0ipxXB zz=}QW(u^2hsgOqDwGBj#NF8-xBkUH`Uj2kIH*L~P`SrLzhksH0&qltmPlBo6UsZLd zK=1(D3;H_ZUR)j1jA%7iIBr{)n7p1Fn{3SV zCQi-xoexfEyXgS!i1ozh@=xFkY|%j2VL2yeU%>?jSrxQ{wV7(&E%g&GHLpiNL;xb7 zrp@BcYOa>+%h#{gM=N6L^=^bnETB>;SGfpT)#5Mmt2b|+0){I6Z`&os#h<}2E~G|K z&E$9QKB~gSMV#HMqsKuQ*EI9m1GPRC}Q~4`p~{94n}w6FbeiR;*MZko>vQ=uV+{Ed*6A>Ga+JN;gJsiK~st9_N(qyop{gA2n1ol5+f08IEcH`q1qH_MwMKm*m0l0Tq%@P0Xs!@C#Y)bBo%8h5Z?&|XtWTA5*D?$Z zYMc&Vg{X%eEKprBTjI~93z0Erugis-r!_KnFe+P*1o3UAW8Wa7XS^YsQ1@N5P}msj zEK14 zmqbgrKVS7La4DA1qugnbycp@a8b^2|A3xmu1lMBgbN-tCE5`WnOP7#pHcMs$zCUw! z`QqF3S(e#B$!{IpCQoqXZkxAMM7+-(33p9b@bH%4EWDPo$4C9)_AB9d1y*d)7hHHs z63r*mWtC|vqrJ7OIo6-#NvibQd|;^RbYmqI&K79%GkK2O5(Y;G^m~_Y=9!wfioOyR zoMzq5NmJ!C_IKgCmCdiPNEXrv;IWh^5@EMx(a%Wyl`Ie15C3y9|CFsbDBX%!9^SqY z4crH`{2~~+JO3uAB*v`M#4elqE8GlNdJX^qRjPC%Hy9`1fk(#exj(7->nQa>55>mMmZNP)E5Kvok~>0kiHHDV_(v^XsM+ zT9nq=U4`hN;)y;THf?$0XJVyUEDO(70!I2n<@ z*J+^!HfBG=046eK>OKn6)Tq3++(VdOsoZ`F_A(&;jpOg@eA zdMLilf%6;pQIh+yOX15^`9u$tetoWbf?|EtC%x|5c@OeyH1|{T!W+U0I)tTc7QW!& zj<4}hNJ7Tu^oa(dtVeS~xAA)#r|!o*B#!xYATODm^!@5An@yhB@nM?chx^>z+PbBv z%W0O6-E(r?Y~M>VbLqD)2Bud?o!(H)j7$3>5@E8Vh1=B0GTc#=KqrCLW@Mz_nVHZS z=O~xjiwmdLHZTwZ&}=Nb!DaEz?{qCKAO@9L8uxQCbi8~VmrFM$f1k|R3O?GAEYUrjH#C=KbaxWcsPnP@L>Ck&mzJbyqtc9~op29n zx?lC+0`*k^Mg>XdM_3O zsz$y&)Fq}laH7V%^qD585s0GHYv*`fo@}4=*+LEOtx{JOU?ThXoUwy~#W55u5Ia5* zUrLGI%U^HADYc|My~Uf50M-KHKqsd}J(7}*C%_pQi;4Ro))O|kf5jX7U0o~4IIr3N z#!m5bX?l$*%|?+^X(aP&qVezpQOWm->DffObUuysj|nAiIhJE^qjCIw9UlXOBHBR5 z1-PGn%M&{6On8GS-S=s-+MYdsu8inN0HfkTv2GA4pUX>f@|2;Dj^DVD3;BbSJ6Vkd z_F08ftO6#tbl7KVhi!Z9ciSUSurTY+Y1vta@T#PL&VZaC&!`+{-c&w3hGW!bf={tn zlj4%>Ha}#8?!vh%YU@$LSDuH?JfB~;Ij4 zX#;~2O1!mm3$nOkX-ISXIc)<EyVdhah1V>Q-u1ZLw)|6I@(c;nv?y ztD81gNs*x3;^nj__9`knH3rF?$N!Wb%!l9WcNo>yeOT7xDnOxCu50)O(OPYSc|%!i zQ-kVQCR?T7nV8w(%kaR*tN(O}k!~>vCG^(xb0yWrkgPX|0()+_uGh#!7`UC5zrv9V zErUqI>}O-lqkS7AwH{B{bpQRU@%-yowd|Rxl8t5nG$l+yrwrzc3Ex@>Evg@22 zXrMEMcnr)h1vDS2wY60n*y|{lT);#_&d&Y}FajzUYTpBtYl}AfY8U==dtg3NKtMnv zV3VS^H|Km!Pmh808@^y+MyC94s!x+p)Y+5=7o{UzQL>GbPCwqNKQ?vJ>eyr#P z{p~l|AH@5DF@9?X=Ux=La%DIly^nd^=r=5$E0F%-NUrTPW)qE*1V%?U?(G$EXv-@*oeZtNE~<&4#hLKK3d9&o1Ac@ zbV2ny0*5<%bL#Kra9M_GcM?KJ&XQNZP*wHbD<3xP8g5voN-pEQ zX3?!1+lx;%5~AndTCCsxY866v*!8}HlfOJE=;3R|4SYKIwo#}sYIz3jL5o1+JRJ6D z3_X|7+^5}^$2&R;G>yp8MM-jVl{oHe&H!SpLS+W!YCBrs2dE)B)M}mH07k0^0s;b% zrUQgRt8*~`N9u2{e5v(A57~n^qZM4Iew=tcju|0^p6+iHoqNa;9Mc336K7j|_Au#V zC=O7%;FPtcLc2d6pq=7g)K;Jc-_Ksh*|7`pG4af@rt5IF5y^aJ4FCADIzI?&+|v~P zi+|^@S>v?p$b@^--_g@TLwsHmJ?qWjcts@7pYR3$XOw4=2erO*5ucGK44=gq-OJdQ z)4Ou+ICnQ@!_eH)w^m^-NtIp|b#f{}oix1auaDu5dy^{M){pWPPuW%On^j2mXZ`{= z$4YV+Z<8X!R+!Dalywq{vXip!eneCudIzKm@?CF{*^*4Dhh=xdn1=+Cdl z9nhj=N%h6kQ&W+h7oxVG1(!U2tH5);>k2E@9kWBfRF|J7c8&SB2yEHPQMkrSj?@e)vdZMnbrqc@}cs zS3Jt3`+|?IKev8ybalvJUu@45+# zopczrfBn%Uu()m)U4gY8=CQ-03)w}7KSq8HUn7V0@Z*z(d|Doz)?nL?o}?=THtiTk z>a9cD-}P)kWGZ#$XMps!zUD>4E4?*`*S9n&P_BHS2+tMWWKYsi`CW_mB8cQ%JHrK!j`=n|KQRs z#$UpZ7~4%E3;C|*qm}_11u}||`o3ng5u9Be`QUVx`^0$1xn#>lPF>-d|2S7z6sUG{ zzkCTksF%HAsQ!fGTU*evV_D27+%0+f_r``Tyq#7SevsuIiz;qs`in;N15Ean zEImME+It#e)M=b0f1?-7I+G>ItC&{gcVpH?JdO9W(n@#q3lbnTwY#mDEvPCps^4 zO^?ejR@!Kc%v#uM~5dHcGgLH<69;n_&4`^^zqL1wFP`KN^_r;bUq zSrZV-$?L*4)Vn{B>=my3Kx4yw5f~c$U12eCZc04)4w!s!EjwhMiyGV~Xa(^rp$rd5 zRgD_U+0J(V$X8??N)n7y{Ltu?UrJ?-wWFirI1Z@e?rb9{5TlfM;?jlm~P- z)B--gN7&dMT@ln8(RKfK!Z9lP-Z0v9PoaKrvW{F03mKe<3A^qlZtbooe5sa88GD8R zwsmxjD?ISh+mGn1Mh~*cY;Jl?c*UBFRrk(PBu5*iZ@ruH%rYJ?Q+S_lZ+1o6oK3u9 z(Fdj9(t$zg#S*nnlSXxdxV{3wI+-nDQ&Rg_NV{8jC+(kC|5)@Txqght0|47#bx8AW zTTsvIeue|#R5UHB_+2o$EgT8@+Hr;HQ}qzqH9m4Mnk3K3CE}h@qFWe_3f!C1#q{Z~ zd$~66dlSEq-79g^i&H_(6?~@R=^p3i9!I0>aFL2fEFH-oXt*v54FC4qgcP1XGVF_W z8X;(zXaG9N^KYl@&?`kh(TevkeYNkXuYocaG6wJ3zA-OA189+_RE0n7;pvlc>K__y zxJ#+U;E-Q2Gh+h^vt~ftGmx!F55}Tnrlh1qd8m^=P%Sm~2Ep7?*|=s47>>O7C5^H1 zf9}x}Jh!ivi#~YC`)phkqI&?5B)HF4sw|B8!{AwLB?s93Pcwgjbd0#z4h)7r|$zfBA9wuZM3@%e)(a3wmPQ`Jh_LG&-o_mL>Wv zhVl>m*fGUG5N(Lp6&3VV{_t(Q{=mjRXJNw~xB+0(+8L?)dEI5n!qf%b%AXs3@II3G zAK{O(DmSl$5Ir_a|B~^gWwKl7k@xZVgL*!F0Jw(CBl8mFvS`aFXtz-+=JjrqWK& zk0R#Tt|UGs)+ww*!d9eu(az>M%d%WI!-FdfO(*I*`p007n(-I3Nb!{2`gvWBx7 z)DTGs^NlNhrRGLxXj*mAVLaB1wi=&fM^Nf4yL&K66dJ5`JuhY;i*x4GOb|q_C(iE3 z3+ClXs!OTU9zye)G;aqi+h0L9<7)M z7iIZNXVozQlOW~2(`_~m^~HDr+P`;wE3(>I^O=p@gZ=cI&-dTAOM(i-&|hzhrV>IuFh z$2+M8b*v)i?fJwWr2;MAuas6>lvijek+dG?Uakm98I__l@M^akxk^!<*}VYhF|b_k z0~*|$3%QqG-Ik*CTXAqpmD-tPHSXN*<-LwJ!-2y4Ti3>*<8;Xppd8wT5pNJ*rxxAq zL>_s|Pa=!Op(NL!pi$B1N89QKz^%p?7Gtw>x+=LwzjQTBP&*r)jc5yAbqi!F;>Q+_ zc_o^hC6fMvm1#N74mr%udOqKEJsFdh6gaLx}84~+p6bJbaJKmZbw8z3Nm z4l|7$wg6nnPl<@W0~cNgOhJqVGnoQIYLWhnR##r=wVXbV{*rKTq3bJKCa!(^P9xpW z;fiZAD5ji`Y+i8}1V!X1l} z4dQu+$!i~oitQjfZ75)Cq#m1#Om<_^442*-rKQj|HG*Y0j47y*07*VZ*L_xpk08H7>-{;@Uy2V?(w`Ry4# zzS2^8vfB}wuAZL79rKQ=vMQ?ym@^v-qR;D_o3Ny$5HByJaS~n#6?+zh@oN1!v0{rc zUTk%fS8tnXmhrPjBgd@)=A|_p&f+H*P8Dd$`?j_B%m^74bKbl7JhXQDox{Rpne+tN z_B{5DW9af92Vs*a6#1%9s{)ZfQhg`UdN|K-C>lwsq*BS!9g7CP~G?7vVMK2AMV}3@f^T%SYK5m_vILYwY|o_@4>hD z)(rv2rCh@ra6Xmg%qFA|YaZ5Me?Fo9%_`#01=r-#N&A#BiqeZR?Y2)CQ{!-Lw@4z# z3WfJtB35XD3?QExV*dx=hPsB>R6psm)nd`jL-yhWFDLn+-m+iqp`&>;U=p$ctS5hrk zNr}{)i__k-eYjyN+5D~ccc#2lN7DbGA|QNeroi!8uXtX>-RAKqP!5@YC#S8D)$mTN zKkCgY!F?T zY!eYmr9hKH`0JS5=SO<=8mSv$&taNX8m{u8dSNK-d3_yP>$~&6VrooSs{Ak`P9e$i zA0M^*FL@Pa8x)*v6osPsjZ;^ph?onlk;hs!$Yh)M@We3#Pu}Cr%QWR8)$|9ob3%cK zyE)~4{>*ynl_5`%JriRNEy4+jrIodrmZ`4KSSZ^o;Ax&dL(#G~09~1zu$)K948c<7 z8p4*k&y_ne3OQ=-HBbFnCGPY_Mo4mKD1nZc)C=t@I7yn-xmk;23E?=I` zQ*wC(-Wf78w&T$w*W6YDEApmM*8EtNXaR&*81)(7Ha0Y!Is8Be(MYpioC14@Oc_8sL|^ zl+x&2Jgvf7PBm3o{j$_EPs*w{P1teCY4Eq!jNFX!} z4Z$=u#F-?X!boPOkOt}Z6-=OEPNrc4bl%M0G3lMm{L!587cTH7Ehhw1rio||Cliwt z-^FC5MxvwLyi(X8NNEPybqVYQa2+MH!Ny`qo2fd@UBRZ?6t#o$fv(fp2cA5(VRkCs zC%l^(xW2*~ow$`h)~}y(kHEF}kVuS{Pd>}@6PP4 zqRR62X>Da7iqNQahrR(^tuswD+y{7v?{Xqpj6&V5sEq0okQV03>A=S@g2U0M|DW!z zI~>dYZ(mkcWTXfgA>GkK$;?bcAr&fng*;@02c_;r$S9-;k(L>KYjZo(2tS2rN7VLNL>o$yx6cudSCLsIf~+R(`^y9qY!?1#eYrBmS&JvOezozKVAG{U(9GJQZxUjN>EX7VN2*lOrptlzG(B4t5sHjIMQ3{YMbtn zs$zF1iL{OSn5aD%S3y3$Z{XyN+ZQFhXJoJq+?2MXZm+eHe+#$u`Hw9hL%fSVOepog zW?4`-CR<_a&NMrD*4ALmhIcnt>uReoEs-7DFo^QFNtq(o{f(`pWE)l zJJm%TnkzJBoZNtyp4j9K4&?)9{oFDsvvnRb)oGt7uZuN4P43;$|8FmLUb#ma8gCZE ziobUzX>SiV_e;7k=`5mTKw2)g!`_~j8{tGx^Mq_3Jx1(3^YhR8L*|C6^NWyr{Mwp^ z|JkLa92zOLPE z<7j8WS&fMbCeQ(khVn>Hq^G1rfuC#D`t^K4)hm%hH=dK&JPCfTyC1gjRM!6ec}&89 zPs%_K2Ny|KlxfU1*-Fj6Qn!SpUFL8(7NyFDJXA`3f$brb= zfnA!PmqMa`5jxif@jT0y?|XAE@en^BKfgD4RV9(1s;6;3Nl|HR+oc`z__5*gsvI}* zCYQ%)8vC}#OA!&E+luPy+JB|PjfJ3r9>{y-`SnpPeaVaTl0ylE-NGRhT;9E`EY8|{ zGuET~b?vFJT05^Ul?@yAVw_s zrD14zxJGZRqVB!>?#fXRa-t7hf%r)#(!EUZ;p$zbu@RE1S2rF`2Tch0a+WaY$R_UK z6%~cOyV@U(26J>@$0xT4${%WK2bA2qOb8y1Fg+#DS603dwP?>VYnhq<{;^R^W4uxM zl-r%|Vi$Vvwx!=b;bOOcrL@UlzC|yq!wrwehK5FMa@sYrLTxv z2nR5$U6)A{mDO@^5kmNGxWSLUpQd8Z_#psK3MwYF! zk|mktY`|??1kE8u$A{5A(rJ_)nN+Yu%+1fU6Aqx|sdHE6lnAW|`kW)E>Xku=lj0uq z>SIQjd_5~$6_=!7A5wp3Xvpb`i7YeW4FVcBRrSoAgv7*QuOlYyBsUom0e(5Q+zvO( zdmvAzs|2lCvxa*?bMiiu;fwfvU6n^}n#e9CPuu~O5b2*MU*_H|Nx-SI#igZL-wJ-- z*5;v^fjNXhIWcu*LK^iZQOp4y7v z43Z6QJ}P0~rTw*p5XS>4!-W%5!!44svRPY?N7i04))Gfiyyn~AYY97u=W>r(mb!(3 zfQXSm`{B(C$5lCQCr_R<$?cH7C8p%cL%ha^{RgjI{KJ-;cn4uzlGnVj@9J4Cj*N^q ztbbNzB6mSb*khxY1c{bH6WD?jxtF8z3K9Lv8 zA3nrBy}rzOcZ$lz2@A$-wS3Tw?-WC1WTe81m{zyacRi)KZ5}~07N%hmc(TB48MI#Ki)a^ zxvY=V@d^l$jc3PtO4pWH$S+>RH1na)_Il%XQ!|KEEDvcYf2kd#`ZWzpTPdt@A@JyxwlMz z;}(p8yT7l8>lArmd)NQTOtS4Kty3P0=5x-Ju|s!E*TP~2HGc5MjaS7RMuwQ;6A~B* z(5uN8HE!!ySqe|29P!VUZJv2fTB3;11iqYu7d#o0a|_Ud2F}TyoVaroqOu+h!+vJr)Ydn?O^( z;#E_ZEU}aRp&^prUz1Z)G{IPlGz@|YJ3n7x)PDy(!;0agh)uO1EraxmEkTf^hIjp< zg{PeL*bFak5<1)t9gWlqrl%-@2VszEzN=@c@nc< zAkdD0pP-jTb4~ff*Vm7B7C;<_EzT#a^6+i zR@i!ZDM%Sm{Im~Tz+!qIKj_CQlXi?+Ye$%z+bADtp37pV-5$}Da?r`it|5XK@kdbW zm?}z`f0keN-u%iPH0t1@)K&mf+)JMp`Qe^$Mcr5HR8%@=?xv@U9iJaw4j;v`B4!a} zup1!BNGo#Omh{BA3LN-l9zw!HoG^jYkj>M%Vj?an$%vpeIXztu=T`)Bbt!4-ZjWbZ zqngoRp!`z3ezl^0z#+>WjeA4)@ogeU3IB{ejJkGy>y+weG1Zib>e&Q zr?Ji@5F!kKRHgfv;PRndO3=jeZ^wrp3DBeCq~F7LroO5Pppb`UKZvU{!p9jaVBERC zL_%UpIWRC#a8rN+h`_=S4eUXY2mErY&4bgCY?l~!0%xRZ1@|1b%)Dd=NmYW{D0V^w z{VIQLcuXZ~h>$7AMS-X04N2mj%Y}pvEmU9Y=h|{~v%rimWvwz{U?5uaEmi(nxgw4(xOmZGQT3%Tpv?gRs>+_PqFY;A`O0g+4o19;h#A1@$`s zdAke@Y7rs`0{G2mtcbVr6xt^ch%5@6b`A{-v&UU5I~25Yk02*F7ybS=dhlg_nt?!L zP)|pPI=19e)|qQJZs@V~e{w<0BdC*;Ui8})6{-ZB7Z;{H{~%t&E1i1a;JyDlXea9ELh*tPI z`uZ{WGb`rE72ywurE*vwC zDvl1Uww>VV@9#J6+#u6Lx32KgWoBVvCkANS!~{U&uNM(Aee7wf8Y2}4;NE+u<8Tew z9Xb9L-vHSlv@ufW=HI6G&hLq^l2lRI2h7GskWkUTb2lt~0K~XFMC#>1_YF<-V^htm z8L97JTiJB85+l{%ze|Aa8L3H%j;G<`aHO|vswoh~lqDj7aDO41CCSjP#6b`|MwgU$ zab?RA?ja~iZrD(KXGvVs@V)Qfj?y2P=j8q(=q_VQP z(xK-K{%wkXO}umqGSfG^ns<%gG2~b+P_W1Hue!RrKhUjx{pqslFYxSMU8}(;=sWhp zh~U|AeGkUFA`tt#deUKp0Lv0SF$i89z@?U%Spa;J>gpCi7u`=kuD%>lXRlo*WZIUd z6>NcCe&@_yK~_rQqr>x%z;G$IJ=hZpVqNyc9nv7SImbhT=nQ>Y*|k;Y+O=z@pIiKq zA0K;s!ziTzepMdaA7GH* zWK))+^|Y%E%C>!BC2h&HJdVRNLL5di+a8lDc0GAoS9pY8!xaTA}@B zMfWa+BDgWWS;rLJjolbiXz^R}5uHhmam!?7`wy?G9UvtkDJ7-vs3DA4E0 zd2ME{X%NBpW^mQ-vOtdbGQNLm1pkS(gS}?63_Li8dCZAG6)+Z;Z6yq?K8$mDhAD-X zq05B)gva`>|CVQ$HE9L5+RTd<^~@~Pd|%MEstyu>Cdv|Z|MC67e^3FSvqYpMLbE1L zA@oX434|G3#n=aZed3O=`}cKlKkFTex-LL4lMR#0C114>pP@hEv;RjGHd3`J7T?em zBV(9E#mA6%8Kx+6H8(c{Xjb@+@Ii4CfS;iX4hxwo%tabI$yv%>KwfK~g!AW@Vo-V! zL5Agi$gbSBEZCFHBPts6?wxDZ+(1;*vu6dd^-|<}*?*P{d=kDc%+imGLBlecKK40? zxDP^5B+f|8SO~JaCuMMZ!P^=AR6#W#DMWlSbruncwy6%X2TV~&HZ-1%jEXuXWsszH zM8;5tpOk*q-?ptDWscg~T0sRn0(j+O%CTYO;X_efZDidjYX$P1Wr)i1on;IR46dyi z4C^;+kZ4)Y$i(D&&!i@I(Se5GbXv_`Fg;5MSpg# z+w`e*v$WP*aL&fU-3{VF0Ucvw?ps!sA#FEzIbvB5REWniu zb4f`N2*Euspg7#^{KB%r>&j+_9TyIgGZ|#`#-pL!qUxq6|2j#YIK!_Kr;*1hw85Dy zKk2ba$;b@g^c$ht8;v)MqbdPuKoP87%BML1M`j{gSmo)VI?w{C9FE3_7(sw``L%Bf zasa#M*9^E@1UOi3>`Lb*w(rO#{h zva+)EDg$9v;l)tw@@y;CSJBbg2{cK;Im@J@JGDkhsiYwtzMp}buNvqb#_}L-q`P>v zX->%V3<$VajyRIL=)12sK-QZ&#ARjKkS8=^@|J->$^-?Z64(%f zuevC*^|P?5!CF=^W)qX6Bb~Mg;EkoYuMSd+o$1j;%0w!IO>_1nY)g6 zx|Npo37iSpS^iu`Y~oe524ROBDpCFP+}s>r`f4DyWl=1ISBreP>1ZH!*>v=0`E5uB zU|QeD$0wN{{PlBZS)YdAsMCTb)(+u?r@i=Q<;SH#=Pm)nThRWt(jWaznS?gKo4eA> zG-CB`%Q=Eqo}j|pBLU;SuEk3F!gW7uxi6*GtYODmeSlh1kyA4tP*l&0%eCc7NZ_Jg@O1Zffh@VW@XaAK$vau5N~&ZNv4lUM7HbNYwA7XN4o9fUe6OkVrwOkS^{pSQPn#M!frAis_d4P^#;6-CL`KROI19}5EA z;wewEw-G4++lP`c6(yFW4KCaekB-4^^>RA~kTN3g+6=#;fJ)()$@(Rv-1|5m2(mET zlV@4Z4d1&B_ctys?j#B8~U6tnWm`kB!Be!9yDlcFhwZm=8;lu0n^z=|m4DYl# z%Eh{5WywxeffLbiQRICC(LERJDkLPN6_E*F8_P}xTMGZrr5|Txzw0LKRunP^c@y;# u)+C|3;8d#ngyZ& literal 0 HcmV?d00001 diff --git a/example/gpu_sft.csv b/example/gpu_sft.csv new file mode 100644 index 0000000..638d630 --- /dev/null +++ b/example/gpu_sft.csv @@ -0,0 +1,51 @@ +step,train/loss,train/grad_norm,train/learning_rate,train/epoch,train/train_runtime,train/train_samples_per_second,train/train_steps_per_second,train/total_flos,train/train_loss +2,1.1492999792099,0.6216375231742859,1.9999999949504854e-06,0.0004617871018126607,,,,, +4,1.0979000329971313,0.681877851486206,3.999999989900971e-06,0.0009235742036253214,,,,, +6,1.1269999742507935,0.784303605556488,6.000000212225132e-06,0.001385361305437982,,,,, +8,1.0542000532150269,0.8737029433250427,7.999999979801942e-06,0.0018471484072506428,,,,, +10,1.2440999746322632,0.7068291902542114,9.999999747378752e-06,0.0023089356254786253,,,,, +12,1.2925000190734863,0.6821666955947876,1.2000000424450263e-05,0.002770722610875964,,,,, +14,1.0843000411987305,0.525643527507782,1.4000000192027073e-05,0.0032325098291039467,,,,, +16,1.0961999893188477,0.43757057189941406,1.5999999959603883e-05,0.0036942968145012856,,,,, +18,1.0614999532699585,0.46141618490219116,1.8000000636675395e-05,0.004156084265559912,,,,, +20,1.332900047302246,0.715879499912262,1.9999999494757503e-05,0.004617871250957251,,,,, +22,1.2070000171661377,0.5926885008811951,1.996917308133561e-05,0.0050796582363545895,,,,, +24,1.2043999433517456,0.5833240747451782,1.9876883015967906e-05,0.005541445221751928,,,,, +26,1.0740000009536743,0.44734400510787964,1.9723698642337695e-05,0.0060032326728105545,,,,, +28,1.1162999868392944,0.3701137900352478,1.9510565834934823e-05,0.006465019658207893,,,,, +30,1.0454000234603882,0.43832680583000183,1.9238796085119247e-05,0.006926806643605232,,,,, +32,1.124899983406067,0.4591037631034851,1.8910064682131633e-05,0.007388593629002571,,,,, +34,1.0686999559402466,0.3873400390148163,1.8526401618146338e-05,0.00785038061439991,,,,, +36,1.0291999578475952,0.40313437581062317,1.8090169760398567e-05,0.008312168531119823,,,,, +38,1.1052000522613525,0.3735405504703522,1.7604059394216165e-05,0.008773955516517162,,,,, +40,1.1555999517440796,0.3818407654762268,1.7071068214136176e-05,0.009235742501914501,,,,, +42,1.0235999822616577,0.4255191683769226,1.6494481315021403e-05,0.00969752948731184,,,,, +44,1.0364999771118164,0.4794503152370453,1.5877853002166376e-05,0.010159316472709179,,,,, +46,1.1344000101089478,0.37273937463760376,1.5224985872919206e-05,0.010621103458106518,,,,, +48,1.0866999626159668,0.417492538690567,1.453990535082994e-05,0.011082890443503857,,,,, +50,1.1038000583648682,0.35408055782318115,1.3826834219798911e-05,0.01154467836022377,,,,, +52,1.1478999853134155,0.3930828273296356,1.3090169886709191e-05,0.012006465345621109,,,,, +54,1.1858999729156494,0.3965947926044464,1.2334453458606731e-05,0.012468252331018448,,,,, +56,1.0096999406814575,0.3860221207141876,1.1564344276848715e-05,0.012930039316415787,,,,, +58,1.114799976348877,0.44393691420555115,1.0784590813273098e-05,0.013391826301813126,,,,, +60,1.079300045967102,0.3605058789253235,9.999999747378752e-06,0.013853613287210464,,,,, +62,1.1766999959945679,0.40689122676849365,9.215408681484405e-06,0.014315400272607803,,,,, +64,1.1075999736785889,0.4002344310283661,8.435655217908788e-06,0.014777187258005142,,,,, +66,1.1866999864578247,0.46947163343429565,7.665546036150772e-06,0.015238975174725056,,,,, +68,1.0311000347137451,0.3296957314014435,6.909830062795663e-06,0.01570076122879982,,,,, +70,1.1088999509811401,0.33858785033226013,6.173165729705943e-06,0.01616254821419716,,,,, +72,1.0720000267028809,0.3967427909374237,5.460095053422265e-06,0.016624337062239647,,,,, +74,1.1460000276565552,0.41202062368392944,4.7750145313329995e-06,0.017086124047636986,,,,, +76,1.0425000190734863,0.38334518671035767,4.1221474020858295e-06,0.017547911033034325,,,,, +78,0.9154000282287598,0.40649303793907166,3.505519543978153e-06,0.018009698018431664,,,,, +80,1.1110999584197998,0.35371580719947815,2.9289321901160292e-06,0.018471485003829002,,,,, +82,1.1672999858856201,0.3381657302379608,2.3959403279150138e-06,0.01893327198922634,,,,, +84,1.2374000549316406,0.3815234303474426,1.909829961732612e-06,0.01939505897462368,,,,, +86,1.2151000499725342,0.38446080684661865,1.4735983313585166e-06,0.01985684596002102,,,,, +88,1.163100004196167,0.40419140458106995,1.0899348126258701e-06,0.020318632945418358,,,,, +90,1.1883000135421753,0.4011874198913574,7.612046601934708e-07,0.020780419930815697,,,,, +92,1.1526999473571777,0.3836020231246948,4.894348535344761e-07,0.021242206916213036,,,,, +94,1.15339994430542,0.452364057302475,2.7630079557638965e-07,0.021703993901610374,,,,, +96,1.062000036239624,0.3502688705921173,1.2311659247643547e-07,0.022165780887007713,,,,, +98,1.0271999835968018,0.4022065997123718,3.0826662111849146e-08,0.022627567872405052,,,,, +100,1.0283000469207764,0.38241174817085266,0.0,0.02308935672044754,183.9481964111328,8.697999954223633,0.5440000295639038,1862467846144.0,1.1177252531051636 diff --git a/example/gpu_sft_loss.png b/example/gpu_sft_loss.png new file mode 100644 index 0000000000000000000000000000000000000000..23adde897acc483c75f15f1ff5fe36dd44e6b6aa GIT binary patch literal 69683 zcmdSBbySpX_%1qvptOjT5{e?-2q*)H5|TqHNTYO@Ff<5AgUAR{gMxr4lG5GM(%mA` zjCAkk9rWAhcm6(SueIOxeap4RH=elTy080s3{g>*B_X^@2!p{$!C>dnFc{9* zMSSo}#S?il@W(x8X)R}UJ5y&@BS#aMl998$wVkuI#k1=!CXP-PcD4fCe7xLuIj@^L zJKHC$=GJQ|MTL|nQJ#~V?s4qkRpEFD8RfC;_!Q`YQHIPXw6K|~V4=ld)WBGS~FW&#RACPul>SsGT7^-RJDj9$2)GbCG5fQ=4&i-=5T1HNe z;Oez&4z0{_)3G|Xb-CW#9CY`b&D(|!e+kzR_}B5S`cQ@8&Ckzk6et-02Oxz-#`@EJ(Z5I;jAX&M?DM4W$LIsrqRi41dZu&Z5l;fM-+a06MTPfSWWTkD5A{LP}T@%hwVvoMYG4JM}4?AmZ>vvBP1 zU2IXFI8T$PcL`O(+sZ!bbORFs&IprWdJ z4t6o9;rG(g10NrsMkk-+WrMkbcJ77<;YpgIJKH<;L_E_i5+@FgE!INUz?K>nj!Zk> z>*?!@c|F~qaL3B@g}16|531Y3&aP#8`h7vQaqHzPl$6q*%1%z3**lP}k3?Z@uzslK zWc!n3rrYMPnW>Z2Df@{RLmxhTkVJOOI~wh-O@P&)VDMPsTNy8N8XP)l7OwV-^89(T zel(I^)PA#5_&KKD``=~7UrwE#?6w`8o~)lXkGDf^<4A_}jJs3lDxae{Mpa#1It-@1{(-K29bW15OQMYkJ5SuP61%8(%mO+KXbHwTp9)`%(5gi8QSs=9M|rJ0 zr-(O3(U1n=`s_6@=DBT}f=N^0;^Nwu>9Vr1N!r+O4Sv-Lint~`d$62Kjq|d{dZYl4 zOyY=l(sNVkTXAvyqB9c>acX5H9VI2@r<|PTr)J54179@fZ0tOA`Z;-1x`Toky=f5Q z`?q#h$AWTlxJz}pwZ1;l(9n3Fnu_DJ(A5-5$&u%M?5>NP31sErL2Mb2k&{QSVZ%$C z1LPm%7tf=%DLI}@&jc~bNJ`d)8!T6Db}0;3Q*rACH-%BpP1pIiJx@+fwxCCbYOtH< zYng71_x7rxVq#l{K=>=E{h-{b+fo>^J&MSmQ z-Dt7lLbGlu-Uib`qq*jYz2CADp{`ZSdadJ@gLggmc)fQDQ`nZ7nVFH(eiv@bFXH&Y zz}7O(a2V_p3MwUvAo>Ky9a`w2^}5}5Gm1g%Gb*hM%qUUFroJ~_A#fXwW`GQyr*Z@5 z1mqX?Ltr#D1v@ijJ~X@IPEoIoXo&G0_BMUUl`ObhNGN!b6l&4AXilL-Z`4c6OzaWG8F*+J*aJ-~h zUuI)t1DuH(Ylbi;e6Oy1%bd)=&%=s}ijdtNlm1YY+QUToWWyO2i-c^)KcR{eI!!qZ zs_EL}cv`-eOkTxwLzYEwmgA-Q3pqx|#?nmnTW!ncsm76W5eK%ck|&2Xr`Ntb`C30z zy>llC?&jFs?J*Z2Y`;GrmzJK6XJTR^;5`ZUI72M6pun*?H!ch9bK++56~u~lpVPye z=+jn&w;BQbTrJ4h!sJM8kbwXE`7^z_f1AN)XT;2*^UlyT$ak_%P6AN03M8T?RQh?- zi}9;=k^Iw&)#9;@r7yY920lO5DljI9<$U_Gy!_rwk~wWK82e%*6zWsysJIH*dNqnQ z3dUCVqE7e7JWn=LqV|yGu{Vx4B~ArwfzQA(jTCz|r-u@iM}OKyypQ(2b&Gl*iFkv6 zINK$gFvwopF9?Z<9L8))*l4X0V%t~H z?n_xA&kIMw(-q=oQ^UCH52sGObzMeG+O{Ed;km3c^}x(*p)sLhrncMX#A&87HI*q_ zuYyWb*8$JAeDQWiqA+sn4C zBh8RE7<_+*x@HfLyXMGI*RhouM54=G`D3d{0C$*#-lteQJ70mErFy3**9bG)RJxo~ zkpWKulhNaYFe0jfyL1|7c-6(k%h>pqrPGSk;l#SPF!}dhL6SPdOAvfPZd}L0mpR5$ z5qWv}49^I2W{c>Tm-@u?_4UR3zP5b){F!h_*NLPWiswT)dV8BGr*wSp_+mO()uaB78bT$gZDxg7!{x!RND%xt z26DYA;|#p-UcGuX75@pYAFTRdAa~Zz=TN8L3gFmu4lEXSd^cc8d6gTnvz?2Ln609r zoo|>Gm~z%f3R#w%Mn^}d9m`CQwB6m-J*L7)t_j`RP8i9~&sTjqR^e*bzfvORu_M*7 z7PfT{rlP4C)b06807^Cv)3xW2a0X{r*SWFmch+Zj>0#rpQ(ojlJY_%Ml(N+&pS%?eGASt zDbmgd;LG#(L&kb~*YOW<@1NhJgS&46jBAFiAm&R=+GA%7PPYwiitqe*NH#O>)ZK(j z27ixA2e5Po_XRt^+&s^2F5Hxjoc3otx_)qQaC`7&-F5HH#3`wj_~$~2m=9!LtZT;r zfy^45%p0ia=!9Lft0a$t|G9YY-aXVNgq5=pUhL0IpL{dCsKyf61m~;uQ-T z`D|9?(#2`und066W^ycWYvsX!{{`j^{`ldWXj@JIp&|R(MgnH!&u#a&zvkw$pDZ_? z`fT;c=9~90dCqef)NOW4kOeh-NSf;D@sl{gM5$Mkf`!kR)({2Yw_eY6B5-GCC-~`& z3#4?l7bNy1X(D8xWC_9|#k_P{S{mE1jj`*b8w+>!-ZSO~V4MdO3~oQkIkmrC1liNv z&MvBaUB|A9a&w^@F^q0*Zr+&oC!4Liz<3^pLZLuPA;iPOqoJX>@9OH>xI*n;=L_z> zV|4gk0}2C}y}2lahdv0M49Mwouv#cYfb}4hI9@~qI?<6{nB4DjO)V~_8WAf1WtxzKP70@ta(T?J-Sm8ec*}T^&48&M+0bOGQfNp7xVLt!{TU%SR z=#KG^kT~dx`Y||o+gK!Gd2vw^fk5;Rm4kr8v}5_EY{C}UhO);bRFHg9r*7Ha-A${h z@o94+BqZFhJKeGC(~kmSX(F#b65wta3?xz4i5JxRZu6A7UfTnzEY?aUtD{9w)VP;M zw&q^CJ*bm!(RZ6ux75D5JAcG_Z*-{o6-Wm{M(K(mM2x9Eetc(nfs&efjzwahpc;Uf zTgHrKZPr3_;BT+3Rce-7>WM~+U+{R)(3^gUMZtV~o;KYaL*hJgVe z2KERQ7x#RoJ`hTVT@t6By59TqpT2%o!mRmmf)Oyb3P)0<(eaq}j1}z6PY^Wm8DYlkWP#^*Y(5uKuQerZ)hEtetM;C0jRDkcSfy-3rvVy)+ZyLw?xkTY>lzM zmwYksLEeZps4rxIMRQkesd-N0G6svpuR-PnMMvw&N?I;1uHh0pMgUcM#>R+B#z#x5pgCH63;KN6~C|30i5C2C^ z>qJbHH9Wx@N7dIuIs0ma0nF9q*6ipuYgSL7>2@Bi@$pg15$5HE!9HbYE4FP1^W)VB zV{cc(9KM8}PyNn+?D2g6&%A?J3hn=0Yqh}UW`a42nI}zTGtPAAYfS(8 zqC(yO90i=S2xMz;$%_EU!@3Yyl*d0n`j6RT)dyKzwQCdK>&JVxdxMr8gB%KbJ9(u$1Eer~IIi3% z!QYiX8@W|zIEyby^BLRu3_DLag4f;2!57Oife~n7N6<}aPtbx-6EG)E#xjO|19=lf zhPJucvdv}PtC1I3GiyE@o~q*6JfaiAy4jL$(iLKp6BwfqG)842iyO$7`bIX&M*hGW zpiw{ddig;p_9P>9AnnHY8n*}%99NQF%EUhm1Gmdic+4hA}W8Vr_ zYgVsjV}z-2;h{xFpiCMZY`%k?#4pnFz&V^k5}1lj3=Sg6S{A#*f>zkUI^i|es#h;M zPlsR$0ch(X3w2Cj(Qb}Wvuyb_i@jT_@a-MHOi+Z6`yt=RX87)GtV!?gv!=N_L$LnW%8R$-_hT;>8P*g(RmRLqkEyqOQ*t zx{~)m+71C(p9+v3ioaiku)2*)=9iUn$!8@?{0wSXzi**$QBW*{9X$$DaC<`oWIB%A z%F9+mIZdNQR{2f~%0~-6CuX|2`v6*8?dup#QL(_!KF1AeUTsnN9v?p^=w+w>DC%HwQ z`+tI$0T4B}w5+eKz29+|L=w~xjat5xUcpzRAVPXrPcg+7v{5KvIhyu}q(P2Yd`aE@9Zd1ElwAPS&f!0fDy77^`ExH1Bu zq*f)y6zb3NlRR=W!i*oyzR4cYf?xGeZsf1y78^GA5m&Bztv3K#xH@DG)OgbZDf`3> zo*R;MaO5Sx)229V*`1yolMl5Bd;sLz+|&l=twVRI#aow1uU{g)X!V%+F)JIZq=^YL z)dCOBZLjw7hqhNxSKimEz5bbeIsL*(*Y*ELy59Dz zid9&x>Ak+8R?C7^&Rj)gWp@Mq+?k09G}Wrpy{vn`Q6q(V`ZJs*t?uHN9`@|V)Zb%P z-(k+~5mYCJH<_E8UrZFX!-r+-SKTX0ypj5@^yREn1&-4>VtoI+C(L+gEz7@g!09d{ zvix^y*pL4H08pXpX!$Ir$87)({o3m4YMYbG%CMhnN6|!SIx8!SAR(jkz%5#S5%#{k{A$YS(E?__+ef@B znVebn5z)+Pq?nMy%9dz;p`9?d_3-O@y>#Ic!2#}P%`r+HSFd3rY8n4DSQfA^oTY(i z)&&nZxk=bV|Dc%-(ZC||Df>{D7~1i3Y#UdyT4GR`^RVNU&AIJEf0GM9M5_4Lb|R;zr$2m( zviiaM%T?M79=G4%eZJ^@9u?PI*0475msa-d{PDlZ(;Zsgysu(9A0!h+x&-Fg7V^5) zjcwV+2w;Sq(TQQ#i1CZ+N`wAI){F+|3uC=ik440DO(QOQKAUFakLsTScXtM=cE=sU zxBmehb8YC=tgf{YddC%kMI_q`Y2PiW2W4=H;nK5jZjZ)Cm})7;ep=A{hpKwA0e(S! z&~|sdP9xrz_T8Fyt}?+%)tK#a*Yjp*OmkVh@;;dkmRa-=c7?7!Xh$Em_N{bfY5P4; zxDlF74DSrkX+wqz_1GrCMlHAs_6@)ggn)Pq((#s!KUG$Jiu< zXFf`INyw1Cn}~%J!7ymed)H3HHqLh|nh4bTZY~d=^X;9p=#Yr*uQDNY`>Z$POp6c# zsaRsACAd3!R_Wbqs`^ARoxmatzbK5=G%Ax;)dNnc>{i`s_#p7>1Fh#NXU*)?ENF?0NJ;3Q z$XDLJfF9QMG}SHaK;ked7DIpri-+W~Z{*5sK7i@Ycr(-ikVwh~IDO`yW`3BtU1|fIoN&wkciuU7#8&1_4 zK!pz{_UrSLFh7a*r^)tTOzC8RA$MM%lS{B}QTvOfv7?pzY2h+Z%+ARg&y(Hi$KvaS zX2P5otH&tvqb&(Rq_0U-|BL@J>LKHDo9m~V**iMe1kH_rI~7Hj@PnFT`E$Sc2c-!L5 zG)CCp-iiH-iKPLmSA8|y%VjD`sT1b#q^Tfyf$HgMJ>RwPFhM1Mmg{dv!E9Qe6}*Sn zC~D0a|DDoruo`WbbjnKzTb8Yc>Wz7&;hJlkf>%L4HlBPUrL{XY<%>rXhs?!jKed2!2GL~O=7Iswyp)K?kt|*c8 z7jj6lYis%Z`K@XFt^phC9gkJl37B*G{N())HIaEI5LNHE($OLw7h9Tqof5mW`?r28AO};h z;Jid=sACtF=i)t4ZjsVb+t;rv-`v-Mi@>A0wnCL3^n~&7uQEis5$xjCF|oW0=;?Tw zVSJiA@z`2*Lt@C6MRyfXMDa}HFy}wbyZo0>u3*1l-OlEapy#xI8MP3HIILs}qd^i{ z8KqZp#+Bi4&me_p5x)0Ea=JVJ6?c!Iby5czTG*qQ(CXUKVE6Ab;3&0@6|LjY&=SLN zqi>Hv^%rzqA%JGFwDE&He2)%TqyRB|q+_4=T1Wj=ma*iSNP#&ywxZ`ze|{>G< zH=BM5s@>{>K%E3jO_Akr7QPL^-}F+fbfF#wXG0pnZeITy+>gOE7cA^?N83s>rNA5; zds)wcwE+6(uUR9maLIMU@qd8{ssYnKaH+-X6@4WnQlRQsx}#-Doh*7ic65ZZ(285% zn(G0Y4WaeCj2@e*qCu^jDtPMx8j7w!=<;RAaGLF&tBN(ZzBzZ&z$Tc|Kac1H!AQy? zqh%mWsKvp^CZliVHb=oDF*ME!Nc}gs&Le(sWK(wV&k?A!ul@s)k3hw%nh;%>9GWdH z^5xk`wm}epXKv~5DPUKiTki*iXI8)4J)14+E3{4mj?)%itJ?3vGoIE&(jguBH_pTC zL%Z;K1)Ss9#K!f zVW@d(3!*M@-%4{UEds{d-0HG%9$xR@$6Q>i{?waR5kG&I>YrbPK=Y^I@)8XQzZFv# zuxZ&9w!h5;@UZQLQdD^6$gV4n^68<*D0^U$_?ZR-EL;q<5Vt|NUeNgJJE2Xpx%@Po zYPe$`vf6XlFv`4B^E|zq;#&9_1cDm2^0A2iQxmcaGv*|h5_fND4@YmQ2J9u2XM98rfa!qRIg6jotaYoeKh;w9TBs>s8Uzog0|z%!|a0a|A^Fjt|0^7zl&7}x{fBD!o=iXy3Bm96^DyZr74{vqF$oxrOR~)|x|V?1 z;RDd=FC1Xqg^=DeJ7+uc$OWoS>8dJm==Ic$Bza|}9M^Y|p;cwb^skoz=~CqVAL$YS zL5@H{r3R|38uiqIWEe1$S%9coqA|vN+A6UIxBo+7zlotS{SGL^oycJ@;!-_M1^|Rf zRRH~r41VrEDuWVxQzt`y7V<6DBX6y2-yKk%S2k>>fN6j#h2o!oDFK>=HBei2G+=JK z$O`1q)Fd&FXnwSylEUz^S%RU+MW63H*-eRQ6CoWq+OOmoLBEHoxi*X#{$eJWoRfeFdG#wzYQm=(YJKi~f>D0sR?)*(=RTSC4QxAp`s$1Q=?Hry?32%Bodyr6xhxKJ+`qYUm4KvA|o( z=#YQD(;`fD9_W`OoZ(!HFVDZ>`cKw{%<IVj!54EeIP6-jtAT(Vo` ze?KJRE|FQ`dZe+4D=!rSMfXsVBXHXrTWBtK` zxn_UM1vu{BJbLPugRE2;D9ux^s3wFhzr`4n8w}06-pCGoQTC+VslBs_E5w4^mX`+Q zD0)`i$ycS_eCESa*b`3q){MxC$GJLKSh*^`!-jnR3S>M|+(jJuOEDPZlF>l0hVn71 zx*t%Z@kA6mqks3-o0KfX#3f9gdUnT2;)`+hX?IS_^G4w}jmhNmq|htw_X(z#Asx`- zAUpAAITnp8vlS_*{PobkJVL5Q%%Lb3o6gInm+YRNO)UX0f~G=J)(`N2gZ|nHcEq~Y zGz6><1B;#|^F!DzkA|yWJ4cAdRe=@}7~5z9Z%udcvfaUAb9rW^K2(T{LVGL>mSbK? z41y~`R8`OMZT%HbHjBfXZIqA;`i_G#P5w{li?Y}{!)aqt*vV_gI(ro@4qg`Cxvclv zBBofLLDlj_MLFM98j8V$cEklVXuXRU9td60Y1*AWtys|f?MimbCQ7Od@`o$-d-V1B zBZkVu0Y#5~I#%@DU&BNH1ei9Dd1GA{5DB|m3)U2%0S1Hu*W*Us&cFR&+3QiOs`N|D@)`2Lb5%OhOR{4u&6$k2Y3svmD^YhyGlsfjO) zynh4IB5UB!=2?X{DvtFso=;Q0?Z5QdvsagI%ZdnH>yTgnf<(>7)mifrS#g#?ekTP4LSp$3onoK7pHUmMr5VXo^TUw zRYEme_(kGo_WncfBiDAsCBmSsQ6|+cC@A9roW<;R03}8tYQ``}(;ci=2~jb!#Qj%l z$p=1><#;d}|2~>rhLc@~X8UMh;EVn0(U-uYh6dSdy_!WBVjW7Ls~tia#m(IqYSebJSHd1NXv3{L!c|Km2ZLn^m^wtgiIT2tWa}0eAMc z+ZoVhjR$1G2x>&h<#=}2(88@SZ>JxVnV;rr)&bT`8vCxkb(O)N42mJ9LJ4otGG&8? zx)$6y52IMFJD$-5@{xuj!D%;J0*ca2I#s*S39W|p3*E<@Ad1u` z-`w?jR-5@l?P6gM-TX>p>vpTlny;>9CSp2Eo)G?*CO`?Uc2R5u9o3J%`P0N#qhD3U zz^Hf9T|fHT^&%7z_-bye=0F*v6=b)V$%v+wEWgb0a)r6rL~Kxw;YY*4xWUET9zBy* z@8D&mH-4;<8(%~I{OFIr zX7TQsz_Zd_RrZTwt{JFzZfZIxPj!N@{mKdA5T_$jq`Yt9or>}|xh9cN5WwFWi?7;rUps_{hMB zD_e*PQ7ctpJ~)VGp#$vG6^O!<`)Hxv#*b#&Mz*~rpj-2m72)Px?$h#28ejMugbwcA zbs+>eNy|$4hg8L*!NF|u$f!uVhCElPl4Wuee&G6VzEf;n|AZGXnA28(wK z#v@rTW5=>cfh$IpCS9hU8uviqTsS^PXEL6{k0q-N`9XReu)EEDugz%c?0MTpAal9G zlU@@OY2q^1^@aQpzPd6NabM*9;foM{5@>a9nZe)wq--$cy){3I?|)93sv<8|los)9 zx;4YY$$@X2cXlgNXWD`@YY0IZhr(!la%zsLGrSK96Z*7d3aj0f z6NB->Lm$-y&6mxoGJ;gX&-5K`UJd(|n*WL1GFYSgpN)HJ{i3?a(3_M zu$*&6xvHjC+&8gH>7vl~J?cWT%eghH+nxvJEP7lR#+~9pTfAePh}j1#O^n3PPqWL+ z(^psVZ%!@CEKCbsc&56@)WMYy^LrI)ZJ69r>a#b0WXY)?TUpze3ebEfN8R_2v4msp z+8b+u>x!cc!M%EeTgauw#k%9&siD|?;D`ypSwLvi`%WK-#GwN(d*F0Mco=v2 zMWA#8lC4jlKi5Cj)C;`8fP{fqwnHZQ6ahxt$|1M3JzuMyqgWK9uu9l*UW>3wmESE%cCi;{b!X}q1bwI>P?dV^ptKU$1lm8%d}z5N}~<3PoE5o-HT?6^aETcEyb3z-8_d&85XeV}0$ z5waN#P)Zb<0eZnkAf2QBInSKw{f<9nbS{gvE&kwjuJG0G5w$Tat({GNeO~A%nz{8$ zOOIi`uc;p7exYVbqW?{^+DKq_L0JSb4GNKfZ$Gm24+FcX9adj{$^U4bPbvzLWo>jp zK~2^y6-|)`^#>oWKC91Ea8AX*py$ z_JKBEm)Q)?#bu{_zyK(7E&Y6A5BwdaY@79ou3Cy79U88X)Y4LsXFn{)2Q&>xU_gTX zz(+y?*1Lr>(9|$v!EPIh#A)MEC9q5MP$itaUKgV_!KfUy2a_Xf4%ARRc@v3MRk8T zvs+fO4X7m-**$2`|c+2PK~!;)kGKo>j7MDzH~0q1-#RU>+#o%gM*=(Tggt*EX{8Kq$!XJV4nNWVERB z7k&FS4T;oY)9t|Yv(Xj*Lnv_HQC#sJUF1jIpswF^i&r9vIs<7K=pr49#gA^(6b$x^ z`DyFe1QZrFur097h)kh`=chc%z3sHTW}}w9cVPzyhlYY1C+8M<^nm0}Eo6#X>a($r z!OG$Y2c0mb;O*5-ja%(uacN@%0uD#btz4xAfBv9WgL-8%PP>4LkvCqEPa-8n%QUN` zgc=;V5d*5`oQk!3knLwpUN#Bt@zt_qLA+DI|I$hbHQ6If+?r#AZ|xx4giTB>tljhj z?dkeowhdhYYpu#{ag9=8go=vmQIE2+&eQ4#BI4qsq%2dpK>BSV1XR8`)d!Y9xNF#% zaHn*4(!=cd>f!EsRH-1mfpg<^7q~KUp(N~FmQ*%07$poS*OJoFIbN0zS z-owTl=cL_sXpi0cR@Xo3as-VS%BT=2X9rr}(1=YR5@)MO9|KB?d(*pk(d@KS)TAl6 zUi$e9^9f~EDo+#2*IS9&xoc194w@Px5F}SeaaT(#*9uUpA-xJ2CtblTV+5c}6AsR5 zLWg&T%bZwZc?=@W&~C4)lFgsciStzzfWViV!lYoURP3Zcs~114MF%>-H`%S94W4mp zsyX-IoI9><0`o@t4$nhfoD!Oh>pm~NtsfmkuL7x@>Ve_31^&3La8A{ZHgu9FjGCtz z95)FH4GmpPc_bJzu3IVc*pKB49rCI|tN5EH`i+Y%c(EPC@N-MX72a5qB9-oi@$XOe zm7kn%C^qjE@L!&(1ko7NRjqgNl|SH=+eknoxnX&G7COnG@4asVPH>QQOPsJK=T8R?~>79mWfpsxU52ZqWU+AXuyhJm;k&tt8~Q; zc{1?g8$hTI4lVFNXJ;URLLjw%Ezta{S8=teEGA-56MigFNf^|JbeF%HJJ|J)fv8$_ zr;#^TR%JAdI9gs_&Tri>){h-iwQb+8?USa~3&_a0J?^zTQSZ)rr5>EG3}#93ybK)< z0*8k33?(fyZ>g99x|o@*9SMr5E>1Fd*-P6Qku-%>ApRi3GGnZ}&JpMU7S>5!-8Z-W zSYTQlCSo@0HgZ6>*!yr$v1l2D#-)KTERj`{eKF@Dwekn2BjA+Mo|rI$B$2hHrFw9B zj+dXGsdBP`Od=deuexErM@OFHo|~PjxyOAeCmN+fs<#}o^g@bmF(0zV;vk;;S+8Sm z;h_uI=337hL2Y6|L5FG|L~CQ*cRFC^}r0TDsHr1sE;6RqzDpkiUQsSf^7F8$fjCM@g5%yGwe^PB59rgJb=?LQ}{V@(2&FS z2ONXPwJNhdvT+^ijeZRT&7=6d;wlypW5&-3?~u{?iq>*+$X5aVnG~uERNoqRs4l=f z0>V>NPtCQJaO6-bBIgahvt(16+5@pAmK{NjK?GD^cK)96!a#Q2q@Nc>%b15N;QoM& zOD}h9ZmCXF|JW5Quc|p)qmSMnv7PhjsAtU-s+R%o&RPO3hudicfodCUI*0Jq5#{@) z!qgX-jNxwBoO3{EaRXG5ttWyg$Ak>Fh^9pBm#nOYF1GkjN&=>7*5WXy=9tUK zwrEIip|qReUiFwVm=@t@f1hps7L$B&i%v>o&dCJKm%){XL3F3&%6jnS`RN}^ppNsh;X{p^(*(6bxcy`68ey-6Cz()Lmxk0xCoqF=dwEqC|$bIr_x$Ph-}dI z<7C%kP1;3{Q5>D5qo5Lu0QGCu4Wb$&bqU~Zz)gr#%!QTXv8`$O51C^rj|a7IlITa1 zSASghezYCvnPe-=*0W?z@rs(puy(dUnr*zNBbjB}ZM3IE;DbMn{%=q63;1M&R;Q(V z?w5lC=9;{F@lr` zJKV&oIR-SGrBGM9lac2nx&G*R#EzL%KWV`PFE$?2{|8B@MoNd`c z2j>7zPcQx|Gb*fsUa0?Ti(8Rd&Wru=c0+-bMW?GWGzbPC_BAVtPKZ3Zr+zJrnoYg+ zjS^eYW|WCZnq*18e`;;gerMCU_id1*UA}7K*yG1sr1y|#04;)+>k{evh7od1zR|kM z8`pH#@0DK_RL9>ySWbeJR*Y|6B zRU)i^BU{X4_Ab)@8OJ3iG3>m{@{!R+BGRJq-!4EJ42v7-oLtvHjEwKSV0iB_9w1DD z?arUPmH_x9RhKd=YtNXosNI^rtLu{q3Xp-VE?R9a1@W7FB}nB^7mCCtlQ30IFVf-3 z3b9{99>vFZ>FDp?&OTP(JXGN-GKTqDQR{NiRp4-2uesm-1r~z}S_~)o7%%zW8ym)t zJrpAj3Ulj9=@bi1EY9+ydL9K++vA2wqso!0$65mNb{z2-)W|4Zbo&Zy2buaa=2{pJ zx8C`fI#lXJW3w@@ky2+F1P6z45CER;((n1_3AIf6?sB@jHf}DjZp&v)P3bUpw}oDi zhZ^3C9XCwilnd$N8^!_)95va6`Q0(Sb8HF;VbofLDy+v(AzJv&Vp$~hrQQb=El*-7 zmzM?!7kD=WMa6O$wkrBEo$qP;u)t`ZWzjRogFIITF9gbt*1lEXN?}<)S0|5mjT95) zk6YiS_%@k`93;8QDK(4Ah~}Fau4m^eU4i6<&bNiB5kcWB{&lY@NSfz?zLL{|n^fwZ zb({?2lVH}0WLx%XIjxG)8732BVIW;z123JKOSEm^>Yfa(DLN4w7&{01CLM2n0UU38 z!|^fD-dKy1&8q8th)c}Z9DS}Wcai|y;Xa9q>2APBA#>*Nj8*@C#ht0$>VWDOtMFmV2HK5r&mGxJuN>f@6;!%6vu##KUNLBH-5{3=!G;5XM9Z=a zKYxl@E#Tews94mFu8nzvX#^)pP`3~zVQCeKp@~0qi_bH=J%bJz2=uLq_~lWdMRs0Q zs>hZwH)Q48&ikJj=e)mG{hB94G*$PuFjX%1Cz+?-7M4OWFe|HWY#rQDI@Ak}??_b? z>S3BPyhz1c;(9czj~At60HaB5SHc|#C%D@}58@?+XYXJ#7MhFfVJ z>9 znis7SmEqLtDQu}DAW1wj14ME_h+~$DaXwMY$P-Hzpff4poNL8Klqn+Xn{C`7-AbYZs&dZl&^X-RKE85#GO^C4DRl)@q(3tC=*bHDy{kit8; z68x85S{m8tAlkomA9hxI%tROL+v)5RybCW(Tyq5-G9zE3*p z0$X_Q%@^}%%`cjLWKwsWGWZg*EtLWB7|$|_C9;v3Ni6W&>roBQt})Q!Ise{t>lJzP zCy9yzi|rqGKAXbn-^jGEBqm|LiSFTJFy=v zoG{iza8Qul_9u9fc1OBG4i0vX@sF9_;8pVUxa6t|lqCMO#GHh^(TpLh-2%uSM!5P;PWz#fTBKcpL=E5OP`alSE5AG~w(apzw ziFBG;?I8xpWW@`P9`YHS>Jd$`(YZ#tJnODq$yg>o%M77gM7Q)3!puCTTPa>&4bves zgybX1ZtQ>BjZoeJP5@*q%7J(Qm_@2M?gWuy8m>GAlW|ZwcGBBF|1Nq3ldti1Y}@1D zZFLlp0`+@N1Swp4QdpIVKDOYLDwfk1XeWAXFPllvkK`qEjb9?U8BSj%dJ}y(oL|Fn zZplY{*gmxJLK*TzR~LX*Qk~wcdXL)k*so7O-!e?LZU&e^b<^jWHBCES2E5% zNeKvMYK;Xy=q4EUuA71;SZ~XFGnGY2{z(l_{uc!k!IL`e=FtVo&8>{3G16Tw1v)FA z3LXRh=|H9$zh)V|V}^W=ad69kXMfga%eGk_$w?YoSZjG0#7H$r^_IK-v65ZX@&VxB z*LCf`<`oy2iMs2GIaG0Db()LcXQB?aWRYbv3)$BOO&cCZ(ZmxfqAYa$OEfa;CQu&|tFYq%DMa6$Jk zWTo4&JeYZ%x3gSTOoUmdU&3NU6AP%+K}Uu)e`0d% zrEvMwm9CLtXG*F#t_MuR9YU2}la%$N85G@YF`L1c{jj64EA6`r#5FRwBQe`QV>3og zbb#dL#@~9RGIo+)K5e;Gsfk5he)R~LQM#3&`!pL3DSO}i8_DWlFW=z7eE)dnS0<*r zd@N~D8#J)cC-|J6Xd9pX`Tb?3CM7^rPxj5b05L<&_@>oN{xd5+apyrLf{BJAqMTRf z%;YolB!z`st5H9!E}eriu6!3#{!48P^Mh%&ux9;sTExgxOP1<=`GNDdnA&U+j6P39 z^fk&a0*!|ZT+`O29c-7c#wQCASNYRy6oYa)1Yqg ztuOdZFl?2wGucg_C>a1`aD_(xoKyF9Zl9rRF+*)}6qQDP9t`9>uuyH_nk;L`h z+tH_rR6i5{1QuB@ZGx`PNCDr5MlEPme#`|7ASekgT3yRDw`~74D*z-AJz}@L^OAG~ zWkq5S$9{iRT3Sb$wLPq=B88zCfGlv<#4vj6<^}!4_QK2#+y6z>TSrB;zwh5zARwTC zAgv-Xv~**FbeD8@HwcIb97+%dkQ$I02_+>ZrBgzXZjtT|fA{$LuIE|n{BaJ;+SG<;sRd&5O0;Z~tGA98yl@GiRni+F*D+$Q^ymW?kN9V16 zdr#J0w+3NzSZi0%ys3p~u9^ETQ_>xB?Jz_Bq@cFNHRg8 zA9+ljp`zt2dG9!lcj_4N{@LyKRsSi&@aO!noaz7nG<&eTbnW!KTq3fyfCoyxH@4ki zFY+bCt_?)|!o6jdsibAc;pP;|veX4> z9oPs{XzTUknasstb-!%p@mJqcP<6wMPrH~Gp9ckc@nN+>vf%~yTkwpt|4!gtF2{k> z(u3n>DXM(mRw_zuMT!(-&~!y^P$=%0*4VCqdTxQay(!l{DE{)|6~p>uU+}IzEZ?`+ zyk*RH|>y z+xb$~`0vIgS>`;#Fg*G8DJpI$VKygQ;7?r$C|vHJt@WqRhL)pU>pA`@ja(P@EVNox z$>hJfbf`jAGT&jy1?9=FrO^L~7na?NbB~~x)lw3c=p04IY01X%g&@ym(QYy%>+nee zpw5ZeVQBI$>0H;nlrYfk`3f4ssy+}|z90g5vF0>rP2KkMhHM#edq3;n{u;6IyHp|x zd050>O5l*@kzG{2-0adBsr%1#nnmL910N$PysaY(X9Xyd^#tS5TV~us@>O^7eMB!k zisRL&Ov5z9m%R0SfA!a>YK{ML{aMz>T1Lw3`}&umXbFeVmZ<}lqqp;jX@@lzhlNNY zfjKW(-|b-#lykwS!eMGtKnM(!2yLmEvNx6NWOtqj#aYZl85vU^1!Xapf(tOsz4s_XZr)FiBlD(yafmu7)yr z2Wvp(-d;jye*u_ODSkP-dIS3@(&B->oe}%6jG4c`gD*8G(Y>(yyD`lUNmX~RDc47? zo;P%(W4gM7yp_%j|5qCofMP?Q##nV5zUeowTrMI|2F7MHR>Jh+A>3C@-ktBMAuwgZ z!F*+=-k})cJXL0>M5d}rkO%dG$k`vQuFe`u`C2-Nm69`L(m98CHaL5;ZG!ab_!I*l z#U`al?$yho#zv?M()(dS-#P30Im*eNYXy?4&3!I4Ki|Q@e2*R@ZwCiw!76Hu8jM~OTGWOv zP6Gy9)Xh@?wM}CyD{+W1`5n+&&DC=pi|Y#=v;6+GH1pM8snbIeUZRB-gOBkma@G42=Tu@C)?E)M4*S5j`MgpSA2HYHw zI)p?t0Ze~x4{K~1qg%<;v8lAN1RKX^|S-p8Iq zPc0xDfd2mUg#V7jtmr*cJSxM6vlzxjsO}IETvx)8N;`dN$1U>AU7YeA`p!SiTsWW4 zW|7^dzspp8C6A@+zrwFYM+N|H2roG z&L-Q~KEBN;{T8nlE*UzyUxq2f(i=?8#Dxo80!#Dj@($#~{HdO_sB~|W1k0R%mjad- z53X>dUaz%#JF}9km+*jJs9Tb!GaK;~W7jK&{lqWVIWMF2lGd(hhu-9$t3^Zh-p;Lq zN6%`UT*8+c%*p&cXeb8?C%&lLv*}lJ&3Qr9n|(Z24`r=tM#PDflk3rkqSgjgYh97Q z*0ZfSAYj0!Nmi}6*XfBlxu11-%6xB7{g|!P5l?|{`glef%%*0`@V8qBI>8OpzXr2O zbwqfwT_R0-{8-0#e&E=G| z-f_RN(foJq;-qrdjvebATX_zD((La`57HgasG`Q_@**({Ju<1{ov z!AgSLBn!e)HUC=5T(C=E&p#E%Mlt*Sn@4Ldw)57wx+yH;f%bv5;JkK!8)y*R7)_6` z6HB(cwM4dSbX))vSQ-P6KjZB3@UkM9U|4b$-q-|vg^E&;Z#cEc1{Nac(#YEwe)%mJYVC20JnHeJ zG;bwgc%8DYmZTY$Gv}2%dP5!-;|xdV0jErUD4QRk<@9g)xhv2_ve!Vq*GRI{M77sI zve!hu(+EGI+CyD9>^QZHY}cbIXgRsYx1waH!dJ=bnZEif4g9;>{|zza3UZ3U2tcmD zw%v1w87cwMdFhpH-UXT`T-0wz+v&DRMZkkudwz=CztjiD8*OvhLyd?R6w*}L%44PK zNt*YsqCRZ92FGlhI{fGSdl@WE0qCbsr5xAa*brBzX@pE@y;}J(++r3qMP!o6^mP&FTbtLr%$PYE9dG)a9OBuqmCDkM^wwib zrhdnU>GJZ)2@)gfTz<5%sRr`R@wHeeAY;&Dw@r`LL_kOKO0}-2bq1?NmZOxaO{I6u z*?Nui;KqM+N5*z6;)%NY;%ii)r+lT4>wT~;iVU=pO`>$+36t#toJ2)F)5pi(Xf=sM z9~X1Zl~E{q&8gL-lXZtBYaoTCf}>ahjWpELVlyVA3v!AmS67Euyd!)274!_!R5-;X za8)INLHFe~zI(pdPaSC?vkbi?bCFB8yhX2Z|Lw^yfMIOo-`ZgiBv zviEIHS9_G_IVj;v06arVZqObA-9^!73z71}azofAUFZ*G~ zn*6XHAMPGJoBVu@97_S-)K@OHg9>Jm&m)_h($I9MbewXw&90=bU-R%WtGyYzgL$^Q z(#e`RmcKmbi1DF8Vx;Pwo|L;$;Db;tOKy(#5gm2Ddy@sMJ{CfxE%#tr{Q7Z^?R_|y zl)Vm+kzXvHa?qK;eWPch0#J#|ieS2WJ5T?;4Rq^sQ{g#QmNd7R#@JJGFoIm_sMvL8 z4Ii8XyiE~yN&MbCLDnFV3{y+yJWa~@(mqvp%+*sdL^O=Xbp?He^AJA4-YiCe@M(x6 zqKLS%1rBl&B-S| zI7zW4PfXe7Z4>uRFRaqNA?o#1-!-DuAfEdVjT-k98L}4)lU;Y2m1xsZl`D4-7a+p;h>Nia0%_xanpP)N>)(18{<-BBxCEQ@4G1wxg`=kz z%oa=7=;_?O*D2D$+E)6A6+AeHp5iiB*ihZ-S_9enC-zT~&gGylG|ts?1sf@7DSrtR z`JcqP{PJz~N5=={C_3jd^Jfe8UW{Vx9`1uxRh}pW z5n1N17w&9WMDd z;aahXe_<4jRjUPYWR~`dHL}GSGxHt=`D~aDh=XHT zP^*jXJ2XA6y`FF?Gna%BwDLgvLbJteQ}ts$m%Cm?+r;s;dDpVX+{}-AMs~jI!pNmi zlg~nY4@+r0%2*eXFe_vJIKo^;3Jz;r8$y5kgmJgXP_qYa2V)e9!7pDI-np3C*tE4CXugyubZ+Z>f_^W3KrAy%ZbswVD}wcLc}tSR|RrlL|BEdjyfdCVxL!C>1iP8(U9FF{HQi& zT{4%F=b~f}F6%3{`f2_8;s|pQ_3^wPy2Buz|8?g^@b%brNn-Cvxj{Y_KC-mO@L>0- z%q)YRaj~}57G}Ko<`_j$=Wa_H?$bJ>cn>MqlXT1N2EF>TyO(V#uxx==GlzA^?Fei? z(Vp9&MX5(i%n_J&5}x(uyp6u1X!mYj=(MpJ1kZN}@oZW!;%{`weOa<=s4N4mYeLcM zyb5+)sO0HD(i(x97Atp<*WaH`3BePZ4}IJdaeKJQvuah{Xx+|*BgrZBr>$d2>L*ou zHgSw?J1K0b(YgfBMk@H7q>zs{VVbH}TDfB|rhx*+!9tYQ3TpNKSCpWTJ4;hy6%(98 zAnh=+Q~%ism7j`{f^s1D=ogy^ZN*Vo-OBuDwkR8iT?Qf*m9pfaFjpMzRM~Zmm)ry~ zCVi$mKsBxVXOs6!C*w8L9qM%L>#!Z_uTa&}(X1iELaNkU>N6ZTcpmEJ`RBi#(%Bdx zhrz{{@J!m{psHJ8^PGr-g0=&u&&0!XR$NnzW+K{=!`kaWF1hmh9{(LpY8vG$is(C@ zmmN=NdRhX9!L(m-BrxnoK9$ z4Po@+LCuh^D5ezF?e$cVS%*uRILOf45T#eZ zln2Hvk;N6sJ4Qx!{O}@X_N^o&qI>)z{Il}c0j*bL=c%p5jYmW8M55?FU5j&zA4@II zH?U=^%kx&Z=6`}i}6(r**CH=jzz}eyyTjfCe5uUX;1+cihH##ODA{D1g|D|FKf4$ zeIAHm*f?9IY3DcoZV~-`4P;Kj`r{slnA&Rt((k`s?_!C^v#HXuUy6v{MsAeYZn>r^>W())YLu1vwmKZEnk*k!k zl;M5c)f(;NgGztQ#Xe>?0;~M`KwHoH<>-k#=RlwH#{ zsEK8!>4a88vr`hYvmHXX_jfab@!e|8t^9Hw*S3V5H_V3wp_YyaK>y}0p5Tbbp0}en zHrWObf|mR+vxSsqYXtg}=;J6AeIe~H3FF1}B$1^_1)oYs@N8%@*LIPSuDGJWVa3OC zoDG_zN&*lV5!M#4%c%>$?ckw&543@!eue|TO6}oPewzk!$8UH8$Oj0LsoHBLuY*b@ zQOkM>*Bkm~cvO$tRoC%cKDB`};=dn;DUFws#)>@dNM(uau4b*NV|{8TK@&Xul&G!0 z?5!@SukRGUvwOdPL8T^|tg~av5L)fW|7ppr!xNM1QigAGF=~dW=!DVV@3u1Xs-bsc zs9sANE7hb9#rQ6QRv$gz&NP3%D~dw$Fpc=XZ=s<93lQf@+K9T3UVzMoRo)h$Z{3a8 zLGQ?mWq5ipfu~@~8rYZ~Kfb6n(}59kCH~L_Q}d+{`&6T@v+)~$#+@zmXl~{<3I<-` z!H1y-myWj2=7V)E* z%Nqt^?!{2*R<5mR2d&8f2qR!5bW^s5>6s0iI0wuv-1j$$--Q=Ee~2T25P@SF9uSdb zVozt`)b>8|Ro9xD8VqH8WyPA!!0$JK9Vm&z!o&Ng`-r4QpAd@DP>6>aZN7jl`ek_k z@^GWpI>#(?uu;x}pdbJb5_MQksXX2b*uj#6Ge<}f#5KEz?m3>N7F4h9O*a?ABs^%J z%C-K>Pt)fCq-T9C9q!KM7|P)7q0Va;Rwj zHR+N=n=L{$kA#flc_b&n!|UO6b^v)C@j;Q~m#_dP`v&+ySJaVJ8ft7(b<5rLXlw%ZS`n4X%mwGpYMx2RYEVOMHq;d)3GavT_SnNh1^FVf ze*eA!6FF5x{&^}TlGeKPBk!HG7SkG%Oq%Ofb7h0zS$w#jDcAYts+PTgf<0SZ&TrjW zEfmWKR@j@@oE~E8JWc3CkQ{JlUg~@8OrOY|#u0#qC!7jvYwJ6aVgcki_`*n_av?_c ztqTny4ur>8+qC2uBdgdL!*^s&bsX%GGvf4x710Y7>IgoMD5NGr81p>#4hfu{hx*N?-J^;9DWF# z#|}(<@V^09D(GwYk}k4rH9vOU`Qg0unkKOgrt&f~o>^}H@t(q;v;18dUfX_<8}ok; zc`6%>N$hzq+j0@6)H#jJZ`>Z{NOdWOI-9Da!_VR~z1^KyMyA5NmvKowN@E{LKr>y6 zDv5$$mk1ERKX?f+i4Yk4{w$E6zXRZyApd9BVg29prYeVTPTpmkb1mWfzwol#tH-`# zG~1~Z;|1RQ_swu`FXh4~Rxak-Wf}*y!GKg|gYJin$t-LhlqEkPH@Z3#B-f6yKZ&8< zyyRLf(WDjHNeG=*2WO0D&iN?mCbM5*+Sr?PvJ}k1U9jHXci;;r4$8TB=!+_$Xp11w zt^mb^SB*|d=hx1Enqpb;h-EyPJ-aV9Dgd3S;@D?hW^A|qGCPHhycFeruulAL=@*qN zo%5sRKI0PR*mYqF&rad$t0?h<;;7k9*21@K^NpPT$Fi6-4)|t|ApYUADZkiGJN+%5 zkIbc?-*0^Id85BZCm0P9&nVIUUaTu&Y>Br#V%jQM($pKlwM8G?S9KC;&QF^>vUNYSIInl2o8>>c>+a64a2;ETeF z{^01c;_Y&0H5rLjB7&ZR2?Z-QePV+dX@D+uW6#SZPQu$|A$eCxRf)Y|Y0gwcXsg-E zofm|OOQti3aX4oow7~OZ8SV7<{BS-t?d(C_PF&&4VuC?y{l#iM-qP{8ABKTH7tYxV z{+7@1N`KzDBU_v$*KOPtwCwI|;#8A;lQ9#~IlBBDSJcc9HXYNr?X?kTyyUT>_(gZp zj^g(kH)Ka~VZtLRbm~fhI@^zvH_$QhOV~MdGTbthBx(aos)^!XeIs@XAl*Q{Ru#X! zKv)@Pe6PC^`&0ya=hZ`!7%|jPsC)Rdw}h@kA^EV2T}i6Q zj56hHA@B>lWle5TMOcVTN?r%n`Go7B{1LdD=rG9s%B?2+IF|wqRt&ux1Ho>}66qv_ z3KIJ(cu;|t53QuoQ0rV`KfnOO*c0hIouwnMW7N7S32-|CoJJ90*HK;R97Pqq!ntspH{SHR-@$U?Alf3WRR2bd20;O`MY8cM=z=btT>o1JaPj zh60^QfV>Z8H!0huJmb#J&m6MlS9X!l8!ySE2xMOl=)g^U@O4B4%zQdC+yp#Zo*bA=gcfD<(5y6s&&6*L+xZ zw_Z~l*1Qb*5D()-8u?+vLQQ+J(g}_lOnJI$@HY$`elzgQWbOQdV1)qxqY(cFuq^VN zEig$mwnkEI2XVf79A&2zMsWk1VHGicMUgPq%32y5JQGHFsUpn$yb<;y4{2MyhHEX!^7V?sVaw=n4f~wQ^4cQD!|@LLV)Q~F3iBeXbT;l>ujGJA z&iJw-G~!k_#PzE5^685sTLND6w77`OOK)tJeR6GV4q3(qe+G@~lHA?$X|Z%Qv7%x@ zdGm+bN=$M0N*g`h`x&99PQw{2^Lru3wJCxrsaFFye|9}d z6a-JA>&r{P(yx|1lc;6SlrXBNRey=h-bNS;0E?eiqg+llg#vu?_F0<&fg=7`zT>n=gt&@q6wT}LR z8~a zndgZ#jwe@gbj2U0&=a)M`|9-+GjYK19tRct?ruR4J!*V)BS`jyrmD!)Xu=2RbXd%T z_gsQ<`n z4J}N154bD0vN?TmCFhrA{rB??0L==`hqrBz3L8v7fID~Hrh~0wl~OFUu>L~++m9*= zBD!Oq4UsJzy}|@=rGn@cut*NfF5;G#*0y;S2nla^W8WHh0;2uwWEx+QmX0TqJYZWz z^}8&lGlFr3N1|wSb>sGh!t24owpKuWti9NgkM;aEu^;BBi!umajI+o#<&+G3Xh9Vr zby983oaF7iV9Ul%as`Y<4^t?^MG>FXS2jV+pyFNL(qv!Esx4YIX83JnGF%y<^U7)2 zc($^R>yR=cCr$$Q2%7A4*heN&bbH=W{RQE&M-ijliKoX_oX()E(4N{Vbjyr}}2<7*4YF{@Oh z5PaXG>+XWogjX+H3~ULyx&xYRPx-j=!BRZunJ3z(q3q>jkU6 z$bNOvZ@kJM>jO;fxmJ`ZGOenwOu}Mwf2gD*pGKH#NpNHjaY*r?fQS<(^5YyeVaG*_ z7vgyME^RJ{Ng`s??$k&F8Qj7GqpliK&zBP4LE7&usmK4yuEW<81k07_7q5)$Z2Ar& zEW;*OEWaa`4|VfEdcC+}DssGPt#J8NB+`ZhMj3RJ@6Ik{tk_Tu7sAd}dS4XEMegr= zywoj!fxFQ`(CTt;UK(XSVpu5U_Snsu|EU%GfT@VNm>Cg#$Fd0=@JwKmzLR}X9E6P9gL)U7sooI(wVp*?X=u&$d%rH z-<-u1`syqY{HjG=K6dtN&_jyQiwSifPHhSQLjD_B#pF1Vidv|h~tVfK$N`gpjP zf9rwS5P>K5{FG*1Zfx7{nX~HsrX*O(qb_PYCN$(0Y|%+?C_w}Q^UsC$pDH4YtDUG1 zAo}iDbPox+FL1{pR%V?H**vZFN>>|2?b+EAfyOKu=A;1<@)PtC_L_=Ab-JU*32=ec z!Py`$`v*f|ttd3=m&Z9Ui8Cbgy)_`OAtb!W(j3AtmpHM>%#& zRg4+_vU#4XZ5)ukDz(UW6s@n6MqEpLH$k2=oqBAbBV!&`dkG93ya}PB50>@q$%mUd zd22BJMs!S`=O+q9curo#xUpR7X~sdpXp+zZ)EC}2DNsRRhH4=mnZFM5vO|#$TFE9` z!ND*0Be>qlY`VZl0KKc5ipFEn1zN?YLV`axVBg@G)sX-YI++57PM=s?j*4-CO1j3a zMC?-#fv~dlDa^iOp5!BQD$y$+VAw*YrA28P_}-pA8LlXDnwP``urxqs(`&l(KT=Iz ze)V*mwR^;vsMVk9n#;wvWk@jQ$Y<7{sy?HiC)fbdqXDc2%5)A?LL6tra-<-;fx%rA zjQGs}`41XLMy~ngGT5~N%ht?Sa~egu#`v{bQjvwuak!_XoAC9^*2LKWrjY4?hq-5t zuNKaa-IipRgq<9fzWJmnEO6I=^vjYZ5o+d?_u<*@pwk*4^*CE)c@I-&GGv(g6=ynm zK-iqc9Lvzfd2-F;$`ln2Xnef7@OsdW08m-B_FQ$PiYewJx^)lG)5-mNNDSZ*bI&u) zxCUskm&Xfb+AY0I5aeK^;}e5#`={4JI63eiWc>PT5~= z?`gpiJ*NK~Nm;Z(%P;N8@j|x7q@B_w^OD1&fG|OQu|_0{*~GZV z)_)XYlp=gV!x%&T0|~!^l~vV{;j5$nwhr^<%49LR-ZiHq z>gz#+6%W*jq~b!|`>9U4z}tssR(@yi28)?We%`27N)Fkw1;eSRSqRKwC8eacVU#ob zUW&GnK1%cr#Gc`1W+v_j#Lrb!d&g}BKDa|@>~Xo2@X0L3LlM<91}e}>(~XwSXK9(j z;AHcAK>^+OOs+dfPcR|_*u_$020Zk96H2gWep%?I803T?Rei6d+>&WtNWR3QZp0dL z$+&byj$=`cX)Kf=UEsx{`xO*m6>cS8M_;Mx#UW4I{y={%Jf z^Bykc9km~l2-~Q5Ljf+hy{gl6{Zu&{Oa)8&q<;s$6 z^_?sA#1UyL5`KlG#DJ5y^ z&`%jngL1xz?49#LH+)J_&m-7?Dlcb*iu??hZHYiDSZFqtN?`m(iQbIu(0y#Azx-?0 zH5-9`<${WFOR$Mue|0a7SnY$Y#osRrc8o1kYh;0Eu#3IlM|wIWFO;c(%8HkU*@5vD z%YT#IRB!svGA$q@ZxYLGOT83a25TS%MX#@kc1e?;zzHJ3X%=EQaEt5l~@!Uap?=bQG_RmW;0PGg-4mG zU`{k{%7}IzZH&|)dNp<(7&3AczouYj7@%}8X%-KI$F)D>h=R}gkq+rn+49t5(Ya#H za1B+09~>H78gj0QH9`MCTq9FiUW_$PoCM{O(!F!4?3Wz!6Hp?2AA(q#1KQf}AJ?7b8ny;>H&3C24xI@tx6Eq7{pjYB_ zFrvDUjry#Vlv^?K@v)4+^ccysE9kibL!!>KwRDud${j5j`jBWjMj6*Yr(<$k$pg=r zjUSQMbgx?__TL?1moH=GC6~KO>!Pk6z&ok{-V;!QZAFgM2hrJ{a_&`nyVmsDmc9E= zfEU)%Q_t$6iQt&;Q0x*u#Xi8NKw;lOCS#Sxv8$3OM!G!>{7^-T*ea{Eeb0T5Pp@+w zvqvdMDFDCMu%6G0V2D@tm2eRhh8z8N-Z{Ogra7TTVt^azC1ENOZSikNQ2PC zw3TiNH)h>c>9Xnp5GN*m@naUh@>N>BFFZPMq?EGus))O;BV{g?^kaH@=~3fssX^Ng zv1abC{vUM1w5{^hWSU+NQXm2Sjc-$!CKJ^#m^`U7OO#4cV95Q!@$vaJ#Duabg9gZ2 z%2y~!N@qc-*e=kfKPF59E+{fNI%=q_$uatz=N~`5#`Ftw=@d09cVi zgG-qaSL?-Hg?f@GS|7Q8D|Fee1eT>vUleiT_5#uIEUtIZ*KSa%stj{yH+p8ldp)6` z0GS|caO*M>=KEPKBaylaHDGSjkj+e4%oQ1x9OPx7GS^!lgW+A?BRp$!%iKw6ILJvc zM~yGYz3r^}6BZZ0Xa>y#&S-yb+aq>t}&-twV zRL@r_rRaR|9>Dm0syTxUTc*WYH?MaWI0WZdZJnM26$|?Nfgj)n0HFw;&A7Kom-k86 zd#SkKJUU%>lV+C$B-ls9Jq((rjyfqcADIKyhG(&nW&b(leiY~}O5w)4%{7m(Y7FSn z<3!kma+sxlRuX(mqQcX8jR7AiP!Ib6$D^UmY1{q{(T{sc(l4MtwkGrr;m%?6;C5T^ z-NR~xw*_+MES@xBJn}>;uAnx2@DrGnofVT!6UM|`{}_SO7`ILG~TW0Kr| z!;fWXhS?#tQ@@bBPjt7tY)O2#n+NhM}bY&Scm!S&uDus?J4KwL9>)Snsk zT!_cddHz`Fc?Yg2pKR~?PGhqa^JA@3c#+>tDD3R0^&n%D< znlq@?j6FXp2|H(rv+oknEWLvC`nJYEoJ^DlWZYuXx?V%WCi6x3t4=aLp3p(XACzgtiatUOrNn>hnoi+AsKVY)O_=Z zPuEEFr*@3dN$Xd!LPDY*>AnFjc;RoR>|v$o&O3zUzOISgDJ9<0izhL z7x)M>RbXmed7TX-fzc)vU(BcwwXDq(q#GJ2YUU} zkVE{o*pTy^J!&S!$Ze3Swhk`FZm(BNzY7k=c6D&0JUs0U03}|8Wp}vu-oV8;Td#F{J^@W!H+E@Je0v9MI3jvdIRU28 z{^OQq64`dKX*5{qQ-+ivN29FNUsL8VFX7&1bc|R0f0)dh<+myOLv$vEb!rrQHxL4% zIaFTtB`ehTl2rnO*(Y_}MyFk&V3VoRs*WGMl-uLE;i%s(^>9YgM1hXo9ZYW zy2UZ`UE^a;%&1?u3~Ff*_q@P=Zu>M^zV(bF`aJFQ~aVB;mHtFP{tsu13wjX3d!rogp^I8B~Q)s7H=TRXfW z$MRdSgztFOf$i*{0I^r#7l5i+#v$ud-ylF- z0)u(#>ut@cdh_*oT6k1s31PI|dh>c;0iETy3_g3mjpkiP9AcfR2mX3MYmc+#tOao! zq7=1vL-e0^SafvC@Y~OO@rlWi|7XJ~17+LaS;?$2I!7Kz4@GyCxgY|dKQor&US4{$akO%Mb zR}qDoY>tOrb7R*=(x05%B<&!!Dw+J`Pg=WCN3rH^4M?Gs_tS?NoZI zF0~ zrEuI@jRFJB&;e?OEj3l8D>dr%#ICXZSvS->4#X9mC6~|UazM9yO8Iy;fp= z3mh?Wo8hcYJ>h|ni6yHBp4aXcr{IH!u`Y{;*I%dYeV%G_x5u}`vU&G1aV_7hMRTKa zWZV=z5|g%hy{hjuM4+9VS^P7&QD^rgpxv%6GDrGlQ&Y&u2X5d|wbb2lzGu z-jEd{2p?H8hac#(#9^jS-S#PGU-_b~0?-euF70)9e#9u}URB<(x@Q|{^hi<^P$n{l zhtO^5nB#o=d8%|m@xDM@QFY%L?5?r6>(6{sFrZFcek+XZn z9roOxB3sCrvpHDvdvCp~y~jyNl6VVk$+e?wgnXd+?L zV`{*2?|&;U^GqRFj6dCQU=rfm1qUy&J3DYD!VB}CwF@jst??j{5lo2o z=z<9JDUmotGDl;l4-9wrU z#41lp{c72`Lun}#p*jYk$lI?NcV$a<_}kwlkJ&$64TyPc$M(f<1}LhHPjdV1%lEwS z{wGwZhW_r7QqGvrnSz;&ZnZPc1n{45ZAueG%fGm@M+8g9YJc}D+uM-)b&)eLyQbDT zz=_%|&(amqcmDg`6gdXuB8EprP8aG;PEg*Y$Thsagr3F?+4FmJF@wDE{zSAxVsq~< zv^_G@Cf=ZamFXB)T~}Q7(>qQ&&NA0jV4q1?Dn*j}e@LTbLF8&>P^&WZotpa(EPw{pdK_O%V8?FVU84;-f6N zeX8YGQSTxFxvbdDtCcVV`z}h)%iK(>(y-yDjQGS_kh;AK+ATq}b5K!yfpbkl4JhQh zcDpc%TK_Hx%m5`aXhAdHITk7F@Osjie3;e&XazQ@)VBk%%>yamV0NnCuw`f0xlf(2 z7>9U#Ppn<_-eTHQZ$Z#=`_WxOI~aEce0&yb;BT~i+3fO&D};zz<(|CH7IA3L zM-~85dK*-lh{jQpZtqSl05Zvdr#`#GA+}uuck2Py|+K1)r|qD$%#x`>EM%`~}7 z)5j{nWiD&4WnfV#FO9o!0wh)WELKWR*-kgr{(0%Tey)w&`4e+78k+Z%;R(H8y6tuO z<;%l)vtvMrA2MpG;r$UBr{4e?P*H%Yx4%NR(u8>6N#)=4U*nij4lqh+clj2@6?drq z8@+=caqouHeFesmvaW3R*Dp)c?C<+(F)u*Y4w~v7xpZTIC*wFILpWii2kxUOvWyph zW1Jsuxe8wDIkm`CXIbwOSY&m0*_x_ecS9rAZ4PiX7_h*k0eAaTO2CE>uD^6P{-e{|@%`rCAu-33Z~q;qulgZkM4cP;?R&LiVyyq~D4*Qb+mkZR zyz;7y_xsB02P>;>@kFJ~+@U$*ckx4(hMMOnTXXexxF@pOTvikl6RZ4tak>rKNHQvZ z+t)qjfzUBzC+H_>6iVmqS`S}u1K$n!$rPyThmP@tvsEgS<9z|7`rl3};+UY>7U0?R z?Q(VB=`FahmJRGN*c(VV(^d}{TI!(jX&wCCn-DaA-{zLaNI{aIef*p1+Ks8KNI9I> z{8kwl0nXLe22`0U)=bx~FBAkOfI!{|BKG!?!zR%zSjd?tiuhlXQfQB zKBCW8HXjfj+)ernD~~3pZ|lm;ZjeK|_v0zi?032gF29q(G~=##_AT%EG>*(Ro0vSd zdO5FoQNK2EAkIXE&=S+yu-G?^Q*uPCCIQB~X7^dQuLG=o`CaOwpTD<~lN@|DI#k*J zC<*kHbI(M=`?*(35^Zy^ag4iU`g!FO$y}YjKQFq|ccv#m#>c`?6cVdtPY%~j4b~{Y zO>nJ$TFB(}%>#@vvRy-x-$PXpit^pahFI#mAD)|RckmB|VpdocEN1ng+-o$gzd5!~ zH|+?>&4zG9%<}X=HxGn?M52?;m!6-uv?}sNjMF+2V`Lwx#BC#4Al4||p-K-pQBG1m zLv3lo7p}Si&6e=#pOcAL+m{4CL2NCo^!}QD3>|~F(q%;jY!tWplgo;Xv54P) z^YZgaim#E9lh1>!u358g=k^(_@LC`ecE9YEe3)Dvef?((Di9QxuJ?T$nPe3`fNv-w zsgX++=0?{M2S)7FyL;^WnL+&EZ!g<{RB;-vY{r8EZms?K44s z3VE5k$6HD}mlb79wg=i_%i1CK`$Y6yaItejq_-^?r0#EGj=3oIyWExDBvB|?!+z%n z`~G>T+roM9$$nGu;3g@zJ4p+T<9xEi8JhT0M;0cKvPV;PeeWWg{ckrVCv>AAWzWp5 zl=eK^v{P($O?|}tT0h~)k0B1omLQ=*4%mxPMF3W)k?ugs_Y9e@ld!W4u{-f+86_a` zeZ6?$Qx~n6Seb}`-QfF|0yld!Op5P#QXUYX&zFe9*zN@|$q&w*aNe@7J2tl+E5n^W z?)7k-bQ@uKke=OUqhboa+wq@){B2HW%UkqKCx8Md_#9eq`(Qey@a%55+vTtn`_vA| zZ-OQC^2HVXjcv8NX10sMQ1cZuUaHFV1c%aq95q%_7i_^hmUlsH^n?CdUM3$I-H6uE zr-|&o1!oa3h$}BY(JN%a;o~vN(<`jeOnh_OW+u11NFy=n!yo6GM}n_w%Khl<2#X_J zL1x<>>urz)UZdQ94-~HV{j?M;K!0>pCZ0XUL^w>U3=aaqSH|nFT#Bp86pJE)VX zB8a^A@H5JUo6|hh%UKqE=A>zoBDyxoVGN>!GGLVRz~-*#i*=^w$H&wFKX2#7u4~yR zBocF5&r?kwN#55FcK+2Aa}C}JE(4%wagc4eiQ^RsXMAb=n2gjwf zvEb?lGaw&1XfoNyOl2 z%qP|LL+|> zaHW4yVhgSI>>b~#IX}b|(1JpNxZ?9SH;s{)(_p}%tR5`MSS%hdHibOFc%E;JJkQ22 z0>jkC=y^SfjZRI)N9gw_UWck{;j>CukSB1!l4TFb`T%-p>WhXkzoWv>unz`2>O**- zPE|gG1qWr=ij%R5>Nb$k<26&_n)H;}z)DyKQu(5Ca{`o5v6Znr>RD;E_ia$YfT^4V zp%Wp`gi02C2f4W5AS4JtM{s`aY7{tCeLw9;d2h$=4HWce#T9~=pKFhtWG7CdG$kLK&O>Z*+sN&0sg=pPw$tCeTHkI}Yd8pE>_Ym2!9 zS!VeaBlrPmIXfBM*)~e`(T5lsINpWzi*U@TBj*K>s(}|g_>BR zfGC16Yd_7Z;oe3MHGkW;Y1gx<{}MWw4vui&UJ;y{d>5nr4e{#aS83CBQ$|pcC!4Zt zerG}Uor5Sj>I&Dj*PkV)<;t%38at#0pPf+n(sc8MUV5nbBk*%ug^{%MSwNuanAn2@ z?hc;c_ofpPqBkckr^#m>?_l3apOqUuD^_j$VW~JxTx4=(F65_-ij ze$pI8vB)q}{4;fJ`oB7boZ2DvGyS0|LQsab5nd2%T6&%$3nh+EGu$3^*X5R}W$kau z*Di##%o2;eqt&~h1!7CpsFK;Ix8p!B$=lt2a`-!RAM&pPSy?$1+k4)EWG!f3#Riu* zA^4K!ntv;|Nv`9oSD(b_>)o;2nyKh=UO%M&OO5nnk-y#Kv}$N0>1StlViBRld^Hwv zQ1?-r=jS`_Y3=MTB;}OTl6p!w4dTUZuY|a=H=wKoo z_4Hi=(}RQS#nR`*&cM)qMu8UlLlxm%-bv$XG7_z|;_`U^Sh-i+?(zFyI=ObP2h%#G zZpH;q1?|~VwErJXR~;7B_IB+6l}m?&(%n4+Sb)+IPz3=5UM;tQrWw~KrcDrm+Il$ejIVkS2kQqh(K`DH$x^sr&&~9|n z_7mR3$uve}2&b>%BgcD4%oeQk^u6fmz%;|6ijox;p;QL0VplX`>id*eprUg3ScUDx zYsf&SfecLB{UpW`TJ1Tas9_NEr-7f__zRdy%Z~JZ>RW9e=g+~YddF)xek5~xcHMqN z((c`h)H&Z9rmxNuMN3;QFCOevAAaripV8MI=k5pEKTO>0ZttJIYA}uoG?L^P&`{m~ zTAU;tc9uL@J|K*M@gCh)&o1)M21WPimxtOE?Is`RzDEX`r?#bz`g_Qj2+<4})v<1#n!K*PqYjr5bDml!e1!Co zYwd0|Ts`^S&C*0#)^X{Vsjfd=d|w@Jk#E+=t`?#SB&zcsk-?K%--bI3|8>_>Y4d;a zDm>(`W5k(A?VhF2O1#7WfEN@SN)X(+xw$`uN?fuoa!^~}y4rx<@4d`EZAEi8Y;EVT zq*%*G&KNXC0sGlb2yI>6H=Y=SaXG}&jTWA(#q*PvdKlm7qN{ImHz&lp@n^eyLj0H6 z@A?xC57GTL_i zWiSgyltsPj>bq__fssN9?EEd&2rHiZc9V-t3pTU(7q1Os6^AP_yPfbI(K;|E zUG2oZ@!8t$@TPsB#7c+@{4Y2}_-W(X)jJ8MgQEE5;ye>BdL}TJSLj?nrhI5f6P$w> zCO;+1e0e;nwA={?2W^i$shq;WNPDB&Te0G$6c6H7;lMntcI^fO|J!oCftAHllqj z9ISkjsHabyu0s^c8U@C-&6wQ8@n+pe>6G5l#V2O{eEkUwsdZ{O$gTo@fFLDBzdsn_ z!+0`kJ~2wBc3_gSbdeoA6yD6V6EddSd?}FGE^?=lXNWGe?uuik zsMvkS)|Xy_w5ryeWd430p=#&liG##I#L~?p;&vV?Be|ILe2Yc4a!q%bqW%2)13J&O z{x41zy6H;|kBfKMS?~J|C?wfZt2dE>Deo%-nBQ%BjURwTiOBxHqZc5mR>PDpQCFcP3WAaal5@L zCim0wSRtm6NA4aN5*#6ubybSyn_l}6mrYscGTiV-$Gf!VK2bpV2@{g7wuKWgU`a{Iy%6H=xuEE@n!$`_o4mc&MVV^ z?SK&6Aeaz$^A?%p&RIdQrBeTVn3j{;3DmzM+^NZvS~_#P9~|bIpY1*IX}My)nkHJT zEjgNhw3jPXx@_S7I96EW84OGvEL~-2nIum6n$1^7praI7B|CDDwE-6`Dxp_3u81UG z{)QRJ|2q>8ZcqG;%*y}7>jmyMyl|?JN1sR?>eaEn+>90+P!>9CVu9P+e~>2%`Fn*SZtWUYEbOV*n^+~1(xa4_^*Vl;w z`8{w_;R#is|EpfM?IK|niThRQ2a^C%8Ro}x}7@@Dekcmj?2H_ zxJ5P=h_Mjl3d_mnsRHAt{_1s4KOtWjIpSP(QJFaWp$hT@16)83n<-3vedE{2 z?(D;BC&08o$!t%>^nrVkUCO~Q5~+2r$jtqBuV8vfdSs;)rM2IbI{STh{%8qV8FtHA zkq^w;Bv1LP+)K{7I20BHkFgGG18J?=WD=;c+km{at!)c9xR|n5s1e-ZL=PJ>Gdhzu zwCC#$-t+o8ku_zR>hN*dvyA6 zt8CdDrdtm2T{^AhHy%xv5^?=>xPhHoZj_ttliQWY+}znVkv$GRxNmwbJIyI)4E{Bg zgQFau%QBqX-_iPdc0G*er`Qj=6r+6ttlomXP6nGU$8b&ROrG(CrEdwFemy@TA1g|FGDA>U&ANhh1kr>X z3zat)^kFGBAWQlQ7%`(BQR9;JyTbo`JF`~td8jH3xSQBZ#L~Jn9`-Au1^OuJHhM#1@@0TjN+UqK5zI z8h6?&k_yt|;dP<;vm-z9xKcv+V3lSIZ zsy}AhyTM^3E<0cZ&V$#>UUfy?J7m<=KhTLMV`$#o{X?HF<1e{DU*&B(>veiuc;1KZ zS>IV>>aEudv%Gs-CQC(YUZ8Yw^Esy2h$G6*Xj z2)91GmTd3YYpc0K5kN;uA1S%Kf^40lHn~t$%>R|y3x;OJHZAZNwPGl<959#iasIN-)#5%AFqtb*h zgf#ceJ9kJVYejqW4Mj4?jI0-+s<0R_syIuVY;*a#&YL0OdAvE=Pi-0;46~M@gq7!} zjYFK6zR_#)J#Ip~OE5OBK&zt)OSC}EYaXb0R%N<*c4N&d1nK&#Sp6tLw@;RX3lklL zb%yLVh?)e3Wv(>v7u$Y}zHbWkUgB0x2g7&MH>BLlL+7n^DN#?{pbw9hVtT<;G$HmB!Sa{dJXNmOvhQ8wPt5fgZz z59-G0&#RNOl>XoT=A`@fAbUR}q*1I=E-19c+uot=Qt}h(QqS9W>ce6uRO1sF)N^mY zw`sbJMSW$HjsSzN zRMyzW{WT9a6?PZ-I!)N6M%swf&JM&A?qS-R&;t8-V@~MhD5@A7vAVbPq3v4>!LFGt z;o`w1rRwShEMiTFNd0Cl$=Yt;-v)8|sTIUV*<|MNT+)^lkykpEHg+tnE7~5ni92eJ ziIaq13mGC%#F`KDD><8;o@F~!#gFnk`$T@xk$+rc8#bnD(HkuV)0^i((qzcecH72J zIyyPbE^#_q2Af~S(5HOfWxqA4N(EjDsH!h(78;}+t7?vPd{>W(C5d?Rty0s&Q~nNQ zFeqjT>!Fq1Fj~+b^Y}u$dVQ0zy4UEBua~X=%*lF3+QFT7{EEr~Cbowm!(U6jZKSks z{RQ)7mi4f%6d{Z=r_V!+Ylh<3R-HYgqT1qmcyNi-90nuzIt8;^XSRn)*W7sbK4InO zZbuAj&#=bNG`tbFzpIFK5lqxWD4o_ABIwS)&)0nP?_Wk8J!#cwA`$rWt+9S5j+|-^ zTa?Yf0NFbz^RWyMD*+H zxw7(S$vxFayzP0iKNXgkIC-)ZEvr!NuK9nj5`6RAp{#9EHJ(4ta`l@5LTUc_aPVq2 zu59CT%TsClPsRd8vN{sHCvIK4)xz(kY~xqnZX-UlVe&b1`X|bDC(ttWCx_-NuYS(G zW?&H1Q_8^SGHa}kkIMFw9_}jKqY7{6cZAD0*!YQM6)}q(mVd7>@e7yhn$fa?Llc`s zglXsIp9j#&4>9oINCiO5fj_bN)V#G^N;e=yEi7T z-nZxRr$i9BjfRTgy^O7z5%WaqOCwt~mRfxtbUMH8t}Nsoj-wkUcWkHnzTQEaZSM@< z3{mb)gB@yw-}rnpZ6n1Ca|4+>D~Yn8x)prdMt&neB2Ih-6gDBJr#UGB*$c?YwsNoU zw8Ar!vsu*lntT^Exe6~K|(p=aGK6)n1|v%oAR-oN?Bz$WQg;IdS?o)i&t)9mTW zZKh?chf=R(c@5ng#T+e}kBJiwkeM%HX{g8Ezv>dniC9~-MRpdQ`^Jh{aJ>oO{(V&cv(ziSmxd1InUXlU|OrzzkeC4Fx&6X0-g%2Sv$GLC_aOI z4xv5@X5atg^9DJ2_w!8#J+4$)Gt(Z&MQpkDFTOA9?mh3eeT`2TuK&z+DX@o#9B$<`z|a^4$DOSQswR9Dbc~ndoH@42cWNyF0ql&EwRwzNsQo0f|h9$5~3X zwDqV(b?R#f&2B?B-pd7qe*H0Tc{Z4QOfRUzAJ6loSL<&8M>&pb<*u#Qz+M6q)9-5) z=(8(0G{=EV=KHxI%%vV=_R(Sx2~~WCjeI4N_-3G&i2qcPRmqrmFsG1h;DV5z{>+$s zGXf3Z_ivEYiId5kusey=PqeEPes%m&-P!mWy@Yg=V;HF1hFWx6H6?z9AMn9+uLD?$ z=tLuBS&X2x-RaB0U|?rWnCS^V(`d%`|anp zB|9mDdX$LS-?v+@0gfg`EUQs;yPSEb=*{Ie8~*x}5~=xbpIPWmu_SW1jD}L8NgY=a zHzpjmPNVWZ^lE1KL}5d-4>ao1>-c82hq}M=mrwhZ6Zi-lmW=<$^5E>n-T6(%JGZCbPFiXIx~sA( z7VIPc3WD+5-!5%XSIN8IKCvCsWPFWd-q~B*#%GQpwXNK*sR0Le@^?1!P~tZ3U$-Sc z=#xk8TEHRKq3^k@Y7*$W>9k?VeZ^=C-7ziPqp#*07A9o6uYzMI_J=3qUj$?iFG9ld z07w{!=!Q-o)ScGu7x=#|=!y=fPlta@(RyXXOy-L1-8#sZrZM%vd0U|tIWwTu>kkz% zTHF`tdNj*J%7c^INy+M;lP*VmR?Dv}mOkpg_7D$qp55ApTtSj|(i4mMqeLPN5ZoYD zxZK+(^oBLn0Ktss^436HIm`H7Fhi;@p{e)zNWpM{&kZX3l~P?W)s&%Nu&Vc{c2vxS z%ww#1Q8j+D_R8_+lfBkPjyHbw6WgtMaB`85tLc~VuXzfr4&?0Wwx%YQTVuv3LpqUV z?Qit5W%PX<4a{4RV+0qw7ljS$Hp;vCw>>wXP0To+lj$z-4Tru6O<{diky@w{(M3sU zKUo-T8UbbBu4lt_KV@WgI4v()?e90J860{Sc}Kc7CaddjnrwshtEn0**(&vQI{s1z z@1^D&gcWq^#|213ep<6%B+J8eIQVfX6!T}ttk2I~6fwx0trtrP##$3bo>M`dsX?tq z@3+mULMPn=Hpnlrg-$#S!fHZv(cO#}z_DH9iyYBpr>k>{%fgh*#;58Bf5x9-@05*<*kS(><9O$XPkDiI+&{<`D`ty7m)fxFw)EggN5 zgC5#n?={g6nR|yJvzY~Iu8;?f#_m)I!6dRXdW9INNy;@Q7;VELS35{~WrHj$%}cQ> zh&6__k$dUgoGEa($#Aw5b9j`tLs3$~!+`9@?|v9+U1_aBo|U`Yze>ZcIk=ZQHxR)b z`abnLk7{)Lq^(+f|I~Qt$v-Han_mj}{Ep+B0h`vz`6zy8CF*mCJx-i{oJJ_0Q-^0V zOqe+HruRKI`#WL=oY04{P{(i>Ep_;TyO3l2UBIyL*wj`@Q1xO~-l(wusT-2Z7s*cj z&L$SJzgX@#V4Ctb{)LT@T;`gE;=r%Qltawk_}Oc3M(u|T>g2-3JLtVCz|1u6lgL`( z8nZU1KEeC2yh*En1Z(+_$=NgCO&B&SJ?>-q)a@}^IweZ(3UQCJLUEuGU?3!dPtm10 zywM%12`RtY&gU`pdgx0k%Q6?hMEv>Ivon8(qE>0tEWyJqW=RAPNgrDENNFQHZDAvT z<@zEI$oP$)AH7_*yl}D(rw+&u24%$RuNAB@x+w)^X0Bmy}b*=TpIF+%Rz+0jd>Uo`8E-bRcg91nlH z8xX*1R=G{%L~wHbdoL+IBR1}hj>G4W!z#y#Lxr(}gOvcfkQ#{FFKP z;Fd;3AAzLypw?07T}?88ioB+Et@+l%33a~*w~>TG7negS(u|?pL|ywc20RWpzouo< zfWqJ71P2A@rhdbOy)<|C5+%Q?(?hMSo1tH+->f2@lu2D}CwVqOp7^PST(B2APm6qO zyZaHtSzcK!8sQy~tu1YJz^!O^b_!vF&f$S_^F#prvOs#7d`j@L-@>CEvUOAEIW^cm zh?x1TF{lI7@Li=pf(Dt&TN#A;PHm=`8*)^xy(vqmVUcfSRuxb&Eva7kJ3)7cM1kJ; zg_x^6q+Fh9RdY$*2cuaCYBJpiU!;OoV+7 zW!jWU)B8HxP+ijS)6dz$#8YeP@jk~qyhbd2{H*S;+2>_I0GE})YVO3wvK_h)0FVA+ zC083eD&DU=`e7ae3iVA0>SgEh!Rlbm`oHafXkkk@=|OVnR(_2zppVGt=`pmg>dRNQ zuNvY6ucnCUG*69B_T=%Ceg(kmn1iqn1!RM{A#UZ@qlzK zOtY@{N71LnYs{YD4L|G0`%3u#&_o2!vfZVgc!dotd@WPYv!f(3xXt{o=9*YW4pg>*vc=Y3q;>kT6_MCG!3>~UjR~>u3t0K#>(gC-12+iw%&N+W}`t&N& z!40a4U38Ip@(p7sgug?{#>J7(tFhLBN#JajOS%vqBP`NDdnw|ILRZu$Rg4A8X9zq? z)Ida~6(=R|vy6VhE8l0)O#DG#xvui)U0KbZPuUIv%rfcU&de^Yc*s>8+kOsx{q{1^|cG(}uT@o8=JcO0~;li$c%lE@R?p6$oVMjc!PM$eAyMDYRNzcQ&H@H)sT& zkA7bawPr#o|7DYQm@La^Jh;NLY{PvS4=ectfQNgJO+eev(I6ejW!PZ8o>jx?-uh$H z4W7}878`(s5CHLB0KrQTLB$EPBecw1wCW00&o6$~I=9V}y`DwWAwFft!P%6vnv4cD zaS;eAOWL+dq>}-)9_n0&)X4Tb={*-QaW(03*~22UzyFSSb;4X+lI%GBQ3mP27CQOx zIu^7;mEAs|u;0QaXA>0Wa_#z$F4qjx+;1K>wsDgoY|Rw&k;auuKu{lM%m($6JFiDx zY0t##5y-_F|LbW{I;nC0w*1ZOhIje~!$dCyV5(pzys|9-s_~~(jCNr?kx8#AS%-zr|E91H{_eQ(hWp5rlTE}sQ*0s%9xy$6;<aI|nBtK z@%(f9pn{?^&tzBRz=m);e^h1KGy*oaTDdefD1ICfYS~7fImrpFJjg!eZUB07`-ZTB z=QkjRMlE)zC5Of}>MC-$lqnGYyANo$^-e%`ib7yMW>K@DG{%d$Uf_w@Uqo(~M-o*CFR_cCvMD3eWRuC&5>^&aZBk|u;@e0V~&*@sTlXmVQY$#uX5+*uvOWRahQ8ruv$PFH#Ekmm0ZND@?L?usq(Ha70Bcv^} zU0*tacPKb(LiZxSWDGj}FqKDhx4)R7w&fiqp#?H2Tl^=EaOk$eSH$v2p>si~%X#lj zhpLm$Ck|Dsh+x%>mLX5~<*WjPEmbvIjd^+1?C>Gr^pj|@%x`!QLu?=5E6+R{$!IFd zttpnDR?vH%hEbb;4IeTn%c_skVWt;VBm^7m$Z8>`7!~13HP#r51iZdTu$9YW&K^+6 zks6o`h&-@#I*)vpT~FLJe@=cm9Jyvk&*0SwYL?|%7knvT{3X@o<6wjlBc_e@O1sGY z0FDPA-uOqtXibGN;W#O2)RhGY+pG5y-<0VD&<@_V=R@f81K)Nf?f^GvTL!nj&HA;s zota|p`C13Z#5~kMHnW>$6oSf<0rf~)KelYwDVNkM71Xu*W~rVrjDx#Z$m^pl0^Dnt|sExP)~53ShypMid)+*~}8 z-=>=LjhaPX9gr61xXgjq)Ix6j`T0ZFDMtY*b=t5N^8W&)8ioBVAw|D4+B3t}?F}w} zWclc2acFK1KK%xR`eUaZUGZ)nZ<9)KQ0SSYT+)jt0?tflCxHfKCZo4HN~YeR(Z@)YSIVH@ZM$6y375`XqcD8vHcmHs zztAabW1kHmt#naq>T}iazA@#Qaxx46b^nCU)9zH?Zs=iM`t#B#9WHZlp$x1pJ9{#_ zRkoQRCZ}r@Uk6|u>lSsN+B{9WCy;h1CjGR1XmH!Y=nfH20gJEVOFw#!H}H~ePnuk9 zy*Es$v$Lx0TI}h+uRVKBg}Ud)M6+HZRQx*z>P{(gFQZvvtQ33!CNC#yzn95$UQ)N0hsJs62VJ z!y&=vQJiJ&>tMJSbx+S(GEvx@g>(e6Lnk8hGfr!#^Y7xpC-1 z{2$5xfQIj$Ij~;=OpX~_Z^H&RsAb{x*n^g#-dib>^?eennU$4Z)l8r;aTw9By7^c_PP|WK0jy93aF|`$O<#as zQ1zEqz53r@nJ*Tdz-7P5vWzE&W^a^!oAPr6{G+nY4q$e;Yk{M-ti-}xy}uW$H$V=s5kVL z_vvZfc*AD2QJxH_&X~KP1XG|QV6Ff#F5_uKKsR)Gw|o2GfPs#PMCwVQXSynXcz1gD zvWYG3V17fo#l??IaS}qG2r&#-r59|>{|UN-N^|(&c4}Lu!q^b`2}-HcE_;3L z+ya#NhNlfkN-$~n?G`C!g%VtFo}KmpJk(8Zf@w#O87xEH@^K~sUO=l3i1=jh>v$tu zUe(jBziz@9d6IpY?wk$a=)kK%J7iwIpq9v3!V02GA8?i12|S4vdsWYp8TdgbyUL$u zn*U8g5)LUSVcE*=rxs=Ud%6}DG9(L?R?k$Xer0onletT9(I=%fO^jUWbSrC^G1{bkr%N& z?jqJ;-bBWvhlncd*UKSZA5*I)Nrz8ZzRsi-WNf&IILvk2c;R01Cuea;%{4R}iazYj zXj~d-Oa}tG=VGquJcR19dUK?A{F*a(s_uK1XD%3YZYVBsLISL2aygWVVyfRnoX`XfZZr9FL%oj}!T^{EHi0gjJ>R?ia{ymD zU43l2Cg2`gVM!*k%(c&;4z}{$4>MOV!Ox=a(lF7}YKLs^ilryCXk`ihYy$y>HAWYB zR|59ln5Sjp#9X3^U-;~HooMux_MQN^(%&|hx<9r-!+CrKv9sm{_G7BOTI};xSKnUQ z4Qm-1H9v6v_R?JYQm06Rb!~4_Df8MXc=PzrgI<{UUI$GV6jtCNaK?A42lak8lmHc+ zu=&HW0b!vvXwpXyNd(PmsyL_mN_Ppfa%cXts!Gf6TY4J-7?y4>uo~JZ-nSWO0K?z4>;6 zG41P#t9Ht#p%a;dv8sB{8j2&3*#-0N-VCPfXoXJC$&th}Kw)82eZ!qPTcGoR2I1j2 zoX5DMl`X*T^7}lo7mC093L=TY(ka8x^FiYU2b-sSA6(ykrON^+nQZOL8RUyl!un(} zfOQAMTH({d9{T$-!(VIC;R}$5#z1lb9$_Ttia7*lIiZQz{L1G=huVBEwLX!2bBs5< ztp4Dx;wJ=efsu7x1D{gwP!SS)3rK{_EUxxIbhX`7_Ick{R&a|!Juz`hv}*&Qw3uU@ zamhp&C9GG^pdJWXa+>mPNgW{1aI7JOcT=*c)wD|GI!}9SP*DH@SS5N$lLj6 zfY*%XhbYIPrVh1ROqT&_>;sD#ZRvxFj#PdF`^_2+;I)}t!~Ik2K?E&mXgaA-*b>VR zZ6e``yKZSn5cfHF;;r@hyiu9mb$Ihp-;Yp((rzJJMGX01MxzCiaL`!fOY4gIWegyM z{4EfwlC!s05K0Z@R)sDFFu|!f+T#1NP=Snuba>&Z!|n4evap)8>4JHKYcnvxT^YBsxp|nT zv~kG22erZP3=q;(mVJ2rflXtOT=B>m8?cGtM`}WZQ4#C_+CF<#{QR@=X%8Pw^WRla zhoFG3q523zCv;;IXdTNx)XvXz$qV2V(Upimv&7A0eJHnQ(n zMrJ3q(0Ll0;Ls|xRp!I@62X+BQRl~-XDK9EtJeSB1soVizruZ%uCv+4gC2DzK<`g( zPyeA^W5ka+s;u_{9Mb0{0xKlTNvb6GqBoE`0fKf>mM ze6mBAIuLZB4ZlSNq2JK&#c|>Q&!7Nyy*a(+Rv-0U{wIhrFG?>FgYvT~KV*-?7X5O2 zXhfgb0z3lapKNFcpsr@4&w}cBB42@@k_Maq3zRsV{G6j6N%^MvrGye5nqu*Et`ol_ zc_l!LQqU1O2|gK2M1+DPdfQF1g~)U$Bn#vH{JCH`-X`s>^X{RsY6<(6oK+&odMF`R z0p2Gr0JR;ZHA+SQ9dD*HI`mz}@P84qJMk9}jF8}70bEwg!dpoS1zC0R8si#XZqpKB@mVo7M`8;?? zeT7txNc~ahlAa{S@GqmB!|9_dyg6TzcPqL%XDnWp{6Q$&QRTB+Gx2%zUVez5paEjO zkZpW$i`L;~oC8?nKA^*5YkY%4>+7LKARwR7rE-1dU2y#Y@r_6q)V+QB(dLeK4u;~V zy~-e0;Rv`7=EmUtB>N)kWp_`?y&FSrqRn+h)@2?tg!P$XJwp@ap#d}KzekGeiN#sU zS$3_!uaLd@UQ;$*iU)ol_?zEA(E*ylXs^^PnlN$N*B;CFAG!spS!aJt218C}rsKhe z+|V%5aI9mn!$i~at6{X3gk4dS%oJ+aS&!K3lt|2Cde;XcAm!TQ1{nlrD1|??U1c*E z3JaNnzaj{a(1rwGBn1HFX$RdxYfkh1x4d6X);OM%KEDRCn>9`H4yTxkI@fRqZ}w&1 zfFcgOkLtq`mYbF)ao0dZK(o&LDuIrUR%y+y44{>;z5#gCBzl#>3HM3QJpQmJJ$I!zQ8(AWznU^ttN7RuWO)Z(#h=^%RZ=o(Pl;L zmY>tEBqT{?x+8gN)tM0O145!zP@s@y;60xx+fw1sGL46wTynDq@RO1A;70*@A0|k7 z`8$QYIA1tib$H?3Kx)^zZ5fKp-Y{#7hkue*`$Hq5KgCy-MXg}sOSP)^nm70@FF;tr z8~M7N?wqvBjs*^V0ph7)gzSGO5`YQ5TZ67on}yVhLKgS1GM)0GwDz z%>96DeFXxR*U1s5*}kkdAOJvURpGa&n_%z^mv8&e@E9|Mq%3(ow#aEr1Ks&a4*z`!nY4`OX2EyTSBem;t1h~7(TAI# z`Go7U*lyaX4GlauZtr+$^A5>jf#ONxubg%yIEB~^P`3uQkY9ShqJ5o*qrN4DkB43? zPE1la^+KMG_ChkWCm4f9iJQ0#xSiUn-vOwA&s+ND?vMG2pS1k4N1+_96t@%GHh_P+ zIn?~SE<|yoV@IXVf@eEErBv@Ea2v`UNh zD+79U28BoD_o>GnouqflCT~i=@$)J2xvgvo0uYSLISgeVC$>ceg}=R@d!KW1Ay0Yd zV9*XQQXul{+EAcab7kD1%bjJsgbh&;c3gP!lp9pV)jKp2KifduS<86Bt%h(gdf?!- zJPxsuPWlUkoJ$cxX^YSvK09n)X~;-$#*;>@#hl{Un{)`W5IT0VhY#OQC<5TQ_!+WPj zSQ>EnY=GF1aY{{T0YdeA!@u0=%=ej3O~?ttg0rV62pF`O-t5#|;fv`blsl}`Z+xk3 zG5~o|eV%X}xuuiEmH6((6)(CTPe7DGL?lVKT8-GC)U^xzOp+b#DMDBQ|ab@mNw` z#vl+&(`yIfBP)piVBxP%UjbxqcML2nVVvB5OBc>52MEh0i_j}hkANG8ButR;%Fb}< zaM*n=ZT04ThzHG}&f{1-u^}3W&D9kJJoC=aRUWx&TWqM87-G7TQhu>)2!-O4h;JcE)^>40gb!doy_yForGw z`Q}mwCo%UD&hbRXuZO#{Q6LU9X+XZ2`{lZJKk3iMUL;&(maDGm6Nqy1od!8M0dqos zfd2$2<|!WsfMJr2QAuwO!D?F;U_E)ea*Va&;0X4wo)4c!A@>+x963wwrYNmwJZ-P5 zUwv7#x7la2rBRHUJajw0$4T<<^<(5?L+q=TV~rUD(Wtlo+xHE^{`MC=+^R+~NX&tEPe}Pn_znlTc`fgpX|xP3gwXl& z7oP?Y+|YascsR0!+#KFbKM2SfA{2ncai6thRKXx4%55T7RpkSI0y zgA0z(%NdXjtRQn}ixGi1vGWeK96&Kh*c0!5!~aUlZ(btWPE)`^xQDl&kKgK1=J58wk@_CSE6xS^tim!=@XVs z?~;><8wwi$mt^4}Zo(#Q1ae-a7$W6z;4w+VK=QRE`bZwPevnsN!ad*E!5aw@ZbEe+ zWY5Ac_#Q9(&}<4shJxW8_d@d3b)MRMbie^a?Rnpig@=CSS1$=8K?#^o(h_UnJc%Ec zH9vMyY(6meP7SRtrsXwZC~$H6U~wPJpML|EK%Zs>zt_6D<3KXan#1lB^3Tp>fyg*Ffov=FfHH)Q7-~uY2 zGlOt8nBOuE#E&YIr*jW6jS%UJis;=K%{oFcNvsY67q@$~CHq~Q;a>rzTF@WiA0~B{ z`_NXdmoXxBKECsr07rX(WhnI-QBf?li0K!z27DawOz#4eFA{&cbY^_fZS4>Ta9p|+ zWnWlYYrJe0Zvyc6!m8rJU1W!=|CET)SLO&(0-@zTXn>0}^jfd5f$yE)st2CX4ku2- zSTXI$tnc%rQUPXi^2ob2gr>6<0C?goeysMhvONwpbFT74F8&|NTw*l%3r-(pW5M*lV`A_0|p4|v#vXHM!HbbU=rqn$&3pa0iDg(MWvu*o1ml|!- zw{V#-5ZjF=z_VzAoQ?FTiUUl;3fzg^m;ZLN|CNv~*_~qyPt5rpLmY6RE$F&SHu2bt85rqap=!^7JQ-G$U`Imh zL{e$@^GDc4kpuMx{Lx!pAAve@O9zEzyeDP{`{psCet-vh++lrnz*%a_=iEB z--2Yh3(kPX0k;=O9F~tQ%SecCQ*fF2P#me1=H8}jQYY8$2$u&))lz0v48Yr7w5nbc zL{Z-0;Iq9qEe&)5avWZu*8};BVu9Ra7y;&n5~w+ggKKJHNMv>^1qv&8U?4BD2;UxH zHUy5KR=Ap_d%IHNCwj0!~SvT%brJBkL#0JJ`13v$USUr5x)9Czc z)EMnhuH50#1p|sSeI>-g4JrhS_e~23@t!sriFG+^Peb=J3Zl9ImyxggR`m!3n=Ok; zSO-^!;z5+o4TlEjBoeaHJAMD{MpD2+L)Jk5?tRuz?3L}Ce?t*#+2*>u)|B{SkPc(r z0cy9uH(u{}SPPIz# z--a!ri2>PU?HdtoK;z?=C_|Y*>t85(g=egOaRFZ7rAseY>TAVzYG;hwJp`dSrj6TM zXkXipV%*)|V?X@!7_-y+sY1@?{Cs!YoYtE>rk1On7J+B+;-|cptU`wAKg$?!(a^~r zUeLkgo^>ObXSv|Wx7%lNr3K9vz~bxhc!I7o>peopGV>XDnJquHfu>CX%@(XCZsHi5 zwonmpt^(~;KHZz?$k9prndb!%p8!z=7WC<9xx{)Ki-6Q-o$cBAU>Hv$XEE;nTx16o zc|ZuDq0;s+DBzFoY3@+5Sdg00QSFztm7r&YU)$zTK8yf?&)M4(g}`BwS2$b3pP;m7 zHB4g^&GG<0zZCI?nhp=oI-;zJ-eP?b(D;NhtS*i7MPW1z$;4=*ajO4P$mDb-20aaq zDIHs;4vKO3$7pQRYb=GMwudLjvOrxToF&Qwarx`uBb!#&SY-CN4n+(kcq6yVbB$<1 z5vco5_^a(VGhM=YLiR`w*TIU)+Wz7~o;nB+xYV-pg1ZNM`4<6C>81Op?FfCS$8XQ+ ziaDyvz)1(3vQOJ_SDt_!;dU^hgOR+@7D#N2yo`G{7*1GrJ%OS7JQLXQPi=g??r{>= z7;k12Qg|`L+O=%%g?(c2SLMkYF%apiaPT%8dy1rg@sE>V;#zX{bbCB~q`8$E|55L} z$GI-9F{p_X#P!ar_8ooKGq-F7?fX6^=|GidA%_s@%Hx<%OdNyR_01;FfzqL8&C1H< zQc~T$s5DXf&>8Do2V8=JQ1KTRi3nFElPbFiO~^tP&sYAcY1zq($RtGZM9BT$8vj_i z4}=dOUPwtbn)zuaSr0n`cSjDf2tW0AIuFD*UL+D5MrH?9c6?pb5}T$<3wPDiWdgl` zrF$*Js`%n`Vr#*SGs*5qE{pPY_3LNb@ghWaWjqH(WN{IiFOHnUd(Lz-Jwl! zb^4t%NVI`6H^w!>fpbiiCG3eZyg0}clYZWbxH69i?YZR;&`_Mv4VmD)#gN;xn!rU< z@OwrB7i4XJRBjTzXgkL>NcJlWBAWX{_qG1A)*8a!|wK zzc+&eL72D~$pQZ22#`iWkEz)(eSidpz!RP@d&*>ehwnnQ^yV;=dir$ug>3}U?eQx1 zJ6bAsxBoRhet{!7eSA%4pTaA7m4Uf*5kfoEsH(kLHzG2gUi8Cp{|>|X?C7TM{=I$M z?LDpi?*V!oyQxtZy85LEb{;EKZrWG~IIRLfJG^fDAj+t@0|*7+wPn0^1rZfTD^$KG znOnI|)+Vc;3$csXao3QbE__sQ<0rbmFDt;%IK_yviok zZ0*$K@^lNGA4)X~QEYbhI1u;o+DSPd{#eK8veESIO{1gKF1rp5kJ-a2fWBbWnPZDu z^$N&zhn_HsP}@Zym!V>DY{kEaDP5>gN*%bTNG=fYs+Y3cTR~x>4(O&o_PTO0U9jc{ zJuu3?K_I}d%7$j_)cEFca*mb0KPYvWo!m24L`Acv$k@d?N)m>M5If}zGO4k^(3W<~ z8SC&!!`=4mUEagyQ7-kf^`E}jv`_KDbkLDNKtG^ z5dwM?hzLlRW>i#=D!qnaZ!}}ULXjf9NDDn!34(%jBovk2r354)@Mcray?3p5*Zt$Y zb@NwN@+-T{p84jR*?SsR**oAL@8FHZmi)fK=#*qVZ`-@AmjkqeuXFAh%^(;zf0O%LDQ&!sVZ*4a^& z^1|68FUNZSVs|l8`od{V;8<@SgAAvlfbWU9kaEl;%6kB%6A28u`v*#%5mFS{hn=6f z;zoiEC^D`4NAC-fA)yd9{p!w~@8&_EOiW@&4@t}5d6|(yv`<`zVczafnm11witNsB z9~Bxl>bzR!xc%L)KqLEAK?;(^T~f!XTLY63KY}rMZ^;hndv)`#bLNi%L;m{b-LNYW z0(y~x;hJ*yf+W>RcEO#AdvIgk;MTIiyDn%3vcu}XcQ7$CteHM4pIL7gK2V%mhO1p? z4LdvD@S2=CPa{9SsKOR<3>)W(IUpds*g%Nz57JW0Vz|~GPJvuI*!Q?Il=f)!!?86| zli@nvM4wluZc*Oz2y=xM+CkbR>=3!FaT^6lCVX!`;J6En7X}k{0NhIZqo3y7Mb{hv zKFJVvK9znTY#y^YapZ@hTZ1mioMG{38%3{tYoPZ&yR{hb!}ZUe7NKLWf4bHMUcB@~ zYw?0PiGSZHe;y}XAO=1Y2A;eBOW^qUU)$Dp7#ji#Zf|Rj$ZfvRuu)wFDGlHlPCcGH z{zmb4286^$Jhk!m24`(Rm&0~vvLig;)Ed+{%*73m!ZA*Ig6Fun6A~traB-f!&msZ5 z2Y!x0Lme2jzcxwu=dZWLU)~#PlYj@@eK%bwpuE`41=` z#+=%?)TAKWIM^x3;n|zi{zg6xQju6%u6v#tI{@!w``&if4laSp@<=!ZN<0Mh=j zc@XU$!0dvw0%+C&jVb-&=Tg`$VmYucx^Y$TlMzzbBc4%mbw%7tjkmW zo!zYJJdn{K1)TPd`GyDV_`oT6+l`b1MqLBaxGTKacR-Ixbr1*>4pdxvzee(3kk^|N zE4;}pK1i}EdkaGl-u-d8^_8$AF1H=)*OYQdcwN|cyY(XidN6sgT|v8Ll6%9QLy8A% zX}|&mex68RLU%EN;}GM~M!MfOj;<4lbu83P$4a!{S#>df@LZmy>*u;+E8RV;uhP>B zySit;I!WFOk;FWLd|9|xz=l5Tmg_sMG=SY87B}05HRdtuZ+V-Q<@lAb$4*vT7saTO zAh|uPZ%vpQe;x;VpDeEwY!d(u2=?8&7Fw?J3&6}?TO4}KMJW)g-c=b6&AbP0eERIg zzRyR=&|BHH`|fDb&|X6C!RXq`Co;Sn)&P^9d->~&f@nV$gI+Y8U1elw7FZ3wh&`iyEFdv>yFyBK_JN<7oaj-I9;K2ep566XRt0=vwr3>~1K!em z_B6KU*?ZLMC=7J?jWwp{cL>h7Q6?&7ilE|>IL@oXCvkd&miYChf_-y_l{v+~up@x! zXyrGO=Tf#2J@wQH>4Q-)H2Zj!>c^*`L6)Kmf2dR^Q5ElfqkX74Xv$T^w9tU5xFQ&Y za&k?Cl3^KZ#Mz?`C%O1aXn)412YghPFM8ZQ0SMT_ju}X+2I(q@rad1=8t`+~^N_^j zLjwb~Py=fRjvQN`G&|WhG&fljF5XUWj+Tpg!eNm%7SXhJbKgP6?Iuzi3=J|0X{(imY6YDI%1kf)vN zNF(D&ysPw4gE1QrL~jG{+cg7qvt=d1xq|Eld+?*fkdCb_K~>?`fG4Tn z`+mx+SL&ogii-Rm(WiHda9 zo}+D7Xng3=247_%0XSc|yn9!w%*8a@KDj4VJ@QX6hYx?37i`9 zdyUYHb0@`0PnG^yd|)s&Q+FV1@%E%__2>Rh(T)~iKrCwpcB`yD+2xDH%*u>$>e)mS z$Z$Txl-|OWUI^2vlx_lu8O{&2V!-*nyQ;Fqad)^yo6sti5!yJ!8Y7lHl{F*%id%!m zp$j5}cGssHvtM*=;2#ro4{2q4Yt((}=aq$dR!T;O2-B@ASm7io%5PMOHa+IwcZ5t8 zR=le2H#2Iisd=YMsYbDD1MA0HwJ>o46dpNEAe8DJ895|>`0(Hp)3dejZ*kHmqZd>L z78X~-GcCvS>2T%;-p#V6nHL$!3*Pkv%)J8#4#bklhpE#)f1amJQz#Ui>KK{!4Gz;tM67DDoVo*7%i+@L&j^hjvz_vhL9jhjN#*h05;$gmT3n& z)(SL7&NOaRzrJWTK)sr7&$U5;g;Q5{Tzq`63#n^EL@L22JL36xgPO{#sF)}zRnwOB zsea$)&ww#bxWBA_3scU$wsieYme<>VuEaS>@BA%C+DQ(yoyl-@#~y$^6MD({Vp78WD6wqXrytda5p za$v2P%c4ESsBheNFiQDYB%~+oVyRgDtsI3BUxEej9C;1uBXNjK>N0#PyxGIFR-VJd zxGp`-OC#%CYJ=1k%)ON^98cfx)U0wV(B=InHt{XXt=ZgL&~9Wx-U&)^$D$p^1Pc8G zmajXhUCSGGmXt1K{$c)Y{$}8i-SCs~BAo{tc4|L$+z#0A-#*_2$XlmY2wbnfBu*Og zMbw3sI}(xvMl6fLjL7<|1DphWTlNym94rzkOitv>(I(u?QTr#7hikG=E3cu z`JGn1e-Q|&`>)Jx@_Lg>NB_nL5`D16AfECQOFIk`yo`7--J5{#t~_@Ol+A`)iK9nt zOnKmi<0v)*AEks{5S9J3397bUzb>9L%Rz z$1@;}7qjKxUt3?PLecs++8iHZ)`@lj9EYX42@*8WJz?NUNDh4L@`OdE1qpgfzpXD> zExmuerX+0Jz5T%OKV5yp?)p+r+YS)KEz3(@2o8ygEI^T5SZiVdEArj=LKu9uCGh_r zKLP`=HK>(q!yO*MtW00XH~Wu%RluAbr5fh%zbpwN9>%}&pyvX5kY^ycF7$X?vrvv( zahvoM@2-&+OgHy(dv5^B2W{f`Knl+tQ;Gskxy8B zhxl8@&HYC(>mUgU{plK3)#KW$LRZji%a`dM_GbT&<^a+CVB5Y?=oR$!!b)F_?cI@f z(`c*F_g)V_{@YB=Mt)(vEW{bE71LjtT~|1u;l&~X`nWQCl>&P^c2s2xF(D5cF4Nku z+=)1_a%eN!L)?9+$^;1c-#bzu`N1iiWslb`Q$?X;$nOa|v^+SP5EV7%HTq6|t^_!n z0}j8Tk=$ee=GKx=y=d$8Aw1PMOy`uE&e= z1ujzYXk5+&!`~NzT2EAZ0ZPWH?fn-ecfV_nYSx4C$|b|FpjOK--|IAR{s3s9Qq_TKNAq z5*-pmJA~~^mg(NWw>I$jqI(C+s#++2JKlTH(_IDiR&lwvRG{LgQJNq|oO^lIyT~Io zzK4JoJlFD2ci9C8j`ymeLxp;Vy^&#MU(u<2d%A$Z(K|2WPz>uQ@c7Cy&M4olB0Qvq znAtcQj&u?bbC~+dTO0Y~JkQ^1Y$AREneb+p{=LS)#}?x%w9rLY{lv9+7|-T++t(pq0QzaW+)r-UEM&}Qq*H*#_8v-1fX~ED zB_|tcF67~7Ew?LbA4j>41RC&t%L6WHCkfjgrnXh~O~3wai|OWA`F)qPN>;3J;Kk*| z;S-zi=I|h4(GZJ^EH`MY{JX)Vng4Rx1;{EyZRS|H1hT^_W8DvrrC4ZJKu1YY!mLD9n4g|DoUD@lRGBpDg8m z8;(jh$jVgY(&P|pUKxLbUo!gF65o>3TTI1)c1hUD9#*w-4{Q%l*6Kxq?3X+^U?hh&giKbwrl z7EW-|pNLiKABwx^mO!C@<;I|DYU>ZiC#ygq#OB6`y-taiq)D5y_qN z)A_E&?L{*kessoACUwf6TW#)5hySF`YIZi{eZ0UD7%havDBqO%9{l_X(iC_ecwu4T zbqg~~&%an4v0H^cHPsK5q#ue~d9{0lF#BqvGbd}*@Fe${b}#Bw1eBV4EFKxa7_hUo zz1l|neknXTDM^b!AQX*Jm+`L4P-of!L;R_jzS@jJknatwn-yyY(z; zvL(x!6mOPu`9!MK%3|v64yMv^(n;-xsFfdO?X?I~sq_ByD?XW@)J^9JAIHbWj{8S2mYD zJ=&hiVEEElXgq*7k)%hE+$TOuYPM0_`gG+F9ZJf}lke@juW4grbMoxkio+4xk!!x} z31c2!VixW{a9~F$^M#o?b|@{&to&1?UI}R(jqaKOl#k2namP{9Ge=G-8^;C ztnbliMy+Q`r|PsBY72b}LG7bMLoX}OFOmtE?%W7aRl9eGiF0k-sL?LTvmQsf{_fnV z8iBu7P2m^lc=LRxNF-n2wAszi%?k4Ke}m$6M#jdkxuFU@nqXrCgDsO;|AP;YI0p5L z-+i4v){ltyXi*JMoVXS5r8&Aopw!vD^Gq775167_kc`CdIW~yA85|pHmT$I<+haja zB9l!>hX{-*#SoFjF&0zkfxiD2Up#Oe=#AFtU}J0RxY3fy>bo<+o^rvxN0)TK4=bhG z@mz=R2WD9Tw~qbo*qST#3k$_jn>$)JuCjYcW?FiV|4fdLKhB5u z&k`T7QDZ*unjfL%4<#o?k;JVC^u~2QUoD&pPki|BLAfu&E!Tf`;-(6sY5{-kbuoK- z=`~Yq%TK78Q}^5^N(#pq&?Bb6QZfO50J+;e;rI;Gvpy%FAU2cUxxnfqQ(@}0NR*k< zF{S$kNH!`fD|77{Q?eXkQ521kY3|IC{8W%`xzZ`pwms%NGvncpAKTj6YU$|cxTu-R z!txdWB)Vlyw_r8b+k2DjSAUgYA{6BUE3#Q_Ero80{@CcE;^M@qKQP~PsiVI|*x(rkxP>B{s5 zLpss{NT0oU`O+AV&rHpvm@BlqKf3?mfh64@jg<$~%tnITnq_?lsNZd}XI{x<6da3Y zw)KHTK6zuU=Gv|GK1ybrWVC;Fycb6un-H}SQAjHk*CsPc^mZjJKj0=1T`xB*@`V`8^IJ=13SbFJ3tasCr*H2s={(NhTKf4E z9AUNeO>6yi-$TF|RMuIA4)3^XEYktz;&b0*azcUzpOyQ= zfXSDg71?P?p<_TrLQ&3G#Mr(&cUoRv9?`f{Q}ud@qoQo_Z$EIW$%a=!^7QN5w{Isg zU3A6pDs#U-Fy5}kzIuMpMtajYs^6OI z?+;X%h9=0{H%0u;8ZVJ__?~@2bpH1pc>=?omxsr8w59Df09K+B0Cd=>J_ru#kTl+!Y)t|?-NxXlpO1o6`)C^Q7S=(rmWRIFx z_gXuhL?UH%E~x!^$1N1vrONSr6f{%Jh~LNlXefE<^`81^G76Qat)7_me=43zCO0XM z}y5Ag0P)AwaE-iVt) z<xar7P$w4xU!W`Z17hSk8&op$MQ;&@#&d%|!s9 ztff_S=&C>8UgI!1e?CUyGx{w(CgyJ#fYi9S4@KQ1ZCPB?Kw>m@S{3{9w0GssIQ(ub zh?DXtjqLcc$Fms52UPac+i@JExSX8&@zv)`GKs7q81(GyIza&e0UE_;vNub{kCyN3 zr80^zkW|wS$+@50#5ZkD`5rw_OwG&YdgS7^MJY;#aYy+MN@e)`s zkRXp^s-X8mn3`ot5uUAA?qMy(28xP`$`d5!`-t;Y7owsNx%cZ!8(ek(lyMGq>{I0^ z?5C>DkFVp@gY{uaWYJ-Spn!1*Ger2vms9m5J{L%YK=N~g_vw{06lWU*vhI4pu6%+3Xh_yCZ&B_~gwyx{wVHPe~-?%mO^ z(Q5dE*#Hg40pc6#@rr?sA)cjFxczzat#_qoty*&xF6Abezs3| zW@dUigjFrNAbewHLw9O2MRK^a3=in=F%PTcbMM#?Pt?RmSnlFDx85)JT3P#j^DYcE zrAr;zD+Vi$)_IbIOlnLXiKN5gVBU4k!dv0sK4?wNzCI!kkS0soCV=7UN z@Sde488&nlqFPxY!Cc!9+rk<|hM4}RD|BcPsh2T~_gEHO9VS>e+^Y^ApK);#+fEp= z=pjEn!+#H zvMBThlUxVtM=F%+_MUoGSFBkW>+d~07CqN3nFo*zy&obYu!@@RV5j~Dc*bhPv!0OQ zFS1=XVBJc(u9Q31AK3X(AtduKCM1 zMzdTepgxG#sa?o}uhJl>|zV+jdIg z@hc35`w{N=Q6q2fGYdmRmK=a**7K2oYJu+TBclcX)TquxVCl(v17J2o2lgZ3QB_s7 zCd$dmHUK*Di*zX$Kz2^SISNgx-9X)I)OA$cE{mX~5XS8}+!E8_D!-7RK-7-bwQ1cQ|ltRf0b`eij95mSwqf}*{#@C{(e$D-vtrCDTW`Y+At#WPkKN2+Zy}Z0q)6>6BD^_!L$2Q}M zR%l)UwvJMmPJ7P0IX6s!kTVK075%u*2{!k&Gr$|`8Hf+!b?7$4sj-of5o;NohF=72 zB;*fE;t^6Dan4&HTK7Qk_~D|6u0IMqV&PMC`aqqwTqA^3U!kDvJ;|ukfSNuMG;3}M zAMre1LIh9^_vDOlUc3Q;WNp&?wDW>VeWG4Is}RtKgfQOhk~^(mS_6%(WTbmtOJ`iL za?sd)!4Yre+QPrb1vtOk1IEdg)rR8drr?f#B)_3O*`S_VaJ2bVhN&$AnAx$R~~+sTA$4z=y= z!jqIZ0r@Rs0u^{eg5I-(HX-2ROGz<&9)0a>mQv}tMZ0TDy@c|Igd0ZV#|8|5#M;G#mOa)wB zVPBL?X0r^g2+{$7Aju^#J=5`J9plfy27!WvWqCGxqp;Tf+Py*G%Fzx)l>05}eNMi; zLbUQ_BV=mj@g%;YvzbPtc>u|vO~L{Y+r@bEsODHlOYW6NkP3nDW;;M#6EM!QRq#mV zi+k{tHqyn?1=g6}q(1vSc_=A>oJ6qAfSQ{YWAMY5!mTdjvY*P~orIO$E3##)*5CQC zh{|x;fEMUb4^w>T+qK8c9y#I_TsCxgXGe|FHOuCW>jLe za~aE1n*^;%qYpLHL}rrp2$iIZ;G~-a7g60Z%^WN1T!g|yM1T?ovwCb>ViOaoDe38X zOMq;|%?OW=Mr->WiD%p<0;KW6lOMG%hvc7)(+B7#ixd&$=w&~5&JrcH!Yn|==-b3~Vf+Z(f;~87B5UX1kh@LptOKB!5LOUz;!(7_O32KHi^ZTy%|HoQ z6E$i$GMnvb^CQkg+H`?;$5y0$s)p$1q_oynb9+#?)|p2fTMyZThWI>hBe@z>^wh_# zdS^g(bAYEyR~I`6%Ia+qq%@?Dj*hyE3Wi;c!^d0CTAzIf;xafmn2p>b`*+t@qducs zJcgRgAf|6LxUgTH6q)qxKKqW6sgqJk{HFeOl7}9AV*}=mBVhj~Vz^%hBmT#JB z41LwfF9#t_2DvM+agZ#Qq=p(V+C#7xLPWb^{pOywo>uw4iGv zV*UKTypXr=hIDM2WBwJeNl318y~i)WWy!Iz$N0{A#{?`eJM|{;K#5H7E^-`KuKsw- zw#PhocLL$Tg9q}kB4DNO7TTDao9mvon->Rq3OQ4QkQyRDGch+$sjk*_1XBi_+c)r{ z>{WqX0LtL*AE6sJZkQK)rXO}_5zA~&{q^gDhr4?}80BwU^;BZyh_h9Q{n|x$hJ=*a zI}j#TsE{`;UEm+P)o(`P%y!?FoP6`2~lStq|aV;c)J|#*o z2*99=oHt&aFO(lP1)h&335G7$eZT;5{NN!s1Hrt==n6+ZxMW<6;5j&Gpm-aR`8P=u z90Q+Jc_98#1C5pnjD;=0MC%!_ZI>NBmEz}q@rC>{QWR=JTngmwm>?#r1B5|p&J@Yy z^`#MDeAWwyMF9t3UI1qx>QpaaVk?V|U&zx1RiYD(wL+|z`&56ellvqFqgBiu6b)9oxJ8eIcBtT1$;sMvPe&11RkV=e*)p>bXhcO!sSwT1|=V?Gr zp^QG5(XjZex32ZJ72e`NnTC{fbm_+cB|Z@!3ek#gZpdB0VCr6_rJWL;tw9T*a#>V%MQ8)lGi|vD!zl^R(`|CkdTn1P~nlM zzL155T7Bw|GB7O2S?~twWjavzqu>nGM!qdRpgQIg3X;*V4iChCx58q7q$YY_I@W*BvDlC^!_E9IQ>l4F%ZV-7HndK>wWP7Sk?M|etsXD6~Wxl zvqA}{NK+t@1_=M}@1MiDeHFBDA&wp6s*jAvAjdmX9*KBvK+QWcsP{9Qv4~S0vqUdNnd>}_T zgB(?G$uqbEe%~<9wSCBbEl^om*$4uC$es8{lo2*RskUH!t$1L`Jy+CFT|=II#}-?3Lic#av2)k| E4<}bZ3jhEB literal 0 HcmV?d00001 diff --git a/example/npu_pretrain.csv b/example/npu_pretrain.csv new file mode 100644 index 0000000..cbcbb23 --- /dev/null +++ b/example/npu_pretrain.csv @@ -0,0 +1,51 @@ +step,train/loss,train/grad_norm,train/learning_rate,train/epoch,train/train_runtime,train/train_samples_per_second,train/train_steps_per_second,train/total_flos,train/train_loss +2,2.7920000553131104,0.035306449979543686,7.999999979801942e-06,0.010457516647875309,,,,, +4,2.8011999130249023,0.03491510450839996,1.5999999959603883e-05,0.020915033295750618,,,,, +6,2.7964000701904297,0.032717395573854446,2.4000000848900527e-05,0.0313725508749485,,,,, +8,2.763700008392334,0.024953875690698624,3.199999991920777e-05,0.041830066591501236,,,,, +10,3.2811999320983887,0.3170815408229828,3.9999998989515007e-05,0.05228758230805397,,,,, +12,2.9409000873565674,0.04423849284648895,3.995128281530924e-05,0.062745101749897,,,,, +14,2.851900100708008,0.03667925298213959,3.9805359847377986e-05,0.07320261746644974,,,,, +16,2.7869999408721924,0.022814607247710228,3.9562950405525044e-05,0.08366013318300247,,,,, +18,2.782599925994873,0.021528413519263268,3.922523319488391e-05,0.0941176488995552,,,,, +20,2.785599946975708,0.017014438286423683,3.87938525818754e-05,0.10457516461610794,,,,, +22,2.7571001052856445,0.015719758346676826,3.827090768027119e-05,0.11503268033266068,,,,, +24,2.762399911880493,0.016948623582720757,3.7658952351193875e-05,0.125490203499794,,,,, +26,2.7411000728607178,0.015535997226834297,3.6960962461307645e-05,0.13594771921634674,,,,, +28,2.7330000400543213,0.012748735956847668,3.6180339520797133e-05,0.14640523493289948,,,,, +30,2.723299980163574,0.014809778891503811,3.532088885549456e-05,0.1568627506494522,,,,, +32,2.7342000007629395,0.01219236571341753,3.4386797779006884e-05,0.16732026636600494,,,,, +34,2.7321999073028564,0.012785322032868862,3.338261376484297e-05,0.17777778208255768,,,,, +36,2.7314000129699707,0.012986919842660427,3.231322989449836e-05,0.1882352977991104,,,,, +38,2.7065999507904053,0.01096824835985899,3.118385939160362e-05,0.19869281351566315,,,,, +40,2.6958999633789062,0.012387535534799099,2.9999999242136255e-05,0.20915032923221588,,,,, +42,2.751499891281128,0.011586200445890427,2.8767422918463126e-05,0.21960784494876862,,,,, +44,2.713099956512451,0.011821281164884567,2.749213126662653e-05,0.23006536066532135,,,,, +46,2.7102999687194824,0.01147585827857256,2.6180339773418382e-05,0.24052287638187408,,,,, +48,2.7019999027252197,0.011368263512849808,2.483843854861334e-05,0.250980406999588,,,,, +50,2.680500030517578,0.009935515932738781,2.3472963221138343e-05,0.26143792271614075,,,,, +52,2.6993000507354736,0.0109846917912364,2.2090569473220967e-05,0.2718954384326935,,,,, +54,2.6940999031066895,0.010465175844728947,2.0697989384643734e-05,0.2823529541492462,,,,, +56,2.7091000080108643,0.01009758748114109,1.9302009604871273e-05,0.29281046986579895,,,,, +58,2.69950008392334,0.01249368954449892,1.7909431335283443e-05,0.3032679855823517,,,,, +60,2.7216999530792236,0.01051376760005951,1.6527035768376663e-05,0.3137255012989044,,,,, +62,2.7158000469207764,0.01054943073540926,1.516156225989107e-05,0.32418301701545715,,,,, +64,2.7214999198913574,0.01076149195432663,1.3819660125591327e-05,0.3346405327320099,,,,, +66,2.7116000652313232,0.010380392894148827,1.2507867722888477e-05,0.3450980484485626,,,,, +68,2.6923000812530518,0.010425001382827759,1.1232576980546582e-05,0.35555556416511536,,,,, +70,2.683199882507324,0.00925016961991787,9.999999747378752e-06,0.3660130798816681,,,,, +72,2.7093000411987305,0.01072422880679369,8.816142326395493e-06,0.3764705955982208,,,,, +74,2.6988000869750977,0.011063243262469769,7.686770914006047e-06,0.38692811131477356,,,,, +76,2.7125000953674316,0.01013101264834404,6.61738795315614e-06,0.3973856270313263,,,,, +78,2.6993000507354736,0.009940676391124725,5.613203938992228e-06,0.40784314274787903,,,,, +80,2.6861000061035156,0.01050259917974472,4.6791110435151495e-06,0.41830065846443176,,,,, +82,2.6828999519348145,0.009912634268403053,3.819659923465224e-06,0.4287581741809845,,,,, +84,2.699199914932251,0.010668900795280933,3.03903811982309e-06,0.43921568989753723,,,,, +86,2.698899984359741,0.009650414809584618,2.3410482299368596e-06,0.44967320561408997,,,,, +88,2.6884000301361084,0.01006452739238739,1.7290908544964623e-06,0.4601307213306427,,,,, +90,2.6928999423980713,0.010409764014184475,1.206147544507985e-06,0.47058823704719543,,,,, +92,2.714200019836426,0.009937116876244545,7.74766078848188e-07,0.48104575276374817,,,,, +94,2.672300100326538,0.009728306904435158,4.370479871340649e-07,0.4915032684803009,,,,, +96,2.7018001079559326,0.010098566301167011,1.9463863054625108e-07,0.501960813999176,,,,, +98,2.7123000621795654,0.009524320252239704,4.8718995060426096e-08,0.5124183297157288,,,,, +100,2.7028000354766846,0.009290286339819431,0.0,0.5228758454322815,788.0635986328125,64.96900177001953,0.12700000405311584,4.629706395531346e+17,2.739542245864868 diff --git a/example/npu_pretrain_loss.png b/example/npu_pretrain_loss.png new file mode 100644 index 0000000000000000000000000000000000000000..65e2d9519b1a48fee5d6b3dec64465a725acee48 GIT binary patch literal 47645 zcmcG$30Tc*`!*b#ZQ9red(L5tXrJ`2zxNT_*p;4NYYBi{&K^jeo z=DBD_!>Ux8mFhk3wN{?t|9;2!eeZF+9nU_Vtk&=T-Q#tg*Lj`SeM5GG-V4Tn(bLjLynVYX_qfq&@LIDTW@1% zX=Q9-zDsn+c2P-@^%rTh%T^RIF|(iF5Vf$pAh!8Pl_pwQborR36$i(fmE^xU7s4v> zuID%85A9d63+`>QwNo*ic>B$<@~?Sw=Pf+6o_2ScCnwkMTr1|kJUoA0Vqy2E%+@l` zhg}h$GKCWjxlXNE5~E@4>DF2q`K)}!++!Ah{wWjK!hb^&gJ`pu3f zd==|AX|HnjvwTOp=CMAzvynEJ`I}q8ivOeL=3Tq7O0e);@?L3SqlWZcV%NVP`W4hnJPbZZ?oSTZJT{K@B;|E` z@Vk^h-2ZpBLsN0_2}2{J?(uJ*wbD%dnq4N%NXz|sGr6KU!>5x<^H-GyOFnwSe>!1 zJ@VIOJTc|vN@1l2LmegEOx~@y^A`(XS3OQ)A9DAqw<^9?Lv$-b`6*rTTm^ zhnoc1r-kPWym%*C|6ab^q*je=HeWxc9Zyb9 zjxH%s?kEk3|MTGWEfNxS?K{4G`xd86_u9F0C+)}hm8j@whbz~o&8lNgMZ2_wpT|1L%12+7Imdo1U##WaTSNtu1zJ@U0dedCc^109@ObNF4xU)CveaJ=~S_M15zhI_7j zI;5L%`DN-*lJ>L2fqTBbk=4~ICqpQzD^{#nRuH=P$^m}o;d{nb-+XVcljyG1wy@^j zd-&+lo(~toR4px^dhyA`ZPQM_SQaXyATM80o1i`NYKfRq0!NtR=!XkS`iqvX+tY;| zrIqKBL$T@Y8XgbOcll7X3&(EyI+=f?qj&gf;*)Z_(|X*mU+0h7;=W{! zgPCai$iVC6TV-UL&Svlb`&Y7QHdBLRj@;({kZqK#QJttOg>F%^Z2Qr(990vajs)#z zCXE?a$Rdhnc5P5i&{9>6Q#p0uz}3WzPm5)~fARVF)bM!h5U-?V6=G z)b*Xyxc6L&>FT$ZJ&Q9S09CC9;9HD2wc*~iq1`ecKq z2Oe?*)ny0d3kwS?1;d;o+FpCaJ5P?rX{JPP*sy_$ba)@i-^dHG}AgY%v4Rv1;yJulBbjMrZa0F zYbu{J6ycaRZz~5!es4=&i*Ri9Q5r3E1U=Z(Kh++fAIvvNtZt6l=eM^*`1%Vi@}@e5 z$J5OkC_FvA8R$a5_y^sRnZB%k(XT~n5!e6~4Euuz4m{qfp7^-@`4S#U%Qp_Mmr7)e z*b2U3L;qM@8%QrdJFD6}(QYPCo zI(EFjIfvJ)2V2z|cBfV?LCepsq=3)yak2gy^iEH$oD!BRh^rEdwf+bTL^_OrAEKq?hY_zO8DIp{f z*`>eE*x2~N-YXIPn>qL8e(zXE|9tTJZ>R3?$p|}5e{FacZ&2X%K*~0;p+8WG+QDMpO6Iz1qT=BE}dvby%|ejn3J)o&E!z}rrA$0ud) zj^BXUSI3Nur!4y>YMJnbNh3~LK8E) zg}*lEG$dH{H#}3sfvLb)PE{CZeuWpUeAeR9^KA$VYf>30&%Mj&RzaC=YmK^o&HWt) zzJY;^!TWH?#`RCm!ZPkK<7;ncgt@~fSKx5>r&3ZL%TD(0BGyY%Qj$w{4adBd@Jepl zWO!128Ly2leM~iunx5M)zht$@WlHmtvj>w(!liY#&8(AkvU4wUp8jFK^pEW?j1P9g z7ywS3PS8qwy6Cqh!P54pfBE&da_oVo%-ZKxor`UoZr;AFN)BR_0vEj?7ZU=jCF;~1 zDZdK-OI0_^#`8hQj zAvG|80b{j67P-bln$pH--TPKl4qTNA^R(N3dAGuOyv-xJI6S*}Tyw8!tw_k;E3ewd z-@QAQcCq3BHkb;qSIG213vG6tan2e4bd#!|H~P`^WY|Ib_Rh|p^6-OH{T{eT_p+J7tw~+2 zFaKWlnQSW9%v94l(XWlql>z?TWo~;Or&4S4rbm)?>RJtcc~7w)j7&Fgiu-%P@^0_r zqF45Ie0lG)C(mh(#Q|L)JCDy68EwA?{W`dbS9?WUTiXM`94~&kMB)lD6V#woiw{rp z<8w^)9KVWtv~CsO=K?m*ce@odeB=o*5Bdag-I5~}d zO<8HiRh#jE;u=w%a+v>N>|-E-{d;CYY1X1Q^CRGXIHyHUDkxN9IMz6K+ccjZDwCaj zOrdqPKDFTGnD?80QvWX;<7?D&WJ8@tHDQp`WT(eJFH7PYKJiUOk)EiVwOaOLx|u1? zXk3xY8tb0gv``tx+P210uaU!w^i_h?j9|wjM~*x%GfmS<`2PKS?fAHz#}F2&z@e{E z(&KYkh}4AyV87SjMtbUO_tvT|kA~Zn;>kLfzZge)vM-C;saq?1biVM%DtYgbSbC`C z-ch(2D~H}Ya?@S4b@{7gLRPL^$>U{Jqb)2ZR`&6+<}Q!V1-I9ATJjmTb@v3{zpr7Z zZ`0TK0SnQz(y_t`0M6W5MnE&n?$eExovD}G4$%9tra@Jo1GZ_yQ#NJUO7Zl>&8Qpe zjL_h`jc&+XE$oKRx5_x;v5YsEPrD+N`sbg2*5WYj<#8<$uiMMBj-Ow@t*j$9lxj33 z!Xx95`g%v0>$H9BP;$TV1&q8VPBleR@x+N+;#;@I>vYa_+xID>W+4a1ravUigUL!_ zV`X0Mh2O?EYs^byQS*i8alC`_`fN<6i*{x%$$FiU)EVaM`z1B5 z8{n?Wd(-jY+=34)BM#qO&2V3%x5a5XAn!+1wPP$$=|r0cf8&EroZ?G1t3=*Ak2TR( zY&?=-eMv<{C6>#l)Ia9Bher=V1rFKqym_m)X+`3{4DqDFi6Z^!Pd4t&Fx4aQ3OxXf zwLn_d1Fic3%ERA;DtHHrz8=0d=a*k&uw|Bg27p6cfUMZ9A>R3L_2(<^EA{JZfCz>?*AHaU7 z#qj&Zccdq#2Z}M$zI^#|`XHTv(%$-tvyGaHJn!p`8pC#9EPuhU6w@zjQN8;)PSJZS z{O>dST%1gN8)n3h!5wwO%d6I^Oy~O5?EVg_w3S#bGvGRBS*R_4*vb?(#pGk9t;+(`NeJD z0YyQMeSuRWNv^7J17>fF8~p@#?mUe}P#x=Q5_cHBxJ@rxW1WP#$d1dFwb!1d^5*KO z9#z5dzO+gd7WR$dli>!7Jip0o?;d^Wc=e>dL77B7tImLWI4g?TdEV8l|Cm|c6LC}2 zl49QRqiIp1G}n60B-!EXaAVlUad=Z^yTA5s>uwcZStpG<(u2nWuWH#|t=l)j{d~wi zccOc16Lwnr;$Q*ErO7fc*Npumcz+~iJB6Z!#jT7A(9`-fqO7b;jIqqraFt0#gu9*t zW0Qx$ugm!!FPHoAq}okfQWB@Or1GB?8>`W+oeSbQS}h2NVz_$6inA%Vj_!=x+3eUa zb0@gPB<-X0&Rx5#5q4sX3Klsd{s2dDr(I(cfZAVc80TeVPRxG-2y0%}v{wCgDNL(o z_)z9VdsJjomrhPc>B__EQt#(0Jkn0-=#TrfQA$d?;10jo$W*8npNZP~dY5j>7<4SOGeOKoVW+8dZg3$1Z;thl8!ixbitzR6D0L z)R8tmaMl?+CUdOWu?A7Z%D5fzpX{i%?ME?dBu#SMqpd~7&=edFPYNR z^RmsGvo+P?)2gjDZ{AGdok4)fJGZ`)FMpeEW@`24rmq!WOiOc*09Nm5yCrppQAKxj z?i&U9(MhAfL_BYbbDG%|ar2V!;*hWJe${Rs*Q`g-Lqa00Y`bJo6p!I#34O53sQ9v0 z(*YB^n|gk!xDlcUH#m)zy2 zuJ>(Zw~l4{rgQ;k02^w7m8Iowz$ALo2wdg?tLJ^0J=J)($G0CpoDk&~GtwJJEv>9Z zK#(;2SXqP6Jinkoibo31L`6pKy@eyWcj^`>40G`nW1JUiIR6@S9Sl(|C-j5yedD~T z%&i2w;TZC~of!(7$z{v=5c*hmhS*Y?&YZ>R-BVUo zS*cShw`Y%9fSyAvIThAj<>A#U{MVR@2 z;E{9Dv2DEKQT4V2l-STQu5y4(Mrv6#qQB}KM_T|Xt7TkE)#6p<<>ei|<*%uUIJ`t= zxcuOrynUJ?tD7AAbCz{V1JYgpZSA?LAEN_pueo=g|EEf~3@$=8Ni^)r>c+=9&ntUm zZWKO(MWhz++qM4nN?#ZY05dMeC=$z~O=gd^xaPgC7t8w*5oDezzi!W^%XZr~Z90sQ z?k1n>k1GVkf^4vUKKaZ1tGUjTaRhgNUM@TSaI<1)@35mF&VHElXyEnb{E5FV#ut=nb zIusf{afHVr=V;mIQPq1Snqra~BemMVJV6JSABwMu1Bki^ryLc!gk$avSKImXIL33p z1!*F>8Bj>_eR}a@YAg;-`;#Ii8=LgLoH0E@_JPwXEH18Y&T$MGeZ6SOk}48{>@k2DGHJ?8 zd$7k$E5oV*i_~jls47pOvcpKXq>41G?-9(A2`2ApO3KM4IdflvX>DArHQhfmt@zqw zdBFMRvR5sxGr9!wBW&0tPu>Iz!NtWDg`QCaI$=EJ@aRVy}whm(EUt z*+pBlvuw<^X*_xO=196x1~$yY(YfVZvR_N$h$&>d7H&m|M9@&n^r(bK9VRvnk^HO9D_5>$!s-oUC=O11)5WT; zGLHFwOIb{bkmFn>9Q!FwEg|kGzjNvv16V@cU)Ayn?8tE#5gysgF`zDz^b3bd{ZN@) z60qxp1NSnpT?AGm)GcLH?oodrZW;~R6L9quUzjp*PpV?bAD-*RiBxjO0yp_N*c8kE>F^zqaQyzH-BqVb_BS{xHkrww*85q-TaK}(g;}x ziYfDJdIRUmPt&dugnT6Dw94?@%Z_jlzDgk6!p6qDQ;&{hGNOZmb8Cswf+gd-A2P}3 z(Df(X+0E0mF(Y}kh{i*sk_Yz2YJZi1>M2TjoPcM@>Q-Y=i^F=NgUl;?Kcp1>kmVQ< zBs7okW0$FkiM*Lfxv9_mGXoYY7% z))Rciqa(LzjHrO_iJl}f(4yhvuA1;F+DW++aiNZVk|lw=j*=%aYlICCk+eFZsi}z- zctH49gxNq{dfa<2KSWsgc`$g)(C;G>0d1DI@qCz%< zev^@NAjb<+5jESWvLb9@Y2=|q+$(&@kSed^Am|_(` zB^s#Eop}KYf^wJJUd?a+^2KiC0r=L!qN1U0@j0#q&U5w_hP7Iy|ytm6nn zhR`fdFUJ8sN&V8L$KnJ zzH!Of%LcanZ-#Yv=go()G;RdlZ4^|}a#nt0gJ-Vcu!@Y%hHRAl98f6urYT^TDtkiB7m?q>Oqxz1TQ z-pXxOF7VQs@D*GuS4IqVmc=7g)GoWmkLu>yn#&RVxtXoCN*(>N6P~5-;V*QsdjS-t7ms8_f9@m|B$_hI@qZefT%u6fONb_TYo8WQ*AL9sjZ#J)3C2}L+# zth^6=k!qWj-j#eVV%=Z8eoeAr2^z_-e%ZycDGT}b^Hbzcg9ZQ^*d|XBkw;5EfK3PH zfBC4BwXYVg_S-=7z7^8xIa8%y#!?6T{+saOE?P(NTEu25gp9m!C)f&{_Qy=?_@lGm z5xKg7riH8;;enBp-16Vn$`5X1k`5AYio2Oh3cc z>9|~~@ACaH$a*F0B?>y8L0isRt?S$M6<@3(QQD$an=ih)w#f6&9d#_;frF&qfAY#W zsCbhD3XcG=^RXibajkb`YU=KwZ*tS)2azzT!AoojA&!Xasa67ELgCsr$!CI>9%-^| z5dv6xD#d!+okgDIu`u@zU(@PRf#U#}jA5sNyj5VbYo1x?H`#r@or(R-xoXwtQAvwT z#%4pnF-B$7#~jCJ;A(w?g5u%rkdWAitalBvr&0g>vv8H%TILJ(oq4K(G;5OJsSjyp z4Qk%LVAj%1SF4x;tJ{Bl;FkCKf1Xhlfs`}4f3W)Z|E}1`rFH)^{9_>_DJRmR<=e<5 z=5qso91wgFko#bR+003b)rpBuszUZ2Nf($)?sgQRZtn1;T0^I!#H0})^8vd*!dJB( z*Ib)kCKfsVpPwR~+DnF*q#I1E-*Sv8B7ulj7W;^?DmMpfAByreTn zjp5D4#NycN^2A1+=o-8Iyy^Fz|orUMfAIqzlC*k)uq$A;URxke{ zuOjpqe)jzi*^E=Zp(0(Szw{So|5lMf9AJ-#`Q1-Xe~xIJIimAp6z0PNc(YT!YLqgC zrOa7Bz#K&!U74FbkAL04JaVb~Q6)7`+xXl5$6$qN-WNA7BMmLILubV_5)#+lI6iF6 zoRE3a7|a^Wr<+W3d+HcHtTB-Ri)EN>&}*H`UbYuxG7rd41q(Q^L_c%+umP43)QX?0 zPz<#$*(AOqAcj3f%vKe2^^UT~_k?s+z^jn!-q_G-;1x z`=bBR`lw8Z$m)hpmR!k=18mdyRp%(>OS}!8y?P7CL(Fv&s~-cyUe-6VFx z)}o)$XQ)UufBPejwW@P&kseHjQqzLnsk4(IN+zRsYsfhh`L-452JGR@C7XT8s2}y7 zYNUzyr2<0y;fL9R`-vR9i8NZwzR8c*|_(2w%T|D^GC_V6iT3Z8xN zpw9QBdK+~4hMG(}F*~q*4znc(io93J8wUI6F~9u0n;{^ts>$90%gFP78jRWYnE(Eu zPC3klm!)g#9c3R!x5W?284%M{#+-UYd%r0l2H6RCY!x`aOd2BsuB2<2QDsQcN4W1Y z{r$r2wY!cqWm+o(J|YxV$#Zcg)F%SPT@0V~U6Q+3(%FZ8{MvVixuJuQ_FTkYNt2+y zYT${HG(7+oyc&5yaZuH>II_nHVzoN((}MDg7cDx4XSks1$Z;HFYQexhKSa>TfEV)1 zKeuD((Vz8fQ=_zK-pQz{u8K!Q9tVAs2?9T@ESm&xAD=YD!X)|?qz0FRux^wMt1(eU z+_W2k8j1ao-maV&>9K}jf@J)kYKS}lPHuJ{(_JOUb@gs zKzewQlCiG}<9tiitU)KUA1iXHorokyyI&t*@rWotxro zen>KRVC&iaRmU3j{vQA+pX3b0GMnVpb*{KK^cy#%CschyK3)j=9pr=t-!(Sg^M3Vo zn|^HIz3B3DWy()2eTVY$Chzt-)d;&>*7KtBN?uxb;^pqfHEvdo>)bX1;*+Rl%O8C= zv$Lh4((-qCxN5&TW4B=;CkRb^ApGZ_`@t35A<5q}FX!Iolli!@ZXB!)0b}@z3m(O$-=qmQ?Du-6)ueZUc_7tPz-Zknn&NCxgvG_8&o1rs0D|cifpj1%?HB=l2*|qw$9GZwhWs@ z^1izzFYol}(=;RjdLSPfZ7n_}^rcf?@|-pwe+}gWa{hmV zd!YjwVPD82*&wf8{ zuTe>kyfW(eCixI4TboJe&a$vN$ZX;XBMfp{3u#KiS52&&qYe)vvRX*G(x&*VJ@2eb zpLe(`6=(V6gWd8AmhoJ+Gebn+UXr(J+-XEjpcU3XYCIbi#1v z-&{j9DMl{8sr(8*xgnnWrScWkuUkVj*k)1sY;LHVAd!f!+I*t?+M-pQ{QODRuFWGt zLaY}vY8ArPMBHMn!5t7BEF!OP{CLV**Uz^lQk}jJzUl;4wYHza%eTtPHY@Wlcv+lV z`JPPQ`TrOJ$#djD$wBB4fZbHbF>??rafm&-8=fVZXMa0_ed&xq{mNu8I4F15W<(1& z3XF`5{2lvZ&cnle=+_aH+_O?qQQU%Ug9Nr5pd;yK0j{L-R zeB=v_akQsPLZ0r7Y`iqo!2-aGh`YjyTh<si1DicUsZ`+*fr4DB@vL220^fGBPX)18 z%^wY&Yi1F{YSJCzbDwK64$b;4q+PP#H!09BW|iLvWkG6=gI)9Nd>UjzdTpLOV!I($ z7i<&-|NCozOe*4?1+$o;l??5Dne9r-^BCGw{eSsK@}5mnRac5t6$K}cd?JJ_e&&%l zLA!TSQoe7D7IE+2hI=kAl`4F8j4}%k*I`WCuqGQKg8A2(sWW`G^4^uoGIA_t;lu$9=%#a&PrM6IfTG}yT-HF{Sf48jsso2^yt{bm zGu}r@EUfKDru?+Og5b^kgYIOb4SN z_93l%KWEus{32BuDD*cO}s!>21kq=m4 zEjfwiDGcxuUz1|R#O*Hx$m>;7>B=slzV|vlu-x4lDC1se17-RplYL5eFlV2kf6%qn z9!EY!sZb89`iR&(+akq=X!Ed8NWpy`%&c^vxL;#p)=G4W{QUJ79=jeaQn0x-N)^z{ zIb?QkF2sA*k$n(F@ZQBN-riXd`#U)k^Xb9U#|3PN#2~HzE7dTwS2EX^2pY}7pnEz`kJ<
`#omqplXeH~w}$iv z)XDE&#f&xP&0~&gYe+=nSB6T{m0PnO8nLV0K_khnAqQSZ3Rk2X%zmGPe7~@vO5XCZ zjRL!bVq9+aL+`kc?$p^c#{8-<08`zGVD^|f*kkq-J{YB(3U_LTdA8#G6#ECZ?Nz%MO5MGZHFna$OQZja1+O9&JhNxgg?jpf1LcEQQjr4CiSQow z>>+Zz|1++bnHs(QX43;!-1md^TqyOFp@x>A)E+ht$IWz+%%E1T({lkKBo(238T@uhiBXT(%a2a?(fxTW(Fy;$4@G5Vb;$+=B&jF!?aeCh zNN6^;f}gu!W%A!J{G-n4(=oo^{JUd2IZyR|ch`yESvYRXd(Yz3w|b%eADST}pT>?G zzWgt0l_VRAij*&2y!gC%9WA`dS4ercO|M2!G6WC|l8~~l)9rku8NM^+{DqIw&yF1O z`_A)S_1vYhz)-Dlwlz(ExP&MHt3mR%cXV_EhC*|EAKKi=wY!XB9LM^QeH;1E2%s~I zt?jKSvROt^K4At!yEAQg4+o6&xP;#;sk&RdJHIAwle>y1*HgX!&WrqHfn`FjFI^Yi zUKC)8`Una#zvNY9g?3#iRcXm}_Ctk2k5I(Z_q)UCJ~hH@7&2h+zX#BRBfd%>&fA|E z%k1Lznl(h@$q$!@2ncQ7d;~G3zd(Y1UUNap{wRYyN8R(1aM$q)f7)!iy=HFSpUgwG z-cHrCT;3bOhS~HYyW4`o0IsnQbkmf%S+MnmEB?+v5Jdo=KrHZMZpd%K0ifIj8*Cm+ z%P@h?8a50a(BHp;0yuoaXY3Rt|(O*ZOG{5vj0zP_xeN4maBNUSc|n-o@ZaLDZ^ zEdvWKXIk_L7DxC4T+o@#clz|!Y}6D&0D!}?rsD|2J9yUprHXfKGMn%a+Zy8$Vg5bJzxj2mPp{C2x#bFnHh~`EGV?l~B#DPG%nI#owes(6F9^ zDzkj$iy1O&-_)Y82}pNdEGPfoVLC>cEEl@Xj>o1Ek8M95t?5U#De9?Q#9D9Whc~AN zDK@MKL}decAC@H$fgJlN%n*?Bsr*=DWGHp5F?in#rAx_M+&sv(-;q!md;PP9L)?FS znrtD^2W@o)h}+o>3Z!&qb&4?HXw;L14LknxV+V?+2zFs58`X}l!3zAQ7H`iEE7%Tp z+8skUNcCSK&D9x;z#_Sl9i(x{0bSnzb63+9kwPEL-)ugFV}=~Vmn4vGQxME*x1Qb7 zYf1YAV_6*(WXDuLNCyd?9XKW=P;J$D;oou36F}ehF4UEzRuMlg$Omly8zGcnK&`x2 zQP1q@K?sVE73s%Pev$(NVwlmj*?O~_DeK>eJMAA;3dyyk)RFc$9?m7_gn08n9me^R z0MNgq+|0kIaFA|UV0UVqV{M{=smLnDeLXj7oGkB}Vc+x@1hS$p>) ziEOU{g(#n&jQI=Ji?Yv52D3vm1u_>uVKOopXRe>nAD$%x#tAuJ^;nI6Wy^X zth*+qY0Q^wdGv5m2oX7>no%J@$MOcWbPrLRgMz;p6b2FxNez?L?Xz+P8%IbOQj!S9 z6h$pmlod@!1qbsicbPkHDN-Ru!&bsCmNB_T4mqyf;Z;k-vL8Y|sSo{y^7-?T1j<4# zU<@exV7IY1(YFEUJ}>Tc9;kD-pug}zx$$sU1xXD3OwWTi& z`X5a8S1Vaiv)++{p>_wLtrh06fSwRZ|IP9w!ETN6B2Oy+%Qx{X`@~K{3@nEh#V-XJ|whq4`;PfZ zAu^8X=X{PRC9`n*{b$*wTASW>*dP0q9r8}U^i`lE#8U8nZGHtCvIanLlRUJAr=itJ zwflCCDIJn^%7DIKr_EFl0x{^{3QJ0AUjDrRWf>5(rOHDItAxZp*Sd8*we~xA-ELjK zVZ*ET9oTx_k+*LB7Er($fYdxK`{7(+QPEP&mbX~q+F_CJdlciVV`7DkoI`p(h)(@h zM(ymc{$-qdZs`>Z4lg3A!Aw2cmn&AN-7XNQ!XOm7{E;Urjo|ME)=HI3L3LR}mJIg_ zyf;ZR0R-fGJ_>CFSXj#hy&Nr~+$IXZ4~7B85Rs7sJvcbnNKpi(Eg&Q%)zH2Jd(Asi zSoOpu&f6~xh{G-prKbB)L9Jw)GM$6cuhG}2NN}9Skv{1C>|(_Rl6m+s@Y>yA{_$q$ z0WZ8&ss110k@nh2m=2-Xi-94#<|lFSEIDS`m?P@VcHVw>gu<&(owH=C`(jipsi3T0 z4GA+dZj(C5*|4GO5a5;E_x#V$BVi1b_i1lZT^CT$+wiQcLt~$}Z+l5o54haNzX{b@ zTeI{A&F2)lR305!3Vr+*nVgtU28fEiBT*PealYextlgN&*^8XSifxid=}a9;klTju z&rFZyc}MceIM|)JzK|QxHCi+&zHRSYR_fo4fBPmH38V#xP+Wv32x@te)yL1~pZW{& z2+>zU5Mp%Vuy^ur=Fo3=N21^;Ya|p>tvDo!ydz;jDcQNG8u{9krH-Oa7~Z=D9iHo3 zYZT~z6zrI_-gpT&Hrr#h_#=1t5{QBjCA$?6Psc#_(PUY)ZBIcW3PrrDQ8Hzzj{-zP zVq2H->}?(Rh;Xnv&()QT8tgn00x9+~h>eIUDyy4otT&}h!CTMeyCD>w^vg~-$wYY~ zAn!WWqpykuwk+e_zoqJ?41qKYvxN80LYjT|*9szA=SVJ!*rt_wj1(Bvm>|Pg0WCL= z@B`vb=v+#MS5RPb^8WVyNE#?PIb|ah=EPw}Du98+qjw~%6QU<|D7p6ZpQKX#1yJZA z89dezWKoO4UXt`8Qc5CaL^V08$fA)H(}n!Z0bh%G=xoLxi2lMwNJ;0Xu`Nq}AGMrCL<)V+nyIoJo?NVQZU`$7uVP;R$% z*DhsLZIGbZ8vIH(1klPSZuvwVi?V|oB<1fpQ=<(;7DeP|C{@)Ug$i%Ic;g2K1}w|g z?JqoQA94fWP9-6eHfmQfL}RD-oyvR>4<9&ifCit?g9;8?`xCgx0NQN*Hq)8}ZFM3j zLTy;l554#rt?nIIGb;`M1$+GhdNLe?14%$^^l8@g&rt@n^YeH1^Z9~HFg~#(xPf1y zRVjYw`x>85eD4e+GCg=Uy*eGM+hWmk>_VMaU3tF*WdDJmxX?|=kJ{oibenh#q5*76 zL$Tf=)WDLf9->@lCR>S$T1TXq+h-S zO(oN8?B(I-<4b(9J?v8C0y}>Kn*s0GsgqV$yb#GZGAh`;HD06)0lD;LCHYlVRelgG zKH*%x@R)2Y?Wwk2K&WtgXP~PtbFLN6DA-pZ>qex0Bpdb-cd(G#K_GrUfNB7!M(>v- zL*j=csR~?Ee1CDPE}Dt!+v^`He7N9&s~NKy9T*meu5b?`I5Ytz2uWiwjEn%o!gVqT zTfxMt7rOP0kV69URFX;eGgfDCw*QUfMnxT4I@Nj*uHe+EQ-rr8B^OA;3!&PFzzEd4 zm|N+f!h~p3QE$TdXqqHWCjWaxrO;DPPHxe5t`Q}60_H{CiJoR&0GMx|3j*xOi2!wV zzLt}tf9SBH;LXEvt2c(oDer#E2ABN1qVDZgGDb`C=h)Ff3yIS*pn)~3f3hFag}Vae zG~wExQ_y5ch%UFg6b^u#*Vgf4NbaIt$Wl+)t=UqGk*;Z4VmdJS+uAE0_>4H zu0IZf$=~NKA6`Z355NwQoV0fOrHWscOOtv5ArwynWx&-2^4Yb{G5hfJu#Cg7 zlA4-7(WdW;o@*Cya5k)dXo0?P&iTLT3T8*Qff`o zgi(!%!g=N)LOO{(92F{d5u~Q-Mn=&tsBk!aX9}AVH#?BKPl9%Ey>J{Hit0Vlup?}Q zXuPbM-M9>sV~nC8SeVS2>B%&Mf@J{(W-Yn8fQ96C72L9P{;di}|NGL*JVS<2J~iDT zrp%N_>mj%=UX-|!%L7QsC-UvJfJnshfCW6Z(?OP$oPt19!|o+_+d#eZmW<7ZAwU~c zRb(jwRr0Mjx-w5v@p*s?Ou^Ln=R29--mW8er;r-0i628{yk04=fRO)_i;fUe-6x|V zAzf}fWU#!wh}p=G?72b^hJ?B%x@{E8$pb{6K6dOMoJjl+{-PSX@|+)oMIW>6Q%HS# z?QV|#3W#N#;hj4um1k>fK6s5xvrV;Uu zw+d|FtEO(qE2?qb6@VL|aElon9Y|1n#(er?Uf?EyLvdP7Hgy!6C(QX8zZ)B_dA##18+-sXOXUh!EIesih)=Vh3kHNWGwC3>zsdS4J+fLFt$BP-l z3w-KP5R^Buiv2qDHP7Ca^$ZzH@CkmBA-yS<;+#7r@!n))nx$~`Bl^ijYVp2i{FLtx zD;b~q(#y(te7po5U73j2O@SynuR$5>xj|orQe&|pzc!iI;0DXtEg&oL1WdC0shVuL zUsLn_XD%ro&n)qkx9poawa4$}s&bbWdJlL)#=hIBcGHu!Cj z!cO>egpn8iSycwT1RJ_Kk(@5GcM>ZDZsk(Ndyp4o^Cz1~yW`rRhPryHud}CW=8BW(qxS9i0*Olk-{wR z=!6N6*%Duq$Ktf5TxMOW$~Ee~!^p{gw@){it_Hcq(mK2Vndb%plBM^Lq{`Wu5?kU) zjsz*zBA2UJqsD_Mf=I;+3=s`#^=_ODpS0lPH^2rZz-PrG;*Y}}8|2CdL`WBLbwOcS z+1P;K{(U4xA%6`5)By9XC20!IL^(` zJ24)ZCRU)>i4zD4JrM#R&zl+zA~zuaOOoDY zzh56B4qKK8U|s_1V~F)_w)?!47v;ooR#ngfkskeJOWKqq2^-fukQ@PjP{9;C_{kQ(7 zA_uyksU&>*<$_{Y@95IP__}&^<-x80ttkALKC&lgzdi#xJ%=3J%L3{x%&GG0m@WcD z??@Ewp{(XZ+X&Eg4}t;jNO*LIGtmoG{4X^q2%26C6HW5*)u)FKnp7vRUDA@$d^Cj# zEas&=Or((lJW{cT`kkd~cOH7yWE0fTk4l~=1P>6~y8%;VSc>rt3%5k*ze%;;Z!JUmSLX|HNrp;}(AE zyP5Hk7XH?Q|E>X@RZZ$Je7(IRa4!kD#fV%=N3Nla9y@yENHDnqop-GvHf&${;>>nylvNXH0Xep~isfQO#9`c=X!E?R4mV;E3W2J>RYO$0 zd}fiK?^K`lv>t;q3b6#Z|@K zMX^m3;tu(p6LR4zQ-ETIhlq}A)}={rfd(1qhurK*s_MW(q@I88{pxLt(H|&9COX)B zF7+4$Owqav=BzJ=Mtt^Qh~i)X*x&wR#{pEjsUxiP7txG}KG~!#>`F76Ri12aBSBGD29wwUjSz`|pH)&9$> z(_j}!qImi{DIY}*Qjr0QBO7J=frBGaVr`0tQHPrZs1L53IgR&@^2I9sZ8`5=F$c<3 zxJInCz8$QnA8m<@KRRyCf<6ydqDBvefJ-dX0t_OXM_(3K8)buzOmp~lyc77(+PGB$v?5d z`Y!lOF2A)8{iySuLp&R(?8y2zJhQ>lDl(g=A$cEzAt1RZWV()31>m+A-d*?=QeT{V z33BN2wN#)CW?6JFedR)7;{8ZkbS~5*U$T>`Mq8ZN8&e61B-Vo(Km9Y26viwBxV)<* zB=SnaN=S@tFma$$~fgo|A!{1qs8|`g02H0CCXb zzab3+(yN8(|Jwh2NbRc}dIBvVdoXs84=Sw)-2ZPIc_SVgEG+Y}e0r(iYWPotvK&CH zun0w5oIDr{yLZBv$t_~^aOK|-*q2UHrjqLdnnVk!^bj_x z8^ffugF`x>e5~zWZs1KilCw2bc5Tp3@}{jtiC`(>R@Ba0GHLWlKK^zGgBUfA3}oK6 zh+&_cVIU6$*gA46)gWPZ)9uBsu$M#zZu1DY{OyK%uY?CZLC={F&RvF)>IDKcoZ7F3 zBO_z+#Cf{m&S`V6*@w4~hvz#+`3P9&HILeNYZ^Bwv=lRQ;=e1ANm)D!>LvM2qUO(P=maLtpQFy}EMTFTnj$UCI zWU+VEDDsttQ&D%zBRVzLR$kwf8^kQad(ls3ST#^M#N>_qSBH;Y3Fp|M2jrYsmwR&HO8v23^bBoZNcD(h}NQS>?7v%yk`SJ};~cu%a6M_sH*Zj2{8-DLz1BK1dvTXwC1~<;LL1_`?DY~;I|N%w znEB?`XUrY<_&oNNP~3y+cHxE{Hmup*i~Z{BQ6p@q{t6E6v#3pjt50d@0DFdA5EpK_ z(|ZE4lf<1ONj{O(oG9Un!(|C-#v@L%jR%m%NvNe7NeWjP79 zI926|rdzjn^pY=?XLPoGkZfC)!=9}0+d}IuKOU#{+UC-T^z5jZEx&{N_58>CfN;fE z&eebF*0$JO;$mqe9v<5?v2E?Z!{R0BWm}KS(LePD)PGg<_Uav*ZJzad1$yCAE8Qz* zS;xgcT`EwkP~ssge1`0e*SU0Ml}(Rnix>H2E`P$SEvf!g%Xt@b^SJ55h!(zf8OtnW zZm+NY^;5~}l+*_H`5WiLI!hy8D{R_T9dT17EB9-Cp(NWVy+|T!)mB~=yAl}e7Vl)H zJ`SfP-*MJTViP<^;rJVu8%;O3 z6$)3f22Xs5T8Yo#(9f>UKBpqyQv6!K-h+CqZ}Qrq2`qY3|h|iqm1nBd-72}fAjqUAxyA;`b-D=lsl2*xE#kug%azs zD%M!4VF#<0ZC})3@)ntp;FKgAtsx)Zps%x!ASB(g+<_ul*986Y-3?}%ZL@z7-f{qn z?Kr=!>fN|Ls+t5P$Y^hsme#@5G8Lpa1a~8zz%5rgowB$Dmk;IL+$}cm=o}TiaoxiVO8t(wC)MEoKGe~NlfVj~um`sQBzed# zntK&_6^M26^K0*k<`&zq1>H&vl;sd^5OqgcPR=jf=6FX!jM;@mg?Y%XjgMk3D56w} zNAJ$9Th+LuT0kE6@d*{tPI;$t8aI1X%Joa|rOa$I)7l&rGBi@`$v)8 zvH!)~S%yWqcJY4b5-Lsq+66OLAs^8y9N{$rKKBzp+mYukrt5w0qI7%8G6n= z?){$koDb*2`Eai5JlDoAd!D+VyVifL^;`FB|1+RiUgZ?Lu>F81uT|VYNm66?OA*cD z29R^@a>F9~iq^5!W3LA zaJDHbD+8gkVERIecAhe@u<9{+dIDo>n_oCunSaE%m=B+6m5J9Pb9;sn~JX(lQ zT%!bOcJKaAfd{W8JkXg@XhmEB5vL}hmyeV<9({qvc^LbykiWaXf!;xb_=k1iAUmEg zWDAeEgQsPAR{z0JT!&$OZ6J_V$*Jxu(pl^Dx(r%qG7w!mNiRK_}ZEL3fFIK2!Y8H#ncB*eY{fpYG1 zdI$SGFff@-g6lDoO-8e*5_4icdeL0C!6(vA>cn=uMd|z zp^CONQDVwUC~q#Arjz?<&gAM z%mpfYNmkH#-?p(T4%(*!LgQIEIaSfI)W9$%G${c*Mv2S4r-1&eiq4n>l9bUZ-2cg- zU4mC>EJctVHVMvPp3%C`8;`;J8cA&$VR6{}?1pxevp$CY?|2du8Mmh#sGq@`hIQ7f zlk!f);i0RR*Sc?bx2fLCRk~*=fh{6&MqOFFhN=QTrr^ro%}N8yAL#-h=n&HgJqKx& z+5kaW0|GDr-l+Ne+Sx8FazzhJYt;nlgn<220>~d|?;sd+It#vdZ;mwA$DQFaSxWDa z@c|AVrK4+Vh+mzNM(h31rnjT}ic1Xlo{aCmMiFHTDEJE~;ngYm3fJup^d-&)X`c5u zu-2r{U2BRoR=AriZNB)tl*MmyzTK0vcN5)uH)~!&JKt=lt76BR|+T31TS{m>y z9~~X#HLBN8nq|1UOgWm{4gZRxlj<2?-fFbL^ZdX9%PCyugR_GeC!IsYiiHB6m%QX) zOd`z0V{6v{zB`xjRzkDQ{x_^!9-3kJq1f(d7$|)<4l9QL#7GokgKP{iaSj99xD@0v zvVm9!kbs~L7NLQ_HsF3#{0IUxB@W9k9lYaJZv#!Cwv@$EI7}S!yXll z)32xA>b@=POb{Kf@9~kUUf@;!VqkV#&`9A|TSVQ~8XQsN;5TbZEwZi8k|AMtwD;H6 z#4fvuW{7||8a}gc_=8!JCP3EHud;X#7RQY-|Nk}dl9XCN8HkR6^&OAq0RjLIS(_J# zlDGmfSZE+E@K}5F+)rq*%QYe$uK=q348O6THT)PikM0YwZgK`> zf{^w;8wulSuAQ8s{N@#_^15t{W6>V0Sec@aIMR8>`304F+Vu&MM4L}NqqXm*N+ zMb^I91I?yoyYHHu-kxCT2=L-SPMj|WCNf}gO~s_bJzCaEYPFSY9d;(y#hosMGzPQ z9N9p70g1>(?`Zwx!R%JtJBpRnWyYR%u^_%)Je3zzuUOkZE;ryIDEYwgvdf0s*!0JR z?9eA|qZvt3ki}BIMqKx4q-{`k3F#LZAo!10qX-aXs6cTFKzPzCN9vc2-8R_;HwHtx z(=DSJ@kemF(1DN$d918RLGhBk+TYG)xZG! zG@rC#_Ywus+UB18J*?Ve0m-*7e$QUS&h2*zmp5c{EJyQh587yp9CE@-L zo4iS&8U38>hD}={w!>>O=GM_l_CdU~ICfps3oqhO;;D6UW}7}i62f|jKFJ8Xk|jL& zop*Ulc;x;T?d*WoQ9+$&(5?9CG7K$~1s^O5LNz4>r&-Dj9&G>CLw-+7iQoz#bQ2b*LVG@r<54*D@8iK_MCHSQ>)_-jh``;wYRm*7Wt zMc+u?Y=@2ew3c4Ap+`C74SvS+v(UE+q-~7izOUmm^!ToK?jYER9K7}2yFbMOCC172 zw+z*F!|-?U!&iVN=r1kO{A}`bo=vIjcFxUWb^CQ`MfJ~`H+e1d8YK!ugM5Z;q~^+C zUH?!`639N?ffnGZL_ibQhbSX6B%p)I55)J}`gT1*xCK0J3K!KiQM(fzh`4s_UP3x* z9FR;wKJ&2_z*Pn!-lbe0Hh8EL6+L<3{!g^`6W~Hi7_aqKw1*{r)FzMQ$lx&YE33@S zU?YkAR^$PV`|MKWYQHqRrq-<+WL!xP?ZeL@aqzxzD#nqYs9FE1GTgZJZNHGezE)z8 z`jMY(>d~YLT){+1{nz)5zdkK&5NEAv%)lV4HL^55#o3iA@BXI@31Z-a)<1WUj^j&asFf*osH&4GT=T4Y_%>Z&|g1waadubOTEYBu zH+`}n; zKC@8#+n)HqTjy_7@$vjd_5G>qw4;$5J-}%vtVNbL!5>i!P;*XV%- zvEK31|6p$_dU7TSzfwTAxfX5U^@HQ)L6CV?b@U@wmi(`A-hm68iORq3u7@N!_PAB$ z{YKT@4+m4N#=WzM(bCu36HPJ*aF)<^eBu!KoAB1l0s3pFF-3 z$6Tb2t4Z5K$Lk8`e^S-a*)fm$_Ilu_&&Df8Mv4yJ+tZ6*zO`UISsDRVq^N+Wt<(GP z@1loJNQi$>_2Fn-#{moXVqE>KW|w*9JZxKcLvQ_f7*K@%b4?H@|43zk6Opa{!z33` zbU<|(?6J;YXWs6dkT1EGc7}fY;4B~~Oz#)GgUgoD^yC(L6xRUN^NS8mk_qRTwEh8n zts*In>HY-g{mJb!ceL^qOz<^89D|-dR8A3gm2`A;la;I9rbV%4cNKfCgo^2EP84yP z1}}}s*^kalxIHDWGL?jy1`<_fB;toB;zunaQ$<6&XBJg zi!PObZ0d1EPNiyVpuqjp_ixMc)VUO$qd2I*UdWi5qxGnDN!7bNFEhcbEY7PG{F`** zTum$#ZXhfNEO_w)lE5zXTb*(M5riFAl3uHVd0Ussvr*A*s&oADE6G3n@?evy&)sUX zHA*LY^LQCgL%wjyU-do@G})`1zc__tmvxF?5Cv@ws5qL?g#PU!=-ixacF-)v29bEr zr?=IpBENZ;aZybjn5>rwMi4fF&2EA+&cKT5{i0L;dep;KAq~GpEY>F3uA#qsM_S-a zhh*NQ;EeQD^0SNa*s&Nst#?+EgaPZ4?LzpeH2k3>d@M<^tcj|b`TFAJ`q#4@bD&5) zoMDJepms=m;fJDQW({d~Q0``HP0fZ=Tyy|GQ}qg*f0@m}~ zobOz#$kBWKlwmz*Rrcp{A7S2UH{u%#N3Dm1EHX<1DlTZ~=l6E>GkmT5Z%bNgMxlU) z_ee`Vta4ysPY%xJ#B1GWV2H)!Q$?EeBO^YOj$wOl^~@fVyw(wZw6fXl6a}6a-wxdG zX0O?%{7?WVzoWHirr($iXPbSi^D2%|wQY)e26E7#8@ZOa(w2D({9YOT!9qCm^M|Ec zEGehC}Fy$NI`e zUS#{Jm9bRFat7%6bs$Dco=D=P&jVcC6ky08_!R0|aF%?9jEl)AVArGko}X{a&3z|| z$?K|4kA@v44!^qre)|MSg^*VY&*G0@yYflIpYIP1#O7bBobP%|q)HpxdFlJF9_i710IM9n^;g*5Onm!|5;6YKCF8k)5^YuY7zSH4SG=(eOn zw`Hlcbe?}!67KBg=_Cu6{9N1imGGwng;|E}84FPXn|MBpn(YZN*cp6>zjh`N*$#;*AF-@H5J7-kJlOSe+%BM*9FEbnfUIc znjNfh+kaIxPL<~1$0hm!fb$GesC+)hD!x}`;kv&B8$%Dp>LFoDtKwvix3+)vNoTW1jl!O zR}cFHh$(DL8#I}PjlwG`*;3ktV6v&GPe&sxGQ?#Z6uHIuvgvVhj!RjUD1YPDy8YTG zzf05Y$EC!cHEoQ2oOAzG2J|uNfdj&Tv!>BQT(Fgiw*`w|hau>C5Cbjqyb>oKS{4+r zRGqP(%bf^Iwq>J6nq*{!emXaJY;S99pUeVLV9dEjhQ%`$Pl!iD{lId@kWkn#p@+5KfXm#@0_-gRS4Wp&Ih8f;2%Q({ zyG`hsuN@tE>{6!sif_3Bp+c(I?#%zSAi+JNqdNus42j~w%o-?eH@&)68x)cE7GrkQj3qT z%B9}pH{>v}I75Un5LFI#Wg1R(!sCU7>Fi6c3|+@Oo0LjJF!G5nae)ah%6xKNTbrF= zh*J@=0F#w9g%K1(B0mniC58W7;Y|7GRZ}aI7=$F2W>W?MrKtIX=!;w z#);*PGlhPvw)HJlLP0+ZOZ896D!&iiQwsAqqtE^DNJLvv*pj_niS^_&Ea*DIVi5VA zW?=C=}!tN^-lt6f;*a>bD(V(DW;g&#)D%;*ID}f3>D zq7>Q#USdBlh!&wuieu(DL-^Ee+J>rSTr`c_T>NxepYqPrIyBT_d?OB$A1 zvnC+mu)AB%i)*f3Qu&PwES0%ckKyUcOTPOt#ixgFvL*%W<{GvA+)MdRILtXi%=VS1 z)^_;7K&<-U(G!Pg?;(rK5PC3p(eZ}|#GM8Q16K-Uuk;TUA~H8`6Xn4EUZ2}&vUc{Y za!Q*j8my_F-b$^`iN()EX}wLGj|Qg@sdPprf?^sQ-X*S%COhfYm3Q5N_)@Ipy(?~X zp)hnK?guRw74U1GyX4$j#*yaAy;1;+MxD71DHrO@Z<_wuN*#&ecBRysIrGPE;H0fu#bHt$ze=GDVS_q9}AG%A=5Y(hfOOmnr zUi&lChI$s`QZ4RT2k7x6o9=PgiD2O(i`=?*eB$6Ra)v8T`(AYXh{o%DM6LJTU-@7o4PqW39o}HYaP(esz09s$ znn6@Jlvovye{^b4Yg+YMQ|W#^?CGswttLN*DzIS}eQ~~aB^zBDYSosO)_OISH&;Iy zbzCBaW0dbWN);<(Va3oPQ>8|u!-38`Uv2>Bam0l9qIDvOv| zf*W*$>49x~ik*McM)RqR*75OH%WfU&u~+rQaoXRxT!{(3X024aab{;#mo9L#9Z@}w zA0D$q#ierXJ`Ai}^mv`h+S{sy=z#AEhP^LcgT%v?_~E(I>>j;hW26`{2y^6_=>S}i?tj$|A|HwbKXb^0 z^wE!uj=mIgyojEmtb0h0I4>Ek=VhEciBw6;N9wt+Y%lDX97K-gd?iB`PM??;{Nyqb zgxkg6d;&L?)`PmA@`^hKr$lWRay&0vaM zsLf1;VB+jBk^o|g3HBwBOFd1k_4l@j$MqA}m`zuL>)UehcXKh)0X6rw&6;r+jm{No zH`BaZ&sFj^LD9fKhIGtnRd4`XO#HF3hIzib-O+OR&e$a=K&wM4==D)) zmebLQ!O_U}=tX}Y^vrJuSCyyM;c5Q4is2mUC=DFy{d8zw|`F0;~i4)>) z_T$I^Fba#iBE_)@`H(^RTB>d+ir*3@=uU`K`i+&u*PNGf(v9j$@BMEts zP`qxyeDBvgmC1Y2Phhbi2w$A~gLEPI-eUpCRgJQy(fqXGuofe=^&ASz3a1)f5HbIl zR_;VeQdQ8acA(!y>;yY73qlr`#?gA%pni+>o{}5t+Ibf>#EH!w{%d0k=Gl=396oFc zii&O;Ck`K9pe{(^047;sbZ5_dni3!1Ku11qmt(_TbN&mAO7tqFV_~*eK8|+F)*yw) zAp+PE%8hNWF9U2ZpD$5*`%{FI8R!)I$7-%I7O8$kU7xi}_V9BTOG0#}3DPycz^o00 zocZwPnlvWsq(u(=hwasW<&7ngY4I`}JN$5~uW^_4xm>!j7MMjC8j3*nnWT#{2VNxB z*uTE@jO46Dvo*28+VV9jk&yRy3Su)C9;xKmL+2~EF*e8Yg4amAdZx1u3IX5=n02}cN<|tS<464Qe9qG0^f?KrC~#8* z2Sm$tNV=HNXMewVe!h$|RoM$ggbwAc0#-ZgJAd2UVgtXE%RLjY3hf`tuBf>t-dz8r6KNvyfuIRyty>0127ZiK2~-O( z#jfR^yMAChiETS)HQvML#1t8z4RSP)t1Hq!XDAc2{EGR`hPqMnu9l-p2(5Y_XH_d< z|F0A5X2Z%;CpbT9Hta&xX3mq=VIJ6|f1 zf7oE_{)M*fv`QVGn8e^j5d2KQ1EavNh%zR9d-3WgYX{g1 z{q1Hfl7FrKGi2Qourr2WmX~IKFa(g@qppaLTpx@h?CG!ki(n3wnVF!RM<;%^Dh=;- zz=q4P?!G0fB{>zRXKO!xw#cznfmrqO8g;wH(0S4=CLZRi4ot$e@Tlq>^OwCsD=xLx&B&=_(Y5|%u)gYiy>G85QLd%4B)P}cTDQ26 z1)76TUH`Ob+eg0esD2zyi)GYSt8pc|^zG&Pia+p&PevI?=Qa0G!{|bsuFTd=V6LsD z+>DAFqRLGRL5GPHv`PYPbb_dj%VV&>NlLQR?pjW>(229$pI*P5z*T`aXT^#bzx-in zRH%uR`7G@DEY;)PexR-)(CtSa=Y_7y(rdN!yznU9m&^=MC)U*#9aEHS*_sO=`0cn)z!iSQVd#?ekxf}qB z|JM8gskk5t+7l!u+5?(LRs2hX9#CDq*x3Y`NT>{0t{@W1!M8vSVBHCJh_Fk6RCabN z3chzsU3MKwjMOUZ^ImAJwEZ+}kS8Ty2=p}VfX1P1!BTCeco8otAbQ!JdBah7`4*w7 zU?rkb_NSjOi3gieR5ohMa!RV2uXZt#FuufzWA2Ca*;zN&sU*xwl=e1MT&!!NvJ5+6 zXYQUYYHl{8M@Fj^20`86z9Aa&;Sm&n@qkI@8Hf8*_ba;{gOya<@YEDb{c_uUj>Jj9 z$B$Lot>AigdOJi`R#3fER_F(_?1s;M5^KmFTiLfMFIykq_`Y`KH#o2`>u>dAUTnH? zDE%;?Y!J9=3k<(sB8XwDGNWSgq_>)jE$Nr?b&*eU81~G$~uq!cYR1 z4%MN1zQS%Fwm&&XB5OAYl>YflgCBQkTMqJ&g?mlgcPx#VmTL_Oxx|{I}R` z8JK{JCKl;2#4k$v0Lv4SP^{7Ox3#x^zXmzfOQ3+=H;~ghzvhi1t_Jxr5!8YtpsE&t z4^pAa&-J?hNE0LlxIGyVE#i81U@0Ud6n|Ek?7>(?96?lkbejvy{VjyGUU^qXxYzm5 zg@i$|^Pjl5xCM_Dc3L0nnuRDGNgPIKc5p+jWgcexC4G2gcQ+ZVr3b5O&!Y#U>ZUKN zto{@+L-3Ue)+S|Y=9q3S$=;S3{nkt0C#}0|rsKGW{7xRJV%SGC%yAg2vNMljTrjsz z(;2Y$2Hi{#HI@Py7OFZLIE(@)nAxzBFN9nge_HRITm`Sq;gp+T*Og-j%|7HZ8$4@& z7u+_4F*unZHoN-}3uRzq7^l|LLwc8=$fwY{kM8g*tFYhhB3(XmypXBjsjOG!#2j;T zWO)C^fk{VNZiIc^FM@;@8_LXPw>dGnkldy3Mgv2YqqJ+%9p*c%^yn|UwGMlH&oNQ= zh&}F;`1mAkH)=RzT(l*QR6Da!_}VzNKJ48P!p@v|t3AmD-;!95PfaD%(a|aVmA`fj z%3|KEXulQ-2n=)uV175|Fo7V2c`E;Wo1ke?|TKfISUmVj-w5S zxA(?!xIXH8Vx3EJ#W8lttE#^3p(I*x%j@Whgn$x0Gb) zj_%ON=k310|9#AW;ad@YOOa)0%Q z??mq2Wx4I~b!xy*t@gDN>a^?gMMI&onR&2`+C8_Hqk{xXQe2fpraZP= z&1xd#0R4yycq99}-;sL_o$YB&$1a*B?V zw^CV^omZ{CxM+LZSJE(2#hKX{E6=ddG2;BFTH4%?@bej#=^9G8;^#^suAey0jb!?Q zyR4zE>I#>~YVe}g(#amsZh-3t726FURTObaIy_G_BY4ca zO+iYL&3O6!oZP_xB^|>5w5)=+6GXB!vCAqP`f;iuf#z(qv~{>9{K#1{q(=uQzd%$7 zlTH?^ZRO&2L4QNi!|-UjNsXiWbmp^@iBylS5u+InRODs`bw}}dNZf_;ZZy@6h=A53 zCefYQ_}y)B6SAvq0s1=_sBib2+&wTKa>X&dP<+1@0W%86p@_u!c;`mFXmN=3ET+g& zMRE|ijSqoY??ZP&b&Bl&`#9pNaXi^sEA32{0U7aCc^XeH2vEDxU z4C=2V9xR`~;l=gPgqjX}ZI*8o|H5>Zb2Wmu4Ew_b@yxc%^gCe(-?D770=N1y~$ z4H@rvB;1Ho?Mmjey^W3{1mSv1AUv0A+#F=ubl2V`A&ygy*ZyiCq+%h))-1lS-EC7q)+(B( z*LLzrkklQ_OrosN)vWMy^Qtt@XOpM(Wy9NwJq*Gaob5Zos}m$bvrSGlk`>zc8V@<% z?GiLGo2jpv5Z#>5RDCp4GC!_GTxb^6&mZq= z#AHZL7CoGw9~;~3May-`f7n0rW*5yR^$d?0NQzw^b-JzKaA%gKMNbAPp# z_n4(bi`3!>_N5ATCp7r*y^#ybOPJ<`Q>xg5`X|^3JT;jhDVxU3eyb%$7NVUg9jU`j zlGp7r(kl=l5lko^zMP!YiVw=#$gKYhhKmc|sUB=jA#(bhl(D4K)Q0VgV)>5(l8(db z@lQ~`D+!jZTX|7|C7N$mwbohpQl{ugaAcKMLThC<+{dW5mzy@-8=NnCZTwZ2#(K8| z2pd1el3&!zrID^ll6KneQo1xBiPmJcHHgE~EaeRJ3+s9Z&dcZ}*)f8s3a|LPM+40)=6Ik<2g7#M z@1+Af9H#22N6R{-9ay{Kj-Rhr{YKl`=RKLBJ!|gs$#z1REJGEv8Z3eM zWolg!WbRFF6q#%$0vkzD&xPqF>B$-y7J7*ZX$F&3}^N*B?Ym3DRSyFRFGP&+E8E{kD$LIjUO%B){6Z|)pZk*X*9X?z}Ra8Yi64Dml+h`+0I z`Ni@94NdG!F&=MolvEpxKz2Y2GUzP2P#KgtJlD0gnh_)57nHI%a$GPpf8DwOt@Ady zyz5KcUzuZ0AZ!#Cf<&x+f^z#k-&_wLwN9cRO3f`(q-SQv*~lB%`=R>OY2>ytL;VwF zk(Ai@cy!!4lY|8A)djK#M3nYlC4-kNSK3jU7%}i35gTESDrfc=yvi&f~DMD_yepFy2t6W#cmL* z@;I2iLxf6~%sp4X3<6N8@6Tw_9WyhtoA>U$2l)UA&517z4QT+~gg&$bRGuCN#XMxr z`}6L^I+_~B81)at;&9cMLvB})!7gK1JA&UQ?&@f{J;u=Grjh?3I1N&Yx!qe!`7Lav zAqJATZ2>wdJ?=7!+vrq>wYvqPFJ8`Xz_NCb+^#MZZ0&lz^gP74 zbSOGd?eF@pK8snCE%elOG~skpHh4;h*k=QY z`L+L5v=`Y9%RU&=(U*qwcn`eVSk1?M8;Yeih$EevaQf2mAt&O+GBl;fOWTm{>m=kV z6N@$=&#b`bszIA`{`o|1XK>}=0O}fLn^8)w{gbUESl+TcA=2hBq`n&hyLM=pc}po& znbPnG?_ma^f|{Bfz_Pg>{ZS8n^JW30fEQ|)T^{BEQ#c&Nqcd=G-vlY<&smRpUMsLL zPIWZUH|Gx~y??c<4cT8)rt8F9wDcCerOSQ0A!yj8IhtacJJ>Gk9>XPHQkIr)U`ygZFyguFHs~3lynBJW=Q4zltczR}&b`~?jWuA* zN2|4FbU(SSSo3{^+pt{->BBy)heU*t5G%)+J`JTN3O1dl(y5y^h)F9ugY^{)j46p& z0^S$7A)3(a5mVEN0V#qF)!)wy(?v~5_nZoA)I2Z5)Hj*V`HPl7x{&eBSz{i_U}4C6+A22EM|wKr38 z`w;#-e$QB7re$Mjb7*G zDTVc949hiV8Jo*w=U>m&VsaA`l!Ikqc$ePA3v~p$7~2=DNE(RCIsV?Ne5ZQ61b;}^ zlu|rd*(0rmfG>F??I7^`C#aLWHPcKP5)u*?6@@NeM((=>BDJ#t?(cb#W`oND=BwAQ z$$brjK}#8YX`qKn;h!z+G?U+iq269jGPtBxzRgYVXWJ(E>_T=^=TG|X4slDJ_U~aj z0?Od7rC{ne`Nu_?9eejWV42YCoTq7|hLNpV!6M$%M}IJ0Ot-F#(!5DDtLGQ9j{g~H z-?3PEaHSigh;u|mZU!oGW=~jW+OXDv-+saR!`5dTC;FPNO26O`;qC= zy|Fb(*YQkRo~YE~0=(W^_E(u~bMC1*gGu~}gh53bxg?)0lH3&9L@Lo~JM<5Kf-`#yt6uf#r1ek!g)caL> z&JL~bS>x!~x_f+l;gVXXwJKa(+(Kj8Lp7g98nWe|A8k8@=i}BiOnHm^HYE!Zi!DDz zwT90ZKe^`FL%&jH7HXj*;?Ir=u84N@Hkvj!c!x>Rcx}h)_RRt%&Z{(2sMyD!>FU+X zHe5==m83q_=?itk)hqVw6+puHx?UlTb556qdlZ;|8SVTM^sbjkg@Vh=fE&~TH$Z*$ zx9#ncB(!2-fK-`S==l8kO~^dR;_CuQB+9gLnVBRo7|g=L!W2Lt?}9SN(&9<~N)(JE zcd032QaB6lXNX%h7%kx{U*rXp>~2lc%R$L9jTbjbTh5CPMoxF&R|a=B8d~V>hV@j7 zN?zrr5t1V$_Z+*H)qs+z})5$(H?{#kPRSZ6D)`@SILN*e|jGN)I&! zmZbCJ*14|xcZJj+@?K|fTK#J19mExABicIs?vvGzarfwzA?xQHZG*ufte`9Pax0&e z(z7K#r-Q)Xo2>V+M7-lH&5BGv7R}o^GQ%}6mMHpJe>wb}dkz$&46tb<{cpLNd zD!Wwh80owk?XEe{MNzfecT2hc&H%Dy$5`t1YE_ixMCXcs>BsX6A#U%&g}Q@ss5g{K zm#^F9nPaVfN6z+`5kW0YF$h8x6DB&;nuo@2ZC#Y$0Laa~BrX#He+NZH#UeTwXaYzu z5LH++T|j9?Mp3vcj`7(u)z_kpyVobv4k;0St|={hobUZ@4=}g9?=VAXuSxeVMT4uu zd9NNq*%B0u10<&JOkBO8IkqU?t8xnMt&qg5-54nHzK_?tE-%w}Cg?@TlCJNaG~!a| zkXnN~I|)S%3AAz*C{cJ^-QF9s$<`7SItn70w5#Qq%KEb#{Hr>o^u|(PSrg_EJM_BM zOa+=ilW^x2vm6HSubv75wDS06Vbx-TFXk~_d`TFtV1i(K(-@uWWCNULTYv935XJiQ zSV4HBs0MUEx;Bes7dNz_Tf#@cQq0Jd0Tqq#>w+WBtjCP{>Nz~S(5Bf~zpEl#9Ub`%X+cE^N zya;BRBCf|1#94KSS1^XQ$6L$~AcL{eBJ3M6ZrTy^Hs5$>&^#Lqi=lDMjW4}1Uo8wY zs7Pc{M#w@1#>`utsG`>WwdLXua&3aG2xE1_>0Op&>D2xUek{ke-OGgbko8vLIl9(v zOeSNtnWg153}-Rhmh+9U@#Q5g#=n!o$RDqP#l&To^tNp0@9G`Q{>kz3j`jpTE(o5tVy|73@-l8g& z5)0P*q)6m15{!pjT5ZM|{13XYBujBmV8qL-cX!7dFpfJd%vhJ_ZYf_ssf#n^UF$6B zSlHb`m3raM&Jk&Hbvj-#uk(wKGdHNw3dj-xlHu#D8AR)>t*kL^Tr~t-`3k-p#RmpjO%gC z`(IQTRr8(;f$r^?nA8TPC0=SJI3x~WCNlS1X;}2piDUA(d`s8kQO8Fgf4--d z7bDi1k#)p2%rIfFtL!^h=R>e{IdQ@+@4}q@wW}6aavxN#{5inHO6yaUmd;yvhySjR zmwMl1)--9$Rn0;jm1l5$9R}@e|M7?-7>KVF3KXOM-4^jiR=$+c=lFa=Y&!0@6xakW zcmomO;-~$E0YPt}ODs4@s~&-uAYdTU-33MAr!0N;^=T4bN4ySuq;wobsHlb^0E28@ znhfJCDs?tDDN7NQ(G2S)gVoOoW3Sh|UyP2`dC<_*I>bGtAH&D`otPRq^O*?u6%YN@ z&Xr@7-F#%lO3vMc=UPH06M+n_VCd&SzkI>C`xwtgQJhX z>A?X)MUSCM3rfBRBkh}-vCw;JGbBqd{OPnV*ZT2-g$nI##~)!Jj`m)kV+k2aVD(JB zUr~~RKNv`3Wf%;mOvL2hOt{+bJ2+RvCJA8DS;AVrSSVq3dr&UyE8Kygym3br!fhrC zR}s1ezBM@NWZ)l_nnjy?S5}D5?cT2o%*a8Qtos+2o@otY{R#;h!U*ifDj?~bRO7i& zp{SA_8s??#J;$VGa{P`7kvhg}S!TpT876ure=MJF&q_BvN0mRIrN#s1+*ldY)QObm zyrg-sGs+(}<=h-RF&FZ4HGj$>fHEwSXZqy1TK0v!((%XA;4EdZUW7SBD&pKoVbs|= z-w+4WyHLO1!@|m*Fod&r_UVl=72kJg;S!{7 za3u%Mp}6Nw7MhFX^JAVP1gyJu9f!M^3-{TlT{T9{mzK$XzhX)fXe0hUX=-@mvfikw z(C-QVW}gfXRByqB_gmf_kf9t~wt;G)*{``)Sqdzt?Fb+6y(N26Bn#p8fO5Ma+Sv+= zf@NdLXV{WW5Q?> zUwrj+GE87=pvd#WHNK2_(Om)o(+K*N^(qRMWhSkwcb!Yr&9%UBSlr#EVt zYb}<>zR3?~6HrNztH0F|#ow|0?P97e0MBMpdyMD%AlP_ePb)ok5Okw>F8RGFkU89? z@DIsN@D}PkP2vPCO;lk4WPf;W`?wR`0Uz7qBAvfuo|3q}D+2@U0cKHi1|4uoVx;83 z-JdJ2B_7f2(M*3I3JQ9Ec`v8}mZ0|Ty5mHv<2W2yA6Se1a}`T;oSN34L<02wO(87? znssy+LLZ3>f3R<)*)?$N^X@Guy`37R((CE_)oEwMG+>T+kA72duoq|v(&wct)O}?m zm*6MGcW`Kxh}Gi$??rMoFrwSdf?)5irAFyDkKdKP1>^XiqpB2-4kNYh@vJcmna>9C zm}_wk*kt{C4ESGKc4qOHgU^LQS^r_3R1ZCq{c~9Oeo+arroh;l4X%*fJ5Urxyf0=i z_^<;3&4fOFzgK;|qJpnlRE_oD*BR>yjWYrskAI#!ER3w@Z|C@X0xNoQ`S;NH?|q)- z3H|d)(dWp!g8j|EN9^K%AC$3+=YPKKvk(7mng8E!pec>7KAiE{J2Nx0cW_V!AYXrF zK9&Tvjet;SQ@i${$}@rpz0g1FzDxPfU$L~IVT;c#yILw+TU)r(K&+}j$IQ%qyV}X( zAz$Fi0~@5@#X|}fo8W|e9)Yk39?pOMwB9s4FZt!-?alSVAj+fa#mkp3A;IW$b{DlU z2fV-QE^aJOH_ktQ{?SBWpYfmL|Gsr~DwzJ~$*%wJzj0i~?{=7^A94re=G%4r`b7?y zcHO0V)iJTL>cd7%Rrn0p0M-5qfJJfvfDWiARHJtM0A3ja$R96({CZP5*)c0KGYWt# zazN_3i>IdyxDwh1rV(=K@Z);bO9+&Bj08oT1jRSm+4R*tJP&(~K}}gKz%Gyio6t|g zHuJl?(STEL2514GcHCnT2|+>1-IhZVQ2BacXNQE2j?QCalmNh*i-KqGumaABfe#>S z{w&pd?BnCp@%#5ZfOYv2!|>GQbnj*Lw$G33es&(7TL3=Z0}81G1L`jNy`Tw%Kzi(| z;}{ZV0RApO(|-eOC__WTn=~{Lph&_JU?vl`6cf_-Iz6h=ha!q@{lyAJ=Iw1H~*D+1LgOw*ih@( z*`b9m3ZfwNn$Uz(P=&()qasY+7xcg!0M$QF#=mic&zHvh)z%Zh2fiy2&>%sJ#smTg z9y1UL-c(mtL+B9*#EnS{E2|HH=ScwhIy|h-XFsMK8XEeCZV8Yt%d&NzY6gL_$CR~p zzivZ9U%!5h51tD=1tl09=j|wv)6ZkqKYpz~ z2LxYIQW8K0i9maELc+l9--lCjK`)^k(F!c0t^lX8Fh7re1>HtB_)oi9X?zC21AGPw zSwQFkoZ|VH)(0B1J2TBPpz3-TC;q4Duc8_;*x?^KNBt30-wDo~@tka{L7`vFuh zUb`Q{36#|+K0VD^I7i1o0+`VH#ni+I%iy?(ZvZRaoB1}yH zlx3pHv!u7Lk7wc<)Op8;uuE->D8hwkn_xKx-roVwsuNdzE-4tE`u%NGl zgDQ}lfH89$Fhs!H?##Bb`5paXfP{gcZd9zW5k1V<3f+-518hgxiyv8%Tr@N^*brBM zb0z$9KXxTAFYm4q4cQSW%0LJ~_mAbZwaxl0F<%W-q%o*wuigwwNOJ2bX?^8tzhngG zJ_8h&E`a<0{?FT=Y)`R3!oYZmO-LI=TZfy*%BQy?FZcDOya1bO6+B>7*c|(&53uEB!Ko=636$Oh_=O z-9!C74v>W&L$W~-gw4*i|iJrtT5=66i3Ps_JSm;C?oF z+Q;?OIM2~ST!8v(?-g126-|TRZEkAX*xfV((h|wTDS&&TqKd=?C0kBGaUzaMIYuSl z&9ZtIz_Ot%+hI8eBS-@wjvWFX#@i1(3 z>nhCP6!6G`6B6O$OM(7R9K~GB4xp~(l9DvSZr^NRpyysY2YuPRk@>jSIN>CH?&@L{>OT0tq5|6szAoSvVjip$pT0vw0-`khz~ zA=gE0iR)8yNXOCo2=NJzQz7(@wCVq7?%KnlPTTP7tR$qF)a1O<(Iy+S&Iw5<)rb(0 z4%R7(PI4OLP`;AV7Rsrj63Ve0T8z}DY-&ZO&?Z!rkQU$lvU~0I{rUa3^G{tDV}8H) z_q@+@Kll60eIq(huWK8Umb~unw)bMu3!eD%bHeyI?9l-f!|>6{dU~w<{Cv3;JH+u& zHR0$~Q&ZEO;Ni7_Gu)N9-VL%xyPGnn)g5O=BU4qQh)Wp^NMb{C zM_$8d`}5nM2L{3{vlbVm*yWmedgiE%-4>)AVp1o z^oVW-0ZKcejHsf1!Q>!&+`dkoSO+(GWo3B67 z#1-Zs5if`4m$0y~YjwDYnh!7eVvfzQf>70g*TKx7LB^-LUxu=n@OK9!g_Kw|+?v1V9iNzYVG4H~p}U49OXY<;1NwF~{Q zI?<;-b&*K)XBmzMj@#kxAx00Un9L|--g!^`o3^)Y;6PvLyUgrtHyG=vZA+1Gwc60U zr9Ze8`n8F??xM?CS+7yYMc>P@HZ#Tsc#8J|G-_T~%p2QWF?V7C+{Er?V$ zS;wtldO`>aAJ?pTwVC=Wm$T3D!s=-vzC zt3)0g2ZRWeuHApVA>&|;;DiBH1TqypsDj@ukV<7sC*;$%lVSIan{?=Y%dOCq2tq^2^;)g

*} zywvO&DQ)6ZdDIzTWjaZBsXWSPTc`2{uNbhUz+ZemzE%))qsAL|yNn!z=ilD3(f$8m z8X}FL?5K&6J=3FmviZ70p*fMpEiM?=T5u(&d~at||IiSV)Mz#8tFF+}ij@8oT(R%n zMge1b0kGkV2t=xxO-w10;)hIJ=T`vMps)#(0WkaypR=SFsTdeIk9_?KxCBu_(XYhVwXAr69iJlnn&|(<&)-k0#3rMx8vXOuAt$cQC0oY=vcVZng+?cv)9oZ z_dP^DHEm2ah|J5&!#r&|8k!EsL&^f4GhVJ#iF@p9dbZ1M^j|P)yYcVXerk`G`QRLvW@&D)EqcUo23~B*T8rAVbT3Z zkC;#%n*||i#*G^{fWsX3_5E(^)qbZX)ko5gntx0Hz87JQ& z`Np}M&(+q}CUwtqL*+2{eh^@TGRu{N1zP{-&$1{(Dhn4{(7EpK)YKeaaki#fifi=_ z(C0GwN|KR?p_~$Z{R>%uM_1GJjL|af@a5u2<)9V~I0_DReA+@YGE`yC6N7?(34O(G zeuaTT@SyF1g9qQWQm6Qm!^0Y*UwWfaez&wqz%@{1u|Sfv$>x3Pl>O+7)Y!8RGmT@~ z>gwuvkMhcu3~Yqa_dh+mR*m}g#YhYjVgM!Pt(0N$lDYhxE|J3%Xn60%``Eg=ZUC_C zka2yQvwNQ{LqFDm5VgN^Kof21lCoUN?Yz17Za@9kZWfCb2Te{P!dz+}ir$&k7v%46 zf_8#7tybr?CC{BZ$9Q5*8+|jsDV8tkPgr)1e#3ajvY`Wb_@t_>G~m+4tA1}97^Dj7 z=zwW6e6pdkB_+r9QxX&uyok=Zvom6*ysj%$)M;bXvS9Jz>1cH6fU=FvOen{5VQoYK zlA)nt_lw(0fLC}MEBx`X1#70brOg5u7LP)2d%D|RN=oYKH3wN8-*&EDZsggsXAPA- zEO+i)0Hv=Ow3EHXSTbb{`oMV(=&}xu4Jbu3M@B}HkHkYCZr{5m`XfJo|M2K&F?39E z28V}SvAvSDcFlzOq+`x=I8>PqMNV;g*D{g4Ms3%Y9kNjyfKr+&dUiZn5wg^|Y&IGm z(c9_m?C*!cCy8pYS}e;XO; zOgMSwOcaC#l>n6BCn`uDVq_awX32xRFd7QNQw$5f!r94lhr1;qU?vbY8Q7J_;^R%h zFUT;NI|Iv{*4pL80Y$^u;?>)?(GY|0mF0+_wn%~_F*i3C1?YJU$JMclD_0hdH5`Jj zpthyep4(d~TY)#3*xSod57;QaAY!RSXo?mpNKY^v6-Ow9mzV2trH{qLRIgDC=At7{ z-}75G&&gHyI&@uk=iDdG)c#fc^Qyg@UKIOEsjEB88yp$v%=(~8??{X}i}H>+ zq`kcz+yxubd^1UXL&H5ljC`+_6<+)bd1&*~aK#+)>yD0%9lX0o$8?DjwjUH}YZXH9 zEE8LW1O*=!0f4ST7XJ@s##lN&r%QYhDmz@*lZy*jqs@7Iqb1( zS0@lCeAA68{SBmWKOE=3-Xw>OFcy5=JfrgCD?e&0KcrdtJ!7A7F8PpPdV1j3r;n3x>o>5sgMAfebmo#@FHq-6j%(o=8Z zdXGCTd4?0#w0*Vn&jmwWQmDM1QkkG_AgG9sxb zJw2VS`NL#QA*2`fk&)$3J#SYe=wdBu(9%AbSf@-6vEu$tE5K#PjvuEql;(kc*B|yY z38U2y`q&+?Er_;4puQY{P_ ztGKk{GXA z6byE87zZ1CQ=%wE2>uA!NhsUNTN>Iq=-3#*WOVGT%q;E9OmuG`4Qy;pEG>9Bx$kib zvfnVav$L`l;^H#@&o4MFZH&0?Jnc~dqu^RSQ?Z4?D2So|T`)@MmV;e@!JdmhQgDo0 z9(9P)H;CpxS^LrZS|i>z3boLLYv_Hi@+!^~(wg^_xQl72>4%F;{(|X~M#$0Ee|~4Y zd#!wlJN^BQXe#Z-nrYf%XQ4R-!T50obHwf@{Pk$xYkO8VvHrSm_6G`m(X0NuMQdxx z3&k0pzJEU~b<)wIFvh>1W&}H&=pj=Mp~R-|zKt$Wi}(-+2B1*%w^SC@Z6j`7Uw1meax+yIDM3U>WkZOzZTZUv}5h z+&suhEBDk^fY#X9xbn|0HB48C8!1BUn9oilVN*G2^yQm3Zyq{{ zo2w{;izvgv;UT6PpKF3yY#;Z5#fUF7`ar3Zt!fApm8+}kUW5CQBz(OROV)iASu%+% z=^7mk(kMaVih-%e7Faz6n;gC}fPwbf_F!aA)W=F7ZlR`ftjyVZ&>(Wsj~Ugtl~`+_ ztqqIRd~4F01f7=bP3;EgZ`;E9WX)b*rkmHWMM+!Mp}<<{X8VzQW`96_{sYG{O#?CT zpK2CGmnB2!B_t%gbQ;3&5)OJ(td90DsbXnqX-`~TMPS-?cBCRABEI5`&quZF?d{d+ z((Tjg7rNetJA!c)T-wP4Ke+WcZS;gK| zfc0FwgKpo?*VV|72G26*-6>21Ljx8e{CeI5ZvubCej9$f(m}IQ`TIWSwZ(CYH0mbRb+G#MC`+kc_ky!3vAffC0llhgEAXQ4I;`Oa& zw#*xpS6PSGikCRq)$*kWHPzJvIn?sCTRwB_ZA4ZCdR`&Sv1RevSsM>=UMr&zc37=# z3S;Kx;kl^MFZ=qnmqE+t$*BOY`ZoUYz-u%QTAkJ0Dj0Uh-NoqL_H<|ktY8`%8dfEv zS%YS&85vLQ?eD9qs#@&~8li!C)YegG1^)c`Gqs>^Y%JJ$wde{k^oDG`hI?Rma7Y^) zHVR5g$=|Z(K5fQhK>wMzgKvECosE%$LhYH)}1GJYoL z*K{`Tt(G@3G8ziLN#yD2DO;fS>rY`uh1>Cg0{zRX#g&yh^MX(%&1`Qu>Y3NiIG=z{`ap9c8e z0pO3Uf~8Z7nl?|Hi-J{=`re{aMNAP^^UZJHzR4CqgEjDWz{ALNR`ypr6Q2weTI;nY z$@3A4Y+Z-ksm6zpO^k+$s)}ZOuRc{G#jAJ@<-G32<#N-s3xF8Q_&Ki0nvWC`t-0-% zc$rrm&2U+7&3FB#s~JDpDM%ewQ&sIoUiE|gKiR^4eo@v3TTFO98x^NX**d84J3^GwMUGc`F#6;rMe5KLXObi}%I z=gxd-lJlhOoclS=vVkfs2cNP~-L1JQBzaT zmeDaXCaG7RS#I4{kfjS2vgEjR<`v6_4%by%?{Q1uKU{7Ye3vD)asPwOHyK6s>w3?g z>gnAYJNQ#jf3y!KhH^VFIhwd4RtaEBGhI3RVgLnKJwTwQaMDD)(cSW0kvLIT)xmP} z;RnA93efXi$pSXhI6^`~S>PLkir`>^qv49<*sPU_D$i!OHP~{e%~}IuhM|C91QXJ{Jw42N_iokwe)VKCE)I^mnMkeROt&1`xKD%@}F66S6n$T zw+W?+12S(PpANIb-V{-$5A@7$`ZF}FSBwgVtvf_^za%9w8tj^f7Z(ea9&NPjwHk?f z3a^!tB)($FSn_&&(`8P0{9XT29i77n(eCkU;OTjJ_p?3QZe1ay7uV8C$T|KVYw_60 zsVJwcw3L>A%<&cgPX^p!XJ=>Y^@%FpM{BVU%;ho;5vN-S$JL^_ER&u)>8g2n04(I* zH?AIwrZP-Q){f12`L;U&krqTj{AK@!!lpskkA~F)K zbuy!6b+}THFt`I?ZHmx+4+{W%HOL0G7PS-3G$~b(x89N>b1@Pze)~f+zCv?B8_{KYc z?X{DlEJ`s?T~^*`ktizAGB7lNSS>#(1b_epOx}#r4XF{E77pbceLtI$gp;k5h>R-N z!yTRRa@Xp7rWcnIF0$CfB(UQshwfVhFc-<_g!7G zv_f_`5K>}Jb}JbA`ub!uN&LjuCp-ziZwcU44Gbvvt{4~?aA%S%8Rc50ef@d~0moc= zU^xb#J4U>GIrSq2RZUT`Cy}mRL<}KMglk5{$=ea9x%f@ws=b}vMET)r4HP`?zxh); zO7#pJhs=54-^>{_+neRD@Qbx5UTDonH-w-oPL?XzV?DUFeCPl&6TQ-1Zq6XuD0n`NF(yC*fZS6B4 zUozS-{{@VJtKx_U<$mG>-|Bqg2to`1O4+TgEpgYYS3DoNOyX-*0(Y?n{03Ld8F*fZ zL-}SrM0wVy!w|XAT*C z@iM7?ENGdD$#u+iKRqe5Y+)hr!2%f0-;H$Nx4=x*$rh}Ql>t(*vpkslWq7FAo~?Yp z&2kDHmbj7<1z=Xe^dkNxtHm#HxDt?nB|I&oOX-1j2p0kX%wk%mu8RvT*bhel>6oGp z*OyfL%d4yPdLbg@o8ynb?gpJ3d!ccRHw1CfqJdHQ=ELOnhocVTboS2oCA9VqonOo<*2)j>iHc^JF)%QM90&k9 zDiF4I31(3sY-ogPGpzafI&B@bTY4p7ua-GHfk7B4UASyhq)YEEA#oX4NT8+r;V^*5 zn*YTM3pJ2~29Cg80O9jp6j^g3WGBBbF%dg{bZ516^X1L89~Bi0P*Af50oLG@Ewhxz zFy%!lV<-;9IMm(tXOy#bFPDQ5uG<{$ z&+Tw{oORgzL9Erjxur8rM)`s1(O!dC{X48H;pu!D{TSdKQ-FkSj*)}o&)2JmXemdd zL&c%S(Tb&eEPtW!=w}W!Z@^DwR1Da! zQWg()A|3}(ZvB^~EB>7@-0kh{)So{e3(!8bw6yHKhz&Cv$hx3bU?H=RJ)N|aQNo#_ z+X96>UMhVEFs%DE9B?J>z#I=2Qo4qQ{3F@^|W z-Uo0NH)f#EoGm9Vs2i#r6KJqbk?={pZn8k#3RZSU?T?4Qoe%sB2%`ecJxeU*?9 z|4@RFh*sn+-|PyYh-k=?;?gE$XlQ8zhdA!5SzB|9ZvQ~E6(Lwi{Z)R)tN@-1u}%1z z8{mtB9})~f?qXb3zLS?}Fw?C1Aof~_$POhG(<&#eYF*1_S|ZWe4fHjDCVu{e2F$e{ zghPU~zT$}`6yT`?<*vf*Vy8#pv@a5cfH!10tZ9}`UbFN)6v}izUdjgl_vWyPD&s8+ zh=L@Ci2C=XDKWdML6mRZN9;(asviPszB7F2dJiFPSi;fmasabN2d_vK)5svoh53PS z#e=vY22fszgHY1a)&icxPL9vY%9`bf(LIZ5E@&FeL4Sqlw^50ppdiP^A1U{^x!=tt zxYKi$Z4m>{^#Q17=ipE?V^cM^Sv6_(1uzUY#q>(Sxj0W6i^2;ag}N&oFZZ>mh>B3` z=-$riXaGd@JH$@mFXIHP4FiAW&-TPy`Jwf+?Gr6Yc9Oury#8aa^oBoYe*#UF_H}@nlS{eiyDy|2gy`Uo_c(U> z`S>ouQh)q--f~PdgD1-WpGV{{PLD`6GQ{}*WNIb!f*1YW6mjwYKS>yWx9k5sjpYC0 z1sAUqh-Yo4Ib#~PxzICQ^zYmmqZXc=pI5UJl&oKvcTA#?Tz%u?F5o*xS0ivVEFUd; z`|KAg1bT*Z9x&-*fC9vD66AMJ~bbnPUY-l7r%}$bw zcJ{);f90xAHDFz2rC-!mkQe_w8qR+9=;b?oF3d2Tz$-+)pN`KbBaYEI8{?i-c@JOk zFeES#5itdW6|p+LYbwG+&=vqF10N|;uoi{&_p1YFy9|qe=T2OA_A;2{%K3`Vlr^$a z<;ki4eDw)h7#aL5uK;6Hd7h1Zf#TnM1EH*tth7i7RqE;X;b@YN{@U3jWKYgkH>M88 znpmAFq&xqX5%8O&_xwtQ0+rUn=UXS94P?H{T=S?BwBH5v`MUhhmWArZ+%DtEL7#6Y z3pcO{*{ZOby7)HaWKLS_zkT#O-^WTOBs_w{0WVTxbosA{C+s5rt%3YkI0#M8+O0yp1+>YJ9oYYx1`K*|~G@E+9XI+oaHi$#uE)uTNZU=wv z#e5c-z*8KQXY0FYk&RwxP1VSSn>GZW%b`hL&9T+;T3QUN<%bq7%X$nofYMA@34Af3 z27L1Ky7>u67#bj*WwS$!K}JS~g_ZTOiOJn|kzI9{g%?`Y>+4pFCl|!SiQZkdgD(`L z^3i9f#n^o1voyoQpCb_wN4#!V;2p#uJp>X8Y-hTQ{1`rJloWjrVznL!cUh{xjPlpUD;W64U3j6qKi0CG3Z$meJL`Td zPo`h5d>2OfFD-YnQ*U)jM~0;}v;aGWK3x~5WhNK?s9Dzi0-49Gje~bPx!!eXPxbs8e{y%wp` zOPI59iDq(2n;fMI2EFct_@rqN&HVJ}#_-99aG89%SYJYQ@8y|g{{~t#(|pk!3JV5l zyxWU?RHD?3QVLdb%Z@Shd00tT0?(%@d$eKL%Si`_^TENv`hZ@@WNyMSJ8VLDe4ju{ zN@}V+XnUz&a@1{O`Z=Jby2i#VObtQ_7!2n32%&8LX7Hg`%cC~;*?pa#q)OQ|CJZlz zZNph&*F`*^)y-WF_3yw>nzuPFA|mk7c@R4{&W%lunzCx3b6{bc)c`5ksfC9ALn>sY{3q%O9ae>Ld%8wapLut38F#e<)R=v zy5cZuPr+F{�iFQ0uCetymem`r#*tmvuKAI7W9R!_6OCWUnj8DtT{g$o2O2-pEpI zppO&S~cf`1>nEYRFwbI0`g)MhBVTl6io-I3F{#JPfAI-;=Hq5^LI3(cPoV%ueUlUq_wsRw=q6R&ci&B_^FJA8uk#JI{k2}%QoQnYyC$Z{ zFL`-n4kOk+S*{?D0tBx7-OdhjMkupvhguT|8G0bk1WABngYZ}=%9F|e$!*FpR4#9g1WPyWR@+3Jn&>!bue z+;Dih8KKu>Ag?o*a|MCascyI8)D_wx;NP`CfS%agf2E@0*TPjnd%8||>Olzd7Zwf< znZb&8EGk@Xwqhe?@^pd?Cyvx<l7QZ-?$onB`Ij|f`@y!S4z>PX)BB}9YW@R z366rC)A-3tgYQzs^y+&?E?SuMa!M5nT>rIR`_DohL7WUnzeO>KH_`+j@CFaHam@P{ zpr2FcEg7CwbU+dfhh}ZP!d<`Y@x#hmLn$}G*q@klHPEuRy=P70HYN9R;koB+Tm}J4 zke%3nVoTmN+nqk}#32PM8F+CQg5 zjiqS2RwsYX<&Yc1q$O3lK|^G(2ehrDyv(mso=3eH259(n^mFUh3{!h5zT7)zo7{7E zSp~;&ed#zsAtq+C_Fot%h2|_!d*=82#8x^bZxja1LZt8zG{sT0pUd&fMLk9^gX~S!<@IYpUn!04XXH!dz$z#jtd0UaW zSwVsbR{LIyNjk(M2b_1f$gp!B0aXZPgXdjMo>391xi~%Pbq#2)Tctz7fAq?h>wo(U)W`FJ6`ZzCC-=>KGFn2Q)`TqtgG9 z)-y0XK=L{E-*)o9xRwQkdPAxWDs0+Y8;4 zZ+**VGm3n1S|V%PU@l*n1Fdd+>eg#9uxaRd{B#8%%yT@-wf~^#*=w?qcgIaI`3HrH zT4DGIm@Ufi#T|88U-EnH|J=YB+7!8yY}kP&6?r~KknX@_9_q zpo}~|qKeX)9Od!*XMCPrA{9cnA*^J)VI6sXtSrz99*=eqGA`pM6s1Rd)J5{M&p+^Q z2ALBt$`E)azItDV!lL7Qg2Kb-zh-`04=e-U9?qFz*l*=_Z8>65%HB4+g$~i`DD-62 z@zZA3tR7oIwW=9;eo?WY#ksGZJ=7xpm?+Lowben@rmwm<9suRz}Z>uqmz%s5wiH116n&H6n36FwOUM$zJ&3j zbaADBUKUWp{hxDcS#l~-0Ro520l_fiNxx)dR};uZwe7zr{ULvPmM? zUXT?3GM=0Q^%objea87uo^>3Z(nIepQ@%c@&9mKon<%+=c+Od3f}lg|=A04E)9k~M zQFD~N^hy}T_=xUN!O)7l@pN#C`1`sGdFZ>uft9@TFZTXLZzAZ8x}~FD+2}9`S?}L= zaki-^ff?kLKE7=PXCh`NfCa}eMx{@^VEGpm#356{b!u&wmH#jl^cr^@l~;SQjS=~R zdkz;ziVop7EquYlushsZ82fMP;|FaKF&)A;ZIGk9^uirmT|FcWB(DC=RQEW6Q{;Eo z0&aDGuBq=IEAE0Y&EP`E7eXxOrytB;WV|%HUor^nDak*?i}})5D{Hmqx@U2sHR4~a zf>4%R;8=m--kV2!X+7`12bg{;*ASfZJHzk$!1#;)v(-t1(3CFHb($Yv$Nx-?DeR`m&e?d|BT#t(v{CXiE^O;OJNoBP>-^Nd5lih#q z{w!cp-F6V}GynP6`j?170awTnH~KjToQZ^!;J-oPE_5!P;*rW7C7|NWL4Oq*(&hDdUJIU=9Z`Oee3@Wd=z)O(A(`OkI@^1T06m3_7e<+4B46<&vT z4;PR|>sUwC9kNeU%WH$YySp`J<4}dgskdE>o)qiv;CPUyJ}8EYP7JPMr1 zEmE5Xc}QqddBNVv$MIu0Tyh2W&u(QPgnBY~3Z26#n8Kpe5*VhHdpYvYG&*39Q)#C}Mk8n7^nIhNtWHbhY&*iPb+khT(r8o@7rd%a@z?S~UCy9sN3 z=Seniq{^8V*38-AW=;6==o<;lYHOFy5pBuoJy9w%h zPr$Ec0{`k-1_AFxOedd*z&c!Z?tdsvWa+<=qyR*X)_1iYX_D0&K^ z;GUu%u&(}sWZ`CP&%@qo5?EkC31*Sbr^kg)+zaB>S3@ONmS1BB0SjHYdaI`Hve3le z;4b@pp*Z**r57_yU@ul4a4wc0NpS_ovy&P36)-hI^rvcEFoNHmoNDh493qQQiC`o@ zM>ZbIq|u=VoGX zOYmW7VLr~ByKa0|TOYcvWn3hxy@rvMD}S>TBsJ+*8V&_P+SwtO@6IjHWN8nL|&U^}^ZjZ%r{ zY!Dce%tkZvJo#(pbCM_-x)#wXP0e}SE|^6c>y;p~%TSR{Hj#(gj5s3c_``vC z6#OdESrLA9DUaKnJ}KqpQVn8i10kN16o}{agi`RqO|!INWn37G#&j(h<;7mKQD?|p zMLCywgQUV)r1vmRws>j}zaV?D9)Df>Wh2AW+muB4D)NJv-g6x!$&AO&nD(vltbV59 zMbke@aYdyE@h`IG?d_}WUyAew)%75MwjmY&wyc3R?wyw?4Lr(7 z&o=ybaKkI&3{RxPL6S5lA--q|$=*p@R3mJ`gRe8#J^KW0r(}Eh)Hi};n(ZpvGgmzw z8Y4gJ-DbT6ZM%mO0eR-V41h5H&4ZZRR|2r^0!G1u+WJVac0(-dI)Gk-W)B6&pT~z$ zqh{@IujIQdnCZm2vwO^ymr@kBhxv;C06MrV_TD#H;pC0@?{|}tXfFy*P6v2&I(2J) zZ`q}xv;b{RV#|>8GD7sNOk*XAZ|j#}E#sv)zxw&`hF^;H`H;A;@+}Ye;^9KrGXa2Y zC9u^2=w6p#7b2C-awC)zO;npLw0ii$`J@C9qvtH7`I18g#i}MISNV4^v-YCx$e&D$ z4`otEtw+`TyET0+9k$|4koRCsTUqbRaQ02hmOpNF{G{~z#Z1)b?@xwK27j|qgZCpV z`W)F`bX81RkLg}Z6z;u={b2q`2?sz2{}ztqsxMRwki~ACoZ6A{lw4`9tE_#z?1<{bhM%F!rTrT+RnNk3o<7b66~E+uM|i1t-9Pvvn>2!&Hb@Q(sqS% zKo;=wJ~IR$$bJ)6FTE~|8d+EasR1OH%N|HPCb~*TOOjuQM;;0M)w+qlohaX{R0qDf zKkEBI&_?o1I^lV^>d4Adh`y^@FQ3Rkm}Up33dMXSMU`IOb%yfg=HFc;VK`2J?kQJI zb+H$JC3s_DN|wJ}@!&mHg=naM%`H}eGs}s?Oy3hwF9noxwd;w{`_)&0Wt;M^@F)tX zroergh0e{qNgR0F8MuoV*d@MzKNoU4dGFyH*sFd~seS(}aRz=_y*^|pn*ENz zQr$hNb5!~kS(>C#y8klAd-JFD&Y=>{us30lUPnhTlMC2xUudAsAYVOXjEd(k_U%#r zYFA^`-nf*<#NooIt2h)_{!Z!nyQUlUFJIyflYd&fExeo1|3z2gN0vDlN5|^aIPlH& z&_HM?`S*2fW}VWifo1Y3sp{;%N!1(VsjESv_En|TN%t%Vkv+J&%5d56*NSdV%&W3| z_~dXdX|(mALp#|N=~h;-k?1b<{(yMAS)A)Z?*xTUnfDxm10Shg`n%9;Xm5 ztppMnSV-=az?QW z)WtB<$p+I`^8#qBvo+~MGpa;%#S zp7nmZkju7}s)S(_1iVs=S2`YeD&~PhnM@^XU_O^6^-JHmnwE!s|AyQ)4pXW5HZFQ) zDBZGKM~kwIi}2f)o9_k$Y$mfxig_&x_41Q}?oInI4NFwwIuP6$W>Cf!?&j$%`Ju%= zR*v70Uv2Ocaj!XX391;mMPs8Sz7Hu_eDc+GJrNj%N;+^^8Q>|_U zPS%iA=PUd&rf5rgvEtuB z{LewhdlQD-A`Wad>+~4P4Pt}mnE`Sfra>Q=U_*s;7cV+9U_9wdUnFbIqwy#bL{X2g zhbuVyFo({Nj|nf9i2)2!NpQlcq5+SjFIXDBpYF1Hn-Sy<-ObFI z#*yDGQcP^x!{YcOGC`TltbNNn0L!B!@;op-1p*-VAQLjGpLzs@9T|!~JT|7Kn{*&q zr*^9lFM^PlPfqdo3;FV%-;Xs0IVV0pUAemIm=mF_!$z@&oobljr!4P29ZulUhRA6MRBjg!72KGdU{2y6e0n2_X?Gcq+b^_?Qxfw1aPG_ zycid|Jj`?tWP_gzr1X|Mq4tauN{?|(eFXE!+ ztr7gZ`13NeGIt{GvPAAv&NSY9&D1$#-oy-~7e=j-5sWDC>qm&Dn+1 zpI*Gu(72Y?=RY*0s&3nMzYuhzNA*0$zY{VvnAR<9s+0-@NI~@7YhEe!ZTB4yYm}x+ zLo%y2+?78p+RxmhVA}aav+;r3PWvkJNw*(Gg+0vOZ3zj4Qc?zPDB`YCiQWI<^5=!o zBgJ(8L7^SeFAM{5=aQpNL%4(!QxNGw1?xv@4QM%hnJXCYv82A#tHf?5Csgjv- z&+O~xmPPAz{uIMtlT*sl8aIKT`y!z@jmiJpqVwc1Xio%JOX?cLPOL{9(Lt$cX~LVi zC2Qq03zz*sCqa`oPx;~K+V`o1hnF96Zhdv_eThfG1=vmH*EmdgbfjpQ$`h3U z5(AEx?B%7Ctw@oin$fwsfu`qy?W|Hk@o@^lu0ypZ4!tW-4(_F|9Re(QIBLv-8D=a{ zLH${Gv(-o zwa#Ok^ma((RDjkIquYLq*3|I$$ph){rO~Rlv6`q5JA9d@Df)1-hak(Emwmd@-Gd?J zYWte_^#>)BOr7#paWv;0js8ihb)BPd{E`!N)_PU*=u%s4B7`jlWJTz^qd<}YXw+4j zM-CLYzybSJkbD?Zr!jGdM?^?)2+26t(2T)gl;9S`#DsS1vF~@U(wiS#rvxHIdeVl- z9u^j=+g;@8$NYX3^Sw%C?P1Sn)!qapj-%GTrKV0<@D?HipD)56y3hd4?J|t{)mjX$ zRnM%>VPD7P4hzH4p1iB0xh7rkt|oMhO1D{Nj3}>!Wpr}Nre400=4m5?d;TjsM z*J|0WCae5nw?o!{-&|@RUr%``P`ku)5i-;bYU=Bjz! zfH;9d{HL85*AQxrm@|@tt3K)tsa0L!`_R^h`%=vHAsy3J^oMG~=`VP#*u(8@jzvJL z8mP81ySO6ZOxmEaZ*Q=}4I8!xZec}ECMT*-GK48Are!_s=~lA42z%pq`Qs|jJ$jK+ zA*~NzWrcHVF15vtjXSfF_6O8G8h_=nG7KzEq4O!B3|=H#e}v)uv+uo@L=8s=TC_pg z%(6N<4Luh@gC`nv2eN_szD40AHsl2O#V1vsSCTCf&TIMyloX7kyGo5uYo?49tRqBj zRk*7~lf6?Jd|4O6o5^rRG(bQ~S-~)1X*|}vB24ZDl}8ybja?-X>m!pV*7d=Dc!+!v zF5jFBXL|wX0_e3zWF0YHNAbXLOR#8Bl_Yrdevj9Er)Df?ZPN=>j5i#{d7*}Y0nG0k zXf)apn3_jV3IYvolGQ*Rp3)4&GuKa3fjknk8Q(sV8NNB)vKg^*RyfGG6wm@hd1e1P z3YRnPE1PJu`*xCU3x9a=fug~@@%LVfbOrLwg^{j=#N?aJ^q|9`X~t%9=uKuF=tRkO zJAMm4m`}!UMlx;$`C0}pXm&y66OD2Tl=_sf9w&AHmOR@Cw{{%B9^>W-QR~1qMGFa) zG~OqYymk72KvJvQC;+Gk7`6_LnUJx3OQPw5Icva&TJwLVrOj0^cCm|oL*ISkm12{< z{%OrCtdwkiZ&AH=dURZSVYk zUe`E~2J>iG^_HVG12mPcnExqQ&f*Fji!37MCVN1Rn!SD%8)~N7Sk(_ zNc{GVsmDh`P-?-kxieKG(Exsz4)INJ1k-z)92Ire0qMm;9H>1%5ta4JLLNzq@@F+m z9lmWSV{bhYjHg~jf#NKhCI)>hQ^CTk*Yka9bVgh@+n|QxuFg)66x3Th5a3!`AIWL1 z2kA=|I#wCLWq@?tezf3db579lIvcxiKO7V3sI3eR$~s>3kmxSdfU4ugDCiVRM?|){ z7M7Q5L07HqY|j6&$H4*IwM0X=OJJ221!#b>K~Cdh2VEf_4)6HiQ4<1-3NPVuZpSw1 z%^1eg=+9;B@PNp;2KG)cPGpN(d~;KuwCSK?h=v2LsI5~@izh~&Q-PhUZ9KwEhs#z$ zy(tX_8Pz~&Eoa>VRReCbnSyGpJ5?P@}wxJ7yzkh1n z$TVv2lPwwH=#DZs6fmu8xOP_aP5rp{6bsN8;AuJf5+#GG`F?^3*AnlDW!a?1kg`DC zST*&B7{4{Y6@CmAgTm;$Bc`N9fqG{HDy*-t>dzIAq>DU-TFXctis)B)#q3WcNz{5I z@<1LXx^_-f4cC2rc_7`7@74P{-rwM+7U}C$IxB**-E+(t-4!{vmj5mSPnE_+kW)x<|wFmM>K=u)XOOk7AkM~{!W_r zGb_W=c7E!I3yenf!A&fo+LvxxDi3^aNk+oXsyV4y*OuE>wJ~6^eaS|Ok@Ee#$DOx? z_`$C8dG0tD94eYeRsUvG@2qzWg#|zByGxgPa0Iu61V??&NbsDXPEh zKM%xz;ffrG+1t4Ur8~(r-eb&NP?w1m7T|ngK&XxZDRWDQtgw@W@QqU;lsb?Ili0c6 zx?DVTbD|aD;;FfvPF9w3q1htrQ0wWb=ho&#aox|0%x+~`^cN*}WF5vV?`Q+bLGpO8>QvBR;F$QVxq@#! z_!$-Abugd6lI_7a@;HUc9nYI76s}jkKJv|8-WVd<--1V8AkGA7X8~JFpxzA?QcGN)et(K zs~5@xRbctnThfLMH0tG1-jFcS>Q#&51uF;8F9nI8C%^@}rLGyZ$DxgnqBFS7h^LUVU%anN;^%Ov@r&n63W z$mk4*Iu8@!@+7_M24yuyPjIK-JAfB`mjg7QelcWgThYdt*nGYKoSj7;1o6(M50wpx4)d?3K&yvjC0kzf`BM&-rNMNYrE&)L!c3S zXnEIk{lFpQ>vXLx+U2s_H7eh0 z>H@x6VO&IPsltH<5-}~=5C&=*s|I7(LVp*?A!s(;!lCz(x`2d zRoL{?;AmNT@emztMIkPxy;yD3!q$*EI-=(PP&s)TGq<6r$%#=HRCM+ z2m1E_lJ-Qc0qW%$ab123RS$&j zc@vpjec@(_vF_zekx=Whpq5-=84Y(yxfWr>6KLbrl&;gYRQnOrzT!uQ8gbQJK3R~} zQEO8H=Ci{jyUUf4r?;>a>t*iaKJ5K0DlJ6rEht6m-zy5rOXbOhf(S>!AKlz64$!w| zL-Fw!pV-qCc4V$xW%zK1pKsu8AOA=`!86YTC0_EU{oCmW!~#ltSw=t!lj1t7;S}9B zQs0*k;EY*io(70oHFUiJsJJh-lhbtf{TLk}yI4VumNc zyI^+8JSQY@Jan$5fF!XqTr@V;Y?o4f^$ugK0t~9*(Cyq!IguzXWu;r2EVJjP?E{4j(By_Ijg9FpAOs!0H}26xQK@b3Jp+in$qmeUCnLP`J{d*` z2lAPMGY!8kS)Uc za}lnqI$O2kBfE%j7O)dn&r}i~G1DulDzr|5rp!J8bJn1lg&*@;*4KB6j6=Z%GqY{= zM>xEiL>M*2>!Po|C$~bg)1k8U+@~un%=eNWMFl7Vd`f<%A6_Lg6{WF|q&u^HCzAbB z{R$}hH)u49IPTsJjK8kM&|cxnv26X+iQQE%msr$oufW_|Ba#Jh03xbZ=6H;~{-Z*H zkEdQKRyWdJjN8-Bx}RjR#qr4!G{38fW>3*IW$uzC1EKQnF!oNM{=_rGMm^*PB@?)5 zA(e2!M507}1sU@v2i!R*Di)E|x}pzST#=`HzFEIO>xyDU^8MAreA4LbgLqxoK1R%e z@m{ohC{4J=`59G{n@bc)v#?SG+&jS>y{!SNI;>JL20@1ag_oK-KNh;+qYRBg_3r71 zJ%4_@j+rL-00h>_hlY~%HdO<6^GZw}F0Mc07gR1q%9EKHOwD0q2g(;;T_wZMF4A=T z2^u7)-j-x>psY(GPq%Npr?;NADu0ME{xecYI+VWj0j@)PINpO_d@acfm_Xyh&zr$P|2AW@n0(@t?w6HC2TsAwZ_Go7v89#RKa%W) z8nu4+$Z*16XL2YNgDWdX!=Er0!yW}b?+7iX<$~B&x<&tf>xfs%SfqD&|M+W5we)ET zAR%N5o%t!^RNkn_MXydu0Ghku~kJL#vTbk-UJM`him{{ zCZVUzOSS{GbG+7{K2hm+6f!wNT)k*-h7517PzMXHlH)!FP3}9m!)6AGai>l8xZ@SH zJfe{B)&A~-Wr(rp{w7Bt3%*24!AIdB9;P*XM0adKpNZFZj&w4?SV2>N?HvD8dIttj z$8Mwq+q6=hU#G-ZsUjA}pAyYC@QwBfc3=m`^tGx!5?5>AD%WVLg4kudTl3_IF@;Pv zNp5^Z_ysKIy17T&#WqpqvsdY47Wp(3@Wbq9(+cUkyUJ|uEtdPNn$$%7o~U#>Ed>qH z-&v?1C?(sxjt2jF!|6T!xIl2pDfHb@_n?$o^lQhkrrYj$L$gMCo% z7|S(m&jQR&Thq8<6_lxy2WGg@WKR@Z=697rC$@isVImhO0mz2S2>yiwkgNV3q zk*6zPYLeE&sdcP%Ic+%-MGGDHZUsDLshV2)zF{X| zD!K4ra7H$#-NqH9S|B4JN0mNt1C>!>+oH8Toj8{-F9wWeu5qM_IZVEr>QWBifEZ78 zzMeb$7iqDt9CRxV3q*-Q@Go6l7kR^)V1oi(nrfm*_R;GY|)Olp5qa>8b~s2E_Q^XDm_5387eFusVB?gucBdrQyEAN+U%<)-WKv8_sd} z(I9b3)~}`LA0BJZSl-W7+_~%@shYJfza;KjWvmpNQ&sKZV^@ekeLXbM z<<_9EsS!sWo+aNyH>i>+SLlc`0IdZ=D6nGjyX0&N>YR_IZ50C6Y{7+^VRbr$M~tN& zUxx?cF8RyM8=%T}S>xu1(&fM#=&Q@}oADbaiIflNAl3Scw&}F_R#a4)!)n7HR*i2> ztZ%G!G5Od!!->vAUzQfu^1mK|{^rnk(>vNuc?r@y(f3Is{4^d$s}>L9K>9f$h~f1l z5ES+$`S=&qk7%99f!5~%pyi`ibSHllYO=3>71ma#&&?%|?Xc4a{W7o#opur`W9^W= zy^a)6QsC$=9Gy7f4y28*d?OZqu}&#cXGP|hH*JmTU4573z1wfKsClN#Q>`EZbK|}` z+!-{HfsiL6x9=u4)yI^*)f2eT&|sdP$t0ifCB(EtNc z{;$$CchBYVlrV7Y2$ntTeKKs|VgjGjtao9)IuEr!ffg-b|~T ziy(mSgmAGW=oPp)VM&wgl^63{{*u`5CI*`WQmjT-*~v3hmTHhE4;s4Ag@@P&it@*r zy3H(f)8NFbd?=2~G2Nr8{o(WN6~;lFkSt*~H54ClA=s?;M#$h`^Dku=FM$mnq%QG^ z-&Q;+wO>xssM$tgJRr3^&pHC00L|8*?E%SRds9UuxBr+tNwD0g9nKWDR1-n^$i0@D0c^&8zuw%HJ*yh(QP{B}fb^-GWjVP*9QXZjcV?E~Q)G z(j^TdEuE6mEe+D$N=W#7uJ3yPm^HIzoCWvXbH3+^y+2#hF^B?hY>Uyir(M3EKoq{W zS^k^8Jt3?`J%f?=-wkD#{!vImpfW4QMt0^p*Q+JFA8P$i~WW3IbrqHbHZ{I9KJ zpKdG0?#=@_`*$?ks_4v|TS^7pPwcrisEe&`qG>eRSL(rWr^w8tq`iiJCG)`U4?PX4 z!U?R;NSQRcXOvt1Le9n02rQ39Z-bXDJEOmrrPRf}`1lf;{8G5xINC1Nz4h#1E=s&n zuIWj*PNT)=rF?1i8YZYios(3vp0w;Qa4eL8=>0iq#oKBg(ig0VrRQC_8;E4?n*<%hOk5DEiaoXVQ>!H|ffhi1lhJCzQMs`eC`L zhpPdU7gYYC3WxtL%su8*Lugf=N0PIq(ox8~7jw9t?S7kt&6H4xyxOh7B8WD_w;h*A z=Xo`Y=*}u##AHkb-$3Xn5KL;rHmg%2g+T&?TcAx+xc^uX@4^7x_u5@Up^TTecONL! z{z$4?eOGd70rTsnZYpi>qbrxfC##99wB`O4+(BwGI}yK|X9oz9kpIf9hY?eT)4`6< zk)gQ@MYcU^$@iY!%RjbqwRz6p z3|R3@ud9G-V|h2@&~S0nrb52?biEV{bGD;FsK>x@MIu?wj=^$8dF2v`zJwHq4Lf#R z5*ur%Rj!4+qE#!ZIwc!0 zQe#WSvOg)KNDL%qos>N=g#({IRdESy1~4AaH}bg7)ziDD6@do+1zdhg=o>#f5Xqg4 zaNQ0kTSY{2Kmm-qOVBB<=G-0XP!&kMG)FBoMY$7R$~|=P?-mfc$(PF9ZUnmSp67zg zh%O45v&22FK2(QRqw1VNcoc>S9?`Y=NZYBPn|ybVgVGn7=E=PBPPyPEbuzCa&#;Vd zP$sc0S*Xeg+SzR*#p1i7v!UcwpynR>(4NbiWy3!J15^5^uG(bxUJFVo6Oi>6CoYk_ zMSdPLG?~z3vEXm}Xd{k+IzkV`jh%OtCRLmP?sbg7y^oKYtDR{(GcI&HzWqefSVI+| zeS%F{vukLN7)#n{rS@5`Eef4vsnOSiQ*{|n2S(nJcSN~zq`uanGf0mpL24!yWF0BQvaBhq%T>2PB4|d z8^uGj7_LrXqr#Sv$Mdh?@&ZjT%6(h@pJYhX?0vuN3J8{A`dspYQ)=a&)KU0p&6BbDu%2&= za$u5psyoBI-T5}|rXfo|k}?8e1gAcox&(xt&l2p7d14kmeUA71vTevXkqMMFoDwD`$PVpwCJyj4n^ST9!bNZ9)J;`oexCJvS<4j+-<0s)%wdLTWOq@x>hhzZU5#04HVPC6<{DLBHz8Ixo! zqU`W-CgW!gzJB4_^i0f2w^E3aMWdYH%wV=l{o;oia!szKT@9YLbx-YHC$lN}BWGH-XH-hlxgtX(l#tD{oopD$bU$Dmbx9$z$#`?4?}}? znB9ls-ye6-Vn8VMMebpOZUpT64;My_W#kqDeIJ7G=gq_qcaEc!GC%SsR=%YKn?bDI zY-h7Ko+GInJbj$ zR;={04GFi8*B3n9n2zE6P)}ZQjXmZYcU>JW8lygJE1wrCI+F=L3IE838zS(OqOiz{ z4TE6u{K}m|#phjeV!enfyLE3S=3q|GZDuUm)FefJ;S5z&hPs2$C@4auuQ%oaTeZ4!kZm zJfLxEsZAp}N?}w+wx|i6EPWg{-%{aTOxH*ivHsCL1nN0v`mt#_&36IiyL*3|`okC1 z-A4EPJO8??(<8!ClNFECFTmk(CegMrEr|Dtq`7+>LTzuLZ%rC^XfTtjeNto@eox;6(m+19slet7m*U4 z!@rzv>5DqCQNzb3_^FMTW1MlF)0*x>(0S%=v%ns}@$BOPA?Q)XaWa=oVN*>{+HD3d z%YCGfwBQ{Y#S8ys;r86Fltx^Vv?r`2sj(gXPKnA@rQ-C6+uIaklLvWBQ|(p3sy1+A z={NJx(Er3{oHXpyNZ3P$eUZl5e5RDWT-m*1!B27D+%DZn>y@}wS{c4^20cTchFN-o zG2<*um|=R@XA{=)=`BLZ=pZkJkd@NjPv4PmIoP{kA4G>O(RBgngUI+ds-mz5#k-k_26T5f%jz> z)3nRYvy<$$UyTS%0j?wGXKGKmETpAGsSMe`i%8SAh+ShqUe#j`w~st(*DfUb#XW6~ z@?6p-{iEm(IWL#lP028wWk!JvoTD`~-JT;QI0Yub^DQ3st>KaQfqP}Mv#d)vXuGC_ za{Pmv?8HEJjzvO+mGwn6swX3Cst*7HC%C8bWw!ZFZTK=x&ipn`2r(DwZRd=a4nlOR z{9|bvx6N&?(%2t=vh_Rt!|j;tm+KDwGmYiJ3MhH{cP_90U67}&VCr6NQE)-FDBwj* zUl*W$?s`h}x-;7O$M@MSmeVzAn%CNWMO@vRfmV(JEA`R%4p?y#B-ylz?uq2Al*fK| z4}A@`weCYznSE4%2$!on{Bdt0Yj^n@Vdyj*@GvCLlgFKWY>E-QQ(fuUZ$344n4%i#m11k8vP zPPpeyS0p54D_>sxR9s(jgmo}+K*%+-{%9|R6Ao1F6%+a_zH)VZ{9GGEvtwXaRgVuE zv0Cw&Ut}WtnMV+E3-|tjrAIODw?~GD%x(sD;(M*Yu$ibVsqGR#I(>Pc1Qrfldp%af#u*JE&Wm zPxsSpe6MwRym%nUL#wI%WaK(j?a+xa)}TOMZaxLRw0?gW6o7+r{=b z;i}!odvqkJ;M)r@8ivcL^JqdUt`6QFQ&c;5D_PtW%<58dx_D*g(py*&VLg7iJm3%+ zv&zFX@}R=M`B1!bSQt|iybXcZKl<4xe)r3LBWvs_M<{aMiL@lxxzN8Y8Ft~=^q(Vw zr8fRd+OJJnUx~PLp<3$-W-js>3-x9+%{6wS+bh*a30{hqx|Rcr);4G*5`MXw{m$6c zZ5e9Rm?1LIxZX#T!nqrdn*rlLz3E?K*9}L*BMOtpE-%n7mKL{`%eYx#xI_*jc9ngY z>XR^@S^U+s$X3o4G1Gmw$Sd}djNc8VF90DKlV4-PQ;^4&z8#;6{bsM(h`2lSlb@2M8*HR=m*jomO$)5FY zW7T6O@>>RF3d}srZ%6TDJ9loJJk~}zTM9sWEfLAvF~=A^mX)k{giGkCC$~z4Pp@SN zT}BR^3@bh2$N>)IUt|n`%#r_oOEveJ_@mLwi~ochTW|uSk0Q$&rm+H_nZ?!V;DKCR z7^JwgUb=QC*5tv9(B(eWQnWK_?=b=Sqg|H(^}AcSY1>5bZAWv=!dB5p`}@Ko0fp82 zT^o%v4AhuT*Iid;5q+r(^<8xCAT%eX?-YVsrz4Xv6I<>?+NL+WV3LTIS2r16WPl41 zPk2>aK%zW`ecpnyrP(H$ksMa|9@X*=^a4=Q;!u<#o(8V zseCV*}HWjf7(VKw}$C@>uOpH70g$-9ki65J7G-P}8r$Pr7yn+K- z_#D1#%QG$SJ#eV1pwGY0Pi=&t0f&cf=SXtKbx^Or;|GBUQj zuAg`1a=xQv9i7e(>M)KXsr_(mmJlI%L|c{fN}*n%9*v3Rlyvb9G2E<2Y@J zLV<_z;KC+-rpV4s=UbQoeWs&YBT7UR1Hua4i}&>G0OBH&uda>?`*dyB)AR19vo1QU z&p%>uBIu%DCBYlK#rt699z19g>c3jQgE`+?5jy6Y^SLNyXuS`!aeACE(E75Z3az*$ z9xgd%($V^|T|UpqbX=St=TxosYd5)J5ITh(n0Tr925P^~Z4f4=knZHP=rfhJO#Vk) zO3I#^hRc8uIG!*-LvgRvfOvWCc(H)*%CK!1c2 zH|rr7N}do{>f{-vveV@kPDi7O2;fisF#OpOhr;l?VWlk-bGtqdb6i8Nv%R=q?#9@* z5gZ)wgn;I4{S^=U2)b#RnMp(K(BS!3iA={UIVk^I_(-2xYVJC#HwQbGk6B6z>bk!q zD`&D~kj~>*-Hlz!o5kHLq1xrue$>g2?KUHgNBpkQ*TB(FlCsL6U%~T9*8W%H0ZO!H z0Dv-EGyx$M4eg(_db@mJ5k>j@d-C_vYY-UPz~2nxArvnMWj?*--A)f|SnTB5BMiM3 zQ1hSjM^Gkm9QRwHt1LL87I`;9fx5QJF}ErodlFlxg8r#m`}Y0Z8S02hmf9@s4M(Y` zyh~yc%f~@sFE{Q|7+A`8DrETOO6g~P`}w}~dW2#R(-h6{4yN^}6>BLR*iwZ1^pOSQ z1@>r9RO?HWzR(k}Mx@ez$?+4`d$V9_VsZ=83i9&r8at#pFJi&)E5c*jz_`+5`4JU5 zTK%8Rg4g8O*$FwO!mPL0pIWKB8MQtjzh6-x+vI*~%cWcY5r8wz3@eCICco83Y2@@p zEW&+%unmPoQ%?E5?f;ospg71}yKeKzhWz1E zD>Ne5G6Vz!CDZNPnR9d6_6`mx67|-2u~2yfphW>64P;Z~JAYdwB7>v)-Iq`OA3&PM z%MK1xAa&ovs7X2k?>{``xuDTQ-7GJUV6x9LP4+2)%TNg1g@To~qSKEA293k2)yR$X z#)LEDnQ2$r{1M@BP;l^@m#V4VWz{|wf6yJv<7auzZHcOCdi`5%>?6%TM5i zl5QI0H5A*qLwt`NwT+eCY!n{@9#b+M8r;1i@+H;DxzFZyS-0@|Nvkrgrz3q9-iaz|o|8><72Bky`Qphs9?`O$-sxcV;aQm} zlTSO2mRy383x$5O^A!htz7Z+z7vm1@@KAgcx*QRL_lH$8kjyb4t^TT9qm6!r$c zDwfa*cu&s0OVQPE68pC6MfZvX%2uh&RA|9Hw4i)QV?Zj1pw*QA3dmsS4L=$!VEGKk zrxcEU=)f`8tqjtsTYg1>{k}d?%cX!IyRV(&I7j%QlR#M6KGF#eX%C=BH!G=VU5CVs zw7(Z?D5O7z)W|-BD2QxW{DJ%cstST&+v$v+&G%|r%YfkVV8b`u^&WmnS?+Vg(3eG{ ztKh+z9lNIH?z3yy9GW1{#I6;&CAm5`9~BCQ&%w(t(AhjSx`|eajRSnSPJefe$s!Fh zbl!uweq0K%&1CmHiRu{uMD;$io7&X@+iRN~Kkrih!*4`8IZB0$KJC<3w{IS|jsNDd zXMf5xB^6yqSbxp${*tf;8EhgQvc-dOj-c`o7WV*Abm8MLc+a|%uN2qqm;LSpp z#j~YIQOa+?Cl3R`bnEHGnoxvss>_YEi)A{$ON{qut`G>YksXH$pzygHAWXWL$0+~g z7Pj7uwAxTnODyO6ATT+yhEKi1dCCG?WMdz`C*NXr=o1^YTD$FXUTrF;01uq!!J!k9 z6Ex%*3fs?lF70)sZzhLV)v&w2Vdl!R>&27hPsO&Zr)MYi6(y?^w|`Y_{}^6dnz~csSitRrf#dl;#+T;eRp&E|2fkcw6cGssxXbsKWBUiqH%9 z{I$`O1nVA_s-KV*6I2rc#Pn8tA&XIJS$~!}<%svQobTgut4<7h;(ldeJS|79>5z}k zq@Q~d0)QgFTvTKV!@k^a(#Suv^3C@+>));@v9gcotI0Qlq_TF(*OPyT-*=O-dB(>+ z#G#Pfl^e4|%QrhHqCfF&kVtq>!UtS96W41jXaj<$3ql5+U;M&l%Y;(oy0SA_bZrV( zx&JV+acIL5D5|hMihJFIsAl-RnyYi?b6;X87E3sehWq7C{xh3a7)yn{j%B&aKTC^k zF9SWci^hYA(Xl(Nu|l{*4|Hs=6zS)u-8_10-k1qlFq3F<=CJ zp&>@HinHp7vLd?Dr~Hh{danJgcD`qkvjnndBK6MbhTzQ&m!S_r}op`@Ln}=dBTyqnpqVKR{BYD&R zq>Om|CRr{ad%qli6xguZ-sXx<{IET{&=RoqR}~DK>f%L=NDB?|?~(km;Ie%y%;3uG zHj;7Q?LqFK=ngC?4-LBjc2p3J(czQ2UtjRNi)l)!m_g>xa;VS3A_f>AwM|9Djz+hP z61dE`ZKiGuM7b5S1Z~bStnxqA>2;?|lvC7G(3W`}VK>8?c3RE#gepZR(>;00-pX`<~7fq6TWEF*j>TLm~Hj9;Po zJ)4OU&Zkbpw0<_RgiWffEY+K7ZBz4xvkz42e+ydWykh&L1(+TAW4L%_oumyz8^Rtr zUEL7S^Q||d)M79@_w z4{5uzDBu&d^^)J6!h%O7U z$|GhV62VufY@~#77z7HQvsxa8D)^FIS7KDq&{VWtvP&-e`GM;)1HlA54x5DG6#V`X z@M=Sqs*rCG%0t*)(t%T;mvBj2a4m&@nBQvr?-YBbmk|kUNvK=nVorKUWol*)xG2BZ z5L?$Qb#;Kq=jgd)& z^R8SreKq2mfn&4cT0S|R2K|!b!gEZ5-1g9@V8i?kSS#7o)nIg78lbOmL%{4x}UpG&IMSm<0WUQ)s75+aSz(I7;L?Y<2?QqL$!En#7ICbiL~u2dn~dHEXK z&*f`c6X!ae2b=hKq^@4t3`GSzxFD zS)@qXZ)>2E3_0t$GvPq<1gGk=1`$t8S_6_ns5fwT)4Xc_SU@Qs!^ZStEV1{CMZPH+ zxqP#;pp)Rhv&jvA))k*@M%W*1_UGSYjYzToCUp1G!|gP1V&`pTf^6gmgVGq)c7jke z#C0-u!MR=GL6*kwQSgAU!gxk_-Ff&CA9P+m?MYQn=7SUu^JblsLeg}O0-=X2E#*KW z1KGZ`wzl%vEMfxdA9%SW$&+&v6SpAjV{UFn^Qr%8m2=X4)#=L_w3heQ>bq{^6ajY+ z`)9_xnjA&O!}!e$?GZ}L{V2^K=CU@7S{ad($$mf{iJ02ZYtU`F^-;Xv$!ty?=id@b*-W=AGxc!UEeDLB2ZH;vCObLCOH>l#~H?HKuEU!5wh30A^{cf@x>l$ zrBd`#j!UP-olah<=%Y$y2t7`^>@L)%it_Rw4i0QEeK>V6(%g1=(VBrMjl3*WW@3(Q zx%i-Z3+7)pjXdUVyTZU?^7PY>Z&8Wqgo;B!Bq)@J_vmzMk>qgCxMZD{RXzd;5WDyhRhL%0ht})Q@8c;UhsjW|MJ^H-gFpQ z+$~AWnAktCSRUq*+HgHw(OeLN(|J&iUup!X83^q6+Ib+$-=B&I*F7ENIsUtei8ntT zyCKpY4_leYtdh41`|woW0bD-f0AathHY;q#ZfPV5PiD4Hrv2VP^2Kb6Yw(~52H}Mlx#zzCYC~6 zok_VA&mCpTy##EcG%2WEljZm;J|&pJ_&GWEeM6?Wit+ac@-0USYR-(Y7##xF{0>f1 zSndVWw>M1D13p8(6Fg)blPOIUT`-}$tZQRM@Sew*R0b0*+P&R=dF6DS7#qH1R2)>P zUh568h6t&PEli*zj}kGfdkHaC>|In$S*M zKnYVj6|P8VbCkuAIEncb71u_=X8ux^be;@IOw9CXxAW9fYD-+gr=%PVeLeI&O~1<3 zWnd>AsxhUhRBpLBU@kQG`p9san*`f&;4n(UsGOcZ_0XT%kFIVu9LNB8I_Fd4)~G@Q z;=Nh~PG)>iFXSwsGEm$AMc;au<3UzD@Zy5#@$ zDbtxW>@3Su+bYu8?9(D2$h$I>z!x)ikVY^ujs`*P7rEc>Vv=e|gbH6U@e29CBBy$Q8h&2R#`Ol7s)*--GjwOuq^8cj~H*nW?>z9m|7o6Xer6A&r+*?}|bRLo5$TxCESrxG)4 zWUIP4kpDr1?2(xlg1;{HL+ZyvZtd9XAd##6oKl7MdIuPD5~XS`yIJCvNSnGzVAII+ z!iXZ%aqfK!6QUhRBwz_iTK~S(5>4nfgxbiPSP>0(JHm#tsOVw`oB6RnQEK4Z!-U`O z1s-)ftCyj`a0Ybk-1~iL+QrFQWK-O+!{^t5H>-(FoECeO=@_jaKn!S3LbHx&_Z+W8 zcjC%BFCUbTaItjEmBzd=NTz;w|NX(tW_Yp`o;Ry0qklWc@2T_zF6J8fZPzGHWLY~L z5O)syL;si<-G+fQNP9NGaC-gF(?N_zwY~x~(R5*;%DI<>9X7LO^3@p>!2LVx^tpAj z1x9)(uAC(jjxxhS_$N=#{A2j~uahRCUe2pV#h`)UwThz}mUm7}PHuxTY#*|XD4YIX z?g1+tPg432UD1S$?aTXij%b%GQSW|Z`I8m4W&EWIDk`E%rDCSe;c@QtV~Y)u$@imc zh6*j~pGXgb#L9k4bJ~EWI!sYZAUw07UkcZ*3bF7bR3-PTzBv zrz;+_Lkr&T%Kx8-SN}>&4hTiL@Mt2YZ|_j~6wsaX-C6;XYUL>7q<8{6<{NqxSLCe9 z9U_8Q%6iWsx`S8O{nu$*!ua_c_6+y4V(qxia5{(g7z4UjP8Cu(H(T!WG7_B-4<8_N zGaYBy0%xVcCTqs6_;yv7-Wcvk@%$~*ZJw8Sq@KP5!NNtZa;msyMjTreesGj^63q?8{=1n8G82>OC{CDAx8py)<5k3D%e3GH&^v4g}{8oX-TcEYV}Wi)07C zS>ia`I;KY9ePlkPc`fPdqVYMwNHvL9zTQ`#j@%)K9ip!>VhTesNk2g94K+8 zNa>4eu^I<5Fio)#3GF9-p=FMotK*~WDQZzTUutL}(JAlq{Za5>t@^bqxaB6g+ZMGN zi!HcmorYPS7A)%`JzIJODoeYdS>=Z>2#H)}BQ6&DF3sp4)`MhRWH!{9E~t z_V}zFv_vDK_S!i3gCpw8QskJ@6KJpSEM_6wD^&lNz*9l zqtF-Me744V!H&5DzyjKRO+vYs869k)MEI)x$cP4oH29n)JZw_)tSeN zM)PT#7=FFTq5;_Ef>l<;^Y`t$V@_(k{hFamMJ11(UV2(gmR&zT-f2bUe=+MclEX9~ zB2ii(>*@gZD)&+U36-F7a(=J}vAwA2WWpj}-IHV5v>+Y&y+Ep)y9jCv4R^z9n*#}F zcZ^DteTGY6^G9Ggpl^Z|;$4z;L&JstX;v5x{xI4Ga(8BCn&>2#q z;~o}Lx$H60UQt3g!1#9Sd1rL~10orJ}xn-+l0e1M;p$Z>{-i}pIv=& zHO_%4P;RvJ^k==7cEL7XF?-Y|Tk;wEKP77o0XwBNtXd4mC$A)BL#}ny1L6WPaWFZ~ z4rYLBb)x$-aVPg}g>Yy{no%YqvkF!3T~FzYmedGoAsXU0pv!sHH=}mTVXSTD;yL}= z{zUf|@%NbVWQI{84wGfITK_f=WBp&+H%pDK@vbA%a8N~nX$WWj&<|(8i}9zrXQE4{ zIvOkoDmshXfRA06q5=oVmN6aPtMK-tB}qLjTr&MR#jhwrS7| z78S7~V`CR#Rh8;Ak4le$ZOQuP<`dL4sCe^;GCGCQP$}KH#uWkKRgQYo!CUIGG!ES8 zX8j|BSEatYoQWUVLO+V)`b#?a3-b}i6m)L)MB3DOLnW_XeFki^Mei4xj&zjk_iA7J zA4eFr-;d(cmrr*;-RV9`rqdc9s?mawk=n#r3o|)RZWu~oyf_<~Z148~>-FJt|B>@L7%?35uIaR*-29za6ME|&T{U_u5@8hO*Sw>Q zZjul0oi2;acDot;p28N{ZK{?gt9bDJ!tFU9E>)^GWh$@B=Y+*A#%TVZhxxE`#y4<| zsE*PI zd~Y!q5@Zbh0F=!BOE2?`T_&~#zR8vM+^%O8K~a_uy-q2ngs}gh8*x-9W97fW~bY>QBRL~49HTq zCZMDR_j8KT44UVF89mr zlIH$kwITVp))tscx9x_$EWW@L7nBy{m%Dk(L$CQ(C8-Ub+8D zVpNCRgC;n}51b>YJBf-Fya+_vL%Iu1<`PpLu^isYWb4axOd06iv>JVX*~E9(ZS64w z6AdeOBP<0wcb2`6fFHLi;ui={?$&)z8YgL_BI?{*>sw@={?(PDs)CIE`AngLb} zv`Fx4b~AEz@f|$>#j){?FuaTw0u{3e3h1wlhCYnDCs&{T(?AcN{Y#-59 z!i-kBxD^>vK-*0{Xa|7X=UZc8Eh|OFH<1{|d>}F@EPi*dwyaEymNOO9K$=>M^dlQP z9{$XFR9LT((=0k>?wtgmZiM!PViVuY@D0H~zl*ifvG}-478ERjTdsy7Dc-;25krww zOODs%OL#iB>F>sXayDNKdi5pTGyB7X2rpF151cqHsJO_J`Za-57=wp(QhAX_2wm}9 zoj3h`@Gp%OB9k%-3UJ^LqZ)cExVd7@YV(v;H?seF-HyJn+>=?Hr zK5b3*F8iW}@Zaxph#F2lD2JsE+V{(17KMo-!Kh?SFK%Dr#O8N2qMr& z5Bs)=yaiRnGS=EM7Hc9b@ADyXGXimV3o4(xw=nksl}1c;+I;h06O6(>-4r@~F#`cp zw=gkOYwZ~zCb=p7E0=(=4?Vx?Ho!9O4tO)ao!K3uBEN^S_u3p69z~e{u=C7Q&FzUbUz*&+xeD- zo`Rq7gJA`KQk=zvrVvYVD^sR!#~qM{mH|=^(YGrU1n0-^G%Yl-w6oM-6d(m(+5U=?d^jws(XFP zn#L-EJ+QQ>BK;B307R_q=_-6wd~_*fd_%(XdM%#TxZaZlROGk&N!@(=T!R}5oRx{^ zQ>NPU%x0$OeA1Je9uW(>-NB=lk0q8%mIb;@DPCVcuWn;6l)d9ZQ*k=fi3Jr=GVFm! zWq9wgvU<6c@WJ#uZ^O{;Q(bbK%OHyQj~J2xuft6zk&DC~FiaQy5r_p??m(JG0YR~pu>NjWh)`X1@{ zwIV!Mxg)SB0cRNEHq%reifmlY(@U?RKT`^KF$1gXA!1DokkOiIH_H3S4c^6EoP3AN z1U1*dS{TWXtY01h*a11Fpr+*XZ@dRVZ$DAtZH_xr*I{r78jD}Y$1O_-|L|?Ua)rXI zc|z?51V|t-IyDkbvNx=V>$D>4(@#$rIiaQ++Y71DYya)|rNPit^F*ySPnGv5J~)}q zaFq2XlyF7^W8rkXn#r*17#-`>WY|RQhu1%LXszJK`H-qlg7;bBb)Hc~yh&n?y@11B zZ*MhHPI&H)YZm7~zG|P8~<+0_@%uQUh%y48j^RzY(TZdolN(OPG)mHx~a!)Yx z`oi5Y*YU9~1#r$W;d2(5rioi7Z!#$0??C@oqO|3Kh=Us;GEC+%z_R8|<6e)JZtI?He~FWh z8EOaQkm(Z@AnHWEK1jg+}85X$&d8lQ!^3lr%@{2u-=R53BtFR) zH~HL?rBJAfafO#tIzMk?v!}2$(Wd5kiFZHpM?sQr59k}Erm?kMUy4r8x4Gwn=|sBA z%?Z6)i~U0YOiu;CR8f(tXeQaCOsBtN#RwvFg0s z30<8(>vC2*5q8QW1WCi*ad~&KmQFMdF21W#1WppB-72RF^X3|eMd@vZUhmj8@MwGo zA!%q7jtl|CZv6W;Ad%)@Yy(=xudA#izy>m`>`ibZt!>%#1$jZzCc1;N2$FX>ClW@C zpw?=T7{J)l-XJ1q$Ypuv5F%UNh%xmgHW*%khwbe?~itZn<7F9a^l`=Fdnk>9KwST z&UkQ1LqQ=!?WO86tKnu)U|q9z_6-?o?Hq7%>kW=p+q765V`9!LiYiW9kSQeGufRD& z&3CSL!e^$9PUA+>eGFO*?IOiC`r~E9qD4rBDBcasdfRA?ADP7%ch`5vEZQ-y%jpa- zIDrS5RH>neynUFo%tQ-*;{D>w^+4O%ALfZjkF`ewKLZd8SsL6&P@)6QpfXJhR+H#e zr+V|JYt7fnev*q6qQV_!+a)YswR zX&a7-&s^P6-5+klJT-%Ay_wY62a&H*#L;C@MyzWHKZKs|HT|{RnO8Dj@4(ThnxZ+$ zs)M>5*$G@+)aF(So)HzYRkEv zJDYlJ2(_*gdPrOy+mJswcrHmp1JqrZLd!QsXI9<{|NQWaCoqyOPj&dBDea=_%6j5r zAOGT|QIjJ10cO5|BF#ZF^B~(%F|T=u3t7e+Hx1ac?S_wrBzdiAskO9FWOXrk{?*VKeZ3=Ss>c29(kgh z2!jPHx@aYlmEiT+|9-D2wQ6!5Jcv{d4C%7ri%UwpH6#f!#f0Qbz43=F{88pc3HlV`6AVG$w(n1;@KKfDgxdy%ZOk1`MP=I-y1kB0!D9~8^an5CT4IhK0F>1n zlCsR^cpznH()aGOy&~HO{i5d1+zW!{uLSlW9k& z&}d^a2k)VWE2&2;_S)(5zYlWV|(zEc*~*rO^AU5mn(5) z*Q8QS2lr-Y;4l7s--Ong6Cl)|2JQnVjr z?!Hf3e31zXS0`+LAd(`sQF~jR-(E&Tp8CMH#Q9-Ffg&udiqHSh)}Oa?|J0f;ezCNg zcvcKp2?zZJG5ihx;!u8OLl&tJ%tt&|O&z|WZa3Wbe}(Ls+lGqNLt8JlaaCNh(w1=( zIlGL?uUaXBiv}_;4`(1qhM-nv&Ij!z3r946Uv&mfKaOeFd$IIN|77?Vl-d&!kdi=s zVbP^@nEUFDz zJp=#H=CP#T-L<@QpT)geSE}57R50%K2DCd_16*>b<&G!To1}}jfk-e`TnI4`i z$)P6MZT#PWw@rKd&=QJwCFBWv>Y)bC-vu#vVg`;tsnC5w_ha&r6YAJ@zF~Xxvg$&O zBhG-1-tGJKi%(3*@_|U2=gt?SB&vN&c22!c~o}PocENjC@8ZvfP*q$CjkyW#g{7oSP8G!^o?XPU(8|%gi z3m zP5SCj$mKf}9_Kw`Pncm$vcqy6WuqSO5mQ3^4S-FBO)du0t zH4+8D0aUbFLn?JR3ttXuZv<5x8A_|6A^;Nh1xDT~7 z%{#wGCLC}-;wSK59@_j(gyN;Mxm(UBSmvf}`qT|PaPS*{3HJZwJjtBq;sh=vn~nbR zdheb^eXCyw&Pa4mj5tK*0?-Qewrs3dT<0POqWr7466(j}P)?{B0NhP@fnl5En&9|M zwIXkMawxDNp*>9a+4VBcBM!WjF8i;|;qR}@1yDVhT~KUm`QYO7HO_0d-cTR)%##b~ zyoP>Gi#oKb26up8t{<3dEy_>+8l;~s@}=*Oo-f`38lB{incx?{@Xll8s9O0`vpz@- z61>)Z@>Z3ann=E3`4Un?{aQvQ#7H1qh4#{U{1Km{yAjF1cw&_z#?~02`u~hfJfD?M z^p)_j+^RK}CUzFRpyxLE&0S;HHmMuXPVM>YDVVU_?3f&9ZI*j0|2gz!iWQ#ZuRbhB z(mN@UU z(YYHs&y|-JlRI0{1AQQLk<&<46la;f29nr z=~w4jg*e^IIX#S(<6m)Jh!mhjp+%U=jlz!&7V&y zduQKO>iCeWvM3~( z_qh=fw!Cc6`Sd?x9g!cp(pUWS=c-9j!>Rz{s~PpY0|WWaZ{4TC zRL5oyx|IXZ@#>=*pZ`*f)M@Hg;Y+8v@BDzRyf3TXify?x1}i$Zoh^_?BI?1nAxW@w z5vky9PYQzn?=<;Bi(7h@jW~r_nzPdV9R;~VjX=L;0rGXt?d?1Vr(iGhwTyMT>R`q~ zRr{FJ#2;5{kP&C5_cXREd{nT=Hm2cqcOdyQBgOf#1px*6WaUz`O^5p5Ec#Y?UOa(~ zDfuo^RQwNFBujE0ZwxI)5t~N(7i|x8hja+BdJ;z)Joq0^U*XsE_x*1mzEQDIX%Ho( zW0Y)wiqbH;OJam{jusJ>Mrw>0N;<$H-Juc!1Ef1gNDLSq0^f_D-{bcW?Dg7x-FwdS zJfF{d&WGFDqgip?vm=4ALC zRFXGJ#3gM_v99aye z8GuY2bPgyhrn5OBNM5Z^tJi&BM1T}>j`1H4R@qcP+6z~GO((EMFNl+Itd+sHbH0k4 zxTIf~!$6tR{H*75jcSXB&cp!m);*c?ufy)ZaIPQ0{4v^X+FrB(&!zAo`NF+dPIhv;`U0Qe1Q?tGJNbx;C*iiFi|%z0kX`dvLok>h88aM>6)NZh8@gvxyFu=nS#QANIAp6~OC{ySW{sXJ!d5U%!CZDhj5b zCscu6bM0J!JHhSGT(TK82nQF$rbnR_bkz(+f1B~3A#IKH3BL-kAhHw=TYbC%dX&s? z3fy|lr0_Sw3V6u4PXlg72mE>AE-m)DT+U}>|J0uoJ9a0^DV7m_cA1WZQIS0OB;I1f zU6Nrn_LpVmtkDK{v-4U|UCTT>r|GjnzJjq<4NIU?nODewHlRQPNdFwH@yoFYG(hMp z$xEzN_7(-V=eVO)(q5=^t~8$^ijLzNxmgp%NzfcKliBB~{$tlpm9N7apA$GA*V}(~ zhf&h^kys0_<)B~pCRWKYQ=9gDh()-h2YaQ3fyFG+oTdI0HXkxVBs&*nP&S>ooIrf= z(wO32p@}YwQTcGEo_8?K^}xUI+*`2{7S!g>gLww$%nJG5l&~DAGXobbE;sZ#!R?l& z!kXs=u>Jb;^jHFNXrXAs?pxqud$HQ_V{PQmG((r2K$8!fIu_Ek2^EQttkLAuzjx*s z+q%UJydxB&&jqzLfj+J>QZUz*tHXQYR6lmXt(8CHlgtEztDu^D3{2IU)f%%jWqlsT zv?Yc<4pjK^{jl$@ax>2`hF8IjLS9R7Wo4_`!mv zgwlsE`V&Eq$by;|tHq9iwO*4$N(V05^*oX2m-x%tzQA@6>>ntd-E@Ds1Uj99JNBrae`0!V6&{S$f zwg2Y}e1H#+XZAc()Ajwl7(#M^d9$51`|eP4!*f5P>{6ALT5B9G-@@-I5nLqr5eMuh zp;6;BUTUs#gSuJ7`ZO@%aVn+Qu|cum}j7<=*N7R>)BU$NDM z`0k3)$@=2FQKd6%4rf-V4z_;1X6nO){K@*~dbHX%R)8_0ss;n3p0MnYF%lWjcSxQm z&9CI!y|{1{Kh-lmnZS4dWSgH9={Yx9-n2gS`mDpli5{+YPZ?Oz<_127#%fm?8Sox7 zS$3r|WN}_HMOM$nD*XnddZ25S55Z^zS`BW0XBvr{rzneqYtePRiPYA)yw}5Hye^)r zzjOx-V{Qej(j>D!LD6lt%73?xpOTZf8G6=^+_hmJWw$9F4<>v$o2`8ej=S-}!jnzq zdVjj%T_F36ljfF5KcMlW{P+7RxT7Cw7k<^-AZp>bv|FCLT8*o&Ar*D&sXj@SgVEZP zEQAZ0ao|{GBN>W7Ni^fSt|X%_r}i2<)OOaC1ZUmcs<&j$fjj9A-z+x-^C>|W^aP`P ztr)0m`yyyQ@bwEmEWzp4)m6E*^$%6zpuH;S@4ybUJ{g78JF2ZOX84GV3%}8xzReT4tqo&by`}?S7US$SbQbHvn zB4T}V1f~dGJ~v$Dmiz74^mNBGWW@tq2F}d9XklT&S)XapQNyr7^D=p$R5)4Hu5VaiU!Y?qqF%W)Fr?i<$* zrLG~HeKOIA>SVEmGSROwT`h5km~Q$cY#YqILBy~OZu6BsRXS;>=J~j zxjCBUz@>amFqZ4i_4k4>nBRhQh41o3xkq*kIDc|%KDaI zknr0>7rr`#o20%xlTbZyuevy~t>E(l522MPJIBxRs-0>LW?%oDq2rIgi@ZL&Mc2xAhwTiO7;Uy?qJ~2&0YGMxw^99H!ljQ!cjds70}(*>#HXu^_M9ljx7VH z-Yo44`rf=lcN50m-sL~PQzv|veeMC_LO|geBiTRzlQr&o6j&6?_!Xo$u*VB`GMZ!7 z!6DXq7ryfGQJS3Fo!4N%C1@(HJ?yy|q)KLVwc{&8?3?B^O3(Qz^tH!*TG8j?jH@7y zt?E#r%9iiOT9*%7XgJ-+vn; z|BiQ6;KBXy4&6)cvZ=IMYI1VN?SHO0LblwVT3fy$=mx1X=*vEJIsa7o$@BYf-adI& z-c`R_uchlXF~wX2tx7i>FeGYsw?2NHyE0g(UD4-}zg)lWdoXsXdTuQLZXShQ57c0e zk|A&<^o7N!`&1O?x^?N0@ySiIJY%D^HqX%DiwgtCqj6l?4#J+P9}cawqsFKt%h?~^ zJx-Gq{_07N$*7aDZR=HJD_NH(DrnQLQ^wz*Z0VGO1zl-S?2Xa4OdKP6AATaRdhJyQ z8=OY-KOQWC^?IA?e}1RQVbY#l$P+zZEg!cm1^*cJ^hE?uR_HF0u4v9@@kMMD@5Ja{ zL5=Cl?(2)M>N41jV(O~cfBscXXxFG&Gp-Ax)9)%%Y6<8zb)K-7`vw^mTTLF8VA2ib z$dtAzzA(SGm+8k<0cm)sVfWr$;3Ax8NCBIVdqV*`-}Xg1yWQnzgA6V_0j3>NA+OMx z(%lxJW@d_5@-p3z3)m{6HnRN#{sG@%x0r1+Z@65;hxc~DI6PWKdi?X}U;VNSttsW! zh~4QVAp)0QhJs2@Jqytfk1i8M=ge}1iH-0LMCL}@FFTK8+Mn0l`doc`fX9`5*tM=Z zE`O>`oa5z^nJCTr2+9}Th-I#aqm_sRq^q3$CNqv>0lV)CVAoE-kl;pnZp`)!Lx zoN$UM7@-!<@iTQzpheQjlPsZb8ENxnT9d+6JXtvqdM#UFiYa7tO&iiHWvk3Yq~aT7 zko`d#E&P4twV~M2Pu#z)u86(s_nUNYEmqkhCTUsB%FxnGo9(i!jDD1+h2J7PvyX}3 z$T|;3Hbpj4-N#O*+VAC3kYgn9Oz=xuc8klTx$Cf+aa}^`^Z<}t$0O+J4icU^M(u#vRk~-!i)SV zIoqj-Y0gndRu>Ul6E+qoJiDx&pxhEShjFY}TpC~7itlWheaIfCg5pJ7{G){$iTqDz zUxaYYdSN`ul&HP8CUv~*gz;c7jCJ2M?e0@pcSQ`{saA9F;TS_7r?p!rwiD9F7vIbm zx%bu>GIopxl$aVjbhWMtDiSp&zDY5oW$0SygoO0v@Ip0l>^cT@`;1);R!0kbjN<}D z>pn#`%5sr-gAx@Pt|Y5+?*!8GXb#IKJAN172N5NZCu?o2a-tq8U$B|p3FR?DnDK1S zE__B^12l`-^q8LUZ{|_>*qar^%H~m*&H;Wun^NQ|x3Gmrn#BmWSL=GZAm+PcW|8=a zDn-?sixfVvx|i|y@p{J6Yj@|Lx95atiB41zzu(#Q7D~bU1T*O-{;(8DTe?QaaH8q^ zBcj261u8yGr;TX?3p5TK?$k(ED_Ez3>1(sy7hS8_#to96O#P?5aP_usljB(l%#xfA zM<4Th(@2(xFy(T*W7FS!#N+qc!@c>Z#MRMg{3lwCZ4&mD z$U)VOW84znCqWEaGAVlOeDt>5Auw}nCr#iXEgZVVK?rT668eru(%CR;1%^pW$4HT3 z&b1Akdwef)>CXxwdQ3!#_~ueSp>4qEZ#T!JCw@@rmDmb(;7-KK-X6K6#L?B<<<&8G z0{@>>1VX3;Gb-j~JGDM>3~8YvdXP8$eObTCsv&Ceclycc%z2{zIuCCOv{|U;7B1q) zon2+2Bj^B6fxHC4VKmHO(Xu$HR7}+gWn_kc9w^n}*vrsn>_9czt}sEk>eDSJQq!e*w8#Ut86m z0H%K)q>}CkUW98Kkv(5d5JNpaY*Mt8oTs_~e;t(|o1~kJmyIPQ=^~$n2oLb7M(4^e zeqw4kcr1BZrcxaF_S@!yhH=2u2#$X#R5%^l>2dckVcj8ZJK#E9wr5m+pYwlW5{0|F zYb#z>Xa`^S=*-gy{yFWAkhUER)-Xf11k+w~3`(u2_lU4x|H zPSu|O4Axv3OS}$(EhWvoQ5&Kql1+=gzw~Yy#d=|~Demfte?o0g(Z%hM_Pa<$l2#30 zW^11o1jD+NKwnN4tHfz%O7>CVU@Cr4bV^Wsd=r5_58W67E z&B~*2l0IG5kBaS>`JPk`D}6Oi?48i_drNzt2o;DaRxWZ6qa4GvwOYbh7-1$I3EHEo z)0|E>krz%1_*ALpikgl-@n&g2A-?&(@bmyzqNbg%Ys0Gv{Y90bgd3^()e3&+OJRwx z?0(~>7?QT{N|jWJ)51g1`kXSyKf`F);t}YH)vNUL{7QUD*1O&khd!uxhnibfhhKWu zAEH}kIfOchyD)!e#q3W@iC2%)Sh4m#zsd^so-ET{;po777W*Y~CHK+ax2bv*&{$sj z`8ldUQs#Tutbd_Ut{3|0cgx}OuU%;f>_mTX#5|-!c*W1U@P}?ggRv)Gr=n+y6ZVkR z^g}pv>>XDT8u*EmJpy`(>-dUT)s*k;HBA+@uPBLJRma10M#A-4kePODxvX(X^Nz2b zcJFk&2sTz9YL*wm#PA8zbeoc4UQ4@)(X7+B9fN(cqG&ClQCQNvN)jHX)!yT?hTrda zn0E59xq3rdr%G{)-TXwmD<(lCUK}^B)U{;6I9Ba3jGXAM z`SCc&nQJ*w>oI1CsoWT?sCkRX=$dAh!?!VKFECkBAhdB$LYe`{&Brb`XkGRIzUmkt62lTszfqpnv=QO9;ZGuULZ zwnYwgnGk)JU*{JgC6332+hqM@Mj1QoEzU%GnnYL3w;fq%<(#-ppZmiKgQp`DdJ{6K6IwPnR| zD|(7xcRRq`*(_gB|67*m7ux(Zw6x|7{}$FKAjCqSi!OrMzpn4r-?G}oJZdf0NweE| zi>UnmBz*jlS8`p5+nBmy_jf3>Ft(=gR!cS5*t?iR`;$;T9lBMiyPebD-2e>+X z|DO7P3Dhf;{^s^$tVRfvWGBYHMP|>sVL4V_F130M*6Hu2*&A6cmo)E*{j+7MTAH6s z#Z-Ykey*RGO&d6a_Zi)CpUs~gnYQCGSy0MDsTAL?D|Zcgm0|o&b~61qL{sGSYp0!* zS2EvGVBk!97Pk%}I(J51xo5IJe#(=M_Q8B-_9ilJ#TTK!*1J*H&CokT9kKW6S{-gP z2u+H&cXJslH>M)8Yfei4C_x{OI!U!mUxY|je0w8urk9oU`G7o;B ze7nNN`mDHqKokdDi%O{5^H1xHi$3eU;Y~H_Y+fP>=RDWvyew9l?_068d^}#kYKjp1 zF<=H;PwiFZ-Bijuu#ItBVK(eS7!Y3! z}Rd{g|&4m?J`W+ctf3^fqZG*R* z#!;5hx%I&paVBP({@v+b&B_O=k0{IEJJbaP{=I4fUct7D%flKrFM~~c+!M9_Qa&A- zy}8h@H^+Vf-bjI0V6)iq7LsdCHkIE>YhCQYyX8bq?k4z@EpT&}9-d-SH}SY^G4Uho z?X{JmsMcK-8$&NQ7hGv%bE=f+ZKMz3)}m#4`=GO8gIS7Xzt8S&{;9L8ZeyQd&)Rbp zSy|IkCvi5mH`rM2$Y!}p6T>67ha18$8ckGBdeph6o(D?)n<1$qp~vqv-I_-S^k6SE z7yO$d+oF{dG6k5*TOR1hI9}$2F%8JR1`rwK~~-X>+llRQ?ek%z^5wZDvA01 zYi6!Nxjzwsxy@}>NIuba;^Fmcl{vezzVG{Oh51DJ)pC*n(6?bu)tc(C7vOI?t6|(k zXmYzXP3~`yr8ch0^g(5H-sSh>&Rv8DeKXDVM}C~#zM&LHCntFeG1&KAx_o6?Nv?IG z3y-|=+sIGZ3If)zW{6oWtMA~{#)MKevGoA$yD(UN*lVR{edDw3bV|aaDODea=c61 zPS;ElBlLBCR#rDUDbmY{-V2%>4p$wvIXB_u`~)C3H=YzvgGjDq&8e*F0R05Qu3mNl zK`@s-p_@lVN#OGme=1n}Kszs$jW%*nwhCEsV9gOQO3xdSk7v6FHDors_;&ZbXP|X^AKCq7 z`CxuCr1yyNCt>f4U(S|WHKdh3qR)r{ji{fBCB-v?#Kp0J z5=+Ki{d>fQap;=-u1)j^A(Y#}LQsil$(-`XEyG(^ICs8^pi*99p-KOFZEZasLmKSU z>2Ifpov7QkX4Q2wsdl)j$kqzg*o)pnjR$dXiqS=QB~cOotjAL++NGtVvA>W)5SiMi z(d{`^7w@;(=;>r0YUUYrcpE>y#Pt^!5Ti-l>JLOd@xPopmXlFhxl_!pE1fZPvWSc$ zDU01>ehKQ;QOpW0d>i=;q2298jyb&k!nL0DE`a^iB@~;VQP5eOH9|68>lRXR@W%=Q zO4l{|vt0D}zRRGd{jg4dWwlC6-|z(xdUeuOKO7_fCK~WZnkvTfKsN3WX*CvivBqs8 z7vUnD7#p!4|Iq8AaIw*!tvLyncGo}-h0O${@%eqDM_}yHTJ3Gc*Z0LLA(&WcTY#VL zmbCZYjxpZERN<^Fdmn@B6f?4D8y3W|bv#MEnU}$KKp57Y9e7jG<;pYmS1sk;gVuW8 zBx)weU8K+vY#wa!1OFtfr+ftfH@E=b__28@fYra(#(M9=iNy|HIxVlc2_VB-UgEx( ze5zwz-uX&;6l*i3HogPu)Z4UhLJ?0#+K&611-D$zq@4S9Bc(Blk*mR2<+^>BlX1|Q1yvFl59r22TdLyeRTm|uq!P123M49y(f zBpAC4iO`2DX?#lMhY>g-hp;r8HN##_gjfcPYSo0LN7+ub&_pji($0B&ZDgzv$2h|6 z|7SvaT0#N-q9IP}R~b3s`27Q%l(Qvlo?N#@Q&Ewa%JBzl&lB*#sWJ2Fsei%`UY1Mw z#|HjmoH$N@FhyV)DqPk;x>Zst9zu&t`1-O_kBGOMaG$z9+YgU+lAda^B=c3S1{2K?59 zTZ?tqN9Zc3W4xbnDs7@}BkAB75xGyQ;GM5>n(O$r@ddB3Ac8H)vXwne5X~x&?g$0g zt_C-5T{7Oax_;-;9v0RCGTNL$OWfn0F2h+y2;l}gE{X{bFuyTfZqXnww)fJhe=?sp zGW(th6zb3&*Nxws$0m+4igJu*SC%4spsB}fHyp*6>8jOBzWf*{;WEY#Ih0yIVr!L? zXj;jqDwqXerQ2RWwMBF7>o5Q>X!TV4j<)CD}T1~+^?MLt;Esh1Y1T6NBul74k^%x8RpyHJhTvz8Glz`Gtk@$cyczJ|?mwNRdKVEWNbVGbgttt?lyW)y|I$(5b z{pedmDqxUI6M#?~otr8*P4VF;?6H)+C5*BZS_t$0oA_6_OcmV%SvKM%(yzOS(TYJZ zHat>-p+Zs%bykSc^rPYJGC~FM<=B9@Ck)Y|DbcYiix)VA5q)=K%zLoutQ-%i>BFNR zK}-w@D%r|nUV$9#tFAA8{X)hP9W`LIbw@|W=wAu8c;t`$aY+n&dYMIl6!IdIcBg_v zx@h!N=B1&?v@)Ia?~QRF_rc(Yu;vt-MbfQ^7l-I*Zgu+HOHvS0K|i*qqmoSK|GP@zYr5uKRK_r(2mR~+JBp*&)Hk_`uld@tG<)* z6QT6HMQCtXoUMT4*xD0q_CtD+e>|!GPffcBOo2sYGnApjHM^+tj8P-5M;o%IaHfqV z*`V^@XrFJ}A6}If+G&g%`ES&8S^Ep-_P)L13vAj!fNtHlXH0?1aMODxL$j5S%=0?b zEJ(K!WT&x^aQ{oKv&&xXZ>=rmn^(W_@W&B1DS|leNBNwgF@e=DjG_<>$OJxaeCbQe zWo4wvuTf(?jzXB6AJ?wS_l|d1BMQi{e4*q7|I3$+f~J%$VIj+kS{5+4=pi8eR=kc+ zs^S5G)JJ)v#fa(&1pD(*nUY5iSxy&Wt5Q`X&igCKG+YKhjBd6ImGt9I?m$8#_@)Wi!$4?6m(~{li88lMfh2a^iiZ+Vj!*smYC+j^F?^scSH5d6z)A z=bGXywwm~VU3O?ccw@pVN*CbhkF2oR>@nk~g=?ydO0ou`A@XADVcUQgbXF-(dA<;% zk5~kCTkT^=WLZjtYmGe_#wfCx`!8-U&;H5)suK4Ekw4BiDwbuES@jr@X4zSy9&to` zM2zyb*NPnhFYDfbP_tj~M#rEzeG)YUfrSCb| zglUHrS7kel62}FGv`-RM5jV;(b;rxnu%g@7O7J*@i&cwI8G??o<;F4gAr>R7IRldX zWXp2gJjsA7Z2>m2+Nm+!=8=J@>d=ib2H4~hrW#aG@NKX5>e%JiJegDQsqdkvF-lK4 zW43q?4()97p;KP||eh!uLeV+KQiBq3hExo@$n&!ofx$#=L7lV<2Cm)ny!&oeIOGW^k+3_*xnv=GXSA`LhU98r;>o)1AiLZ>T1|83n{!Yznn ztlZ5fW8%M(_j-du7<{N_{K!0mu9|t6B*t7u&|_UxiMa2(cma?YPXAu>X!+poCV4Hz zplalSZnvulzmuL$+~8sahZeKD@h8FH{Hn%PQq$f`%UFH8p_;-e5#)bW!HrZgPlyT$ z6yQ~JM%C?5qGjRS1C@DQMXL!l6>a4XL+_PdN95M3HxvMo;x|QhLFjXd%L663-9u2|cD+-B&GP-@ z5l10_cndwn9j#}NrL?H}Q%+`(_ck)T%Di|Wq<*B@qrNW8;v#Z&z&q3F zM(1)OU}&m_3-eM^D7Uw0k}y^Ss%t^YyR+J}uj;eMP-a{zh!S zexJa(2jFJTJ*2Bj>8mzB3c=Z{wCgCUPGs0FCHCikOhZ-zKSjoAG&D?#@=ysN4(r&q*LRe z>-ZxDO87qKri0XNWV{?kT5(2y4zX zUP@2;&gPCbWIG3CZe<6tCAKrvlOx&7s)XRVSk81Np)CB-OyG}fX2ar>>^KaUB?UDLpy2xfX zmF%9}f>&PY<}zr?+34og$B$fC?xC&AjXw4Bd7XW4HOWF?i`x2 zE({s!Wb$I1yHPUkSkTz_yYXATH5~Bb?yG~9Iq4=V5=9e#r@Km}th+G*FCvsXjO+o~ z_?@f4k7bd7r67eyg*2D;%wk{2zupE&g3!rL?;d>Orm1`D1l^>Kl*JRUCR=ubjB(+k5z; z74LaP=kTuMm1fh&9XuVbR+2lb9jk2`t)h!};Q$K7HQ*PkEBmC|q~Jw1cn={}tnr#a z?&ye2Ik(xZ?vJ`a3uJ57MsP548$y(ZT(RGgP5V%C?u(m9ij9XQv#RA|vV)p@v-Y$! za>txN=)G(rK3l2suo=&puAetT-6@X*QsF_iqe23q5l>Aubo)Cm1|k4_ym5a_^~ic3jUmV%=*t?7feHS=R1|lJ1&E)>(bBf zfv{@I=;E&=!w9`vqIA1GDF-GWlTZgnDlAkYG~EfHxB^m|@D@X6+kUWvt#7>_h&m7K z$uUAkc0(_f=y^|q2#wg(I>=Bj zQhMxnxjV9f^l_%J6=#qB+3GI}Y1HI8FFfkPzCivGFuTKher|X~)gDHX3x=K zjm7~VSe+lB*J^J2{rfl&&BMZN6*&$oYnJDE_ zA7;7DJZ~cG*_6S43XOFv$(i}%=VOvxHe?O&#TG-2`;oYZ2%iaf0Uj>w}pxd^N|gdW!FiI;l}M0I zp!XW`wMD{D#_juK@13S67OdUK0l-A7!hO{hI;2NrcvM2KE>dAcnJcjuxOEZBTLFP! zmN(4`Do2cr+KsBx0L#uzMRe8w(sd1{0wdTy5?XD~$!yhJb4eJD1U$lVB%kh}4X|V^ z4S%hj=HIG1Id!!vW|YLlaQm^Dl$k82`qV959ut8jjb3wSeSh|93zjuhf>NT;3ojc^ zHrF5?J!ww8+NU+LTa&%86WL!19e7*r_M0atm%%b|G;)B@R{L9R(kRdD zg^TDtX7!n5dlUe1w=&~H!HLf~W2GB*0sppZy4Kb^d7laFH?nxq_)0}N*(9qAzf#St zo|JT@;6`6m-TlxwS4kJqFwO_MJ=*sIh1Q$5g1)odL_(&Xs{Io2rJ-R!F*jTEL|r~S z*tn}@_MmsY$*+60^U1U=V0dxH{!vy5;M^PhS|zy)3;Fzw&g~OFe`DZsxaXlhA4G|G zc!Tm+IpH*NX?+eMO4A(@0dOLKpQ>0xgKS%FJAjoZ>v(5;DTU6${nJpo2u-`othoW` z|3_`5A8l{1x?DDKNY>rvTiik~Pl7t6aspCvP>)O{X8h&{ovn6$4et zP4Mi5G^mf}rd8|26z3i&u$|%MD@xxG%kDH*o}UlRdQeMq)ku!~V>xymM{CS$T@3^( z?%B>-gG((r0fd{VY9XWi-ZDe)p9!Q=$Pq0ZQ6Bfag{s(T@KWRzu>^@khg#oPiM!-y zf&|$>tF6EqB@&3nRsq(pFHjYZs{-97UlJ?CagiTLUg}o)TXSYx7u(=kf)LE+=Z3!j zbg{AxH&G`#4#P}6ZGhRsOnx1YXqmM|ba|C5=3V`r$TBa(cRnv2$93*i($$m6B zF^&r>)j;nKmLx%1gxCOK&6Ca+m;`nfU-VFK>>l2QXY62K%45ANAn`sZAgFXIN)?5= zyn+XUQehP2o00mCVsvL{FTcH?PbJJsA6Jk|Y#2`IZwqo5lK?+wxAkR8K~#|_0jdNL z9j+n@n?eDIMKi~b&g0aAN6f}SOx^Y3O1YKcr=-zbAT+es4KiW$(A3%Xz61`>y!@4K z7Qly5CmHVh_7+HN6j$B?&xQPgFw;3r&A?*F#{amT8&wdDYe7R_=z{SG4^ZvG_tZ|m zm7n^p$1@k!eANw}kcqMi_34~{03l$lO#??${5&1S*hkB5fB1D75ljjt!({l&EJLTS znP`>AkU-{%BLaqahzx!RRfw%UU-$!(IxjQgC?NHF1}3Xy0}W!h_1}>(^##*FGwZ<; zvZvD&Nq4s9l6*Yxi!3MVv`NfRwK0{K`5r&rVRXl&&bele^d8<80agWoPlGl2D+r-L@{CKvs{A(=y*J0iZljBePp`>gnFp zFqmJ`(C30s6WJ5ORNf39cj@(A5HBXnl$06WR5RT5{4W-KsW>45-lvslBn7@wXahbs z&--qaRpD_o#D;LyTe~NAjFx9XyF571Ubl5{jqFu)=Kz?dd?I`gmDT(1ww!=^tj}&v z;rN*sqKC=)T@pGwZ2V51he;D!z9kNbiuOLCek?xtxxC*Xn|k7-#`7K`kKyb!x2XUBz z?iA5o$EZGcLWfgN%IOi$5H;sHO>5P1$o$U%CqRRF>xw`V>qv3NVJ|Ew%0F~RjksKZ zYb&f7FQ9|c-4QSkN3r%*gP+B)D#kBOf@@500?N`;sE4mGVu-NxLEXT}fO!r&NP)*< z!J<6acxxnfylpVF*P>frA)@4a5xmy9`*Qu!)X4T3;IuGsE(0ih`p>6PcJG9*`Co^$ zMRibefLx8IrF7{_IrEjVNM}7DU}Gb4Q|}4+z;@_CtqLv$#i4t|hG{)61qy{~K<6Yn zijDN)N2ktBJ0`(+7BFh^7lrm#`#kp!d#$u7cz?Hc5n?G_YjNidWgIqpWQtRZ+CAax zeDF6ksgysG7A}i5z8l%R^^TjcwDJ4`TozykqH8j`?hcV&r*seLeEch4HZT{)&8u1g zm}OXiHMzBD3AhOtiS-1sWAK8};M^4-r&?b_u*^X7IQRHFq3m{IoW6mEJ!jQoSu-hoW~0-9(uFTZ#?mb+yejpQm$~?pVla{1Hv>0) zC)=}%*JNZ|zADXO3>WHopl6V(3U4C9mk2C9N@TO^F3sfd;e|4yM$AR2xmwAlasW25FCPKweKsua#y=9X_S zcm%V`{a)!X29%#kz3tt5-WOYCn2Nr91zt!@vm}?vztwoC9!&D%ETKR1PweoY(MQt# z>|;!U@lN9{v);o~3a+Tp9mYSp_BO7OgB(A!UCjmOdD6++{c%DWCl5m}0&_`@98wlr z8}ye#nL1xXH$B%P&;l}s&64<3ymX*zkm`ugF)bH5^P3jGPffQ1Z-cusKqj)MxcD12 zv^FM)(BbX<-;%JlBlFsVLp^gY>2`aLuZ@o6A9pj;dez|4>Y^q&G){J*oGCNcU?YOq z_22u)zU=vZHasSiT3n_SK_C2H>g%ME6GG`ZY7U|52o&`*3idHZQ6bnXcpko^chpN+gGUehCiF*BW%ksN_KTnPT z8BA#}BX zY3VC!nFaj2Hu)Y`U=l4HL>+zGScbv|Dq6c=9!gG0E0q&P*@nJ>UXPS~HB&)IKIa-n zi{whz&UrGNo?WNVRm*MNzR7s{ObRAY;2=E)Sr7$3S8saU-DG)I*8l*W%OV{$>zdE3W; zJae8c%z`w{3=*dF(-zGv3$>4`)`OqW6mQ1oxXPe+dR{&p<5Zi`J;3)Wujx#FgC%lB zg6ZWp(o{({@BJ)+@vrLp$cpWr=H>pPuh8%J4@;E8LrnE8OIwGyJ<0xI+?2@yi-Dlm7m6QQ{o)+MS>K`KI13M`+ZL z0ZRmx1E6YZL!Z2ma=wfqcW>h1%_-VX;)=NcN)*&?OmH@B5R;e}jC%^6WJ7`j2z8c{ zR#$>J&OPZud+P*uogONt?asXLmQJ%@cua{CoGs+A5po1z#Nm;;txQ1l-5X*O!58X| zH0hDc0`03nluhV&eh}~U3@~#0nnZwT<{?V+q%99o=A?D2;52o{Vdws$f~XT!iw9uN zqC_AT9zHoZtGC+{?N!z8|B=a0vS*J?USgTUP z*Hs**+XkkvBFZh#nYgfNdM$D2nWurde_x7Hqp~jYGk{iSlA+)cE>pp@iXdvGK z)V)O0uXd`|&eUL$FW3_$R8u?Gg$!$A^l(5R1$y`bLD<aJZ=u@MAK8vE;78Bsl`kEQ-3hPeCx1|Ow#Q=-K&VuK)(xJ)oGxzrqL+qpku9?! z0uz8u1CGdJEdBY(qO?%YkVZVR_`N-)=L_Q%G%XWi>6R-B;72Jq$0o|h_Kx&*NDMQP ze$XuUNL(TJgWv$}E!OiFP4?HNtCG>ee|hcXYURK5jk7N}V$#+E-iVNxnuZqnZ<9PCTA?PZ7YbQEm2|sc4SOK2$}2Cn-F=?J8qr~)cr#;NO-1KQJ&uf4^ z*Sj_zF&|f?NW8Kn$)5&tt0WrGtsSeZoM$}hK`UX)rmY5qUT8p>?u4(9VYPwsz9;#) zC-3_C^K}Mb)||sDH`cW5hX+3beX--}5-X2A0+#(rcy5|GdikKRc3BTR=P#%hg3{^gkYD}cxN%S6nYH^$vXcW6YU?S=Bvkw>d>Di$)x9sQS zBFLHMaaXyt1DZ{_#)*p6DgmM+b1oaz$4(7EX)FrTT4peA>V5lBc*)5lfW!CrgfVRQ zI@|DtwrNJDbFZ%@d%Y;(nIdg|0xrM6{ffvakM^Ie{nyu0oB)|* zcXgGuz}_H~5SGEZb7mm>5-8trvf<{lGwFakx@RJ0NbSF(a^)?B#5|zWa(`aTl-~Pi zF)Cpwd&30x&duVQ{<|MtY5*zGW+7my1iq zLc#$6JVz_KRY0B~%F36W&;Kog1K|5!32U{_UT*-67yW$u4<$UGICNyZcjgk7*h)tU zw&9*s=8l3tEOG?(4#rrVSDCxpVNo@C6_``=7KCWbGX7>u4Y5`!tCT+^P=tITsd(h+ zYt88Gwo4iUP9Y@R$?n`a%OqeX{97T)QQPqQ4CUbMKR|;bK>0o4Ge>LyI}K>uk^#|! zf5)!y9yMOjPlWnjIB2^DOja*MV8{X#D!*vIKuDkIWcT0~Pes1I8lTf^&Db^pjxnpe z@8XgewGRNl#I7t#Bm(vSTU6~J+U%ChlDD3MLi7<9DS4 zpG*5MN_eb$;;6rDtn{uI431ael@FvI)F{r@u3a9D7NJ=mw-vpYi{~>#|Kv2&qEb!pv?`N->kp*!s z#!SU_EYo*;YfmckL(gm2xK>`pxK}xy?m)H`^GDYXj}3O5?!3dCw&&xHKrOzM0iBv_Pkb^15Q?JRg=J}XSibr6 zuOh06t~BLR?mIv^%?R3k3bMCW34anUr-ZJ!PRYB~UI0@B{CtW;waR-9CEEO7Co=T5 z^+LUP==yS)*M8eF8wszKmpgEeLbHJNbXoe2`qbz1zzU}}^_fsnv$tnAz42+mc(MR; zYO7xzn3nqQ0Pxt>RjN1COyt^hlhlVnda-hP-}@_#G*|nKnLcdEEB!RiRl{ly7`5gkq4#OdPwKDE;gn&<$d zsqg*P>hl-tzk~G6Rv!cBK0RTT+^RycDeQFQi4Y2O;~TyP$xt{WXTDFJ6HS?;Z-aZnlqD!agH`8}Q6HLLb&f zP%@>yTe7VW3gRCn+wUGnHTE@r{ICW+LUH;>)yte^xF_#;1LX3T;W(Z3pa79{$**P{ z9%$V^fS){mf7wz$Fk*R;4( z=p1)+T)Nlzx;-+7fVBjMzbrF0ur`UwAqjS{Ow}WdWNP^$aIu^_SDfLfp#6Q@&pa`8W`APOry&o7- z039J`3-6w+7ru;1Jx;rDH^BQZ9{X%Y-{8X1~q@R>>V~bzRQrt2Sy|V$u#?PN z9z+4U@V+UE7hT0XA=YpEi*}G|q7!k%2Snj}axON&MZRDnsAWi9x zXgi%E_~=*i+_x7kD*Qk1vn2j}AAPBWYbC5iEA{N*zdmei$K)$)hqhZMEXCGznVIX~ z0)#Gk)boDZ`1D>q2JykgHd>_O`dMM}FMWr!ZR$zvnX2~Lit+b9IhrnN)G$_?d@VLs zlQ?viY+gq^dc~uFG(9f1`@iD8GpxxiST}-#-K~I#1=t#TQ$guPq(y|Fpn%jZ2uklD z5Q^A9Y7npi(xnIzItW4C5Q>7-&^ywbG$E8bUxdBSx%bDp&vTx0ZytYO@|E??nl1((ts7qtrr&nSUrw|` za0iOzTI(_5rb=x^5Zp(*O8axm-?v9B1ii#gSG(}18$%nJ+5P5zb1~8{zX83n)A;=F z+rG1Mrr#ggBc=_?k!YN<@|OcZ1BTZPA@A_room{8oLHlzlgVFCcc`~LNSoI z;clUa1(+`e)5S`8shY3#hA%?x2FK{bYi5r2ns2+5lXtxpOLC#C-syCCkQWZy{QkDz zZG5DPI!p-V$7<;7TH<8$(g_?Hgt53eQTc0gk30d!8sMUpw7wp!CG6IsF#{IeScg`b z-e5Q_b$~Z7p-9XJ!w!AyP^N(ZR$0uO8Jy(hIZWP&2{#-RONQMvk@YMO)kaK9cC%nn zcsX#NK&O0xQnu5a9Xm*FA0um$A%E0FhECW6C_M0w$nPc@*Z(~JjFwcy*_p&Mc+bsF zr=jOh(s?k_oK!VKIaKN-xkThtyU{2?G9ZO#*3?WA1IzB}K?DY&h5|Lz1{iL+OYI;% zdJU3Wc;2BX5m+A1v5Obv`$7z{_NXcr|MM}R^}G1Q+jl8FJP{O_f*ac-1*`m1jgORJ zvMSBL2ZF7d+9`G?jRdLT9cZdAMmZ<}NacY4(?cDGvAGrm-p^$Z!TqaYB2f?`Ub`O7 zKA8Hba-iJ5CNPkTM;aG%8j>V%L^zZ=BhPu}FNB#w(HnhF*NC2X!zdSZ%X$25|2hy3 zLHGsa5FMt_fdv78(tST*ujFbV*Oe*M_6Q!6EGwk5Ic9ejzQn~ehR>Pqj7b|X=b({0 zjhG-*sOA+YkkT;F@ckeQWZh6ELCL&Q(^kS=%K7`?9ytd@*HG*!z&=#D&(n$FTvONT%t*|Kr8V5X!8(si5nQB*LG(S&y)eZ#? z8Ofkb)d<}O!W-D&5Z( ztnvBS&5x{}vzOi*5e?ye20Y$|$k>mCq&M5x?p2+VfX$_-v%0&&=d`X}i`yskQ6qM> zIR%;m9U9B;v{79>XXMtus~g@vh)>M3B&WT9uO?LXD?Z=4Gnt9+=G~@bHTO+QZ|uT6 za?x_FLRxGpRDRH6ujbO8y$=^%C|fSY!kaNEO13lYIb36;kLydFOwTHu))&bW&`IpL zfs9Gze&qd`w{q`>vAU1`S-SSKR$}J6K*Sj~-OJ zUCE1yVO<|!CWfMwzu)_NZDrnhV`W4YkF*I7oIZh^sxlpzq9c-AcA1vv{-m4c6v~*e zTZ-cYbhJ}EeR!AOrT4e;c&8=IV{!TDv3B#=Z$p`g^7x>E*acT6^mgZGsO4bl{$MFigVbdz%4ER}l6_59Fj4C!?Jo25L!jjmG4sJ{?3P`hkm z;^wAfnB`j289w*CydgoEg{jIVxJt#%V-@e3v$`R_l~{YfIo zG2P#X;tX*O-bqInY;bP(&Lb(|bbh3sJ_>~^rl#0_u(GgL`Fz~?_&Dd9PHO1b2<^{Z zB)Yyz`yGY$`s~bF^_k86%)#bbZ3Z_Rv%Y;XF+(0wutgdT^Yr344R()~ zwzjH-p#yV$74FOgcng$V^2_ik*yqEs{^IbN*&#^P@u>+;?2xlRJ^Q@;!LEI1jJ36O zfKq}zxxLsq-NXR7bd`v9WmAt=dw#U>ls}~H9<=Pxjf`kEq2X5F8FNIZ%(IBb*XMND z(-@EGqQZ#`y#jT7uKVKn)#>BN*@1gS#f1@C_xNOh_nc=uT}SkR8lD{5v%qm3;T94i zb42N%#uQILtEy$&c7h^w@_C3JgB}OpM#~V5D1Qp8PpST1VWgik8(YI&2c zd2~`_!U=t716$npU|;eFXb%j|!KSch$+>e$ax2Q0yv!v_xR$Bt!BzBrXaVt{r1?+Ip=;UbF; zC7-6>P5x!Rsec|W#H^5pbhYfyB8o1=uHaClw5!G*J4|F$ec^Uu1l}Fj(j=1xdbtfBLLe z7s%z&cC+`BFGe=_ z7Aqf{@)V$20AQTtU<9jzsCSc1B|$2tEL#WvQLUc;>_7$MSb#(%#Ef!OC8q0VQ||N1Ia7?2p&qqk8Ait&$}dd-kAUq4ptkZYMjHYK14$n#xk}-=p<5TjWRNX>iXLxvq~Uqtlz$%VsnC!nCP>)0ImdQQ z2Bel%Ln${=xb1EONTzdfX#yz2nU zPu04U#$car9`H7Dywq3&p2I{pvWByLq#E|k0e+PGxxGAK>r;?uyUp7#P@$|PiK>PR zQpdzC^#yiMeGKG6d^!D=jD4V74U%*SoEv9z;ZGnj_-Jj941)aUh0Qi^&y3#!$^Uy} zs7_sWBbc~!IYWVHB>DQ=uGoAWtp=6YMrN35_hD<#>LKNU%!%EfYt`^~8ZPecgUm!LDPn zgVn+90sr|pVXn@g1-Ijq!8NS;r}M5SBq3LZo* zuUHP`>EfYuM~@H_#5-&oxd6HL+#|Y`EXIit_X{HV2RV!FEA`EJXh{f&{E)Z``nhd` zRiI!%4Vg+LX9*t-^yi=?lO8OXzJG>%J0p|IL1AWgl>Oi`5oL6 zoyx=4N6e{C@t#%59I#jj&DuzX?c+J)h(;3h;A?~Ig$()*uJI$;$)CmA3krNLi#@*RLW_4 z3}~CaYWxGHjjuENeICHRG~}oNl@B#SMyceYjL$+LAx)Q2sU*pL$PbK%sf8w=cxxt6 ziPS_R*rD_NO6Gp{$Wt(HJ%uUw9Q?R3#Jh5F+Y_C<@tl`5^{RrYz5Ot17nA~0ulG_s zs$L5=bDG{y97`dOCvG;qf>>AM?N%_1k~4!Gnt>gu#%hMn=d|2!%mwMq2fxq%ANVW2 zj|PjCSxFQmt~s=j$$X4^?={3L7Ek4O0-TggB1t+7J(Yph;g;2S4qgF)8+Yy`-OMt4 zUou^+nytfM;c*CgE|i8CJb|y)>JL$3s-8`eV!pNIkFPH+Y;0Q7v&zztb^-3QpXl?T zgH}s(^VMH-hTiMW22Kr%2!dYoB1Q@tr1|>pSl?JClA#A>x-#p1g-z)E#uuCQcH;!% zOHrun@y6$44F4|zT-koo30BtD3uOHSgA$jqjGVgTxQ_@f7Ajr9n|*tg6uYsIvO40u zF(R_AM7=a>`f?ltZ+u1LA=3|yeA=<^?(ZwgS2%C+TQKrL5gAJDtNhd}bdEJ)} zSsqOYu^6lj+DqqG3?#t?-0DIDMO|jabe-hA_5?k~XS5Okxc~YF<@eQxN*(~Ky|6IK zO#Jl-JqFiy=$7@~@Qn7}SivV!W{BSMAw?h>OygKeY$gOLxIB}8th(GtS{T3~0H|)vcl*t?pOuYk*dt7fjh#TOud$*H2}ytMP@k1vcGjFrZU1o^GE*A_D)z;CDeQl^NF~RMS{;7~rNRTNZI+3Q@jhBZf41M6>K~87S%r`7A4hJ$ z%g46}%}~=IID5t{bkou@{pR~yNo8d+2Drr*eB#faKhfUQa0=z~SraB>eB1&V_V%?+F-F+Dt{J$O=HA|{6Sg0(ow#VFarv?z`rhio7-mwMI-hcX z@jks?vTNT_6+CojXADvrym+h^iKM&zEv&6uV08@kh+e-a+IWa+bx|l16Nnc&`ZDd_ z$>^`J(B)WPVPM`}?2He-b1vy~|Cc9=CB5tFjoDrrzg5qU_i13t&z}wk4%Hm)Lt)BJ zWCPbRPz$Fn;+@)~(sO?^;1eygN-uKMue_}HRJwfmvIJ;T-tCHjL^fNro@HXSTk(NI zhg7eWVUB6s{g)v5XWA!P-`n%ma;C9E-VY(6!tdxJYr+}s1Dn?{jb%8>LY-@wX12~% zZ3E{BXh8gUn83;)FE==b*pJ!uGam65^!7fX+6L_`F~ z;*&QSrzBnOl7dgMc*w9D*;%1%PBQzPAnig}k-wv6NRNAGv~_jcpBEHNhV53{@oSY# z%nb2bNFZ42LAOYVfSV8_q=Jrrw2#?heRVN&?os3&48DxgJu_Z;!q%zFb>e-&c(CNO zhN9x5TeohZFT^vvQb^aoFV zXWN*iFP*DA0nFmQqx|2QwDseVVI#asK?07@w@>3mHMd?`0e-L%%lJQwd+C)6*izyMSg&Ssh`AMzoSYN>c9C{ z%ISrnaP0WcS|M#5u2eoQ^r*nCMCfd)R$5xx-gAL{81j_D^->8~ub<%(6B#*LdU|}s z-mKMCcLUG4id8WALY?^xHoZ+v3H6>ogb`{HE&?6&oL7dUCPaxVYCpX8u+y@F+0c#T zs3kNvHsbB1gZD5#Q)a_yfnCot`}!iz`0Tr}rj!X#?PhbCd6okQ4ivS9C?%{GcvD0L z^zyGlSL3wI%+~uO(J3&a-1|HZCANm0QC=9sdf!TU;}Gz<{{zrcW_52;3reS~ZxOw4 zpQnC(m6Pd&KJ(M1%Z4~GIj!R@=}dbcL2eE#sPJ@twI`=l+lNaCcLP4aDdPF_yLE5G zi-#lioJPOLF|OUzy`AvRbMQp(2Z34wM?2Vq+$bR<8+yM>diYYex!aVguruvFitI_O z-fQ-HjiW?y#~*FbO!tGOg@sa7DNw}u^mKIkovad0RfdIjaRSu`>HL;p^-sMNHt)b& z)eWBuSgs zos|4zcN>0kdfJ9`aeaM#33#zPs)SO*EL;swN}Jq4_pc}3RgJc^XxsT8-&{;g z5(KK3+@zM?h&xVdVx?OIg@o8}o15d>#PN>c`u#&CS-2_7;Q6Ov-XlQjz3Akae>vOd zcc{aQ*c)joDk9RB67BNJ;O0#;RlsDz9N!cBLAt zBx}y<^{YQ{Qa03LB-Xo3pgKm@yQ~yw+lFC8nn))uJ^a-C%H+4#qLTT8!n7;XlD#Xz zxc*1X9SB)J3EEdx2Iz{D(e2f^`RD2GlBveV>w0(`P?GHpcM*>R3uh z3~2=37N}@C^l)HxA`4!x3G8;~;2cb1l%YG%J`egtWpneCiJkolM!!BYTVY(()Pkg? zq@;@1DwvH+lZjEPOn|3AUfXH#+T{}QITp8$M#5gtf$G8pb(ZhE) zHr{r1c2>LldY!TyVkVDxh)jyIx|X`+RIb*HV5p0Mfk8sc5!sEo5QAmX?Qj z*S%{$Mjd2?HJ+A%+6JF+yYfT0AMzX79PMB{gD1)oltVhen&0>M(~lLa7@v}&O#D?L zgvN+=JB3edtjv#006>UBcYANHvviBHNAvRX0(sBS88iR=a z$G6ub9n-XIzkCu+q~%6K9(ytcDJ_C2F-l5GT3z$K7sRh9_a zsfgfEFht@%icd0dJJHh8uC`=(abSvPKkuU^iCXn+TVQ503O)`CtODXxp6V)U%ImJm zKKE0)rn$K}v9yX|E+5JF;1loEMyAHc-*#H5I>#_G9)n(4 z_qre|syguEl2=FWz3?)a%<3vu@3kr#4t9%D&SO2QXg-|M9e*G5f%d;M+_ zBAtwY?_$U+<=ORrZm17BJb3B^R}U%M*K=W5$Q!mXjt#PH7Q({ZP^B!-Xd4;Df=LlK zu&-c}u{r+k8t^xgi@`wpY!8{G zD66zo`V3{KX(wdM-fphdZ$3vOWW7@t>71XuS!aIF=EgGL@gR$-j=Z1lrGqwK&iTq0 z*!Gq+#M+jvmDNmyzkzf`zEyiX;u-=JP8at-K9LJ%rNLSJqQ+>?Vy5d@{}Z;> zfr@8T6$}ar0ub5sx@UpYx+53k+H*ezdRMcOE}mrOzNv|u3bE2QFaVEG*(G@NS!#MZ z9|z?%4>E4hhD4LY%?bzvHXl2z@$TSBv9|zSZUHQFpU~?~bNx7YDwrZ}rcpKhab6us zH>~#2%f4&E%HrfZ?XJH44naoXhW@QvX|bDYBe%fdgHMCCl0um-!KTExr^PqCE1P{J z6qs3??R$F`u+M1UC%>O=QWp2_HSnaoi3dwG)meb;rSvT4+7Bv`R=0+XHIsDg^pxvM zRe?^xDtLfK44Dl63TSak7wXZwoE*(41DBV8U~|C&;~<0R`2GusH_yDD6+lnybxqB1 zna!1YteYg5-QqBEA-dabFbMpyOV`Yt+d^1~rviQfUY30&Tjy)`DH7e@mT#iH86loS zXAtm~toAw=M{y}#XjGN({PjTwOwYtJakI2eYW}M7A(7yHpa^A4TshH2)-vp~T4<#* zsi4H!3dh^-Y#Q&}-QAr)5iPzvK5&a*>hHBuYb4<=8PxaUQrznICo+kdnSwa*&@X12 z1^rEE9?kqso`QLVq#@yBNr|3v@JV|sP)_*;fHxC$k~7bHXUOZzy;%U+1ACbFK3d3v zbPtjMsrLYmQHpKgT8}n!tLRNdxYF1-=Bq0xhkrl?JXuVjv{ZfgZLXCHUfp4s#VflFmhs z&1_81hydUq5s^^4F}s@gOs!J#F?&z|0f0{nBMM0exy%%UOr?TctyDZo@*99>N&$vJ z;`IVJSIG1L^1>&ACG~P*iDdz;T)%p3z10;HFbPeYynTXtY@qt40!}z31FlzJp8*Q;Nvev2toxsSzdZ+-snz*);0E2+MdN zQ}|~RF)`6};d4b*HZ|0|Xi!ho)#^&dWXEaFszsebo9Su@){&s;pN+xdPy@KXPe=}eMgJ;!D zLHs|fxdsihbxK^`qsIuff|_C{)z{T&??E;4!FC`XdtHeX156__xTyEKIZ8)JI|IHz z?{ybUBj_g%!JyNd-kUR<7<`@dvi^#v45l$c+mHv)FF?}KYOYu(Yhec~elRkCvsB5M zwD->8A7z8PRz8%%unzbNj~0No6TjyO(5AlQw+r}z*sOwrf{6{vOcdI{wc9zhXKEqm znh0VhI|=;o&jV~3f`6<#^C1aonWdBcInva~x#fm#k1BiKwO|G{wfsbbEo7 z=lVD`9A#vqN`KTs$X>rR;^3F2U|q!lLrj_y)8R%Wbi+K+AR?SBMPhh*12oZ3$Z zR|@f78@#%~D|cmKVWAD=m5=rqu89bqxrW2xmPoQ2_IlU5$YP+OB7iSM;e?`*7|9>* zpsuiraxTY(3RELlYBDbFPw0ADl92GF46Fnk2OwFwesF#dXz+Yq7VcXJe@?8(Bq_VB`(I*%^g}`qT;uPV2BD0ZaL-y1sKl&-aC7t+5_E`0T zaa#KNQrMc;+75YTz7?TFQlBI_J0(@2Qplc176yXs9*=Mfq-m+Pm0HHY0@&lEHYA&<})1Xe6oW4#*fiS;8*2x8Dw7gvn^^*0x=T(LpjkTXaM1OPtZ z@SW?#M@GTdHuv=zcUeQIeWsSMP*jgo_w(~x1-;5Z?zt7uD>54-B7+KF`?doF4JDRP zueQ%+rssXdGiL1FZacAzdriq>-{USrVMp4#cgGG~sb~U14yu6+VM@wJ2>}6non2fk zfJ3_lt_=Zd(yOBhfK7ra$KZwt>dv$2mHB7N-P*v7DH=}^WJgjbgJHEXXR_+D- z+0x#gT34qI-JHj`WLNxw@=Sn2r<+J6))ASJg0e*c#P_b&Z?0jTU0u^lOAWv%6fbpH za$q(Wu$zfb*u30$HxD4$j zjddv`cR(Pnjznk2j~_?q*$}#P^<0l-8^91M5D9mD_{&dSda=2Ig0LMD_A^Kv%O{4^ zgZ11W8ADutB)@JvOz@q(n~*Jes{d|8>92}va(}1 zc%}+JD9b>hbTxE!Q^DtVyDyjY3Sf>OJNA8babhvZ*7I_R)X%3-PL^SrS)`iW zseZ`jI@42%M-V3TAPOq+GBPsi8dg(N+f%^9!*k8r`aJ^gVQA`Zyz9&=*8R(2G(qSs$or!K!opc5VwJ#%IM= zb4ZSp>76>4{p$f%V;ZsEt4)bZ{i!K2SAIzW$=d`aN9Zw-8g~3XjP@^Tnk)2A0A*wd zFh~Y7(1)CLeFZ|m6YE9s@$&fQhK4Y^&M>P~B&3BHXyiqG6VxYt`t+%AIWI3ye91(C z|7a6rL=-WN7_R6Xy?{Z3DqdU2(0T%qH}(L}HgYZ(w8@_tmoU zX(I)TJoB|i8`pt=Nj1@B|3gFU#CiV_Iye{bI=7nPZlLu>OmeV1MLh$ekFhZvoCSlDbK*V+h;f!*@ z{vZ{hUPNyvHf7*QV;iOldSlkS!A5i6HAQN4O{<_TE*?RZNSqDqdkWY;zGM5-5)v*0 zqCf(Ro~8VPF{A`_WA=%x*aLOE6Jlr0WhpNOKhDbExN&2w+?N(9SxXOdWB=njtZ!@S z`LDvoKu}+R3QEWVqyuXMAnjI;SA^FJQ>`Y*^S^S^BFu0&eB^FX mtqlC-le?AD{r~vaugx90c)F9}X~I;ffl*Xf$dosJ@V@}O59p-; literal 0 HcmV?d00001 diff --git a/example/requirements.txt b/example/requirements.txt new file mode 100644 index 0000000..aef37e1 --- /dev/null +++ b/example/requirements.txt @@ -0,0 +1,8 @@ +transformers==4.46.3 +tokenizers==0.20.3 +accelerate==1.1.1 +deepspeed==0.16.2 +datasets==3.1.0 +safetensors==0.4.5 +pyarrow==17.0.0 +tensorboard==2.18.0 diff --git a/example/run.sh b/example/run.sh new file mode 100644 index 0000000..d8287db --- /dev/null +++ b/example/run.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +MODEL_PATH="/model/BitCPM-CANN-1B-unquantized" +DATA_PATH="/dataset/c4-pro/data/000_1_7.parquet" +OUTPUT_DIR="./output" +DS_CONFIG="./ds_config_z2.json" + +NUM_GPUS=8 +BATCH_SIZE_PER_GPU=8 +GRAD_ACCUM_STEPS=8 +MAX_SEQ_LENGTH=1024 + +export ASCEND_RT_VISIBLE_DEVICES=8,9,10,11,12,13,14,15 +export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 +export DS_SKIP_CUDA_CHECK=1 +torchrun --nproc_per_node=$NUM_GPUS train.py \ + --model_name_or_path $MODEL_PATH \ + --data_path $DATA_PATH \ + --max_seq_length $MAX_SEQ_LENGTH \ + --output_dir $OUTPUT_DIR \ + --per_device_train_batch_size $BATCH_SIZE_PER_GPU \ + --gradient_accumulation_steps $GRAD_ACCUM_STEPS \ + --max_steps 100 \ + --learning_rate 4e-5 \ + --lr_scheduler_type cosine \ + --warmup_ratio 0.1 \ + --weight_decay 1e-2 \ + --logging_steps 2 \ + --save_steps 500 \ + --save_total_limit 3 \ + --bf16 \ + --deepspeed $DS_CONFIG \ + --gradient_checkpointing \ + --seed 42 \ + --dataloader_num_workers 4 \ + --report_to tensorboard \ + --logging_dir /data/tensorboard/pretrain \ + --gradient_checkpointing_kwargs '{"use_reentrant": false}' diff --git a/example/run_sft.sh b/example/run_sft.sh new file mode 100644 index 0000000..597180d --- /dev/null +++ b/example/run_sft.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +MODEL_PATH="/model/BitCPM-CANN-1B-unquantized" +DATA_PATH="/dataset/HuggingFaceH4_ultrachat_200k/data/train_sft-00000-of-00003-a3ecf92756993583.parquet" +OUTPUT_DIR="./output_sft" +DS_CONFIG="./ds_config.json" + +NUM_GPUS=8 +BATCH_SIZE_PER_GPU=2 +GRAD_ACCUM_STEPS=1 +MAX_SEQ_LENGTH=8192 + +export ASCEND_RT_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 +export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 +export DS_SKIP_CUDA_CHECK=1 + +torchrun --nproc_per_node=$NUM_GPUS train_sft.py \ + --model_name_or_path $MODEL_PATH \ + --data_path $DATA_PATH \ + --max_seq_length $MAX_SEQ_LENGTH \ + --output_dir $OUTPUT_DIR \ + --per_device_train_batch_size $BATCH_SIZE_PER_GPU \ + --gradient_accumulation_steps $GRAD_ACCUM_STEPS \ + --max_steps 100 \ + --learning_rate 2e-5 \ + --lr_scheduler_type cosine \ + --warmup_ratio 0.2 \ + --weight_decay 0.0 \ + --logging_steps 2 \ + --save_steps 500 \ + --save_total_limit 3 \ + --bf16 \ + --deepspeed $DS_CONFIG \ + --gradient_checkpointing \ + --seed 42 \ + --dataloader_num_workers 4 \ + --report_to tensorboard \ + --logging_dir /data/tensorboard/sft \ + --train_on_prompt false \ + --gradient_checkpointing_kwargs '{"use_reentrant": false}' diff --git a/example/train.py b/example/train.py new file mode 100644 index 0000000..842a39b --- /dev/null +++ b/example/train.py @@ -0,0 +1,203 @@ +""" +Continual pretraining script for CPM-2B model using DeepSpeed + HuggingFace Trainer. +""" + +import os +import json +import math +import logging +from dataclasses import dataclass, field +from typing import Optional + +import contextlib + +import torch +from datasets import load_dataset +from transformers import ( + AutoModelForCausalLM, + AutoTokenizer, + AutoConfig, + Trainer, + TrainingArguments, + HfArgumentParser, + DataCollatorForLanguageModeling, + set_seed, +) + +import deepspeed +_orig_no_sync = deepspeed.DeepSpeedEngine.no_sync + +@contextlib.contextmanager +def _patched_no_sync(self): + try: + with _orig_no_sync(self): + yield + except AssertionError: + yield + +deepspeed.DeepSpeedEngine.no_sync = _patched_no_sync + +logger = logging.getLogger(__name__) + + +@dataclass +class ModelArguments: + model_name_or_path: str = field( + metadata={"help": "Path to pretrained model or model identifier"} + ) + torch_dtype: Optional[str] = field( + default="bfloat16", + metadata={"help": "torch dtype for model weights (float16, bfloat16, float32)"}, + ) + + +@dataclass +class DataArguments: + data_path: str = field( + metadata={"help": "Path to training data (parquet file or directory)"} + ) + max_seq_length: int = field( + default=4096, + metadata={"help": "Maximum sequence length for training"}, + ) + text_column: str = field( + default="text", + metadata={"help": "Name of the text column in the dataset"}, + ) + preprocessing_num_workers: int = field( + default=8, + metadata={"help": "Number of workers for data preprocessing"}, + ) + + +def tokenize_and_group(dataset, tokenizer, data_args): + """Tokenize texts and group into chunks of max_seq_length.""" + + column_names = dataset.column_names + text_column = data_args.text_column + if text_column not in column_names: + candidates = [c for c in column_names if "text" in c.lower()] + if candidates: + text_column = candidates[0] + else: + text_column = column_names[0] + logger.warning(f"Column '{data_args.text_column}' not found, using '{text_column}'") + + def tokenize_function(examples): + return tokenizer(examples[text_column], add_special_tokens=False) + + tokenized_dataset = dataset.map( + tokenize_function, + batched=True, + num_proc=data_args.preprocessing_num_workers, + remove_columns=column_names, + desc="Tokenizing", + ) + + block_size = data_args.max_seq_length + + def group_texts(examples): + concatenated = {k: sum(examples[k], []) for k in examples.keys()} + total_length = len(concatenated["input_ids"]) + total_length = (total_length // block_size) * block_size + + result = { + k: [t[i : i + block_size] for i in range(0, total_length, block_size)] + for k, t in concatenated.items() + } + result["labels"] = result["input_ids"].copy() + return result + + grouped_dataset = tokenized_dataset.map( + group_texts, + batched=True, + num_proc=data_args.preprocessing_num_workers, + desc="Grouping texts", + ) + + return grouped_dataset + + +def main(): + parser = HfArgumentParser((ModelArguments, DataArguments, TrainingArguments)) + model_args, data_args, training_args = parser.parse_args_into_dataclasses() + + logging.basicConfig( + format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", + datefmt="%Y-%m-%d %H:%M:%S", + level=logging.INFO if training_args.local_rank in [-1, 0] else logging.WARN, + ) + logger.info(f"Training args: {training_args}") + + set_seed(training_args.seed) + + dtype_map = { + "float16": torch.float16, + "bfloat16": torch.bfloat16, + "float32": torch.float32, + } + torch_dtype = dtype_map.get(model_args.torch_dtype, torch.bfloat16) + + logger.info(f"Loading tokenizer from {model_args.model_name_or_path}") + tokenizer = AutoTokenizer.from_pretrained( + model_args.model_name_or_path, + trust_remote_code=True, + ) + if tokenizer.pad_token is None: + tokenizer.pad_token = tokenizer.eos_token + + logger.info(f"Loading model from {model_args.model_name_or_path}") + model = AutoModelForCausalLM.from_pretrained( + model_args.model_name_or_path, + torch_dtype=torch_dtype, + trust_remote_code=True, + attn_implementation="sdpa", + ) + model.config.use_cache = False + + logger.info(f"Loading dataset from {data_args.data_path}") + if os.path.isfile(data_args.data_path): + raw_dataset = load_dataset("parquet", data_files=data_args.data_path, split="train") + elif os.path.isdir(data_args.data_path): + parquet_files = [ + os.path.join(data_args.data_path, f) + for f in os.listdir(data_args.data_path) + if f.endswith(".parquet") + ] + raw_dataset = load_dataset("parquet", data_files=parquet_files, split="train") + else: + raise ValueError(f"Data path not found: {data_args.data_path}") + + logger.info(f"Dataset loaded: {len(raw_dataset)} samples, columns: {raw_dataset.column_names}") + + train_dataset = tokenize_and_group(raw_dataset, tokenizer, data_args) + logger.info(f"Processed dataset: {len(train_dataset)} samples of length {data_args.max_seq_length}") + + data_collator = DataCollatorForLanguageModeling( + tokenizer=tokenizer, + mlm=False, + ) + + trainer = Trainer( + model=model, + args=training_args, + train_dataset=train_dataset, + data_collator=data_collator, + ) + + logger.info("Starting training...") + train_result = trainer.train( + resume_from_checkpoint=training_args.resume_from_checkpoint + ) + + trainer.save_model() + trainer.save_state() + + metrics = train_result.metrics + metrics["train_samples"] = len(train_dataset) + trainer.log_metrics("train", metrics) + trainer.save_metrics("train", metrics) + + +if __name__ == "__main__": + main() diff --git a/example/train_sft.py b/example/train_sft.py new file mode 100644 index 0000000..e672481 --- /dev/null +++ b/example/train_sft.py @@ -0,0 +1,424 @@ +""" +Supervised fine-tuning script using DeepSpeed + HuggingFace Trainer. +""" + +import json +import logging +import os +from dataclasses import dataclass, field +from typing import Any, Dict, List, Optional, Tuple + +import contextlib + +import torch +from datasets import load_dataset +from transformers import ( + AutoModelForCausalLM, + AutoTokenizer, + HfArgumentParser, + Trainer, + TrainingArguments, + set_seed, +) + +import deepspeed +_orig_no_sync = deepspeed.DeepSpeedEngine.no_sync + +@contextlib.contextmanager +def _patched_no_sync(self): + try: + with _orig_no_sync(self): + yield + except AssertionError: + yield + +deepspeed.DeepSpeedEngine.no_sync = _patched_no_sync + +logger = logging.getLogger(__name__) + +IGNORE_INDEX = -100 + + +@dataclass +class ModelArguments: + model_name_or_path: str = field( + metadata={"help": "Path to pretrained model or model identifier"} + ) + torch_dtype: Optional[str] = field( + default="bfloat16", + metadata={"help": "torch dtype for model weights (float16, bfloat16, float32)"}, + ) + + +@dataclass +class DataArguments: + data_path: str = field(metadata={"help": "Path to SFT data file or directory"}) + max_seq_length: int = field( + default=4096, + metadata={"help": "Maximum sequence length for training"}, + ) + prompt_column: Optional[str] = field( + default=None, + metadata={"help": "Prompt/instruction column name. Auto-detected if omitted."}, + ) + input_column: Optional[str] = field( + default=None, + metadata={"help": "Optional extra input/context column name"}, + ) + response_column: Optional[str] = field( + default=None, + metadata={"help": "Response/output column name. Auto-detected if omitted."}, + ) + messages_column: Optional[str] = field( + default=None, + metadata={"help": "Chat messages column name. Auto-detected if omitted."}, + ) + system_column: Optional[str] = field( + default=None, + metadata={"help": "Optional system prompt column name"}, + ) + train_on_prompt: bool = field( + default=False, + metadata={"help": "Whether to compute loss on prompt/user tokens"}, + ) + add_eos_token: bool = field( + default=True, + metadata={"help": "Append eos_token to plain prompt/response examples"}, + ) + preprocessing_num_workers: int = field( + default=8, + metadata={"help": "Number of workers for data preprocessing"}, + ) + + +class SFTDataCollator: + def __init__(self, tokenizer, pad_to_multiple_of: Optional[int] = 8): + self.tokenizer = tokenizer + self.pad_to_multiple_of = pad_to_multiple_of + + def __call__(self, features: List[Dict[str, List[int]]]) -> Dict[str, torch.Tensor]: + max_length = max(len(feature["input_ids"]) for feature in features) + if self.pad_to_multiple_of: + multiple = self.pad_to_multiple_of + max_length = ((max_length + multiple - 1) // multiple) * multiple + + input_ids = [] + attention_mask = [] + labels = [] + pad_token_id = self.tokenizer.pad_token_id + + for feature in features: + length = len(feature["input_ids"]) + pad_length = max_length - length + input_ids.append(feature["input_ids"] + [pad_token_id] * pad_length) + attention_mask.append([1] * length + [0] * pad_length) + labels.append(feature["labels"] + [IGNORE_INDEX] * pad_length) + + return { + "input_ids": torch.tensor(input_ids, dtype=torch.long), + "attention_mask": torch.tensor(attention_mask, dtype=torch.long), + "labels": torch.tensor(labels, dtype=torch.long), + } + + +def load_sft_dataset(data_path: str): + if os.path.isfile(data_path): + extension = os.path.splitext(data_path)[1].lstrip(".").lower() + if extension == "jsonl": + extension = "json" + if extension not in {"parquet", "json", "csv", "txt"}: + raise ValueError(f"Unsupported data file extension: {extension}") + return load_dataset(extension, data_files=data_path, split="train") + + if os.path.isdir(data_path): + data_files = [] + extension = None + for name in os.listdir(data_path): + current_extension = os.path.splitext(name)[1].lstrip(".").lower() + if current_extension == "jsonl": + current_extension = "json" + if current_extension in {"parquet", "json", "csv", "txt"}: + extension = extension or current_extension + if current_extension == extension: + data_files.append(os.path.join(data_path, name)) + if not data_files or extension is None: + raise ValueError(f"No supported data files found in: {data_path}") + return load_dataset(extension, data_files=sorted(data_files), split="train") + + raise ValueError(f"Data path not found: {data_path}") + + +def choose_column( + column_names: List[str], explicit: Optional[str], candidates: List[str] +) -> Optional[str]: + if explicit: + if explicit not in column_names: + raise ValueError(f"Column '{explicit}' not found. Available columns: {column_names}") + return explicit + for name in candidates: + if name in column_names: + return name + return None + + +def parse_messages(value: Any) -> List[Dict[str, str]]: + if isinstance(value, str): + value = json.loads(value) + if not isinstance(value, list): + raise ValueError("messages/conversations column must be a list or JSON string") + + messages = [] + for item in value: + if not isinstance(item, dict): + raise ValueError("Each message must be a dict") + + role = item.get("role", item.get("from")) + content = item.get("content", item.get("value")) + if role == "human": + role = "user" + elif role == "gpt": + role = "assistant" + + if role is None or content is None: + raise ValueError("Each message must contain role/from and content/value") + messages.append({"role": str(role), "content": str(content)}) + + return messages + + +def tokenize_text(tokenizer, text: str) -> List[int]: + return tokenizer(text, add_special_tokens=False)["input_ids"] + + +def apply_chat_template(tokenizer, messages: List[Dict[str, str]], add_generation_prompt: bool) -> str: + if tokenizer.chat_template is None: + raise ValueError( + "The tokenizer has no chat_template. Use prompt/response columns or set a chat_template." + ) + return tokenizer.apply_chat_template( + messages, + tokenize=False, + add_generation_prompt=add_generation_prompt, + ) + + +def encode_prompt_response( + example: Dict[str, Any], + tokenizer, + data_args: DataArguments, + prompt_column: str, + input_column: Optional[str], + response_column: str, +) -> Tuple[List[int], List[int]]: + prompt = str(example[prompt_column]) + if input_column and example.get(input_column): + prompt = prompt + "\n" + str(example[input_column]) + response = str(example[response_column]) + + messages = [] + if data_args.system_column and example.get(data_args.system_column): + messages.append({"role": "system", "content": str(example[data_args.system_column])}) + messages.append({"role": "user", "content": prompt}) + messages.append({"role": "assistant", "content": response}) + + if tokenizer.chat_template is not None: + full_text = apply_chat_template(tokenizer, messages, add_generation_prompt=False) + prompt_text = apply_chat_template(tokenizer, messages[:-1], add_generation_prompt=True) + input_ids = tokenize_text(tokenizer, full_text) + prompt_length = len(tokenize_text(tokenizer, prompt_text)) + else: + response_text = response + if data_args.add_eos_token and tokenizer.eos_token: + response_text += tokenizer.eos_token + full_text = prompt + "\n" + response_text + input_ids = tokenize_text(tokenizer, full_text) + prompt_length = len(tokenize_text(tokenizer, prompt + "\n")) + + labels = input_ids.copy() + if not data_args.train_on_prompt: + labels[:prompt_length] = [IGNORE_INDEX] * min(prompt_length, len(labels)) + return input_ids, labels + + +def encode_messages( + example: Dict[str, Any], + tokenizer, + data_args: DataArguments, + messages_column: str, +) -> Tuple[List[int], List[int]]: + messages = parse_messages(example[messages_column]) + + if tokenizer.chat_template is not None: + full_text = apply_chat_template(tokenizer, messages, add_generation_prompt=False) + input_ids = tokenize_text(tokenizer, full_text) + labels = [IGNORE_INDEX] * len(input_ids) + + if data_args.train_on_prompt: + labels = input_ids.copy() + else: + for index, message in enumerate(messages): + if message["role"] != "assistant": + continue + before_text = apply_chat_template( + tokenizer, messages[:index], add_generation_prompt=True + ) + after_text = apply_chat_template( + tokenizer, messages[: index + 1], add_generation_prompt=False + ) + start = len(tokenize_text(tokenizer, before_text)) + end = len(tokenize_text(tokenizer, after_text)) + labels[start:end] = input_ids[start:end] + else: + labels = [] + input_ids = [] + for message in messages: + part = f"{message['role']}: {message['content']}\n" + if data_args.add_eos_token and message["role"] == "assistant" and tokenizer.eos_token: + part += tokenizer.eos_token + part_ids = tokenize_text(tokenizer, part) + input_ids.extend(part_ids) + if data_args.train_on_prompt or message["role"] == "assistant": + labels.extend(part_ids) + else: + labels.extend([IGNORE_INDEX] * len(part_ids)) + + return input_ids, labels + + +def preprocess_sft_dataset(raw_dataset, tokenizer, data_args: DataArguments): + column_names = raw_dataset.column_names + messages_column = choose_column( + column_names, data_args.messages_column, ["messages", "conversations"] + ) + prompt_column = choose_column( + column_names, + data_args.prompt_column, + ["prompt", "instruction", "question"], + ) + input_column = choose_column( + column_names, + data_args.input_column, + ["input", "context"], + ) + response_column = choose_column( + column_names, + data_args.response_column, + ["response", "output", "answer", "chosen"], + ) + + if messages_column: + logger.info(f"Using chat messages column: {messages_column}") + elif prompt_column and response_column: + logger.info(f"Using prompt column '{prompt_column}' and response column '{response_column}'") + else: + raise ValueError( + "Cannot infer SFT data format. Provide either messages/conversations or " + "prompt/instruction plus response/output columns." + ) + + def encode_batch(examples): + batch_input_ids = [] + batch_labels = [] + batch_attention_mask = [] + + batch_size = len(next(iter(examples.values()))) + for i in range(batch_size): + example = {name: values[i] for name, values in examples.items()} + if messages_column: + input_ids, labels = encode_messages(example, tokenizer, data_args, messages_column) + else: + input_ids, labels = encode_prompt_response( + example, tokenizer, data_args, prompt_column, input_column, response_column + ) + + input_ids = input_ids[: data_args.max_seq_length] + labels = labels[: data_args.max_seq_length] + if not input_ids or all(label == IGNORE_INDEX for label in labels): + continue + + batch_input_ids.append(input_ids) + batch_labels.append(labels) + batch_attention_mask.append([1] * len(input_ids)) + + return { + "input_ids": batch_input_ids, + "attention_mask": batch_attention_mask, + "labels": batch_labels, + } + + return raw_dataset.map( + encode_batch, + batched=True, + num_proc=data_args.preprocessing_num_workers, + remove_columns=column_names, + desc="Tokenizing SFT data", + ) + + +def main(): + parser = HfArgumentParser((ModelArguments, DataArguments, TrainingArguments)) + model_args, data_args, training_args = parser.parse_args_into_dataclasses() + + logging.basicConfig( + format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", + datefmt="%Y-%m-%d %H:%M:%S", + level=logging.INFO if training_args.local_rank in [-1, 0] else logging.WARN, + ) + logger.info(f"Training args: {training_args}") + + set_seed(training_args.seed) + + dtype_map = { + "float16": torch.float16, + "bfloat16": torch.bfloat16, + "float32": torch.float32, + } + torch_dtype = dtype_map.get(model_args.torch_dtype, torch.bfloat16) + + logger.info(f"Loading tokenizer from {model_args.model_name_or_path}") + tokenizer = AutoTokenizer.from_pretrained( + model_args.model_name_or_path, + trust_remote_code=True, + ) + if tokenizer.pad_token is None: + tokenizer.pad_token = tokenizer.eos_token + + logger.info(f"Loading model from {model_args.model_name_or_path}") + model = AutoModelForCausalLM.from_pretrained( + model_args.model_name_or_path, + torch_dtype=torch_dtype, + trust_remote_code=True, + attn_implementation="sdpa", + ) + model.config.use_cache = False + + logger.info(f"Loading SFT dataset from {data_args.data_path}") + raw_dataset = load_sft_dataset(data_args.data_path) + logger.info(f"Dataset loaded: {len(raw_dataset)} samples, columns: {raw_dataset.column_names}") + + train_dataset = preprocess_sft_dataset(raw_dataset, tokenizer, data_args) + logger.info(f"Processed dataset: {len(train_dataset)} samples") + + trainer = Trainer( + model=model, + args=training_args, + train_dataset=train_dataset, + data_collator=SFTDataCollator(tokenizer), + ) + + logger.info("Starting SFT training...") + train_result = trainer.train( + resume_from_checkpoint=training_args.resume_from_checkpoint + ) + + trainer.save_model() + trainer.save_state() + + metrics = train_result.metrics + metrics["train_samples"] = len(train_dataset) + trainer.log_metrics("train", metrics) + trainer.save_metrics("train", metrics) + + +if __name__ == "__main__": + main() diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..1d3bb1e --- /dev/null +++ b/generation_config.json @@ -0,0 +1,8 @@ +{ + "do_sample": true, + "top_p": 0.8, + "temperature": 0.8, + "bos_token_id": 1, + "eos_token_id": [2,73440], + "pad_token_id": 2 +} diff --git a/modeling_llama.py b/modeling_llama.py new file mode 100644 index 0000000..26d845c --- /dev/null +++ b/modeling_llama.py @@ -0,0 +1,1598 @@ +# coding=utf-8 +# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved. +# +# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX +# and OPT implementations in this library. It has been modified from its +# original forms to accommodate minor architectural differences compared +# to GPT-NeoX and OPT used by the Meta AI team that trained the model. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import math +from typing import List, Optional, Tuple, Union + +import torch +import torch.nn.functional as F +import torch.utils.checkpoint +from torch import nn + +from transformers.activations import ACT2FN +from transformers.cache_utils import Cache, DynamicCache, StaticCache +from transformers.generation import GenerationMixin +from transformers.modeling_attn_mask_utils import AttentionMaskConverter +from transformers.modeling_flash_attention_utils import _flash_attention_forward +from transformers.modeling_outputs import ( + BaseModelOutputWithPast, + CausalLMOutputWithPast, + QuestionAnsweringModelOutput, + SequenceClassifierOutputWithPast, + TokenClassifierOutput, +) +from transformers.modeling_rope_utils import ROPE_INIT_FUNCTIONS +from transformers.modeling_utils import PreTrainedModel +from transformers.pytorch_utils import ALL_LAYERNORM_LAYERS +from transformers.utils import ( + add_code_sample_docstrings, + add_start_docstrings, + add_start_docstrings_to_model_forward, + is_flash_attn_greater_or_equal_2_10, + logging, + replace_return_docstrings, +) +from .configuration_llama import LlamaConfig + + +logger = logging.get_logger(__name__) + +_CHECKPOINT_FOR_DOC = "meta-llama/Llama-2-7b-hf" +_CONFIG_FOR_DOC = "LlamaConfig" + + +def get_quantizer(quant_type="none", bit=4, group_size=128): + if quant_type == "intsym": + return SteIntSymQuantizerGPTQ(bit, group_size) + elif quant_type == "ternary": + return SteTernaryQuantizer(group_size) + elif quant_type == "none": + return NoQuantizer() + else: + raise ValueError(f"Unsupported quantization type: {quant_type}") + +class SteIntSymQuantizerGPTQ(nn.Module): + def __init__(self, bit=4, group_size=-1): + super().__init__() + self.bit = bit + self.group_size = group_size + + def forward(self, x): + org_w_shape = x.shape + + if self.group_size > 0: + assert org_w_shape[-1] % self.group_size == 0 + x = x.reshape(-1, self.group_size) + elif self.group_size == -1: + assert org_w_shape[-1] % self.group_size == 0 + x = x.reshape(-1, x.shape[-1]) + elif self.group_size == 0: + x = x.reshape(1, -1) + + assert x.dim() == 2 + + xmax = x.max(dim=1, keepdim=True)[0] + xmin = x.min(dim=1, keepdim=True)[0] + abs_max_val = torch.maximum(torch.abs(xmin), xmax) # ไธŽQuantizer็š„xmax่ฎก็ฎ—ไธ€่‡ด + scales = abs_max_val * 2 / (2 ** self.bit - 1) # ๅˆ†ๅญๅˆ†ๆฏ้ƒฝๅฏน้ฝ + + max_int = 2 ** (self.bit - 1) - 1 + min_int = - (2 ** (self.bit - 1)) + + assert torch.isnan(scales).sum() == 0 + + x_q = (torch.clamp(torch.round(x / scales), min_int, max_int)) * scales + + assert torch.isnan(x_q).sum() == 0 + + x = x.reshape(org_w_shape) + x_q = x_q.reshape(org_w_shape) + + return x + (x_q - x).detach() + +class SteTernaryQuantizer(nn.Module): + def __init__(self, group_size): + super().__init__() + self.group_size = group_size + + def forward(self, x): + org_w_shape = x.shape + if self.group_size > 0: + assert x.shape[-1] % self.group_size == 0 + x = x.reshape(-1, self.group_size) + elif self.group_size == -1: + x = x.reshape(-1, x.shape[-1]) + + assert x.dim() == 2 + + scales = 1.0 / (x.abs().mean(dim=1, keepdim=True).clamp_(min=1e-5)) + x_q = (torch.clamp(torch.round(x * scales),-1,1) / scales) + + assert torch.isnan(x_q).sum() == 0 + + x = x.reshape(org_w_shape) + x_q = x_q.reshape(org_w_shape) + + return x + (x_q - x).detach() + +class NoQuantizer(nn.Module): + def __init__(self): + super().__init__() + + def forward(self, x): + return x + +class LinearQuantizer(nn.Linear): + def __init__(self, in_features, out_features, bias=False, quant_type="ternary", bit=4, group_size=-1): + super().__init__(in_features, out_features, bias) + self.quantizer = get_quantizer(quant_type, bit, group_size) + + def forward(self, x): + weight_tensor = self.quantizer(self.weight) + x = torch.nn.functional.linear(x, weight_tensor) + if self.bias is not None: + x = x + self.bias + return x + +class LlamaRMSNorm(nn.Module): + def __init__(self, hidden_size, eps=1e-6): + """ + LlamaRMSNorm is equivalent to T5LayerNorm + """ + super().__init__() + self.weight = nn.Parameter(torch.ones(hidden_size)) + self.variance_epsilon = eps + + def forward(self, hidden_states): + input_dtype = hidden_states.dtype + hidden_states = hidden_states.to(torch.float32) + variance = hidden_states.pow(2).mean(-1, keepdim=True) + hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon) + return self.weight * hidden_states.to(input_dtype) + + def extra_repr(self): + return f"{tuple(self.weight.shape)}, eps={self.variance_epsilon}" + + +ALL_LAYERNORM_LAYERS.append(LlamaRMSNorm) + + +class LlamaRotaryEmbedding(nn.Module): + def __init__( + self, + dim=None, + max_position_embeddings=2048, + base=10000, + device=None, + scaling_factor=1.0, + rope_type="default", + config: Optional[LlamaConfig] = None, + ): + super().__init__() + # TODO (joao): remove the `if` below, only used for BC + self.rope_kwargs = {} + if config is None: + logger.warning_once( + "`LlamaRotaryEmbedding` can now be fully parameterized by passing the model config through the " + "`config` argument. All other arguments will be removed in v4.46" + ) + self.rope_kwargs = { + "rope_type": rope_type, + "factor": scaling_factor, + "dim": dim, + "base": base, + "max_position_embeddings": max_position_embeddings, + } + self.rope_type = rope_type + self.max_seq_len_cached = max_position_embeddings + self.original_max_seq_len = max_position_embeddings + else: + # BC: "rope_type" was originally "type" + if config.rope_scaling is not None: + self.rope_type = config.rope_scaling.get("rope_type", config.rope_scaling.get("type")) + else: + self.rope_type = "default" + self.max_seq_len_cached = config.max_position_embeddings + self.original_max_seq_len = config.max_position_embeddings + + self.config = config + self.rope_init_fn = ROPE_INIT_FUNCTIONS[self.rope_type] + + inv_freq, self.attention_scaling = self.rope_init_fn(self.config, device, **self.rope_kwargs) + self.register_buffer("inv_freq", inv_freq, persistent=False) + self.original_inv_freq = self.inv_freq + + def _dynamic_frequency_update(self, position_ids, device): + """ + dynamic RoPE layers should recompute `inv_freq` in the following situations: + 1 - growing beyond the cached sequence length (allow scaling) + 2 - the current sequence length is in the original scale (avoid losing precision with small sequences) + """ + seq_len = torch.max(position_ids) + 1 + if seq_len > self.max_seq_len_cached: # growth + inv_freq, self.attention_scaling = self.rope_init_fn( + self.config, device, seq_len=seq_len, **self.rope_kwargs + ) + self.register_buffer("inv_freq", inv_freq, persistent=False) # TODO joao: may break with compilation + self.max_seq_len_cached = seq_len + + if seq_len < self.original_max_seq_len and self.max_seq_len_cached > self.original_max_seq_len: # reset + self.register_buffer("inv_freq", self.original_inv_freq, persistent=False) + self.max_seq_len_cached = self.original_max_seq_len + + @torch.no_grad() + def forward(self, x, position_ids): + if "dynamic" in self.rope_type: + self._dynamic_frequency_update(position_ids, device=x.device) + + # Core RoPE block + inv_freq_expanded = self.inv_freq[None, :, None].float().expand(position_ids.shape[0], -1, 1) + position_ids_expanded = position_ids[:, None, :].float() + # Force float32 (see https://github.com/huggingface/transformers/pull/29285) + device_type = x.device.type + device_type = device_type if isinstance(device_type, str) and device_type != "mps" else "cpu" + with torch.autocast(device_type=device_type, enabled=False): + freqs = (inv_freq_expanded.float() @ position_ids_expanded.float()).transpose(1, 2) + emb = torch.cat((freqs, freqs), dim=-1) + cos = emb.cos() + sin = emb.sin() + + # Advanced RoPE types (e.g. yarn) apply a post-processing scaling factor, equivalent to scaling attention + cos = cos * self.attention_scaling + sin = sin * self.attention_scaling + + return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype) + + +class LlamaLinearScalingRotaryEmbedding(LlamaRotaryEmbedding): + """LlamaRotaryEmbedding extended with linear scaling. Credits to the Reddit user /u/kaiokendev""" + + def __init__(self, *args, **kwargs): + logger.warning_once( + "`LlamaLinearScalingRotaryEmbedding` is deprecated an will be removed in v4.46. Please use " + "`LlamaRotaryEmbedding`, which now also does linear scaling (simply pass the model config to __init__)." + ) + kwargs["rope_type"] = "linear" + super().__init__(*args, **kwargs) + + +class LlamaDynamicNTKScalingRotaryEmbedding(LlamaRotaryEmbedding): + """LlamaRotaryEmbedding extended with Dynamic NTK scaling. Credits to the Reddit users /u/bloc97 and /u/emozilla""" + + def __init__(self, *args, **kwargs): + logger.warning_once( + "`LlamaDynamicNTKScalingRotaryEmbedding` is deprecated an will be removed in v4.46. Please use " + "`LlamaRotaryEmbedding`, which now also does dynamic ntk scaling (simply pass the model config to " + "__init__)." + ) + kwargs["rope_type"] = "dynamic" + super().__init__(*args, **kwargs) + + +def rotate_half(x): + """Rotates half the hidden dims of the input.""" + x1 = x[..., : x.shape[-1] // 2] + x2 = x[..., x.shape[-1] // 2 :] + return torch.cat((-x2, x1), dim=-1) + + +def apply_rotary_pos_emb(q, k, cos, sin, position_ids=None, unsqueeze_dim=1): + """Applies Rotary Position Embedding to the query and key tensors. + + Args: + q (`torch.Tensor`): The query tensor. + k (`torch.Tensor`): The key tensor. + cos (`torch.Tensor`): The cosine part of the rotary embedding. + sin (`torch.Tensor`): The sine part of the rotary embedding. + position_ids (`torch.Tensor`, *optional*): + Deprecated and unused. + unsqueeze_dim (`int`, *optional*, defaults to 1): + The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and + sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note + that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and + k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes + cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have + the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2. + Returns: + `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding. + """ + cos = cos.unsqueeze(unsqueeze_dim) + sin = sin.unsqueeze(unsqueeze_dim) + q_embed = (q * cos) + (rotate_half(q) * sin) + k_embed = (k * cos) + (rotate_half(k) * sin) + return q_embed, k_embed + + +class LlamaMLP(nn.Module): + def __init__(self, config): + super().__init__() + self.config = config + self.hidden_size = config.hidden_size + self.intermediate_size = config.intermediate_size + self.gate_proj = LinearQuantizer(self.hidden_size, self.intermediate_size, bias=config.mlp_bias, quant_type="ternary", bit=4, group_size=-1) + self.up_proj = LinearQuantizer(self.hidden_size, self.intermediate_size, bias=config.mlp_bias, quant_type="ternary", bit=4, group_size=-1) + self.down_proj = LinearQuantizer(self.intermediate_size, self.hidden_size, bias=config.mlp_bias, quant_type="ternary", bit=4, group_size=-1) + self.act_fn = ACT2FN[config.hidden_act] + + def forward(self, x): + if self.config.pretraining_tp > 1: + slice = self.intermediate_size // self.config.pretraining_tp + gate_proj_slices = self.gate_proj.weight.split(slice, dim=0) + up_proj_slices = self.up_proj.weight.split(slice, dim=0) + down_proj_slices = self.down_proj.weight.split(slice, dim=1) + + gate_proj = torch.cat( + [F.linear(x, gate_proj_slices[i]) for i in range(self.config.pretraining_tp)], dim=-1 + ) + up_proj = torch.cat([F.linear(x, up_proj_slices[i]) for i in range(self.config.pretraining_tp)], dim=-1) + + intermediate_states = (self.act_fn(gate_proj) * up_proj).split(slice, dim=2) + down_proj = [ + F.linear(intermediate_states[i], down_proj_slices[i]) for i in range(self.config.pretraining_tp) + ] + down_proj = sum(down_proj) + else: + down_proj = self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x)) + + return down_proj + + +def repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor: + """ + This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch, + num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim) + """ + batch, num_key_value_heads, slen, head_dim = hidden_states.shape + if n_rep == 1: + return hidden_states + hidden_states = hidden_states[:, :, None, :, :].expand(batch, num_key_value_heads, n_rep, slen, head_dim) + return hidden_states.reshape(batch, num_key_value_heads * n_rep, slen, head_dim) + + +class LlamaAttention(nn.Module): + """Multi-headed attention from 'Attention Is All You Need' paper""" + + def __init__(self, config: LlamaConfig, layer_idx: Optional[int] = None): + super().__init__() + self.config = config + self.layer_idx = layer_idx + if layer_idx is None: + logger.warning_once( + f"Instantiating {self.__class__.__name__} without passing a `layer_idx` is not recommended and will " + "lead to errors during the forward call if caching is used. Please make sure to provide a `layer_idx` " + "when creating this class." + ) + + self.attention_dropout = config.attention_dropout + self.hidden_size = config.hidden_size + self.num_heads = config.num_attention_heads + self.head_dim = getattr(config, "head_dim", self.hidden_size // self.num_heads) + self.num_key_value_heads = config.num_key_value_heads + self.num_key_value_groups = self.num_heads // self.num_key_value_heads + self.max_position_embeddings = config.max_position_embeddings + self.rope_theta = config.rope_theta + self.is_causal = True + + self.q_proj = LinearQuantizer(self.hidden_size, self.num_heads * self.head_dim, bias=config.attention_bias, quant_type="ternary", bit=4, group_size=-1) + self.k_proj = LinearQuantizer(self.hidden_size, self.num_key_value_heads * self.head_dim, bias=config.attention_bias, quant_type="ternary", bit=4, group_size=-1) + self.v_proj = LinearQuantizer(self.hidden_size, self.num_key_value_heads * self.head_dim, bias=config.attention_bias, quant_type="ternary", bit=4, group_size=-1) + self.o_proj = LinearQuantizer(self.num_heads * self.head_dim, self.hidden_size, bias=config.attention_bias, quant_type="ternary", bit=4, group_size=-1) + + # TODO (joao): remove in v4.46 (RoPE is computed in the model, not in the decoder layers) + self.rotary_emb = LlamaRotaryEmbedding(config=self.config) + + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_value: Optional[Cache] = None, + output_attentions: bool = False, + use_cache: bool = False, + cache_position: Optional[torch.LongTensor] = None, + position_embeddings: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, # will become mandatory in v4.46 + **kwargs, + ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]: + bsz, q_len, _ = hidden_states.size() + + if self.config.pretraining_tp > 1: + key_value_slicing = (self.num_key_value_heads * self.head_dim) // self.config.pretraining_tp + query_slices = self.q_proj.weight.split( + (self.num_heads * self.head_dim) // self.config.pretraining_tp, dim=0 + ) + key_slices = self.k_proj.weight.split(key_value_slicing, dim=0) + value_slices = self.v_proj.weight.split(key_value_slicing, dim=0) + + query_states = [F.linear(hidden_states, query_slices[i]) for i in range(self.config.pretraining_tp)] + query_states = torch.cat(query_states, dim=-1) + + key_states = [F.linear(hidden_states, key_slices[i]) for i in range(self.config.pretraining_tp)] + key_states = torch.cat(key_states, dim=-1) + + value_states = [F.linear(hidden_states, value_slices[i]) for i in range(self.config.pretraining_tp)] + value_states = torch.cat(value_states, dim=-1) + + else: + query_states = self.q_proj(hidden_states) + key_states = self.k_proj(hidden_states) + value_states = self.v_proj(hidden_states) + + query_states = query_states.view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2) + key_states = key_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2) + value_states = value_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2) + + if position_embeddings is None: + logger.warning_once( + "The attention layers in this model are transitioning from computing the RoPE embeddings internally " + "through `position_ids` (2D tensor with the indexes of the tokens), to using externally computed " + "`position_embeddings` (Tuple of tensors, containing cos and sin). In v4.46 `position_ids` will be " + "removed and `position_embeddings` will be mandatory." + ) + cos, sin = self.rotary_emb(value_states, position_ids) + else: + cos, sin = position_embeddings + query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) + + if past_key_value is not None: + # sin and cos are specific to RoPE models; cache_position needed for the static cache + cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} + key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx, cache_kwargs) + + key_states = repeat_kv(key_states, self.num_key_value_groups) + value_states = repeat_kv(value_states, self.num_key_value_groups) + attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) / math.sqrt(self.head_dim) + + if attention_mask is not None: # no matter the length, we just slice it + causal_mask = attention_mask[:, :, :, : key_states.shape[-2]] + attn_weights = attn_weights + causal_mask + + # upcast attention to fp32 + attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query_states.dtype) + attn_weights = nn.functional.dropout(attn_weights, p=self.attention_dropout, training=self.training) + attn_output = torch.matmul(attn_weights, value_states) + + if attn_output.size() != (bsz, self.num_heads, q_len, self.head_dim): + raise ValueError( + f"`attn_output` should be of size {(bsz, self.num_heads, q_len, self.head_dim)}, but is" + f" {attn_output.size()}" + ) + + attn_output = attn_output.transpose(1, 2).contiguous() + + attn_output = attn_output.reshape(bsz, q_len, -1) + + if self.config.pretraining_tp > 1: + attn_output = attn_output.split(self.hidden_size // self.config.pretraining_tp, dim=2) + o_proj_slices = self.o_proj.weight.split(self.hidden_size // self.config.pretraining_tp, dim=1) + attn_output = sum([F.linear(attn_output[i], o_proj_slices[i]) for i in range(self.config.pretraining_tp)]) + else: + attn_output = self.o_proj(attn_output) + + if not output_attentions: + attn_weights = None + + return attn_output, attn_weights, past_key_value + + +class LlamaFlashAttention2(LlamaAttention): + """ + Llama flash attention module. This module inherits from `LlamaAttention` as the weights of the module stays + untouched. The only required change would be on the forward pass where it needs to correctly call the public API of + flash attention and deal with padding tokens in case the input contains any of them. + """ + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + # TODO: Should be removed once Flash Attention for RoCm is bumped to 2.1. + # flash_attn<2.1 generates top-left aligned causal mask, while what is needed here is bottom-right alignement, that was made default for flash_attn>=2.1. This attribute is used to handle this difference. Reference: https://github.com/Dao-AILab/flash-attention/releases/tag/v2.1.0. + # Beware that with flash_attn<2.1, using q_seqlen != k_seqlen (except for the case q_seqlen == 1) produces a wrong mask (top-left). + self._flash_attn_uses_top_left_mask = not is_flash_attn_greater_or_equal_2_10() + + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: Optional[torch.LongTensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_value: Optional[Cache] = None, + output_attentions: bool = False, + use_cache: bool = False, + cache_position: Optional[torch.LongTensor] = None, + position_embeddings: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, # will become mandatory in v4.46 + ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]: + if isinstance(past_key_value, StaticCache): + raise ValueError( + "`static` cache implementation is not compatible with `attn_implementation==flash_attention_2` " + "make sure to use `sdpa` in the mean time, and open an issue at https://github.com/huggingface/transformers" + ) + + output_attentions = False + + bsz, q_len, _ = hidden_states.size() + + query_states = self.q_proj(hidden_states) + key_states = self.k_proj(hidden_states) + value_states = self.v_proj(hidden_states) + + # Flash attention requires the input to have the shape + # batch_size x seq_length x head_dim x hidden_dim + # therefore we just need to keep the original shape + query_states = query_states.view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2) + key_states = key_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2) + value_states = value_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2) + + if position_embeddings is None: + logger.warning_once( + "The attention layers in this model are transitioning from computing the RoPE embeddings internally " + "through `position_ids` (2D tensor with the indexes of the tokens), to using externally computed " + "`position_embeddings` (Tuple of tensors, containing cos and sin). In v4.46 `position_ids` will be " + "removed and `position_embeddings` will be mandatory." + ) + cos, sin = self.rotary_emb(value_states, position_ids) + else: + cos, sin = position_embeddings + query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) + + if past_key_value is not None: + # sin and cos are specific to RoPE models; cache_position needed for the static cache + cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} + key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx, cache_kwargs) + + # TODO: These transpose are quite inefficient but Flash Attention requires the layout [batch_size, sequence_length, num_heads, head_dim]. We would need to refactor the KV cache + # to be able to avoid many of these transpose/reshape/view. + query_states = query_states.transpose(1, 2) + key_states = key_states.transpose(1, 2) + value_states = value_states.transpose(1, 2) + + dropout_rate = self.attention_dropout if self.training else 0.0 + + # In PEFT, usually we cast the layer norms in float32 for training stability reasons + # therefore the input hidden states gets silently casted in float32. Hence, we need + # cast them back in the correct dtype just to be sure everything works as expected. + # This might slowdown training & inference so it is recommended to not cast the LayerNorms + # in fp32. (LlamaRMSNorm handles it correctly) + + input_dtype = query_states.dtype + if input_dtype == torch.float32: + if torch.is_autocast_enabled(): + target_dtype = torch.get_autocast_gpu_dtype() + # Handle the case where the model is quantized + elif hasattr(self.config, "_pre_quantization_dtype"): + target_dtype = self.config._pre_quantization_dtype + else: + target_dtype = self.q_proj.weight.dtype + + logger.warning_once( + f"The input hidden states seems to be silently casted in float32, this might be related to" + f" the fact you have upcasted embedding or layer norm layers in float32. We will cast back the input in" + f" {target_dtype}." + ) + + query_states = query_states.to(target_dtype) + key_states = key_states.to(target_dtype) + value_states = value_states.to(target_dtype) + + attn_output = _flash_attention_forward( + query_states, + key_states, + value_states, + attention_mask, + q_len, + position_ids=position_ids, + dropout=dropout_rate, + sliding_window=getattr(self, "sliding_window", None), + use_top_left_mask=self._flash_attn_uses_top_left_mask, + is_causal=self.is_causal, + ) + + attn_output = attn_output.reshape(bsz, q_len, -1).contiguous() + attn_output = self.o_proj(attn_output) + + if not output_attentions: + attn_weights = None + + return attn_output, attn_weights, past_key_value + + +class LlamaSdpaAttention(LlamaAttention): + """ + Llama attention module using torch.nn.functional.scaled_dot_product_attention. This module inherits from + `LlamaAttention` as the weights of the module stays untouched. The only changes are on the forward pass to adapt to + SDPA API. + """ + + # Adapted from LlamaAttention.forward + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_value: Optional[Cache] = None, + output_attentions: bool = False, + use_cache: bool = False, + cache_position: Optional[torch.LongTensor] = None, + position_embeddings: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, # will become mandatory in v4.46 + **kwargs, + ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]: + if output_attentions: + # TODO: Improve this warning with e.g. `model.config.attn_implementation = "manual"` once this is implemented. + logger.warning_once( + "LlamaModel is using LlamaSdpaAttention, but `torch.nn.functional.scaled_dot_product_attention` does not support `output_attentions=True`. Falling back to the manual attention implementation, " + 'but specifying the manual implementation will be required from Transformers version v5.0.0 onwards. This warning can be removed using the argument `attn_implementation="eager"` when loading the model.' + ) + return super().forward( + hidden_states=hidden_states, + attention_mask=attention_mask, + position_ids=position_ids, + past_key_value=past_key_value, + output_attentions=output_attentions, + use_cache=use_cache, + cache_position=cache_position, + position_embeddings=position_embeddings, + ) + + bsz, q_len, _ = hidden_states.size() + + query_states = self.q_proj(hidden_states) + key_states = self.k_proj(hidden_states) + value_states = self.v_proj(hidden_states) + + query_states = query_states.view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2) + key_states = key_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2) + value_states = value_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2) + + if position_embeddings is None: + logger.warning_once( + "The attention layers in this model are transitioning from computing the RoPE embeddings internally " + "through `position_ids` (2D tensor with the indexes of the tokens), to using externally computed " + "`position_embeddings` (Tuple of tensors, containing cos and sin). In v4.46 `position_ids` will be " + "removed and `position_embeddings` will be mandatory." + ) + cos, sin = self.rotary_emb(value_states, position_ids) + else: + cos, sin = position_embeddings + query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin) + + if past_key_value is not None: + # sin and cos are specific to RoPE models; cache_position needed for the static cache + cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} + key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx, cache_kwargs) + + key_states = repeat_kv(key_states, self.num_key_value_groups) + value_states = repeat_kv(value_states, self.num_key_value_groups) + + causal_mask = attention_mask + if attention_mask is not None: + causal_mask = causal_mask[:, :, :, : key_states.shape[-2]] + + # SDPA with memory-efficient backend is currently (torch==2.1.2) bugged with non-contiguous inputs with custom attn_mask, + # Reference: https://github.com/pytorch/pytorch/issues/112577. + if query_states.device.type == "cuda" and causal_mask is not None: + query_states = query_states.contiguous() + key_states = key_states.contiguous() + value_states = value_states.contiguous() + + # We dispatch to SDPA's Flash Attention or Efficient kernels via this `is_causal` if statement instead of an inline conditional assignment + # in SDPA to support both torch.compile's dynamic shapes and full graph options. An inline conditional prevents dynamic shapes from compiling. + is_causal = True if causal_mask is None and q_len > 1 else False + + attn_output = torch.nn.functional.scaled_dot_product_attention( + query_states, + key_states, + value_states, + attn_mask=causal_mask, + dropout_p=self.attention_dropout if self.training else 0.0, + is_causal=is_causal, + ) + + attn_output = attn_output.transpose(1, 2).contiguous() + attn_output = attn_output.view(bsz, q_len, -1) + + attn_output = self.o_proj(attn_output) + + return attn_output, None, past_key_value + + +LLAMA_ATTENTION_CLASSES = { + "eager": LlamaAttention, + "flash_attention_2": LlamaFlashAttention2, + "sdpa": LlamaSdpaAttention, +} + + +class LlamaDecoderLayer(nn.Module): + def __init__(self, config: LlamaConfig, layer_idx: int): + super().__init__() + self.hidden_size = config.hidden_size + + self.self_attn = LLAMA_ATTENTION_CLASSES[config._attn_implementation](config=config, layer_idx=layer_idx) + + self.mlp = LlamaMLP(config) + self.input_layernorm = LlamaRMSNorm(config.hidden_size, eps=config.rms_norm_eps) + self.post_attention_layernorm = LlamaRMSNorm(config.hidden_size, eps=config.rms_norm_eps) + + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_value: Optional[Cache] = None, + output_attentions: Optional[bool] = False, + use_cache: Optional[bool] = False, + cache_position: Optional[torch.LongTensor] = None, + position_embeddings: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, # will become mandatory in v4.46 + **kwargs, + ) -> Tuple[torch.FloatTensor, Optional[Tuple[torch.FloatTensor, torch.FloatTensor]]]: + """ + Args: + hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)` + attention_mask (`torch.FloatTensor`, *optional*): + attention mask of size `(batch_size, sequence_length)` if flash attention is used or `(batch_size, 1, + query_sequence_length, key_sequence_length)` if default attention is used. + output_attentions (`bool`, *optional*): + Whether or not to return the attentions tensors of all attention layers. See `attentions` under + returned tensors for more detail. + use_cache (`bool`, *optional*): + If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding + (see `past_key_values`). + past_key_value (`Tuple(torch.FloatTensor)`, *optional*): cached past key and value projection states + cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*): + Indices depicting the position of the input sequence tokens in the sequence + position_embeddings (`Tuple[torch.FloatTensor, torch.FloatTensor]`, *optional*): + Tuple containing the cosine and sine positional embeddings of shape `(batch_size, seq_len, head_dim)`, + with `head_dim` being the embedding dimension of each attention head. + kwargs (`dict`, *optional*): + Arbitrary kwargs to be ignored, used for FSDP and other methods that injects code + into the model + """ + residual = hidden_states + + hidden_states = self.input_layernorm(hidden_states) + + # Self Attention + hidden_states, self_attn_weights, present_key_value = self.self_attn( + hidden_states=hidden_states, + attention_mask=attention_mask, + position_ids=position_ids, + past_key_value=past_key_value, + output_attentions=output_attentions, + use_cache=use_cache, + cache_position=cache_position, + position_embeddings=position_embeddings, + **kwargs, + ) + hidden_states = residual + hidden_states + + # Fully Connected + residual = hidden_states + hidden_states = self.post_attention_layernorm(hidden_states) + hidden_states = self.mlp(hidden_states) + hidden_states = residual + hidden_states + + outputs = (hidden_states,) + + if output_attentions: + outputs += (self_attn_weights,) + + if use_cache: + outputs += (present_key_value,) + + return outputs + + +LLAMA_START_DOCSTRING = r""" + This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the + library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads + etc.) + + This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. + Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage + and behavior. + + Parameters: + config ([`LlamaConfig`]): + Model configuration class with all the parameters of the model. Initializing with a config file does not + load the weights associated with the model, only the configuration. Check out the + [`~PreTrainedModel.from_pretrained`] method to load the model weights. +""" + + +@add_start_docstrings( + "The bare LLaMA Model outputting raw hidden-states without any specific head on top.", + LLAMA_START_DOCSTRING, +) +class LlamaPreTrainedModel(PreTrainedModel): + config_class = LlamaConfig + base_model_prefix = "model" + supports_gradient_checkpointing = True + _no_split_modules = ["LlamaDecoderLayer"] + _skip_keys_device_placement = ["past_key_values"] + _supports_flash_attn_2 = True + _supports_sdpa = True + _supports_cache_class = True + _supports_quantized_cache = True + _supports_static_cache = True + + def _init_weights(self, module): + std = self.config.initializer_range + if isinstance(module, nn.Linear): + module.weight.data.normal_(mean=0.0, std=std) + if module.bias is not None: + module.bias.data.zero_() + elif isinstance(module, nn.Embedding): + module.weight.data.normal_(mean=0.0, std=std) + if module.padding_idx is not None: + module.weight.data[module.padding_idx].zero_() + + +LLAMA_INPUTS_DOCSTRING = r""" + Args: + input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): + Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide + it. + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) + attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): + Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + + [What are attention masks?](../glossary#attention-mask) + + Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + If `past_key_values` is used, optionally only the last `input_ids` have to be input (see + `past_key_values`). + + If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`] + and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more + information on the default strategy. + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, + config.n_positions - 1]`. + + [What are position IDs?](../glossary#position-ids) + past_key_values (`Cache` or `tuple(tuple(torch.FloatTensor))`, *optional*): + Pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention + blocks) that can be used to speed up sequential decoding. This typically consists in the `past_key_values` + returned by the model at a previous stage of decoding, when `use_cache=True` or `config.use_cache=True`. + + Two formats are allowed: + - a [`~cache_utils.Cache`] instance, see our + [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache); + - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of + shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`). This is also known as the legacy + cache format. + + The model will output the same cache format that is fed as input. If no `past_key_values` are passed, the + legacy cache format will be returned. + + If `past_key_values` are used, the user can optionally input only the last `input_ids` (those that don't + have their past key value states given to this model) of shape `(batch_size, 1)` instead of all `input_ids` + of shape `(batch_size, sequence_length)`. + inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): + Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This + is useful if you want more control over how to convert `input_ids` indices into associated vectors than the + model's internal embedding lookup matrix. + use_cache (`bool`, *optional*): + If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see + `past_key_values`). + output_attentions (`bool`, *optional*): + Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned + tensors for more detail. + output_hidden_states (`bool`, *optional*): + Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for + more detail. + return_dict (`bool`, *optional*): + Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. + cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*): + Indices depicting the position of the input sequence tokens in the sequence. Contrarily to `position_ids`, + this tensor is not affected by padding. It is used to update the cache in the correct position and to infer + the complete sequence length. +""" + + +@add_start_docstrings( + "The bare LLaMA Model outputting raw hidden-states without any specific head on top.", + LLAMA_START_DOCSTRING, +) +class LlamaModel(LlamaPreTrainedModel): + """ + Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`LlamaDecoderLayer`] + + Args: + config: LlamaConfig + """ + + def __init__(self, config: LlamaConfig): + super().__init__(config) + self.padding_idx = config.pad_token_id + self.vocab_size = config.vocab_size + + self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size, self.padding_idx) + self.layers = nn.ModuleList( + [LlamaDecoderLayer(config, layer_idx) for layer_idx in range(config.num_hidden_layers)] + ) + self.norm = LlamaRMSNorm(config.hidden_size, eps=config.rms_norm_eps) + self.rotary_emb = LlamaRotaryEmbedding(config=config) + self.gradient_checkpointing = False + + # Initialize weights and apply final processing + self.post_init() + + def get_input_embeddings(self): + return self.embed_tokens + + def set_input_embeddings(self, value): + self.embed_tokens = value + + @add_start_docstrings_to_model_forward(LLAMA_INPUTS_DOCSTRING) + def forward( + self, + input_ids: torch.LongTensor = None, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_values: Optional[Union[Cache, List[torch.FloatTensor]]] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + cache_position: Optional[torch.LongTensor] = None, + ) -> Union[Tuple, BaseModelOutputWithPast]: + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states + ) + use_cache = use_cache if use_cache is not None else self.config.use_cache + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + if (input_ids is None) ^ (inputs_embeds is not None): + raise ValueError("You must specify exactly one of input_ids or inputs_embeds") + + if self.gradient_checkpointing and self.training and use_cache: + logger.warning_once( + "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`." + ) + use_cache = False + + if inputs_embeds is None: + inputs_embeds = self.embed_tokens(input_ids) + + # kept for BC (non `Cache` `past_key_values` inputs) + return_legacy_cache = False + if use_cache and not isinstance(past_key_values, Cache): + return_legacy_cache = True + if past_key_values is None: + past_key_values = DynamicCache() + else: + past_key_values = DynamicCache.from_legacy_cache(past_key_values) + logger.warning_once( + "We detected that you are passing `past_key_values` as a tuple of tuples. This is deprecated and " + "will be removed in v4.47. Please convert your cache or use an appropriate `Cache` class " + "(https://huggingface.co/docs/transformers/kv_cache#legacy-cache-format)" + ) + + if cache_position is None: + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + cache_position = torch.arange( + past_seen_tokens, past_seen_tokens + inputs_embeds.shape[1], device=inputs_embeds.device + ) + if position_ids is None: + position_ids = cache_position.unsqueeze(0) + + causal_mask = self._update_causal_mask( + attention_mask, inputs_embeds, cache_position, past_key_values, output_attentions + ) + hidden_states = inputs_embeds + + # create position embeddings to be shared across the decoder layers + position_embeddings = self.rotary_emb(hidden_states, position_ids) + + # decoder layers + all_hidden_states = () if output_hidden_states else None + all_self_attns = () if output_attentions else None + next_decoder_cache = None + + for decoder_layer in self.layers: + if output_hidden_states: + all_hidden_states += (hidden_states,) + + if self.gradient_checkpointing and self.training: + layer_outputs = self._gradient_checkpointing_func( + decoder_layer.__call__, + hidden_states, + causal_mask, + position_ids, + past_key_values, + output_attentions, + use_cache, + cache_position, + position_embeddings, + ) + else: + layer_outputs = decoder_layer( + hidden_states, + attention_mask=causal_mask, + position_ids=position_ids, + past_key_value=past_key_values, + output_attentions=output_attentions, + use_cache=use_cache, + cache_position=cache_position, + position_embeddings=position_embeddings, + ) + + hidden_states = layer_outputs[0] + + if use_cache: + next_decoder_cache = layer_outputs[2 if output_attentions else 1] + + if output_attentions: + all_self_attns += (layer_outputs[1],) + + hidden_states = self.norm(hidden_states) + + # add hidden states from the last decoder layer + if output_hidden_states: + all_hidden_states += (hidden_states,) + + next_cache = next_decoder_cache if use_cache else None + if return_legacy_cache: + next_cache = next_cache.to_legacy_cache() + + if not return_dict: + return tuple(v for v in [hidden_states, next_cache, all_hidden_states, all_self_attns] if v is not None) + return BaseModelOutputWithPast( + last_hidden_state=hidden_states, + past_key_values=next_cache, + hidden_states=all_hidden_states, + attentions=all_self_attns, + ) + + def _update_causal_mask( + self, + attention_mask: torch.Tensor, + input_tensor: torch.Tensor, + cache_position: torch.Tensor, + past_key_values: Cache, + output_attentions: bool, + ): + if self.config._attn_implementation == "flash_attention_2": + if attention_mask is not None and 0.0 in attention_mask: + return attention_mask + return None + + # For SDPA, when possible, we will rely on its `is_causal` argument instead of its `attn_mask` argument, in + # order to dispatch on Flash Attention 2. This feature is not compatible with static cache, as SDPA will fail + # to infer the attention mask. + past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0 + using_static_cache = isinstance(past_key_values, StaticCache) + + # When output attentions is True, sdpa implementation's forward method calls the eager implementation's forward + if self.config._attn_implementation == "sdpa" and not using_static_cache and not output_attentions: + if AttentionMaskConverter._ignore_causal_mask_sdpa( + attention_mask, + inputs_embeds=input_tensor, + past_key_values_length=past_seen_tokens, + is_training=self.training, + ): + return None + + dtype, device = input_tensor.dtype, input_tensor.device + sequence_length = input_tensor.shape[1] + if using_static_cache: + target_length = past_key_values.get_max_cache_shape() + else: + target_length = ( + attention_mask.shape[-1] + if isinstance(attention_mask, torch.Tensor) + else past_seen_tokens + sequence_length + 1 + ) + + # In case the provided `attention` mask is 2D, we generate a causal mask here (4D). + causal_mask = self._prepare_4d_causal_attention_mask_with_cache_position( + attention_mask, + sequence_length=sequence_length, + target_length=target_length, + dtype=dtype, + device=device, + cache_position=cache_position, + batch_size=input_tensor.shape[0], + ) + + if ( + self.config._attn_implementation == "sdpa" + and attention_mask is not None + and attention_mask.device.type == "cuda" + and not output_attentions + ): + # Attend to all tokens in fully masked rows in the causal_mask, for example the relevant first rows when + # using left padding. This is required by F.scaled_dot_product_attention memory-efficient attention path. + # Details: https://github.com/pytorch/pytorch/issues/110213 + min_dtype = torch.finfo(dtype).min + causal_mask = AttentionMaskConverter._unmask_unattended(causal_mask, min_dtype) + + return causal_mask + + @staticmethod + def _prepare_4d_causal_attention_mask_with_cache_position( + attention_mask: torch.Tensor, + sequence_length: int, + target_length: int, + dtype: torch.dtype, + device: torch.device, + cache_position: torch.Tensor, + batch_size: int, + **kwargs, + ): + """ + Creates a causal 4D mask of shape `(batch_size, 1, query_length, key_value_length)` from a 2D mask of shape + `(batch_size, key_value_length)`, or if the input `attention_mask` is already 4D, do nothing. + + Args: + attention_mask (`torch.Tensor`): + A 2D attention mask of shape `(batch_size, key_value_length)` or a 4D attention mask of shape + `(batch_size, 1, query_length, key_value_length)`. + sequence_length (`int`): + The sequence length being processed. + target_length (`int`): + The target length: when generating with static cache, the mask should be as long as the static cache, + to account for the 0 padding, the part of the cache that is not filled yet. + dtype (`torch.dtype`): + The dtype to use for the 4D attention mask. + device (`torch.device`): + The device to plcae the 4D attention mask on. + cache_position (`torch.Tensor`): + Indices depicting the position of the input sequence tokens in the sequence. + batch_size (`torch.Tensor`): + Batch size. + """ + if attention_mask is not None and attention_mask.dim() == 4: + # In this case we assume that the mask comes already in inverted form and requires no inversion or slicing. + causal_mask = attention_mask + else: + min_dtype = torch.finfo(dtype).min + causal_mask = torch.full( + (sequence_length, target_length), fill_value=min_dtype, dtype=dtype, device=device + ) + if sequence_length != 1: + causal_mask = torch.triu(causal_mask, diagonal=1) + causal_mask *= torch.arange(target_length, device=device) > cache_position.reshape(-1, 1) + causal_mask = causal_mask[None, None, :, :].expand(batch_size, 1, -1, -1) + if attention_mask is not None: + causal_mask = causal_mask.clone() # copy to contiguous memory for in-place edit + mask_length = attention_mask.shape[-1] + padding_mask = causal_mask[:, :, :, :mask_length] + attention_mask[:, None, None, :] + padding_mask = padding_mask == 0 + causal_mask[:, :, :, :mask_length] = causal_mask[:, :, :, :mask_length].masked_fill( + padding_mask, min_dtype + ) + + return causal_mask + + +class LlamaForCausalLM(LlamaPreTrainedModel, GenerationMixin): + _tied_weights_keys = ["lm_head.weight"] + + def __init__(self, config): + super().__init__(config) + self.model = LlamaModel(config) + self.vocab_size = config.vocab_size + self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False) + + # Initialize weights and apply final processing + self.post_init() + + def get_input_embeddings(self): + return self.model.embed_tokens + + def set_input_embeddings(self, value): + self.model.embed_tokens = value + + def get_output_embeddings(self): + return self.lm_head + + def set_output_embeddings(self, new_embeddings): + self.lm_head = new_embeddings + + def set_decoder(self, decoder): + self.model = decoder + + def get_decoder(self): + return self.model + + @add_start_docstrings_to_model_forward(LLAMA_INPUTS_DOCSTRING) + @replace_return_docstrings(output_type=CausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC) + def forward( + self, + input_ids: torch.LongTensor = None, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_values: Optional[Union[Cache, List[torch.FloatTensor]]] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + labels: Optional[torch.LongTensor] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + cache_position: Optional[torch.LongTensor] = None, + num_logits_to_keep: int = 0, + **loss_kwargs, + ) -> Union[Tuple, CausalLMOutputWithPast]: + r""" + Args: + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. + + num_logits_to_keep (`int`, *optional*): + Calculate logits for the last `num_logits_to_keep` tokens. If `0`, calculate logits for all + `input_ids` (special case). Only last token logits are needed for generation, and calculating them only for that + token can save memory, which becomes pretty significant for long sequences or large vocabulary size. + + Returns: + + Example: + + ```python + >>> from transformers import AutoTokenizer, LlamaForCausalLM + + >>> model = LlamaForCausalLM.from_pretrained("meta-llama/Llama-2-7b-hf") + >>> tokenizer = AutoTokenizer.from_pretrained("meta-llama/Llama-2-7b-hf") + + >>> prompt = "Hey, are you conscious? Can you talk to me?" + >>> inputs = tokenizer(prompt, return_tensors="pt") + + >>> # Generate + >>> generate_ids = model.generate(inputs.input_ids, max_length=30) + >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0] + "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you." + ```""" + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states + ) + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + # decoder outputs consists of (dec_features, layer_state, dec_hidden, dec_attn) + outputs = self.model( + input_ids=input_ids, + attention_mask=attention_mask, + position_ids=position_ids, + past_key_values=past_key_values, + inputs_embeds=inputs_embeds, + use_cache=use_cache, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + cache_position=cache_position, + ) + + hidden_states = outputs[0] + if self.config.pretraining_tp > 1: + lm_head_slices = self.lm_head.weight.split(self.vocab_size // self.config.pretraining_tp, dim=0) + logits = [F.linear(hidden_states, lm_head_slices[i]) for i in range(self.config.pretraining_tp)] + logits = torch.cat(logits, dim=-1) + else: + # Only compute necessary logits, and do not upcast them to float if we are not computing the loss + logits = self.lm_head(hidden_states[:, -num_logits_to_keep:, :]) + + loss = None + if labels is not None: + loss = self.loss_function(logits=logits, labels=labels, vocab_size=self.config.vocab_size, **loss_kwargs) + + if not return_dict: + output = (logits,) + outputs[1:] + return (loss,) + output if loss is not None else output + + return CausalLMOutputWithPast( + loss=loss, + logits=logits, + past_key_values=outputs.past_key_values, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + ) + + +@add_start_docstrings( + """ + The LLaMa Model transformer with a sequence classification head on top (linear layer). + + [`LlamaForSequenceClassification`] uses the last token in order to do the classification, as other causal models + (e.g. GPT-2) do. + + Since it does classification on the last token, it requires to know the position of the last token. If a + `pad_token_id` is defined in the configuration, it finds the last token that is not a padding token in each row. If + no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot guess the + padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take the last value in + each row of the batch). + """, + LLAMA_START_DOCSTRING, +) +class LlamaForSequenceClassification(LlamaPreTrainedModel): + def __init__(self, config): + super().__init__(config) + self.num_labels = config.num_labels + self.model = LlamaModel(config) + self.score = nn.Linear(config.hidden_size, self.num_labels, bias=False) + + # Initialize weights and apply final processing + self.post_init() + + def get_input_embeddings(self): + return self.model.embed_tokens + + def set_input_embeddings(self, value): + self.model.embed_tokens = value + + @add_start_docstrings_to_model_forward(LLAMA_INPUTS_DOCSTRING) + def forward( + self, + input_ids: Optional[torch.LongTensor] = None, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_values: Optional[Union[Cache, List[torch.FloatTensor]]] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + labels: Optional[torch.LongTensor] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + ) -> Union[Tuple, SequenceClassifierOutputWithPast]: + r""" + labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): + Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., + config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If + `config.num_labels > 1` a classification loss is computed (Cross-Entropy). + """ + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + transformer_outputs = self.model( + input_ids, + attention_mask=attention_mask, + position_ids=position_ids, + past_key_values=past_key_values, + inputs_embeds=inputs_embeds, + use_cache=use_cache, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + hidden_states = transformer_outputs[0] + logits = self.score(hidden_states) + + if input_ids is not None: + batch_size = input_ids.shape[0] + else: + batch_size = inputs_embeds.shape[0] + + if self.config.pad_token_id is None and batch_size != 1: + raise ValueError("Cannot handle batch sizes > 1 if no padding token is defined.") + if self.config.pad_token_id is None: + sequence_lengths = -1 + else: + if input_ids is not None: + # if no pad token found, use modulo instead of reverse indexing for ONNX compatibility + sequence_lengths = torch.eq(input_ids, self.config.pad_token_id).int().argmax(-1) - 1 + sequence_lengths = sequence_lengths % input_ids.shape[-1] + sequence_lengths = sequence_lengths.to(logits.device) + else: + sequence_lengths = -1 + + pooled_logits = logits[torch.arange(batch_size, device=logits.device), sequence_lengths] + + loss = None + if labels is not None: + loss = self.loss_function(logits=logits, labels=labels, pooled_logits=pooled_logits, config=self.config) + + if not return_dict: + output = (pooled_logits,) + transformer_outputs[1:] + return ((loss,) + output) if loss is not None else output + + return SequenceClassifierOutputWithPast( + loss=loss, + logits=pooled_logits, + past_key_values=transformer_outputs.past_key_values, + hidden_states=transformer_outputs.hidden_states, + attentions=transformer_outputs.attentions, + ) + + +@add_start_docstrings( + """ +The Llama Model transformer with a span classification head on top for extractive question-answering tasks like +SQuAD (a linear layer on top of the hidden-states output to compute `span start logits` and `span end logits`). + """, + LLAMA_START_DOCSTRING, +) +class LlamaForQuestionAnswering(LlamaPreTrainedModel): + base_model_prefix = "transformer" + + # Copied from transformers.models.bloom.modeling_bloom.BloomForQuestionAnswering.__init__ with Bloom->Llama + def __init__(self, config): + super().__init__(config) + self.transformer = LlamaModel(config) + self.qa_outputs = nn.Linear(config.hidden_size, 2) + + # Initialize weights and apply final processing + self.post_init() + + def get_input_embeddings(self): + return self.transformer.embed_tokens + + def set_input_embeddings(self, value): + self.transformer.embed_tokens = value + + @add_start_docstrings_to_model_forward(LLAMA_INPUTS_DOCSTRING) + def forward( + self, + input_ids: Optional[torch.LongTensor] = None, + attention_mask: Optional[torch.FloatTensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_values: Optional[Union[Cache, List[torch.FloatTensor]]] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + start_positions: Optional[torch.LongTensor] = None, + end_positions: Optional[torch.LongTensor] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + **kwargs, + ) -> Union[Tuple, QuestionAnsweringModelOutput]: + r""" + start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): + Labels for position (index) of the start of the labelled span for computing the token classification loss. + Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence + are not taken into account for computing the loss. + end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): + Labels for position (index) of the end of the labelled span for computing the token classification loss. + Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence + are not taken into account for computing the loss. + """ + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + outputs = self.transformer( + input_ids, + attention_mask=attention_mask, + position_ids=position_ids, + past_key_values=past_key_values, + inputs_embeds=inputs_embeds, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + + sequence_output = outputs[0] + + logits = self.qa_outputs(sequence_output) + start_logits, end_logits = logits.split(1, dim=-1) + start_logits = start_logits.squeeze(-1).contiguous() + end_logits = end_logits.squeeze(-1).contiguous() + + loss = None + if start_positions is not None and end_positions is not None: + loss = self.loss_function(start_logits, end_logits, start_positions, end_positions, **kwargs) + + if not return_dict: + output = (start_logits, end_logits) + outputs[2:] + return ((loss,) + output) if loss is not None else output + + return QuestionAnsweringModelOutput( + loss=loss, + start_logits=start_logits, + end_logits=end_logits, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + ) + + +@add_start_docstrings( + """ + The Llama Model transformer with a token classification head on top (a linear layer on top of the hidden-states + output) e.g. for Named-Entity-Recognition (NER) tasks. + """, + LLAMA_START_DOCSTRING, +) +class LlamaForTokenClassification(LlamaPreTrainedModel): + def __init__(self, config): + super().__init__(config) + self.num_labels = config.num_labels + self.model = LlamaModel(config) + if getattr(config, "classifier_dropout", None) is not None: + classifier_dropout = config.classifier_dropout + elif getattr(config, "hidden_dropout", None) is not None: + classifier_dropout = config.hidden_dropout + else: + classifier_dropout = 0.1 + self.dropout = nn.Dropout(classifier_dropout) + self.score = nn.Linear(config.hidden_size, config.num_labels) + + # Initialize weights and apply final processing + self.post_init() + + def get_input_embeddings(self): + return self.model.embed_tokens + + def set_input_embeddings(self, value): + self.model.embed_tokens = value + + @add_start_docstrings_to_model_forward(LLAMA_INPUTS_DOCSTRING) + @add_code_sample_docstrings( + checkpoint=_CHECKPOINT_FOR_DOC, + output_type=TokenClassifierOutput, + config_class=_CONFIG_FOR_DOC, + ) + def forward( + self, + input_ids: Optional[torch.LongTensor] = None, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_values: Optional[List[torch.FloatTensor]] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + labels: Optional[torch.LongTensor] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + ) -> Union[Tuple, TokenClassifierOutput]: + r""" + labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): + Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., + config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If + `config.num_labels > 1` a classification loss is computed (Cross-Entropy). + """ + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + outputs = self.model( + input_ids, + attention_mask=attention_mask, + position_ids=position_ids, + past_key_values=past_key_values, + inputs_embeds=inputs_embeds, + use_cache=use_cache, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + sequence_output = outputs[0] + sequence_output = self.dropout(sequence_output) + logits = self.score(sequence_output) + + loss = None + if labels is not None: + loss = self.loss_function(logits, labels, self.config) + + if not return_dict: + output = (logits,) + outputs[2:] + return ((loss,) + output) if loss is not None else output + + return TokenClassifierOutput( + loss=loss, + logits=logits, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + ) diff --git a/pytorch_model.bin b/pytorch_model.bin new file mode 100644 index 0000000..64d1553 --- /dev/null +++ b/pytorch_model.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5ad186deeb7a0b84bcd58a5b9ad90baa300c719cf363b215dc999f9d11ef031f +size 3244413702 diff --git a/qat-convert.py b/qat-convert.py new file mode 100644 index 0000000..a23cca6 --- /dev/null +++ b/qat-convert.py @@ -0,0 +1,176 @@ +import torch +import torch.nn as nn +from tqdm import tqdm +import os +import safetensors + +class SteTernaryQuantizer(nn.Module): + def __init__(self, group_size): + super().__init__() + self.group_size = group_size + + def forward(self, x): + org_w_shape = x.shape + if self.group_size > 0: + assert x.shape[-1] % self.group_size == 0 + x = x.reshape(-1, self.group_size) + elif self.group_size == -1: + x = x.reshape(-1, x.shape[-1]) + assert x.dim() == 2 + scales = 1.0 / (x.abs().mean(dim=1, keepdim=True).clamp_(min=1e-5)) + x_q = (torch.clamp(torch.round(x * scales),-1,1) / scales) + assert torch.isnan(x_q).sum() == 0 + x = x.reshape(org_w_shape) + x_q = x_q.reshape(org_w_shape) + return x_q + +class SteIntQuantizer(nn.Module): + def __init__(self, bit, group_size): + super().__init__() + self.bit = bit + self.group_size = group_size + + def forward(self, x): + org_w_shape = x.shape + if self.group_size > 0: + assert org_w_shape[-1] % self.group_size == 0 + x = x.reshape(-1, self.group_size) + elif self.group_size == -1: + x = x.reshape(-1, x.shape[-1]) + + assert x.dim() == 2 + + abs_max_val = x.abs().amax(dim=1, keepdim=True) + max_int = 2 ** (self.bit - 1) - 1 + min_int = - (2 ** (self.bit - 1)) + scales = abs_max_val.clamp(min=1e-5) / max_int + + assert torch.isnan(scales).sum() == 0 + + x_q = (torch.clamp(torch.round(x / scales), min_int, max_int)) * scales + + assert torch.isnan(x_q).sum() == 0 + + x = x.reshape(org_w_shape) + x_q = x_q.reshape(org_w_shape) + + return x_q + +class SteInt2Quantizer(nn.Module): + def __init__(self, group_size): + super().__init__() + self.group_size = group_size + + def forward(self, x): + org_w_shape = x.shape + if self.group_size > 0: + assert x.shape[-1] % self.group_size == 0 + x = x.reshape(-1, self.group_size) + elif self.group_size == -1: + x = x.reshape(-1, x.shape[-1]) + + assert x.dim() == 2 + + scales = 1.0 / (x.abs().mean(dim=1, keepdim=True).clamp_(min=1e-5) * 1) + x_q = (torch.clamp(torch.round(x * scales),-2,1) / scales) + + assert torch.isnan(x_q).sum() == 0 + + x = x.reshape(org_w_shape) + x_q = x_q.reshape(org_w_shape) + + return x_q + +def quantize_model_bin(input_bin_path, output_bin_path, quant_type="ternary", bit=2, group_size=128, device="cuda" if torch.cuda.is_available() else "cpu"): + """ + ็›ดๆŽฅๅฏนPyTorchๆจกๅž‹binๆ–‡ไปถ่ฟ›่กŒ้‡ๅŒ–ใ€‚ + + Args: + input_bin_path: ่พ“ๅ…ฅๆจกๅž‹binๆ–‡ไปถ่ทฏๅพ„ + output_bin_path: ่พ“ๅ‡บ้‡ๅŒ–ๅŽ็š„ๆจกๅž‹binๆ–‡ไปถ่ทฏๅพ„ + quant_type: ้‡ๅŒ–็ฑปๅž‹ ("ternary" ๆˆ– "int") + bit: ๆ•ดๆ•ฐ้‡ๅŒ–็š„ไฝๆ•ฐ (ไป…ๅœจ quant_type="int" ๆ—ถไฝฟ็”จ) + group_size: ้‡ๅŒ–ๅˆ†็ป„ๅคงๅฐ + device: ่ฟ่กŒ่ฎพๅค‡ + """ + print(f"ๅŠ ่ฝฝๆจกๅž‹ๆ–‡ไปถ: {input_bin_path}...") + if input_bin_path.endswith(".bin"): + state_dict = torch.load(input_bin_path, map_location=device) + elif input_bin_path.endswith(".safetensors"): + state_dict = safetensors.load_file(input_bin_path) + elif os.path.isdir(input_bin_path) and os.path.exists(os.path.join(input_bin_path, "pytorch_model.bin")): + state_dict = torch.load(os.path.join(input_bin_path, "pytorch_model.bin"), map_location=device) + elif os.path.isdir(input_bin_path) and os.path.exists(os.path.join(input_bin_path, "model.safetensors")): + state_dict = safetensors.load_file(os.path.join(input_bin_path, "model.safetensors")) + else: + raise ValueError(f"ไธๆ”ฏๆŒ็š„ๆจกๅž‹ๆ–‡ไปถ็ฑปๅž‹: {input_bin_path}") + + print(f"ๅบ”็”จ {quant_type} ้‡ๅŒ–...") + if quant_type == "ternary": + quantizer = SteTernaryQuantizer(group_size=group_size) + elif quant_type == "int": + quantizer = SteIntQuantizer(bit=bit, group_size=group_size) + elif quant_type == "int2": + quantizer = SteInt2Quantizer(group_size=group_size) + else: + raise ValueError(f"ไธๆ”ฏๆŒ็š„้‡ๅŒ–็ฑปๅž‹: {quant_type}") + + # ็ปŸ่ฎก้œ€่ฆ้‡ๅŒ–็š„ๅ‚ๆ•ฐๆ•ฐ้‡ + total_params = sum(1 for k, v in state_dict.items() if ("weight" in k and "layer" in k) or ("fc" in k)) + + # ๅบ”็”จ้‡ๅŒ– + with torch.no_grad(): + for name, param in tqdm(state_dict.items(), total=total_params, desc="้‡ๅŒ–ไธญ"): + if (("weight" in name and "layer" in name and param.dim() == 2) or ("fc" in name and param.dim() == 2)): + # ๅฏนๆƒ้‡่ฟ›่กŒ้‡ๅŒ– + original_weight = param.data.clone() + quantized_weight = quantizer(original_weight) + state_dict[name] = quantized_weight + + # ๆ‰“ๅฐๅ‰ๅ‡ ไธชๅฑ‚็š„็ปŸ่ฎกไฟกๆฏ + if total_params > 0: + total_params -= 1 + if total_params > total_params - 5: + print(f"ๅฑ‚: {name}") + print(f" ๅŽŸๅง‹่Œƒๅ›ด: {original_weight.min():.4f} ๅˆฐ {original_weight.max():.4f}") + print(f" ้‡ๅŒ–ๅŽ่Œƒๅ›ด: {quantized_weight.min():.4f} ๅˆฐ {quantized_weight.max():.4f}") + print(f" ๅ‡ๆ–น่ฏฏๅทฎ: {((original_weight - quantized_weight)**2).mean():.8f}") + + # ไฟๅญ˜้‡ๅŒ–ๅŽ็š„ๆจกๅž‹ + print(f"ไฟๅญ˜้‡ๅŒ–ๅŽ็š„ๆจกๅž‹ๅˆฐ: {output_bin_path}...") + if output_bin_path.endswith(".bin"): + torch.save(state_dict, output_bin_path) + elif output_bin_path.endswith(".safetensors"): + safetensors.save_file(state_dict, output_bin_path) + else: + os.makedirs(os.path.dirname(output_bin_path), exist_ok=True) + output_bin_path = os.path.join(output_bin_path, "pytorch_model.bin") + torch.save(state_dict, output_bin_path) + print("ๅฎŒๆˆ!") + +def main(): + import argparse + parser = argparse.ArgumentParser(description="้‡ๅŒ–PyTorchๆจกๅž‹binๆ–‡ไปถ") + parser.add_argument("--input_bin", type=str, required=True, help="่พ“ๅ…ฅๆจกๅž‹binๆ–‡ไปถ่ทฏๅพ„") + parser.add_argument("--output", type=str, required=True, help="่พ“ๅ‡บ้‡ๅŒ–ๅŽ็š„ๆจกๅž‹binๆ–‡ไปถ่ทฏๅพ„") + parser.add_argument("--quant_type", type=str, default="ternary", choices=["ternary", "int", "int2"], help="้‡ๅŒ–็ฑปๅž‹") + parser.add_argument("--bit", type=int, default=2, help="ๆ•ดๆ•ฐ้‡ๅŒ–็š„ไฝๆ•ฐ") + parser.add_argument("--group_size", type=int, default=-1, help="้‡ๅŒ–ๅˆ†็ป„ๅคงๅฐ") + parser.add_argument("--device", type=str, default="cuda" if torch.cuda.is_available() else "cpu", help="่ฟ่กŒ่ฎพๅค‡") + parser.add_argument("--config", type=str, default="", help="model config file") + + args = parser.parse_args() + os.makedirs(args.output, exist_ok=True) + quantize_model_bin( + input_bin_path=args.input_bin, + output_bin_path=os.path.join(args.output, "pytorch_model.bin"), + quant_type=args.quant_type, + bit=args.bit, + group_size=args.group_size, + device=args.device + ) + if args.config: + os.system(f"cp {args.config}/* {args.output}") + print(f"ๅคๅˆถ{args.config}ๆ–‡ไปถๅˆฐ{args.output}") +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/special_tokens_map.json b/special_tokens_map.json new file mode 100644 index 0000000..8619dda --- /dev/null +++ b/special_tokens_map.json @@ -0,0 +1,81 @@ +{ + "additional_special_tokens": [ + { + "content": "<|im_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + { + "content": "<|im_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + { + "content": "<|tool_call|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + { + "content": "<|execute_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + { + "content": "<|execute_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + { + "content": "<|fim_prefix|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + { + "content": "<|fim_middle|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + { + "content": "<|fim_suffix|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + } + ], + "bos_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "unk_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + } +} diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..ef897dc --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b00802b71a613e3f7df3899fe9643a3ff949736d333a2b892448a974383fe372 +size 3676758 diff --git a/tokenizer.model b/tokenizer.model new file mode 100644 index 0000000..3acef16 --- /dev/null +++ b/tokenizer.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bb74d51116831c3bf65db812c553f94ab0c88dcf97a5bbb37e3504f6d359c530 +size 1181204 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..92c77dc --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,116 @@ +{ + "add_bos_token": true, + "add_eos_token": false, + "added_tokens_decoder": { + "0": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "1": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "2": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "73440": { + "content": "<|im_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "73441": { + "content": "<|im_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "73442": { + "content": "<|tool_call|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "73443": { + "content": "<|execute_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "73444": { + "content": "<|execute_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "73445": { + "content": "<|fim_prefix|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "73446": { + "content": "<|fim_middle|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "73447": { + "content": "<|fim_suffix|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + } + }, + "additional_special_tokens": [ + "<|im_end|>", + "<|im_start|>", + "<|tool_call|>", + "<|execute_start|>", + "<|execute_end|>", + "<|fim_prefix|>", + "<|fim_middle|>", + "<|fim_suffix|>" + ], + "bos_token": "", + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "legacy": true, + "model_max_length": 1000000000000000019884624838656, + "pad_token": null, + "sp_model_kwargs": {}, + "spaces_between_special_tokens": false, + "tokenizer_class": "LlamaTokenizer", + "unk_token": "", + "use_default_system_prompt": false, + "chat_template": "{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}" +}