From 2c13ce153c5f57248506591a615e77a84b5b379e Mon Sep 17 00:00:00 2001 From: ModelHub XC Date: Tue, 21 Jul 2026 15:26:10 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=EF=BC=8C=E7=94=B1ModelHub=20XC=E7=A4=BE=E5=8C=BA=E6=8F=90?= =?UTF-8?q?=E4=BE=9B=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model: MohdNihal03/qwen2.5-coder-1.5b-CodeSLM-Nihal Source: Original Platform --- .gitattributes | 38 + README.md | 106 +++ adapter/README.md | 62 ++ adapter/adapter_config.json | 48 ++ adapter/adapter_model.safetensors | 3 + adapter/chat_template.jinja | 59 ++ adapter/tokenizer.json | 3 + adapter/tokenizer_config.json | 30 + adapter/training_args.bin | 3 + adapter/training_log.json | 1279 +++++++++++++++++++++++++++++ chat_template.jinja | 54 ++ config.json | 61 ++ generation_config.json | 14 + loss_curve.png | 3 + model.safetensors | 3 + token_reduction.png | Bin 0 -> 43800 bytes tokenizer.json | 3 + tokenizer_config.json | 30 + 18 files changed, 1799 insertions(+) create mode 100644 .gitattributes create mode 100644 README.md create mode 100644 adapter/README.md create mode 100644 adapter/adapter_config.json create mode 100644 adapter/adapter_model.safetensors create mode 100644 adapter/chat_template.jinja create mode 100644 adapter/tokenizer.json create mode 100644 adapter/tokenizer_config.json create mode 100644 adapter/training_args.bin create mode 100644 adapter/training_log.json create mode 100644 chat_template.jinja create mode 100644 config.json create mode 100644 generation_config.json create mode 100644 loss_curve.png create mode 100644 model.safetensors create mode 100644 token_reduction.png create mode 100644 tokenizer.json create mode 100644 tokenizer_config.json diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..f82056b --- /dev/null +++ b/.gitattributes @@ -0,0 +1,38 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +loss_curve.png filter=lfs diff=lfs merge=lfs -text +adapter/tokenizer.json filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..27c556e --- /dev/null +++ b/README.md @@ -0,0 +1,106 @@ +--- +license: apache-2.0 +base_model: Qwen/Qwen2.5-Coder-1.5B-Instruct +tags: + - code + - python + - qlora + - lora + - peft + - qwen2 + - text-generation +datasets: + - sahil2801/CodeAlpaca-20k +language: + - en +pipeline_tag: text-generation +library_name: transformers +--- + +# qwen2.5-coder-1.5b-CodeSLM-Nihal + +A **QLoRA fine-tune of Qwen2.5-Coder-1.5B-Instruct** into a **terse, code-first Python assistant**, +built as a hands-on, phase-by-phase fine-tuning learning project on a single 8 GB consumer GPU +(NVIDIA RTX 5060). + +The base model is a capable coder but chronically verbose — every answer trails a prose essay. This +fine-tune trains it to reply with **correct, minimal code and nothing else.** + +## Headline result + +![Output length: 78% shorter after fine-tuning](./token_reduction.png) + +On 18 held-out prompts (greedy decoding), average output length dropped **from 272 to 60 tokens +(−78%)** while core algorithm correctness was preserved. + +## Training curve + +![Training vs. validation loss](./loss_curve.png) + +| Checkpoint (epoch) | Validation loss | +|---|---| +| 0.13 | 0.5157 | +| 0.38 | 0.5020 | +| 0.88 | 0.4927 | +| 1.00 | 0.4928 | + +Train and validation loss tracked each other the entire run — clean convergence, **no overfitting.** +Final train loss **0.4906**; validation mean token-accuracy **~85.4%**. + +> These are training/next-token metrics plus a qualitative 18-prompt comparison. No standardized code +> benchmark (HumanEval/MBPP) was run. + +## Training details + +| | | +|---|---| +| Base | `Qwen/Qwen2.5-Coder-1.5B-Instruct` | +| Method | QLoRA (4-bit NF4 base + LoRA), completion-only loss | +| Dataset | `sahil2801/CodeAlpaca-20k` → Qwen ChatML (19,020 train / 1,002 val) | +| LoRA | r=16, α=32, dropout=0.05, targets q/k/v/o/gate/up/down | +| Trainable params | 18.46M (~1.18%) | +| Schedule | 1 epoch, 1,189 steps, LR 2e-4 cosine + 3% warmup | +| Batch | 2 × 8 grad-accum = effective 16 | +| Optimizer | paged_adamw_8bit, bf16, gradient checkpointing, max_len 1024 | +| Hardware | 1× RTX 5060 (8 GB), ~50 min, peak VRAM 3.32 GB | + +## Files in this repo + +- **Root** — merged fp16 model (Transformers format); load directly with `from_pretrained`. +- **`adapter/`** — the standalone LoRA adapter (~36 MB) to apply onto the base yourself. +- **`gguf/`** — `…-f16.gguf` (full precision) and `…-Q4_K_M.gguf` (~986 MB) for llama.cpp / Ollama. + +## Usage + +### Transformers +```python +from transformers import AutoModelForCausalLM, AutoTokenizer +m = "MohdNihal03/qwen2.5-coder-1.5b-CodeSLM-Nihal" +tok = AutoTokenizer.from_pretrained(m) +model = AutoModelForCausalLM.from_pretrained(m, device_map="auto") +msgs = [{"role": "user", "content": "Write a Python function that reverses a string without slicing."}] +ids = tok.apply_chat_template(msgs, add_generation_prompt=True, return_tensors="pt").to(model.device) +print(tok.decode(model.generate(ids, max_new_tokens=128)[0][ids.shape[1]:], skip_special_tokens=True)) +``` + +### Ollama +``` +ollama run mohdnihalll03/qwen2.5-coder-1.5b-codeslm-nihal +``` + +Prompt format: Qwen2.5 ChatML. Suggested: `temperature 0.2`, stop on `<|im_start|>` / `<|im_end|>`. + +## Limitations + +- **Style over correctness:** fine-tuning changed formatting far more than correctness. A few subtle + base-model bugs persist (an email-regex character-class quirk; a `@timer` decorator missing + `functools.wraps`), and one bracket-matching answer regressed to a logic bug on empty-stack input. + **Review generated code before use.** +- Python-focused; 1.5B params + 4-bit quantization — not a substitute for a large frontier model. +- Occasional instruction drift (`print` vs `return`, tabulation vs memoization). + +## Attribution + +- Base: Qwen2.5-Coder-1.5B-Instruct (© Alibaba Cloud, Apache-2.0) +- Data: `sahil2801/CodeAlpaca-20k` +- Fine-tuned by **Nihal** as a QLoRA learning project (TRL / PEFT / bitsandbytes). diff --git a/adapter/README.md b/adapter/README.md new file mode 100644 index 0000000..6baa0c5 --- /dev/null +++ b/adapter/README.md @@ -0,0 +1,62 @@ +--- +base_model: Qwen/Qwen2.5-Coder-1.5B-Instruct +library_name: peft +model_name: lora-adapter +tags: +- base_model:adapter:Qwen/Qwen2.5-Coder-1.5B-Instruct +- lora +- sft +- transformers +- trl +licence: license +pipeline_tag: text-generation +--- + +# Model Card for lora-adapter + +This model is a fine-tuned version of [Qwen/Qwen2.5-Coder-1.5B-Instruct](https://huggingface.co/Qwen/Qwen2.5-Coder-1.5B-Instruct). +It has been trained using [TRL](https://github.com/huggingface/trl). + +## Quick start + +```python +from transformers import pipeline + +question = "If you had a time machine, but could only go to the past or the future once and never return, which would you choose and why?" +generator = pipeline("text-generation", model="None", device="cuda") +output = generator([{"role": "user", "content": question}], max_new_tokens=128, return_full_text=False)[0] +print(output["generated_text"]) +``` + +## Training procedure + + + + + +This model was trained with SFT. + +### Framework versions + +- PEFT 0.19.1 +- TRL: 1.7.0 +- Transformers: 5.12.1 +- Pytorch: 2.11.0+cu128 +- Datasets: 5.0.0 +- Tokenizers: 0.22.2 + +## Citations + + + +Cite TRL as: + +```bibtex +@software{vonwerra2020trl, + title = {{TRL: Transformers Reinforcement Learning}}, + author = {von Werra, Leandro and Belkada, Younes and Tunstall, Lewis and Beeching, Edward and Thrush, Tristan and Lambert, Nathan and Huang, Shengyi and Rasul, Kashif and Gallouédec, Quentin}, + license = {Apache-2.0}, + url = {https://github.com/huggingface/trl}, + year = {2020} +} +``` \ No newline at end of file diff --git a/adapter/adapter_config.json b/adapter/adapter_config.json new file mode 100644 index 0000000..cb2cb51 --- /dev/null +++ b/adapter/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen2.5-Coder-1.5B-Instruct", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 32, + "lora_bias": false, + "lora_dropout": 0.05, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 16, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "up_proj", + "v_proj", + "gate_proj", + "down_proj", + "q_proj", + "o_proj", + "k_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/adapter/adapter_model.safetensors b/adapter/adapter_model.safetensors new file mode 100644 index 0000000..804abd6 --- /dev/null +++ b/adapter/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dc1c086d050c1087e0f503123de03e7ba81afca44876da8596a5abb91ec7a29b +size 36981856 diff --git a/adapter/chat_template.jinja b/adapter/chat_template.jinja new file mode 100644 index 0000000..64e3784 --- /dev/null +++ b/adapter/chat_template.jinja @@ -0,0 +1,59 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0]['role'] == 'system' %} + {{- messages[0]['content'] }} + {%- else %} + {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }} + {%- endif %} + {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0]['role'] == 'system' %} + {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }} + {%- else %} + {{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif (message.role == "assistant" and not message.tool_calls) %} + {{- '<|im_start|>' + message.role + '\n' }} + {%- generation %} + {{- message.content + '<|im_end|>' + '\n' }} + {%- endgeneration %} + {%- elif message.role == "assistant" %} + {{- '<|im_start|>' + message.role }} + {%- if message.content %} + {{- '\n' + message.content }} + {%- endif %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {{- tool_call.arguments | tojson }} + {{- '}\n' }} + {%- endfor %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} +{%- endif %} diff --git a/adapter/tokenizer.json b/adapter/tokenizer.json new file mode 100644 index 0000000..34510ff --- /dev/null +++ b/adapter/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3fd169731d2cbde95e10bf356d66d5997fd885dd8dbb6fb4684da3f23b2585d8 +size 11421892 diff --git a/adapter/tokenizer_config.json b/adapter/tokenizer_config.json new file mode 100644 index 0000000..6028fd5 --- /dev/null +++ b/adapter/tokenizer_config.json @@ -0,0 +1,30 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "local_files_only": false, + "model_max_length": 32768, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/adapter/training_args.bin b/adapter/training_args.bin new file mode 100644 index 0000000..861f852 --- /dev/null +++ b/adapter/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3ce4ce8b4e89d28f67ace5f5e759fdd2a53979ebbb5a30ad399c82f6526bcca6 +size 5777 diff --git a/adapter/training_log.json b/adapter/training_log.json new file mode 100644 index 0000000..ccf108f --- /dev/null +++ b/adapter/training_log.json @@ -0,0 +1,1279 @@ +[ + { + "loss": 0.6644901275634766, + "grad_norm": 0.69140625, + "learning_rate": 5e-05, + "entropy": 0.41775863207876685, + "num_tokens": 17886.0, + "mean_token_accuracy": 0.8308710962533951, + "epoch": 0.008412197686645636, + "step": 10 + }, + { + "loss": 0.5578639030456543, + "grad_norm": 0.6328125, + "learning_rate": 0.00010555555555555557, + "entropy": 0.5046202793717385, + "num_tokens": 35672.0, + "mean_token_accuracy": 0.8467379540205002, + "epoch": 0.016824395373291272, + "step": 20 + }, + { + "loss": 0.49953713417053225, + "grad_norm": 0.42578125, + "learning_rate": 0.0001611111111111111, + "entropy": 0.5429252408444881, + "num_tokens": 52882.0, + "mean_token_accuracy": 0.848870538175106, + "epoch": 0.025236593059936908, + "step": 30 + }, + { + "loss": 0.4875448703765869, + "grad_norm": 0.388671875, + "learning_rate": 0.00019999665919423514, + "entropy": 0.49813074097037313, + "num_tokens": 70413.0, + "mean_token_accuracy": 0.8602197140455246, + "epoch": 0.033648790746582544, + "step": 40 + }, + { + "loss": 0.5065551280975342, + "grad_norm": 0.396484375, + "learning_rate": 0.0001999372733012762, + "entropy": 0.5205996319651603, + "num_tokens": 87742.0, + "mean_token_accuracy": 0.8523474678397178, + "epoch": 0.04206098843322818, + "step": 50 + }, + { + "loss": 0.5336020946502685, + "grad_norm": 0.58984375, + "learning_rate": 0.0001998036980255298, + "entropy": 0.5422930240631103, + "num_tokens": 106667.0, + "mean_token_accuracy": 0.8395245373249054, + "epoch": 0.050473186119873815, + "step": 60 + }, + { + "loss": 0.584792947769165, + "grad_norm": 0.4453125, + "learning_rate": 0.00019959603252786895, + "entropy": 0.5817637398838997, + "num_tokens": 123772.0, + "mean_token_accuracy": 0.8319423466920852, + "epoch": 0.058885383806519455, + "step": 70 + }, + { + "loss": 0.5139331817626953, + "grad_norm": 0.314453125, + "learning_rate": 0.00019931443097074569, + "entropy": 0.5499239668250084, + "num_tokens": 142732.0, + "mean_token_accuracy": 0.8438791155815124, + "epoch": 0.06729758149316509, + "step": 80 + }, + { + "loss": 0.48215503692626954, + "grad_norm": 0.353515625, + "learning_rate": 0.000198959102403747, + "entropy": 0.5112259805202484, + "num_tokens": 161271.0, + "mean_token_accuracy": 0.8517454102635383, + "epoch": 0.07570977917981073, + "step": 90 + }, + { + "loss": 0.5477577209472656, + "grad_norm": 0.423828125, + "learning_rate": 0.00019853031060840508, + "entropy": 0.5490782242268324, + "num_tokens": 179202.0, + "mean_token_accuracy": 0.835928975045681, + "epoch": 0.08412197686645637, + "step": 100 + }, + { + "loss": 0.5046371936798095, + "grad_norm": 0.37890625, + "learning_rate": 0.0001980283739023765, + "entropy": 0.49583916887640955, + "num_tokens": 197597.0, + "mean_token_accuracy": 0.8535919323563576, + "epoch": 0.09253417455310199, + "step": 110 + }, + { + "loss": 0.4952244758605957, + "grad_norm": 0.421875, + "learning_rate": 0.00019745366490313607, + "entropy": 0.5152403030544519, + "num_tokens": 214819.0, + "mean_token_accuracy": 0.8491240337491035, + "epoch": 0.10094637223974763, + "step": 120 + }, + { + "loss": 0.5076200485229492, + "grad_norm": 0.390625, + "learning_rate": 0.00019680661025136078, + "entropy": 0.49193753488361835, + "num_tokens": 232053.0, + "mean_token_accuracy": 0.8580134496092796, + "epoch": 0.10935856992639327, + "step": 130 + }, + { + "loss": 0.5707052230834961, + "grad_norm": 0.462890625, + "learning_rate": 0.00019608769029420895, + "entropy": 0.5594139508903027, + "num_tokens": 248970.0, + "mean_token_accuracy": 0.8354306548833847, + "epoch": 0.11777076761303891, + "step": 140 + }, + { + "loss": 0.5284650325775146, + "grad_norm": 0.48828125, + "learning_rate": 0.00019529743872873014, + "entropy": 0.5149690546095371, + "num_tokens": 266885.0, + "mean_token_accuracy": 0.8453154772520065, + "epoch": 0.12618296529968454, + "step": 150 + }, + { + "eval_loss": 0.5157255530357361, + "eval_runtime": 43.5089, + "eval_samples_per_second": 23.03, + "eval_steps_per_second": 5.769, + "eval_entropy": 0.5205607345973353, + "eval_num_tokens": 266885.0, + "eval_mean_token_accuracy": 0.8498107833691327, + "epoch": 0.12618296529968454, + "step": 150 + }, + { + "loss": 0.5166601181030274, + "grad_norm": 0.3984375, + "learning_rate": 0.00019443644220566985, + "entropy": 0.5193028658628464, + "num_tokens": 285094.0, + "mean_token_accuracy": 0.8514553621411324, + "epoch": 0.13459516298633017, + "step": 160 + }, + { + "loss": 0.4895184993743896, + "grad_norm": 0.396484375, + "learning_rate": 0.00019350533989396395, + "entropy": 0.5498913679271936, + "num_tokens": 302878.0, + "mean_token_accuracy": 0.8499076500535011, + "epoch": 0.14300736067297581, + "step": 170 + }, + { + "loss": 0.489929723739624, + "grad_norm": 0.34765625, + "learning_rate": 0.00019250482300624543, + "entropy": 0.4781280752271414, + "num_tokens": 321038.0, + "mean_token_accuracy": 0.8518652722239495, + "epoch": 0.15141955835962145, + "step": 180 + }, + { + "loss": 0.5677974700927735, + "grad_norm": 0.392578125, + "learning_rate": 0.00019143563428571634, + "entropy": 0.560076616331935, + "num_tokens": 338613.0, + "mean_token_accuracy": 0.8355910241603851, + "epoch": 0.1598317560462671, + "step": 190 + }, + { + "loss": 0.47907423973083496, + "grad_norm": 0.380859375, + "learning_rate": 0.00019029856745476508, + "entropy": 0.486137630045414, + "num_tokens": 355794.0, + "mean_token_accuracy": 0.8546374544501305, + "epoch": 0.16824395373291273, + "step": 200 + }, + { + "loss": 0.5049150466918946, + "grad_norm": 0.515625, + "learning_rate": 0.0001890944666257392, + "entropy": 0.5356668088585138, + "num_tokens": 374388.0, + "mean_token_accuracy": 0.8522498905658722, + "epoch": 0.17665615141955837, + "step": 210 + }, + { + "loss": 0.5084699153900146, + "grad_norm": 0.474609375, + "learning_rate": 0.00018782422567431063, + "entropy": 0.48354478403925893, + "num_tokens": 393466.0, + "mean_token_accuracy": 0.8485782667994499, + "epoch": 0.18506834910620398, + "step": 220 + }, + { + "loss": 0.5076002597808837, + "grad_norm": 0.357421875, + "learning_rate": 0.0001864887875758985, + "entropy": 0.5002748906612396, + "num_tokens": 411947.0, + "mean_token_accuracy": 0.8519161671400071, + "epoch": 0.19348054679284962, + "step": 230 + }, + { + "loss": 0.5514575004577636, + "grad_norm": 0.60546875, + "learning_rate": 0.00018508914370564243, + "entropy": 0.5646139703691005, + "num_tokens": 429099.0, + "mean_token_accuracy": 0.8357668131589889, + "epoch": 0.20189274447949526, + "step": 240 + }, + { + "loss": 0.49529438018798827, + "grad_norm": 0.58984375, + "learning_rate": 0.00018362633310244574, + "entropy": 0.518617831915617, + "num_tokens": 446977.0, + "mean_token_accuracy": 0.8446412399411202, + "epoch": 0.2103049421661409, + "step": 250 + }, + { + "loss": 0.5090048313140869, + "grad_norm": 0.408203125, + "learning_rate": 0.00018210144169763522, + "entropy": 0.502267974242568, + "num_tokens": 465009.0, + "mean_token_accuracy": 0.851601280272007, + "epoch": 0.21871713985278654, + "step": 260 + }, + { + "loss": 0.5145918369293213, + "grad_norm": 0.41796875, + "learning_rate": 0.00018051560150880952, + "entropy": 0.5074335142970086, + "num_tokens": 481925.0, + "mean_token_accuracy": 0.8518189460039138, + "epoch": 0.22712933753943218, + "step": 270 + }, + { + "loss": 0.47620458602905275, + "grad_norm": 0.29296875, + "learning_rate": 0.00017886998979947522, + "entropy": 0.5017431110143662, + "num_tokens": 499792.0, + "mean_token_accuracy": 0.8548987463116646, + "epoch": 0.23554153522607782, + "step": 280 + }, + { + "loss": 0.507429552078247, + "grad_norm": 0.39453125, + "learning_rate": 0.00017716582820509423, + "entropy": 0.5222229663282633, + "num_tokens": 517055.0, + "mean_token_accuracy": 0.8444660127162933, + "epoch": 0.24395373291272346, + "step": 290 + }, + { + "loss": 0.5420027732849121, + "grad_norm": 0.39453125, + "learning_rate": 0.00017540438182619083, + "entropy": 0.5492073997855187, + "num_tokens": 536355.0, + "mean_token_accuracy": 0.8354690104722977, + "epoch": 0.25236593059936907, + "step": 300 + }, + { + "eval_loss": 0.5092939138412476, + "eval_runtime": 43.2398, + "eval_samples_per_second": 23.173, + "eval_steps_per_second": 5.805, + "eval_entropy": 0.4988216485872687, + "eval_num_tokens": 536355.0, + "eval_mean_token_accuracy": 0.849776790911458, + "epoch": 0.25236593059936907, + "step": 300 + }, + { + "loss": 0.536237096786499, + "grad_norm": 0.91015625, + "learning_rate": 0.00017358695828919264, + "entropy": 0.5142859980463982, + "num_tokens": 553515.0, + "mean_token_accuracy": 0.8491675749421119, + "epoch": 0.2607781282860147, + "step": 310 + }, + { + "loss": 0.5729623317718506, + "grad_norm": 0.44921875, + "learning_rate": 0.00017171490677570138, + "entropy": 0.558933037891984, + "num_tokens": 571228.0, + "mean_token_accuracy": 0.8405497163534165, + "epoch": 0.26919032597266035, + "step": 320 + }, + { + "loss": 0.5896058559417725, + "grad_norm": 0.5703125, + "learning_rate": 0.00016978961702091548, + "entropy": 0.572591795027256, + "num_tokens": 589110.0, + "mean_token_accuracy": 0.8290358498692513, + "epoch": 0.277602523659306, + "step": 330 + }, + { + "loss": 0.5112470626831055, + "grad_norm": 0.400390625, + "learning_rate": 0.0001678125182819466, + "entropy": 0.5400734946131707, + "num_tokens": 606112.0, + "mean_token_accuracy": 0.8408979564905167, + "epoch": 0.28601472134595163, + "step": 340 + }, + { + "loss": 0.4602771282196045, + "grad_norm": 0.4296875, + "learning_rate": 0.0001657850782767974, + "entropy": 0.4790476318448782, + "num_tokens": 623646.0, + "mean_token_accuracy": 0.8584060147404671, + "epoch": 0.29442691903259727, + "step": 350 + }, + { + "loss": 0.5069724559783936, + "grad_norm": 0.48046875, + "learning_rate": 0.00016370880209478684, + "entropy": 0.5200437907129526, + "num_tokens": 641902.0, + "mean_token_accuracy": 0.8489004954695701, + "epoch": 0.3028391167192429, + "step": 360 + }, + { + "loss": 0.5578237533569336, + "grad_norm": 0.369140625, + "learning_rate": 0.00016158523107923317, + "entropy": 0.5649974066764116, + "num_tokens": 659952.0, + "mean_token_accuracy": 0.8369685173034668, + "epoch": 0.31125131440588855, + "step": 370 + }, + { + "loss": 0.47066588401794435, + "grad_norm": 0.359375, + "learning_rate": 0.00015941594168322282, + "entropy": 0.47163805216550825, + "num_tokens": 676446.0, + "mean_token_accuracy": 0.8630838245153427, + "epoch": 0.3196635120925342, + "step": 380 + }, + { + "loss": 0.4772009372711182, + "grad_norm": 0.400390625, + "learning_rate": 0.0001572025442993157, + "entropy": 0.4667525518685579, + "num_tokens": 694154.0, + "mean_token_accuracy": 0.8570062339305877, + "epoch": 0.3280757097791798, + "step": 390 + }, + { + "loss": 0.5249220848083496, + "grad_norm": 0.375, + "learning_rate": 0.00015494668206405483, + "entropy": 0.5329773716628552, + "num_tokens": 711669.0, + "mean_token_accuracy": 0.8415302649140358, + "epoch": 0.33648790746582546, + "step": 400 + }, + { + "loss": 0.4875462055206299, + "grad_norm": 0.5234375, + "learning_rate": 0.00015265002963816846, + "entropy": 0.4823704816401005, + "num_tokens": 728983.0, + "mean_token_accuracy": 0.858619874715805, + "epoch": 0.3449001051524711, + "step": 410 + }, + { + "loss": 0.44158411026000977, + "grad_norm": 0.37890625, + "learning_rate": 0.00015031429196336982, + "entropy": 0.4349179297685623, + "num_tokens": 745793.0, + "mean_token_accuracy": 0.8672094404697418, + "epoch": 0.35331230283911674, + "step": 420 + }, + { + "loss": 0.44835238456726073, + "grad_norm": 0.390625, + "learning_rate": 0.00014794120299667737, + "entropy": 0.4709930635988712, + "num_tokens": 763754.0, + "mean_token_accuracy": 0.8656966030597687, + "epoch": 0.3617245005257624, + "step": 430 + }, + { + "loss": 0.46076221466064454, + "grad_norm": 0.375, + "learning_rate": 0.00014553252442319538, + "entropy": 0.45636184960603715, + "num_tokens": 782350.0, + "mean_token_accuracy": 0.8606420174241066, + "epoch": 0.37013669821240797, + "step": 440 + }, + { + "loss": 0.41022934913635256, + "grad_norm": 0.32421875, + "learning_rate": 0.00014309004434831036, + "entropy": 0.41649786606431005, + "num_tokens": 801737.0, + "mean_token_accuracy": 0.8772262930870056, + "epoch": 0.3785488958990536, + "step": 450 + }, + { + "eval_loss": 0.5019798278808594, + "eval_runtime": 43.2448, + "eval_samples_per_second": 23.17, + "eval_steps_per_second": 5.804, + "eval_entropy": 0.49474223777354953, + "eval_num_tokens": 801737.0, + "eval_mean_token_accuracy": 0.8509889845829086, + "epoch": 0.3785488958990536, + "step": 450 + }, + { + "loss": 0.4886730670928955, + "grad_norm": 0.40625, + "learning_rate": 0.0001406155759702739, + "entropy": 0.4626791954040527, + "num_tokens": 818998.0, + "mean_token_accuracy": 0.8578041106462478, + "epoch": 0.38696109358569925, + "step": 460 + }, + { + "loss": 0.43723235130310056, + "grad_norm": 0.36328125, + "learning_rate": 0.00013811095623415804, + "entropy": 0.4580305341631174, + "num_tokens": 837490.0, + "mean_token_accuracy": 0.8644093602895737, + "epoch": 0.3953732912723449, + "step": 470 + }, + { + "loss": 0.5131179332733155, + "grad_norm": 0.44921875, + "learning_rate": 0.00013557804446818133, + "entropy": 0.5031312361359597, + "num_tokens": 853819.0, + "mean_token_accuracy": 0.8500203013420105, + "epoch": 0.4037854889589905, + "step": 480 + }, + { + "loss": 0.4966330051422119, + "grad_norm": 0.46875, + "learning_rate": 0.00013301872100341912, + "entropy": 0.5123794697225094, + "num_tokens": 870402.0, + "mean_token_accuracy": 0.8551265612244606, + "epoch": 0.41219768664563616, + "step": 490 + }, + { + "loss": 0.5083879947662353, + "grad_norm": 0.408203125, + "learning_rate": 0.00013043488577792184, + "entropy": 0.5123440526425839, + "num_tokens": 887321.0, + "mean_token_accuracy": 0.8488352715969085, + "epoch": 0.4206098843322818, + "step": 500 + }, + { + "loss": 0.4652432918548584, + "grad_norm": 0.447265625, + "learning_rate": 0.0001278284569262781, + "entropy": 0.4698960553854704, + "num_tokens": 905696.0, + "mean_token_accuracy": 0.8606894016265869, + "epoch": 0.42902208201892744, + "step": 510 + }, + { + "loss": 0.4380183219909668, + "grad_norm": 0.408203125, + "learning_rate": 0.00012520136935566904, + "entropy": 0.43917837329208853, + "num_tokens": 924737.0, + "mean_token_accuracy": 0.8651972591876984, + "epoch": 0.4374342797055731, + "step": 520 + }, + { + "loss": 0.46619858741760256, + "grad_norm": 0.51171875, + "learning_rate": 0.0001225555733094719, + "entropy": 0.4861061923205853, + "num_tokens": 941252.0, + "mean_token_accuracy": 0.8588027134537697, + "epoch": 0.4458464773922187, + "step": 530 + }, + { + "loss": 0.4590827465057373, + "grad_norm": 0.32421875, + "learning_rate": 0.00011989303291947826, + "entropy": 0.4769519090652466, + "num_tokens": 959549.0, + "mean_token_accuracy": 0.8566611051559448, + "epoch": 0.45425867507886436, + "step": 540 + }, + { + "loss": 0.5028058528900147, + "grad_norm": 0.390625, + "learning_rate": 0.00011721572474780205, + "entropy": 0.49060247242450716, + "num_tokens": 977934.0, + "mean_token_accuracy": 0.8463574901223183, + "epoch": 0.46267087276551, + "step": 550 + }, + { + "loss": 0.5106627941131592, + "grad_norm": 0.353515625, + "learning_rate": 0.00011452563631955997, + "entropy": 0.5243487633764744, + "num_tokens": 995534.0, + "mean_token_accuracy": 0.8489840433001519, + "epoch": 0.47108307045215564, + "step": 560 + }, + { + "loss": 0.5040833473205566, + "grad_norm": 0.431640625, + "learning_rate": 0.00011182476464741359, + "entropy": 0.511018768697977, + "num_tokens": 1012521.0, + "mean_token_accuracy": 0.8465926513075829, + "epoch": 0.4794952681388013, + "step": 570 + }, + { + "loss": 0.520852518081665, + "grad_norm": 0.380859375, + "learning_rate": 0.0001091151147490677, + "entropy": 0.5150163471698761, + "num_tokens": 1029532.0, + "mean_token_accuracy": 0.8415179714560509, + "epoch": 0.4879074658254469, + "step": 580 + }, + { + "loss": 0.3955108404159546, + "grad_norm": 0.404296875, + "learning_rate": 0.00010639869815882668, + "entropy": 0.42485857382416725, + "num_tokens": 1047599.0, + "mean_token_accuracy": 0.8708899393677711, + "epoch": 0.49631966351209256, + "step": 590 + }, + { + "loss": 0.5156550884246827, + "grad_norm": 0.404296875, + "learning_rate": 0.00010367753143431278, + "entropy": 0.4836477965116501, + "num_tokens": 1066584.0, + "mean_token_accuracy": 0.8515340134501457, + "epoch": 0.5047318611987381, + "step": 600 + }, + { + "eval_loss": 0.5016828179359436, + "eval_runtime": 44.4349, + "eval_samples_per_second": 22.55, + "eval_steps_per_second": 5.649, + "eval_entropy": 0.5061054474922765, + "eval_num_tokens": 1066584.0, + "eval_mean_token_accuracy": 0.8509582224120182, + "epoch": 0.5047318611987381, + "step": 600 + }, + { + "loss": 0.5101516246795654, + "grad_norm": 0.412109375, + "learning_rate": 0.00010095363465945549, + "entropy": 0.5069868132472038, + "num_tokens": 1084792.0, + "mean_token_accuracy": 0.8516661524772644, + "epoch": 0.5131440588853838, + "step": 610 + }, + { + "loss": 0.5655738353729248, + "grad_norm": 0.43359375, + "learning_rate": 9.822902994486313e-05, + "entropy": 0.5374718602746725, + "num_tokens": 1102585.0, + "mean_token_accuracy": 0.8460393868386745, + "epoch": 0.5215562565720294, + "step": 620 + }, + { + "loss": 0.5003692626953125, + "grad_norm": 0.380859375, + "learning_rate": 9.550573992668964e-05, + "entropy": 0.4945683263242245, + "num_tokens": 1118590.0, + "mean_token_accuracy": 0.8570186719298363, + "epoch": 0.5299684542586751, + "step": 630 + }, + { + "loss": 0.4934247970581055, + "grad_norm": 0.35546875, + "learning_rate": 9.278578626511173e-05, + "entropy": 0.49867704808712005, + "num_tokens": 1136773.0, + "mean_token_accuracy": 0.8551020383834839, + "epoch": 0.5383806519453207, + "step": 640 + }, + { + "loss": 0.45653066635131834, + "grad_norm": 0.361328125, + "learning_rate": 9.007118814353019e-05, + "entropy": 0.4587596397846937, + "num_tokens": 1154768.0, + "mean_token_accuracy": 0.8645354241132737, + "epoch": 0.5467928496319664, + "step": 650 + }, + { + "loss": 0.43801026344299315, + "grad_norm": 0.3515625, + "learning_rate": 8.736396076960973e-05, + "entropy": 0.436482329107821, + "num_tokens": 1171141.0, + "mean_token_accuracy": 0.8680778846144677, + "epoch": 0.555205047318612, + "step": 660 + }, + { + "loss": 0.48572382926940916, + "grad_norm": 0.33984375, + "learning_rate": 8.466611387927075e-05, + "entropy": 0.4897559147328138, + "num_tokens": 1188848.0, + "mean_token_accuracy": 0.8528486609458923, + "epoch": 0.5636172450052577, + "step": 670 + }, + { + "loss": 0.4595578670501709, + "grad_norm": 0.423828125, + "learning_rate": 8.197965024474269e-05, + "entropy": 0.48105588629841806, + "num_tokens": 1205462.0, + "mean_token_accuracy": 0.8587836340069771, + "epoch": 0.5720294426919033, + "step": 680 + }, + { + "loss": 0.5170315265655517, + "grad_norm": 0.296875, + "learning_rate": 7.930656418778692e-05, + "entropy": 0.5295548569411039, + "num_tokens": 1223225.0, + "mean_token_accuracy": 0.8485119223594666, + "epoch": 0.580441640378549, + "step": 690 + }, + { + "loss": 0.49773259162902833, + "grad_norm": 0.439453125, + "learning_rate": 7.664884009919337e-05, + "entropy": 0.4959330677986145, + "num_tokens": 1241341.0, + "mean_token_accuracy": 0.8523301362991333, + "epoch": 0.5888538380651945, + "step": 700 + }, + { + "loss": 0.45866031646728517, + "grad_norm": 0.333984375, + "learning_rate": 7.400845096564895e-05, + "entropy": 0.4629350196570158, + "num_tokens": 1260070.0, + "mean_token_accuracy": 0.8623251214623451, + "epoch": 0.5972660357518401, + "step": 710 + }, + { + "loss": 0.45479936599731446, + "grad_norm": 0.466796875, + "learning_rate": 7.138735690507232e-05, + "entropy": 0.45315413922071457, + "num_tokens": 1277817.0, + "mean_token_accuracy": 0.8606215000152588, + "epoch": 0.6056782334384858, + "step": 720 + }, + { + "loss": 0.4651636600494385, + "grad_norm": 0.50390625, + "learning_rate": 6.87875037115016e-05, + "entropy": 0.4955283936113119, + "num_tokens": 1296319.0, + "mean_token_accuracy": 0.8529913663864136, + "epoch": 0.6140904311251314, + "step": 730 + }, + { + "loss": 0.458605432510376, + "grad_norm": 0.375, + "learning_rate": 6.621082141061594e-05, + "entropy": 0.46719060391187667, + "num_tokens": 1314222.0, + "mean_token_accuracy": 0.8618179276585579, + "epoch": 0.6225026288117771, + "step": 740 + }, + { + "loss": 0.4238747119903564, + "grad_norm": 0.37890625, + "learning_rate": 6.365922282696246e-05, + "entropy": 0.46433731317520144, + "num_tokens": 1332578.0, + "mean_token_accuracy": 0.8674190878868103, + "epoch": 0.6309148264984227, + "step": 750 + }, + { + "eval_loss": 0.49650052189826965, + "eval_runtime": 44.1675, + "eval_samples_per_second": 22.686, + "eval_steps_per_second": 5.683, + "eval_entropy": 0.49466295991523335, + "eval_num_tokens": 1332578.0, + "eval_mean_token_accuracy": 0.8537447065471178, + "epoch": 0.6309148264984227, + "step": 750 + }, + { + "loss": 0.4777510166168213, + "grad_norm": 0.296875, + "learning_rate": 6.113460216395276e-05, + "entropy": 0.46779244504868983, + "num_tokens": 1350642.0, + "mean_token_accuracy": 0.8585006296634674, + "epoch": 0.6393270241850684, + "step": 760 + }, + { + "loss": 0.43058485984802247, + "grad_norm": 0.390625, + "learning_rate": 5.8638833597683295e-05, + "entropy": 0.4450514350086451, + "num_tokens": 1368256.0, + "mean_token_accuracy": 0.8651673465967178, + "epoch": 0.647739221871714, + "step": 770 + }, + { + "loss": 0.4754780292510986, + "grad_norm": 0.2294921875, + "learning_rate": 5.6173769885622816e-05, + "entropy": 0.48611559383571146, + "num_tokens": 1385497.0, + "mean_token_accuracy": 0.8574813604354858, + "epoch": 0.6561514195583596, + "step": 780 + }, + { + "loss": 0.48357625007629396, + "grad_norm": 0.380859375, + "learning_rate": 5.3741240991200214e-05, + "entropy": 0.47957778461277484, + "num_tokens": 1404550.0, + "mean_token_accuracy": 0.8487858697772026, + "epoch": 0.6645636172450052, + "step": 790 + }, + { + "loss": 0.5497960567474365, + "grad_norm": 0.54296875, + "learning_rate": 5.134305272531394e-05, + "entropy": 0.5210029605776072, + "num_tokens": 1421268.0, + "mean_token_accuracy": 0.840529902279377, + "epoch": 0.6729758149316509, + "step": 800 + }, + { + "loss": 0.4972085952758789, + "grad_norm": 0.443359375, + "learning_rate": 4.898098540577122e-05, + "entropy": 0.5074868854135275, + "num_tokens": 1438077.0, + "mean_token_accuracy": 0.8451508775353431, + "epoch": 0.6813880126182965, + "step": 810 + }, + { + "loss": 0.42497959136962893, + "grad_norm": 0.578125, + "learning_rate": 4.665679253565191e-05, + "entropy": 0.4287653032690287, + "num_tokens": 1455329.0, + "mean_token_accuracy": 0.870225128531456, + "epoch": 0.6898002103049422, + "step": 820 + }, + { + "loss": 0.49147672653198243, + "grad_norm": 0.302734375, + "learning_rate": 4.437219950157927e-05, + "entropy": 0.5095056083053351, + "num_tokens": 1472973.0, + "mean_token_accuracy": 0.8493282213807106, + "epoch": 0.6982124079915878, + "step": 830 + }, + { + "loss": 0.49974665641784666, + "grad_norm": 0.400390625, + "learning_rate": 4.2128902292862625e-05, + "entropy": 0.5031959179788827, + "num_tokens": 1491004.0, + "mean_token_accuracy": 0.8519662648439408, + "epoch": 0.7066246056782335, + "step": 840 + }, + { + "loss": 0.4568824768066406, + "grad_norm": 0.369140625, + "learning_rate": 3.9928566242463516e-05, + "entropy": 0.46861373744904994, + "num_tokens": 1507930.0, + "mean_token_accuracy": 0.8673135682940483, + "epoch": 0.7150368033648791, + "step": 850 + }, + { + "loss": 0.4371347427368164, + "grad_norm": 0.3515625, + "learning_rate": 3.777282479072023e-05, + "entropy": 0.4369157709181309, + "num_tokens": 1525250.0, + "mean_token_accuracy": 0.8657293424010277, + "epoch": 0.7234490010515248, + "step": 860 + }, + { + "loss": 0.48940362930297854, + "grad_norm": 0.54296875, + "learning_rate": 3.56632782727477e-05, + "entropy": 0.48742267247289417, + "num_tokens": 1542980.0, + "mean_token_accuracy": 0.8546120315790177, + "epoch": 0.7318611987381703, + "step": 870 + }, + { + "loss": 0.4359768867492676, + "grad_norm": 0.51171875, + "learning_rate": 3.360149273041358e-05, + "entropy": 0.4555686455219984, + "num_tokens": 1560118.0, + "mean_token_accuracy": 0.8625203266739845, + "epoch": 0.7402733964248159, + "step": 880 + }, + { + "loss": 0.47966947555541994, + "grad_norm": 0.326171875, + "learning_rate": 3.158899874977221e-05, + "entropy": 0.5010597914457321, + "num_tokens": 1577819.0, + "mean_token_accuracy": 0.8502609938383102, + "epoch": 0.7486855941114616, + "step": 890 + }, + { + "loss": 0.48172292709350584, + "grad_norm": 0.353515625, + "learning_rate": 2.9627290324819635e-05, + "entropy": 0.47771124467253684, + "num_tokens": 1595465.0, + "mean_token_accuracy": 0.8525931060314178, + "epoch": 0.7570977917981072, + "step": 900 + }, + { + "eval_loss": 0.49366605281829834, + "eval_runtime": 44.2235, + "eval_samples_per_second": 22.658, + "eval_steps_per_second": 5.676, + "eval_entropy": 0.49088804530669966, + "eval_num_tokens": 1595465.0, + "eval_mean_token_accuracy": 0.8535285991026585, + "epoch": 0.7570977917981072, + "step": 900 + }, + { + "loss": 0.4764974117279053, + "grad_norm": 0.318359375, + "learning_rate": 2.7717823748412786e-05, + "entropy": 0.4569669846445322, + "num_tokens": 1612331.0, + "mean_token_accuracy": 0.859618866443634, + "epoch": 0.7655099894847529, + "step": 910 + }, + { + "loss": 0.5832089900970459, + "grad_norm": 0.5078125, + "learning_rate": 2.586201653117687e-05, + "entropy": 0.5448153771460056, + "num_tokens": 1629141.0, + "mean_token_accuracy": 0.8399338677525521, + "epoch": 0.7739221871713985, + "step": 920 + }, + { + "loss": 0.4664586067199707, + "grad_norm": 0.5234375, + "learning_rate": 2.4061246349202836e-05, + "entropy": 0.468376400321722, + "num_tokens": 1646892.0, + "mean_token_accuracy": 0.8635450974106789, + "epoch": 0.7823343848580442, + "step": 930 + }, + { + "loss": 0.515719985961914, + "grad_norm": 0.390625, + "learning_rate": 2.2316850021316383e-05, + "entropy": 0.4714161459356546, + "num_tokens": 1664928.0, + "mean_token_accuracy": 0.8522258847951889, + "epoch": 0.7907465825446898, + "step": 940 + }, + { + "loss": 0.46754937171936034, + "grad_norm": 0.361328125, + "learning_rate": 2.0630122516677952e-05, + "entropy": 0.4566764455288649, + "num_tokens": 1683193.0, + "mean_token_accuracy": 0.860313655436039, + "epoch": 0.7991587802313355, + "step": 950 + }, + { + "loss": 0.45092215538024905, + "grad_norm": 0.53515625, + "learning_rate": 1.900231599345006e-05, + "entropy": 0.49635875867679713, + "num_tokens": 1700570.0, + "mean_token_accuracy": 0.8524207696318626, + "epoch": 0.807570977917981, + "step": 960 + }, + { + "loss": 0.5068153381347656, + "grad_norm": 0.375, + "learning_rate": 1.743463886924569e-05, + "entropy": 0.49028092734515666, + "num_tokens": 1718365.0, + "mean_token_accuracy": 0.8554800346493721, + "epoch": 0.8159831756046267, + "step": 970 + }, + { + "loss": 0.4151425838470459, + "grad_norm": 0.341796875, + "learning_rate": 1.592825492404816e-05, + "entropy": 0.453617250174284, + "num_tokens": 1736733.0, + "mean_token_accuracy": 0.8672206476330757, + "epoch": 0.8243953732912723, + "step": 980 + }, + { + "loss": 0.47887310981750486, + "grad_norm": 0.439453125, + "learning_rate": 1.448428243626796e-05, + "entropy": 0.48988366313278675, + "num_tokens": 1756048.0, + "mean_token_accuracy": 0.8519709303975105, + "epoch": 0.832807570977918, + "step": 990 + }, + { + "loss": 0.41326084136962893, + "grad_norm": 0.37109375, + "learning_rate": 1.3103793352578153e-05, + "entropy": 0.425517388433218, + "num_tokens": 1773583.0, + "mean_token_accuracy": 0.8752173602581024, + "epoch": 0.8412197686645636, + "step": 1000 + }, + { + "loss": 0.44759268760681153, + "grad_norm": 0.283203125, + "learning_rate": 1.1787812492144611e-05, + "entropy": 0.4636589106172323, + "num_tokens": 1791572.0, + "mean_token_accuracy": 0.8631374597549438, + "epoch": 0.8496319663512093, + "step": 1010 + }, + { + "loss": 0.48625845909118653, + "grad_norm": 0.423828125, + "learning_rate": 1.05373167858418e-05, + "entropy": 0.49628445636481044, + "num_tokens": 1809222.0, + "mean_token_accuracy": 0.8517171651124954, + "epoch": 0.8580441640378549, + "step": 1020 + }, + { + "loss": 0.4556222438812256, + "grad_norm": 0.3828125, + "learning_rate": 9.353234551018787e-06, + "entropy": 0.45326445177197455, + "num_tokens": 1827208.0, + "mean_token_accuracy": 0.8658925801515579, + "epoch": 0.8664563617245006, + "step": 1030 + }, + { + "loss": 0.49567365646362305, + "grad_norm": 0.466796875, + "learning_rate": 8.236444802354194e-06, + "entropy": 0.4830797161906958, + "num_tokens": 1844866.0, + "mean_token_accuracy": 0.8538633629679679, + "epoch": 0.8748685594111462, + "step": 1040 + }, + { + "loss": 0.49363341331481936, + "grad_norm": 0.365234375, + "learning_rate": 7.187776599311225e-06, + "entropy": 0.49590680450201036, + "num_tokens": 1863508.0, + "mean_token_accuracy": 0.8481405600905418, + "epoch": 0.8832807570977917, + "step": 1050 + }, + { + "eval_loss": 0.492710679769516, + "eval_runtime": 44.3795, + "eval_samples_per_second": 22.578, + "eval_steps_per_second": 5.656, + "eval_entropy": 0.4946926224279214, + "eval_num_tokens": 1863508.0, + "eval_mean_token_accuracy": 0.8535925515619407, + "epoch": 0.8832807570977917, + "step": 1050 + }, + { + "loss": 0.46772065162658694, + "grad_norm": 0.330078125, + "learning_rate": 6.2080084306775255e-06, + "entropy": 0.48166475519537927, + "num_tokens": 1882313.0, + "mean_token_accuracy": 0.859215559065342, + "epoch": 0.8916929547844374, + "step": 1060 + }, + { + "loss": 0.4697853088378906, + "grad_norm": 0.419921875, + "learning_rate": 5.297867636646625e-06, + "entropy": 0.4738619901239872, + "num_tokens": 1899903.0, + "mean_token_accuracy": 0.8621121436357498, + "epoch": 0.900105152471083, + "step": 1070 + }, + { + "loss": 0.48142337799072266, + "grad_norm": 0.369140625, + "learning_rate": 4.458029868870106e-06, + "entropy": 0.4744772035628557, + "num_tokens": 1917878.0, + "mean_token_accuracy": 0.8594217240810395, + "epoch": 0.9085173501577287, + "step": 1080 + }, + { + "loss": 0.40869989395141604, + "grad_norm": 0.4296875, + "learning_rate": 3.6891185888811354e-06, + "entropy": 0.42849237024784087, + "num_tokens": 1935443.0, + "mean_token_accuracy": 0.8725451439619064, + "epoch": 0.9169295478443743, + "step": 1090 + }, + { + "loss": 0.4973776817321777, + "grad_norm": 0.29296875, + "learning_rate": 2.9917046052618913e-06, + "entropy": 0.4887580588459969, + "num_tokens": 1953376.0, + "mean_token_accuracy": 0.8516494154930114, + "epoch": 0.92534174553102, + "step": 1100 + }, + { + "loss": 0.5040076732635498, + "grad_norm": 0.333984375, + "learning_rate": 2.3663056498984035e-06, + "entropy": 0.5119045183062554, + "num_tokens": 1970989.0, + "mean_token_accuracy": 0.8522839710116387, + "epoch": 0.9337539432176656, + "step": 1110 + }, + { + "loss": 0.5222736358642578, + "grad_norm": 0.458984375, + "learning_rate": 1.8133859936374065e-06, + "entropy": 0.5279249057173729, + "num_tokens": 1987794.0, + "mean_token_accuracy": 0.8435431227087975, + "epoch": 0.9421661409043113, + "step": 1120 + }, + { + "loss": 0.5045926570892334, + "grad_norm": 0.494140625, + "learning_rate": 1.333356101630434e-06, + "entropy": 0.5016367115080357, + "num_tokens": 2006103.0, + "mean_token_accuracy": 0.8468659013509751, + "epoch": 0.9505783385909569, + "step": 1130 + }, + { + "loss": 0.4517922878265381, + "grad_norm": 0.41015625, + "learning_rate": 9.265723286211114e-07, + "entropy": 0.4635258123278618, + "num_tokens": 2024653.0, + "mean_token_accuracy": 0.858749409019947, + "epoch": 0.9589905362776026, + "step": 1140 + }, + { + "loss": 0.4429300308227539, + "grad_norm": 0.333984375, + "learning_rate": 5.933366544018881e-07, + "entropy": 0.46379459239542487, + "num_tokens": 2042110.0, + "mean_token_accuracy": 0.8633494853973389, + "epoch": 0.9674027339642481, + "step": 1150 + }, + { + "loss": 0.48291401863098143, + "grad_norm": 0.404296875, + "learning_rate": 3.338964596363958e-07, + "entropy": 0.4801469299942255, + "num_tokens": 2059221.0, + "mean_token_accuracy": 0.8571181014180184, + "epoch": 0.9758149316508938, + "step": 1160 + }, + { + "loss": 0.4716052055358887, + "grad_norm": 0.3828125, + "learning_rate": 1.4844434221414904e-07, + "entropy": 0.4888222087174654, + "num_tokens": 2076725.0, + "mean_token_accuracy": 0.8568236321210861, + "epoch": 0.9842271293375394, + "step": 1170 + }, + { + "loss": 0.4568983554840088, + "grad_norm": 0.4453125, + "learning_rate": 3.7117974273603865e-08, + "entropy": 0.4492193166166544, + "num_tokens": 2093837.0, + "mean_token_accuracy": 0.8625048115849495, + "epoch": 0.9926393270241851, + "step": 1180 + }, + { + "eval_loss": 0.49275869131088257, + "eval_runtime": 44.373, + "eval_samples_per_second": 22.581, + "eval_steps_per_second": 5.657, + "eval_entropy": 0.4945634504950854, + "eval_num_tokens": 2109469.0, + "eval_mean_token_accuracy": 0.8536091024182232, + "epoch": 1.0, + "step": 1189 + }, + { + "train_runtime": 2984.9918, + "train_samples_per_second": 6.372, + "train_steps_per_second": 0.398, + "total_flos": 2.585949872638771e+16, + "train_loss": 0.49064892395090115, + "epoch": 1.0, + "step": 1189 + } +] \ No newline at end of file diff --git a/chat_template.jinja b/chat_template.jinja new file mode 100644 index 0000000..9840d40 --- /dev/null +++ b/chat_template.jinja @@ -0,0 +1,54 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0]['role'] == 'system' %} + {{- messages[0]['content'] }} + {%- else %} + {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }} + {%- endif %} + {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0]['role'] == 'system' %} + {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }} + {%- else %} + {{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {{- '<|im_start|>' + message.role }} + {%- if message.content %} + {{- '\n' + message.content }} + {%- endif %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {{- tool_call.arguments | tojson }} + {{- '}\n' }} + {%- endfor %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} +{%- endif %} diff --git a/config.json b/config.json new file mode 100644 index 0000000..8227b12 --- /dev/null +++ b/config.json @@ -0,0 +1,61 @@ +{ + "architectures": [ + "Qwen2ForCausalLM" + ], + "attention_dropout": 0.0, + "bos_token_id": 151643, + "dtype": "float16", + "eos_token_id": 151645, + "hidden_act": "silu", + "hidden_size": 1536, + "initializer_range": 0.02, + "intermediate_size": 8960, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 32768, + "max_window_layers": 28, + "model_type": "qwen2", + "num_attention_heads": 12, + "num_hidden_layers": 28, + "num_key_value_heads": 2, + "pad_token_id": null, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000.0, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": true, + "transformers_version": "5.12.1", + "use_cache": true, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..89d0ec9 --- /dev/null +++ b/generation_config.json @@ -0,0 +1,14 @@ +{ + "bos_token_id": 151643, + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "repetition_penalty": 1.1, + "temperature": 0.7, + "top_k": 20, + "top_p": 0.8, + "transformers_version": "5.12.1" +} diff --git a/loss_curve.png b/loss_curve.png new file mode 100644 index 0000000..868fb0d --- /dev/null +++ b/loss_curve.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:daf98241987116a07a4cf7c39e2040bb0d4500fffc479f66966eaddf8957b6ce +size 146432 diff --git a/model.safetensors b/model.safetensors new file mode 100644 index 0000000..6999d6c --- /dev/null +++ b/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:49101962d7ca1d1056ba5c063e3618649e8cd8f878d6717d7b47b1d6e895ec41 +size 3087466808 diff --git a/token_reduction.png b/token_reduction.png new file mode 100644 index 0000000000000000000000000000000000000000..3f27a9668b96200f3194a150e65667f52f9b9373 GIT binary patch literal 43800 zcmd?RXHZmK7d6<3f{CE02$GZ}NX|J4l7oQcAd*3H&LAiv5+r9N2T77MCM4%1StRGs zWN4V(yzhMTRZZ1YP1RR3Q#C)HY9GCQ@44rkz4zK{tf z#KJ&f=IChaz{kvN^*^p)vavU1j&8}afxBF@mC|%TAP8@x|HCj%?T2UO2S|&Hs=6kv zO=3IVc|rJXI}t0h*JG2@mxAKzr4MhU2yfx9Nqt4oH{`xd%gcT2-2BFyg8q^2^E;xM z+{+D5m%^Wkl}^_vy2y_P?2a0pH90JFDvW!@Pp)+*A53}a;RgPE2v0^J#T_5b}(O)A9{fq(P!)Uq)QUnhB+T({GY#+|g9yc>p^F%X1}sRBLQvn}&q zTgW1J<~wAIbZRzN<+Qu>+->G9VW#;&wsO6z9rCSqfd;GjMD^E?MC{LS$pqr^ zihb6{I7edt*)*YOQt0cV2I!N>zm+ZF&@Q^D$z*d+dk(B41Kz!B|5Ds&!Sw0|ArphQ zw|4}yzEGZeamr0nK2=a{KXfw z_$Ef$HtWqR5etc(TysjyO{b;xJMB+y6A=;FpyJt0JIkHdwUCNJo`<$>+s%Ys!u$w| z#hRIS4{jHZH+WQ)n-Bcf`Pkw#-+p(r)HpbP%IiQkUeKe`aqSQ5l#u)P@pOqBqh+RH zmXq1?iJe!rvt(m4ZKfNoJ6LW1ij;W`7iiwb#}7B~Aumnkby#Rkdm7le`10)ZXyJFo zltydO?(Yx%=XX&}bh@=1BQLgi2Jy!}PE~kq)GW%eXjED29<6*SmK*r^MZLH^h=8^; z%no(DM@>)PemRuG0O7lqVKrX)&z2`5S*yz0YQIpYrgXx6wJ<1U)(@ME<@@KburPD! zQk!X^p!e^?4O>FnB+0#~BtG0>?KnQM-I%CxZLtW+!1Cv`9Om%*dxTt^&x+M!8!tEM z)5QtW+x0n^bg`fII*8x-Iko%F-(s{htK4FUUK`t=m6f&Bumu-+n3t0h!Die}bsbCi zWQ`r^1Ft9-EEA8!xzF+~X;!z+SyT5;n1qNKrMs!c?~g=F9Fqfc5LZ38^X=N#EYPyG*ju(_daSC*cixBOb=GUxxB|M zV*ccl-R$r9Z_klAIC;UUnVigb#LmO2hq|2ByCHS_(zqy}Xq0?jev(xoP$L6HgX|v%Tf&S~|}A{dgv_ZUf1J$HgAX=rHr&aUe6%)zgQEhI3vF6EXZ-Onp5UPy|F_)@a4^gjQh#@O;%)IWm3 z=UI|Iikw{7yMFff0~^P=uEY_NoXr`=s)>JbY{nmE54UHFy!MWgi(Pi++QfVI*2;Qn zE6+E5&SlF{E#y8Nj@GjGsvg&Q#a6r?WgBfmHXKaJI@c-K)NemaUmTh~ zla8d}P2e#9=dQnN(?1K-sfD3te31=n(>8a!yVaZyEs2MWUa$9gLs$_?&LEs#M;4Gaw0VrUr`DR_??QWFlu&bf3) zauidkQdJH{ysWEd-rW?&>+T8Ni0$iH&m0Lmb{pxDN)d1)iKc&HFK~7^8=4^*R;wq> z!}Dh*@9I^nf3so2&ZtOOZ*GZ}&P47Wr;pxWwMp?;B(kfjs>Tg>C8sCxtevl2jnX9RhNZhbJ z4Q)eeidwP0U;hR+c0q<}g@M%?8L@!k@rg$*PV*Yp*Ld?tOxD-D3 z6&6W;=b4XIj~^{ofVm(Z-L-GC+_drD zQW4wyuuLi_SfpKLO?1zUj`6ATc*L<^zcjO$X-|sP2138lb95VZFkQ!FD--p2hR0$%(%FhdEgN7 zX;ZYqVu(cebn9vQjek+Sb-_Oa@mx;NPBv+j!ajcFs33Cp-u)q-+NAOIRrYUT%AQRE zMFE4#<>c8Op_9(~?6Y;gbI1Usl#-zw(+@WqRc@bsTdf&#h=ZD2rpw$-&`<6;xxYVd z<1yh89?6Tzo4or@UzwwIfVOO#K_PlE-Gmyc^6{O=+AUqOpm?$xlg&8sK3@?^WKx<`d1dMyGd4mO1}41T4|3s z^u)1J%BM~ap58NF++@p|rmNM(G;q=L+8d(646NSW+vRlBFDyJdUM=q0OiT%8)UM=# zhrOy?#zE*h6H@WeHhdmIx;6~IxVG#Rh~!l()?0TbFAQna6IrUqV0YbmsS;eg=rG;r zRYdHZynDb;*&`~Gx!cg1>&sHO_>dYSEpxj@y+FetbYNWzvv{+N>%CU|7}S2et@kdZ85N&eqSO+h*)!OSbpKh>=bL_dk5|J) zOnNCMHIbhtB#gDXi;>oiTrRR!U;IDZeNro^=+2r$7jq!~GKWx(>YAw#(T3LL!Mw=vG3SQJGf1j+f(nTiD?;cxzM0p^O+c&QIO*6=AuAup z^1Z<0Xg-#G*G8_xl(*qr_{Kd3TdRTd;DBA7X@Ckv4&>aqQp={?S<@Axs5%BpOff6m zyc6@nin@h9;?~tG^&1WV189k;%zVSuh=}l4x`Q%a!b ztUEfK4eP467|LfeYQ?WQaaQ{CAGAilj=Nf-T)}VnX1GTOM z&PYgg?0sJo_C0o&`XWB{D|dIEB^eERe2(vQT&w8(EG;-zWn)8jzBi%|F_Nr#>R3<~ zRl8m}-l@~VFmz{Ip*!kbfvE`VzEt%gX-Eb4OOJu|Jz%BeSIg6G%jwUTT zm%CM$Hzw>_NKJ~CyE!9uozwk?^2cqOCQlEkB(Po761i=h>jd+dNxT%gsJ`4zNep?w zolt}PF-gj({!B)em;J;c0yeRUvRtuoM@;nzj>h)Q+dvR%0r<A$7z z7IK#>4~|Lr0^VY2^390{;*uJ{7BS%oh881^Le@rg`HH#)y3kCWr|zweB-Yd=S}4G4 zars$Kpx9^25+fTs`k(fE^`0s|Nc`*Bw1-^UC`n_d-lZ{B{%I4%sLj>ZWBe=$ zQ!ye286Mx$usv;=ThJKVeiK@wG~rJA3gIGmb>59_wf=1nn+B6@8Z&^1Xb@QMf zOkplw(bwABoSr_lU2R4J>d;*tnapDAX+@r#B6|{Vh18&d_Q$0w z@ulu>y@e{oTD9AvVj1LEmr_-YkyeJ)=T?x@fGypp^|hL?oQSt!f6OvsG-*-3#d+X> zhR$7dsem<(#o&!xGT+eP+{&aw!?n*(#g|I%9ycz96Lc?zd=U@2ZStBEh3XBKBMnmu zP;@5A)ub)#IRP zu9>XyG4XVE=y%b4mV}*^I-S`q0r^t*q=ug`e$OjH%lt?udy&LUUQ>BmJ$^4m(+d?& zOFE+D{@Ok@damfJz!i)%`K7S*F(@P-68o2u^%oIe?Gl}hW5>GIez*ndHks{yRXEn5 zT@>lJY-{AahQdx7HIShF)%TiDlOjJ`8SM9-%wW6b!qR1xTW#l zrrngEf-CnB0btsd9^rFGoY3{xT*-H=#Bvw)nD6g3xJX)_CT&h1O#7t%vtMeC)H7RY z{)Cm-&A)bhl&jZJYp$H1Drn`NR}t8~W7KTQ`=rSuVvLzmsLuJE$>D>x!@2;TIWOVp z(Fs*XH%7k?PBf{}v?8B!v!-sLRr_8bD|NYG4YEk@M{rfOe#EuJo4n=&sUIC79U5&; z@i$vd8A$M|d~x0WXHrD?w3~TpE*lDx(H~3xllQ*_eXb?**gev|5@3Fmq;W*Xg950| zj@+26>!3~bd}U!@t}`~*O_!JO_Ty(45s!lnMu)|2PQnd$^>e6toZOMn^*q88*z#TG zz88!15EZ5zt9oQ05?6AsuSVB0xq?VjCuNI_O42e`MY*V4PI^zf?eVdj@|Ej%SWub~ zvuk+yo1f~=Sgu*t{o5yRAhYt${~pi9@3cSQ;GIp(X>n&WUB+L*$H!+NmbC(fF#stO zZ_yhH;} z@O2UtHTOH_zuXW5b;xrYGsFpvz-M^r# zmXbOs)3Nxg{eWVoIbH95U50RGcS3c+=dSA&nyLHJ{b$0*$=d^p!qnFzlDhEr*za(s zeN>tF#U{;3xEFOmL3`cei;jYd-v9`%80aGs=xzJb9j>u?ovsUDXv!Gkxm9AV)n%lYLX( zLJrFOel7cT%J}dUS<~znUAu8CY03Z$>xOZsjk^=^+mf*-TaE;U?dPOwfJ6Huua@K= zRZrL+f8?>ZGGvVuSUUHxNDTCl!7+e#ymMAv&#ie~o1_RoL1tFu*p4NmX1)5M2Rd5Czu2crrj><9BrP?0ZEs{3x9O5@ITYfW4Yg^H&4z_j9s*Ib z@Lc&2Xtu4EkE9FBJrxQ(@sg6{opyb8Y8F%2T}CT}15{+x__llS%@6^#F3d$gPG;Ww0*S{np>t*^5jRy8*C|?WZpH4I{>G zBHcNTK9unmrj+xhbmZ}1&*brF{5;)EJ^K{|KH7>@(IAP9GnHm=U)il9CPznUfsn?bU6GlDAVRr zNZQzuv?vO7yOCM9cFtMYitEd@(DL&F?=vTT(suD;7Y?5OJ>mc}{PlX?gaM;{Ua4h| zv=IX>?YyS%#~L<>WtR_Jq{Fkj=iBa#BLYLg%#CJRtY%7I;@>$vzKhZym=HN$ip!lmAaS_u%q9G5v z`8VZuHWfRL8m12XaH-CRwQXv=x>a%%n#W!C*G7Z(iTE6IEj?=N=OY+WhajGiP?qYt zJL)zY6aURRul+HY%9-o-dHM45Ea1nc4F{{S^0t}P)m5{x^1LIIoheY53+yploR0V| zD9@~CT0$K}+o(+c>S@>6khaEsU_S^*V-tF;Wqg85kuI7tSlJ^z&w-|#6sp3tPTBJL z4DfzL5D#FP1HXpbk3BUUxl|qAg`U*S<5Wl5!bJ*dWuzXOl9g_(JS#rfoIbC`9LU)d z$TWI+Y_iPLQrviK5#gf^fPoY;4QJS%jvKwjRjX&4ugu8Bx(;((Hp)Y4oNY(=V-iB- zH<>I3?pYnDyRb!elw@_W?iH<4h6%hX{mq;Dv?aTLf5ly$hnG(wIy5B9P~My%a&?cO zuP!-(QSs#S5-GEnVDc)`jB9YAE73!ocR*}2sR39ElL?9jl^mBPOiWC9b$1$SjT<%}Nr#ksqXRCgF(!06 z+ok7;deQ6l!Y;dVHOX_+inS40%TzVK?j)|+{<_4mX_<2qD~rOgz4K>1(#k}%oOXWf z!CxB>HZ0C3sN=VP9Z=-h(M0wk7fJRXQkT2Mnq;N`1;@KMcCKSNRNnRqSkx|;x85cj zjVBvMsgjl@yIzAmZ%vZrR_a8x)6E#)Rx0%MBzyOmiQPsCJJbAb{Qyn7SqEmJ+W@gD zifOP(^gOWQnd*6C^b_@L_p31Uk%iq&HdGxaQRx1Sp+!ZeqfaJMBR!?V`IW$n*DGp~ zk;=%jQ><D4ksom!-dp_&GJvb|HyjyURUojDzslm@Pg zShnn)DEIiE6)(H>ZBm&IxVnrerTmc15c_figGwTigC}yvOer>|`%sE}JvhCKclXg# zY~!Ke>xw@YFC!4H>5$&qfQ*$V*jOIO38qbdhc8mSFVUzP24PjBF;I_iK(860@MifIgCHDE~!d zqt{70(4LV38_?0RfrgT&nip{y3rhxgd!ui@81}$2-xNd(kcfhR^R_%B5l@qayv8p3 zNw-BmiS-1oq8~IC*VQl*-bcU)Mnn5Z%a;u#St&>tdBDHdhDnF~{ri#~m?DpaR9FQB z;`NKCXj;x~wXn(vLdgE?)J-M$0O)~sV7VVXfBrl>n*}KNDOSj8?0&j%$$W9sSwwHI zl7pu|77i5^l{lH#Zl8E@WQ-yDe&1wxYu~Z(#xm>2Jd*ez1&rw0bkj5kQkCC%jith_ z#$hoA_L?*h{W>j4z_kwUynXw&4d6@G^NeTaLNpb*4`H7p!u#w!aSZIcBpMnTnmamV zKip$n(P}d5d&E9sl4*y@pVg&U=e$mA(^O7&@fvDnx+3?9i0B@0TSr0ZS1;0Qut@j_ zxoo7uYrN8`C_8%tI0>pLZ}b1o-@+5FG)|W>&w){WrC3_II0~Q`6{Ft&u9-~k&yr27 z*}oyz!#ZL~oDj}s3EI+*Qy4Z3%Qvc29%lxmjepc(yugPppSW!QYj!=65EqZD*~fNl znVz2R{Q2|efYVL%WN378aRww{s3I55vY(UI8uVvMvw=!9TJKh7F1E6`NXYCuu#hRu z$s1CquN5XOqx@5b>V?Bq^jFP|6N$yh$;r3(_BahMO*xNSl|pxxr{C0wW(nAArUXX3 zYjvTW`LT}VX(4jlh#mgV{99CByGqdiSE=QZYMslbb78S=U1j_#{;iDy%`%$* zeOb7;)PPD%&Rr-1Da}O4=ggDWZO5qI&{64Rv+<-`pmDXRuK&Mp!YGLyz}&DKA%N^; z-vP0ymhB^OJ8Is?%d`YZh5Ai;moPE2KxQj4Tp2EWQm9qI7WB#y6~n0AX*X704t>;Z zVq%MTpKNidKmG3m)x+&7k9aBwboveyZt5D{vNsm5;;koZo3(xZe#0<_8f?iERDJy} zJ=GVr!U*V)GEd?LZE2#V>6b62H&ZRN=phNWb&}`dOkjVllO_AJ&ma-ZwM9xo-KECF z#^vC2z2-0aB2Sq*(yd%ERS2!%IwWyD@mfj?G#bd3x5lh1C;86?YVYF=;JwBPB0mDb zrUf@fZ$kuCh%Y|+^EG7B|0SR8BP*8>t)94nPb+PvnS4&S0}FKP)No07D1h>e&dq(? zmdIVKTBM^1G}^-k_q_;EyJ-2&fE4Wqc%<574JzV%w7Onfi9L*m!qAO(_c{(4Gl(ah zd#{M43dgPM z9zJ~+SEB++YVPlM|274$<6Y@p_sKe!p5FhAvCoJj@pIeb{tOV9`U|zWmq*KTTs>Nv z@b28ZM+2P(dzaYkF#s^#)VkHa#mCx}1v=HX-z$oMAuM&@TVY^Chn{jEvDi1P^5rZV z3+ej-dEdT?5ad!jE_BA<5kAjJ^(JP%uS)ssW9!bvBb}fCJ=ogbmUgadYVt{U%94qe z>2qlL{k#1_SJQ=uX2G7+)Iz^rCK<-Vmkz|e=H!jl92sUZWgk|)@_CBuJPpLX%gkK- z-^Yhtl+uk-r`2KCRXx|4w|B4|ArrdS&)=q&X~o172%n#an~AJ|fr4uvr0aH&AjoYd zp&|~ZXnk3R?p|A*q0>i zkIqkMrCyI+vFnuA>)3I5y2&R$MV3AI_pe{oF%I+X#tFeDPP{PaaOcYbzVc=vQ@Spb z)E@h5LEQV(YoldZkg4SwlSsu;--A%yKO?+5K=Ia4~drU_kQR-L~f;33U|0E)}u0Er|y;Eg2c2T8H)< z;4CYZycCVROhY#o)}tHKv$SNEr|~tNMA&g35SrPk z=Rwc49_dNoe4@va`wovhQaS2do`$)bK9ln!(;`=JRD-TKaqEkVB5=*}|8 zu-Ae>&iB3mgM#=VibM<5+u6gWguHk9lj`a(~W*n{*|_Yt?}fL}Gtq($eBO zr&4DmZCHVJRrlZ&^*oe1>zv;q4`|0+03XuaF8~(|gGPx6E-4=sgl!DS)`=~UkK5*& z?xAn1+|rTuTm|DU!|`-) zZ!a3O?YI^N;+w;{&z}OWqSvl`{Rs%T>}8v4xVT)%_3^4;+Bj~TlpzVkUq7@l0}ZuB z=;xM*e`LN9g+{@p<`|U~JA9QdmED>pmtHlO0?jywE%v0QZj4-@>yHr`m_O#$#wt2G z9{>Tpmb@VoO&?ED3<(HaPN@JR9n(>j){_4v&_?=&uj*8E6-OvRQs{Z`!gyxofn%G;N@pAn%7-&nBT(pz#nJm*m1ugkbuO|rX_C96v*niEQ%IB0nj)KC> zy}f_NtX2fM%^Y{^tD&U%RDI3zja1r`MhWl;#&@& z)4<3m6C}bHXpa#4kT5dU}^3 zjtqK6zo|f2C}F>Cnvc$c( zl8e%+KQjo}9|^SH%j>)r_vP9$cvzkgGU>2j_FHPA$M;Rem6rrJa83#Bl>@WK&%vwr zo~SXjq0j*iTt7z`XW|mXMVFx<69lUL_CJY{#$$nwSms!e?5Jh1GG=iskgAJz#~O+@ zU<#C0jhu)J9GQJ>ouJwuUyxmqhWKlJss@Yb<+`~0kR}Stc+8O_kB)}sw^Jv$f{ehv zquk`}6?u1%=%z(>pJ}R>8YG68$M+qvUIW14Bzy|MX@N&yU!T38S>Fd0UG(xIF|$1d zP&r6k3s7ryBd5Y!nmWEu$d51drOdmJ?;pIk5Xr3wfS`=H1sza0snXM{2$B7WjFyLk zzH&Lck5zM(^27T6vN5(?YU|;j+ZE*ZT&z3hN1%G_Vy=eT4jT7Q)k-F{Gibg@Hwl8 zepUuFaB5ieig0K5|3$oKGu*h!8MzceEjtID%63Am^XjlXy?PM?7*Sbf3D7wn!4^n8%VXvv(#4T4 zIs-I^?&@a4f!-0IWL^JZ@EU2}W7c~uaSMGvik7m>cS`<3IF~uc%^){))k1?BznAKB zRs%jhO|UeYTHqh7jT-)s%c&>^6GUH>naGdS0t20x$%-d*_<6$iCk`+o*k#_ph^gJV zAb{)zWG{)CPJerIuwaYXk_COJL>I7b_5n?Xfw**KN6K^QbBYaj&TE%*T zBRkE(gk1uUAGf}_jeZ-EZc}OHcmNK4orDY;6lmH3+J*SWH!c%)ftKXDIUKVW@Q@<# za)1_r>$p`_yth9NF$c>kF}F1XBu>t-pFe)gg$bXrRQPu#a#y9lN6%6CELdjq!APu@ z;&bMPHr+}yA;FIng!bZP`@Mqvo(c5!KQM5;_|Hs9BJGzx`);}ZUoG2iCL8+ZhzHz6 zPA0d9zAvJ~1pHV3ib_ktC2VrBJn+_A(1XQ-9a;nJ?p*V3Y;3$7fc{>%%B3IZtpcql zH5*(1P0_vv4;MPU`s#77*--flx}tRfVv6a&v1@_K0@!d53kVQ_AS(mP^F=w z`qc?Uj4aE=<<8fy%EmHTEdalyRxg0Q>RY~V3Ctjx3VWhf5Dvg?>f*{0SrLtDTZ1F8 zRvPuB2(rEY!vcoq%Jlo_FTJ#kp^Con(L_PB3ur&zbGJ_t+T0gasB7dNYpm=SDbij< z#1$GLusDWTYPLL@sJ6=(_u|~X;1JLY1j@zh50mui8Om=zTXEoZG#4*fPoBs*_>Gn2hj^)PNSh3wE{z{ySh=2 zRA9%$!;3xHs8b~7Fr!!kuP`+3l3P#M}ZOQUeraIu@{oP^aC7Zac&U(krO$u zM@vn-p=L2WxrqHgLDU#%ED?`=>a7z&Nq zpQR~`72H8DN4WnbF{?-UeyLcVKu2&#^24Stebs!u2B1h;{l|+-1~iqga04ml1{`_h zG(`bjgHufO4fGXjzao5lTUsQsa7iA+lseSu611anUAB*b-=!<1^`7X9USh+- zdjag9FY6JCgaMBXA^T zaW#-rl0K3P>uFWKHvdg0r%1sGTqhByg=)mOlj{wfd!XLT7B+bvl}-QxEd{Yryk#&? zMKXq2e}mq}J@kK&!ga21+>i^Jx|b1Txj<5=5kmN6zA-izs8F*7PAGf02mnsD^E%p{ z^!+6;!t>yAmUAi8f>z&+6z4kHH)=uYb9Q8kwvikjA1AbITzXa*J@~E2s4XI!Mupr5 z40+0xuQf+2trB-Pr-h$}UPP$&F#c~+kF6lulyI1t1m8k{{}&w*pzqpLeg+1H=j*8h zFlJ*gM`vrAN=d!9M17)^%mk@En{F98p!s(XtZl8vYaDXW-gT<_KH_d@esYfD2GJ6U zvk1;Z1Y$IJ9usBmBUbqJ1gcJ!YMyfTFcak)(P&kcTn_)dS|C{K_kcv*RAWCrB}Qep3XRVj(Z95ngrhh+0zyodGYzGs`MnsCtClXB7XN%8F z8^^vYXErmMAIf4Nc!eki16aqHSqg6t|1}N(n~pNt2>gIz`#T|Fz$~V zCa-cD$h*S?Q*ai&m292PNR4b&$x-o?`*VgWwHkC#_8C*%?ns@kIBVY!+;t*ymLJjV7TT^p?)gGOQnJnqcky@vw%}@9CKYFaxH4xTBr4hxTPVdKA&y><>$BXlYU~Zf%UQqW!q1vQC%em(`kvcw^D2@!En^Okj;P0qmD0pa ziZDCs999PD*u(gZr(iw)-oa2nvzc zu7=Baok*S4I{oSx*Jj$BpvD$DY#sdDEbQ!2pO_I_d3K<4Wu;m1omz>gm%2zU3e)fr zWixFzixMH1RZQ8FGnOz^N0fA>CLQ7%Qj-jSr57JK7JF*9zB6Y~zGbZ%_GOxyY~_VIRqie*)v%QEzfTHOrjK+QFPB)G(LL^ z4ydv}#Ob`<|FS^ii?(iuM8YIPUo3Nn-+{24?Q8LR?>98px|;0o(%-kyZ@P5lO#6LE zbS`pH*9+gUNDsAH(@>weLx5@?@9KI19hJdy2>}`7GC8<6Ccv4X;CVH5s*?-@6?mMAEdG8mst5)}QBuJb5qV%V_>SZzuU_ z56*KKj;qNQNF zghY3bVEykt7WhALNn;3*3w0K^X7{BpBaHN+;t4t|e5^nG7rd~I?(6aKyG0m5RMK-_ zrY3@t6i?=4}_c+N_`965*i)v!BGV%0N+4{ajBj2gpTrGr-+X$o5gWHzpuU9CzlM=?gGdsW1&{|dWtyO4RZCd2jUkjkSV zX0JV^;kNhTl9N8|=@M&M@i5jg)%|@be+iZG!=_Oax9RfbQlDcVPTzc^7F-(Xt|5oT z`K`|4F22o)nk?mW?}hCd=B5b4X>amPH^|;|L6Nt2#y6FZw!M|p)T^iSf1BapJczKDd?jUync_tEW1+(3lYA@;gCa#2{2DNUfEO~MW_B$>|Y_WD^dN! z}^G?~vQ&F=b%ZQa#9+cknGLyFf^62ki*0fb9S$SN(fu{HtN0z}}#WJ+w)5`i%nd zycmf1?Cs8;+aZ{7_r{{wsF|#%9vNWc{ubs^XHubzB_Y+`npkPNU0A#~@p$CEAHB%v=xyHFG>Uu@f%n-XNl?$8n*ljF-KD8M?JfSm}j4D-cdtdW@VyBrm0pz^IHWR<9 zrH8Li*!l;Y)}E#|xvW+ioiibPLlQr(xca(GI>n;7nK%Cax)lN4+39>NC}<&?!!T@M zXuoI`$vP~tgD0nCPInd+_p7%uBnhZPAr6@JSnhK_?$pgJ(ZS&*u<`mcL)nHEaLv3) z)XF(urqX87;y8ZGjetvH@5$lxH<(KUdq#mDM)X$@t~4FLA40d3RRC%>oLpo&+L>qf z`wZ0qfQHF9C`hkfYy&f|l!lF)K>WkCzCoU*cQshpwZ7JQ{)2dOQ@ELzY%9F zQkT>uo!y?2`!yf#eA;R7$gVcilD)5gZ21xf-_q~5T^_##0$bM~f4L2D9#m)J~f2Z~N{^HX~!QAI2DJfnsgSd`nfz^hmd7c&iDx6eLn_I|k6=xm~S-wi5 zdLzoLub($~{`F47v$G#%WotyeyT$}|POT065{**ii5;!TYN$HVd z^4Y=LhD_C11v+A#qy+g?j+t_&*!Q+`Z;eoT~gb#_9DUCI~~O| zy_4Jjp3q>LPkdGn>mff?#Ii&9F7EvC-P)S*`K|QI*98_ZyG_%v_aH!?=LT0b>rhWL z)T#*{uY@ENl9{*3IoW(H?|WKV&*xz*i^y)@>vuDiGTtfA#bncUXXg8N>HjHd6Bu5| zqaK@zUpsq&El77|Whfd-Ujh{eE@=ykB3YaIm+YvGiRae`>mCmIkRvEo-FVAGB=2cJ zqZ~g7&8zWbty4cVo;6^={kLwKkd(yp5Y9b?L*o|jzdqhf(3u#^!+Qy_ z^r!SaGZ#JmbBE_R6SqT8V#&S|TWP*i5EDnrL>#*bBBRQWCRV2`za|XYt_Xwl_L?yBo zYk4rZXSdu-04D{N>zu8CKyp>Pi-&G(U(b01Ti1WplQ80%DnT+e;Q}UAJGo6Z{XYp_`2zaKV0B49TEXJPzR&I91l^u;g&j zV3taY{3+069|dp@{HPe3KZ&GOjDhoeR30C31;2V6SjTgiM@TabK$kXJVHt}SSj&Ph z2!P5@)U){Sz@-ES_~~sL_F}$_nu4g5De8}RyqEAx1G~!FLwkU^Mz|m;TYCA2+$T=VSHTp+)T~Xq;*XEqXyZ2v)8wg`UVd16% z24eJt{RyT1sgMgo3OFd_J1^vcRzEb8<-1sYkL)1;LN!86#NUS?;Uh|d9KM`KR2&FX zJ-`y6HVDMk)FnaqVpC9DhDz3rs!$_gWxegRk+kONmM)G;JbHk5sec)0!9U_($889< z&lYpeSK`Y`Eq~le2)pFlh5q@7mjqWyQzsG|3mU8jDPs+)B zQrlcE)iFJ%jbfo*CktI-BwOe5_mZ&ZUVtKT&Udv zWtAE*!;fma5ScE1V?%c>G{p!&p9(_bOrm?$nJo*@2TQXX85j>cngzoDTSQEFANZIlpQkMw>kio+_ zcSdky+A)vY8(pT@I}EV#+EVgtq(h$nY!kSS;FW`r2TO4j{6+?CT@SeSy5>O1aYJvi z)>qBi(+_7_LOX)Uy*WA-M~WhU>U&e)C3E*ZxP-6|m&3aX_Cb0$jVX=X1`@KXcL#za zDsSsA>bBi!wT?#BcLs7mcae_ShwFtrZxP{7jod|YF-S#8fkt9SnpK#lezKyJls%lG zG?_ee5e^rimXSL9tEEPWhX~h=Y&T=?huDjEbKi&b6W_u8O&=zZ+WQ=rXujH6-dR5B z(We{*yA=!y>=KYJsMx_)FT*w}Cv$+loLTYfD`msotLh7Vmg+^Vp|P~>6(uYR!U!XW zn9Bilw+ILf7eFagBfJ8?;L@RXzGZ_3-Eb$utwJotmm*2!ySm-qe)x-m>3qqT7!QPVnNKPJ|DocB1H^&JE98 zz9hmGX^de2M0=KAgZsiZI2Mh-DAAuGiB~4X#LUcm?fUfywCV<3V_Ci{=tOTlcji7N z7(Q!{c}Pih{Nu_e=FAe07Bcu2yvW zqQ=@KnfypGDM~j|Kk|>|2m>P_xNt15&i0G5U5jj^ntKiA>Stc(qzAxdyT&}~N{|sR zp0%G_)igQkAGWgp`%^J#IN}GHR8Fvl{-Eg3x1c zcnvS;)b6-R1N*pb^zM_co#iZHX`l?1^9?-|s1dF_vI!s6U<;$qB-5!C#A)K*yvcFR zbY-O2hV04YaEZZF75-{a2y4K|KM%#LG`$*bNu))fF`q|Em8YU zZy@9dPc*NF=6FY!fYxHlQAmyet*Q;puR>kwFV+_>TL;s2H2iEv;yh?HL_D^iq0BUY zc!Y`YHG+J|$SD~Xt|Rf`wCoX=nsXE1iyvVm^Kf%)kg24$G=XR|%#JAmQwt6LAS?Fl zhoB(VinkDo5&qbC31dGl{3_6vR2ZA||JSJef99m`-QahXsthHegoe$8*Jb+^4Ki2% z>{N{q2l1DVGXU*-0sGhhH;VyFuBWGGC)cFp!5$JB31-v;tCRipvd_O@8mQ5%szr}b z;ks1U&d|U^laD|CESV`#AY3WX3gNpx%eKrcXKQ|7IO6$F9@O5Tz<~1kc%u4Kl+Tv; z$j&?|zwvn;`THv&5eUSHfwbd`Y&jzKbExWb@D@RplE8Rb<-_EdcrW!nWG@CqF(&3N z@acNXb^Nkl)v9V@o_gytwJi2eA`k*0pvu6|sKK#N>awX<1OD5&rM`2%q}5rl*#dP2 zhX}(#TaFic3n`C79?el%NPNNopACDMns<8B+?N*v6&qWG?DG=|PDzr85)Tm#_$`;C z-2p}UX%YF`85O>Khz713LoEpAilgymXd0@kCx28?1%qi4Psrp z?7QY`KpTFCG}F69mZ0-6KnDA3F|s}QU-A<|S{wue@JRfe2sB3#e-#VzyW90C4Sjl{ z;}T0=#~!`WlYK`vKp)L;OyOIeO3oboZpv-=eTjWZQS6eDYY@3M)coab zX$hL$43hfQFW>x#Yn8fqzgN;Cal$^{>?5q$Fd)IZ)oDel8`C2dB$emB=0YonE@sye zBDgqLE6zVVGfab%=q@whaslz6N~=txmi?l6bh-~R0i?r20Y5d9|M5PnnD@~Kfm%&fburIuX;|Fu-jRU%<6lj|l zjP*e%{{{;j(`b#a_w>v>?lm=9ecy-1Ihs|Q$u882LH{C(pE-RA#zycaP-D!1lfxKJ zKore`3H5(t?=7RU+}?fRhmcT8QR$LUT1g231qta80V$VAhcrm4D53~RH-e-}gGeKw zG)Q+S4H5#<@m`a){`>5`&pz)M=Zt;E`{i9<)(3g&zUMvXb^Yp!ZAuV#UqhK?A$2>T zbjZ$sXVQ|V?y$7>z+OhKC5JF9ZP_Y}1rAblB0F$FZIxqC4y+P8ja4R8N->7O*BV!% zQ~ULsI+MNi$*=ZHu`)(wEMi89mz>#@$;L50$DbSu@fmpTxX_3&ChvbKMp*I#fVE6T z_E%32^8X>u60!8N&^w7~#5$&mLpAm{wKdADDEil)Inzvir{EX42L!q$!Q?LNs3x!~ z96+~P3YfC8&zHQs%OE^>i3?OLEYbnRojnhQ`us@;?ERP5d-PDVda7AUmpIva4+ z{pHpPOo5}+XXiz2-Omq`!!M~r=hB&}{?2XK3vL+*yBr|OPK(%wF z_%))eLEoSNsVhi?uCVL_8j)s5DhnMs1bvwz>@#PwU9Zxv0sR?mytOgkJ>!gA&|rN@ zB7|yPYGFFtQ29N{BaO|9!*t!gs%PaH6+Fd>4AL{|n8qNidWk|Sji<@k`YmQ ziqZ6_@C0Fr$XFQ^1s6R6*-Awa?>!8nxo~p>85uw5MVoLF>>;e>&Kk&?8-IR%r4WLH zk)l3l)2Uw@OdocC+|w$O7w|EB$sHw*KZ|uDCwzaMCp*uj^S!krNiu0l;+Pd&Qj8uO z`U2Qntaktuo^PO4$k;{^5#`9v<=E)bLLcK=>KisW zS%Q}DL8LJMh&!9E$5-tD`pm481klFzf#82_N$Q)F;ke(;zmA1irlmoTZmKJ3Di1z)COZb&8ppS^22^{p}Cwi^cI@j7?)DeX77p8jb0q zA`%re|6YUpe5iUA^e2rQ1#p6d?Vxf(HaJ}W*~x?Bazfc%xi*-t5RQ-#Ze5{Y^%O>) zOrY!!r7(2+BJe>=GhOKo_&$!!L~!n|LC5?u!pT#H?#w$drl59Q7t)i44}@$-%x6F$ z;Z$CU)Gt+#-=W0C4XKVH&=K;}K6A~{Hx*&y(s!wXBa7E4AAR1E&UrFRpDG}G8&P2d zIdb-_rk#h+kSpr?J?)-lRyiCko3b+FFW3nCM;U`%g=VUyAoZ#^2s-e%!-(U}7dm{2GyBL=Q!2f?07OU6OxzY56%5g?Zc zGnU0qmPD4L;5%SqX=y$ON*N@9%u3dW(%lnevao(B3T>cZ1_A4Y+n)dP=Vd{P3p+oN zuCpswzB?{We`QaK(*Coo2ft^Sm%P)v3k*I~EI-SD>2tmVrP55*$6|c}WXDjtBLc*o^J@2@$**D#-EVCN@ zLY36A^M?ji(f#=}8TCqMmbj(sNdqw7O78fI&_gj`1L3qP@ZD{*R3{E5Wl`FXQBcB~(8 ze6)_gYmVYw(aWtniRuSW?a_JT5W z3W5Upt|tQ!K2V{FHgLB9(dU1G2}hYh`kMCBFTiPcnNalJ6Bq(2G|dMfCzmri$xpNx z){1_`U)HVv5%PhFsQ~t4#991Sxi0CA$$JM4tl-<0g||F8fEjh1n*tME>F$O_c49a1 zYqoeHu0iLnf|rZ@`+Ob+lCFb%EBdWUVdQjs!d=tWXeMYMb0+`f_dQv_Y=~C<1e1}n zZ>IFkIjeb>wsdOY8o0Yux!;vvr`FbyHYZah#Z7aHxi(eJuPF9@QH?o#j=_+&@x|&2 zT-t;EWHS5J%a<=pmf>A;8e(s501lHLlR>{od(;7WT0N@}#dICzz{%vkYrF3J;^ku; z{AM_fTUV)WJ^fKK_NayEA(R_@Mp^B2tow87gu0s_bR`P+g$XZd>H#s{Jdf{-TR|)0 znMY4Nmm|zrf#*iZ?+YfD=*c3N%MjStvwZ>=H+4>MYiZc*n|9C8&1*c&SN#*@Ep=Xg z6#7p_iRUD0)d$Zgrq?#sg)RQQ8h@efl*3iX=MzEZc`_jYdql2u>r#NJ=#)jjrx(__=& z4AwwogAi4(JthJoGkb4BS1jUZrxwyfE7sh48=7KRgbF!{B39_PpP{EAqnGd;XeB41 zgtrW#KTNx&@aMGj*TwO;_D%|(W~fjVluTGIp7&kaHtjtH{cNL>C9q*R&So9os+lvW zs=;W10O`vukVMq}{sJI$ahKFeg0!f=-`Q048Rzn+6TjquC*0q8dP12TW|~4yaJ`oI`y7tqiRs6NQkqKubf_f#JSlRODaTw&=!%6#iF8^?c}UR0hA0 z+dHjIvm0nlBO+T%%5Y_IQ`tv#Eu4#K$MVo|6dNNF!}@v(vNis}6KZZ%VPR=Fwq62R zN(Yx%jzO3K`9(#bJ;o?8B8HK@WZ#?tSmsZY0kj#RxFNc0XL;EKWnYI>H&hv8*Ul+M zJ(z7FPi>zfWHZL~jc*R6u7$%)0W@Wr0`pbyg<_QBUTDK6+wZXb=3T*Tv2gwGm4QY~ z2ssE3b&_wKumDj~${#qR3#|HiG(B`6+BQqKFntg(bw*BezL?ef4O82$j%@{OaUvUo z6dL&*%oSJ|2NozY;G7;%T88txajx^-0|3;Kj|BCt$8cv&RrC0`A|OS=b!wOBB5SLy zsrZ#nV5S7)0Q&ZNu4bb*k&5Lpgu(UExg9Z=es+#a z1JU7KT)*~w0^m`3AZe0~h8>o&_S)~QD0uh+jJ{x->h7IhPTY@`iNE@V28&VdccZV` z&hBA%>>t>Njxn+UwdD)X(q`t~KsOnsTXmiqm8Ir0P?o!yNKu|UhjN6k1S^JLy60>U zUqpF2Tl@BVqDRfewD$Xczhc+Q>J7_H0cB>qdoe;6QifwjIwhV$r1Be}59FcaD}t2S zmKHc=C$x&(`2((c>jxd$ebTKedhmgE-|e{;{r9LaT*hH|B73|agDG3pg@G>68?1|_ z+fepnI3t$qa}juFb|;6gCF4Br8{dKlsU~~nZuc=JR)F|{o7UmaX{E;TaU-Ou25b#% z#qh$!kky>#w-;I0HY^B7%R4@bt^Upk&CpU{*QrkI{fgr&u#Wt8MGQe7&(P9Z&kUE? zZM_JS_)i;R49CL$i9tRmKVXZ^BE7J&Q+CD!M{a9GxQ*h>o>qjX*83VG4~$=;L6WJV(O$yk9HY1!NK z@e_h+h(sf}bh`u@D}}fLnO&6`^Y+$o?0ML01W#ZrPU}>~Yr(ejVoNeg8-V!;1M+lvu2M3D5MtLNH8JFWntXkE4zc|#LxhI-|~YidDEj9fA z>f~>CIavdE#eb|Hr;2ee`u5^pf%CR}fVg|}$Kkzi0!^9~pK)*J)Cb^As8`SmSa+NU zF+rQ>-los8ke&^QGeO_#i@z{ZTG)FNBQCq0e;;0cDS+^la7*dQbD?Fq$_QxDg##o{ zC@WBiDgg+>m<%(Z%rUCx)Y1ccC?$F#+y(w2gzEJxEzyq~@>L6K{vW5V??q0v6}+sT z++aYe#=Qsk`i6hLcwJnik1@88iarBr(x%IkllPs~;a$5vP8ryw!Go{gGOL{(`L3|) zMDrblgz#T#BZ{hWf9f@Xp}`apiC9LAME?EruP5sq|JEx zeQ_!X?DRy`0*!*k*I?ImnV~t0HiF3PI@!iz&g7pjf+7>h*;JBL{byS+e+lKY+(zmJSo*i1S}C!~dMO zdHRQ-6?K;~8r*94FctYV!32m5ch(il&u#}|GAA%5=M<}){|A0wV+)E;>08e8Q+g7> z#nT?HaVoap)+82ue116xbiZ-ztPnO}0?dgNX*%RSJ3Gx~fcpFb%G2XtuQ6t&KEAw$9d^08OQtPJpL$&_$h? z{{Hw#TwX~jFp&P5Ub;#G2hx6yfQwa|qKO$7A3tvAA5LGm1I)myQjmiAsf4iGNbpc#J(}B;|Fe)C;+a)UW3G*F@w{^>J+=<%C(&7S2V&+(8k@>W5b9f$>I4m z_irUFzXl4S0Fh4^-4y-YM2+nkww&p(b&BRJR!RF!tN~8Oz?UzA;xjaT%^rpe=^Nh?%JmMmKTz=xnIScLfmsJV@ay65%gx+(;1fRn z1ZQjyEx*PmA}S^*Mqku_YtB)~Z6?lDIRnrrI!%v!)%H#VF-S@bdO2O*1zaii$3=MM zFBVtua9!H>*l;{UbB+~6K?Rf3glv~Kh)))jxnUg0U>gN-X6Y^g+Z3=gP=7}P!vz5y z@LI9vyTvapB+5LR7fnUh>J|v@u$E@JC0Nx+~+(TG+o@bc#3J4U5;$d(n-4?E?cLtBa^@2?YC zo4>k;f_mbEK4eI@Qa;P??2M5jD_QvXG@$Qif!xUoH;{?SB^fxVAq%idNDM3KiP4@R*|Uan^B zE*_ly4vo1j)1c9dj3kuWGy&Y4^9~O`DKW9KLlO`#^}|SQa`b?{H=kg=_x-#mtABMg z`)uFG$+K9i`@VU=(3n;M<926)A}XibG|e9HQ%sFd#le2+x6B`*t+E-}{%ogI# zxU426S3oz3Q;Q-ynj^V6p&&x3=Ph95(;wc*&*`=Sa8MXs7{)7L-bn2m==@1eW*m8+ znfXCO3pwjT?FkhMOouJsE|5+ytE;R3(aW)dnGWu784t9^CUn4j6&{<*25(4}7C<*z z7#q{UC6{oGQ|zKrn+=O35>$pi$+D98Wa@-I$J`ZF)&?Hb|3Em-Llr7jV$sXjz9qG;~;_M=7OpN z-;MRnKM@Oc{_=?$tOkplRXd;b2g`%IPt|Z^!{g{nThD7}80Ow1e5sMk${{8pS1I%Z z1bx6EPX7WaOc?!t^BUZGA56(%C?R1g0-Q1V*>5tkdgu?HAO8V92Wm$bwxX>>tdFp| zJ$p{mSV8gccK+}8e-CW8M!|iL{Lpf9&@5>-CplhK(4sD;~23r4?2sPz)nUKTzIQ>5H$^$N|&?-3W!z_{O<) zK*!fm5EGmnQ_#woR@8y1aiXl@0CW~83Pb@m3TMH>^y>HOkQv$WcVdJIhcXIM-qK44Q)V zI@*vb`%j#Qf%q3720wyVl(Yv@njxTqkt*ubO2|@2FhxUk|4ppv7KH=_ID=}E*6{|y zS(RluZ@*_L8TT$fySEjj1+5h5#Z>>7{PnFrRyLz$x3Qr|0g`~vb9?nH-McRtcrMg| zpor2iZI8F;vlxVQJznYOL`@<0Veu7cfG9&_C*Uw=dbLi@q@zc;{xo?V4HkU3px*+* zR-vT51z;TPhwSO@nPou7pph&Rhuildj>UBfK;%deck4+&7(K)}yrq@!)LDaEDX=TT ziNKb(51z?16w1~|W~r$?O8<8^`38+7C|&|2QTTC|L_bg*X<#w$hC=-d0yRst6Ru z&0~V&3|5&UgmcZ)%q=B+CIC3_xO3Xw*@>yS1&I#4nk}Ea4<7ONvt=(A*^Umi3R@3^ zmH@QlO1r_K4hpf_l(aF}kkEfMRt z>h9g34Lh4notWsi7r&!;l8P5lm79l~x!CcYK zEOdJ0Hjw`BBuLDCCnr9J0gl1e;(2pZ$dMIXPu*poX^HCS?Z1LtBp4lT*Ul6T#|?tt zYwP?i{W*Tk?NJvSK}3T#yNmdK@C&0LMXv?orKiEs5r!sEe477{RJ#I{Tac&GCBWM2 z{U}DkNP6Ci?T#Z=iP2wrK?*NPIDgpQdcS!Z9=dC5+PQ2X)|}T1y6U3uqUxT|`p#Eb z=M33-4lQ|!gAL7YVYLQ=9z7{KU&FX(3-{?JdHx~b<62V|`?C25uLA+)UT9?6#VBA~ z*yjfi6@}g3$z~JmFF%>{JScN@xzQnX`Agf>a1{*;6J?0#3Phu0%Y7pmM zH}Cy|!~ouUtB5u@9yJS7T%`gvD zUcg=n9MH#OrD$wFGf5j+0cW=cv?{32pd#S{>$hBmGs1;Xn|y5kEy7!WEj$(id&KzC zdg^p^cvn6_gemPG+}Le=%o%7s`7fSmWNOboRCG~()LR^%3;ZpJlPiwsQn01k!H7Fo z1a?*Aa8^$G^5O+SVGejU*e>_{xD-FRmFC}f2)0+TL9ZgaB*%%p&qb0GaDFS-#(@(% zw!p&1*rCfo=@yXWjU)sFkK?E3;#Z~`TXPLsz7CHtMI=!&8X6i#MxK%)foBZh7nUQi zBfxAAi7Q{iPZAJ>pj>Q-yGey5^p`8HAEp3o7K~<{E%lXlDq;N9*$=Zn49uQJoj#mm zA@(2JCf0aCzX7O%X=VU&Ae%`K>r(245lVC zUDb7e%d2jyZ&79g$WzRaA`@fH{SRyNyFILdt3gJl5jKEZJzo{kLUJJ?aOSE7{#6m+ z&onqnOpUjo2SI^dyeR4g2%=^%_Lw{X$s308!sBxs@_(3At;94OCBU26n4_S{A|x8S zj$?8kz&WrAWl%kZxV`wR_9^iqj>(p~`?l*wP@w+;;0VbVvF}B!Q@qvWi$3QAdPx z$b;(WY5o?UFkim3G^LW93PC9$69wXXg{EyLfW@H32=tgTDaL9IEbxueks~`D54I{y zBE;aW`Bs^}eUyRpkh9vk`G7N)*_Idx&xJ{9NCFa?!dn66;Fr zRu6(+vFNPLSl~LTTOOAU-y?A1XmV!abT`!3n@lT0_035B4}bIZKkzqWSmpSBU|lP! z08I&k^L9YNq`a=4&Fu-2u4!=f>Z${di80|De=C)>Gh+Um_#|HoNWcpmYgtMkZPL!t z=Q&F0z|5^M0aFN`LtZzk%f`M`dOu+zS|~_n6dD(j$&GuiMPLEb|bI;bzh{~*FWFdc-Pk#6Q(}{5g8#!On`PGcqG!# ztMi9I;E&Ryt!0}?cs@fa!_6K2!xwtK6nlEN>4~iM9&vG#V@OXDOu~*I3uV{MAo^<@ zFyOBeB{y7PW}b$kI`Y`(E)Lfv_4+e^hVUoEwocp(dz&7FJAZ`RStqWGUtn}v1JNL4 z6x}_~qi-JChJ235y8hu6^mM)g-=Ug+F2M{XZE4?9++m{rSm~>ELN|h7_+So$vLf-V z5yvigmOs3LmG3LCMobU_Le&7HLr>E|kQ)7py{dab)&HRi7U@Jj7886Chv(v;EP1WY zT;I~)4+5?-YrbGZ<^V=x5!85KOv*e3t>hyz=v}$}86++s%QYzT~HI6sKTko3~M#m3zB^cdrNY-(U zUmoO;PT69shyD)si$Y^3YOw!)TMY^KHTkyuclEi(ScM-C2wYXYl`p8oPb6RpD38B( zf&PC@t`v|Q|1;N{62045d~@vm4)wpu3#^(K&v-0RuMmXXLF>NqKl=9(;d$eGR6}#7JFXM=Q{2c|`Xl z41h#2xFAtc03mZ47&!{yj1Y)AwTf#PG$nDcO8GyBxr7*A#*&h~y1BIFu-%fY-|}_& zvjNZ{pXcqO;h9eeg+{#*{4P1=D`;XQ_-KFkJ%cS6g=Yb|rLZ|5NAE6!dKk!)Rkk0* zs9r~b2pDmdd5`7Dl#T{N2o`amRU4)2Tw}^fk|I zxL9C_F!Wdw6tVy5Cj&vb-hQBT4TJXa7W|+5#0xa$kL&IyuCX#iUb?w`hAVRuc9BT) zBS>gseeJ_-OcxtyO8zfaNJ{n<*z-RT(IK;EYk_N`xQ6Q*Cuk;agf^f&5a@>&_4=qQ z;es2)3hvo`tXl-7j(zz8ssg=1;OQp#;*r-U*v0!riMb%YC**xKX&&`4rM9Z$y|b=} z);mP`ay5X4xAmPm_;%e}@fkeH>~#0AM4x^Ebukk56y%lyR3hU}a~u(Yr-@VZxyU`M zj}d#2@ObHq>lyDjXxPm0UpNw4tbkO6U$K{O{`D^9%+d!KYOzNwWd2?1IT6)`a~xWi zscsaL-U9$1fiG(i7sC1|=+7d#uwu4h<*sx8SEzQN=j`4P#@7D(f;VvX&}Fl@BP!( zhP3PVM*0;RN579Q_obP}%hZq`kDX;4UIkDGOsWcrp-03Qs%KwYoGCl=po}XcK>jIJsRCN?S7Fdk?cw;Vb#Smv1z{rgCk-P7FNW)dpW0ja2}b zX8xOy=yFRwOb`WOLNmCJ(_n^M0A3I;bdYB`v@VU6L4?zl&h-V#XYWyiieIKW2cp7& zzEZLCt0u?WM57CKb9f|moqri$uxx@DZ#kP%sQ%cPDbbhDpkaxeb8}q;JiuxOM01)c zHiZ7kYH(b>Dv1|^{RiX<)Q8kHqJO_pB$)QDgFbLlk7TsO2n>jyM!JiwKpsXXW}5!_ z5T+Z+e9;GHt#uNRA4R}LCvA>E^f<}AsPo(&|NR{lT1nN1-KcWBbB5SNRD8$|w;pdV z_O$J_EX|O~a`Pf|_N)QxL{n^TRisV#co=U+H7A_uueQX2DGPi1ma%{BIOf9$2Jca| zsQE_>!1z}2?TM$Ty6JZTr~L5aU*B4O@LW41t^#AhLfg)Ly!@XC<2)ugrNCcq7Ekm? zHpbtsX4qL?+6tsw(&ITt##m@utyxi7VPey0;l7!BJI4l){cyxq9{Sr~Tdl!?^y}8M z5%+*yU4vAs2aV%6Cajj|90y-X);OZ$bffZLLwkPzO|1RsLl^E3cOC~zK- zF9nCL?k1<2)U6=O3;b0F+K~tQh2l}wA-LPyu{Mqe6=|K7qo6vksyh0ph(OS3lu^+p zD_EE*M9K}rT1QaV35i9wK)_o#wSvQkgLO~nx)cGU7#QV@ixr1US~h2m&Nfu*%HV*n zK$P6*JcEzV;P0O;4)^NV3aijb2K>kQsPlRB$H9Fe_oqm~+Z4Rz$PJ1Y7LPGEMcVxQ z&|Gcq{@oTa7d@>THHCja8*=R44NDsD6)*O$?FcST0UNGRr6*{xnDc9l8PM_b5oD{h ze=o4jBD~8`0{1&S!(WMd@|yZ{WH8`Mehk;x486TgMT9tR)soN_rGETp*OvN3ig!f6 z_hi4fGRm4xM7BL=*X;WaSR$CQ&NY{Bimm==fTA?qYNu4UPRaoXo; zIZ}izv*)(#daE{OeN{|d=*IcV>*HOhOx^}sjCs)@Z$Hf6TTCgcY<`$3gHTM=Lz~3s zismO`25xsNEs7mib3!?-a4QC4`eK*UdHnxc4DKF^pj1j-{t%Wb|7r&Mm5WQ9ZcyI6 z@$pZ7v8ZRBIe&*>;O`C0!LOqC{VizF6rToBiTJ&JNEap@@|Sp zU%q*(N|oAc?J<-MG8OfFgwOVO??y%vuL>Lqq$B3VfpP^VMMwL6wY6hKBFT$0yy5Fs z{Q*~LbG$&!lV23*+xySkxx2e79EujVops+WqTO@Ji9d9YAtX|+rKWpugn9gYB1CNi*b&~;ACj&ut#*AMWk--!G+q`*v7XP~_;bW`jk~1w|$L1_X z%5&Xlh3oGGBY5e5SKctn5nh!6-&rU$t1P8n19|r}AU&&O7!-3{AX5wlkB=fcksx{x zC4C=DWT9Lp#!99^!HE#fEZ!960;yRSPZH3P9LEQ&L7njJ?h!yzO)z=n0)yAao9*rG z)ZaEx0%Lpw<)oqmkz8?U7o2x2dRxt$xzWbo`}^;pvA|i7Q~ztER1*PaDC+Pmuc1Dh zN_@2a3N5-wspMGq`0yYLJ0;9*zKiuv-WFg$3Xq^^FbpFPc|jN+3Ny2%6TNpW$SIFM z?jmEB5*`HbKKAqXZvhmB5)#58$~GL#M@;}GP9vf5izw(qFup|BiSO|hWzB+GQ)#>! zWF!ip*iMDWs({VA0I|i40~BnVBuCx?=7(D-|Mg&Rt0USKB}8Mo0(>u1fmqhOVjAj} z7@}0zxL=hVLK3>EAqt}FnAC?f`^Sq_OiejUKw&8*U#1sRub&cY2X&r z00R*O@Ta24VVF(Bgr@%m-2WCtw>TsTqnTURr(0$>S%6*{Z_R~(zI0*4DPmwJN~V!K z+AjouJ@Z%@yiXGZt8npFtUhVFB>Hed>dxB?tkjnd(tA_+v zGBUPR#B2cLbSB|a_UT{U8Nnz#6Gi>Qy!1k=I1=GxoOY}8ol%C5*vITq)LoVc1RKAl z3PVi`M*M`rgEQM;U@!sIhz4Y}A&?JoAxK>a1H)Y&5PCsk4w@66b04NmbE7Ed5t#9A zDu0qLvkGWBHZpyP1LR~^R#Q{ct|~!q8p4HLi@W@Jk_7n~lo(5(6oU{LsUGTT1dfTm zL8ND$DzF1RL{hn7Yf$BbyaJ8Z4fR1JGKTThX|ggx0%uVudb56X4}F1*dZea<6RSb6 z-cJk`jqu}Syl}{aPIt)%os>e5>hT7keSuNw1^_p0;d(<{MNrwydl?M^q`;n+`;x6!X%BJP2}@=!wbMK1~Y#{D*8I4`?0~uyA;A*1d{^67zsjv zW)yXM9B0X;qXwivg!(_PkY%VPB7-1y#P_?S`Ohtot~QNeDHM?^vG_PnR?qnW-kHayd_oU7>F$9yH|_r4sA68JJQpU27*n>sgJs9wf2<8TdBwY_UG`?$5V{pP(kTn zT)GAIO3e_g33r?oHodmYsN?lS^aG7HqFUEdy9kC$Sa-;1iuQIG8iyd3A5-X;tX@5? zMlr^kED`9AVE)hYucgvs0l7wY9R$GiBdad z>|_EN*a~Y%k((-v(@1Dr`A2&P%oAaw89=L}#)ylP9Q3rOz~i2mm&Z~9tIhx|CMZQh z!M~&cQF>{BIsdd6bDJdIfl3P{kt0W6lfs5+E}q|`Lo8Q_4!4%UsRBt? z2*Y}Kfa4H!eTYVjfFeJyEE?l$J%hr20aa(yV<00loUV(b;VGCa9fb_xa3R{_q0QtBT5^tFBgF~j<<50Ua?#TsEa z;9Da{06J7A(l04^d4wVBNqp~DVY*8volPk4_YWQIVYUW)xBgsC`En@i3pb#4IJ1R)eZoR;Cu3&A=wL$_MWhx&zo6=>Ff*J9`QHo4z;Au=KvKiKv4W{RY zMWeJPQerdAhQ_}iOu2h;3hL<6fy5&&F;0UUVTR^f;fu=PC05QZ@%2O^{1pvde#IZi z%XFy#%MS_6Ym$RVd1jc3HMR+DeeC3jGPrM?e~p$4j~&8|bU>g((|tqt=?J&I6_~J~ z4uuQLmK}V5W>Ha5@(v=PlTp*V^#|ywuu(gpN=>Up9B1n99(|W4jroHjZr*`*Fu)rh zuITbr7~PpNO9SB>b+8tf?Bt-7q@&<9RQf$00(X(c#~*ZR6f3R*KUNaE#@R=;t32Hm z4?n^5t~)zFRmR4~W`qqHK))^ebrxq+niy_(>{VIF-%1U@2(-)r6=)&)ofn`v^`l@! zm!s)5)%$C&Xso!}I&PqE+b|OfsZge zQh(2KDJlI+->lcfWBgWw`(B3-klq5b6c_;_h=2>#1k-t99?arnLWFj`=u1r7M35BW zW&7{F+eSySViLt8P^0(ZIw<5XNQN_E{whcZ>7=5_d9|bjW z_x`7}@Iylp{|DdoD#7JdCup%4z0v%uTG{H1pnW_@rVvVxv@|=QsZnnG2~~Tyf%kxK z?`d}5*4gFudm~x!6bHdoXX~`j&(F6_QtgLbeQ@M9u65$&cRg}$o29l2>HWo6sa>dddN&?#$BNKk%N?zR!5+F>;k+U z;q1Ca%#`ygmCskXFmT}o|4_#D6^Ss=6a@U? z=t(+N|I!ftjTI-3%DP=3$YQHbml=@v%o(U;e4eBg-Fssmy8`JSpPQ#>3FT(>US-CN zqwJFwb7U`Z+6NmXeeSt{F6t{bO=n7SM^Q2W7{D|jbYQ;ooxd){maFtdo2-=T+q>my z+}cHdUaAnzuOxxw)Z|m^{_sY0W23wUm`;m8xHo1sXP0wWCAKGrtpNC@b^HUZutxJP ziUcdEOl9SGFkpL#PCTr|#Sp@vwa@fc7pQC=p;+`7lGRul*|;Y2FfE(&@0FDkikh;J z97I!N!=D7KI)#C3f-E&C&eLFa`vr0vOW$G%zb$QVHZz{yrg9sm zTa7GdRR6B|1E|Yo$*Eg=>+`{3W5w~Haw5bl-c-e3=q-KSnp{}@eC(R?tr5MO7~e|p-VL`3^8W$rR_JI6 zXogG=0a1!=jU(LIP2cAm*tt8!v*+v}EXcoS*8)=3zfXB@YJKc}5XRRCT(fr&O6oLL zfi3qIZ2whmW(xlC1F$tY`xktMxUW_L;~ETqd_62SB~0e^2LR(wp}Ez{vd)yJW=WZV zf$mHW?ty&?QTj8o(|^KY0V*}wA8uMKQo6Lv4bK#^1*bsdfejpVra6LdMFYbM z<5hq}XbLG{GGlJJWPBe^Cc$ENRq@|`x)A)GZ=tyr3am$yZ7rcjD)%Q3ptMNfXEUFzU#5)9&Z>Yi);sD=GSy#!}eD!^{TOt61 z$pmYHA_T}P8Hp{CSHpwjc2U;HY{T^yzf<~i{_P=Pstdlvj`dVH&4*ffS@Or4Zj7hv zstIzIxYW4;aX#65){=r`shd>4|Id0mrg1rbuoEO>w18?0YgT#g{Cx1F+nxYU$3@YZ z`T+7EplUvUvXWJw{VWeF+fm?nn#ysAFQC^K#~c?HzZ!V|%5V4?VFhmAAdo874>!bc zw~1JoiozTKdjC~8VO>1pgOZ|F9sa-$lJn=-2h#cWI!@SuC|WND0HwY^41$)gG%**- zogg&T2O05m>>U%TRd+qWRpccHVuZGWbzz|_u6Hp9CD}UpwLfNOXLqx6G&6$U0gq`~ z3eCL<)mLYAX@Y?leKqq5$_epwgNhLx4|eaEXD1f&CU^aMJA`|4MZ=Q(Y_@XG6hDmq zxw7aSSnFr#bKvX_mhLnNEi;Xm_PTaWJI>!5n0`~*w9ih}XuLwzGjOF}@e(LHud&i& zxmg62UNeBjg%hnHqdTSCl~Qbm+|JckVzV^w#+hqdE=^`+s|b|pTFym#A#rdM-uux@ z8?NnxMBTa`TWA#QGU*VutdUW<>fCk}Rm@j@Zl5?d+q)HR`Au{0t9y6L-%ny(VR{%) z8&UTjNViX?sVDbB)>7$41LwVJ0^+KoB%@lb0fkPfwn6vNm>*oCqC+fie{XK8bYAIy z*LH71MAH$4l|sWWG(I|HG!o;~O1I>Zp(|k3_mLga(Ot5~!OhbYo~58MGHR|5!^&V> zP>6@?Mse}or(<(H_jrpqZ`|_f58bWo;hQH`u9V5SlbcxjWj^(K3|?SaC4^JKRVXQZVuvsQ;n&*|+C^QJMkjjuKQ@o94NgKzM*l zwBs(+X$^O)U6%9;4*}-Nn?fauZ6Cwchm4v=3{-W(PZROw!=P4e`JODD1}D`q^0%iC zQ%>MsTtsLRt%!q6%DZ=nN@Sy)8>=io%cXN!F8%;2$(U+Kz_=W?SMS+hrA5OQ(55#6 z`R7^QRcQ7mgmA@7Z9?mC7nGby=>?X9C@W+Kp*5hNn!!c`Q#1-p6tdu210g!dWya<; z3QqhcxMTH};D~{Q`bUluXz&sUyN`zk0Ou37FgsM~Q3CAD#ik{ch686){oRbDxI0E( zf@7kP4uyDEAaRgf0c9BdT%tRoH-_WUWPA0;8l3MfaK369Xk!wuSK~t|Z01b?o{yv{ z??;nxxqKo(q;X3}CmK499#1=A+Yud*VJ6H^!zHiUS#sN*&p?Wtd2fk>rOuT)s7p{z zVX06iOmf<3>#64Esq~Hp!3j)cP%`xV{dG~I#L)*1cpmr3*TVG;4Gl|Gu#Uo!<)8s% z-BA#Um9}m7VowzEhA+>4OMn(vABJ&qlR|_uDxiJ`9ZftXZJh)*r?hWnu(07{|2R=) zP$QQ@r8j_`sn8M36pFyCJ|zz?6Eb|`WQdKrx^aISs`~QLDqtt&A~C-6n?A zta2_}P%wy9R91%ViGJ?VNZZj4aJwV6lF(VdgZW-f-f~*dU;hRSz|2wvRqN|| zP`WB@InLYpKruaB{S~fWC+zwq4-jx&C_0aZJn$Fu1z+2YE>+xy z(ImHAoHu{zI`(U<{InpB8@tz2Uh-&GGp@%nEEw)fG@EzUA80R2D?enpu4q`*Jtua6 zXEs(WUyuKTD@UB5m8m7pAD#*1)&e~!%Fr7YknS0hI@lj|>Q+X0i{tD81bE%rTDRo4 zoHdyI4Apd~_zc|8C~$U_Mgr|?b{Gucis`bk7609A;=n}|(* z+%`qeWqlazDt>>H85Xt~GVxaYBc@v|xm>o$W6_iOCQG4boc>_U5cWyP!Tnmq0rTB9 zw@cveKAR~S*Zt53(P@qeZ>yzu=_;UV>(*mto?A}@$s20)EMUNB$`TBZi-MLtwvSTp zZlw0mduE6Omkp$30z%=|Z+qko?UzfGx5~}{%SevFhAUp;D_^omBqc93rn~ykIK_q< z7kmWWtuSPc9py4ifvUIIz-J`U?I$6XkahGad5 z!!s%%*;#X$8_Gp%m|a74rp6VL{cxXBe>DFdrB(PH_|yJhX85SnrMTUm-a?3zQv_3* zLM#p=s3Fip!6~qCXBk!WZIAsnd=%nz+oG>XLza1(362UTQ7(>Sfw&%2ag3cp20jot z6KAZ3BYQB~yTavK6mV~3Tvs*SfL<$@{@Rq!(Owc^`p>SK1$e4kL;Aboim9a*iH9J@ zqmZ^A8Y0UwVl;ga>F))qgYbS1(cK3n$V|DOU6rZp(78-F=V#JCZPK*8Fs*H*Wr=v4 zMmG^LmaQcJGzp&jm{-=L9izeFhZyr}c4C!{79A_e_#)CoL(1|Gf<0muWy%ooz6?4# z&AVBWDfR}XYe$>y;!f{5UQV8TmQK}~cuOlhEuu5!!Cz@w)!5v6A6r8+KT7g(^6|A; zf7LFUlM6Y!G-&#Y#YXSm&48kiPE)nBORvHe2G@Q%{t$<`uPbd8G4sOAD|?S;_v78| zW-49HU|!pZN7ZbJ;ppgSy8=+L@iZpYIX7u!ebdwu+;Oatx9c+2K4xbo=Vf48&i$y@4*%PfQ6{p3N$jk7bL zQE@I+o8mzOd~W9kGf0kQi&alNIb~VZ4?FDq<|rHmVIVQR=ye3M$%dvWAe>ra)D@&f z%lv~r=Xb6L6dp$*l=GpQ@{cHrwH&}!#OrR81U{;F0{H2nil;brJ_oOUP zN(PJtLj=Asn)LS}=!e8t2TsVdNr->6kkv_BD?7XhGG*1WvNDd%>V@|oP8!V$_Um_t z($-$v9H^TPgvz9EI>ZoSlz5D;Jf9{@bm~&Lo$Ccq8jnv%KkzfVt+d*PN!L7Fq-VMO z%t>J*bQH3Qv-uQIMqi(+)9KDgF&*JpI%<13ZKmfqh&nXDf%Pq?-T7>R|GYHL78_{9zBf-HV zl23%`4L3ym@<3YRq{JdhA>x<9{n4;;$e9d|m*A?ZIufzZ{R}YbIdZ;8i(RmdlWL4q{}kyA=&pn|KRS-kyo z5BiHZW_(VUg#b!O1t(tQgi%IQQu`!H+zB zpj1zA_3#CAkojEvGqR;GyS&)mOt|9NNuE#$ovapae?nJ1o9F=(N!o7BKpG8`3OQaI z8EL+^2P_5Gpfg@S;TOlB{PK*1>2VJCr-9mQs;e(+dSLZ0$jr8CuN!(}wg4@>wO0>^ z@bHP}R)6~2uKM|2J#}<(&YaAzZt!OKhub7fHeZiU_pIX#y%zNwkQp+8$?HwP0nZK9 z{}TD4!!Y!0@e6%N^cCh9sP3%pV~7132d02ywjxq`8`#s&{ZPH`-PpRh!R0`3^RC-c zalbzP73g*67H3$De=U3=`VQ+TO2WVUizDTg`;d6LEJAyVexCL zTw777&m*`wT;H&4^Gj4stMIx%L$>%*QTK?W_;UoF&imH862w{X3dQ%BT0E0(^nG@{ z6DE2t27@+>v{)dKxt(!-#gI&n-bma8^m-s;woRBD0d^(Ts0AD$KRuSA9&`h z*yg}}1B_6SV4J8P-0u86$uBpzfyOW(>wFTE?LhF@YN8ZMok|P;ko*@6lqFK4AKuqW ziP|XCNZ<(t9N0k&P?^LyenUb&K7V!!|C`J}^HWoG*SR`~GLvNKDNhC!>p;Rr%QXVW zRO@S!3r4_2B<#hB+Gkp&oRxij!B9FHuW@M-6gy905w%|n70!e5#7VjO8W>K#UDLo} z_rpDWN+O#1Q^bLetci)2Cs&p+q+3=3YcGt3tR<}RsoyS07q!HNA2gqhx+AA7x;9C6 znhItxwy;XMG5UE(9~yKD5nd@S87wrrV}QT(SckYYaw-qHS?_ZV^$Q~ElEKt3GiP@M zx(!)2{C7T1iL1$rq(6+hv-pAPDyf!}^n(E~27wIdNS% z+_Ooi@_Nf-8Cp4l{2#{Ex(?;i`w}d?s$zITy=$ZM{Ayd9F<>ljQAJK)&CB1?9f$-; ze!y;%vryN_st;<0LS{TKerK)xQxy~|i(A9iY-+NGaYkVjdwwrj53bBgC0=m$Y_d0d zCpnuB1Me&KutXvQN!42FdA?IHe=@bq&~;|0qt2HKngUG+UvS+fbGlrmOcA~iRGQB_ z;y&G@Jx$IieaW9z(}5AI{M)zf$MMo<%};N=qif{H53he!yA`I_5_rJE)g9Bwbk-)> zsM>q^N=s>tLh|~~Y8|Nu9gMmvUHK~E`}_LlVARX>?>uR;1d={~0E&`j zTBy~QM5}Uga!yV@&LdVM5(-xH!z&2gYE!6t^ZI(jE6R50T1GJp?~J!T5G3=PPi!+O zw|nX4|MjfApHfiti0e8{wemx4%VGtBg}>NH>FbD?@#-X>yQFX_5cS-V_RovieCb8v z@H2l;L$N2F>S4wGhrE=umkE+b$!Z9nz4(jc-PI1DMx3@@T_?iH%%g+GVV0g|Cy|h8 z$e&&Qu9`p*7I1y82Xs%2*?4Y~61@1%OIDQj-Gk_f>s9-{l@ycaD+Jy3cn2FcS8wCW z9CTdC4;5Sf%)Av(@bKA0g&NW5$&~vod5gH;$ug6}Np5gtpN)che`7H`xu&y;b5~`e z|K+E>hf0ak@;=IBr1;dcL!=w$o_WyM`Td}zp*l|+G+0^TlKJceOX^>vrRXw09u#TR z$i6X8?*EL6`htafw8ObV`{b7w3jHX#E+ow0PHYTX32~y)ceg0pnea$5xKtIoax!%1 z1OkNkJQ}19VY1o_4r!rR3ArNJjpiV$tgx&$a2aGhh$CROd9uK%K62q9@zpURO3Jr> zI&C+`seV!tojd7zVPUyq(BE&?-AKYgn_&WV4-=z8CX6ZR%7{X1PRj@!EY>?;p8$iD-~4o*5<99*%^JztX!EE zuk<)?4Zl9~K|1~p_9?T6z&EyAq_Ob{#jW5p=eOz$rMR>&(6+zpK3CcjNPDC6(~y9S z5IZ9c4!eaZzYgAw(x*CJtZ{TEY{Z$0luy?E-`umf6of77&v))esD?@0!7R}NwFc>n zm=A>HY#Y0hU+$%de}%qxG*l(N_hF0*Cu5nv=0wL|Ya+!gW!cIP->wg_d;ES7*yDIXzM2YbqhkYOZtIf8tr4xn}Qs43gWW^TsGF)iGw(= zZk=KhqP5-(>Lt4JvF9|uayVOKEceNK6Yak!ytw%v-7_Z2SOL4O(o!j1-M1?rzY*f7 zwn}&b5^=8MHx(|6uGf?Gq4#>VpKhf0{&{F0Bws$^ePhf)-S|^cXUgx74k@kp?AUER z!d$grZi;`}vNn8~j3#m1g^}M`4L+bxnMO3J}u_os61}K2Lk8p6j;z{Dqnl@}p7G9Fz%EejORP(>6 z&mqsvB5E0E#DDqJ^Yxef;I^bqqchSRc$|aD&SaGq(FV#Ezdi{Bx@?Ql#J2*o@9)b5ex)>z=g9k-g$bX+AD%__q z>siAZ?lf$>fqCNcs&|4VR++Kh{{@$gWxw3STf^uFm5@T;JwAwqPpeiOe1c%xivX_i zb5e%og^iy@UrlZ2JfJX~|8~BZX`SH4NtVyU9oD^>^CIoOc!*H6> zj|{UiB>P8AtWo+uHLxB^Nt^8+`L2-h*B(|1#hq)J&BfPF_dPLr)4$91HjTPrNcarr zHdJFbjpS@0;!{b#-y;t+}k% z)t;JtM=1j?t?a;Fzo)upg$yOhTgWIiEVxl#fzLv z`g%qFui-2D_?9u55&45w_vmW1OYU5I__6p+gwhY{C6TJ966v!3xE_tRw>uv9 zty7c9nyF#mFJXA-tNB9Rg;Sa|13!7}1g;U8TXsvcR{zSLTHsob&0Eedw14#X=NjPd zN-NWsz%ixpl|7f{U%dX&W@+P#ehul?zt8in&F}xaxIEgv`h;Tpj=zCsPxFIipMU?gkIEwCq*ebsR5DL;dZMb8$;Up{q! z^U3zBK=9!!x8g1KTDJYguTG~r-i^-M@#~;!{o&5V{5;K0H@KRD9{vN4-KC~!N}tkg zEiEWC(3m(ebjCa(z9|y7G8XqPdhWL1J#duH|NDO6^>RkXS#yAK(!Y9rRDN%GeKl}` z#Ojoyiba?3ePO}9h~I+^h*4s$wz_PAGCySM0On%l9{&l z$hp?}=HI6kyE#r``5xAFWdje#$*gJ9FVsD;xe~PM$-H{tB*?pnZ{|B++e}!cc7Mtq zfue*A@7?J-pU(JeX4@LS@J-t`X;Gt==4t2Tzh9v^|YT#N{l}`k^wmG}- zh?*!(bA9s1u*PLuM}pz$2yfM3jzG`y^6lD+hu-(kn(oxWE1dR3;9pXk&awkKF8+se zOpUuaScEnR%", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "local_files_only": false, + "model_max_length": 32768, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +}