From c728e525055579e455aa7d2528dc06cf4a5350bf Mon Sep 17 00:00:00 2001 From: dongxinyu03 Date: Wed, 10 Dec 2025 12:05:39 +0800 Subject: [PATCH] Initial commit for vLLM-Kunlun Plugin --- .gitignore | 53 + .python-version | 1 + .readthedocs.yaml | 16 + CHANGELOG.md | 24 + LICENSE.txt | 201 ++ README.md | 314 +++ build.sh | 25 + docs/Makefile | 25 + docs/README.md | 57 + docs/envs.py | 183 ++ docs/requirements-docs.txt | 10 + docs/source/_templates/sections/header.html | 58 + docs/source/community/contributors.md | 38 + docs/source/community/governance.md | 51 + docs/source/community/user_stories/index.md | 3 + docs/source/community/versioning_policy.md | 3 + docs/source/conf.py | 144 ++ .../developer_guide/contribution/index.md | 70 + .../evaluation/accuracy/accuracy_kernel.md | 271 +++ .../evaluation/accuracy/accuracy_server.md | 240 +++ .../evaluation/accuracy/index.md | 10 + .../evaluation/accuracy_report/GLM-4.5-Air.md | 18 + .../evaluation/accuracy_report/GLM-4.5.md | 18 + .../accuracy_report/InternVL3_5-30B-A3B.md | 18 + .../accuracy_report/Qwen2.5-VL-7B-Instruct.md | 18 + .../evaluation/accuracy_report/index.md | 10 + .../developer_guide/evaluation/index.md | 8 + .../feature_guide/Kunlun_Graph.md | 76 + .../developer_guide/feature_guide/index.md | 9 + .../developer_guide/performance/index.md | 7 + .../performance_benchmark/benchmark_kernel.md | 147 ++ .../performance_benchmark/benchmark_server.md | 199 ++ .../performance_benchmark/index.md | 11 + .../performance_benchmark/profiling.md | 418 ++++ docs/source/faqs.md | 39 + docs/source/index.md | 69 + docs/source/installation.md | 129 ++ .../logos/vllm-kunlun-logo-text-dark.png | Bin 0 -> 178337 bytes .../logos/vllm-kunlun-logo-text-light.png | Bin 0 -> 178337 bytes docs/source/quick_start.md | 200 ++ docs/source/tutorials/index.md | 9 + docs/source/tutorials/multi_xpu_GLM-4.5.md | 153 ++ .../multi_xpu_Qwen3-Coder-480B-A35B(W8A8).md | 132 ++ docs/source/tutorials/single_xpu_Qwen3-8B.md | 168 ++ .../user_guide/configuration/env_vars.md | 17 + docs/source/user_guide/configuration/index.md | 9 + .../user_guide/feature_guide/graph_mode.md | 82 + docs/source/user_guide/feature_guide/index.md | 11 + docs/source/user_guide/feature_guide/lora.md | 27 + .../user_guide/feature_guide/quantization.md | 45 + docs/source/user_guide/release_notes.md | 3 + .../source/user_guide/support_matrix/index.md | 10 + .../support_matrix/supported_features.md | 14 + .../support_matrix/supported_models.md | 33 + pyproject.toml | 30 + requirements.txt | 34 + setup.py | 66 + setup_env.sh | 11 + vllm_kunlun/__init__.py | 180 ++ vllm_kunlun/compilation/__init__.py | 0 vllm_kunlun/compilation/wrapper.py | 148 ++ vllm_kunlun/csrc/dispatch_utils.h | 49 + vllm_kunlun/csrc/utils.cpp | 32 + vllm_kunlun/csrc/xops.h | 241 +++ vllm_kunlun/distributed/__init__.py | 0 .../distributed/kunlun_communicator.py | 102 + vllm_kunlun/lora/ops/kunlun_ops/__init__.py | 16 + vllm_kunlun/lora/ops/kunlun_ops/lora_ops.py | 443 +++++ vllm_kunlun/lora/punica_wrapper/__init__.py | 0 .../lora/punica_wrapper/punica_kunlun.py | 547 ++++++ vllm_kunlun/models/__init__.py | 68 + vllm_kunlun/models/glm.py | 24 + vllm_kunlun/models/glm4.py | 301 +++ vllm_kunlun/models/glm4_1v.py | 1597 +++++++++++++++ vllm_kunlun/models/glm4_moe.py | 716 +++++++ vllm_kunlun/models/gpt_oss.py | 630 ++++++ vllm_kunlun/models/intern_vit.py | 480 +++++ vllm_kunlun/models/internlm2.py | 450 +++++ vllm_kunlun/models/interns1.py | 869 +++++++++ vllm_kunlun/models/interns1_vit.py | 431 +++++ vllm_kunlun/models/internvl.py | 1404 ++++++++++++++ vllm_kunlun/models/llama.py | 643 ++++++ vllm_kunlun/models/model_loader/__init__.py | 0 .../model_loader/bitsandbytes_loader.py | 24 + vllm_kunlun/models/qwen2.py | 498 +++++ vllm_kunlun/models/qwen2_5_vl.py | 1351 +++++++++++++ vllm_kunlun/models/qwen2_vl.py | 1510 +++++++++++++++ vllm_kunlun/models/qwen3.py | 530 +++++ vllm_kunlun/models/qwen3_moe.py | 836 ++++++++ vllm_kunlun/ops/__init__.py | 21 + vllm_kunlun/ops/_kunlun_ops.py | 597 ++++++ vllm_kunlun/ops/activation.py | 23 + vllm_kunlun/ops/attention/__init__.py | 3 + .../ops/attention/backends/__init__.py | 3 + .../ops/attention/backends/kunlun_attn.py | 803 ++++++++ vllm_kunlun/ops/attention/backends/utils.py | 604 ++++++ vllm_kunlun/ops/attention/layer.py | 274 +++ vllm_kunlun/ops/fused_moe/__init__.py | 0 vllm_kunlun/ops/fused_moe/layer.py | 310 +++ vllm_kunlun/ops/layernorm.py | 60 + vllm_kunlun/ops/linear.py | 24 + vllm_kunlun/ops/paged_attn.py | 305 +++ vllm_kunlun/ops/quantization/__init__.py | 0 vllm_kunlun/ops/quantization/awq.py | 128 ++ .../quantization/compressed_tensors_moe.py | 333 ++++ vllm_kunlun/ops/quantization/gptq.py | 108 ++ vllm_kunlun/ops/rotary_embedding.py | 180 ++ vllm_kunlun/ops/sample/__init__.py | 0 vllm_kunlun/ops/sample/sampler.py | 1431 ++++++++++++++ vllm_kunlun/ops/vocab_parallel_embedding.py | 477 +++++ vllm_kunlun/patches/__init__.py | 0 vllm_kunlun/patches/eval_frame.py | 1723 +++++++++++++++++ vllm_kunlun/patches/performance.png | Bin 0 -> 140656 bytes vllm_kunlun/patches/vLLM_Kunlun.jpg | Bin 0 -> 178337 bytes vllm_kunlun/platforms/__init__.py | 5 + vllm_kunlun/platforms/envs.py | 111 ++ vllm_kunlun/platforms/kunlun.py | 289 +++ vllm_kunlun/platforms/version.py | 8 + vllm_kunlun/tests/__init__.py | 0 vllm_kunlun/utils.py | 402 ++++ vllm_kunlun/v1/__init__.py | 0 vllm_kunlun/v1/attention/__init__.py | 3 + vllm_kunlun/v1/attention/backends/__init__.py | 3 + .../v1/attention/backends/kunlun_attn.py | 706 +++++++ vllm_kunlun/v1/sample/__init__.py | 0 vllm_kunlun/v1/sample/ops/__init__.py | 0 vllm_kunlun/v1/sample/ops/penalties.py | 91 + .../v1/sample/ops/topk_topp_sampler.py | 198 ++ vllm_kunlun/v1/worker/__init__.py | 0 vllm_kunlun/v1/worker/block_table.py | 174 ++ vllm_kunlun/vllm_utils_wrapper.py | 1254 ++++++++++++ 131 files changed, 28816 insertions(+) create mode 100644 .gitignore create mode 100644 .python-version create mode 100644 .readthedocs.yaml create mode 100644 CHANGELOG.md create mode 100644 LICENSE.txt create mode 100644 README.md create mode 100644 build.sh create mode 100644 docs/Makefile create mode 100644 docs/README.md create mode 100644 docs/envs.py create mode 100644 docs/requirements-docs.txt create mode 100644 docs/source/_templates/sections/header.html create mode 100644 docs/source/community/contributors.md create mode 100644 docs/source/community/governance.md create mode 100644 docs/source/community/user_stories/index.md create mode 100644 docs/source/community/versioning_policy.md create mode 100644 docs/source/conf.py create mode 100644 docs/source/developer_guide/contribution/index.md create mode 100644 docs/source/developer_guide/evaluation/accuracy/accuracy_kernel.md create mode 100644 docs/source/developer_guide/evaluation/accuracy/accuracy_server.md create mode 100644 docs/source/developer_guide/evaluation/accuracy/index.md create mode 100644 docs/source/developer_guide/evaluation/accuracy_report/GLM-4.5-Air.md create mode 100644 docs/source/developer_guide/evaluation/accuracy_report/GLM-4.5.md create mode 100644 docs/source/developer_guide/evaluation/accuracy_report/InternVL3_5-30B-A3B.md create mode 100644 docs/source/developer_guide/evaluation/accuracy_report/Qwen2.5-VL-7B-Instruct.md create mode 100644 docs/source/developer_guide/evaluation/accuracy_report/index.md create mode 100644 docs/source/developer_guide/evaluation/index.md create mode 100644 docs/source/developer_guide/feature_guide/Kunlun_Graph.md create mode 100644 docs/source/developer_guide/feature_guide/index.md create mode 100644 docs/source/developer_guide/performance/index.md create mode 100644 docs/source/developer_guide/performance/performance_benchmark/benchmark_kernel.md create mode 100644 docs/source/developer_guide/performance/performance_benchmark/benchmark_server.md create mode 100644 docs/source/developer_guide/performance/performance_benchmark/index.md create mode 100644 docs/source/developer_guide/performance/performance_benchmark/profiling.md create mode 100644 docs/source/faqs.md create mode 100644 docs/source/index.md create mode 100644 docs/source/installation.md create mode 100644 docs/source/logos/vllm-kunlun-logo-text-dark.png create mode 100644 docs/source/logos/vllm-kunlun-logo-text-light.png create mode 100644 docs/source/quick_start.md create mode 100644 docs/source/tutorials/index.md create mode 100644 docs/source/tutorials/multi_xpu_GLM-4.5.md create mode 100644 docs/source/tutorials/multi_xpu_Qwen3-Coder-480B-A35B(W8A8).md create mode 100644 docs/source/tutorials/single_xpu_Qwen3-8B.md create mode 100644 docs/source/user_guide/configuration/env_vars.md create mode 100644 docs/source/user_guide/configuration/index.md create mode 100644 docs/source/user_guide/feature_guide/graph_mode.md create mode 100644 docs/source/user_guide/feature_guide/index.md create mode 100644 docs/source/user_guide/feature_guide/lora.md create mode 100644 docs/source/user_guide/feature_guide/quantization.md create mode 100644 docs/source/user_guide/release_notes.md create mode 100644 docs/source/user_guide/support_matrix/index.md create mode 100644 docs/source/user_guide/support_matrix/supported_features.md create mode 100644 docs/source/user_guide/support_matrix/supported_models.md create mode 100644 pyproject.toml create mode 100644 requirements.txt create mode 100644 setup.py create mode 100644 setup_env.sh create mode 100644 vllm_kunlun/__init__.py create mode 100644 vllm_kunlun/compilation/__init__.py create mode 100644 vllm_kunlun/compilation/wrapper.py create mode 100644 vllm_kunlun/csrc/dispatch_utils.h create mode 100644 vllm_kunlun/csrc/utils.cpp create mode 100644 vllm_kunlun/csrc/xops.h create mode 100644 vllm_kunlun/distributed/__init__.py create mode 100644 vllm_kunlun/distributed/kunlun_communicator.py create mode 100644 vllm_kunlun/lora/ops/kunlun_ops/__init__.py create mode 100644 vllm_kunlun/lora/ops/kunlun_ops/lora_ops.py create mode 100644 vllm_kunlun/lora/punica_wrapper/__init__.py create mode 100644 vllm_kunlun/lora/punica_wrapper/punica_kunlun.py create mode 100644 vllm_kunlun/models/__init__.py create mode 100644 vllm_kunlun/models/glm.py create mode 100644 vllm_kunlun/models/glm4.py create mode 100644 vllm_kunlun/models/glm4_1v.py create mode 100644 vllm_kunlun/models/glm4_moe.py create mode 100644 vllm_kunlun/models/gpt_oss.py create mode 100644 vllm_kunlun/models/intern_vit.py create mode 100644 vllm_kunlun/models/internlm2.py create mode 100644 vllm_kunlun/models/interns1.py create mode 100644 vllm_kunlun/models/interns1_vit.py create mode 100644 vllm_kunlun/models/internvl.py create mode 100644 vllm_kunlun/models/llama.py create mode 100644 vllm_kunlun/models/model_loader/__init__.py create mode 100644 vllm_kunlun/models/model_loader/bitsandbytes_loader.py create mode 100644 vllm_kunlun/models/qwen2.py create mode 100644 vllm_kunlun/models/qwen2_5_vl.py create mode 100644 vllm_kunlun/models/qwen2_vl.py create mode 100644 vllm_kunlun/models/qwen3.py create mode 100644 vllm_kunlun/models/qwen3_moe.py create mode 100644 vllm_kunlun/ops/__init__.py create mode 100644 vllm_kunlun/ops/_kunlun_ops.py create mode 100644 vllm_kunlun/ops/activation.py create mode 100644 vllm_kunlun/ops/attention/__init__.py create mode 100644 vllm_kunlun/ops/attention/backends/__init__.py create mode 100644 vllm_kunlun/ops/attention/backends/kunlun_attn.py create mode 100644 vllm_kunlun/ops/attention/backends/utils.py create mode 100644 vllm_kunlun/ops/attention/layer.py create mode 100644 vllm_kunlun/ops/fused_moe/__init__.py create mode 100644 vllm_kunlun/ops/fused_moe/layer.py create mode 100644 vllm_kunlun/ops/layernorm.py create mode 100644 vllm_kunlun/ops/linear.py create mode 100644 vllm_kunlun/ops/paged_attn.py create mode 100644 vllm_kunlun/ops/quantization/__init__.py create mode 100644 vllm_kunlun/ops/quantization/awq.py create mode 100644 vllm_kunlun/ops/quantization/compressed_tensors_moe.py create mode 100644 vllm_kunlun/ops/quantization/gptq.py create mode 100644 vllm_kunlun/ops/rotary_embedding.py create mode 100644 vllm_kunlun/ops/sample/__init__.py create mode 100644 vllm_kunlun/ops/sample/sampler.py create mode 100644 vllm_kunlun/ops/vocab_parallel_embedding.py create mode 100644 vllm_kunlun/patches/__init__.py create mode 100644 vllm_kunlun/patches/eval_frame.py create mode 100644 vllm_kunlun/patches/performance.png create mode 100644 vllm_kunlun/patches/vLLM_Kunlun.jpg create mode 100644 vllm_kunlun/platforms/__init__.py create mode 100644 vllm_kunlun/platforms/envs.py create mode 100644 vllm_kunlun/platforms/kunlun.py create mode 100644 vllm_kunlun/platforms/version.py create mode 100644 vllm_kunlun/tests/__init__.py create mode 100644 vllm_kunlun/utils.py create mode 100644 vllm_kunlun/v1/__init__.py create mode 100644 vllm_kunlun/v1/attention/__init__.py create mode 100644 vllm_kunlun/v1/attention/backends/__init__.py create mode 100644 vllm_kunlun/v1/attention/backends/kunlun_attn.py create mode 100644 vllm_kunlun/v1/sample/__init__.py create mode 100644 vllm_kunlun/v1/sample/ops/__init__.py create mode 100644 vllm_kunlun/v1/sample/ops/penalties.py create mode 100644 vllm_kunlun/v1/sample/ops/topk_topp_sampler.py create mode 100644 vllm_kunlun/v1/worker/__init__.py create mode 100644 vllm_kunlun/v1/worker/block_table.py create mode 100644 vllm_kunlun/vllm_utils_wrapper.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..283eae9 --- /dev/null +++ b/.gitignore @@ -0,0 +1,53 @@ +# Virtualenv +/.venv/ +/venv/ + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] + +# C extensions +*.so + +# Distribution / packaging +/bin/ +/build/ +/develop-eggs/ +/dist/ +/eggs/ +/lib/ +/lib64/ +/output/ +/parts/ +/sdist/ +/var/ +/*.egg-info/ +/.installed.cfg +/*.egg +/.eggs + +# AUTHORS and ChangeLog will be generated while packaging +/AUTHORS +/ChangeLog + +# BCloud / BuildSubmitter +/build_submitter.* +/logger_client_log + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +.tox/ +.coverage +.cache +.pytest_cache +nosetests.xml +coverage.xml + +# Translations +*.mo + +# Sphinx documentation +docs/_build diff --git a/.python-version b/.python-version new file mode 100644 index 0000000..d35e171 --- /dev/null +++ b/.python-version @@ -0,0 +1 @@ +3.10.10 \ No newline at end of file diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 0000000..4c59f79 --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,16 @@ +version: 2 + +build: + os: ubuntu-22.04 + tools: + python: "3.12" + +sphinx: + configuration: docs/source/conf.py + fail_on_warning: false + +formats: [] + +python: + install: + - requirements: docs/requirements-docs.txt \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..78979c5 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,24 @@ +Changelog +===# Change Chinese to English comments +The following records all changes worth noting in the project, formatted based on [Keep a Changelog]. + +This project version follows [Semantic Versioning] and [PEP-440]. + +[Unreleased] +--- +### Added +- This records new content added +### Changed +- This records changed content + +0.1.0 - 2025-08-12 +--- +### Added +- Create project + + +[Unreleased]: http://icode.baidu.com/repos/baidu/hac-aiacc/vllm-kunlun/merge/0.1.0...master + +[Keep a Changelog]: https://keepachangelog.com/zh-CN/1.0.0/ +[Semantic Versioning]: https://semver.org/lang/zh-CN/ +[PEP-440]: https://www.python.org/dev/peps/pep-0440/ diff --git a/LICENSE.txt b/LICENSE.txt new file mode 100644 index 0000000..261eeb9 --- /dev/null +++ b/LICENSE.txt @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/README.md b/README.md new file mode 100644 index 0000000..2cc74eb --- /dev/null +++ b/README.md @@ -0,0 +1,314 @@ +![vLLM Kunlun Logo](vllm_kunlun/patches/vLLM_Kunlun.jpg) + +

+ Documentation | + Users Forum | + slack | +

+ +--- + +## Latest News🔥 +- [2025/11] +- [2025/11] +- [2025/11] +- [2025/11] Initial release of vLLM Kunlun + +--- + +# Overview + +vLLM Kunlun (vllm-kunlun) is a community-maintained hardware plugin designed to seamlessly run vLLM on the Kunlun XPU. It is the recommended approach for integrating the Kunlun backend within the vLLM community, adhering to the principles outlined in the [RFC]: Hardware pluggable. This plugin provides a hardware-pluggable interface that decouples the integration of the Kunlun XPU with vLLM. + +By utilizing the vLLM Kunlun plugin, popular open-source models, including Transformer-like, Mixture-of-Expert, Embedding, and Multi-modal LLMs, can run effortlessly on the Kunlun XPU. + +--- +## Prerequisites + +- **Hardware**: Kunlun3 P800 +- **OS**: Ubuntu 22.04 +- **Software**: + - Python >=3.10 + - PyTorch ≥ 2.5.1 + - vLLM (same version as vllm-kunlun) + +--- +## Supported Models + + +

Generaltive Models

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ModelSupportQuantizationLoRAPiecewise Kunlun GraphNote
Qwen2/2.5✅✅✅
Qwen3✅✅✅
Qwen3-Moe/Coder✅✅✅✅
QwQ-32B✅✅
LLama2/3/3.1✅✅
GLM-4.5/Air✅✅✅✅
Qwen3next⚠️comming soon
Gpt oss⚠️comming soon
Deepseek v3/3.2⚠️comming soon
+ +

Multimodal Language Models

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ModelSupportQuantizationLoRAPiecewise Kunlun GraphNote
Qianfan-VL✅✅
Qwen2.5VL✅✅
InternVL2.5/3/3.5✅✅
InternVL3.5✅✅
InternS1✅✅
Qwen2.5 omini⚠️comming soon
Qwen3vl⚠️comming soon
+ + + +## Performance Visualization 🚀 +### High-performance computing at work: How different models perform on the Kunlun3 P800. + +Current environment: 16-way concurrency, input/output size 2048. + + +![Models and tgs](./vllm_kunlun/patches/performance.png) + +## Getting Started + +Please use the following recommended versions to get started quickly: + +| Version | Release type | Doc | +|----------|---------------|-----| +| v0.10.1.1 | Latest stable version | [QuickStart](./docs/_build/html/quick_start.html) and [Installation](./docs/_build/html/installation.html) for more details | + +--- + +## Contributing + +See [CONTRIBUTING]() for more details, which is a step-by-step guide to help you set up the development environment, build, and test. + +We welcome and value any contributions and collaborations: +- Open an [Issue]() if you find a bug or have a feature request + +## License + +Apache License 2.0, as found in the [LICENSE](./LICENSE) file. \ No newline at end of file diff --git a/build.sh b/build.sh new file mode 100644 index 0000000..d7d6c92 --- /dev/null +++ b/build.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +set -euo pipefail + +echo "========= build enter =========" + +echo "$PATH" +WORK_DIR=$(cd $(dirname $0) && pwd) && cd $WORK_DIR + +echo_cmd() { + echo $1 + $1 +} + +echo "========= build vllm =========" + +echo_cmd "rm -rf output" +echo_cmd "mkdir -p output" + +cd ${WORK_DIR} +rm -rf output/.scm/ +tar -zcvf ../vllm-kunlun.tar.gz ../vllm-kunlun/ +mv ../vllm-kunlun.tar.gz ./output/ + +echo "========= build exit =========" \ No newline at end of file diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 0000000..fe062fc --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,25 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = source +BUILDDIR = _build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +intl: + sphinx-intl build + @$(SPHINXBUILD) -b html -D language=zh_CN "$(SOURCEDIR)" "$(BUILDDIR)/html/zh-cn" $(SPHINXOPTS) $(O) + diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 0000000..3496a6d --- /dev/null +++ b/docs/README.md @@ -0,0 +1,57 @@ +## 🚀 Installation + +```bash + +uv venv myenv --python 3.12 --seed +source myenv/bin/activate + + + # Step 1: Enter the docs directory +cd docs + +# Step 2: Install dependencies (using uv) +uv pip install -r requirements-docs.txt + +# Install sphinx-autobuild (if not in requirements file) +uv pip install sphinx-autobuild + +# Run from the docs directory: +sphinx-autobuild ./source ./_build/html --port 8000 + +# Step 1: Clean up old files +make clean + +# Step 2: Build HTML +make html + +# Step 3: Local preview +python -m http.server -d _build/html/ + +Browser access: http://localhost:8000 + +🌍 Internationalization +Internationalization translation process (taking Chinese as an example) + +# Step 1: Extract translatable text (generate .pot) +sphinx-build -b gettext source _build/gettext + +# Step 2: Generate/update Chinese .po file +sphinx-intl update -p _build/gettext -l zh_CN + +# Step 3: Manually translate .po file +# Use a text editor to open source/locale/zh_CN/LC_MESSAGES/*.po +# Fill in the Chinese translation in msgstr "" + +# Step 4: Compile and build Chinese documentation +make intl + +# Step 5: View the effect +python -m http.server -d _build/html + + +Browser access: + +English version: http://localhost:8000 +Chinese version: http://localhost:8000/zh-cn + +``` diff --git a/docs/envs.py b/docs/envs.py new file mode 100644 index 0000000..dc01993 --- /dev/null +++ b/docs/envs.py @@ -0,0 +1,183 @@ +# +# Copyright (c) 2025 Baidu Technologies Co., Ltd. All Rights Reserved. +# This file is a part of the vllm-kunlun project. +# +# This file is mainly Adapted from vllm-project/vllm/vllm/envs.py +# Copyright 2023 The vLLM team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import os +from typing import Any, Callable, Dict + +# The begin-* and end* here are used by the documentation generator +# to extract the used env vars. + +# begin-env-vars-definition + +env_variables: Dict[str, Callable[[], Any]] = { + # max compile thread number for package building. Usually, it is set to + # the number of CPU cores. If not set, the default value is None, which + # means all number of CPU cores will be used. + "MAX_JOBS": lambda: os.getenv("MAX_JOBS", None), + # The build type of the package. It can be one of the following values: + # Release, Debug, RelWithDebugInfo. If not set, the default value is Release. + "CMAKE_BUILD_TYPE": lambda: os.getenv("CMAKE_BUILD_TYPE"), + # Whether to compile custom kernels. If not set, the default value is True. + # If set to False, the custom kernels will not be compiled. Please note that + # the sleep mode feature will be disabled as well if custom kernels are not + # compiled. + "COMPILE_CUSTOM_KERNELS": lambda: bool( + int(os.getenv("COMPILE_CUSTOM_KERNELS", "1")) + ), + # The CXX compiler used for compiling the package. If not set, the default + # value is None, which means the system default CXX compiler will be used. + "CXX_COMPILER": lambda: os.getenv("CXX_COMPILER", None), + # The C compiler used for compiling the package. If not set, the default + # value is None, which means the system default C compiler will be used. + "C_COMPILER": lambda: os.getenv("C_COMPILER", None), + + "SOC_VERSION": lambda: os.getenv("SOC_VERSION", "KUNLUNP800"), + # If set, vllm-kunlun will print verbose logs during compilation + "VERBOSE": lambda: bool(int(os.getenv("VERBOSE", "0"))), + # /usr/local/Kunlun/kunlun-toolkit/latest + "KUNLUN_HOME_PATH": lambda: os.getenv("KUNLUN_HOME_PATH", None), + # The path for XCCL library, it's used by pyxccl communicator backend. If + # not set, the default value is libxccl.so。 + "XCCL_SO_PATH": lambda: os.environ.get("XCCL_SO_PATH", None), + # The version of vllm is installed. This value is used for developers who + # installed vllm from source locally. In this case, the version of vllm is + # usually changed. For example, if the version of vllm is "0.9.0", but when + # it's installed from source, the version of vllm is usually set to "0.9.1". + # In this case, developers need to set this value to "0.9.0" to make sure + # that the correct package is installed. + "VLLM_VERSION": lambda: os.getenv("VLLM_VERSION", None), + # Whether to enable the trace recompiles from pytorch. + "VLLM_KUNLUN_TRACE_RECOMPILES": lambda: bool( + int(os.getenv("VLLM_KUNLUN_TRACE_RECOMPILES", "0")) + ), + # Whether to enable fused_experts_allgather_ep. MoeInitRoutingV3 and + # GroupedMatmulFinalizeRouting operators are combined to implement EP. + "VLLM_ENABLE_FUSED_EXPERTS_ALLGATHER_EP": lambda: bool( + int(os.getenv("VLLM_ENABLE_FUSED_EXPERTS_ALLGATHER_EP", "0")) + ), + # Whether to enable the model execute time observe profile. Disable it when + # running vllm kunlun in production environment. + "VLLM_KUNLUN_MODEL_EXECUTE_TIME_OBSERVE": lambda: bool( + int(os.getenv("VLLM_KUNLUN_MODEL_EXECUTE_TIME_OBSERVE", "0")) + ), + # Some models are optimized by vllm kunlun. While in some case, e.g. rlhf + # training, the optimized model may not be suitable. In this case, set this + # value to False to disable the optimized model. + "USE_OPTIMIZED_MODEL": lambda: bool(int(os.getenv("USE_OPTIMIZED_MODEL", "1"))), + # The tolerance of the kv cache size, if the difference between the + # actual kv cache size and the cached kv cache size is less than this value, + # then the cached kv cache size will be used. + "VLLM_KUNLUN_KV_CACHE_MEGABYTES_FLOATING_TOLERANCE": lambda: int( + os.getenv("VLLM_KUNLUN_KV_CACHE_MEGABYTES_FLOATING_TOLERANCE", 64) + ), + # Whether to enable the topk optimization. It's enabled by default. Please set to False if you hit any issue. + # We'll remove this flag in the future once it's stable enough. + "VLLM_KUNLUN_ENABLE_TOPK_TOPP_OPTIMIZATION": lambda: bool( + int(os.getenv("VLLM_KUNLUN_ENABLE_TOPK_TOPP_OPTIMIZATION", "1")) + ), + # `LLMDataDistCMgrConnector` required variable. `DISAGGREGATED_PREFILL_RANK_TABLE_PATH` is + # used for llmdatadist to build the communication topology for kv cache transfer, it is + # a required variable if `LLMDataDistCMgrConnector` is used as kv connector for disaggregated + # pd. The rank table can be generated by adopting the script `gen_ranktable.sh` + # in vllm_kunlun's example folder. + "DISAGGREGATED_PREFILL_RANK_TABLE_PATH": lambda: os.getenv( + "DISAGGREGATED_PREFILL_RANK_TABLE_PATH", None + ), + # `LLMDataDistCMgrConnector` required variable. `VLLM_KUNLUN_LLMDD_RPC_IP` is used as the + # rpc communication listening ip, which will be used to receive the agent metadata from the + # remote worker. + "VLLM_KUNLUN_LLMDD_RPC_IP": lambda: os.getenv( + "VLLM_KUNLUN_LLMDD_RPC_IP", "0.0.0.0" + ), + # `LLMDataDistCMgrConnector` required variable. `VLLM_KUNLUN_LLMDD_RPC_PORT` is used as the + # rpc communication listening port, which will be used to receive the agent metadata from the + # remote worker. + "VLLM_KUNLUN_LLMDD_RPC_PORT": lambda: int( + os.getenv("VLLM_KUNLUN_LLMDD_RPC_PORT", 5557) + ), + # Whether to enable mla_pa for deepseek mla decode, this flag will be removed after its available torch_npu is public accessible + # and the mla_pa will be the default path of deepseek decode path. + "VLLM_KUNLUN_MLA_PA": lambda: int(os.getenv("VLLM_KUNLUN_MLA_PA", 0)), + # Whether to enable MatmulAllReduce fusion kernel when tensor parallel is enabled. + "VLLM_KUNLUN_ENABLE_MATMUL_ALLREDUCE": lambda: bool( + int(os.getenv("VLLM_KUNLUN_ENABLE_MATMUL_ALLREDUCE", "0")) + ), + # Whether to enable FlashComm optimization when tensor parallel is enabled. + # This feature will get better performance when concurrency is large. + "VLLM_KUNLUN_ENABLE_FLASHCOMM1": lambda: bool( + int(os.getenv("VLLM_KUNLUN_ENABLE_FLASHCOMM1", "0")) + ), + # Whether to enable MLP weight prefetch, only used in small concurrency. + "VLLM_KUNLUN_ENABLE_PREFETCH_MLP": lambda: bool( + int(os.getenv("VLLM_KUNLUN_ENABLE_PREFETCH_MLP", "0")) + ), + # buffer size for gate up prefetch + "VLLM_KUNLUN_MLP_GATE_UP_PREFETCH_SIZE": lambda: int( + os.getenv("VLLM_KUNLUN_MLP_GATE_UP_PREFETCH_SIZE", 18 * 1024 * 1024) + ), + # buffer size for down proj prefetch + "VLLM_KUNLUN_MLP_DOWN_PREFETCH_SIZE": lambda: int( + os.getenv("VLLM_KUNLUN_MLP_DOWN_PREFETCH_SIZE", 18 * 1024 * 1024) + ), + # Whether to enable dense model and general optimizations for better performance. + # Since we modified the base parent class `linear`, this optimization is also applicable to other model types. + # However, there might be hidden issues, and it is currently recommended to prioritize its use with dense models. + "VLLM_KUNLUN_ENABLE_DENSE_OPTIMIZE": lambda: bool( + int(os.getenv("VLLM_KUNLUN_ENABLE_DENSE_OPTIMIZE", "0")) + ), + # Whether to enable mlp optimize when tensor parallel is enabled. + # this feature in eager mode will get better performance. + "VLLM_KUNLUN_ENABLE_MLP_OPTIMIZE": lambda: bool( + int(os.getenv("VLLM_KUNLUN_ENABLE_MLP_OPTIMIZE", "0")) + ), + # Determine the number of physical devices in a non-full-use scenario + # caused by the initialization of the Mooncake connector. + "PHYSICAL_DEVICES": lambda: os.getenv("PHYSICAL_DEVICES", None), + # Whether to enable msMonitor tool to monitor the performance of vllm-kunlun. + "MSMONITOR_USE_DAEMON": lambda: bool(int(os.getenv("MSMONITOR_USE_DAEMON", "0"))), + # Timeout (in seconds) for delayed KVCache block release. In the prefill + # node, if a request is marked for delayed KV block release and the blocks + # are not freed within this timeout, they will be forcibly released. + "VLLM_KUNLUN_KVCACHE_DELAY_FREE_TIMEOUT": lambda: int( + os.getenv("VLLM_KUNLUN_KVCACHE_DELAY_FREE_TIMEOUT", 250) + ), + "VLLM_KUNLUN_ENABLE_MLAPO": lambda: bool( + int(os.getenv("VLLM_KUNLUN_ENABLE_MLAPO", "0")) + ), + # Whether to enable transpose weight and cast format to FRACTAL_NZ. + "VLLM_KUNLUN_ENABLE_NZ": lambda: int(os.getenv("VLLM_KUNLUN_ENABLE_NZ", 1)), + # Decide whether we should enable CP parallelism. + "VLLM_KUNLUN_ENABLE_CONTEXT_PARALLEL": lambda: bool( + int(os.getenv("VLLM_KUNLUN_ENABLE_CONTEXT_PARALLEL", "0")) + ), +} + +# end-env-vars-definition + + +def __getattr__(name: str): + # lazy evaluation of environment variables + if name in env_variables: + return env_variables[name]() + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + + +def __dir__(): + return list(env_variables.keys()) diff --git a/docs/requirements-docs.txt b/docs/requirements-docs.txt new file mode 100644 index 0000000..b8d3ac7 --- /dev/null +++ b/docs/requirements-docs.txt @@ -0,0 +1,10 @@ +sphinx +sphinx-argparse +sphinx-book-theme +sphinx-copybutton +sphinx-design +sphinx-togglebutton +myst-parser +msgspec +sphinx-substitution-extensions +sphinx-intl \ No newline at end of file diff --git a/docs/source/_templates/sections/header.html b/docs/source/_templates/sections/header.html new file mode 100644 index 0000000..d81d58d --- /dev/null +++ b/docs/source/_templates/sections/header.html @@ -0,0 +1,58 @@ + + + + diff --git a/docs/source/community/contributors.md b/docs/source/community/contributors.md new file mode 100644 index 0000000..5f44922 --- /dev/null +++ b/docs/source/community/contributors.md @@ -0,0 +1,38 @@ +# Maintainers and Acknowledgments + +## Maintainers + +| Name | Github ID | Date | +| :----------: | :----------------------------------------------: | :-----: | +| Xinyu Dong | [@xyDong0223](https://github.com/xyDong0223) | 2025/11 | +| Qian Bao | [@baoqian426](https://github.com/baoqian426) | 2025/11 | +| Zhennan Chen | [@chanzhennan](https://github.com/chanzhennan) | 2025/11 | +| Yili Chen | [@chenyili0619](https://github.com/chenyili0619) | 2025/11 | +| Hanyu Jin | [@Hanyu-Jin](https://github.com/Hanyu-Jin) | 2025/11 | +| Donghua Li | [@ldh2020](https://github.com/ldh2020) | 2025/11 | + +## Acknowledgments + +| Name | +| :------------: | +| Haowen Han | +| Tianyu Ma | +| Jizhong Yuan | +| Yucheng Liang | +| Hanshuo Yang | +| Wei Li | +| Hao Wang | +| Zhihui Wang | +| Hao Wang | +| YingZhuo Zhao | +| Wanli Yang | +| Xin Zhao | +| Yuqi Lin | +| Xiaokang Cheng | +| Zeyu You | +| Jingyu Zhang | +| Lidang Jiang | +| Yijin Qiao | +| Chenchao Hu | +| Weijie Hong | +| Song Jiang | \ No newline at end of file diff --git a/docs/source/community/governance.md b/docs/source/community/governance.md new file mode 100644 index 0000000..cf0f288 --- /dev/null +++ b/docs/source/community/governance.md @@ -0,0 +1,51 @@ +# Governance + +## Mission + +As a vital component of vLLM, the vLLM Kunlun project is dedicated to providing an easy, fast, and cheap LLM Serving for everyone on Kunlun XPUs and to actively contributing to the enrichment of vLLM. + +## Principles + +vLLM Kunlun follows the vLLM community's code of conduct: [vLLM - CODE OF CONDUCT](https://github.com/vllm-project/vllm/blob/main/CODE_OF_CONDUCT.md) + +## Governance - Mechanics + +vLLM Kunlun is an open-source project under the vLLM community, where the authority to appoint roles is ultimately determined by the vLLM community. It adopts a hierarchical technical governance structure. + +- Contributor: + + **Responsibility:** Help new contributors on boarding, handle and respond to community questions, review RFCs and code. + + **Requirements:** Complete at least 1 contribution. A contributor is someone who consistently and actively participates in a project, including but not limited to issue/review/commits/community involvement. + + The contributor permissions are granted by the [vllm-kunlun]'s repo `Triage` on GitHub, including repo read and clone, issue and PR management, facilitating efficient collaboration between community developers. + +- Maintainer: + + **Responsibility:** Develop the project's vision and mission. Maintainers are responsible for shaping the technical direction of the project and ensuring its long-term success. With code merge permissions, they lead roadmap planning, review community contributions, make ongoing code improvements, and actively participate in community engagement—such as regular meetings and events. + + **Requirements:** Deep understanding of ‌vLLM‌ and ‌vLLM Kunlun‌ code bases, with a commitment to sustained code contributions and competency in ‌design, development, and PR review workflows‌. + + - **Review quality‌:** Actively participate in community code reviews, ensuring high-quality code integration. + - **Quality contribution‌:** Successfully develop and deliver at least one major feature while maintaining consistent high-quality contributions. + - **Community involvement‌:** Actively address issues, respond to forum inquiries, participate in discussions, and engage in community-driven tasks. + +The approval from existing Maintainers is required. The vLLM community has the final decision-making authority. +Maintainers will be granted write access to the [vllm-kunlun] GitHub repo. This includes permission to read, clone, and push to the repository, as well as manage issues and pull requests. + +## Nominating and Removing Maintainers + +### The Principles + +- Membership in vLLM Kunlun is given to individuals on merit basis after they demonstrate their strong expertise in vLLM/vLLM Kunlun through contributions, reviews, and discussions. + +- For membership in the maintainer group, individuals have to demonstrate strong and continued alignment with the overall vLLM/vLLM Kunlun principles. + +- Maintainers who have been inactive for a long time may be transitioned to **emeritus** status under lenient criteria. + +- The membership is for an individual, not a company. + +### Nomination and Removal + +- Nomination: Anyone can nominate a candidate to become a maintainer, including self-nominations. All existing maintainers are responsible for reviewing and evaluating each nomination. The nominator should provide relevant information about the nominee's qualifications—such as review quality, quality contribution, and community involvement—among other strengths. +- Removal: Anyone may nominate an individual for removal from the maintainer role, including self-nominations. All current maintainers are responsible for reviewing and evaluating such nominations. The nominator should provide relevant information about the nominee—such as prolonged inactivity, misalignment with the project's overall direction, or other factors that may render them unsuitable for the maintainer position. \ No newline at end of file diff --git a/docs/source/community/user_stories/index.md b/docs/source/community/user_stories/index.md new file mode 100644 index 0000000..8026466 --- /dev/null +++ b/docs/source/community/user_stories/index.md @@ -0,0 +1,3 @@ +# User stories + +Comming soon... \ No newline at end of file diff --git a/docs/source/community/versioning_policy.md b/docs/source/community/versioning_policy.md new file mode 100644 index 0000000..7b2e7aa --- /dev/null +++ b/docs/source/community/versioning_policy.md @@ -0,0 +1,3 @@ +# Versioning policy + +Comming soon... \ No newline at end of file diff --git a/docs/source/conf.py b/docs/source/conf.py new file mode 100644 index 0000000..d2413be --- /dev/null +++ b/docs/source/conf.py @@ -0,0 +1,144 @@ +# +# Copyright (c) 2025 Baidu Technologies Co., Ltd. All Rights Reserved. +# Copyright 2023 The vLLM team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# This file is a part of the vllm-kunlun project. +# Adapted from vllm-project/vllm/docs/source/conf.py +# + +# -- Path setup -------------------------------------------------------------- + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +import json +import os + +# import sys +# sys.path.insert(0, os.path.abspath('.')) + +# -- Project information ----------------------------------------------------- + +project = "vllm-kunlun" +copyright = "2025, vllm-kunlun team" +author = "the vllm-kunlun team" + +# The full version, including alpha/beta/rc tags +release = "" + +# -- General configuration --------------------------------------------------- + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. + +# Copy from https://github.com/vllm-project/vllm/blob/main/docs/source/conf.py +extensions = [ + "sphinx.ext.napoleon", + "sphinx.ext.intersphinx", + "sphinx_copybutton", + "sphinx.ext.autodoc", + "sphinx.ext.autosummary", + "myst_parser", + "sphinxarg.ext", + "sphinx_design", + "sphinx_togglebutton", + "sphinx_substitution_extensions", +] + +myst_enable_extensions = ["colon_fence", "substitution"] + +# Change this when cut down release +myst_substitutions = { + # the branch of vllm, used in vllm clone + # - main branch: 'main' + # - vX.Y.Z branch: 'vX.Y.Z' + "vllm_version": "0.10.1.1", + # the branch of vllm-kunlun, used in vllm-kunlun clone and image tag + # - main branch: 'main' + # - vX.Y.Z branch: latest vllm-kunlun release tag + "vllm_kunlun_version": "0.10.1.1", + # the newest release version of vllm-kunlun and matched vLLM, used in pip install. + # This value should be updated when cut down release. + "pip_vllm_kunlun_version": "0.10.1.1", + "pip_vllm_version": "0.10.1.1", + # vllm version in ci + "ci_vllm_version": "0.10.1.1", +} + +# For cross-file header anchors +myst_heading_anchors = 5 + +# Add any paths that contain templates here, relative to this directory. +templates_path = ["_templates"] + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = "en" +locale_dirs = ["locale/"] +gettext_compact = False +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path. +exclude_patterns = [ + "_build", + "Thumbs.db", + ".DS_Store", + ".venv", + "README.md", + "user_guide/release.template.md", + "**/*.zh.md", +] + +# -- Options for HTML output ------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_title = project +html_theme = "sphinx_book_theme" +html_logo = "logos/vllm-kunlun-logo-text-light.png" +html_theme_options = { + "path_to_docs": "docs/source", + "repository_url": "https://github.com/xxxxx/vllm-kunlun", + "use_repository_button": True, + "use_edit_page_button": True, +} +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +# html_static_path = ['_static'] + +READTHEDOCS_VERSION_TYPE = os.environ.get("READTHEDOCS_VERSION_TYPE") +if READTHEDOCS_VERSION_TYPE == "tag": + # remove the warning banner if the version is a tagged release + header_file = os.path.join( + os.path.dirname(__file__), "_templates/sections/header.html" + ) + # The file might be removed already if the build is triggered multiple times + # (readthedocs build both HTML and PDF versions separately) + if os.path.exists(header_file): + os.remove(header_file) + + +def setup(app): + pass + + +if __name__ == "__main__": + print(json.dumps(myst_substitutions)) diff --git a/docs/source/developer_guide/contribution/index.md b/docs/source/developer_guide/contribution/index.md new file mode 100644 index 0000000..e5b0a5e --- /dev/null +++ b/docs/source/developer_guide/contribution/index.md @@ -0,0 +1,70 @@ +# Contributing + +## Building and Testing +It's recommended to set up a local development environment to build vllm-kunlun and run tests +before you submit a PR. + +#### Run models locally + +After completing Run lint setup which is shown in quicksatrt, you can run your changed locally: + +```{code-block} bash + :substitutions: + +python -m vllm.entrypoints.openai.api_server \ + --host 0.0.0.0 \ + --port 8356 \ + --model /your_modified_models\ + --trust-remote-code \ + --tensor-parallel-size 1 \ + --no-enable-prefix-caching \ + --no-enable-chunked-prefill \ + --distributed-executor-backend mp \ + --served-model-name your_modified_models \ + --compilation-config '{"splitting_ops": ["vllm.unified_attention_with_output_kunlun", + "vllm.unified_attention", "vllm.unified_attention_with_output", + "vllm.mamba_mixer2"]}' \ +``` +Please save a screenshot of your service running successfully, and attach an accuracy report. + +#### Submit the commit + +```bash +# Commit changed files using `-s` +git commit -sm "your commit info" +``` + +🎉 Congratulations! You have completed the development environment setup. + + +## PR Title and Classification + +Only specific types of PRs will be reviewed. The PR title is prefixed appropriately to indicate the type of change. Please use one of the following: + +- `[Attention]` for new features or optimization in attention. +- `[Communicator]` for new features or optimization in communicators. +- `[ModelRunner]` for new features or optimization in model runner. +- `[Platform]` for new features or optimization in platform. +- `[Worker]` for new features or optimization in worker. +- `[Core]` for new features or optimization in the core vllm-kunlun logic (such as platform, attention, communicators, model runner) +- `[Kernel]` for changes affecting compute kernels and ops. +- `[Bugfix]` for bug fixes. +- `[Doc]` for documentation fixes and improvements. +- `[Test]` for tests (such as unit tests). +- `[CI]` for build or continuous integration improvements. +- `[Misc]` for PRs that do not fit the above categories. Please use this sparingly. + +:::{note} +If the PR spans more than one category, please include all relevant prefixes. +::: + +## Others + +If you find any problem when contributing, you can join our slack group to talk with us and then feel free to submit a PR to improve the doc to help other developers. + +:::{toctree} +:caption: Index +:maxdepth: 1 +testing +multi_node_test +::: \ No newline at end of file diff --git a/docs/source/developer_guide/evaluation/accuracy/accuracy_kernel.md b/docs/source/developer_guide/evaluation/accuracy/accuracy_kernel.md new file mode 100644 index 0000000..054364a --- /dev/null +++ b/docs/source/developer_guide/evaluation/accuracy/accuracy_kernel.md @@ -0,0 +1,271 @@ +## Operator accuracy test + +### torch_xray + +torch_xray is an operator precision analysis tool that can dump module-level input-output precision comparisons and automatically construct operator unit tests. + +#### 1.Download and install + +***\*python3.10:\**** + +bos:/klx-sdk-release-public/xpytorch/dev_kl3/torch_xray/latest/torch_xray-999.9.9-cp310-cp310-linux_x86_64.whl + +[https://su.bcebos.com/klx-sdk-release-public/xpytorch/dev_kl3/torch_xray/latest/](https://su.bcebos.com/klx-sdk-release-public/xpytorch/dev_kl3/torch_xray/latest/torch_xray-999.9.9-py3-none-any.whl)torch_xray-999.9.9-cp310-cp310-linux_x86_64.whl + +***\*python3.8:\**** + +bos:/klx-sdk-release-public/xpytorch/dev_kl3/torch_xray/latest/torch_xray-999.9.9-cp38-cp38-linux_x86_64.whl + +[https://su.bcebos.com/klx-sdk-release-public/xpytorch/dev_kl3/torch_xray/latest/](https://su.bcebos.com/klx-sdk-release-public/xpytorch/dev_kl3/torch_xray/latest/torch_xray-999.9.9-py3-none-any.whl)torch_xray-999.9.9-cp38-cp38-linux_x86_64.whl + +Note that the same installation package must be used when using it in different environments. + +#### 2.Use + +##### Dump module-level inputs and outputs and compare their precision. + +Below is a sample code snippet used to dump the input and output of the vision module and compare the errors in the vllm framework. + +```bash +from torch_xray import PrecisionDebugger + +def execute_model( + self, + scheduler_output: "SchedulerOutput", + intermediate_tensors: Optional[IntermediateTensors] = None, + ) -> Union[ModelRunnerOutput, AsyncModelRunnerOutput, IntermediateTensors]: + # dump_path # Path to store dump results + # rank # Rank that needs to be dumped + # step # Setting the inference value to 1 is sufficient. + # model # The module to be dumped must be of type nn.module + debugger = PrecisionDebugger(dump_path="dump-vision", hook_name="dump", rank=[0], step=[1], model=self.model.visual, dump_torch_api=False) + debugger.start() + ........ +``` + +The results directory will generate an h5 file and a csv file. + +```bash +-rw-r--r-- 1 root root 471231309 Oct 31 13:12 globalrank-0_localrank-0.h5 +-rw-r--r-- 1 root root 71 Oct 31 13:11 globalrank-0_localrank-0_summary.csv +``` + +##### Data processing + +```bash +summary xxx.h5 sum.txt +``` + +The generated h5 file is processed using the summary command to generate a txt file in which the results are presented in tabular form. + +```bash ++-------+------+------+-----------------------------------------------------------+-------------+-------------+--------------+-------------+ +| Index | Step | Rank | Module | Min | Max | Mean | Std | ++-------+------+------+-----------------------------------------------------------+-------------+-------------+--------------+-------------+ +| 0 | 1 | 0 | patch_embed.proj.Conv3d.0.forward_params.weight | -0.0776367 | 0.0795898 | 6.8e-06 | 0.0072608 | +| 1 | 1 | 0 | patch_embed.proj.Conv3d.0.forward_params.bias | -3.046875 | 2.953125 | 0.0113748 | 0.3257138 | +| 2 | 1 | 0 | patch_embed.proj.Conv3d.0.forward_input.0 | -0.7490234 | 0.7021484 | 0.3302804 | 0.2339017 | +| 3 | 1 | 0 | patch_embed.proj.Conv3d.0.forward_output.0 | -4.0078125 | 5.1210938 | 0.0147052 | 0.3815643 | +| 4 | 1 | 0 | pos_embed.Embedding.0.forward_params.weight | -13.8125 | 20.25 | 0.0010043 | 0.2428094 | +| 5 | 1 | 0 | pos_embed.Embedding.0.forward_input.0 | 0.0 | 2303.0 | 1153.9191895 | 714.594360 | +| 6 | 1 | 0 | pos_embed.Embedding.0.forward_output.0 | -13.8125 | 20.25 | 0.0007552 | 0.2643428 | +| 7 | 1 | 0 | rotary_pos_emb.Qwen2_5_VisionRotaryEmbedding.0.forward... | 0.0 | 25.0 | 1.7337022 | 3.9271674 | +| 8 | 1 | 0 | blocks.0.norm1.LayerNorm.0.forward_params.weight | -0.5351562 | 3.140625 | 0.4660275 | 0.7907906 | +| 9 | 1 | 0 | blocks.0.norm1.LayerNorm.0.forward_params.bias | -2.359375 | 2.921875 | 0.0013793 | 0.1879374 | +| 10 | 1 | 0 | blocks.0.norm1.LayerNorm.0.forward_input.0 | -15.65625 | 20.21875 | 0.0155256 | 0.4382802 | +| 11 | 1 | 0 | blocks.0.norm1.LayerNorm.0.forward_output.0 | -6.1640625 | 6.7460938 | 0.0006746 | 0.2708515 | +| 12 | 1 | 0 | blocks.0.attn.qkv.QKVParallelLinear.0.forward_params.bias | -6.125 | 6.1875 | -0.0292423 | 0.8602651 | +| 13 | 1 | 0 | blocks.0.attn.qkv.QKVParallelLinear.0.forward_input.0 | -6.1640625 | 6.7460938 | 0.0006746 | 0.2708515 | +| 14 | 1 | 0 | blocks.0.attn.qkv.QKVParallelLinear.0.forward_output.0 | -6.5859375 | 7.6171875 | -0.0125549 | 1.0678084 | +| 15 | 1 | 0 | blocks.0.attn.proj.RowParallelLinear.0.forward_params... | -3.578125 | 3.203125 | -0.0043617 | 0.4846557 | +| 16 | 1 | 0 | blocks.0.attn.proj.RowParallelLinear.0.forward_input.0 | -1.9130859 | 1.4375 | 0.0005577 | 0.0947055 | +| 17 | 1 | 0 | blocks.0.attn.proj.RowParallelLinear.0.forward_output.0 | -9.109375 | 7.3867188 | -0.0034284 | 0.4465481 | +| 18 | 1 | 0 | blocks.0.norm2.LayerNorm.1.forward_params.weight | -0.1376953 | 14.5625 | 1.9166113 | 3.017405 | +| 19 | 1 | 0 | blocks.0.norm2.LayerNorm.1.forward_params.bias | -1.6328125 | 3.84375 | 0.0062865 | 0.2443586 | +| 20 | 1 | 0 | blocks.0.norm2.LayerNorm.1.forward_input.0 | -8.5859375 | 11.109375 | 0.0120974 | 0.4243064 | +| 21 | 1 | 0 | blocks.0.norm2.LayerNorm.1.forward_output.0 | -12.015625 | 14.265625 | -0.0012364 | 0.4973041 | +| 22 | 1 | 0 | blocks.0.mlp.linear_fc1.ColumnParallelLinear.0.forwar... | -9.4375 | 0.7304688 | -2.4200516 | 1.6754951 | +| 23 | 1 | 0 | blocks.0.mlp.linear_fc1.ColumnParallelLinear.0.forwar... | -12.015625 | 14.265625 | -0.0012364 | 0.4973041 | +| 24 | 1 | 0 | blocks.0.mlp.linear_fc1.ColumnParallelLinear.0.forwar... | -12.59375 | 13.0625 | -2.1465943 | 1.8433502 | +| 25 | 1 | 0 | blocks.0.mlp.act_fn.GELU.0.forward_input.0 | -12.59375 | 13.0625 | -2.1465943 | 1.8433502 | ++-------+------+------+-----------------------------------------------------------+-------------+-------------+--------------+-------------+ +``` + +##### Accuracy Comparison + +```bash +# The results are stored in result.csv +compare xpu.h5 gpu.h5 result.csv +``` + +The `compare` command is used to process the H5 files generated on the GPU and XPU, resulting in a CSV file. This CSV file is then downloaded to the local machine and opened with Excel, yielding a result similar to the image below. + +If you encounter a "no matched keys" problem, please refer to the instructions at the end of this article for a solution. + + +##### Example of results + +```bash ++-------+--------+-----------------------------------------------------------+--------+-----------+-------------+-------------+--------+ +| Index | Status | Module (Bench/Target) | Cosine | RMSE | IsClose (%) | Max Err (t) | GtNum | ++-------+--------+-----------------------------------------------------------+--------+-----------+-------------+-------------+--------+ +| 0 | | patch_embed.proj.Conv3d.0.forward_params.weight | 1 | 0 | 100 | 0 | 0 | +| 1 | | patch_embed.proj.Conv3d.0.forward_params.bias | 1 | 0 | 100 | 0 | 0 | +| 2 | | patch_embed.proj.Conv3d.0.forward_input.0 | 1 | 0 | 100 | 0 | 0 | +| 3 | | patch_embed.proj.Conv3d.0.forward_output.0 | 1 | 9.90E-06 | 100 | 0.001953 | 267 | +| 4 | | pos_embed.Embedding.0.forward_params.weight | 1 | 0 | 100 | 0 | 0 | +| 5 | | pos_embed.Embedding.0.forward_input.0 | 1 | 0 | 100 | 0 | 0 | +| 6 | | pos_embed.Embedding.0.forward_output.0 | 1 | 0 | 100 | 0 | 0 | +| 7 | | rotary_pos_emb.Qwen2_5_VisionRotaryEmbedding.0.forward... | 1 | 0 | 100 | 0 | 0 | +| 8 | | blocks.0.norm1.LayerNorm.0.forward_params.weight | 1 | 0 | 100 | 0 | 0 | +| 9 | | blocks.0.norm1.LayerNorm.0.forward_params.bias | 1 | 0 | 100 | 0 | 0 | +| 10 | | blocks.0.norm1.LayerNorm.0.forward_input.0 | 1 | 1.14E-05 | 100 | 0.00390625 | 216 | +| 11 | | blocks.0.norm1.LayerNorm.0.forward_output.0 | 1 | 1.84E-05 | 99.98 | 0.0078125 | 1585 | +| 12 | | blocks.0.attn.qkv.QKVParallelLinear.0.forward_params.bias | 1 | 0 | 100 | 0 | 0 | +| 13 | | blocks.0.attn.qkv.QKVParallelLinear.0.forward_input.0 | 1 | 1.84E-05 | 99.98 | 0.0078125 | 1585 | +| 14 | | blocks.0.attn.qkv.QKVParallelLinear.0.forward_output.0 | 1 | 0.0002776 | 99.53 | 0.00390625 | 119074 | +| 15 | | blocks.0.attn.proj.RowParallelLinear.0.forward_params... | 1 | 0 | 100 | 0 | 0 | +| 16 | | blocks.0.attn.proj.RowParallelLinear.0.forward_input.0 | 1 | 3.40E-05 | 99.07 | 0.0012207 | 52482 | +| 17 | | blocks.0.attn.proj.RowParallelLinear.0.forward_output.0 | 1 | 0.0001283 | 99.07 | 0.00390625 | 50591 | +| 18 | | blocks.0.norm2.LayerNorm.1.forward_params.weight | 1 | 0 | 100 | 0 | 0 | +| 19 | | blocks.0.norm2.LayerNorm.1.forward_params.bias | 1 | 0 | 100 | 0 | 0 | +| 20 | | blocks.0.norm2.LayerNorm.1.forward_input.0 | 1 | 0.0001437 | 99.01 | 0.0039062 | 31376 | +| 21 | Fail | blocks.0.norm2.LayerNorm.1.forward_output.0 | 1 | 0.0002779 | 98.72 | 0.015625 | 40770 | +| 22 | | blocks.0.mlp.linear_fc1.ColumnParallelLinear.0.forward... | 1 | 0 | 100 | 0 | 0 | +| 23 | Fail | blocks.0.mlp.linear_fc1.ColumnParallelLinear.0.forward... | 1 | 0.0002779 | 98.72 | 0.015625 | 40770 | +| 24 | | blocks.0.mlp.linear_fc1.ColumnParallelLinear.0.forward... | 1 | 0.000779 | 98.67 | 0.0078125 | 196313 | +| 25 | | blocks.0.mlp.act_fn.GELU.0.forward_input.0 | 1 | 0.000779 | 98.67 | 0.0078125 | 196313 | +| 26 | | blocks.0.mlp.act_fn.GELU.0.forward_output.0 | 1 | 0.0001012 | 98.08 | 0.0039062 | 153508 | ++-------+--------+-----------------------------------------------------------+--------+-----------+-------------+-------------+--------+ +``` + +Generally, the main focus is on Min Err/Max Err. + +##### Indicator Explanation + +To be improved... + +#### The dump operator is tested and run. + +```bash +X_DEBUG=0x102 # trace operator name、arguments shape、dtype、data_range +X_DEDUP=True # Remove duplicates based on shape and dtype. +X_DUMP_NUM # The default value is 0, meaning no tensor data is saved. Setting it to n means that n parameters are randomly selected from each operator to save the actual parameters. +``` + +Below is a sample code snippet that dumps information such as the size and dtype of the forward operator of Qwen3_VisionTransformer. During runtime, an xray_debug directory will be automatically created in the current directory to store the dump results. + +```bash +from torch_xray import begin_dump, end_dump +............. + +class Qwen3_VisionTransformer(nn.Module): + + def __init__( + self, + vision_config: Qwen3VLVisionConfig, + norm_eps: float = 1e-6, + quant_config: Optional[QuantizationConfig] = None, + prefix: str = "", + use_data_parallel: bool = False, + ) -> None: + super().__init__() + self.hidden_size = vision_config.hidden_size + .......... + def forward( + self, + x: torch.Tensor, + grid_thw: list[list[int]], + ) -> torch.Tensor: + # Start dump + # X_DEBUG=0x102 # trace operator name、arguments shape、dtype、data_range + # X_DEDUP=True # Remove duplicates based on shape and dtype. + # The default value is 0, meaning no tensor data is saved. Setting it to n means that n parameters are randomly selected from each operator to save the actual parameters. + begin_dump(X_DEBUG=0x102, X_DEDUP=True, X_DUMP_NUM=5) + + hidden_states = x.to(device=self.device, dtype=self.dtype) + hidden_states = self.patch_embed(hidden_states) + ........... + + # End dump + end_dump(clear_context=True) + return hidden_states +``` +This is the file directory. +```bash +├── xary_debug/ +│ ├── proc_xxx/ # Process-based storage results +│ ├── dump/ # The dumped tensor +│ ├── dump.json # Information needed to generate unit tests, such as input/output size and dtype. +``` + +##### Generate unit test + +jprof --cpu_init --blacklist --factory=load dump.json + +Create a pytests directory in the current directory to store unit tests. + +##### Run unit test + +The GPU only needs to copy the XPU's pytests directory and execute it. + +Since the unit test program defaults to finding the actual dumped tensors using relative paths, this step must be performed in the xary_debug/ directory. + +```bash +# detail_compare_path stores the unit test results. +pytest --detail_compare_path=./xxx.csv proc_xxx/pytests/ --seed 42 +``` + +##### Results Comparison + +```bash +# After obtaining two result CSV files, compare them and generate result.csv. +summary_diff_check ./xpu.csv ./gpu.csv ./result.csv +``` + +##### Example of results + +```bash ++------------+-----------------------+-------------+-------------+-----------+----------+---------+---------+----------+ +| name | op_name | dtype | shape | min-val | max-val | is_pass | xpu_max | gpu_max | ++------------+-----------------------+-------------+-------------+-----------+----------+---------+---------+----------+ +| 00004-aten | aten.linspace.default | torch.float | [10] | 0 | 47 | pass | 0 | 1.91E-06 | +| 00005-aten | aten.linspace.default | torch.float | [26] | 0 | 47 | pass | 0 | 0 | +| 00027-aten | aten.add.Tensor | torch.int64 | [10, 26] | 0 | 0 | pass | 0 | 0 | +| 00028-aten | aten.add.Tensor | torch.int64 | [10, 26] | 0 | 0 | pass | 0 | 0 | +| 00037-aten | aten.add.Tensor | torch.float | [260, 1152] | -29.09375 | 33.75 | pass | 0 | 0 | +| 00038-aten | aten.add.Tensor | torch.float | [260, 1152] | -27.1875 | 37.625 | pass | 0 | 0 | +| 00047-aten | aten.add.Tensor | torch.float | [260, 1152] | -28.98438 | 42.34375 | pass | 0 | 0 | +| 00082-aten | aten.sub.Tensor | torch.int32 | [1] | 0 | 0 | pass | 0 | 0 | ++------------+-----------------------+-------------+-------------+-----------+----------+---------+---------+----------+ +``` + +The main focus is on the values ​​of gpu_1e-1, xpu_1e-1, etc., which represent the number of elements whose error between the gpu/xpu result and the cpu result exceeds the order of 1e-n. This serves as the primary basis for determining whether there is a problem with the operator's precision. + +#### Replenish + +##### Bypassing the issue of differing naming conventions between Kunlun Card and GPU modules, which prevents diff calculation. + +```bash +# +blocks.0.mlp.linear_fc1.ColumnParallelLinear.0.forward_params.bias +# +blocks.0.mlp.linear_fc1.ColumnParalleLinear.forward_params.bias +``` + +As shown in the figure above, due to various reasons, the module names dumped by the GPU and XPU are often different, and the compare command cannot be used to identify them directly. + +```python +for step in steps: # (['/'] for group creation order h5py >= 3.10.0) + # for bench_key, target_key in get_matched_names( + # list(dump_ben[str(step)].keys()), + # list(dump_tar[str(step)].keys()), + # fuzzy_match, + # ): + for bench_key, target_key in zip( + list(dump_ben[str(step)].keys()), + list(dump_tar[str(step)].keys()), +): +``` + +Modify torch_xray/compare/compare.py to skip the get_matched_name step. This modification will allow for line-by-line comparison even if module names differ, producing a compare result. However, it's crucial to ensure that the number of rows in the GPU and XPU dumps is consistent. \ No newline at end of file diff --git a/docs/source/developer_guide/evaluation/accuracy/accuracy_server.md b/docs/source/developer_guide/evaluation/accuracy/accuracy_server.md new file mode 100644 index 0000000..b0d0d1f --- /dev/null +++ b/docs/source/developer_guide/evaluation/accuracy/accuracy_server.md @@ -0,0 +1,240 @@ +## Overall accuracy test + +### EvalScope + +#### 1.Download and install + +EvalScope supports use in Python environments. Users can install EvalScope via pip or from source code. Here are examples of both installation methods: + +```bash +#pip +pip install evalscope[perf] -U +#git +git clone https://github.com/modelscope/evalscope.git +cd evalscope +pip install -e '.[perf]' +``` + +#### 2.Dataset preparation script + +```python +from evalscope.collections import CollectionSchema, DatasetInfo, WeightedSampler +from evalscope.utils.io_utils import dump_jsonl_data +import os # Step 1: Import the os module + +schema = CollectionSchema( + name="VL-Test", + datasets=[ + CollectionSchema( + name="PureText", + weight=1, + datasets=[ + DatasetInfo( + name="mmlu_pro", + weight=1, + task_type="exam", + tags=["en"], + args={"few_shot_num": 0}, + ), + DatasetInfo( + name="ifeval", + weight=1, + task_type="instruction", + tags=["en"], + args={"few_shot_num": 0}, + ), + DatasetInfo( + name="gsm8k", + weight=1, + task_type="math", + tags=["en"], + args={"few_shot_num": 0}, + ), + ], + ), + CollectionSchema( + name="Vision", + weight=2, + datasets=[ + DatasetInfo( + name="math_vista", + weight=1, + task_type="math", + tags=["en"], + args={"few_shot_num": 0}, + ), + DatasetInfo( + name="mmmu_pro", + weight=1, + task_type="exam", + tags=["en"], + args={"few_shot_num": 0}, + ), + ], + ), + ], +) + + +# get the mixed data +mixed_data = WeightedSampler(schema).sample(1000) + +output_path = "outputs/vl_test.jsonl" # Step 2: Define the output file path +output_dir = os.path.dirname(output_path) # Step 3: Obtain the directory name +if not os.path.exists(output_dir): # Step 4: Check if the directory exists + os.makedirs(output_dir, exist_ok=True) # Step 5: Automatically create directories + + +# dump the mixed data to a jsonl file +dump_jsonl_data(mixed_data, output_path) # Step 6: Securely write to the file +``` + +Dataset composition visualization: + +``` +┌───────────────────────────────────────┐ +│ VL-Test (1000 samples) │ +├─────────────────┬─────────────────────┤ +│ PureText │ Vision │ +│ (333 samples) │ (667 samples) │ +├─────────────────┼─────────────────────┤ +│ • mmlu_pro │ • math_vista │ +│ • ifeval │ • mmmu_pro │ +│ • gsm8k │ │ +└─────────────────┴─────────────────────┘ +``` + +#### 3.Test + +```python +from dotenv import dotenv_values + +from evalscope import TaskConfig, run_task +from evalscope.constants import EvalType + +task_cfg = TaskConfig( + model="Qwen2.5-VL-7B-Instruct", + api_url="http://localhost:8804/v1", + api_key="EMPTY", + eval_type=EvalType.SERVICE, + datasets=[ + "data_collection", + ], + dataset_args={ + "data_collection": { + "local_path": "../outputs/vl_test.jsonl", + } + }, + eval_batch_size=5, + generation_config={ + "max_tokens": 30000, # The maximum number of tokens that can be generated should be set to a large value to avoid output truncation. + "temperature": 0.6, # Sampling temperature (recommended value from qwen report) + "top_p": 0.95, # top-p sampling (recommended value from qwen report) + "top_k": 20, # Top-k sampling (recommended value from qwen report) + "n": 1, # Number of responses generated per request + "repetition_penalty": 1.0, # 1.0 = Penalty disabled, >1.0 = Penalty repeated. + }, +) + +run_task(task_cfg=task_cfg) +``` + +Parameter Tuning Guide: + +| Parameter | Current value | Effect | Adjustment suggestions | +| ----------------- | ------------- | ---------------------------------------- | -------------------------------------------------------- | +| `temperature` | 0.6 | Control output diversity | Math problems ↓ 0.3 / Creative writing ↑ 0.9 | +| `top_p` | 0.95 | Filtering low-probability tokens | Reduce "nonsense" | +| `eval_batch_size` | 5 | Number of requests processed in parallel | With sufficient video memory, it can be increased to 10. | + +Run the test: + +```bash +#!/bin/bash +# ======================================== +# Step 1: Set the log file path +# ======================================== +LOG_FILE="accuracy_$(date +%Y%m%d_%H%M).log" + +# ======================================== +# Step 2: Execute the Python script and capture all output +# Meaning of 2>&1: +# - 2 represents standard error output (stderr) +# ->& represents redirection and merging +# - 1 represents standard output (stdout) +# Function: Merges error messages into standard output as well. +# ======================================== +python accuracy.py 2>&1 | tee "$LOG_FILE" + +# ======================================== +# Step 3: Check Execution Status +# ${PIPESTATUS[0]} Get the exit code of the first command (Python) in the pipeline +# ======================================== +EXIT_CODE=${PIPESTATUS[0]} +if [ $EXIT_CODE -eq 0 ]; then + echo "✅ Evaluation completed! Log saved to: $LOG_FILE" +else + echo "❌ Evaluation failed! Exit code: $EXIT_CODE Please check the log: $LOG_FILE" +fi +``` + +#### 4.Common problem fixes + +##### 4.1 NLTK resource missing fix + +```bash +Resource punkt_tab not found. +``` + +Solution： + +```python +import nltk +import os + +# Step 1: Set the download path (select a writable directory) +download_dir = "/workspace/myenv/nltk_data" +os.makedirs(download_dir, exist_ok=True) + +# Step 2: Configure NLTK data path +nltk.data.path.append(download_dir) + +# Step 3: Download necessary resources +print("🔽 Start downloading punkt_tab resource...") +try: + nltk.download("punkt_tab", download_dir=download_dir) + print("✅ Download successful!") +except Exception as e: + print(f"❌ Download failed: {e}") + print("💡 Alternative: Download manually from GitHub") + print( + " URL: https://raw.githubusercontent.com/nltk/nltk_data/gh-pages/packages/tokenizers/punkt_tab.zip" + ) +``` + +repair: + +```bash +# Activate environment +source /workspace/myenv/bin/activate + +# Run the repair script +python fix_nltk.py + +# Rerun the test +bash run_accuracy_test.sh +``` + +#### 5.Results Display + +```bash ++-------------+---------------------+--------------+---------------+-------+ +| task_type | metric | dataset_name | average_score | count | ++-------------+---------------------+--------------+---------------+-------+ +| exam | acc | mmmu_pro | 0.521 | 334 | +| math | acc | math_vista | 0.6066 | 333 | +| exam | acc | mmlu_pro | 0.5405 | 111 | +| instruction | prompt_level_strict | ifeval | 0.6937 | 111 | +| math | acc | gsm8k | 0.8288 | 111 | ++-------------+---------------------+--------------+---------------+-------+ +``` diff --git a/docs/source/developer_guide/evaluation/accuracy/index.md b/docs/source/developer_guide/evaluation/accuracy/index.md new file mode 100644 index 0000000..5b9a025 --- /dev/null +++ b/docs/source/developer_guide/evaluation/accuracy/index.md @@ -0,0 +1,10 @@ +# Accuracy + +This document details the accuracy testing methods for vllm-kunlun and the analysis of the results. + +:::{toctree} +:caption: Accuracy +:maxdepth: 1 +accuracy_server +accuracy_kernel +::: diff --git a/docs/source/developer_guide/evaluation/accuracy_report/GLM-4.5-Air.md b/docs/source/developer_guide/evaluation/accuracy_report/GLM-4.5-Air.md new file mode 100644 index 0000000..f6198fa --- /dev/null +++ b/docs/source/developer_guide/evaluation/accuracy_report/GLM-4.5-Air.md @@ -0,0 +1,18 @@ +# GLM-Air-4.5 + +* vLLM Version: vLLM: 0.10.1.1 , vLLM-KunLun Version: v0.10.1.1 +* Software Environment:OS: Ubuntu 22.04, PyTorch ≥ 2.5.1 +* Hardware Environment: KunLun P800 +* Parallel mode:TP8 + +```bash ++-------------+----------+---------------+---------+-----+--------+---------+ +| Model | Dataset | Metric | Subset | Num | Score | Cat.0 | ++-------------+----------+---------------+---------+-----+--------+---------+ +| GLM-4.5-Air | math_500 | AveragePass@1 | Level 1 | 43 | 0.9302 | default | +| GLM-4.5-Air | math_500 | AveragePass@1 | Level 2 | 90 | 0.9222 | default | +| GLM-4.5-Air | math_500 | AveragePass@1 | Level 3 | 105 | 0.8762 | default | +| GLM-4.5-Air | math_500 | AveragePass@1 | Level 4 | 128 | 0.8984 | default | +| GLM-4.5-Air | math_500 | AveragePass@1 | Level 5 | 134 | 0.8955 | default | ++-------------+----------+---------------+---------+-----+--------+---------+ +``` diff --git a/docs/source/developer_guide/evaluation/accuracy_report/GLM-4.5.md b/docs/source/developer_guide/evaluation/accuracy_report/GLM-4.5.md new file mode 100644 index 0000000..7ae7eac --- /dev/null +++ b/docs/source/developer_guide/evaluation/accuracy_report/GLM-4.5.md @@ -0,0 +1,18 @@ +# GLM-4.5 + +* vLLM Version: vLLM: 0.10.1.1 , vLLM-KunLun Version: v0.10.1.1 +* Software Environment:OS: Ubuntu 22.04, PyTorch ≥ 2.5.1 +* Hardware Environment: KunLun P800 +* Parallel mode:TP8 + +```bash ++---------+----------+---------------+---------+-----+--------+---------+ +| Model | Dataset | Metric | Subset | Num | Score | Cat.0 | ++---------+----------+---------------+---------+-----+--------+---------+ +| GLM-4.5 | math_500 | AveragePass@1 | Level 1 | 43 | 0.9302 | default | +| GLM-4.5 | math_500 | AveragePass@1 | Level 2 | 90 | 0.8111 | default | +| GLM-4.5 | math_500 | AveragePass@1 | Level 3 | 105 | 0.7143 | default | +| GLM-4.5 | math_500 | AveragePass@1 | Level 4 | 128 | 0.6172 | default | +| GLM-4.5 | math_500 | AveragePass@1 | Level 5 | 134 | 0.5149 | default | ++---------+----------+---------------+---------+-----+--------+---------+ +``` \ No newline at end of file diff --git a/docs/source/developer_guide/evaluation/accuracy_report/InternVL3_5-30B-A3B.md b/docs/source/developer_guide/evaluation/accuracy_report/InternVL3_5-30B-A3B.md new file mode 100644 index 0000000..84f99b4 --- /dev/null +++ b/docs/source/developer_guide/evaluation/accuracy_report/InternVL3_5-30B-A3B.md @@ -0,0 +1,18 @@ +# InternVL3_5-30B-A3B + +* vLLM Version: vLLM: 0.10.1.1 , vLLM-KunLun Version: v0.10.1.1 +* Software Environment:OS: Ubuntu 22.04, PyTorch ≥ 2.5.1 +* Hardware Environment: KunLun P800 +* Parallel mode:TP8 + +``` ++-------------+---------------------+--------------+---------------+-------+ +| task_type | metric | dataset_name | average_score | count | ++-------------+---------------------+--------------+---------------+-------+ +| exam | acc | mmmu_pro | 0.5449 | 334 | +| math | acc | math_vista | 0.6847 | 333 | +| exam | acc | mmlu_pro | 0.6126 | 111 | +| instruction | prompt_level_strict | ifeval | 0.7658 | 111 | +| math | acc | gsm8k | 0.9369 | 111 | ++-------------+---------------------+--------------+---------------+-------+ +``` \ No newline at end of file diff --git a/docs/source/developer_guide/evaluation/accuracy_report/Qwen2.5-VL-7B-Instruct.md b/docs/source/developer_guide/evaluation/accuracy_report/Qwen2.5-VL-7B-Instruct.md new file mode 100644 index 0000000..b0cb732 --- /dev/null +++ b/docs/source/developer_guide/evaluation/accuracy_report/Qwen2.5-VL-7B-Instruct.md @@ -0,0 +1,18 @@ +# Qwen2.5-VL-7B-Instruct + +* vLLM Version: vLLM: 0.10.1.1 , vLLM-KunLun Version: v0.10.1.1 +* Software Environment:OS: Ubuntu 22.04, PyTorch ≥ 2.5.1 +* Hardware Environment: KunLun P800 +* Parallel mode:TP1 + +``` ++-------------+---------------------+--------------+---------------+-------+ +| task_type | metric | dataset_name | average_score | count | ++-------------+---------------------+--------------+---------------+-------+ +| exam | acc | mmmu_pro | 0.521 | 334 | +| math | acc | math_vista | 0.6066 | 333 | +| exam | acc | mmlu_pro | 0.5405 | 111 | +| instruction | prompt_level_strict | ifeval | 0.6937 | 111 | +| math | acc | gsm8k | 0.8288 | 111 | ++-------------+---------------------+--------------+---------------+-------+ +``` \ No newline at end of file diff --git a/docs/source/developer_guide/evaluation/accuracy_report/index.md b/docs/source/developer_guide/evaluation/accuracy_report/index.md new file mode 100644 index 0000000..6131052 --- /dev/null +++ b/docs/source/developer_guide/evaluation/accuracy_report/index.md @@ -0,0 +1,10 @@ +# Accuracy Report + +:::{toctree} +:caption: Accuracy Report +:maxdepth: 1 +Qwen2.5-VL-7B-Instruct +InternVL3_5-30B-A3B +GLM-4.5 +GLM-4.5-Air +::: diff --git a/docs/source/developer_guide/evaluation/index.md b/docs/source/developer_guide/evaluation/index.md new file mode 100644 index 0000000..c071515 --- /dev/null +++ b/docs/source/developer_guide/evaluation/index.md @@ -0,0 +1,8 @@ +# Accuracy + +:::{toctree} +:caption: Accuracy +:maxdepth: 1 +accuracy/index +accuracy_report/index +::: diff --git a/docs/source/developer_guide/feature_guide/Kunlun_Graph.md b/docs/source/developer_guide/feature_guide/Kunlun_Graph.md new file mode 100644 index 0000000..66471ce --- /dev/null +++ b/docs/source/developer_guide/feature_guide/Kunlun_Graph.md @@ -0,0 +1,76 @@ +# Kunlun Graph + +## Why we need Kunlun Graph? + +When in LLM inference, each token requires nearly thousand operator executions, and when host launching operators are slower than device, it will cause host bound. In severe cases, the device will be idle for more than half of the time. To solve this problem, we use graph in LLM inference. + +``` +eager mode: + +host: | launch op1 | launch op2 | launch op3 | launch op4 | launch op5 | + +device: | run op1 |free| run op2 |free| run op3 |free| run op4 |free| run op5 | + + | <----- total time -----> | + +graph mode: + +host: | launch graph | + +device: | run op1 | run op2 | run op3 | run op4 | run op5 | + + | <----- total time -----> | + +``` + +## How to use Kunlun Graph? + +Kunlun Graph is enabled by default in V1 Engine, just need to check that `enforce_eager` is not set to `True`. + +## How it works? + +In short, graph mode works in two steps: **capture and replay**. When engine starts, we will capture all of the ops in model forward and save it as a graph, and when req come in, we just replay the graph on devices, and waiting for result. + +But in reality, graph mode is not that simple. + +### Padding and Bucketing + +Due to graph can only replay the ops captured before, without doing tiling and checking graph input, we need to ensure the consistency of the graph input, but we know that model input's shape depends on the request scheduled by Scheduler, we can't ensure the consistency. + +Obviously, we can solve this problem by capturing the biggest shape and padding all of the model input to it. But it will bring a lot of redundant computing and make performance worse. So we can capture multiple graphs with different shape, and pad the model input to the nearest graph, which will greatly reduce redundant computing. But when `max_num_batched_tokens` is very large, the number of graphs that need to be captured will also become very large. But we know that when intensor's shape is large, the computing time will be very long, and graph mode is not necessary in this case. So all of things we need to do is: +1. Set a threshold; +2. When `num_scheduled_tokens` is bigger than the threshold, use `eager_mode`; +3. Capture multiple graphs within a range below the threshold; + +``` +| graph1 | +| graph2 | +| graph3 | +| graph4 | # the threshold + +| input1 | pad | # use graph1 +| input2 | # don't need pad +| input3 | pad | # use graph4 +| input4 | # use eager mode + +``` + +### Piecewise and Full graph + +Due to the increasing complexity of the attention layer in current LLM, we can't ensure all types of attention can run in graph. In MLA, prefill_tokens and decode_tokens have different calculation method, so when a batch has both prefills and decodes in MLA, graph mode is difficult to handle this situation. + +vLLM solves this problem with piecewise graph mode. We use eager mode to launch attention's ops, and use graph to deal with others. But it also bring some problems: The cost of launching ops has become large again, although much smaller than eager mode, but it will also lead to host bound when cpu is poor or `num_tokens` is small. + + +## How it be implemented? + +vLLM has already implemented most of the modules in graph mode. You can see more details at: [CUDA Graphs](https://docs.vllm.ai/en/latest/design/cuda_graphs.html) + +When in graph mode, vLLM will call `current_platform.get_static_graph_wrapper_cls` to get current device's graph model wrapper, so what we need to do is to implement the graph mode wrapper on Kunlun: `Kunlun Graph Wrapper`. + +vLLM has added `support_torch_compile` decorator to all models, this decorator will replace the `__init__` and `forward` interface of the model class, and when `forward` called, the code inside the `vllm_kunlun.compilation` will be executed, and it will do capture or replay as mentioned above. + +## Limitation + +1. `FULL` and `FULL_AND_PIECEWISE` are not supported now; +3. `use_inductor` is not supported now; diff --git a/docs/source/developer_guide/feature_guide/index.md b/docs/source/developer_guide/feature_guide/index.md new file mode 100644 index 0000000..c7cb064 --- /dev/null +++ b/docs/source/developer_guide/feature_guide/index.md @@ -0,0 +1,9 @@ +# Feature Guide + +This section provides an overview of the features implemented in vLLM-Kunlun. Developers can refer to this guide to understand how vLLM-Kunlun works. + +:::{toctree} +:caption: Feature Guide +:maxdepth: 1 +Kunlun_Graph +::: diff --git a/docs/source/developer_guide/performance/index.md b/docs/source/developer_guide/performance/index.md new file mode 100644 index 0000000..fd4fb7a --- /dev/null +++ b/docs/source/developer_guide/performance/index.md @@ -0,0 +1,7 @@ +# Performance + +:::{toctree} +:caption: Performance +:maxdepth: 1 +performance_benchmark/index +::: diff --git a/docs/source/developer_guide/performance/performance_benchmark/benchmark_kernel.md b/docs/source/developer_guide/performance/performance_benchmark/benchmark_kernel.md new file mode 100644 index 0000000..0330a02 --- /dev/null +++ b/docs/source/developer_guide/performance/performance_benchmark/benchmark_kernel.md @@ -0,0 +1,147 @@ +## Operator performance + +### XProfiler + +#### 1.Download and install + +- The download link for the x86_64 platform installation package xre-Linux-x86_64 is: + +`https://klx-sdk-release-public.su.bcebos.com/xre/kl3-release/5.0.21.26/peermem/xre-Linux-x86_64-5.0.21.26.run` + +`https://klx-sdk-release-public.su.bcebos.com/xre/kl3-release/5.0.21.26/peermem/xre-Linux-x86_64-5.0.21.26.tar.gz` + +- If the client is using bdCentOS, we recommend using the following download link: + +`https://klx-sdk-release-public.su.bcebos.com/xre/kl3-release/5.0.21.26/xre-bdcentos-x86_64-5.0.21.26.tar.gz` + +After downloading and extracting, you can directly execute `xpu-installer` and `install_rt.sh` to install. + +#### 2.Start using + +XProfiler supports three modes: 1) fork mode; 2) time mode; and 3) daemon mode. After execution, XProfiler will generate two types of JSON files: + +- xprofiler.settings.json: Records the event configuration for this trace. + +- xprofiler.trace.json: Records the results of this trace. + +The specific modes will be introduced below. + +##### fork mode + +The fork pattern is used to track the entire time period from the start to the end of a user program. This pattern is suitable for most inference tasks and is the simplest to use. An example is shown below: + +```bash +/xxxx/xxxx/xprofiler -r500 --xpu=0 python test.py +``` + +- --r: Sets the trace time resolution in nanoseconds (ns). The default is 100. If an "out of space error" occurs, try increasing the -r value to 500. + +- --xpu: Specifies the acquisition device ID, supporting multi-card configuration. --xpu=all enables all cards; the default is card 0. + +More parameters can be found in the command-line parameters section later. + +##### time mode + +The time mode is used to track user programs for a period of time. This method is suitable for tasks that need to run for a long time. + +Using the -t or --time command-line parameter, XPorfiler will run for the specified time and then exit, in seconds. In this mode, the application needs to be started separately. An example is as follows: + +(1) Starting XPorfiler + +```bash +/xxxx/xxxx/xprofiler -r 500 --xpu=0 -t600 # Time mode collects events within a specified time period, measured in seconds (s). +``` + +A temporary .sock file will be generated in the execution directory. The path needs to be configured in the environment variables. + +(2) Start the program + +```bash +export XPU_ENABLE_PROFILER_TRACING=1 +export XPU_TRACING_OUTPUT_NAME=/xprofiler.sock +# Start your own program +python xxx.py +``` + +##### deamon mode + +The daemon mode is used to track the event timeline of a specified code segment, eliminating interference from redundant information. The startup command is the same as in fork mode. + +(1) Insert start and stop interfaces. + +```python +import xtorch_ops +# Only capture events during the generate phase +xtorch_ops.kunlun_profiler_start() + outputs = llm.generate( + inputs, + sampling_params=sampling_params, + lora_request=lora_request, + ) +xtorch_ops.kunlun_profiler_end() +``` + +(2) Launch X profiler in a terminal + +```python +# Specify the output file as the trace_output file in the current path. +/xxxx/xxxx/xprofiler-Linux_x86_64-2.0.2.0/bin/xprofiler -r 500 --xpu=0 -e ./trace_output -d +``` + +After startup, a .sock file will be generated in the current directory. + +```bash +xprofiler.sock +``` + +(3) Launch your own program on another terminal. + +```python +export XPU_ENABLE_PROFILER_TRACING=1 +# Here, the path to the .sock file from step 2 is used for assignment. +export XPU_TRACING_OUTPUT_NAME=/xprofiler.sock +# Start your own program +python xxx.py +``` + +Note: If you want to specify a particular card to run on, you must import the XPU_VISIBLE_DEVICES environment variable in the terminal in steps 2 and 3; otherwise, you will not be able to capture the data. + +##### More parameters + +| parameters | Example | default value | describe | +| -------------------------- | --------------------------------------- | ------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| -b or --buffer-size | -b=512 | 256 | Specifies the size of the trace buffer in MB. This is generally not required. However, if there are many trace signals, the buffer size can be increased appropriately to avoid OOS (Out of Size). | +| -x or --xpu | -x=0--xpu=0 | 0 | Set the card number to be tracked; multiple cards or all cards can be set. | +| -t or --time | -t=10 | off | Enable time mode, in seconds, to capture information over a specified period. | +| -d or --deamonize | -r500 | 0 | Enable daemon mode to retrieve events in the background. | +| -r or --export-profile | -e ./trace_output-e ./output/trace.json | ./ | Record the trace results to a document or folder. If this parameter is not specified, a default xprofiler.trace.json file will be generated in the execution directory. | +| -S or --settings | -S xprofiler.trace.json | off | xprofiler reads a JSON file containing the events that need to be traced. If this parameter is not configured, xprofiler enables `--profile-api-trace` and `--sse-trace` by default. | +| -A or --profiler-api-trace | -A | on | Get driver events. | +| -s or --sse-trace | -s | on | Get all SSE events. | +| -C or --cluster-trace | -C | off | Retrieve all cluster events. | +| -n or --sdnn-trace | -n | off | Get all SDNN events. | +| -c or --sdnn-cluster-trace | -c | off | Retrieve all SDNN cluster events. | +| -E or --cache-trace | -E | off | Get bandwidth statistics events. | +| -u or --debug | -u44:open log，debug level-u0:close log | 33 | Debug the interface and enable driver event/device event logging.。 | + +#### 3.View Results + +The generated xprofiler.trace.json file can be viewed and analyzed using a visual interface. Two tools are introduced here. + +##### Chrome browser + +Enter chrome://tracing/ in your browser (you may need to enable developer tools the first time you access this site), and click "load" in the top left corner to import the file. Interface display. + +![img](https://rte.weiyun.baidu.com/wiki/attach/image/api/imageDownloadAddress?attachId=89aef70f112a4394adcac8b03ef994db&docGuid=WFoZOcuqnSXJIE) + +##### prefetto ui + +Search directly, or visit[Perfetto UI](https://ui.perfetto.dev/#!/viewer?local_cache_key)，The interface is as follows。 + +![img](https://rte.weiyun.baidu.com/wiki/attach/image/api/imageDownloadAddress?attachId=895a715344e9473c9ee93518c3064b27&docGuid=WFoZOcuqnSXJIE) + +#### 4.Performance Analysis + +With various performance data available, analysis and optimization can then be performed based on the results. + +(Further details to be added later) diff --git a/docs/source/developer_guide/performance/performance_benchmark/benchmark_server.md b/docs/source/developer_guide/performance/performance_benchmark/benchmark_server.md new file mode 100644 index 0000000..b16615e --- /dev/null +++ b/docs/source/developer_guide/performance/performance_benchmark/benchmark_server.md @@ -0,0 +1,199 @@ +## vLLM server performance + +### vLLM benchmark CLI + +You can directly use vLLM's CLI benchmark. For more details, please refer to[vLLM Developer Guide Benchmark Suites](https://docs.vllm.ai/en/stable/contributing/benchmarks.html) + +#### 1.Online testing + +##### 1.1Start the vLLM server + +Server startup script reference + +```bash +USE_ORI_ROPE=1 VLLM_USE_V1=1 python -m vllm.entrypoints.openai.api_server \ + --host 0.0.0.0 \ + --port xxxx \ + --model /xxxx/xxxx/model\ + --gpu-memory-utilization 0.9 \ + --trust-remote-code \ + --max-model-len 32768 \ + --tensor-parallel-size 1 \ + --dtype float16 \ + --max_num_seqs 128 \ + --max_num_batched_tokens 32768 \ + --max-seq-len-to-capture 32768 \ + --block-size 128 \ + --no-enable-prefix-caching \ + --no-enable-chunked-prefill \ + --distributed-executor-backend mp \ + --served-model-name modelname \ + --compilation-config '{"splitting_ops": ["vllm.unified_attention_with_output_kunlun", + "vllm.unified_attention", "vllm.unified_attention_with_output", + "vllm.mamba_mixer2"]}' \ +``` + +##### 1.2Execute test + +To run the test script, you can refer to the code below. + +```bash +#!/bin/bash +# Run benchmark tests +python -m vllm.entrypoints.cli.main bench serve \ + --host 127.0.0.1 \ + --port xxxx \ + --backend vllm \ + --model modelname \ + --dataset-name random \ + --num-prompts 500 \ + --random-input-len 1024 \ + --random-output-len 1024 \ + --tokenizer /xxxx/xxxx/model \ + --ignore-eos 2>&1 | tee benchmark.log +``` + +##### 1.3Result + +The following content will be displayed after the process is complete. + +```bash +========== Serving Benchmark Result ========== +Successful requests: 500 +Benchmark duration (s): 144.89 +Total input tokens: 510414 +Total generated tokens: 512000 +Request throughput (req/s): 3.45 +Output token throughput (tok/s): 3533.68 +Total Token throughput (tok/s): 7056.42 +----------Time to First Token---------- +Mean TTFT (ms): 57959.61 +Median TTFT (ms): 43551.93 +P99 TTFT (ms): 116202.52 +----------Time per Output Token (excl. 1st token)---------- +Mean TPOT (ms): 33.30 +Median TPOT (ms): 34.15 +P99 TPOT (ms): 35.59 +----------Inter-token Latency---------- +Mean ITL (ms): 33.30 +Median ITL (ms): 29.05 +P99 ITL (ms): 46.14 +============================================ +``` + +Key Parameter Explanation: + +| index | meaning | Optimization Objective | +| --------------------------- | ------------------------------------| ---------- | +| ***\*Output Throughput\**** | Output token generation rate | ↑ The higher the better | +| ***\*Mean TTFT\**** | First Token Delay (Time To First Token) | ↓ The lower the better | +| ***\*P99 TTFT\**** | 99% of requests have delayed first token. | ↓ The lower the better | +| ***\*Mean TPOT\**** | Average generation time per output token | ↓ The lower the better | +| ***\*P99 TPOT\**** | 99% of requests' time per token generation | ↓ The lower the better | +| ***\*ITL\**** | Delay between adjacent output tokens | ↓ The lower the better | + +#### 2.Offline testing + +Comming soon... + +### EvalScope + +EvalScope is a comprehensive model testing tool that can test not only model accuracy but also performance. For more information, please visit [website address missing].[EvalScope](https://evalscope.readthedocs.io/en/latest/index.html)，A brief introduction follows. + +#### 1.Download and install + +EvalScope supports use in Python environments. Users can install EvalScope via pip or from source code. Here are examples of both installation methods: + +```bash +#pip +pip install evalscope[perf] -U +#git +git clone https://github.com/modelscope/evalscope.git +cd evalscope +pip install -e '.[perf]' +``` + +After downloading, some modules may be missing, causing the program to fail to run. Just follow the prompts to install them. + +#### 2.Start using + +The following demonstrates the performance test of the Qwen3-8B in a single-card scenario. + +##### 2.1Start the server + +The first step is to start the server. The example script is shown below. + +```bash +USE_ORI_ROPE=1 VLLM_USE_V1=1 python -m vllm.entrypoints.openai.api_server \ + --host 0.0.0.0 \ + --port xxxx \ + --model /xxxx/xxxx/Qwen3-8B\ + --gpu-memory-utilization 0.9 \ + --trust-remote-code \ + --max-model-len 32768 \ + --tensor-parallel-size 1 \ + --dtype float16 \ + --max_num_seqs 128 \ + --max_num_batched_tokens 32768 \ + --max-seq-len-to-capture 32768 \ + --block-size 128 \ + --no-enable-prefix-caching \ + --no-enable-chunked-prefill \ + --distributed-executor-backend mp \ + --served-model-name Qwen3-8B \ + --compilation-config '{"splitting_ops": ["vllm.unified_attention_with_output_kunlun", + "vllm.unified_attention", "vllm.unified_attention_with_output", + "vllm.mamba_mixer2"]}' \ +``` + +##### 2.2 Start EvalScope + +Start EvalScope to begin performance testing. + +```bash +evalscope perf \ + --parallel 1 10\#The number of concurrent requests can be tested at once, separated by spaces. + --number 10 20\#The total number of requests per request, aligned with spaces and the concurrency count. + --model Qwen3-8B \ + --url http://127.0.0.1:xxxx/v1/chat/completions \ + --api openai \ + --dataset random \ + --max-tokens 1024 \ + --min-tokens 1024 \ + --prefix-length 0 \ + --min-prompt-length 1024 \ + --max-prompt-length 1024 \ + --tokenizer-path /xxxx/xxxx/Qwen3-8B\ + --extra-args '{"ignore_eos": true}' +``` + +##### 2.3Results Analysis + +The following figure shows the results. You can view other data from a single test through the logs. For the specific meaning of the parameters, please refer to the parameter interpretation in the vLLM benchmark test. + +```bash +Performance Test Summary Report + +Basic Information: ++-------------------+------------------------+ +| Model | Qwen3-8B | +| Total Generated | 30,720.0 tokens | +| Total Test Time | 199.79 seconds | +| Avg Output Rate | 153.76 tokens/sec | ++-------------------+------------------------+ + +Detailed Performance Metrics ++-------+------+------------+------------+-----------+-----------+-----------+-----------+-----------+---------------+ +| Conc. | RPS | Avg Lat.(s)| P99 Lat.(s)| Gen. Toks/s| Avg TTFT(s)| P99 TTFT(s)| Avg TPOT(s)| P99 TPOT(s)| Success Rate | ++-------+------+------------+------------+-----------+-----------+-----------+-----------+-----------+---------------+ +| 1 | 0.07 | 16.191 | 16.475 | 70.40 | 0.080 | 0.085 | 0.016 | 0.016 | 100.0% | +| 10 | 0.53 | 18.927 | 19.461 | 540.87 | 0.503 | 0.562 | 0.018 | 0.019 | 100.0% | ++-------+------+------------+------------+-----------+-----------+-----------+-----------+-----------+---------------+ + +Best Performance Configuration +Highest RPS: Concurrency 10 (0.53 req/sec) +Lowest Latency: Concurrency 1 (16.191 seconds) + +Performance Recommendations: +* The system seems not to have reached its performance bottleneck, try higher concurrency +``` diff --git a/docs/source/developer_guide/performance/performance_benchmark/index.md b/docs/source/developer_guide/performance/performance_benchmark/index.md new file mode 100644 index 0000000..92dbb1d --- /dev/null +++ b/docs/source/developer_guide/performance/performance_benchmark/index.md @@ -0,0 +1,11 @@ +# Performance_benchmark + +This document details the performance testing methods for vllm-kunlun and the analysis of the results to ultimately optimize performance. The main considerations are server throughput and operator performance. + +:::{toctree} +:caption: Performance +:maxdepth: 1 +benchmark_server +benchmark_kernel +profiling +::: \ No newline at end of file diff --git a/docs/source/developer_guide/performance/performance_benchmark/profiling.md b/docs/source/developer_guide/performance/performance_benchmark/profiling.md new file mode 100644 index 0000000..81b705d --- /dev/null +++ b/docs/source/developer_guide/performance/performance_benchmark/profiling.md @@ -0,0 +1,418 @@ +## Profiling + + + +### 🔧 Action Plan（Three Phases） +#### Phase 1️⃣: Multi-Device Log Redirection Configuration +##### Background +By default, kernel logs from all 8 XPU devices are interleaved and emitted to [stdout], resulting in: +- It becomes impossible to distinguish which log originates from which device. +- Timestamps become interleaved, making it difficult to analyze the temporal relationships. +- Single-device bottlenecks are masked by global aggregation. + +##### Solution +During model initialization, create separate log files for each device. +##### Code Explanation (embedded in qwen2.py) +```python +import os # ← Ensure this is imported at the top of the file +from vllm.distributed import get_tensor_model_parallel_rank # ← Import function to get the tensor model parallel rank + +class Qwen2Model(nn.Module): + + def __init__(self, + *, + vllm_config: VllmConfig, + prefix: str = "", + decoder_layer_type: type[nn.Module] = Qwen2DecoderLayer): + super().__init__() + + # ========== [Expert Solution] Kunlun XPU Multi-Device Log Redirection ========== + try: + # Step 1: Get the current XPU device's rank (0~7) + rank = get_tensor_model_parallel_rank() + + # Step 2: Create log directory (works with your get_kernel_time_ex.py) + log_dir = "./xpu_logs" + os.makedirs(log_dir, exist_ok=True) + + # Step 3: Generate a separate log file for each device + log_file = os.path.join(log_dir, f"rank_{rank}.log") + + # Step 4: Core operation – redirect file descriptors + # os.O_TRUNC: Clear previous logs on each run to avoid mixing outputs + fd = os.open(log_file, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o664) + os.dup2(fd, 1) # Redirect stdout → rank_X.log + os.dup2(fd, 2) # Redirect stderr → rank_X.log + os.close(fd) # Close original file descriptor; redirection persists + + # Optional: print a confirmation message (will go into rank_X.log) + print(f"[Qwen2Model Init] Rank {rank} log redirected to {log_file}") + + except Exception as e: + # Fallback mechanism: failure to redirect logs does not affect model loading + print(f"[WARNING] Failed to redirect log for rank: {e}", flush=True) + # ========== End of log redirection code ========== + +``` +##### ⚠️ Common Issues +**Q1**:Why not use Python's `logging` module? +**A**:The XPU runtime kernel logs are emitted from the C++ layer and cannot be captured by Python’s `logging` module. Redirection via low-level file descriptors is required. +**Q1**:Will logs be lost if the model fails to load?? +**A**:The `try-except` block ensures that if log redirection fails, it falls back to the default behavior without affecting model startup. + +#### Phase 2️⃣: Profiling Environment Activation +##### 🚀 vLLM Launch +```bash +unset XPU_DUMMY_EVENT +export XPU_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 +export XPU_USE_MOE_SORTED_THRES=1 +export XFT_USE_FAST_SWIGLU=1 +export XMLIR_CUDNN_ENABLED=1 +export XPU_USE_DEFAULT_CTX=1 +export XMLIR_FORCE_USE_XPU_GRAPH=1 +export XPU_USE_FAST_SWIGLU=1 +export VLLM_HOST_IP=$(hostname -i) +echo "VLLM_HOST_IP: $VLLM_HOST_IP" + +export XMLIR_ENABLE_MOCK_TORCH_COMPILE=false + +export XPUAPI_DEBUG=0x1 # Enable kernel performance logging +export XPURT_DISPATCH_MODE=PROFILING # Activate profiling mode + +USE_ORI_ROPE=1 VLLM_USE_V1=1 python -m vllm.entrypoints.openai.api_server \ +      --host 0.0.0.0 \ +      --port 8000 \ +      --model /models/Qwen2.5-72B-Instruct \ +      --gpu-memory-utilization 0.9 \ +      --trust-remote-code \ +      --max-model-len 32768 \ +      --tensor-parallel-size 8 \ +      --dtype float16 \ +      --max_num_seqs 512 \ +      --max_num_batched_tokens 32768 \ +      --max-seq-len-to-capture 32768 \ +      --block-size 128 \ +      --no-enable-prefix-caching \ +      --no-enable-chunked-prefill \ +      --distributed-executor-backend mp \ +      --served-model-name Qwen2.5-72B-Instruct \ +      --compilation-config '{"splitting_ops": ["vllm.unified_attention_with_output_kunlun", + "vllm.unified_attention", "vllm.unified_attention_with_output", + "vllm.mamba_mixer2"]}' 2>&1 | tee output_p800.log + +``` + + +##### 🚀 Client Load Testing +```bash +#!/bin/bash + +# Define test combinations array (concurrency x input length x output length) +TEST_COMBINATIONS=( + "8x1024x1024" # Medium-low concurrency +) + +# Create result directory +RESULT_DIR="bench_$(date +%Y%m%d_%H%M)" +mkdir -p $RESULT_DIR + +# Summary results file +SUMMARY_FILE="$RESULT_DIR/summary_results.csv" +echo "num_prompts,input_len,output_len,throughput,latency_mean,latency_p50,latency_p90,latency_p99" >$SUMMARY_FILE + +# Progress counter +TOTAL_TESTS=${#TEST_COMBINATIONS[@]} +CURRENT_TEST=0 + +# Loop through different test combinations +for COMBINATION in "${TEST_COMBINATIONS[@]}"; do + # Parse combination parameters + NUM_PROMPTS=$(echo $COMBINATION | cut -d'x' -f1) + INPUT_LEN=$(echo $COMBINATION | cut -d'x' -f2) + OUTPUT_LEN=$(echo $COMBINATION | cut -d'x' -f3) + + # Update progress + CURRENT_TEST=$((CURRENT_TEST + 1)) + + echo "==========================================================" + echo "Test progress: $CURRENT_TEST/$TOTAL_TESTS ($(printf "%.1f" $(echo "$CURRENT_TEST/$TOTAL_TESTS*100" | bc -l))%)" + echo "Current test configuration: concurrency=$NUM_PROMPTS, input length=$INPUT_LEN, output length=$OUTPUT_LEN" + echo "==========================================================" + + OUTPUT_FILE="$RESULT_DIR/p800_${NUM_PROMPTS}_${INPUT_LEN}_${OUTPUT_LEN}.log" + + # Run benchmark + python3 -m vllm.entrypoints.cli.main bench serve \ + --host 127.0.0.1 \ + --port 8000 \ + --backend vllm \ + --model Qwen2.5-72B-Instruct \ + --dataset-name random \ + --num-prompts $NUM_PROMPTS \ + --random-input-len $INPUT_LEN \ + --random-output-len $OUTPUT_LEN \ + --tokenizer /ssd1/models/Qwen2.5-72B-Instruct \ + --ignore-eos 2>&1 | tee $OUTPUT_FILE + + # Wait 15 seconds to let the service recover + echo "Waiting 15 seconds before the next round..." + sleep 15 + + # Extract key performance metrics from output and append to summary file + THROUGHPUT=$(grep "Throughput" $OUTPUT_FILE | awk '{print $2}') + LATENCY_MEAN=$(grep "Mean latency" $OUTPUT_FILE | awk '{print $3}') + LATENCY_P50=$(grep "p50 latency" $OUTPUT_FILE | awk '{print $3}') + LATENCY_P90=$(grep "p90 latency" $OUTPUT_FILE | awk '{print $3}') + LATENCY_P99=$(grep "p99 latency" $OUTPUT_FILE | awk '{print $3}') + + echo "$NUM_PROMPTS,$INPUT_LEN,$OUTPUT_LEN,$THROUGHPUT,$LATENCY_MEAN,$LATENCY_P50,$LATENCY_P90,$LATENCY_P99" >>$SUMMARY_FILE +done + +# Output summary report +echo "==========================================================" +echo "Benchmark completed! Results saved in: $RESULT_DIR" +echo "==========================================================" + + +``` + +#### Phase 3️⃣: Log Analysis and Bottleneck Identification +```lua +xpu_logs/ +├─ rank_0.log +├─ rank_1.log +├─ rank_2.log +├─ rank_3.log +├─ rank_4.log +├─ rank_5.log +├─ rank_6.log +└─ rank_7.log + +``` +##### 🔍 Script Workflow (op_log.py) +**Input**:Raw Kernel Logs (Sample Format) +``` +[XPURT_PROF] void xblas_xpu3::fc_cdnn_infer 123456 ns +[XPURT_PROF] void kl3_all_reduce 987654 ns +``` +**Processing logic** +:::::{tab-set} +::::{tab-item} op_log.py + + +```python +""" +A better version of 'get_op_time.py', get more level dump and support kl3. +  +Usage: python3 get_kernel_time_ex.py --help +""" +  +import os +import sys +import re +  +unit_factors = [0.9, 1.3, 1.45] # kunlun1, kunlun2, kunlun3 +patterns = ["\[XPURT_PROF\] (\S+)\s+\S+\s+(\S+) ns", "\[XPURT_PROF\] (\S+)\s+(\S+)\s+\S+ ns"] +tab_space_num = int(4) +  +def get_total_time(res): +    total_time = 0.0 +    for i in res.values(): +        total_time += i +    return  total_time +  +def print_info_op(res, cnt, unit, op): +    total_time = get_total_time(res) +    total_cnt = 0 +    # print detailed op time +    lis=sorted(res.items(), key=lambda d:d[1], reverse=True) +    if sys.version_info.major == 2: +        import commands +        for i in range(len(lis)): +            (status, cmd_output) = commands.getstatusoutput("c++filt {}".format(lis[i][0])) +            if status == 0: +                formt_type = (cmd_output.split('('))[0] +            total_cnt += cnt[lis[i][0]] +    elif sys.version_info.major == 3: +        import subprocess +        for i in range(len(lis)): +            (status, cmd_output) = subprocess.getstatusoutput("c++filt {}".format(lis[i][0])) +            if status == 0: +                formt_type = (cmd_output.split('('))[0] +            total_cnt += cnt[lis[i][0]] +    print(f"{op} {total_time / unit} {total_cnt}") +  +def print_info_kernel(res, cnt, unit): +    total_time = get_total_time(res) +    total_cnt = 0 +    print("Total time(ms) is {}".format(total_time / unit)) +    # print detailed op time +    lis=sorted(res.items(), key=lambda d:d[1], reverse=True) +    if sys.version_info.major == 2: +        print("{:<90}{:<10}{:<15}{:<15}".format("Op type", "count", "time(ms)", "%")) +        import commands +        for i in range(len(lis)): +            (status, cmd_output) = commands.getstatusoutput("c++filt {}".format(lis[i][0])) +            if status == 0: +                formt_type = (cmd_output.split('('))[0] +            print("{:<90}{:<10}{:<15}{:<15.5}".format(formt_type, cnt[lis[i][0]], lis[i][1] / unit, \ +                lis[i][1] / total_time * 100)) +            total_cnt += cnt[lis[i][0]] +    elif sys.version_info.major == 3: +        print("{:<90}{:<10}{:<20}{:<20}".format("Op type", "count", "time(ms)", "%")) +        import subprocess +        for i in range(len(lis)): +            (status, cmd_output) = subprocess.getstatusoutput("c++filt {}".format(lis[i][0])) +            if status == 0: +                formt_type = (cmd_output.split('('))[0] +            print("{:<150}{:<10}{:<25}{:<20.5}".format(formt_type, cnt[lis[i][0]], lis[i][1] / unit, \ +                lis[i][1] / total_time * 100)) +            total_cnt += cnt[lis[i][0]] +  +    print("Total count is {}".format(total_cnt)) +  +def count_head_spaces(s: str) -> int: +    +    count = 0 +    for char in s: +        if char == ' ': +            count += 1 +        else: +            break +    return count +  +def process_line(lines, pattern1, unit_factor, dump_level): +    """ process a line in a file with profiling info +  +    Args: +        unit_factor: A factor differentiated by KUNLUN1 and KUNLUN2 +  +    """ +    res = {} +    cnt = {} +    op = "init_op" +    unit = unit_factor * 1000 * 1000 # ns -> ms +    wait_next_one = False +    for i in range(len(lines)): +        cur_line = lines[i] +        if "gtest_" in cur_line: +            cur_level = count_head_spaces(cur_line) / tab_space_num +            if cur_level == dump_level: +                wait_next_one = False +                print_info_op(res, cnt, unit, op) +                # clear buf +                res = {} +                cnt = {} +                op = cur_line.lstrip().rstrip() +            elif cur_level < dump_level: +                wait_next_one = True +                # skip record kernel time untime next one +                continue +        if wait_next_one: +            # skip record kernel time +            continue +        match = re.match(pattern1, lines[i]) +        if match: +            op_type = match.group(1) +            op_time = match.group(2) +            if op_type in res: +                res[op_type] += float(op_time) +                cnt[op_type] += 1 +            else: +                res[op_type] = float(op_time) +                cnt[op_type] = 1 +  +    # get left total time +    if dump_level == -1: +        print_info_kernel(res, cnt, unit) +    else: +        print_info_op(res, cnt, unit, op) +    return res +  +def process_file(file_name, pattern2, unit_factor, dump_level = -1): +    """ Process a file line by line +  +    Iteratively process each line in the target file. +  +    """ +  +    with open(file_name, "r") as f: +        lines = f.readlines() +        f1_res_list = process_line(lines, pattern2, unit_factor, dump_level) +  +if __name__ == '__main__': +    import argparse +  + +    parser = argparse.ArgumentParser() +  + +    group = parser.add_mutually_exclusive_group() +    group.add_argument('-xpu1', action='store_true', help='指定为 xpu1') +    group.add_argument('-xpu2', action='store_true', help='指定为 xpu2') +    group.add_argument('-xpu3', action='store_true', help='指定为 xpu3') +    parser.add_argument('--level', type=int, default=-1, help='指定 dump 缩进级别（默认为 -1）') + +    parser.add_argument('filename', help='要处理的文件名') +  + +    args = parser.parse_args() +  + +    filename = args.filename +    xpu_version = 0 +    if args.xpu2: +        xpu_version = 1 +    if args.xpu3: +        xpu_version = 2 +    dump_level = args.level +    print(f'Filename: {filename}') +    print(f'-xpu option: {xpu_version}') +    print(f'--level option: {dump_level}') +  +    unit_factor = unit_factors[xpu_version] +    pattern_idx = 0 +    if xpu_version > 0: +        pattern_idx = 1 +    process_file(filename, patterns[pattern_idx], unit_factor, dump_level) +  +``` + +:::: + +::::{tab-item} op_log.sh + + + +```bash + +for i in {0..7}; do +    python op_log.py -xpu3 xpu_logs/rank_${i}.log > analysis_rank${i}.log +    echo "Rank ${i} 分析完成" +done + + +for i in {0..7}; do +    echo "=== Rank $i ==="  +    head -n 6 analysis_rank${i}.log | tail -n 5 +done +``` +:::: +::::: +##### 📈 Output Example (analysis_rank0.log) +``` +Filename: xpu_logs/rank_0.log +-xpu option: 2 +--level option: -1 +Total time(ms) is 53742.29571862069 +Op type                                                                                   count     time(ms)            %                    +void xblas_xpu3::fc_cdnn_infer                                                     661569    22736.262780689656       42.306               +void kl3_all_reduce                                                                                                                          176134    14782.525712413793       27.506               +void kl3_all_reduce_butterfly                                                                                                                164864    4197.28395862069         7.81            +``` +##### 🚨 Troubleshooting Guide +|Symptom|Cause|Solution| +|-|-|-| +|`xpu_logs` directory is empty|XPUAPI_DEBUG not enabled|Verify that the environment variable is correctly set| +All 8 log files have identical content|Multi-process backend not activated|Ensure `--distributed-executor-backend` mp is specified| +|Throughput drops >15%|Profiling overhead too high|Enable profiling only during analysis; disable in production| \ No newline at end of file diff --git a/docs/source/faqs.md b/docs/source/faqs.md new file mode 100644 index 0000000..a6169b7 --- /dev/null +++ b/docs/source/faqs.md @@ -0,0 +1,39 @@ +# FAQs + +## Version Specific FAQs + +- [[v0.10.1.1] FAQ & Feedback] + +## General FAQs + +### 1. What devices are currently supported? + +Currently, **ONLY** Kunlun3 series(P800) series are supported + +Below series are NOT supported yet: + +- Kunlun4 series(M100 and M300) +- Kunlun2 series(R200) +- Kunlun1 series + +We will support the kunlun4 M100 platform in early 2026. + +### 2. How to get our docker containers? + +**base**:`docker pull wjie520/vllm_kunlun:v0.0.1`. + + +### 3. How vllm-kunlun work with vLLM? + +vllm-kunlun is a hardware plugin for vLLM. Basically, the version of vllm-kunlun is the same as the version of vllm. For example, if you use vllm 0.10.1.1, you should use vllm-kunlun 0.10.1.1 as well. For main branch, we will make sure `vllm-kunlun` and `vllm` are compatible by each commit. + + +### 4. How to handle the out-of-memory issue? + +OOM errors typically occur when the model exceeds the memory capacity of a single XPU. For general guidance, you can refer to [vLLM OOM troubleshooting documentation](https://docs.vllm.ai/en/latest/getting_started/troubleshooting.html#out-of-memory). + +In scenarios where XPUs have limited high bandwidth memory (HBM) capacity, dynamic memory allocation/deallocation during inference can exacerbate memory fragmentation, leading to OOM. To address this: + +- **Limit `--max-model-len`**: It can save the HBM usage for kv cache initialization step. + +- **Adjust `--gpu-memory-utilization`**: If unspecified, the default value is `0.9`. You can decrease this value to reserve more memory to reduce fragmentation risks. See details in: [vLLM - Inference and Serving - Engine Arguments](https://docs.vllm.ai/en/latest/serving/engine_args.html#vllm.engine.arg_utils-_engine_args_parser-cacheconfig). diff --git a/docs/source/index.md b/docs/source/index.md new file mode 100644 index 0000000..6818df8 --- /dev/null +++ b/docs/source/index.md @@ -0,0 +1,69 @@ +# Welcome to vLLM Kunlun Plugin + +:::{figure} ./logos/vllm-kunlun-logo-text-light.png +:align: center +:alt: vLLM +:class: no-scaled-link +:width: 70% +::: + +:::{raw} html + +

+vLLM Kunlun Plugin + +

+ +

+ +Star +Watch +Fork +

+::: + +vLLM Kunlun (vllm-kunlun) is a community-maintained hardware plugin designed to seamlessly run vLLM on the Kunlun XPU. It is the recommended approach for integrating the Kunlun backend within the vLLM community, adhering to the principles outlined in the [[RFC]: Hardware pluggable](https://github.com/vllm-project/vllm/issues/11162). This plugin provides a hardware-pluggable interface that decouples the integration of the Kunlun XPU with vLLM. + +By utilizing the vLLM Kunlun plugin, popular open-source models, including Transformer-like, Mixture-of-Expert, Embedding, and Multi-modal LLMs, can run effortlessly on the Kunlun XPU. + +## Documentation + +% How to start using vLLM on Kunlun XPU? +:::{toctree} +:caption: Getting Started +:maxdepth: 1 +quick_start +installation +tutorials/index.md +faqs +::: + +% What does vLLM Kunlun Plugin support? +:::{toctree} +:caption: User Guide +:maxdepth: 1 +user_guide/support_matrix/index +user_guide/configuration/index +user_guide/feature_guide/index +user_guide/release_notes +::: + +% How to contribute to the vLLM Kunlun project +:::{toctree} +:caption: Developer Guide +:maxdepth: 1 +developer_guide/contribution/index +developer_guide/feature_guide/index +developer_guide/evaluation/index +developer_guide/performance/index +::: + +% How to involve vLLM Kunlun +:::{toctree} +:caption: Community +:maxdepth: 1 +community/governance +community/contributors +community/versioning_policy +community/user_stories/index +::: diff --git a/docs/source/installation.md b/docs/source/installation.md new file mode 100644 index 0000000..0bcde7d --- /dev/null +++ b/docs/source/installation.md @@ -0,0 +1,129 @@ +# Installation + +This document describes how to install vllm-kunlun manually. + +## Requirements + +- **OS**: Ubuntu 22.04 +- **Software**: + - Python >=3.10 + - PyTorch ≥ 2.5.1 + - vLLM (same version as vllm-kunlun) + +## Setup environment using container +We provide a clean, minimal base image for your use`wjie520/vllm_kunlun:v0.0.1`.You can pull it using the `docker pull` command. +### Container startup script + +:::::{tab-set} +:sync-group: install + +::::{tab-item} start_docker.sh +:selected: +:sync: pip +```{code-block} bash + :substitutions: +#!/bin/bash +XPU_NUM=8 +DOCKER_DEVICE_CONFIG="" +if [ $XPU_NUM -gt 0 ]; then + for idx in $(seq 0 $((XPU_NUM-1))); do + DOCKER_DEVICE_CONFIG="${DOCKER_DEVICE_CONFIG} --device=/dev/xpu${idx}:/dev/xpu${idx}" + done + DOCKER_DEVICE_CONFIG="${DOCKER_DEVICE_CONFIG} --device=/dev/xpuctrl:/dev/xpuctrl" +fi +export build_image="wjie520/vllm_kunlun:v0.0.1" +docker run -itd ${DOCKER_DEVICE_CONFIG} \ + --net=host \ + --cap-add=SYS_PTRACE --security-opt seccomp=unconfined \ + --tmpfs /dev/shm:rw,nosuid,nodev,exec,size=32g \ + --cap-add=SYS_PTRACE \ + -v /home/users/vllm-kunlun:/home/vllm-kunlun \ + -v /usr/local/bin/xpu-smi:/usr/local/bin/xpu-smi \ + --name "$1" \ + -w /workspace \ + "$build_image" /bin/bash +``` +:::: +::::: +## Install vLLM-kunlun +### Install vLLM 0.10.1.1 +``` +conda activate python310_torch25_cuda + +pip install vllm==0.10.1.1 --no-build-isolation --no-deps +``` +### Build and Install +Navigate to the vllm-kunlun directory and build the package: +``` +git clone https://github.com/baidu/vLLM-Kunlun # TODO: replace with Github Url to install vllm-kunlun + +cd vllm-kunlun + +pip install -r requirements.txt + +python setup.py build + +python setup.py install + +``` +### Replace eval_frame.py +Copy the eval_frame.py patch: +``` +cp vllm_kunlun/patches/eval_frame.py /root/miniconda/envs/python310_torch25_cuda/lib/python3.10/site-packages/torch/_dynamo/eval_frame.py +``` +## Update xpytorch +``` +wget https://klx-sdk-release-public.su.bcebos.com/kunlun2aiak_output/0830/xpytorch-cp310-torch251-ubuntu2004-x64.run + +bash xpytorch-cp310-torch251-ubuntu2004-x64.run +``` + +## Install custom ops +``` +pip install \ +https://xtorch_ops + +pip install \ +https://xspeedgate_ops-0.0.0-cp310-cp310-linux_x86_64.whl +``` + +## Quick Start + +### Set up the environment + +``` +chmod +x /workspace/vllm-kunlun/setup_env.sh && source /workspace/vllm-kunlun/setup_env.sh +``` + +### Run the server +:::::{tab-set} +:sync-group: install + +::::{tab-item} start_service.sh +:selected: +:sync: pip +```{code-block} bash + :substitutions: +python -m vllm.entrypoints.openai.api_server \ + --host 0.0.0.0 \ + --port 8356 \ + --model /models/Qwen3-8B\ + --gpu-memory-utilization 0.9 \ + --trust-remote-code \ + --max-model-len 32768 \ + --tensor-parallel-size 1 \ + --dtype float16 \ + --max_num_seqs 128 \ + --max_num_batched_tokens 32768 \ + --max-seq-len-to-capture 32768 \ + --block-size 128 \ + --no-enable-prefix-caching \ + --no-enable-chunked-prefill \ + --distributed-executor-backend mp \ + --served-model-name Qwen3-8B \ + --compilation-config '{"splitting_ops": ["vllm.unified_attention_with_output_kunlun", + "vllm.unified_attention", "vllm.unified_attention_with_output", + "vllm.mamba_mixer2"]}' \ +``` +:::: +::::: \ No newline at end of file diff --git a/docs/source/logos/vllm-kunlun-logo-text-dark.png b/docs/source/logos/vllm-kunlun-logo-text-dark.png new file mode 100644 index 0000000000000000000000000000000000000000..bd8bedd6bf6e0e6516a1dabca552142c4afcb6d5 GIT binary patch literal 178337 zcmeEv2Ut_d`u7O|gS3c%paMZaQF?Cz0TmEbilV59QIRGfU3!8dAVp#W6%>%Jh_TQ_ zB=jP(AXPd6lnx0c1k%2PuDg5h?)~4pyI;BA^KkU!OeQmPX6C%@_r5a+=4U1WqG{;w z>;M2qjsVgC0I&n>kj(%TOhLeZ0Aw4$_ALznmm%AKNne5N{<#bbSVjo|KY<zvZtf;J}t)!%_q$;MUsjZ};t)ve2_7(uZz;;*_m6erlvHrL>WZP}X_k0Ko z6iEG1MoCdI74mDJQdxfViEA|#`XkM%p0VNQGGN1N$8YYer2g&BYG5vtz%uCMxzFXrVXaovK7R| zz7_Ft(G@>=#6oPpyO+I~y1b&iqPQN1n3$N3*F^{IqX&$Bt`2_F7618;etv%Pek$_r zUXBV%T3T8PipmPg%5q={Iq!hWKIi@AE_+LSZ{$Zi2kgD=yqrCJoZT;rt+jjpg1fJe zuDJNxjea?P?x&l_FE{$bq5V;%i*~=%@$mI>{noXMb_({c_HOoa9((oRX59lB(5@N!C&Lr!v3x_1D_&URLhzu6qBDety%|&o%z3 zu_NvmogD)Hxhp>??R)j#l=)5Vi+0*R&OWa8-=5hoQ~B-T8MylVWA56(*Jj_7l)|Jd&L+<&~Cj>13p?0c!-P{MaI`_>{zXL=mqFn`dj9tU%nWw*)x{V0p0=0+xm z4FL$qtSmz3Jv?03IN9y8kJnM-ePSn1ofdg&WAGOmcq-a?c=`M+KrH-xJl4{| zU_QSW*eNg#$_9Yn@q0RBEq&p8y7*fetE2nDG9SUTn4Q!4i(q;hOv_&QO}^uA(r&K4 zYxi9{)_TunfBE<^@VX}Ggn8PKZcPB-*}ipx=xF7-mSz-V%DFFcTQUKrus-03f@G*e*+X43XD-r-2 z4*>uI>`M~ZFZU||ARPe!`40iWE*Aje%fYs~001)%06O4QbZvnEC9)7;BN770AA5^F9hg|f&h}K5WqPf0>B#}fNLKFs2qd<8%7|2=~oC4Hvs`!ry;-=A_O?S00EN8 z5MX!(0_>tgfJ>kzc>!es<~Oha>KrUUAU6w8wuuE$6=DH~#94q3Dl9;%DGR`NmIXNA z#sV~=S%4D{S%9!)766{X0(51w0NZodwBp*qoB<*%oZNlfy`9`W#8l)J0nL3TM_AX` z2ms84A*TW8_MOJUU~v|JMVLhoU|*Alpa4)6WN9(R>j2BQ1IoezWraeatZb~{!3JYnJ7DavZwLF&2gkRA zbM4^%<$!?o!HOHe|J)m48-L0Fj}y#(P}R0D%K=_?R&}mbC`24!;e|kXA8pvI2 zkZ;GgA%GvC24`j40At_C!MO=wfk2@wtWY+v*WfQ9YucTamu>S7C4&uomgiyOp8U#J zZ$Dv|*!QYJ;OHl!q{;=aYa2PX2nuc8wsV)%?mg0~YU&!ATH5;$7#bNLG%-D9b^L@i z$R2i}1a@?CzU1xWd&SQ`ATaEDctqrlsOUR)@5LwFfABCdEj=SMD?8`uv)6C(3kr*h z-j0VF`DrA6-s*3b1hSf_!%L<3M} z%Q_FO)6QR_oprhUUH3qlpVoP3orl(W=x@*fl-at@Lw{rb)@AWJ4uCS-)_G{1ht~Db z-=F~~vwfY1{>J>R>!Ec$1Z8%t^UyjEt?Qw`K?6``=Q-Zw z*F%4U2G-+4>vD8mm;OB(SdS0=z4=?GzxDZBj}NWWz`7jxdo-{fANqUqw@!cS^S2%! zTBm_u{H*|M?ui)}uu0o~du1satK{2ier3a8o7L5voCX zLAGz>S}Q-OCj?}idv+uJDMO!!W|cTlkVOePRNSc+b7OisPuw{>`9#m{+ORw3ivpB3 zsv)DLXmINiX4*}x(TyOd+kAJUr0#UI@m$u~u)>mr2x8rA^z@Nrzrf zmo_I2>+MKSI6qf9P)~QHxRCZ&FaZHo-eCl6fc~uiOr5IRd=pnp#VI~Y5bvbSCyD;B z;%mm)*_UOA2m+>>32X@#Ct0B-iG4{g`OkK%r0kmKWdgFJA`sv)Huebn8COi%~u_AzA}$_Qddbk zZmt~p`f29zHvE%GRV&NciarCqfyd8I)2gYzN}qK}{O{&sJ&LwYBS2cBGn-uKv58ZG zWtn#A&I^|k<10^}?x}5URJ|@OF4iEKtO(Ga9Wh&NJMt=#u}YAWo4K8t;LL7yz#u`^ z46uNKoQD@>0?rCOdyi7yJZ&Vna~~x78?{=1n{$(-OoW9m2oI%=+-?Afdd*iU3R|mB zeu+%YVX2Mv0I#DC?3f0%mV-VIr^e7*)b227{3_hgr zAQ3iOPj$VR{cMLcUBreD@&}s}@7D18CsnrE>vINNtiGYJV=KKkE;RNID3QjphJpv~ zLR;Ux5ia76t2)FPeeIs)^G^|??@a{+PMbBkqE~4$luQC9MpP)Wx@>!j?&jG~*tr=g z=+|gNax*NQi?=55V$e&!O$QejXKK(aG&#zX;gsviyJM=$gzicMHjTS3>?jTQrWT1n zw(dl1-QRqqoY6^>rDPMZzqUE~DQTH}_v(V7IMp%iQ=~*RV-E>Y^~jiKShoNX1eZ&3 zms(X>zPx2PF(4jkZ*7{JjoIJsl#{+DenZ~xwJKN%An#Z=w*Dhy6ZYQ}0?kVK@ECP> zka&yzdA)Eii$i(!tkeME7obO3{@{mDop)Sx(TUpcGVVTnNULVSyo> z^Iv8}96>PwXf_iV9%lmKS4Rd{x2j=iJSj{dCKWzO{ZpkEAK6LGXP!9mQ!9_=NGJZ@ z4iaZwm8AcDYatQb7}sF2(f5OmtNXzKXhR@BBzCqA#3l^bw1> zh*m1RG$$<@*!?F@yG`uPfcMOVu>@XV9y?Q&gPK6SUXESjmjCLqHwW{Y_I9Q}?Y73` zi5tDA4od8+p-p zmzV%W1jz&hjcKxQP;9j{65s?X3-T$dVUf(fuTkrZw$H49UI;&wUy zzFi{5w7}<3Zq%;h?wYZo)B2j4^)skfG5S1zHsYA2R15QN<6qtPB!Zol7h4%B0h3%{ zV6^@8@R=Ctp5;OxsX!9lhkk-zI}UM>Kz~G_jDqVYz%!bl@Io zVi7lnAfKkE>6j4PM)abdQj$);FKsE1Vti~LdG|KKX{4w`I?b1>dPBsKo3~Wnc+86q1#GqW_U7@_rGlAfTug6}8R15*ZQ51gdq-UF5M& z^9H<{3ojnH7F`c~xYHhW&btTy@V)skw%QhDS<-QtcH~+9vbTw9rh=o@z?jsfx$?8+ zS3vfv`cv%n=Vt29m0nzRc$AGCi)NTJ0Z=Y1p2Kdw#7H60k>=D{JemodYcjUzz&oVC zQIkc-&Yo%T&51}HHcBws=qsKhTYWbifa@#iBkVpI@&lItt>+Qd2g?(tbe77*=5}0O zey^7M?3Y79K|ap*3Ht;#t8R6;-~m8sacM4%xj{0{3i4l0iiIDlpRKSwZY+7XoCPn5 zcgFQxni@Xd&e%Xy{g^Cx&0Kzxpy_7NmLsABNMPrdg*jFkTtj`1ymp;}Tx+&UhV=^B z_-e{nyH|PZUK1l`(&=f^n^?_YPg1K-$F440NA@=D}`Z+;5>po{1~xvb@^WiwMQdUKS>l@*KGq2V>iA=sP)h1r2%dRp3=Y zf8d-o{iHf%!EUP%jZKc*T9OUd~qzr)F-HIoTN=c<+*Q{^P{Y^d*Wl=};vI z&~%jGQNbE|fjj%t5raKuuqOVPnPj+;kVN!i{qnh4vD*_9z0ou_iVbOh4U(N^E*Mp+ zw95)8N;q7-KY+b?w$F$4;=N#dxUL42YD0W8d_SBa>wf{UnQ|$3@ldXj`?ofQzqKhG zCqv1t#&VN%v`|z0*6Q1AkY5;6^sQ7Jr{{5<4=Jy5GmleNMbixpMW;oo`DP}1GI5RA z*9Pl`%)d2cVE;{flyUVKigY%ul?hNEAeTi@IDr)fCctKKO(HW`z2ux!B38MJB*OpJoEr zT$n&Z=^4!Dx)3Ii=9U9puA^kbtdrA5XZpcdVFy;*x%))!v2EngwXCx zF@br}cKp<=JDt7rhzmH@-+Tr5Ee(G;6+T*o*D--J&}c>s@&Yo}lV(Ga$d;oGAMvho zRSnrMHFj%X4B0v9?cj!TlgY^r+Jtov=4DDPFA+cDmUa8tnE-7>EQ_;7c#^AUp9<#5 z#~@FY02_R`yO0l~U28lc?+AKwF-9rbK(8jY{@a^&4}xzAl7Jbs7dk9P85XLpUA zXS8;G*=16%<5Ry3z`F!K(+o(3c_fM2p24`Ky1{@KXB)4KP@^dug_?s60`+vW4Hxpp>(tY;6srtmr_tn!9(K}_QtEHdCW!!Cnn0QJXY(F?FVaR&l zF5Ci!GJ_FN;OrJC%qTL{6GjM3po&^#yJ3UV66fbAgxf_ip%Dp0$ytNxvCkZYC zP_d1KRpu%b$lpeF))U|Kt;U^-B**K?-*`$jN4Lq5xW7^`LpC>)MZ(DgXLpEAaz(}; z#xw%{V{?deGQ&^ujZ%58T#YNmtUeX=;tCnG;K4NWheFz-8DoTGq4Lp_0#_@H8ZJ}I zrfE=>kGUhcB})VNTg7#{<7v^W0_f~ykd_rel@_{o@@-X{z;Kl9pRUUbR=Cs30~E zn@6B`Qyk!Y{uf3*Mjw`K(PQmgfFwj4ifr}L+RCM9AZ>s%6rczY?+@2SF?jqFk(>NI zuP3y=HV(;tClE0&Z`Ou=C)Ti=^5M}ETRh%{v5j_^h_5l{wDHfaED}HIu-zg22A|DG zSTlc2_x*VZHj|^=7+%UvLZ?&&Nv}2KedPTr`N4k2S(;o8|AI97Lsq=C8N+EYP+Ik= z%!11FpA}TUmfw??+zw32EF2@pf@GW&yqlhgmYzgy3_e9b@lFQDB?&xl_Rh}3bw9A( zcUQhmcDzp}mQ>@9qUkjbA?WM_icFx)6}ciEigX;u)xs$u_sfy1yrt)vK=5bG{BZ=4 zst}44>E4DXPTw*bKi&BHph&D?k!`m@{d4rb*UBymCBi(0A0QJv_T$gQCgzAi4r8NYXI|#~{&$piqt7OjGGBMGVI@G6Akj ze%iJl$!Yafl7%lUqut*gi^Uq8f&Hz13Lyz(6u<<=jYyI71#nCaR+y^kG6)FoZjQjI zU#pnFog63`9ZfsO1R}eGp-kY(_FBYpQzR4EX~_iOuPHDluumfvIhO=yn5M$JS7k_H z41_%q&baO_gc(#v(Bh5%`@3kZ3Zh#%sq&l0EDGAr5shPR&t}C5TqxAsu#ewSP%1XL zG8B1nJZp!nvXSP*#7rIk>hI9`p68x%o(JoMmPev#tMn55Wd&}=hh!c%!Tkw?$0#@R zY;I>vU>A&?^5*HUi7e{--dk?$m+`EO5QENkFVQ#DIuw+(=gECJ?4Wg-SHufa!S<$w zaU4$Dc~*c4q&7t|*o>J#jWsc4L}3dhrLJrjo|F>1B=gwrROc?2+;_c&gR-q!qj%e& zvpE(5`sDx@j~z=W@;!!PS}MZ?43gk94lEOhPG@Y!aez7Yj)mYDk8Bp8iTK0n^kgZ{!~Gg{dTbq1 zpM2ko+ZX4T7P{fK{Ixv6c7&zw0)p?bX`G)gqfei=a%; zJU>=jj4JFNy>re%Of^7`GdSC8<_bUu(w^LmlH_}YdE^6)ONl?C&&F^0`%~$G!m;QO zrQ)#`G(Y67FK+w<-*y+eQD@INXn?ob?XtF(83!0l`1n6hCGM*rgHt5^#Y5_;Zrf)g z)MJ&~+z5+(@(Oazj40d(Sq^})1VTw^B96Yn6-ksvGNiHp^fYio=`68exn;LfgVeKq zI}746ojp~cg__wW!ZTYnkGnpCOq?dsTLOtni!-POByIBzOsy+LH)uPz3pOi73lWd8 zF;#@MhKl^g+-KWP-&c3|#+gfkiapgQ7eCToN#bcJWHT7@9z zp3y)mxYn>4c`-@H6O2)z6OeH6>5)YN+8dt07TEGDCa}>u95b#03Wlp7x09?emB$IF zRj$vNAu9~+IEGD#|C|MEIg24rOoS($ZY!U<>$eFi!~`ytus;^@lh|nmt?rS`PIM!aIB z>1=Y)SzPOPN8q}V|0j%m0ZIFNvxjG4Un+XMbm6XCBQ3+}8Rfdht{{igmC+yYLUxpH zqN5?nwz^Rs=Ne|#2@KK8y7m3)&MU{?3)0(b{i%SW0s^Wz%7=jslvf&6O{a%m+#j*$ zJs=WSzqHq*i8Bzc=4XT{y~0z8_qS`7(1Kp2U>;ep=Klt2{1$fnNE^{3@t0_?LAJX` zJHO33+y@i3?Eq+MYfoimKMS{G0-}n@S#fuB?Q>_K%fYuoWR{>O+NXVN)u*i6znqkv zK6mG9z{w#=OZl(vsVw;WDRD@Cg)$}DQ4+j*d!e3LwwK`}$BYw7aC!Ff8R|<;_?(L9 z*HE1#t=hnF=+qoTv@#mGS~V|(xb@wF{J~Xl^E}M{6ry?airO3xrfVRZ31r^sTBw^? z!i{?~Hm)#%UGE9atJHQTu=x5kKt1{=apMnjjKjL9FhS}0{kz62Z`@AAS6{MW4R07p zojBDrEXR62>F68FP_N6I08LPW;I;8XIr^B&bRx(SPa_eL-42?xa#}cgM_57BS&kVA zksA{$7Op{R8~}o7Xos3Eg6#Y5eXIP~ ztKw~i$!Rvu+qSa2QIzo7V}|>Pr|fcgn>UrrO1f7v@WeMq=X61dQP(k@ z8o>`z9A_SJkp)(rf3H6NNTvL#yvQExC2I)$(zR2prnaA@A$Ls5M`#SI*Loi}duTPA z(n{!0kzTe?LxKfS`**tR{~Ga7J{k0J@=uR7#|Nj$GO+STppTgZPp!SV+#zIMC-aP z->LVQ9o~t%)5_vyoC2O0@*n#ho)boxDNzEgbAqKP1;f~|vfbF~vmzMvffEOD&n3aA zUd-K>BeV6&rh3ZRx(4>{3aOs;Be7`F_^JnAJ_n<>`RknFf7);=-BnLKlOsA$(4i2C zDcedrL@WZaMX!H$Wf99khwb*^FLxC^G;ZdP)>dX5IQR5BSP>+q40lEA!PmeFC4kl& za7w)Fk|= zt)BJ1*Hx_cVQ-QA12<;|!_S%nLWn~A%oBn_w?_F{FUf6nH<#}D`^fPQa+X4plHO-1 z+Zf(D$+eYdlK^W8MiR+e8ZrWp7FFrc!HhKRvpsgE^Q(r2-_e_y-H{M0ASb$*{TWMm zEg*b+E;@z3G=Ox!7WAm`>PcMknk?zeVc2~f8$G}}9W6N_Y>AEM^?-HlDxH!Hh3hNo zh6Aj3+gsO!|57kO)h7cWQlsg<#4%zL;C){$5Ny(vTi&tckeR9UV0j`Aj1E(?sES`K9C+~ zEYKgUMu{9T57$#_qXiME$5IYtI8jaA$QsM_^ov3jhirQmEQ%g94 zF~Pb9E==1_s!bws5(YAN-OqDYj=B)9@~TV1=V)rVe+EM4$q+W!dg(27E2Db=Mt@$) zQB;_vD^+{?;=#nP5vZ?mX2y@*-#wHY?J*sRr{>jCI`2w!I${J>F9# z`d~(1R0v;yoi4hY5{m{zlv4O6!WOCK{Gj3H0`GQY#7Kd5G)-C+&oIFaqpaFy^Fz!Q zOx}HHviFoLjTt8?oj^QzwpA}5LVpK3m`|`Yp)as8U~I+c2_b4oXQP;G-9j0+q?z@6=AZ!7U2g zH*blUJ;iG~GwP8M6$hyc_NyDU|Gr`pHi#r19u)FFRE{Z_3okVvx#@03oW;7?t^~E^GI;)R?ZmOU1OSB^8>co3aeN6XFB@Mv2A2AqyCJjtOv|r^odFqKevpKVo>WG9_xp4^f5M<{wmZGdsYD=yW>sMLHR?*YEyUdZLNWd+PSf)0cZZ zTe^q|CZM5K#b6I3+;t<)4P#=XnLxeysWzYTe*CblyJR&yyO#|>4?mU4snNRyX#X5J z__Hevew6=jda2$Y;is-CHncg&G{=ymY{WSRUd^1op)(Y@y{;dX$qz5Vn|@m%3kgDG zQ>^D4X?!Tgk>L?W?cO;$pBw602)2zTv;2m!5eJ&RBoG+ei7Yds3RB@N+J6C*>h1rd z$Ovu%LGF#yLfYd+Wut{SZH$1}p}d*5$Cg-GbY)hGXPtf+QqUy?VI!>_W2peWnwn{ojC(A0XtnSBzu{D7zvH6jrH8peIl5?rBi<{`ULN7Vn9ym?x8G zCEJe~e;Nu(dVAD5PKZIhk?we|m?8dbANGvJ23vkFo>rwn*rYL`E&I~98D^PH!USbr z+R~GiBxBRx-6&8~T3>To>xr+Av=RG`dz*|8TQ5P0@Zs>tTI6*Gf^ya2SW~;)J=B4e z`Px-v0zs*FIW(RLT#RO%@t=BY6?c7=YCVad@mY^-A4kh~lySSL2;_IQPimXK?Rhn5 zgPThed7~;J!6?2up^qrYx3X09t!DzdOdvroUhbC=&~JzTEnTMxeW~Nv)|z7R@l)E? zozl;qnK)>2z_g@Z3Z=rEQIy@;wF(piG`|6nYWm3d8Ku?enm+tv;u%1#LUVkl+pD;b znUMwPO?S%;>d&f7&7q9Y9Vvo?Bgon&zv{v_?yasFjE}AQKe0Vd7qAAcpr#V{b3<2f zk8&X2`0446=u=+~%C(oi*q0_fD_b3!$}?KHL3e5pSH^u%$dE#og#uH80s^(x&*2jeSy)ze5u+P zRA>s1BPBc_vmv*N?W1~akJbU$@@Qypv}SqTVGEcPRf*BmneygJSf10B#Oadpj@jW7 zY*=&#=4?cuasS+Kg2k5B6{}k>-2pmvcYYmPf$+YVed0}4Y%sU=Q+UF z^v|G<{G)058`s*u4eI@2S*6JR6Q){IH8OdjXX9C)OW3ZIpEb|gb5XNSSG()$@{9@9 zB0><)rMLae+&R&~lN~#>co*UB5D^rNr|N3Cz+$jM<>x-Ch%nDa_jeNl{Y>EY$e31> z@H?KV6N7294(Rrj0Oh3@DiCQGwbP0YPTpCeoN%X7XrNmU>zV?1-5Lmh%dqCs1*A?NA z?hPi!wN_&}vUy(?d9;K*u~n_b-oo$TU`PrUb1hy=M%68{+Ukkr@wDmAy1C^lWE6S} z|b2u9)L99!9{CN$xU| z)}5P`HVqgyeD++vxVBpKSaRy?q8Yy`R8+Dc*r1EMfd7fnLtuj~uUK3HLcs=uet^bd z6mPJ0HE}?q=h*Vgd5V26^*Oyayf`h{-n`qY_1VN=lPK2ED1Il}Atdw%)BUH{qoNTB zw-!~wU^S+~8Z$~>iy)odCdEif#r>pBUkkCm0F8wiFpgBJYEIWMKt1wVTSx?1)dKxc zX<%T#hlu$F+Fj!Tggv`rIT$A38oYZb5EjDMV4*t*3){Mu@&RcTPyp7>&D;iU#$8XS z$AeF_tIlywS)b@SE7P$n!bx>vH16Q&t>iKUjB609e4xLYCeVi=l`v+( z6_Nb;5Sj$0GKZ25Dh4zj?BP2sL&KORjg3YwN=<TlxzZ)?)m_$Vb{q-jf~oB3(tS6p^?x!}j*l9!3>$Gll>e;VhvkIq>Z$Cn1um|&Rf zS~{j7ns$f@6vG)Qs*?a?A5Y(%QZC}2HM?Mtm`%H>twG21V#Yh3|KrzG_}sapt-rCl z_S{VFe`nD>f}meo=_1F|Gtr{xj|lErw3z?=P{R&xp8J{A<+hjn{6EyNPw#z~%xe^K zV}lfEeIBDA$j2#;$eX2yPb?pQ%gqy&mXWsirs)(m-fUwWYrS$~5l1lx=j%O;ePtTK z&Xh0`6{G-?!mwGYyDw-y+|g|NQuLKOy0G)jPR)t<95y^BcAVx}}QgbirfU&V)YGb8wZu<{`7XKu<(EU?P#P=c zi0{(@wu*%PKeeB^&3|s@ruo*L56fZzo_oMK4q|hU;E-U|Tn~~^0E(R(Te=HM*rsti zILUe1BqQ;+MGhbW9YOfsi`Ut#NRSFwvc2UkwoZ9ag$cv)Kj_jtRIQ2EHC5M;ARF4dc!0q)*W0MKn1Zg)Ll%A~?>O=ahj zd%GjXj+8Ps66$Fp$^va-Lo*U9Q2Gk9x|Yoq%W;dn2Jn~TFFkkRPAJD>^L5mgQg7n^x)R4~Y~<+}=aPZDyC^B!2?;8p z#3lp>>0r9c9&#-^SDE9Xl6( zoGLb7djN6L(lX0fB=RsQiuB@lez(~=%a7Kj@=rR`!O0J$=alYc0_-U>Qlvvw73#+J zBixrYt%c|frSWMaZH+R~#As>}gTt2aiGc5Q96ploT)4eqab}-N>}9G$iknOdb@eAC z2gUT;3W^{8?|*mQjL%h|NWvrVFY$Jbw&o6ll!K+lxtH`uwqJLU5oTO@K$BT9tIN*B zI3S4Tcy25Qx)#roU!J5x=l0dHL(KCh9%{kxsyjiaqyGYg_-*fy|4lDr{#V$`0#k9! z5*x`770Gt*RWt0HAN~$BB6(Xmma%(X=K^<>BlH9S7 z86rXz2Q7w%U?gHm5DOa0p0pgO9l4UCI=G=s?NUj|P5j*_F4re}%xXBdRe98X426`> z@8o(QBIsTa{07y6nHeLVqghfE)Zb^=M5)ce0unP{v~Mtx+xK-|VAgx{%2ne5r*^wc zjPA(91ckR2$pM@Oi5EVpM6upaK=Ms8I7#aFhi5jRBrHF@NGmHqR~{77gdr+DuOXwn z-gA4mdug4ywl^%+?)K_cY$kv?f#K5e%LqO~V=f3);b9$uYIBb`cC$&B)yNT}$ zy!ZtfzXLOrh}a6JnU%pNmjPF~=T7giR+5~>f)nw#tuTclE=&C?ae;B%fZo3dL9yys z*QOLn)$Sms58b3czRoyNFr3Axd-b{ZV)PzCjbmJ1>igJP?RXkc7d7h3B@nC=uvJ|C?J@{&%2s4xs={Li9$=IA?G`-gj`M&ro-U4Ii9_B&}8j8DXm!mJy zxzu;5=f#&Q+4~$^9w|=g<#mqzQ>X_B;!#g*kLYLks-P@p(YFaeV}2N;|9>Ds${2%# zJM&Ut-Sk+Hea5N2vwb4R%+Cwto6F2tYwfk*yNa^&|1ENCyImMRvotZWvdRI_T0nEm zVZ3A@Y6!ErRy|S~jS86t27SR-$T4*EZ(6a`O7Z}@uH9=2rR0PMRl31x-pUfc9 z;>a&gSx3X>ty_q7Tq7!dt{$2~lglCE@7{c9S_s^FMWpKmUC`_Pj3w2a9)Gz^h^AgD zPGL+PZ%@_Q#B-BXR=eJKKw>tQgZB9g{~pM`(R0EhouP_Kaigz(^6K{rp*wr_;NMb^ zVK2|z07DYDPq{(2CdddV`>ZA7(H(mZEyJiJYkvqmZ+0hv46mGM~9Pc?}IpH z$STz+447%R+SrC#9^6>VUc2W~Vqqu<`^0`|AV|gH2qH2wFtQR8b9Ogn@9+(pA)#x* z?aRgJ)k-7r?e%Z>^5{*6fy?Qoz1z2(m|ukn$7$9PCeCQZ*+$k8vcqaBOdrK3s#gt5 zTjV_Zbj8JPnMz2h#&ZT+5#JX@2g`U!Mw~A^X#)bYd%>WslCCgn#BnEM+oC1vDya0) za{e(x;3}qL)!St93t!J1*;iinc{Y`GX<9_tR9#uw*hr1)JES1uA2ci#kr#Z7ko&~j z(I=HuRonlmi*m}G)tZF3Q6@61=`PS@K2X(2xbHrTN4C%h*D#*r&ls;|W;`qGr#&4j zX?msqFiro?T@v{uGeegfz(}Tm4(S!$-^TCa{zdq14N?a-S18=n)1ar6?f4vQLGhC- z)&BZepjuJ4on=_8$Wm3|o(>|u821)Qn8g=rlju(g3vtL%UW9usUa`a{Efi^vAn9f zZ)>Im;b%a-W1tHKW9+<4;3k5&jOj%%t_8vv91=`m8?9=scVNH&Lho9TLoWsk)s1}R zfU6T3wc2RC_?1W#eMAp(T9@QkixEZIGl31kCn#ZZr9I>#`craI!F8KTblp?ip1MLs z52P>0*g$C&g98kW?vf!XFouw;!EK=1?AjRsdEPxDWf|PV*7o&@F->4Qkh=(>!FsLIqqwf1SGuN!Nyd%{u0^ib_r92ox%15#d7$)yST`!)n+nOaytuBzY*H$XX%3;AQ^NXUiJ$kZw>W z>N#BCg*dR_ama2I`836yDif@OtvJhZ^|CK#eALqWOI9pX$4d08yqCH+Z#mB|HWSO& z(fG~*-GF@USe4RkOqirtSH9ZGco+1g0q63%dh?jYTYdc{C7Zgp*A=hrdn%Q1#bLE9 zlnc_D@O?izL{a7|I9V}d2O@0d5b>v}ay``>HvOs2J|Sl{tIkZ^_ZLdgyI%X5<>b6T zS4+74s!=q+5Ju|-vr?j{nzVgo6nBz%)pSuIUbmp#Cb4z3BZ{}r<}K96J#74g*K>%F zsLZsH4D~quIno{nFgj4&4n*td;kf4Gg>d!Ayypfld>UU1#*d%fsxIBcyTevrN86~8 zb`7M9og_{&h7z6=tVxN_0+HQhg0au#$dH1LcYI&^Y#OeX2|~YUa_LP0<)M(#d!SSh zn8HIDYBYU%3Pap^a5F;~Rgj#SeQW1+TIX_MM%d1R>HyJ;%8&(~bjZEj-RDQG!0^)> zsvM(zkOy2mgNb}aX|C|1Co)Lq%bf5tb?tejdH`Vzf;lhmyA!=wQxKfq z#EPLGfbMu{Q&TWn3UVUoFRaij(?Nr(JRe4r#fPg09 zs_gq;t}fODt1$tdDEe#U_*MewrV=)02+(ao$I}NXnzj_V9+XdxptqRliJ(~ialv{- zjX7l(ZpM)4M_{}m$<&^9xK$z_2D~kGu{Dv3ztH#ewh)kf(z^UQq-7M7+(CbZ{=6y; zy18E8uX(Te;;Sfk=L)AV>)L{kwJJGj5=cbM=xNg*VCTZAK7xgiwO8ZKD zdTrG#$@TtaSgTDoYyht@4o0^u>m-T_fDi$yHo7rI0Dno&Uq;W0CyF@FJ%vGN5yt?6L2Jq_cEUESahMq^>1iV8;%$5VPG7*~@KAnjFS2&j&J&?T9Q zr)^ds*IAgf=^)E>xX(O=*V<=F9dl+q_*l`Sx(>8}Y6+Bkkx1Su^Yo6Uu z69}>Jyv3u%pa0Dn_QUl43s=bh(?1S#>(-vo3-O)hCWdO>t&b1?ViDuzkwZE7`5pdJ zjLK!ysu(65o!<4OZcSevK(I1_&u2m2Fd2uEPWsfXjB@OrS;6f1zRybGU+p@}C7Qol z1=}HGDnM=?9gs&(+ktV1PwMTERoEZKPX6cjKEWChzsK(`t0we9p5a?jPf?qx5 zj%q6M{$=4F0<-~$hp;P%Qa5l@Zw!i#K;=)NVi3Q}`}4ca)1r4nS9XGqgi+n6n6AFT zRo>@`o&Wgt+qMCjagfFM7-r_6AftzmfVS4QG7lU`tryK#w2+__BY(s5mjFh>=y4z3 zlGzhN(nbcN!}vs0&zqr-DXeJaNyheq8(C3kO#Y%nZ#Y$?N@Mm-o1&%@_I20*K69F7 zgK=gsNbhV=iVO%sw-YvzvSQr6x}0TSP}pSj)>(ColYFmSc;);Fd(j(>G%E;&*HCO$ zmk0I|IEOiX8rZ`QySB$Em1mg{iCw~kr)=^;+z8x*$#=T{)TxM`RyGM#XH`Yq?+pPhAy#0Us}xmqn(^} zng9cZ<%cX3%~+BGk=O7Wd~pIcA6`u^D?Q$8EW+sZ%8(vayp7BYa_WwR^oaVdc_!U~c#IP?1c( zK&0u4%=WUwj(&)-RnZPQ7o!phLlRB|^A~g!XkKwaD=1BN1>J+i_1~xS@b$Sxl{Vok zTVK`yA!%2Q`-Miyb+{#51%|T6nD$7wh#r%GSk?ZD49u>9hN@o7 z=eOs>6Td|%QqX}>_Oc3Fi z=?hGt*JO$We-{FqPlZpRs=(z=kT(2OCxQ-G*Q5|FQ7h`bwXkz9u;i&dUBP=G8I2afcY&4nC$xVQU{sgx zSBvSB+xOx|3bn6k!Gv(N2y6+Xq@Vc}&A@Vl$A&3~x#A=YZ0D-reo0-9k4m^CcUp6dK$u z2yJe+H^1GYsU^H^Vc`sp^8vyBdaZ%!<`YFrZ>F1wpA(A%zb|=4>usZ167ScT^B4~! zIr>*_SjpAAI|sgZLo3dzp{H@CLP*+ls4L?|Kr^@uo`@T3F4senz=-!I!(n&WKtW#h zNxlPh%YsT{bgPC>_sDf0;fJlK2FvGq_E(~WPMD*t4 z-se)fsu7$@Z@C7f*dG!5SL?bOKhldL{A(k$r6YFCx6=?$7BdA{l#pYqzWg*Yt^r>B z`A)RKpp0*BXQKz- zG}p}za_fMy@!Iy;$Hkj0VAxM*--DYd7f4h_RCX+eXaJgowV(ov2%ylx&GK!vNQxL9 z3}>rCMpdnGm?|9_8r&%FJc6KYLV;VK5|M;tG!pYkm4s!mD}c6m6A-{HFf~YQMk4Uz ziTft?7)d zVZjb2;Jifw`2`kUQ6PERh=yQV_yss+eDg@w>cL4=!y8M==Itzs@FFssW1 zvYYl{DbgCW2e@iHH53enESTkC!6-}Q(i{h)CWU7a&gWc|Xq4$fFB&^FSWfQZ4M=RF zMV0LlIOQiH!xeY^1oCe?O{}+)V0C4D6x7g+b=`ng`}tNM-Oc?kaPnD5X`{#S#`4n# ztqfV;YyzZMLRHG7X-A333M|WH3U8VCCC>?MQLJE+YE$E0pMw|7ULV@C!V8nFhM?+E zGm%sjg-U)!x!yDIdVZozu(iLfYISqNgXcq<)vxWMnD3;n$dR;~za5@!0%+x0Tahx}e2TddGF%X;iiUY~TlCDwx1^Q60+INdI0! zUDZvs0$n>xS2pU)3U=y3^DZS}ULIR)iXD+Ngms~Z&23CW^qP~5ZhvY%fv57pY# zTd7nQ&M%EgF=%i^4j7dUM@V|D>brJ&cl1(~0x;h{+MygS>Rf70<9i z&(W^W{J?`c)6@U-l7})AIu3sk4`FnGC>c?;jpi}I5No?*;=}E4v@Jw?DAY7LX(qFW z^%LJUK!WF4<5Gd@vqBBPvz`V9Ur$o1z+t7Z&_bRDYf!LRg-OqyvDdT|`}Scv`t&If z!tH<)Q|*c;*o+@olh0Ef(DkAl>CekV)T44Fx4FltW&{KyU$|~Kbjp9%R;9+C#!yMK zK4n_D^}H0xn&5b|jEgdLjN(UvAHHkSx%gE52y{L+@?L^qt>{)kHX&b;fK>S2H8G0| zp+CYqx!*{6V_j_nUf?fo+|k)dzy2KEkkjM%wW!zvJ;B(5P*ztTQ5&*)TW1O|`az$I zdyfKP5cFLPG!G-HruyqACE8V2wxYYU`olLIl;7kdlM;W=&9_QHM0d&iC(P<#IA$~I zjmC@&i4R;)aMOKc4$}N}U$1zF&%=k|$(c@YrNgj(nc@edP#;(Sdf7ZWn$e6$pb?Bu zgFHTSHwNJfn_XP)9WTwlg1GeZ9fwY%%XQWnpZbdN5Bv8>8>GU$(;nc@w5>tmZ;AsN zPpXS>3w9Zrajz(PZdcUhGxc8i#PL2$1#zcSui}3az-}GOL`!0-ur%crB(ZRguGx|{ zB7z?wI!;B?3E+!sQsE$ZQar~ZFyncX2+EEaFyOfxbQg&FJ2IjNo`B*=2*09?_gCCq zo(ivoQ#_X=j)Cs7@VA&cx;|*CjeekqQeebajE%uNh|QIe9T{E55v~cfdsi-OQhoJg z?}qZG(CB?v9{~UkwYFz?%MbCM?%jfZ-2C$C9`FT7H!)i=L$^;`*J7f}WGGLc(JV_} zz9s49dgl~84^;KuS@K~}(x9m)fxXzmLlYXkfc&Qp%PfaMTfsfCG{oF7D0dw(0U|A73E|0i!72H$KOdla&GJRs|!Rq1tUHpx5syp+YhRYw~`*x7kYU_MV0VINh3p7%^?fK2?Pi9<=%zgo$&k8GUKYHVcrebSK{(v^0;;Jt#P-tM*mO#zAd>5*#bc*L8=`_i+^ zD~Q*aH!0OPK}uaYXy9)`i=EkX|9HD(b4%$y*8J}t4UpUTvg;L z(s3ZLuYFH?H5T^vU3p8(MAn-~O1nNN1VJQMCVrI@j>5t)3$i{xAaB=cSX3gMeA#{L zk6<9lvI)X7#ZqAnZlj4SJSIactn>fxwY$%duGlTbc2L~tOUARFS%Tu+_rA4wx(nv> z2Vizsv4~mAA0fPSoBmh+3(^FHABpf4r2i1B<^X|)AUO6C4j})$>7O*_|4||=bVUyj zln6nlppH^XSQELXnETJ~6@CpQ^G<-WiZNc{K!Ql;1ff)_V(=JBi`9%X#bbI#-kUZW|{wth-G{5p#-^_NG{4fDKUc4}r2*?zn|qUx~a zDP~g3)Quin>)>G(%#$&>(Os)q3$vM7yb}de?BR{RDM6n1y4QHvnC_&+-lX*V{3*Dj zZO(rCjkagqd7^PNToZNMbT10u?oJiMH13-q+;=DLH4C8;!VXEBd)eU(5~W=<4Yjcx zNDVeELGxMpb;MH$?ay4I6>Rh<_u3E0J=`9}<%ddRS4q!uH?HbH+Wn62AK?{hZPqf3 z+_h+(F~dG2I1wWWLfPHdC?V@c`yEC(3Lb4*&JOv6Tr-|j?|El7h_n?90_wvLVWCLb zLMvUmMbFZDvYD4p2u2}M=~PO7o8-Z`uK`=%6M;9tmEFqfJ5@b0g&wji`~++6hn@`B z5WL0KtzK>n?ZV_o)RZC=8TOqOs$RR|HM)oBde=mQaW{)a;@acZl_n+ouVu#9Pg&3V|*`&%T}1G;iCHVM98K z9pl}cg?NvL{;>`RhL>R&vMF{;Imn%UPzni15$Fx!w0Wi^*@(3pJm+FC0Q%jWieSZ* z0%rfvxp>xa2gH|< z-?S(W&@k~-=cHwZNL?MFe>2Wv>nMG3#QudabEpa^`2zRAYpb$sks5~}xV)&X-pGhzs)otI+2 zc!sVw6s-2?Wi^YlM5n^UXWzBkmr508anUnEcC7J@hM+@S7oxw)#R5y?0KFx>@`bpX z7q`7Vc!ZCZcVf&=uzKJH48*n~Ty+(DvKU3`bJ6&e$9!fQS#BUhMAf`#mwL`h+)Rq2 zXv*9Lsr+))=Bs)3ZaPu{V(RVPg;HJLuNx_B-cjgo6H(|YKj3(s`00$=GdFNgH10By z60BLG^{#62$S&hlyb?t(2N>lHz}rig{47u57o8S6eb4Sf8Wt3opq%{g{uAUav}32d zyuU(9sL+|UoI$hXEN7L3s`$5qtsxlyaRFzCKgwGcJ1>Lk-ZPLaU+q{^x34d2OSwBX ze@1p)U|{>{4GOlZsfP@K*o%fjL}S@RHV$t1GhYFR0h;^9hj*QGh!YhLkG1u7<_`zqmpKXRn;AdW8#t*>4KRyfWORR%^+tBJDZ*ujuN#1F&5W4n=euniW$-b6) z5a2K?g)p8kq3i_;DuF4w=RLzcJKyq=?lz(DeUJpciM6+no$bOsQ^wP)*YM-|%c_mH z`%7&-yQg0=)DoW{gIn(m^6_)RBFwaJOTM7kKTuPjaNRvqhd5pkvr@Yxj~BSx$iXh_ zBbmZ?<6c+UtKBEnCBLMZdf$F$>g(+6tukjCtzBYZr7nd&5bQ@4_`u+%hz($028_`= zEDZ^Ev!}2UH&t|B-2y?v(F&cHcfvTnGm*Ni`@!x+JQtN-*N5PvsK52|J?L~LF5*k> zz=ZmhzO17J?ka^~9tx}mv;Xu9bhKC6uqwvh=`e4MLrLvv#6otgKwp&khU|ls<4tGY z-pjJO#*74NYA3cPVi^0= zA5Aw-9*8QdO9R9ZcY{h;Cp9Q$&R7qI-Ao&OhxBTGotc-15 zh9CG^W{e9=G>rdzh5qUNky*#T$>GI*gM#p#U>;otqV_1{eBH&Lns(l;re6gfcp8qZ zD+{KMzjd$=UGj)4i#1OxOHNLGa+fvIGK>aCeAfG}XX;3=bMO&Bo_ z?5%!|daqHvk8#yL#H<4%>DO-3;S`om+2W&l1Aa5>iHo%BzvZ zGmDlJvJ1J;v6N34!O9&o?h7Gu`1iaYZW+an49gC(94&FuSJ6{kP^rlukPCd!LqaM+ z#tMPUZ$EX9`M!=BC6)OZF?Oee8|IE4!mR1JgCe04Sac`fu^Jt~-Y;1~ zm2_~7ozev0OfakpgcR?bF#t4MEccYJ#&IsQD;b?K?zTnINguvxjcZ_TdOwE(=BUK5jx!~E&DA1`^w0EDtQI7cVAsB< z?KR_cS94Y6)yB;C)(J07s%{DHx@F93qN92X~=QlaQj!4G2?B#X2ej> z_9>J!s~eLqTZ0=+I~$=*jm|9K&wXWB=2mKxFRl4S*f%HP26<_+k(im$<%Kfs0@SB< ziaBd1iLU8yy5fjF=%k{ARhWdAo1El zkTJXYsX2a^$5i>@1ms;s<|AUDG4ExYv+NzlPzuZCmu$gxSkFuY4kT%9riaq9oh!)Te4|j0!-X!1|4el zo^VRq&CH(@@MukI)RzBfa<`1|PJ0vLXHKpmiy4GCPhjfW_|=vO6tl-pa($b9ADfuY z^i*T0(Ydqm6Rhv>gTeM>_y&rCYjBU~8mhxPUv4wMH&+^@(Vqr@eF5v7! zjDDTn=f?C@Hc%yh-oCzuA0kjCPQuBxqx%7j%>ENbRw2X&LYM$i{*Mw17wNb5HD;#z zz*nJwmPpfOCI$Q883t5#BJ3eT!hiA`V3X4K>QY{x6TEHt@ZRZf+ll66N<3V zbof-$cxkGrC?wfZcI&Y94iI5iU`RW@LC_vps$}tE48YwVJqtxH?SoUh0AU0mh5tyM zgfYPI{|l}o8UXe+N*K3xEUI9panWlGxbvyz%1r3zkqdkCAo=;0*AU(A0e0J>TLN~- z2w_SDpJ=Nv*kvVq6IUV{Qkb>wXm*)iRfBZxR08V93auxt(yl4Lt@7#oa6)omMyr z0D^yUj(^GL%z#sm;}$C^nH#UEkH(jGGOkv1e>mFZeYH@MH}8|B=69L>;!3=82%m9+ zomM(VmO$3rt3Al-K_BfN7%-(*Z_e^~uZj@5hjnNtT!3I1W5}=-L|yVFe#&%W2n=?SD0CV_bqd6fq$9qejPi-@0LOz zuZ5-|$Yze%AXtwsO>9X{6)Bq)1j7T<42!bX};*4+xHg zZK=9W&7offbp!8X-QKNt#|@m2i3>?V*^x>@8Mx;i*rjdltTwby@FivjODxzl##!Q<$)=J038 zvB8Bah$}+_K|z;DJE8bq!W;SbiR{lhXhj@P?yE|}+B~ne^uM6%U;I!HNpbE*@1;*uVbCsW#6=vJ3KtM-7~tCguZ{W?tiW_j2|ddOWm+&`$DV)T%4b@oG& zVal_U2LwYhje~;;+G(+ZzRI)CjT2vn=t>b3<>aKtdf#RVr;3l(gt(tfzmVpPUBsr}N!eX^U~rGQ+3F9%BZxfkk2&8viVk+ztsiNINfxitNX%s zzq(0y?moylrruYBdTj9n5_)WptF+i^0f`Tj`|*`~vCF2!ea+FuBfQKDc};2)g9HT- zgQO95bT`0Z@2jLpC+#nj)7~E(v6xElmJ!^OLM?ugz}!}ll6e>&QDD+;eFj2l62nu$ zbR$%=q!I*(xgkgBCxWd(>@Q8pyg(89{zh7Oy)hw&J^4+`VpwAJ4@in^zR{b$wNLCE zQ7W-5)hF%Fn8Gp(F-NhR+Kv9CwMlfU`I;7e591b+m$hr)>YSgjTlv^&eht?2_Z1EP z>E4@0V!;BJMO3Cw>ntGk$= zwBd5&z&_cX$U(0MCUeePPL!s{>o_>I?ivOEP*mPYWq2~NSkUImTcm_+f#>2`RS1M= zJ`chw!o(l>9c#!i+QmtOcP`)7eV|K&^^+E9Y<*wi4n@W_)*}+A-{tyUiJr@mkO0y3 zw?CoqA%Z>Xej-u;Ei;Q?XKT3M9g=w|0ZJ?9^el73+%MTW$jsm!KU1xINWD@SFkj7= z%o*JAL;~Z;DAzJt-1z|A2Y%r5RR6Ncv_^(SDQCN}*mH&FvsZ4i=7{WTZjHU5jh3W< zaxb5B+|d{XzRMSl-=%Uvc0;Ojyz8s^gi1(8sT?VPjqBk@>Uu&j>F8Zx1=K}Ojx7=v zt^+zzXP*Mw7X9yl%U}B%vizBdLM9ic1_|hc6#z0r1>#uGKxF9jUm$h<>g*)ljsg%jo{&m1_V z+*rpR8a*Rh8c_9F@Y*xm3+xg8kxQR3W)dJvDM%=;Gf*+wNk#a)Z18*3dPBR++o0*i zVNwOKDgFUj&sotNUz%H4u{ytEUs_gbtI)S!&OQL1eJ5aae(|nePzWa5sx50SUe7DQ zd)jg4`O-5Tfo!jy;){tBj$qG(5_==2Tc@0aMJuAwLs^TYo-Jehrw`%!?v0kIO`Y#b7EK{4k=M;B z2)h^KUVHTOo|k$VT49=*^K4;h_uga~N58pm66wQZzN_~qy-nePy7N71G|`AFSd*OF z_aN8^lJs?U5AW8k$a9;b(d8<_DV1lS(8}sG$U1NC9t{g#5sUk!c0!k|r0gxCRX%#Z z%V}6cZfBhg7NNjv+If=ME#VHIo_})eBf9F9cr_UwIIHZl5P0hbPK+I{?-B4-eV)cf zeMugz1%*X5S5nfIy0KZS{G={jnjnjIV}yM2TlNH>lZri^!u3*?*uOAVjerSlD< z(Gfm&HJ4^JLI~u?8{HJG@ZP=faTlWxq?Yr0b&uu*?T}jl{&B zC^K@VOR)FvRIFh0zKDwrEcB(ROBJ7PxFHdwFSWQ8(zM1)#67fzQ6Htt;r{k(V8=QgomN=gQey}%q}*zr!>T+ z9`h0S9*kQMT@TJaq{NLcU$@$cH}`DvBkT-<*Kg5p|2%5VSW zp#Q_B9^%*zub9zfL@X#kP7)@R7(Cn?4dkXqm>|{f0yQzjF@uCY)+_a%XsLJLYHD#s zg2-8m{#O#A&?3n;%-AHdsGZ3x3HW2%z*;qFL52;&SbEEVpCzz9%9Kf}hzv!12YGfZ zYY4e^84tjz-C#9{bFQL#fdxjNnWqLH=ku}hYKZeMH#EE8l|EN=I%iK7jNInF=hDR{ z)pL7`anA*JGo~_rnm#n4+ff1)vh<6=$2+Q|{PopN2iA*zc9i3AR(<&VZ3ja6n2`1! z#0s&muy0a{vG2B{9rKx*etx+!wrMaj)i(K^cboJ_b&hePp1%3E-)OYFg4j>5Zeeh% zB;(;$!9Nr}X&;H!wE7IVp-*(w*yj8vGL`Yu-K^b*z>&khAw=ga5^Id^^1phP0xOVp z&Y3rRZ{~VaL-~oam4|7YXYn0SgZu&U0!BDUx*n#+q>4RwS4k;(=6-ANIf{C@n!JUh z*UmM1fN^MNJElmxE_*Ic>aFb_;|XLO8VL}YN>*=`RB1eG-=Q_+4vWbx=S&hQ;N*pF z@5CPv2Tc*7^J3O1xMyDn8kL)bK-E$RTBvar?hdlMaL7FI10X%NEn_KES!5 zODSHoqa0F`d>x-b#Td!8eVE4{dLqero3)+(Zc~2cDE167uQrXJRavz^SOW0l+}TLD zy)0o;9%=A2c7GFpXqC%KD0{oAL{`A0h)n-X5eE+>aL4nZ{SDI!;fhzJn3n9z-a_dd4BEuB^E>$QSrcZ9J=?c#BAn?ru5aQeu&iBe0EBgB4H7tAPa3xAPR~ruvmV)c zAt%+!SWTB9%8lw->;Q)TXZj+3Qka1MGp7-3{55;9cT@0-&Kp2*7Ozdc0Ls7UI6&PX zp&G;zrzLq zc)5)%%Z#4*0nwJpV&4J>%MUnK{D73!1*0D0dgFp|HQGJ2Ya1t1$)fmh%vI^APc|Vx zAU>#<>znQjZWTHmxk04bdVWH?DPHw_g~4~Z+rZ;I&%*n4CyEPIlGMd}*r)&x-;v$L z>ZG5dUb<}D@SfrBDBwCPf-@p~Mv_I15rvFKzsVbsEHE&LAIgk2unV5q);pl|Q`TKF zffE9g&S>QFhUAteNF~&09qbNZQey%$f-vnlfKKMGIqQGqe-`oHBDRuGN*K-O2QO{! z+0FWxSc!pS$ojeZBou&a>vT$o6b5rYL59xJH4_5(pgkNnJm+>`ZanT5@@)8b?H9KF zofP)Jd4>JAJBR`j6zMGN%}Wi*&^hq}6AvMc(~&alSs9%g)jm9fCrY^9JU^Qet6fbx zzg9zm%M+Yr7k=!edU1-#LrhimdbP6R)&S+wMg=vQ!rsdC z85)YAlcDhIp~yWGkxWJAbG4J+*!yNmb{NdKm0?hac3r@}eXcthy_)dlpuTB~I)xR{ z)1Rjej_j+yM@@dX8>wc#bTG6+@xbHb_r|WCiKr_&(E9*OQ7q6a#g_Vt()(7yn1d>9 zT(jC!CQE%nXhD)K=-7@5)tc1_yNAx&vJEK|``trBT5bei(aUfRzMRYTLS9Dr$xjQ z_}`y|mO$4?w7(&Voc&B>Eg=X=Oae)k`$71I+$d)Dr4xhuFTH6X=)0Z&`5upKs}cVO z&%i(@9cj#|;r3oa27GXtWq==tGQ$9Dd)|U&UyjTD*JTx*1`=5r%nXyptc9Sv zbJ0?mKW_rj{-2n>|AF_g$wU<3iEMyb4CRHH)y#Lc{O9)$ztGdakfv?bR$-ZO$jQl{ zM3a7NB$F+RopwWLhxFlDaR;`ODiecw>rA*lg(EuXwex{iADgV9R!1*(h}qTK=VR^* z@3d3SO>z!7br#+#+eiqvh^Se9MBA)uCDXlgG7dZ`X~NlyElW>9OC0*`JMRkRw9Id0 zvUVGg9eqZjB-|kd2R&^_bNT%F80XX zEp6qk2oQ{6+1gz|M&6soE*yGkei}0%Ho-^jBb#edcqdzBK4qqMD|+Id4-Y}qx<|Pi zO-)!^LOG;=>)R_SV)f0N$*EP@yB^TnTW3`KY#Z~+0Bs9Lku~KD9|y8<%xO) zgiyFBJ>uwt4cwlBm(gXJQO_So=Y|_d48dePo4K_QAT_P#yN`wmggnAIi;Ql>yj7N%2eFF-_`07(?|uouBYum^No0BMPmy z2Slf)RB3YiB`Se}JWCDA2xq@scFsp4YU@B8dL=+-VgIq4>Xw`%ge+I6c6S9bZNNcA zqinFBR74)Ms7>7w!)Ic`!`8^N#}oA&7&aN*v(+Pi$fo3&J$kZiHJm|0++2{f;k+l6 zis`<_dl}OwwTbVcHp)Ue@){v86>S$HoY0tD{tnV7!!NE!rxnmR#tzd&6i zf{=GyptYP++%2{2)HR;{;l(alTb+GSl6Cw)0X>aKN291d6dU@G0fnPm*TCj=MNt9$ zRNJKN@{mCBc1(H9E8`vR1EU*0WWvJy8>d0p2kVkdU1@8QOuFmYhDu?Dfd?yTXLp3` zL!CgkAPXKWFCH0V`dzQ+c+)}1ml-R%I$ox9^qf@C=B*~-(M-klkbj+Fe$Mj$sUK6g zlcC185`Np$2Rls9D6g>+R+<*M>6(tXodtPm4(YPN<{^jZ(%Fa8%i>sN!5+XDoF$uz zH#+_~FCarkS&AnhQ(Ahcy<_6bDSkc`u1ddKk(dJP;t^Xi&JZC?CRw)gj%Yl8qb%3o zII+z!;tujI^0DweT(gkgr5>N?^RB3CCC^q^l&Io^+{s<4#nuR9%LAVNcDn#=54bun@*f9Ki1D+e|U;E zkc@#gdF@onORpVQMSha>*&q7-nS+dCM#TYTYdRukV^7PWDDOv`#Si*+G> z@RL{3>>u~pn~H5?kO_gvI?Mx>EHwuYl}V@@`D>AHiP&{qU|- zak{vF-MC4FCFSBBYEa!Bn_}&9cbV>wk*8XYu;^YSRL%$oS@uDd@GS6ysrB7=b)cr_ zaBDMlYSQh)OsOI@^P%ub;>z>Z9SMr21Rrt}WAq7PCPG$gjq zXCt34{lci@d4cZNO}QbCMEH>2IQD+|zi(s1c!2$)x+=p}D|5 z&82DxFQ4-$jaa=A=!r5I7#*1V<5*i1<@<#bEov@@u7yNCfsLWQ>dY+reW>oz%5#34YdOKoP9(Uo3?cqL1L1aEJEM&p9BZ^0mOnBUWW7{1sApPq!+1 zBi4>r^~^C+$f(6{6sYFa#bCR)1JrQD!LLP_yz5Nu-IU@^6Ub5sqJV6__?V1puslrY zNhhzj+UN(qx4vvsKJ6_FVK17wP76ezN~YS?2Uty6?wR84MXWFs8L+ZeDnY7YKr>Tv zVGWV{Ca)mZU8m1wqN>w9cX{#FA)Q>iYqmBEv)af9PI$r(h(*rv*&ytEhy8ui&(7mxz-o63R}mqfANK_vi)hSUCESDpuK%v z0Dp{D>Qg(0c}wygyf0t#b_Av8YcTviR{U~OEnoAFl{=sm=2^Vw7R`H2I61H{-X8@r z+G?!OyfM`s)UfueiHu8I*!ev+>hft7m@hqW?0dtFFnizKsquFk0>=aQb#M7HK+@W7 zv-3uXsGRqgS-#^p(eG{sCSng9JlGelavwvM(0>%kN;}2#7 zwD-M6KOhO9gXNz$U13Vkr-+ef9(aHD!SOit1|o0}4!ugvamlNH_)3dLy-SOkuE4PP z2^$uIKOnV}WGJxm<+pdI1I5|@@f-Eu`obg(c$C}^p+FxJ5msZ!DdEwT)wA%>(chj? z_-|Vo%@|3fgD#4?q>#9W4vWc@FHeyCs_{vrZ!%#tJbPC()Ru2$vw#k;?}f9AwF5ooL`c9j*h-?cipJOa8jL2eA9`2vTDcB;5L zgGY>h!U^{YQzN9W<2`>27wI#vH2q!heVI=Vo>8!<7RtB>n%a+sasgX6F#qk?g^-9h zVWyyWI@#0thX;r1%hsRvTb*mXvXJ`%#t33xX*;YO%Xpk{$h}U>cS}>dT8JJsL5dma z=~aVaT!JtXb`YUF4W1yjv#{$Bub5Z9i!6V98sWC=K>Rp#QHP=eD-{g z_WSx=hF3$~f;p&gB79(Vtm-AXeQYXmpkSV@%5kc{PpQ~|l|Me!BeV4?#t;Hpv%h<- z_34Ja@=WHG-roXfUV*r3WIOa@wuI8+keV@cl@6{}V^8rgL$I@_WF?Jqtp45`rRUz? zF2qdv7I!OB1yDtRmS$!Cm0C%j*JuR~nZMigyyFju8KT?!l7z~wEk5Kq66(>fTgpZ| z1*Y2cEXFQk7!z~_;d_UX6D2{k@J`$e<$ZP!l}|YMyxET-u90m*TUuq_P@ibK&zF=a7Y6b zyKPlP%zMid-2@le`>V}gUNbb@-F6EKe&@K)r7A?lKXIN#4oENyMh6qeb_|v^=1kzY z-bW1|5bHUtMGL+U`shq&ssXRdOW6-krUvdoU#0EvPGS}faSt$#+XxY|1I4ENvWwj0;;T2e_4~Ki zZAz!Sv?1JWAG?eC?y&hc+1M`x@;?t9{UQ_m==g6D+}Opg^m^wW5{bW37TGE4`mU?w ztWl<7WR)t}7da_s*NvR-W;g}|bB+qVvH57_yECq%?0(lSB>-FNU~P(yyT}EJr{1c& zQ~Ku+rEO@{ahUM*E9Bz`DhfAApB;v2ZdF=?9(`OkUb+UQg(ucJ=d5tnU2*L8886i!uMWADsM(s>-@| z0JX_PHdY;dgQdmE^`m5*iVEF5sS6hm+U(P2X(^9RHT z&}FjHSRyH47@j3FF0hU)39!(FQn(w447`ZTv_0U2K*9946M|C&SB+@Uh;cXt<}tdu z&TOXpvAA(NZKJKfFppniM7xlZ8ehD>*iG`2yWSX`mvsC)JA?~)RFW8WD%5!teb9OS zwZE@H$01YK^DeQT#V#Hg^=})3u<;lrl{S&P^(Ij_y8}_LiJk{{nY5CtMqUlkv1tv` zTHJ1D>t3@*J|TE$)&9&-wku}{r*?mpIK2)rah`?NB8#vMKOo%x$NSEcarYIqx|RxW zr38Jxapbn8_lc~1RVxRNkF#9&~Zwa7d2Bubz z(t1rB?o;)MhyU_LVT#y`6O3aCfODEu&Nh2eJp#0dq?*^Q>1Y6t_5g5s~yc}#LKuvZcC_jE66WZNgh39vN!Anxh|tOZr{!|W&8k~?QEavs8ctW z6(bL4s;9KdyY-naNDXqSlBPV(xy!T9p{T9DvOA#Lmf7=2t1}OkCA_bIE)(H$?W|){ z%6Gqb+*Bk>ib6Pj^}|tr@3nNEsvuU;7QDni=-$RPuN+1@*+NK#EJrfT2!-nUpjASM ze;>f+)9wu3?QvvmZOH^n;t0*c2$gZ=ce)RRaeg_bpT~ruHTySs_Wz05dEEvW74gp0 zyMVh5V5yF0gcnOQ^Z!H-{5Smh7fkvSO8=iF1b!Cl4V?E(Z+I>O_fJR-MK+>vlKuZI(CL z&q>f89XT;lzqp9{rUw}nKg2;eK_I5%b~G$Ke#>y{JY1h-t_B5BO~i4Kd_W)PgJ)Z_rofnAP&0dz7JN%Qa#F_@%7&m&1HSj z=7ifsj+L-*Gh`A9k0Kk$-rFJ4i&k>>9nuT$QZSf^O%*Rs4w3idkLy=nPlbJhLH$wM zWb+4Wgq^c;0V=-Hj(el?H&=*fUmzC+?XX_)BW1=A;w5vTJSuE;5ciAmx>sGRk7iE9 zF?iBghQdRvhE^C;$O}~&kwpW&P8r-FPWqp2q4pVBoEWaAH@-Z;n%5#fi^t?TL9cXA zSsI)kUa}V#cv7#m|5Ge96`xyy1SCu$RaT`9m<@JZuP86fKj-PE;7h^pn7Z+}-Q@{r z4(JVV5v7W~6GrU*B8>;o3LT18+nke42en1Mxn$%AnAk{`n%z1d!Uau7$6XN+7?N!M zoS!284Pk2Iw?l0*yKPff;yPpfj%^s&ZU@bT?o31LypS@2Uqy+$YSy5so z6RZ`&&B-Y>V?zw+zPou-AG?W#1p3+y_yqeN=hK(3n0j-@)4$B$!m!f)!5`=j;3bQQ zC;MHbbK#~&sBhMH=6`DcBYUD92lrK+z>ZJwQop^Q=P!P^Zi)ZHIZ-wwYi|D`H#gC* z*9h=3sGT5NlEQPR0qu6(N;g^T8XWX_#R+ zp(t35xKJEU4m6a{m5nq^HL(RXz#SJl>B#hrSjAUD^KdKioyH*<;BS!< z!Iv~d9!{wCubK3bd06EUdKPi?jv$cvi67(bB~`Ig8Y@>EMg)3g%P>v3QSDJ{Q^qv& z_x_j-GwSlVxQHCpfz(lj49fbh;dYQY*=n$*Wo>5Da7dFwt^d6~h}zv{n@M$3{qNeJ zC7fnywJX2<`rU0-8^R2-&EBZScOrMPs{0m5o)ZM_w~mc;R|%ndy4328J(xajs(J4S z8}lW4A45&f;H`tl2iGN-|<^fceHUA?~Cbp76rr7>9xz5IMSs9$Hb zQn}^v_9k~^kt;woeTelH$y;??N08?BY|uIhRB{}gZq#v&Zg30V0tlL&hiY6LDhZr2j3_qrnTv~x|c%qUK;$+-I#-Z%0N8{1*^Ey=z ziA~-q#x~t2nk43e$WfVE<6gl}I)u&WZKWR8_RSNbe7j3mO_Ptxo6braUs#*(pubO#NkQiL*qn8I)RLsSk=dH-oC0uZ}GG0&Dxrd?EhAw zG;Xl#i^X(2COv=s;IsU?s37G_G&5-O{+L_WeU+8*Md7v&o=it;4}B9Bcx|Gp%-X(o z8SZ$G6$oqY+<1O{$7GX1}K-mAq54Qlel_lVJXjry)l)CRa2WaJnKKIq ze4^r6OPB;SFY_IDREsj7P~au~je1v^lC>Iqs~2iJ)p>vE$3iQyDbc%q-4Rn1q6$42@6<7@K|RY>FK_%DR< zF~i5NUddxlp`NmiR&Lf|ip(1cT#I_iX6=@bJn#3oeggpv8YaF4=7A)}kOEl=u63PD z;#J@6Ez}(LU9Eq_d9$mLGKiP(!pr8<_;3s{_1UN>LYAE4!9iMU{kXM0w5R1c!ww%n z-dx0y#jIAkX0DcW9m8zdVH_oF=P2*v_FxF8<3FHc9IQ z{hmR4%^0u{pLq1EnY%xjt$ua&TLoSHDxqBjK%D)pox5vH81UkfQ2MGVk4Q;*3dsen z<;+`=g%OJ;ME3#nvG*g2lvC2vxp#zxU>Z8xz2?ksaIw{_5V40_#rdvOAM({uMHVzI z%8_R_tf`)0iWk1V6)!|4S?C)SEnhB&EQU)>zPWc#^eykjLz9b_7Gut%2BwIK_+rgtTzg0p?iYuPKcxYofB!@ z^MD^mU~v1f^k0sF2KPXIL#myt8pB5rh%X5%w?;V$^R=5lAchC8Yy!;GeN zqMgBWba^6#xmja$Kk+s)qUh9?-(b|mh(7zf;k)Cau1U%^Bd1<(Vc8%g0`V=1Z2XC$ zJH1Lo)<_!+)$}fqos-%=+4y8%`DjyYcjmOoqKMd|_QO0o7pe8F1;q6xBVCB07N#Xp ztoKPI#$#&+DdXZ|B1;b1{kx^k75gAY3No2FMvVtjdFvd z4BcKL+^zh&TA)>VD7wL=f5GFnPqkkY`JFM;hQ5bYU&Y6a5B_6y4tKj(4U>CR$lm;F z&DDd44w$)A%MZOq%5CYtm#^C$i3><Π5~x|yksgSK6OGxQcPF{!tisetxS^|L2p z9>EIGF-SSh&V2+2453^S0w89S&hdEi?naFMu2Awn1Fg3iG$=+Z9Cfl+= zfRIrsK|tkfxiRk(h;I;knQ`bp<_KJB8e@Bn>JO^m9)3_PEXgYgKhY(=D7E-)r|0I~ zA~CE5{SpiAMJBKw;T|H%r-y();l@TJ1Bh)RzB2$u|HghC#@GvJB)gGgn@CQeKYu~J z_XEQ2N&xI|cL#jEEH$JZ$PZ}8(T73lYsj6P_WSpx-8sj%Ny9{pp&~X>WJ^N$yjz+~cyS>hpJb$r#uRyZQ?UHZS$00n< z(>wpzH^Mhz(=19k3l9b(Tn9;fElwV9StgoSFB&RQtiG6@!_OaBg42aXLS2b=s_6cS zBkhblYndAn>UN^W8SCOH;RNGX@ysB^WZ|(VIvT7)Vv7; z*)bpi0M)T*zpKyBIXl8kjgAde&1vxoytNI@+9k>s$Mj`2VDbRa2Es3ZDimHVg!4I@ zTo8Mx7>n#}35|1iPca=lYS`Jhy82O2BGd<&Z?y25C_A7u)FYM=sN*3y-?u zjLXgj-P-cdD7`Poe&5t9(cYK#7Qz9kPX7m;fFo%w^OBK~6NCkZI`Pxs!!l;}8t2Yd zmycvk+&+EA>k+zfY^=(joD&R?j%fCg5*fVZm*|+qm$`#(w#=7rW;gn6W6N{vslxX* zOYvU@e_a^n3y|}PJ6iiS;e_TbkrUlD(|upshn19iKX9ndG&AH-U2`B3WDk|x0*GZ8 zoP_p!gphGQ0;NfZKX9J5-+}HvHh+U_YXx}@)zNZdJJA-nT8DxI{2ep@{}4&9R3#>i zlSU6!uxI5jAE*-RoR*vusd$t}Gk*?9O87qAzO^=!7qQodYdlN24h3hLHH%~dhs@1* z#B-u2$tWWWa>L^%3wZ)pVEA-!c>$Q;p1P#2=ldi-`WHf1kPOh0Tq!etw!Z}rW3%0$u6*b}nQctcIc;X2Ut#j*#;0c%@Vcu^XrDMrrmS$XvoSe?7b)gqK zbFedKg#f_$uX5J^%J1+Wo&PNmeVfuEN{n%x{JG1g;G1j5W2=i#e98N5`jE@^Yxwug z+i#XE3;O#a_fvgm4B0oU17c5~0T4Qvgs4XQl<%BrG*ZzCB5$V6l?8V$k*6KV%Kbfb zS*_V%omh{cxcEhur=D(0XSvo<9&c6Ml>SAkU>6&z!S&8;`)Is@6*ysSn}49d+T8Ktq<^$xU1q#3%WkC37woZ$M3 z**M1MGiP)@g9Ea7@^>26kwb5*w@2AyyBSfJYMKWANc^MEAhkD`NyLfn=lv-|(6f>&@Q>=J+GpBHuh4(1#cxX`#+HZg zK)?Knv?1g`J?YFuv=MP5Tig=H9bVCWZh@=3NAN74Ka9UjGO) zGWKCXJCDPu0RCrjKD>)LF>+FSK2)j(SHw(AEd%=LQ(N{AnD#A(gzhTg0sIedTqFUe zAp4MC9Dcj^{X~h}%<)~?^|8ftlM)GZ!t7JXWp_}&{Kg8roUkgk=tT)Z?_uogWt?+7 zfxX&ui7u%#v!~cx_i_tlW^*4j5xc}T5{f(o?2>T6N2Lg%Jm;8kJ)lf1964^3xrh`g z*!%KPgnNo-sMh6Zbq>V&1?c}`?@i#L?EALyk)kA%klj?2EhGt9hLkl~N-8l$3kk`- z&4^I8NkR#uEJM;{Uo&+)LsOGnx1XTts2a_5wHD^b{TZkzf;nkyDy)@6s z&!W`{+(wF|eD8gUyHnw(KJbcnUcMbOz)r$_H=ZbdT!07}-PNhKC*|0Xut1xt-4KMI z?-4*w(jK*qE;_7QGWKK<9H95f>rYdbW1pK1(h;eYEPPxf;lt4?5|kSy_4 z8ZxE$g6^4;khzwO3YTHlVwD^GqOmSNAWSJx5SGUaky#t+0C;RO2BsN#Tu&GQ8UW8P z4g|ake^K*JKYri$qqW(?={-yD4cR|_47Up%mCgneee=`aX0ZrA;3u0Za zdV3@?enB_93N^-~teL?y&}R0}!B9BBl~kAf(I$zVgiV`GL#$NLb`v zqbHNeb5AU5zp>63PM@dOXKjDRuf0^H0keuL<@Dx6x!A29VNCXHug&XUJJ8KA-~7+s zw)cCbOL$DajEUsiOnn`i8>;vAttoP@vBMOtW%zi(klYg&TcGDOc!LcAcI&lf3HH*S zMjsP9j-bp8SHo86w;b;hcAa@u^G_NuSQ4Bs_Do8uq!}(gVbaxP?Mis)HB_$}=Ghf& z6C1)eE$;%7hj6xu&*9j42)?Cr-qxEvE67-`5XzKK|E|fBDX9f zat<+k{pF>skxl8J2e$uj$J_IYyY!W>0IIoiQtppcpLZs3r-=aBwRiA8-itZKrjjOS zYrJI&i{{-th~EB%UdnWWziQ?Y z2m=qI&htlabN_0s$^5^gZ@B0T8$$^-jzJtm!u&YtnQE#;!AWE`J$!<$(SA9~7k|Vz zGScPFDbvTo1!hf;zOwE`bnXV(z&m7373luE(0CF4$-SLn(ZM>l)Sy!DrL@FDJx4o3 ze_bgwN;^vq_Yr-@6jp2eG{`%1B=)GxQn%=FmlwlQ62PD!Ve1V@kpr=e`ZFHkzFBso zN+tbHONYI0b$?hlGLsI+n%0>Xwx*p39MSMJVO4=xtl3k|kUbbq{C(WhN96W~ewKY2 zy=mz4owpu2o@%jZYz_+ODluQXa|pOOvSq$!il!PMJ4ZF>*vg}$e&>lj??7f-+U?eK zE%M`=yBgE`gZeY92zx8_Avd9zOcKbqBwGO3gh}^Tq{<>&5c~1>QBR+frK>yMIG*ll zJYD>uh5z)R)mylCQY)0UOA3|))`rfUk5VLUq9f}dO1E#?ROAJ)scRIN$j@&9pF)tG?;HUI>SJVZI;?r| zl!s-w5{Fe$5)b{;56E5Pkg>eb+A$Wd*~w#(?-qOL7wn@@FgDO#1HDT>{WnSH{S3ReSHU>V~KOk3sK-3TfGiS1dU$R*I zWqMNOuJ4+2U&`9+2I*;u*2N<)+B*9mJ7K32E-(l)ywf`ien5&>Xxt0uiNztZ8XSz) zGlgH{d;(u>v>&MFj!SkdIHu;`C=pH<>nFtB?ZwvG}S+k z?p_z^X$JNFf>Sw4CVh%~9YysD;wTi7-BONYindt3vGmR;y=3p%J7IjOg30aFq`{Gn<&O<^>R3r}Lsr6(0gFvMvW^NLH!vF~eNO)9^%JDrE# z;1n?#QC3bcpFs7#JtU9nJ=Y)~;p>5A7jZFw2=9XMF{f|q3jVu1MbJaSnp|)bEcm$p z?t3cM;J3E4GG|sVcpXE7csgl(jKw})prJ5(_-IaS?0e@pdxSFcC&-Sk7kMrp0K=rc zqrk|ruB<*jv_Md@P}tBcS=Cb2@b*cgQ>&jY>-{`$OuCmq1F|`uVB9{Spih5QArC{V zc{}eKt6Q=;Ae*-9UbtT2bM(5Zhw;l&=FKkLu1fP}*{k5BJb{C{#XjU020NptuWR3> z$z$bC*&n{&f16tL?96QWH9A?89j}<}mKqHnculk4@*210TOzZ? z{Xv-c*?(w1kFH5F0!W1h00J@wU-!bDV9#EJ96j_#zsTg`SvtlbKQdrl@MW^}_#mZ) zwAh0$L_L6oE2+_v#mb7NmqDAE)hBl#28{~#djn*K@RT0YD245DvaSWjiF#KB257AH zg=JE&q`n(mhWXMDN*7g=L{PBn^nPII%tQVLRx|1L!+o1-C~N2ewQXWr(NMrv`?mt> zv1$!W<(BS)1o}&$cMaIB|L65hr>1i!dZs{Ed}}@1-`fdzq)SVG?oH!TkQ+Is*PEwa znp3vA63E-~-j9zUl9p;$PH2l(7U{ZvZmjRKP5@scDDjb1$dA*3`aXHHEOx&;u#>kI*fs95@YMx759@CCyOVGAzK zay8ejKI*sAe9nHdol~m-d_l^Ke=_s$_!r-Y{^{7!X@1#v@-N0GrC;r z5q<)}bEZt?+U2w`k385!qjS3=h}W^~$Fonn+w1gvzh91c547uw9y&R0d^nIQ7bW0V z#{Mp;M-k#^PQ+bx#+74+Ze2NBZM_A8 zH=xeSO}#-?!9cgCR4zN?Ynye)&7pLklfg2M_g9Vo~?B4 zj__9vqxJ)fl9g{XrO9^wGQa+qp*hl5uW!@L>J9L3&Lc?kt(CBApYdQVaTL z?97%3Fetdm$ba3LLHf*?T9>Xmm_u^N{`T-flxQ2gyajn1-TaxyfwlJFyY6#^;iX2i z5w7B`5x)0!l(8|`uM~x@+ks0__4_VM|0DXaLAeL6*FVV5r$FoECgED=VRgvBU)dYh zz;cy`y8Wo>2Q#-9sLtg^QGsvP5MbE+GrP}B=5>yL@s|Jlp#e>VJK(oYsNoC`X}!1pcgI%=N+l`1)7C#~(&Lg8K4>1{gP=Do_+= zi#DEh)FLS7iHj3}8(0kyR3i2PLO*0q(jG|M{raJ!GTcmce!2wNNTOU-S%1LB`iO1L zw=eeMke9<6x(J=fn!Mz?9rB2l&?vRTIb`PgL42U!d01?wUHpr0!|XQ|8pam5=+AgL zK7Vqc)Vefc@r6ibRJB1RyS~GFdxX#_I~U>A#`Fa-i5jE*@Y}D6-+C{Is|Ruf0Hd9^ z?;>m?Zr;2pVmt|W*ipW)hpd_NTSDN7k;SNL1;)vz*uF~E@BM69he1X3%G6Fr-iJ`! z0SO^?l@$bTZnseXYAj)@(Wf8aC3DGY9aep7=Rb!{n4@OiJ<2=hpB_9pC!ja zPSqzC2fJbT>`rbCA`9>B+sG$kWG9#dTsmJ0e^BL&>S5|Ec0w^B0$~IbT*SRbKu+VT z^0_@}Bw8)y*dAAx`u)-_hmc0?nSkh^fsh?|TO{1iG>@!EcvA=4FKe(xI{2Z86W}UV z{&Fo&uU4pE55h)R@bT1zh0kWBDKq@bVJoZF<7)!hMhEWHYSjsF_rbWt7bk}I&7M>_H{xOW<=Fm7b1NKN))>J5M@9IEy{rp{RBag z4KulKrQZMU&>838&}i&ds`$S8*98Be+oI3X!) zMA)F`)UkAe=){9cc$eFNqUBRu&V%~TwRwjvLk=b4MEP(|VC2vO+*T+XU?$h(S@m4% zpMdg^$HtGg1&y~#EnMq)bNN%Ox=`X@*IEh=weOC>U!qN-Pv{|r|Zhor7_FDWdud>=k0gksRMy^>yYMl4?t3i*5 zsTxA6Zu?8G|EebNhvLL<&LO|Q78vctWG0<=6q--gFJYeMdXdXGswKLrm<}f3i*m6U zPkMXNiGG)+po7xd&|}f#>M*OP{@wR-PU>j_K^k8>JcN%SWYIo)h>knzPNMr1tdhK-c1|{u#8`$lS zf%tRHYhzGpMIk4a%b+P`_9)=$v$oi*R*Glfbjn^((J(+_0OccI`kpIeWO-+XZYl$e znL5P!QA^Z#J$_5%lbRuQw-=*IEt$7}eZ)!!<@pG0EBA?S?fLBRM|gaI*Bg=txqrf_ zHOotfWEan{yz2P*2FDLb$=yT#CKhwAoz71rYqYRWd3H9NqFi&=ls}-jZ*63Mjeono zNmKd(d5WJi043n+aMY?gj4;MH0{RZ6hCovpI3-ORTU+kjBn5qYPwt{M-N#~&-bS5+ zFNw0_CU>2WAHRcY?+&9%S#uI1&-=J>t%Eb7(Bx;8sH|D+t{Kn|W@Ge$%IX?sXN`MoPfs|#=EJ|p~pe@+}5z3?+Sm5umSK`&d?v=w(yQ+=4ll0+D z`6Xv$>=xpHm+s56vKhv&wBR#|sboclZr@CzOJrGy635_j{2cNNo4XgZ2LE)7t3rQf zz)z3BGaKr3?UC$60{@3Muu&1`H=OD(vwiz3B*bxB40E!z(k`{57CJc}%TM)x$$@sA zc$H}*`98Y4_i92|#M)!Srpc0(j8N?kf9}Az?biL@>c-tYmpF|1Cj*YY%NicmzQ<+! za8fzd?CWCnWv1^JAOKJ@1D&|DYGnx*=t%B2Ar zFQ{sQ4acU9;adCDDBi7!fi6jij=V5f7P@JhjZnukuYE;5<1}hyHDEZ?(EW~{q$#oz zme=aAqLrb?#VuImJng)pL=u594w8H<;O$yr9N+0UiLztf;#NE9N{JcZyNVV|r$(mZy7 z^T6XR^S4x#*HegKbzFo?+sOIQ`Du&U>|G6B=0dj#tm`;0vKCd1Or4~44%>SG&-*fg zPiu<4J(34mP1zpay@kF#|3W#tm8LQtk|NaapVTMhXZ-)iv_CGQ=F1(7UI3Cg=%fd( z;CS7(%NN0>1lSL7pj+>PbbQ{QwU~EYECx^= ze)}(J*U_=gIG%gucQ&q;hQY}P*A`1=W%8S2xF<4!Iw)?W(**Pe9$x zmJ2zpKiPh;k*7ag)^_U9DIY)Q_IQ}Hf{+4(ISOxc$h_$WyN7W077CrKra{D*Pf*kf zrYb&;DFBS(E8RBa;ns`vr8R+gXFHP6nFLVWuS)aR`}>Ug_sn{GDGAlyW`05#fZdgi zU-h*<0esu`f!Wfi!(vt#8HK90Kuwb>00e0*AkKJ`f@?eTM{&$VZzl&6pT=aMzRIbW zCQKJ^c+%*StP>1m?E1=LAyvjto1Voue106qqD8XYYm?{T)6-<#Z%`y2|G*I{9dLz@ zsinyKJo|SDK=Sk2>x>66lS0;Pgxm1D-Ag3>4-1!cuJ;=him!YUJQt(9%O+ICWBjE_ z8@}JU3LXF)pHTOVLp|UeNTI9#am@p)rKR!vJpJ7sij|Z;-vt%5{lu2amLjnuejHYX z2u0-=*Ji^)VH^cSN81O?!^4Il<&|Zk2X5SG;yL%?G^-|$=Jv{;v&xY`9^f3Z54P$y z#?MMOW{86(^`v^QUA`b*1ZP3q7<3plg6>2>=xm#;*zpruo1t}##?(whs6JplFFxpt1xOi!&;v5yqW?6mf>&?-j@p55Ph zUrEpBz{8snQA6#5Czyu7hp=JIFJ+!>x%}D!csl*6K*{ z)pw#NKJImV8H*-Scn0M=%-%^fRRQ_wHQM*I#~xm0wIO(@S`tPdrW4mjr^JKpWCPnr^z(XJhh=MvEV-*>ZU|ww`K|wO+YKO>I0s5(E;xR)b)TCJFQn8)3{+-lw zQyg(az1L=M#o9OVA>!0_)Z@+i69eHji9DJR2xQ&>Vj9fH{3(SQV%GT&dR7gM85fn8 zj5Cy*qaz49XG~ZLThxo+VS{4@*n0m9_Z<5pY3HQ^U4tFZ`lrsZKuA0jh2LnpJq^V~ zX#JTcV3k34Fe#g!rMuVS5e+5H5XrM~bCQypMWIP{`a23&EfxZAtbdy7w5laK9ezoa(ZU$p$6dujPBbVCg^it=9 zN>>gix~J@be=7v(4<5r1ruJ@3Gd>qeK9-S6&!X<9ohaSa6xD53B-mz5IgRz^&dt#`UOy#QH@DY$BTI2TEB-#u7Ag0mNX2Sq@81`TYm{Cr)zhutY|2q4@nc%8?7O zgARSkyoA22NyD)X0jt>h=~q+JfhVV-e(Y=v@0AtVLcuQ^dw&^obw3_+@dsoyPUrY6 z(Z(A$fE$vl53NNp7fLIOtuDKNTUsALMvOj4i5jox%A!o-1v!{)g_weWK-}DD!Kf>A zSY9YQi@3bQHU0Cbp={dNI8td+Ljl8U%wE|H`pm}J8OOW2I9?Xgxg6|}54RDm-@gv; zja*a~rbA}2l_#V(8prPiTWXOl$MI0TH%{(+f|tsat7S6buik#{+8@3?m5#hZwyf<3 z4QH~^6Al-i75Y*gctpwPx}@R&x!Ql_rt~JP_Ta-CUeDj$HZSN-U7aw8Nu}KveRkPH z@Px|(q7|ceV{*Xnv;}YZBh=A56ikMfbQbPp4S_3wG8|KiWAPmtYnXsVi@Hpkll!Uho%v6}k=I<8rdI+4BU!tk|N z2hU~HmkQcBP+b7|0$>^mFmCo^x7N8K1;1UE|ABI-+avntsFrR+9tWwi#&N$h{YaPt zTcu9!{Z8pX1jac5Mj;I zP``qR_ukh;gI#7dtM;^j zp79Y@|oR{_c~Y8OND{uoQ;Q9C^6sm!mlMsIjeZ#51&9@XaV zL{+_0zOjB2`zZxl4tnU}d~a?Lkud~k9gIIFwAX!~smZkB`ohz9>iO*AF7WjM+ke#{Mdf>|FV&*UtiAc)*fF@r%5By!52gl4C$5wD zU?-e-R|-hszrzRgZ3 zVyjV42E`xSm@Apolah2ySXZ1sp6D(vVcm^i%oK*;8pp`u*y^`a?cq5@X7f;w=y4aG z8rB=x-(8(7@9d0#Nw7QzO5TAYAo$ExZW5MYnm9}EU`0yW{jGE1{|a@}tU@D;3!OQY z-?FW$?>K27PETGAQEyBco@P((g@S4OZ!)N1;PLvd8B21gXkuj?qdWxQ)nI@|PnUZR z*L8idPPrxL)PSeMgI{BIYCL(XoD)U^lSSYSAmlt{`vYPc?n3r9VklCge}is*hj9LS zxli;|;-&c$>Jm(EbnK+ROBqO7S_Y}Ky4H*e5XFx4*9n-%VOe&suEmg3Wi=xz(@^D>pWtNakwFbTMOD*d$h*V)S(ti@}x4c=!lB(Pgow-POw?zp=@x#7=G#@bU4 z`}xMR4YKc_&3r-WiKQ$?XCsG5;^Z-xaz7t)yzxdqzNPsY#bVKkMLz6=bQdOS*nmWG zU;tKCpsQ}}heBsbe-P_Ho0{Oa;1Rmcud~P=it1XWWgf5#*B?{*W{MZNhY#E#J zSOo?GZ!i$V*h5y~O;+&`a$GGaj6mP`WoB)un}=cucB(CFvY*Eokz?5fDsrM3YS5Q* zTb^W0GHL7+Z!qtUm3p6-uc!^mhtXThIW4FG!HIzO!NQgOddfpHtq(9aK>b$Hzenc` zhXiw1$#-&Uy~5_5$oS@#Ro!!5a0!qb5`&8u6$nm4B948<8t~clVQ|Z61xtI3kz~j` z`<+#dD@)ttD=Bvx)YXbE7H&Ll&ydugk9mB9$7|u~dq1x)^j{W2PG&Bh761BaI|a>O zRMjWcs#w3-*gHk>Cex)K4S{yZeGW;L`|}(JU={Ya=jSVR=+dLtdLHWWw0*BUNr(#A z>)_Le!M=|&YW-@$1i+S0AN5>VyscJixy-jD4QVajS1z6R?HRAsn~(v767IsG8M3&x zV+HhNv-mt?HL-64R|5+^=v(;(#1vJQf)$@neshg_U1w90bdQus=6Z`>$MFta^qy?_ z`WeK{)(c0yt#hfaej=Z2%B+g=vnYbTiV?_5U%1$RK+3I)1dmRBbtcUB#WQw3GH;x| z>Oai;1$J3$9L;)tiF!IXqc(qevEZ5fdQHTYX#vOJPw+d%VZsA#b$b)t=A_s1uhH?H zdyH7CU+Br##mCniQCARhfzMP}#5=%?b?eJVzN8P>mH8w;(LUZKK`AIl5t}K;<(?$o zieMboKoDC|Z`7I-??iW!71y6px@^xiXu`A%uJL|k(MYtIAmsza5#K&TR^kJ~IaFZR zZkeLl&eUf&7-EwZjpm>xj^3D-#(BkC^l+Btv~JWr`vLLGcccaT%kG_{CYl4AW!!Eb zz!r@DD0uc!DY8eH%9Wf(n zKxm6sf17Q8qu63Xq{l{|Upxyt&PIul>zrl1v+>~uCd~}gcAB#%uYjnQBf^ww=}NUk zwtJ*sG3IIbHa~N1RF6e{Dj+NhHn>LeOEwRY8_NUIJ-4xZmR$M`Ul&qT${Z>co}_@vx7gg&j1@bfZ&=Vlt0%4QA=DrUx5EYQr`% zhn1TSePr2<`-PSgvx$-vf4-+twCi)@MDY?Q@zO+4qX;@%-#24N+I(5^{J1D4mM^K( z_rV>JlyTdcd@xAp_uc3j#!qJ*7F&!i(#Px6j-3{Yv4VCHl!1nW zL6L~XxtEuHbL&};RQ$5|(c{1+BiH0ZMBUH|@r;&i@|#P=-N)a;PHjF)laE;E7klJV z_1)K^7QqE~N`hXrFBl6sxG(zxOEn4!Dx}c&I+qQ_dzP$UwL7OyFt}RtvsUl)2vv~6 zwidm%;(Pja??*KrY*;>aN+ z6QS`81h?Xf)H4~TD^`a;0Q<~T!$n#PX5Q}s}w=OfTz!h36v~gDmRgJNp z-+lV)P*I2j42f1Tq-5W1D*KZuqkQDr*j2yAZbY za?v%iT6Ctt_ey2xM2>HL!iO`^kOFFsu}SHqgrn#hUx1GO;?t8#DmA!A(sSq_L|$W* zhU6>7NGmbIN@!vTuYzv)n1Qh%qsgx*x- zSt%(3H&YeFLs+UdBEpAZUPU`cem%YtJSYTip^%ighnYttLR;1AFttMRa$gtyS(&q{ zrcqL}{0zrBulnPcNA&R_(s!ocAwLJUoZ!DGLNJhY`Zhe=zVCyaT3fEJKz_MA+j~|s z{c)y+K|H)t8TqYf@Inmv-o)PX{ywE){?=M$gDkAkJCct&lhDfzs{JhpLA*VNpJp9r zCEHOMIgv7C*1-FV8R6P%m4~OAV+G&F`{pv7@Ujg>r5_!TM|x)aHWn5lbr|!-hU#AC*Mw}@B{iATdFcG1Z`rterA+l@UUbT>!W9z}F{g_63lZlH^$XnR!-{S1cxG8w zW9><+R#sMeKp0i(548OH<*eOWJ#o>NTYHX;ypxHH{3YF+1%n;>;_2>sq;@B#B=Exv zh5!W399e>Fh3A{c*X~uOSxz=hl@afJ_2Oqr?VfVMNNJ4TRrXlvaOoY(H(u<&L#Sz0 zyEfT0xxLO+;i>K^q+bcK@pTkGbzR7v%6WE`MI$+u7Cwm$d-QeYc(zn*rp(c!Bs#nt z+!T2aQLGE1N)y>z&}hARqIGr9jpb_`fj%kx)Vpxye6y-N>hPNqUqc@VCH}354rA1&X(BW~0w!+SCEDTB8(PJj6A zoJYMNMG@KHCrdUZtc;I6$)W8bv}+c0oHB%$J3k0Iso$9%+h%=DkXd%|Dls|mvmkSa zk68zkk<(c4nS>EM8-#HL?f~67gNneA;A9La-+J{tJW9kn%TIs+&221$_1BZ zfpufDoAFXKAkegfyJUP4PQZ=e$j9czNq~osp|FCA&reZ>6zB{WLzs(t8`ZIGIZ0MV z)1|lV)WgD4!7y(g0zH_{3701z0oNrCeAH`*FwOZrZ6{CB#!A zy)s&D^kWW*!2q*|S-7@$GguLJ{cZ4EQsOB*kn-TkRA~<^;2#ooZgsiC{y~1R?YzIR zCJfvxgv-ub@`z`sO%R55|B{4gQb!?Re}41lNMij<)mD7irN4H{(u5gXOhIV}TQC91 zyG;8T2gZHf4)Q8e)o(P0`EiZ63w=AEK_1fTR@1WIS?PC$lAfrPt1DslruUexx#I_m zL-%$@y=0>Ca?wYV$=r!li%kWaH!sz3uE)P?wvcL73acQv@@4u1l! z37~YO;JT!%NdF3kw7TC8PzkXg2epj-11Mq`d?L9L07RXq2q?zx2XJyd#BT;v8`>)@ zR|Y~SA~M;Y6!jhko5~Q#SwvA~{D-bq!U(gW6-U#mQ@kt;^EwM!92w{1FwVj`1=bD) z=sby7Q%_X|A*ML74aGvueacgcv=6B2o9peLko}kk!iw;{^A`T{n1KF{ z$SVL-ccQ8t){8<(frN{NpMuK?kFJM^2Rb@0k3l>dlYu@U0Dn?;;4|@LQIWKMd|q6k zT@`e3Md16Dg|>@*`JdikZ@uQwh^3XGYrCTnUBGS+cIg03nEWi2$cUV@^u?go!%rEv zYzmpME=B%9V)*L}_-`bKzy3Y!N6&vBnwB~}8x|J>A*J8cI7h}!q`q4K?uGOl6yu|ak(XcE|6H08G?D7oAy_Tu&9q< zlaG9vDmmCZaycr6p@N%g=PtQ(=URD^#9)%Kx3?#vUz@bNuQg6?XO`hq2u3!@?p+w| zD`osQNDZ=13EMDcLp2rPHvlvP?vR{?X8J#}&FR`Ta$6F&82m8aM#QSjR_cNvYD1DS zm-T(`uge806D&f#q4=U?Xz`4(q9E1mHYOf40u_f}S*z3FCzzKn9X#lI-&NL22Czj+ z7^_b)g!|;(L6PK!l{Z9nhaotqC~{*XDiysjm!B064T0?So@L^+w}xj-`^UVE}-vP+;x=OVn{^dkFtaQ%Eb## zvsiE%L8(k%{Xqc5nWRtn0U=@QWBbJ~40$;lm-al;Z6O7!J>~s=;R)Iu*N0-dx^>-e z6W!-&O4fnbC6EP4XSY9cuNFj=myU_`7wUaT5qsoVuraDfQZN7%tupmp$md?jj&F8- zaZ5cq{PX#E-F3II=wHjee~mTwlEwepn+rl`{{1fa zHDG(CCpO{Np=iSu~QLW1=fL9W*{EhKIqCYnw=D+<9;m1ZC;Rd1v3bE3%Hw;V7A1P#wgpY(igs!IRc&WDUKac;u9da} znWvN;X(1{{V-E)#s^-0WA6d72_#rE@r2YhZavD=VL_~TCGoU+OMNTKaAG+3?Q?UfC z9&$IVK#zQJ;}|k@jShMZ_o}7l@-U22{Kn5jZFwy!>v@2P-To?&HAhQop9c zVL(}xE|T+qy@JpMYQ9m2t2MBqp#GCA^|#voHiyIhy9T}szY~N=>9Ey@q%o50Rn*)< z0)6o>^+dts`+I3Z1aqAc3LnQ{Rvku#0Po{7c>Eb2#m%C-7Z4q@G_Ey1f` zTDJ?BNa_E@$2sgz?Haf`jiq&1Gu|1rnP% z5oMSvbZ{OSNW6F7?3ER*EG#&<$?=7smt9N6T1Q?}0Kp8u2mHE`?G)>*RPsT?&7do# zhFOfUlV7Vl4FbwJA$&MD-18rh+lWC`OL}UB>f}BL%PyHyo}DYV3iXzbdTZ~o9eZI3 zG-0hxnEu6@(mz{|HJN5e94EU>ysj{nMQ1IXCZ#HzPd_h|uY(zrTGd^Z^M8vj^i$79 zRcp#LpeA612YGT@M`tMYXycJIV+`sQDs*m|m=<2xFjEbtuN=DcT0be2{(NcUUKtTz zlkGKG*<$3Kihd!8z7oW@M;sBq91|5wfrqw-NKI;F*Krpk7fTbuR_&3#u*3DRK>~Iu z@0(q*zQd|o&=q=VrObV%m-z&6YZ8op$vv$(WWTqGTV%1h z+{C3P(V`c*hyDyHweGpNxhoo1=vR0!PRO?%LdPycO&m~=>_MgIvov* za%%M4cwWR^nbnE;B0rbLhtOi<{*)X?E=?luGpG8{5VxV6>vQAYr%k>*)P!%eJ}hax z)Yyf;PUE8T`W+^8giTq7%+|#_L=>*Saa>hDgx(W0H$(BjWvv@S=d6YEt(0mh?jPUS z^KBh23_cv;&jaBhkNQtBSYqdAE(;s0*u%X)%VM^dCi3dt38baNHXK z@aMCMmAzihHF1U+WQPj{T}u{er=O=uZXJTqjTI|~IXG0=h+|L3gfs;GWQ*<*b#OwS z2IXB1W1E}K5iT=wBO;6D{`J7JM4U8CEPh&71cQF)u8q6EmeWJMS9wRBEy^Ywzxnx1 z*u=y#Q#&~jTTl!7N|##BlUG(1zmnsnTSUKn{aE=G1?aYLK|e!+acz>?cjU=F8Mgt= zt#gso#s-aPhth%dxk7?c-Pq83(wu$SA z-nX&GUD$~-Wm*}}Vi}%w{hDR>@##b#_HmC7HB4>38@Rb-Zi1>r_4OhOyz%eSCHe#%Y#@m9;O=Wih^fUA^2aFOyoBvyuDigDd+|{Xx2_ZzE&o)sl- zr@bRE-wz{J4CUCu)Z2?w;F*YBDBJekWUfWZ4@l&o9H&F)QZ#2bo#k7jw$K9&58%^@Rr7l48f|~Q`3)!mp2T~8%X=Zzh0tx`j! z1~J}iq<)1fpg80aTs0NEuU%LUn^b9^uc&MWLAvOiLL!wl?$YObIK7^n%aqv#_bfrm;CvdLFaUp(!;NzO5+eMd{VhhDsHpah>oZ1Sh$YVY%?_5z9)P zvdIrfi*hP8H>GL{hr2($r6Tj+x-n) zRg^$mdrG_(YuMuvdcNZVbGx-X`BM38X&MADKq!Lu+Jm_hnPV(~Qt;9h(wI@V<4@kI zP2g^miWhog?wZyY{aO1{KD_Sf=wIDm<8E4|uBe z!tLR*sZ+{!>tqZ@R?RQt!o%egP^~ezjfAykV|N!my5bV?;-xCjdHPs; z2R$Z;tbXf7!;}TS(?j6ZG`owL2~f{)AOx*%H_ZOdK={jLn)*JO!*h1bTcA53@FM_B zmb_{i6{n2pN?M3ScM9O*?Xm*hBZI_Hx);zebpjfubK4tYUT0s$fIO9q5pwC4U+KrdNZlpEI-Kf zQGW{3qy`99X)~r#6Pp90EF{T497im?qR1Sca!t5ISIzI^0sxko*@}9wF ztjt=>JQ%Yh0R9<8HmCs6fYuu*jSc++5|+w1fMWxyJa=nB+?5JCda}t9RjL1Pt$Brr z2o3;oWPOzS$5fCs`4k>kl!2S+##}55nHP18=x7pS8(`g;rJhYbINybT>r6gOdViPj z15)40B^Pe_dB+^fgv+D&men7Sv1Ak*gg*K175+x|93)LW%N`0O6!`If5)Rz}_S_7^ zl{!3CaAS-vn@|xF!1m~IPDfxi;7D});zy$>^u33YPo(zkvu;B`0>t97G_xc@j&Uf zxK=1x4Er>skjhI&8C}9G?n7Mbe!70rN`3i^OMM5fivp3Jg%0gC6|h*$B6_TVB(R8uODo4if6e)quGIVz4p#>l>J5M%YCB8pn z)-@hgvQ7hiyeSX0+T@2#&9sH`4W17LcIj1fT4c#oF)t)Q81t2E^O%7#R=+)Io}UP9 z==)xclyJQmtW)fKueM=c_=WDJf`L8qgW5uzvmcv!B-&>z=9t$&7dczx56CfTz%w6* zQ~bKKw^(rtQ-6LlWe30>cR@(}jm7DxD`R=sB`j5%3?n@t+fTrFXa=2bt-w{I?ae)J zd(|ef5W8EK@7&x!9^rsQpkkn#Tl zaN7lPGboSbBgoEU17nP^#?T*-l~Oo)1TaAnaMF#Ue&_zYucPWXuT;JRKS;Fq$%-uBmB_k2Q=Br0h4xcgt3 zR5g@VPID1kYd)LiPzq_0hpPIQF6T;+bieD_P0i<1l{J2S$7AC>m+A%3CtB zld<@H*+4DE?fw~%?e{>^nETUFYIwqpePUXZ@F&;5nU8qdejJP&HiYGU4j+#SfWpyP zxVcC8y#NC!rkKQ3jj?UCzBkIq&i*!?GOW?RtDG2(Mr>ZMJR5%>2$It{kbh=2F%NAG_d;@zIynW9dffVW9cqryf7&L};F7dYD1 z7hwU-c#o1`=u`t-fHEC6>E}C)~>333wBW={f@+9L^}V)zAN8%e}_&b7%)!UZ);p}DJ%eZ zVzkya1n`H%sXdT51q1BAD{=bNCCVX+xcTfR!81q^$739jU>>n~)#*t+zZ5d_GZst`fYn+8?n8^=u25Yp2kxHD8dxtoNRJ$do7-|x z=2g8P%jC$mt1S8-Wp9jF$=lzR5q^qrRiRH(|1-LQ9c zA5d)5jgu2kYA@FtPXv~4iC#Z7Vi1iLDT8Ih_f&|D!U*DlYJ6X2T6iKnmF#rbh7n4G z8LafKA>B{){&ppx!(rUVrlPo}NfFL9djHm=XvuGj(B>3>h~jsLh?^c$BC!mYyL?|J ziRSW(N->UReCd@u?GhpaYPq@WC?FFo)x7) zJI?&G!rXuIzM3>C5d^Q^rCY4(%5U?DlTpak{F0t(AE^aq-|W@<-F{cbC$QLJn!|=Y;eDWu9&Waq)1ATx%B!a z2Ad?|wwLH+WN}@Rz;I)_SIe>4`?P28HRWYpYPOmXJYQh9mvT^G&+?o>RJ+P;sp!#xBYuiSF&sw z|Ca33NHwfG6Ppohw_DrA_oQVVe+jd1tO?LwCxiU1ld#eV=b-USEj}mEQsqv}py^(OJ}OiF$>bdy=`1<0Cf`EUJG% z_|~;D7Knnr14@GGIwQJ+jzE83A9hk`Uz68O^=y(eN5wVV3;ml4((U`mwcRZ(%Zq-j zT!cHvZM-4`X0nD-;d@Saav55`CK^uYhF5Yqpazm9pLi?WTl6FZksJ!zS(F-^t8Asb z`IqIbZK~o`jUl&b*cs;j;u*w%3i4h^v>>i-Zu$lx-?W2npHeD{I*%B&8UID9fZQWgD_3 zWzU*rjGbY|GKN{s=hJyD_jz5{`TJe>bwAg0-_PrHUw`y6rti#{?`L_xKgatxjyKDL z{LxC8cly{sBN=yO%d^jxobCoI5g-Q=?uza0vERmys0_{*Tc~t>(?2bIn5UqiY0&+Q zwp7J0m|NO7cct1^23hvYHBQBYXxEqD*`F6{#-GgFl<=9bigl%1xllP~J>_NseoBqf zYUR7^g*!Y}{Xewz7xXngovGrSK7af6Wxyu04-#g|vUXEFWlP@demZEB*w~O^d-UdEofs;*j?b7FrIZwTHO3zq;?F5pLj;t=Hn$3Gs zJwDyozpA~p(!lSDf{Dpm1OBGaflGA9Tj(CWKnuxxHH0QF;!Z!Ds;P+UlEM1b1wtLB zo|#_}_O4RSDdxZT2#C7OCRV4#4b<=N{F)v##6I@U8AXDRi^4d~AT? zy^%rI{tK^ndrm!jhY9KQIDO)?g=aE1-em>!^x1mci{@~_1__^OlUFLX#hl}}I=_Em zNyw|6bbgn{s4vn6&=RA8LF7C~hEe{_*{%M;%fZq;o`aO_gdbv!=2-VkwGDmJBI0JX zc-OU#6{RhzbJ{-JB!y(ob|ds)o72aa0((b;yvW!cJm12a4_f3b2Pe;rETQy1YizGp z_+FGgukGX12Jt|SBiyXLm(*T30o_=APM^yRqSZ`a>`Rk5-5%!wXN>htCObI=1G}DuWi?TN@Dpatr zdFk6XApK+yVo_e|(QTD)SR22ndh51mp9icHW7;!5Ql=AShMq@v1wnlAU-FVl&|I#o z1Vp7|;|)|F?FBK1L@#@hnKkO9FPa$OviAa7b(WAJI+D&@Wvzd^R> z*|HnV=DN;+=rT95(}1|XTl$oGQ(>v&xcME3)-4I zFTLrDjxTV!TR*-w2ACoiz1h=V5j*)WvX7nXR*O3`c($`3tD>qZ`ATVNgsxbY%?c2~J+lG~mM%6J}6Ri)C$i3AR4&@t0|oC`xB zzE7DS%n91tl~ivpCv5NcZV8oc%Pke_2D)=z+3?SI&KtmU%*ZCxAv%OW!u5xZ;(mW@+y5%f{Il4-_@O8OSgdOR zxT<&ttp!oi2=-=asjqOEb~R@kuh}@30Y@UzRR(%889F;cy@c#fxkzgYU-1}__++IV z=s(JI>MCKpu>CS{iuhFIGt9{?E1%iqnBu=KoVQ+a%jsUz3?uM7_szdMG-~m0pjW~9 za`UI&)(Yl!*70S@zphqU0Ee-y;4BN^fGJOVZp=$O)p8_M3-`%v zAgA7Xx=iVXAN|zWL9Nk4A`|7BRXCmF^&f}peFo;{FljlG4pLh2ks%U)ROH2c04eKf zlgn2a5j51v14n%`XMC?3J8*RNrXIgHg1p&O8azh@(TF%2Gw`mT(^b8vYtOAc;I0zX z4{08$gRZoM4Q{ayLFMEww+6peiM{ZsVZjVDtz|h4d!-_C{ccY89U;+l)J8WKO9R{S zxN`v84(%wIIwhIFXu+I0Gs`#f&0Y=j8lqmr^|x3t(99FF$eqP=pbi?29+OKm(tcz% zRQrZ!oB!r;eqMigdJZEzs;~?5`1#%6@RpyL%YXYXUV8&*86hJg^>mxiHvGNx?<%^!7frN2(!^>IW7dpDldw!VeQSikw zKU>4Mf6m|`v>_RakuT`VeNfXw2M)@qowDX6;rV93u!|_15X}}Ov>+G<6taLqUHLsZ z?Nr0=i*KipK2CT*WNU{ny*;zj6O85TtIl!nMwNy=Fp;64!=0^VG^I}ghxxHXh0djY zu|eypW2zF67RhxOa!gZZI?RX>*68V%q83bGx={2aD{)e$ITWCa)aj^JVyV%N z)ww};`WQ*Zxan1{fm3ibmSJLqBGOR{=B>5sqB-fN_(sl-lJE!ALchH>?s zhxe;7Kv|0?x0BPsRP{5vpjY3|80g$P*siaOhZ3Pt^9tFU$9l)ru+O1dIW^_C*nX{xxz~sTTDyu|@3m9H&BB14pcg-Rr95Ny z-%rMWGFcJ7_WXO`B!tn`O<7qrD_QMG*&r)wFa6kk>y4eVgO<`)lp34gFC&_wm$~DI z+SWzrxO!y)#16kZff^yAD!3ym0pb;}qa3Wl2Ni>%kK-uwRoLyEncFoSsn_<Al_3J$+%xOr7+D zWI6T(&X7F%2O?HxPS@HQ%$I6o*bXo=dc+8w zX8q_oV&BLX_Yy#?#BFzpE272GA8#GcAx)~{rAt~hKVzg0`>TPz;IdD9IO}j!g-jMl{@Zcsg_&S%hhN#fz?K{Icp!5 zDsCF>It?%`mA~Po{yi{61PyV#IDCO_J6J;0p^mmE=?6+5z-AuYB6$6DhlH_?aO75Y zn)fEEbS=UD#B1I5$Am!Ej(IGegsH+)O`!E-F*v5!B5}o}1ctrW@vmDsaFH;@Q>8a)-KTrlP4z!Ccli?Xn!6MAN7(f(we_H zZkGHgzFwH&O~LfaXWqHuL!k_!BCx!m0(t_=kE$Dq8KFLRw&K`FkuG~5I6RMA+O`qj_{&MzO^xKUKdM*U8@f-ts1}ky!n9( zTK#r9`IggHCDy@Tr1B^=Fc?>e2JC{T;>*qYTk<~Tiopb*NL42v{EaF97liibPt`xP zTgB3)Gw0Y!R5K1ASPgp-ApN{H5ygrB2>jLu{yesbKQdhHU2d8I^cFA9R&Xp;s3Ns( zFuTE!yt9TeE&j=_{9mOPh<`A9=p94e-9SuW%KqNq`Ol2O5L5w#09KfAfIJf}K`UIC zT~Mt)G(NCs;6Vung_=`pbUdi({8iCixOc4g6K-pe6E!w{3$56z`IWJ%FOS=+!-kV5 z?>xDp1@tJJpxT2qAX$PpPW4^_Oxg|wqJ?V%hd&6;fT&2*?qXaNb?B7G8`G4d1z#t< zs!m+2m5WC#tq_3w@c-f$|NZMWtA53Q0-+}RipGe9AF>W30i;P<9cdrsos7meC8 z^R%*yKJ#Z(?3G6e{isnw*lz9c4&DMEBM;c9d9;4j9mmlDM?xh@Ky%;kQ|9ioVRl*QE* zt@PzNzP@$(g#NyQaB_*ZRD;@wMx_U|lzJ50Z&S@|x^mA!)e4`@I9)yIlPXJ1V(UN1 z?{GCe#fQJE?6{EI@KGlb4c$>Faq-SE8_f(#E$++{$#L~Vx%t}M_#AVJt{&InR4A2SgRPUmt#2>#( z+qA|sYaFQR$vZUGk+sOoeLtWPDGcv`chlJ^k;jEwRc5TdB8zgS@~ao{s1JR zVBMCFz&Pb$kTLrMYwt_{Ql#JkOugupbyr_ab@~Znf2(|obhM~WTo4xxK{l(0qPLeb zDlR{E1x)zQxbJBV3VPR4!p zGL>&4k1Xaa(5-(#h3|)PHFYK_Rw}D56o+uNqerqGUyQ~q{#XS;P;aNDVeL* z4N(Oh6V6@SeEm6AT#SRa9%x@V=u|(D4V09F;w4IRkL&OD%wBFIO2V|q-)w+B;&@)@ zS{lmquwT^Jj4?BLP(DBvu3jYE{*}yY*|ksiM0`F1DcRPoQUU2{Ns zqj|#EviQr_WO6n!A~MWVPs_7SBtp)3=`} zx?2YBVc!bgbyWP+Hp8C$ZVMf!27AT!N~z3m1i$zVDXmZa#)hhtVt&C*@%c>|=FQVO zhzWxxXke-yB~;x(8rf95p&Z$)Qdy@I7?2Zc+Ic*;Rm1cH>ew3k6LvrAGL6vi?aHj~ zp^707i5;#vC(jm_SmHiJeZ(yKet(zO)LgHnKmpIesNi`yA)^D@n7MaE8JKdHVw!U&Mk1Wcyan>>^PDv?C z%-Th#7S8PX9kwv?9ydRd!&2z#t;fm=+c8S%2MiA#m5Mb1*uvG-CK}3WgUeu^#5^_w z&cMYJG|~Iv$FK)0W40UPhWfxvS!(<%x^Ow;cj&^6ze5)sNrFO`E);!eSX{9qj)prB zh%QMm?+lohMK398y7rGw$A9#k;XPx0m}OOISw(4L=j4{DNbj5D@%jjfytr&T&(N&x zTR;;84pccGjXH9wKp?`Jh#-_kA+f?iPI z5t9a@t++WP)rc_YYx3dnC5CjbmEPBCpY)HfWQ9J|HhlXk5H>i`CV8*nI#gHqE6km+ zp!`2ff`_V%u>gEOuSM4clJ=5Zb*fY46|i=cVs|HP#^lB}bwRSF1nk42c5K^pd+ zjR(BL)@mMFTZFDE;}#>3X<+rfTXkfGsYKGfqGEz8YCCL_ZVLs9w~5yhzVn?E1cp4^ zKdo|{kdaXFNC>but$(KOm;9A}jGog{)wSSRZ0ydEk$#pOCU-B=J{-~k5(sD%hb+#C zP_v0-IC^IJkqr-+yg&HGIfbdc)osD(Qvqjuq4Mqwru)Xe8esP$1n!YC%KbTw+NC=( zI3OlR<+NvVqY?_pp)KDGPaQ0 zH0i9OnSt!AMlGO~H4F)ew%*76RaY5UxdzjSwW4@Ij>3$}?1n;y<^x8e_#>&L8uC>9 z&853XBte4mnJ~3R#;}C(V~r2;eqe(5X}oD;cET;kcLh|AT@92fubr0H!RdFRX8Q9=K} z@L`?}(xLU%BVkg<>IQa|c|gH;)zTCk_-{Lg-72f{m*9uVxb8)YZjss3bjQXZ0R~b{6`2x?R-$eA!cX?rRFQ<{)o|9S^^zzWweI_gn1j zGw^H}q$GVDKu;&JJ@&i1@!ktZHnP{H_Lk$xEhn^w`=hCyP7p{HQ?e>E#KYvOd-vs zvyOMFxci{He0@@-=ZhtW+ehjOmF9P!1MOz{q0tn4nH8L6b11B|>P0->&y&{dCtu2El#`6#Fk3RiADltlWbI-7tQGRSvEOKe{}S5byj862*R zJ>I|L#zL^aqXBTc#r6=Hm0Uxn_caCj8H zBds~umZyg)i2+4AhBD|m56ZVO{5&q4UM=x}rhH#b?3!ci*0SPnb@&YLIB4|Zx?J-p zXiHDsL@W$~ebISrX(Oe)?ynT|lGr*mmM-g{^Ga7$v=4({32&6xSsc@qYR#tipmYI; z?yt}i;v9ccO%qt&Tbe4apeep}AaNx`sp7Z#N!&#C4FH+G95*l7BTaMd$G5gBta;FO zYA!9Ofv}B%)kk>jVfl(G8hml1pgQH@rJY5TCR0~>SnU^|gD%Vqw<7dThPC!2$E&R9 zo3HmSEq00?o6L)7+10G6HusWk_B}}b)8+3CsC}8;LR`G&9g%f_(E2<{R%s)25>=$w z&TFXm$H<0oernI({`wo8RB0Ln#~mC~GZKI=BgjBLu)F#;Art5_&DJ$hAWNr<;(+K6 z@bRC3232RZ`2JQO`?Iupj=LKj%kpN(k<4)RTRgZK*^{N=whg`l-(S^wZZ6ufe8%pq zIR6Rq3xZQt185Kg5~H`e+l$&*d$6)Ib54tWHlwh^yZ}Rv!UV|!43kg~t?vhPVx^T9 z?#@`yLyy(s!2mVAPv<WQ6PrH$#f&LxI-E_rK6dSZ+~x5mt$hbfqrr|}=)zg^wIP&Rfeh+{^maT0|%Zwq2t zN3R;QF91haA=IK6`wdvnb;{T=kC7W%lfa?*FO^jt-4eDSREt85_f~3x_H_Yx&SPUt z8#1Sd_@a%mcZ!v|7KNHyv;@<)`6f5T=y%KT_e=7hJw_eK@bF}3H3;|UBjr%{<* zoKE~xyCoQ>brn9{;AI@ZT(80r%ud z;`ZwOe%d7U$ker1_u=8K_~v_ZaaoDZXFuz2tJzz0?TwCp07hyqk(Lv7T$?on-(inIX6e)=TBC_9?2JmZc|wzmeVWq3 znsxf#jP}FM-R6dvT%Mhs`$r2b3_~87KNr#?ft__O0Bt$i#?2s%3FfDYh8540D0N~z z_CsDkTNL0du_anN(V(|zqS0aB)%{wGr@0M20kSg|X@>hO(M|_Hz)oG^zg{E`L*&TK zwWGTrUZUfORM17HZb6{Lw6s((fj)PiraJTE#MVZG%(9kgrFmi~*vNY$SUc!$P}4Wj z`)b95vNdR};P=aM32)hRf~_?UNJWAs(_W)fZJQWhUIj&j-Jn#_fwmCVM4R2qdpeyG!PU~cD2G)dTu;3`(+q{nTKI^wz9;Y&1(J@og zxNW*)w+>yFFuS~B(D^G2NZ)P+9^BVPG7E2$nqYkOnw>vpW0M~HRuCt{p{{#LA}J2(G&h6OsNcm1N1`o1eUX)t*v>*}w5eQ9HVrymY;)+ozZlF(bzmRi-SIyO2k%7_U(KWf zxNOjQ_r>BA-?Df4yL#Gx!e!*ZE`QJCJMW{YtuLT3a^QUGjb{LdizrRX8Z`6| z^jAFL;qeoPLoVbt{RYHsu$da1ZyV4jH=Q7Zv0p?+Vmz+ZCz4hA>PMCukv}Op9vM5a zt;qLSJoT9=M%eHEDc`Zd&qeYd*h8^qKdC*x!EfZZBCDxfZDMU2gmLAaEQ`mknvVXq z1t;8lK}n7Rp-$25g-ARR#u9(w?VQnK(qA8j|H+_C3>*9V!p?sl|H%g&`nCPvm&d<) zMzOHCa^b_hCD8^m0q7dz9zo`uwF{=wqSEbIT-~%E;fMz12Pl$aoLHx+Q!II`WO@$| z(&fH?DJ4EWjVO!}70lw<$0g_Lv-vjguh5{OB8l(39vG#TXr(VLEmQ;Son069-mLJ- zg*~)WW+^8?cxH`my!{s)fyDpp2($(D-GDQl^Wpfl%y8@&YE56ua_N``nE({jL_0*o z)DPA(_D7bL8smRBrmDJouxI zZdb0RIL7(@i8}!F6o0KNHmcQJeD>R45;5XEMEV zh3Y1kEO$|HYiCVHy_*%Pfp&^{@VWZP@)uRJ!tav3RNrV#B0XyRm`f8WW*kFbz^6K` zi1LnN3CWgbzJw8=+x=(^cowwP6MvY>x2?;h!r5_jXyZgKdU^;8P?=h@i6=pl)P{Et zWs~ZQ-XpxTT&P5J zLySF5EZnd<==Q2l;NS=YK7YSTvLuyt0gPhDQ1B>P#;(LZJGtOU6NI0SMF;5ryHzqpON&r8Kfk8JaY7AJ7sv~Uv zb~tdyqxFUc`FQAzga)0+_6yy1W20zS?zLkGkndGAyOIM zj%`DZvsrf#Ml5^AB6Y7lV}Id0;!}t;SLTp264A{4Gyg1~d&tH4WrgtL9X$I^`eABH zRHbvqke^q5kNI%l(2L7*4}pg;61-2oCOjJ(2F{fCHClH!EZeW_$$GI-_;uZV4=MF> z+9Ahm{r7CdRMv)r7{@z#fhdLD45l-WTF-X>lwoAW4Ruk{oeS5ov(Q5M_!U5Z+SpC4 z>vBJ=58b}L-nirFCj*I4yqY>OdkMwUFgid3px-C5?M)>60Ntv?VNM9K$l6qON~k3zNG>g%b{_(KGFqIjZq)D zRfEx{`Mgm~#IR{m?#sDh*^2`7xhJ)eyenlf51-7}r-1H;Z&@R;m8M@buz~_iqam5{ z0J?6H94Y9Aqf+giI^sG~6mZcW9=}ZP;QdIhRgPLwuRBO)37>g8Lr-uB7xx!~B;!Z4KdxSLEsmA>3=@ocrD&3ImoVOSmo<&br2YoG7nI zG+fwwZQ1y!XhPrWp{7K$DlCvNu7!Rt`tf9*cE@5y;CWS`a#K62?LK2cwX4o?qh-BZ z`BX9E0S(yzEc4XLdMh2Vz477=m3*FVx1z)^Ilx`wc>{`SS|ni_=O-c?nbm2p`(T1o zJD-TKFW;-Z`qs~n^!BZJxVNE_Jl5E{c(%DUvjS|#g-L&6#{DN>{S$Gi5%A3CAkYvY zCmV)EV1GbV3&a3|?!(UX0nN4Sxyq>XWo$j7;Db;%enJE4YQ$A)(eqilGf^DL_|@ZM=>uMx5MP)EKOrRu$zjuD8|e zQEek)io$`?m8gSsCfHHaTrq+{`g>~^`^zjNb-WCp?VftDS;$`BAQ|z(M!yVyj>}2_^LwtKV z{s%Q=J%SFb4VQ}VRW@%uXJ)p815OsZ4?-jlRxqvMi{G9{g>hMsgR@oQ?0ij_cC*MMO9MBggt|~?(Qyjq z*)kBYegn0r^-4zDo6A8?jE1Y@EL2g-*SB5nBFU@$nd9z~Ztu>(-Xm9YCd9=FGoM;} zI(vGwwa>~$Zjrm>5Usl9!ESy;q!bKgqa)Z9x`m(g}nP+C_B|?*eK@R0#A5O>^bcZXh{v6&Z+WPp|lZwH!+{u_sNB z2Mc(HwFW2`6w4P~EjyzF2?8|m zI|Jk`JOKVpwF|EVCVqwNaUgRf?EOyw?rlb zfzfP|M2|bMcDm9>7%j|w407!Nvd-9Tm#dD~xpev*(|W)LH>`hH@6EMT*$mw4nb<2+ ztIZYYWw4X-d?b(I>uf#o2zy)|Oq)=%;O?LyWK-mJ!1(w7jf?&4iyE*P}dG-Nqa{5kw<181_3TyrJtkVh=CPJ>H~I@=Xwp@XE2(sF}eX89ujT zX9i5Kj>pFlhP-TU{>nfq>f9=DxlG>k+wOq9_7nDmT#3=dcXuC*>YO5ZTc1m0#7|j9 zE;23p7ht;{xI6T}y9w58!L2Dx@++k!h=Y`1T0gF{_A`(wdey2i)3jlaI@ONu#Zzs~ z)3C|A=tzHgw~JNgQQw5TT3z_D5kAIs5|l)GXVFgqsESkyHYeLKPW z+u=US9TN6tlM4gI;fsc|j-l(@m?6N25ocN$$k|$hBCG3)A~v8ZV4fWO9R@jUOXq|w z=X45&0z&>z{MLbglpmLqt^1_LfeO~pF?uxEbaFXe7(T2PW06!ASrBw+0|SgjC`UR9 zi`U1W*Lm#MKh)y=#uR=jSa(1<;ildp;wnLl{tQT>wSyS2|#-5q#uV zr5|nR!Nep1Mz2(Za`AY8m1Sr$bs6L_-|(KH5=d~y!rpq@hp!LCMUwTrtxg)~#09d0 zAm-0K?^)0#0K;K1Y(Xs;O95cPAlUJq2HK%%ramll~FNt}0hCGV(v^ovw_IU|lEraNj zu5MuT)3+%!gaZ!z#Z3Uhw^qse(&~O zqx{T;hk9~rtL0csC-IR7Ly5pjR|o0T6#+S*#VUb8C!lvu6sJ+d0Feax!_;lxCUvK- zCC0Wr+pSm2t&TrX-0jDB!{PH=#I8tPLgZP>#n-(G0L{~lIJ7XrD)$&Kp0<}% zm~pGbW|Ii{iv93XcfzENt^Py#NIl+(&p%h(zN_qDYzwkP8Ei=d&q!aTk5a>54qB9+ z&FoomMjqSkNG>!wJX*c&~hdgj!cGOLA9gS z0&?{j*g?NdTMPBZgkhC*rQi2>Kg+B5+?sYZaMDncBs94+a3_PGJyg9;Mn1v5yn;W?lyeyb9~o2U&4nbcjsLA-We89w1J`uol(X- z%i>`;yU{SCw{`;s@(f~v5e=NPQfK|HB_~`hzu%yWk@rY`BpG}7IFA~m32Z?z?83w9 z{eA|UJwXw+dBdxXg}vLTb&Z%S;A_-bG9<$g1=RL?y}$#)EGn-PfA@=_A!_xRl$U;aVw2_Dbfr9-Cz4r3rr}yzZyP(s1OaBw`2<}h z3=mA%Z*=+l8Hp5fOvA>plpIFlH`qBl#aP>ycOyrGWwK$^;aA!3FMoqCH>O!W9R|B&c0x~Dn|UJ0hw3yxL9MvR$R_)GIp^fY4|Hxn&YWx3 zi0l!9;!kRk`7(+|(aVEl7{-AnindqYH%}6U`;)b4>eo*L12#6#nm1pKKlY8Dwf)$& zFZXjK`Dg^jzD0_!GDVr@UyS6lKg!^)O*D4O-0U?m^26!Y+Ps$3_^t4SdPM}!0P(m? zw9Qkift}9TkMccb1-`aoFP?|X(_y}#HaCFlyYjS~IkG*ikhd$Tmy+WuX zFcd{8VN88&DwrSPD+wM6rcbOH=Lucx3_sKwJKb&S5 zyE)FN(NY-%u9{aWDy$dDkn+((s^A!kdnw%~+UhIsbwgLC!&Z0u$#sP)ZLCK9ayQXN z@_MW^s_hCq8{!JfB`p59%>z?gj^%2?g5ncHjTXq^dq#5@jiNY>nMFzO=~zj#1Si@k zpHdJ=1_sWSSUMFMChm_NHcvkKF#BZL(E1&5$iL0(3o#ZmI5NR5;v8@za}>9hL%kZ) z+)OTpD^0YPxa+t`HUr?5yR)oYqw+o}lvad?B5s^P*2>@F`y9S0mha(~2H%S;e@-2- zlo&(2-XzDdb1tXjM<~~8&S$khG z=4u9>?{199*DKqFemtepEOEE#q7O$5}ZItfW4O(1}7IOd3 z#Ml4T8tJbc|2D|_H~LAe7Wy-4XV6jhy)pR5dn(Sklb7hRo0H_Gq3k)?!}z%BQ~2b3bGqxJ0vCFlKl8wW9?xEhTp@m1UE z6341y7VgX8?8C8lP8LF3(LO4g&1-DuRnGm!<8PXlZ9tCnB=paiC=BsrcjCT7ho<5? zJNVzX$?EeV#*!H)7^OU?&0N-y{dIL)$MA9XyWf8>=@|&F{a(0L{9=6Vxjejsj6Tct zbDP(eiS^)H-laLaeueRRkG@@~mfeZAO4uNfFakEtqV`CtPtG@2(x8NmjOQAsMW_XN z_nOzN>u`6d#tK`hg|%(lFa|Tb#Gaow%S-gz;78}Y()O+)ETv)DVcF_i7Og+hp_DCe)hUox{-mV5PNv_hxK+ql18HL5uzefY9Eb ze^FgjVRq?iV)JAFTB`gf`t?dI;0i z_Q+ERs4=l#vnm+ppndFgtOQHWse-ziWb2~*QJ=k_P=4tNekhO3m-%8a?uUCTug>=i z$}m3Wuqv%iBrfLDb=Ur`z-E?xOxe_Q(1JdRtz~lYtIkxXlEb#TbjkLhgV}X2{Q^D- z=zihhdw4?ZG`Dof9aGJ{NyBA+ZASLt*OJ6V7*f`WulH(dzhAj;8K;qJX?6A@>^3jF zMDDsU?v=KsVPkOK1S)={1R1HjOTJt%+@a_AdGw}lHf!n?Q3!2-*bJR$xw6s_Ab!Xs z?whAd?dblGhV+@ae#s}SCPZk~+_gQ|1D)zU9)Rg;_X2hQ_|@D!S+Hwj%TEjC=N~p| z-qi_V-R6>8QYJf5_K=aW=p)YLyGf*EXlUkP(Ger~nR09#eu3~=i=RD710Wn@umK~q z+8m@0yn;+lF4(?_lyHay)W$c_sEXbEh)4Vz^g^?nD|EtKD;G74Cvk~<6ft}awRC9% z!vDZ3)!RB?4!b8K?xwhpvGPC2Uy;1R&ySe;K*!kc$3J3v_7)sEi%{ zzg}JR`^xUwfZf)o+JR|=Awa(D{>~LT4jT{7plE2EUI_?j_CK?~3_`147Zz)w$E`8V zt2e&FU3!z7*+zXIPA!dTJ?h{J2z^#z8@=O5)=r4SC*n%2{=yu^)Vdk8(z0fI23iTX z{5s@3iyLt?>T{n;!}?%VNY)KmF)pMAC8OJ9B(Z|H7D(i2)p zyz8HJ@BrcCQ{3>irq_E5&9GUtVWI^9IMXpyGbUj`$tr9tr9o`s9m6fhRwRC8GTsgA zx8$Tuxyt3DZYp}xr-!o*ryN}8E07#qGi#c$K$t1 z5dfT7_!ycUy?c@K(8RvMUYmIeYUW}94E0(h0PC`LoQG38wGi1@3K%#9YTT7k0`L_! z;#b(2p>jw;VXg@%cHc^$? zaby6f9?GK!5XGR;QfNu9avQ*b00P{(?G23M9Ee<=!Tczj2a8x9PRap)7lQNf1g6f+ zffKC6e9TTqjfb#Q0VZ|o1bE4oecXK#W$P47t1@fX9LES>2c-Cm&`K_P#e0a@=ECuW zM$0fu7dX42J|Z}TX9Gk`Xs)-Kmvsw#z&w*CYRY&`IpR`-SP*5-+gjADOX;$yc=~yeY&Mu zT{Sxze?l+Te5(x_sptr8X%+_Hc4q|oapC<_joJs!|z#ZASH-twae zi~T9|6b8gR&r&U>ErvP?+wD867PsezPHReVh`qQvW%&7|<^>+_DZW#RA;G`spLyEc zYL?@!1DY@Xtk#5%L7xk#NgN_@%NB70tkkPNd(VjO;OKjV;{gN=K*#AkEsh_P55}Pf z?fEMPOZ`4Z|5G;b zs?5kotsFQd8#A4u=D{G~RZ-N027?|&Z~))I4r*Ru9o?84*>LDqYC^dwuWNh6jLO4d>Camk?BOyC5?B@a^U=U{0WhCy;h@JlyFcA@4m0J1n^fVqMO7**L6 z>uFg7jww0;YETL7a9Vs7aae?H3;t|y$Hu>{EHcH^JlFzC>u2mo7Wp`ZpxYr}THM(1 zWykI=<3v8<+VC5y&Rmb?m=i|}iA#PPDeMqc9DCPywr!b5F6Y``;7Q%z!IRjly(KJe z#u`nO?!}04pvjRM7W)_7;#=~ZeRZ(gZB z$$@Oq8LYDm8!{AOogDd$9DK+`bJb0~3;j08J#@BlW6$HQFOo7}Th%V)^Q8lwr332S z2n7Yw7J4qj$%8z?^V!WqR>>{@=JMgdv?Elm^1TD1L2hXs=U{02vuF4pth{sDtErpz zVTCZZgMOF&qFNqw3>7ZWl?^&x-hbdy%=x!Zaeb>2_63`#wL*305Kj?G(Ld4s8y*7= zq$$cN>exsVmZ!Y`$b8I&&g?55sueGJuRc0-OqsR(2v+!tp<^5Ls(aY1i%1Qf#H6Ed58fN?D5x(W1PATC(ffLVC?nJF402{z4K-f zALexJ$eCb8=!WbNSsJ1EGmkQS=~;|puUWd?o2|;fUao3a+hy;jwl8JPF`6}eF2ny$ zE{FT!F-76kgDQ`X!9igZ;stPS%Kb{7u>y3h{ehw+lO9!@*9Qo0d+Z8!cBpSzxjJ;| z_Lmcq*Q&$WAhtON2hwdAZLb)duPt6L%{UQ+ncP0nRPU&%0YuAW9p{}Kn^Y#Fjh@G_ zL^A2x(g)fl=xJ1*CvFa>@5$2ii{*HyOmP#&HE?t(>H5Mo%~m-P*0fO#HLr}@Sqg48 zl^$F!7#f1AByMF8PSDySy@_IeR&*VMGd*sg73R*zLE85-O61j$=xm^2~ecvjXXKd7!)Zz@WJ>N|fFgiX;Tx*Nl8 zY%vn6UiR$r+xJBuyMv^*w#d5Nk-p`)D3#y(>10cNpzXaB!q>(7x<7kv5XHf&6AE(8 zHGJcmy80s}CB@>3xYUN5`O-_pkm#`Ri+qP+JGq+N_dc#|griLHQv*a7;%E}SsEaYx z;K}L4>b?^of&PEC^HfS@GiYmBo&^Ge(LkN_#az_>#Y0;rl$LYoz^RR8z&s!1*oobJ zv)ZvLOxUNuB0Q_c~ z{d((=!j=28K^4vf8#NYyb+|$uC9))FluOW&$fXm=9y|Q5PH0wUn^-5C z1)$!nqkhs|mpuxvJlaF6GfUC?Bc5v&=zNL(gkL(S%!Y=b>AZkhY7EC{9~5bDOcMBT zluhFDGMoC7*d3ij5cu5MMAYBY?xGM)3e(4Ra`c5wTRgpkrckli^T40AckZd9vCKqh z1n#aG%z@aBUS6lc>nUW6lwVjw)??g-?n0+1-<#X6iGYjiccM2OMine6%jHk{5ALrs zyL^+l4gaND-a#i_P|SypAB{hJr~^<%*^$@-zvmks^Y|+9c5a0LP-q7;gzDYl!?gOm z&{FEnQg;LN2L+pWs9!(6dZ&AO*amub4NwJ0?@1$}>7rd<0}qw-XZP&2y@D?;{fwx- za*?GmgNbL5p0O>Xq@{FRN|1@)Iaa!D_YPwR)nbR+O*r>A7HjXuxV@++-F8xNz+>U0 zV!k=hu^L?*;YJ~snN@68M7)f>n95O#Z(96Cf9J*DL}kAxZleFbi{;!(EAj7TI(o?a zn4hR1>ZjfLf3Hj1etwe#?$L6EWFVqt2J;3gN};$>14r?nG+7elyttDG4NIQowLk80 zJ97WV*^8PMsE>Nc_uXKGsWh|x_Z&zK&)U_@QHAOnP&eAvSv?TdW-p^1KIHwMOa5Qn z7X#pM&Qyom7K&n_cJ&i8iR2N-E2Osup(F>NYr&tp4<)rok8)T28l;VPvURsebE zWyhZ+X8*Nq7?E;%4mrO_6Dk@&M0xb;QiRPnQaRV+Z&ztKMZ=8#Tmbn)(|J6iFN)k& z2iDoJBJEHCdEG;{?7Qm!!`_?6L%sI@<0FX(5mA<@$Qn|%vJ90iWobdkR7eQPo-uEf zeTzb+j6#x_$WqAI*Rp5dC0T|U%NSvhZh zzt?5ZI-^<3?-0N>;hnIrk`6$;`G>`H?7tgwihLjXVjXPSs5`*n>^{_3JplE-yA1pV zbFVQjk$zr)8GkS4w2(?%_jg(ct4|ObYQ*}u)5}*7v;&C&8ePx+dpYaxA;XMIhv}@r z06~bY!ja@5eEJGGbg2CHni+nI%F9OAhX+G*fZjBqKTG5+^=Dp8<(x%=sk6w?iJR?) z(_m)JBSjl?iJy9kEaWz3v}ro}EWCbwQH!i$Si}(E*7q<-#_RMu zdBp44E{W)QajRvbV19z0<2cRiFoJH45hIXXy?swmdY?JlW^uG$?GbAFsLtFWp8Mj0 z9=XGR*KYr&K=mh!r5mraJ>oKRe|mcQCK(wSo;G%o3_HjE;FNm~Vwo?7>TMjZM=hN` zyg7Qi_mvwkYI$KT>Kr+140#8=8{#IvbQip^I2xfibYL-V@fpYM8MMhT4vs!dZv*6m zcCR+;1_eB;*wB`HY<62Xw_K6`(pWLOh$YyF;f{?WISvbpX5T_jGL?;dPBKLmZ+A0= zOU25rs6r2yb?ajX#8%BMF>EN&IQ*+a?MAr=7blZw$#$2NWD(H;HoexdA&XG zQ8emeGxvuNw-UwU>)$P$8XUCqa;sBk-vjgDr6p8R_a@o$yJlaK94MWkA*b`mXouu+ zw-x_fG3&@@3?F60m}LBQo^T%A(3gDwmwE*|T>^}z9hLfZ5}fxB;peiM3Jau5^le=t z)Q{dYf|+_cR^1mm8Pk4Tzqpx4?BVm7-E~i$)2Ed>tgVMnf_&Dh()2`K%1imqOi=<8 z8i#H{e3z~nfuO=LZ6ICi1=6(-Uh>M+ywDza4bAc^6Z}o+Pv%ldHP-vK50Q%&fw>gI z+Xl?7W`1=ys?n#nb4>nM1KjHlw@9zC)gpK@9<^MsHj_5CEF=Lv2NJ~nCEZccx_qW? zf)B*~fvLq6Q%-w%ilHL_Hf?|mfv5dX<*~oHZ|_$J2hQV~@J8{Ui}>)Bv1ib92`hC1 z+ull+TA-%%L?LFYPXL~r2k%-__G5zKmK*A!i>X`pr&xwt_$4{my1#oEE?cwmnO+Fc zp}8Ft9!E?e6tn-6;%w||AI@s`h(n(V_jFfB@4=;`1nXL&7-)~k zW>v$&1Y{^U#%(2A8WwjH_)87R9C($JX8tsYrEb%{7mV=MtS==}qx4-=ZGv|#mJy&_ zTxyIj#4?{cC1!a#+;}V~m|Zu!8gKsOf~d|~+^74CEV+w6%|YTwzW`m2vj$7yQ5>P| zdGtP_%$pPfSfq@soG z0G|n9qC5(4+MXa>NJ>3*-E38cqP79?0`U7v$ZhZ&+l`rDE4wcB;~pc)GCkA;L7M#@MgZ0HOIr9S3jXg8Q8H9eG8AK06rBA_A+^3Zp*Pu~s}l<~DyO+l=A_ zYGgdwoK;QkX?H&bC(%MdKvyyR5XAiCa(lKh{jmZYjLB=sZjL}lq@jM zU>l*STm(gf4C2zLcnzYjejMPA)lm3OfXNoE;d8GvTOcj+_NhiXun3H!xccG(br|kP zyr-zr+i+aCtC+AvXnVQPknhdfB2~&(1A)^zo!MGPIauS`LqCLmvEoAZ$l*^@WKb%! zmuO)`5XywyB>S>Pb7>)T5#d>gCI>3DyD2M4CIL-}Qp^&1H?P_c{ zO_H`d-m-edKK#&&hcDzZ)nNDt-=yd2X3jJ|wJAWGC7TgND2iyH zw>@=6-SXtafw@N+sp9e{w`g9zH`%o1>p)zko$G8p$(#0!F1YbK95MilQ++=o5V-T{ zrAh5C$zCuyrYifOcLL_9+_!e3R{{_c(CAytqV$G+0+!|t7?Br%qiC<$@&?LN)k@t5 zW`g@S7-D^)lXf0(9(_xH0KR`U;0uB|kv-kGpGG2Nvk*exN_-{l06C4~cb?v15u&Jk z%dXRbM3yjAJ~nyIVC!B{n1^R{=Ni*@-(pZ0S!AY4z`+ESxQ1BQVuwMyk#LPos1vlu z8hb~m%H+5!&RFMiKc@%zpmOj^FJ^rB*cLK|x&5d4(nouQyS4zQ13-!!K)K|dniS33 zXFJ8N$P`K+ULCv7SvnKLV0(Q#%V6euKP?MqhYf`n8dcX5jG`}F8c+5 z*C5qH2PG;F#c9HOL+6MTZK?__gDz+Rc0`{3Urvl_LzxX`_(#rd#@1~28}}xu9bCJp zXD7PdX*k5a^&8AG(~-U(0tmC&xcJ;V;_iX^awd!id%h5uFR^`!s+ztLnkr_ndprRy z(IvJ!P3aY&Y5d5(6dkHsJ~@ssOt$JBVo%zckGCupNN4+EIaqhN=tL{!RmKJVGwd+* zF6h@!IkeUW2R%M>bcDh=jafe{ga(jwr3(T)3Ke&?QuXBgpw2dIgV7zZdpt)K09AS# zm|Q1qQaR`Qj-rtbXQ#tE7wN|^-#6Jj3wRkh{QlSF!Ll;BX+B~hCpZP;O2*>u2eBrM z1|8Jp$9`6&v$9b{hS1F;{QHwNi>?yHBZdv|n~b*_xVX!vSW>z_$I7V{SaM$cMDuGp zuc>CAN{8;&L9A46u%^_YJ5qMP?erMwyQdk^WAf=)$!)9d+$ZNpPT8K3NezD2=O$IE zaOcu1ZCxV3*%5bM!#pO|VhP!gTrhsw$javq-5tXhhnntMRo_~)ZD7+^b5MW6m3z&sT;yFW$4+hL#y7g~TdL=;WWuHO zE{1NCo`1{%_LN773&C^=A89@E?a}PXE)Vrv9nu4mJ+~(t-u2H?5AAI#NjVoKtd-mU zef8!4FOXSWMkXcmtJ4}Nxdios`Xid4Obed{Wx$?u4CIER5p-JpN&2Rrz*|j*=`}NA zBn1WTo1mHla5n@=;}e4Hbu%ZCb#nM6fKbK~ZRmBopk*r$h9$mWX6m_htvaX^@ADPW zEw>BbfqSt)mf!?&^18Hik8rdIN+oFIG&wo5@(%O$gR^ZlWtLX*RXI&BY7Psl^zGdm z61#Q-9J`};t9*lHQe0`f3u!a2`KKR~=B_c4MS2Q^kkz|Cu7<8qq@$J3blUq_mKwYO z{@IU23gh*9x9Bhjv7^J#74YJicN7_NEPZ<`|HDGLkp9{Vzgdswj~OQi6(?WXe}O@{ zQQJxabR`&QU^@~7*x!|i0;peCFKJx4I?Q=_B)n+*Wvzj2N2w-qJ9{>dnNFIxZ3B>` z-U%PD{YlJm*Zf31{A$Dpvd(|oS*U)wvh^tGKt@qH;bNDKOAR$d)xF z%uxkwa9*;R$!*j26?1NY`+lmq`*GWCsfOwo^=ltpQ$ld}a2q}^c>F8+hZM2U3Tw4@ zDNAE9*^-nK=1$=Xeh;C^`Mw=ruB}43!8+!sCbnFG_3SpqA@Q1inanT`wgyFZxiiHv zGcEJoS_(TVrRH^`d!(AG@9Odc9C`KsJRG_6--;uDr=P}llpiHO9mA4!QQD$yPbp$) z51;M1Byw*FD^$X@g3XI{Tp9`2*bUsZW2@b@UDEJgBjv>^^8i2Ab+x)>Hig~~LXeA0 zJ?Tq3u z1sbQ3X3Tm-XXP z2&<-zIMQ4Q+t_dPJbE=)5d9h|yV**%?O8PHU{pTuesmW~_-^z_HuO2RhpJM!*p?OU ze=(_HIe|a3(=6jC4%6ECy9q(kzrirp?U4_E?a`jfStB=&(E0_(6hS+831CHH%o)I) zx4?MQrCTPzaq-`D0amB(v<}#R-+>zPja7=5Jpf)= z(CtgOnRB31h{6$SGO~t~vKc|YQ%G&~M*PA`Y>xSIXFo`qJ*X~Gv7GTNV8?goFN=T6 z1;!O1us;U_YM7_7>q*~Wt0G9O;u_$!Y<~F-_UI`vLkCfypPg35{|>VJ^_dF)gF4lg zl;6)1zz$dczFV0B!y4sp{H)(k6QKLhe$TE<6YluS!p0x?Kf~HXaKbnSHcq-@ND4oW z)NzY?e11fWn!TP%&LDV?_0%~_wH_zmO9wK=Uh_f!To!b@n{&~n3}u(%crk>e$mWSh zgmwxGy~+HH+IAf!Q08-Vd=k5B%K|)*zZT}dA-GtBO^>Y1a?%5&$6Lt~FPKT!%NnL% zIiy1)&PccZ0d0F%p_~$Y1dn<9O+(%-u&I>WIc{)njom)d5S@#DvqHP{}(K?6yC!Amx4X4%FJe{1T$;4Aa~3FH#DnV-WK6a{ERA%7mJCp(JN-Awku}Eza65;!!}r(E&s~AmY?yNq;p@dh z0oYe<96oivwa{Ll%<=ngE;LIwO**o9m5+dNytWR{Cxo&bkTM^#Xk`snVouFRLQhRG8)=eKum9fvMKKY3MDyUA z7w|*QYWIzy?2k(8J!jqk72ddX>4gdXODW9Ux}rBFZ0#=Jt-K$y`G%Iztsa7E(~~i; zl!sK+7)bWPIL-?(nusVq`ZY-j{)Vsi$#LiC<~1}|*E}-J&L-}8sr)jk8M~j|#Ul9s9Mti<9Qe{~HCk3Ok`83uW4%#eZhj8S*Q`Bf^CvFZ7mY1&s zB~+{RAv(X6S@YK@6L_L>%ZXq$|z1e$(U+PGccFJoXP@H7zB{}vcLPf;4Yg4?t~ zyW}otJU|yji4?uN;2Lkxx*d5U^Fdl~QFl~z-ra`;Ri@Qc&I*p-2Mf&z({}dmFRUvw zFP!Kc9BjB-%k4iEf4uM<=NUa-+6YR6FiP+aR68=Z|H#E1`gxt99UYs)(2<%#P5_2h z&t9tq19%8(Jo_Dg1kAq#R(eeG?{PSdna4N$GV>s^*aG#Ta@E2*0UW+NgFit+!W-}* zpf)Ar7Hvn^aI;mb-(ZFS_*~t>gleRmMK>ck5RRD8*ip>xs-h&)ux*KK;WkH?<>n$* z<-ICHY`BM&{3;_S~w$-3TQLm!^z6-1tGp062qKS8NL5QyA zR-NgD135HjbSL5sb2Xwrc8p4AA>tqDdhXWi*U8&<&DEnciH*`=K6>9dVt?db(etp; zmxln`5?+9qn5rs!`S%>{WqM@a}2bon4rYFfUxtO0&&(d<7gUWWDO=9 z5FkcCKdakmF-LLq`|5shD)%hiD02+c1DK$l!cVP?s62dCX2a~XymASL)h4a-?qt)j zxPZ2qxQzY;^@^pFKu+mqj(gox6RkX^jJGHB)U7wCn{IcktNLQ5dPc(Y+WyYrGkz?I zpKh|()L(armzoAuUIetV6*&sb0lqHtf^ms6ZygGTZjQXE zr{4WLUdRNot8W+}|1OdGhV&Ps{$!E`*^P3Jss?Q)ck>qDik9}bd%mza>;3%VYm;{X z#z#7C!1xh-X|G%`hG#ATcKF11$lmCGLH1((q=BiTcabsB z*ttT=8I&F^p?sh2gwf`v9wlWlmBO5zX8StN6`iiPS=lP(A||Ohc;_+d)yzmY-=9G3 z5n<5(LhUs^`Z+HRT+~i1C@;Kn!h2`OcI6tf$QzMOQH+v246CkGaMYd13_x7V#@@l~ zC>I<~exkHmz1sfKRkq#T#v@7k%fR3saoZP0MkQuWd!;IRrgW>IH7PVZsIJq&HaA1Q zXY+>-H`iKmio;a#f4~Z1kXPe(^zW1*CTWpp9VQuFccfM$Z-r)mlJF(cA3Iy@*6UE> zTu|)oedx8`PkF~i3?@8zBp!w(BYY&~w2vR@0$K3|dDdhxx3KwDFWK+()wSG0NuD*k zB`gn6H%5RiBR@hao+zL4rZ%4}B(7%ykH!o;Q8`FfRb)D7XnSt5R zI+qox9UB!o=rhhofL^l#2+gxE>4})%<(L{XPwe^5AeAPw2x&v;i3}L?@1T#Kv?s!t z1c)D3+)7PB0Ma_Y9e6UkwK0Ft*uZTvbvDfCzvGFsqr%T{L9wh*LuVS)Lo237ThK{N zF!_o)c>psyEl$x{8NtJe-VTV%Li>!uMmY8sccR6n3Df|5h|n#Gq{*VU;;h9|x3_=r znXtX18xp%B4qM@=XXLMQAIMAP98J2|Rv5c&)~p;(WXh;Z&YJFgpEH!x<;+{dUE@`v zFA&gFvBLaZdO4c}4%$HhGb3-V=I~X=hP`QhfjZokdjzJC|Ml#tUZ-SFpa zfa8}9`Z&E1s?d(#bw0%J8w>#j9J8t}WQ*VWisZld!ILtdBjTlPL6)-|%k zvcnfUvDa?ZhbShGaLkgtcY9@I8j^~=yiE!YS1RZ_rN_PA7R9+4ByX%{QhDL$J3U7R z_-Y;vM|-l9Fi`qeWu;=JxqNLk?&&s#n<7u8zBXx@3Ub9}eTIImV<4&(?-;zk^x0Rf zz`WCMXMe;&9{W_r09CUAU~lj%&MJ&vr~6-+qyL><@HdvhKl%Et>)#I|W_CPi;$U`J zkr9)!a@nbIgq%EDrki?4djcKt7A6pp4-C~!E2dp)eKE;NNrE|lh+#SLOa+cXOx+_2& zojVq@xBe7;<=i(|Y0Rg^thKtCVWRbWL?p^6oBZ+TIDy-G-%xPd&GUI}TOc zKGdR1k$KOY*^Ep};mW7%yzLU*tTBB!c0m|%%B&gMf|3yr)!5-9U4<8Jet(SdrB(6q zFEg^X#{TUrCSPGfO#ZRkoCF=iV63Sy5HUX03crdyG``JLYSrqaoE{~65%G*{!?a>m z6pfIkt;aI*%t&DG>aBy*pk%a>dkQX|juuKYKc|0CLQ zY%`K9rN>;M!LkdKH=PDYg}=exS>IesO;XO_U?|UA}OvrZ@Zhv5EeK3CX@_y?(9wM8)u|sF1?-Fsh z)zl|iIjzXZn(hrZ|BRCdVOfl!Q2+*e3AEhs>XGevR*DgH#|;<^d-yXM6XI~y3Nv7tDij*H5gVJ_LX_u zd|BbSi+>t9&}Zlz{NXil&J%qsS^%nweVCx}*Pm{z2uZ#CxK&%0`D}fmb*INifN41w zL`(G>ln(TnvN>Jbz_f;VZfzi1xlIW+hRInbeR365lEuwj3Llk?yA|1-#r2Eg7fFlf zf`*5AheY!cLnbj#j)(_HuTV&CKZ9Gh$gwrYqye~98@|8JCNCT}8Kv=-Emz<62ASf*wgc4ZSLCxAaIWhmV5kxO0G6@Whqpz%icz|&qr=HB z@?3rZce0+4^ox6PZ*K}fBz}XDC0*x^e1oxy)E+`zCSL}dteun^W%#t|-A!jt;O@p> zshD<>-&}J`#7>3b@R}A+e(j;tj0!h(j%RN=fB*JfEx{m;gZ{qQg&lRnT!i}AGPx`t z@jBC;>(3)|EcNYu?QcF!OO!l_*aTQgO+_6CHL;%=NjXIz@EPiQ8ON4`|W>^12boDkB6&U<2j&K?|5QkRxxrp9QJzZX31Lw3vkALxET_1|F z+Nxuud@yQjC~+GCp@cB+>LIv};`qpEW2+GbI6)t2<79(w57n^wIjl^hS&Vu4^eWTM zT}Dl3_XMe6XO~Elv>d83#SA4IdNHyopXe1MQa}`r$4C~xtJ_SJ{5)V3mAT4cya*g3 z*CPz&2c~0S^e0(1M_Sh^=Mk)E=1D*g)TQ#wl03UKSn2JLaFN$#lzc45w{<#MTyNsM zpA^5{6~F4S?M z(xjalueiuAUfNSYvwDLI#5rJtS8?oWu3XXk_l;MZ1e@WHdd}H!4W2N!2{>5-7%tpp zp74edyf{UUdY<+`?aDAWIsAFafL@EokoAy$=ZXaHTOZpx-T>)x@#;Fa6xKH8t7+@y7Wf|I(OnX+CvvXKT%^g59?=@5}5ff3jyQt)2fyvVUF#j9v~t z{y^lY{oKcA*6i*VihI;92pW-$wmB4U-M@U#MnYC8k#Ybr?8`jF?3gEDv}jK(o)C{l zt5fy5H6+ONiPle^;Re1AaZDK{pS4{#aeHaA5>n&+;`I-po?K{D5&yEf&|e-#dkh4b zoFIXIBx)Z4Q=OyuIY>E{qHFe$rL&-yf3#8UZNDKy-|or5*d!ptN`popBHv_8)x-Ob z>k&CJ9D!jbta(dmR+E{$`Um!srrxQLrK`p*kH08h(uORvXX%jC8inLQdxB!89HJh| zC%cWlC~K80Z-3!41@hp){+je#0la+V?F`6c#*CJ~hx7e2t@cJ?DiWNZl%da! z<$O?2@B;m@Y&QAp@;ZqB0wX0pxAXhg-h^^j~kUft~azMx-jb6aEHKjp>gZ z`_vkM@^zg&#n_zT*!GNuJyo{nig|dYlvkkWiQqUL4i;17v#f&v`q~ct47^<gYQHXRHi0E5O$y$ASl0 zIGuE7n*o=~e#17=u)FyZm6fc`)L?o$;teqA>%)x8j8x;o60NP`zFb5YM=1j>?J+%l z{0NNq`lYODkxj3FMvfEj{cExH6l$Lqp*-$(9Er`!Uo9gyA^>q0B1_3pF-A45xq!|2 zT>WSnjwDQ6y`#a0?xMqptFfM&J3QEh7iSCx`nZPnUO14u&+6qCmkSl6W-p+B6lD_t zxOb(V&Z!MXNQiLsN$W=u~kzz`M@y-DlQ9MaTqoF9rc6aPR=GE4DV-Gz;2xx4?r+NqxrZu zIiK1I3wZId0cT==T?(AYf1))>cw&D-IZe2z@WX3+XY>71sxA7L&$|`NcyC->O}KKDMjz`qZC`grTY60AIQ~$dtAQB z#w!>*DNP!ny#|LZE7XZlbD#7Ms>su=&QwIYs@-_ZvC?__x^shuyxy1YzI>{ijdB7p z$u9=q4=r~LSsAQOK5Q<9dtJYG(Y|^i)LNOGHSQm?qn z*LJg|MhYtvuodJ{R^V>a#eu@ztQyCP5(~z9HfMyV@3~)8WK!Z6T7M!$QD!0F{Vc_u zU>59Qw8_UnhoBoWs?={5T0l81TPH%@*?lA3g!3k^j!_OuhQdUz-mN_MbWDF`go}nb zOAf~#%g!39U(&W?sP>QO3xk)Me$in-b+hU*7|VUVX|u~DaJd#H?X@2J1qk;60s)qV z#)%mu3D$vFwlrJ0`MZ++E{3(xUK*n6cL~(fbY5CBx)r{t2^0^#Pg4O+dNU8h+_??T z#lKX12&Je{zZW0&)z9@5ji?uP{mh(n+h9%_{f#+kh5ZSB+-yS^|49wsyYR;_(n;kS zB#ge7-hwD-4WSEA_46o-_~I~+;-V4&+4?*7ToZZwz1~s(HREiRBqB=onaD0CYavw$D0Gu$6x69m zCty29fPLgk0EPh`pt7}S@wHsF3$I-A2t)VAiidcctqv_wvhCIe)OHj?5x7`&C(}=; zH`QN{4=pC3__zX2INwh;u4HEV2K%HHg(_{;;az(|VWT(Q!r;#nV+J%!FI*y^ z!$iu7LZ9tjWQt)jCgcH`#wOX}^WMsi^lAllkyC92kurQbg&{3fRl}-IdV7rHv|U-} z!nrgVj)Ca7gA$DT%B^Dm&BMw`u?!rI+sGt%nV8uThus=9;Av%6j!@+N1X{W z&f_-b_V!zD?d)yagd0K}M}7nbF7_PEf5xEF#MQ+4p(4-o))vH@O1>>v(7-XW{P_ssLA$Il=LR$pU_ThGv*iUP7M*% zk)PR}i;PA5o?o9?%j9ka?9SAm*qvNsX*B%`b?u*_R>0`o_cNn&A+(TkWEpQ@_yq4V zH=lORKImFZx4y`wUHn`@oF^#&ou~wd!Z(CsIu1D540$3;-~?khS)3G0(VON*>5$!- z%9Dx8Mz=q2uu9uc09NUaV9Nxpu&Zir`Nu@6b;)4Ew3*)K32j3UBRQz**uK_GM+>i< z7yUzhLpp7(o7>GVN{enFqAZmgk-nE~{$%LOFGbqWw@?3elyU==*d)m(!b`;BCAxv) zp`70*Wd0-RiK#&xYv#5bKa!7CDn8F!Sx6pff5{YJzLt(V@tcXd;!R%RTo;eq+z#37 zW;uyvKYpnC3x5QP-ehL5ggm^>R_aq#SM}M1cL}jmA|Hc!&Ac^_!VWSU!i2Ka$k6+B zvjrq|U%99_n@~!71+5ECJq;&B_$1MNFMNZIJb=iL=<%@~yPHPNg2ef5b{Xe+l+~GjMFamMs6wqMiQr>m z%$KL9``k30YG zYb7J<4kRg5UA9a?zIVhBLF&etBkT~t!(wD*f{g)zIWpzbinKsvpQzH!UHb;q7WM^P zFkaS3SoZwM>H&cF3}c9b;_3w+pq<@6j)eIr=A1R{7AcHi^cmR0Ap11_bd5B8qYV}E-I z=XdSLy`X7+w8KwBcbw*d7q{Ri*N^@zVE-qj z<=;cu|Np+`@O(Tyy&~h13QpKZpc=`grtCR}B?R7FiYCnLD|so$qT$xd@ZehpsMLH<7}7N{93MDeK|8>p~i^AifmRx%67N+ zD4Plm4_LE}F3HATkw>Qa_>FT&hY(HCBw-{EbpA{1y>T>O=EIJuphE59HFzuiVW#8m zPqgPr-}7nx_8oV1-8AoOi{^#lJL7#e;gC3ccpuJ)d^!^wMm@2wl;}KX9&dG3l|Pwo zUj4mLvb?E)&V>Fjyo;=;*!@PPq1&^uxt^EWu)XrS(bnBD40Xy}tFNVJHvd#hF&l>> z@>}GR>`uQzn7v6#il@>z;dkDu&bZF{)KXd^pRv><(VxU%)0!}k3~Y64`=;O9>HN`a z4~8U{A27d2GgY}97>4mGaYWR&X0R$wlG_n1Zwgu&#Dy6N+?inoKR{PmH#}iY9pXy)W%w> z2NQw;hTj9;37jB$3rdOhREUP0buTO5V@h`m_N8M)j z6l=4Q34aMr7Ddjvl}00K7^kxveDGAE>>e}=y#m1iHg!T;M7A+^r+##nP+51rXn%UV zrIqD}xAGskjPoU&a~Vbt-R@$gM1t%DnC*1|75d|@`}a3KM+3R1qMV%^IyQ@8^_e>H zx~IUY0!5KIHYB^phK7$h$ZsFQZ>KJeO|MYbErg*FD6r2DQbf0=$d-Vu;aGWTjt{r5 zucce@`OdrRug>0o3pXmTc`lDshN@#$utnFAQ9DCg>3&d^2Hl>vPC+9U_e00bh3{YT zOy#V{1dkAOuzjnQt6u9#H93$avFr6{GBW98XF393&k#Q)9AL#T`-o;rP!1>)APiI& z(YXjaf$uAA_lGNN(k$F8=T@(ao4U-QZ3cTTM{L9uYraflg$Z}fNThB-oLUEvyf`V0 z7OBBP?>gENheeqcb!C>c;ced(D4FG4e`d#@dm2-8U8R zs8RnByaTg3?vB|Af(5;ke0CID{T@CR8kqJ-hD|GaJm2!+sg65|x)0`WZb3V+-_l`% z*?@EM&r*j?mv*nUl9FkU&_d`gI7cRdGX|e{B+%F6y{RbtdZ$YS@qkR%?E>bgC%LeO zH8Z$Q$97KRqaZ73+k5Vs>09Sh9w0wZ_LmvE)G7Nq?(kw}RB2H)je5*MUI?TM(&UXm zBuK3c41J600{X*&-ij#TtYvo^pNe%+)7|+lmzIP#$V67NzoiHjn8x9Ci|)n_X}+7v z@in3zDJAKHrKXi1EHxd#ec|rx3JR9SdQd&<8Y4)p5*(rcR()*W9jTNxHS%(Iaul%0V3?~J$Dc6wo>H5jL85+&%HhNgem$~OUhx8AGr`p7a?C+ zsAYdib~P^UdR068bn|Q9?)hEH&-&B2;(8-!GS33`McupWR-b8erd=Dh_x$A$SVYd7Fm?WnG?*ZYjQCQ zjm*WF%ew-*6M+k^8Y4gn49PYbL)vack@20_aUQE3&KVc9)1~3_ZyHu}Kxz6--g_G2 zJem!(^%lq;&%11N2hE#Lpq_o1^Rebqv#JEq`UJqxx&J21i7IauOeA*nFPenX)K<{e5jym$RLdIiAwqHP1g(v=X z<)*NSsGXu?yqdzjym^5-%oZ?8$V^?mn+}8`8yp$c6VC6T&!@%P^W9WXn-l#h8#6$jSXU2QW^=pV={V1?+ z`4WoP0`2yt19HTLbE7ngu{F`|MiKlXZ#tK=Ew70d<_8{2-1?R_E$Di+&ZvoVh<7O+ zPB?+NllM{JB2|~j6DP@(Z~)I05tGWz&zW@wIL{Auv@zeIw~o9iu3G5BR?LEk(SE;y z3uTyVB_;2mql=59QM<&G7~Se&5-e%Ko8ALG94x(7hKFGXu{bkzeHYVAkD zQ)J7CQ1Hq{WaXimLu#wL(T`=mbk@AOc}iX{@M-J*`|^g#$9iVf#Xc?bFpxtq;DZG> zKEZK&660d7JEPCe_AIVe;*r>IFgB`Wwhv#$QTT1okGH03R!h_$Su|!_Kg;;oQJg6@VD zBFq^wF!`klYXmDefIk8EwbGK%GUQcDa@+(PyTi&8^fXpZc;rmHO1~*g=mi6-3bxIP zvnh%bcpW!@a!}Aodi}cpN+GT23D&nLKfSTk^qxT45o;y)20^pw^h<%UvybBM!j8KA z^c?#Y=6Ao$lOl%FAR_C*JJUNXP&T?m?_gO`r#}4nX2))VW*%WLM@r zdyuaVa+>HD8W~G4;sUNTP z*R<{SPIdK{Cr`1a81H_2`LSo_&kUQ?%YT97@UN6J{_lQm=5IZJ&8+$ZM7lppy!&zg zA3weA7lOhk+J?)t?xhTL;qR(hQDeY{CM3zKMsD> zhyHxq{TO#Y+HPYE{HPE87zaO|1^?O@_)#DF*V^yL`1{d*Kk7q2#=wu~z`r&Ie$mtZV-`Bg=hg8IXff=tzD~;GD)Dlzyyt?OLvXo%t(0n(p2`C|v0DGWH`Bjv$=p>`NQ7j|IB$5V}|)@wnr# z0Pbz4EBkZI=DWV67LzK~i;^yn1v#q%5C#(1WZ*Fx+kbm36mK+k(WrK0+x~Kj(mjaw z#M?EiDsk1Vc~Uem%0VuR zcFrhzxM}d>$0N~`4Lb4ilP~MM1qmnDBVg-N5!5nZ>&u%>vLa$4JRs(;q;e^~^)FAI zGc6u2e-(-RSQ>Zb^at4%p2ORS$QNv-qJm!EU@CY`&VR)7oi1_s&?FeT%<6#WPd&bx z;7&5`QE=s?8Wndxf7j!@zpU`+`E+jykzmrnn@h(FruaeD?pNGZx&EawB;L*jI|FAM z{WKO@9dA+XP0Mx`%9HUDm7H92Hz*BD1`#i*oLMjaX20+}kMSN-j@|(k76W}`pUufm zPk8uhj|LhmAJJ^sDH1bIw?*Pq!aql`7>`{DpV8!DSmEVe%OuRwSz{>(lm#(2iLg3G z51+Vr*7DQ*P5~z$k8JlC4g0H<3v9b#7kXp(#9w1O!j@vM!;Z9`gvI|sM}Td}?N0sT zAs$&PIIbC`62s){?@|^?w#7kvY9!=D63*}1<8CDzODr872~(8zhiid^-WBQD;zd0E z`bx5D_#`x+?jU(pqr$8ONjM{mxZ)atAaUkz1UBVB3YgajL=mDY0Q!0vxRYt#g3()& zdZ-beikgfOo`3gW|D~bW|Lp%J?*r4I;7mR2gt$mk@+F_FDkr8)ULO~=KL-m9hPVm zI!uaeFlU?#%~XHExRMeZ+aMriE>k+L?nS-i;0VJ#IJOyQar5~t>TSl%I1`hnExMU& zR26(g{C7JR5%&W7*Hr0-c?}IJ`+)wOOyD*WgyceSAaNYIew^Kp_m6ki+0#0P2M0 z7Su8Dg38>GQ{><`SR3XgG_Cyv9Sc=D;y!nyWWK>{=pgz0`Fm5G3N#u$vgpl+A&Jjm z)=lg+;azQ`qi3^JJKBCJXJr)Jq!jmv4hOM1Y|~{%l#^5=+#8+v(?)guOO?B+lV$j0 zt}=XtJq%yvkpooBji|u-GlvjTbSVv{p?-2Usfi5vdu4e#yh5FB&qzGNqi|N+JG<(_ z1x}_5RzWP4g2v3f7=oTrG(-Se{cDuikdh~ju_wuqnF5O2224C<)UEw=mK$SR!u>6@ zd1)tQA+1_y2b{O;2o~LpD-<5ks_ZZ zGHR1^fcXfL8>LnWj3Brj68ob>R3o;w$w$YoPFzpBb?6an`IiE3k-KqGUQ@g3weZMh zurM!7gi;R}^^5h4c_q2y^vLeEGU<%>kLHl|bHnDmbU&ZaCFHAMr{%^+YAywV zpicKR?9|xRF%QD6?mj>6y2~~k>QAVM)?IzTN*RDHIH|QhpG3 z+XG*g8-}LO`!Q3uFVMB2NC;SNJ%zqzH!X6KWk#q88xL2xVPI}*j75TRCa)_>U z+_D^v*H;r>3fv80JoPOaR*Xj2Lu4G$sbYk&9PWtYrtfoPG;H?>TWzTHZdM2l8x3Zb zI}jawo~jSyhN-}TGkF|=Z?2lqv^uUFoqni+;r7XnXE%y1p3F+hSMpCe5=->u!`k&L z$BY(t5p~C1YhC+s6QU_%Jsr;OwZfwBBI=Dgof#Ewh$s#CNvq_0)szy_-W{k(+#hR6 zIRj!1w}Uhp*N7#bQtOy&uO_a(A&>&ueA^wm^YdTU3-h8HC--j4cODWMWq*Nc~qwn@lUW(@_j5%wsX? zZ!6%}@;w7#mC6%IKb1TlOY^w^O;>f`9wB=L=td%Ay0j=0=xZ@{+6h>r zgQwGJqAtDc0B(8A0r=L=0RuDfI6@k|P6D@V*~U!rIC`do12_%+llS_kKRW|*yF-B; z|3n&{Rpl_n8D&DkhX@9kHF;zRv<3Nbexnf{qf(%J9u|$TKjGxqPYR$Af z6#Fvt6fi_RhI4x@34&=Wi3?ON_ zEZ7xacv0kBu*>xY=lwgh)MPbGbp^R%IG;A)kxlkg9&wUzGU+_g9~(~Pni@u_5_c4M zai6NY;4x76xvzzG690u+?e#WJ3H^0T`c?+YnHjV&@zuy1SID<67VaSfqPj{NrKszdHs42i_z9AA4^e4rSl}506xeG8IB%DvA=4 zHQOjzN|822Ohws3k}b?>30WtEB3nphnPkm2_9fMbvSuB!XNECOX6Eeq=(>K_@4D~r zeH{1w9LMv0j_3Iu-#@N$%r(t9KIi9re_rp`>-B!W)m&KR&}t4$XOpnDLys^bc&sPs zWY?1VE0)5D#cihE7rL`jb?bU}1)(iXN8E*9$~)WCs@6?#OEWje1yD4(B)oZlp^oD| z6Xr5N876z82s{fhmJ`8_^p+Mh;`LP97~P@ zi^er_eci)0b1KRO&Qx!U*h@*;hF)@%ln!r|NZHC-kwc$DD{t@P{XR!gZ6Rj1kGq1F z=$w;(kp^0x%`lWAJ%YwSwFZziIXFeB3oi!eG@6ZjyF@F=+rHXMa*7rQl#mE^u~$$;-b`AO-Gp`v&RYs282V*%1mY zbF7^m$vCnci!Tr{wi)qYLu(+WeA*9N5Zw;Wz<__8D2Lqx4Hx4U9JF?Va0H+|_~({6 zxaU}^-u#MsIGM?}WXzT-Bz~EPWlUL#4)cFfV?|#!gfo5&5K#WEgkp?!1gQpKq za;_%L8=e~c#|s>Tc1@Fdw9ws3AHhjyV^`|oWt7npG(`A7iXeh!l0VqF$aeihTqYDz z;ec|d`u!honTu}hP~q$sCe9EU$=`TGnJTP{-`f2ccy8Ky+G6Ph!T5O^`A*3OND4zW zo;l`>y+aJz)}OxLtw0{98z5{!7~bKh1-@m#C!hVetvPDy@ax0@W)0{gXZQB9^HaMAuf2|Bl!Z;* z8tB=)qdEe7YLbr-Mv!Dp-cG)SPI?3H?`+(8OYd4%fZ6~te`{{cbB|lN<2dp>i!X_C z72eOh3>%)SettY-txwj7@HfNVlCiIk9_BUHz4G(2pjGdsSDef(qG#1IB2cR9@6XU* z|FOYe7Syca)^EH0tR%CIjifkfHVBUbt<2M51Q(*U(ascQ3D+$%jnAL-Ick@9^>XAj z-cAbvH!_Q=v0yBz=7TV6*h9EjzG!OLonC zuScyG-+O<@B}ctRw<-(wIjGd${($x4>ps*5tB@2IAyHHsdc2d-Tbk0(D1(dn+Y0&{ z%b)vUezwWq!amwZB0_$<(z=lKKB`^*EnhYEr(1B&DMiS!9ag}2^nl>2Td|DK@M|Xw zo_#}dD#G_gpRfz;M4Xmm9Qu}Ul^s*PeIR3gJB&3bo{A7rw=NZ@%8&YKrTv{md|B$e z*q40PX=tJt1*xV|%}FbVa9g2S9C%@$c4hFZfc>pH1JjeBOiT{%5=~eRSg0inmBCm3DXGdbZ!$SwG zf`x9LCrd;%9C{(uDlyDkT_ZGjgk}?>v3cZ^PLr7I*Bc%89rq1_sd4Z7@3`n}Yn4RB zPNGCY>t0Ft<5~sh+L-p9A&dezcw_3s)F6t=$}i`QpDP~I+`3eA?Y5Wph{B`T@e9eH z2f_E>Ak40C#@eRxwNgnVjo-icD78BoSqmG_^|zeD*u@6jxbt4@q&{)AjzfxX01S^B zdbA zc}4Sm;*`rN`?`N#JUbG(YwmPgw%$tP*tEV1Z)LqOW*@7W_b#c-C4#TnD0PChc;+GR z-HjX9v9Z};#CfNHD(Ze3Cuoei7BOEcdo~L!paGfNAI@-PC$ny0>VKia9=o=2b%V1G#2sEV z`(&#N(k}(6Q_&(El146;1Ak7TydWYDdJ}S^dZoxWe|HI^{u)bNr{ zj@fXdWnNpfaV@J9IFI}k)%xJTj6b-w$no&QgZH*?G!pRA;H@KR6%pV=DzlFH_6)NF z$)i=riE=-!iUSlfh7J{V9(|Ed;T_DI2}7~By+}u=mU~f9wEl=y{Ju)csA(fPn8a1I zlcv%zzhi1pq@ymm5NA{%ys5d3+qzX@^|0-!H1ydQBvE89RC^n;A@g2qwE-gy>Ie7H z!uQs^mv4M|wRNiCrHsqcfKwTT=)8@9@DMnj>&w-KTv3^mt(IyFTIyxTA7iwnp@$ww zFC>lVvjk=sH)wNT&1@nlT^gMCW(7ZgO*7Bg`tdGTs=alzeopHaTlKfZ&LLiyaGZ8K zb|aYBtk-$D)9I^*V7x+BW91wD4H6M?;#1lpPu4PF@Sl$+*qr-*&;A04(sCzm^==XI zM*?Fbc&P<80fY@dZdGPx)eJo3(NJrw4|s{lBga-XGhH-fazd?RJ|HbB2$N&)q;s&7 zLADkM+6?lWSSr!lD9!cGz5TTXp3KpdblIegQ{KEOg*uKhonDL5?_^uzlA?ttcHi{V zj1PnRhfC*1n94p7tq4ME33>)Bnskkq#7}{yi%xk7ZGP(}dI2v=j#_W*2%H5#`V>tQ zefGk{VG=l|&~$Pr_)~U_L}k2l(rR`&eAK*<9q%QR$^IUen|6wiSJxvkD*aYgpP}z7 zF!tBMS;SCr@rVH>a^uzHP>zedxd!X}Gdr^&J z+-Gg`(VTOx8uF^kk6JXnO`BH}Sb~CvVN;#@u%?UK9j&nhOL0RR>=jDqOHQh*Hl~Vh zI_lvMqGbtYlaj;JG}wPw2C9E8`JvOUTX{9tB$lP`(KFVE?bUPEC4R*QrWRpqNrY9M z8Qf-#3s$3>ZuB|b@K02)aO~}$F#9N87Vh3zdjEzIZ-A%VBSkw~ezNqEE&PRBAeeQ!Pdd}1Z^BbCTpW`&U zV(zNwzqv|Gwgv56d8f|lHMm7Sd$2CqWV6F|p^?YOEgL~#y?nVe=c?^4&7bdI9}6QJmBvqx zd@~dX+aNn(zi>ArEp4lr^x_LL!kx+#!VT71IgSqKd@g%-JwkLBe9&XB_V*k85zC+O zp-uOy>@3{ z@zlk;(m!yZMY9Q}*M@>2C_=%5>u54_3{t)U?P~3TIFCC~Uz$2nQw#zKap(sqW_;L9 z8_~bi3@gzpU+HGN3=0oqsp1AmKb$w3hpq3L&1~G-ko5CMf5aolj3LFcmd_1)tZjtT z(6?TYgnYcfrg;O4ytvlpFpS;^2R4jq1eLw_RLt>0JHBV0tT1MpycUZadkfXx+h->@ zhuQE!4^K1Z54qzl#{dCN4aJ6EkR*^@Q0>h%C058nCmW_1bysdzptybdRz7qkT{9v+ zVv+K_-=9wGgg9B+RO5!^bwiDo9NoDZx|!F|1A}1fq|5f^bMAb_3F7yjANjcfC12+MLnUYovscQ4U%^otLqExb$xf_%7Xss6%P>M`NbKQa4&@|IYOu> zZnv}-+uIdcQ&bvzy0S!%sVL`i!179*-o8-6h&IN_|8S3`RWlXT~V$VdC@El;kXsir9QVrJ2bHvRUY|NWUGauuwLY zI&L`+_Spqv(KXSx9s`n{;TNgzNOf|e4p?3dw^75HtsW<$_g4Ae5ATf={bDWTIdV(f zfq2HW3rtcZg;Yzx3VLAUKFoH=6KvRdZh9x5n&KU*QTmf7cZEG?Z8_h*C|DYB)~EhR zaLM-!K>7_k4?k{m_3}Du-d`&;-EnH~#YUkhQAN0W z)VAb9iS)kbcT`m5P|!JQ=4--e8v1LPG*x*KoU%5eZ8C(~*VcZ%O-T)HeA|%@*@Oc_ zX;zjw)MbR{ELEpL(8P?TLv<&OXpUdXca#XZTuhLRftecp`pj(=V)XzKB)_Q48G)Ld zgMOiQRccjB!BVSLh|EZ`zE{I_`wjRuacI-R&xgmXmd}-HpM`KPTXRZ<8lf#i!ib0imQ(8p zoIMhsEuJ?KRBV@Lm3%Mquu1idSNjr9c}L8_xkRJ^Yn>YLmgUP%@e!n{w#3-)3V#0B zK3)Fd*V$hvm;1_u@-43{^p}NgIVY!oH#vSaq$)JiKNN;qz<_}VFO#VH!`?aNrio+aO2G|)P&t#Ej=|T`qoRX@)X6_@utEtS?h|g zFF(e#7^@$@xxm-6!(rJ9;nAVh-k;wD>kNw#Ttk;ylNxq?w1=hb9G~V=Tn27EX<-_3 zx4)(SF z-=Nl9*c@qQ18j8mK_31t&=EvPV?X7vv&P@B^cSB=`lSDW^+`idJ6Fj_KKREgtZ*_` zzs2gZqpGDXDrR?hK5#Gsla>k1hS#yjP{bb;WOJ-_!ceJTy_CfM6Ky$XYsjm~`xuv2 z*L~ok;M%!Z0^*m{Y-V+69NWrUAFL|9;Rkxvr9yYEo)J8swJgMok8tw!=l zP?m$jf|^sIXBmJoSHXnl{3T5_@|BLC^eVjpr#!hE@qMG4Q3n;zAJo(13dqllxc`In zh|tG30!a88Qk0fFjJs!H9OC9RQXF08b*E>g#MK5#>@RX&QexL{uTlMsK0eh!N;ZaN zxN2R~VM^_g*!`!dhEpf}}GJdA$Kpjpe9n_b%N<0;`zT-s8c3}*%w{*a2n+F&>M z7$)9Ix=_{G(QPV@1jJ(}yY(@35-|$<2tSW@cPz11pjrV$na5YBDYfwDYcRFF2*+pp z>Jwp5Z^z&Y|ATj4ylGY(ySwT9e*T|OU>Hu?x@t{!MYXD24W>7?aqvh{Di~72WA(x| z%iKEf+}Gow5@%eKJeSGMwgouL4|l7R7pDl>3&bDu2#*S_wo;d-Qjc9n4-e?F++K)u zf7@Kd{Wm#vh(yd zC?_wMWYnhCw>ILN!InPYrcYvrYbe=%Y_LmwS!~a_J7~a50ygQvGqD%6FI-#1f`^DoyCcv3$>?#SWOy2zLIMw<5eVXW3EnKLZ!&UhMNzPI~Is{KBe{b2e zxJkV21pkesaTdf5MK*fMF&?thE3s5c^XgYMsl0UW;GLJ>u>visO0S>M-kw^}aBn|8YN$d~ zf`_eVW^UGZq9oo$;z<9ShUljB^GfHG-}u*X+(qr?@Kd{N6AWfPc$(`JP`g9!q`&Q; zzuH$=SmH0RFKpAsuP04vxHroCx8!~2{%hca$am9)R`7d1rZ7t1ae@G~37-JwN(;uZ zjXVTc@1R<1Xzb@Fv~9%{IDQ8g^>U!eegFYeQu4Dj>8eZt(ipDUs^MX(2Rje zBm_yMzhUG5$PAy>SY_Y2FqH>pr-T{zUqFodN0-@AY9!=|CUk-HR)~|K3>yucl+qJM7b*00$PguZ0^z^&8 zA0B(|A)GO#Mh5`jkWIDrEj8oJ{o%F_VGuS`3R@`>=LX1@@ zu3S!z`|Z9#1wZlg5>KgF+FpDs9{}AGh_xs8&yP9|<~fxoL@I?u<9pv&9s%wFic6g^ z%G}{f)6r<%fSOz4R*7^7U`e#r0q~HVhU}}X%(|Uk78X=KE0zH}dY^5P?_P}c&UUAo z!+hK7s#@%f?tWvNu=c~3hA{!i=C`m1wf|J0{+FVni&t)PevCPh;nP`$YSnnUVIWj8 zst3hMa~>hGwo9I7t~bRBR*PRd<=%6&7k2t`USf&I!(FJ1j=g>6ftdXCE}(L6H)cTb z?ad>pyWpm#nH&B47kexlM8q)_hU#Ys&PTjWXBiuW#GY`l(vc1v)zFxL;V7CR3VPo3 z)Y{`|*1GHJ&!U#Zp>xHNXgQg1g&k=g!KdWEOEr4R%_cLgd@iz|R9XP7KUnU-*Gf5! zHa2mjBGkC&Cj~k1Xxn9;=l5cy2IBhq6UMQSInyvXcG;iDs$v@b@Da~_pKZPWwQ4O@_ff%CxIQ_+f0Xv(^3 z!ByqrP4PRMdiULEc>Lj+`uipEBVWT^3db zp-&ydFBV?1>)PAy#i%JHtX|i2Sk($Gbz*2T@1j>&z=;RokQOii{5ythi7f-PC(>AJ zHQ>y;gA$PEGSVwqY=6?&f)+p2(rE(pja=!yHCbIcN|r)bkw)zk?}mp)Bv^HbODf0 z>mPx)H+4d@oH@WLSxlUsgI*)|QLAe>&Ve`8#OP(pkQoZ`!VZ@~3rao`m?d;KmhPB* zA4}E2uH>aIAA){bzMqF>1zC~MW@r|Z4^8%;o`i;jp+%56|GWrbPZbJ}tnmb${e7f3 z0(@8?tQYAX^sz}I^GvC%cR!z}q1xObkNbA4S(rG^1o+N*J+2rVay!($xY@60YBz0B zg%S1AFBRtIprg)lO>xcg&cUq;br{eQ$fEaHjBn@8BuEYnj?f6;Tu7cf9tl z^E>MUZZ(hiD5e1$lCpuFq@liLd6!di&iAvoUiMp8XXH#TZioycR9Rp2^^Bszqf12H$aEO`Dt&T`j*VikA?DwRA1fREp-n40S!jj z=@G+H4tL5d5hDf<7E#Y!xM@6_BNd$e>P3q3$J&lT!KpgM`hr>^Vu3igcZcww*R|-% zMN~R<_x!k&YIFr~UDv8>8hkZAunFg|vQLIllfVi48ETRPBakB%&uaNv4TnU^u+Dth z2+c;?b7X%nmm0ZIt`{RLA?YV^<-)h-)2!Hc)i4RnLpNN!(|?I77pb8lbyZKSYnp4l zXnv$0%|QNfZAP@g^h48S#hz80FjfQjB2ST)a+PN1ho+?Mm2%n57(%6>oyiTdLQTye*EU3C&%ZmEpND<_M+xU*UA0fy6jvHVdbR!%m=szG`bnXC=O%29cb_r zj0fA8^RZ6$m3~6`Wzz8=XxmUBrdia6aUUK41b(?;%f3$8^D;&Qo7c^sefqH6&)?5g z?DT~Hz8tr8q>?!`2Khok=~wzx%xK3?sfhsF&towM`Hvn=8S!t?dcxuUclYamAH^7U zNI1!ULKTdx>hM#Q3)j+k3tLdz8nN4#j%s}xNfukaaQ9OYw)cje09VDkhq!yv@Ez)p zkQ}(bQO!A+UbAfWvI^Spgr`k0LGa$AZmrBhd@*wkMtHLGiu%#KP`!-jKDzPiY2yt; zVS}(+kURPBPG{1rCcq?97oTvja}Y9SK^wmd>PZK;c51!A?4>`v z&41@5Q34Y~JE5^SxD9DslbTOi6OR+M;6lJv5}R-gIbixU1dQeugrCy5OBp+{+W!ls zF9Ug_I66oH5H|o`UqC?XUMfW*%@D>0l+2(0KujvIpR5Vz&Y?D)e>b-NQ>ujoll~JK z9tBxRFY9;2KfVtp>s@quAV(bUi-VWI9cx}7l8LYsVea-&pEF4_hA^#aHtyqR%5$TBP)-)9IDQEJ#@NB`r_kcqO0rPg%Lt< zPf`f#(N~H<7E=yS7E@&BRV$c#iWrr}OWDZ0fzhswvuX#Dvpm}@F?uwMV z95>j0KEkjjb!OPcZ?;;@=V*sjN0GOf(Z<{FpATy`Hk55WJn#QLDcUcCJ6)S=bfOL< z3U0CH$GvFTqr2zs)+^Wz3YF-Bi0-~XRR&;SJ!e#QY5P5>RtS45@Mg(tB0PBPNh~Aq zRgXexnmAMS0N1-Pwp-0^FuD-)nbd@Z_mRTr;ljl@|3Rfdob8H*__feGPP@?~hGpR= zK0nw>YpBk{C26gPLzwzV18(O5g0ULmq0jN_6sEB*NpEMu7uma#fgTAOXB6IDVrIyQ zY`IM;n&n=RW4vd_XmBtu`)H)rmyZ`$C4}a_h-P7lUOMEr@0&jeH~rR&M#gP4STxdU~s7 z3&;A6mrYhcC|;je5H`UWq$(00JIWNdK6bw|W$FSQ zq}m)(c`Dn0Gb1nSqp3VNk2mif6}wpD${%6M@yQqD*z2Rh4r1>lAJEse5(~*y+mSH_ zNgV8-S}54A}ITjq^)@zuhK}`zXoUnM&wOsOzC1Qlj9HH%iQdjD2%f9)h6! z6>9DDG)Edl)qlT^@&&5$df*z2;i3?kB4p&TS@-M`{s;h%eAqF6&k3Yy@fRv{XE9S# zdxLxl9h*?Bl~{9~*5`*M48aE*Gu?5en1-_S?*f@$ZmqoEpZR6hfB@mzY_D-7g_L()+<`?rc#9WN}JaA=UqD`FndRNTdoM^ZJgfX*A#{I5M zXUfjsxqs9^6sQz)&6FBq$|y~)5!-jtM+$IC3ijPtYLt0(X!=yk#0Q7)nNZn$*-S)0 zRot609wp`}l!i8dKQVQXj?_|8LxnMKy@T(odYO9YCaj~&%ZPvdoa@$yP>FMR@6dt> zs>zXzx%ohds!JC045%Br0yuMNXP%<4+&-;X)K{mB3#=XA6{G7#qEwHAHF1c7u0z(i zUY?Pm*^FVK`!X8`fjNM0NN0O@F@vzY+|8|Z>uj%W-s3U-ZK|BH@ekMbzYZNWki=`v2&r31h9OVv@@j_hqOUmNyD&r(0S3-*Gzo{4ZPi~jBu686 zO{G_sMw?+06@p7DO79$wo(Mk0VL7~o8+rQ+b(7pTQBNu+)MH_T9jbJaz~m2g=WUad zMlut}cw9C)r_OfHsVgLBT}2-{1urAq#mv1^&^9d*3;Y+cEd>UZSBGJNC1}5aqrTpm zc^k_Vx0mw@(1mSxA4p*$FmH#Nb5eWz8Ie?S*!X+TqOF(je<_An7W$1U4-xjR=Rl(R zZ7kJf)qyY9?Aa=qQlA>6m7!Dd_&7L3PX7a)^>=FP|Ae3aXClHM_NG^3li0`e&8K5x z*ZE3n@ z?)!rCY-9L-`-wMvh}$vwrO;|ZJAS#PdNxJpO;pnEWnve$Uw`-!05?n(xOrqQlSekc zE9Cr_P?PA_J^0lP=k?nr9FN3c15T*9;98+sGR=LNfrpoefiY!s)t>fZgYNT>0)%G| zd=!ipKvw*p=RZ)j96razm_7K-sBWRtq%iQ@+c?SJ#aUPW1d-9T>vu=E=R84O@(wnj z3^GqhNyMvOLb++x(`Wi}P3u#sCfLY~VUO8@(Y(t|6SJ!mq2PRY=%C-{PU=(Ie=F{5 z)Wv%H!qYC8Ni@6b|!gY5Rc@5rojRM!=ffNo#KkM;nk0XmT@Q5!ji zzUQzS%MQY?2YwJug=qlU9K`JU1xGJHC8T3~+XYUhWbwah8)f+HT zL&fM|MeiJEtPZu4uzh;vT1LO!Y$U&NY65$oE59yO{%H!=lW8?hE>W{mP2=+8Iv!>Q z#;uaUojRn#dP-~{G5r8uTQnk0B!F4tYZ5%Q0wK*q^Z~au30^=1NbsXLb`3;d$^I?D zuLo-jBzOxT!2^GlQ+XPC)*1k%z_hG?koM6nn1AmyoS}OoJRpHTP&neu!~o36R+=te zsRvsF&>OL3l^~VH0?^SBu7*NhGCMjv{^=SY=HbK2ukM4mik<~2uRM<5?sBrr@(%Ck z-oH?=x@;Z0@EKW3a=ai8m4`IKom^lZ=my0C7MVAlJ$uTIvXgXNZT@Tz(cJ9v=m5%ro(!<-x zbIGEY+?yF&^_N>7p*ZiHv6S7&K@eaIL$&hNN>JS>pk1(zkjuo3*5=#PM#ikOeD4Nj zCw)stvHh*nAHH&|3h44Teb$FZ{f_-Y33?_NP;o&^&O&g2i(}U0WtCl#W4YR!ogIXu z$-{5XLffmcG+a2=78ki%8or@2!E8CIxXIgGNhHZ_w%Ve&+*>bepRqPq7Nm!vnx;l^ zj!ewXIZ(-G-pkt$ZFjD2-ACkAOxpTx;7F~|*1Z85_pRMU6>#w+>Z#NK8JZy@!B})U z{)dI&$fs;ZWkmfl)aS~#H4LMN8#H=4HFl1Ro7Bq_~Rl;2pKX#e81OgAU3Mw7*~GYm z`sOmxU~46s2W#h$AppvJTEa8?k2ukJW=^b(BW?GHgcmElEQiSi%Bp;*XSk>L=`3fu?Z0Xgw|)e;=`P`JM2Afu5K#1y$L;-;+R`NJRZ;- zl*T*H&|*haLQFf)&dXiJ>da(?I>u2$+2T>@$F~yAnL)B|>q&@~+nRTtCX?f1e&Z zDj4XPJQuTvb2`zU)zM2m`uTaQQ&sA{e2qF=&Mr{Uf$p0)&Cb!qoH<@v<;l7mgk_`~s!VbO8-OMUSJJ?o@T& zpkHdz>+G82ePs2aLodK_miLa@M?6o5qB%x&K1V zm{Q|t*MFfJJ81+!f`kQL0FnCHUqv*;FqY4GT0%Ah0StZCQIOXFOuRk^mbRf5GOl)L zn7G;(IFPnUs4dNAp&s|eeCcVf5boI{c9W=m>!ucs*pY@3;#^l*I($2BV-WSi2omn# z&zm~pxU=iX#*Hn9UmW+Rud6cpU8v0`K|wf%|BW8 zeh|}amNG6|P-Gq9lW&@?#8A42*`~w2U0V% z;DzDF&fW|Mw&f}mWTTTMvuDasdjlT$`)jR9V;w+kxx1#eq%4rGNP}tXJV0zd5DipQ zt6(6_WG+A9SkVE?{;#NBH`WWNb{}?(Ry4#!P5(lz$~df6;HIJEKP1dVZ(qBwo&7V~ z7_uZwkO$EtJtQ%HGr(DYGQiOP9R@f#XC)ZK)s6{dS3=z|IM;%Fvk5i!iLa43wg!Xv z1C8*Q0Q7No{xH z(L=|rzW|kg4Cx@fY)RmB`p&_I$XZJ<05a>(l$2mNeY_qN_$#Iyx?({+dR|&e*mRPo zpt%bF<>94qZDBe;>bGkl%;<=o8hwe5NjYR@<+?yQw$Tq8itZzgBs>*_b6@)**uHw`qFUkhJS3`DCKegHnd$J4ix+c@=!oi1@2mD~tWs?l;8V=N?+; zTI@5|BNiy}{6Qj$eeCaU_)N{MGJ7X&IpGPgl2H*zKZZXTJB~DGvjg)Smg|wtjuadh znYr7Ot4?|KN0MxDHst6_JZso;cAS))i#vsRw#l)zTKV#|C9beF{WBm6gn${=g;}j3 zt0}??S0512XrdT~BRfsQaQKt@s9Z1};~8sCW#qmq&d%^TH9lY$nI3^y~fB25p zo%@t56m-`xb||`A^tCTxg_Q#bo#kVsx7T(Kcd~r5`I+s9&7SXh;N!1%r~~F>#cS?6!G?_u-QSqqtQEDhAn9KQeb< zy5Ag#u@~K1#ihk3EyaeMZ9igz4O`b4O?$Q&q>C3W)^LqMv&&2+mNmNoFHW`^DNrIO zyD;*gh6Lt#Y4KH@>nFv?kA*F+?=<_zumk4|kTN`w&ZwB#Wh&Va(8EZNj@8S9c0X<~ znQiaoC$4?$*^v48l~~xDS80bfh$hqzg{1JnxRE4?%ac>*9hDJ&8|!LyV_VpM$$(~* zpnJ{txp=2phf>s*VJl52?l%Lvb9r(jgD+t8wkUzUp6hnZ>F+kTF zGx-%U>^%WhVU-^LwYo8FT_tmr%!(m_@<S;v8MMAY8t4Jxc`w|#2djzAw+jLW!-f$?&k5|4qEjN-= zx09vv;)*A9D)johPMPb4C6WAxGLk#8ZdG^zpDj0b7B zD0O`Rh+-mz6H3yBjTD?8rA(w=!u*X<37X(0z{d?s8^_ zua+WVr`cD)D<5?E@U3Rv?+)5l-pVT9vGG!M+k^LFA$HpYRh5GBH-~c#bvKTmr zeyf73!2EiFHAP2(F#a!?$A8aAZlI+8J4SNvq@5jFM;p^B$rA8Dj2)UVOCv9=09a)# zF;rOFt%kGW({nw^6h@wtgs(yy^+&|#^Z>)7;o-d;nxYq4b#Hv`zR^Bftp<1K6$LKlZfoB zKInOf0vzM>JAKqD9v2~kT}xYcw?Lt}PL`rWdc9?)%T5&k&L}Sik)nWbb7aX9n74c` z6`e~vMO5k%L{dhhpze=gnU@)sv?9`Wl8tkSm{OVR{?9ohk+SC55(O#_PSM}*dHq6} zMOrT6K-8{PmGHveBm|MgcVc^&10}|W?FF^*+Bd5Q{~-bY?SlWi5cdD`pV9v_K=F@w z01&}CmEBOt$|#8d!BXT}G;BgD#)i%wPTOXiA_c*x=*c<78p+3d$h(4sOa%Hqh6Vn7 zk*%!@-8jr8HlUfa0I=3pstUqK@;5VWoIA9&FSUMVh+cBCO*!^8f7l&L?D(0tZ^_&* zjh-oT%%xI6SctTupl5T^uMClY%xG<|LCz21oW=?+dHa@^HW=19 zCU^6eYdt6PtFFK0Gxf1!_uZ*+u@iatioAQsCTf28FtlkB*+QMg*mM+0^Ej$~z#4jA ziC1Ovx=WEc6Q^ITwFLOzy?3N(a>y`t)u%IOWgPsrccIo z`?{K5lsxsnFY;1FXo7#$Dk5L)9M>>V&;)rdc~)vt2|6N#KBCghJFbIBM7)#=8aKou zMY=*%ohrrR<)*C`&D-inDlTdgp7#Zoj#dcL^MRGu>&*&j8(z>h#NRGsI>Bf6#GFFi zvu0-)`14eWcfj}CJ7sBFj}0yVasi3UpJG&0U<%C@hJ7_wQS2H{eifr<+2^n0FLJu3~Vl+Q@iat?J$Oc#k-pdhK>8U zF2EOFpBl`RSpGHFQT0RFmt5tk<(V)7>lZTo|Eo8k7 z#>5{vX%9InW2c@>fn>JE0nVY49{F6QUKmY@8#TY^fY}a9ng>s`KCx=6?3CQ_B3l2_ zkZ@#ox-eIuHfA8;J{C9LOn+&Znt*z|>9bbGiW^Hho{A2qLyyEZw^zE2^sev2JyM*p zINiGVyno>2y92&L(y01^jg|e3B)GzvDn6_qQGMvcWa%!k1cfWPPo?*>>OT}kM2+N& zX%K$?@WVcngAWcheW7_XUUwHUc?d>HXE(IWwI65uBtA~^0B-8eW>cfEyN$$fKkWgBJICWVwEz~ zS&hz5qrNwlzvg>+?Pym?_EzWne15-stHyIvMsLPcB0l_OzwPS9%?>uVwzxB>dgrrhe3$|Qy?MTF0z`c{|fYc zon(DBnzHlpxPF3Ui0p-wD{9kHtOv4V~^u5zffF&|btbho@aE>(}dsg}cg(a&@X2c{V_ z=gX9bLm21b=2xCOmOoau{|v-=LKkk;4cdLgpKZAA)lO-9!W*H2;s63h`-?3K6_TA( z2=v+Bo)rMr)*R`bpr8W;hiw|9hXK`{{3buiMH;_+@yp1$M_%LkGG(SezsxyRq0^NsU@kbse_X#Qeiks*Sn6=2&Z zncPwkw|X!qJ>R=AbwSzzKwcmtzex1M2HZ7*4j~AtW|ZXTHiH|gnbPI~oFP~J5cRy= z%2Ck&acRvQpUra`u>;MVRx}7f7>2wq_{W2!EZ|)hWIa;>ZHcI0VLOAeAVtz!6;_IV z9f2tXEHq!7wK2WzQ=sbLd^-mYfMDuXxfR85Ip%If5?iv8*u~8P3`{^P)S7t}>KVxb zl%eK03V#ml@qa}aK~kV!T8tp0p|fm#GN&oGLPY74Y|LZgG}VKi#5^zO~og06lmMH09ovOLnKn4u}o3=G+YKU>ferR zJfiFbBq`6~cbC3v;3{D(1<F2>dJonQSbmm7s2XYm2P(+BY<;|p%K`AO2+K{pc7`WGxtBcx1jaag0`IviDzO; z8VvjIed^)Zqr{#iLU|~)|6aNP9fx(&JTrxPhiS}-JFG$Go>nI91%%;gLQAvgBOi>D z^oj~QR)$i^evRL*b1*{ObG|;R&bYkB{)p()Nss%D5Lou&pMi3r(5=+)T^id%Z^zcI zevCe#@XT?}I$pzj{y0Gg*KAYcyrbV!fs#;96dc@EQjQ5%)HJ$A-`+ci|JRhu`hRE4{ZAkN0IUqI8OD}{Vn}K*D8Tk@yRxVTU{5s%r zhNxHf1u{+-oY$72Y(2bZ32RfS$_m1p08ST#m92!A6vS^=DH0Ascz(ia>V)x2xh3r9 zRPD`uq^F=xMxT|e0-C@a?V4TljM~>mdC)&tpuBULmr-NN&#V5 z0WlDaf@YmTy|DwX@GI)PSa3WrT5_;rK!`9^WtXsk%4DgAL6U}vy<}Jd%X-!XB8t^U z5Khyx@bYW#0*Lm%zNZMsRm6ggXWRvvN=C@!JtyEVy(R#(6=k)Ao_?K z?(Sc!{`dBh|EK)S^*__nrT$8f^7WgmdD^S@!O|8~!O$CgQ~14|`9*DI)T1}bBbxeM zzxW-y(BtQj|7vWih6Q|B_T5E=H;ZjyAJZx7ETpd z7-(cXFlA&=38SEMv-b!g2$8g1FM`=I<=z)zo0obpTbs-W74s)$+~|2Nzyci_t?`>M zn|;jNwN$6J-+4=G%Q1Qkq?Q7<*|@K{QRRVk@ce1&_xif>qbF5D!j0VZ<^AhJof1vYtGzav z!gb+Nn8(ihzHk-v|3{lO9D@v&VPXJs6 zq8;%P@iRymJ%9*10YunrjV{A_U6Oeh7jJhBxeNMk2K< zi4*^`xpt>{)8OT+^Tal5X4+|RIwPCjXR%{J!xrTiEds8Pw&X9VJRaqTIdEHZ zU7hVZ5_lJ76K!I)1zGo1a#2Yxr2I*Vx_Rj`dn7tQOC_WJ6J6g5wJK_H1XY<;abP`pU=s!AfnMRn~oZElv4(PG;qA*`SN$M0a>sl0dc8q^Q zVfktp5||(wtd765m^vFrwd@~8&n^vIPH--C1|5r_i;9KYS*jD z4Zb3OfblUo*3NK4CT_MCG}Te9jcIOT9E)W}8{T*&j(02sX7N;bALErHWH-cTmpg7% zP1XJRm2=*3E0WSiuX6@XRAA|n1a>-8WZKm73N$L;#)0?+3NpsOrpRko>hPKB>tnEWfw{WmtuGSS}b=WW_TDk01k zgK*eaILsyoI-R@>RCJypPuB}Kf(xrRM@H2DU}{j7?-7m5;elP0nR2TwOQ4|-HfT=d zG91g)U|k70|4z_yH!U?_NqI%oMMC9r?j?%bWJ}G_GG>0}(niu-Qm;N}7DbxVrG}h^ zqes~B-DNuyyPoF7cNPnM72Z1$^KiSwS(eGW<@IRrd>$H_1tP)be1Ew)1?r6PRgL2> z`XADCi+JLzI$xX%b$_GQX1`H_azG3kkLeyum{n(9aTt0>XoNM6D|~VpW_y1qr0J!K zovwQ295cS{u1&G5nbw#>)279(D9GB-DA$M@J#2*R>k74?$Tg+4N1>6FSH+#6vcG+FQY;Z6H6L3ltGO`BM zG$6?h_~vuCtP2tJ6dxUr&?N^IU!_-!<|LQ&z1ypA#coLKJ?H+hn$BU2Y>FOj`B}Y_ zb((tO{i0&kgSJaO=6*^dIPrT#_Ip>l*f%D=s`=#x=pcrA3XrrrLH!bB_q1|<-iN&% zT6=n`A6-qg*MFr~=b)mM^vIJdJ&}~-CiG{Caz=E3?2i$B+!^mx4&!5UxaT0T#PsDD zvs3U~L$K?~3eILR>*yKI(_n%p2KR-KbuI9qZo|T>H~Un6?FR->h1qy$@dUQ zzJ>p@{j?2h6z<6Uzs;&~9o_$?%Q0|EPTmJc6ku+N zpTOg<6|mYoYc2{rLWn<-3wi`c`4=!^ExZEW9wb$=I)29lo_r ze88bkV4#Zd^lrFM_zyEa8AORN$+Zc=hgPKQ zE2WCJtb{ijcX(yZe|36xif3~Ro}dstk;t>Ri331N;7lCFy2L*Vv zg%`{l6kIvB; z`1a3fU`Lc^ZM)+6ujr`^Wr_H|P)<4NGW0O-IOy)c%ux{_ASPs6dV<4FXFfx$`tj&g zccB`R_`!2LOsUFdx6wA<`e}PW?%r}G6qFG2A?+Qf?{;2fkOP@FHCsQu7b@1Ns$UrzR<}xm(PBaMpz#-Ae z%h|GF&ZM*#L435a!z;mJLi&op#Q@wCioMa5S~W_-%@*ebU0xhBsHU*mqZj0PCdF#fV1&5MD=< zRvJH?9G7iIh@mxmSBl%C{NrEi`U-5_vQffxH!WSiq@W{?{Z(-*1cqEiKYZ33Jc7Bpjevj>$7JOM;g z%%mL-rSZ1d|Cnrsrmoy)$8cTmjIBKwpF1nK+@e*0APhgl{?v!$Li_koTuFip186C- z?!AeM+O$%!jH0n4C8G_cj*!x~g!!@-L>nl#HG6QCvB}`x%7WpR^~it?>Y-2l)E@IC zOuVcJ0NEhl)L@%XG55o#Yh;r7k84D4`0Rab+-Zrf?6s}5zvKQNztsEdojzS9>`CfQy%mQaGqokikxMh z05e5oGW)@t8+2_Qh~W*6rlQyLoBX4!1CF*snm`9YI1uOn07B!S^-<99wQl@B9JkQR zV*X~s-?!=iTmR2}qi?J%!a>E%F(^i%dZBG2dnvCoOJDT)1!2bJI@7q%jJn6QPZYAh zHE}!mOkJemCaP(UtQ_=lkg;N%ruIiwYY?NOH60}i8IMB>`R(4ebqkFc3nb`1-8_P? z90-_Ot+}qWYR)Qzk7dcqE+dK1mJ{fm6z+nOAf2U?sVBO;E2eU^ItNEcnDWA%7;^9M zqn3lb`fkuD&|5E~VW`B>N~ef@rQZEIyCRIJT7?Q*j|#?TSH#Ayo_RfZ$PO`*h)i(7 zmn<_8(1*^Ns}I)0^{jjT3j+%vRPzjN*!j5=)C6uj-2S@q1^?vWnNRa^U1CJbO~=YH zgH{C5wabug+`Bp4exo2~{wpdekkB5>vQK}I8#;g#)Kqqwm*O{5!3k80Z`o^QBH1uu zShs4gjiI8~j;(!p{RG5M<1P5l36x)xti03Z^$t1_r-f=H z;xn8X(9lnVNu!aHASKkAyd5*z3Sqy-jC-M#r_h(iza8L_B?b(+vWa}~^iZMe_7*b?HjcOYG5MS?NDfe)l}{&FN6HZ#Z;>u(rc zlV-vJZ;CF8t!v6u#zqc+{7{>rJkR#a0eI{hk$ng=egor)UE3+M{W}D2^&LWOy&C!k zUv+S&!#E;_+mxp@d7Ehts8V$0qMkJ=H*BT8IBjaZ)$R-@AE@^7Yd(G!(dekNbd-K= zJW<6evUq-W*~?%g2)kgqS8FPvKqf&jpyyX^G9hT@C4yUX4b?(_qp6w1RSBS zVl|h_=_jotU1~FxNlTI;)56uB>cn8z?=OY}f5KIG+zP=h@qd~aF)eAH$2txk31^X9; zze5E085cq7VK+;90O%+ zvzv9lL$D2Lbq`POXje$^YY8HArhSL(<823UIZ)M%r2pbw=#LrBH>`BdW9c7o$A zXSYQ~80+BS=5?i(V~MKn$Woq0vjf8#I>l{5uY75V){DmNB_g1@01v~11#W@%gdWd0 zhBl{Sqe3NTg41t54T_^VNWBUYZoU8MpjqbC#_%|$wH>Tvrf4x*hjouFMOSJHuy!Yw z+|3>|z3!pPY36;A`>0iT%Koi@w>h^EjD4kt)WE2UL8=-<^CREhvgxHSTeT8G zzxto@^i!*Tgv*1E-3R&8FZ4-ZbHt9GP;o1k1AOMIO6;nkrl;4wLu^9$ z)>haX9T_V02V<}Xlu%GS$ufxuPtbFGLUOQsvoyCnr|TC$ea;oBtZqq{XuShSnb&S* z<-9&>2KgD3{Sop#vE18(IS)eWFOuJ<^fM&Cz)>ov%SJE@;k`^SCD!lvSI-L_LJCZb z9PvF~WM8!jx^cPaZI6&nVl$$Lk9}{QR*n-myu$EH_?RqiN=c?-@w3bKEeEA`mwq{} zchYpvhIi2I`KDcXZ2PArnjz~cTZ-L{-=yb3LPp&md5}A8Kk;g0_c#oR?oI!83jOWIzVzb{&aghTVGFZipLEt4dXB*??^K-N65mYFIL=j= zl=91I+pVX0!g=R;=aBCROpy?p4r-u)bb^YCc-G1^pnBpIAIe*wJXXtC{G>Kv%MLcw zC8SA%_{>L~9DS1FTaT4tOUwXqRe$fD-Tp5GJoXo=+n>d$4!!YddR`|HAitfNcLTx% zu3$KVJx~_py#^nMCTY@-E3!>f8?*xj4iSgVMlPlc40D}Kags4G9f2nrvST|Y_4=vb zNLc!=>bdAKtmIdFm&mc>4z|a4u)5mpy6&!s^FzJkNCPVyMYKfwb0P>M-1xi&PwqZO+Ye|CtTfMIhyQ>Kd{IIA=2S zQA$x0mQT-8t$?1=V4;=jP3EfA6W%>tvoZakI4hU)_33-VAiZ;uj-~VykdV4xE(4)9 zo2%|wJ1E1SjBjtWHWy79`8r}e=^0;j=AMub!AWUt9}CQlV|gP`dinTkT@d;t9Czw( zG=P^MsTA)|s9FoH*lcLmcQWSD3)HIn<~`4vG|sA-vG1{f8*l*?HCl;}R&IITXhii+ zl~PP2Nx7wcNt9wc&I>C|4vI~>A52JbmiOgmdxdIY>Om3+aif1lsW?b$sYyrkB}&DP z!bCl>tF>TWx*+2R%}OVqqnCZ?B}fI7yKRfr$U$8gqxQ2u?{@JfD^482EQ zK_@grHKn2N*p>dfv5f4e!31HAn~8YEKv)AjN)~1SoPc^0prZHEAG5{jtRpLr{Gdta znUHje! zpT*)DbyM#%uF6W@mT><*{9~1HFly?yGC3%Yc(754T+_3+OoO7(=z~!0-7vNw#e@T*66jca4AF+D5~ET0?SS{J zx1rs6Sz@4<*z6VqH*?vRFoPE07;q&2;Mhr{BH8BHgx@%JPWG?3Dw^2#G&0|A(mgjB$zHzNh?3m@Fj%yN_!FS?rx@uo72$gQ_Go_# z54ZPaZO%T@Zm;c#`$b?+NC2y(`2(PYY*Ay~_zv+jY?S4z9DRV+ zAz~aEAzO-@b!VIi;sHW~K4e*OrgqK7?(JA1lC@Wj&(6$^Mf~7=eLmrW{siUp~G%C`VCO z-x`(mwez`F{)*xm@f)R8xV4%YFezkWC@11k14MO71nG0?yD_Qh_eB5YJ+7%oUOxO{ zR9TMFvv8SxEA$tp3T0Ode8VVHm!6p0_{=T%saMuU+xarqJ9(d|Bpbb) zUj$&5>c|h+IRNV6$^v7y&H7azTfj)W3C6!|(Nu)Pr2KM*M4ZF<*x7ZCsmUit41rQ=ziCrkT z?o~X`x*vqBOeP2#d#5(oI!SeVBhf@c9M^sHIkhiF(%Hku5b&=g^h%2nZpwjnh;^e* zIJ8wOzIdRNx+`hl_Qm;HqiaHkx7=Pby#)I}3wnD1t`;aU)U@FgglW$z0gjRfVgNxi zpm5cYjp0fDqaxze4NXB8Pwu>f6McwaS~+;>ExT{M=M%T4#OC$)*9-1%;g^Q(2|;ao zKUp_r&s3R~!gCGxZ@5z+tTjkvvTFx)z&iUV1W*v`TLX7 z-xe$j$=j|2^b`wFs{Fg?b-F&!l)lpHDp;M1*zZSW*c$APIvFawp`Gh%s6yAA-d5-H zn!R!II~;eibCgWmpts5%G2Ng%&gIPP$*0|3_MP> zDB~dL(_kr0)$Yd6p6kUgYv4ZOK>@>T2E!D68c@X{$f#&1j^0YvCR+E=B>q9;#2nuZ zcXp3oZIrSz!}Wq%zQ9IVS@aIp-C3p-b=TpS6n42-<&(3TnH`))A|GGDHbeta@Qxq* zqxC(~`s;8IvKJ+d~-A2LpDZ6odR$C;A+?LO=-}y z{B>25y7aqkX%(Kfd)1??;e0Hx#S0In^RyrZ;1^Xm=u#x^RD)=oJ9dzBzTq>+UW0(* z5RH(k6a%+h6Ipu*!{iU~bx!d9-^SS8f1l}-M`Pj7tLOZmq7PF|iVD!{GySVRP5zjj zW+y#wAJwfZM|%rwl{^Hp{WQ?P)Ers$_Sd1Ucdg&e;? zR2bgb5_dH914OWiXl4f@q*+&WH`&;d5VMI4D|$5m6c@pgmo;^wlKmdHJ9hEUCb65U zEOlZgV=_DH3{LMvy!}|r$~kK)Qg|Ulw6ilUE$p04f{<-Z774!*;|ZfH+@-Ofgiv{C z=R=frLK2sT=#~?;@d0-hG38(2uafmhTeVtvhA)AAdJ&QX6y^f+17&%q8)DTId{cV$ z(!I12zn7Wkd-r}OsI_0VG?I`H7E7dS^2b3IgJZsZ(oke2vJAo8V%vZhT-aNCjdA#_ z9UbQs;aOE9T1C#afA{I*N70f`0ms8tO?B%fAoK#-Mb=GCUc3u34qJh59zH-6p@_Zq zA7kl|Osc4wZQ{=~=faYt9n>WX-d5`~)5bdkiXhEnm64Hykf+32h< z<1|>MQ|KOKxG-9?fBC~M3#!3g&AGuHH^ckejc;e9EtNX&0zMYYk;5|om-dNn2>^^d zi%HfK9!JI03H71{$+{Ai+k_peJS^Jm3fhYj_nEx?x-tXE|Jl^K zQxKM=^g!zUzL^#oq4G1Nin`d98GAIJYN6z5o0pN|9GhGB!|td-I9U7WMGbXQ)RM?+ z<0x_Wfcqbl3!Bc@mcG8>T76%hxi^7#raMX3H;!pV{|2T!cM2|M-kWK}(4!{=x!A$( z)rh@Nz#>lT%|*>YhZ-WX?Y6kAKtNXXFFDdh0X|-I9q_>U5w_;&avVmf#S{*&P>~ZiNcZmRdHl|Jw zQ2thk&rzu=rj&+H2p{2LiCX>fZQIowh2N@uktTn5&?c+5c-{&sv{~EslPoV-b`p@1 z{*1Pe_IS2pCtyi?s8My_SF7lWPp_(&u1)*O&8{1)Mr1X-Ja9IaVnK82a_Sy0Ndz9h zZp=;X`xC3eUjr>3rnv_cCha%WmA5%$*vfny1{VR7Li;DiF;*&k468+}dC+*BoV+Cn zZ-Q;o?pWG#Enp>IY4=8`>R`I(L&3!@O&pcBP4KZ>WO$WkL{{uL%Q2ZniCF9ZTFn%* zcavkKEce`+bFX#HH}Jk_+?ThMZ@%?xCBTYf z*338A69nnRa-^N({MCHk@{IgDyzL?(n+14q;Bd+*%)kTkXbYAo-Z-+0Za0>RXu9uM z5V6-&v8MFMMT=Z~&n36jb63S|;E9kBzfQaTtjdpj=NH)=qpDG?Z)mH4-8c{U|TrU>5yiy5&01(OyJ=CH^c}k_V|6RuG`U zL51U8fki-ZgUebVD**tAqtaIU!I!oIo4PH;jaEj-GQB7>Z*;2gk4A{@YV{#*r4`=i z$|9xropJW~MRe^ExSx-7XY#^LWEl$^5vnj1Xdv5np|4UYs=Nkbwj-ne{Md7l4O`>+ zNr560g-=5MR8od9c+adSNBi)lfd>fNZ5LS#zfc0cqP+=2Y$S;%FkzGh`#wK?%f-&w zh2qBv&HD+LP*Oq`n-Xq@0^*#b;OfKRc@)<*$+!Tdr{3dNnd+28Ec(DnXGB|KW(>zQ0pZz{CEThU55!b34bOFm6?~ zHv~j$rZGejGHvl2#kor zjH83&&&P&~63$(?3$gA5Dodb@MEi15W4=y6ftBIbYNFQdCuurqiIoG0SCb7XUTfYw zqQSc_glk4|o$2HjJ+a}0e%P4JofA;=ClcG8yY8+Rha3}dmbS9^Rr5~{@=d143u)A> z3?3W$`$HcEA1m_YNvt3FfyB3l{9usdfI!Y!cmDcQ#ogdX98RRJzIpDdS5XHag-RY2 z*)LltFS;T9ZoYeA=XBpK`yAFp_|63B;7Cqfr{TsMhNq+<8^9@tp9bHUwg4B|&S3(1 z18fWK3R?eo{pL8jfDwPRR4zC_E7;#iH`i)4Md)d5ih~;@ z7>if}5(a|!u`0H9RR0@UL4&Od^?5x2Z{cmWf^NN!<|JyJXLr831!cc0fw6g`$F5zh zqvud1-FyU2xCbi(jNN2FI%(X9K$#6A57|zxFhxvh;pMDHaQ8%KdkU*{D9EExtsr{Y zj^q8w!!GkKFRb1UYBs)oM^L+VZ5r8uWeVGlJsjPDxg;y-@HHI>cJ=9XqxEoH37b(?1JyqeoCuNYSKpWe2T&X>I7fxETeX;Ur@&(6eFtrQ*A zY*;GX1t3pXW~tOxDM#^U+ZoDA^VJO|&-?OhN;BG-4CJ;_Re1@!BZ$<i1(Vqaym%N8CY%Mz5nc`7UNW&EsXjH9x(BkR*etx(ObVAd}W-Z*(bodvy;xvN$-n{n;NukSjfEa9W zP7e1Odj$9qC2V4i_1?fNXCEV|0xIRTy57p`+J^yEW*^FHnZbYHeYLi>^)uB*Am^uh>uGl20T~jW^E(6u z_^GQhNP=R(b10Y{eWKr7{Pn08E-3$#P|Dx|B8?a#oLP3Y43WS^+V6XN6p zD(_Xidett0HhZ}x%E8Q_^O`&KA(O=WrYfRf1gz8@w$rw=psIrF8r2&Y;!461T~;-Y ztC^dlwe#OA;;|2Bv%c(izHvqM$Z>^-WHs_F*j;e_Onhln9^)v}w2~SaH=6~Ib`pP) zblt>G=vi>)&=HLQwE$0wi~KwxX{O3D2D=^&-mm%x2d*);(}B zm7tZ>#6ecm7ib!+RNOvE5_udJA%b_Sa5!_9O;6IaqVsijVlJWvTn%3^oYE+%^*)w- zX}2qs4m$EpZjdy)SAZOE!EhjtbQ+3i%qbn~QWi#Zd5)D;#+=_?9eW|xH2VmLA^CaH zpo!WIK)U>I@AZ%O{U81w`nMS9-~He}ypR7+k8|kX(gS~wp})t_|5G1;ejoTRYdrM| literal 0 HcmV?d00001 diff --git a/docs/source/logos/vllm-kunlun-logo-text-light.png b/docs/source/logos/vllm-kunlun-logo-text-light.png new file mode 100644 index 0000000000000000000000000000000000000000..bd8bedd6bf6e0e6516a1dabca552142c4afcb6d5 GIT binary patch literal 178337 zcmeEv2Ut_d`u7O|gS3c%paMZaQF?Cz0TmEbilV59QIRGfU3!8dAVp#W6%>%Jh_TQ_ zB=jP(AXPd6lnx0c1k%2PuDg5h?)~4pyI;BA^KkU!OeQmPX6C%@_r5a+=4U1WqG{;w z>;M2qjsVgC0I&n>kj(%TOhLeZ0Aw4$_ALznmm%AKNne5N{<#bbSVjo|KY<zvZtf;J}t)!%_q$;MUsjZ};t)ve2_7(uZz;;*_m6erlvHrL>WZP}X_k0Ko z6iEG1MoCdI74mDJQdxfViEA|#`XkM%p0VNQGGN1N$8YYer2g&BYG5vtz%uCMxzFXrVXaovK7R| zz7_Ft(G@>=#6oPpyO+I~y1b&iqPQN1n3$N3*F^{IqX&$Bt`2_F7618;etv%Pek$_r zUXBV%T3T8PipmPg%5q={Iq!hWKIi@AE_+LSZ{$Zi2kgD=yqrCJoZT;rt+jjpg1fJe zuDJNxjea?P?x&l_FE{$bq5V;%i*~=%@$mI>{noXMb_({c_HOoa9((oRX59lB(5@N!C&Lr!v3x_1D_&URLhzu6qBDety%|&o%z3 zu_NvmogD)Hxhp>??R)j#l=)5Vi+0*R&OWa8-=5hoQ~B-T8MylVWA56(*Jj_7l)|Jd&L+<&~Cj>13p?0c!-P{MaI`_>{zXL=mqFn`dj9tU%nWw*)x{V0p0=0+xm z4FL$qtSmz3Jv?03IN9y8kJnM-ePSn1ofdg&WAGOmcq-a?c=`M+KrH-xJl4{| zU_QSW*eNg#$_9Yn@q0RBEq&p8y7*fetE2nDG9SUTn4Q!4i(q;hOv_&QO}^uA(r&K4 zYxi9{)_TunfBE<^@VX}Ggn8PKZcPB-*}ipx=xF7-mSz-V%DFFcTQUKrus-03f@G*e*+X43XD-r-2 z4*>uI>`M~ZFZU||ARPe!`40iWE*Aje%fYs~001)%06O4QbZvnEC9)7;BN770AA5^F9hg|f&h}K5WqPf0>B#}fNLKFs2qd<8%7|2=~oC4Hvs`!ry;-=A_O?S00EN8 z5MX!(0_>tgfJ>kzc>!es<~Oha>KrUUAU6w8wuuE$6=DH~#94q3Dl9;%DGR`NmIXNA z#sV~=S%4D{S%9!)766{X0(51w0NZodwBp*qoB<*%oZNlfy`9`W#8l)J0nL3TM_AX` z2ms84A*TW8_MOJUU~v|JMVLhoU|*Alpa4)6WN9(R>j2BQ1IoezWraeatZb~{!3JYnJ7DavZwLF&2gkRA zbM4^%<$!?o!HOHe|J)m48-L0Fj}y#(P}R0D%K=_?R&}mbC`24!;e|kXA8pvI2 zkZ;GgA%GvC24`j40At_C!MO=wfk2@wtWY+v*WfQ9YucTamu>S7C4&uomgiyOp8U#J zZ$Dv|*!QYJ;OHl!q{;=aYa2PX2nuc8wsV)%?mg0~YU&!ATH5;$7#bNLG%-D9b^L@i z$R2i}1a@?CzU1xWd&SQ`ATaEDctqrlsOUR)@5LwFfABCdEj=SMD?8`uv)6C(3kr*h z-j0VF`DrA6-s*3b1hSf_!%L<3M} z%Q_FO)6QR_oprhUUH3qlpVoP3orl(W=x@*fl-at@Lw{rb)@AWJ4uCS-)_G{1ht~Db z-=F~~vwfY1{>J>R>!Ec$1Z8%t^UyjEt?Qw`K?6``=Q-Zw z*F%4U2G-+4>vD8mm;OB(SdS0=z4=?GzxDZBj}NWWz`7jxdo-{fANqUqw@!cS^S2%! zTBm_u{H*|M?ui)}uu0o~du1satK{2ier3a8o7L5voCX zLAGz>S}Q-OCj?}idv+uJDMO!!W|cTlkVOePRNSc+b7OisPuw{>`9#m{+ORw3ivpB3 zsv)DLXmINiX4*}x(TyOd+kAJUr0#UI@m$u~u)>mr2x8rA^z@Nrzrf zmo_I2>+MKSI6qf9P)~QHxRCZ&FaZHo-eCl6fc~uiOr5IRd=pnp#VI~Y5bvbSCyD;B z;%mm)*_UOA2m+>>32X@#Ct0B-iG4{g`OkK%r0kmKWdgFJA`sv)Huebn8COi%~u_AzA}$_Qddbk zZmt~p`f29zHvE%GRV&NciarCqfyd8I)2gYzN}qK}{O{&sJ&LwYBS2cBGn-uKv58ZG zWtn#A&I^|k<10^}?x}5URJ|@OF4iEKtO(Ga9Wh&NJMt=#u}YAWo4K8t;LL7yz#u`^ z46uNKoQD@>0?rCOdyi7yJZ&Vna~~x78?{=1n{$(-OoW9m2oI%=+-?Afdd*iU3R|mB zeu+%YVX2Mv0I#DC?3f0%mV-VIr^e7*)b227{3_hgr zAQ3iOPj$VR{cMLcUBreD@&}s}@7D18CsnrE>vINNtiGYJV=KKkE;RNID3QjphJpv~ zLR;Ux5ia76t2)FPeeIs)^G^|??@a{+PMbBkqE~4$luQC9MpP)Wx@>!j?&jG~*tr=g z=+|gNax*NQi?=55V$e&!O$QejXKK(aG&#zX;gsviyJM=$gzicMHjTS3>?jTQrWT1n zw(dl1-QRqqoY6^>rDPMZzqUE~DQTH}_v(V7IMp%iQ=~*RV-E>Y^~jiKShoNX1eZ&3 zms(X>zPx2PF(4jkZ*7{JjoIJsl#{+DenZ~xwJKN%An#Z=w*Dhy6ZYQ}0?kVK@ECP> zka&yzdA)Eii$i(!tkeME7obO3{@{mDop)Sx(TUpcGVVTnNULVSyo> z^Iv8}96>PwXf_iV9%lmKS4Rd{x2j=iJSj{dCKWzO{ZpkEAK6LGXP!9mQ!9_=NGJZ@ z4iaZwm8AcDYatQb7}sF2(f5OmtNXzKXhR@BBzCqA#3l^bw1> zh*m1RG$$<@*!?F@yG`uPfcMOVu>@XV9y?Q&gPK6SUXESjmjCLqHwW{Y_I9Q}?Y73` zi5tDA4od8+p-p zmzV%W1jz&hjcKxQP;9j{65s?X3-T$dVUf(fuTkrZw$H49UI;&wUy zzFi{5w7}<3Zq%;h?wYZo)B2j4^)skfG5S1zHsYA2R15QN<6qtPB!Zol7h4%B0h3%{ zV6^@8@R=Ctp5;OxsX!9lhkk-zI}UM>Kz~G_jDqVYz%!bl@Io zVi7lnAfKkE>6j4PM)abdQj$);FKsE1Vti~LdG|KKX{4w`I?b1>dPBsKo3~Wnc+86q1#GqW_U7@_rGlAfTug6}8R15*ZQ51gdq-UF5M& z^9H<{3ojnH7F`c~xYHhW&btTy@V)skw%QhDS<-QtcH~+9vbTw9rh=o@z?jsfx$?8+ zS3vfv`cv%n=Vt29m0nzRc$AGCi)NTJ0Z=Y1p2Kdw#7H60k>=D{JemodYcjUzz&oVC zQIkc-&Yo%T&51}HHcBws=qsKhTYWbifa@#iBkVpI@&lItt>+Qd2g?(tbe77*=5}0O zey^7M?3Y79K|ap*3Ht;#t8R6;-~m8sacM4%xj{0{3i4l0iiIDlpRKSwZY+7XoCPn5 zcgFQxni@Xd&e%Xy{g^Cx&0Kzxpy_7NmLsABNMPrdg*jFkTtj`1ymp;}Tx+&UhV=^B z_-e{nyH|PZUK1l`(&=f^n^?_YPg1K-$F440NA@=D}`Z+;5>po{1~xvb@^WiwMQdUKS>l@*KGq2V>iA=sP)h1r2%dRp3=Y zf8d-o{iHf%!EUP%jZKc*T9OUd~qzr)F-HIoTN=c<+*Q{^P{Y^d*Wl=};vI z&~%jGQNbE|fjj%t5raKuuqOVPnPj+;kVN!i{qnh4vD*_9z0ou_iVbOh4U(N^E*Mp+ zw95)8N;q7-KY+b?w$F$4;=N#dxUL42YD0W8d_SBa>wf{UnQ|$3@ldXj`?ofQzqKhG zCqv1t#&VN%v`|z0*6Q1AkY5;6^sQ7Jr{{5<4=Jy5GmleNMbixpMW;oo`DP}1GI5RA z*9Pl`%)d2cVE;{flyUVKigY%ul?hNEAeTi@IDr)fCctKKO(HW`z2ux!B38MJB*OpJoEr zT$n&Z=^4!Dx)3Ii=9U9puA^kbtdrA5XZpcdVFy;*x%))!v2EngwXCx zF@br}cKp<=JDt7rhzmH@-+Tr5Ee(G;6+T*o*D--J&}c>s@&Yo}lV(Ga$d;oGAMvho zRSnrMHFj%X4B0v9?cj!TlgY^r+Jtov=4DDPFA+cDmUa8tnE-7>EQ_;7c#^AUp9<#5 z#~@FY02_R`yO0l~U28lc?+AKwF-9rbK(8jY{@a^&4}xzAl7Jbs7dk9P85XLpUA zXS8;G*=16%<5Ry3z`F!K(+o(3c_fM2p24`Ky1{@KXB)4KP@^dug_?s60`+vW4Hxpp>(tY;6srtmr_tn!9(K}_QtEHdCW!!Cnn0QJXY(F?FVaR&l zF5Ci!GJ_FN;OrJC%qTL{6GjM3po&^#yJ3UV66fbAgxf_ip%Dp0$ytNxvCkZYC zP_d1KRpu%b$lpeF))U|Kt;U^-B**K?-*`$jN4Lq5xW7^`LpC>)MZ(DgXLpEAaz(}; z#xw%{V{?deGQ&^ujZ%58T#YNmtUeX=;tCnG;K4NWheFz-8DoTGq4Lp_0#_@H8ZJ}I zrfE=>kGUhcB})VNTg7#{<7v^W0_f~ykd_rel@_{o@@-X{z;Kl9pRUUbR=Cs30~E zn@6B`Qyk!Y{uf3*Mjw`K(PQmgfFwj4ifr}L+RCM9AZ>s%6rczY?+@2SF?jqFk(>NI zuP3y=HV(;tClE0&Z`Ou=C)Ti=^5M}ETRh%{v5j_^h_5l{wDHfaED}HIu-zg22A|DG zSTlc2_x*VZHj|^=7+%UvLZ?&&Nv}2KedPTr`N4k2S(;o8|AI97Lsq=C8N+EYP+Ik= z%!11FpA}TUmfw??+zw32EF2@pf@GW&yqlhgmYzgy3_e9b@lFQDB?&xl_Rh}3bw9A( zcUQhmcDzp}mQ>@9qUkjbA?WM_icFx)6}ciEigX;u)xs$u_sfy1yrt)vK=5bG{BZ=4 zst}44>E4DXPTw*bKi&BHph&D?k!`m@{d4rb*UBymCBi(0A0QJv_T$gQCgzAi4r8NYXI|#~{&$piqt7OjGGBMGVI@G6Akj ze%iJl$!Yafl7%lUqut*gi^Uq8f&Hz13Lyz(6u<<=jYyI71#nCaR+y^kG6)FoZjQjI zU#pnFog63`9ZfsO1R}eGp-kY(_FBYpQzR4EX~_iOuPHDluumfvIhO=yn5M$JS7k_H z41_%q&baO_gc(#v(Bh5%`@3kZ3Zh#%sq&l0EDGAr5shPR&t}C5TqxAsu#ewSP%1XL zG8B1nJZp!nvXSP*#7rIk>hI9`p68x%o(JoMmPev#tMn55Wd&}=hh!c%!Tkw?$0#@R zY;I>vU>A&?^5*HUi7e{--dk?$m+`EO5QENkFVQ#DIuw+(=gECJ?4Wg-SHufa!S<$w zaU4$Dc~*c4q&7t|*o>J#jWsc4L}3dhrLJrjo|F>1B=gwrROc?2+;_c&gR-q!qj%e& zvpE(5`sDx@j~z=W@;!!PS}MZ?43gk94lEOhPG@Y!aez7Yj)mYDk8Bp8iTK0n^kgZ{!~Gg{dTbq1 zpM2ko+ZX4T7P{fK{Ixv6c7&zw0)p?bX`G)gqfei=a%; zJU>=jj4JFNy>re%Of^7`GdSC8<_bUu(w^LmlH_}YdE^6)ONl?C&&F^0`%~$G!m;QO zrQ)#`G(Y67FK+w<-*y+eQD@INXn?ob?XtF(83!0l`1n6hCGM*rgHt5^#Y5_;Zrf)g z)MJ&~+z5+(@(Oazj40d(Sq^})1VTw^B96Yn6-ksvGNiHp^fYio=`68exn;LfgVeKq zI}746ojp~cg__wW!ZTYnkGnpCOq?dsTLOtni!-POByIBzOsy+LH)uPz3pOi73lWd8 zF;#@MhKl^g+-KWP-&c3|#+gfkiapgQ7eCToN#bcJWHT7@9z zp3y)mxYn>4c`-@H6O2)z6OeH6>5)YN+8dt07TEGDCa}>u95b#03Wlp7x09?emB$IF zRj$vNAu9~+IEGD#|C|MEIg24rOoS($ZY!U<>$eFi!~`ytus;^@lh|nmt?rS`PIM!aIB z>1=Y)SzPOPN8q}V|0j%m0ZIFNvxjG4Un+XMbm6XCBQ3+}8Rfdht{{igmC+yYLUxpH zqN5?nwz^Rs=Ne|#2@KK8y7m3)&MU{?3)0(b{i%SW0s^Wz%7=jslvf&6O{a%m+#j*$ zJs=WSzqHq*i8Bzc=4XT{y~0z8_qS`7(1Kp2U>;ep=Klt2{1$fnNE^{3@t0_?LAJX` zJHO33+y@i3?Eq+MYfoimKMS{G0-}n@S#fuB?Q>_K%fYuoWR{>O+NXVN)u*i6znqkv zK6mG9z{w#=OZl(vsVw;WDRD@Cg)$}DQ4+j*d!e3LwwK`}$BYw7aC!Ff8R|<;_?(L9 z*HE1#t=hnF=+qoTv@#mGS~V|(xb@wF{J~Xl^E}M{6ry?airO3xrfVRZ31r^sTBw^? z!i{?~Hm)#%UGE9atJHQTu=x5kKt1{=apMnjjKjL9FhS}0{kz62Z`@AAS6{MW4R07p zojBDrEXR62>F68FP_N6I08LPW;I;8XIr^B&bRx(SPa_eL-42?xa#}cgM_57BS&kVA zksA{$7Op{R8~}o7Xos3Eg6#Y5eXIP~ ztKw~i$!Rvu+qSa2QIzo7V}|>Pr|fcgn>UrrO1f7v@WeMq=X61dQP(k@ z8o>`z9A_SJkp)(rf3H6NNTvL#yvQExC2I)$(zR2prnaA@A$Ls5M`#SI*Loi}duTPA z(n{!0kzTe?LxKfS`**tR{~Ga7J{k0J@=uR7#|Nj$GO+STppTgZPp!SV+#zIMC-aP z->LVQ9o~t%)5_vyoC2O0@*n#ho)boxDNzEgbAqKP1;f~|vfbF~vmzMvffEOD&n3aA zUd-K>BeV6&rh3ZRx(4>{3aOs;Be7`F_^JnAJ_n<>`RknFf7);=-BnLKlOsA$(4i2C zDcedrL@WZaMX!H$Wf99khwb*^FLxC^G;ZdP)>dX5IQR5BSP>+q40lEA!PmeFC4kl& za7w)Fk|= zt)BJ1*Hx_cVQ-QA12<;|!_S%nLWn~A%oBn_w?_F{FUf6nH<#}D`^fPQa+X4plHO-1 z+Zf(D$+eYdlK^W8MiR+e8ZrWp7FFrc!HhKRvpsgE^Q(r2-_e_y-H{M0ASb$*{TWMm zEg*b+E;@z3G=Ox!7WAm`>PcMknk?zeVc2~f8$G}}9W6N_Y>AEM^?-HlDxH!Hh3hNo zh6Aj3+gsO!|57kO)h7cWQlsg<#4%zL;C){$5Ny(vTi&tckeR9UV0j`Aj1E(?sES`K9C+~ zEYKgUMu{9T57$#_qXiME$5IYtI8jaA$QsM_^ov3jhirQmEQ%g94 zF~Pb9E==1_s!bws5(YAN-OqDYj=B)9@~TV1=V)rVe+EM4$q+W!dg(27E2Db=Mt@$) zQB;_vD^+{?;=#nP5vZ?mX2y@*-#wHY?J*sRr{>jCI`2w!I${J>F9# z`d~(1R0v;yoi4hY5{m{zlv4O6!WOCK{Gj3H0`GQY#7Kd5G)-C+&oIFaqpaFy^Fz!Q zOx}HHviFoLjTt8?oj^QzwpA}5LVpK3m`|`Yp)as8U~I+c2_b4oXQP;G-9j0+q?z@6=AZ!7U2g zH*blUJ;iG~GwP8M6$hyc_NyDU|Gr`pHi#r19u)FFRE{Z_3okVvx#@03oW;7?t^~E^GI;)R?ZmOU1OSB^8>co3aeN6XFB@Mv2A2AqyCJjtOv|r^odFqKevpKVo>WG9_xp4^f5M<{wmZGdsYD=yW>sMLHR?*YEyUdZLNWd+PSf)0cZZ zTe^q|CZM5K#b6I3+;t<)4P#=XnLxeysWzYTe*CblyJR&yyO#|>4?mU4snNRyX#X5J z__Hevew6=jda2$Y;is-CHncg&G{=ymY{WSRUd^1op)(Y@y{;dX$qz5Vn|@m%3kgDG zQ>^D4X?!Tgk>L?W?cO;$pBw602)2zTv;2m!5eJ&RBoG+ei7Yds3RB@N+J6C*>h1rd z$Ovu%LGF#yLfYd+Wut{SZH$1}p}d*5$Cg-GbY)hGXPtf+QqUy?VI!>_W2peWnwn{ojC(A0XtnSBzu{D7zvH6jrH8peIl5?rBi<{`ULN7Vn9ym?x8G zCEJe~e;Nu(dVAD5PKZIhk?we|m?8dbANGvJ23vkFo>rwn*rYL`E&I~98D^PH!USbr z+R~GiBxBRx-6&8~T3>To>xr+Av=RG`dz*|8TQ5P0@Zs>tTI6*Gf^ya2SW~;)J=B4e z`Px-v0zs*FIW(RLT#RO%@t=BY6?c7=YCVad@mY^-A4kh~lySSL2;_IQPimXK?Rhn5 zgPThed7~;J!6?2up^qrYx3X09t!DzdOdvroUhbC=&~JzTEnTMxeW~Nv)|z7R@l)E? zozl;qnK)>2z_g@Z3Z=rEQIy@;wF(piG`|6nYWm3d8Ku?enm+tv;u%1#LUVkl+pD;b znUMwPO?S%;>d&f7&7q9Y9Vvo?Bgon&zv{v_?yasFjE}AQKe0Vd7qAAcpr#V{b3<2f zk8&X2`0446=u=+~%C(oi*q0_fD_b3!$}?KHL3e5pSH^u%$dE#og#uH80s^(x&*2jeSy)ze5u+P zRA>s1BPBc_vmv*N?W1~akJbU$@@Qypv}SqTVGEcPRf*BmneygJSf10B#Oadpj@jW7 zY*=&#=4?cuasS+Kg2k5B6{}k>-2pmvcYYmPf$+YVed0}4Y%sU=Q+UF z^v|G<{G)058`s*u4eI@2S*6JR6Q){IH8OdjXX9C)OW3ZIpEb|gb5XNSSG()$@{9@9 zB0><)rMLae+&R&~lN~#>co*UB5D^rNr|N3Cz+$jM<>x-Ch%nDa_jeNl{Y>EY$e31> z@H?KV6N7294(Rrj0Oh3@DiCQGwbP0YPTpCeoN%X7XrNmU>zV?1-5Lmh%dqCs1*A?NA z?hPi!wN_&}vUy(?d9;K*u~n_b-oo$TU`PrUb1hy=M%68{+Ukkr@wDmAy1C^lWE6S} z|b2u9)L99!9{CN$xU| z)}5P`HVqgyeD++vxVBpKSaRy?q8Yy`R8+Dc*r1EMfd7fnLtuj~uUK3HLcs=uet^bd z6mPJ0HE}?q=h*Vgd5V26^*Oyayf`h{-n`qY_1VN=lPK2ED1Il}Atdw%)BUH{qoNTB zw-!~wU^S+~8Z$~>iy)odCdEif#r>pBUkkCm0F8wiFpgBJYEIWMKt1wVTSx?1)dKxc zX<%T#hlu$F+Fj!Tggv`rIT$A38oYZb5EjDMV4*t*3){Mu@&RcTPyp7>&D;iU#$8XS z$AeF_tIlywS)b@SE7P$n!bx>vH16Q&t>iKUjB609e4xLYCeVi=l`v+( z6_Nb;5Sj$0GKZ25Dh4zj?BP2sL&KORjg3YwN=<TlxzZ)?)m_$Vb{q-jf~oB3(tS6p^?x!}j*l9!3>$Gll>e;VhvkIq>Z$Cn1um|&Rf zS~{j7ns$f@6vG)Qs*?a?A5Y(%QZC}2HM?Mtm`%H>twG21V#Yh3|KrzG_}sapt-rCl z_S{VFe`nD>f}meo=_1F|Gtr{xj|lErw3z?=P{R&xp8J{A<+hjn{6EyNPw#z~%xe^K zV}lfEeIBDA$j2#;$eX2yPb?pQ%gqy&mXWsirs)(m-fUwWYrS$~5l1lx=j%O;ePtTK z&Xh0`6{G-?!mwGYyDw-y+|g|NQuLKOy0G)jPR)t<95y^BcAVx}}QgbirfU&V)YGb8wZu<{`7XKu<(EU?P#P=c zi0{(@wu*%PKeeB^&3|s@ruo*L56fZzo_oMK4q|hU;E-U|Tn~~^0E(R(Te=HM*rsti zILUe1BqQ;+MGhbW9YOfsi`Ut#NRSFwvc2UkwoZ9ag$cv)Kj_jtRIQ2EHC5M;ARF4dc!0q)*W0MKn1Zg)Ll%A~?>O=ahj zd%GjXj+8Ps66$Fp$^va-Lo*U9Q2Gk9x|Yoq%W;dn2Jn~TFFkkRPAJD>^L5mgQg7n^x)R4~Y~<+}=aPZDyC^B!2?;8p z#3lp>>0r9c9&#-^SDE9Xl6( zoGLb7djN6L(lX0fB=RsQiuB@lez(~=%a7Kj@=rR`!O0J$=alYc0_-U>Qlvvw73#+J zBixrYt%c|frSWMaZH+R~#As>}gTt2aiGc5Q96ploT)4eqab}-N>}9G$iknOdb@eAC z2gUT;3W^{8?|*mQjL%h|NWvrVFY$Jbw&o6ll!K+lxtH`uwqJLU5oTO@K$BT9tIN*B zI3S4Tcy25Qx)#roU!J5x=l0dHL(KCh9%{kxsyjiaqyGYg_-*fy|4lDr{#V$`0#k9! z5*x`770Gt*RWt0HAN~$BB6(Xmma%(X=K^<>BlH9S7 z86rXz2Q7w%U?gHm5DOa0p0pgO9l4UCI=G=s?NUj|P5j*_F4re}%xXBdRe98X426`> z@8o(QBIsTa{07y6nHeLVqghfE)Zb^=M5)ce0unP{v~Mtx+xK-|VAgx{%2ne5r*^wc zjPA(91ckR2$pM@Oi5EVpM6upaK=Ms8I7#aFhi5jRBrHF@NGmHqR~{77gdr+DuOXwn z-gA4mdug4ywl^%+?)K_cY$kv?f#K5e%LqO~V=f3);b9$uYIBb`cC$&B)yNT}$ zy!ZtfzXLOrh}a6JnU%pNmjPF~=T7giR+5~>f)nw#tuTclE=&C?ae;B%fZo3dL9yys z*QOLn)$Sms58b3czRoyNFr3Axd-b{ZV)PzCjbmJ1>igJP?RXkc7d7h3B@nC=uvJ|C?J@{&%2s4xs={Li9$=IA?G`-gj`M&ro-U4Ii9_B&}8j8DXm!mJy zxzu;5=f#&Q+4~$^9w|=g<#mqzQ>X_B;!#g*kLYLks-P@p(YFaeV}2N;|9>Ds${2%# zJM&Ut-Sk+Hea5N2vwb4R%+Cwto6F2tYwfk*yNa^&|1ENCyImMRvotZWvdRI_T0nEm zVZ3A@Y6!ErRy|S~jS86t27SR-$T4*EZ(6a`O7Z}@uH9=2rR0PMRl31x-pUfc9 z;>a&gSx3X>ty_q7Tq7!dt{$2~lglCE@7{c9S_s^FMWpKmUC`_Pj3w2a9)Gz^h^AgD zPGL+PZ%@_Q#B-BXR=eJKKw>tQgZB9g{~pM`(R0EhouP_Kaigz(^6K{rp*wr_;NMb^ zVK2|z07DYDPq{(2CdddV`>ZA7(H(mZEyJiJYkvqmZ+0hv46mGM~9Pc?}IpH z$STz+447%R+SrC#9^6>VUc2W~Vqqu<`^0`|AV|gH2qH2wFtQR8b9Ogn@9+(pA)#x* z?aRgJ)k-7r?e%Z>^5{*6fy?Qoz1z2(m|ukn$7$9PCeCQZ*+$k8vcqaBOdrK3s#gt5 zTjV_Zbj8JPnMz2h#&ZT+5#JX@2g`U!Mw~A^X#)bYd%>WslCCgn#BnEM+oC1vDya0) za{e(x;3}qL)!St93t!J1*;iinc{Y`GX<9_tR9#uw*hr1)JES1uA2ci#kr#Z7ko&~j z(I=HuRonlmi*m}G)tZF3Q6@61=`PS@K2X(2xbHrTN4C%h*D#*r&ls;|W;`qGr#&4j zX?msqFiro?T@v{uGeegfz(}Tm4(S!$-^TCa{zdq14N?a-S18=n)1ar6?f4vQLGhC- z)&BZepjuJ4on=_8$Wm3|o(>|u821)Qn8g=rlju(g3vtL%UW9usUa`a{Efi^vAn9f zZ)>Im;b%a-W1tHKW9+<4;3k5&jOj%%t_8vv91=`m8?9=scVNH&Lho9TLoWsk)s1}R zfU6T3wc2RC_?1W#eMAp(T9@QkixEZIGl31kCn#ZZr9I>#`craI!F8KTblp?ip1MLs z52P>0*g$C&g98kW?vf!XFouw;!EK=1?AjRsdEPxDWf|PV*7o&@F->4Qkh=(>!FsLIqqwf1SGuN!Nyd%{u0^ib_r92ox%15#d7$)yST`!)n+nOaytuBzY*H$XX%3;AQ^NXUiJ$kZw>W z>N#BCg*dR_ama2I`836yDif@OtvJhZ^|CK#eALqWOI9pX$4d08yqCH+Z#mB|HWSO& z(fG~*-GF@USe4RkOqirtSH9ZGco+1g0q63%dh?jYTYdc{C7Zgp*A=hrdn%Q1#bLE9 zlnc_D@O?izL{a7|I9V}d2O@0d5b>v}ay``>HvOs2J|Sl{tIkZ^_ZLdgyI%X5<>b6T zS4+74s!=q+5Ju|-vr?j{nzVgo6nBz%)pSuIUbmp#Cb4z3BZ{}r<}K96J#74g*K>%F zsLZsH4D~quIno{nFgj4&4n*td;kf4Gg>d!Ayypfld>UU1#*d%fsxIBcyTevrN86~8 zb`7M9og_{&h7z6=tVxN_0+HQhg0au#$dH1LcYI&^Y#OeX2|~YUa_LP0<)M(#d!SSh zn8HIDYBYU%3Pap^a5F;~Rgj#SeQW1+TIX_MM%d1R>HyJ;%8&(~bjZEj-RDQG!0^)> zsvM(zkOy2mgNb}aX|C|1Co)Lq%bf5tb?tejdH`Vzf;lhmyA!=wQxKfq z#EPLGfbMu{Q&TWn3UVUoFRaij(?Nr(JRe4r#fPg09 zs_gq;t}fODt1$tdDEe#U_*MewrV=)02+(ao$I}NXnzj_V9+XdxptqRliJ(~ialv{- zjX7l(ZpM)4M_{}m$<&^9xK$z_2D~kGu{Dv3ztH#ewh)kf(z^UQq-7M7+(CbZ{=6y; zy18E8uX(Te;;Sfk=L)AV>)L{kwJJGj5=cbM=xNg*VCTZAK7xgiwO8ZKD zdTrG#$@TtaSgTDoYyht@4o0^u>m-T_fDi$yHo7rI0Dno&Uq;W0CyF@FJ%vGN5yt?6L2Jq_cEUESahMq^>1iV8;%$5VPG7*~@KAnjFS2&j&J&?T9Q zr)^ds*IAgf=^)E>xX(O=*V<=F9dl+q_*l`Sx(>8}Y6+Bkkx1Su^Yo6Uu z69}>Jyv3u%pa0Dn_QUl43s=bh(?1S#>(-vo3-O)hCWdO>t&b1?ViDuzkwZE7`5pdJ zjLK!ysu(65o!<4OZcSevK(I1_&u2m2Fd2uEPWsfXjB@OrS;6f1zRybGU+p@}C7Qol z1=}HGDnM=?9gs&(+ktV1PwMTERoEZKPX6cjKEWChzsK(`t0we9p5a?jPf?qx5 zj%q6M{$=4F0<-~$hp;P%Qa5l@Zw!i#K;=)NVi3Q}`}4ca)1r4nS9XGqgi+n6n6AFT zRo>@`o&Wgt+qMCjagfFM7-r_6AftzmfVS4QG7lU`tryK#w2+__BY(s5mjFh>=y4z3 zlGzhN(nbcN!}vs0&zqr-DXeJaNyheq8(C3kO#Y%nZ#Y$?N@Mm-o1&%@_I20*K69F7 zgK=gsNbhV=iVO%sw-YvzvSQr6x}0TSP}pSj)>(ColYFmSc;);Fd(j(>G%E;&*HCO$ zmk0I|IEOiX8rZ`QySB$Em1mg{iCw~kr)=^;+z8x*$#=T{)TxM`RyGM#XH`Yq?+pPhAy#0Us}xmqn(^} zng9cZ<%cX3%~+BGk=O7Wd~pIcA6`u^D?Q$8EW+sZ%8(vayp7BYa_WwR^oaVdc_!U~c#IP?1c( zK&0u4%=WUwj(&)-RnZPQ7o!phLlRB|^A~g!XkKwaD=1BN1>J+i_1~xS@b$Sxl{Vok zTVK`yA!%2Q`-Miyb+{#51%|T6nD$7wh#r%GSk?ZD49u>9hN@o7 z=eOs>6Td|%QqX}>_Oc3Fi z=?hGt*JO$We-{FqPlZpRs=(z=kT(2OCxQ-G*Q5|FQ7h`bwXkz9u;i&dUBP=G8I2afcY&4nC$xVQU{sgx zSBvSB+xOx|3bn6k!Gv(N2y6+Xq@Vc}&A@Vl$A&3~x#A=YZ0D-reo0-9k4m^CcUp6dK$u z2yJe+H^1GYsU^H^Vc`sp^8vyBdaZ%!<`YFrZ>F1wpA(A%zb|=4>usZ167ScT^B4~! zIr>*_SjpAAI|sgZLo3dzp{H@CLP*+ls4L?|Kr^@uo`@T3F4senz=-!I!(n&WKtW#h zNxlPh%YsT{bgPC>_sDf0;fJlK2FvGq_E(~WPMD*t4 z-se)fsu7$@Z@C7f*dG!5SL?bOKhldL{A(k$r6YFCx6=?$7BdA{l#pYqzWg*Yt^r>B z`A)RKpp0*BXQKz- zG}p}za_fMy@!Iy;$Hkj0VAxM*--DYd7f4h_RCX+eXaJgowV(ov2%ylx&GK!vNQxL9 z3}>rCMpdnGm?|9_8r&%FJc6KYLV;VK5|M;tG!pYkm4s!mD}c6m6A-{HFf~YQMk4Uz ziTft?7)d zVZjb2;Jifw`2`kUQ6PERh=yQV_yss+eDg@w>cL4=!y8M==Itzs@FFssW1 zvYYl{DbgCW2e@iHH53enESTkC!6-}Q(i{h)CWU7a&gWc|Xq4$fFB&^FSWfQZ4M=RF zMV0LlIOQiH!xeY^1oCe?O{}+)V0C4D6x7g+b=`ng`}tNM-Oc?kaPnD5X`{#S#`4n# ztqfV;YyzZMLRHG7X-A333M|WH3U8VCCC>?MQLJE+YE$E0pMw|7ULV@C!V8nFhM?+E zGm%sjg-U)!x!yDIdVZozu(iLfYISqNgXcq<)vxWMnD3;n$dR;~za5@!0%+x0Tahx}e2TddGF%X;iiUY~TlCDwx1^Q60+INdI0! zUDZvs0$n>xS2pU)3U=y3^DZS}ULIR)iXD+Ngms~Z&23CW^qP~5ZhvY%fv57pY# zTd7nQ&M%EgF=%i^4j7dUM@V|D>brJ&cl1(~0x;h{+MygS>Rf70<9i z&(W^W{J?`c)6@U-l7})AIu3sk4`FnGC>c?;jpi}I5No?*;=}E4v@Jw?DAY7LX(qFW z^%LJUK!WF4<5Gd@vqBBPvz`V9Ur$o1z+t7Z&_bRDYf!LRg-OqyvDdT|`}Scv`t&If z!tH<)Q|*c;*o+@olh0Ef(DkAl>CekV)T44Fx4FltW&{KyU$|~Kbjp9%R;9+C#!yMK zK4n_D^}H0xn&5b|jEgdLjN(UvAHHkSx%gE52y{L+@?L^qt>{)kHX&b;fK>S2H8G0| zp+CYqx!*{6V_j_nUf?fo+|k)dzy2KEkkjM%wW!zvJ;B(5P*ztTQ5&*)TW1O|`az$I zdyfKP5cFLPG!G-HruyqACE8V2wxYYU`olLIl;7kdlM;W=&9_QHM0d&iC(P<#IA$~I zjmC@&i4R;)aMOKc4$}N}U$1zF&%=k|$(c@YrNgj(nc@edP#;(Sdf7ZWn$e6$pb?Bu zgFHTSHwNJfn_XP)9WTwlg1GeZ9fwY%%XQWnpZbdN5Bv8>8>GU$(;nc@w5>tmZ;AsN zPpXS>3w9Zrajz(PZdcUhGxc8i#PL2$1#zcSui}3az-}GOL`!0-ur%crB(ZRguGx|{ zB7z?wI!;B?3E+!sQsE$ZQar~ZFyncX2+EEaFyOfxbQg&FJ2IjNo`B*=2*09?_gCCq zo(ivoQ#_X=j)Cs7@VA&cx;|*CjeekqQeebajE%uNh|QIe9T{E55v~cfdsi-OQhoJg z?}qZG(CB?v9{~UkwYFz?%MbCM?%jfZ-2C$C9`FT7H!)i=L$^;`*J7f}WGGLc(JV_} zz9s49dgl~84^;KuS@K~}(x9m)fxXzmLlYXkfc&Qp%PfaMTfsfCG{oF7D0dw(0U|A73E|0i!72H$KOdla&GJRs|!Rq1tUHpx5syp+YhRYw~`*x7kYU_MV0VINh3p7%^?fK2?Pi9<=%zgo$&k8GUKYHVcrebSK{(v^0;;Jt#P-tM*mO#zAd>5*#bc*L8=`_i+^ zD~Q*aH!0OPK}uaYXy9)`i=EkX|9HD(b4%$y*8J}t4UpUTvg;L z(s3ZLuYFH?H5T^vU3p8(MAn-~O1nNN1VJQMCVrI@j>5t)3$i{xAaB=cSX3gMeA#{L zk6<9lvI)X7#ZqAnZlj4SJSIactn>fxwY$%duGlTbc2L~tOUARFS%Tu+_rA4wx(nv> z2Vizsv4~mAA0fPSoBmh+3(^FHABpf4r2i1B<^X|)AUO6C4j})$>7O*_|4||=bVUyj zln6nlppH^XSQELXnETJ~6@CpQ^G<-WiZNc{K!Ql;1ff)_V(=JBi`9%X#bbI#-kUZW|{wth-G{5p#-^_NG{4fDKUc4}r2*?zn|qUx~a zDP~g3)Quin>)>G(%#$&>(Os)q3$vM7yb}de?BR{RDM6n1y4QHvnC_&+-lX*V{3*Dj zZO(rCjkagqd7^PNToZNMbT10u?oJiMH13-q+;=DLH4C8;!VXEBd)eU(5~W=<4Yjcx zNDVeELGxMpb;MH$?ay4I6>Rh<_u3E0J=`9}<%ddRS4q!uH?HbH+Wn62AK?{hZPqf3 z+_h+(F~dG2I1wWWLfPHdC?V@c`yEC(3Lb4*&JOv6Tr-|j?|El7h_n?90_wvLVWCLb zLMvUmMbFZDvYD4p2u2}M=~PO7o8-Z`uK`=%6M;9tmEFqfJ5@b0g&wji`~++6hn@`B z5WL0KtzK>n?ZV_o)RZC=8TOqOs$RR|HM)oBde=mQaW{)a;@acZl_n+ouVu#9Pg&3V|*`&%T}1G;iCHVM98K z9pl}cg?NvL{;>`RhL>R&vMF{;Imn%UPzni15$Fx!w0Wi^*@(3pJm+FC0Q%jWieSZ* z0%rfvxp>xa2gH|< z-?S(W&@k~-=cHwZNL?MFe>2Wv>nMG3#QudabEpa^`2zRAYpb$sks5~}xV)&X-pGhzs)otI+2 zc!sVw6s-2?Wi^YlM5n^UXWzBkmr508anUnEcC7J@hM+@S7oxw)#R5y?0KFx>@`bpX z7q`7Vc!ZCZcVf&=uzKJH48*n~Ty+(DvKU3`bJ6&e$9!fQS#BUhMAf`#mwL`h+)Rq2 zXv*9Lsr+))=Bs)3ZaPu{V(RVPg;HJLuNx_B-cjgo6H(|YKj3(s`00$=GdFNgH10By z60BLG^{#62$S&hlyb?t(2N>lHz}rig{47u57o8S6eb4Sf8Wt3opq%{g{uAUav}32d zyuU(9sL+|UoI$hXEN7L3s`$5qtsxlyaRFzCKgwGcJ1>Lk-ZPLaU+q{^x34d2OSwBX ze@1p)U|{>{4GOlZsfP@K*o%fjL}S@RHV$t1GhYFR0h;^9hj*QGh!YhLkG1u7<_`zqmpKXRn;AdW8#t*>4KRyfWORR%^+tBJDZ*ujuN#1F&5W4n=euniW$-b6) z5a2K?g)p8kq3i_;DuF4w=RLzcJKyq=?lz(DeUJpciM6+no$bOsQ^wP)*YM-|%c_mH z`%7&-yQg0=)DoW{gIn(m^6_)RBFwaJOTM7kKTuPjaNRvqhd5pkvr@Yxj~BSx$iXh_ zBbmZ?<6c+UtKBEnCBLMZdf$F$>g(+6tukjCtzBYZr7nd&5bQ@4_`u+%hz($028_`= zEDZ^Ev!}2UH&t|B-2y?v(F&cHcfvTnGm*Ni`@!x+JQtN-*N5PvsK52|J?L~LF5*k> zz=ZmhzO17J?ka^~9tx}mv;Xu9bhKC6uqwvh=`e4MLrLvv#6otgKwp&khU|ls<4tGY z-pjJO#*74NYA3cPVi^0= zA5Aw-9*8QdO9R9ZcY{h;Cp9Q$&R7qI-Ao&OhxBTGotc-15 zh9CG^W{e9=G>rdzh5qUNky*#T$>GI*gM#p#U>;otqV_1{eBH&Lns(l;re6gfcp8qZ zD+{KMzjd$=UGj)4i#1OxOHNLGa+fvIGK>aCeAfG}XX;3=bMO&Bo_ z?5%!|daqHvk8#yL#H<4%>DO-3;S`om+2W&l1Aa5>iHo%BzvZ zGmDlJvJ1J;v6N34!O9&o?h7Gu`1iaYZW+an49gC(94&FuSJ6{kP^rlukPCd!LqaM+ z#tMPUZ$EX9`M!=BC6)OZF?Oee8|IE4!mR1JgCe04Sac`fu^Jt~-Y;1~ zm2_~7ozev0OfakpgcR?bF#t4MEccYJ#&IsQD;b?K?zTnINguvxjcZ_TdOwE(=BUK5jx!~E&DA1`^w0EDtQI7cVAsB< z?KR_cS94Y6)yB;C)(J07s%{DHx@F93qN92X~=QlaQj!4G2?B#X2ej> z_9>J!s~eLqTZ0=+I~$=*jm|9K&wXWB=2mKxFRl4S*f%HP26<_+k(im$<%Kfs0@SB< ziaBd1iLU8yy5fjF=%k{ARhWdAo1El zkTJXYsX2a^$5i>@1ms;s<|AUDG4ExYv+NzlPzuZCmu$gxSkFuY4kT%9riaq9oh!)Te4|j0!-X!1|4el zo^VRq&CH(@@MukI)RzBfa<`1|PJ0vLXHKpmiy4GCPhjfW_|=vO6tl-pa($b9ADfuY z^i*T0(Ydqm6Rhv>gTeM>_y&rCYjBU~8mhxPUv4wMH&+^@(Vqr@eF5v7! zjDDTn=f?C@Hc%yh-oCzuA0kjCPQuBxqx%7j%>ENbRw2X&LYM$i{*Mw17wNb5HD;#z zz*nJwmPpfOCI$Q883t5#BJ3eT!hiA`V3X4K>QY{x6TEHt@ZRZf+ll66N<3V zbof-$cxkGrC?wfZcI&Y94iI5iU`RW@LC_vps$}tE48YwVJqtxH?SoUh0AU0mh5tyM zgfYPI{|l}o8UXe+N*K3xEUI9panWlGxbvyz%1r3zkqdkCAo=;0*AU(A0e0J>TLN~- z2w_SDpJ=Nv*kvVq6IUV{Qkb>wXm*)iRfBZxR08V93auxt(yl4Lt@7#oa6)omMyr z0D^yUj(^GL%z#sm;}$C^nH#UEkH(jGGOkv1e>mFZeYH@MH}8|B=69L>;!3=82%m9+ zomM(VmO$3rt3Al-K_BfN7%-(*Z_e^~uZj@5hjnNtT!3I1W5}=-L|yVFe#&%W2n=?SD0CV_bqd6fq$9qejPi-@0LOz zuZ5-|$Yze%AXtwsO>9X{6)Bq)1j7T<42!bX};*4+xHg zZK=9W&7offbp!8X-QKNt#|@m2i3>?V*^x>@8Mx;i*rjdltTwby@FivjODxzl##!Q<$)=J038 zvB8Bah$}+_K|z;DJE8bq!W;SbiR{lhXhj@P?yE|}+B~ne^uM6%U;I!HNpbE*@1;*uVbCsW#6=vJ3KtM-7~tCguZ{W?tiW_j2|ddOWm+&`$DV)T%4b@oG& zVal_U2LwYhje~;;+G(+ZzRI)CjT2vn=t>b3<>aKtdf#RVr;3l(gt(tfzmVpPUBsr}N!eX^U~rGQ+3F9%BZxfkk2&8viVk+ztsiNINfxitNX%s zzq(0y?moylrruYBdTj9n5_)WptF+i^0f`Tj`|*`~vCF2!ea+FuBfQKDc};2)g9HT- zgQO95bT`0Z@2jLpC+#nj)7~E(v6xElmJ!^OLM?ugz}!}ll6e>&QDD+;eFj2l62nu$ zbR$%=q!I*(xgkgBCxWd(>@Q8pyg(89{zh7Oy)hw&J^4+`VpwAJ4@in^zR{b$wNLCE zQ7W-5)hF%Fn8Gp(F-NhR+Kv9CwMlfU`I;7e591b+m$hr)>YSgjTlv^&eht?2_Z1EP z>E4@0V!;BJMO3Cw>ntGk$= zwBd5&z&_cX$U(0MCUeePPL!s{>o_>I?ivOEP*mPYWq2~NSkUImTcm_+f#>2`RS1M= zJ`chw!o(l>9c#!i+QmtOcP`)7eV|K&^^+E9Y<*wi4n@W_)*}+A-{tyUiJr@mkO0y3 zw?CoqA%Z>Xej-u;Ei;Q?XKT3M9g=w|0ZJ?9^el73+%MTW$jsm!KU1xINWD@SFkj7= z%o*JAL;~Z;DAzJt-1z|A2Y%r5RR6Ncv_^(SDQCN}*mH&FvsZ4i=7{WTZjHU5jh3W< zaxb5B+|d{XzRMSl-=%Uvc0;Ojyz8s^gi1(8sT?VPjqBk@>Uu&j>F8Zx1=K}Ojx7=v zt^+zzXP*Mw7X9yl%U}B%vizBdLM9ic1_|hc6#z0r1>#uGKxF9jUm$h<>g*)ljsg%jo{&m1_V z+*rpR8a*Rh8c_9F@Y*xm3+xg8kxQR3W)dJvDM%=;Gf*+wNk#a)Z18*3dPBR++o0*i zVNwOKDgFUj&sotNUz%H4u{ytEUs_gbtI)S!&OQL1eJ5aae(|nePzWa5sx50SUe7DQ zd)jg4`O-5Tfo!jy;){tBj$qG(5_==2Tc@0aMJuAwLs^TYo-Jehrw`%!?v0kIO`Y#b7EK{4k=M;B z2)h^KUVHTOo|k$VT49=*^K4;h_uga~N58pm66wQZzN_~qy-nePy7N71G|`AFSd*OF z_aN8^lJs?U5AW8k$a9;b(d8<_DV1lS(8}sG$U1NC9t{g#5sUk!c0!k|r0gxCRX%#Z z%V}6cZfBhg7NNjv+If=ME#VHIo_})eBf9F9cr_UwIIHZl5P0hbPK+I{?-B4-eV)cf zeMugz1%*X5S5nfIy0KZS{G={jnjnjIV}yM2TlNH>lZri^!u3*?*uOAVjerSlD< z(Gfm&HJ4^JLI~u?8{HJG@ZP=faTlWxq?Yr0b&uu*?T}jl{&B zC^K@VOR)FvRIFh0zKDwrEcB(ROBJ7PxFHdwFSWQ8(zM1)#67fzQ6Htt;r{k(V8=QgomN=gQey}%q}*zr!>T+ z9`h0S9*kQMT@TJaq{NLcU$@$cH}`DvBkT-<*Kg5p|2%5VSW zp#Q_B9^%*zub9zfL@X#kP7)@R7(Cn?4dkXqm>|{f0yQzjF@uCY)+_a%XsLJLYHD#s zg2-8m{#O#A&?3n;%-AHdsGZ3x3HW2%z*;qFL52;&SbEEVpCzz9%9Kf}hzv!12YGfZ zYY4e^84tjz-C#9{bFQL#fdxjNnWqLH=ku}hYKZeMH#EE8l|EN=I%iK7jNInF=hDR{ z)pL7`anA*JGo~_rnm#n4+ff1)vh<6=$2+Q|{PopN2iA*zc9i3AR(<&VZ3ja6n2`1! z#0s&muy0a{vG2B{9rKx*etx+!wrMaj)i(K^cboJ_b&hePp1%3E-)OYFg4j>5Zeeh% zB;(;$!9Nr}X&;H!wE7IVp-*(w*yj8vGL`Yu-K^b*z>&khAw=ga5^Id^^1phP0xOVp z&Y3rRZ{~VaL-~oam4|7YXYn0SgZu&U0!BDUx*n#+q>4RwS4k;(=6-ANIf{C@n!JUh z*UmM1fN^MNJElmxE_*Ic>aFb_;|XLO8VL}YN>*=`RB1eG-=Q_+4vWbx=S&hQ;N*pF z@5CPv2Tc*7^J3O1xMyDn8kL)bK-E$RTBvar?hdlMaL7FI10X%NEn_KES!5 zODSHoqa0F`d>x-b#Td!8eVE4{dLqero3)+(Zc~2cDE167uQrXJRavz^SOW0l+}TLD zy)0o;9%=A2c7GFpXqC%KD0{oAL{`A0h)n-X5eE+>aL4nZ{SDI!;fhzJn3n9z-a_dd4BEuB^E>$QSrcZ9J=?c#BAn?ru5aQeu&iBe0EBgB4H7tAPa3xAPR~ruvmV)c zAt%+!SWTB9%8lw->;Q)TXZj+3Qka1MGp7-3{55;9cT@0-&Kp2*7Ozdc0Ls7UI6&PX zp&G;zrzLq zc)5)%%Z#4*0nwJpV&4J>%MUnK{D73!1*0D0dgFp|HQGJ2Ya1t1$)fmh%vI^APc|Vx zAU>#<>znQjZWTHmxk04bdVWH?DPHw_g~4~Z+rZ;I&%*n4CyEPIlGMd}*r)&x-;v$L z>ZG5dUb<}D@SfrBDBwCPf-@p~Mv_I15rvFKzsVbsEHE&LAIgk2unV5q);pl|Q`TKF zffE9g&S>QFhUAteNF~&09qbNZQey%$f-vnlfKKMGIqQGqe-`oHBDRuGN*K-O2QO{! z+0FWxSc!pS$ojeZBou&a>vT$o6b5rYL59xJH4_5(pgkNnJm+>`ZanT5@@)8b?H9KF zofP)Jd4>JAJBR`j6zMGN%}Wi*&^hq}6AvMc(~&alSs9%g)jm9fCrY^9JU^Qet6fbx zzg9zm%M+Yr7k=!edU1-#LrhimdbP6R)&S+wMg=vQ!rsdC z85)YAlcDhIp~yWGkxWJAbG4J+*!yNmb{NdKm0?hac3r@}eXcthy_)dlpuTB~I)xR{ z)1Rjej_j+yM@@dX8>wc#bTG6+@xbHb_r|WCiKr_&(E9*OQ7q6a#g_Vt()(7yn1d>9 zT(jC!CQE%nXhD)K=-7@5)tc1_yNAx&vJEK|``trBT5bei(aUfRzMRYTLS9Dr$xjQ z_}`y|mO$4?w7(&Voc&B>Eg=X=Oae)k`$71I+$d)Dr4xhuFTH6X=)0Z&`5upKs}cVO z&%i(@9cj#|;r3oa27GXtWq==tGQ$9Dd)|U&UyjTD*JTx*1`=5r%nXyptc9Sv zbJ0?mKW_rj{-2n>|AF_g$wU<3iEMyb4CRHH)y#Lc{O9)$ztGdakfv?bR$-ZO$jQl{ zM3a7NB$F+RopwWLhxFlDaR;`ODiecw>rA*lg(EuXwex{iADgV9R!1*(h}qTK=VR^* z@3d3SO>z!7br#+#+eiqvh^Se9MBA)uCDXlgG7dZ`X~NlyElW>9OC0*`JMRkRw9Id0 zvUVGg9eqZjB-|kd2R&^_bNT%F80XX zEp6qk2oQ{6+1gz|M&6soE*yGkei}0%Ho-^jBb#edcqdzBK4qqMD|+Id4-Y}qx<|Pi zO-)!^LOG;=>)R_SV)f0N$*EP@yB^TnTW3`KY#Z~+0Bs9Lku~KD9|y8<%xO) zgiyFBJ>uwt4cwlBm(gXJQO_So=Y|_d48dePo4K_QAT_P#yN`wmggnAIi;Ql>yj7N%2eFF-_`07(?|uouBYum^No0BMPmy z2Slf)RB3YiB`Se}JWCDA2xq@scFsp4YU@B8dL=+-VgIq4>Xw`%ge+I6c6S9bZNNcA zqinFBR74)Ms7>7w!)Ic`!`8^N#}oA&7&aN*v(+Pi$fo3&J$kZiHJm|0++2{f;k+l6 zis`<_dl}OwwTbVcHp)Ue@){v86>S$HoY0tD{tnV7!!NE!rxnmR#tzd&6i zf{=GyptYP++%2{2)HR;{;l(alTb+GSl6Cw)0X>aKN291d6dU@G0fnPm*TCj=MNt9$ zRNJKN@{mCBc1(H9E8`vR1EU*0WWvJy8>d0p2kVkdU1@8QOuFmYhDu?Dfd?yTXLp3` zL!CgkAPXKWFCH0V`dzQ+c+)}1ml-R%I$ox9^qf@C=B*~-(M-klkbj+Fe$Mj$sUK6g zlcC185`Np$2Rls9D6g>+R+<*M>6(tXodtPm4(YPN<{^jZ(%Fa8%i>sN!5+XDoF$uz zH#+_~FCarkS&AnhQ(Ahcy<_6bDSkc`u1ddKk(dJP;t^Xi&JZC?CRw)gj%Yl8qb%3o zII+z!;tujI^0DweT(gkgr5>N?^RB3CCC^q^l&Io^+{s<4#nuR9%LAVNcDn#=54bun@*f9Ki1D+e|U;E zkc@#gdF@onORpVQMSha>*&q7-nS+dCM#TYTYdRukV^7PWDDOv`#Si*+G> z@RL{3>>u~pn~H5?kO_gvI?Mx>EHwuYl}V@@`D>AHiP&{qU|- zak{vF-MC4FCFSBBYEa!Bn_}&9cbV>wk*8XYu;^YSRL%$oS@uDd@GS6ysrB7=b)cr_ zaBDMlYSQh)OsOI@^P%ub;>z>Z9SMr21Rrt}WAq7PCPG$gjq zXCt34{lci@d4cZNO}QbCMEH>2IQD+|zi(s1c!2$)x+=p}D|5 z&82DxFQ4-$jaa=A=!r5I7#*1V<5*i1<@<#bEov@@u7yNCfsLWQ>dY+reW>oz%5#34YdOKoP9(Uo3?cqL1L1aEJEM&p9BZ^0mOnBUWW7{1sApPq!+1 zBi4>r^~^C+$f(6{6sYFa#bCR)1JrQD!LLP_yz5Nu-IU@^6Ub5sqJV6__?V1puslrY zNhhzj+UN(qx4vvsKJ6_FVK17wP76ezN~YS?2Uty6?wR84MXWFs8L+ZeDnY7YKr>Tv zVGWV{Ca)mZU8m1wqN>w9cX{#FA)Q>iYqmBEv)af9PI$r(h(*rv*&ytEhy8ui&(7mxz-o63R}mqfANK_vi)hSUCESDpuK%v z0Dp{D>Qg(0c}wygyf0t#b_Av8YcTviR{U~OEnoAFl{=sm=2^Vw7R`H2I61H{-X8@r z+G?!OyfM`s)UfueiHu8I*!ev+>hft7m@hqW?0dtFFnizKsquFk0>=aQb#M7HK+@W7 zv-3uXsGRqgS-#^p(eG{sCSng9JlGelavwvM(0>%kN;}2#7 zwD-M6KOhO9gXNz$U13Vkr-+ef9(aHD!SOit1|o0}4!ugvamlNH_)3dLy-SOkuE4PP z2^$uIKOnV}WGJxm<+pdI1I5|@@f-Eu`obg(c$C}^p+FxJ5msZ!DdEwT)wA%>(chj? z_-|Vo%@|3fgD#4?q>#9W4vWc@FHeyCs_{vrZ!%#tJbPC()Ru2$vw#k;?}f9AwF5ooL`c9j*h-?cipJOa8jL2eA9`2vTDcB;5L zgGY>h!U^{YQzN9W<2`>27wI#vH2q!heVI=Vo>8!<7RtB>n%a+sasgX6F#qk?g^-9h zVWyyWI@#0thX;r1%hsRvTb*mXvXJ`%#t33xX*;YO%Xpk{$h}U>cS}>dT8JJsL5dma z=~aVaT!JtXb`YUF4W1yjv#{$Bub5Z9i!6V98sWC=K>Rp#QHP=eD-{g z_WSx=hF3$~f;p&gB79(Vtm-AXeQYXmpkSV@%5kc{PpQ~|l|Me!BeV4?#t;Hpv%h<- z_34Ja@=WHG-roXfUV*r3WIOa@wuI8+keV@cl@6{}V^8rgL$I@_WF?Jqtp45`rRUz? zF2qdv7I!OB1yDtRmS$!Cm0C%j*JuR~nZMigyyFju8KT?!l7z~wEk5Kq66(>fTgpZ| z1*Y2cEXFQk7!z~_;d_UX6D2{k@J`$e<$ZP!l}|YMyxET-u90m*TUuq_P@ibK&zF=a7Y6b zyKPlP%zMid-2@le`>V}gUNbb@-F6EKe&@K)r7A?lKXIN#4oENyMh6qeb_|v^=1kzY z-bW1|5bHUtMGL+U`shq&ssXRdOW6-krUvdoU#0EvPGS}faSt$#+XxY|1I4ENvWwj0;;T2e_4~Ki zZAz!Sv?1JWAG?eC?y&hc+1M`x@;?t9{UQ_m==g6D+}Opg^m^wW5{bW37TGE4`mU?w ztWl<7WR)t}7da_s*NvR-W;g}|bB+qVvH57_yECq%?0(lSB>-FNU~P(yyT}EJr{1c& zQ~Ku+rEO@{ahUM*E9Bz`DhfAApB;v2ZdF=?9(`OkUb+UQg(ucJ=d5tnU2*L8886i!uMWADsM(s>-@| z0JX_PHdY;dgQdmE^`m5*iVEF5sS6hm+U(P2X(^9RHT z&}FjHSRyH47@j3FF0hU)39!(FQn(w447`ZTv_0U2K*9946M|C&SB+@Uh;cXt<}tdu z&TOXpvAA(NZKJKfFppniM7xlZ8ehD>*iG`2yWSX`mvsC)JA?~)RFW8WD%5!teb9OS zwZE@H$01YK^DeQT#V#Hg^=})3u<;lrl{S&P^(Ij_y8}_LiJk{{nY5CtMqUlkv1tv` zTHJ1D>t3@*J|TE$)&9&-wku}{r*?mpIK2)rah`?NB8#vMKOo%x$NSEcarYIqx|RxW zr38Jxapbn8_lc~1RVxRNkF#9&~Zwa7d2Bubz z(t1rB?o;)MhyU_LVT#y`6O3aCfODEu&Nh2eJp#0dq?*^Q>1Y6t_5g5s~yc}#LKuvZcC_jE66WZNgh39vN!Anxh|tOZr{!|W&8k~?QEavs8ctW z6(bL4s;9KdyY-naNDXqSlBPV(xy!T9p{T9DvOA#Lmf7=2t1}OkCA_bIE)(H$?W|){ z%6Gqb+*Bk>ib6Pj^}|tr@3nNEsvuU;7QDni=-$RPuN+1@*+NK#EJrfT2!-nUpjASM ze;>f+)9wu3?QvvmZOH^n;t0*c2$gZ=ce)RRaeg_bpT~ruHTySs_Wz05dEEvW74gp0 zyMVh5V5yF0gcnOQ^Z!H-{5Smh7fkvSO8=iF1b!Cl4V?E(Z+I>O_fJR-MK+>vlKuZI(CL z&q>f89XT;lzqp9{rUw}nKg2;eK_I5%b~G$Ke#>y{JY1h-t_B5BO~i4Kd_W)PgJ)Z_rofnAP&0dz7JN%Qa#F_@%7&m&1HSj z=7ifsj+L-*Gh`A9k0Kk$-rFJ4i&k>>9nuT$QZSf^O%*Rs4w3idkLy=nPlbJhLH$wM zWb+4Wgq^c;0V=-Hj(el?H&=*fUmzC+?XX_)BW1=A;w5vTJSuE;5ciAmx>sGRk7iE9 zF?iBghQdRvhE^C;$O}~&kwpW&P8r-FPWqp2q4pVBoEWaAH@-Z;n%5#fi^t?TL9cXA zSsI)kUa}V#cv7#m|5Ge96`xyy1SCu$RaT`9m<@JZuP86fKj-PE;7h^pn7Z+}-Q@{r z4(JVV5v7W~6GrU*B8>;o3LT18+nke42en1Mxn$%AnAk{`n%z1d!Uau7$6XN+7?N!M zoS!284Pk2Iw?l0*yKPff;yPpfj%^s&ZU@bT?o31LypS@2Uqy+$YSy5so z6RZ`&&B-Y>V?zw+zPou-AG?W#1p3+y_yqeN=hK(3n0j-@)4$B$!m!f)!5`=j;3bQQ zC;MHbbK#~&sBhMH=6`DcBYUD92lrK+z>ZJwQop^Q=P!P^Zi)ZHIZ-wwYi|D`H#gC* z*9h=3sGT5NlEQPR0qu6(N;g^T8XWX_#R+ zp(t35xKJEU4m6a{m5nq^HL(RXz#SJl>B#hrSjAUD^KdKioyH*<;BS!< z!Iv~d9!{wCubK3bd06EUdKPi?jv$cvi67(bB~`Ig8Y@>EMg)3g%P>v3QSDJ{Q^qv& z_x_j-GwSlVxQHCpfz(lj49fbh;dYQY*=n$*Wo>5Da7dFwt^d6~h}zv{n@M$3{qNeJ zC7fnywJX2<`rU0-8^R2-&EBZScOrMPs{0m5o)ZM_w~mc;R|%ndy4328J(xajs(J4S z8}lW4A45&f;H`tl2iGN-|<^fceHUA?~Cbp76rr7>9xz5IMSs9$Hb zQn}^v_9k~^kt;woeTelH$y;??N08?BY|uIhRB{}gZq#v&Zg30V0tlL&hiY6LDhZr2j3_qrnTv~x|c%qUK;$+-I#-Z%0N8{1*^Ey=z ziA~-q#x~t2nk43e$WfVE<6gl}I)u&WZKWR8_RSNbe7j3mO_Ptxo6braUs#*(pubO#NkQiL*qn8I)RLsSk=dH-oC0uZ}GG0&Dxrd?EhAw zG;Xl#i^X(2COv=s;IsU?s37G_G&5-O{+L_WeU+8*Md7v&o=it;4}B9Bcx|Gp%-X(o z8SZ$G6$oqY+<1O{$7GX1}K-mAq54Qlel_lVJXjry)l)CRa2WaJnKKIq ze4^r6OPB;SFY_IDREsj7P~au~je1v^lC>Iqs~2iJ)p>vE$3iQyDbc%q-4Rn1q6$42@6<7@K|RY>FK_%DR< zF~i5NUddxlp`NmiR&Lf|ip(1cT#I_iX6=@bJn#3oeggpv8YaF4=7A)}kOEl=u63PD z;#J@6Ez}(LU9Eq_d9$mLGKiP(!pr8<_;3s{_1UN>LYAE4!9iMU{kXM0w5R1c!ww%n z-dx0y#jIAkX0DcW9m8zdVH_oF=P2*v_FxF8<3FHc9IQ z{hmR4%^0u{pLq1EnY%xjt$ua&TLoSHDxqBjK%D)pox5vH81UkfQ2MGVk4Q;*3dsen z<;+`=g%OJ;ME3#nvG*g2lvC2vxp#zxU>Z8xz2?ksaIw{_5V40_#rdvOAM({uMHVzI z%8_R_tf`)0iWk1V6)!|4S?C)SEnhB&EQU)>zPWc#^eykjLz9b_7Gut%2BwIK_+rgtTzg0p?iYuPKcxYofB!@ z^MD^mU~v1f^k0sF2KPXIL#myt8pB5rh%X5%w?;V$^R=5lAchC8Yy!;GeN zqMgBWba^6#xmja$Kk+s)qUh9?-(b|mh(7zf;k)Cau1U%^Bd1<(Vc8%g0`V=1Z2XC$ zJH1Lo)<_!+)$}fqos-%=+4y8%`DjyYcjmOoqKMd|_QO0o7pe8F1;q6xBVCB07N#Xp ztoKPI#$#&+DdXZ|B1;b1{kx^k75gAY3No2FMvVtjdFvd z4BcKL+^zh&TA)>VD7wL=f5GFnPqkkY`JFM;hQ5bYU&Y6a5B_6y4tKj(4U>CR$lm;F z&DDd44w$)A%MZOq%5CYtm#^C$i3><Π5~x|yksgSK6OGxQcPF{!tisetxS^|L2p z9>EIGF-SSh&V2+2453^S0w89S&hdEi?naFMu2Awn1Fg3iG$=+Z9Cfl+= zfRIrsK|tkfxiRk(h;I;knQ`bp<_KJB8e@Bn>JO^m9)3_PEXgYgKhY(=D7E-)r|0I~ zA~CE5{SpiAMJBKw;T|H%r-y();l@TJ1Bh)RzB2$u|HghC#@GvJB)gGgn@CQeKYu~J z_XEQ2N&xI|cL#jEEH$JZ$PZ}8(T73lYsj6P_WSpx-8sj%Ny9{pp&~X>WJ^N$yjz+~cyS>hpJb$r#uRyZQ?UHZS$00n< z(>wpzH^Mhz(=19k3l9b(Tn9;fElwV9StgoSFB&RQtiG6@!_OaBg42aXLS2b=s_6cS zBkhblYndAn>UN^W8SCOH;RNGX@ysB^WZ|(VIvT7)Vv7; z*)bpi0M)T*zpKyBIXl8kjgAde&1vxoytNI@+9k>s$Mj`2VDbRa2Es3ZDimHVg!4I@ zTo8Mx7>n#}35|1iPca=lYS`Jhy82O2BGd<&Z?y25C_A7u)FYM=sN*3y-?u zjLXgj-P-cdD7`Poe&5t9(cYK#7Qz9kPX7m;fFo%w^OBK~6NCkZI`Pxs!!l;}8t2Yd zmycvk+&+EA>k+zfY^=(joD&R?j%fCg5*fVZm*|+qm$`#(w#=7rW;gn6W6N{vslxX* zOYvU@e_a^n3y|}PJ6iiS;e_TbkrUlD(|upshn19iKX9ndG&AH-U2`B3WDk|x0*GZ8 zoP_p!gphGQ0;NfZKX9J5-+}HvHh+U_YXx}@)zNZdJJA-nT8DxI{2ep@{}4&9R3#>i zlSU6!uxI5jAE*-RoR*vusd$t}Gk*?9O87qAzO^=!7qQodYdlN24h3hLHH%~dhs@1* z#B-u2$tWWWa>L^%3wZ)pVEA-!c>$Q;p1P#2=ldi-`WHf1kPOh0Tq!etw!Z}rW3%0$u6*b}nQctcIc;X2Ut#j*#;0c%@Vcu^XrDMrrmS$XvoSe?7b)gqK zbFedKg#f_$uX5J^%J1+Wo&PNmeVfuEN{n%x{JG1g;G1j5W2=i#e98N5`jE@^Yxwug z+i#XE3;O#a_fvgm4B0oU17c5~0T4Qvgs4XQl<%BrG*ZzCB5$V6l?8V$k*6KV%Kbfb zS*_V%omh{cxcEhur=D(0XSvo<9&c6Ml>SAkU>6&z!S&8;`)Is@6*ysSn}49d+T8Ktq<^$xU1q#3%WkC37woZ$M3 z**M1MGiP)@g9Ea7@^>26kwb5*w@2AyyBSfJYMKWANc^MEAhkD`NyLfn=lv-|(6f>&@Q>=J+GpBHuh4(1#cxX`#+HZg zK)?Knv?1g`J?YFuv=MP5Tig=H9bVCWZh@=3NAN74Ka9UjGO) zGWKCXJCDPu0RCrjKD>)LF>+FSK2)j(SHw(AEd%=LQ(N{AnD#A(gzhTg0sIedTqFUe zAp4MC9Dcj^{X~h}%<)~?^|8ftlM)GZ!t7JXWp_}&{Kg8roUkgk=tT)Z?_uogWt?+7 zfxX&ui7u%#v!~cx_i_tlW^*4j5xc}T5{f(o?2>T6N2Lg%Jm;8kJ)lf1964^3xrh`g z*!%KPgnNo-sMh6Zbq>V&1?c}`?@i#L?EALyk)kA%klj?2EhGt9hLkl~N-8l$3kk`- z&4^I8NkR#uEJM;{Uo&+)LsOGnx1XTts2a_5wHD^b{TZkzf;nkyDy)@6s z&!W`{+(wF|eD8gUyHnw(KJbcnUcMbOz)r$_H=ZbdT!07}-PNhKC*|0Xut1xt-4KMI z?-4*w(jK*qE;_7QGWKK<9H95f>rYdbW1pK1(h;eYEPPxf;lt4?5|kSy_4 z8ZxE$g6^4;khzwO3YTHlVwD^GqOmSNAWSJx5SGUaky#t+0C;RO2BsN#Tu&GQ8UW8P z4g|ake^K*JKYri$qqW(?={-yD4cR|_47Up%mCgneee=`aX0ZrA;3u0Za zdV3@?enB_93N^-~teL?y&}R0}!B9BBl~kAf(I$zVgiV`GL#$NLb`v zqbHNeb5AU5zp>63PM@dOXKjDRuf0^H0keuL<@Dx6x!A29VNCXHug&XUJJ8KA-~7+s zw)cCbOL$DajEUsiOnn`i8>;vAttoP@vBMOtW%zi(klYg&TcGDOc!LcAcI&lf3HH*S zMjsP9j-bp8SHo86w;b;hcAa@u^G_NuSQ4Bs_Do8uq!}(gVbaxP?Mis)HB_$}=Ghf& z6C1)eE$;%7hj6xu&*9j42)?Cr-qxEvE67-`5XzKK|E|fBDX9f zat<+k{pF>skxl8J2e$uj$J_IYyY!W>0IIoiQtppcpLZs3r-=aBwRiA8-itZKrjjOS zYrJI&i{{-th~EB%UdnWWziQ?Y z2m=qI&htlabN_0s$^5^gZ@B0T8$$^-jzJtm!u&YtnQE#;!AWE`J$!<$(SA9~7k|Vz zGScPFDbvTo1!hf;zOwE`bnXV(z&m7373luE(0CF4$-SLn(ZM>l)Sy!DrL@FDJx4o3 ze_bgwN;^vq_Yr-@6jp2eG{`%1B=)GxQn%=FmlwlQ62PD!Ve1V@kpr=e`ZFHkzFBso zN+tbHONYI0b$?hlGLsI+n%0>Xwx*p39MSMJVO4=xtl3k|kUbbq{C(WhN96W~ewKY2 zy=mz4owpu2o@%jZYz_+ODluQXa|pOOvSq$!il!PMJ4ZF>*vg}$e&>lj??7f-+U?eK zE%M`=yBgE`gZeY92zx8_Avd9zOcKbqBwGO3gh}^Tq{<>&5c~1>QBR+frK>yMIG*ll zJYD>uh5z)R)mylCQY)0UOA3|))`rfUk5VLUq9f}dO1E#?ROAJ)scRIN$j@&9pF)tG?;HUI>SJVZI;?r| zl!s-w5{Fe$5)b{;56E5Pkg>eb+A$Wd*~w#(?-qOL7wn@@FgDO#1HDT>{WnSH{S3ReSHU>V~KOk3sK-3TfGiS1dU$R*I zWqMNOuJ4+2U&`9+2I*;u*2N<)+B*9mJ7K32E-(l)ywf`ien5&>Xxt0uiNztZ8XSz) zGlgH{d;(u>v>&MFj!SkdIHu;`C=pH<>nFtB?ZwvG}S+k z?p_z^X$JNFf>Sw4CVh%~9YysD;wTi7-BONYindt3vGmR;y=3p%J7IjOg30aFq`{Gn<&O<^>R3r}Lsr6(0gFvMvW^NLH!vF~eNO)9^%JDrE# z;1n?#QC3bcpFs7#JtU9nJ=Y)~;p>5A7jZFw2=9XMF{f|q3jVu1MbJaSnp|)bEcm$p z?t3cM;J3E4GG|sVcpXE7csgl(jKw})prJ5(_-IaS?0e@pdxSFcC&-Sk7kMrp0K=rc zqrk|ruB<*jv_Md@P}tBcS=Cb2@b*cgQ>&jY>-{`$OuCmq1F|`uVB9{Spih5QArC{V zc{}eKt6Q=;Ae*-9UbtT2bM(5Zhw;l&=FKkLu1fP}*{k5BJb{C{#XjU020NptuWR3> z$z$bC*&n{&f16tL?96QWH9A?89j}<}mKqHnculk4@*210TOzZ? z{Xv-c*?(w1kFH5F0!W1h00J@wU-!bDV9#EJ96j_#zsTg`SvtlbKQdrl@MW^}_#mZ) zwAh0$L_L6oE2+_v#mb7NmqDAE)hBl#28{~#djn*K@RT0YD245DvaSWjiF#KB257AH zg=JE&q`n(mhWXMDN*7g=L{PBn^nPII%tQVLRx|1L!+o1-C~N2ewQXWr(NMrv`?mt> zv1$!W<(BS)1o}&$cMaIB|L65hr>1i!dZs{Ed}}@1-`fdzq)SVG?oH!TkQ+Is*PEwa znp3vA63E-~-j9zUl9p;$PH2l(7U{ZvZmjRKP5@scDDjb1$dA*3`aXHHEOx&;u#>kI*fs95@YMx759@CCyOVGAzK zay8ejKI*sAe9nHdol~m-d_l^Ke=_s$_!r-Y{^{7!X@1#v@-N0GrC;r z5q<)}bEZt?+U2w`k385!qjS3=h}W^~$Fonn+w1gvzh91c547uw9y&R0d^nIQ7bW0V z#{Mp;M-k#^PQ+bx#+74+Ze2NBZM_A8 zH=xeSO}#-?!9cgCR4zN?Ynye)&7pLklfg2M_g9Vo~?B4 zj__9vqxJ)fl9g{XrO9^wGQa+qp*hl5uW!@L>J9L3&Lc?kt(CBApYdQVaTL z?97%3Fetdm$ba3LLHf*?T9>Xmm_u^N{`T-flxQ2gyajn1-TaxyfwlJFyY6#^;iX2i z5w7B`5x)0!l(8|`uM~x@+ks0__4_VM|0DXaLAeL6*FVV5r$FoECgED=VRgvBU)dYh zz;cy`y8Wo>2Q#-9sLtg^QGsvP5MbE+GrP}B=5>yL@s|Jlp#e>VJK(oYsNoC`X}!1pcgI%=N+l`1)7C#~(&Lg8K4>1{gP=Do_+= zi#DEh)FLS7iHj3}8(0kyR3i2PLO*0q(jG|M{raJ!GTcmce!2wNNTOU-S%1LB`iO1L zw=eeMke9<6x(J=fn!Mz?9rB2l&?vRTIb`PgL42U!d01?wUHpr0!|XQ|8pam5=+AgL zK7Vqc)Vefc@r6ibRJB1RyS~GFdxX#_I~U>A#`Fa-i5jE*@Y}D6-+C{Is|Ruf0Hd9^ z?;>m?Zr;2pVmt|W*ipW)hpd_NTSDN7k;SNL1;)vz*uF~E@BM69he1X3%G6Fr-iJ`! z0SO^?l@$bTZnseXYAj)@(Wf8aC3DGY9aep7=Rb!{n4@OiJ<2=hpB_9pC!ja zPSqzC2fJbT>`rbCA`9>B+sG$kWG9#dTsmJ0e^BL&>S5|Ec0w^B0$~IbT*SRbKu+VT z^0_@}Bw8)y*dAAx`u)-_hmc0?nSkh^fsh?|TO{1iG>@!EcvA=4FKe(xI{2Z86W}UV z{&Fo&uU4pE55h)R@bT1zh0kWBDKq@bVJoZF<7)!hMhEWHYSjsF_rbWt7bk}I&7M>_H{xOW<=Fm7b1NKN))>J5M@9IEy{rp{RBag z4KulKrQZMU&>838&}i&ds`$S8*98Be+oI3X!) zMA)F`)UkAe=){9cc$eFNqUBRu&V%~TwRwjvLk=b4MEP(|VC2vO+*T+XU?$h(S@m4% zpMdg^$HtGg1&y~#EnMq)bNN%Ox=`X@*IEh=weOC>U!qN-Pv{|r|Zhor7_FDWdud>=k0gksRMy^>yYMl4?t3i*5 zsTxA6Zu?8G|EebNhvLL<&LO|Q78vctWG0<=6q--gFJYeMdXdXGswKLrm<}f3i*m6U zPkMXNiGG)+po7xd&|}f#>M*OP{@wR-PU>j_K^k8>JcN%SWYIo)h>knzPNMr1tdhK-c1|{u#8`$lS zf%tRHYhzGpMIk4a%b+P`_9)=$v$oi*R*Glfbjn^((J(+_0OccI`kpIeWO-+XZYl$e znL5P!QA^Z#J$_5%lbRuQw-=*IEt$7}eZ)!!<@pG0EBA?S?fLBRM|gaI*Bg=txqrf_ zHOotfWEan{yz2P*2FDLb$=yT#CKhwAoz71rYqYRWd3H9NqFi&=ls}-jZ*63Mjeono zNmKd(d5WJi043n+aMY?gj4;MH0{RZ6hCovpI3-ORTU+kjBn5qYPwt{M-N#~&-bS5+ zFNw0_CU>2WAHRcY?+&9%S#uI1&-=J>t%Eb7(Bx;8sH|D+t{Kn|W@Ge$%IX?sXN`MoPfs|#=EJ|p~pe@+}5z3?+Sm5umSK`&d?v=w(yQ+=4ll0+D z`6Xv$>=xpHm+s56vKhv&wBR#|sboclZr@CzOJrGy635_j{2cNNo4XgZ2LE)7t3rQf zz)z3BGaKr3?UC$60{@3Muu&1`H=OD(vwiz3B*bxB40E!z(k`{57CJc}%TM)x$$@sA zc$H}*`98Y4_i92|#M)!Srpc0(j8N?kf9}Az?biL@>c-tYmpF|1Cj*YY%NicmzQ<+! za8fzd?CWCnWv1^JAOKJ@1D&|DYGnx*=t%B2Ar zFQ{sQ4acU9;adCDDBi7!fi6jij=V5f7P@JhjZnukuYE;5<1}hyHDEZ?(EW~{q$#oz zme=aAqLrb?#VuImJng)pL=u594w8H<;O$yr9N+0UiLztf;#NE9N{JcZyNVV|r$(mZy7 z^T6XR^S4x#*HegKbzFo?+sOIQ`Du&U>|G6B=0dj#tm`;0vKCd1Or4~44%>SG&-*fg zPiu<4J(34mP1zpay@kF#|3W#tm8LQtk|NaapVTMhXZ-)iv_CGQ=F1(7UI3Cg=%fd( z;CS7(%NN0>1lSL7pj+>PbbQ{QwU~EYECx^= ze)}(J*U_=gIG%gucQ&q;hQY}P*A`1=W%8S2xF<4!Iw)?W(**Pe9$x zmJ2zpKiPh;k*7ag)^_U9DIY)Q_IQ}Hf{+4(ISOxc$h_$WyN7W077CrKra{D*Pf*kf zrYb&;DFBS(E8RBa;ns`vr8R+gXFHP6nFLVWuS)aR`}>Ug_sn{GDGAlyW`05#fZdgi zU-h*<0esu`f!Wfi!(vt#8HK90Kuwb>00e0*AkKJ`f@?eTM{&$VZzl&6pT=aMzRIbW zCQKJ^c+%*StP>1m?E1=LAyvjto1Voue106qqD8XYYm?{T)6-<#Z%`y2|G*I{9dLz@ zsinyKJo|SDK=Sk2>x>66lS0;Pgxm1D-Ag3>4-1!cuJ;=him!YUJQt(9%O+ICWBjE_ z8@}JU3LXF)pHTOVLp|UeNTI9#am@p)rKR!vJpJ7sij|Z;-vt%5{lu2amLjnuejHYX z2u0-=*Ji^)VH^cSN81O?!^4Il<&|Zk2X5SG;yL%?G^-|$=Jv{;v&xY`9^f3Z54P$y z#?MMOW{86(^`v^QUA`b*1ZP3q7<3plg6>2>=xm#;*zpruo1t}##?(whs6JplFFxpt1xOi!&;v5yqW?6mf>&?-j@p55Ph zUrEpBz{8snQA6#5Czyu7hp=JIFJ+!>x%}D!csl*6K*{ z)pw#NKJImV8H*-Scn0M=%-%^fRRQ_wHQM*I#~xm0wIO(@S`tPdrW4mjr^JKpWCPnr^z(XJhh=MvEV-*>ZU|ww`K|wO+YKO>I0s5(E;xR)b)TCJFQn8)3{+-lw zQyg(az1L=M#o9OVA>!0_)Z@+i69eHji9DJR2xQ&>Vj9fH{3(SQV%GT&dR7gM85fn8 zj5Cy*qaz49XG~ZLThxo+VS{4@*n0m9_Z<5pY3HQ^U4tFZ`lrsZKuA0jh2LnpJq^V~ zX#JTcV3k34Fe#g!rMuVS5e+5H5XrM~bCQypMWIP{`a23&EfxZAtbdy7w5laK9ezoa(ZU$p$6dujPBbVCg^it=9 zN>>gix~J@be=7v(4<5r1ruJ@3Gd>qeK9-S6&!X<9ohaSa6xD53B-mz5IgRz^&dt#`UOy#QH@DY$BTI2TEB-#u7Ag0mNX2Sq@81`TYm{Cr)zhutY|2q4@nc%8?7O zgARSkyoA22NyD)X0jt>h=~q+JfhVV-e(Y=v@0AtVLcuQ^dw&^obw3_+@dsoyPUrY6 z(Z(A$fE$vl53NNp7fLIOtuDKNTUsALMvOj4i5jox%A!o-1v!{)g_weWK-}DD!Kf>A zSY9YQi@3bQHU0Cbp={dNI8td+Ljl8U%wE|H`pm}J8OOW2I9?Xgxg6|}54RDm-@gv; zja*a~rbA}2l_#V(8prPiTWXOl$MI0TH%{(+f|tsat7S6buik#{+8@3?m5#hZwyf<3 z4QH~^6Al-i75Y*gctpwPx}@R&x!Ql_rt~JP_Ta-CUeDj$HZSN-U7aw8Nu}KveRkPH z@Px|(q7|ceV{*Xnv;}YZBh=A56ikMfbQbPp4S_3wG8|KiWAPmtYnXsVi@Hpkll!Uho%v6}k=I<8rdI+4BU!tk|N z2hU~HmkQcBP+b7|0$>^mFmCo^x7N8K1;1UE|ABI-+avntsFrR+9tWwi#&N$h{YaPt zTcu9!{Z8pX1jac5Mj;I zP``qR_ukh;gI#7dtM;^j zp79Y@|oR{_c~Y8OND{uoQ;Q9C^6sm!mlMsIjeZ#51&9@XaV zL{+_0zOjB2`zZxl4tnU}d~a?Lkud~k9gIIFwAX!~smZkB`ohz9>iO*AF7WjM+ke#{Mdf>|FV&*UtiAc)*fF@r%5By!52gl4C$5wD zU?-e-R|-hszrzRgZ3 zVyjV42E`xSm@Apolah2ySXZ1sp6D(vVcm^i%oK*;8pp`u*y^`a?cq5@X7f;w=y4aG z8rB=x-(8(7@9d0#Nw7QzO5TAYAo$ExZW5MYnm9}EU`0yW{jGE1{|a@}tU@D;3!OQY z-?FW$?>K27PETGAQEyBco@P((g@S4OZ!)N1;PLvd8B21gXkuj?qdWxQ)nI@|PnUZR z*L8idPPrxL)PSeMgI{BIYCL(XoD)U^lSSYSAmlt{`vYPc?n3r9VklCge}is*hj9LS zxli;|;-&c$>Jm(EbnK+ROBqO7S_Y}Ky4H*e5XFx4*9n-%VOe&suEmg3Wi=xz(@^D>pWtNakwFbTMOD*d$h*V)S(ti@}x4c=!lB(Pgow-POw?zp=@x#7=G#@bU4 z`}xMR4YKc_&3r-WiKQ$?XCsG5;^Z-xaz7t)yzxdqzNPsY#bVKkMLz6=bQdOS*nmWG zU;tKCpsQ}}heBsbe-P_Ho0{Oa;1Rmcud~P=it1XWWgf5#*B?{*W{MZNhY#E#J zSOo?GZ!i$V*h5y~O;+&`a$GGaj6mP`WoB)un}=cucB(CFvY*Eokz?5fDsrM3YS5Q* zTb^W0GHL7+Z!qtUm3p6-uc!^mhtXThIW4FG!HIzO!NQgOddfpHtq(9aK>b$Hzenc` zhXiw1$#-&Uy~5_5$oS@#Ro!!5a0!qb5`&8u6$nm4B948<8t~clVQ|Z61xtI3kz~j` z`<+#dD@)ttD=Bvx)YXbE7H&Ll&ydugk9mB9$7|u~dq1x)^j{W2PG&Bh761BaI|a>O zRMjWcs#w3-*gHk>Cex)K4S{yZeGW;L`|}(JU={Ya=jSVR=+dLtdLHWWw0*BUNr(#A z>)_Le!M=|&YW-@$1i+S0AN5>VyscJixy-jD4QVajS1z6R?HRAsn~(v767IsG8M3&x zV+HhNv-mt?HL-64R|5+^=v(;(#1vJQf)$@neshg_U1w90bdQus=6Z`>$MFta^qy?_ z`WeK{)(c0yt#hfaej=Z2%B+g=vnYbTiV?_5U%1$RK+3I)1dmRBbtcUB#WQw3GH;x| z>Oai;1$J3$9L;)tiF!IXqc(qevEZ5fdQHTYX#vOJPw+d%VZsA#b$b)t=A_s1uhH?H zdyH7CU+Br##mCniQCARhfzMP}#5=%?b?eJVzN8P>mH8w;(LUZKK`AIl5t}K;<(?$o zieMboKoDC|Z`7I-??iW!71y6px@^xiXu`A%uJL|k(MYtIAmsza5#K&TR^kJ~IaFZR zZkeLl&eUf&7-EwZjpm>xj^3D-#(BkC^l+Btv~JWr`vLLGcccaT%kG_{CYl4AW!!Eb zz!r@DD0uc!DY8eH%9Wf(n zKxm6sf17Q8qu63Xq{l{|Upxyt&PIul>zrl1v+>~uCd~}gcAB#%uYjnQBf^ww=}NUk zwtJ*sG3IIbHa~N1RF6e{Dj+NhHn>LeOEwRY8_NUIJ-4xZmR$M`Ul&qT${Z>co}_@vx7gg&j1@bfZ&=Vlt0%4QA=DrUx5EYQr`% zhn1TSePr2<`-PSgvx$-vf4-+twCi)@MDY?Q@zO+4qX;@%-#24N+I(5^{J1D4mM^K( z_rV>JlyTdcd@xAp_uc3j#!qJ*7F&!i(#Px6j-3{Yv4VCHl!1nW zL6L~XxtEuHbL&};RQ$5|(c{1+BiH0ZMBUH|@r;&i@|#P=-N)a;PHjF)laE;E7klJV z_1)K^7QqE~N`hXrFBl6sxG(zxOEn4!Dx}c&I+qQ_dzP$UwL7OyFt}RtvsUl)2vv~6 zwidm%;(Pja??*KrY*;>aN+ z6QS`81h?Xf)H4~TD^`a;0Q<~T!$n#PX5Q}s}w=OfTz!h36v~gDmRgJNp z-+lV)P*I2j42f1Tq-5W1D*KZuqkQDr*j2yAZbY za?v%iT6Ctt_ey2xM2>HL!iO`^kOFFsu}SHqgrn#hUx1GO;?t8#DmA!A(sSq_L|$W* zhU6>7NGmbIN@!vTuYzv)n1Qh%qsgx*x- zSt%(3H&YeFLs+UdBEpAZUPU`cem%YtJSYTip^%ighnYttLR;1AFttMRa$gtyS(&q{ zrcqL}{0zrBulnPcNA&R_(s!ocAwLJUoZ!DGLNJhY`Zhe=zVCyaT3fEJKz_MA+j~|s z{c)y+K|H)t8TqYf@Inmv-o)PX{ywE){?=M$gDkAkJCct&lhDfzs{JhpLA*VNpJp9r zCEHOMIgv7C*1-FV8R6P%m4~OAV+G&F`{pv7@Ujg>r5_!TM|x)aHWn5lbr|!-hU#AC*Mw}@B{iATdFcG1Z`rterA+l@UUbT>!W9z}F{g_63lZlH^$XnR!-{S1cxG8w zW9><+R#sMeKp0i(548OH<*eOWJ#o>NTYHX;ypxHH{3YF+1%n;>;_2>sq;@B#B=Exv zh5!W399e>Fh3A{c*X~uOSxz=hl@afJ_2Oqr?VfVMNNJ4TRrXlvaOoY(H(u<&L#Sz0 zyEfT0xxLO+;i>K^q+bcK@pTkGbzR7v%6WE`MI$+u7Cwm$d-QeYc(zn*rp(c!Bs#nt z+!T2aQLGE1N)y>z&}hARqIGr9jpb_`fj%kx)Vpxye6y-N>hPNqUqc@VCH}354rA1&X(BW~0w!+SCEDTB8(PJj6A zoJYMNMG@KHCrdUZtc;I6$)W8bv}+c0oHB%$J3k0Iso$9%+h%=DkXd%|Dls|mvmkSa zk68zkk<(c4nS>EM8-#HL?f~67gNneA;A9La-+J{tJW9kn%TIs+&221$_1BZ zfpufDoAFXKAkegfyJUP4PQZ=e$j9czNq~osp|FCA&reZ>6zB{WLzs(t8`ZIGIZ0MV z)1|lV)WgD4!7y(g0zH_{3701z0oNrCeAH`*FwOZrZ6{CB#!A zy)s&D^kWW*!2q*|S-7@$GguLJ{cZ4EQsOB*kn-TkRA~<^;2#ooZgsiC{y~1R?YzIR zCJfvxgv-ub@`z`sO%R55|B{4gQb!?Re}41lNMij<)mD7irN4H{(u5gXOhIV}TQC91 zyG;8T2gZHf4)Q8e)o(P0`EiZ63w=AEK_1fTR@1WIS?PC$lAfrPt1DslruUexx#I_m zL-%$@y=0>Ca?wYV$=r!li%kWaH!sz3uE)P?wvcL73acQv@@4u1l! z37~YO;JT!%NdF3kw7TC8PzkXg2epj-11Mq`d?L9L07RXq2q?zx2XJyd#BT;v8`>)@ zR|Y~SA~M;Y6!jhko5~Q#SwvA~{D-bq!U(gW6-U#mQ@kt;^EwM!92w{1FwVj`1=bD) z=sby7Q%_X|A*ML74aGvueacgcv=6B2o9peLko}kk!iw;{^A`T{n1KF{ z$SVL-ccQ8t){8<(frN{NpMuK?kFJM^2Rb@0k3l>dlYu@U0Dn?;;4|@LQIWKMd|q6k zT@`e3Md16Dg|>@*`JdikZ@uQwh^3XGYrCTnUBGS+cIg03nEWi2$cUV@^u?go!%rEv zYzmpME=B%9V)*L}_-`bKzy3Y!N6&vBnwB~}8x|J>A*J8cI7h}!q`q4K?uGOl6yu|ak(XcE|6H08G?D7oAy_Tu&9q< zlaG9vDmmCZaycr6p@N%g=PtQ(=URD^#9)%Kx3?#vUz@bNuQg6?XO`hq2u3!@?p+w| zD`osQNDZ=13EMDcLp2rPHvlvP?vR{?X8J#}&FR`Ta$6F&82m8aM#QSjR_cNvYD1DS zm-T(`uge806D&f#q4=U?Xz`4(q9E1mHYOf40u_f}S*z3FCzzKn9X#lI-&NL22Czj+ z7^_b)g!|;(L6PK!l{Z9nhaotqC~{*XDiysjm!B064T0?So@L^+w}xj-`^UVE}-vP+;x=OVn{^dkFtaQ%Eb## zvsiE%L8(k%{Xqc5nWRtn0U=@QWBbJ~40$;lm-al;Z6O7!J>~s=;R)Iu*N0-dx^>-e z6W!-&O4fnbC6EP4XSY9cuNFj=myU_`7wUaT5qsoVuraDfQZN7%tupmp$md?jj&F8- zaZ5cq{PX#E-F3II=wHjee~mTwlEwepn+rl`{{1fa zHDG(CCpO{Np=iSu~QLW1=fL9W*{EhKIqCYnw=D+<9;m1ZC;Rd1v3bE3%Hw;V7A1P#wgpY(igs!IRc&WDUKac;u9da} znWvN;X(1{{V-E)#s^-0WA6d72_#rE@r2YhZavD=VL_~TCGoU+OMNTKaAG+3?Q?UfC z9&$IVK#zQJ;}|k@jShMZ_o}7l@-U22{Kn5jZFwy!>v@2P-To?&HAhQop9c zVL(}xE|T+qy@JpMYQ9m2t2MBqp#GCA^|#voHiyIhy9T}szY~N=>9Ey@q%o50Rn*)< z0)6o>^+dts`+I3Z1aqAc3LnQ{Rvku#0Po{7c>Eb2#m%C-7Z4q@G_Ey1f` zTDJ?BNa_E@$2sgz?Haf`jiq&1Gu|1rnP% z5oMSvbZ{OSNW6F7?3ER*EG#&<$?=7smt9N6T1Q?}0Kp8u2mHE`?G)>*RPsT?&7do# zhFOfUlV7Vl4FbwJA$&MD-18rh+lWC`OL}UB>f}BL%PyHyo}DYV3iXzbdTZ~o9eZI3 zG-0hxnEu6@(mz{|HJN5e94EU>ysj{nMQ1IXCZ#HzPd_h|uY(zrTGd^Z^M8vj^i$79 zRcp#LpeA612YGT@M`tMYXycJIV+`sQDs*m|m=<2xFjEbtuN=DcT0be2{(NcUUKtTz zlkGKG*<$3Kihd!8z7oW@M;sBq91|5wfrqw-NKI;F*Krpk7fTbuR_&3#u*3DRK>~Iu z@0(q*zQd|o&=q=VrObV%m-z&6YZ8op$vv$(WWTqGTV%1h z+{C3P(V`c*hyDyHweGpNxhoo1=vR0!PRO?%LdPycO&m~=>_MgIvov* za%%M4cwWR^nbnE;B0rbLhtOi<{*)X?E=?luGpG8{5VxV6>vQAYr%k>*)P!%eJ}hax z)Yyf;PUE8T`W+^8giTq7%+|#_L=>*Saa>hDgx(W0H$(BjWvv@S=d6YEt(0mh?jPUS z^KBh23_cv;&jaBhkNQtBSYqdAE(;s0*u%X)%VM^dCi3dt38baNHXK z@aMCMmAzihHF1U+WQPj{T}u{er=O=uZXJTqjTI|~IXG0=h+|L3gfs;GWQ*<*b#OwS z2IXB1W1E}K5iT=wBO;6D{`J7JM4U8CEPh&71cQF)u8q6EmeWJMS9wRBEy^Ywzxnx1 z*u=y#Q#&~jTTl!7N|##BlUG(1zmnsnTSUKn{aE=G1?aYLK|e!+acz>?cjU=F8Mgt= zt#gso#s-aPhth%dxk7?c-Pq83(wu$SA z-nX&GUD$~-Wm*}}Vi}%w{hDR>@##b#_HmC7HB4>38@Rb-Zi1>r_4OhOyz%eSCHe#%Y#@m9;O=Wih^fUA^2aFOyoBvyuDigDd+|{Xx2_ZzE&o)sl- zr@bRE-wz{J4CUCu)Z2?w;F*YBDBJekWUfWZ4@l&o9H&F)QZ#2bo#k7jw$K9&58%^@Rr7l48f|~Q`3)!mp2T~8%X=Zzh0tx`j! z1~J}iq<)1fpg80aTs0NEuU%LUn^b9^uc&MWLAvOiLL!wl?$YObIK7^n%aqv#_bfrm;CvdLFaUp(!;NzO5+eMd{VhhDsHpah>oZ1Sh$YVY%?_5z9)P zvdIrfi*hP8H>GL{hr2($r6Tj+x-n) zRg^$mdrG_(YuMuvdcNZVbGx-X`BM38X&MADKq!Lu+Jm_hnPV(~Qt;9h(wI@V<4@kI zP2g^miWhog?wZyY{aO1{KD_Sf=wIDm<8E4|uBe z!tLR*sZ+{!>tqZ@R?RQt!o%egP^~ezjfAykV|N!my5bV?;-xCjdHPs; z2R$Z;tbXf7!;}TS(?j6ZG`owL2~f{)AOx*%H_ZOdK={jLn)*JO!*h1bTcA53@FM_B zmb_{i6{n2pN?M3ScM9O*?Xm*hBZI_Hx);zebpjfubK4tYUT0s$fIO9q5pwC4U+KrdNZlpEI-Kf zQGW{3qy`99X)~r#6Pp90EF{T497im?qR1Sca!t5ISIzI^0sxko*@}9wF ztjt=>JQ%Yh0R9<8HmCs6fYuu*jSc++5|+w1fMWxyJa=nB+?5JCda}t9RjL1Pt$Brr z2o3;oWPOzS$5fCs`4k>kl!2S+##}55nHP18=x7pS8(`g;rJhYbINybT>r6gOdViPj z15)40B^Pe_dB+^fgv+D&men7Sv1Ak*gg*K175+x|93)LW%N`0O6!`If5)Rz}_S_7^ zl{!3CaAS-vn@|xF!1m~IPDfxi;7D});zy$>^u33YPo(zkvu;B`0>t97G_xc@j&Uf zxK=1x4Er>skjhI&8C}9G?n7Mbe!70rN`3i^OMM5fivp3Jg%0gC6|h*$B6_TVB(R8uODo4if6e)quGIVz4p#>l>J5M%YCB8pn z)-@hgvQ7hiyeSX0+T@2#&9sH`4W17LcIj1fT4c#oF)t)Q81t2E^O%7#R=+)Io}UP9 z==)xclyJQmtW)fKueM=c_=WDJf`L8qgW5uzvmcv!B-&>z=9t$&7dczx56CfTz%w6* zQ~bKKw^(rtQ-6LlWe30>cR@(}jm7DxD`R=sB`j5%3?n@t+fTrFXa=2bt-w{I?ae)J zd(|ef5W8EK@7&x!9^rsQpkkn#Tl zaN7lPGboSbBgoEU17nP^#?T*-l~Oo)1TaAnaMF#Ue&_zYucPWXuT;JRKS;Fq$%-uBmB_k2Q=Br0h4xcgt3 zR5g@VPID1kYd)LiPzq_0hpPIQF6T;+bieD_P0i<1l{J2S$7AC>m+A%3CtB zld<@H*+4DE?fw~%?e{>^nETUFYIwqpePUXZ@F&;5nU8qdejJP&HiYGU4j+#SfWpyP zxVcC8y#NC!rkKQ3jj?UCzBkIq&i*!?GOW?RtDG2(Mr>ZMJR5%>2$It{kbh=2F%NAG_d;@zIynW9dffVW9cqryf7&L};F7dYD1 z7hwU-c#o1`=u`t-fHEC6>E}C)~>333wBW={f@+9L^}V)zAN8%e}_&b7%)!UZ);p}DJ%eZ zVzkya1n`H%sXdT51q1BAD{=bNCCVX+xcTfR!81q^$739jU>>n~)#*t+zZ5d_GZst`fYn+8?n8^=u25Yp2kxHD8dxtoNRJ$do7-|x z=2g8P%jC$mt1S8-Wp9jF$=lzR5q^qrRiRH(|1-LQ9c zA5d)5jgu2kYA@FtPXv~4iC#Z7Vi1iLDT8Ih_f&|D!U*DlYJ6X2T6iKnmF#rbh7n4G z8LafKA>B{){&ppx!(rUVrlPo}NfFL9djHm=XvuGj(B>3>h~jsLh?^c$BC!mYyL?|J ziRSW(N->UReCd@u?GhpaYPq@WC?FFo)x7) zJI?&G!rXuIzM3>C5d^Q^rCY4(%5U?DlTpak{F0t(AE^aq-|W@<-F{cbC$QLJn!|=Y;eDWu9&Waq)1ATx%B!a z2Ad?|wwLH+WN}@Rz;I)_SIe>4`?P28HRWYpYPOmXJYQh9mvT^G&+?o>RJ+P;sp!#xBYuiSF&sw z|Ca33NHwfG6Ppohw_DrA_oQVVe+jd1tO?LwCxiU1ld#eV=b-USEj}mEQsqv}py^(OJ}OiF$>bdy=`1<0Cf`EUJG% z_|~;D7Knnr14@GGIwQJ+jzE83A9hk`Uz68O^=y(eN5wVV3;ml4((U`mwcRZ(%Zq-j zT!cHvZM-4`X0nD-;d@Saav55`CK^uYhF5Yqpazm9pLi?WTl6FZksJ!zS(F-^t8Asb z`IqIbZK~o`jUl&b*cs;j;u*w%3i4h^v>>i-Zu$lx-?W2npHeD{I*%B&8UID9fZQWgD_3 zWzU*rjGbY|GKN{s=hJyD_jz5{`TJe>bwAg0-_PrHUw`y6rti#{?`L_xKgatxjyKDL z{LxC8cly{sBN=yO%d^jxobCoI5g-Q=?uza0vERmys0_{*Tc~t>(?2bIn5UqiY0&+Q zwp7J0m|NO7cct1^23hvYHBQBYXxEqD*`F6{#-GgFl<=9bigl%1xllP~J>_NseoBqf zYUR7^g*!Y}{Xewz7xXngovGrSK7af6Wxyu04-#g|vUXEFWlP@demZEB*w~O^d-UdEofs;*j?b7FrIZwTHO3zq;?F5pLj;t=Hn$3Gs zJwDyozpA~p(!lSDf{Dpm1OBGaflGA9Tj(CWKnuxxHH0QF;!Z!Ds;P+UlEM1b1wtLB zo|#_}_O4RSDdxZT2#C7OCRV4#4b<=N{F)v##6I@U8AXDRi^4d~AT? zy^%rI{tK^ndrm!jhY9KQIDO)?g=aE1-em>!^x1mci{@~_1__^OlUFLX#hl}}I=_Em zNyw|6bbgn{s4vn6&=RA8LF7C~hEe{_*{%M;%fZq;o`aO_gdbv!=2-VkwGDmJBI0JX zc-OU#6{RhzbJ{-JB!y(ob|ds)o72aa0((b;yvW!cJm12a4_f3b2Pe;rETQy1YizGp z_+FGgukGX12Jt|SBiyXLm(*T30o_=APM^yRqSZ`a>`Rk5-5%!wXN>htCObI=1G}DuWi?TN@Dpatr zdFk6XApK+yVo_e|(QTD)SR22ndh51mp9icHW7;!5Ql=AShMq@v1wnlAU-FVl&|I#o z1Vp7|;|)|F?FBK1L@#@hnKkO9FPa$OviAa7b(WAJI+D&@Wvzd^R> z*|HnV=DN;+=rT95(}1|XTl$oGQ(>v&xcME3)-4I zFTLrDjxTV!TR*-w2ACoiz1h=V5j*)WvX7nXR*O3`c($`3tD>qZ`ATVNgsxbY%?c2~J+lG~mM%6J}6Ri)C$i3AR4&@t0|oC`xB zzE7DS%n91tl~ivpCv5NcZV8oc%Pke_2D)=z+3?SI&KtmU%*ZCxAv%OW!u5xZ;(mW@+y5%f{Il4-_@O8OSgdOR zxT<&ttp!oi2=-=asjqOEb~R@kuh}@30Y@UzRR(%889F;cy@c#fxkzgYU-1}__++IV z=s(JI>MCKpu>CS{iuhFIGt9{?E1%iqnBu=KoVQ+a%jsUz3?uM7_szdMG-~m0pjW~9 za`UI&)(Yl!*70S@zphqU0Ee-y;4BN^fGJOVZp=$O)p8_M3-`%v zAgA7Xx=iVXAN|zWL9Nk4A`|7BRXCmF^&f}peFo;{FljlG4pLh2ks%U)ROH2c04eKf zlgn2a5j51v14n%`XMC?3J8*RNrXIgHg1p&O8azh@(TF%2Gw`mT(^b8vYtOAc;I0zX z4{08$gRZoM4Q{ayLFMEww+6peiM{ZsVZjVDtz|h4d!-_C{ccY89U;+l)J8WKO9R{S zxN`v84(%wIIwhIFXu+I0Gs`#f&0Y=j8lqmr^|x3t(99FF$eqP=pbi?29+OKm(tcz% zRQrZ!oB!r;eqMigdJZEzs;~?5`1#%6@RpyL%YXYXUV8&*86hJg^>mxiHvGNx?<%^!7frN2(!^>IW7dpDldw!VeQSikw zKU>4Mf6m|`v>_RakuT`VeNfXw2M)@qowDX6;rV93u!|_15X}}Ov>+G<6taLqUHLsZ z?Nr0=i*KipK2CT*WNU{ny*;zj6O85TtIl!nMwNy=Fp;64!=0^VG^I}ghxxHXh0djY zu|eypW2zF67RhxOa!gZZI?RX>*68V%q83bGx={2aD{)e$ITWCa)aj^JVyV%N z)ww};`WQ*Zxan1{fm3ibmSJLqBGOR{=B>5sqB-fN_(sl-lJE!ALchH>?s zhxe;7Kv|0?x0BPsRP{5vpjY3|80g$P*siaOhZ3Pt^9tFU$9l)ru+O1dIW^_C*nX{xxz~sTTDyu|@3m9H&BB14pcg-Rr95Ny z-%rMWGFcJ7_WXO`B!tn`O<7qrD_QMG*&r)wFa6kk>y4eVgO<`)lp34gFC&_wm$~DI z+SWzrxO!y)#16kZff^yAD!3ym0pb;}qa3Wl2Ni>%kK-uwRoLyEncFoSsn_<Al_3J$+%xOr7+D zWI6T(&X7F%2O?HxPS@HQ%$I6o*bXo=dc+8w zX8q_oV&BLX_Yy#?#BFzpE272GA8#GcAx)~{rAt~hKVzg0`>TPz;IdD9IO}j!g-jMl{@Zcsg_&S%hhN#fz?K{Icp!5 zDsCF>It?%`mA~Po{yi{61PyV#IDCO_J6J;0p^mmE=?6+5z-AuYB6$6DhlH_?aO75Y zn)fEEbS=UD#B1I5$Am!Ej(IGegsH+)O`!E-F*v5!B5}o}1ctrW@vmDsaFH;@Q>8a)-KTrlP4z!Ccli?Xn!6MAN7(f(we_H zZkGHgzFwH&O~LfaXWqHuL!k_!BCx!m0(t_=kE$Dq8KFLRw&K`FkuG~5I6RMA+O`qj_{&MzO^xKUKdM*U8@f-ts1}ky!n9( zTK#r9`IggHCDy@Tr1B^=Fc?>e2JC{T;>*qYTk<~Tiopb*NL42v{EaF97liibPt`xP zTgB3)Gw0Y!R5K1ASPgp-ApN{H5ygrB2>jLu{yesbKQdhHU2d8I^cFA9R&Xp;s3Ns( zFuTE!yt9TeE&j=_{9mOPh<`A9=p94e-9SuW%KqNq`Ol2O5L5w#09KfAfIJf}K`UIC zT~Mt)G(NCs;6Vung_=`pbUdi({8iCixOc4g6K-pe6E!w{3$56z`IWJ%FOS=+!-kV5 z?>xDp1@tJJpxT2qAX$PpPW4^_Oxg|wqJ?V%hd&6;fT&2*?qXaNb?B7G8`G4d1z#t< zs!m+2m5WC#tq_3w@c-f$|NZMWtA53Q0-+}RipGe9AF>W30i;P<9cdrsos7meC8 z^R%*yKJ#Z(?3G6e{isnw*lz9c4&DMEBM;c9d9;4j9mmlDM?xh@Ky%;kQ|9ioVRl*QE* zt@PzNzP@$(g#NyQaB_*ZRD;@wMx_U|lzJ50Z&S@|x^mA!)e4`@I9)yIlPXJ1V(UN1 z?{GCe#fQJE?6{EI@KGlb4c$>Faq-SE8_f(#E$++{$#L~Vx%t}M_#AVJt{&InR4A2SgRPUmt#2>#( z+qA|sYaFQR$vZUGk+sOoeLtWPDGcv`chlJ^k;jEwRc5TdB8zgS@~ao{s1JR zVBMCFz&Pb$kTLrMYwt_{Ql#JkOugupbyr_ab@~Znf2(|obhM~WTo4xxK{l(0qPLeb zDlR{E1x)zQxbJBV3VPR4!p zGL>&4k1Xaa(5-(#h3|)PHFYK_Rw}D56o+uNqerqGUyQ~q{#XS;P;aNDVeL* z4N(Oh6V6@SeEm6AT#SRa9%x@V=u|(D4V09F;w4IRkL&OD%wBFIO2V|q-)w+B;&@)@ zS{lmquwT^Jj4?BLP(DBvu3jYE{*}yY*|ksiM0`F1DcRPoQUU2{Ns zqj|#EviQr_WO6n!A~MWVPs_7SBtp)3=`} zx?2YBVc!bgbyWP+Hp8C$ZVMf!27AT!N~z3m1i$zVDXmZa#)hhtVt&C*@%c>|=FQVO zhzWxxXke-yB~;x(8rf95p&Z$)Qdy@I7?2Zc+Ic*;Rm1cH>ew3k6LvrAGL6vi?aHj~ zp^707i5;#vC(jm_SmHiJeZ(yKet(zO)LgHnKmpIesNi`yA)^D@n7MaE8JKdHVw!U&Mk1Wcyan>>^PDv?C z%-Th#7S8PX9kwv?9ydRd!&2z#t;fm=+c8S%2MiA#m5Mb1*uvG-CK}3WgUeu^#5^_w z&cMYJG|~Iv$FK)0W40UPhWfxvS!(<%x^Ow;cj&^6ze5)sNrFO`E);!eSX{9qj)prB zh%QMm?+lohMK398y7rGw$A9#k;XPx0m}OOISw(4L=j4{DNbj5D@%jjfytr&T&(N&x zTR;;84pccGjXH9wKp?`Jh#-_kA+f?iPI z5t9a@t++WP)rc_YYx3dnC5CjbmEPBCpY)HfWQ9J|HhlXk5H>i`CV8*nI#gHqE6km+ zp!`2ff`_V%u>gEOuSM4clJ=5Zb*fY46|i=cVs|HP#^lB}bwRSF1nk42c5K^pd+ zjR(BL)@mMFTZFDE;}#>3X<+rfTXkfGsYKGfqGEz8YCCL_ZVLs9w~5yhzVn?E1cp4^ zKdo|{kdaXFNC>but$(KOm;9A}jGog{)wSSRZ0ydEk$#pOCU-B=J{-~k5(sD%hb+#C zP_v0-IC^IJkqr-+yg&HGIfbdc)osD(Qvqjuq4Mqwru)Xe8esP$1n!YC%KbTw+NC=( zI3OlR<+NvVqY?_pp)KDGPaQ0 zH0i9OnSt!AMlGO~H4F)ew%*76RaY5UxdzjSwW4@Ij>3$}?1n;y<^x8e_#>&L8uC>9 z&853XBte4mnJ~3R#;}C(V~r2;eqe(5X}oD;cET;kcLh|AT@92fubr0H!RdFRX8Q9=K} z@L`?}(xLU%BVkg<>IQa|c|gH;)zTCk_-{Lg-72f{m*9uVxb8)YZjss3bjQXZ0R~b{6`2x?R-$eA!cX?rRFQ<{)o|9S^^zzWweI_gn1j zGw^H}q$GVDKu;&JJ@&i1@!ktZHnP{H_Lk$xEhn^w`=hCyP7p{HQ?e>E#KYvOd-vs zvyOMFxci{He0@@-=ZhtW+ehjOmF9P!1MOz{q0tn4nH8L6b11B|>P0->&y&{dCtu2El#`6#Fk3RiADltlWbI-7tQGRSvEOKe{}S5byj862*R zJ>I|L#zL^aqXBTc#r6=Hm0Uxn_caCj8H zBds~umZyg)i2+4AhBD|m56ZVO{5&q4UM=x}rhH#b?3!ci*0SPnb@&YLIB4|Zx?J-p zXiHDsL@W$~ebISrX(Oe)?ynT|lGr*mmM-g{^Ga7$v=4({32&6xSsc@qYR#tipmYI; z?yt}i;v9ccO%qt&Tbe4apeep}AaNx`sp7Z#N!&#C4FH+G95*l7BTaMd$G5gBta;FO zYA!9Ofv}B%)kk>jVfl(G8hml1pgQH@rJY5TCR0~>SnU^|gD%Vqw<7dThPC!2$E&R9 zo3HmSEq00?o6L)7+10G6HusWk_B}}b)8+3CsC}8;LR`G&9g%f_(E2<{R%s)25>=$w z&TFXm$H<0oernI({`wo8RB0Ln#~mC~GZKI=BgjBLu)F#;Art5_&DJ$hAWNr<;(+K6 z@bRC3232RZ`2JQO`?Iupj=LKj%kpN(k<4)RTRgZK*^{N=whg`l-(S^wZZ6ufe8%pq zIR6Rq3xZQt185Kg5~H`e+l$&*d$6)Ib54tWHlwh^yZ}Rv!UV|!43kg~t?vhPVx^T9 z?#@`yLyy(s!2mVAPv<WQ6PrH$#f&LxI-E_rK6dSZ+~x5mt$hbfqrr|}=)zg^wIP&Rfeh+{^maT0|%Zwq2t zN3R;QF91haA=IK6`wdvnb;{T=kC7W%lfa?*FO^jt-4eDSREt85_f~3x_H_Yx&SPUt z8#1Sd_@a%mcZ!v|7KNHyv;@<)`6f5T=y%KT_e=7hJw_eK@bF}3H3;|UBjr%{<* zoKE~xyCoQ>brn9{;AI@ZT(80r%ud z;`ZwOe%d7U$ker1_u=8K_~v_ZaaoDZXFuz2tJzz0?TwCp07hyqk(Lv7T$?on-(inIX6e)=TBC_9?2JmZc|wzmeVWq3 znsxf#jP}FM-R6dvT%Mhs`$r2b3_~87KNr#?ft__O0Bt$i#?2s%3FfDYh8540D0N~z z_CsDkTNL0du_anN(V(|zqS0aB)%{wGr@0M20kSg|X@>hO(M|_Hz)oG^zg{E`L*&TK zwWGTrUZUfORM17HZb6{Lw6s((fj)PiraJTE#MVZG%(9kgrFmi~*vNY$SUc!$P}4Wj z`)b95vNdR};P=aM32)hRf~_?UNJWAs(_W)fZJQWhUIj&j-Jn#_fwmCVM4R2qdpeyG!PU~cD2G)dTu;3`(+q{nTKI^wz9;Y&1(J@og zxNW*)w+>yFFuS~B(D^G2NZ)P+9^BVPG7E2$nqYkOnw>vpW0M~HRuCt{p{{#LA}J2(G&h6OsNcm1N1`o1eUX)t*v>*}w5eQ9HVrymY;)+ozZlF(bzmRi-SIyO2k%7_U(KWf zxNOjQ_r>BA-?Df4yL#Gx!e!*ZE`QJCJMW{YtuLT3a^QUGjb{LdizrRX8Z`6| z^jAFL;qeoPLoVbt{RYHsu$da1ZyV4jH=Q7Zv0p?+Vmz+ZCz4hA>PMCukv}Op9vM5a zt;qLSJoT9=M%eHEDc`Zd&qeYd*h8^qKdC*x!EfZZBCDxfZDMU2gmLAaEQ`mknvVXq z1t;8lK}n7Rp-$25g-ARR#u9(w?VQnK(qA8j|H+_C3>*9V!p?sl|H%g&`nCPvm&d<) zMzOHCa^b_hCD8^m0q7dz9zo`uwF{=wqSEbIT-~%E;fMz12Pl$aoLHx+Q!II`WO@$| z(&fH?DJ4EWjVO!}70lw<$0g_Lv-vjguh5{OB8l(39vG#TXr(VLEmQ;Son069-mLJ- zg*~)WW+^8?cxH`my!{s)fyDpp2($(D-GDQl^Wpfl%y8@&YE56ua_N``nE({jL_0*o z)DPA(_D7bL8smRBrmDJouxI zZdb0RIL7(@i8}!F6o0KNHmcQJeD>R45;5XEMEV zh3Y1kEO$|HYiCVHy_*%Pfp&^{@VWZP@)uRJ!tav3RNrV#B0XyRm`f8WW*kFbz^6K` zi1LnN3CWgbzJw8=+x=(^cowwP6MvY>x2?;h!r5_jXyZgKdU^;8P?=h@i6=pl)P{Et zWs~ZQ-XpxTT&P5J zLySF5EZnd<==Q2l;NS=YK7YSTvLuyt0gPhDQ1B>P#;(LZJGtOU6NI0SMF;5ryHzqpON&r8Kfk8JaY7AJ7sv~Uv zb~tdyqxFUc`FQAzga)0+_6yy1W20zS?zLkGkndGAyOIM zj%`DZvsrf#Ml5^AB6Y7lV}Id0;!}t;SLTp264A{4Gyg1~d&tH4WrgtL9X$I^`eABH zRHbvqke^q5kNI%l(2L7*4}pg;61-2oCOjJ(2F{fCHClH!EZeW_$$GI-_;uZV4=MF> z+9Ahm{r7CdRMv)r7{@z#fhdLD45l-WTF-X>lwoAW4Ruk{oeS5ov(Q5M_!U5Z+SpC4 z>vBJ=58b}L-nirFCj*I4yqY>OdkMwUFgid3px-C5?M)>60Ntv?VNM9K$l6qON~k3zNG>g%b{_(KGFqIjZq)D zRfEx{`Mgm~#IR{m?#sDh*^2`7xhJ)eyenlf51-7}r-1H;Z&@R;m8M@buz~_iqam5{ z0J?6H94Y9Aqf+giI^sG~6mZcW9=}ZP;QdIhRgPLwuRBO)37>g8Lr-uB7xx!~B;!Z4KdxSLEsmA>3=@ocrD&3ImoVOSmo<&br2YoG7nI zG+fwwZQ1y!XhPrWp{7K$DlCvNu7!Rt`tf9*cE@5y;CWS`a#K62?LK2cwX4o?qh-BZ z`BX9E0S(yzEc4XLdMh2Vz477=m3*FVx1z)^Ilx`wc>{`SS|ni_=O-c?nbm2p`(T1o zJD-TKFW;-Z`qs~n^!BZJxVNE_Jl5E{c(%DUvjS|#g-L&6#{DN>{S$Gi5%A3CAkYvY zCmV)EV1GbV3&a3|?!(UX0nN4Sxyq>XWo$j7;Db;%enJE4YQ$A)(eqilGf^DL_|@ZM=>uMx5MP)EKOrRu$zjuD8|e zQEek)io$`?m8gSsCfHHaTrq+{`g>~^`^zjNb-WCp?VftDS;$`BAQ|z(M!yVyj>}2_^LwtKV z{s%Q=J%SFb4VQ}VRW@%uXJ)p815OsZ4?-jlRxqvMi{G9{g>hMsgR@oQ?0ij_cC*MMO9MBggt|~?(Qyjq z*)kBYegn0r^-4zDo6A8?jE1Y@EL2g-*SB5nBFU@$nd9z~Ztu>(-Xm9YCd9=FGoM;} zI(vGwwa>~$Zjrm>5Usl9!ESy;q!bKgqa)Z9x`m(g}nP+C_B|?*eK@R0#A5O>^bcZXh{v6&Z+WPp|lZwH!+{u_sNB z2Mc(HwFW2`6w4P~EjyzF2?8|m zI|Jk`JOKVpwF|EVCVqwNaUgRf?EOyw?rlb zfzfP|M2|bMcDm9>7%j|w407!Nvd-9Tm#dD~xpev*(|W)LH>`hH@6EMT*$mw4nb<2+ ztIZYYWw4X-d?b(I>uf#o2zy)|Oq)=%;O?LyWK-mJ!1(w7jf?&4iyE*P}dG-Nqa{5kw<181_3TyrJtkVh=CPJ>H~I@=Xwp@XE2(sF}eX89ujT zX9i5Kj>pFlhP-TU{>nfq>f9=DxlG>k+wOq9_7nDmT#3=dcXuC*>YO5ZTc1m0#7|j9 zE;23p7ht;{xI6T}y9w58!L2Dx@++k!h=Y`1T0gF{_A`(wdey2i)3jlaI@ONu#Zzs~ z)3C|A=tzHgw~JNgQQw5TT3z_D5kAIs5|l)GXVFgqsESkyHYeLKPW z+u=US9TN6tlM4gI;fsc|j-l(@m?6N25ocN$$k|$hBCG3)A~v8ZV4fWO9R@jUOXq|w z=X45&0z&>z{MLbglpmLqt^1_LfeO~pF?uxEbaFXe7(T2PW06!ASrBw+0|SgjC`UR9 zi`U1W*Lm#MKh)y=#uR=jSa(1<;ildp;wnLl{tQT>wSyS2|#-5q#uV zr5|nR!Nep1Mz2(Za`AY8m1Sr$bs6L_-|(KH5=d~y!rpq@hp!LCMUwTrtxg)~#09d0 zAm-0K?^)0#0K;K1Y(Xs;O95cPAlUJq2HK%%ramll~FNt}0hCGV(v^ovw_IU|lEraNj zu5MuT)3+%!gaZ!z#Z3Uhw^qse(&~O zqx{T;hk9~rtL0csC-IR7Ly5pjR|o0T6#+S*#VUb8C!lvu6sJ+d0Feax!_;lxCUvK- zCC0Wr+pSm2t&TrX-0jDB!{PH=#I8tPLgZP>#n-(G0L{~lIJ7XrD)$&Kp0<}% zm~pGbW|Ii{iv93XcfzENt^Py#NIl+(&p%h(zN_qDYzwkP8Ei=d&q!aTk5a>54qB9+ z&FoomMjqSkNG>!wJX*c&~hdgj!cGOLA9gS z0&?{j*g?NdTMPBZgkhC*rQi2>Kg+B5+?sYZaMDncBs94+a3_PGJyg9;Mn1v5yn;W?lyeyb9~o2U&4nbcjsLA-We89w1J`uol(X- z%i>`;yU{SCw{`;s@(f~v5e=NPQfK|HB_~`hzu%yWk@rY`BpG}7IFA~m32Z?z?83w9 z{eA|UJwXw+dBdxXg}vLTb&Z%S;A_-bG9<$g1=RL?y}$#)EGn-PfA@=_A!_xRl$U;aVw2_Dbfr9-Cz4r3rr}yzZyP(s1OaBw`2<}h z3=mA%Z*=+l8Hp5fOvA>plpIFlH`qBl#aP>ycOyrGWwK$^;aA!3FMoqCH>O!W9R|B&c0x~Dn|UJ0hw3yxL9MvR$R_)GIp^fY4|Hxn&YWx3 zi0l!9;!kRk`7(+|(aVEl7{-AnindqYH%}6U`;)b4>eo*L12#6#nm1pKKlY8Dwf)$& zFZXjK`Dg^jzD0_!GDVr@UyS6lKg!^)O*D4O-0U?m^26!Y+Ps$3_^t4SdPM}!0P(m? zw9Qkift}9TkMccb1-`aoFP?|X(_y}#HaCFlyYjS~IkG*ikhd$Tmy+WuX zFcd{8VN88&DwrSPD+wM6rcbOH=Lucx3_sKwJKb&S5 zyE)FN(NY-%u9{aWDy$dDkn+((s^A!kdnw%~+UhIsbwgLC!&Z0u$#sP)ZLCK9ayQXN z@_MW^s_hCq8{!JfB`p59%>z?gj^%2?g5ncHjTXq^dq#5@jiNY>nMFzO=~zj#1Si@k zpHdJ=1_sWSSUMFMChm_NHcvkKF#BZL(E1&5$iL0(3o#ZmI5NR5;v8@za}>9hL%kZ) z+)OTpD^0YPxa+t`HUr?5yR)oYqw+o}lvad?B5s^P*2>@F`y9S0mha(~2H%S;e@-2- zlo&(2-XzDdb1tXjM<~~8&S$khG z=4u9>?{199*DKqFemtepEOEE#q7O$5}ZItfW4O(1}7IOd3 z#Ml4T8tJbc|2D|_H~LAe7Wy-4XV6jhy)pR5dn(Sklb7hRo0H_Gq3k)?!}z%BQ~2b3bGqxJ0vCFlKl8wW9?xEhTp@m1UE z6341y7VgX8?8C8lP8LF3(LO4g&1-DuRnGm!<8PXlZ9tCnB=paiC=BsrcjCT7ho<5? zJNVzX$?EeV#*!H)7^OU?&0N-y{dIL)$MA9XyWf8>=@|&F{a(0L{9=6Vxjejsj6Tct zbDP(eiS^)H-laLaeueRRkG@@~mfeZAO4uNfFakEtqV`CtPtG@2(x8NmjOQAsMW_XN z_nOzN>u`6d#tK`hg|%(lFa|Tb#Gaow%S-gz;78}Y()O+)ETv)DVcF_i7Og+hp_DCe)hUox{-mV5PNv_hxK+ql18HL5uzefY9Eb ze^FgjVRq?iV)JAFTB`gf`t?dI;0i z_Q+ERs4=l#vnm+ppndFgtOQHWse-ziWb2~*QJ=k_P=4tNekhO3m-%8a?uUCTug>=i z$}m3Wuqv%iBrfLDb=Ur`z-E?xOxe_Q(1JdRtz~lYtIkxXlEb#TbjkLhgV}X2{Q^D- z=zihhdw4?ZG`Dof9aGJ{NyBA+ZASLt*OJ6V7*f`WulH(dzhAj;8K;qJX?6A@>^3jF zMDDsU?v=KsVPkOK1S)={1R1HjOTJt%+@a_AdGw}lHf!n?Q3!2-*bJR$xw6s_Ab!Xs z?whAd?dblGhV+@ae#s}SCPZk~+_gQ|1D)zU9)Rg;_X2hQ_|@D!S+Hwj%TEjC=N~p| z-qi_V-R6>8QYJf5_K=aW=p)YLyGf*EXlUkP(Ger~nR09#eu3~=i=RD710Wn@umK~q z+8m@0yn;+lF4(?_lyHay)W$c_sEXbEh)4Vz^g^?nD|EtKD;G74Cvk~<6ft}awRC9% z!vDZ3)!RB?4!b8K?xwhpvGPC2Uy;1R&ySe;K*!kc$3J3v_7)sEi%{ zzg}JR`^xUwfZf)o+JR|=Awa(D{>~LT4jT{7plE2EUI_?j_CK?~3_`147Zz)w$E`8V zt2e&FU3!z7*+zXIPA!dTJ?h{J2z^#z8@=O5)=r4SC*n%2{=yu^)Vdk8(z0fI23iTX z{5s@3iyLt?>T{n;!}?%VNY)KmF)pMAC8OJ9B(Z|H7D(i2)p zyz8HJ@BrcCQ{3>irq_E5&9GUtVWI^9IMXpyGbUj`$tr9tr9o`s9m6fhRwRC8GTsgA zx8$Tuxyt3DZYp}xr-!o*ryN}8E07#qGi#c$K$t1 z5dfT7_!ycUy?c@K(8RvMUYmIeYUW}94E0(h0PC`LoQG38wGi1@3K%#9YTT7k0`L_! z;#b(2p>jw;VXg@%cHc^$? zaby6f9?GK!5XGR;QfNu9avQ*b00P{(?G23M9Ee<=!Tczj2a8x9PRap)7lQNf1g6f+ zffKC6e9TTqjfb#Q0VZ|o1bE4oecXK#W$P47t1@fX9LES>2c-Cm&`K_P#e0a@=ECuW zM$0fu7dX42J|Z}TX9Gk`Xs)-Kmvsw#z&w*CYRY&`IpR`-SP*5-+gjADOX;$yc=~yeY&Mu zT{Sxze?l+Te5(x_sptr8X%+_Hc4q|oapC<_joJs!|z#ZASH-twae zi~T9|6b8gR&r&U>ErvP?+wD867PsezPHReVh`qQvW%&7|<^>+_DZW#RA;G`spLyEc zYL?@!1DY@Xtk#5%L7xk#NgN_@%NB70tkkPNd(VjO;OKjV;{gN=K*#AkEsh_P55}Pf z?fEMPOZ`4Z|5G;b zs?5kotsFQd8#A4u=D{G~RZ-N027?|&Z~))I4r*Ru9o?84*>LDqYC^dwuWNh6jLO4d>Camk?BOyC5?B@a^U=U{0WhCy;h@JlyFcA@4m0J1n^fVqMO7**L6 z>uFg7jww0;YETL7a9Vs7aae?H3;t|y$Hu>{EHcH^JlFzC>u2mo7Wp`ZpxYr}THM(1 zWykI=<3v8<+VC5y&Rmb?m=i|}iA#PPDeMqc9DCPywr!b5F6Y``;7Q%z!IRjly(KJe z#u`nO?!}04pvjRM7W)_7;#=~ZeRZ(gZB z$$@Oq8LYDm8!{AOogDd$9DK+`bJb0~3;j08J#@BlW6$HQFOo7}Th%V)^Q8lwr332S z2n7Yw7J4qj$%8z?^V!WqR>>{@=JMgdv?Elm^1TD1L2hXs=U{02vuF4pth{sDtErpz zVTCZZgMOF&qFNqw3>7ZWl?^&x-hbdy%=x!Zaeb>2_63`#wL*305Kj?G(Ld4s8y*7= zq$$cN>exsVmZ!Y`$b8I&&g?55sueGJuRc0-OqsR(2v+!tp<^5Ls(aY1i%1Qf#H6Ed58fN?D5x(W1PATC(ffLVC?nJF402{z4K-f zALexJ$eCb8=!WbNSsJ1EGmkQS=~;|puUWd?o2|;fUao3a+hy;jwl8JPF`6}eF2ny$ zE{FT!F-76kgDQ`X!9igZ;stPS%Kb{7u>y3h{ehw+lO9!@*9Qo0d+Z8!cBpSzxjJ;| z_Lmcq*Q&$WAhtON2hwdAZLb)duPt6L%{UQ+ncP0nRPU&%0YuAW9p{}Kn^Y#Fjh@G_ zL^A2x(g)fl=xJ1*CvFa>@5$2ii{*HyOmP#&HE?t(>H5Mo%~m-P*0fO#HLr}@Sqg48 zl^$F!7#f1AByMF8PSDySy@_IeR&*VMGd*sg73R*zLE85-O61j$=xm^2~ecvjXXKd7!)Zz@WJ>N|fFgiX;Tx*Nl8 zY%vn6UiR$r+xJBuyMv^*w#d5Nk-p`)D3#y(>10cNpzXaB!q>(7x<7kv5XHf&6AE(8 zHGJcmy80s}CB@>3xYUN5`O-_pkm#`Ri+qP+JGq+N_dc#|griLHQv*a7;%E}SsEaYx z;K}L4>b?^of&PEC^HfS@GiYmBo&^Ge(LkN_#az_>#Y0;rl$LYoz^RR8z&s!1*oobJ zv)ZvLOxUNuB0Q_c~ z{d((=!j=28K^4vf8#NYyb+|$uC9))FluOW&$fXm=9y|Q5PH0wUn^-5C z1)$!nqkhs|mpuxvJlaF6GfUC?Bc5v&=zNL(gkL(S%!Y=b>AZkhY7EC{9~5bDOcMBT zluhFDGMoC7*d3ij5cu5MMAYBY?xGM)3e(4Ra`c5wTRgpkrckli^T40AckZd9vCKqh z1n#aG%z@aBUS6lc>nUW6lwVjw)??g-?n0+1-<#X6iGYjiccM2OMine6%jHk{5ALrs zyL^+l4gaND-a#i_P|SypAB{hJr~^<%*^$@-zvmks^Y|+9c5a0LP-q7;gzDYl!?gOm z&{FEnQg;LN2L+pWs9!(6dZ&AO*amub4NwJ0?@1$}>7rd<0}qw-XZP&2y@D?;{fwx- za*?GmgNbL5p0O>Xq@{FRN|1@)Iaa!D_YPwR)nbR+O*r>A7HjXuxV@++-F8xNz+>U0 zV!k=hu^L?*;YJ~snN@68M7)f>n95O#Z(96Cf9J*DL}kAxZleFbi{;!(EAj7TI(o?a zn4hR1>ZjfLf3Hj1etwe#?$L6EWFVqt2J;3gN};$>14r?nG+7elyttDG4NIQowLk80 zJ97WV*^8PMsE>Nc_uXKGsWh|x_Z&zK&)U_@QHAOnP&eAvSv?TdW-p^1KIHwMOa5Qn z7X#pM&Qyom7K&n_cJ&i8iR2N-E2Osup(F>NYr&tp4<)rok8)T28l;VPvURsebE zWyhZ+X8*Nq7?E;%4mrO_6Dk@&M0xb;QiRPnQaRV+Z&ztKMZ=8#Tmbn)(|J6iFN)k& z2iDoJBJEHCdEG;{?7Qm!!`_?6L%sI@<0FX(5mA<@$Qn|%vJ90iWobdkR7eQPo-uEf zeTzb+j6#x_$WqAI*Rp5dC0T|U%NSvhZh zzt?5ZI-^<3?-0N>;hnIrk`6$;`G>`H?7tgwihLjXVjXPSs5`*n>^{_3JplE-yA1pV zbFVQjk$zr)8GkS4w2(?%_jg(ct4|ObYQ*}u)5}*7v;&C&8ePx+dpYaxA;XMIhv}@r z06~bY!ja@5eEJGGbg2CHni+nI%F9OAhX+G*fZjBqKTG5+^=Dp8<(x%=sk6w?iJR?) z(_m)JBSjl?iJy9kEaWz3v}ro}EWCbwQH!i$Si}(E*7q<-#_RMu zdBp44E{W)QajRvbV19z0<2cRiFoJH45hIXXy?swmdY?JlW^uG$?GbAFsLtFWp8Mj0 z9=XGR*KYr&K=mh!r5mraJ>oKRe|mcQCK(wSo;G%o3_HjE;FNm~Vwo?7>TMjZM=hN` zyg7Qi_mvwkYI$KT>Kr+140#8=8{#IvbQip^I2xfibYL-V@fpYM8MMhT4vs!dZv*6m zcCR+;1_eB;*wB`HY<62Xw_K6`(pWLOh$YyF;f{?WISvbpX5T_jGL?;dPBKLmZ+A0= zOU25rs6r2yb?ajX#8%BMF>EN&IQ*+a?MAr=7blZw$#$2NWD(H;HoexdA&XG zQ8emeGxvuNw-UwU>)$P$8XUCqa;sBk-vjgDr6p8R_a@o$yJlaK94MWkA*b`mXouu+ zw-x_fG3&@@3?F60m}LBQo^T%A(3gDwmwE*|T>^}z9hLfZ5}fxB;peiM3Jau5^le=t z)Q{dYf|+_cR^1mm8Pk4Tzqpx4?BVm7-E~i$)2Ed>tgVMnf_&Dh()2`K%1imqOi=<8 z8i#H{e3z~nfuO=LZ6ICi1=6(-Uh>M+ywDza4bAc^6Z}o+Pv%ldHP-vK50Q%&fw>gI z+Xl?7W`1=ys?n#nb4>nM1KjHlw@9zC)gpK@9<^MsHj_5CEF=Lv2NJ~nCEZccx_qW? zf)B*~fvLq6Q%-w%ilHL_Hf?|mfv5dX<*~oHZ|_$J2hQV~@J8{Ui}>)Bv1ib92`hC1 z+ull+TA-%%L?LFYPXL~r2k%-__G5zKmK*A!i>X`pr&xwt_$4{my1#oEE?cwmnO+Fc zp}8Ft9!E?e6tn-6;%w||AI@s`h(n(V_jFfB@4=;`1nXL&7-)~k zW>v$&1Y{^U#%(2A8WwjH_)87R9C($JX8tsYrEb%{7mV=MtS==}qx4-=ZGv|#mJy&_ zTxyIj#4?{cC1!a#+;}V~m|Zu!8gKsOf~d|~+^74CEV+w6%|YTwzW`m2vj$7yQ5>P| zdGtP_%$pPfSfq@soG z0G|n9qC5(4+MXa>NJ>3*-E38cqP79?0`U7v$ZhZ&+l`rDE4wcB;~pc)GCkA;L7M#@MgZ0HOIr9S3jXg8Q8H9eG8AK06rBA_A+^3Zp*Pu~s}l<~DyO+l=A_ zYGgdwoK;QkX?H&bC(%MdKvyyR5XAiCa(lKh{jmZYjLB=sZjL}lq@jM zU>l*STm(gf4C2zLcnzYjejMPA)lm3OfXNoE;d8GvTOcj+_NhiXun3H!xccG(br|kP zyr-zr+i+aCtC+AvXnVQPknhdfB2~&(1A)^zo!MGPIauS`LqCLmvEoAZ$l*^@WKb%! zmuO)`5XywyB>S>Pb7>)T5#d>gCI>3DyD2M4CIL-}Qp^&1H?P_c{ zO_H`d-m-edKK#&&hcDzZ)nNDt-=yd2X3jJ|wJAWGC7TgND2iyH zw>@=6-SXtafw@N+sp9e{w`g9zH`%o1>p)zko$G8p$(#0!F1YbK95MilQ++=o5V-T{ zrAh5C$zCuyrYifOcLL_9+_!e3R{{_c(CAytqV$G+0+!|t7?Br%qiC<$@&?LN)k@t5 zW`g@S7-D^)lXf0(9(_xH0KR`U;0uB|kv-kGpGG2Nvk*exN_-{l06C4~cb?v15u&Jk z%dXRbM3yjAJ~nyIVC!B{n1^R{=Ni*@-(pZ0S!AY4z`+ESxQ1BQVuwMyk#LPos1vlu z8hb~m%H+5!&RFMiKc@%zpmOj^FJ^rB*cLK|x&5d4(nouQyS4zQ13-!!K)K|dniS33 zXFJ8N$P`K+ULCv7SvnKLV0(Q#%V6euKP?MqhYf`n8dcX5jG`}F8c+5 z*C5qH2PG;F#c9HOL+6MTZK?__gDz+Rc0`{3Urvl_LzxX`_(#rd#@1~28}}xu9bCJp zXD7PdX*k5a^&8AG(~-U(0tmC&xcJ;V;_iX^awd!id%h5uFR^`!s+ztLnkr_ndprRy z(IvJ!P3aY&Y5d5(6dkHsJ~@ssOt$JBVo%zckGCupNN4+EIaqhN=tL{!RmKJVGwd+* zF6h@!IkeUW2R%M>bcDh=jafe{ga(jwr3(T)3Ke&?QuXBgpw2dIgV7zZdpt)K09AS# zm|Q1qQaR`Qj-rtbXQ#tE7wN|^-#6Jj3wRkh{QlSF!Ll;BX+B~hCpZP;O2*>u2eBrM z1|8Jp$9`6&v$9b{hS1F;{QHwNi>?yHBZdv|n~b*_xVX!vSW>z_$I7V{SaM$cMDuGp zuc>CAN{8;&L9A46u%^_YJ5qMP?erMwyQdk^WAf=)$!)9d+$ZNpPT8K3NezD2=O$IE zaOcu1ZCxV3*%5bM!#pO|VhP!gTrhsw$javq-5tXhhnntMRo_~)ZD7+^b5MW6m3z&sT;yFW$4+hL#y7g~TdL=;WWuHO zE{1NCo`1{%_LN773&C^=A89@E?a}PXE)Vrv9nu4mJ+~(t-u2H?5AAI#NjVoKtd-mU zef8!4FOXSWMkXcmtJ4}Nxdios`Xid4Obed{Wx$?u4CIER5p-JpN&2Rrz*|j*=`}NA zBn1WTo1mHla5n@=;}e4Hbu%ZCb#nM6fKbK~ZRmBopk*r$h9$mWX6m_htvaX^@ADPW zEw>BbfqSt)mf!?&^18Hik8rdIN+oFIG&wo5@(%O$gR^ZlWtLX*RXI&BY7Psl^zGdm z61#Q-9J`};t9*lHQe0`f3u!a2`KKR~=B_c4MS2Q^kkz|Cu7<8qq@$J3blUq_mKwYO z{@IU23gh*9x9Bhjv7^J#74YJicN7_NEPZ<`|HDGLkp9{Vzgdswj~OQi6(?WXe}O@{ zQQJxabR`&QU^@~7*x!|i0;peCFKJx4I?Q=_B)n+*Wvzj2N2w-qJ9{>dnNFIxZ3B>` z-U%PD{YlJm*Zf31{A$Dpvd(|oS*U)wvh^tGKt@qH;bNDKOAR$d)xF z%uxkwa9*;R$!*j26?1NY`+lmq`*GWCsfOwo^=ltpQ$ld}a2q}^c>F8+hZM2U3Tw4@ zDNAE9*^-nK=1$=Xeh;C^`Mw=ruB}43!8+!sCbnFG_3SpqA@Q1inanT`wgyFZxiiHv zGcEJoS_(TVrRH^`d!(AG@9Odc9C`KsJRG_6--;uDr=P}llpiHO9mA4!QQD$yPbp$) z51;M1Byw*FD^$X@g3XI{Tp9`2*bUsZW2@b@UDEJgBjv>^^8i2Ab+x)>Hig~~LXeA0 zJ?Tq3u z1sbQ3X3Tm-XXP z2&<-zIMQ4Q+t_dPJbE=)5d9h|yV**%?O8PHU{pTuesmW~_-^z_HuO2RhpJM!*p?OU ze=(_HIe|a3(=6jC4%6ECy9q(kzrirp?U4_E?a`jfStB=&(E0_(6hS+831CHH%o)I) zx4?MQrCTPzaq-`D0amB(v<}#R-+>zPja7=5Jpf)= z(CtgOnRB31h{6$SGO~t~vKc|YQ%G&~M*PA`Y>xSIXFo`qJ*X~Gv7GTNV8?goFN=T6 z1;!O1us;U_YM7_7>q*~Wt0G9O;u_$!Y<~F-_UI`vLkCfypPg35{|>VJ^_dF)gF4lg zl;6)1zz$dczFV0B!y4sp{H)(k6QKLhe$TE<6YluS!p0x?Kf~HXaKbnSHcq-@ND4oW z)NzY?e11fWn!TP%&LDV?_0%~_wH_zmO9wK=Uh_f!To!b@n{&~n3}u(%crk>e$mWSh zgmwxGy~+HH+IAf!Q08-Vd=k5B%K|)*zZT}dA-GtBO^>Y1a?%5&$6Lt~FPKT!%NnL% zIiy1)&PccZ0d0F%p_~$Y1dn<9O+(%-u&I>WIc{)njom)d5S@#DvqHP{}(K?6yC!Amx4X4%FJe{1T$;4Aa~3FH#DnV-WK6a{ERA%7mJCp(JN-Awku}Eza65;!!}r(E&s~AmY?yNq;p@dh z0oYe<96oivwa{Ll%<=ngE;LIwO**o9m5+dNytWR{Cxo&bkTM^#Xk`snVouFRLQhRG8)=eKum9fvMKKY3MDyUA z7w|*QYWIzy?2k(8J!jqk72ddX>4gdXODW9Ux}rBFZ0#=Jt-K$y`G%Iztsa7E(~~i; zl!sK+7)bWPIL-?(nusVq`ZY-j{)Vsi$#LiC<~1}|*E}-J&L-}8sr)jk8M~j|#Ul9s9Mti<9Qe{~HCk3Ok`83uW4%#eZhj8S*Q`Bf^CvFZ7mY1&s zB~+{RAv(X6S@YK@6L_L>%ZXq$|z1e$(U+PGccFJoXP@H7zB{}vcLPf;4Yg4?t~ zyW}otJU|yji4?uN;2Lkxx*d5U^Fdl~QFl~z-ra`;Ri@Qc&I*p-2Mf&z({}dmFRUvw zFP!Kc9BjB-%k4iEf4uM<=NUa-+6YR6FiP+aR68=Z|H#E1`gxt99UYs)(2<%#P5_2h z&t9tq19%8(Jo_Dg1kAq#R(eeG?{PSdna4N$GV>s^*aG#Ta@E2*0UW+NgFit+!W-}* zpf)Ar7Hvn^aI;mb-(ZFS_*~t>gleRmMK>ck5RRD8*ip>xs-h&)ux*KK;WkH?<>n$* z<-ICHY`BM&{3;_S~w$-3TQLm!^z6-1tGp062qKS8NL5QyA zR-NgD135HjbSL5sb2Xwrc8p4AA>tqDdhXWi*U8&<&DEnciH*`=K6>9dVt?db(etp; zmxln`5?+9qn5rs!`S%>{WqM@a}2bon4rYFfUxtO0&&(d<7gUWWDO=9 z5FkcCKdakmF-LLq`|5shD)%hiD02+c1DK$l!cVP?s62dCX2a~XymASL)h4a-?qt)j zxPZ2qxQzY;^@^pFKu+mqj(gox6RkX^jJGHB)U7wCn{IcktNLQ5dPc(Y+WyYrGkz?I zpKh|()L(armzoAuUIetV6*&sb0lqHtf^ms6ZygGTZjQXE zr{4WLUdRNot8W+}|1OdGhV&Ps{$!E`*^P3Jss?Q)ck>qDik9}bd%mza>;3%VYm;{X z#z#7C!1xh-X|G%`hG#ATcKF11$lmCGLH1((q=BiTcabsB z*ttT=8I&F^p?sh2gwf`v9wlWlmBO5zX8StN6`iiPS=lP(A||Ohc;_+d)yzmY-=9G3 z5n<5(LhUs^`Z+HRT+~i1C@;Kn!h2`OcI6tf$QzMOQH+v246CkGaMYd13_x7V#@@l~ zC>I<~exkHmz1sfKRkq#T#v@7k%fR3saoZP0MkQuWd!;IRrgW>IH7PVZsIJq&HaA1Q zXY+>-H`iKmio;a#f4~Z1kXPe(^zW1*CTWpp9VQuFccfM$Z-r)mlJF(cA3Iy@*6UE> zTu|)oedx8`PkF~i3?@8zBp!w(BYY&~w2vR@0$K3|dDdhxx3KwDFWK+()wSG0NuD*k zB`gn6H%5RiBR@hao+zL4rZ%4}B(7%ykH!o;Q8`FfRb)D7XnSt5R zI+qox9UB!o=rhhofL^l#2+gxE>4})%<(L{XPwe^5AeAPw2x&v;i3}L?@1T#Kv?s!t z1c)D3+)7PB0Ma_Y9e6UkwK0Ft*uZTvbvDfCzvGFsqr%T{L9wh*LuVS)Lo237ThK{N zF!_o)c>psyEl$x{8NtJe-VTV%Li>!uMmY8sccR6n3Df|5h|n#Gq{*VU;;h9|x3_=r znXtX18xp%B4qM@=XXLMQAIMAP98J2|Rv5c&)~p;(WXh;Z&YJFgpEH!x<;+{dUE@`v zFA&gFvBLaZdO4c}4%$HhGb3-V=I~X=hP`QhfjZokdjzJC|Ml#tUZ-SFpa zfa8}9`Z&E1s?d(#bw0%J8w>#j9J8t}WQ*VWisZld!ILtdBjTlPL6)-|%k zvcnfUvDa?ZhbShGaLkgtcY9@I8j^~=yiE!YS1RZ_rN_PA7R9+4ByX%{QhDL$J3U7R z_-Y;vM|-l9Fi`qeWu;=JxqNLk?&&s#n<7u8zBXx@3Ub9}eTIImV<4&(?-;zk^x0Rf zz`WCMXMe;&9{W_r09CUAU~lj%&MJ&vr~6-+qyL><@HdvhKl%Et>)#I|W_CPi;$U`J zkr9)!a@nbIgq%EDrki?4djcKt7A6pp4-C~!E2dp)eKE;NNrE|lh+#SLOa+cXOx+_2& zojVq@xBe7;<=i(|Y0Rg^thKtCVWRbWL?p^6oBZ+TIDy-G-%xPd&GUI}TOc zKGdR1k$KOY*^Ep};mW7%yzLU*tTBB!c0m|%%B&gMf|3yr)!5-9U4<8Jet(SdrB(6q zFEg^X#{TUrCSPGfO#ZRkoCF=iV63Sy5HUX03crdyG``JLYSrqaoE{~65%G*{!?a>m z6pfIkt;aI*%t&DG>aBy*pk%a>dkQX|juuKYKc|0CLQ zY%`K9rN>;M!LkdKH=PDYg}=exS>IesO;XO_U?|UA}OvrZ@Zhv5EeK3CX@_y?(9wM8)u|sF1?-Fsh z)zl|iIjzXZn(hrZ|BRCdVOfl!Q2+*e3AEhs>XGevR*DgH#|;<^d-yXM6XI~y3Nv7tDij*H5gVJ_LX_u zd|BbSi+>t9&}Zlz{NXil&J%qsS^%nweVCx}*Pm{z2uZ#CxK&%0`D}fmb*INifN41w zL`(G>ln(TnvN>Jbz_f;VZfzi1xlIW+hRInbeR365lEuwj3Llk?yA|1-#r2Eg7fFlf zf`*5AheY!cLnbj#j)(_HuTV&CKZ9Gh$gwrYqye~98@|8JCNCT}8Kv=-Emz<62ASf*wgc4ZSLCxAaIWhmV5kxO0G6@Whqpz%icz|&qr=HB z@?3rZce0+4^ox6PZ*K}fBz}XDC0*x^e1oxy)E+`zCSL}dteun^W%#t|-A!jt;O@p> zshD<>-&}J`#7>3b@R}A+e(j;tj0!h(j%RN=fB*JfEx{m;gZ{qQg&lRnT!i}AGPx`t z@jBC;>(3)|EcNYu?QcF!OO!l_*aTQgO+_6CHL;%=NjXIz@EPiQ8ON4`|W>^12boDkB6&U<2j&K?|5QkRxxrp9QJzZX31Lw3vkALxET_1|F z+Nxuud@yQjC~+GCp@cB+>LIv};`qpEW2+GbI6)t2<79(w57n^wIjl^hS&Vu4^eWTM zT}Dl3_XMe6XO~Elv>d83#SA4IdNHyopXe1MQa}`r$4C~xtJ_SJ{5)V3mAT4cya*g3 z*CPz&2c~0S^e0(1M_Sh^=Mk)E=1D*g)TQ#wl03UKSn2JLaFN$#lzc45w{<#MTyNsM zpA^5{6~F4S?M z(xjalueiuAUfNSYvwDLI#5rJtS8?oWu3XXk_l;MZ1e@WHdd}H!4W2N!2{>5-7%tpp zp74edyf{UUdY<+`?aDAWIsAFafL@EokoAy$=ZXaHTOZpx-T>)x@#;Fa6xKH8t7+@y7Wf|I(OnX+CvvXKT%^g59?=@5}5ff3jyQt)2fyvVUF#j9v~t z{y^lY{oKcA*6i*VihI;92pW-$wmB4U-M@U#MnYC8k#Ybr?8`jF?3gEDv}jK(o)C{l zt5fy5H6+ONiPle^;Re1AaZDK{pS4{#aeHaA5>n&+;`I-po?K{D5&yEf&|e-#dkh4b zoFIXIBx)Z4Q=OyuIY>E{qHFe$rL&-yf3#8UZNDKy-|or5*d!ptN`popBHv_8)x-Ob z>k&CJ9D!jbta(dmR+E{$`Um!srrxQLrK`p*kH08h(uORvXX%jC8inLQdxB!89HJh| zC%cWlC~K80Z-3!41@hp){+je#0la+V?F`6c#*CJ~hx7e2t@cJ?DiWNZl%da! z<$O?2@B;m@Y&QAp@;ZqB0wX0pxAXhg-h^^j~kUft~azMx-jb6aEHKjp>gZ z`_vkM@^zg&#n_zT*!GNuJyo{nig|dYlvkkWiQqUL4i;17v#f&v`q~ct47^<gYQHXRHi0E5O$y$ASl0 zIGuE7n*o=~e#17=u)FyZm6fc`)L?o$;teqA>%)x8j8x;o60NP`zFb5YM=1j>?J+%l z{0NNq`lYODkxj3FMvfEj{cExH6l$Lqp*-$(9Er`!Uo9gyA^>q0B1_3pF-A45xq!|2 zT>WSnjwDQ6y`#a0?xMqptFfM&J3QEh7iSCx`nZPnUO14u&+6qCmkSl6W-p+B6lD_t zxOb(V&Z!MXNQiLsN$W=u~kzz`M@y-DlQ9MaTqoF9rc6aPR=GE4DV-Gz;2xx4?r+NqxrZu zIiK1I3wZId0cT==T?(AYf1))>cw&D-IZe2z@WX3+XY>71sxA7L&$|`NcyC->O}KKDMjz`qZC`grTY60AIQ~$dtAQB z#w!>*DNP!ny#|LZE7XZlbD#7Ms>su=&QwIYs@-_ZvC?__x^shuyxy1YzI>{ijdB7p z$u9=q4=r~LSsAQOK5Q<9dtJYG(Y|^i)LNOGHSQm?qn z*LJg|MhYtvuodJ{R^V>a#eu@ztQyCP5(~z9HfMyV@3~)8WK!Z6T7M!$QD!0F{Vc_u zU>59Qw8_UnhoBoWs?={5T0l81TPH%@*?lA3g!3k^j!_OuhQdUz-mN_MbWDF`go}nb zOAf~#%g!39U(&W?sP>QO3xk)Me$in-b+hU*7|VUVX|u~DaJd#H?X@2J1qk;60s)qV z#)%mu3D$vFwlrJ0`MZ++E{3(xUK*n6cL~(fbY5CBx)r{t2^0^#Pg4O+dNU8h+_??T z#lKX12&Je{zZW0&)z9@5ji?uP{mh(n+h9%_{f#+kh5ZSB+-yS^|49wsyYR;_(n;kS zB#ge7-hwD-4WSEA_46o-_~I~+;-V4&+4?*7ToZZwz1~s(HREiRBqB=onaD0CYavw$D0Gu$6x69m zCty29fPLgk0EPh`pt7}S@wHsF3$I-A2t)VAiidcctqv_wvhCIe)OHj?5x7`&C(}=; zH`QN{4=pC3__zX2INwh;u4HEV2K%HHg(_{;;az(|VWT(Q!r;#nV+J%!FI*y^ z!$iu7LZ9tjWQt)jCgcH`#wOX}^WMsi^lAllkyC92kurQbg&{3fRl}-IdV7rHv|U-} z!nrgVj)Ca7gA$DT%B^Dm&BMw`u?!rI+sGt%nV8uThus=9;Av%6j!@+N1X{W z&f_-b_V!zD?d)yagd0K}M}7nbF7_PEf5xEF#MQ+4p(4-o))vH@O1>>v(7-XW{P_ssLA$Il=LR$pU_ThGv*iUP7M*% zk)PR}i;PA5o?o9?%j9ka?9SAm*qvNsX*B%`b?u*_R>0`o_cNn&A+(TkWEpQ@_yq4V zH=lORKImFZx4y`wUHn`@oF^#&ou~wd!Z(CsIu1D540$3;-~?khS)3G0(VON*>5$!- z%9Dx8Mz=q2uu9uc09NUaV9Nxpu&Zir`Nu@6b;)4Ew3*)K32j3UBRQz**uK_GM+>i< z7yUzhLpp7(o7>GVN{enFqAZmgk-nE~{$%LOFGbqWw@?3elyU==*d)m(!b`;BCAxv) zp`70*Wd0-RiK#&xYv#5bKa!7CDn8F!Sx6pff5{YJzLt(V@tcXd;!R%RTo;eq+z#37 zW;uyvKYpnC3x5QP-ehL5ggm^>R_aq#SM}M1cL}jmA|Hc!&Ac^_!VWSU!i2Ka$k6+B zvjrq|U%99_n@~!71+5ECJq;&B_$1MNFMNZIJb=iL=<%@~yPHPNg2ef5b{Xe+l+~GjMFamMs6wqMiQr>m z%$KL9``k30YG zYb7J<4kRg5UA9a?zIVhBLF&etBkT~t!(wD*f{g)zIWpzbinKsvpQzH!UHb;q7WM^P zFkaS3SoZwM>H&cF3}c9b;_3w+pq<@6j)eIr=A1R{7AcHi^cmR0Ap11_bd5B8qYV}E-I z=XdSLy`X7+w8KwBcbw*d7q{Ri*N^@zVE-qj z<=;cu|Np+`@O(Tyy&~h13QpKZpc=`grtCR}B?R7FiYCnLD|so$qT$xd@ZehpsMLH<7}7N{93MDeK|8>p~i^AifmRx%67N+ zD4Plm4_LE}F3HATkw>Qa_>FT&hY(HCBw-{EbpA{1y>T>O=EIJuphE59HFzuiVW#8m zPqgPr-}7nx_8oV1-8AoOi{^#lJL7#e;gC3ccpuJ)d^!^wMm@2wl;}KX9&dG3l|Pwo zUj4mLvb?E)&V>Fjyo;=;*!@PPq1&^uxt^EWu)XrS(bnBD40Xy}tFNVJHvd#hF&l>> z@>}GR>`uQzn7v6#il@>z;dkDu&bZF{)KXd^pRv><(VxU%)0!}k3~Y64`=;O9>HN`a z4~8U{A27d2GgY}97>4mGaYWR&X0R$wlG_n1Zwgu&#Dy6N+?inoKR{PmH#}iY9pXy)W%w> z2NQw;hTj9;37jB$3rdOhREUP0buTO5V@h`m_N8M)j z6l=4Q34aMr7Ddjvl}00K7^kxveDGAE>>e}=y#m1iHg!T;M7A+^r+##nP+51rXn%UV zrIqD}xAGskjPoU&a~Vbt-R@$gM1t%DnC*1|75d|@`}a3KM+3R1qMV%^IyQ@8^_e>H zx~IUY0!5KIHYB^phK7$h$ZsFQZ>KJeO|MYbErg*FD6r2DQbf0=$d-Vu;aGWTjt{r5 zucce@`OdrRug>0o3pXmTc`lDshN@#$utnFAQ9DCg>3&d^2Hl>vPC+9U_e00bh3{YT zOy#V{1dkAOuzjnQt6u9#H93$avFr6{GBW98XF393&k#Q)9AL#T`-o;rP!1>)APiI& z(YXjaf$uAA_lGNN(k$F8=T@(ao4U-QZ3cTTM{L9uYraflg$Z}fNThB-oLUEvyf`V0 z7OBBP?>gENheeqcb!C>c;ced(D4FG4e`d#@dm2-8U8R zs8RnByaTg3?vB|Af(5;ke0CID{T@CR8kqJ-hD|GaJm2!+sg65|x)0`WZb3V+-_l`% z*?@EM&r*j?mv*nUl9FkU&_d`gI7cRdGX|e{B+%F6y{RbtdZ$YS@qkR%?E>bgC%LeO zH8Z$Q$97KRqaZ73+k5Vs>09Sh9w0wZ_LmvE)G7Nq?(kw}RB2H)je5*MUI?TM(&UXm zBuK3c41J600{X*&-ij#TtYvo^pNe%+)7|+lmzIP#$V67NzoiHjn8x9Ci|)n_X}+7v z@in3zDJAKHrKXi1EHxd#ec|rx3JR9SdQd&<8Y4)p5*(rcR()*W9jTNxHS%(Iaul%0V3?~J$Dc6wo>H5jL85+&%HhNgem$~OUhx8AGr`p7a?C+ zsAYdib~P^UdR068bn|Q9?)hEH&-&B2;(8-!GS33`McupWR-b8erd=Dh_x$A$SVYd7Fm?WnG?*ZYjQCQ zjm*WF%ew-*6M+k^8Y4gn49PYbL)vack@20_aUQE3&KVc9)1~3_ZyHu}Kxz6--g_G2 zJem!(^%lq;&%11N2hE#Lpq_o1^Rebqv#JEq`UJqxx&J21i7IauOeA*nFPenX)K<{e5jym$RLdIiAwqHP1g(v=X z<)*NSsGXu?yqdzjym^5-%oZ?8$V^?mn+}8`8yp$c6VC6T&!@%P^W9WXn-l#h8#6$jSXU2QW^=pV={V1?+ z`4WoP0`2yt19HTLbE7ngu{F`|MiKlXZ#tK=Ew70d<_8{2-1?R_E$Di+&ZvoVh<7O+ zPB?+NllM{JB2|~j6DP@(Z~)I05tGWz&zW@wIL{Auv@zeIw~o9iu3G5BR?LEk(SE;y z3uTyVB_;2mql=59QM<&G7~Se&5-e%Ko8ALG94x(7hKFGXu{bkzeHYVAkD zQ)J7CQ1Hq{WaXimLu#wL(T`=mbk@AOc}iX{@M-J*`|^g#$9iVf#Xc?bFpxtq;DZG> zKEZK&660d7JEPCe_AIVe;*r>IFgB`Wwhv#$QTT1okGH03R!h_$Su|!_Kg;;oQJg6@VD zBFq^wF!`klYXmDefIk8EwbGK%GUQcDa@+(PyTi&8^fXpZc;rmHO1~*g=mi6-3bxIP zvnh%bcpW!@a!}Aodi}cpN+GT23D&nLKfSTk^qxT45o;y)20^pw^h<%UvybBM!j8KA z^c?#Y=6Ao$lOl%FAR_C*JJUNXP&T?m?_gO`r#}4nX2))VW*%WLM@r zdyuaVa+>HD8W~G4;sUNTP z*R<{SPIdK{Cr`1a81H_2`LSo_&kUQ?%YT97@UN6J{_lQm=5IZJ&8+$ZM7lppy!&zg zA3weA7lOhk+J?)t?xhTL;qR(hQDeY{CM3zKMsD> zhyHxq{TO#Y+HPYE{HPE87zaO|1^?O@_)#DF*V^yL`1{d*Kk7q2#=wu~z`r&Ie$mtZV-`Bg=hg8IXff=tzD~;GD)Dlzyyt?OLvXo%t(0n(p2`C|v0DGWH`Bjv$=p>`NQ7j|IB$5V}|)@wnr# z0Pbz4EBkZI=DWV67LzK~i;^yn1v#q%5C#(1WZ*Fx+kbm36mK+k(WrK0+x~Kj(mjaw z#M?EiDsk1Vc~Uem%0VuR zcFrhzxM}d>$0N~`4Lb4ilP~MM1qmnDBVg-N5!5nZ>&u%>vLa$4JRs(;q;e^~^)FAI zGc6u2e-(-RSQ>Zb^at4%p2ORS$QNv-qJm!EU@CY`&VR)7oi1_s&?FeT%<6#WPd&bx z;7&5`QE=s?8Wndxf7j!@zpU`+`E+jykzmrnn@h(FruaeD?pNGZx&EawB;L*jI|FAM z{WKO@9dA+XP0Mx`%9HUDm7H92Hz*BD1`#i*oLMjaX20+}kMSN-j@|(k76W}`pUufm zPk8uhj|LhmAJJ^sDH1bIw?*Pq!aql`7>`{DpV8!DSmEVe%OuRwSz{>(lm#(2iLg3G z51+Vr*7DQ*P5~z$k8JlC4g0H<3v9b#7kXp(#9w1O!j@vM!;Z9`gvI|sM}Td}?N0sT zAs$&PIIbC`62s){?@|^?w#7kvY9!=D63*}1<8CDzODr872~(8zhiid^-WBQD;zd0E z`bx5D_#`x+?jU(pqr$8ONjM{mxZ)atAaUkz1UBVB3YgajL=mDY0Q!0vxRYt#g3()& zdZ-beikgfOo`3gW|D~bW|Lp%J?*r4I;7mR2gt$mk@+F_FDkr8)ULO~=KL-m9hPVm zI!uaeFlU?#%~XHExRMeZ+aMriE>k+L?nS-i;0VJ#IJOyQar5~t>TSl%I1`hnExMU& zR26(g{C7JR5%&W7*Hr0-c?}IJ`+)wOOyD*WgyceSAaNYIew^Kp_m6ki+0#0P2M0 z7Su8Dg38>GQ{><`SR3XgG_Cyv9Sc=D;y!nyWWK>{=pgz0`Fm5G3N#u$vgpl+A&Jjm z)=lg+;azQ`qi3^JJKBCJXJr)Jq!jmv4hOM1Y|~{%l#^5=+#8+v(?)guOO?B+lV$j0 zt}=XtJq%yvkpooBji|u-GlvjTbSVv{p?-2Usfi5vdu4e#yh5FB&qzGNqi|N+JG<(_ z1x}_5RzWP4g2v3f7=oTrG(-Se{cDuikdh~ju_wuqnF5O2224C<)UEw=mK$SR!u>6@ zd1)tQA+1_y2b{O;2o~LpD-<5ks_ZZ zGHR1^fcXfL8>LnWj3Brj68ob>R3o;w$w$YoPFzpBb?6an`IiE3k-KqGUQ@g3weZMh zurM!7gi;R}^^5h4c_q2y^vLeEGU<%>kLHl|bHnDmbU&ZaCFHAMr{%^+YAywV zpicKR?9|xRF%QD6?mj>6y2~~k>QAVM)?IzTN*RDHIH|QhpG3 z+XG*g8-}LO`!Q3uFVMB2NC;SNJ%zqzH!X6KWk#q88xL2xVPI}*j75TRCa)_>U z+_D^v*H;r>3fv80JoPOaR*Xj2Lu4G$sbYk&9PWtYrtfoPG;H?>TWzTHZdM2l8x3Zb zI}jawo~jSyhN-}TGkF|=Z?2lqv^uUFoqni+;r7XnXE%y1p3F+hSMpCe5=->u!`k&L z$BY(t5p~C1YhC+s6QU_%Jsr;OwZfwBBI=Dgof#Ewh$s#CNvq_0)szy_-W{k(+#hR6 zIRj!1w}Uhp*N7#bQtOy&uO_a(A&>&ueA^wm^YdTU3-h8HC--j4cODWMWq*Nc~qwn@lUW(@_j5%wsX? zZ!6%}@;w7#mC6%IKb1TlOY^w^O;>f`9wB=L=td%Ay0j=0=xZ@{+6h>r zgQwGJqAtDc0B(8A0r=L=0RuDfI6@k|P6D@V*~U!rIC`do12_%+llS_kKRW|*yF-B; z|3n&{Rpl_n8D&DkhX@9kHF;zRv<3Nbexnf{qf(%J9u|$TKjGxqPYR$Af z6#Fvt6fi_RhI4x@34&=Wi3?ON_ zEZ7xacv0kBu*>xY=lwgh)MPbGbp^R%IG;A)kxlkg9&wUzGU+_g9~(~Pni@u_5_c4M zai6NY;4x76xvzzG690u+?e#WJ3H^0T`c?+YnHjV&@zuy1SID<67VaSfqPj{NrKszdHs42i_z9AA4^e4rSl}506xeG8IB%DvA=4 zHQOjzN|822Ohws3k}b?>30WtEB3nphnPkm2_9fMbvSuB!XNECOX6Eeq=(>K_@4D~r zeH{1w9LMv0j_3Iu-#@N$%r(t9KIi9re_rp`>-B!W)m&KR&}t4$XOpnDLys^bc&sPs zWY?1VE0)5D#cihE7rL`jb?bU}1)(iXN8E*9$~)WCs@6?#OEWje1yD4(B)oZlp^oD| z6Xr5N876z82s{fhmJ`8_^p+Mh;`LP97~P@ zi^er_eci)0b1KRO&Qx!U*h@*;hF)@%ln!r|NZHC-kwc$DD{t@P{XR!gZ6Rj1kGq1F z=$w;(kp^0x%`lWAJ%YwSwFZziIXFeB3oi!eG@6ZjyF@F=+rHXMa*7rQl#mE^u~$$;-b`AO-Gp`v&RYs282V*%1mY zbF7^m$vCnci!Tr{wi)qYLu(+WeA*9N5Zw;Wz<__8D2Lqx4Hx4U9JF?Va0H+|_~({6 zxaU}^-u#MsIGM?}WXzT-Bz~EPWlUL#4)cFfV?|#!gfo5&5K#WEgkp?!1gQpKq za;_%L8=e~c#|s>Tc1@Fdw9ws3AHhjyV^`|oWt7npG(`A7iXeh!l0VqF$aeihTqYDz z;ec|d`u!honTu}hP~q$sCe9EU$=`TGnJTP{-`f2ccy8Ky+G6Ph!T5O^`A*3OND4zW zo;l`>y+aJz)}OxLtw0{98z5{!7~bKh1-@m#C!hVetvPDy@ax0@W)0{gXZQB9^HaMAuf2|Bl!Z;* z8tB=)qdEe7YLbr-Mv!Dp-cG)SPI?3H?`+(8OYd4%fZ6~te`{{cbB|lN<2dp>i!X_C z72eOh3>%)SettY-txwj7@HfNVlCiIk9_BUHz4G(2pjGdsSDef(qG#1IB2cR9@6XU* z|FOYe7Syca)^EH0tR%CIjifkfHVBUbt<2M51Q(*U(ascQ3D+$%jnAL-Ick@9^>XAj z-cAbvH!_Q=v0yBz=7TV6*h9EjzG!OLonC zuScyG-+O<@B}ctRw<-(wIjGd${($x4>ps*5tB@2IAyHHsdc2d-Tbk0(D1(dn+Y0&{ z%b)vUezwWq!amwZB0_$<(z=lKKB`^*EnhYEr(1B&DMiS!9ag}2^nl>2Td|DK@M|Xw zo_#}dD#G_gpRfz;M4Xmm9Qu}Ul^s*PeIR3gJB&3bo{A7rw=NZ@%8&YKrTv{md|B$e z*q40PX=tJt1*xV|%}FbVa9g2S9C%@$c4hFZfc>pH1JjeBOiT{%5=~eRSg0inmBCm3DXGdbZ!$SwG zf`x9LCrd;%9C{(uDlyDkT_ZGjgk}?>v3cZ^PLr7I*Bc%89rq1_sd4Z7@3`n}Yn4RB zPNGCY>t0Ft<5~sh+L-p9A&dezcw_3s)F6t=$}i`QpDP~I+`3eA?Y5Wph{B`T@e9eH z2f_E>Ak40C#@eRxwNgnVjo-icD78BoSqmG_^|zeD*u@6jxbt4@q&{)AjzfxX01S^B zdbA zc}4Sm;*`rN`?`N#JUbG(YwmPgw%$tP*tEV1Z)LqOW*@7W_b#c-C4#TnD0PChc;+GR z-HjX9v9Z};#CfNHD(Ze3Cuoei7BOEcdo~L!paGfNAI@-PC$ny0>VKia9=o=2b%V1G#2sEV z`(&#N(k}(6Q_&(El146;1Ak7TydWYDdJ}S^dZoxWe|HI^{u)bNr{ zj@fXdWnNpfaV@J9IFI}k)%xJTj6b-w$no&QgZH*?G!pRA;H@KR6%pV=DzlFH_6)NF z$)i=riE=-!iUSlfh7J{V9(|Ed;T_DI2}7~By+}u=mU~f9wEl=y{Ju)csA(fPn8a1I zlcv%zzhi1pq@ymm5NA{%ys5d3+qzX@^|0-!H1ydQBvE89RC^n;A@g2qwE-gy>Ie7H z!uQs^mv4M|wRNiCrHsqcfKwTT=)8@9@DMnj>&w-KTv3^mt(IyFTIyxTA7iwnp@$ww zFC>lVvjk=sH)wNT&1@nlT^gMCW(7ZgO*7Bg`tdGTs=alzeopHaTlKfZ&LLiyaGZ8K zb|aYBtk-$D)9I^*V7x+BW91wD4H6M?;#1lpPu4PF@Sl$+*qr-*&;A04(sCzm^==XI zM*?Fbc&P<80fY@dZdGPx)eJo3(NJrw4|s{lBga-XGhH-fazd?RJ|HbB2$N&)q;s&7 zLADkM+6?lWSSr!lD9!cGz5TTXp3KpdblIegQ{KEOg*uKhonDL5?_^uzlA?ttcHi{V zj1PnRhfC*1n94p7tq4ME33>)Bnskkq#7}{yi%xk7ZGP(}dI2v=j#_W*2%H5#`V>tQ zefGk{VG=l|&~$Pr_)~U_L}k2l(rR`&eAK*<9q%QR$^IUen|6wiSJxvkD*aYgpP}z7 zF!tBMS;SCr@rVH>a^uzHP>zedxd!X}Gdr^&J z+-Gg`(VTOx8uF^kk6JXnO`BH}Sb~CvVN;#@u%?UK9j&nhOL0RR>=jDqOHQh*Hl~Vh zI_lvMqGbtYlaj;JG}wPw2C9E8`JvOUTX{9tB$lP`(KFVE?bUPEC4R*QrWRpqNrY9M z8Qf-#3s$3>ZuB|b@K02)aO~}$F#9N87Vh3zdjEzIZ-A%VBSkw~ezNqEE&PRBAeeQ!Pdd}1Z^BbCTpW`&U zV(zNwzqv|Gwgv56d8f|lHMm7Sd$2CqWV6F|p^?YOEgL~#y?nVe=c?^4&7bdI9}6QJmBvqx zd@~dX+aNn(zi>ArEp4lr^x_LL!kx+#!VT71IgSqKd@g%-JwkLBe9&XB_V*k85zC+O zp-uOy>@3{ z@zlk;(m!yZMY9Q}*M@>2C_=%5>u54_3{t)U?P~3TIFCC~Uz$2nQw#zKap(sqW_;L9 z8_~bi3@gzpU+HGN3=0oqsp1AmKb$w3hpq3L&1~G-ko5CMf5aolj3LFcmd_1)tZjtT z(6?TYgnYcfrg;O4ytvlpFpS;^2R4jq1eLw_RLt>0JHBV0tT1MpycUZadkfXx+h->@ zhuQE!4^K1Z54qzl#{dCN4aJ6EkR*^@Q0>h%C058nCmW_1bysdzptybdRz7qkT{9v+ zVv+K_-=9wGgg9B+RO5!^bwiDo9NoDZx|!F|1A}1fq|5f^bMAb_3F7yjANjcfC12+MLnUYovscQ4U%^otLqExb$xf_%7Xss6%P>M`NbKQa4&@|IYOu> zZnv}-+uIdcQ&bvzy0S!%sVL`i!179*-o8-6h&IN_|8S3`RWlXT~V$VdC@El;kXsir9QVrJ2bHvRUY|NWUGauuwLY zI&L`+_Spqv(KXSx9s`n{;TNgzNOf|e4p?3dw^75HtsW<$_g4Ae5ATf={bDWTIdV(f zfq2HW3rtcZg;Yzx3VLAUKFoH=6KvRdZh9x5n&KU*QTmf7cZEG?Z8_h*C|DYB)~EhR zaLM-!K>7_k4?k{m_3}Du-d`&;-EnH~#YUkhQAN0W z)VAb9iS)kbcT`m5P|!JQ=4--e8v1LPG*x*KoU%5eZ8C(~*VcZ%O-T)HeA|%@*@Oc_ zX;zjw)MbR{ELEpL(8P?TLv<&OXpUdXca#XZTuhLRftecp`pj(=V)XzKB)_Q48G)Ld zgMOiQRccjB!BVSLh|EZ`zE{I_`wjRuacI-R&xgmXmd}-HpM`KPTXRZ<8lf#i!ib0imQ(8p zoIMhsEuJ?KRBV@Lm3%Mquu1idSNjr9c}L8_xkRJ^Yn>YLmgUP%@e!n{w#3-)3V#0B zK3)Fd*V$hvm;1_u@-43{^p}NgIVY!oH#vSaq$)JiKNN;qz<_}VFO#VH!`?aNrio+aO2G|)P&t#Ej=|T`qoRX@)X6_@utEtS?h|g zFF(e#7^@$@xxm-6!(rJ9;nAVh-k;wD>kNw#Ttk;ylNxq?w1=hb9G~V=Tn27EX<-_3 zx4)(SF z-=Nl9*c@qQ18j8mK_31t&=EvPV?X7vv&P@B^cSB=`lSDW^+`idJ6Fj_KKREgtZ*_` zzs2gZqpGDXDrR?hK5#Gsla>k1hS#yjP{bb;WOJ-_!ceJTy_CfM6Ky$XYsjm~`xuv2 z*L~ok;M%!Z0^*m{Y-V+69NWrUAFL|9;Rkxvr9yYEo)J8swJgMok8tw!=l zP?m$jf|^sIXBmJoSHXnl{3T5_@|BLC^eVjpr#!hE@qMG4Q3n;zAJo(13dqllxc`In zh|tG30!a88Qk0fFjJs!H9OC9RQXF08b*E>g#MK5#>@RX&QexL{uTlMsK0eh!N;ZaN zxN2R~VM^_g*!`!dhEpf}}GJdA$Kpjpe9n_b%N<0;`zT-s8c3}*%w{*a2n+F&>M z7$)9Ix=_{G(QPV@1jJ(}yY(@35-|$<2tSW@cPz11pjrV$na5YBDYfwDYcRFF2*+pp z>Jwp5Z^z&Y|ATj4ylGY(ySwT9e*T|OU>Hu?x@t{!MYXD24W>7?aqvh{Di~72WA(x| z%iKEf+}Gow5@%eKJeSGMwgouL4|l7R7pDl>3&bDu2#*S_wo;d-Qjc9n4-e?F++K)u zf7@Kd{Wm#vh(yd zC?_wMWYnhCw>ILN!InPYrcYvrYbe=%Y_LmwS!~a_J7~a50ygQvGqD%6FI-#1f`^DoyCcv3$>?#SWOy2zLIMw<5eVXW3EnKLZ!&UhMNzPI~Is{KBe{b2e zxJkV21pkesaTdf5MK*fMF&?thE3s5c^XgYMsl0UW;GLJ>u>visO0S>M-kw^}aBn|8YN$d~ zf`_eVW^UGZq9oo$;z<9ShUljB^GfHG-}u*X+(qr?@Kd{N6AWfPc$(`JP`g9!q`&Q; zzuH$=SmH0RFKpAsuP04vxHroCx8!~2{%hca$am9)R`7d1rZ7t1ae@G~37-JwN(;uZ zjXVTc@1R<1Xzb@Fv~9%{IDQ8g^>U!eegFYeQu4Dj>8eZt(ipDUs^MX(2Rje zBm_yMzhUG5$PAy>SY_Y2FqH>pr-T{zUqFodN0-@AY9!=|CUk-HR)~|K3>yucl+qJM7b*00$PguZ0^z^&8 zA0B(|A)GO#Mh5`jkWIDrEj8oJ{o%F_VGuS`3R@`>=LX1@@ zu3S!z`|Z9#1wZlg5>KgF+FpDs9{}AGh_xs8&yP9|<~fxoL@I?u<9pv&9s%wFic6g^ z%G}{f)6r<%fSOz4R*7^7U`e#r0q~HVhU}}X%(|Uk78X=KE0zH}dY^5P?_P}c&UUAo z!+hK7s#@%f?tWvNu=c~3hA{!i=C`m1wf|J0{+FVni&t)PevCPh;nP`$YSnnUVIWj8 zst3hMa~>hGwo9I7t~bRBR*PRd<=%6&7k2t`USf&I!(FJ1j=g>6ftdXCE}(L6H)cTb z?ad>pyWpm#nH&B47kexlM8q)_hU#Ys&PTjWXBiuW#GY`l(vc1v)zFxL;V7CR3VPo3 z)Y{`|*1GHJ&!U#Zp>xHNXgQg1g&k=g!KdWEOEr4R%_cLgd@iz|R9XP7KUnU-*Gf5! zHa2mjBGkC&Cj~k1Xxn9;=l5cy2IBhq6UMQSInyvXcG;iDs$v@b@Da~_pKZPWwQ4O@_ff%CxIQ_+f0Xv(^3 z!ByqrP4PRMdiULEc>Lj+`uipEBVWT^3db zp-&ydFBV?1>)PAy#i%JHtX|i2Sk($Gbz*2T@1j>&z=;RokQOii{5ythi7f-PC(>AJ zHQ>y;gA$PEGSVwqY=6?&f)+p2(rE(pja=!yHCbIcN|r)bkw)zk?}mp)Bv^HbODf0 z>mPx)H+4d@oH@WLSxlUsgI*)|QLAe>&Ve`8#OP(pkQoZ`!VZ@~3rao`m?d;KmhPB* zA4}E2uH>aIAA){bzMqF>1zC~MW@r|Z4^8%;o`i;jp+%56|GWrbPZbJ}tnmb${e7f3 z0(@8?tQYAX^sz}I^GvC%cR!z}q1xObkNbA4S(rG^1o+N*J+2rVay!($xY@60YBz0B zg%S1AFBRtIprg)lO>xcg&cUq;br{eQ$fEaHjBn@8BuEYnj?f6;Tu7cf9tl z^E>MUZZ(hiD5e1$lCpuFq@liLd6!di&iAvoUiMp8XXH#TZioycR9Rp2^^Bszqf12H$aEO`Dt&T`j*VikA?DwRA1fREp-n40S!jj z=@G+H4tL5d5hDf<7E#Y!xM@6_BNd$e>P3q3$J&lT!KpgM`hr>^Vu3igcZcww*R|-% zMN~R<_x!k&YIFr~UDv8>8hkZAunFg|vQLIllfVi48ETRPBakB%&uaNv4TnU^u+Dth z2+c;?b7X%nmm0ZIt`{RLA?YV^<-)h-)2!Hc)i4RnLpNN!(|?I77pb8lbyZKSYnp4l zXnv$0%|QNfZAP@g^h48S#hz80FjfQjB2ST)a+PN1ho+?Mm2%n57(%6>oyiTdLQTye*EU3C&%ZmEpND<_M+xU*UA0fy6jvHVdbR!%m=szG`bnXC=O%29cb_r zj0fA8^RZ6$m3~6`Wzz8=XxmUBrdia6aUUK41b(?;%f3$8^D;&Qo7c^sefqH6&)?5g z?DT~Hz8tr8q>?!`2Khok=~wzx%xK3?sfhsF&towM`Hvn=8S!t?dcxuUclYamAH^7U zNI1!ULKTdx>hM#Q3)j+k3tLdz8nN4#j%s}xNfukaaQ9OYw)cje09VDkhq!yv@Ez)p zkQ}(bQO!A+UbAfWvI^Spgr`k0LGa$AZmrBhd@*wkMtHLGiu%#KP`!-jKDzPiY2yt; zVS}(+kURPBPG{1rCcq?97oTvja}Y9SK^wmd>PZK;c51!A?4>`v z&41@5Q34Y~JE5^SxD9DslbTOi6OR+M;6lJv5}R-gIbixU1dQeugrCy5OBp+{+W!ls zF9Ug_I66oH5H|o`UqC?XUMfW*%@D>0l+2(0KujvIpR5Vz&Y?D)e>b-NQ>ujoll~JK z9tBxRFY9;2KfVtp>s@quAV(bUi-VWI9cx}7l8LYsVea-&pEF4_hA^#aHtyqR%5$TBP)-)9IDQEJ#@NB`r_kcqO0rPg%Lt< zPf`f#(N~H<7E=yS7E@&BRV$c#iWrr}OWDZ0fzhswvuX#Dvpm}@F?uwMV z95>j0KEkjjb!OPcZ?;;@=V*sjN0GOf(Z<{FpATy`Hk55WJn#QLDcUcCJ6)S=bfOL< z3U0CH$GvFTqr2zs)+^Wz3YF-Bi0-~XRR&;SJ!e#QY5P5>RtS45@Mg(tB0PBPNh~Aq zRgXexnmAMS0N1-Pwp-0^FuD-)nbd@Z_mRTr;ljl@|3Rfdob8H*__feGPP@?~hGpR= zK0nw>YpBk{C26gPLzwzV18(O5g0ULmq0jN_6sEB*NpEMu7uma#fgTAOXB6IDVrIyQ zY`IM;n&n=RW4vd_XmBtu`)H)rmyZ`$C4}a_h-P7lUOMEr@0&jeH~rR&M#gP4STxdU~s7 z3&;A6mrYhcC|;je5H`UWq$(00JIWNdK6bw|W$FSQ zq}m)(c`Dn0Gb1nSqp3VNk2mif6}wpD${%6M@yQqD*z2Rh4r1>lAJEse5(~*y+mSH_ zNgV8-S}54A}ITjq^)@zuhK}`zXoUnM&wOsOzC1Qlj9HH%iQdjD2%f9)h6! z6>9DDG)Edl)qlT^@&&5$df*z2;i3?kB4p&TS@-M`{s;h%eAqF6&k3Yy@fRv{XE9S# zdxLxl9h*?Bl~{9~*5`*M48aE*Gu?5en1-_S?*f@$ZmqoEpZR6hfB@mzY_D-7g_L()+<`?rc#9WN}JaA=UqD`FndRNTdoM^ZJgfX*A#{I5M zXUfjsxqs9^6sQz)&6FBq$|y~)5!-jtM+$IC3ijPtYLt0(X!=yk#0Q7)nNZn$*-S)0 zRot609wp`}l!i8dKQVQXj?_|8LxnMKy@T(odYO9YCaj~&%ZPvdoa@$yP>FMR@6dt> zs>zXzx%ohds!JC045%Br0yuMNXP%<4+&-;X)K{mB3#=XA6{G7#qEwHAHF1c7u0z(i zUY?Pm*^FVK`!X8`fjNM0NN0O@F@vzY+|8|Z>uj%W-s3U-ZK|BH@ekMbzYZNWki=`v2&r31h9OVv@@j_hqOUmNyD&r(0S3-*Gzo{4ZPi~jBu686 zO{G_sMw?+06@p7DO79$wo(Mk0VL7~o8+rQ+b(7pTQBNu+)MH_T9jbJaz~m2g=WUad zMlut}cw9C)r_OfHsVgLBT}2-{1urAq#mv1^&^9d*3;Y+cEd>UZSBGJNC1}5aqrTpm zc^k_Vx0mw@(1mSxA4p*$FmH#Nb5eWz8Ie?S*!X+TqOF(je<_An7W$1U4-xjR=Rl(R zZ7kJf)qyY9?Aa=qQlA>6m7!Dd_&7L3PX7a)^>=FP|Ae3aXClHM_NG^3li0`e&8K5x z*ZE3n@ z?)!rCY-9L-`-wMvh}$vwrO;|ZJAS#PdNxJpO;pnEWnve$Uw`-!05?n(xOrqQlSekc zE9Cr_P?PA_J^0lP=k?nr9FN3c15T*9;98+sGR=LNfrpoefiY!s)t>fZgYNT>0)%G| zd=!ipKvw*p=RZ)j96razm_7K-sBWRtq%iQ@+c?SJ#aUPW1d-9T>vu=E=R84O@(wnj z3^GqhNyMvOLb++x(`Wi}P3u#sCfLY~VUO8@(Y(t|6SJ!mq2PRY=%C-{PU=(Ie=F{5 z)Wv%H!qYC8Ni@6b|!gY5Rc@5rojRM!=ffNo#KkM;nk0XmT@Q5!ji zzUQzS%MQY?2YwJug=qlU9K`JU1xGJHC8T3~+XYUhWbwah8)f+HT zL&fM|MeiJEtPZu4uzh;vT1LO!Y$U&NY65$oE59yO{%H!=lW8?hE>W{mP2=+8Iv!>Q z#;uaUojRn#dP-~{G5r8uTQnk0B!F4tYZ5%Q0wK*q^Z~au30^=1NbsXLb`3;d$^I?D zuLo-jBzOxT!2^GlQ+XPC)*1k%z_hG?koM6nn1AmyoS}OoJRpHTP&neu!~o36R+=te zsRvsF&>OL3l^~VH0?^SBu7*NhGCMjv{^=SY=HbK2ukM4mik<~2uRM<5?sBrr@(%Ck z-oH?=x@;Z0@EKW3a=ai8m4`IKom^lZ=my0C7MVAlJ$uTIvXgXNZT@Tz(cJ9v=m5%ro(!<-x zbIGEY+?yF&^_N>7p*ZiHv6S7&K@eaIL$&hNN>JS>pk1(zkjuo3*5=#PM#ikOeD4Nj zCw)stvHh*nAHH&|3h44Teb$FZ{f_-Y33?_NP;o&^&O&g2i(}U0WtCl#W4YR!ogIXu z$-{5XLffmcG+a2=78ki%8or@2!E8CIxXIgGNhHZ_w%Ve&+*>bepRqPq7Nm!vnx;l^ zj!ewXIZ(-G-pkt$ZFjD2-ACkAOxpTx;7F~|*1Z85_pRMU6>#w+>Z#NK8JZy@!B})U z{)dI&$fs;ZWkmfl)aS~#H4LMN8#H=4HFl1Ro7Bq_~Rl;2pKX#e81OgAU3Mw7*~GYm z`sOmxU~46s2W#h$AppvJTEa8?k2ukJW=^b(BW?GHgcmElEQiSi%Bp;*XSk>L=`3fu?Z0Xgw|)e;=`P`JM2Afu5K#1y$L;-;+R`NJRZ;- zl*T*H&|*haLQFf)&dXiJ>da(?I>u2$+2T>@$F~yAnL)B|>q&@~+nRTtCX?f1e&Z zDj4XPJQuTvb2`zU)zM2m`uTaQQ&sA{e2qF=&Mr{Uf$p0)&Cb!qoH<@v<;l7mgk_`~s!VbO8-OMUSJJ?o@T& zpkHdz>+G82ePs2aLodK_miLa@M?6o5qB%x&K1V zm{Q|t*MFfJJ81+!f`kQL0FnCHUqv*;FqY4GT0%Ah0StZCQIOXFOuRk^mbRf5GOl)L zn7G;(IFPnUs4dNAp&s|eeCcVf5boI{c9W=m>!ucs*pY@3;#^l*I($2BV-WSi2omn# z&zm~pxU=iX#*Hn9UmW+Rud6cpU8v0`K|wf%|BW8 zeh|}amNG6|P-Gq9lW&@?#8A42*`~w2U0V% z;DzDF&fW|Mw&f}mWTTTMvuDasdjlT$`)jR9V;w+kxx1#eq%4rGNP}tXJV0zd5DipQ zt6(6_WG+A9SkVE?{;#NBH`WWNb{}?(Ry4#!P5(lz$~df6;HIJEKP1dVZ(qBwo&7V~ z7_uZwkO$EtJtQ%HGr(DYGQiOP9R@f#XC)ZK)s6{dS3=z|IM;%Fvk5i!iLa43wg!Xv z1C8*Q0Q7No{xH z(L=|rzW|kg4Cx@fY)RmB`p&_I$XZJ<05a>(l$2mNeY_qN_$#Iyx?({+dR|&e*mRPo zpt%bF<>94qZDBe;>bGkl%;<=o8hwe5NjYR@<+?yQw$Tq8itZzgBs>*_b6@)**uHw`qFUkhJS3`DCKegHnd$J4ix+c@=!oi1@2mD~tWs?l;8V=N?+; zTI@5|BNiy}{6Qj$eeCaU_)N{MGJ7X&IpGPgl2H*zKZZXTJB~DGvjg)Smg|wtjuadh znYr7Ot4?|KN0MxDHst6_JZso;cAS))i#vsRw#l)zTKV#|C9beF{WBm6gn${=g;}j3 zt0}??S0512XrdT~BRfsQaQKt@s9Z1};~8sCW#qmq&d%^TH9lY$nI3^y~fB25p zo%@t56m-`xb||`A^tCTxg_Q#bo#kVsx7T(Kcd~r5`I+s9&7SXh;N!1%r~~F>#cS?6!G?_u-QSqqtQEDhAn9KQeb< zy5Ag#u@~K1#ihk3EyaeMZ9igz4O`b4O?$Q&q>C3W)^LqMv&&2+mNmNoFHW`^DNrIO zyD;*gh6Lt#Y4KH@>nFv?kA*F+?=<_zumk4|kTN`w&ZwB#Wh&Va(8EZNj@8S9c0X<~ znQiaoC$4?$*^v48l~~xDS80bfh$hqzg{1JnxRE4?%ac>*9hDJ&8|!LyV_VpM$$(~* zpnJ{txp=2phf>s*VJl52?l%Lvb9r(jgD+t8wkUzUp6hnZ>F+kTF zGx-%U>^%WhVU-^LwYo8FT_tmr%!(m_@<S;v8MMAY8t4Jxc`w|#2djzAw+jLW!-f$?&k5|4qEjN-= zx09vv;)*A9D)johPMPb4C6WAxGLk#8ZdG^zpDj0b7B zD0O`Rh+-mz6H3yBjTD?8rA(w=!u*X<37X(0z{d?s8^_ zua+WVr`cD)D<5?E@U3Rv?+)5l-pVT9vGG!M+k^LFA$HpYRh5GBH-~c#bvKTmr zeyf73!2EiFHAP2(F#a!?$A8aAZlI+8J4SNvq@5jFM;p^B$rA8Dj2)UVOCv9=09a)# zF;rOFt%kGW({nw^6h@wtgs(yy^+&|#^Z>)7;o-d;nxYq4b#Hv`zR^Bftp<1K6$LKlZfoB zKInOf0vzM>JAKqD9v2~kT}xYcw?Lt}PL`rWdc9?)%T5&k&L}Sik)nWbb7aX9n74c` z6`e~vMO5k%L{dhhpze=gnU@)sv?9`Wl8tkSm{OVR{?9ohk+SC55(O#_PSM}*dHq6} zMOrT6K-8{PmGHveBm|MgcVc^&10}|W?FF^*+Bd5Q{~-bY?SlWi5cdD`pV9v_K=F@w z01&}CmEBOt$|#8d!BXT}G;BgD#)i%wPTOXiA_c*x=*c<78p+3d$h(4sOa%Hqh6Vn7 zk*%!@-8jr8HlUfa0I=3pstUqK@;5VWoIA9&FSUMVh+cBCO*!^8f7l&L?D(0tZ^_&* zjh-oT%%xI6SctTupl5T^uMClY%xG<|LCz21oW=?+dHa@^HW=19 zCU^6eYdt6PtFFK0Gxf1!_uZ*+u@iatioAQsCTf28FtlkB*+QMg*mM+0^Ej$~z#4jA ziC1Ovx=WEc6Q^ITwFLOzy?3N(a>y`t)u%IOWgPsrccIo z`?{K5lsxsnFY;1FXo7#$Dk5L)9M>>V&;)rdc~)vt2|6N#KBCghJFbIBM7)#=8aKou zMY=*%ohrrR<)*C`&D-inDlTdgp7#Zoj#dcL^MRGu>&*&j8(z>h#NRGsI>Bf6#GFFi zvu0-)`14eWcfj}CJ7sBFj}0yVasi3UpJG&0U<%C@hJ7_wQS2H{eifr<+2^n0FLJu3~Vl+Q@iat?J$Oc#k-pdhK>8U zF2EOFpBl`RSpGHFQT0RFmt5tk<(V)7>lZTo|Eo8k7 z#>5{vX%9InW2c@>fn>JE0nVY49{F6QUKmY@8#TY^fY}a9ng>s`KCx=6?3CQ_B3l2_ zkZ@#ox-eIuHfA8;J{C9LOn+&Znt*z|>9bbGiW^Hho{A2qLyyEZw^zE2^sev2JyM*p zINiGVyno>2y92&L(y01^jg|e3B)GzvDn6_qQGMvcWa%!k1cfWPPo?*>>OT}kM2+N& zX%K$?@WVcngAWcheW7_XUUwHUc?d>HXE(IWwI65uBtA~^0B-8eW>cfEyN$$fKkWgBJICWVwEz~ zS&hz5qrNwlzvg>+?Pym?_EzWne15-stHyIvMsLPcB0l_OzwPS9%?>uVwzxB>dgrrhe3$|Qy?MTF0z`c{|fYc zon(DBnzHlpxPF3Ui0p-wD{9kHtOv4V~^u5zffF&|btbho@aE>(}dsg}cg(a&@X2c{V_ z=gX9bLm21b=2xCOmOoau{|v-=LKkk;4cdLgpKZAA)lO-9!W*H2;s63h`-?3K6_TA( z2=v+Bo)rMr)*R`bpr8W;hiw|9hXK`{{3buiMH;_+@yp1$M_%LkGG(SezsxyRq0^NsU@kbse_X#Qeiks*Sn6=2&Z zncPwkw|X!qJ>R=AbwSzzKwcmtzex1M2HZ7*4j~AtW|ZXTHiH|gnbPI~oFP~J5cRy= z%2Ck&acRvQpUra`u>;MVRx}7f7>2wq_{W2!EZ|)hWIa;>ZHcI0VLOAeAVtz!6;_IV z9f2tXEHq!7wK2WzQ=sbLd^-mYfMDuXxfR85Ip%If5?iv8*u~8P3`{^P)S7t}>KVxb zl%eK03V#ml@qa}aK~kV!T8tp0p|fm#GN&oGLPY74Y|LZgG}VKi#5^zO~og06lmMH09ovOLnKn4u}o3=G+YKU>ferR zJfiFbBq`6~cbC3v;3{D(1<F2>dJonQSbmm7s2XYm2P(+BY<;|p%K`AO2+K{pc7`WGxtBcx1jaag0`IviDzO; z8VvjIed^)Zqr{#iLU|~)|6aNP9fx(&JTrxPhiS}-JFG$Go>nI91%%;gLQAvgBOi>D z^oj~QR)$i^evRL*b1*{ObG|;R&bYkB{)p()Nss%D5Lou&pMi3r(5=+)T^id%Z^zcI zevCe#@XT?}I$pzj{y0Gg*KAYcyrbV!fs#;96dc@EQjQ5%)HJ$A-`+ci|JRhu`hRE4{ZAkN0IUqI8OD}{Vn}K*D8Tk@yRxVTU{5s%r zhNxHf1u{+-oY$72Y(2bZ32RfS$_m1p08ST#m92!A6vS^=DH0Ascz(ia>V)x2xh3r9 zRPD`uq^F=xMxT|e0-C@a?V4TljM~>mdC)&tpuBULmr-NN&#V5 z0WlDaf@YmTy|DwX@GI)PSa3WrT5_;rK!`9^WtXsk%4DgAL6U}vy<}Jd%X-!XB8t^U z5Khyx@bYW#0*Lm%zNZMsRm6ggXWRvvN=C@!JtyEVy(R#(6=k)Ao_?K z?(Sc!{`dBh|EK)S^*__nrT$8f^7WgmdD^S@!O|8~!O$CgQ~14|`9*DI)T1}bBbxeM zzxW-y(BtQj|7vWih6Q|B_T5E=H;ZjyAJZx7ETpd z7-(cXFlA&=38SEMv-b!g2$8g1FM`=I<=z)zo0obpTbs-W74s)$+~|2Nzyci_t?`>M zn|;jNwN$6J-+4=G%Q1Qkq?Q7<*|@K{QRRVk@ce1&_xif>qbF5D!j0VZ<^AhJof1vYtGzav z!gb+Nn8(ihzHk-v|3{lO9D@v&VPXJs6 zq8;%P@iRymJ%9*10YunrjV{A_U6Oeh7jJhBxeNMk2K< zi4*^`xpt>{)8OT+^Tal5X4+|RIwPCjXR%{J!xrTiEds8Pw&X9VJRaqTIdEHZ zU7hVZ5_lJ76K!I)1zGo1a#2Yxr2I*Vx_Rj`dn7tQOC_WJ6J6g5wJK_H1XY<;abP`pU=s!AfnMRn~oZElv4(PG;qA*`SN$M0a>sl0dc8q^Q zVfktp5||(wtd765m^vFrwd@~8&n^vIPH--C1|5r_i;9KYS*jD z4Zb3OfblUo*3NK4CT_MCG}Te9jcIOT9E)W}8{T*&j(02sX7N;bALErHWH-cTmpg7% zP1XJRm2=*3E0WSiuX6@XRAA|n1a>-8WZKm73N$L;#)0?+3NpsOrpRko>hPKB>tnEWfw{WmtuGSS}b=WW_TDk01k zgK*eaILsyoI-R@>RCJypPuB}Kf(xrRM@H2DU}{j7?-7m5;elP0nR2TwOQ4|-HfT=d zG91g)U|k70|4z_yH!U?_NqI%oMMC9r?j?%bWJ}G_GG>0}(niu-Qm;N}7DbxVrG}h^ zqes~B-DNuyyPoF7cNPnM72Z1$^KiSwS(eGW<@IRrd>$H_1tP)be1Ew)1?r6PRgL2> z`XADCi+JLzI$xX%b$_GQX1`H_azG3kkLeyum{n(9aTt0>XoNM6D|~VpW_y1qr0J!K zovwQ295cS{u1&G5nbw#>)279(D9GB-DA$M@J#2*R>k74?$Tg+4N1>6FSH+#6vcG+FQY;Z6H6L3ltGO`BM zG$6?h_~vuCtP2tJ6dxUr&?N^IU!_-!<|LQ&z1ypA#coLKJ?H+hn$BU2Y>FOj`B}Y_ zb((tO{i0&kgSJaO=6*^dIPrT#_Ip>l*f%D=s`=#x=pcrA3XrrrLH!bB_q1|<-iN&% zT6=n`A6-qg*MFr~=b)mM^vIJdJ&}~-CiG{Caz=E3?2i$B+!^mx4&!5UxaT0T#PsDD zvs3U~L$K?~3eILR>*yKI(_n%p2KR-KbuI9qZo|T>H~Un6?FR->h1qy$@dUQ zzJ>p@{j?2h6z<6Uzs;&~9o_$?%Q0|EPTmJc6ku+N zpTOg<6|mYoYc2{rLWn<-3wi`c`4=!^ExZEW9wb$=I)29lo_r ze88bkV4#Zd^lrFM_zyEa8AORN$+Zc=hgPKQ zE2WCJtb{ijcX(yZe|36xif3~Ro}dstk;t>Ri331N;7lCFy2L*Vv zg%`{l6kIvB; z`1a3fU`Lc^ZM)+6ujr`^Wr_H|P)<4NGW0O-IOy)c%ux{_ASPs6dV<4FXFfx$`tj&g zccB`R_`!2LOsUFdx6wA<`e}PW?%r}G6qFG2A?+Qf?{;2fkOP@FHCsQu7b@1Ns$UrzR<}xm(PBaMpz#-Ae z%h|GF&ZM*#L435a!z;mJLi&op#Q@wCioMa5S~W_-%@*ebU0xhBsHU*mqZj0PCdF#fV1&5MD=< zRvJH?9G7iIh@mxmSBl%C{NrEi`U-5_vQffxH!WSiq@W{?{Z(-*1cqEiKYZ33Jc7Bpjevj>$7JOM;g z%%mL-rSZ1d|Cnrsrmoy)$8cTmjIBKwpF1nK+@e*0APhgl{?v!$Li_koTuFip186C- z?!AeM+O$%!jH0n4C8G_cj*!x~g!!@-L>nl#HG6QCvB}`x%7WpR^~it?>Y-2l)E@IC zOuVcJ0NEhl)L@%XG55o#Yh;r7k84D4`0Rab+-Zrf?6s}5zvKQNztsEdojzS9>`CfQy%mQaGqokikxMh z05e5oGW)@t8+2_Qh~W*6rlQyLoBX4!1CF*snm`9YI1uOn07B!S^-<99wQl@B9JkQR zV*X~s-?!=iTmR2}qi?J%!a>E%F(^i%dZBG2dnvCoOJDT)1!2bJI@7q%jJn6QPZYAh zHE}!mOkJemCaP(UtQ_=lkg;N%ruIiwYY?NOH60}i8IMB>`R(4ebqkFc3nb`1-8_P? z90-_Ot+}qWYR)Qzk7dcqE+dK1mJ{fm6z+nOAf2U?sVBO;E2eU^ItNEcnDWA%7;^9M zqn3lb`fkuD&|5E~VW`B>N~ef@rQZEIyCRIJT7?Q*j|#?TSH#Ayo_RfZ$PO`*h)i(7 zmn<_8(1*^Ns}I)0^{jjT3j+%vRPzjN*!j5=)C6uj-2S@q1^?vWnNRa^U1CJbO~=YH zgH{C5wabug+`Bp4exo2~{wpdekkB5>vQK}I8#;g#)Kqqwm*O{5!3k80Z`o^QBH1uu zShs4gjiI8~j;(!p{RG5M<1P5l36x)xti03Z^$t1_r-f=H z;xn8X(9lnVNu!aHASKkAyd5*z3Sqy-jC-M#r_h(iza8L_B?b(+vWa}~^iZMe_7*b?HjcOYG5MS?NDfe)l}{&FN6HZ#Z;>u(rc zlV-vJZ;CF8t!v6u#zqc+{7{>rJkR#a0eI{hk$ng=egor)UE3+M{W}D2^&LWOy&C!k zUv+S&!#E;_+mxp@d7Ehts8V$0qMkJ=H*BT8IBjaZ)$R-@AE@^7Yd(G!(dekNbd-K= zJW<6evUq-W*~?%g2)kgqS8FPvKqf&jpyyX^G9hT@C4yUX4b?(_qp6w1RSBS zVl|h_=_jotU1~FxNlTI;)56uB>cn8z?=OY}f5KIG+zP=h@qd~aF)eAH$2txk31^X9; zze5E085cq7VK+;90O%+ zvzv9lL$D2Lbq`POXje$^YY8HArhSL(<823UIZ)M%r2pbw=#LrBH>`BdW9c7o$A zXSYQ~80+BS=5?i(V~MKn$Woq0vjf8#I>l{5uY75V){DmNB_g1@01v~11#W@%gdWd0 zhBl{Sqe3NTg41t54T_^VNWBUYZoU8MpjqbC#_%|$wH>Tvrf4x*hjouFMOSJHuy!Yw z+|3>|z3!pPY36;A`>0iT%Koi@w>h^EjD4kt)WE2UL8=-<^CREhvgxHSTeT8G zzxto@^i!*Tgv*1E-3R&8FZ4-ZbHt9GP;o1k1AOMIO6;nkrl;4wLu^9$ z)>haX9T_V02V<}Xlu%GS$ufxuPtbFGLUOQsvoyCnr|TC$ea;oBtZqq{XuShSnb&S* z<-9&>2KgD3{Sop#vE18(IS)eWFOuJ<^fM&Cz)>ov%SJE@;k`^SCD!lvSI-L_LJCZb z9PvF~WM8!jx^cPaZI6&nVl$$Lk9}{QR*n-myu$EH_?RqiN=c?-@w3bKEeEA`mwq{} zchYpvhIi2I`KDcXZ2PArnjz~cTZ-L{-=yb3LPp&md5}A8Kk;g0_c#oR?oI!83jOWIzVzb{&aghTVGFZipLEt4dXB*??^K-N65mYFIL=j= zl=91I+pVX0!g=R;=aBCROpy?p4r-u)bb^YCc-G1^pnBpIAIe*wJXXtC{G>Kv%MLcw zC8SA%_{>L~9DS1FTaT4tOUwXqRe$fD-Tp5GJoXo=+n>d$4!!YddR`|HAitfNcLTx% zu3$KVJx~_py#^nMCTY@-E3!>f8?*xj4iSgVMlPlc40D}Kags4G9f2nrvST|Y_4=vb zNLc!=>bdAKtmIdFm&mc>4z|a4u)5mpy6&!s^FzJkNCPVyMYKfwb0P>M-1xi&PwqZO+Ye|CtTfMIhyQ>Kd{IIA=2S zQA$x0mQT-8t$?1=V4;=jP3EfA6W%>tvoZakI4hU)_33-VAiZ;uj-~VykdV4xE(4)9 zo2%|wJ1E1SjBjtWHWy79`8r}e=^0;j=AMub!AWUt9}CQlV|gP`dinTkT@d;t9Czw( zG=P^MsTA)|s9FoH*lcLmcQWSD3)HIn<~`4vG|sA-vG1{f8*l*?HCl;}R&IITXhii+ zl~PP2Nx7wcNt9wc&I>C|4vI~>A52JbmiOgmdxdIY>Om3+aif1lsW?b$sYyrkB}&DP z!bCl>tF>TWx*+2R%}OVqqnCZ?B}fI7yKRfr$U$8gqxQ2u?{@JfD^482EQ zK_@grHKn2N*p>dfv5f4e!31HAn~8YEKv)AjN)~1SoPc^0prZHEAG5{jtRpLr{Gdta znUHje! zpT*)DbyM#%uF6W@mT><*{9~1HFly?yGC3%Yc(754T+_3+OoO7(=z~!0-7vNw#e@T*66jca4AF+D5~ET0?SS{J zx1rs6Sz@4<*z6VqH*?vRFoPE07;q&2;Mhr{BH8BHgx@%JPWG?3Dw^2#G&0|A(mgjB$zHzNh?3m@Fj%yN_!FS?rx@uo72$gQ_Go_# z54ZPaZO%T@Zm;c#`$b?+NC2y(`2(PYY*Ay~_zv+jY?S4z9DRV+ zAz~aEAzO-@b!VIi;sHW~K4e*OrgqK7?(JA1lC@Wj&(6$^Mf~7=eLmrW{siUp~G%C`VCO z-x`(mwez`F{)*xm@f)R8xV4%YFezkWC@11k14MO71nG0?yD_Qh_eB5YJ+7%oUOxO{ zR9TMFvv8SxEA$tp3T0Ode8VVHm!6p0_{=T%saMuU+xarqJ9(d|Bpbb) zUj$&5>c|h+IRNV6$^v7y&H7azTfj)W3C6!|(Nu)Pr2KM*M4ZF<*x7ZCsmUit41rQ=ziCrkT z?o~X`x*vqBOeP2#d#5(oI!SeVBhf@c9M^sHIkhiF(%Hku5b&=g^h%2nZpwjnh;^e* zIJ8wOzIdRNx+`hl_Qm;HqiaHkx7=Pby#)I}3wnD1t`;aU)U@FgglW$z0gjRfVgNxi zpm5cYjp0fDqaxze4NXB8Pwu>f6McwaS~+;>ExT{M=M%T4#OC$)*9-1%;g^Q(2|;ao zKUp_r&s3R~!gCGxZ@5z+tTjkvvTFx)z&iUV1W*v`TLX7 z-xe$j$=j|2^b`wFs{Fg?b-F&!l)lpHDp;M1*zZSW*c$APIvFawp`Gh%s6yAA-d5-H zn!R!II~;eibCgWmpts5%G2Ng%&gIPP$*0|3_MP> zDB~dL(_kr0)$Yd6p6kUgYv4ZOK>@>T2E!D68c@X{$f#&1j^0YvCR+E=B>q9;#2nuZ zcXp3oZIrSz!}Wq%zQ9IVS@aIp-C3p-b=TpS6n42-<&(3TnH`))A|GGDHbeta@Qxq* zqxC(~`s;8IvKJ+d~-A2LpDZ6odR$C;A+?LO=-}y z{B>25y7aqkX%(Kfd)1??;e0Hx#S0In^RyrZ;1^Xm=u#x^RD)=oJ9dzBzTq>+UW0(* z5RH(k6a%+h6Ipu*!{iU~bx!d9-^SS8f1l}-M`Pj7tLOZmq7PF|iVD!{GySVRP5zjj zW+y#wAJwfZM|%rwl{^Hp{WQ?P)Ers$_Sd1Ucdg&e;? zR2bgb5_dH914OWiXl4f@q*+&WH`&;d5VMI4D|$5m6c@pgmo;^wlKmdHJ9hEUCb65U zEOlZgV=_DH3{LMvy!}|r$~kK)Qg|Ulw6ilUE$p04f{<-Z774!*;|ZfH+@-Ofgiv{C z=R=frLK2sT=#~?;@d0-hG38(2uafmhTeVtvhA)AAdJ&QX6y^f+17&%q8)DTId{cV$ z(!I12zn7Wkd-r}OsI_0VG?I`H7E7dS^2b3IgJZsZ(oke2vJAo8V%vZhT-aNCjdA#_ z9UbQs;aOE9T1C#afA{I*N70f`0ms8tO?B%fAoK#-Mb=GCUc3u34qJh59zH-6p@_Zq zA7kl|Osc4wZQ{=~=faYt9n>WX-d5`~)5bdkiXhEnm64Hykf+32h< z<1|>MQ|KOKxG-9?fBC~M3#!3g&AGuHH^ckejc;e9EtNX&0zMYYk;5|om-dNn2>^^d zi%HfK9!JI03H71{$+{Ai+k_peJS^Jm3fhYj_nEx?x-tXE|Jl^K zQxKM=^g!zUzL^#oq4G1Nin`d98GAIJYN6z5o0pN|9GhGB!|td-I9U7WMGbXQ)RM?+ z<0x_Wfcqbl3!Bc@mcG8>T76%hxi^7#raMX3H;!pV{|2T!cM2|M-kWK}(4!{=x!A$( z)rh@Nz#>lT%|*>YhZ-WX?Y6kAKtNXXFFDdh0X|-I9q_>U5w_;&avVmf#S{*&P>~ZiNcZmRdHl|Jw zQ2thk&rzu=rj&+H2p{2LiCX>fZQIowh2N@uktTn5&?c+5c-{&sv{~EslPoV-b`p@1 z{*1Pe_IS2pCtyi?s8My_SF7lWPp_(&u1)*O&8{1)Mr1X-Ja9IaVnK82a_Sy0Ndz9h zZp=;X`xC3eUjr>3rnv_cCha%WmA5%$*vfny1{VR7Li;DiF;*&k468+}dC+*BoV+Cn zZ-Q;o?pWG#Enp>IY4=8`>R`I(L&3!@O&pcBP4KZ>WO$WkL{{uL%Q2ZniCF9ZTFn%* zcavkKEce`+bFX#HH}Jk_+?ThMZ@%?xCBTYf z*338A69nnRa-^N({MCHk@{IgDyzL?(n+14q;Bd+*%)kTkXbYAo-Z-+0Za0>RXu9uM z5V6-&v8MFMMT=Z~&n36jb63S|;E9kBzfQaTtjdpj=NH)=qpDG?Z)mH4-8c{U|TrU>5yiy5&01(OyJ=CH^c}k_V|6RuG`U zL51U8fki-ZgUebVD**tAqtaIU!I!oIo4PH;jaEj-GQB7>Z*;2gk4A{@YV{#*r4`=i z$|9xropJW~MRe^ExSx-7XY#^LWEl$^5vnj1Xdv5np|4UYs=Nkbwj-ne{Md7l4O`>+ zNr560g-=5MR8od9c+adSNBi)lfd>fNZ5LS#zfc0cqP+=2Y$S;%FkzGh`#wK?%f-&w zh2qBv&HD+LP*Oq`n-Xq@0^*#b;OfKRc@)<*$+!Tdr{3dNnd+28Ec(DnXGB|KW(>zQ0pZz{CEThU55!b34bOFm6?~ zHv~j$rZGejGHvl2#kor zjH83&&&P&~63$(?3$gA5Dodb@MEi15W4=y6ftBIbYNFQdCuurqiIoG0SCb7XUTfYw zqQSc_glk4|o$2HjJ+a}0e%P4JofA;=ClcG8yY8+Rha3}dmbS9^Rr5~{@=d143u)A> z3?3W$`$HcEA1m_YNvt3FfyB3l{9usdfI!Y!cmDcQ#ogdX98RRJzIpDdS5XHag-RY2 z*)LltFS;T9ZoYeA=XBpK`yAFp_|63B;7Cqfr{TsMhNq+<8^9@tp9bHUwg4B|&S3(1 z18fWK3R?eo{pL8jfDwPRR4zC_E7;#iH`i)4Md)d5ih~;@ z7>if}5(a|!u`0H9RR0@UL4&Od^?5x2Z{cmWf^NN!<|JyJXLr831!cc0fw6g`$F5zh zqvud1-FyU2xCbi(jNN2FI%(X9K$#6A57|zxFhxvh;pMDHaQ8%KdkU*{D9EExtsr{Y zj^q8w!!GkKFRb1UYBs)oM^L+VZ5r8uWeVGlJsjPDxg;y-@HHI>cJ=9XqxEoH37b(?1JyqeoCuNYSKpWe2T&X>I7fxETeX;Ur@&(6eFtrQ*A zY*;GX1t3pXW~tOxDM#^U+ZoDA^VJO|&-?OhN;BG-4CJ;_Re1@!BZ$<i1(Vqaym%N8CY%Mz5nc`7UNW&EsXjH9x(BkR*etx(ObVAd}W-Z*(bodvy;xvN$-n{n;NukSjfEa9W zP7e1Odj$9qC2V4i_1?fNXCEV|0xIRTy57p`+J^yEW*^FHnZbYHeYLi>^)uB*Am^uh>uGl20T~jW^E(6u z_^GQhNP=R(b10Y{eWKr7{Pn08E-3$#P|Dx|B8?a#oLP3Y43WS^+V6XN6p zD(_Xidett0HhZ}x%E8Q_^O`&KA(O=WrYfRf1gz8@w$rw=psIrF8r2&Y;!461T~;-Y ztC^dlwe#OA;;|2Bv%c(izHvqM$Z>^-WHs_F*j;e_Onhln9^)v}w2~SaH=6~Ib`pP) zblt>G=vi>)&=HLQwE$0wi~KwxX{O3D2D=^&-mm%x2d*);(}B zm7tZ>#6ecm7ib!+RNOvE5_udJA%b_Sa5!_9O;6IaqVsijVlJWvTn%3^oYE+%^*)w- zX}2qs4m$EpZjdy)SAZOE!EhjtbQ+3i%qbn~QWi#Zd5)D;#+=_?9eW|xH2VmLA^CaH zpo!WIK)U>I@AZ%O{U81w`nMS9-~He}ypR7+k8|kX(gS~wp})t_|5G1;ejoTRYdrM| literal 0 HcmV?d00001 diff --git a/docs/source/quick_start.md b/docs/source/quick_start.md new file mode 100644 index 0000000..22de6de --- /dev/null +++ b/docs/source/quick_start.md @@ -0,0 +1,200 @@ +# Quickstart + +## Prerequisites + +### Supported Devices + +- Kunlun3 P800 + +## Setup environment using container + +:::::{tab-set} +::::{tab-item} Ubuntu + +```{code-block} bash + :substitutions: +#!/bin/bash +XPU_NUM=8 +DOCKER_DEVICE_CONFIG="" +if [ $XPU_NUM -gt 0 ]; then + for idx in $(seq 0 $((XPU_NUM-1))); do + DOCKER_DEVICE_CONFIG="${DOCKER_DEVICE_CONFIG} --device=/dev/xpu${idx}:/dev/xpu${idx}" + done + DOCKER_DEVICE_CONFIG="${DOCKER_DEVICE_CONFIG} --device=/dev/xpuctrl:/dev/xpuctrl" +fi +export build_image="wjie520/vllm_kunlun:v0.0.1" +docker run -itd ${DOCKER_DEVICE_CONFIG} \ + --net=host \ + --cap-add=SYS_PTRACE --security-opt seccomp=unconfined \ + --tmpfs /dev/shm:rw,nosuid,nodev,exec,size=32g \ + --cap-add=SYS_PTRACE \ + -v /home/users/vllm-kunlun:/home/vllm-kunlun \ + -v /usr/local/bin/xpu-smi:/usr/local/bin/xpu-smi \ + --name "$1" \ + -w /workspace \ + "$build_image" /bin/bash +``` + +:::: +::::: + +Start docker: + +```bash +#start +bash ./rundocker.sh +#Enter container +docker exec -it bash +``` + +The default working directory is `/workspace`. With the fully provisioned environment image we provide, you can quickly start developing and running tasks within this directory. + +## Set up system environment + +``` +#Set environment +chmod +x /workspace/vllm-kunlun/setup_env.sh && source /workspace/vllm-kunlun/setup_env.sh +``` + +## Usage + +You can start the service quickly using the script below. + +:::::{tab-set} +::::{tab-item} Offline Batched Inference + +With vLLM installed, you can start generating texts for list of input prompts (i.e. offline batch inferencing). + +Try to run below Python script directly or use `python3` shell to generate texts: + + + +```python +import os +from vllm import LLM, SamplingParams + +def main(): + model_path = "/models/Qwen3-8B" + + llm_params = { + "model": model_path, + "tensor_parallel_size": 1, + "trust_remote_code": True, + "dtype": "float16", + "enable_chunked_prefill": False, + "distributed_executor_backend": "mp", + } + + llm = LLM(**llm_params) + + messages = [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "What is your name?" + } + ] + } + ] + + sampling_params = SamplingParams( + max_tokens=200, + temperature=1.0, + top_k=50, + top_p=1.0, + stop_token_ids=[181896] + ) + + outputs = llm.chat(messages, sampling_params=sampling_params) + + response = outputs[0].outputs[0].text + print("=" * 50) + print("Input content:", messages) + print("Model response:\n", response) + print("=" * 50) + +if __name__ == "__main__": + main() +``` + +:::: + +::::{tab-item} OpenAI Completions API + +vLLM can also be deployed as a server that implements the OpenAI API protocol. Run +the following command to start the vLLM server with the +[Qwen3-8B]model: + + + +```bash +python -m vllm.entrypoints.openai.api_server \ + --host 0.0.0.0 \ + --port 8356 \ + --model /models/Qwen3-8B\ + --gpu-memory-utilization 0.9 \ + --trust-remote-code \ + --max-model-len 32768 \ + --tensor-parallel-size 1 \ + --dtype float16 \ + --max_num_seqs 128 \ + --max_num_batched_tokens 32768 \ + --max-seq-len-to-capture 32768 \ + --block-size 128 \ + --no-enable-prefix-caching \ + --no-enable-chunked-prefill \ + --distributed-executor-backend mp \ + --served-model-name Qwen3-8B \ + --compilation-config '{"splitting_ops": ["vllm.unified_attention_with_output_kunlun", + "vllm.unified_attention", "vllm.unified_attention_with_output", + "vllm.mamba_mixer2"]}' \ +``` + +If you see a log as below: + +```bash +(APIServer pid=51171) INFO: Started server process [51171] +(APIServer pid=51171) INFO: Waiting for application startup. +(APIServer pid=51171) INFO: Application startup complete. +(Press CTRL+C to quit) +``` + +Congratulations, you have successfully started the vLLM server! + +You can query the model with input prompts: + +```bash +curl http://localhost:8356/v1/completions \ + -H "Content-Type: application/json" \ + -d '{ + "model": "Qwen3-8B", + "prompt": "What is your name?", + "max_tokens": 7, + "temperature": 0 + }' + +``` + +vLLM is serving as a background process, you can use `kill -2 $VLLM_PID` to stop the background process gracefully, which is similar to `Ctrl-C` for stopping the foreground vLLM process: + + + +```bash + VLLM_PID=$(pgrep -f "vllm serve") + kill -2 "$VLLM_PID" +``` + +The output is as below: + +``` +INFO: Shutting down FastAPI HTTP server. +INFO: Shutting down +INFO: Waiting for application shutdown. +INFO: Application shutdown complete. +``` + +Finally, you can exit the container by using `ctrl-D`. +:::: +::::: diff --git a/docs/source/tutorials/index.md b/docs/source/tutorials/index.md new file mode 100644 index 0000000..49a805c --- /dev/null +++ b/docs/source/tutorials/index.md @@ -0,0 +1,9 @@ +# Tutorials + +:::{toctree} +:caption: Deployment +:maxdepth: 1 +single_xpu_Qwen3-8B +multi_xpu_GLM-4.5 +multi_xpu_Qwen3-Coder-480B-A35B(W8A8) +::: diff --git a/docs/source/tutorials/multi_xpu_GLM-4.5.md b/docs/source/tutorials/multi_xpu_GLM-4.5.md new file mode 100644 index 0000000..da07b74 --- /dev/null +++ b/docs/source/tutorials/multi_xpu_GLM-4.5.md @@ -0,0 +1,153 @@ +# Multi XPU (GLM-4.5) + +## Run vllm-kunlun on multi XPU + +Setup environment using container: + +```bash +docker run -itd \ + --net=host \ + --cap-add=SYS_PTRACE --security-opt=seccomp=unconfined \ + --ulimit=memlock=-1 --ulimit=nofile=120000 --ulimit=stack=67108864 \ + --shm-size=128G \ + --privileged \ + --name=glm-vllm-01011 \ + -v ${PWD}:/data \ + -w /workspace \ + -v /usr/local/bin/:/usr/local/bin/ \ + -v /lib/x86_64-linux-gnu/libxpunvidia-ml.so.1:/lib/x86_64-linux-gnu/libxpunvidia-ml.so.1 \ + iregistry.baidu-int.com/hac_test/aiak-inference-llm:xpu_dev_20251113_221821 bash + +docker exec -it glm-vllm-01011 /bin/bash +``` + +### Offline Inference on multi XPU + +Start the server in a container: + +```{code-block} bash + :substitutions: +import os +from vllm import LLM, SamplingParams + +def main(): + + model_path = "/data/GLM-4.5" + + llm_params = { + "model": model_path, + "tensor_parallel_size": 8, + "trust_remote_code": True, + "dtype": "float16", + "enable_chunked_prefill": False, + "distributed_executor_backend": "mp", + } + + llm = LLM(**llm_params) + + messages = [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "Hello, who are you?" + } + ] + } + ] + + sampling_params = SamplingParams( + max_tokens=100, + temperature=0.7, + top_k=50, + top_p=1.0, + stop_token_ids=[181896] + ) + + outputs = llm.chat(messages, sampling_params=sampling_params) + + response = outputs[0].outputs[0].text + print("=" * 50) + print("Input content:", messages) + print("Model response:\n", response) + print("=" * 50) + +if __name__ == "__main__": + main() + +``` + +::::: + +If you run this script successfully, you can see the info shown below: + +```bash +================================================== +Input content: [{'role': 'user', 'content': [{'type': 'text', 'text': 'Hello, who are you?'}]}] +Model response: + +Well, the user asked a rather direct question about identity. This question seems simple, but there could be several underlying intentions—perhaps they are testing my reliability for the first time, or they simply want to confirm the identity of the conversational partner. From the common positioning of AI assistants, the user has provided a clear and flat way to define identity while leaving room for potential follow-up questions.\n\nThe user used "you" instead of "your", which leans towards a more informal tone, so the response style can be a bit more relaxed. However, since this is the initial response, it is better to maintain a moderate level of professionalism. Mentioning +================================================== +``` + +### Online Serving on Single XPU + +Start the vLLM server on a single XPU: + +```{code-block} bash +python -m vllm.entrypoints.openai.api_server \ + --host localhost \ + --port 8989 \ + --model /data/GLM-4.5 \ + --gpu-memory-utilization 0.95 \ + --trust-remote-code \ + --max-model-len 131072 \ + --tensor-parallel-size 8 \ + --dtype float16 \ + --max_num_seqs 128 \ + --max_num_batched_tokens 4096 \ + --max-seq-len-to-capture 4096 \ + --block-size 128 \ + --no-enable-prefix-caching \ + --no-enable-chunked-prefill \ + --distributed-executor-backend mp \ + --served-model-name GLM-4.5 \ + --compilation-config '{"splitting_ops": ["vllm.unified_attention_with_output_kunlun", "vllm.unified_attention", "vllm.unified_attention_with_output", "vllm.mamba_mixer2"]}' > log_glm_plugin.txt 2>&1 & +``` + +If your service start successfully, you can see the info shown below: + +```bash +(APIServer pid=51171) INFO: Started server process [51171] +(APIServer pid=51171) INFO: Waiting for application startup. +(APIServer pid=51171) INFO: Application startup complete. +``` + +Once your server is started, you can query the model with input prompts: + +```bash +curl http://localhost:8989/v1/chat/completions \ + -H "Content-Type: application/json" \ + -d '{ + "model": "GLM-4.5", + "messages": [ + {"role": "user", "content": "Hello, who are you?"} + ], + "max_tokens": 100, + "temperature": 0.7 + }' +``` + +If you query the server successfully, you can see the info shown below (client): + +```bash +{"id":"chatcmpl-6af7318de7394bc4ae569e6324a162fa","object":"chat.completion","created":1763101638,"model":"GLM-4.5","choices":[{"index":0,"message":{"role":"assistant","content":"\nThe user asked, \"Hello, who are you?\" This is a question about my identity. First, I need to confirm the user's intent. They might be using this service for the first time or have never interacted with similar AI assistants before, so they want to know my background and capabilities.\n\nNext, I should ensure my answer is clear and friendly, focusing on key points: who I am, who developed me, and what I can do. I should avoid technical jargon and keep the response conversational so it's easy to understand.\n\nAdditionally, the user may have potential needs, such as wanting to know what I am capable of.","refusal":null,"annotations":null,"audio":null,"function_call":null,"tool_calls":[],"reasoning_content":null},"logprobs":null,"finish_reason":"length","stop_reason":null}],"service_tier":null,"system_fingerprint":null,"usage":{"prompt_tokens":11,"total_tokens":111,"completion_tokens":100,"prompt_tokens_details":null},"prompt_logprobs":null,"kv_tr +``` + +Logs of the vllm server: + +```bash +(APIServer pid=54567) INFO: 127.0.0.1:60338 - "POST /v1/completions HTTP/1.1" 200 OK +(APIServer pid=54567) INFO 11-13 14:35:48 [loggers.py:123] Engine 000: Avg prompt throughput: 0.5 tokens/s, Avg generation throughput: 0.7 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 0.0% +``` diff --git a/docs/source/tutorials/multi_xpu_Qwen3-Coder-480B-A35B(W8A8).md b/docs/source/tutorials/multi_xpu_Qwen3-Coder-480B-A35B(W8A8).md new file mode 100644 index 0000000..1f15a0f --- /dev/null +++ b/docs/source/tutorials/multi_xpu_Qwen3-Coder-480B-A35B(W8A8).md @@ -0,0 +1,132 @@ +# Multi XPU (Qwen3-Coder-480B-A35B(W8A8)) + +## Run vllm-kunlun on Multi XPU + +Setup environment using container: + +```bash +# !/bin/bash +# rundocker.sh +XPU_NUM=8 +DOCKER_DEVICE_CONFIG="" +if [ $XPU_NUM -gt 0 ]; then + for idx in $(seq 0 $((XPU_NUM-1))); do + DOCKER_DEVICE_CONFIG="${DOCKER_DEVICE_CONFIG} --device=/dev/xpu${idx}:/dev/xpu${idx}" + done + DOCKER_DEVICE_CONFIG="${DOCKER_DEVICE_CONFIG} --device=/dev/xpuctrl:/dev/xpuctrl" +fi + +export build_image="xxxxxxxxxxxxxxxxx" + +docker run -itd ${DOCKER_DEVICE_CONFIG} \ + --net=host \ + --cap-add=SYS_PTRACE --security-opt seccomp=unconfined \ + --tmpfs /dev/shm:rw,nosuid,nodev,exec,size=32g \ + --cap-add=SYS_PTRACE \ + -v /home/users/vllm-kunlun:/home/vllm-kunlun \ + -v /usr/local/bin/xpu-smi:/usr/local/bin/xpu-smi \ + --name "$1" \ + -w /workspace \ + "$build_image" /bin/bash +``` + +### Preparation Weight + +* Pull Qwen3-Coder-480B-A35B-Instruct bf16 weights +* Modify the weights configuration.json file and add the fields quantization_config and compression_config. + +```json +{ + "architectures": [ + "Qwen3MoeForCausalLM" + ], + "attention_dropout": 0.0, + "decoder_sparse_step": 1, + "eos_token_id": 151645, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 6144, + "initializer_range": 0.02, + "intermediate_size": 8192, + "max_position_embeddings": 262144, + "max_window_layers": 62, + "mlp_only_layers": [], + "model_type": "qwen3_moe", + "moe_intermediate_size": 2560, + "norm_topk_prob": true, + "num_attention_heads": 96, + "num_experts": 160, + "num_experts_per_tok": 8, + "num_hidden_layers": 62, + "num_key_value_heads": 8, + "output_router_logits": false, + "qkv_bias": false, + "rms_norm_eps": 1e-06, + "rope_scaling": null, + "rope_theta": 10000000, + "router_aux_loss_coef": 0.0, + "shared_expert_intermediate_size": 0, + "sliding_window": null, + "tie_word_embeddings": false, + "torch_dtype": "bfloat16", + "transformers_version": "4.51.0", + "use_cache": true, + "use_qk_norm": true, + "use_sliding_window": false, + "vocab_size": 151936, + "quantization_config": { + "quant_method": "compressed-tensors" + }, + "compression_config": { + "format": "pack_quantized", + "config_groups": { + "linear_w8a8": { + "targets": ["Linear"], + "weights": { + "type": "int", + "num_bits": 8, + "strategy": "channel", + "group_size": null, + "symmetric": true, + "dynamic": false + }, + "input_activations": { + "type": "int", + "num_bits": 8, + "strategy": "token", + "group_size": null, + "symmetric": true, + "dynamic": true + } + } + }, + "ignore": [], + "sparsity_config": null + } +} + +``` + +### Online Serving on Multi XPU + +Start the vLLM server on multi XPU: + +```bash +python3 -m vllm.entrypoints.openai.api_server \ + --host 0.0.0.0 \ + --port 8898 \ + --model /Qwen/Qwen3-Coder-480B-A35B-Instruct \ + --dtype float16 \ + --trust-remote-code \ + --tensor-parallel-size 8 \ + --block-size 128 \ + --max-model-len 40960 \ + --max-num-seqs 512 \ + --max-num-batched-tokens 40960 \ + --max-seq-len-to-capture 40960 \ + --distributed-executor-backend mp \ + --enable-chunked-prefill=False \ + --no-enable-prefix-caching \ + --disable-log-requests \ + --gpu-memory-utilization 0.85 +``` \ No newline at end of file diff --git a/docs/source/tutorials/single_xpu_Qwen3-8B.md b/docs/source/tutorials/single_xpu_Qwen3-8B.md new file mode 100644 index 0000000..154bc08 --- /dev/null +++ b/docs/source/tutorials/single_xpu_Qwen3-8B.md @@ -0,0 +1,168 @@ +# Single XPU (Qwen3-8B) + +## Run vllm-kunlun on Single XPU + +Setup environment using container: + +```bash +# !/bin/bash +# rundocker.sh +XPU_NUM=8 +DOCKER_DEVICE_CONFIG="" +if [ $XPU_NUM -gt 0 ]; then + for idx in $(seq 0 $((XPU_NUM-1))); do + DOCKER_DEVICE_CONFIG="${DOCKER_DEVICE_CONFIG} --device=/dev/xpu${idx}:/dev/xpu${idx}" + done + DOCKER_DEVICE_CONFIG="${DOCKER_DEVICE_CONFIG} --device=/dev/xpuctrl:/dev/xpuctrl" +fi + +export build_image="xxxxxxxxxxxxxxxxx" + +docker run -itd ${DOCKER_DEVICE_CONFIG} \ + --net=host \ + --cap-add=SYS_PTRACE --security-opt seccomp=unconfined \ + --tmpfs /dev/shm:rw,nosuid,nodev,exec,size=32g \ + --cap-add=SYS_PTRACE \ + -v /home/users/vllm-kunlun:/home/vllm-kunlun \ + -v /usr/local/bin/xpu-smi:/usr/local/bin/xpu-smi \ + --name "$1" \ + -w /workspace \ + "$build_image" /bin/bash +``` + +### Offline Inference on Single XPU + +Start the server in a container: + +```{code-block} bash +from vllm import LLM, SamplingParams + +def main(): + + model_path = "/models/Qwen3-8B" + + llm_params = { + "model": model_path, + "tensor_parallel_size": 1, + "trust_remote_code": True, + "dtype": "float16", + "enable_chunked_prefill": False, + "distributed_executor_backend": "mp", + } + + llm = LLM(**llm_params) + + messages = [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "tell a joke" + } + ] + } + ] + + sampling_params = SamplingParams( + max_tokens=200, + temperature=1.0, + top_k=50, + top_p=1.0, + stop_token_ids=[181896] + ) + + outputs = llm.chat(messages, sampling_params=sampling_params) + + response = outputs[0].outputs[0].text + print("=" * 50) + print("Input content:", messages) + print("Model response:\n", response) + print("=" * 50) + +if __name__ == "__main__": + main() + +``` + +::::: + +If you run this script successfully, you can see the info shown below: + +```bash +================================================== +Input content: [{'role': 'user', 'content': [{'type': 'text', 'text': 'tell a joke'}]}] +Model response: + + +Okay, the user asked me to tell a joke. First, I need to consider the user's needs. They might just want to relax or need some entertainment. Next, I need to choose a suitable joke that is not too complicated, easy to understand, and also interesting. + + +The user might expect the joke to be in Chinese, so I need to ensure that the joke conforms to the language habits and cultural background of Chinese. I need to avoid sensitive topics, such as politics, religion, or anything that might cause misunderstanding. Then, I have to consider the structure of the joke, which usually involves a setup and an unexpected ending to create humor. + +For example, I could tell a light-hearted story about everyday life, such as animals or common scenarios. For instance, the story of a turtle and a rabbit racing, but with a twist. However, I need to ensure that the joke is of moderate length and not too long, so the user doesn't lose interest. Additionally, I should pay attention to using colloquial language and avoid stiff or complex sentence structures. + +I might also need to check if this joke is common to avoid repetition. If the user has heard something similar before, I may need to come up with a different angle. +================================================== +``` + +### Online Serving on Single XPU + +Start the vLLM server on a single XPU: + +```{code-block} bash +python -m vllm.entrypoints.openai.api_server \ + --host 0.0.0.0 \ + --port 9000 \ + --model /models/Qwen3-8B\ + --gpu-memory-utilization 0.9 \ + --trust-remote-code \ + --max-model-len 32768 \ + --tensor-parallel-size 1 \ + --dtype float16 \ + --max_num_seqs 128 \ + --max_num_batched_tokens 32768 \ + --max-seq-len-to-capture 32768 \ + --block-size 128 \ + --no-enable-prefix-caching \ + --no-enable-chunked-prefill \ + --distributed-executor-backend mp \ + --served-model-name Qwen3-8B \ + --compilation-config '{"splitting_ops": ["vllm.unified_attention_with_output_kunlun", + "vllm.unified_attention", "vllm.unified_attention_with_output", + "vllm.mamba_mixer2"]}' \ +``` + +If your service start successfully, you can see the info shown below: + +```bash +(APIServer pid=118459) INFO: Started server process [118459] +(APIServer pid=118459) INFO: Waiting for application startup. +(APIServer pid=118459) INFO: Application startup complete. +``` + +Once your server is started, you can query the model with input prompts: + +```bash +curl http://localhost:9000/v1/completions \ + -H "Content-Type: application/json" \ + -d '{ + "model": "Qwen3-8B", + "prompt": "What is your name?", + "max_tokens": 100, + "temperature": 0 + }' +``` + +If you query the server successfully, you can see the info shown below (client): + +```bash +{"id":"cmpl-80ee8b893dc64053947b0bea86352faa","object":"text_completion","created":1763015742,"model":"Qwen3-8B","choices":[{"index":0,"text":" is the S, and ,","logprobs":null,"finish_reason":"length","stop_reason":null,"prompt_logprobs":null}],"service_tier":null,"system_fingerprint":null,"usage":{"prompt_tokens":5,"total_tokens":12,"completion_tokens":7,"prompt_tokens_details":null},"kv_transfer_params":null} +``` + +Logs of the vllm server: + +```bash +(APIServer pid=54567) INFO: 127.0.0.1:60338 - "POST /v1/completions HTTP/1.1" 200 OK +(APIServer pid=54567) INFO 11-13 14:35:48 [loggers.py:123] Engine 000: Avg prompt throughput: 0.5 tokens/s, Avg generation throughput: 0.7 tokens/s, Running: 0 reqs, Waiting: 0 reqs, GPU KV cache usage: 0.0%, Prefix cache hit rate: 0.0% +``` diff --git a/docs/source/user_guide/configuration/env_vars.md b/docs/source/user_guide/configuration/env_vars.md new file mode 100644 index 0000000..b71c61d --- /dev/null +++ b/docs/source/user_guide/configuration/env_vars.md @@ -0,0 +1,17 @@ +# Environment Variables + +vllm-kunlun uses the following environment variables to configure the system: + +| *Environment Variables* | ***\*Recommended value\**** | ***\*Function description\**** | +| ---------------------------------------- | ----------------- | ------------------------------------------------------------ | +| `unset XPU_DUMMY_EVENT` | | ***\*Unsets\**** `XPU_DUMMY_EVENT` variable, usually done to ensure real XPU events are used for synchronization and performance measurement. | +| `export XPU_VISIBLE_DEVICES` | `0,1,2,3,4,5,6,7` | ***\*Specify visible XPU Devices\****. Here, 8 devices (0 to 7) are specified for inference tasks. This is required for multi-card or distributed inference. | +| `export XPU_USE_MOE_SORTED_THRES` | `1` | Enables the Moe Model ***\*Sort Optimization\****.Setting to `1` usually enables this performance optimization. | +| `export XFT_USE_FAST_SWIGLU` | `1` | Enables the ***\*Fast SwiGLU Ops\****. SwiGLU is a common activation function, and enabling this accelerates model inference. | +| `export XPU_USE_FAST_SWIGLU` | `1` | Enables the ***\*Fast SwiGLU Ops\****. Similar to `XFT_USE_FAST_SWIGLU`, this enables the fast SwiGLU calculation in Fused MoE Fusion Ops. | +| `export XMLIR_CUDNN_ENABLED` | `1` | Enables XMLIR (an intermediate representation/compiler) to use the ***\*cuDNN compatible/optimized path\**** (which may map to corresponding XPU optimized libraries in the KunlunCore environment). | +| `export XPU_USE_DEFAULT_CTX` | `1` | Sets the XPU to use the default context. Typically used to simplify environment configuration and ensure runtime consistency. | +| `export XMLIR_FORCE_USE_XPU_GRAPH` | `1` | ***\*Forces the enablement of XPU Graph mode.\****. This can capture and optimize the model execution graph, significantly boosting inference performance. | +| `export VLLM_HOST_IP` | `$(hostname -i)` | ***\*Sets the host IP address for the vLLM service\****. This uses a shell command to dynamically get the current host's internal IP. It's used for inter-node communication in a distributed environment. | +| `export XMLIR_ENABLE_MOCK_TORCH_COMPILE` | `false` | ***\*Disable Mock Torch Compile Function\****. Set to `false` to ensure the actual compilation and optimization flow is used, rather than mock mode. | +| `FUSED_QK_ROPE_OP` | `0` | ***\*Control whether to use the Fused QK-Norm and RoPE implementation\****. Default is `0` (use original/standard RoPE). Setting to `1` may be used to enable QWEN3. | \ No newline at end of file diff --git a/docs/source/user_guide/configuration/index.md b/docs/source/user_guide/configuration/index.md new file mode 100644 index 0000000..0c245c9 --- /dev/null +++ b/docs/source/user_guide/configuration/index.md @@ -0,0 +1,9 @@ +# Configuration Guide + +This section provides a detailed configuration guide of vLLM Kunlun. + +:::{toctree} +:caption: Configuration Guide +:maxdepth: 1 +env_vars +::: diff --git a/docs/source/user_guide/feature_guide/graph_mode.md b/docs/source/user_guide/feature_guide/graph_mode.md new file mode 100644 index 0000000..463d703 --- /dev/null +++ b/docs/source/user_guide/feature_guide/graph_mode.md @@ -0,0 +1,82 @@ +# Graph Mode Guide + +This guide provides instructions for using Kunlun Graph Mode with vLLM Kunlun. Please note that graph mode is available both on V1 and V0 Engine. All supported models are highly compatible with Kunlun Graph. + +## Getting Started + +From vLLM-KunLun-0.10.1.1 with V1 Engine, vLLM Kunlun will run models in graph mode by default to keep the same behavior with vLLM. If you hit any issues, please feel free to open an issue on GitHub and fallback to the eager mode temporarily by setting `enforce_eager=True` when initializing the model. + +There is a graph mode supported by vLLM Kunlun: + +- **KunlunGraph**: This is the default graph mode supported by vLLM Kunlun. In vLLM-KunLun-0.10.1.1, Qwen, GLM and InternVL series models are well tested. + + +## Using KunlunGraph + +KunlunGraph is enabled by default. Take Qwen series models as an example, just set to use V1 Engine(default) is enough. + +Offline example: + +```python +import os + +from vllm import LLM + +model = LLM(model="models/Qwen3-8B-Instruct") +outputs = model.generate("Hello, how are you?") +``` + +Online example: + +```shell +vllm serve Qwen3-8B-Instruct +``` + +## Using KunlunGraph + +Enabling Kunlun Graph on the Kunlun platform requires the use of splitting ops. + +Online example: + +```shell +python -m vllm.entrypoints.openai.api_server \ + --host 0.0.0.0 \ + --port 8000 \ + --model /models/Qwen3-8B-Instruct\ + --gpu-memory-utilization 0.9 \ + --trust-remote-code \ + --max-model-len 32768 \ + --tensor-parallel-size 1 \ + --dtype float16 \ + --no-enable-prefix-caching \ + --no-enable-chunked-prefill \ + --distributed-executor-backend mp \ + --served-model-name Qwen3-8B-Instruct \ + --compilation-config '{"splitting_ops": ["vllm.unified_attention_with_output_kunlun", + "vllm.unified_attention", "vllm.unified_attention_with_output", + "vllm.mamba_mixer2"]}' \ +``` + + +## Fallback to the Eager Mode + +If `KunlunGraph` fail to run, you should fallback to the eager mode. + +Online example: + +```shell +python -m vllm.entrypoints.openai.api_server \ + --host 0.0.0.0 \ + --port 8000 \ + --model /models/Qwen3-8B-Instruct\ + --gpu-memory-utilization 0.9 \ + --trust-remote-code \ + --max-model-len 32768 \ + --tensor-parallel-size 1 \ + --dtype float16 \ + --no-enable-prefix-caching \ + --no-enable-chunked-prefill \ + --distributed-executor-backend mp \ + --served-model-name Qwen3-8B-Instruct \ + --enforce_eager +``` \ No newline at end of file diff --git a/docs/source/user_guide/feature_guide/index.md b/docs/source/user_guide/feature_guide/index.md new file mode 100644 index 0000000..5141a9e --- /dev/null +++ b/docs/source/user_guide/feature_guide/index.md @@ -0,0 +1,11 @@ +# Feature Guide + +This section provides a detailed usage guide of vLLM Kunlun features. + +:::{toctree} +:caption: Feature Guide +:maxdepth: 1 +graph_mode +quantization +lora +::: diff --git a/docs/source/user_guide/feature_guide/lora.md b/docs/source/user_guide/feature_guide/lora.md new file mode 100644 index 0000000..6f2688a --- /dev/null +++ b/docs/source/user_guide/feature_guide/lora.md @@ -0,0 +1,27 @@ +# LoRA Adapters Guide + +## Overview + +Like vLLM, vllm_kunlun supports LoRA as well. The usage and more details can be found in [vLLM official document ](https://docs.vllm.ai/en/latest/features/lora.html). + +You can refer to [Supported Models ](https://docs.vllm.ai/en/latest/models/supported_models.html#list-of-text-only-language-models)to find which models support LoRA in vLLM. + +Currently, only vLLM v0 mode (including eager and CUDA Graph modes) supports multi-LoRA inference in vllm_kunlun. + +## Example + +We provide a simple LoRA example here: + +```bash +export ENABLE_KUNLUN_LARGE_OPS=0 + +USE_ORI_ROPE=0 VLLM_USE_V1=0 vllm serve qwen3-8b \ + --enable-lora \ + --max-lora-rank 64 \ + --lora-modules lora1=/path/to/lora1 lora2=/path/to/lora2 +``` + + +## Custom LoRA Operators + +We have implemented LoRA-related custom operators for Kunlun hardware, such as `bgmv_shrink`, `bgmv_expand`, `sgmv_shrink`, and `sgmv_expand`. The implementation can be found in `vllm_kunlun/lora/ops/kunlun_ops/lora_ops.py`. \ No newline at end of file diff --git a/docs/source/user_guide/feature_guide/quantization.md b/docs/source/user_guide/feature_guide/quantization.md new file mode 100644 index 0000000..9851cd3 --- /dev/null +++ b/docs/source/user_guide/feature_guide/quantization.md @@ -0,0 +1,45 @@ +# Quantization Guide +>Note: This feature is currently experimental. In future versions, there may be behavioral changes around configuration, coverage, performance improvement. + +Like vLLM, we now support quantization methods such as compressed-tensors, AWQ, and GPTQ, enabling various precision configurations including W8A8, W4A16, and W8A16. These can help reduce memory consumption and accelerate inference while preserving model accuracy. + + +## Usages + +### Compressed-tensor +To run a `compressed-tensors` model with vLLM-kunlun, you should first add the below configuration to the model's `config.json`: + +```Bash +"quantization_config": { + "quant_method": "compressed-tensors" + } +``` + +Then you run `Qwen/Qwen3-30B-A3B` with dynamic W8A8 quantization with the following command: + +```Bash +python -m vllm.entrypoints.openai.api_server \ + --model Qwen/Qwen3-30B-A3B \ + --quantization compressed-tensors +``` + +### AWQ + +To run an `AWQ` model with vLLM-kunlun, you can use `Qwen/Qwen3-32B-AWQ` with the following command: + +```Bash +python -m vllm.entrypoints.openai.api_server \ + --model Qwen/Qwen3-32B-AWQ \ + --quantization awq +``` + +### GPTQ + +To run a `GPTQ` model with vLLM-kunlun, you can use `Qwen/Qwen2.5-7B-Instruct-GPTQ-Int4` with the following command: + +```Bash +python -m vllm.entrypoints.openai.api_server \ + --model Qwen/Qwen2.5-7B-Instruct-GPTQ-Int4 \ + --quantization gptq +``` + diff --git a/docs/source/user_guide/release_notes.md b/docs/source/user_guide/release_notes.md new file mode 100644 index 0000000..d7f7d51 --- /dev/null +++ b/docs/source/user_guide/release_notes.md @@ -0,0 +1,3 @@ +# Release Notes + +Comming soon... \ No newline at end of file diff --git a/docs/source/user_guide/support_matrix/index.md b/docs/source/user_guide/support_matrix/index.md new file mode 100644 index 0000000..ae5ac4c --- /dev/null +++ b/docs/source/user_guide/support_matrix/index.md @@ -0,0 +1,10 @@ +# Features and Models + +This section provides a detailed matrix supported by vLLM-Kunlun. + +:::{toctree} +:caption: Support Matrix +:maxdepth: 1 +supported_models +supported_features +::: diff --git a/docs/source/user_guide/support_matrix/supported_features.md b/docs/source/user_guide/support_matrix/supported_features.md new file mode 100644 index 0000000..a32e700 --- /dev/null +++ b/docs/source/user_guide/support_matrix/supported_features.md @@ -0,0 +1,14 @@ +# Supported Features + +The feature support principle of vLLM-KunLun is: **aligned with the vLLM**. We are also actively collaborating with the community to accelerate support. + +You can check the [support status of vLLM V1 Engine][v1_user_guide]. Below is the feature support status of vLLM-KunLun: + +## Features Supported +|Feature|Status|Note| +|-|-|-| +|Tensor Parallel|🟢 Functional|| +|Experts Parallel|🟢 Functional|| +|Graph Mode|🟢 Functional|| +|Quantization| 🟢 Functional|| +|LoRA|⚠️ Need Test|Only LLM models| \ No newline at end of file diff --git a/docs/source/user_guide/support_matrix/supported_models.md b/docs/source/user_guide/support_matrix/supported_models.md new file mode 100644 index 0000000..f3a6141 --- /dev/null +++ b/docs/source/user_guide/support_matrix/supported_models.md @@ -0,0 +1,33 @@ +# Supported Models + +## Generative Models + +| Model | Support | W8A8 | LoRA | Tensor Parallel | Expert Parallel | Data Parallel | Piecewise Kunlun Graph | +| :------------ | :------------ | :--- | :--- | :-------------- | :-------------- | :------------ | :--------------------- | +| Qwen2 | ✅ | | ✅ | ✅ | | ✅ | ✅ | +| Qwen2.5 | ✅ | | ✅ | ✅ | | ✅ | ✅ | +| Qwen3 | ✅ | | ✅ | ✅ | | ✅ | ✅ | +| Qwen3-Moe | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | +| Qwen3-Coder | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | +| QwQ-32B | ✅ | | | ✅ | | ✅ | ✅ | +| LLama2 | ✅ | | | ✅ | | ✅ | ✅ | +| LLama3 | ✅ | | | ✅ | | ✅ | ✅ | +| LLama3.1 | ✅ | | | ✅ | | ✅ | ✅ | +| GLM-4.5 | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | +| GLM-4.5-Air | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | +| Qwen3-next | 🔜Comming soon | | | | | | | +| gpt-oss | 🔜Comming soon | | | | | | | +| DeepSeek-V3 | 🔜Comming soon | | | | | | | +| DeepSeek-V3.2 | 🔜Comming soon | | | | | | | + +## Multimodal Language Models +| Model | Support | W8A8 | LoRA | Tensor Parallel | Expert Parallel | Data Parallel | Piecewise Kunlun Graph | +| :----------- | :------------ | :--- | :--- | :-------------- | :-------------- | :------------ | :--------------------- | +|Qianfan-VL | ✅ | | | ✅| |✅ |✅| +| Qwen2.5VL | ✅ | | | ✅ | | ✅ | ✅ | +| InternVL2.5 | ✅ | | | ✅ | | ✅ | ✅ | +| InternVL3 | ✅ | | | ✅ | | ✅ | ✅ | +| InternVL3.5 | ✅ | | | ✅ | | ✅ | ✅ | +| InternS1 | ✅ | | | ✅ | | ✅ | ✅ | +| Qwen2.5-Omni | 🔜Comming soon | | | | | | | +| Qwen3-VL | 🔜Comming soon | | | | | | | \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..790842d --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,30 @@ +[build-system] +requires = ["hatchling>=1.22"] +build-backend = "hatchling.build" + +[project] +name = "vllm-kunlun" +version = "0.10.1.1" +description = "vLLM Kunlun3 backend plugin" +readme = "README.md" +requires-python = ">=3.10" +license = { text = "MIT" } +authors = [{ name = "kunlun"}] +dependencies = [] + +[project.scripts] +vllm-kunlun = "vllm_kunlun.cmdline:main" + +[project.entry-points."vllm.platform_plugins"] +kunlun = "vllm_kunlun:register" + +[project.entry-points."vllm.general_plugins"] +kunlun_model = "vllm_kunlun:register_model" + +[tool.hatch.build] +packages = ["vllm_kunlun"] +include = ["vllm_kunlun/conf/*", "vllm_kunlun/data/*"] + +[tool.hatch.build.targets.wheel] +packages = ["vllm_kunlun"] +output-dir = "output/dist" \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..4c951bd --- /dev/null +++ b/requirements.txt @@ -0,0 +1,34 @@ +setuptools==80.9.0 +black==23.3.0 +blake3==1.0.5 +cachetools==6.1.0 +cbor2==5.7.0 +cloudpickle==3.1.1 +compressed-tensors==0.10.2 +diskcache==5.6.3 +gguf==0.17.1 +mistral_common==1.8.3 +msgspec==0.19.0 +numba==0.61.2 +openai==1.99.1 +openai-harmony==0.0.4 +opencv-contrib-python==4.12.0.88 +partial-json-parser==0.2.1.1.post6 +prometheus_client==0.22.1 +pybase64==1.4.1 +pyzmq==27.0.1 +ray==2.48.0 +setproctitle==1.3.7 +watchfiles==1.1.0 +pydantic==2.11.7 +tokenizers>=0.21.2 +uvloop==0.21.0 +prometheus-fastapi-instrumentator==7.1.0 +transformers>=4.56.1 + +hatchling>=1.25 +build>=1.0.3 +pytest +mock + + diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..1584f7c --- /dev/null +++ b/setup.py @@ -0,0 +1,66 @@ +# +# setup.py for vllm_kunlun +# + +import os +import shutil +from setuptools import find_packages, setup +from torch.utils.cpp_extension import CppExtension, BuildExtension + +ROOT_DIR = os.path.dirname(__file__) + +ext_modules = [ + CppExtension( + name='vllm_kunlun._kunlun', + sources=['vllm_kunlun/csrc/utils.cpp'], + include_dirs=[ + 'vllm_kunlun/csrc', + "/usr/local/cuda/include", + ], + library_dirs=["/usr/local/cuda/lib64"], + extra_compile_args=['-O3'], + ) +] + +class CustomBuildExt(BuildExtension): + def run(self): + super().run() + for ext in self.extensions: + ext_path = self.get_ext_fullpath(ext.name) + file_name = os.path.basename(ext_path) + target_path = os.path.join("vllm_kunlun", file_name) + + if os.path.exists(target_path): + os.remove(target_path) + shutil.copyfile(ext_path, target_path) + print(f"[BuildExt] Copied {ext_path} -> {target_path}") + + +if __name__ == '__main__': + + setup( + name='vllm_kunlun', + version="v1.0", + author="vLLM-Kunlun team", + license="Apache 2.0", + description="vLLM Kunlun3 backend plugin", + packages=find_packages(exclude=("docs", "examples", "tests*")), + package_data={ + 'vllm_kunlun': ['_kunlun.so', 'so/*.so', 'include/*.h'] + }, + python_requires=">=3.10", + ext_modules=ext_modules, + cmdclass={ + 'build_ext': CustomBuildExt, + }, + entry_points={ + 'vllm.platform_plugins': ["kunlun = vllm_kunlun:register"], + 'vllm.general_plugins': [ + "kunlun_model = vllm_kunlun:register_model", + "kunlun_quant = vllm_kunlun:register_quant_method" + ], + "console_scripts": [ + "vllm_kunlun = vllm_kunlun.entrypoints.main:main" + ] + } + ) diff --git a/setup_env.sh b/setup_env.sh new file mode 100644 index 0000000..235b227 --- /dev/null +++ b/setup_env.sh @@ -0,0 +1,11 @@ +unset XPU_DUMMY_EVENT +export XPU_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 +export XPU_USE_MOE_SORTED_THRES=1 +export XFT_USE_FAST_SWIGLU=1 +export XMLIR_CUDNN_ENABLED=1 +export XPU_USE_DEFAULT_CTX=1 +export XMLIR_FORCE_USE_XPU_GRAPH=1 +export XPU_USE_FAST_SWIGLU=1 +export VLLM_HOST_IP=$(hostname -i) +export XMLIR_ENABLE_MOCK_TORCH_COMPILE=false +export FUSED_QK_ROPE_OP=0 \ No newline at end of file diff --git a/vllm_kunlun/__init__.py b/vllm_kunlun/__init__.py new file mode 100644 index 0000000..a255d20 --- /dev/null +++ b/vllm_kunlun/__init__.py @@ -0,0 +1,180 @@ +# +# Copyright (c) 2025 Baidu, Inc. All Rights Reserved. +# Author: Xinyu Dong +# Email: dongxinyu03@baidu.com +# This file is a part of the vllm-kunlun project. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""vllm kunlun init""" +from .platforms import current_platform +import sys +import importlib +import warnings +import builtins +import os +import time +import vllm.envs as envs + +OLD_IMPORT_HOOK = builtins.__import__ + + +def _custom_import(module_name, globals=None, locals=None, fromlist=(), level=0): + try: + start_time = time.time() + + # Module mapping table + module_mappings = { + "vllm.model_executor.layers.fused_moe.layer": "vllm_kunlun.ops.fused_moe.layer", + "vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors_moe": "vllm_kunlun.ops.quantization.compressed_tensors_moe", + "vllm.compilation.wrapper": "vllm_kunlun.compilation.wrapper", + } + + # Keep the original imported modules + original_imports = [ + "vllm.model_executor.layers.fused_moe.base", + "vllm.model_executor.layers.fused_moe.config", + "vllm.model_executor.layers.fused_moe.layer", + ] + + if module_name in original_imports: + if module_name == "vllm.model_executor.layers.fused_moe.layer" and fromlist: + if "FusedMoEMethodBase" in fromlist: + return OLD_IMPORT_HOOK( + module_name, + globals=globals, + locals=locals, + fromlist=fromlist, + level=level, + ) + + if module_name in module_mappings: + if module_name in sys.modules: + return sys.modules[module_name] + target_module = module_mappings[module_name] + module = importlib.import_module(target_module) + sys.modules[module_name] = module + sys.modules[target_module] = module + return module + + relative_mappings = { + ( + "compressed_tensors_moe", + "compressed_tensors", + ): "vllm_kunlun.ops.quantization.compressed_tensors_moe", + ("layer", "fused_moe"): "vllm_kunlun.ops.fused_moe.layer", + } + + if level == 1: + parent = globals.get("__package__", "").split(".")[-1] if globals else "" + key = (module_name, parent) + if key in relative_mappings: + if module_name in sys.modules: + return sys.modules[module_name] + target_module = relative_mappings[key] + module = importlib.import_module(target_module) + sys.modules[module_name] = module + sys.modules[target_module] = module + return module + + except Exception: + pass + + return OLD_IMPORT_HOOK( + module_name, globals=globals, locals=locals, fromlist=fromlist, level=level + ) + + +def import_hook(): + """Apply import hook for VLLM Kunlun""" + if not int(os.environ.get("DISABLE_KUNLUN_HOOK", "0")): + builtins.__import__ = _custom_import + + try: + modules_to_preload = [ + "vllm_kunlun.ops.quantization.compressed_tensors_moe", + "vllm_kunlun.ops.fused_moe.custom_ops", + "vllm_kunlun.ops.fused_moe.layer", + "vllm_kunlun.ops.quantization.fp8", + ] + for module_name in modules_to_preload: + importlib.import_module(module_name) + except Exception: + pass + + +def register(): + """Register the Kunlun platform""" + from .utils import redirect_output + from .vllm_utils_wrapper import ( + direct_register_custom_op, + patch_annotations_for_schema, + ) + + import_hook() + if envs.VLLM_USE_V1: + patch_V1blockTable() + patch_V1top_p_K() + patch_V1penalties() + else: + patch_sampler() + return "vllm_kunlun.platforms.kunlun.KunlunPlatform" + + +def register_model(): + """Register models for training and inference""" + from .models import register_model as _reg + + _reg() + + +def patch_sampler(): + try: + custom_sampler = importlib.import_module("vllm_kunlun.ops.sample.sampler") + sys.modules["vllm.model_executor.layers.sampler"] = custom_sampler + print("[vllm_kunlun] sampler patched ->", custom_sampler.__file__) + except Exception as e: + warnings.warn(f"[vllm_kunlun] sampler patch failed: {e!r}") + + +def patch_V1top_p_K(): + try: + custom_sampler = importlib.import_module( + "vllm_kunlun.v1.sample.ops.topk_topp_sampler" + ) + sys.modules["vllm.v1.sample.ops.topk_topp_sampler"] = custom_sampler + print("[vllm_kunlun] V1sampler top p & k patched ->", custom_sampler.__file__) + except Exception as e: + warnings.warn(f"[vllm_kunlun] V1 sampler top p & k patch failed: {e!r}") + + +def patch_V1penalties(): + try: + custom_sampler = importlib.import_module("vllm_kunlun.v1.sample.ops.penalties") + sys.modules["vllm.v1.sample.ops.penalties"] = custom_sampler + print("[vllm_kunlun] V1sampler penalties patched ->", custom_sampler.__file__) + except Exception as e: + warnings.warn(f"[vllm_kunlun] V1 sampler penalties patch failed: {e!r}") + + +def patch_V1blockTable(): + try: + custom_sampler = importlib.import_module("vllm_kunlun.v1.worker.block_table") + sys.modules["vllm.v1.worker.block_table"] = custom_sampler + print("[vllm_kunlun] V1 block table patched ->", custom_sampler.__file__) + except Exception as e: + warnings.warn(f"[vllm_kunlun] V1 block table patch failed: {e!r}") + + +# Automatically apply patches when modules are imported +import_hook() diff --git a/vllm_kunlun/compilation/__init__.py b/vllm_kunlun/compilation/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/vllm_kunlun/compilation/wrapper.py b/vllm_kunlun/compilation/wrapper.py new file mode 100644 index 0000000..2de4fa7 --- /dev/null +++ b/vllm_kunlun/compilation/wrapper.py @@ -0,0 +1,148 @@ +# +# Copyright (c) 2025 Baidu, Inc. All Rights Reserved. +# Author: Bao Qian +# Email: baoqian@baidu.com +# This file is a part of the vllm-kunlun project. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os +import sys +from abc import abstractmethod +from contextlib import contextmanager +from types import CodeType +from typing import Callable, Optional + +import torch + +import vllm.envs as envs +from vllm.logger import init_logger + +logger = init_logger(__name__) + + +class TorchCompileWrapperWithCustomDispatcher: + """ + A wrapper class for torch.compile, with a custom dispatch logic. + Subclasses should: + 1. Implement the forward method + 2. Implement the dispatch logic in the __call__ method + It can use `self.compiled_codes` to access the compiled bytecode, + and `with self.dispatch_to_code(index):` to dispatch to + the compiled code. + 3. Implement the `__init__` method to determine how to call + `torch.compile` over the forward method. + """ + + def __init__(self, + compiled_callable: Optional[Callable] = None, + compilation_level: int = 0): + from vllm.config import get_current_vllm_config + vllm_config = get_current_vllm_config() + self.vllm_config = vllm_config + if compiled_callable is None: + # default compilation settings + # compiling the forward method + + backend = vllm_config.compilation_config.init_backend(vllm_config) + options = None + if isinstance(backend, str) and backend == "inductor": + options = get_current_vllm_config( + ).compilation_config.inductor_compile_config + + compiled_callable = torch.compile( + self.forward, + fullgraph=envs.VLLM_TEST_DYNAMO_FULLGRAPH_CAPTURE, + backend=backend, + options=options) + + self.compiled_callable = compiled_callable + self.original_code_object = self.__class__.forward.__code__ + self.compiled_codes: list[CodeType] = [] + torch._dynamo.convert_frame.register_bytecode_hook(self.bytecode_hook) + + # read the env var to determine whether to use the custom dispatcher + # subclasses can use this to switch between the custom dispatcher + # and the default Dynamo guard mechanism. + from vllm.config import CompilationLevel + self.use_custom_dispatcher: bool = \ + compilation_level >= CompilationLevel.DYNAMO_ONCE + + def __call__(self, *args, **kwargs): + """Implement the dispatch logic here, beyond the torch.compile level. + NOTE: this function can have additional arguments beyond the forward + method, for directly dispatching to the compiled code. + """ + return self.compiled_callable(*args, **kwargs) + + @abstractmethod + def forward(self, *args, **kwargs): + ... + + def bytecode_hook(self, old_code: CodeType, new_code: CodeType): + """Hook to save the compiled bytecode for direct execution.""" + if old_code is not self.original_code_object: + return + # code borrowed from https://github.com/thuml/depyf/blob/f4ad79fadee27ea113b4c75202db1eb1a11c0dbc/depyf/explain/enable_debugging.py#L25 + frame = sys._getframe() + while frame and frame.f_back: + frame = frame.f_back + code_name = frame.f_code.co_name + file_name = frame.f_code.co_filename.split(os.path.sep)[-1] + if code_name == "_compile" and file_name == "convert_frame.py": + break + frame = frame.f_locals["frame"] + assert frame.f_code == old_code + + if frame.f_locals["self"] is not self: + return + + self.compiled_codes.append(new_code) + local_cache_dir = self.vllm_config.compilation_config.local_cache_dir + if isinstance(local_cache_dir, str): + decompiled_file = os.path.join(local_cache_dir, + "transformed_code.py") + if not os.path.exists(decompiled_file): + try: + # usually the decompilation will succeed for most models, + # as we guarantee a full-graph compilation in Dynamo. + # but there's no 100% guarantee, since decompliation is + # not a reversible process. + import depyf + src = depyf.decompile(new_code) + with open(decompiled_file, "w") as f: + f.write(src) + + logger.debug("Dynamo transformed code saved to %s", + decompiled_file) + except Exception: + pass + # if self.vllm_config.compilation_config.use_cudagraph and \ + # "update" in new_code.co_names: + # import depyf + # src = depyf.decompile(new_code) + # msg = "Assigning / modifying buffers of nn.Module during forward pass is not allowed when using cudagraph inside the compiler because it will cause silent errors. Please use eager mode or fix the code. The following code contains clues about which buffer is being modified (please search for the usage of the function `update`):\n" + src # noqa + # raise RuntimeError(msg) + + @contextmanager + def dispatch_to_code(self, index: int): + """Context manager to dispatch to the compiled code. + Why does this work? Because Dynamo guarantees that the compiled + bytecode has exactly the same arguments, cell variables, and free + variables as the original code. Therefore we can directly switch + the code object in the function and call it. + + See https://dev-discuss.pytorch.org/t/what-is-the-relationship-requirement-among-original-bytecode-transformed-bytecode-and-bytecode-returned-by-hooks-in-dynamo/1693/7 for more details. + """ # noqa + self.__class__.forward.__code__ = self.compiled_codes[index] + yield + self.__class__.forward.__code__ = self.original_code_object \ No newline at end of file diff --git a/vllm_kunlun/csrc/dispatch_utils.h b/vllm_kunlun/csrc/dispatch_utils.h new file mode 100644 index 0000000..03414b7 --- /dev/null +++ b/vllm_kunlun/csrc/dispatch_utils.h @@ -0,0 +1,49 @@ +/* + * Adapted from + * https://github.com/pytorch/pytorch/blob/v2.0.1/aten/src/ATen/Dispatch.h + */ +#pragma once + +#include + +#define VLLM_DISPATCH_CASE_FLOATING_TYPES(...) \ + AT_DISPATCH_CASE(at::ScalarType::Float, __VA_ARGS__) \ + AT_DISPATCH_CASE(at::ScalarType::Half, __VA_ARGS__) \ + AT_DISPATCH_CASE(at::ScalarType::BFloat16, __VA_ARGS__) + +#define VLLM_DISPATCH_FLOATING_TYPES(TYPE, NAME, ...) \ + AT_DISPATCH_SWITCH(TYPE, NAME, VLLM_DISPATCH_CASE_FLOATING_TYPES(__VA_ARGS__)) + +// TODO(luka/varun): use FP8_TYPE macro after refactoring +#ifndef USE_ROCM + #define VLLM_DISPATCH_CASE_QUANT_TYPES(...) \ + AT_DISPATCH_CASE(at::ScalarType::Float8_e4m3fn, __VA_ARGS__) \ + AT_DISPATCH_CASE(at::ScalarType::Char, __VA_ARGS__) +#else + #define VLLM_DISPATCH_CASE_QUANT_TYPES(...) \ + AT_DISPATCH_CASE(at::ScalarType::Float8_e4m3fnuz, __VA_ARGS__) \ + AT_DISPATCH_CASE(at::ScalarType::Char, __VA_ARGS__) +#endif + +#define VLLM_DISPATCH_QUANT_TYPES(TYPE, NAME, ...) \ + AT_DISPATCH_SWITCH(TYPE, NAME, VLLM_DISPATCH_CASE_QUANT_TYPES(__VA_ARGS__)) + +#define VLLM_DISPATCH_CASE_FLOATING_AND_BYTE_TYPES(...) \ + AT_DISPATCH_CASE(at::ScalarType::Float, __VA_ARGS__) \ + AT_DISPATCH_CASE(at::ScalarType::Half, __VA_ARGS__) \ + AT_DISPATCH_CASE(at::ScalarType::BFloat16, __VA_ARGS__) \ + AT_DISPATCH_CASE(at::ScalarType::Byte, __VA_ARGS__) + +#define VLLM_DISPATCH_FLOATING_AND_BYTE_TYPES(TYPE, NAME, ...) \ + AT_DISPATCH_SWITCH(TYPE, NAME, \ + VLLM_DISPATCH_CASE_FLOATING_AND_BYTE_TYPES(__VA_ARGS__)) + +#define VLLM_DISPATCH_CASE_INTEGRAL_TYPES(...) \ + AT_DISPATCH_CASE(at::ScalarType::Byte, __VA_ARGS__) \ + AT_DISPATCH_CASE(at::ScalarType::Char, __VA_ARGS__) \ + AT_DISPATCH_CASE(at::ScalarType::Short, __VA_ARGS__) \ + AT_DISPATCH_CASE(at::ScalarType::Int, __VA_ARGS__) \ + AT_DISPATCH_CASE(at::ScalarType::Long, __VA_ARGS__) + +#define VLLM_DISPATCH_INTEGRAL_TYPES(TYPE, NAME, ...) \ + AT_DISPATCH_SWITCH(TYPE, NAME, VLLM_DISPATCH_CASE_INTEGRAL_TYPES(__VA_ARGS__)) diff --git a/vllm_kunlun/csrc/utils.cpp b/vllm_kunlun/csrc/utils.cpp new file mode 100644 index 0000000..8e12d2c --- /dev/null +++ b/vllm_kunlun/csrc/utils.cpp @@ -0,0 +1,32 @@ +#include "xops.h" +#include "dispatch_utils.h" +#include +torch::Tensor weak_ref_tensor(torch::Tensor& tensor) { + // Ensure tensor is on CUDA + if (!tensor.is_cuda()) { + throw std::runtime_error("Tensor must be on CUDA device"); + } + + // Get the raw data pointer + void* data_ptr = tensor.data_ptr(); + + // Get tensor sizes and strides + std::vector sizes = tensor.sizes().vec(); + std::vector strides = tensor.strides().vec(); + + // Get tensor options (dtype, device) + auto options = tensor.options(); + + // Create a new tensor from the raw data pointer + auto new_tensor = torch::from_blob(data_ptr, sizes, strides, options); + + return new_tensor; +} + +TORCH_LIBRARY(_kunlun, m) { + m.def("weak_ref_tensor", &weak_ref_tensor); +} + +PYBIND11_MODULE(_kunlun, m) { + m.def("weak_ref_tensor", &weak_ref_tensor); +} \ No newline at end of file diff --git a/vllm_kunlun/csrc/xops.h b/vllm_kunlun/csrc/xops.h new file mode 100644 index 0000000..a4c49b6 --- /dev/null +++ b/vllm_kunlun/csrc/xops.h @@ -0,0 +1,241 @@ +#ifndef OPS_H +#define OPS_H +#include +#include +void rms_norm_xpu(torch::Tensor &output, + torch::Tensor &input, + torch::Tensor &weight, + double eps); +// inplace +void fused_add_rms_norm_xpu(torch::Tensor& input, // [..., hidden_size] + torch::Tensor& residual, // [..., hidden_size] + torch::Tensor& weight, // [hidden_size] + double epsilon); + +void silu_and_mul_xpu(torch::Tensor &output, + torch::Tensor &input); + + +void quick_gelu_xpu(torch::Tensor &output, + torch::Tensor &input); + +// neox && gptj +void rotary_embedding(torch::Tensor &positions, + torch::Tensor& query, + torch::Tensor& key, + int64_t head_size, + torch::Tensor& cos_sin_cache, + bool is_neox); + +void batched_rotary_embedding(torch::Tensor &positions, + torch::Tensor& query, + torch::Tensor& key, + int64_t head_size, + torch::Tensor& cos_sin_cache, + bool is_neox, + int64_t rot_dim, + torch::Tensor& offsets); + +// x = 16 // sizeof(cache dtype) +void paged_attention_v1_xpu( + torch::Tensor& out, // [num_seqs, num_heads, head_size] + torch::Tensor& query, // [num_seqs, num_heads, head_size] + torch::Tensor& key_cache, // [num_blocks, num_kv_heads, block_size, head_size] + torch::Tensor& value_cache, // [num_blocks, num_kv_heads, block_size, head_size] + int64_t num_kv_heads, + double scale, + torch::Tensor& block_tables, // [num_seqs, max_num_blocks_per_seq] + torch::Tensor& seq_lens, // [num_seqs] + torch::Tensor& seq_lens_host, // [num_seqs] + int64_t block_size, + int64_t max_seq_len, + const c10::optional& alibi_slopes, // [num_heads] + const std::string& kv_cache_dtype, + double k_scale, + double v_scale, + int64_t tp_rank, int64_t blocksparse_local_blocks, // no used but to keep same with vllm-offficial + int64_t blocksparse_vert_stride, int64_t blocksparse_block_size, // no used but to keep same with vllm-offficial + int64_t blocksparse_head_sliding_step // no used but to keep same with vllm-offficial + ); + +void reshape_and_cache( + torch::Tensor& key, // [num_tokens, num_heads, head_size] + torch::Tensor& value, // [num_tokens, num_heads, head_size] + torch::Tensor& + key_cache, // [num_blocks, num_heads, head_size/x, block_size, x] + torch::Tensor& + value_cache, // [num_blocks, num_heads, head_size, block_size] + torch::Tensor& slot_mapping, // [num_tokens] + const std::string& kv_cache_dtype, + const double k_scale, + const double v_scale); + +void flash_attention_context_vllm_xpu( + torch::Tensor& query, // [num_tokens, num_heads, head_size] + torch::Tensor& key, // [num_tokens, num_kv_heads, head_size] + torch::Tensor& value, // [num_tokens, num_kv_heads, head_size] + torch::Tensor& out, // [num_tokens, num_heads, head_size] + torch::Tensor& seq_lod, // [batch_size + 1] + torch::Tensor& seq_lod_host, // [batch_size + 1] + int64_t max_seq_len, + int64_t max_kv_len, + double scale, + const c10::optional& alibi_slopes, // [num_heads], + const c10::optional& key_cache, // [num_blocks, num_kv_heads, block_size, head_size] + const c10::optional& value_cache, // [num_blocks, num_kv_heads, block_size, head_size] + const c10::optional& block_tables, // [num_seqs, max_num_blocks_per_seq] + const c10::optional& kv_prefix_start_loc, // [lod of prefix] + const c10::optional& kv_prefix_start_loc_host, // [lod of prefix] + const c10::optional is_causal // use causal mask or not, default true +); + +void paged_attention_v2_xpu( + torch::Tensor &out, + torch::Tensor &exp_sums, + torch::Tensor &max_logits, + torch::Tensor &tmp_out, + torch::Tensor &query, // [num_seqs, num_heads, head_size] + torch::Tensor & + key_cache, // [num_blocks, num_kv_heads, block_size, head_size] + torch::Tensor & + value_cache, // [num_blocks, num_kv_heads, block_size, head_size] + int64_t num_kv_heads, + double scale, + torch::Tensor &block_tables, // [num_seqs, max_num_blocks_per_seq] + torch::Tensor &seq_lens, // [num_seqs] + torch::Tensor& seq_lens_host, // [num_seqs] + int64_t block_size, int64_t max_seq_len, + const c10::optional &alibi_slopes, // [num_heads] + const std::string &kv_cache_dtype, double k_scale, double v_scale, + int64_t tp_rank, int64_t blocksparse_local_blocks, // no used but to keep same with vllm-offficial + int64_t blocksparse_vert_stride, int64_t blocksparse_block_size, // no used but to keep same with vllm-offficial + int64_t blocksparse_head_sliding_step // no used but to keep same with vllm-offficial + ); + +void weight_only_quant_matmul_xpu( + torch::Tensor &x, + torch::Tensor &out, + torch::Tensor &qweight, + torch::Tensor &qscale +); + + + +void multi_latent_attention_xpu( + torch::Tensor q, + torch::Tensor kv_rope_cache, + torch::Tensor out, + torch::Tensor block_tables, + torch::Tensor seq_lens, + double scale, + int64_t max_seq_len +); + +void outplace_fused_experts_xpu( + torch::Tensor &hidden_states, + torch::Tensor &output, + torch::Tensor &w1, + torch::Tensor &w2, + torch::Tensor &topk_weights, + torch::Tensor &topk_ids +); + +void outplace_fused_experts_sorted_xpu( + torch::Tensor &hidden_states, + torch::Tensor &output, + torch::Tensor &w1, + torch::Tensor &w2, + torch::Tensor &topk_weights, + torch::Tensor &topk_ids +); + + +void grouped_topk_xpu(torch::Tensor &router_logits, + torch::Tensor& score_bias, + torch::Tensor& topk_weight, + torch::Tensor& topk_ids, + double scale, + int64_t expert_group_num, + int64_t moe_topk_group, + int64_t moe_top_k); + +void topk_softmax_xpu(torch::Tensor &topk_weights, /* [m, topk] */ + torch::Tensor& topk_indices, /* [m, topk] */ + torch::Tensor& token_expert_indices, /* no used in xpu */ + torch::Tensor& gating_output /* [m, n] */ + ); +torch::Tensor weak_ref_tensor(torch::Tensor& tensor); + +void dynamic_scaled_int8_quant_xpu(torch::Tensor &out, + torch::Tensor &x, + torch::Tensor &input_scale, + const c10::optional& input_azp +); +void cutlass_scaled_mm_xpu(torch::Tensor& out, torch::Tensor const& a, + torch::Tensor const& b, torch::Tensor const& a_scales, + torch::Tensor const& b_scales, + std::optional const& bias); + +void castte_xpu( + torch::Tensor& input, // [num_tokens, hidden_dim] + torch::Tensor& ouput, // [num_tokens, hidden_dim] + torch::Tensor& scale // [1] +); + +void castte_per_token_xpu( + torch::Tensor& input, // [num_tokens, hidden_dim] + torch::Tensor& ouput, // [num_tokens, hidden_dim] + torch::Tensor& scale // [num_tokens] +); + +void fc_fusion_castte_xpu( + torch::Tensor& x, // [num_tokens, in_dim] + torch::Tensor& ouput, // [num_tokens, out_dim] + torch::Tensor& x_scale, // [1] + torch::Tensor& qweight, // [out_dim, in_dim] + torch::Tensor& qscale, // [1] + const c10::optional& bias // [out_dim] +); + +void fc_fusion_castte_per_token_xpu( + torch::Tensor& x, // [num_tokens, in_dim] + torch::Tensor& ouput, // [num_tokens, out_dim] + torch::Tensor& x_scale, // [num_tokens] + torch::Tensor& qweight, // [out_dim, in_dim] + torch::Tensor& qscale, // [1] + const c10::optional& bias // [out_dim] +); + +// trival cutlass +bool cutlass_scaled_mm_supports_fp8_xpu(int64_t cuda_device_capability); +bool cutlass_scaled_mm_supports_block_fp8_xpu(int64_t cuda_device_capability); + +void outplace_split_norm_rope_xpu( + torch::Tensor &qkv, + torch::Tensor &cos_sin_cache, + torch::Tensor &q_weight, + torch::Tensor &k_weight, + torch::Tensor &positions, + torch::Tensor &q_emb_out, + torch::Tensor &k_emb_out, + torch::Tensor &v_out, + const int64_t emb_batch_size, + const int64_t max_seqlen, + const int64_t head_num, + const int64_t kv_head_num, + const int64_t head_dim, + const int64_t rotary_dim +); + +void moe_fc_int8( + torch::Tensor &hidden_states, // dtype : bfloat16 + torch::Tensor &output, + torch::Tensor &w1, + torch::Tensor &w1_scale, + torch::Tensor &w2, + torch::Tensor &w2_scale, + torch::Tensor &topk_weights, + torch::Tensor &topk_ids +); + +#endif // OPS_H \ No newline at end of file diff --git a/vllm_kunlun/distributed/__init__.py b/vllm_kunlun/distributed/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/vllm_kunlun/distributed/kunlun_communicator.py b/vllm_kunlun/distributed/kunlun_communicator.py new file mode 100644 index 0000000..aac3b8a --- /dev/null +++ b/vllm_kunlun/distributed/kunlun_communicator.py @@ -0,0 +1,102 @@ +# +# Copyright (c) 2025 Baidu, Inc. All Rights Reserved. +# Author: Bao Qian, Dong Xinyu +# Email: baoqian@baidu.com, dongxinyu03@baidu.com +# This file is a part of the vllm-kunlun project. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""kunlun_communicator""" +from contextlib import contextmanager +from typing import Optional + +import torch +from torch.distributed import ProcessGroup +from vllm.distributed.device_communicators.base_device_communicator import DeviceCommunicatorBase +from vllm.distributed.device_communicators.cuda_communicator import CudaCommunicator + +class KunlunCommunicator(CudaCommunicator): + """KunlunCommunicator""" + def __init__(self, + device, + device_group, + cpu_group, + unique_name): + """ + Initializes the CUDA Communicator. + + Args: + cpu_group (ProcessGroup): The CPU process group. + device (Optional[torch.device], optional): The device to use. Defaults to None. + device_group (Optional[ProcessGroup], optional): The device process group. Defaults to None. + unique_name (str, optional): The unique name of this communicator. Defaults to "". + + Raises: + ValueError: If both ``device`` and ``device_group`` are not specified. + """ + DeviceCommunicatorBase.__init__(self, cpu_group, device, device_group, unique_name) + self.ca_comm = None + self.disabled = False + with torch.cuda.device(device): + self.stream = torch.cuda.Stream() + + # A small all_reduce for warmup. + data = torch.zeros(1, device=device) + self.all_reduce(data) + self.stream.synchronize() + del data + + def all_reduce(self, input_): + """all_reduce""" + return DeviceCommunicatorBase.all_reduce(self, input_) + + def all_gather(self, input_, dim): + """all_gather""" + return DeviceCommunicatorBase.all_gather(self, input_, dim) + + def gather(self, input_, dst, dim): + """gather""" + return DeviceCommunicatorBase.gather(self, input_, dst, dim) + + def send(self, tensor, dst): + """send""" + DeviceCommunicatorBase.send(self, tensor, dst) + + def recv(self, size, dtype, src): + """recv""" + return DeviceCommunicatorBase.recv(self, size, dtype, src) + + def destroy(self): + """destroy""" + pass + + @contextmanager + def change_state(self, enable, stream): + """ + A context manager to change the state of the communicator. + """ + if enable is None: + # guess a default value when not specified + enable = self.available + + if stream is None: + stream = self.stream + + old_disable = self.disabled + old_stream = self.stream + + self.stream = stream + self.disabled = not enable + yield + + self.disabled = old_disable + self.stream = old_stream \ No newline at end of file diff --git a/vllm_kunlun/lora/ops/kunlun_ops/__init__.py b/vllm_kunlun/lora/ops/kunlun_ops/__init__.py new file mode 100644 index 0000000..ccf44c7 --- /dev/null +++ b/vllm_kunlun/lora/ops/kunlun_ops/__init__.py @@ -0,0 +1,16 @@ +"""# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project""" + +from vllm_kunlun.lora.ops.kunlun_ops.lora_ops import (bgmv_expand,bgmv_expand_slice, bgmv_shrink, + sgmv_expand, sgmv_expand_slice, + sgmv_shrink) + + +__all__ = [ + "bgmv_expand", + "bgmv_expand_slice", + "bgmv_shrink", + "sgmv_expand", + "sgmv_expand_slice", + "sgmv_shrink" +] \ No newline at end of file diff --git a/vllm_kunlun/lora/ops/kunlun_ops/lora_ops.py b/vllm_kunlun/lora/ops/kunlun_ops/lora_ops.py new file mode 100644 index 0000000..7196eca --- /dev/null +++ b/vllm_kunlun/lora/ops/kunlun_ops/lora_ops.py @@ -0,0 +1,443 @@ +# +# Copyright (c) 2025 Baidu, Inc. All Rights Reserved. +# +# Author: Wang Hao +# Email: wanghao129@baidu.com +# +# This file is a part of the vllm-kunlun project. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""kunlun_ops for lora""" + +import torch +from torch._C import dtype + + +def sgmv_shrink( + inputs: torch.Tensor, + lora_a_weights: torch.Tensor, + output_tensor: torch.Tensor, + block_statistic: torch.Tensor, + sorted_tokens_num_lod: torch.Tensor, + moe_index: torch.Tensor, + expert_m: torch.Tensor, + b_seq_start_loc: torch.Tensor, + seq_len_tensor: torch.Tensor, + lora_indices_tensor: torch.Tensor, + batches: int, + max_seq_length: int, + token_nums: int, + scaling: float, +): + """ + sgmv_shrink + """ + + expert_num = 9 + device = inputs.device + + lora_ids = lora_indices_tensor.repeat_interleave(seq_len_tensor, dim=0).to( + device=device, dtype=torch.int32 + ) + + lora_ids.masked_fill_(lora_ids < 0, expert_num - 1).unsqueeze_(1) + + + + torch.ops._C.gen_block_statistic(lora_ids, block_statistic) + + + inputs_sorted = torch.zeros_like(inputs, dtype=inputs.dtype, device=device) + torch.ops._C.moe_pre_sorted( + inputs, + lora_ids, + block_statistic, + inputs_sorted, + moe_index, + expert_m, + sorted_tokens_num_lod + ) + + + output_tensor.unsqueeze_(1) + + torch.ops._C.moe_fc( + x=inputs_sorted, + weight=lora_a_weights, + sorted_tokens_num_lod=sorted_tokens_num_lod, + sorted_tokens_idx=moe_index, + moe_topk=1, + y=output_tensor, + act=None, + x_perchannel_max=None, + w_perchannel_max=None, + topk_ids=None, + topk_w=None, + bias=None, + tgemm_type=None, + tweight_type=None, + scale_n=0, + scale_k=0, + use_pack_int4=False + ) + + output_tensor.squeeze_(1).mul_(scaling) + + return output_tensor + + +def sgmv_expand(inputs: torch.Tensor, + lora_b_weights: torch.Tensor, + output_tensor: torch.Tensor, + block_statistic: torch.Tensor, + sorted_tokens_num_lod: torch.Tensor, + moe_index: torch.Tensor, + b_seq_start_loc: torch.Tensor, + seq_len_tensor: torch.Tensor, + lora_indices_tensor: torch.Tensor, + batches: int, + max_seq_length: int, + token_nums: int, + add_inputs: bool = False): + """ + sgmv_expand + """ + + + expert_num = 9 + device = inputs.device + + + lora_ids = lora_indices_tensor.repeat_interleave(seq_len_tensor, dim=0).to( + device=device, dtype=torch.int32 + ) + + lora_ids.masked_fill_(lora_ids < 0, expert_num - 1).unsqueeze_(1) + + out = torch.zeros((token_nums, 1, slice_size), dtype=inputs.dtype, device=device) + + + torch.ops._C.moe_fc( + x=inputs, + weight=lora_b_weights, + sorted_tokens_num_lod=sorted_tokens_num_lod, + sorted_tokens_idx=moe_index, + moe_topk=1, + y=out, + act=None, + x_perchannel_max=None, + w_perchannel_max=None, + topk_ids=None, + topk_w=None, + bias=None, + tgemm_type=None, + tweight_type=None, + scale_n=0, + scale_k=0, + use_pack_int4=False + ) + + output_post = out.squeeze(1) + torch.ops._C.moe_post( + output_post, + moe_index.unsqueeze(1), + normed_scale, + normed_scale, + output_post + ) + + + common_len = min(output_post.shape[1], output_tensor.shape[1]) + + limit = min(output_post.shape[0], output_tensor.shape[0]) + + + if add_inputs: + output_tensor[:limit, :common_len] += output_post[:limit, :common_len] + else: + output_tensor[:limit, :common_len] = output_post[:limit, :common_len] + + return output_tensor + + +def sgmv_expand_slice(inputs: torch.Tensor, + lora_b_weights: torch.Tensor, + output_tensor: torch.Tensor, + block_statistic: torch.Tensor, + sorted_tokens_num_lod: torch.Tensor, + moe_index: torch.Tensor, + normed_scale: torch.Tensor, + b_seq_start_loc: torch.Tensor, + seq_len_tensor: torch.Tensor, + lora_indices_tensor: torch.Tensor, + batches: int, + max_seq_length: int, + token_nums: int, + slice_offset: int, + slice_size: int, + add_inputs: bool = False): + + """ + sgmv_expand_slice + """ + + expert_num = 9 + device = inputs.device + + lora_ids = lora_indices_tensor.repeat_interleave(seq_len_tensor, dim=0).to( + device=device, dtype=torch.int32 + ) + + lora_ids.masked_fill_(lora_ids < 0, expert_num - 1).unsqueeze_(1) + + + out = torch.zeros((token_nums, 1, slice_size), dtype=inputs.dtype, device=device) + + + torch.ops._C.moe_fc( + x=inputs, + weight=lora_b_weights, + sorted_tokens_num_lod=sorted_tokens_num_lod, + sorted_tokens_idx=moe_index, + moe_topk=1, + y=out, + act=None, + x_perchannel_max=None, + w_perchannel_max=None, + topk_ids=None, + topk_w=None, + bias=None, + tgemm_type=None, + tweight_type=None, + scale_n=0, + scale_k=0, + use_pack_int4=False + ) + + output_post = out.squeeze(1) + torch.ops._C.moe_post( + output_post, + moe_index.unsqueeze(1), + normed_scale, + normed_scale, + output_post + ) + + + slice_end = slice_offset + slice_size + actual_slice_size = min(slice_size, output_tensor.shape[1] - slice_offset) + + limit = min(output_post.shape[0], output_tensor.shape[0]) + + + if add_inputs: + output_tensor[:limit, slice_offset:slice_end] += output_post[:limit, :actual_slice_size] + else: + output_tensor[:limit, slice_offset:slice_end] = output_post[:limit, :actual_slice_size] + + return output_tensor + + +def bgmv_shrink( + inputs: torch.Tensor, # [m, hidden_dim] + lora_a_weights: torch.Tensor, # [n, 1, r, hidden_dim] + output_tensor: torch.Tensor, # [m, r] + block_statistic: torch.Tensor, + sorted_tokens_num_lod: torch.Tensor, + moe_index: torch.Tensor, + expert_m: torch.Tensor, + lora_indices_tensor: torch.Tensor, # [m] + scaling: float = 1.0 +) -> torch.Tensor: + """ + bgmv_shrink + """ + + expert_num = 9 + + lora_ids = lora_indices_tensor.to(dtype=torch.int32, device=inputs.device) + lora_ids.masked_fill_(lora_ids < 0, expert_num - 1) + + torch.ops._C.gen_block_statistic(lora_ids.unsqueeze(1), block_statistic) + + inputs_sorted = torch.empty_like(inputs, dtype=inputs.dtype, device=inputs.device) + + torch.ops._C.moe_pre_sorted( + inputs, + lora_ids.unsqueeze(1), + block_statistic, + inputs_sorted, + moe_index, + expert_m, + sorted_tokens_num_lod + ) + + output_tensor.unsqueeze_(1) # Change to [m, 1, r] + torch.ops._C.moe_fc( + x=inputs_sorted, + weight=lora_a_weights, + sorted_tokens_num_lod=sorted_tokens_num_lod, + sorted_tokens_idx=moe_index, + moe_topk=1, + y=output_tensor, + act=None, + x_perchannel_max=None, + w_perchannel_max=None, + topk_ids=None, + topk_w=None, + bias=None, + tgemm_type=None, + tweight_type=None, + scale_n=0, + scale_k=0, + use_pack_int4=False + ) + + output_tensor.squeeze_(1).mul_(scaling) + + return output_tensor + + +def bgmv_expand(inputs: torch.Tensor, + lora_b_weights: torch.Tensor, + output_tensor: torch.Tensor, + block_statistic: torch.Tensor, + sorted_tokens_num_lod: torch.Tensor, + moe_index: torch.Tensor, + lora_indices_tensor: torch.Tensor, + add_inputs: bool = True): + """" + bgmv_expand + """ + + + expert_num = 9 + device = inputs.device + + + + + lora_ids = lora_indices_tensor.to(dtype=torch.int32, device=inputs.device) + lora_ids.masked_fill_(lora_ids < 0, expert_num - 1) + + out = torch.zeros((inputs.shape[0], 1, slice_size), dtype=inputs.dtype, device=device) + + + torch.ops._C.moe_fc( + x=inputs, + weight=lora_b_weights, + sorted_tokens_num_lod=sorted_tokens_num_lod, + sorted_tokens_idx=moe_index, + moe_topk=1, + y=out, + act=None, + x_perchannel_max=None, + w_perchannel_max=None, + topk_ids=None, + topk_w=None, + bias=None, + tgemm_type=None, + tweight_type=None, + scale_n=0, + scale_k=0, + use_pack_int4=False + ) + + + + + + + output_post = out.squeeze(1) + torch.ops._C.moe_post(output_post, moe_index.unsqueeze(1), normed_scale, normed_scale, output_post) + + + limit = output_tensor.shape[0] + if output_post.shape[0] == 1 and output_tensor.shape[0] != 1: + limit = 1 + + # LoRA adapter and model may add different amounts of padding to output + common_len = min(output_post.shape[1], output_tensor.shape[1]) + + if add_inputs: + output_tensor[:, :common_len] += output_post[:limit, :common_len] + else: + output_tensor[:, :common_len] = output_post[:limit, :common_len] + + return output_tensor + + +def bgmv_expand_slice( + inputs: torch.Tensor, + lora_b_weights: torch.Tensor, + output_tensor: torch.Tensor, + block_statistic: torch.Tensor, + sorted_tokens_num_lod: torch.Tensor, + moe_index: torch.Tensor, + normed_scale: torch.Tensor, + lora_indices_tensor: torch.Tensor, + slice_offset: int, + slice_size: int, + add_inputs: bool = True +): + """ + bgmv_expand_slice + """ + + expert_num = 9 + device = inputs.device + + + + + lora_ids = lora_indices_tensor.to(dtype=torch.int32, device=inputs.device) + lora_ids.masked_fill_(lora_ids < 0, expert_num - 1) + + out = torch.zeros((inputs.shape[0], 1, slice_size), dtype=inputs.dtype, device=device) + + torch.ops._C.moe_fc( + x=inputs, + weight=lora_b_weights, + sorted_tokens_num_lod=sorted_tokens_num_lod, + sorted_tokens_idx=moe_index, + moe_topk=1, + y=out, + act=None, + x_perchannel_max=None, + w_perchannel_max=None, + topk_ids=None, + topk_w=None, + bias=None, + tgemm_type=None, + tweight_type=None, + scale_n=0, + scale_k=0, + use_pack_int4=False + ) + + + output_post = out.squeeze(1) + torch.ops._C.moe_post(output_post, moe_index.unsqueeze(1), normed_scale, normed_scale, output_post) + + + slice_end = slice_offset + slice_size + actual_slice_size = min(slice_size, output_tensor.shape[1] - slice_offset) + limit = min(output_post.shape[0], output_tensor.shape[0]) + + + if add_inputs: + output_tensor[:limit, slice_offset:slice_end] += output_post[:limit, :actual_slice_size] + else: + output_tensor[:limit, slice_offset:slice_end] = output_post[:limit, :actual_slice_size] + + return output_tensor diff --git a/vllm_kunlun/lora/punica_wrapper/__init__.py b/vllm_kunlun/lora/punica_wrapper/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/vllm_kunlun/lora/punica_wrapper/punica_kunlun.py b/vllm_kunlun/lora/punica_wrapper/punica_kunlun.py new file mode 100644 index 0000000..0d85ede --- /dev/null +++ b/vllm_kunlun/lora/punica_wrapper/punica_kunlun.py @@ -0,0 +1,547 @@ +# +# Copyright (c) 2025 Baidu, Inc. All Rights Reserved. +# Author: Wang Hao +# Email: wanghao129@baidu.com +# This file is a part of the vllm-kunlun project. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Based on: +Chen, L., Ye, Z., Wu, Y., Zhuo, D., Ceze, L., & Krishnamurthy, A. (2023). +Punica: Multi-Tenant LoRA Serving. +https://arxiv.org/abs/2310.18547 +""" + +from typing import TYPE_CHECKING, Optional, Union, final + +import torch + + +# SPDX-License-Identifier: Apache-2.0 +from typing import Callable, Optional, Tuple, Union + + +from vllm_kunlun.lora.ops.kunlun_ops import ( + bgmv_expand, + bgmv_expand_slice, + bgmv_shrink, + sgmv_expand, + sgmv_expand_slice, + sgmv_shrink, +) + +from vllm.lora.punica_wrapper.punica_base import PunicaWrapperBase +import time + + +# The platforms that are compatible with the PyTorch-native implementation can +# inherit this class +class PunicaWrapperKunlun(PunicaWrapperBase): + """ + PunicaWrapperKunlun with moe_fc + """ + + def __init__( + self, + max_num_batched_tokens: int, + max_batches: int, + device: Union[torch.device, str], + **kwargs, + ): + PunicaWrapperBase.__init__(self, max_num_batched_tokens, max_batches, device) + + def _shrink_prefill( + self, + y: torch.Tensor, + x: torch.Tensor, + w_t_all: torch.Tensor, + block_statistic: torch.Tensor, + sorted_tokens_num_lod: torch.Tensor, + moe_index: torch.Tensor, + scale: float, + ): + + expert_m = torch.zeros(9, dtype=torch.int32, device=x.device) + + sgmv_shrink( + x, + w_t_all, + y, + block_statistic, + sorted_tokens_num_lod, + moe_index, + expert_m, + *self.prefill_metadata, + scale, + ) + + def _shrink_decode( + self, + y: torch.Tensor, + x: torch.Tensor, + w_t_all: torch.Tensor, + block_statistic: torch.Tensor, + sorted_tokens_num_lod: torch.Tensor, + moe_index: torch.Tensor, + scale: float, + ): + + expert_m = torch.zeros(9, dtype=torch.int32, device=x.device) + bgmv_shrink( + x, + w_t_all, + y, + block_statistic, + sorted_tokens_num_lod, + moe_index, + expert_m, + self.token_lora_indices, + scale, + ) + + def _expand_prefill( + self, + y: torch.Tensor, + x: torch.Tensor, + w_t_all: torch.Tensor, + block_statistic: torch.Tensor, + sorted_tokens_num_lod: torch.Tensor, + moe_index: torch.Tensor, + add_inputs: bool, + ): + + sgmv_expand( + x, + w_t_all, + y, + block_statistic, + sorted_tokens_num_lod, + moe_index, + *self.prefill_metadata, + add_inputs, + ) + + def _expand_decode( + self, + y: torch.Tensor, + x: torch.Tensor, + w_t_all: torch.Tensor, + block_statistic: torch.Tensor, + sorted_tokens_num_lod: torch.Tensor, + moe_index: torch.Tensor, + add_inputs: bool, + ): + bgmv_expand( + x, + w_t_all, + y, + block_statistic, + sorted_tokens_num_lod, + moe_index, + self.token_lora_indices, + add_inputs, + ) + + def _expand_slice_prefill( + self, + y: torch.Tensor, + x: torch.Tensor, + w_t_all: torch.Tensor, + block_statistic, + sorted_tokens_num_lod: torch.Tensor, + moe_index: torch.Tensor, + y_offset: int, + y_slice_size: int, + add_inputs: bool, + ): + + normed_scale = torch.ones([y.size(0), 1], dtype=torch.float32, device=x.device) + + sgmv_expand_slice( + x, + w_t_all, + y, + block_statistic, + sorted_tokens_num_lod, + moe_index, + normed_scale, + *self.prefill_metadata, + y_offset, + y_slice_size, + add_inputs, + ) + + def _expand_slice_decode( + self, + y: torch.Tensor, + x: torch.Tensor, + w_t_all: torch.Tensor, + block_statistic: torch.Tensor, + sorted_tokens_num_lod: torch.Tensor, + moe_index: torch.Tensor, + y_offset: int, + y_slice_size: int, + add_inputs: bool, + ): + + normed_scale = torch.ones([y.size(0), 1], dtype=torch.float32, device=x.device) + + bgmv_expand_slice( + x, + w_t_all, + y, + block_statistic, + sorted_tokens_num_lod, + moe_index, + normed_scale, + self.token_lora_indices, + y_offset, + y_slice_size, + add_inputs, + ) + + def _apply_expand( + self, + y: torch.Tensor, + x: torch.Tensor, + w_t_all: torch.Tensor, + block_statistic, + sorted_tokens_num_lod: torch.Tensor, + moe_index: torch.Tensor, + y_offset: int, + y_slice_size: int, + add_inputs: bool = True, + ): + """ + Perform the ` y[:,y_offset:y_offset+y_slice_size]+=x@w_t_all` + computation, which is suitable for the + GEMM of lora'b. + """ + + expand_slice_fun: Callable = ( + self._expand_slice_prefill if self.is_prefill else self._expand_slice_decode + ) + expand_slice_fun( + y, + x, + w_t_all, + block_statistic, + sorted_tokens_num_lod, + moe_index, + y_offset, + y_slice_size, + add_inputs, + ) + + def _apply_shrink( + self, + y: torch.Tensor, + x: torch.Tensor, + w_t_all: torch.Tensor, + block_statistic: torch.Tensor, + sorted_tokens_num_lod: torch.Tensor, + moe_index: torch.Tensor, + scale: float, + ): + """ + Perform the ` y+=x@w_t_all` computation, which is suitable for the + GEMM of lora'a. + When `is_prefill is` true, it indicates that it is currently the + prefill stage, and the `_shrink_prefill` function should be called. + Otherwise, it is the decode stage, and the _shrink_decode function + should be called. + """ + y_org = y + y = y.view(-1, y.shape[-1]) + + shrink_fun: Callable = ( + self._shrink_prefill if self.is_prefill else self._shrink_decode + ) + + shrink_fun( + y, x, w_t_all, block_statistic, sorted_tokens_num_lod, moe_index, scale + ) + + y = y.view_as(y_org) + + def add_shrink( + self, + y: Union[Tuple[torch.Tensor, ...], torch.Tensor], + x: torch.Tensor, + lora_a_stacked: Tuple[torch.Tensor, ...], + block_statistic: torch.Tensor, + sorted_tokens_num_lod: torch.Tensor, + moe_index: torch.Tensor, + scale: float, + **kwargs, + ): + """ + Performs GEMM for multiple slices of lora_a. + When `is_prefill is` true, it indicates that it is currently the + prefill stage, and the `_shrink_prefill` function should be called. + Otherwise, it is the decode stage, and the _shrink_decode function + should be called. + + Semantics: + for i in range(len(lora_a_stacked)): + y[i] += (x @ lora_a_stacked[i]) * scale + + Args: + y (Union[Tuple[torch.Tensor, ...], torch.Tensor]): Output tensors + x (torch.Tensor): Input tensor + lora_a_stacked (Tuple[torch.Tensor, ...]): lora_a's weights + scale (float): Scaling factor for the operation + """ + + x = x.view(-1, x.shape[-1]) + + for slice_idx in range(len(lora_a_stacked)): # Each slice represents a layer + + self._apply_shrink( + y[slice_idx], + x, + lora_a_stacked[slice_idx], + block_statistic, + sorted_tokens_num_lod, + moe_index, + scale, + ) + + def add_expand( + self, + y: torch.Tensor, + x: Union[Tuple[torch.Tensor, ...], torch.Tensor], + lora_b_stacked: Tuple[torch.Tensor, ...], + block_statistic: torch.Tensor, + sorted_tokens_num_lod: torch.Tensor, + moe_index: torch.Tensor, + lora_bias_stacked: Optional[Tuple[torch.Tensor, ...]], + output_slices: Tuple[int, ...], + offset_start: int = 0, + add_inputs=True, + **kwargs, + ) -> None: + """ + Performs GEMM and bias addition for multiple slices of lora_b. + + Semantics: + for i in range(len(lora_b_stacked)): + slice = output_slices[i] + y[:, offset:offset+slice] += x[i] @ lora_b_stacked[i] + + lora_bias_stacked[i] + offset += slice + + Args: + y (torch.Tensor): Output tensor. + x (Union[Tuple[torch.Tensor, ...], torch.Tensor]): Input tensors + lora_b_stacked (Tuple[torch.Tensor, ...]): lora_b's weight + lora_bias_stacked (Optional[Tuple[torch.Tensor, ...]]): + bias's weight + output_slices (Tuple[int, ...]): Every slice's size + add_inputs (bool): Defaults to True. + """ + + y_org = y + y = y.view(-1, y.shape[-1]) + offset_left = offset_start + + if lora_bias_stacked is not None: + self._apply_bias( + self.token_lora_indices, y, output_slices, lora_bias_stacked + ) + + for slice_idx in range(len(lora_b_stacked)): + self._apply_expand( + y, + x[slice_idx], + lora_b_stacked[slice_idx], + block_statistic, + sorted_tokens_num_lod, + moe_index, + offset_left, + output_slices[slice_idx], + add_inputs=add_inputs, + ) + offset_left += output_slices[slice_idx] + + y = y.view_as(y_org) + + def add_lora_embedding( + self, + y: torch.Tensor, + x: torch.Tensor, + lora_b_stacked: torch.Tensor, + add_inputs: bool = True, + **kwargs, + ) -> None: + """ + Applies lora specifically for VocabParallelEmbeddingWithLoRA. + + Semantics: + y += x @ lora_b_stacked + + Args: + y (torch.Tensor): Output tensor. + x (torch.Tensor): Input tensor. + lora_b_stacked (torch.Tensor): lora_b's weights. + add_inputs (bool): Default to True. + """ + + expand_fun: Callable = ( + self._expand_prefill if self.is_prefill else self._expand_decode + ) + expand_fun(y, x, lora_b_stacked, add_inputs) + + def add_lora_linear( + self, + y: torch.Tensor, + x: torch.Tensor, + lora_a_stacked: Tuple[torch.Tensor, ...], + lora_b_stacked: Tuple[torch.Tensor, ...], + lora_bias_stacked: Optional[Tuple[torch.Tensor, ...]], + scale: float, + output_slices: Tuple[int, ...], + *, + buffer: Optional[Tuple[torch.Tensor, ...]] = None, + **kwargs, + ) -> None: + """ + Applicable to linear-related lora. + + Semantics: + for i in range(len(lora_a_stacked)): + y[i] += ( + x[i].unsqueeze(0) + @ lora_a_stacked[indices[i], layer_idx, :, :] + @ lora_b_stacked[indices[i], layer_idx, :, :] + * scale + ).squeeze(0)+lora_bias_stacked[i] + + Args: + y (torch.Tensor): Output tensor. Will be changed in-place. + x (torch.Tensor): Input tensor + lora_a_stacked (Tuple[torch.Tensor, ...]): lora_a's weight. + lora_b_stacked (Tuple[torch.Tensor, ...]): lora_b's weight. + lora_bias_stacked (Optional[Tuple[torch.Tensor, ...]]): lora's bias. + scale (float): Scaling factor. + output_slices (Tuple[int, ...]): Every slice's size. + buffer (Optional[Tuple[torch.Tensor, ...]]): Defaults to None. + """ + + if self.no_lora: + return + + expert_num = 9 + block_statistic = torch.zeros( + [12, expert_num], dtype=torch.int32, device=x.device + ) + sorted_tokens_num_lod = torch.zeros( + expert_num + 1, dtype=torch.int32, device=x.device + ) + token_nums = x.size(0) + moe_index = torch.zeros(token_nums, dtype=torch.int32, device=x.device) + + assert len(lora_a_stacked) == len(lora_b_stacked) == len(output_slices) + if lora_bias_stacked is not None: + assert len(lora_bias_stacked) == len(output_slices) + y = self._apply_bias( + self.token_lora_indices, y, output_slices, lora_bias_stacked + ) + + if buffer is None: + r = lora_b_stacked[0].size(-1) + buffer = tuple( + torch.zeros((x.size(0), r), dtype=torch.float16, device=x.device) + for _ in range(len(output_slices)) + ) + # [tensor.squeeze_(1) for tensor in lora_a_stacked] + new_lora_a_stacked = tuple(lora_a.squeeze(1) for lora_a in lora_a_stacked) + self.add_shrink( + buffer, + x, + new_lora_a_stacked, + block_statistic, + sorted_tokens_num_lod, + moe_index, + scale, + **kwargs, + ) + # [tensor.unsqueeze_(1) for tensor in lora_a_stacked] + + # [tensor.squeeze_(1) for tensor in lora_b_stacked] + new_lora_b_stacked = tuple(lora_b.squeeze(1) for lora_b in lora_b_stacked) + self.add_expand( + y, + buffer, + new_lora_b_stacked, + block_statistic, + sorted_tokens_num_lod, + moe_index, + None, + output_slices, + add_inputs=True, + **kwargs, + ) + # [tensor.unsqueeze_(1) for tensor in lora_b_stacked] + + def add_lora_logits( + self, + y: torch.Tensor, + x: torch.Tensor, + lora_a_stacked: torch.Tensor, + lora_b_stacked: torch.Tensor, + scale, + *, + buffer: Optional[torch.Tensor] = None, + **kwargs, + ) -> None: + """ + Applies lora specifically for LogitsProcessorWithLoRA. + + Semantics: + buffer = (x @ lora_a_stacked) * scale + y += buffer @ lora_b_stacked + + Args: + y (torch.Tensor): Output tensor. + x (torch.Tensor): Input tensor. + lora_a_stacked (torch.Tensor): lora_a's weights. + lora_b_stacked (torch.Tensor):lora_b's weights. + scale (float): Scaling factor. + buffer (Optional[torch.Tensor]):Default to None. + """ + y_org = y + y = y.view(-1, y.shape[-1]) + x = x.view(-1, x.shape[-1]) + + if lora_a_stacked.dim() == 2: + lora_a_stacked = lora_a_stacked.unsqueeze(0) + if lora_b_stacked.dim() == 2: + lora_b_stacked = lora_b_stacked.unsqueeze(0) + + r = lora_a_stacked.size(-1) + + if buffer is None: + buffer = torch.zeros((x.size(0), r), dtype=torch.float32, device=x.device) + + indices = self.sampler_indices + if indices.max() >= lora_a_stacked.size(0): + indices = torch.clamp(indices, 0, lora_a_stacked.size(0) - 1) + + lora_a_reshaped = lora_a_stacked.transpose(1, 2) + lora_b_reshaped = lora_b_stacked.transpose(1, 2) + + bgmv_shrink(x, lora_a_reshaped, buffer, indices, scale) + bgmv_expand(buffer, lora_b_reshaped, y, indices, add_inputs=True) + + y = y.view_as(y_org) diff --git a/vllm_kunlun/models/__init__.py b/vllm_kunlun/models/__init__.py new file mode 100644 index 0000000..2d3b08f --- /dev/null +++ b/vllm_kunlun/models/__init__.py @@ -0,0 +1,68 @@ +from vllm import ModelRegistry + + +def register_model(): + # from .demo_model import DemoModel # noqa: F401 + from .qwen2_vl import Qwen2VLForConditionalGeneration #noqa: F401 + from .qwen2_5_vl import Qwen2_5_VLForConditionalGeneration #noqa: F401 + from .qwen3 import Qwen3ForCausalLM #noqa: F401 + from .qwen3_moe import Qwen3MoeForCausalLM #noqa: F401 + + # ModelRegistry.register_model( + # "DemoModel", + # "vllm_kunlun.model_executor.models.demo_model:DemoModel") + + ModelRegistry.register_model( + "Qwen2VLForConditionalGeneration", + "vllm_kunlun.models.qwen2_vl:Qwen2VLForConditionalGeneration") + + ModelRegistry.register_model( + "Qwen2_5_VLForConditionalGeneration", + "vllm_kunlun.models.qwen2_5_vl:Qwen2_5_VLForConditionalGeneration") + + ModelRegistry.register_model( + "Qwen3ForCausalLM", + "vllm_kunlun.models.qwen3:Qwen3ForCausalLM") + + ModelRegistry.register_model( + "Qwen3MoeForCausalLM", + "vllm_kunlun.models.qwen3_moe:Qwen3MoeForCausalLM") + + ModelRegistry.register_model( + "GlmForCausalLM", + "vllm_kunlun.models.glm:GlmForCausalLM") + + ModelRegistry.register_model( + "GptOssForCausalLM", + "vllm_kunlun.models.gpt_oss:GptOssForCausalLM") + ModelRegistry.register_model( + "InternLM2ForCausalLM", + "vllm_kunlun.models.internlm2:InternLM2ForCausalLM") + + ModelRegistry.register_model( + "Qwen2ForCausalLM", + "vllm_kunlun.models.qwen2:Qwen2ForCausalLM") + + ModelRegistry.register_model( + "InternVLChatModel", + "vllm_kunlun.models.internvl:InternVLChatModel") + + ModelRegistry.register_model( + "InternS1ForConditionalGeneration", + "vllm_kunlun.models.interns1:InternS1ForConditionalGeneration") + + ModelRegistry.register_model( + "Glm4MoeForCausalLM", + "vllm_kunlun.models.glm4_moe:Glm4MoeForCausalLM") + + ModelRegistry.register_model( + "Glm4ForCausalLM", + "vllm_kunlun.models.glm4:Glm4ForCausalLM") + + ModelRegistry.register_model( + "Glm4vForConditionalGeneration", + "vllm_kunlun.models.glm4_1v:Glm4vForConditionalGeneration") + + +def register_quant_method(): + """to do""" \ No newline at end of file diff --git a/vllm_kunlun/models/glm.py b/vllm_kunlun/models/glm.py new file mode 100644 index 0000000..d61e089 --- /dev/null +++ b/vllm_kunlun/models/glm.py @@ -0,0 +1,24 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Inference-only HF format GLM-4 model compatible with THUDM weights.""" +from vllm.config import VllmConfig +# from vllm.model_executor.models.llama import LlamaForCausalLM +from .llama import LlamaForCausalLM #noqa: F401 + +from vllm.model_executor.models.utils import PPMissingLayer + +class GlmForCausalLM(LlamaForCausalLM): + + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): + print("glm for causalLM initialization!!!!", flush=True) + vllm_config.model_config.hf_config.partial_rotary_factor = 0.5 + super().__init__(vllm_config=vllm_config, prefix=prefix) + # Hack Llama model to fit HF format GLM implementation + # Attention difference between GLM and Llama: + # 1. Half partial rotary_dim and no Neox style. + # 2. There is no bias for o_proj in attention + for layer in self.model.layers: + if not isinstance(layer, PPMissingLayer): + layer.self_attn.rotary_emb.is_neox_style = False + layer.self_attn.o_proj.bias = None + layer.self_attn.o_proj.skip_bias_add = True diff --git a/vllm_kunlun/models/glm4.py b/vllm_kunlun/models/glm4.py new file mode 100644 index 0000000..816ab6e --- /dev/null +++ b/vllm_kunlun/models/glm4.py @@ -0,0 +1,301 @@ +# +# Copyright (c) 2025 Baidu, Inc. All Rights Reserved. +# Adapted from vllm/model_executor/models/glm4.py +# Copyright 2023 The vLLM team. +# +# This file is a part of the vllm-kunlun project. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Inference-only GLM-4-0414 model compatible with HuggingFace weights.""" +from collections.abc import Iterable +from typing import Optional, Union + +import torch +from torch import nn +from transformers import Glm4Config + +from vllm.attention import AttentionType +from vllm_kunlun.ops.attention.layer import Attention +from vllm.compilation.decorators import support_torch_compile +from vllm.config import CacheConfig, VllmConfig +from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size +from vllm.model_executor.layers.layernorm import RMSNorm +from vllm.model_executor.layers.linear import (QKVParallelLinear, + RowParallelLinear) +from vllm.model_executor.layers.logits_processor import LogitsProcessor +from vllm.model_executor.layers.quantization import QuantizationConfig +from vllm.model_executor.layers.rotary_embedding import get_rope +from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead +from vllm.model_executor.sampling_metadata import SamplingMetadata +from vllm.sequence import IntermediateTensors + +from vllm.model_executor.models.interfaces import SupportsLoRA, SupportsPP +from vllm_kunlun.models.llama import LlamaMLP as Glm4MLP +from vllm_kunlun.models.llama import LlamaModel +from vllm.model_executor.models.utils import AutoWeightsLoader, PPMissingLayer, maybe_prefix + + +class Glm4Attention(nn.Module): + + def __init__(self, + config: Glm4Config, + hidden_size: int, + num_heads: int, + num_kv_heads: int, + max_position: int = 4096 * 32, + head_dim: Optional[int] = None, + qkv_bias: bool = False, + rope_theta: float = 10000, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + rope_scaling: Optional[tuple] = None, + prefix: str = "", + attn_type: str = AttentionType.DECODER) -> None: + super().__init__() + self.hidden_size = hidden_size + tp_size = get_tensor_model_parallel_world_size() + self.total_num_heads = num_heads + assert self.total_num_heads % tp_size == 0 + self.num_heads = self.total_num_heads // tp_size + self.total_num_kv_heads = num_kv_heads + if self.total_num_kv_heads >= tp_size: + # Number of KV heads is greater than TP size, so we partition + # the KV heads across multiple tensor parallel GPUs. + assert self.total_num_kv_heads % tp_size == 0 + else: + # Number of KV heads is less than TP size, so we replicate + # the KV heads across multiple tensor parallel GPUs. + assert tp_size % self.total_num_kv_heads == 0 + partial_rotary_factor = getattr(config, "partial_rotary_factor", 0.5) + self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size) + self.head_dim = head_dim or hidden_size // self.total_num_heads + self.rotary_dim = self.head_dim + self.q_size = self.num_heads * self.head_dim + self.kv_size = self.num_kv_heads * self.head_dim + self.scaling = self.head_dim**-0.5 + self.rope_theta = rope_theta + self.qkv_proj = QKVParallelLinear( + hidden_size, + self.head_dim, + self.total_num_heads, + self.total_num_kv_heads, + bias=qkv_bias, + quant_config=quant_config, + prefix=f"{prefix}.qkv_proj", + ) + self.o_proj = RowParallelLinear( + self.total_num_heads * self.head_dim, + hidden_size, + bias=False, + quant_config=quant_config, + prefix=f"{prefix}.o_proj", + ) + self.rotary_emb = get_rope( + self.head_dim, + rotary_dim=self.rotary_dim, + max_position=max_position, + base=self.rope_theta, + rope_scaling=rope_scaling, + partial_rotary_factor=partial_rotary_factor, + is_neox_style=False, + ) + self.attn = Attention(self.num_heads, + self.head_dim, + self.scaling, + num_kv_heads=self.num_kv_heads, + cache_config=cache_config, + quant_config=quant_config, + prefix=f"{prefix}.attn", + attn_type=attn_type) + + def forward( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + ) -> torch.Tensor: + qkv, _ = self.qkv_proj(hidden_states) + q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1) + q, k = self.rotary_emb(positions, q, k) + attn_output = self.attn(q, k, v) + output, _ = self.o_proj(attn_output) + return output + + +class Glm4DecoderLayer(nn.Module): + + def __init__( + self, + config: Glm4Config, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + prefix: str = "", + ) -> None: + super().__init__() + self.hidden_size = config.hidden_size + rope_theta = getattr(config, "rope_theta", 1000000) + rope_scaling = getattr(config, "rope_scaling", None) + + self.self_attn = Glm4Attention( + config=config, + hidden_size=self.hidden_size, + num_heads=config.num_attention_heads, + max_position=config.max_position_embeddings, + num_kv_heads=config.num_key_value_heads, + rope_theta=rope_theta, + qkv_bias=getattr(config, 'attention_bias', False), + head_dim=getattr(config, 'head_dim', None), + cache_config=cache_config, + quant_config=quant_config, + rope_scaling=rope_scaling, + prefix=f"{prefix}.self_attn", + attn_type=AttentionType.DECODER, + ) + self.mlp = Glm4MLP( + hidden_size=self.hidden_size, + intermediate_size=config.intermediate_size, + hidden_act=config.hidden_act, + quant_config=quant_config, + prefix=f"{prefix}.mlp", + ) + self.input_layernorm = RMSNorm(config.hidden_size, + eps=config.rms_norm_eps) + self.post_attention_layernorm = RMSNorm(config.hidden_size, + eps=config.rms_norm_eps) + self.post_self_attn_layernorm = RMSNorm(config.hidden_size, + eps=config.rms_norm_eps) + self.post_mlp_layernorm = RMSNorm(config.hidden_size, + eps=config.rms_norm_eps) + + def forward( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + residual: Optional[torch.Tensor], + ) -> tuple[torch.Tensor, torch.Tensor]: + # Self Attention + if residual is None: + residual = hidden_states + hidden_states = self.input_layernorm(hidden_states) + else: + hidden_states, residual = self.input_layernorm( + hidden_states, residual) + hidden_states = self.self_attn( + positions=positions, + hidden_states=hidden_states, + ) + + hidden_states = self.post_self_attn_layernorm(hidden_states) + + # Fully Connected + hidden_states, residual = self.post_attention_layernorm( + hidden_states, residual) + hidden_states = self.mlp(hidden_states) + hidden_states = self.post_mlp_layernorm(hidden_states) + + return hidden_states, residual + + +ALL_DECODER_LAYER_TYPES = { + "attention": Glm4DecoderLayer, +} + + +@support_torch_compile( + dynamic_arg_dims={ + "input_ids": 0, + "positions": -1, + "intermediate_tensors": 0, + "inputs_embeds": 0, + }) +class Glm4Model(LlamaModel): + + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): + super().__init__(vllm_config=vllm_config, + prefix=prefix, + layer_type=Glm4DecoderLayer) + + +class Glm4ForCausalLM(nn.Module, SupportsLoRA, SupportsPP): + packed_modules_mapping = { + "qkv_proj": [ + "q_proj", + "k_proj", + "v_proj", + ], + "gate_up_proj": [ + "gate_proj", + "up_proj", + ], + } + + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): + super().__init__() + config = vllm_config.model_config.hf_config + quant_config = vllm_config.quant_config + lora_config = vllm_config.lora_config + + self.config = config + self.lora_config = lora_config + + self.quant_config = quant_config + self.model = Glm4Model(vllm_config=vllm_config, + prefix=maybe_prefix(prefix, "model")) + + if get_pp_group().is_last_rank: + if config.tie_word_embeddings: + self.lm_head = self.model.embed_tokens + else: + self.lm_head = ParallelLMHead(config.vocab_size, + config.hidden_size, + quant_config=quant_config, + prefix=maybe_prefix( + prefix, "lm_head")) + else: + self.lm_head = PPMissingLayer() + + self.logits_processor = LogitsProcessor(config.vocab_size) + + self.make_empty_intermediate_tensors = ( + self.model.make_empty_intermediate_tensors) + + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor: + return self.model.get_input_embeddings(input_ids) + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + intermediate_tensors: Optional[IntermediateTensors] = None, + inputs_embeds: Optional[torch.Tensor] = None, + ) -> Union[torch.Tensor, IntermediateTensors]: + hidden_states = self.model(input_ids, positions, intermediate_tensors, + inputs_embeds) + return hidden_states + + def compute_logits( + self, + hidden_states: torch.Tensor, + sampling_metadata: SamplingMetadata, + ) -> Optional[torch.Tensor]: + logits = self.logits_processor(self.lm_head, hidden_states, + sampling_metadata) + return logits + + def load_weights(self, weights: Iterable[tuple[str, + torch.Tensor]]) -> set[str]: + loader = AutoWeightsLoader( + self, + skip_prefixes=(["lm_head."] + if self.config.tie_word_embeddings else None), + ) + return loader.load_weights(weights) diff --git a/vllm_kunlun/models/glm4_1v.py b/vllm_kunlun/models/glm4_1v.py new file mode 100644 index 0000000..dfd5ecf --- /dev/null +++ b/vllm_kunlun/models/glm4_1v.py @@ -0,0 +1,1597 @@ +# +# Copyright (c) 2025 Baidu, Inc. All Rights Reserved. +# Adapted from vllm/model_executor/models/glm4_1v.py +# Copyright 2023 The vLLM team. +# +# This file is a part of the vllm-kunlun project. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Inference-only GLM-4V model compatible with HuggingFace weights.""" + +import math +from collections.abc import Iterable, Mapping, Sequence +from functools import partial +from typing import Annotated, Any, Callable, Literal, Optional, Union + +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F +from einops import rearrange +from transformers import BatchFeature +from transformers.models.glm4v.configuration_glm4v import Glm4vVisionConfig +from transformers.models.glm4v.image_processing_glm4v import ( + Glm4vImageProcessor, smart_resize) +from transformers.models.glm4v.video_processing_glm4v import ( + Glm4vVideoProcessor) +from transformers.video_utils import VideoMetadata + +from vllm.config import VllmConfig +from vllm.distributed import parallel_state +from vllm.distributed import utils as dist_utils +from vllm.logger import init_logger +from vllm.model_executor import SamplingMetadata +from vllm.model_executor.layers.layernorm import RMSNorm +from vllm.model_executor.layers.linear import (ColumnParallelLinear, + MergedColumnParallelLinear, + QKVParallelLinear, + RowParallelLinear) +from vllm.model_executor.layers.quantization import QuantizationConfig +from vllm.model_executor.model_loader.weight_utils import default_weight_loader +from vllm.model_executor.models.module_mapping import MultiModelKeys +from vllm.multimodal import MULTIMODAL_REGISTRY +from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig, + MultiModalKwargs, VideoItem) +from vllm.multimodal.parse import (ImageSize, MultiModalDataItems, + MultiModalDataParser) +from vllm.multimodal.processing import (BaseMultiModalProcessor, + BaseProcessingInfo, PromptReplacement, + PromptUpdate, PromptUpdateDetails) +from vllm.multimodal.profiling import BaseDummyInputsBuilder +from vllm.platforms import _Backend +from vllm.sequence import IntermediateTensors +from vllm.transformers_utils.config import uses_mrope +from vllm.utils.tensor_schema import TensorSchema, TensorShape + +from vllm_kunlun.ops.activation import SiluAndMul +from vllm.model_executor.models.interfaces import (MultiModalEmbeddings, SupportsLoRA, + SupportsMultiModal, SupportsPP) +from vllm_kunlun.models.qwen2_vl import _qwen2vl_field_config, apply_rotary_pos_emb_vision +from vllm.model_executor.models.utils import (AutoWeightsLoader, WeightsMapper, + init_vllm_registered_model, maybe_prefix, + merge_multimodal_embeddings) +from vllm.model_executor.models.vision import get_vit_attn_backend + +logger = init_logger(__name__) + +# For profile run +_MAX_FRAMES_PER_VIDEO = 600 + +# === Vision Inputs === # + +import torch +import torch.nn.functional as F + +def grid_sample(input, grid, **kwargs): + try: + return F.grid_sample(input, grid, **kwargs) + except RuntimeError: + # if grid_sample is not implemented on XPU, falling back to CPU. + result = F.grid_sample(input.cpu(), grid.cpu(), **kwargs).to(device=input.device, dtype=input.dtype).contiguous() + return result + +class Glm4vImagePixelInputs(TensorSchema): + """ + Dimensions: + - np: Number of patches + - cpp: Number of channels * patch_size * patch_size + - ni: Number of images + - g: Grid dimensions (3 for grid_t, grid_h, grid_w) + """ + type: Literal["pixel_values"] = "pixel_values" + + pixel_values: Annotated[torch.Tensor, TensorShape("np", "cpp")] + image_grid_thw: Annotated[torch.Tensor, TensorShape("ni", 3)] + + +class Glm4vImageEmbeddingInputs(TensorSchema): + """ + Dimensions: + - f: Number of image features (varies based on image resolution) + - h: Hidden size (must match language model backbone) + - n: Number of images + - g: Grid dimensions (3 for grid_t, grid_h, grid_w) + """ + type: Literal["image_embeds"] = "image_embeds" + + image_embeds: Annotated[torch.Tensor, TensorShape("f", "h")] + image_grid_thw: Annotated[torch.Tensor, TensorShape("n", 3)] + + +Glm4vImageInputs = Union[Glm4vImagePixelInputs, Glm4vImageEmbeddingInputs] + + +class Glm4vVideoPixelInputs(TensorSchema): + """ + Dimensions: + - np: Number of patches + - ctpp: Number of channels * temporal_patch_size * + patch_size * patch_size + - f: Number of frames + - g: Grid dimensions (3 for grid_t which is usually 1 for processed + video, grid_h, grid_w) + """ + type: Literal["pixel_values_videos"] = "pixel_values_videos" + + pixel_values_videos: Annotated[torch.Tensor, TensorShape("np", "ctpp")] + video_grid_thw: Annotated[torch.Tensor, TensorShape("f", 3)] + + +class Glm4vVideoEmbeddingInputs(TensorSchema): + """ + Dimensions: + - p: Number of video patches across all frames + - h: Hidden size (must match language model backbone) + - f: Number of frames + - g: Grid dimensions (3 for grid_t which is usually 1 for processed + video, grid_h, grid_w) + """ + type: Literal["video_embeds"] = "video_embeds" + + video_embeds: Annotated[torch.Tensor, TensorShape("p", "h")] + video_grid_thw: Annotated[torch.Tensor, TensorShape("f", 3)] + + +Glm4vVideoInputs = Union[Glm4vVideoPixelInputs, Glm4vVideoEmbeddingInputs] + +# === Vision Encoder === # + + +class Glm4vVisionMLP(nn.Module): + + def __init__( + self, + in_features: int, + hidden_features: int, + bias: bool = False, + quant_config: Optional[QuantizationConfig] = None, + prefix: str = "", + ): + super().__init__() + self.gate_up_proj = MergedColumnParallelLinear( + input_size=in_features, + output_sizes=[hidden_features] * 2, + bias=bias, + quant_config=quant_config, + prefix=f"{prefix}.gate_up_proj") + self.down_proj = RowParallelLinear(hidden_features, + in_features, + bias=bias, + quant_config=quant_config, + prefix=f"{prefix}.down_proj") + self.act_fn = SiluAndMul() + + def forward(self, x: torch.Tensor): + x, _ = self.gate_up_proj(x) + x = self.act_fn(x) + x, _ = self.down_proj(x) + return x + + +def all_gather_interleave(local_tensor, hidden_size: int, tp_size: int): + """All-gather the input tensor interleavely across model parallel group.""" + import torch.distributed as dist + + gathered_tensors = [torch.zeros_like(local_tensor) for _ in range(tp_size)] + dist.all_gather( + gathered_tensors, + local_tensor, + group=parallel_state.get_tp_group().device_group, + ) + + gathered_tensors_split = [ + torch.split(tensor, hidden_size // tp_size, -1) + for tensor in gathered_tensors + ] + ordered_tensors = [ + tensor for pair in zip(*gathered_tensors_split) for tensor in pair + ] + result_tensor = torch.cat(ordered_tensors, dim=-1) + return result_tensor + + +class Glm4vVisionAttention(nn.Module): + + def __init__( + self, + embed_dim: int, + num_heads: int, + projection_size: int, + quant_config: Optional[QuantizationConfig] = None, + prefix: str = "", + ) -> None: + super().__init__() + # Per attention head and per partition values. + self.tp_size = parallel_state.get_tensor_model_parallel_world_size() + self.tp_rank = parallel_state.get_tensor_model_parallel_rank() + self.hidden_size_per_attention_head = dist_utils.divide( + projection_size, num_heads) + self.num_attention_heads_per_partition = dist_utils.divide( + num_heads, self.tp_size) + + self.qkv = QKVParallelLinear( + hidden_size=embed_dim, + head_size=self.hidden_size_per_attention_head, + total_num_heads=num_heads, + total_num_kv_heads=num_heads, + bias=False, + quant_config=quant_config, + prefix=f"{prefix}.qkv", + ) + self.proj = RowParallelLinear( + input_size=projection_size, + output_size=embed_dim, + quant_config=quant_config, + prefix=f"{prefix}.proj", + bias=False, + ) + + # Detect attention implementation. + self.attn_backend: _Backend = get_vit_attn_backend(support_fa=True) + if self.attn_backend not in { + _Backend.FLASH_ATTN, + _Backend.TORCH_SDPA, + _Backend.XFORMERS, + }: + raise RuntimeError( + f"GLM-4V does not support {self.attn_backend} backend now.") + + def split_qkv(self, qkv: torch.Tensor) -> tuple[torch.Tensor, ...]: + # [s, b, 3 * head * head_dim] + seq_len, bs, _ = qkv.shape + if self.tp_size > 1: + qkv = all_gather_interleave(qkv, self.qkv.hidden_size, + self.tp_size) + + # [s, b, 3 * head * head_dim] -> 3 * [s, b, head * head_dim] + q, k, v = qkv.chunk(3, dim=2) + + # 3 * [s, b, head * head_dim] + if self.tp_size > 1: + splitter = partial( + dist_utils.split_tensor_along_last_dim, + num_partitions=self.tp_size, + ) + q = splitter(q)[self.tp_rank] + k = splitter(k)[self.tp_rank] + v = splitter(v)[self.tp_rank] + + # 3 * [s, b, head * head_dim] -> 3 * [s, b, head, head_dim] + new_shape = ( + seq_len, + bs, + self.num_attention_heads_per_partition, + self.hidden_size_per_attention_head, + ) + q, k, v = (x.view(*new_shape) for x in (q, k, v)) + return q, k, v + + def forward( + self, + x: torch.Tensor, + cu_seqlens: torch.Tensor, + rotary_pos_emb: torch.Tensor, + max_seqlen: Optional[int] = None, # Only used for Flash Attention + seqlens: Optional[list[int]] = None, # Only used for xFormers + ) -> torch.Tensor: + # [s, b, c] --> [s, b, head * 3 * head_dim] + x, _ = self.qkv(x) + + # [s, b, 3 * head * head_dim] -> 3 * [s, b, head, head_dim] + q, k, v = self.split_qkv(x) + batch_size = q.shape[1] + + q, k, v = (rearrange(x, "s b ... -> b s ...").contiguous() + for x in (q, k, v)) + if rotary_pos_emb is not None: + q = apply_rotary_pos_emb_vision(q, rotary_pos_emb) + k = apply_rotary_pos_emb_vision(k, rotary_pos_emb) + + if self.attn_backend == _Backend.FLASH_ATTN: + # from vllm_flash_attn.flash_attn_interface import ( + # flash_attn_varlen_func) + from flash_attn import flash_attn_varlen_func + + q, k, v = (rearrange(x, "b s ... -> (b s) ...") for x in [q, k, v]) + + output = flash_attn_varlen_func( + q, + k, + v, + cu_seqlens_q=cu_seqlens, + cu_seqlens_k=cu_seqlens, + max_seqlen_q=max_seqlen, + max_seqlen_k=max_seqlen, + dropout_p=0, + causal=False, + ) + + context_layer = rearrange(output, + "(b s) ... -> b s ...", + b=batch_size) + elif self.attn_backend == _Backend.TORCH_SDPA: + # Execute attention entry by entry for speed & less VRAM. + outputs = [] + for i in range(1, len(cu_seqlens)): + start_idx = cu_seqlens[i - 1] + end_idx = cu_seqlens[i] + q_i = q[:, start_idx:end_idx] + k_i = k[:, start_idx:end_idx] + v_i = v[:, start_idx:end_idx] + q_i, k_i, v_i = (rearrange(x, "b s h d -> b h s d") + for x in [q_i, k_i, v_i]) + output_i = F.scaled_dot_product_attention(q_i, + k_i, + v_i, + dropout_p=0.0) + output_i = rearrange(output_i, "b h s d -> b s h d ") + outputs.append(output_i) + context_layer = torch.cat(outputs, dim=1) + elif self.attn_backend == _Backend.XFORMERS: + from xformers import ops as xops + from xformers.ops.fmha.attn_bias import BlockDiagonalMask + + attn_bias = BlockDiagonalMask.from_seqlens(q_seqlen=seqlens, + kv_seqlen=None, + device=q.device) + + context_layer = xops.memory_efficient_attention_forward( + q, k, v, attn_bias=attn_bias, p=0, scale=None) + + context_layer = rearrange(context_layer, + "b s h d -> s b (h d)").contiguous() + + output, _ = self.proj(context_layer) + return output + + +class Glm4vVisionBlock(nn.Module): + + def __init__( + self, + dim: int, + num_heads: int, + mlp_hidden_dim: int, + norm_layer: Optional[Callable[[int], nn.Module]] = None, + quant_config: Optional[QuantizationConfig] = None, + prefix: str = "", + ) -> None: + super().__init__() + if norm_layer is None: + norm_layer = partial(nn.LayerNorm, eps=1e-6) + self.norm1 = norm_layer(dim) + self.norm2 = norm_layer(dim) + self.attn = Glm4vVisionAttention( + embed_dim=dim, + num_heads=num_heads, + projection_size=dim, + quant_config=quant_config, + prefix=f"{prefix}.attn", + ) + self.mlp = Glm4vVisionMLP( + dim, + mlp_hidden_dim, + bias=False, + quant_config=quant_config, + prefix=f"{prefix}.mlp", + ) + + def forward( + self, + x: torch.Tensor, + cu_seqlens: torch.Tensor, + rotary_pos_emb: torch.Tensor, + max_seqlen: Optional[int] = None, # Only used for Flash Attention + seqlens: Optional[list[int]] = None, # Only used for xFormers + ) -> torch.Tensor: + x = x + self.attn( + self.norm1(x), + cu_seqlens=cu_seqlens, + rotary_pos_emb=rotary_pos_emb, + max_seqlen=max_seqlen, + seqlens=seqlens, + ) + + x = x + self.mlp(self.norm2(x)) + return x + + +class Glm4vVisionPatchEmbed(nn.Module): + + def __init__( + self, + patch_size: int = 14, + temporal_patch_size: int = 1, + in_channels: int = 3, + hidden_size: int = 1536, + ) -> None: + super().__init__() + self.patch_size = patch_size + self.temporal_patch_size = temporal_patch_size + self.hidden_size = hidden_size + + kernel_size = (temporal_patch_size, patch_size, patch_size) + self.proj = nn.Conv3d( + in_channels, + hidden_size, + kernel_size=kernel_size, + stride=kernel_size, + bias=True, + ) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + L, C = x.shape + x = x.view(L, -1, self.temporal_patch_size, self.patch_size, + self.patch_size) + x = self.proj(x).view(L, self.hidden_size) + return x + + +class Glm4vPatchMerger(nn.Module): + + def __init__( + self, + d_model: int, + context_dim: int, + quant_config: Optional[QuantizationConfig] = None, + bias: bool = False, + prefix: str = "", + ) -> None: + super().__init__() + self.hidden_size = d_model + self.proj = ColumnParallelLinear(self.hidden_size, + self.hidden_size, + bias=bias, + gather_output=True, + quant_config=quant_config, + prefix=f"{prefix}.proj") + self.post_projection_norm = nn.LayerNorm(self.hidden_size) + self.gate_up_proj = MergedColumnParallelLinear( + input_size=self.hidden_size, + output_sizes=[context_dim] * 2, + bias=bias, + quant_config=quant_config, + prefix=f"{prefix}.gate_up_proj", + ) + self.down_proj = RowParallelLinear( + context_dim, + self.hidden_size, + bias=bias, + quant_config=quant_config, + prefix=f"{prefix}.down_proj", + ) + self.act_fn = SiluAndMul() + self.extra_activation_func = nn.GELU() + + def forward(self, x: torch.Tensor): + x, _ = self.proj(x) + x = self.extra_activation_func(self.post_projection_norm(x)) + gate_up, _ = self.gate_up_proj(x) + x = self.act_fn(gate_up) + x, _ = self.down_proj(x) + return x + + +class Glm4vVisionEmbeddings(nn.Module): + + def __init__(self, config: Glm4vVisionConfig): + super().__init__() + self.config = config + self.embed_dim = config.hidden_size + self.image_size = config.image_size + self.patch_size = config.patch_size + + self.num_patches = (self.image_size // self.patch_size)**2 + self.num_positions = self.num_patches + self.position_embedding = nn.Embedding(self.num_positions, + self.embed_dim) + self.register_buffer( + "position_ids", + torch.arange(self.num_positions).expand((1, -1)), + persistent=False, + ) + + def forward(self, embeddings, lengths, image_shapes, h_coords, + w_coords) -> torch.Tensor: + pos_embed_weight = self.position_embedding.weight + hidden_size = pos_embed_weight.shape[1] + total_seq = h_coords.shape[0] + device = pos_embed_weight.device + + # Move coordinates to correct device + h_coords, w_coords = h_coords.to(device), w_coords.to(device) + + # Handle empty sequence case + if total_seq == 0: + adapted_pos_embed = torch.empty(0, + hidden_size, + device=device, + dtype=pos_embed_weight.dtype) + else: + # Convert inputs to tensors if needed + if isinstance(lengths, list): + lengths = torch.tensor(lengths, + device=device, + dtype=torch.long) + if not isinstance(image_shapes, torch.Tensor): + image_shapes = torch.tensor(image_shapes, + device=device, + dtype=torch.long) + + # Prepare 2D position embedding + orig_size_sq = pos_embed_weight.shape[0] + orig_size = int(orig_size_sq**0.5) + pos_embed_2d = (pos_embed_weight.view( + orig_size, orig_size, + hidden_size).permute(2, 0, + 1).unsqueeze(0).to(device=device, + dtype=torch.float32)) + + # Calculate target dimensions for each patch + target_h = torch.cat([ + image_shapes[i, 1].repeat(lengths[i]) + for i in range(len(lengths)) + ]).to(device=device, dtype=torch.float32) + target_w = torch.cat([ + image_shapes[i, 2].repeat(lengths[i]) + for i in range(len(lengths)) + ]).to(device=device, dtype=torch.float32) + + # Normalize coordinates to [-1, 1] range for grid_sample + h_coords = h_coords.to(device=device, dtype=torch.float32) + w_coords = w_coords.to(device=device, dtype=torch.float32) + norm_w = ((w_coords + 0.5) / target_w) * 2 - 1 + norm_h = ((h_coords + 0.5) / target_h) * 2 - 1 + + # Create sampling grid + grid = (torch.stack((norm_w, norm_h), + dim=-1).unsqueeze(0).unsqueeze(2)) + + # Perform bicubic interpolation + interpolated_embed_fp32 = grid_sample( + pos_embed_2d, + grid, + mode="bicubic", + align_corners=False, + padding_mode="border", + ) + + # Reshape and convert back to original dtype + adapted_pos_embed_fp32 = ( + interpolated_embed_fp32.squeeze(0).squeeze(-1).permute(1, 0)) + adapted_pos_embed = adapted_pos_embed_fp32.to( + pos_embed_weight.dtype).to(embeddings.device) + + # Add adapted position encoding to embeddings + embeddings = embeddings + adapted_pos_embed + return embeddings + + +class Glm4vVisionRotaryEmbedding(nn.Module): + + def __init__(self, dim: int, theta: float = 10000.0) -> None: + super().__init__() + self.dim = dim + self.theta = theta + inv_freq = 1.0 / (theta + **(torch.arange(0, dim, 2, dtype=torch.float) / dim)) + self.register_buffer("inv_freq", inv_freq, persistent=False) + self._seq_len_cached = 0 + self._freqs_cached = None + + def update_freqs_cache(self, seqlen: int) -> None: + if seqlen > self._seq_len_cached: + seqlen *= 2 + self._seq_len_cached = seqlen + self.inv_freq = 1.0 / (self.theta**(torch.arange( + 0, + self.dim, + 2, + dtype=torch.float, + device=self.inv_freq.device, + ) / self.dim)) + seq = torch.arange(seqlen, + device=self.inv_freq.device, + dtype=self.inv_freq.dtype) + freqs = torch.outer(seq, self.inv_freq) + self._freqs_cached = freqs + + def forward(self, seqlen: int) -> torch.Tensor: + self.update_freqs_cache(seqlen) + return self._freqs_cached[:seqlen] + + +class Glm4vVisionTransformer(nn.Module): + + def __init__( + self, + vision_config: Glm4vVisionConfig, + norm_eps: float = 1e-6, + quant_config: Optional[QuantizationConfig] = None, + prefix: str = "", + ) -> None: + super().__init__() + + patch_size = vision_config.patch_size + temporal_patch_size = vision_config.temporal_patch_size + in_channels = vision_config.in_channels + depth = vision_config.depth + self.hidden_size = vision_config.hidden_size + self.num_heads = vision_config.num_heads + + self.patch_size = vision_config.patch_size + self.spatial_merge_size = vision_config.spatial_merge_size + self.out_hidden_size = vision_config.out_hidden_size + + self.patch_embed = Glm4vVisionPatchEmbed( + patch_size=patch_size, + temporal_patch_size=temporal_patch_size, + in_channels=in_channels, + hidden_size=self.hidden_size, + ) + + norm_layer = partial(RMSNorm, eps=norm_eps) + head_dim = self.hidden_size // self.num_heads + self.rotary_pos_emb = Glm4vVisionRotaryEmbedding(head_dim // 2) + self.blocks = nn.ModuleList([ + Glm4vVisionBlock( + dim=self.hidden_size, + num_heads=self.num_heads, + mlp_hidden_dim=vision_config.out_hidden_size, + norm_layer=norm_layer, + quant_config=quant_config, + prefix=f"{prefix}.blocks.{layer_idx}", + ) for layer_idx in range(depth) + ]) + self.merger = Glm4vPatchMerger( + d_model=vision_config.out_hidden_size, + context_dim=vision_config.intermediate_size, + quant_config=quant_config, + bias=False, + prefix=f"{prefix}.merger", + ) + self.embeddings = Glm4vVisionEmbeddings(vision_config) + + self.post_conv_layernorm = RMSNorm(vision_config.hidden_size, + eps=vision_config.rms_norm_eps) + self.downsample = nn.Conv2d( + in_channels=vision_config.hidden_size, + out_channels=vision_config.out_hidden_size, + kernel_size=vision_config.spatial_merge_size, + stride=vision_config.spatial_merge_size, + ) + self.post_layernorm = RMSNorm(vision_config.hidden_size, + eps=vision_config.rms_norm_eps) + + self.attn_backend: _Backend = get_vit_attn_backend(support_fa=True) + + @property + def dtype(self) -> torch.dtype: + return self.patch_embed.proj.weight.dtype + + @property + def device(self) -> torch.device: + return self.patch_embed.proj.weight.device + + def rot_pos_emb(self, grid_thw: torch.Tensor) -> torch.Tensor: + pos_ids = [] + for t, h, w in grid_thw: + hpos_ids = torch.arange(h).unsqueeze(1).expand(-1, w) + wpos_ids = torch.arange(w).unsqueeze(0).expand(h, -1) + hpos_ids = (hpos_ids.reshape( + h // self.spatial_merge_size, + self.spatial_merge_size, + w // self.spatial_merge_size, + self.spatial_merge_size, + ).permute(0, 2, 1, 3).flatten()) + wpos_ids = (wpos_ids.reshape( + h // self.spatial_merge_size, + self.spatial_merge_size, + w // self.spatial_merge_size, + self.spatial_merge_size, + ).permute(0, 2, 1, 3).flatten()) + pos_ids.append( + torch.stack([hpos_ids, wpos_ids], dim=-1).repeat(t, 1)) + pos_ids = torch.cat(pos_ids, dim=0) + max_grid_size = grid_thw[:, 1:].max() + rotary_pos_emb_full = self.rotary_pos_emb(max_grid_size) + rotary_pos_emb = rotary_pos_emb_full[pos_ids].flatten(1) + return rotary_pos_emb, pos_ids + + def compute_attn_mask_seqlen( + self, + cu_seqlens: torch.Tensor, + ) -> tuple[Optional[int], Optional[list[int]]]: + max_seqlen, seqlens = None, None + seqlens = (cu_seqlens[1:] - cu_seqlens[:-1]).tolist() + if self.attn_backend == _Backend.FLASH_ATTN: + max_seqlen = (cu_seqlens[1:] - cu_seqlens[:-1]).max().item() + return max_seqlen, seqlens + + def forward( + self, + x: torch.Tensor, + grid_thw: torch.Tensor, + ) -> torch.Tensor: + # patchify + x = x.to(device=self.device, dtype=self.dtype) + x = self.patch_embed(x) + x = self.post_conv_layernorm(x) + + # compute position embedding + rotary_pos_emb, image_type_ids = self.rot_pos_emb(grid_thw) + # compute cu_seqlens + cu_seqlens = torch.repeat_interleave(grid_thw[:, 1] * grid_thw[:, 2], + grid_thw[:, 0]).cumsum( + dim=0, dtype=torch.int32) + cu_seqlens = F.pad(cu_seqlens, (1, 0), "constant", 0) + + # pre-compute seqlens for attn mask to reduce cuMemcpy operations + max_seqlen, seqlens = self.compute_attn_mask_seqlen(cu_seqlens) + x = self.embeddings(x, seqlens, grid_thw, image_type_ids[:, 0], + image_type_ids[:, 1]) + + # transformers + x = x.unsqueeze(1) + for blk in self.blocks: + x = blk( + x, + cu_seqlens=cu_seqlens, + rotary_pos_emb=rotary_pos_emb, + max_seqlen=max_seqlen, + seqlens=seqlens, + ) + + # adapter + x = self.post_layernorm(x) + + x = x.view(-1, self.spatial_merge_size, self.spatial_merge_size, + x.shape[-1]) + x = x.permute(0, 3, 1, 2) + x = self.downsample(x).view(-1, self.out_hidden_size) + x = self.merger(x) + + return x + + def load_weights(self, weights: Iterable[tuple[str, + torch.Tensor]]) -> set[str]: + stacked_params_mapping = [ + # (param_name, shard_name, shard_id) + ("attn.qkv.", "attn.q.", "q"), + ("attn.qkv.", "attn.k.", "k"), + ("attn.qkv.", "attn.v.", "v"), + ("gate_up_proj", "gate_proj", 0), + ("gate_up_proj", "up_proj", 1), + ] + params_dict = dict(self.named_parameters(remove_duplicate=False)) + loaded_params: set[str] = set() + + for name, loaded_weight in weights: + for param_name, weight_name, shard_id in stacked_params_mapping: + if weight_name not in name: + continue + name = name.replace(weight_name, param_name) + + param = params_dict[name] + weight_loader = param.weight_loader + weight_loader(param, loaded_weight, shard_id) + break + else: + param = params_dict[name] + weight_loader = getattr(param, "weight_loader", + default_weight_loader) + weight_loader(param, loaded_weight) + loaded_params.add(name) + return loaded_params + + +class Glm4vProcessingInfo(BaseProcessingInfo): + + def get_hf_config(self): + return self.ctx.get_hf_config() + + def get_tokenizer(self): + return self.ctx.tokenizer + + def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]: + return {"image": None, "video": 1} + + def get_image_processor(self, **kwargs: object) -> Glm4vImageProcessor: + return self.get_hf_processor(**kwargs).image_processor + + def get_video_processor(self, **kwargs: object) -> Glm4vVideoProcessor: + return self.get_hf_processor(**kwargs).video_processor + + def _get_vision_info( + self, + *, + image_width: int, + image_height: int, + num_frames: int = 16, + do_resize: bool = True, + max_image_pixels: int = 28 * 28 * 2 * 30000, + ) -> tuple[ImageSize, int]: + hf_config = self.get_hf_config() + vision_config = hf_config.vision_config + patch_size = vision_config.patch_size + merge_size = vision_config.spatial_merge_size + temporal_patch_size = vision_config.temporal_patch_size + if do_resize: + resized_height, resized_width = smart_resize( + num_frames=num_frames + if num_frames > temporal_patch_size else temporal_patch_size, + height=image_height, + width=image_width, + factor=patch_size * merge_size, + max_pixels=max_image_pixels, + ) + preprocessed_size = ImageSize(width=resized_width, + height=resized_height) + else: + preprocessed_size = ImageSize(width=image_width, + height=image_height) + + # NOTE: Frames are padded to be divisible by `temporal_patch_size` + # https://github.com/huggingface/transformers/blob/v4.48.3/src/transformers/models/qwen2_vl/image_processing_qwen2_vl.py#L294 + padded_num_frames = num_frames + num_frames % temporal_patch_size + + grid_t = max(padded_num_frames // temporal_patch_size, 1) + grid_h = preprocessed_size.height // patch_size + grid_w = preprocessed_size.width // patch_size + + num_patches = grid_t * grid_h * grid_w + num_vision_tokens = num_patches // (merge_size**2) + + return preprocessed_size, num_vision_tokens + + def get_image_size_with_most_features(self) -> ImageSize: + max_image_size, _ = self._get_vision_info(image_width=9999999, + image_height=9999999) + return max_image_size + + def get_num_image_tokens( + self, + *, + image_width: int, + image_height: int, + ) -> int: + _, num_image_tokens = self._get_vision_info( + image_width=image_width, + image_height=image_height, + max_image_pixels=28 * 28 * 2 * 6144, + ) + return num_image_tokens + + def get_max_image_tokens(self) -> int: + target_width, target_height = self.get_image_size_with_most_features() + + return self.get_num_image_tokens( + image_width=target_width, + image_height=target_height, + ) + + def get_num_video_tokens( + self, + *, + image_width: int, + image_height: int, + num_frames: int, + ) -> int: + _, num_video_tokens = self._get_vision_info( + image_width=image_width, + image_height=image_height, + num_frames=num_frames, + max_image_pixels=28 * 28 * 2 * 30000, + ) + return num_video_tokens + + def _get_max_video_frames(self, max_tokens: int) -> int: + target_width, target_height = self.get_image_size_with_most_features() + + num_frames = 0 + + while True: + next_num_frames = num_frames + 1 + next_max_tokens = self.get_num_video_tokens( + image_width=target_width, + image_height=target_height, + num_frames=next_num_frames, + ) + if next_max_tokens > max_tokens or next_max_tokens == 0: + break + + num_frames = next_num_frames + + return num_frames + + def get_num_frames_with_most_features( + self, + seq_len: int, + mm_counts: Mapping[str, int], + ) -> int: + max_images = mm_counts.get("image", 0) + max_videos = mm_counts.get("video", 0) + + max_image_tokens = self.get_max_image_tokens() * max_images + max_total_frames = self._get_max_video_frames(seq_len - + max_image_tokens) + max_frames_per_video = min(max_total_frames // max(max_videos, 1), + _MAX_FRAMES_PER_VIDEO) + + return max(max_frames_per_video, 1) + + def _get_video_second_idx(self, metadata: dict[str, Any], + total_frames: int) -> list[int]: + video_processor = self.get_video_processor() + + video_fps = metadata.get("fps", video_processor.fps) + meta_frames = metadata.get("total_num_frames", total_frames) + max_frame_idx = meta_frames - 1 + duration = metadata.get("duration", + round(max_frame_idx / video_fps) + 1) + if duration <= video_processor.max_duration: + n = int(math.floor(duration * video_processor.fps)) + frame_indices = [ + min( + max_frame_idx, + int(math.ceil(i * video_fps / video_processor.fps)), + ) for i in range(n) + ] + else: + num_samples = int(video_processor.max_duration * + video_processor.fps) + if num_samples >= meta_frames: + frame_indices = list(range(meta_frames)) + else: + target_seconds = np.linspace(0, + duration, + num_samples, + endpoint=True) + frame_indices = [ + min(max_frame_idx, int(math.ceil(t * video_fps))) + for t in target_seconds + ] + + seen, uniq = set(), [] + for idx in frame_indices: + if idx not in seen: + seen.add(idx) + uniq.append(idx) + if len(uniq) & 1: + uniq.append(uniq[-1]) + frame_indices = uniq + + full_second_idxs = [int(idx / video_fps) for idx in frame_indices] + timestamps_list = full_second_idxs[::2] + selected_timestamps = [] + for idx in range(0, len(timestamps_list)): + selected_timestamps.append(timestamps_list[idx]) + return selected_timestamps + + +class Glm4vDummyInputsBuilder(BaseDummyInputsBuilder[Glm4vProcessingInfo]): + + def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str: + num_images = mm_counts.get("image", 0) + num_videos = mm_counts.get("video", 0) + + hf_config = self.info.get_hf_config() + hf_processor = self.info.get_hf_processor() + tokenizer = self.info.get_tokenizer() + + image_token: str = hf_processor.image_token + video_token_ids = [ + hf_config.video_start_token_id, + hf_processor.video_token_id, + hf_config.video_end_token_id, + ] + video_token = tokenizer.decode(video_token_ids) + + return image_token * num_images + video_token * num_videos + + def get_dummy_mm_data( + self, + seq_len: int, + mm_counts: Mapping[str, int], + ) -> MultiModalDataDict: + num_images = mm_counts.get("image", 0) + num_videos = mm_counts.get("video", 0) + + target_width, target_height = ( + self.info.get_image_size_with_most_features()) + target_num_frames = self.info.get_num_frames_with_most_features( + seq_len, mm_counts) + return { + "image": + self._get_dummy_images(width=target_width, + height=target_height, + num_images=num_images), + "video": + self._get_dummy_videos( + width=target_width, + height=target_height, + num_frames=target_num_frames, + num_videos=num_videos, + ), + } + + def _get_dummy_videos( + self, + *, + width: int, + height: int, + num_frames: int, + num_videos: int, + ) -> list[VideoItem]: + video = np.full((num_frames, width, height, 3), 255, dtype=np.uint8) + video_items = [] + for i in range(num_videos): + video_metadata = { + "fps": 2.0, + "duration": num_frames / 2.0, + "total_num_frames": num_frames, + "video_backend": "opencv", + } + video_item = (video.copy(), video_metadata) + video_items.append(video_item) + + return video_items + + +class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]): + + def _get_data_parser(self) -> MultiModalDataParser: + return MultiModalDataParser(video_needs_metadata=True) + + def _call_hf_processor( + self, + prompt: str, + mm_data: Mapping[str, object], + mm_kwargs: Mapping[str, object], + tok_kwargs: Mapping[str, object], + ) -> BatchFeature: + mm_data = dict(mm_data) + processor = self.info.get_hf_processor(**mm_kwargs) + + # GLM-4.1V use `image_token_id` as video placeholder, we need to + # replace it with `video_token_id` for video processing. So we + # separate video processing from image processing. + if ("videos" in mm_data and isinstance(mm_data["videos"], list) + and len(mm_data["videos"]) > 0): + video_grid_thw_lst = [] + pixel_values_videos_lst = [] + for item in mm_data.pop("videos", []): + video_array, metadata = item + + # FIXME(Isotr0py): Activate the below logic after we can disable + # resampling from video loader backend. + # assert metadata["total_num_frames"] == len(video_array), ( + # f"Total frames {metadata['total_num_frames']} does not " + # f"match the length of video array {len(video_array)}.") + + # NOTE: Temporary workaround for resampled videos. + # this can cause a divergence with HF implementation if + # the input video is resampled in advance. + + if metadata["total_num_frames"] != len(video_array): + logger.warning( + "Total frames in metadata " + "(%s) does not match the length of " + "video array %s. This can " + "be because the video is resampled " + "in advance. This may cause " + "a divergence with HF implementation.", + metadata["total_num_frames"], + len(video_array), + ) + metadata["total_num_frames"] = len(video_array) + metadata = VideoMetadata(**metadata) + + video_mm_data = dict() + video_mm_data["videos"] = [[video_array]] + video_mm_data["video_metadata"] = [[metadata]] + + video_outputs = super()._call_hf_processor( + prompt="<|begin_of_video|><|video|><|end_of_video|>", + mm_data=video_mm_data, + mm_kwargs=mm_kwargs, + tok_kwargs=tok_kwargs, + ) + input_ids = video_outputs.pop("input_ids") + input_ids[input_ids == processor.image_token_id] = ( + processor.video_token_id) + video_placeholder = processor.tokenizer.batch_decode( + input_ids)[0] + prompt = prompt.replace( + "<|begin_of_video|><|video|><|end_of_video|>", + video_placeholder, + ) + + video_grid_thw_lst.append(video_outputs["video_grid_thw"]) + pixel_values_videos_lst.append( + video_outputs["pixel_values_videos"]) + video_outputs = dict( + pixel_values_videos=torch.cat(pixel_values_videos_lst), + video_grid_thw=torch.cat(video_grid_thw_lst), + ) + else: + video_outputs = dict() + + processed_outputs = super()._call_hf_processor( + prompt=prompt, + mm_data=mm_data, + mm_kwargs=mm_kwargs, + tok_kwargs=tok_kwargs, + ) + combined_outputs = dict( + processed_outputs, + **video_outputs, + ) + return BatchFeature(combined_outputs) + + def _get_mm_fields_config( + self, + hf_inputs: BatchFeature, + hf_processor_mm_kwargs: Mapping[str, object], + ) -> Mapping[str, MultiModalFieldConfig]: + return _qwen2vl_field_config(hf_inputs) + + def _get_prompt_updates( + self, + mm_items: MultiModalDataItems, + hf_processor_mm_kwargs: Mapping[str, Any], + out_mm_kwargs: MultiModalKwargs, + ) -> Sequence[PromptUpdate]: + hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs) + image_processor = self.info.get_image_processor( + **hf_processor_mm_kwargs) + tokenizer = self.info.get_tokenizer() + hf_config = self.info.get_hf_config() + + boi_token_id = hf_config.image_start_token_id + eoi_token_id = hf_config.image_end_token_id + + bov_token_id = hf_config.video_start_token_id + eov_token_id = hf_config.video_end_token_id + + merge_length = image_processor.merge_size**2 + + def get_image_replacement_glm4v(item_idx: int): + grid_thw = out_mm_kwargs["image_grid_thw"][item_idx] + assert isinstance(grid_thw, torch.Tensor) + + num_tokens = int(grid_thw.prod()) // merge_length + return [hf_processor.image_token_id] * num_tokens + + def get_video_replacement_glm4v(item_idx: int): + grid_thw = out_mm_kwargs["video_grid_thw"][item_idx] + assert isinstance(grid_thw, torch.Tensor) + + video, metadata = mm_items["video"][item_idx] + timestamps = self.info._get_video_second_idx(metadata, len(video)) + frames_idx_token = [ + tokenizer.encode(str(i), add_special_tokens=False) + for i in timestamps + ] + num_tokens_per_frame = int(grid_thw[1:].prod()) // merge_length + placeholder = [] + placeholder.append(bov_token_id) + for frame_idx in frames_idx_token: + placeholder.append(boi_token_id) + placeholder.extend([hf_processor.video_token_id] * + num_tokens_per_frame) + placeholder.append(eoi_token_id) + placeholder.extend(frame_idx) + placeholder.append(eov_token_id) + return PromptUpdateDetails.select_token_id( + placeholder, + embed_token_id=hf_processor.video_token_id, + ) + + return [ + PromptReplacement( + modality="image", + target=hf_processor.image_token, + replacement=get_image_replacement_glm4v, + ), + PromptReplacement( + modality="video", + target="<|begin_of_video|><|video|><|end_of_video|>", + replacement=get_video_replacement_glm4v, + ), + ] + + +@MULTIMODAL_REGISTRY.register_processor( + Glm4vMultiModalProcessor, + info=Glm4vProcessingInfo, + dummy_inputs=Glm4vDummyInputsBuilder, +) +class Glm4vForConditionalGeneration(nn.Module, SupportsMultiModal, + SupportsLoRA, SupportsPP): + packed_modules_mapping = { + "qkv_proj": [ + "q_proj", + "k_proj", + "v_proj", + ], + "gate_up_proj": ["gate_up_proj"] + } + + # To ensure correct weight loading and mapping. + hf_to_vllm_mapper = WeightsMapper( + orig_to_new_prefix={ + "lm_head.": "language_model.lm_head.", + "model.language_model.": "language_model.model.", + "model.visual.": "visual.", + }) + + @classmethod + def get_placeholder_str(cls, modality: str, i: int) -> Optional[str]: + if modality.startswith("image"): + return "<|begin_of_image|><|image|><|end_of_image|>" + if modality.startswith("video"): + return "<|begin_of_video|><|video|><|end_of_video|>" + + raise ValueError("Only image or video modality is supported") + + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): + super().__init__() + config = vllm_config.model_config.hf_config + quant_config = vllm_config.quant_config + multimodal_config = vllm_config.model_config.multimodal_config + + self.config = config + self.multimodal_config = multimodal_config + + self.visual = Glm4vVisionTransformer( + config.vision_config, + norm_eps=getattr(config, "rms_norm_eps", 1e-5), + quant_config=quant_config, + prefix=maybe_prefix(prefix, "visual"), + ) + + if config.model_type == "glm4v": + architectures = ["Glm4ForCausalLM"] + elif config.model_type == "glm4v_moe": + architectures = ["Glm4MoeForCausalLM"] + else: + architectures = None + + self.language_model = init_vllm_registered_model( + vllm_config=vllm_config, + hf_config=config.text_config, + prefix=maybe_prefix(prefix, "language_model"), + architectures=architectures) + + self.make_empty_intermediate_tensors = ( + self.language_model.make_empty_intermediate_tensors) + + def _validate_and_reshape_mm_tensor(self, mm_input: object, + name: str) -> torch.Tensor: + if not isinstance(mm_input, (torch.Tensor, list)): + raise ValueError( + f"Incorrect type of {name}. Got type: {type(mm_input)}") + if isinstance(mm_input, torch.Tensor): + if mm_input.ndim == 2: + return mm_input + if mm_input.ndim != 3: + raise ValueError(f"{name} should be 2D or batched 3D tensor. " + f"Got ndim: {mm_input.ndim} " + f"(shape={mm_input.shape})") + return torch.concat(list(mm_input)) + else: + return torch.concat(mm_input) + + def _parse_and_validate_image_input( + self, **kwargs: object) -> Optional[Glm4vImageInputs]: + pixel_values = kwargs.pop("pixel_values", None) + image_embeds = kwargs.pop("image_embeds", None) + image_grid_thw = kwargs.pop("image_grid_thw", None) + + if pixel_values is None and image_embeds is None: + return None + + if pixel_values is not None: + pixel_values = self._validate_and_reshape_mm_tensor( + pixel_values, "image pixel values") + image_grid_thw = self._validate_and_reshape_mm_tensor( + image_grid_thw, "image grid_thw") + + return Glm4vImagePixelInputs( + type="pixel_values", + pixel_values=pixel_values, + image_grid_thw=image_grid_thw, + ) + + if image_embeds is not None: + image_embeds = self._validate_and_reshape_mm_tensor( + image_embeds, "image embeds") + image_grid_thw = self._validate_and_reshape_mm_tensor( + image_grid_thw, "image grid_thw") + + return Glm4vImageEmbeddingInputs( + type="image_embeds", + image_embeds=image_embeds, + image_grid_thw=image_grid_thw, + ) + + def _parse_and_validate_video_input( + self, **kwargs: object) -> Optional[Glm4vVideoInputs]: + pixel_values_videos = kwargs.pop("pixel_values_videos", None) + video_embeds = kwargs.pop("video_embeds", None) + video_grid_thw = kwargs.pop("video_grid_thw", None) + + if pixel_values_videos is None and video_embeds is None: + return None + + if pixel_values_videos is not None: + pixel_values_videos = self._validate_and_reshape_mm_tensor( + pixel_values_videos, "video pixel values") + video_grid_thw = self._validate_and_reshape_mm_tensor( + video_grid_thw, "video grid_thw") + + return Glm4vVideoPixelInputs( + type="pixel_values_videos", + pixel_values_videos=pixel_values_videos, + video_grid_thw=video_grid_thw, + ) + + if video_embeds is not None: + video_embeds = self._validate_and_reshape_mm_tensor( + video_embeds, "video embeds") + video_grid_thw = self._validate_and_reshape_mm_tensor( + video_grid_thw, "video grid_thw") + + return Glm4vVideoEmbeddingInputs( + type="video_embeds", + video_embeds=video_embeds, + video_grid_thw=video_grid_thw, + ) + + def _process_image_input( + self, image_input: Glm4vImageInputs) -> tuple[torch.Tensor, ...]: + grid_thw = image_input["image_grid_thw"] + assert grid_thw.ndim == 2 + + if image_input["type"] == "image_embeds": + image_embeds = image_input["image_embeds"].type(self.visual.dtype) + else: + pixel_values = image_input["pixel_values"].type(self.visual.dtype) + image_embeds = self.visual(pixel_values, grid_thw=grid_thw) + + merge_size = self.visual.spatial_merge_size + sizes = grid_thw.prod(-1) // merge_size // merge_size + return image_embeds.split(sizes.tolist()) + + def _process_video_input( + self, video_input: Glm4vVideoInputs) -> tuple[torch.Tensor, ...]: + grid_thw = video_input["video_grid_thw"] + assert grid_thw.ndim == 2 + + device = self.visual.device + flat_grid_thw = torch.cat([ + torch.tensor([[1, h, w]] * t, device=device) + for t, h, w in grid_thw + ]) + if video_input["type"] == "video_embeds": + video_embeds = video_input["video_embeds"].type(self.visual.dtype) + else: + pixel_values_videos = video_input["pixel_values_videos"].type( + self.visual.dtype) + video_embeds = self.visual(pixel_values_videos, + grid_thw=flat_grid_thw) + + # Split concatenated embeddings for each video item. + merge_size = self.visual.spatial_merge_size + sizes = grid_thw.prod(-1) // merge_size // merge_size + + return video_embeds.split(sizes.tolist()) + + def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict: + mm_input_by_modality = {} + + # Preserve the order of modalities if there are multiple of them + # from the order of kwargs. + for input_key in kwargs: + if (input_key in ("pixel_values", "image_embeds") + and "image" not in mm_input_by_modality): + mm_input_by_modality["image"] = ( + self._parse_and_validate_image_input(**kwargs)) + if (input_key in ("pixel_values_videos", "video_embeds") + and "video" not in mm_input_by_modality): + mm_input_by_modality["video"] = ( + self._parse_and_validate_video_input(**kwargs)) + return mm_input_by_modality + + def get_language_model(self) -> torch.nn.Module: + return self.language_model + + def get_multimodal_embeddings( + self, **kwargs: object) -> Optional[MultiModalEmbeddings]: + mm_input_by_modality = self._parse_and_validate_multimodal_inputs( + **kwargs) + if not mm_input_by_modality: + return None + + # The result multimodal_embeddings is tuple of tensors, with each + # tensor correspoending to a multimodal data item (image or video). + multimodal_embeddings: tuple[torch.Tensor, ...] = () + + # NOTE: It is important to iterate over the keys in this dictionary + # to preserve the order of the modalities. + for modality in mm_input_by_modality: + multimodal_input = mm_input_by_modality[modality] + if modality == "image": + vision_embeddings = self._process_image_input(multimodal_input) + multimodal_embeddings += vision_embeddings + if modality == "video": + video_embeddings = self._process_video_input(multimodal_input) + multimodal_embeddings += video_embeddings + return multimodal_embeddings + + def get_input_embeddings( + self, + input_ids: torch.Tensor, + multimodal_embeddings: Optional[MultiModalEmbeddings] = None, + ) -> torch.Tensor: + inputs_embeds = self.language_model.get_input_embeddings(input_ids) + if (multimodal_embeddings is not None + and len(multimodal_embeddings) != 0 + and all(embed.numel() > 0 for embed in multimodal_embeddings)): + inputs_embeds = merge_multimodal_embeddings( + input_ids, + inputs_embeds, + multimodal_embeddings, + [self.config.image_token_id, self.config.video_token_id], + ) + return inputs_embeds + + def get_input_embeddings_v0( + self, + input_ids: torch.Tensor, + image_input: Optional[Glm4vImageInputs] = None, + video_input: Optional[Glm4vVideoInputs] = None, + ) -> torch.Tensor: + inputs_embeds = self.get_input_embeddings(input_ids) + if image_input is not None: + image_embeds = self._process_image_input(image_input) + inputs_embeds = merge_multimodal_embeddings( + input_ids, + inputs_embeds, + image_embeds, + placeholder_token_id=self.config.image_token_id, + ) + + if video_input is not None: + video_embeds = self._process_video_input(video_input) + inputs_embeds = merge_multimodal_embeddings( + input_ids, + inputs_embeds, + video_embeds, + placeholder_token_id=self.config.video_token_id, + ) + return inputs_embeds + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + intermediate_tensors: Optional[IntermediateTensors] = None, + inputs_embeds: Optional[torch.Tensor] = None, + **kwargs: object, + ) -> Union[torch.Tensor, IntermediateTensors]: + """Run forward pass for GLM-4V. + + Args: + input_ids: Flattened (concatenated) input_ids corresponding to a + batch. + positions: Flattened (concatenated) position ids corresponding to a + batch. + **NOTE**: If mrope is enabled (default setting for GLM-4V + opensource models), the shape will be `(3, seq_len)`, + otherwise it will be `(seq_len,). + pixel_values: Pixel values to be fed to a model. + `None` if no images are passed. + image_grid_thw: Tensor `(n_images, 3)` of image 3D grid in LLM. + `None` if no images are passed. + pixel_values_videos: Pixel values of videos to be fed to a model. + `None` if no videos are passed. + video_grid_thw: Tensor `(n_videos, 3)` of video 3D grid in LLM. + `None` if no videos are passed. + second_per_grid_ts: Tensor `(num_videos)` of video time interval ( + in seconds) for each grid along the temporal dimension in the + 3D position IDs. `None` if no videos are passed. + """ + if intermediate_tensors is not None: + inputs_embeds = None + + # NOTE: In v1, inputs_embeds is always generated at model runner from + # `get_multimodal_embeddings` and `get_input_embeddings`, this + # condition is only for v0 compatibility. + elif inputs_embeds is None: + image_input = self._parse_and_validate_image_input(**kwargs) + video_input = self._parse_and_validate_video_input(**kwargs) + + if image_input is None and video_input is None: + inputs_embeds = None + else: + if uses_mrope(self.config): + assert positions.ndim == 2 and positions.size(0) == 3, ( + "multimodal section rotary embedding requires " + f"(3, seq_len) positions, but got {positions.size()}") + inputs_embeds = self.get_input_embeddings_v0( + input_ids, + image_input=image_input, + video_input=video_input) + input_ids = None + + hidden_states = self.language_model.model( + input_ids=input_ids, + positions=positions, + intermediate_tensors=intermediate_tensors, + inputs_embeds=inputs_embeds, + ) + return hidden_states + + def compute_logits( + self, + hidden_states: torch.Tensor, + sampling_metadata: SamplingMetadata, + ) -> Optional[torch.Tensor]: + return self.language_model.compute_logits(hidden_states, + sampling_metadata) + + def load_weights(self, weights: Iterable[tuple[str, + torch.Tensor]]) -> set[str]: + loader = AutoWeightsLoader(self) + return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper) + + def get_mm_mapping(self) -> MultiModelKeys: + """ + Get the module prefix in multimodal models + """ + return MultiModelKeys.from_string_field( + language_model="language_model.model", + connector="visual.merger.", + tower_model="visual.", + ) + + +@MULTIMODAL_REGISTRY.register_processor( + Glm4vMultiModalProcessor, + info=Glm4vProcessingInfo, + dummy_inputs=Glm4vDummyInputsBuilder, +) +class Glm4vMoeForConditionalGeneration(Glm4vForConditionalGeneration): + packed_modules_mapping = { + "qkv_proj": [ + "q_proj", + "k_proj", + "v_proj", + ], + "gate_up_proj": [ + "gate_proj", + "up_proj", + ], + } diff --git a/vllm_kunlun/models/glm4_moe.py b/vllm_kunlun/models/glm4_moe.py new file mode 100644 index 0000000..cf658ae --- /dev/null +++ b/vllm_kunlun/models/glm4_moe.py @@ -0,0 +1,716 @@ +# +# Copyright (c) 2025 Baidu, Inc. All Rights Reserved. +# Adapted from vllm/model_executor/models/glm4_moe.py +# Copyright 2023 The vLLM team. +# +# This file is a part of the vllm-kunlun project. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Inference-only GLM-4.5 model compatible with HuggingFace weights.""" +import os +import typing +from collections.abc import Callable, Iterable +from itertools import islice +from typing import Any, Optional, Union + +import torch +from torch import nn +from transformers.models.glm4_moe import Glm4MoeConfig + +from vllm_kunlun.ops.attention.layer import Attention +from vllm.compilation.decorators import support_torch_compile +from vllm.config import CacheConfig, VllmConfig, get_current_vllm_config +from vllm.distributed import (get_ep_group, get_pp_group,get_dp_group,get_tp_group, + get_tensor_model_parallel_world_size) +from vllm.logger import init_logger +from vllm_kunlun.ops.activation import SiluAndMul +from vllm_kunlun.ops.fused_moe.layer import FusedMoE +from vllm.model_executor.layers.layernorm import RMSNorm +from vllm.model_executor.layers.linear import (MergedColumnParallelLinear, + QKVParallelLinear, + RowParallelLinear) +from vllm.model_executor.layers.logits_processor import LogitsProcessor +from vllm.model_executor.layers.quantization import QuantizationConfig +from vllm.model_executor.layers.rotary_embedding import get_rope +from vllm.model_executor.layers.vocab_parallel_embedding import ( + ParallelLMHead, VocabParallelEmbedding) +from vllm.model_executor.model_loader.weight_utils import ( + default_weight_loader, maybe_remap_kv_scale_name) +from vllm.model_executor.sampling_metadata import SamplingMetadata +from vllm.sequence import IntermediateTensors + +from vllm.model_executor.models.interfaces import SupportsLoRA, SupportsPP +from vllm.model_executor.models.utils import (AutoWeightsLoader, PPMissingLayer, is_pp_missing_parameter, + make_empty_intermediate_tensors_factory, make_layers, + maybe_prefix) +from vllm_kunlun.ops.rotary_embedding import Split_Norm_Rope + +logger = init_logger(__name__) + + +class Glm4MoeMLP(nn.Module): + + def __init__( + self, + hidden_size: int, + intermediate_size: int, + hidden_act: str, + quant_config: Optional[QuantizationConfig] = None, + reduce_results: bool = True, + prefix: str = "", + ) -> None: + super().__init__() + self.gate_up_proj = MergedColumnParallelLinear( + hidden_size, [intermediate_size] * 2, + bias=False, + quant_config=quant_config, + prefix=f"{prefix}.gate_up_proj") + self.down_proj = RowParallelLinear(intermediate_size, + hidden_size, + bias=False, + quant_config=quant_config, + reduce_results=reduce_results, + prefix=f"{prefix}.down_proj") + if hidden_act != "silu": + raise ValueError(f"Unsupported activation: {hidden_act}. " + "Only silu is supported for now.") + self.act_fn = SiluAndMul() + + def forward(self, x): + gate_up, _ = self.gate_up_proj(x) + x = self.act_fn(gate_up) + x, _ = self.down_proj(x) + return x + +class Glm4MoE(nn.Module): + + def __init__( + self, + config: Glm4MoeConfig, + quant_config: Optional[QuantizationConfig] = None, + prefix: str = "", + enable_eplb: bool = False, + ): + super().__init__() + self.tp_size = get_tensor_model_parallel_world_size() + self.routed_scaling_factor = config.routed_scaling_factor + + self.ep_group = get_ep_group().device_group + self.ep_rank = self.ep_group.rank() + self.ep_size = self.ep_group.size() + self.n_routed_experts: int = config.n_routed_experts + self.n_shared_experts: int = config.n_shared_experts + + if config.hidden_act != "silu": + raise ValueError(f"Unsupported activation: {config.hidden_act}. " + "Only silu is supported for now.") + # NOTE In the transformers implementation, the gate isn't an nn.Linear, + # so we cannot use ReplicatedLinear here. + # See: https://github.com/huggingface/transformers/blob/v4.55.1/src/transformers/models/glm4_moe/modeling_glm4_moe.py#L260 + self.gate = nn.Linear( + config.hidden_size, + config.n_routed_experts, + bias=False, + dtype=torch.float32, + ) + self.gate.e_score_correction_bias = nn.Parameter( + torch.empty(config.n_routed_experts, dtype=torch.float32)) + + # Load balancing settings. + vllm_config = get_current_vllm_config() + parallel_config = vllm_config.parallel_config + self.enable_eplb = enable_eplb + + self.n_redundant_experts = parallel_config.num_redundant_experts + self.n_logical_experts = self.n_routed_experts + self.n_physical_experts = (self.n_logical_experts + + self.n_redundant_experts) + self.n_local_physical_experts = self.n_physical_experts // self.ep_size + + self.physical_expert_start = (self.ep_rank * + self.n_local_physical_experts) + self.physical_expert_end = (self.physical_expert_start + + self.n_local_physical_experts) + + self.experts = FusedMoE( + num_experts=config.n_routed_experts, + top_k=config.num_experts_per_tok, + hidden_size=config.hidden_size, + intermediate_size=config.moe_intermediate_size, + reduce_results=False, + renormalize=config.norm_topk_prob, + quant_config=quant_config, + use_grouped_topk=True, + num_expert_group=config.n_group, + topk_group=config.topk_group, + prefix=f"{prefix}.experts", + scoring_func="sigmoid", + e_score_correction_bias=self.gate.e_score_correction_bias, + enable_eplb=self.enable_eplb, + num_redundant_experts=self.n_redundant_experts) + + if config.n_shared_experts is not None: + intermediate_size = (config.moe_intermediate_size * + config.n_shared_experts) + self.shared_experts = Glm4MoeMLP( + hidden_size=config.hidden_size, + intermediate_size=intermediate_size, + hidden_act=config.hidden_act, + quant_config=quant_config, + reduce_results=self.experts.must_reduce_shared_expert_outputs( + ), + prefix=f"{prefix}.shared_experts", + ) + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + num_tokens, hidden_dim = hidden_states.shape + hidden_states = hidden_states.view(-1, hidden_dim) + + if self.n_shared_experts is not None: + shared_output = self.shared_experts(hidden_states) + else: + shared_output = None + + router_logits = self.gate(hidden_states.to(dtype=torch.float32)) + kunlun_linear_weights = self.gate.weight + final_hidden_states = self.experts( + hidden_states=hidden_states, + router_logits=router_logits, + linear_weights=kunlun_linear_weights) * self.routed_scaling_factor + if shared_output is not None: + final_hidden_states = final_hidden_states + shared_output + if self.tp_size > 1: + final_hidden_states = ( + self.experts.maybe_all_reduce_tensor_model_parallel( + final_hidden_states)) + return final_hidden_states.view(num_tokens, hidden_dim) + + +class Glm4MoeAttention(nn.Module): + + def __init__( + self, + config: Glm4MoeConfig, + hidden_size: int, + num_heads: int, + num_kv_heads: int, + rope_theta: float = 10000, + rope_scaling: Optional[dict[str, Any]] = None, + max_position_embeddings: int = 131072, + head_dim: Optional[int] = None, + rms_norm_eps: float = 1e-05, + qkv_bias: bool = False, + use_qk_norm: bool = False, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + prefix: str = "", + ) -> None: + super().__init__() + self.hidden_size = hidden_size + tp_size = get_tensor_model_parallel_world_size() + self.total_num_heads = num_heads + assert self.total_num_heads % tp_size == 0 + self.num_heads = self.total_num_heads // tp_size + self.total_num_kv_heads = num_kv_heads + if self.total_num_kv_heads >= tp_size: + # Number of KV heads is greater than TP size, so we partition + # the KV heads across multiple tensor parallel GPUs. + assert self.total_num_kv_heads % tp_size == 0 + else: + # Number of KV heads is less than TP size, so we replicate + # the KV heads across multiple tensor parallel GPUs. + assert tp_size % self.total_num_kv_heads == 0 + self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size) + self.head_dim = head_dim or (hidden_size // self.total_num_heads) + self.q_size = self.num_heads * self.head_dim + self.kv_size = self.num_kv_heads * self.head_dim + self.scaling = self.head_dim**-0.5 + self.rope_theta = rope_theta + self.max_position_embeddings = max_position_embeddings + self.use_qk_norm = use_qk_norm + + self.qkv_proj = QKVParallelLinear(hidden_size, + self.head_dim, + self.total_num_heads, + self.total_num_kv_heads, + bias=qkv_bias, + quant_config=quant_config, + prefix=f"{prefix}.qkv_proj") + + self.o_proj = RowParallelLinear(self.total_num_heads * self.head_dim, + hidden_size, + bias=False, + quant_config=quant_config, + prefix=f"{prefix}.o_proj") + + self.partial_rotary_factor = getattr(config, "partial_rotary_factor", 0.5) + self.rotary_emb = get_rope( + self.head_dim, + rotary_dim=self.head_dim, + max_position=max_position_embeddings, + base=rope_theta, + rope_scaling=rope_scaling, + partial_rotary_factor=self.partial_rotary_factor, + ) + self.attn = Attention( + self.num_heads, + self.head_dim, + self.scaling, + num_kv_heads=self.num_kv_heads, + cache_config=cache_config, + quant_config=quant_config, + prefix=f"{prefix}.attn", + ) + + if self.use_qk_norm: + self.q_norm = RMSNorm(self.head_dim, eps=rms_norm_eps) + self.k_norm = RMSNorm(self.head_dim, eps=rms_norm_eps) + + def forward( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + ) -> torch.Tensor: + qkv, _ = self.qkv_proj(hidden_states) + + if os.getenv('USE_ORI_ROPE') == "1" or not self.use_qk_norm: + q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1) + if self.use_qk_norm: + q = self.q_norm(q.reshape(-1, self.num_heads, + self.head_dim)).reshape(q.shape) + k = self.k_norm(k.reshape(-1, self.num_kv_heads, + self.head_dim)).reshape(k.shape) + q, k = self.rotary_emb(positions, q, k) + else: + # Rope fusion operators + q, k, v = Split_Norm_Rope(qkv, + self.rotary_emb.cos_sin_cache, + self.q_norm.weight, + self.k_norm.weight, + positions, + self.max_position_embeddings, + self.num_heads, + self.num_kv_heads, + self.head_dim, + partial_rotary_factor=self.partial_rotary_factor, + ) + + attn_output = self.attn(q, k, v) + output, _ = self.o_proj(attn_output) + return output + + +class Glm4MoeDecoderLayer(nn.Module): + + def __init__( + self, + config: Glm4MoeConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + prefix: str = "", + enable_eplb: bool = False, + ) -> None: + super().__init__() + self.hidden_size = config.hidden_size + rope_theta = getattr(config, "rope_theta", 10000) + rope_scaling = getattr(config, "rope_scaling", None) + max_position_embeddings = getattr(config, "max_position_embeddings", + 131072) + # DecoderLayers are created with `make_layers` which passes the prefix + # with the layer's index. + layer_idx = int(prefix.split(sep='.')[-1]) + self.layer_idx = layer_idx + + self.self_attn = Glm4MoeAttention( + config=config, + hidden_size=self.hidden_size, + num_heads=config.num_attention_heads, + num_kv_heads=config.num_key_value_heads, + rope_theta=rope_theta, + rope_scaling=rope_scaling, + max_position_embeddings=max_position_embeddings, + head_dim=config.head_dim, + rms_norm_eps=config.rms_norm_eps, + qkv_bias=config.attention_bias, + cache_config=cache_config, + quant_config=quant_config, + prefix=f"{prefix}.self_attn", + use_qk_norm=config.use_qk_norm, + ) + + if (config.n_routed_experts is not None + and layer_idx >= config.first_k_dense_replace): + self.mlp = Glm4MoE( + config=config, + quant_config=quant_config, + prefix=f"{prefix}.mlp", + enable_eplb=enable_eplb, + ) + else: + self.mlp = Glm4MoeMLP(hidden_size=config.hidden_size, + intermediate_size=config.intermediate_size, + hidden_act=config.hidden_act, + quant_config=quant_config, + prefix=f"{prefix}.mlp") + + self.input_layernorm = RMSNorm(config.hidden_size, + eps=config.rms_norm_eps) + self.post_attention_layernorm = RMSNorm(config.hidden_size, + eps=config.rms_norm_eps) + self.routed_scaling_factor = config.routed_scaling_factor + + def forward( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + residual: Optional[torch.Tensor], + ) -> tuple[torch.Tensor, torch.Tensor]: + if residual is None: + residual = hidden_states + hidden_states = self.input_layernorm(hidden_states) + else: + hidden_states, residual = self.input_layernorm( + hidden_states, residual) + hidden_states = self.self_attn(positions=positions, + hidden_states=hidden_states) + hidden_states, residual = self.post_attention_layernorm( + hidden_states, residual) + hidden_states = self.mlp(hidden_states) + return hidden_states, residual + + +@support_torch_compile( + dynamic_arg_dims={ + "input_ids": 0, + "positions": -1, + "intermediate_tensors": 0, + "inputs_embeds": 0, + }) +class Glm4MoeModel(nn.Module): + + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): + super().__init__() + + config = vllm_config.model_config.hf_config + cache_config = vllm_config.cache_config + quant_config = vllm_config.quant_config + enable_eplb = vllm_config.parallel_config.enable_eplb + self.config = config + + self.vocab_size = config.vocab_size + + if get_pp_group().is_first_rank: + self.embed_tokens = VocabParallelEmbedding( + config.vocab_size, + config.hidden_size, + prefix=f"{prefix}.embed_tokens") + else: + self.embed_tokens = PPMissingLayer() + + self.start_layer, self.end_layer, self.layers = make_layers( + config.num_hidden_layers, + lambda prefix: Glm4MoeDecoderLayer( + config=config, + cache_config=cache_config, + quant_config=quant_config, + prefix=prefix, + enable_eplb=enable_eplb, + ), + prefix=f"{prefix}.layers") + + if get_pp_group().is_last_rank: + self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) + else: + self.norm = PPMissingLayer() + self.make_empty_intermediate_tensors = ( + make_empty_intermediate_tensors_factory( + ["hidden_states", "residual"], config.hidden_size)) + + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor: + return self.embed_tokens(input_ids) + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + intermediate_tensors: Optional[IntermediateTensors] = None, + inputs_embeds: Optional[torch.Tensor] = None, + ) -> Union[torch.Tensor, IntermediateTensors]: + if get_pp_group().is_first_rank: + if inputs_embeds is not None: + hidden_states = inputs_embeds + else: + hidden_states = self.get_input_embeddings(input_ids) + residual = None + else: + assert intermediate_tensors is not None + hidden_states = intermediate_tensors["hidden_states"] + residual = intermediate_tensors["residual"] + + for i in range(self.start_layer, self.end_layer): + layer = self.layers[i] + hidden_states, residual = layer(positions, hidden_states, residual) + + if not get_pp_group().is_last_rank: + return IntermediateTensors({ + "hidden_states": hidden_states, + "residual": residual + }) + + hidden_states, _ = self.norm(hidden_states, residual) + return hidden_states + + def make_empty_intermediate_tensors( + self, batch_size: int, dtype: torch.dtype, + device: torch.device) -> IntermediateTensors: + return IntermediateTensors({ + "hidden_states": + torch.zeros((batch_size, self.config.hidden_size), + dtype=dtype, + device=device), + "residual": + torch.zeros((batch_size, self.config.hidden_size), + dtype=dtype, + device=device), + }) + + def get_expert_mapping(self) -> list[tuple[str, str, int, str]]: + # Params for weights, fp8 weight scales, fp8 activation scales + # (param_name, weight_name, expert_id, shard_id) + return FusedMoE.make_expert_params_mapping( + ckpt_gate_proj_name="gate_proj", + ckpt_down_proj_name="down_proj", + ckpt_up_proj_name="up_proj", + num_experts=self.config.n_routed_experts) + + def load_weights(self, weights: Iterable[tuple[str, + torch.Tensor]]) -> set[str]: + stacked_params_mapping = [ + # (param_name, shard_name, shard_id) + ("qkv_proj", "q_proj", "q"), + ("qkv_proj", "k_proj", "k"), + ("qkv_proj", "v_proj", "v"), + ("gate_up_proj", "gate_proj", 0), + ("gate_up_proj", "up_proj", 1), + ] + + params_dict = dict(self.named_parameters()) + loaded_params: set[str] = set() + expert_params_mapping = self.get_expert_mapping() + for name, loaded_weight in weights: + spec_layer = get_spec_layer_idx_from_weight_name(self.config, name) + if spec_layer is not None: + continue + for (param_name, weight_name, shard_id) in stacked_params_mapping: + # Skip non-stacked layers and experts (experts handled below). + if weight_name not in name: + continue + # We have mlp.experts[0].gate_proj in the checkpoint. + # Since we handle the experts below in expert_params_mapping, + # we need to skip here BEFORE we update the name, otherwise + # name will be updated to mlp.experts[0].gate_up_proj, which + # will then be updated below in expert_params_mapping + # for mlp.experts[0].gate_gate_up_proj, which breaks load. + if (("mlp.experts." in name) and name not in params_dict): + continue + name = name.replace(weight_name, param_name) + # Skip loading extra bias for GPTQ models. + if name.endswith(".bias") and name not in params_dict: + continue + if is_pp_missing_parameter(name, self): + continue + + param = params_dict[name] + weight_loader = param.weight_loader + weight_loader(param, loaded_weight, shard_id) + break + else: + is_expert_weight = False + for mapping in expert_params_mapping: + param_name, weight_name, expert_id, shard_id = mapping + if weight_name not in name: + continue + + # Anyway, this is an expert weight and should not be + # attempted to load as other weights later + is_expert_weight = True + + # Do not modify `name` since the loop may continue here + # Instead, create a new variable + name_mapped = name.replace(weight_name, param_name) + + if is_pp_missing_parameter(name_mapped, self): + continue + + param = params_dict[name_mapped] + # We should ask the weight loader to return success or not + # here since otherwise we may skip experts with other + # available replicas. + weight_loader = typing.cast(Callable[..., bool], + param.weight_loader) + success = weight_loader(param, + loaded_weight, + name_mapped, + shard_id=shard_id, + expert_id=expert_id, + return_success=True) + if success: + name = name_mapped + break + else: + if is_expert_weight: + # We've checked that this is an expert weight + # However it's not mapped locally to this rank + # So we simply skip it + continue + + # Skip loading extra bias for GPTQ models. + if name.endswith(".bias") and name not in params_dict: + continue + + # Remapping the name of FP8 kv-scale. + name = maybe_remap_kv_scale_name(name, params_dict) + if name is None: + continue + + if is_pp_missing_parameter(name, self): + continue + + param = params_dict[name] + weight_loader = getattr(param, "weight_loader", + default_weight_loader) + weight_loader(param, loaded_weight) + loaded_params.add(name) + + return loaded_params + + +class Glm4MoeForCausalLM(nn.Module, SupportsPP, SupportsLoRA): + packed_modules_mapping = { + "qkv_proj": [ + "q_proj", + "k_proj", + "v_proj", + ], + "gate_up_proj": [ + "gate_proj", + "up_proj", + ], + } + + fall_back_to_pt_during_load = False + + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): + super().__init__() + config = vllm_config.model_config.hf_config + quant_config = vllm_config.quant_config + self.config = config + self.quant_config = quant_config + self.model = Glm4MoeModel(vllm_config=vllm_config, + prefix=maybe_prefix(prefix, "model")) + if get_pp_group().is_last_rank: + self.lm_head = ParallelLMHead(config.vocab_size, + config.hidden_size, + quant_config=quant_config) + else: + self.lm_head = PPMissingLayer() + self.logits_processor = LogitsProcessor(config.vocab_size) + self.make_empty_intermediate_tensors = ( + self.model.make_empty_intermediate_tensors) + self.expert_weights = [] + + # Set MoE hyperparameters + self.num_moe_layers = (config.num_hidden_layers - + config.first_k_dense_replace) + self.num_expert_groups = config.n_group + + self.moe_layers: list[FusedMoE] = [] + example_moe = None + for layer in self.model.layers: + if isinstance(layer, PPMissingLayer): + continue + + assert isinstance(layer, Glm4MoeDecoderLayer) + if isinstance(layer.mlp, Glm4MoE): + # Pick last one layer since the first ones may be dense layers. + example_moe = layer.mlp + self.moe_layers.append(layer.mlp.experts) + + if example_moe is None: + raise RuntimeError("No Glm4MoE layer found in model.layers.") + + self.num_logical_experts = example_moe.n_logical_experts + self.num_physical_experts = example_moe.n_physical_experts + self.num_local_physical_experts = example_moe.n_local_physical_experts + self.num_routed_experts = example_moe.n_routed_experts + self.num_shared_experts = example_moe.n_shared_experts + self.num_redundant_experts = example_moe.n_redundant_experts + + def set_eplb_state( + self, + expert_load_view: torch.Tensor, + logical_to_physical_map: torch.Tensor, + logical_replica_count: torch.Tensor, + ) -> None: + for layer_idx, layer in enumerate(self.moe_layers): + # Register the expert weights. + self.expert_weights.append(layer.get_expert_weights()) + layer.set_eplb_state( + moe_layer_idx=layer_idx, + expert_load_view=expert_load_view, + logical_to_physical_map=logical_to_physical_map, + logical_replica_count=logical_replica_count, + ) + + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor: + return self.model.get_input_embeddings(input_ids) + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + intermediate_tensors: Optional[IntermediateTensors] = None, + inputs_embeds: Optional[torch.Tensor] = None, + ) -> Union[torch.Tensor, IntermediateTensors]: + hidden_states = self.model(input_ids, positions, intermediate_tensors, + inputs_embeds) + return hidden_states + + def compute_logits( + self, + hidden_states: torch.Tensor, + sampling_metadata: SamplingMetadata, + ) -> Optional[torch.Tensor]: + logits = self.logits_processor(self.lm_head, hidden_states, + sampling_metadata) + return logits + + def load_weights(self, weights: Iterable[tuple[str, + torch.Tensor]]) -> set[str]: + loader = AutoWeightsLoader(self) + return loader.load_weights(weights) + + def get_expert_mapping(self) -> list[tuple[str, str, int, str]]: + return self.model.get_expert_mapping() + + +def get_spec_layer_idx_from_weight_name(config: Glm4MoeConfig, + weight_name: str) -> Optional[int]: + if hasattr(config, + "num_nextn_predict_layers") and (config.num_nextn_predict_layers + > 0): + layer_idx = config.num_hidden_layers + for i in range(config.num_nextn_predict_layers): + if f"layers.{layer_idx+i}." in weight_name: + return layer_idx + i + return None \ No newline at end of file diff --git a/vllm_kunlun/models/gpt_oss.py b/vllm_kunlun/models/gpt_oss.py new file mode 100644 index 0000000..cc18587 --- /dev/null +++ b/vllm_kunlun/models/gpt_oss.py @@ -0,0 +1,630 @@ +# +# Copyright (c) 2025 Baidu, Inc. All Rights Reserved. +# Adapted from vllm/model_executor/models/gpt_oss.py +# Copyright 2023 The vLLM team. +# +# This file is a part of the vllm-kunlun project. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from collections.abc import Iterable +from typing import Optional + +import torch +import torch.distributed as dist +from torch import nn +from transformers import GptOssConfig + +from vllm.attention import Attention, AttentionType +from vllm.compilation.decorators import support_torch_compile +from vllm.config import CacheConfig, VllmConfig +from vllm.distributed import (get_ep_group, get_tensor_model_parallel_rank, + get_tensor_model_parallel_world_size) +from vllm.model_executor.layers.fused_moe import FusedMoE +from vllm.model_executor.layers.layernorm import RMSNorm +from vllm.model_executor.layers.linear import (QKVParallelLinear, + RowParallelLinear) +from vllm.model_executor.layers.logits_processor import LogitsProcessor +from vllm.model_executor.layers.quantization import QuantizationConfig +from vllm.model_executor.layers.rotary_embedding import get_rope +from vllm.model_executor.layers.vocab_parallel_embedding import ( + ParallelLMHead, VocabParallelEmbedding) +from vllm.model_executor.model_loader.weight_utils import default_weight_loader +from vllm.model_executor.sampling_metadata import SamplingMetadata +from vllm.sequence import IntermediateTensors +from vllm.utils import cdiv + +from .utils import extract_layer_index, maybe_prefix + + +class OAIAttention(nn.Module): + + def __init__( + self, + config: GptOssConfig, + quant_config: Optional[QuantizationConfig] = None, + cache_config: Optional[CacheConfig] = None, + prefix: str = "", + ): + super().__init__() + self.layer_idx = extract_layer_index(prefix) + self.head_dim = config.head_dim + self.num_attention_heads = config.num_attention_heads + self.num_key_value_heads = config.num_key_value_heads + self.hidden_size = config.hidden_size + + self.rotary_emb = get_rope( + self.head_dim, + rotary_dim=self.head_dim, + max_position=config.max_position_embeddings, + base=config.rope_theta, + dtype=torch.float32, + rope_scaling={ + "rope_type": + "yarn", + "factor": + config.rope_scaling["factor"], + "original_max_position_embeddings": + config.rope_scaling["original_max_position_embeddings"], + "beta_fast": + config.rope_scaling["beta_fast"], + "beta_slow": + config.rope_scaling["beta_slow"], + }, + is_neox_style=True, + ) + + tp_size = get_tensor_model_parallel_world_size() + + self.sinks = torch.nn.Parameter( + torch.empty(config.num_attention_heads // tp_size, + dtype=torch.bfloat16, + requires_grad=False)) + + self.norm = RMSNorm(config.hidden_size, eps=1e-5) + + self.q_size = self.num_attention_heads * self.head_dim // tp_size + self.kv_size = self.num_key_value_heads * self.head_dim // tp_size + self.scaling = self.head_dim**-0.5 + self.rope_theta = config.rope_theta + + self.qkv = QKVParallelLinear( + hidden_size=self.hidden_size, + head_size=self.head_dim, + total_num_heads=self.num_attention_heads, + total_num_kv_heads=self.num_key_value_heads, + quant_config=quant_config, + prefix=f"{prefix}.qkv_proj", + ) + + self.o_proj = RowParallelLinear( + input_size=self.num_attention_heads * self.head_dim, + output_size=self.hidden_size, + quant_config=quant_config, + prefix=f"{prefix}.o_proj", + ) + + self.num_local_attention_heads = config.num_attention_heads // tp_size + self.num_local_key_value_heads = config.num_key_value_heads // tp_size + + # Only apply sliding window to every other layer + sliding_window = (config.sliding_window if self.layer_idx % + 2 == 0 else None) + self.attn = Attention( + self.num_local_attention_heads, + self.head_dim, + self.scaling, + num_kv_heads=self.num_local_key_value_heads, + cache_config=cache_config, + quant_config=quant_config, + per_layer_sliding_window=sliding_window, + attn_type=AttentionType.DECODER, + prefix=f"{prefix}.attn", + sinks=self.sinks, + ) + + def forward(self, hidden_states: torch.Tensor, + positions: torch.Tensor) -> torch.Tensor: + t = self.norm(hidden_states) + + qkv, _ = self.qkv(t) + q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1) + q, k = self.rotary_emb(positions, q, k) + v = v.contiguous() + attn_output = self.attn(q, k, v) + output, _ = self.o_proj(attn_output) + + return output + hidden_states + + +class MLPBlock(torch.nn.Module): + + def __init__( + self, + config: GptOssConfig, + layer_idx: int, + quant_config: QuantizationConfig, + prefix: str = "", + ): + super().__init__() + self.layer_idx = layer_idx + self.num_experts = config.num_local_experts + self.experts_per_token = config.num_experts_per_tok + self.world_size = dist.get_world_size() if dist.is_initialized() else 1 + self.norm = RMSNorm(config.hidden_size, eps=1e-5) + self.router = torch.nn.Linear(config.hidden_size, + config.num_local_experts, + dtype=torch.bfloat16) + assert config.intermediate_size % self.world_size == 0 + self.experts = FusedMoE(num_experts=config.num_local_experts, + top_k=config.num_experts_per_tok, + hidden_size=config.hidden_size, + intermediate_size=config.intermediate_size, + reduce_results=True, + renormalize=True, + quant_config=quant_config, + prefix=f"{prefix}.experts", + apply_router_weight_on_input=False, + has_bias=True, + activation="swigluoai") + + def forward(self, x: torch.Tensor) -> torch.Tensor: + t = self.norm(x) + g = self.router(t) + t = self.experts(hidden_states=t, router_logits=g) + return x + t + + +class TransformerBlock(torch.nn.Module): + + def __init__( + self, + config: GptOssConfig, + quant_config: QuantizationConfig, + prefix: str = "", + ): + super().__init__() + self.layer_idx = extract_layer_index(prefix) + self.attn = OAIAttention(config, prefix=f"{prefix}.attn") + self.mlp = MLPBlock(config, + self.layer_idx, + quant_config=quant_config, + prefix=f"{prefix}.mlp") + + def forward(self, hidden_states: torch.Tensor, + positions: torch.Tensor) -> torch.Tensor: + attn_output = self.attn(hidden_states, positions) + output = self.mlp(attn_output) + return output + + +@support_torch_compile +class GptOssModel(nn.Module): + + def __init__( + self, + *, + vllm_config: VllmConfig, + prefix: str = "", + ): + super().__init__() + self.config = vllm_config.model_config.hf_config + self.quant_config = vllm_config.quant_config + self.config.hidden_size = self.config.hidden_size + self.embedding = VocabParallelEmbedding( + self.config.vocab_size, + self.config.hidden_size, + ) + self.layers = torch.nn.ModuleList([ + TransformerBlock( + self.config, + quant_config=self.quant_config, + prefix=maybe_prefix(prefix, f"block.{layer_idx}"), + ) for layer_idx in range(self.config.num_hidden_layers) + ]) + self.norm = RMSNorm(self.config.hidden_size, eps=1e-5) + + def forward(self, input_ids: torch.Tensor, + positions: torch.Tensor) -> torch.Tensor: + x = self.embedding(input_ids) + for layer in self.layers: + x = layer(x, positions) + x = self.norm(x) + return x + + +class GptOssForCausalLM(nn.Module): + + def __init__( + self, + vllm_config: VllmConfig, + prefix: str = "", + ): + super().__init__() + self.vllm_config = vllm_config + self.model_config = vllm_config.model_config.hf_config + self.model = GptOssModel( + vllm_config=vllm_config, + prefix=maybe_prefix(prefix, "model"), + ) + self.lm_head = ParallelLMHead( + self.model_config.vocab_size, + self.model_config.hidden_size, + ) + self.logits_processor = LogitsProcessor(self.model_config.vocab_size) + + def forward(self, + input_ids: torch.Tensor, + positions: torch.Tensor, + intermediate_tensors: Optional[IntermediateTensors] = None, + inputs_embeds: Optional[torch.Tensor] = None) -> torch.Tensor: + assert intermediate_tensors is None + assert inputs_embeds is None + return self.model(input_ids, positions) + + def compute_logits(self, hidden_states: torch.Tensor, + sampling_metadata: SamplingMetadata) -> torch.Tensor: + logits = self.logits_processor(self.lm_head, hidden_states, + sampling_metadata) + return logits + + def _load_weights_mxfp4( + self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: + rename_mapping = { + "self_attn": "attn", + "input_layernorm.weight": "attn.norm.weight", + "post_attention_layernorm.weight": "mlp.norm.weight", + "embed_tokens": "embedding", + } + + def maybe_rename(name: str) -> str: + for remap_name, new_name in rename_mapping.items(): + if remap_name in name: + return name.replace(remap_name, new_name) + return name + + params_dict = dict(self.named_parameters()) + loaded_params: set[str] = set() + mxfp4_block = 32 + + tp_rank = get_tensor_model_parallel_rank() + tp_size = get_tensor_model_parallel_world_size() + intermediate_size = self.model_config.intermediate_size + intermediate_size_block = intermediate_size // mxfp4_block + per_rank_intermediate_size_block = cdiv(intermediate_size_block, + tp_size) + per_rank_intermediate_size = (per_rank_intermediate_size_block * + mxfp4_block) + + # Calculate common slicing bounds for current rank + tp_rank_start = tp_rank * per_rank_intermediate_size + tp_rank_end = min((tp_rank + 1) * per_rank_intermediate_size, + intermediate_size) + + # Attention heads per rank + heads_per_rank = self.model_config.num_attention_heads // tp_size + head_start = tp_rank * heads_per_rank + + use_ep = self.vllm_config.parallel_config.enable_expert_parallel + ep_size = get_ep_group().world_size + ep_rank = get_ep_group().rank + num_experts = self.model_config.num_local_experts + experts_per_rank = num_experts // ep_size + ep_rank_start = ep_rank * experts_per_rank + ep_rank_end = (ep_rank + 1) * experts_per_rank + + for name, weight in weights: + # FIXME(woosuk): Remove this after testing. + weight = weight.cuda() + + if "gate_up_proj_blocks" in name: + # Handle MLP gate and up projection weights + new_name = name.replace("gate_up_proj_blocks", "w13_weight") + + # flat weight from (E, 2 * N, block_size, entry_per_block) + # to (E, 2 * N, -1), shouldn't trigger copy for contiguous + weight = weight.view(num_experts, 2 * intermediate_size, + -1).contiguous() + + # Extract gate and up projection parts + # since the weight is shuffled, we can slice directly + if use_ep: + narrow_weight = weight[ep_rank_start:ep_rank_end, ...] + else: + narrow_weight = weight[:, + 2 * tp_rank_start:2 * tp_rank_end, + ...] + + param = params_dict[new_name] + weight_loader = getattr(param, "weight_loader", + default_weight_loader) + weight_loader(param, + narrow_weight, + weight_name=new_name, + shard_id=None, + expert_id=None) + loaded_params.add(new_name) + + elif "down_proj_blocks" in name: + # Handle MLP down projection weights + new_name = name.replace("down_proj_blocks", "w2_weight") + # same flatten here, but since 2 mx4 value are packed in 1 + # uint8, divide by 2 + weight = weight.view(num_experts, -1, + intermediate_size // 2).contiguous() + if use_ep: + narrow_weight = weight[ep_rank_start:ep_rank_end, ...] + else: + narrow_weight = weight[..., + tp_rank_start // 2:tp_rank_end // 2] + + param = params_dict[new_name] + weight_loader = getattr(param, "weight_loader", + default_weight_loader) + weight_loader(param, + narrow_weight, + weight_name=new_name, + shard_id=None, + expert_id=None) + loaded_params.add(new_name) + + elif "gate_up_proj_scales" in name: + # Handle MLP gate and up projection weights scale + new_name = name.replace("gate_up_proj_scales", + "w13_weight_scale") + if use_ep: + narrow_weight = weight[ep_rank_start:ep_rank_end, ...] + else: + narrow_weight = weight[:, + 2 * tp_rank_start:2 * tp_rank_end, + ...] + + param = params_dict[new_name] + weight_loader = getattr(param, "weight_loader", + default_weight_loader) + weight_loader(param, + narrow_weight, + weight_name=new_name, + shard_id=None, + expert_id=None) + loaded_params.add(new_name) + + elif "down_proj_scales" in name: + # Handle MLP down projection weights + new_name = name.replace("down_proj_scales", "w2_weight_scale") + if use_ep: + narrow_weight = weight[ep_rank_start:ep_rank_end, ...] + else: + narrow_weight = weight[..., tp_rank_start // + mxfp4_block:tp_rank_end // + mxfp4_block] + + param = params_dict[new_name] + weight_loader = getattr(param, "weight_loader", + default_weight_loader) + weight_loader(param, + narrow_weight, + weight_name=new_name, + shard_id=None, + expert_id=None) + loaded_params.add(new_name) + elif "gate_up_proj_bias" in name: + # Handle MLP gate and up projection biases + new_name = name.replace("gate_up_proj_bias", "w13_bias") + + # Extract gate and up projection bias parts + if use_ep: + narrow_weight = weight[ep_rank_start:ep_rank_end, ...] + else: + narrow_weight = weight[:, + 2 * tp_rank_start:2 * tp_rank_end] + + param = params_dict[new_name] + weight_loader = getattr(param, "weight_loader", + default_weight_loader) + weight_loader(param, + narrow_weight, + weight_name=new_name, + shard_id=None, + expert_id=None) + loaded_params.add(new_name) + + elif "down_proj_bias" in name: + # Handle MLP down projection bias + new_name = name.replace("down_proj_bias", "w2_bias") + param = params_dict[new_name] + weight_loader = getattr(param, "weight_loader", + default_weight_loader) + if use_ep: + weight = weight[ep_rank_start:ep_rank_end, ...] + else: + # (only load on rank 0 to avoid duplication) + if tp_rank != 0: + weight.zero_() + weight_loader(param, + weight, + weight_name=new_name, + shard_id=None, + expert_id=None) + loaded_params.add(new_name) + elif "sinks" in name: + # Handle attention sinks (distributed across ranks) + name = name.replace("self_attn", "attn") + param = params_dict[name] + narrow_weight = weight.narrow(0, head_start, heads_per_rank) + param.data.copy_(narrow_weight) + loaded_params.add(name) + elif "q_proj" in name or "k_proj" in name or "v_proj" in name: + shard_id = ("q" if "q_proj" in name else + "k" if "k_proj" in name else "v") + name = name.replace("self_attn", "attn") + param_name = name.replace(f"{shard_id}_proj", "qkv") + param = params_dict[param_name] + weight_loader = param.weight_loader + weight_loader(param, weight, loaded_shard_id=shard_id) + loaded_params.add(param_name) + else: + # Handle all other weights with potential renaming + renamed_name = maybe_rename(name) + if renamed_name not in params_dict: + continue + param = params_dict[renamed_name] + weight_loader = getattr(param, "weight_loader", + default_weight_loader) + weight_loader(param, weight) + loaded_params.add(renamed_name) + + return loaded_params + + def _load_weights_other( + self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: + rename_mapping = { + "self_attn": "attn", + "input_layernorm.weight": "attn.norm.weight", + "post_attention_layernorm.weight": "mlp.norm.weight", + "embed_tokens": "embedding", + } + + def maybe_rename(name: str) -> str: + for remap_name, new_name in rename_mapping.items(): + if remap_name in name: + return name.replace(remap_name, new_name) + return name + + params_dict = dict(self.named_parameters()) + loaded_params: set[str] = set() + + tp_rank = get_tensor_model_parallel_rank() + tp_size = get_tensor_model_parallel_world_size() + intermediate_size = self.model_config.intermediate_size + + per_rank_intermediate_size = cdiv(intermediate_size, tp_size) + # Calculate common slicing bounds for current rank + tp_rank_start = tp_rank * per_rank_intermediate_size + tp_rank_end = min((tp_rank + 1) * per_rank_intermediate_size, + intermediate_size) + + # Attention heads per rank + heads_per_rank = self.model_config.num_attention_heads // tp_size + head_start = tp_rank * heads_per_rank + + use_ep = self.vllm_config.parallel_config.enable_expert_parallel + ep_size = get_ep_group().world_size + ep_rank = get_ep_group().rank + num_experts = self.model_config.num_local_experts + experts_per_rank = num_experts // ep_size + ep_rank_start = ep_rank * experts_per_rank + ep_rank_end = (ep_rank + 1) * experts_per_rank + + for name, weight in weights: + if ".experts.gate_up_proj" in name and "bias" not in name: + # Handle MLP gate and up projection weights + new_name = name.replace(".experts.gate_up_proj", + ".experts.w13_weight") + + # Extract gate and up projection parts + # since the weight is shuffled, we can slice directly + if use_ep: + narrow_weight = weight[ep_rank_start:ep_rank_end, ...] + else: + narrow_weight = weight[:, :, + 2 * tp_rank_start:2 * tp_rank_end] + + narrow_weight = narrow_weight.permute(0, 2, 1).contiguous() + param = params_dict[new_name] + + param.copy_(narrow_weight) + loaded_params.add(new_name) + + elif ".experts.down_proj" in name and "bias" not in name: + # Handle MLP down projection weights + new_name = name.replace(".experts.down_proj", + ".experts.w2_weight") + + if use_ep: + narrow_weight = weight[ep_rank_start:ep_rank_end, ...] + else: + narrow_weight = weight[:, tp_rank_start:tp_rank_end, :] + narrow_weight = narrow_weight.permute(0, 2, 1).contiguous() + param = params_dict[new_name] + + param.copy_(narrow_weight) + loaded_params.add(new_name) + + elif "gate_up_proj_bias" in name: + # Handle MLP gate and up projection biases + new_name = name.replace("gate_up_proj_bias", "w13_bias") + + # Extract gate and up projection bias parts + if use_ep: + narrow_weight = weight[ep_rank_start:ep_rank_end, ...] + else: + narrow_weight = weight[:, + 2 * tp_rank_start:2 * tp_rank_end] + + param = params_dict[new_name] + + param.copy_(narrow_weight) + loaded_params.add(new_name) + + elif "down_proj_bias" in name: + # Handle MLP down projection bias + new_name = name.replace("down_proj_bias", "w2_bias") + + if use_ep: + weight = weight[ep_rank_start:ep_rank_end, ...] + else: + # (only load on rank 0 to avoid duplication) + if tp_rank != 0: + weight.zero_() + param = params_dict[new_name] + param.copy_(weight) + loaded_params.add(new_name) + elif "sinks" in name: + # Handle attention sinks (distributed across ranks) + name = name.replace("self_attn", "attn") + param = params_dict[name] + narrow_weight = weight.narrow(0, head_start, heads_per_rank) + param.data.copy_(narrow_weight) + loaded_params.add(name) + elif "q_proj" in name or "k_proj" in name or "v_proj" in name: + shard_id = ("q" if "q_proj" in name else + "k" if "k_proj" in name else "v") + name = name.replace("self_attn", "attn") + param_name = name.replace(f"{shard_id}_proj", "qkv") + param = params_dict[param_name] + weight_loader = param.weight_loader + weight_loader(param, weight, loaded_shard_id=shard_id) + loaded_params.add(param_name) + else: + # Handle all other weights with potential renaming + + renamed_name = maybe_rename(name) + if renamed_name not in params_dict: + continue + param = params_dict[renamed_name] + weight_loader = getattr(param, "weight_loader", + default_weight_loader) + weight_loader(param, weight) + loaded_params.add(renamed_name) + + return loaded_params + + def load_weights(self, weights: Iterable[tuple[str, + torch.Tensor]]) -> set[str]: + quant_method = (self.model_config.quantization_config['quant_method'] + if hasattr(self.model_config, "quantization_config") + else None) + if quant_method == "mxfp4": + return self._load_weights_mxfp4(weights) + else: + return self._load_weights_other(weights) diff --git a/vllm_kunlun/models/intern_vit.py b/vllm_kunlun/models/intern_vit.py new file mode 100644 index 0000000..e04f284 --- /dev/null +++ b/vllm_kunlun/models/intern_vit.py @@ -0,0 +1,480 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +# adapted from https://huggingface.co/OpenGVLab/InternVL2-4B/blob/main/modeling_intern_vit.py +# -------------------------------------------------------- +# InternVL +# Copyright (c) 2023 OpenGVLab +# Licensed under The MIT License [see LICENSE for details] +# -------------------------------------------------------- +from collections.abc import Iterable +from functools import partial +from typing import Optional + +import torch +import torch.nn as nn +import torch.nn.functional as F +from transformers import PretrainedConfig + +from vllm_kunlun.ops.attention.layer import MultiHeadAttention +from vllm.distributed import (divide, get_tensor_model_parallel_rank, + get_tensor_model_parallel_world_size, + split_tensor_along_last_dim, + tensor_model_parallel_all_gather) +from vllm.model_executor.layers.activation import get_act_fn +from vllm.model_executor.layers.layernorm import RMSNorm +from vllm.model_executor.layers.linear import (ColumnParallelLinear, + QKVParallelLinear, + RowParallelLinear) +from vllm.model_executor.layers.quantization import QuantizationConfig +from vllm.model_executor.model_loader.weight_utils import default_weight_loader + +NORM2FN = { + 'rms_norm': RMSNorm, + 'layer_norm': nn.LayerNorm, +} + + +class InternVisionEmbeddings(nn.Module): + + def __init__(self, config: PretrainedConfig): + super().__init__() + self.config = config + self.embed_dim = config.hidden_size + self.image_size = config.image_size + self.patch_size = config.patch_size + + self.class_embedding = nn.Parameter(torch.randn(1, 1, self.embed_dim)) + + self.patch_embedding = nn.Conv2d(in_channels=3, + out_channels=self.embed_dim, + kernel_size=self.patch_size, + stride=self.patch_size) + + self.num_patches = (self.image_size // self.patch_size)**2 + self.num_positions = self.num_patches + 1 + + self.position_embedding = nn.Parameter( + torch.randn(1, self.num_positions, self.embed_dim)) + + def _get_pos_embed(self, pos_embed: torch.Tensor, H: int, W: int): + target_dtype = pos_embed.dtype + pos_embed = pos_embed.float().reshape( + 1, self.image_size // self.patch_size, + self.image_size // self.patch_size, -1).permute(0, 3, 1, 2) + pos_embed = F.interpolate(pos_embed, + size=(H, W), + mode='bicubic', + align_corners=False) + return pos_embed.reshape(1, -1, H * W).permute(0, 2, + 1).to(target_dtype) + + def _get_position_embedding(self, H: int, W: int) -> torch.Tensor: + position_embedding = self.position_embedding + if self.num_patches == H * W: + return position_embedding + + return torch.cat( + [ + position_embedding[:, :1, :], + self._get_pos_embed(position_embedding[:, 1:, :], H, W), + ], + dim=1, + ) + + def forward(self, pixel_values: torch.FloatTensor) -> torch.Tensor: + target_dtype = self.patch_embedding.weight.dtype + patch_embeds = self.patch_embedding(pixel_values.to( + target_dtype)) # shape = [*, channel, width, height] + batch_size, _, height, width = patch_embeds.shape + patch_embeds = patch_embeds.flatten(2).transpose(1, 2) + class_embeds = self.class_embedding.expand(batch_size, 1, + -1).to(target_dtype) + embeddings = torch.cat([class_embeds, patch_embeds], dim=1) + position_embedding = self._get_position_embedding(height, width) + embeddings = embeddings + position_embedding.to(target_dtype) + return embeddings + + +class InternVisionPatchModel(nn.Module): + + def __init__(self, config: PretrainedConfig): + super().__init__() + self.config = config + self.embeddings = InternVisionEmbeddings(config) + + def get_input_embeddings(self): + return self.embeddings + + def forward( + self, + pixel_values: Optional[torch.Tensor] = None, + pixel_embeds: Optional[torch.Tensor] = None, + ) -> torch.FloatTensor: + if pixel_values is None and pixel_embeds is None: + raise ValueError( + 'You have to specify pixel_values or pixel_embeds') + + if pixel_embeds is not None: + hidden_states = pixel_embeds + elif pixel_values is not None: + if pixel_values.ndim == 4: + hidden_states = self.embeddings(pixel_values) + else: + raise ValueError( + f'wrong pixel_values size: {pixel_values.shape}') + + return hidden_states + + +class InternParallelAttention(nn.Module): + """Multi-headed attention from 'Attention Is All You Need' paper""" + + def __init__( + self, + config: PretrainedConfig, + quant_config: Optional[QuantizationConfig] = None, + *, + num_dummy_heads: int = 0, + prefix: str = "", + ) -> None: + super().__init__() + + self.config = config + self.embed_dim = config.hidden_size + self.num_heads = config.num_attention_heads + self.head_dim = self.embed_dim // self.num_heads + if self.head_dim * self.num_heads != self.embed_dim: + raise ValueError( + f'embed_dim must be divisible by num_heads ' + f'(got `embed_dim`: {self.embed_dim} and `num_heads`:' + f' {self.num_heads}).') + + self.tp_size = get_tensor_model_parallel_world_size() + self.tp_rank = get_tensor_model_parallel_rank() + + # Additional dummy heads are used to enable TP for common GPU counts. + self.dummy_dim = (num_dummy_heads + self.num_heads) * self.head_dim + self.num_heads_per_partition = divide(num_dummy_heads + self.num_heads, + self.tp_size) + + self.scale = self.head_dim**-0.5 + self.qkv = QKVParallelLinear( + self.embed_dim, + self.head_dim, + num_dummy_heads + self.num_heads, + bias=config.qkv_bias, + quant_config=quant_config, + prefix=f"{prefix}.qkv", + ) + + self.qk_normalization = config.qk_normalization + + if self.qk_normalization: + self.q_norm = RMSNorm(self.dummy_dim, + eps=config.layer_norm_eps, + var_hidden_size=self.embed_dim) + self.k_norm = RMSNorm(self.dummy_dim, + eps=config.layer_norm_eps, + var_hidden_size=self.embed_dim) + + self.proj = RowParallelLinear( + self.dummy_dim, + self.embed_dim, + quant_config=quant_config, + prefix=f"{prefix}.proj", + ) + + self.attn = MultiHeadAttention(self.num_heads_per_partition, + self.head_dim, self.scale) + + def _apply_qk_norm(self, q: torch.Tensor, k: torch.Tensor): + if self.tp_size > 1: + q = tensor_model_parallel_all_gather(q.contiguous()) + k = tensor_model_parallel_all_gather(k.contiguous()) + q = self.q_norm(q) + k = self.k_norm(k) + if self.tp_size > 1: + splitter = partial(split_tensor_along_last_dim, + num_partitions=self.tp_size) + q = splitter(q)[self.tp_rank] + k = splitter(k)[self.tp_rank] + return q, k + + def forward(self, x: torch.Tensor) -> torch.Tensor: + B, N, _ = x.shape + qkv, _ = self.qkv(x) + q, k, v = qkv.chunk(3, dim=-1) + + if self.qk_normalization: + q, k = self._apply_qk_norm(q, k) + + out = self.attn(q, k, v) + out, _ = self.proj(out) + return out + + +class InternSdpaAttention(nn.Module): + """Multi-headed attention from 'Attention Is All You Need' paper""" + + def __init__( + self, + config: PretrainedConfig, + *, + num_dummy_heads: int = 0, + ) -> None: + super().__init__() + + self.config = config + self.embed_dim = config.hidden_size + self.num_heads = config.num_attention_heads + self.head_dim = self.embed_dim // self.num_heads + if self.head_dim * self.num_heads != self.embed_dim: + raise ValueError( + f'embed_dim must be divisible by num_heads ' + f'(got `embed_dim`: {self.embed_dim} and `num_heads`:' + f' {self.num_heads}).') + + # Additional dummy heads are used to enable TP for common GPU counts. + self.dummy_dim = (num_dummy_heads + self.num_heads) * self.head_dim + + self.scale = self.head_dim**-0.5 + self.qkv = nn.Linear(self.embed_dim, + 3 * self.dummy_dim, + bias=config.qkv_bias) + + self.qk_normalization = config.qk_normalization + + if self.qk_normalization: + self.q_norm = RMSNorm(self.dummy_dim, + eps=config.layer_norm_eps, + var_hidden_size=self.embed_dim) + self.k_norm = RMSNorm(self.dummy_dim, + eps=config.layer_norm_eps, + var_hidden_size=self.embed_dim) + + self.proj = nn.Linear(self.dummy_dim, self.embed_dim) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + B, N, C = x.shape + qkv = self.qkv(x) + q, k, v = qkv.chunk(3, dim=-1) + + q = q.view(B, N, self.num_heads, self.head_dim) + k = k.view(B, N, self.num_heads, self.head_dim) + v = v.view(B, N, self.num_heads, self.head_dim) + + if self.qk_normalization: + B_, N_, H_, D_ = q.shape + q = self.q_norm(q.flatten(-2, -1)).view(B_, N_, H_, D_) + k = self.k_norm(k.flatten(-2, -1)).view(B_, N_, H_, D_) + q = q.transpose(1, 2) + k = k.transpose(1, 2) + v = v.transpose(1, 2) + + x = F.scaled_dot_product_attention(q, k, v, scale=self.scale) + x = x.transpose(1, 2).reshape(B, N, -1) + + x = self.proj(x) + return x + + +class InternMLP(nn.Module): + + def __init__( + self, + config: PretrainedConfig, + quant_config: Optional[QuantizationConfig] = None, + prefix: str = "", + ) -> None: + super().__init__() + + self.config = config + self.activation_fn = get_act_fn(config.hidden_act) + self.fc1 = ColumnParallelLinear(config.hidden_size, + config.intermediate_size, + bias=True, + quant_config=quant_config, + prefix=f"{prefix}.fc1") + self.fc2 = RowParallelLinear(config.intermediate_size, + config.hidden_size, + bias=True, + quant_config=quant_config, + prefix=f"{prefix}.fc2") + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + hidden_states, _ = self.fc1(hidden_states) + hidden_states = self.activation_fn(hidden_states) + hidden_states, _ = self.fc2(hidden_states) + + return hidden_states + + +class InternVisionEncoderLayer(nn.Module): + + def __init__( + self, + config: PretrainedConfig, + quant_config: Optional[QuantizationConfig] = None, + *, + num_dummy_heads: int = 0, + prefix: str = "", + ) -> None: + super().__init__() + + self.embed_dim = config.hidden_size + self.intermediate_size = config.intermediate_size + self.norm_type = config.norm_type + + self.attn = self._init_attn(config, + quant_config, + num_dummy_heads=num_dummy_heads, + prefix=f"{prefix}.attn") + + self.mlp = InternMLP(config, + quant_config=quant_config, + prefix=f"{prefix}.mlp") + self.norm1 = NORM2FN[self.norm_type](self.embed_dim, + eps=config.layer_norm_eps) + self.norm2 = NORM2FN[self.norm_type](self.embed_dim, + eps=config.layer_norm_eps) + + self.ls1 = nn.Parameter(config.initializer_factor * + torch.ones(self.embed_dim)) + self.ls2 = nn.Parameter(config.initializer_factor * + torch.ones(self.embed_dim)) + + def _init_attn( + self, + config: PretrainedConfig, + quant_config: Optional[QuantizationConfig], + *, + num_dummy_heads: int, + prefix: str = "", + ): + # fallback to sdpa attention if tp unavailable + tp_size = get_tensor_model_parallel_world_size() + num_heads = config.num_attention_heads + + if (num_heads + num_dummy_heads) % tp_size == 0: + return InternParallelAttention(config, + quant_config=quant_config, + num_dummy_heads=num_dummy_heads, + prefix=prefix) + + return InternSdpaAttention(config, num_dummy_heads=num_dummy_heads) + + def forward( + self, + hidden_states: torch.Tensor, + ): + hidden_states = hidden_states + self.attn( + self.norm1(hidden_states)) * self.ls1 + + hidden_states = hidden_states + self.mlp( + self.norm2(hidden_states)) * self.ls2 + + return hidden_states + + +class InternVisionEncoder(nn.Module): + + def __init__( + self, + config: PretrainedConfig, + quant_config: Optional[QuantizationConfig] = None, + *, + num_hidden_layers_override: Optional[int] = None, + num_dummy_heads: int = 0, + prefix: str = "", + ): + super().__init__() + + self.config = config + + if num_hidden_layers_override is None: + num_hidden_layers = config.num_hidden_layers + else: + num_hidden_layers = num_hidden_layers_override + + self.layers = nn.ModuleList([ + InternVisionEncoderLayer(config, + quant_config, + num_dummy_heads=num_dummy_heads, + prefix=f"{prefix}.layers.{layer_idx}") + for layer_idx in range(num_hidden_layers) + ]) + + def forward(self, inputs_embeds: torch.Tensor): + + hidden_states = inputs_embeds + for encoder_layer in self.layers: + hidden_states = encoder_layer(hidden_states) + + return hidden_states + + +class InternVisionModel(nn.Module): + + packed_modules_mapping = { + "qkv": ["qkv"], + } + + def __init__( + self, + config: PretrainedConfig, + quant_config: Optional[QuantizationConfig] = None, + *, + num_hidden_layers_override: Optional[int] = None, + num_dummy_heads: int = 0, + prefix: str = "", + ) -> None: + super().__init__() + + self.config = config + + self.embeddings = InternVisionEmbeddings(config) + self.encoder = InternVisionEncoder( + config=config, + quant_config=quant_config, + num_hidden_layers_override=num_hidden_layers_override, + num_dummy_heads=num_dummy_heads, + prefix=f"{prefix}.encoder", + ) + + def get_input_embeddings(self): + return self.embeddings + + def forward( + self, + pixel_values: Optional[torch.Tensor] = None, + pixel_embeds: Optional[torch.Tensor] = None, + ) -> torch.FloatTensor: + if pixel_values is None and pixel_embeds is None: + raise ValueError( + 'You have to specify pixel_values or pixel_embeds') + + if pixel_embeds is not None: + hidden_states = pixel_embeds + elif pixel_values is not None: + if pixel_values.ndim == 4: + hidden_states = self.embeddings(pixel_values) + else: + raise ValueError( + f'wrong pixel_values size: {pixel_values.shape}') + + encoder_outputs = self.encoder(inputs_embeds=hidden_states) + + return encoder_outputs + + def load_weights(self, weights: Iterable[tuple[str, + torch.Tensor]]) -> set[str]: + params_dict = dict(self.named_parameters()) + loaded_params: set[str] = set() + for name, loaded_weight in weights: + param = params_dict[name] + weight_loader = getattr(param, "weight_loader", + default_weight_loader) + weight_loader(param, loaded_weight) + loaded_params.add(name) + return loaded_params diff --git a/vllm_kunlun/models/internlm2.py b/vllm_kunlun/models/internlm2.py new file mode 100644 index 0000000..7f1c241 --- /dev/null +++ b/vllm_kunlun/models/internlm2.py @@ -0,0 +1,450 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from collections.abc import Iterable +from functools import partial +from typing import Any, Optional, Union + +import torch +from torch import nn +from transformers import PretrainedConfig + +# from vllm.attention import Attention +from vllm_kunlun.ops.attention.layer import Attention +from vllm.compilation.decorators import support_torch_compile +from vllm.config import CacheConfig, VllmConfig +from vllm.distributed import (get_pp_group, get_tensor_model_parallel_rank, + get_tensor_model_parallel_world_size, + split_tensor_along_last_dim, + tensor_model_parallel_all_gather) +# from vllm.model_executor.layers.activation import SiluAndMul +from vllm_kunlun.ops.activation import SiluAndMul +from vllm.model_executor.layers.layernorm import RMSNorm +from vllm.model_executor.layers.linear import (MergedColumnParallelLinear, + QKVParallelLinear, + RowParallelLinear) +from vllm.model_executor.layers.logits_processor import LogitsProcessor +from vllm.model_executor.layers.pooler import DispatchPooler, Pooler +from vllm.model_executor.layers.quantization import QuantizationConfig +from vllm.model_executor.layers.rotary_embedding import get_rope +from vllm.model_executor.layers.vocab_parallel_embedding import ( + ParallelLMHead, VocabParallelEmbedding) +from vllm.model_executor.model_loader.weight_utils import default_weight_loader +from vllm.model_executor.sampling_metadata import SamplingMetadata +from vllm.sequence import IntermediateTensors + +from vllm.model_executor.models.interfaces import SupportsLoRA, SupportsPP, default_pooling_type +from vllm.model_executor.models.utils import (is_pp_missing_parameter, + make_empty_intermediate_tensors_factory, make_layers, + maybe_prefix) + + +class InternLM2MLP(nn.Module): + + def __init__( + self, + hidden_size: int, + intermediate_size: int, + hidden_act: str, + quant_config: Optional[QuantizationConfig] = None, + prefix: str = "", + ) -> None: + super().__init__() + self.gate_up_proj = MergedColumnParallelLinear( + hidden_size, + [intermediate_size] * 2, + bias=False, + quant_config=quant_config, + prefix=f"{prefix}.gate_up_proj", + ) + self.w2 = RowParallelLinear( + intermediate_size, + hidden_size, + bias=False, + quant_config=quant_config, + prefix=f"{prefix}.w2", + ) + if hidden_act != "silu": + raise ValueError(f"Unsupported activation: {hidden_act}. " + "Only silu is supported for now.") + self.act_fn = SiluAndMul() + + def forward(self, x): + gate_up, _ = self.gate_up_proj(x) + x = self.act_fn(gate_up) + x, _ = self.w2(x) + return x + + +class InternLM2Attention(nn.Module): + + def __init__( + self, + hidden_size: int, + num_heads: int, + num_kv_heads: int, + rope_theta: float = 10000, + rope_scaling: Optional[dict[str, Any]] = None, + max_position_embeddings: int = 8192, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + prefix: str = "", + ) -> None: + super().__init__() + self.hidden_size = hidden_size + self.tp_size = get_tensor_model_parallel_world_size() + self.tp_rank = get_tensor_model_parallel_rank() + self.total_num_heads = num_heads + assert self.total_num_heads % self.tp_size == 0 + self.num_heads = self.total_num_heads // self.tp_size + self.total_num_kv_heads = num_kv_heads + if self.total_num_kv_heads >= self.tp_size: + # Number of KV heads is greater than TP size, so we partition + # the KV heads across multiple tensor parallel GPUs. + assert self.total_num_kv_heads % self.tp_size == 0 + else: + # Number of KV heads is less than TP size, so we replicate + # the KV heads across multiple tensor parallel GPUs. + assert self.tp_size % self.total_num_kv_heads == 0 + self.num_kv_heads = max(1, self.total_num_kv_heads // self.tp_size) + self.head_dim = hidden_size // self.total_num_heads + self.q_size = self.num_heads * self.head_dim + self.kv_size = self.num_kv_heads * self.head_dim + self.key_value_groups = int(self.num_heads / self.num_kv_heads) + self.scaling = self.head_dim**-0.5 + self.rope_theta = rope_theta + self.max_position_embeddings = max_position_embeddings + + self.wqkv = QKVParallelLinear( + hidden_size, + self.head_dim, + self.total_num_heads, + self.total_num_kv_heads, + bias=False, + quant_config=quant_config, + prefix=f"{prefix}.wqkv", + ) + self.wo = RowParallelLinear( + self.total_num_heads * self.head_dim, + hidden_size, + bias=False, + quant_config=quant_config, + prefix=f"{prefix}.wo", + ) + + self.rotary_emb = get_rope( + self.head_dim, + rotary_dim=self.head_dim, + max_position=max_position_embeddings, + base=rope_theta, + rope_scaling=rope_scaling, + ) + self.attn = Attention( + self.num_heads, + self.head_dim, + self.scaling, + num_kv_heads=self.num_kv_heads, + cache_config=cache_config, + quant_config=quant_config, + prefix=f"{prefix}.attn", + ) + + def split_qkv(self, qkv: torch.Tensor): + seq_len = qkv.shape[0] + if self.tp_size > 1: + qkv_map = [self.q_size, self.kv_size, self.kv_size] * self.tp_size + qkv = tensor_model_parallel_all_gather(qkv) + qkv = torch.split(qkv, qkv_map, dim=-1) + qkv = qkv[::3] + qkv[1::3] + qkv[2::3] + qkv = torch.cat(qkv, dim=-1) + + qkv = qkv.view(seq_len, self.total_num_kv_heads, + self.key_value_groups + 2, self.head_dim) + q, k, v = torch.split(qkv, [self.key_value_groups, 1, 1], dim=-2) + q = q.reshape(seq_len, self.q_size * self.tp_size) + k = k.reshape(seq_len, self.kv_size * self.tp_size) + v = v.reshape(seq_len, self.kv_size * self.tp_size) + + if self.tp_size > 1: + splitter = partial(split_tensor_along_last_dim, + num_partitions=self.tp_size) + q = splitter(q)[self.tp_rank] + k = splitter(k)[self.tp_rank] + v = splitter(v)[self.tp_rank] + return q, k, v + + def forward( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + ) -> torch.Tensor: + qkv, _ = self.wqkv(hidden_states) + q, k, v = self.split_qkv(qkv) + q, k = self.rotary_emb(positions, q, k) + attn_output = self.attn(q, k, v) + output, _ = self.wo(attn_output) + return output + + +class InternLMDecoderLayer(nn.Module): + + def __init__( + self, + config: PretrainedConfig, + cache_config: Optional[CacheConfig] = None, + quant_config: Optional[QuantizationConfig] = None, + prefix: str = "", + ) -> None: + super().__init__() + self.hidden_size = config.hidden_size + rope_theta = getattr(config, "rope_theta", 10000) + rope_scaling = getattr(config, "rope_scaling", None) + max_position_embeddings = getattr(config, "max_position_embeddings", + 8192) + self.attention = InternLM2Attention( + hidden_size=self.hidden_size, + num_heads=config.num_attention_heads, + num_kv_heads=config.num_key_value_heads, + rope_theta=rope_theta, + rope_scaling=rope_scaling, + max_position_embeddings=max_position_embeddings, + cache_config=cache_config, + quant_config=quant_config, + prefix=f"{prefix}.attention", + ) + self.feed_forward = InternLM2MLP( + hidden_size=self.hidden_size, + intermediate_size=config.intermediate_size, + hidden_act=config.hidden_act, + quant_config=quant_config, + prefix=f"{prefix}.feed_forward", + ) + self.attention_norm = RMSNorm(config.hidden_size, + eps=config.rms_norm_eps) + self.ffn_norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) + + def forward( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + residual: Optional[torch.Tensor], + ) -> tuple[torch.Tensor, torch.Tensor]: + # Self Attention + if residual is None: + residual = hidden_states + hidden_states = self.attention_norm(hidden_states) + else: + hidden_states, residual = self.attention_norm( + hidden_states, residual) + hidden_states = self.attention( + positions=positions, + hidden_states=hidden_states, + ) + + # Fully Connected + hidden_states, residual = self.ffn_norm(hidden_states, residual) + hidden_states = self.feed_forward(hidden_states) + return hidden_states, residual + + +@support_torch_compile +class InternLM2Model(nn.Module): + + def __init__( + self, + *, + vllm_config: VllmConfig, + prefix: str = "", + layer_type: type[InternLMDecoderLayer] = InternLMDecoderLayer): + super().__init__() + + config = vllm_config.model_config.hf_config + cache_config = vllm_config.cache_config + quant_config = vllm_config.quant_config + + self.config = config + self.vocab_size = config.vocab_size + self.tok_embeddings = VocabParallelEmbedding( + config.vocab_size, + config.hidden_size, + ) + self.start_layer, self.end_layer, self.layers = make_layers( + config.num_hidden_layers, + lambda prefix: layer_type( + config, cache_config, quant_config, prefix=prefix), + prefix=f"{prefix}.layers") + self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) + self.make_empty_intermediate_tensors = ( + make_empty_intermediate_tensors_factory( + ["hidden_states", "residual"], config.hidden_size)) + + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor: + return self.tok_embeddings(input_ids) + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + intermediate_tensors: Optional[IntermediateTensors] = None, + inputs_embeds: Optional[torch.Tensor] = None, + ) -> Union[torch.Tensor, IntermediateTensors]: + if get_pp_group().is_first_rank: + if inputs_embeds is not None: + hidden_states = inputs_embeds + else: + hidden_states = self.get_input_embeddings(input_ids) + residual = None + else: + assert intermediate_tensors is not None + hidden_states = intermediate_tensors["hidden_states"] + residual = intermediate_tensors["residual"] + for layer in self.layers[self.start_layer:self.end_layer]: + hidden_states, residual = layer(positions, hidden_states, residual) + if not get_pp_group().is_last_rank: + return IntermediateTensors({ + "hidden_states": hidden_states, + "residual": residual + }) + hidden_states, _ = self.norm(hidden_states, residual) + return hidden_states + + +class InternLM2ForCausalLM(nn.Module, SupportsPP, SupportsLoRA): + packed_modules_mapping = { + "wqkv": ["wqkv"], + "gate_up_proj": ["w1", "w3"], + } + + def __init__(self, + *, + vllm_config: VllmConfig, + prefix: str = "", + model_type: type[InternLM2Model] = InternLM2Model): + super().__init__() + config = vllm_config.model_config.hf_config + quant_config = vllm_config.quant_config + lora_config = vllm_config.lora_config + + self.config = config + self.quant_config = quant_config + self.lora_config = lora_config + + self.model = model_type(vllm_config=vllm_config, + prefix=maybe_prefix(prefix, "model")) + self.output = ParallelLMHead(config.vocab_size, + config.hidden_size, + quant_config=quant_config, + prefix=maybe_prefix(prefix, "output")) + if self.config.tie_word_embeddings: + self.output.weight = self.model.tok_embeddings.weight + self.logits_processor = LogitsProcessor(config.vocab_size) + self.make_empty_intermediate_tensors = ( + self.model.make_empty_intermediate_tensors) + + def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor: + return self.model.get_input_embeddings(input_ids) + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + intermediate_tensors: Optional[IntermediateTensors], + inputs_embeds: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + hidden_states = self.model(input_ids, positions, intermediate_tensors, + inputs_embeds) + return hidden_states + + def compute_logits( + self, + hidden_states: torch.Tensor, + sampling_metadata: SamplingMetadata, + ) -> Optional[torch.Tensor]: + logits = self.logits_processor(self.output, hidden_states, + sampling_metadata) + return logits + + def load_weights(self, weights: Iterable[tuple[str, + torch.Tensor]]) -> set[str]: + stacked_params_mapping = [ + # (param_name, shard_name, shard_id) + ("gate_up_proj", "w1", 0), + ("gate_up_proj", "w3", 1), + ] + params_dict = dict(self.named_parameters()) + loaded_params: set[str] = set() + for name, loaded_weight in weights: + if "rotary_emb.inv_freq" in name: + continue + for (param_name, weight_name, shard_id) in stacked_params_mapping: + if weight_name not in name: + continue + name = name.replace(weight_name, param_name) + # Skip loading extra bias for GPTQ models. + if name.endswith(".bias") and name not in params_dict: + continue + if is_pp_missing_parameter(name, self): + continue + param = params_dict[name] + weight_loader = param.weight_loader + weight_loader(param, loaded_weight, shard_id) + break + else: + # Skip loading extra bias for GPTQ models. + if name.endswith(".bias") and name not in params_dict: + continue + if is_pp_missing_parameter(name, self): + continue + param = params_dict[name] + weight_loader = getattr(param, "weight_loader", + default_weight_loader) + weight_loader(param, loaded_weight) + loaded_params.add(name) + return loaded_params + + +@default_pooling_type("ALL") +class InternLM2ForRewardModel(InternLM2ForCausalLM): + + is_pooling_model = True + + def __init__( + self, + *, + vllm_config: VllmConfig, + prefix: str = "", + model_type: type[InternLM2Model] = InternLM2Model, + ): + super().__init__(vllm_config=vllm_config, + prefix=prefix, + model_type=model_type) + + for attr in ("output", "logits_processor"): + delattr(self, attr) + + config = vllm_config.model_config.hf_config + self.v_head = RowParallelLinear( + config.hidden_size, + 1, + bias=False, + input_is_parallel=False, + prefix=maybe_prefix(prefix, "v_head"), + ) + + pooler_config = vllm_config.model_config.pooler_config + assert pooler_config is not None + + self.pooler = DispatchPooler( + {"encode": Pooler.for_encode(pooler_config)}, ) + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + intermediate_tensors: Optional[IntermediateTensors] = None, + inputs_embeds: Optional[torch.Tensor] = None, + ) -> Union[torch.Tensor, IntermediateTensors]: + hidden_states = self.model(input_ids, positions, intermediate_tensors, + inputs_embeds) + logits, _ = self.v_head(hidden_states) + return logits diff --git a/vllm_kunlun/models/interns1.py b/vllm_kunlun/models/interns1.py new file mode 100644 index 0000000..8fd23d8 --- /dev/null +++ b/vllm_kunlun/models/interns1.py @@ -0,0 +1,869 @@ +# +# Copyright (c) 2025 Baidu, Inc. All Rights Reserved. +# Adapted from vllm/model_executor/models/interns1.py +# Copyright 2023 The vLLM team. +# +# This file is a part of the vllm-kunlun project. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from collections.abc import Iterable, Mapping, Sequence +from typing import Literal, Optional, TypedDict, Union + +import regex as re +import torch +import torch.nn as nn +from transformers import BatchFeature, InternVLProcessor, PretrainedConfig +from transformers.activations import ACT2FN +from transformers.models.got_ocr2.image_processing_got_ocr2_fast import ( + GotOcr2ImageProcessorFast) + +from vllm.config import VllmConfig +from vllm.model_executor.layers.quantization import QuantizationConfig +from .interns1_vit import InternS1VisionModel +from vllm.model_executor.models.module_mapping import MultiModelKeys +from vllm.model_executor.sampling_metadata import SamplingMetadata +from vllm.multimodal import MULTIMODAL_REGISTRY +from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig, + MultiModalKwargs, NestedTensors) +from vllm.multimodal.parse import (ImageEmbeddingItems, ImageProcessorItems, + ImageSize, MultiModalDataItems) +from vllm.multimodal.processing import (BaseMultiModalProcessor, + BaseProcessingInfo, PromptReplacement, + PromptUpdate, PromptUpdateDetails) +from vllm.multimodal.profiling import BaseDummyInputsBuilder +from vllm.sequence import IntermediateTensors +from vllm.model_executor.models.interfaces import (MultiModalEmbeddings, SupportsLoRA, + SupportsMultiModal, SupportsPP) +from vllm.model_executor.models.utils import (AutoWeightsLoader, WeightsMapper, flatten_bn, + init_vllm_registered_model, maybe_prefix, + merge_multimodal_embeddings) + + +class InternS1MultiModalProjector(nn.Module): + + def __init__(self, config): + super().__init__() + self.layer_norm = nn.LayerNorm(config.vision_config.hidden_size * + int(1 / config.downsample_ratio)**2) + self.linear_1 = nn.Linear( + config.vision_config.hidden_size * + int(1 / config.downsample_ratio)**2, + config.text_config.hidden_size) + self.act = ACT2FN[config.projector_hidden_act] + self.linear_2 = nn.Linear(config.text_config.hidden_size, + config.text_config.hidden_size) + + def forward(self, image_features): + hidden_states = self.layer_norm(image_features) + hidden_states = self.linear_1(hidden_states) + hidden_states = self.act(hidden_states) + hidden_states = self.linear_2(hidden_states) + return hidden_states + + +class InternS1ImagePixelInputs(TypedDict): + type: Literal["pixel_values"] + pixel_values: torch.Tensor + """ + Shape: + `(batch_size * num_images * (1 + num_patches), num_channels, height, width)` + """ + + +class InternS1ImageEmbeddingInputs(TypedDict): + type: Literal["image_embeds"] + data: Union[torch.Tensor, list[torch.Tensor]] + """ + A tensor of shape `(num_images, total_image_feature_size, hidden_size)` + or a list of tensors of shape `(total_image_feature_size, hidden_size)` + + `hidden_size` must match the hidden size of language model backbone. + """ + + +InternS1ImageInputs = Union[InternS1ImagePixelInputs, + InternS1ImageEmbeddingInputs] + + +class InternS1VideoPixelInputs(TypedDict): + type: Literal["pixel_values_videos"] + pixel_values: torch.Tensor + """ + Shape: + `(batch_size * num_video * num_frames, num_channels, height, width)` + """ + + num_patches: torch.Tensor + """Shape: `(batch_size * num_images)`""" + + +class InternS1VideoEmbeddingInputs(TypedDict): + type: Literal["video_embeds"] + data: Union[torch.Tensor, list[torch.Tensor]] + """ + A tensor of shape `(num_videos, total_video_feature_size, hidden_size)` + or a list of tensors of shape `(total_video_feature_size, hidden_size)` + + `hidden_size` must match the hidden size of language model backbone. + """ + + +InternS1VideoInputs = Union[InternS1VideoPixelInputs, + InternS1VideoEmbeddingInputs] + + +def resolve_interns1_min_max_num( + min_dynamic_patch: int, + max_dynamic_patch: int, + dynamic_image_size: bool, + use_thumbnail: bool, +) -> tuple[int, int]: + min_dynamic_patch = min_dynamic_patch if dynamic_image_size else 1 + max_dynamic_patch = max_dynamic_patch if dynamic_image_size else 1 + + if use_thumbnail and max_dynamic_patch != 1: + max_dynamic_patch += 1 + + return min_dynamic_patch, max_dynamic_patch + + +def get_interns1_target_ratios( + min_num: int, + max_num: int, +) -> list[tuple[int, int]]: + target_ratios = {(i, j) + for n in range(min_num, max_num + 1) + for i in range(1, n + 1) + for j in range(1, n + 1) if min_num <= i * j <= max_num} + return sorted(target_ratios, key=lambda x: x[0] * x[1]) + + +class InternS1ProcessingInfo(BaseProcessingInfo): + """ProcessingInfo for InternS1-style models.""" + + def get_hf_processor(self, **kwargs: object) -> InternVLProcessor: + return self.ctx.get_hf_processor(InternVLProcessor, **kwargs) + + def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]: + return {"image": None, "video": None} + + def get_num_image_tokens( + self, + *, + image_width: int, + image_height: int, + processor: Optional['GotOcr2ImageProcessorFast'] = None, + ) -> int: + if processor is None: + processor = self.get_hf_processor().image_processor + + if not isinstance(processor, GotOcr2ImageProcessorFast): + raise ValueError(f'GotOcr2ImageProcessorFast is expected but got ' + f'{type(processor)}') + num_image_patches = processor.get_number_of_image_patches( + image_height, image_width, images_kwargs=dict()) + num_image_tokens = self.get_hf_processor( + ).image_seq_length * num_image_patches + return num_image_tokens + + def resolve_target_ratios(self, use_thumbnail: Optional[bool] = None): + image_processor = self.get_hf_processor().image_processor + min_dynamic_patch = image_processor.min_patches + max_dynamic_patch = image_processor.max_patches + # HF format's InternVL processor uses `crop_to_patches` which is + # equivalent to `use_thumbnail` in original format. + use_thumbnail = image_processor.crop_to_patches + dynamic_image_size = True + min_num, max_num = resolve_interns1_min_max_num( + min_dynamic_patch, + max_dynamic_patch, + dynamic_image_size, + use_thumbnail=use_thumbnail) + + return get_interns1_target_ratios(min_num, max_num) + + def get_image_size_with_most_features(self) -> ImageSize: + processor = self.get_hf_processor() + + hf_config = self.ctx.get_hf_config() + base_height, base_width = hf_config.vision_config.image_size + target_ratios = self.resolve_target_ratios() + + largest_feature_size, largest_feature_pinpoint = 0, None + for wr, hr in target_ratios: + width, height = base_width * wr, base_height * hr + + feat_size = self.get_num_image_tokens( + image_width=width, + image_height=height, + processor=processor.image_processor, + ) + if feat_size > largest_feature_size: + largest_feature_size = feat_size + largest_feature_pinpoint = ImageSize(width=width, + height=height) + + assert not (largest_feature_size == 0 or largest_feature_pinpoint + is None), ("Cannot have a largest feature size of 0!") + + return largest_feature_pinpoint + + def get_max_image_tokens(self) -> int: + processor = self.get_hf_processor() + target_width, target_height = self.get_image_size_with_most_features() + + return self.get_num_image_tokens( + image_width=target_width, + image_height=target_height, + processor=processor.image_processor, + ) + + def get_num_frames_with_most_features( + self, + seq_len: int, + mm_counts: Mapping[str, int], + ) -> int: + max_images = mm_counts.get("image", 0) + max_videos = mm_counts.get("video", 0) + + processor = self.get_hf_processor() + + max_image_tokens = self.get_max_image_tokens() * max_images + max_total_frames = (seq_len - + max_image_tokens) // processor.image_seq_length + max_frames_per_video = max_total_frames // max(max_videos, 1) + + return max(max_frames_per_video, 1) + + +class InternS1DummyInputsBuilder(BaseDummyInputsBuilder[InternS1ProcessingInfo] + ): + """DummyInputsBuilder for InternS1-style models.""" + + def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str: + num_images = mm_counts.get("image", 0) + num_videos = mm_counts.get("video", 0) + image_token = self.info.get_hf_processor().image_token + video_token = self.info.get_hf_processor().video_token + + return image_token * num_images + video_token * num_videos + + + def get_dummy_mm_data( + self, + seq_len: int, + mm_counts: Mapping[str, int], + ) -> MultiModalDataDict: + """Generates dummy multimodal data on Kunlun3 platform for performance analysis and warmup. + + Retrieves visual resolution based on configuration (defaulting to 224x224) + and generates resized dummy data for images and videos. + + Args: + seq_len: Sequence length (unused). + mm_counts: A mapping of multimodal type counts, containing "image" + and "video" keys. + + Returns: + MultiModalDataDict: A dictionary containing the generated dummy image + and video data, structured as: + { + "image": dummy_image_data, + "video": dummy_video_data + } + + Author: + Dong Xinyu + """ + config = self.info.get_hf_config() + img_size = getattr(config.vision_config, "image_size", None) + if isinstance(img_size, (tuple, list)) and len(img_size) == 2: + cfg_h, cfg_w = int(img_size[0]), int(img_size[1]) + else: + cfg_h, cfg_w = 224, 224 + + target_width = min(cfg_w, 224) + target_height = min(cfg_h, 224) + target_num_frames = 1 + + num_images = mm_counts.get("image", 0) + num_videos = mm_counts.get("video", 0) + + return { + "image": self._get_dummy_images( + width=target_width, + height=target_height, + num_images=num_images, + ), + "video": self._get_dummy_videos( + width=target_width, + height=target_height, + num_frames=target_num_frames, + num_videos=num_videos, + ), + } + + +class InternS1MultiModalProcessor( + BaseMultiModalProcessor[InternS1ProcessingInfo]): + """ Basic image-only MultiModalProcessor for InternS1-style models.""" + + def _call_hf_processor( + self, + prompt: str, + mm_data: Mapping[str, object], + mm_kwargs: Mapping[str, object], + tok_kwargs: Mapping[str, object], + ) -> Mapping[str, NestedTensors]: + mm_data = dict(mm_data) + videos = mm_data.pop("videos", []) + images = mm_data.pop("images", []) + assert isinstance(videos, list) + assert isinstance(images, list) + + hf_processor = self.info.get_hf_processor(**mm_kwargs) + tokenizer = hf_processor.tokenizer + video_token_id = tokenizer.encode(hf_processor.video_token, + add_special_tokens=False) + assert len(video_token_id) == 1 + video_token_id = video_token_id[0] + + prompt = re.sub(hf_processor.image_token, "", + prompt) + prompt = re.sub(hf_processor.video_token, "", + prompt) + + image_outputs = {} + if images: + image_pixel_values = [] + for image in images: + processed_outputs = super()._call_hf_processor( + prompt=hf_processor.image_token, + mm_data={"images": image}, + mm_kwargs=mm_kwargs, + tok_kwargs=tok_kwargs, + ) + image_pixel_values.append( + processed_outputs.pop("pixel_values")) + + input_ids = processed_outputs.pop("input_ids") + image_placeholder = tokenizer.batch_decode(input_ids)[0] + prompt = prompt.replace("", + image_placeholder, 1) + + num_patches = [len(item) for item in image_pixel_values] + image_outputs: dict[str, NestedTensors] = { + "pixel_values": torch.concat(image_pixel_values), + "image_num_patches": torch.tensor(num_patches), + "image_token_id": torch.tensor(hf_processor.image_token_id), + } + + video_outputs = {} + if videos: + video_pixel_values = [] + for video in videos: + processed_outputs = super()._call_hf_processor( + prompt=hf_processor.video_token, + mm_data={"videos": video}, + mm_kwargs=mm_kwargs, + tok_kwargs=tok_kwargs, + ) + video_pixel_values.append( + processed_outputs.pop("pixel_values")) + + input_ids = processed_outputs.pop("input_ids") + input_ids[input_ids == + hf_processor.image_token_id] = video_token_id + + video_placeholder = tokenizer.batch_decode(input_ids)[0] + prompt = prompt.replace("", + video_placeholder, 1) + + num_frames = [len(item) for item in video_pixel_values] + video_outputs: dict[str, NestedTensors] = { + "pixel_values_videos": torch.concat(video_pixel_values), + "video_num_patches": torch.tensor(num_frames), + "video_token_id": torch.tensor(video_token_id), + } + + prompt = re.sub("", hf_processor.image_token, + prompt) + prompt = re.sub("", hf_processor.video_token, + prompt) + text_outputs = tokenizer(prompt, **tok_kwargs, return_tensors="pt") + + combined_outputs = dict( + **text_outputs, + **image_outputs, + **video_outputs, + ) + return BatchFeature(combined_outputs) + + def _get_mm_fields_config( + self, + hf_inputs: Mapping[str, NestedTensors], + hf_processor_mm_kwargs: Mapping[str, object], + ) -> Mapping[str, MultiModalFieldConfig]: + + image_num_patches = hf_inputs.get("image_num_patches", torch.empty(0)) + video_num_patches = hf_inputs.get("video_num_patches", torch.empty(0)) + num_images = len(image_num_patches) + num_videos = len(video_num_patches) + + return dict( + pixel_values=MultiModalFieldConfig.flat_from_sizes( + "image", image_num_patches), + image_num_patches=MultiModalFieldConfig.batched("image"), + image_embeds=MultiModalFieldConfig.batched("image"), + image_token_id=MultiModalFieldConfig.shared("image", num_images), + pixel_values_videos=MultiModalFieldConfig.flat_from_sizes( + "video", video_num_patches), + video_num_patches=MultiModalFieldConfig.batched("video"), + video_token_id=MultiModalFieldConfig.shared("video", num_videos), + ) + + def _get_prompt_updates( + self, + mm_items: MultiModalDataItems, + hf_processor_mm_kwargs: Mapping[str, object], + out_mm_kwargs: MultiModalKwargs, + ) -> Sequence[PromptUpdate]: + hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs) + img_context_token = hf_processor.image_token + start_image_token = hf_processor.start_image_token + end_image_token = hf_processor.end_image_token + video_token = hf_processor.video_token + + if "video_num_patches" in out_mm_kwargs: + video_num_patches = out_mm_kwargs["video_num_patches"] + assert isinstance(video_num_patches, torch.Tensor) + video_num_patches = video_num_patches.tolist() + else: + video_num_patches = [] + + if "image_num_patches" in out_mm_kwargs: + image_num_patches = out_mm_kwargs["image_num_patches"] + assert isinstance(image_num_patches, torch.Tensor) + image_num_patches = image_num_patches.tolist() + else: + image_num_patches = [] + + def get_replacement_interns1_image(item_idx: int): + images = mm_items.get_items( + "image", (ImageEmbeddingItems, ImageProcessorItems)) + + if isinstance(images, ImageEmbeddingItems): + feature_size = images.get_feature_size(item_idx) + else: + num_patches = image_num_patches[item_idx] + feature_size = num_patches * hf_processor.image_seq_length + + repl_features = img_context_token * feature_size + repl_full = start_image_token + repl_features + end_image_token + return PromptUpdateDetails.select_text(repl_full, + img_context_token) + + def get_replacement_interns1_video(item_idx: int): + num_patches = video_num_patches[item_idx] + repl_features = video_token * hf_processor.image_seq_length + repl_features_with_sep = (start_image_token + repl_features + + end_image_token) + # num_patches is equal to num_frames + repl_full = '\n'.join([ + f'Frame{i+1}: {repl_features_with_sep}' + for i in range(num_patches) + ]) + + return PromptUpdateDetails.select_text(repl_full, video_token) + + return [ + PromptReplacement( + modality="image", + target=img_context_token, + replacement=get_replacement_interns1_image, + ), + PromptReplacement( + modality="video", + target=video_token, + replacement=get_replacement_interns1_video, + ), + ] + + +@MULTIMODAL_REGISTRY.register_processor( + InternS1MultiModalProcessor, + info=InternS1ProcessingInfo, + dummy_inputs=InternS1DummyInputsBuilder) +class InternS1ForConditionalGeneration(nn.Module, SupportsMultiModal, + SupportsPP, SupportsLoRA): + + # To ensure correct weight loading and mapping. + hf_to_vllm_mapper = WeightsMapper( + orig_to_new_prefix={ + "lm_head.": "language_model.lm_head.", + "model.language_model.": "language_model.model.", + "model.vision_tower.": "vision_tower.", + "model.multi_modal_projector.": "multi_modal_projector.", + }) + + @classmethod + def get_placeholder_str(cls, modality: str, i: int) -> Optional[str]: + # transformers InternVLProcessor uses as the seperator + # refer to https://github.com/huggingface/transformers/blob/f90de364c2484c7c325bbe05befdcf487bd75b63/src/transformers/models/internvl/processing_internvl.py#L116 + if modality.startswith("image"): + return '' + if modality.startswith("video"): + return "