From 427b17e2da1d6d967e7581b40b6bd169cdb324c8 Mon Sep 17 00:00:00 2001 From: Rui Kang Date: Thu, 23 Oct 2025 15:56:07 +0800 Subject: [PATCH] [Misc] Add a model loader that utilizes HCCL for weight loading (#2888) ### What this PR does / why we need it? This PR introduces a new model loader called Netloader, which leverages high-bandwidth P2P direct transfer between NPU cards to achieve weight loading. Netloader is implemented as a plugin through the newly added 'register_model_loader' function in vLLM 0.10. It facilitates the process of weight loading by sending weights from a pre-loaded model (server) to an empty model of a newly started instance (client). The server operates concurrently with normal inference tasks through sub-threads and the 'stateless_init_torch_distributed_process_group' in vLLM. The client initiates a transfer request after verifying that the model and partitioning method are the same as the server's, and uses HCCL's collective communication (send/recv) to load the weights in the order they are stored in the model. Application Scenarios: 1. Significantly Reduces Inference Instance Startup Time By reusing the weights of already loaded instances and performing high-speed transfers directly between computing cards, this method reduces model loading latency compared to traditional remote/local pull methods. 2. Reduces Network and Storage Pressure Avoids the need to repeatedly download weight files from remote repositories, reducing the impact on centralized storage and network traffic, thereby enhancing overall system stability and service quality. 3. Improves Resource Utilization and Reduces Costs Accelerating the loading process reduces reliance on redundant computing pools, allowing computing resources to be elastically scaled and reclaimed as needed. 4. Enhances Business Continuity and High Availability In fault recovery scenarios, new instances can quickly take over existing services, avoiding prolonged business interruptions and improving the system's high availability and user experience. ### Does this PR introduce _any_ user-facing change? Netloader utilizes the existing --load-format=netloader and --model-loader-extra-config to be activated. The model-loader-extra-config needs to be input as a JSON string (as it is now) Afterwards, you can check whether the outputs for the same sentence are consistent when the temperature is set to 0. Signed-off-by: destinysky - vLLM version: v0.11.0rc3 - vLLM main: https://github.com/vllm-project/vllm/commit/v0.11.0 --------- Signed-off-by: destinysky --- .../images/netloader_flowchart.png | Bin 0 -> 40208 bytes .../images/netloader_timing_diagram.png | Bin 0 -> 37598 bytes docs/source/user_guide/feature_guide/index.md | 1 + .../user_guide/feature_guide/netloader.md | 97 ++++ setup.py | 3 +- .../model_loader/netloader/test_netloader.py | 215 +++++++++ .../netloader/test_netloader_elastic.py | 432 ++++++++++++++++++ .../netloader/test_netloader_load.py | 114 +++++ .../netloader/test_netloader_utils.py | 61 +++ vllm_ascend/__init__.py | 5 + vllm_ascend/model_loader/__init__.py | 0 .../model_loader/netloader/__init__.py | 20 + .../netloader/executor/__init__.py | 0 .../netloader/executor/elastic_load.py | 170 +++++++ .../netloader/interaction/__init__.py | 0 .../netloader/interaction/elastic.py | 408 +++++++++++++++++ vllm_ascend/model_loader/netloader/load.py | 84 ++++ .../model_loader/netloader/netloader.py | 324 +++++++++++++ vllm_ascend/model_loader/netloader/utils.py | 66 +++ 19 files changed, 1999 insertions(+), 1 deletion(-) create mode 100644 docs/source/user_guide/feature_guide/images/netloader_flowchart.png create mode 100644 docs/source/user_guide/feature_guide/images/netloader_timing_diagram.png create mode 100644 docs/source/user_guide/feature_guide/netloader.md create mode 100644 tests/ut/model_loader/netloader/test_netloader.py create mode 100644 tests/ut/model_loader/netloader/test_netloader_elastic.py create mode 100644 tests/ut/model_loader/netloader/test_netloader_load.py create mode 100644 tests/ut/model_loader/netloader/test_netloader_utils.py create mode 100644 vllm_ascend/model_loader/__init__.py create mode 100644 vllm_ascend/model_loader/netloader/__init__.py create mode 100644 vllm_ascend/model_loader/netloader/executor/__init__.py create mode 100644 vllm_ascend/model_loader/netloader/executor/elastic_load.py create mode 100644 vllm_ascend/model_loader/netloader/interaction/__init__.py create mode 100644 vllm_ascend/model_loader/netloader/interaction/elastic.py create mode 100644 vllm_ascend/model_loader/netloader/load.py create mode 100644 vllm_ascend/model_loader/netloader/netloader.py create mode 100644 vllm_ascend/model_loader/netloader/utils.py diff --git a/docs/source/user_guide/feature_guide/images/netloader_flowchart.png b/docs/source/user_guide/feature_guide/images/netloader_flowchart.png new file mode 100644 index 0000000000000000000000000000000000000000..4efe6b2b806fe0143a525bbacb3ec76d80eb0cf8 GIT binary patch literal 40208 zcmZ5{1yojB*EJwWDc$uTA>G{|(%l_OBi$V$AT8b9E!`pA-QC^Y{0F`F`hNfL42R>m z_t~}fT64|0o&Z^Ck+*O-aA06yZ^cAE%7cNuG6e&JK!AY&-Uw@$WPyPpfr)(-P;>%6 zXp%C+6RFcs%m0 zQatcSP0f>tii&BigM-W?yptV0Jp9rV-{{Hpxas`)Cd^Xzl{Y7HS|R5-8Q51sFbG0f zScr?TLg{CJxma)1G%CvD=ts}|0Z+cC8@+}{99)=*05Gk|cs_6>Nbdq%{sy4t&YiKMnb-U|JJL zNr0Ek^Y~S-|Fi=B_NIjeK8S)B=*=Gu)B11!9T51put&YO5-;oh^Q@p?^)Y=56Pz`( z!udPN*Fs>c>uEjhv@l+O`+E;SBJmmuLkavlo0rkD3($Epg#KF{@DEC9uvNWIT$rPm zwX|9xdBp-lWeNfx+^J+Sh5L74FbF6l{u@=W4$6Ely1(<0r3I$AdFJcz&mcX*VDg0Y z%)rP9QVPH1{tlf7;r;0iNCzLkiE|H(u0qmCDTOp*V~|V6(+FBs+qDpJZ1X<#-c;`? z-3lXY!d>Qki1SGARg z5-eJVm*)=pn~;b>;QqD&esh?(qx=$g*-P?i{l(uJ&6U4_#EYbFnp4qyfHH9%Un%ofG7eAe;OME`CAA=dU41R|4?~g4)~WR)&6@BFnq! zA~+%&E5m}Z_wJvs){`IkOl1byW8Hv1pb>0jrQ{ooSLdT zrJEyu@oaO;n*6Sb2}`=9j7S?+g>GCq=up-zB7Fk7d;U-Yy8X%VRUf`X%9fJOfHo>BsRG? z`W+}6{rZtlvBaOY2Dw52vq9*^Yp3lPv?FM%$4SoC(CDVabLQi=(iu zIyt`8rrAf$-LF4mGU#bhfGBo8fwo*U``~7Hdpf!l5)TC@+?gS*QheweKSCBmG`CF6 zoEm|hFhP&w^y>#^wLAjrpKWY|Nkcq*oqe=B;_o0R-KR+8rdXk(Vi$*!EZ}ju!z+bX zm>*rZDHJB2{H(ue-Zt83(lxWAYI!ktZ(7BY69}hm^3zHHI@5HoLdK;;#`e;->WuF2 z>S%86Zf?lJRcC4wvW8xU`lx!owCO@Z!){<+oornAe2CLzY4+B@(^qeNYct;0eDcO+ zS@j{i$vyyvV)NUgYyX8w)~~tYtNBvLp0X-?9YNc7XvQ`s2Y=j%3yBvmGavu>cs#*d zcVmmMlQ5vkUijmEuoC-a_iv%jC1Z8hbV&nR72HtoX&;nJjRW3z9`Q_!YG41_Kfm1E z#BCfL9H%}XFi3z-HFc|QQRq`xNM-d--56;ATUBgSI%AK!#y?vO}F; zTUq76i{;Dm{8Mq<(}Us>&e>vt{Be~L6{KT{6r;NnDjfyuN2^tn$($U1JLxiKW^sO1IhnywC!ulA z{C@kJa=g7SRPIs>(np&llbFp)k0Wa($B*avVomj;`CUm|R?cHZuRiZra=AoJXm2l% zJ5iL=Sp7o6?s)y%ccuZ?e-iDik%%Yx11P~iwLttv^i)qD>Vhi1&Z2QTY+HTUqb+Qr zEM=vFEDR2@ThjSN+P%8!lEs`V#Fo3huE}cKx>V37n7*iSwZNHRPGqIf#zv^Fk_@u!3sRx-7DO6CRq}b7 z8Oj#l5}tmB`!_#g+M8*$4?vC3h6-f(*^JmW;wu zS8OZGmZG*VzZrf%M5zmHaz(wri)5;mEb_p3-8GGO9$KQ`?ZIc+d zd{;X^J3E0+X2PShrP7zk4+w}nowriVlGkp-ez{sbqJYoCLc1HG4__OiLg%G)c8b>a zq!|`DXwSfg&VGhWy}U<*L_c-H7inS>C{<5gE=y%FIhqc6ib$7CYv81Gdq0?4bT2}6 zK&|--2&CTi$Ui*Wb_AOo@SQ+=91(|F36`pNBc{G#Ti`#6kB_+Bq$}?$jQ!r+COUUW z?wd%22*n!QeqIC(lvuJ{ysBCBCrrp5Jn|va*)<{`gLJv>I*4$0HV(yZb{B6iT-C|s zaA|BjopFHa{u67VP=9Rarz1|Pj{2LzZ8lM2@Fqa>^P2Ca(uip>k2e;14q9qH!=>7< z@WyclNn#bhc~US%9NteK>zzNCBx>=-v&^h>?5`DM)sm^+i z^v#oP2Mz}c>If6I`yMW1&%bDZ&T!6$F%6{$hWnZ5EO$~P6Y3VY4-TkF_|u#qMqnU_ zKk^fR1@K1Q;a{Y0YF{hCgC@Um|I&zllwb-_-Gt>epq*t<*eG!3_`YE{eXT;f+bY3h zews<7p#ZWtEY%UUV{+~4?JT{C|0rzR3twwE?Ra&}a$yORuPl+`TvzOGlS^yv{?(po zR^x5N%MOgZ1;0q&5p1ZlIE&p?xQS~o3kY+ZYpDcW5(>c-PeYEa7@>tm?6$g5Y(QBX z0!enwe1lrbCks4LRbw*`>7S2tPmiquo%RK_HM+O6*Rn+5PyVIP z`x%HrAoVqS<>YR^She1FV}~@Vv`Yq=bxxgPI6j@zVkn}};7C|-5LPIVe!gD2yMZoi zDN5#HB~Gm2m0HaJR0aNfBM1se}E*IHxbaA`YVy$e_%IC1T29{_yzjmKpy?~ z){DOi0HPaL94_{D7lC1n2R5)qBhbi5SQro|sQ=*2@BGhgB7wu9p*?ak?~?xj9<4YDZakd@tl^Xl}8QW}zFlRNVez zy?Y4tN>Cc?$fp$y#_q^BF}7l`ccoh$Ysu01>r;E*LuC;Am0t4+fI+?%^LiRK+VM?#JEM-w7wlCw zRo`|_Czs9^Zf*kN;~g(qJwCfV2}^>dAL~qfUMFHbr$vv~X--55$20%_u)R_D#44QN z98^cQv2%L%fV;ka&^i$CI{{dE-`Hf(M^yURoJX5rx{iT!y;Om3k`haA5%U~T+k7S} z>&`7#ciYtPL+mu@aAveC#97OI_3G%&YBJ)YUP7^T6{AaX1e$ZqWHCD?+j>d;2iJbeZWYQ-hkt&FymM z32|b~GKKP?L?m$DqhiEOyf4>yqRHCCbWgRaTE$5E8$@{%7pT~@%$8^>j#ETPeQ?-_ zNQEhTdtQ4fIrTPg)}=uJ;ir0_ZUX!-DW{6#)7ZKx$FPjDy(J5U5Q@q6G_%G_4mPhG znF-5`$zjz&JG)P$@ek=AN?Es-%`#7z5R+NrqjIEc?I;s$_WcDPLXdjY~3 zMhAj5s-`zLXFr8vKWr2Bg7^_YwTO4`w8{@@6c$)>a(L%0TX^WrgZ||Lmee$KG&GiH zZ-waEveU0m23suIs+%oq9EHrxjcS_g#i(F)syL;zQG*%|ulBE=jL^bmztsqJH~Pke zu&MB=gt*=xYL%Bv;yCZW=f>tymN^y??z|#+lA1Riu;(E!EG~V3pQIYCmlwg^mqL)Q zIoPXdc@E@@$1aF}rG?>{*XP>bqGm3);5K9$wU~TGcdwx$Bgbm#j1vn#BK;&(Na$f8Lv|d;JzEFW3Ldb30Sy+TM&```effZ!YuYRzB?G zR40{$3^OH<-tFS&_|Gi)!`~1L^l58L!%S-!u6>+FhDD)z4xwK&w5gq^969 zuZ1#Z?1u9OWieV}yJUp#VoMxj^aH8)(?Z5*=^GuAoSY$2vpePB5np1J6w?J&W@P_V zOWN^my76Q0iWbOQ16z<}ljl2qozLzDA>O?^Wv1>P-GRo@1DZ0AteU)G@M=Y=2^&_` zJuIXFJLNZeI2J&*5dp0gB6-xs)5}D1ggML24gC&2@nz1kjNSuHKaaZ)$UMiGER%D7 zRy>!oUR(2ALo4eWuPZUt?$j__{ij-ut zlXi3CIqQ@#EXh=HrUjL&Ui;~#mJB7urM8q!cdnIZ76DTye#VjCCOu=bpfyGM!^Mb_ ztqdAuOMlwF)uB0`{`FRm`#A?%R==Y=jEbWvmJ~d&J~nbUI&}^F0Q6~C~xETOO@qDZR7_q>7LX>d34?FsmE^Vb!cqtuG^%%>E|{52Sd2#2$Q4yXs*>sCgC(b z%dN=q2{r`WSeLq*Ox#>D>7kOtqsnI?K5dQssXdjaruVRO!KuC(=Qu|mee3qz@k1fF z`!{t@58DA%As=5hDG$%P#PyRUTm!a8T!13O^0R5h8~))oYspNb61CwDD4v8&%r~OB zdTruERGw@oJbD3(pG#Nu)ZzQpQQQr(#5M=db9^eE`b2NGd{)B2ooO5?_P({~`0wsvDljVK~2$Q8BVOP?P##BjT!!yJ@CL zS;*ELFKqAjF$$v-9+IDs&?~bZbtfD1D)(n zf$0=Gb=u1Aa(mlQIaR(Z>Jc~Mbn@UW{_OYVMo(eWMErBy&5(Fst>bupY}{!E!cW$o zPnYa|)(|hLcJzDQr+B{O6Yhp@oK6Ze*0aD`5%` zZ8U04^-CeW&o!Fv^>=XPxy_%UoX&`6=jA9;fDwIy3>u$zIlTS6!{*du?q}992mUQ( zBz%qpr+lOMaZYvNIVEX9Ez=>LaVe(QI;8T=Xg9aX!p8b<3Pzk**141Cs;$qC^U&cM zpMmM@;-ib(_;|KMa?!7Y;Ks~u^6PpN;{0VIo>9?@VE0`N2?S7dgvbn8(b8DR{5+Ma zTXIG>^@w!1=+IuaS(?(E4N{O{@G~meVlAhYRA_KqlIneyokUi@V5ak$zRfLl$i0*E zswMNq%s3tYdsi>K-{8mrg7muW8yVm|Q3^-49Qo}Ju{1Un@{Ke(uV(r4sz@it~uUZQ<9+uiNa+roM4r7Hd%R zxn((TbuJcnS%0}))K*R<@xuAKkr^U$V}x>FYitOUhqk)5^tK5r@3hJu^a0s4{nZPE z-xKmOG%K8kpN7Bm`ZC^aQ3hHi9!=7^H}HJEmnnbH#uuB`n{Qy?1B~yvnyJx=%+3VgtX<D5(WhQ6<_$pME^Xxsz9EIHx=Wv3|u@VA~2no_qyMJEE6xi zOIYb|!!$>TOE{40Hu&2YIPmY259F;mYXa~8Fyx*vt#APOa{>qq?A+#1GXK*X0%)k5 zhFq`#;RgO+itKF%0RtMpCB*z^Hhw-}jga2W$ow}mY!md5e}}Or^g1a+DE`i=7y$Nf zfA0iTK*u2dzon;934Dr||CcrR|8y#-RAM)`s{?01rP2VwxSa@O zA}Z^Cg&9#wzGiymjFiUjV@nRcaaIKR5&!3X23B$0AEW zWcd|m4%#yswC{@|1S3pN;SW~%ufQb@!Mq^q6YBBq|2FyW@P5L)+4=Dr$E21)KS`dP zC+hpSHefUdz65>$d;!?)@2$(vPo*u-kH@7tEdukUo{wMV#)T=OwBOPBFQb18YhU)u z1BW3&WK~c6?GLEG21Ym{0@%$x3~tVE z`-g;0>jO5uiWnHiK`QWn>i6&yfOqE||4)w~H^RId2u>r3x zEp#RjAPYW%HUA-dCcXkRkvcOmCHUXfY$98+5WCqque|v?GC$!9pQOm$0|{80Hzo2P z@lO`&WsW0oC*Xfu`uPD3X%pc7M_tsld+`#B)gps0S6cT1CanwDoq+FDN;rJOps3 z-$(!$#OFnMmEHLCuaDr@MT4D48-f9S8Tp%~1JM-P^N+lNbVemO7aZ`=My92+82{?hMoO&!%9v2SIcxP@-9)tsjFwBfeY! zQoz@%qNwIU0aAgWSJ#mKl7pEfV!qtZT88e;rnJG(o{g=ZK}!yg3&>C@a4k~{!8Ho^ zq|41+F=mp$U?q2(yE$PJ$%$PK?a$14G?+Z@hp^o(F%;5{4$ww?Ros_>-1oDtftkd? zs7QHq;1g!_FWctFRRlOa5|p%ejf9MA&SnNev3Lf1t74>ODDaS|i^BW6OpJ`i(oBtI zDa?YorS1i>-&{-37!?MDb8*kPhfF!JZO?O~3lU2nq}#r@575!tcx?*5dBp|TK?Gn{ z)Y$YBKCR?PJ58ZzjAL(37-SaYy<&J+BmGyCcu)~qT^uP21cv#~Ytx1}RTJ>`-Fz}) zI)Ib_04OiSu#)Z0qgtJ@9BIDL_K>^hHP0Ea^ze>SxX2TqfAWx zloU%zDx6*0ZemBwNAH%p<=&`rQeBK+s&OPe7j}EWi$d;U*}ROUp?;!;RdK$ZQll$_ zZ)d*R-S@=A-)CxD3$_=>7(RO8^KgKea?SIt!{g>|gcv8L$9x4}EU!llSh%#VK_DH? zB20c*&NoOaI0$=^704f3F@kWhHzgvoyC;2xb>DP3JEFMN+}1Y}uD_3T4pP(23r0iBZMn>KjS0Z3ww&Bg=o{NGI94WL#ff*Y9CyULU7Sf836DrlPTN z+L}X+=a0oZ>!SEz$~C1T{>S+10Ar~PfBQ5;U{M(ceT>Zw*bj_slC7Ir_ovSy5n)X~ zL`57(w#=slT|O@KUc-sSwA2_>LNKIr)?pE5CSNidLetkuPMz>CR zMJ08XpQuH>7NUWkOtW?;T3Jr#U6`&A{)^zn#MXe-24^zXGXF*tsBtG$LvMCQIJeZG9o+{%5FooY(+ z%lzU+sQU-A>?D9a660@reA~0&g9?)tvC90Cw!cr-q>J9(4f1>_Tl5GqkAfXx`Qfoj z^e0ddCIdYNr^C{=i}AthUysp6l-L$x@j7T>qrC(`aKW@Lc!65eUvO9opBi5RFaPt) zv4Os*LU>a{62AH;n4*&b2w?3k>(QF(8fxxae;`8alIwl+-=q;NPA(wncSIBs zM^SsOF~f^g;zo|W)MJpilpat_39iM?d6%DZZi5j6Kpv1J+8$jLx{R(JGm#r!a<#CO zIYckRcN*XIa6(;=Q%jP)gwhB>DZ{*E#5vgK0tgKD0u&(JE1AB%8KZ+}2*PBWjE4S& z8p(yV^}KRbit^=l=3}zlmPE=!((EatR(^^F;?S;UNmK1(=X>(lF$;MMUT;{OmY|q(wA@pp_o_14t}Is!rena z-gtAXRw-TH(SgOaX-V zq9xnpMKNE!o)q`nY|FA(6?&OL>_{|A$|~n@rscXvB38M0z9D_FC8z?)W)E9^(R68B zZ5w!ADj1rrl?-orUW2#2LaI#gfgh%Nwl#lL9BTdO6w&Od!6FIijIJ)X=V*z`UPcZW zPdC6%XQxpoKU0{ zYU$arOmow*yeTBYf*_YhDPnfJw~okoUsMmamN;NWni8CFzlc6obz97YeE&JuIc*|Z zrnI*^!m9QpKzIW}bVR1F!g91v6(oF?@gaohbX~@OmI_)7;6wp{U~_wwfp5@H*m_MR zWLT1GiN9xndd8XMOazCK<+D!4ED@-tak??LJM(efyquC|h2}L&{mja0aRq0;D?YWB zCMQ)tJNdo$k(p9n&E?0Y>kA?}$!~hIZdL}i<8R}5l%DQ?jg2vd1x|RL6J73lT(*UF zs&ZWQIeW_n2j`s@pRq?BRl0Ne30ymRWd90aC0RzoT{p_2T|3)~m{zt^6OBL!I8Ptn zJh*eImM`Jg>Tox>=6L(Pv;vP&ooBecUWIvCdTmkW!#k}Qc*2;*DVb8pZHn$$P7f@q zN%Gv`k(K}>KE0b!6Tzf6KY-POxB2gaBe6g15`MO~Xy3b!Uc;S?-rM7|hgRag$R`h7 zC+}njK!_iuPDU|D}wu|xUBWqEql>4kLR`#=V z@&NR1M5!}a`1lP4{u8q_ic zD9P2_yj(2(5FKnO-Qv{P#R1);2e`L43Jm5)WXPx(0@*t@ld*C;5TX+@P>kkgF|;Q4 zJ^~<*=k}Q|nIkIi2o&wD$h1X670ogwn}x=wu{;JQs*(1f0jKqt_6B#7i2iX(zvms! z=?ZS>fOMG0HxkCwocuL}FLM`>l&hBY44!j+YXZ>w+(-0PFwg_xl#YtH6as$4nblE- ztYvHUB^7q5xyzRl`fVp#GOAq-gA>f%SRpR*&UEtG?c5raNvc8zf_ZTmmHQdPyXX=m zWSsJX;jj4+mWt_(^cOZyI24&;J?rc6lt1@=cL)GA0ZZe6>2nN}(ys-RzX*z!zJCMi zsvNch)W*95CqHa#c z=(n)wd6Zdo_X;V+h>HiaP1V}s%H5ILrg9%@1sz$!rTsL z%n@-peFJkeQk?ywIt$v==RjeV!nj$QL8P#rw*-r{9qA zzrpOHFE5?pcAIjLsqM*EZdUep)u7kSJm5`&QiCemrrQM0`{@#)P3M}EGc7T}aL1^g zixK^r16uFsl?&xA*ve|}iO`<-bqQYpG$dCrLNFMj_pF{Zij4fi`l6%5>1?rv3Q({2 zz( zz2m^uA%y57y>hAC%~Dv!5NdbZG}w?pTIvgciDWfX@&ZG2Rc&XV2*B$r|LhRDC6%hl z)$TtfJ@upxdq*D;C#Q6?B<>iUKc#sR^k8GLZ8yz}X(*XkCKvHJ!`W<@;|nsG1*uAk zn@bEyu@s*}C$jwNW?77tnE%`;f<46m<7~MUaz&9!+CIcCL)gRVxL9oa^6<`68a)-` zorpPZ-C>z0CjUgl^`|UTU=N*QKt5rMI<>^=t6aBwYWvC;zy3Rc-pQXI5mI$j3Xyzn46JS^}<^t>EpX9k(9Q&bXXe}ngOG@sGj*a zgjVz6J-Nu(EM^ltMbv^Z1?7$zQFp9YDR9d6gv4eJP9)PdXHOY~<*maxCzB=I7m3=XzK<)#;jJm@lbU6838R(5(>&L0Q zCVyji@ruvPi@y}|=6vs&Ql@lHmH;yFg;9hF9AVIebeeKL4`Q4yRD|m}%D7i(wi{wY zb&*h~+@wwdQgJKTlT3!(LnvW5PeOVt2x1gogd`@sJPdGregbN#D{GcxuYH+l{PSma z2gRv~7;rA=a{&4VK;G1#pPEQ*_yh|887VvY*-tK22)FSF`^<6OCW>j!$w}?k{9Ps7 zCzewgOlE|#<`Be(`0j3}CiuHlX$8?DH>#5{-a3S`gy62&t=7gA;@*a7aKAYkT@nDO z;sqGerbWp959cEZwo0tm3NR7BdEGs6uuBNs)a_E$t+Lw5WtT=7CTGXb(s51Ao#r>Ff=EQXlov`PGsCa)+? zZW1jM#>byO-b2{Oq{DIkA$iMD`Hd2im7YnFc`*jvj=Ki-6y)b-Tox_v=dY<0x}%J% zn{b&aIa*Hpk}xinBjz#|s00vwgVETZRuQKeE>t13@_rLjzmW4(jX2qn(5r$^%2xbl zU2oXN*sIPS=ct?R=X*PobsnNJV}~DuArQsoHPZZ{3O=l;5=!Plyl^H3$oxj?RT!|h zJ|_&2S#jfRGRf4!$dw;qM)>s(zOS<_yV>rJLC=^mEZoStVdEDqikl@AW-8ULKQ_1sN_2^+O47{G# zqRtU_s`$GNGzt^K_}FVvo;UuUW~?o@D?Y+`2q3qUUsqRbGcN~{b_JR73?}%D_{A!4 zsB$eNj?b;g$@lf)QpZ||AE)wQ|N1(c4Q;hY6~lAlWK6}1@II{256xd9{m5d{pm2Pl zAD*}UgFZgb9B|KqB3V*$QZG5~PuMi3Dig&MtW~|$5E~EkXBh=Ffr3M*#}xvy?@6i) z<~2IUrAfqv3RKJE4CYTf;o?nA<;SJf&WlUWKqX^|C0=-Kle=Tf?P^R>|Lm>PhkR+% z!}3cGVI;NG>W@-)!eD;HSan{vVG>HC6 zqhmWgwdWDMzc|TG?SL-MwX`VaqOxUoW_I*Ix|Mlm=1G8*{qi#5JTh>*9CQnnZ(#;3 zp*%!+>Z!x3wzJ8SuVuuvjO%S@b2hTCk+^ZtwnD!4JU^s9S8)9Lr<`BwWfq1K025rJ zbyFYf&uc-~vDj%G@3~o=2+L=)5V6{x&y;(E4=^S#hQN#JRp*VwZ^VgoKp~9Go6|}` zxxcZd;>Mq(hzk!$~%rm)80`er;|%|x8(z-$e5 z^${OjX`b>tn~x4-R^5R`$`ESql*8-pe+S}+h?mS{8vJHr^#KMXp<9aCmxMJfa~R&V z(C=jO<**SevGzpc@gspuFbLP!^mUx%#CX#uXmkd-`R`b5;l@&Nv<4YR9sc$;BZYE{4HdVuP zAVhlS1^N4I=)w0;N8^=i-O<{*QPhnbv@n`)?(w%ng{zHwZ^GVgM;k2ZwL9m@dwR^3 zx`PXWf4;|rqdLqLEa$Yu%$@3Q=lXXth~R}HkxyvJ`6i8Ujqluw_=#Jgejm$A+Kq%o z01(dpxy36K|3lMbsR1uVs(0nA|2Ew5YyIgTwm5-p@W-S5`Mdr-;I_Zm0U7JdMvlOP zM*&11){ExjU$wcy_LmNR^9{gn-nyj`ayS1n$v_Whm|&-yn?K$V*VXxlJ=MI7#^T*~ zn?J`Q_%ThrH*>r;4Zb0)9Lt>*lKM;w=O zbRlbAo9@H((E1-Kfj$A!bUa)9T&SSpL6vdsdhvSY^v?8$`ua)P=VB)7J*mUd>}$hI zjjrj@u7#W0hdv|Oj5;d_WPPtp2obA46nt7KWc{9eXM2+Yj=GCu70dfv!{tV9z}fU9 zeOz*(#~f9P{5Izwwyk{ClnQQ^=(EBGCSlsq1eDy7ZLw+B?G2Y!d79Kr+kOmlWdpSL zh={J3(wfnL6_GL}-y5WJDF!d}}Y&Q9O664ziG@Ks2c;W@Lh<{3G!5~;jWEEq@jjv)n zGokTwUy9C_)VRecez7(zE!{d1u^;ctdD|Z(E(02iU%iB&Wr_{mDeYt$7K~y%nA>mG zECLp9v1yhBHMaUS=KF}-IWrhgrc^$3jmZ_q%;%##dxJ*Y_n{ZXfn@q!byasI#wSM# zQRwsxg^k%!Fyl*+D!K2M-)|=%DjaW_T-Rf7=~Rkt@ODa-k&=B2}z8Xbm5KC8a$E zE9`0|CX=T=a_fR$$p7=V!H(GvYc(C6fE+ zg{xfnhi4SscYTpoC zP#Ftn)Zt{86pzVL?a|wH$3mTnPrr$6Liz&8g7NHT7rSTZ?4c&l^jg@af#h|GKxE8n zpD$i_#$0(YjDEX$SF7veli0^!ygxoJO($$9yUeSZS3J)~g|1GKW@XB|IzA07>lQUw zG-+6nVK~koPTx~)i<7rW& zFaM7LWD2BlYVGw4;$X{JrbILs8{Khmir8Bi@a;^IVI1#xL^dd=`T~7`fK&XSX zoh{u^+gJ;B>UWLr_jnqMj|p|&*y!+G-Wwgv0U7f)SBPtC)my}5T^8`wN%#CwTCq4G z+UK)V5|W7K(k;T}>*vqxLKRwA))~4c#GK(S&STSa@8b_XIlpXO%{%S*PSS9Hd+0#J z>3Q?>lg^l;Fl!OWSG|?QbvHv&DdMQ*=j`9n8s^~*KdU!O&&v09D;Ry2l+tw4@yw_m zv|K$n+yfu&N|My1nyh%zcl`0228|{HGZH71O{P`)TAh0hLPx3G+{GhSHPA96aqdAu z;^ln%9?|#AQp4<=Ccupa_)mE3Y?Pr;knA{)Cg(@5p(Iv6M{y;gcDQEU=s+ zz9*y2c0gbr%jdI~=&tOge}~^;ez(+7i~Dw6f06n2eHMv@ZAE!%K*UnB^x#lDF6m~x z>JYsv@?FKWu*BLQH?0fv@Ob=e8tQje{8_8O&S4SpddA#q2zfQrohNSnF!7gylkv); zyKq&qnUdX?H41k6E{5b@%Be zc!lNKT|Pd>6N;yu2Ui#Kk*jtt01~L~Xb%uwbZZDv{ti(5oZh>;o_Fn_5=0p!Y>mVM zyL@qHL?#n$0ub9{(eWv80{4}Ad}L2)!9!(wGf2mMUh-e~?rxYLR#Hw;hXnEPk5EMO zV-`k5Bi4^g3iyh5wpV6s_}DpbKS?W0MT(N?^T4@xj^ap@_*paC)Mb-}IjAZoz<( zrZpZ?e|Xkri(|Kg5_XQd+V+acq+zbd3E}NuJa*pVnmw~7Twl;b>855Fuc;%L%#Q;i zJZP6=cey=9f_iDzxv0M@TWOIQeu2G+>}@p zv9DKLXSN^twSFRr)3n>Hl=5AYY?ie^tD$>nn(d>u!5iEMA|Kv7>l;yPf>H~HebJ)s z>l;Bw=BUN@*km~CP(vy75_=L^=6tHB4+*rJy_LypIRf^Y#*mHl(&9qC?H(>?MZII0 zDtnbWT-KC9JNR186trR%bI|QKNTo?{zWaG)X~6eNjmuDq&+h2#WlMm5s}|nsC@_!9 zSyB%n}lR(!htODbX@KRh!$uY#UV={)P4AMZdP zcG;l-aUGDnJ-2=W5J}bq!V0Jr5ax&I29gJ{<@mxa+y`u_7qC^aDEAwagL%Ir^lG(+ z!7(AxDl)rak0|yQ|EJW#OT;UufMIRtSF39kka89&ySC!~hHxGbc>)ZO^h#Q9M2soC z0i1=B_JQ;-^a83E+Pd$mX-xk|==oP)_^rJF2wfW{jT%Prg<~_VL2UdZ^8AMGfiL=P z_-p4z?kB(N_Nf<4qtsE1?D%T(Pw@a2CB%;8{e&g}Bp^8l2bO@e7RXMw`YV-}@pU{7 zQrCW`zrS;?SKu{Fq@V0^Ak(wnuIh>DRusBdS)W^~*Pq%adLEPmweG2`(`=y^T>oGI zAP^)2dB)+#yb-r=VZLhz!n6JD80ZbaZxmCOE?66yC)t&OU*Jl(`(eVoVFJHFc)_5L z@4SX8`I16_YL?d>D@)Q+(&4AubUtNWDs-s|eRaJGmAODTfB8iO+tm)zpLFI$ZF&Qw zLJYDl65>=LCk!FdI%m?_PS55zUo@NaEF2azo(4_7NU~T^OV3;#-i%3iGyJ!`oa4>f zuYlDDa!t*2td*n`TWH5bPs0zErS~D?Ebzgef_oeCJZg+1WVC5H&Yl9R^4*d<*^y`Ob z4wTi~h=}5LZod3(pT*AJY###qG^bt2y9r4Y6S)Nw)(*|=8tOM*{6xTd0*w250aHPk z4pt94($;zukaDkHUeb!H+nv31xPL*rLbkJaqhV(xUxBdVv|r9-O7OOdZcc-VXcSHJa+7|JxI`$zO zeG_+v%HC0;o9C=0Z#%$wMzgCXY%uJu4MyjL@5?>!)6lh$$28&)E?d5ssFdbAw*xMl z{*|TVmk6+a*NwvI!a;`V_#{Nsb9*bZ7BlMf)hYEw70X^*?2H81OYH=+5<9LbH9(GReS^_Y>oI(6BpQ&^`?Y$Ip(>`F2E{<4zo4!G!ZAH2{jBfSJS0GXY_in8E8hGIxojQ0AD=MaXjX@H^;9G=G)sMi!;o zi~qv|IcnT^$gS?4+GtvMomj|H5-b-SEQuJxz+x1&<;IQbdnz`%h=aKJ>&0ax?Hnk+QW6JIBPXT4UrXgB1W8EuRq+dgY7i;&R+9o0Y4OLILDQStgCr=U7CxXsd@Zf)cw248*o%U zTKBpy1_{!<-w7@~9e9j4`el)@ZlRyg55ruvvD%X(^mcW3i7hW#omcfKge<;sy<6`5 z=934dthc7tFtU2+m_;1b@liuPlg8MYnw0CP%8wz}v^q3OlV~l5n9Rpv6lxt$Li~8g zXc)rglh$pe?Ma@yUE!1jQ#OkMr(l821iEQ5UHQ^`WST_@p&tCX9)oSuPa#9l?@XiJ zu6RYxI$nwRq)HTM9SorhRXYlpC((q4u zKn>9A57mgmsn*=4?+&28IDh(}ISnDTqCm2D(Xywzo1p?a6Fll?s#c9`VXDkhClU8h zBe?(E3%y$heu3Nd7uH=NoxJh~X2X31|)&)vNjY_A7z- z*6Z2fGWc#aJts2z^ycZRlaJHFz6I-|y+p*(X8n|A&Un=z#}w`T4bA-iXlT~RI0rEs zY6_mAPLEu%$)wk^6AM~wk(s>~;tWw;^zoh;|I99eK8Y&3*HnTPkjl5kA%8sh+<#z? zNn1q-J4SWx#hb;gXsrv@P`??K2w6^5iemL~7TZHRC#gA8ay@Lz9KdL;!MWNBY(qR@ z!u;Yq33zku7`_yzA2J%emcQjE;q30_+RYe#emnNgp_%4|WE30??#+>Yq@MUr3fDfz za`OmCh4tIZZ@2v0CBjJ8)|y7c$%iHE5?-xVV2Vug!ShlH1l|HmS#qOWZhGP81bf|TL}vIQ?J&OUM)#*@0=eJL_rvJ=WZ>4{Eac_)RpV8Wg7 zElYoP>~l(Fzxp#2^97e#t4Ls?W0&e)6jTL7giAA>tO;N8jH`_*BvR=YkO*RlHIfU? zp~=9Ph;<_i)DKhDB~M-l*-#MVkJ$)|`XN(w&nvJDpR_jLMr3&3r3EaNd+~9@odGs! z3egGE5$FNP{j}x@UY50?YHO^}g1YY&4Z)0`V_XVhUMYN%WUyekp%IB$M2~uL5;bj+ zymqDEwjLaI#GIIgSy$x{l#?oFO@k9D;RK@(Iixcs(Yy5RIcK!=OVUYBvbgVU8)s`C zT$WD)c<5f>nxZOF^N1e~t>Y?OdXEJwHGZ0BN~D@f2}4jPx=#!dQjE)R{G^LhC$X}qAD32;N>$dNcW~=Gtrh<#51h&DPxe# zeN}xh=B6368_NlOj#GX#53lRkgl2MFU5%FW6B>;SRIDp#s_B8AYRAY4F z%mp;s@vQVUcLr|!kVuQC)X(?)PEL8gfB)Q4o?!`kZlYm}k$Fv=gL;lZr1 z(YJ>sXm@09GuCxSz@}mP6#JkF=B)~nMxeXCrxz>?VEE*WhryC+#xgGr0DL}ch(0y3lC>eJLHnx>Zu;+8XH6pDrG&@Hvi+_)*@Dcib&zt@wByGZrj7N@l(7$cZ+nS4aU4^}!kP3t`S$x%la2)azvkv0B@>6Hre7yMQOwa4bvHk>+HkF- zGu~zGoOS>q``uWHpJj;dSH^Nr55IfA!(=4$-wiF?xz~{L-lARO4SxUPWz9$Liy-o? ze8c?b3N|4RoK}c2uO@RqHC5I!d>O@WAS9 zyCD`YNuw_Dnx3|%Fz5mg*QGE-*U#%SbH-W}z3gR3j&Cb~Fz88ICJ%}0a2>o4-I1e}pTn!koc} zxNd3-zR=WQIUtaAQcD(8DOW#QSa_x*cs*p?uxkFR!qtQ%t}K-F*}YGFM3IwY3_Ysq zK&hp}mflH@M3Fs~(F<9>Re^_MGZweKuX?WEzk5J21xv-wsR(UuTZDZe(X*VCr&W9u zD_;}kdayAmG`|tu9j(U0)6(G0|COSeXjQVSsWEOBq(58IDSG4ZWHGv((+D7i0Kq}(0_L+rVs#@cBCSm!_@%A_^ z@?n_}h5hYXwg&pW{X{nZbE;ftoZlgr^~8MNM>!qVdl=8&iZ)dwVLKQ_?DRQh_r@i6V_qCm?HbR&wtxqAWPt>2`1<~A2?JSd z*}ffSc+am*#4r+oFBV~Q--R%ep8N5=JMph^=MTN@-8s=7<6T#^QZ&3CJ06cjlZq`( zKV5rd`=Th)S_W=Oi5T(NJ~6(}YoD4Lz9vyPwd8x{)=3z(Lww^;cS$(9l;9GBAQAb6 z@%Dj~UIu1BMBoEAX}82r1o=03YQ&_iCL#%X0g^OBNI}O`H0VtVZ+qedDvkJiH~LOF zu(JnzLJg#F>DPN^pK}Xq3FNM^GT4YS_y^gLQ=HOwiK8j>O(l`GdlwBylm%<}scSe? z`zR{4+Ps~v$Jprccs23KqTuh z<8TQLX;OZ{$k^wod~y)-0ej~xoB{b`Sb}bG|7_29i<#=%9 zEF_n3&X_Tett-_V*AgE>D-QU?hKR8uP_q)6tws7&SiNqrcGNorgWRR^$hSb?gJ35wj&t`!)sVybfZ!1^P!_ErTMRU?YY~evzMwop$F`h0;0=5#3*iXxm)t4V zxz6pHrPS!IYp;saUU~4bcj1pKQ9@+0%soer<*C0X{tzKf<`%}8BbITH-4hXC9H$SN zMI2E%{8NFGOuq5Ev2xNtRPcJ#1}6(#PGH%RVu`Z1LcZL@7Hw{U*+ntDZ{pWIYBq1j zy64sH({=0XWP%@Q%`{#_SdZ*fR;n8MD`PX8wI?8UMwKfYaLC*fe#Fp^V9bPai((7_ z{1f*0U$(2JduBB2bak0izQXO3%jN{1$J3;q6^sucXLQfmtK+Ze#r;GJ0=PieIg9Q< zID}b{VVY*zu+O0m0(KOG2eSjgLU~4T)bl1xj*HjTMHQ}VAs9j8`9fpg8JeB27QWay zeNJrT1aemb@172@3ec&Rpikc7-?_XjwIjWguyf*)Z?mS>I>H-tb|IJ_h6-_fFGA(k z<=Y&}>_&fuG=63BnpCSwN&*=Od-HHZlCaxKwuI4_T<)TeTcmcn4X zf3)SZ4^J8DLsF;<7?{9BgzTehpOTGJ1 z%Manq-~`nH`=p;%c!*gc%b9+`MD5?22;AyM1miN|?n47ekv%xm%e!mHcivj}6Gy4l zrr|P~Ow|)E7{{454L6BgJd`IT)hc|(OPbx<4Mv`Q6X5!I0ZZ8biTmV4=KaM(=hwP@ zod*FJ7Y^>n)aE?)!&h5NCC!1^j1+|+Q2FRq1vtg#rfhsV)hhjX-^fVppbzTMqJt$+ zHQ&E-ELwG>Ll`h+Vfb6PuY?aAxJ$~<199U+1>0$q*-MKG>^0W;BqRA{Nvlq8_zO>6 zj|7TUJBTwkS(#B48OkkElAEhh-zQUhgLrE`eI}7h`;ehOEY3Nh=}M#KP$@Upj}}@V zEz+w9+VDEs-1&Uz(-cFFL2P!e)TA5<;$az{ikJykNWvo=XDQURjhF?w?2?5j&OT}3@aaJNSVce`c zR6(G8N;qG6v&h~GJlEUdg$O2uU&ple^v ze9J{{eG?M0S1#5@pFK)9=Bq)UNfDRAkHI})2$?rN&3=^?!4VrUG}*-=L1Is9?odQe zx`mc2^@2Qw!ZJl$B1a!X*8fBi)IlZjPE%A)gg)4WB=LZ*MZkeYNoqdm4Zh||vVNs0 zcYrxw((2R@Q=f763r%zPjRLT6;T{mu~_D4*pl=oV@F!O}YVib7rAb;XMy zS!ez+DwXCl;Pi5Ac$)(NBfFY3%T^n{MY@DKW%cb48u6aoU20**y7aH?`rPoNO3wNF zta10M-+vt<9nUy(8mIS>i+D&Tc9q(?v$OAViLUY3vsiUKLk@fE4*)aOC;&#GhlGhp zl5)bfl|u^Q_}U_Uulmwhg55IH208BHRvv6sqfz{xW@Kpl^8Dx4ceDBBUxirEc>CUp zwwZ@Zr0UyTNU`G&LQ)9ll2(CU%OQ(a7^Id*U4wYS5AC*b27Zoy96c%5;iO*FmXL|7 zf9I`$3wM^)`L14pqkJYQQ}Baf1RN&r0@YxQICMcOr}oSh+?III_dTd>@K{)XjQvpa zt+83z(P)hVEAiRbmHXQs$FNmjmqPbOnQpUFaV*&0oc@0PD>aPysNnZpngrY9!|67u zA!|AtULib*+P9qxGzG)!IsI7ZU1T^L{_<|c)9S(eP17O-{`RX@jRN2EB?Ovk-(ctQ zFt$T6#iNGkTJxr_iOMIid%n>l&oB|najKs$PdLhcRmDZz$@sw-qtmUcE^NBeGo)CE ztLAR~Jr9)cg;pt9&cKY}SJEENY5_T-21)*e*I`WFyFUcVh2LqFz~Bzdtr85J53V%h zuD<;B2#vQSd)pTy>=&b~C%Pzfiyeq$?hnAaYCL{{h?+!lA4WB5c5jzwDzK2{V8}Yo zzAGZ3`JflJ7;_ghh?{#4k)Ftb$YC>>HqwBe)sy!p%0-{tLm@4p=Uag@-aU?4W9(Hj6)={zIg zu0g2h<}kBGB<`-lba0jQr^Sk=)+{&k&RFsS8py3HTlto@_~+-I)kp@diD%PXh)NEE zs9LkZhLHoVb%tZH()qT*c@y=PNSA|OVWcg-<{;myt)udtjeZ?g$HJ8~z(mRsET6LI za-dHEfxq@5j4Q-#{5ty=z(UccT|M3&DW%IR<{UvMU#oadlIZP|FH@k}m8$LYo)5O@ zd{EO*`N9R| zq#j3DhS2N`PC-&?YV1PErM2lWk66n4SSm+huZS(Urgs?2!D7W&xGc2=5E4A?u zD3O0b)k=`qz#g0Satn=mZW@R%%>lQ$*b{3WwC|(IC0Hyjo;$4iVT<_G?cKWCRGiF} zU{{*+-uGzXF8LsIa0nZWUwFHsiIavTWIg)m<>^iwCW5>+S+{hSev^1CeOsy6cD{je z`f1=$DCN(gAJQA7bVWLx!RU}xuHVMh%U8#V*cc-=DkVTWW9ljsGX_$Oqp~hFa~dRM z&o$*Heht#Mn$Wh1YK6#Fz1EaVvn|Fq>{uB;H$u>rEmF zfeCio{>|*hf$&-szI<@y+u#|asAy>#6StH!X}MW-iP;?%SAB);e>RNJ z6@+Wj=+#<==45@FQohYlmor5!89~cbAYy0uMxv6q5qMTTiDy{pdy5!azZR9JbloJ^C2&i9$dFo<)MT7V0$6z;COL91=^$fZQu`)?Nuqc2# zhKSdz`7Gpmakm<}t=pm(u#I{0Q9-X`^bIe)%Nb<0Ge4oSI+cjj=~45YqK_iihBwMNmV((wZE zI_5`7_#~=cg1VbTqDJA-=C7}P0A3ysg%6YOBQu;3D7HtDKii1$K5M;eVUrY8D|^Bx z@xv#TaXUOZvSO@3++(XCzysSkB3h4XPwg_e-Xg?dSS{{YUWfZ|Y!fh!uzk2Tr+>j& zt3DH}*JSC&gAQs7YX;G?#e`i{LNq1Dk;tx$Vt*A9i)PTY<@Xlg*mFqxQD4$FutnD| z7$iZ)HE68-SRx^{l;Ev7LCI@0!81v67Le-s2k?okc zYtRkx9{4@BXlWINy`AbJ%%>~sW6xq18+&qVi_oBn1)UqqY5R}qd)7ljBti_QS{wMJ zKl-?dU8F%DNX6zdn-RTLfVS&K-BsT_YL*gyd%NTxy()X@9xaOt$ve^a^8d6jTKoC~I3vfWcg>5jlOo7Qk8T)j{JOzRis;V5{Qwx$q1I;@zvIU zJDRAJYGi$z6xXizb+_Sm5ZQpk*+tZ2BBvRMbL>IYX-}-u4H)a6r8N8+z(1jQjJnh1 z(R`2kf}A)5p>>!-Hp%k>U-f}uRc#zs4o$F1W#zpKM#qk zb{A_a&`@HEJr?0Afm!ug-~;re6#d2)_;{?ug^_z5iHqDrnDjD+IPP?J!fbI&&Mz*7b33O^~``APvn$&)VO+;`e@alWc0S^tAvSa%LPl zZluq1h=!3I4W|!TDErI~`}FVu+m8J{`}wTvWnUtYF zEc3@5H2Cr}QO?vso499FdM_5DVg_`x8QbnP_Pm1ob~V_j1}S`Sd8OFoboJwoq|3SQ zoR$XXttXk2K{q_Q)UFQ1y3(BZkHKJf{E7xjFE1Lq*vfGc>n}*3a-rJ0kxM8y=H0*! z`aWuqQ-3sow996n(M<-YY1+Q~dK^|?kfK_>=7QBoC4m%6588&SFckVUS0l3)KX!*M z;J@JN_o^nQ*M7-te^Zr!P1s0;jM0pn`v*)bx#IosTyzm!LlHC)n-+BaLDEw!O-^db z(u!t(?T~1A^XG!{f=h;;x011`5&ffw3qg~evb0O1L zrQT1FxBZ>aKqy9=cQ}7e;jk|yA(<$d9;mMIdzjQp=E*M^Ix3ow*{C+$;}r&deKI_ z@5cvJUz-hFABvaCG6uxtHhPmI$u}(T@`Xa;sv57&nT(O(m2j0I6xJb{&(O#S$Ryj{ zBu1~>&8ge0(DvmFzOy!)1wuEDgDO(!JPwjq3n&tfbB(Ru4&O(#x67?%M~C$%&X<)d zw!2AqHS}dE&TytujghE^O)|~ZlVI+(C=)ePRpB-ArRPG|A0>f_?y^uIC}s)n=B&IH zdTkx9UVu`(%uQB9n%FeX@SPMony|7+DRFIxUJ0+Ye$ijIwo>6I@`Gq`%+dkI` zO~dkVI%XNh>kW3QYw&X-WOvSW^w~Nlz(-HXvKT+@mA>Ae3g4F@1P)%o87p27XF1lV zSY|eI0CA1^Fy@t7OMRzeJK?w_&D9UInf)HPQ$xNnFrUV=hNugoI!kSFP@>%1q|Q^2 z>g|*)5Z8pYEqW9;Kp!{RpC;O3+f1dzk9uP^&H$lEJNXzQZKAY-2)!#tWYqnMB);|- zNMdkl8LzG>+`7cGWwI`gHYWG7s8)<(u1qNYDDBc^7IUQVjeaGoghzVqVa%e%qF*|+ zc-YAM6_&k2-gJSu8|wJBqt#m0l*r3-!(nNoR|rS?!-K;Xi*%}ev$nXVbi$WnRFmWR zAofTaw=!l9tOeDC$j&hk1#%jP3pxJkZexV|&WQXW^$3o!*>Sw(s}Tw)Cj*ndXSFa< zBr+e>zIgz_v(3Rp*K)%;zeum@Vq}M0ma>e_#FUPjWLC079aq;pp0s?+i)nO^?QiW~|-}dk1s4_frc=JFMN~qT?G9z1_nb*;$RKt}w?v#bPt-ZVgrPqdbLEBmh3e%!HrASrpe6{T$hxD&4OG21xEJXZ--L4~%9a^+F z<(@ewImqx`<~Y>{UZwy+-&v zyggO$^E=2V9H8qiZf%Mcp&M2;eJpLiR0(2PdZ%Nz{#+vQHg&m-q^G-o&GL#S%!v0I zeL3HnX)-_9jM^@DB6 z=Xj-k`WuoE28$LwDkE#OpP>k32G-59gZDDaqczT>6|!Vi!6HOEbdm?rQpEG{zKFE} zDmAepeOVY0^yF{wEBvbFA_)1z{Ub9;(KE|N<84j9(n{khqKOm9VMB6XNTVg2^EUnh z3YqxEV(D+=!e^)rr|_H~0N8C@*zsz-w0~QN{YkQP3rz=p$Q|8ghcF#i9bpBHr>#lu^0V za(y?AR%B&rvx96G)L?O7J*8}b224{9GX~F&nX@!KH+sruNzIqmw{W$e5Pb8Z$zRbR z9}-LTqMd^n7{e;0r07%Jeh&M8j<`~doht~q2u^~IB;G8Okr)hSDYO?RL;0!vH>^zL-meXmV7_WO_p^ol?`0n_U(=xAgMi{hl{8ECPwwRda#0Z2d$S zOQdzeQu#jyz!si<-*saMgoc(TST%LY;e#wka@r+vxZbGLA#8}=9E*e`CS`ag{*OIyKDXp} z7RC*WpEY18^q+;kM82^~K;S0nKd%J7g)`_;NWOc(P$C1;iJTWhK5U-WAMiv+R5VHJ ztUh?-P*9$cV)*i;D06_sGIAuitW*8+OCK&iOZtU$2FROY?PZ3B?1mCf(NUGS7t^k6 zu&2pMERaN2a7>Px`l4x0=Z(f8RpglEugZ^lE7-jyx2RV!g1G}EY@iAOuES*duNu)VBwD==!Y3(Bos4q%!EnzOap4aNeZE9 zW5z~p&cW_skj;GIJAf6F&3>PI@%gny08b%sd=T@`lq&7WS*;Zd*Sm4tRo6m{e6q_@Fk4U#7t%$ixVkHP z_gF5uuNGeP6txG9XLB6Z1UR*}!S2)7J3fwbENiQu*2lEn`4a=b$`e`HIl3>X^MWcF zv0Gq2VHk&q5TNpeiM-E>ZI^7-V0%Yi=p9{EZ{)NGGkhxqVaS)x3!My81h|{4CJTm} znOX?5hL69Y1BYRIQ|XBu2k?10_}Rhig`{beE=(^gL7{AY$q-WjdARCorE2J-dWc~}@j>z6q)YV`IB&)o zuBRe?y5h{|0?{1fXM9Ap6V3@anaQMBUQ9eQf z%Od)(0VDAoga>O}4x0125x2KLCMS9GvS{{|2T93Wd&mH|Hz3vxr$sG^}9F2b`MU=8WyPhg| zn=t%40vLaf_uh3oU+OhHV+H;R=QGO{a^0fch;FM%-Ol}v2X;eE1j2W;4|HNQq6|uo z5E{;t-0R=iEs$SZ@>iNFMtD1gD_P1tPW{jV#4Vu6HwfXYm2$drr*q zdSBo0QlckcUgr;d!_>xnS$hQ+Z+bJ85w_VBcTW={JWngV>^a|53E~F=!1>I<00;B& zFp*A~VK%f?(XiKfmiN{&E3&W?#bayk!);mY&cROu_C?s3xd*j(sOO6gT z;STqnR&GZouc*C+;UF>dMi;M}6_`!#@09jcqb1D_-NlA~?+3*{wAR)L&vfT)5h>&F zA

`og$69Iym?;LjN2LgfX5uS*jYZc4{lvYZnysBQ`P}&-f@GbQ*vPt0KbUq6 z>m4EfuoZh0t?e6zG(p-4Q z8Cun#^Vq&5onF-2`U+NCZPXXtV6Uh}knmEreW~jlhtmYP79oycToLE-zLE~+xnD^r zL#psDb{Yy0ajV_qr?P#_6R@&eu8=KOfTXWRK4U+Y&|A(&5C)9P=fJpJBg$_#jDVKW zxjlQiGk?b#KqzrCKN> zieOsG^t=B?=F(3#?m&>xtt8}HSxH`0&wdx^eFyBM>3V0{anN7?qE|CX-Mk5al0^V2 z3;wPtxH4Pb%$!JB-Z*b#^y*SgxjqbdfL{~$$e~g*R@If z_%v(dr;0kn`4XcSb3U)ZWTxVYN?_p@a8-j&RPGuh2NuxF z(eS(RwtiHPLbT$01GT_Akny&54GX zpyhL?>{rs)Cx$igGskwjrUZB{pY{y9|KEYFU3k#v`kal9M|8IdUCA)SBmo@BT;NdF zOI_kxfCqULsZ1?f9iy_0FzvUcv+#Zo**Sfqz6f5Ltvl>z{oSmwMo#66vzON!)ETZW zd-+i?pt`qapV#dtglWN^k zUzF)k>dB$>pt46pl`W=EnP@u2LuzEn(tC_=MS28xBCN0>_`p|#WP1I)sB9qK9N;qg z3WZ<9qlb@JRXX55*ltF;khlA-5(~zp$xEBL9s-&9$BaCR)>O9C1u%{CQ1N#wS5)vz^WZVatsw1qbM| z2AmHVW&?Q(>BheUQRVWDek^Ld(&lb5IAxuJ%z}-QQ_{0^4#$3`5vuzQwU8m2vZmy; z@mr3kZ;zkXxU34nUO(i{w4VzNjB3|1=7%`iyV z-6kSyY+_dZ1;g$-UP`~@sdEqs`_hD~0*{s{n%&Li-Zf03-OWX@pLgcUr^4xyWJ)H{ zrX3}NwZNKz<$P)qcFsNLS;&7YsNpT#Fz2U@h(_s!X_6z|fLrgSEdZQ(hG7obMlIz`bWu_%={|!nkPl4Nwg&^-KK#! zl>d5rpwe@L+&FSp%&)8z>O9^|R}!j2Dp}V*=hPxwG*o4D-LIyne4>_vgGeDD6I*#7 ze|bEfbQ7AC${tfE0;|zdKqm7a2|Ho$Zd)F2fhVcj1YOuky_g9i0~|(bsUyEr1mBWE z|1@-i8%bYlxu1W`y%|jn+kwz52j&}AKl&W-b^QGgsJi6$%4B5p(ld;zled%q4fq0Y zREc^D&!^gAKwAJ_&J%}H=M8~kkVnuL0r04TCW-zXChOVSgPQ-**96Sv6bzGpnd6UU zJqHiWkfsqq^*@S#iF$Rr{CIu|KyuFa*o<$_OFg7r<|uu~Xx2B58@!LLujWx$aO@%T zd7lLId}Fmcv9mY*NHni$DSu@gMztvRvltfW<=}K!g5L@P|Nk|qwG0bL3Mis`(em5p z1(xdLzx*ozAp4h2-UF0v)qAb+6@Xj(84eV38OPS^^N^ zNbnx_V>bB5f=v;Fzi&xb1)AeRrj~Ph4D*jNfu~4-;g)tCZ=d=QctQ#+5h1i6xK8dT zQ@X4FRE&iJF0}qB4e{bHb{bHq9YAI-M}M>ahs>>%2EnyY0;to?2T}2_;BW7MenRqi zmHC`Vjtugj1^`4HU|ZZAD9`_|gwIg6rHC2;H`9I*5!@5r<9EB#QFvB~pK?PbP(9y0 z32F~WQ@A8>{IsM0tq!0T>WZF#EfMZt@%V_s08!Yf#ency;C`GBkjMA{i9jE?S@Jo; z`QHZ0F#td9O{OY;L;M#m59Cgg0fJPSsVM!XE&+}>@F@EVF0#u>xc-N64(Nd$kc5-n z3i1y=AL!O};B&^KQm*_H{ohH@`W8LE?g+qEFSX{s{`W@+Sd72?_H+s8pTC*nd~m?s zu@IQRVdYtW%bMgZ$;7LE@Me0e5tvq%Ebe>$3vC<4%g z5VLaM|CId21gmij?f9vd0bhC;v>XLW{u@AS7BeL!xPRXa(9FRYbgt?Vtv|g7mXHXz z8y*sl<&Pc>P&|ir!RfnAtX`sjekGFo1YAh7z;1>3A8NU$C$QjA;+Oqr83UnFHmU%R z6n*LYpPP-qU9(W153FW?DkA5@f!k;zru4hvPdd60{CEarL0b7@{vA*GRN(8()s%j- zkVPR_08CW=bKu8G;}=ff`1ZB0%t|o}K7Xm@SpZRMjk1Pk_bJbp=&c5{oF)n- z7%L<2RsimA4~D!19+)*YmT&Z*LVY;8K)d~GPkwU#dm*0(Tx%jguaQ>-s4wqSL{CLzWsl-0|Chm4%0{H!2n8SdC-nbzKOa5N zMB5ebxBsdKSp9rk)9-$x(tbDS>25&)V088h|ie3!Ye?}py6>A6I5{J>>26O@k!E%!B#$Ak}Gj!>R9Do}sl3b5<| z?6Wb<8WbY)m{k^se9+Tw=La8LEBn9VdIEdHQP2FISBPw5=_DM2|AB4;gMu4{Z*&!q zn>N*i0VvQB@ti;$>Zw@BpQ-A}mjhO-lC80k@yLm4%MIqiIot;#xmS?+{}emdhMiz_ zO-^cf%j2$#Q}cM1_K*f_@g9&5{70Qn5`In_dgJ{1*A}n=R}nObaKA@(yQt?w8ge$Fmpn1wF^x+n z@X{h$YxbvY;UX&E)7Z*~1r&m50$cl^6}ug5##?gm=K5D7K_^UI$(NLl9>Ld*>2$JDx3c4`F~e?VBn+hI1D3^ z{r%d1goZ<4cE#sk%4hm3AOZ*QYQZH>t0xBCa{&l&86OdFi3L;(*Opyyb@pYPv=*23 zyr2KB+(3)bfR2`e$a>A7D8%Bf%QdsP8FzH(OI!plb#KdgfmVL~>Xl2WR_N?mzN$-X z>NO5`iOLbeziN zO;+=^bpBv5C1vq9=|RB=ZY06X1QZ4&u$W4*0MMEJisHqKhiMwLa0!$BfAsos8!p@4 zR|0c92lRE!+*!ft2>i9eUnj>pvhu9Gms{u8IR6{J-zN9!LMOx zdr^|A8h-f5*Bg2{G`?hpzJ*N{oy=n7L=y#J2bP0R7!HgS(vTkxvg$v08AF{b8G31^ zdm=(2c!aNmQ(z5VuH<(<&-DV4U@pTtein%nffswmM~6VM)TO3)+x}PKDL654Ed!Q~ z3a&>1s7Wp5E3kphsxGhC8gE&)HUVHi^izdU2rqv0_|%6y-n$i0)0dtl93uudQyXfL zTEXr#1HhT|rX_I+-_{;%4AUk0 zsaU4I^dnvCh|V&)XhIgxue#~AahgRWOA0ONN$?#`Dz4c8fYaxKw&$l(cEx8as;=*I z$1df@&Ky6x*c;Mg@E_7W8h)~G$hqR_rcl)QcJ zq^fNi@>+rufP)EJ${Z!<=q+lx>luD(*l=v7TfAPHRYb>jmy8N6z71b^li2O3V#0Z{ zaEmPySq`!UnqFC2CvLr4gN>Hi4rb8VE7GVM?Ep!xdkNahrpb!CCblb!Cnp>)i5L6j z3$hwicY=~{%vzcnNZ53%9}Nnvi!{>4h0W$oSZXvwT}X3}G8Sexl33D8mYnf>_7~NF zGn>6h;oznE;r1=0|E z4VE^$fm*O`-zMr}N_oOs-mY^h0*pzK76-eLcMa8&s``SZjidbQ2rnXDk_d7ib*?|Iw z>He=@X%5+Mnj97Rm-JMn=v-X3Tzu)dMy#XMX>u)@Jps<@CX2|K~7_@y#t{rmdGO>-khb4HvQ+hFbwsn_`hFS=TEf?F(@cIDBTvBdaEdEfNZ zQ&zF?e$7#fWy5cZI3bq%8`MIF@Ck?JuRgqR&8X?hRY^{wt{;z)lStkFb#`nZ z;2yRPsk~A4QvPYT;YANSqI7;7&c6B0QC$05;IAUzF?CQo0@HU}(1D4qqbva9=egAv z>Utd`a23DThXjXhok6&m0#sftr>NlwdQf%g2eB(UOEzxw@(d<93er@BUmR}UqYmRJ zueXMLNF=E)2k$_=-jK@4#`(K`(dl>lBC@5MUY;99?ssbxlvC?D?`#1krXd!ON4%rh*e&c%=VCuw?ABJ7 z5s#&`ThE?-B)8C0-7#S65-pxyM$fV&u1;C3pQ#?KzpXZSv|7oN_u4iUG+=Na{OTQi z+(GKJPsX5mj>uNZg~%=0VLs#L$4tsmcQI4RSvoQxiqB-ViH&Jgl@8Io4tP(Q44 zH(c~TOyv|%-8URHT8KzXYtPNeY7|82{#pkj3gWw^SZk8#Ce^N+n{4fwa)d5u#r`oX z@L58)-BX>H%{ENvj~mMK+b%v*g(48pT*lMHDoV38@6EXicXBz zUY>mu$CZ$BJ`1?)UMWUja?J4{@5p{zjU_7Td+^@-i6#kb!e0onzk}~+dQCs(J4Jy|^(cZvWkJBsS6nSM$jD?N*Kc~DVa6K;? z{%Bxa{W%R1$#DXkg&Y#UCo|7)ds#lZmrYwYC#rz%|7n5EARP|-pO*mFZiBs} z6aqajo{=A~UcA@S(*uHUe;7#|(9h+qXpty|vQ1BZ-7=_EGr==Zs?|@)Ii*#0?)})O zN_ev$ZHa=#_fZYJlQx$8QM?LAdI>%Dh5iuB5*_cKJxBUK-e37FNLZ`KVR|brt0$OJ4Y-n=mcboUyXpao$ahtnFycAn&*Mr#;^z_uvt_?O?M^QAFB2Zqi z7eLX(20D=j2uZ5E5R$pboIL(Mxot-APT-4e9V}rvQcD|uWR*nUbV5z^#F%QyIVGly zepHdV%}fT$+gP|A#=;t*g%1;CpR*e6S{{nB1oZ;_LBnIH1e~HR>@{D8HDp@UScgi6 z5R}^-RZfXrl2DQyNoOLrzDovwuk|Tzi}|EYyS?)hGC4&15w2Bt64+)Jf#83%%%Rkt zHgFUKA7;{9^f4Y$$w==N)OOpx#HV#zr+XG|NG*ILaPlHae9r!i!nECnNW4%y4Jc5; zId?au9(SgjCw_7rBRiOqO+Y||_z=4*#r!U?YiL!?Gul(0lHUSaeD|YCO-}%m(L6WHBO%YQUD$Qyw<|W2`ghyG>n$;>i>gSpiD44&2zfau{`FLHXHF z<0%^ItYLA7?`&AA3$AO&xblMoMtrw-GO@!-_3exgV(-hG_klSk0B!n;=V4d11&@%| z%bW-r)_t4v)uJL?7y!DDHFIf7cP!~0FyD6XgDbPEXmn_qT#LqdEpbK_m{)3k+?ryX z<@96FHS@dx+BfsZe6U|DZ_43K)UGGi(HSTAa=MelFL=vFrSh&FO^;|RjpxK}d_fOd zFDQR3*XC1u`YEpgMNxJjR3kK6lsll#_l=Gi(^mN%bF$5DmP`Z#Ye}ZK5erZ@S??r_ zug9l(CYL|7VUDVoGt|Atw~Wp_60}y0CYk)P6;RIQ=iN5Na^Uuy5;OpU%i*AXs8yX> z@N*4E*biSvz2_wqCQYC@cbF)x`-5fqs?uzZKnYQ zv3~u1N?)nriG9$t=faZ0~g)VtIFv zI(88#B%abZ)%H6N7$}J@QVzF0i|H0|RW@jl`Z`DA-9Iz+u!$NnPb7CX*{f2^D6-S7 zpYObT=+RJ%+%!4I4%*3<^vb0xowvIXjs&CY zuNB*%!nwb_T3sDGm%K9;ac-P{?T^LQSO)=tpe`=NuT-m&pmp>eMcY=-rt}M|l#W)| z|21{p;cT{Fm>A^?Q4xFA-oDym)~GG9N0nMdshVx0sJ%jBQ?+W;EUod?reAB91}#Br z(;|LWXzBXBsgCcu@<*=sI&a?R$&)#;>A3XRFxd6Pm|_ZNt}w zv^`(N_9U)ZfLzY92?6`Eeszm}XT26_>b$M0903 z6_k57JH*vvc|fMem#516&T{r)rAkSiCW~3{!&@9V)%*}4;U~E?2f4{S)|EQQVa!nJ z*5j7FtbXzA2iKJv&Yc*bas{m~u;HTSKM8ILm4jb&eF9b7g-o5vG}-ToR}LtT>Kz2Y zK;`}1UK;YW=)sM8!6uiQ`_4`~77OQ$W;-_R*E}SQsx3K4WEL!zIn->t{h^Yxgu`xyAKi z#-JwO*HCMsaHH_zUsZmMtrIZZaY#oiYK?`VAf#+mB9ks^U`do3n%La)i7+MA72Gc~ ztYpnGWmCSoW*L+pr~A&?JgRXoh9husy3ctXfeDwQlXyR zQb-+}i<&a4t_o$|7ZG<+(OT1Kaj6tgc*o^c-nz z`3B`|l%HjfuQ(u@J11`blu43hV4t{oG*jv_S#(vdyG^fam>|`0!GS5iNNm=e zgTa(RGH5m^-6ToSS5aE0{q33Rn}53x@86f;2+R`%0bPa}7dz=7OOVRFh zoHQwFKx8IxmzE4Tk&>o_yd;lHvheOe_CdFk&#=Y1jlDAZKLluj5cjR># z1R&c3T9vnOr|qSYA|RFL18{5Z5P%~Xyuc^)DI1q{*Gi-mxi9Lcc+It|1L6)MY|BA7n!)kZ%m!|E(3Du*V; zynaQd`leiqR&E2U>L^78&`7D{CfrtGg|MRo z@a)JxpQR9w{31Fe-WwTe1p!4>GfJLIsmBU?Hdm&ZfzHD*esGJXKEu$xxhl2_AWcf& zRTuuy6nvtALj_OPY(q)S2qEJ9`xX~!72z|lKTP69F`J&BjrR-H&pn{KRAXASpLt0!1bZ;?+vK_j(yIIn<5_cIMT2NGEBTgg6aJTfF9Fw}mi-&F!Md1n`w>Rz| zL2|lNutcY~-@KBHY9$y_hNLuw+oW$ayHbgv5WYKw<=p1)E}U2V^Y zU#f>LsURd|v#CvUGx}ntS;^iTnHI#uiy5ng?b&pT>(gR8G$TXLCIsm%bR}Pbt9=ra* zPDZ19e-(RNuOasL({t%11ypzbngxl`p471^rh&Qb;}OntiR=h9APge&KLFpok|7}< zzJDrk(0`Gf^;!k|jMJvu1zVTeBy2^%`a>0s^hMJ8N&C>BQM=#m4b(ir$qxo5S->d%fg$(*=^%Ne^3Os_G{P>Qy&WbOOX0hWX7p zE3X2>m%-#oNWL=?iq%i%&1je3FoW$#Uo1)ySRO{lNuu)E$7DHuVX+I*RO)I-uRa%o zgWX$APkrE)i0ZA(+jC`%Pv2vz{nT51U?(>Mi$$r@j+zn#bLnXNBxNmrLB2nVVn3md zQuTnQ;kFj}5_z(h({&2n^{`puV8h#`w_koXPHF(OUB(fC8-y6YhcPp}9x&8W)vblH zqu9!VQE0KKQ|2UgT7AMK-FePI>*)~o!jOT{I@(<9E*`_H@JT& z&cJg%(GaF7LTIsGJlf?{ScE(W(&5&yNfdBohk{^=8=N#TBJa}{xc=oDc=t_2rJby& zw4wm!f!-?EP39P5m`gHT75v95Hg&onxBv>b@F4bXFiBN$F9zKhaCl^ONo>xATw_03 zc}Hxfba;XrwrVlUyTdGyEzLB$qY*YA^%;lVR&kBYAmudYi``NLxN~Pt=(K z44~`o%3DYf!wFmcw9@(xoM#D|L3VWlQEl`Gwc`c=1O{e)Zqf1JV(c-=DmW|iE3s982yz?ml0JzJFiV@Q`1$*KXF0^9u|lz}e- zN-`{euqaG26lB!;!MhSk`hjoz@d|(ZGg}~F2=>tuB{k_feE~TRzz|^$>>;OPHV}wZ z1AszDU^46fQn~yx10E6r`mPCxj=z5=*1@}hH`Js<1_jwPj_4HTlOPXVAv4r7)2-LO G68(QI8EDD? literal 0 HcmV?d00001 diff --git a/docs/source/user_guide/feature_guide/images/netloader_timing_diagram.png b/docs/source/user_guide/feature_guide/images/netloader_timing_diagram.png new file mode 100644 index 0000000000000000000000000000000000000000..0af05812d855112c20a1d58a869e9890f19c8ada GIT binary patch literal 37598 zcmZs?1zc2Z*F6kKgEZ384bokMw9=i@oq}`@Qqn0MN~@G~BPiY7-Kli*odM&1-tX`4 zIO1XEI#=viYwbPn6y>EJBNHM+K|wv1krr2mf`WyGf`XnygaPirp$eWuL1942h>NPa zLGLy`cF@&uS-6Ouq?Cu6jG3U1j&QSv=YlTi)6|55=MXT8_<-{fk;ah9x}KGOsK8AQ zy25fFXO37Vrcp5F? z8iHl0T0;dBcDpR!%(?0}``lbUinx95d#!|rewN|6U%PQTPkEK*Sr2YL-?+qr#>Du~ zWg!FOJ5)G0F&QOJD#HCySp>F*61T4UM~Y$lrB;(*msnQoopkNi4`I@a6INv+d#;x^ z7>juuz7(N*-=uH86ch`BWy>%~5Z3A(n|#v1;jHg{1<(S@q!%Zd7RvG}ppq#F;Kvx= zqt0HOR7R09eI7-J-k1I5#x;{ zyFm1#oRYBOpnq42pIH^_rb?5wNbU(y+u`G4t1k};cignEY>?8)5pEAoNtKTkU)=~{dyZYZ}Ml*otPIF9z zKa5`_58A;4J_zO~2i`Al7cl!j2OnIz@YfrQ9i*B1?PF`my?l(ijX}>yBwu=Y*jOb&hn#)-;}8 z1Lx+sgh`2ntq)ww(~Pa?g%4XiN(2o|%|$9@F!MrZ%f{g7ZK^fbtL5ue(G?sb9+Eez z23iqHMR%s#U4;X*g}0e$m#Z`-kTb{HLPB)CftlP-JRk(S!~EmK##B$ z*#}#AN-%Xre6D%z^1edz;BBTRk%@w~6HV_(Bx0ngR)kP|6oN+4j<22@+t_bgnfIik znGmq0X2czt9bZ$;B=dF8jCGt!P59f9=<1c;k{qR*gV%hj{l+s4{xo53998{!y z!{A5w-{j4ZBAkhJf6mTdZ~7>@`V2c29k0!X!%alK=22jKU1v_H^l3Gj#Mk>h{n)k0 zP$V%(sQH7V$MNcB7yZV?kmkL2gDHUBro)OKeL>CY$dK^Zt+s_@4qBzhzV5Z%LNCb? zO@fJIKi$k3F7tvT&!&pKO{tJe0Z;c%ow5H-Wh&|F70B_dHf;-JN7-LXJ%BWEnSd!^ zyty6Gw^lV00auT2&u6{UUOt>yz8=271|Q~uqL4AQr{h0MZMfYXqEs%frqXQH=mG56>B+8?v+_dIEgc}LfTikdd zZ(ouh6knQ#!D03@E;<*7C*V~VU%hedbfp<7t(#aQsaA1@z{cDog%9c?g8BAaUEuS8 z3}U*`91ASSlq}eQ7@)lyMB47ym}K`pNr-sD#kb!przHcnrr7;zqeEna8o?*g^ULC= zb373$vorqWUQE2J6ttO>(>Dh5Lbix+h`)!3e2Jdv!RvxjR^MB%7kPH0wauQ0U7DoT zCs2QNVX)Ibz1!t~0djLr)hNmHB~e{}qh`0&CG)82pu&k*h|iMd(>sW5U%;AtdmWFu zNnaogO*F!$z>DY5Kw{ltr&XRPPqxSF&4DMGwr}1(mkm|IXneImNL64OzzmnIV|F?M zUA_P6lw(eNQR1?6<}n@lGK1g{?rl9JDo6v)p#Y8kYQRqQ+uq#Zks z`+93g!N4)=M_O~e=@Re^n*W~W86NAK{9)|m#3fF4N-PakY|MVM3+$D9N4{ETD|fcK zs+|*~wt8n}y$X6)MiP5~``yjO?_j!7(v2he&CHFKV!AW{;Dn$h!JJ{ ztXjn(wrvkptbca3EX;Hd$2S=+X!03KzT?hU^)sY61|B{7_kwueOu|rlh8g36fVkMR zqiJ#4+U;IMY$PQ_^XQ4yN}1sI8OVTI|s3-i5BNplj*|c2vEi_ zr_d>7$>{ZduTn9fv+X^;J*YG^mkRSpHTo2F7H0!qFBox@2n)5=Gv*Zw%Ll z>(6ICRa$l2g|ZjEMi#KiBa+oI04#KWSb+M=lmn zt2L1|A$kh4Q#xT_H^Ys>y3lk$ASCe-t>3sWFCS_0Xrq+hO#5hWC+~+v9+70Qy#ljJ zBn5U-?0c4Ulv#ngVIhfU9cVw-+d$TWeEZKiSQ4zZ)C#>~5zH<8OLaSv*RGbecxztH zw=mo-!Va{T@Z!U^h*~stc9JTaPH49Y&UVoqmh~I8yiNHvAIKC4k1=Y>u6W>kSO~rU zSr_L%Z5H4_U+vT=kK1e~$*(9Mn%#8O($;>VU;A?5hx)F=hBz(bra%0Opd7W=bS_Lb zk$rc(DMma0Z0Om+*$RomO`gyJ)Ag+s*fnVVsj}z9jz#-BzN561gK449^HL-#u9RCh z&s+IFDhDufNyB$0OPA4^)Kz83SKFjEjn?Ad-R`CO?#+~qq<||YMc+(?o&_f;M}#{u z>09I(+*aT=(~4PL-898(sC)TYw>%GuKy|lM2s`fZW6wiB+$caRg(?h%`0871#B1Nw zl{>X3c#4bE{u4MRc1Ukjw^)wh%vkSYuApB9HEd2GPHeT+t~&D$Dj|y<#2tEbxDJcz zoz*gGHH-Ag$2GZD_3gC zyNZC=h?=h$n%$Cn)3lRlDW+{>_{DqzJpT)Kbs_t8V~s}N4oZz-blaAyyXI*3(_SIz zANCoM?o1Iys@3LwJ8QLacXH$LFS`8+)aP9*pLtPHimNly<1#A)!+)@t7dUw69ukY0Bf3Vx>4Xc$SG zic7wG@qAJYJ!hrEetzSuL4`CGVMeN169oYQ_o>pF3EA;P7!&J`m|3DHOqI*g+vVHU zxUy0gK`VLxFER%M?6vB|M=!TYI?4x{{FPp&&(t($4qOGt@sE_zgvOJZqjgK)-o^ zlq{2|*WCp%)PWUP1 zh3v8Fk{7;_l;{%)CA)0VH0L`Vavpp;}tfkV0_4UUyC& zH+*+ur#=g1B!shCr|_oN(l@~e%_+Ngc$dmz3|E9POKQ19J{Sbo(sevC3HTl?>#%$C zfcr(pEacj}P zYyZZ2qVp1FE?1*XCBJGR%C~riBjvC=&oE=~magsd^P{>AFZ!9&cV$Pl6Wix2DdYFd zK|2bx?W=RPMokcjc6b}S+w^JX?!(k7V-kA6T^q|X%W+^ZtbXcY-RgWjObMpl9l7oy zzb>~Jiv=y}*Mlpru$cyrp{DY2&O>j!rmYLfF|b$b-KO=W%v^(qk4xX{sGQ-}ijvx( z9M2nV6(PslPiWeO^W}A3Y(F!_$L;K1qvs?({$gKe__dj5_t{rolwzx`#BG{K^G{wf z6utYt?_{^ZXMI0Rd3-CLlT80G(dCKyK4L#4~Sn%y!?;F0>NWCTH-&2p&pcaU^dv1L9 zT;D^ySldE+v64RXW#B7H$38_%txH_G4DYEP$Gi+LmT<6IFa;^W?N=3f_@;pVd>w%-xH8oaCfy5Y#R= z9n3pB80{Cmt~O%XCC<6}()p@!KzUMfYl7*DYKXgfzj&MUSC@eOyvUAr$*1Iv}v44P^BC(h~c_2;ceT!qx@kaBz~;zJo(b@ zVDyg`FtLPc{=yV7e~`rzwW*B47f$ zx2%r51&$|QcZt3xD3+KHp>E9Z5dX|a_Eqil_-LJgIO*E{WImo=GXKSXm(2z2?jDLQ zZGzQ!!$_iSLo2#$VS9N8E|;DJUecb}oDBONY5y+~cWv?&5tFM0zB}jC*vkWfqOk8) zie=XFXKTJCpGlVaFntn!F(;f88qO88bJcLZZPNoeFk1$0vsO4I7xq;PG`7(g*BRn^ z`CIx9GpMgeoWOfNvPo1rfgSU!$&?#-NBng12f^9!Lb3KOIasQbXoY!}XczkTCZNxb?$wZ&g1AV%|f*@<7-wnTU2HBx1oAsbu1?ba|~`Y zu}zrSPgUg$Ju@^*!eehv)ozk?Ygt_OJl}p9w63vQd-W+D)vZLSfgy!b*vVtr#E13c zESyCH;wUZj)#d~P{uc%K!xw)PhOBucRRIFTqRf%F@ErQl^sx#ZEjtE77ToEoo|FPN zQIa~=k024)$^3?P?CeU(wwrv zSD4IK@0h7f`-_)?K`rx+b|v()`|&6Qj#pPXjnA5Cv-X?=eKh5sqU!we(RltTX2zm3 zV)Q78x8qGyp^=C0}>Mf&+gso3G`z-VSsz39AsEYaFH+GoPZ9hV@k}g;RT$?%mjLgR?v?7 zYf{dOWln)#&Pm-(2+X8f`u5e=*3OIO-l)-@9#GUalT6<1~waz5Gn`(@>1l|fnM>7euH-LCbs@33vQ1H zNa(Rx%gnXHUGbxXXZ4tMXLh9_7ccg91mvwc3o+}vzcdRP|47z2sFu;uI{+Wv3P&r+ zPK9UeaLdu2xav4ij;R^&A85p7a{88cqD`p%HVsD%MfT`Nq2j=bi)33+I`?G!pqKXA zQk(TN&IU9Ig=o2`PCEQ`4GR;W+i9l4RQ?n3@+;(@mCJzoJU+tZw1vp3^X@ZFxo zoZ7G{iGJ!dFhKzI786~dP<#z@a$C*J*tD3(&g)3M)#$#UGyXkoPH*7KoC(6vq-1+@ zcQU}SKC8}c*JCNiI3KpYN+xht!$U|auz`rc1~NNllSrX3he1KB4`2282Y9QhG_P?F zVivF`s)?zPk_AaV!tuOPJ{t>|SzmWw*jflpYbqh|ZI6Y^^DGW`J2_^(eDtG#?L>QS z^m`TKSocm4_GVW`f6B2}wPpB9029Kj(!(BR1p5~>%n||b}uH9!KhIHGkjrSs{ZfIFUYglC=_1; zAFz_qaeCO#2Z1dFM1onM$?k9%<{d>!q{PPywh~>596c8`zSm<5A4A)DTtpE_TCEhB z6_Uctz7gUZt0zHd4g0N@(%;fc)PBIz5!xVE5p1X6D$^1QCYok92{pl2{jAAe`qQ^V z*r#+B7LH%86#0D%i6L_(Q*UPjv*xQh&I_YAU*YijV{5n8M2zvjsrJMl-^F=2c6+&P zZ+96opB$lz!TW|X(M?2coa>78gZ|Hd7<5XtL{LmDPgYP(er{yt8g;dZ(!Ou{r-O0| zKo=>%Zl?ok=uZ-gMwGD8Ct!bA-X{iO)mfqZ5@ZO_LVSE#oL{6%NN{VNTj|1a$rWEx z%KaQ;L}?7!*uU2xFKj$2XMbaCK5*|u5_J1#|K38B!9^a${Rck>ZjT1}nPGy> zsDNW2IQTS9^`BA5U?c$7#0WY*FT-RUR)Y=&&@lUD6 zZ;SuU9azMmBdj(Rfko}rC%C`!gsh;Y30T%{m^3o*?#EIfw7=jF3 zR}%pSx&?e7zF(g5pPmiDW*?9^4cG?<8`c{B?LTBI9s_f6c4AxwG;@$V3>JjL-e0N_ zz^Q!tkYIQZ%OUFYTdZi{>_E1_q7aX6&%Zfp=2Ri#Q2jJ|bC~-w4o+>99!OK0U0u+* zh7ulo1)W9Ml7?Rp!P1juYffZ3k-My-DMpGBw!g5cu7pUyQrvjHC|yva!^`+(Gdnd*6S zs4K=I+E0oM7*eo*Pvix7)Ee;j{Bp@<2?PVr~$ zgR#KHqW8!3N0a2aRLh z+6jK+%zp+43>Zt1U6G2e@OKme31LMmKt8F625g?82dRiAS{QPbtBcDs$%c}!DX~O3 zb2N9+)K?qpV@jw|ujVRjjl1nNmBz<5#nE`v37lHUwT{$5(H)N|`?ZzimxP6r*b=nY zH#aw}OB&Y|&L587M+Fy?F4iw2jY50iJ3kB_Q~gg4BnEO&OzTL;C*8w(4^tOGcWW)3}*}*&g^z zPWIR|_tC8)heT7E8SU&zTlp3`a;22<*{^9F{kA0MUN@H(j)|qunv_0qT=*#a)dX(8 zKJsB2UWRGRtwe;3x$1rnp`VYAf9GXaLxZB*YonR1`Ig?ZkU(Y6>$K0!KDoZW9**X# zo{o;r-IB;P!gi+DA<~5PL^0dZz;*8cS?@EtY?W6-7JrNz8nb``au79Tn&Cf-&kr^3 zwySApS?47)Q#A$#isgE#<6O1EohMU4fo-R(Sks*CZO^ua_}L4ARzax{tk z&N`2alO^9Tr0H?U)5aNAZ=|c3nRO&LZ#9^%zuC>nCNH^`L8<%pz4oPPBApN`@4>;v{B>96HPXr1yyroW{_Ua0*QN~bQ!NiQh{t4Ez$ioXJFqqoXEA7=Mqu5g z6x4Pe_+CeC6lYoaUtDyB^`!1ti?_tZgiLKW-^K8;I+!DN-Q7r?r8oUDA6BaR9DgdozFlKF zqV+wbt&I!pd5eIq_|w~cx2&TeQ^fbS+V`s3mvU!^XyF{bk>Q`CMTdd(I^$6T?Aum4 z5RKbfZ)WBO!p{hpvE!!u9F5vrGCZVj--<8Jr-t>?UCBAvFfM(TLl~FlkX7i6C;C;t z)U|NDvVJP&Ba^!Wgy{DUN5JKF+JZdNYiiQ2e zctNPlAwryX_VIb=MJgw~mRU^T%gzuX+Qp=-PO(VhSLuvD9xoIT!}B4~dnPyCcqPzg zdVP)OB+KOM`j?OvqY;5}6rPvhQ*--C=F^BH*aEwrrWl{^4Z)pzo$&&ZGsc{Auj;$( zI&yR!r6yl>?9hJ7pV(s+YdqWbv{lv1z83E)%1aO=(OA@$IbmUDb|1-83x^vyU@&4- z8z$VBLJEH|ZSbN?Is$sF~QzGK3U^%diz*5P!9R1tK!x(zC9PQ7PzNC8=#Z?02F6i9;yhpcAn! zP}_XnxTB!=$3+t3it)or?fnSdpXlW{Vc=#H#f%?)?jCbNS@bwmEC`v!qeBcaCd;0s)ijA7TnwiSMZU^}h zKO|mhz@?+j46T8jQH84~k;M($-rgSDMKv)ur&kQbFZV)C@{=@E{lK(rsxOQ%ERwYP zb(~rn&$^Ohkv1!{P2PM1x8mn0IqKna45?L*!B!-os-9uSIAbDW8gw*PSG!tRghcpq zFHQw*)c7G!Sa8UzKJUPy9w^2 zyqlYtnBd{&2A*a&TeH-2gWR1)?qJIFq`=13cCOyl-pVSpuH?Nf?J9h?3mL9&{J((E zCkGsg(}I3c@l&KXZ{A=`sZ38#pPoc9dsQ8IdU)`1cINjYwJ?@69BzLusj9AyK4 zKRY{%!;hvaU9gujok6Po2HJ^joK~s}1vO$v`#!{)HB2CBKNX310)Bx_Mu(oV3T0?x zgGx?xCt9mdJ6oL$<#~rFLo_I!DeDL8cML^tsxN@LYU0<1>}eYE$0A4ZpxssAfb(uu z1X?$+o)j{}aFK%+g#`s=VMT<5_7tSUBLl3P;=l!0GLM2B{6NzV>%9c+9~I)jqqd^Y zbXM8?o`l?R^1K>GCb3>f=P>h1y7#cSVpxfaPJfw?2 zIVd}AGebMO@a#oHQ&W?Cd3iZDlH9gx&DU2Ho-W5Lzj);sE3g7^wl`Ilwi&a+zdvd9 z?scq`8!fd&F)oT`bgp}D#~MwrrKhF!0R>J#QR9#0f@x>~uXtx){GJAaY$C)#O%KrY z|MX5kc5iSSw&M2x**o!&loYL(i}*AD3y~-kT>)Deonwv>yC2{jwgTq<$>4T~GqUZ+ zHT=g}J*df=36lUw2;b!H*l3B!#Kbfyg0iS_ox8WYi;9X$c1Hc_0W?EiLAgL|E5PYe z|C(K?9?>#6smY$aT6cg7z=VPyMrGbyxetDR|9Szuw^K^e5widvy1w52?kmfZfWS>p zPfufGV;a0-{zNAI(ybwTG55IC((%@ar>7?!87kVlI$t+@>!gzeCD3D2T+9M0=&9e5 zz$?h3Pri>qzKm8OtT zB|1D~HYaIXi7X@wMgr+M*&7_N6_(rOR{pyQyeCg;l=EAQeH9c2wzsYNEp0Of$HyhB zT5!{0#1!Y}=l2f|XvN6Z7PFwdAGH5_@IRRdY_vBfSFQhh4I-HDHvuOH4M%7i+hJj0 z#>U35zjpUG-awHI2Q@Ydc}mc3U@#2^vbz+-Aw_dkR991~{%gPidjmve5-$cvJPQUE zaWFVI7;ySpWm+#M)h`TR=(eQ^>IwV^uDj^9m3mJAkq_5G(y%;&j^biBG zy%dB0m!afMJ9YhsHv?uy3~60@)Drs%l8r{Op)dYCWetMcwN)g=FAVaU^OflVry@T6+~OMdP#T{qw!ssyFI5Vi?ogUzA*O)r8)L0v6|iW6xR ziXZg=$W_)iS*#IDO&8h-jOm}dK?Y#Or?Midtp$D+zACRP7Fxf4 z2$R6;6~NC+(X&F|^n@US3_LxrcBjhE_UE#FR@xsgUje;D5fQ-0pQRu^WB{?z+ffyT zz`ET@H$nN_@^YuwuV35SM=42HRaKSpadAncKUlmUKlLPTg$NJN#K=h7=q8!QUrVyk zf~3&XaqFc@8i(V}A#7UtipomN8|)D5X>A5g9Zn6d-y9|Cu z+Yuin^*6;fU=!p$uSJi({a3tSvcif+!m5t3-jtW(R|PoO!B@a9j9@2Y z&G9n#+!kNOZoT;U4pu7)S%#`|~F&?rOXNo^$hHpl}Ba9mn^? z&wq@KV$cdO6yoE^m-?_Ts9LIobZ%WxGBMI3+_$R~H{uPvz0-v}zl>Bk7LRVdpu$Q| zPe1-B7RQfOS1Ps{MF>|b!u>QVxNZ~3yn`R`ypf++{EMR%5Rru0bEnb)fJ;V*(o(-c zP)agjTxL!6+B!EF2F4kUUM|Nw9n|g*Gm4e~!C>C&qOH}}zl8A^l+5-6jjax$QvWqEJKaAnPoc;JUYCIn)q9QU;X~-R4}j^` zlPpuCXUVeta=g~%_GZEJKqp`r>O6|94kjhsZsI7HW1y-2)ZL>==PfT8#&NPgVE{n0 z7Le>veZaO+&G#5YDEp`(;+Dck_bQ)-b)3b!aW2x>NZ++!s#(9u&Cw--X(?ywIgifP zepUTvBP&UE{_Z81I-faI?)0W9RLbJxb@yx>Hc89+^>8RgPA44W89I{)L@oS?xid-g zY0ecD9}i@g5BuRhCCRNPv!vdy`P$s|?Kb+JS`-lqsJD?C&|FU3UFz2hRn`*=xG-&) z0uW9}u|{9PW7P8UC5s#tly^&*wCg z)x;+@t@fsqo>MXm?+V(zOPWH4)3gKmIjGXO>w{b;v`rCHW1IUQ?Nv5%5skOsBN z9&1`x--$SmBncaQMT+~29OjJoTlwyIy@>0q_4?J)r(FKC^dUMo$~w=hw8|4XBfH^5 z>F?(u$raD7tSFI6S~zPB<;S^9{x6;owx#IoHef%q6cM zdnmPK_4PZU)~*-BiA~2xTKc}$tHF2Uoo?^XH zETp)08pP(XO0pFh`2UKhvc}EJ?zBIHh1#*^5h>k@P8Z>{uVRanJ2xio;C8MDn+ML_Rr1D^^SLR^{ijA_t`HZ=&%VbhxQq27|Xv({v}-AC2zbs8gTUs zEWLXRjUDM1^JS`jAkYle8Y6i?E~zb4`g?<32$8d7OcQE~ao0HP8e49s7VoSJxu<#)`pe>Re#@~r!R%Sx3ub|$@Q-+8w zeIG+(^;0cZuXd^nlt%W}#iIHM;TH6KKCbdjld-(^4_U*@IT^McyzFJh+hzvD0E_kDv(b+P)k=~`MhQKU`&9xt!t1lC2(B~fW4LZOcuX4^ZGcGzL zmyp0Q@fd5&b0HbO;zZcOXn#A%pivFMZFFbk5L^5U7=TEj2s5StPlyvf`{X}&F~DZD z06LyJC>^JT;iAdf`nwH;kZowM1yvDS!fjNBj$1t;M z0T5%cfAx9nkK|bfn?nom0)HSmRV>O)#QqH=z>Kxi{q&k+p?gICa*g{X{;Q@Hy#Vd} zTQ(Td(`ZNmsmR`=`xWd#xj2P`eLm5Ta*Hm*PeCig1WBpf%d)<@+w@u>5b*r|EYx()RmAt4qMBLL|GAiw>FB` zo~HSU$_8c{0C&eWonQWm6A+M?2C|g`@Ex;#-#v@^>qpLFz-;Dbk@<%IcQ(9Cz@pH? z8rE?i_Cx|kyZjzt;sTqnH1q4cET&Z27L!hTF=BFdCvgM$Q-KPsBgSA{+@lw5p+H)V z(V-1*)x?CU_V*~_aho&96q=j<4Y)_K3_PF#3> z0=Bp4ha{yA(S~ZHLX*$82k!hW{$njIls~&@GZo9}zxm#*o}H|MvI%ofxf#!=l{`0g z%z?5@XCp>?r~wmDyBbty$7q@Wv>fmH(D=r?ip4K;*Dbs5QQ_gdJ$MTJ#e6`UdLLRX zlItGTzDsSrfG|HHxyHGMAs3JLPuyNI*N`}NHmi267(Ll`akQfpwsnC56EzhL<9Q45 zaC=0RsvW6sPb=l1Y*dP%Z?fF?>zX`$9gIV7L0M4zpayOZYr=0(Ni^z;)hwFg)Zutc102x^vPA&mG+pdW*E zHRg$J_tRc%JBHEK?2_0?wME_ctd^PsA@(Ba>`8LvN^irP{f@U*1oO4X;STHe%-AZK znR=;7ZJjzR1lp0787RF_0=0_*hyy}cpeHoWpbcav^B;^@xx zzJ;oxym)@{#=_nvkY8>@TlzEK7VqpEgr;)m@Gd`3%fP-U_63?%UK*kr-<|QBcPbm2 z8`FJ9Wfy?>U)P6GpRLeY{-#)fD?LVMpqH55vN%({=k)+x%WC5SLt>r@#ULGQhSy;i zMEdr{W@)~5%s$se@acwoaAF1k4`2jcotHiG)#b6F7&AK>@Rs73xx;o^*L!BC-Qsfb z!xS4^rHhw9`mxinRTEwPu=g|mCA&r`J3j2C{R%iE%cE1Wm0%L>w5=rh-ZV}(>(U6@ppu?&QE@WvT<+YhYiu>FEv|D#4N=>@5BXjZ(J>4oP9E>+b zm&hk-?hktW<)O^{C^h(O!S^o95=0S$kB`5Y(wG$(2zML!`mxJU?~{*@>0U4`c z(OL?w(TZ24RWn9sd=V&^Ww}V+b2J9wXTRLim_gM^s(#6c^>RnuAsTEEIY@(jEHi8( zLm%84QQ5gPGSpeQJcv6~!@(;34QD~*K^{NCvsuJgR&38C2e^PQCsYZN*z-P#E%x-e z8y=3$H$t5}s2~vNJ((cil^Uxw$Ya&3YEzTiueXJ@OC0(oOt`4tC!2?DI`CNF1EfM7 z1SnYecf2+G`}>X&Ka8aRqxe41R||C@XAqwD+r1(&eH0z*+R^+h@rh%7{j>CV6$(XP z*vX-Ty*&~_!evq+b2vZ&AjVT%B8-sK(9n>9=Z(m*3{)LFOSpj}uSTXDu|{!DRT?zG z;csgCExF$7t!?xA_D5-QAbXxhHud2wJ05sl_o-Fbr+*~g8bC<~G+y7z9O$=i-{!!k zNnSpuxcwsq6ch2-ojW%Bv)_6D|1@6prO@67&@f5>1Yc(tvDHY3xv!EB4h@N#C|Fus z_hiRDaf)r>26Qa+V0cSQvj}WIjaTk#V;cIl_V%>4lMhN38=1^v7MY3$GsG-;d7&fz z)()f~cP68thP)I59|e?^u?5&%BdqzPJ1m&Nq>+~Ii=c|YqD?-bOW;s+Ky zhz!9!t$O2xK%OE@fSNf)I;#%D0`Pjt$nNg0`13e^+~O}0of#xIfF?46Tm(xV$bYF? zbBRnxSr)iSTCQu0&1JK%^Shos2n)%|-{eNI{^C9HtU6eeMlA^J8W;HN{2O*X1d0}6 z;O)DQ3e@T=|5Jy+0Ivhsk<2SfJ_wNx$a+^^P6 zBf2w+@HH&K&U2cqB9lcj9^JbCpoE{W6;PNDrHKzI|57$^Y!x`bAu>DESBeAF>Ir3O zc{vAw$bo8yHV@Vf0T=_D3zE6Ben2Mv3jx#vh-4Q0Fxt_-2f=}WH2ksO;#?NlZ>-8# z@KGLPSD+cga}9b4DZ{m-Khas-Beip{&U>s8ukc&I0v!E?x%tNOG6ncHf{4ec zY_hw%Tj(qdS?&#>Q(-8U9VwT^78cTPs~3F!J=t?$o6qS8vng1BHEBn*0Q!59 z*x1m}(8!2bZc))=L?R-hD)~5MIb__KRV!y{^2@6$8Qj>e)oiIKj6d1O53n}BsOR?@ zY=Hm_95Dr(!9=DC3^HyC4ntK}LYXu?0)h-yC+w3aPa=u={1xvTngLsY^f!6o+z$hy z#0-(l784WOy;La}6m(igMai-hc@COWmqwz9LF|IU0)_{B(2y;NXPp8QAI+#WgM<&T z5mIK&J+S!6zrqDWfqMm&O!NqsrG5K0F_B8Pe6#D(l|c+gpW_rVG!VMDG4m zjJZckyk9=L|3+i?&{BMn6*yd8*~=85$380o(W2Vm$ABKsL1T5q{()-$gk4aev78%QpjsIdex67Yl+veVoOdv)~6NRPIs!37P zYuK99m0ec^G2*PcsE{vzB_vKkXl9_b1EO`WH&$yW1Nqenb8>Eugor5YhY^CFde~2Qy#DNVI^@2=mS}e5Ci-xLXeH^=Ad&~M9lQ=Hu+$!ur~^!?R0QlX9@4QQBqRh_ zjB3aM0s_Lj-NKclwqSH~Gc&}sC}O_n9GV!f>RoJKzh+C8#hMLfC z@;uv@larg$`r`k)tLeUYdEPJVV?*_OqaHFNuu(W*EF=d+5DiEe1R`Yzj1^b{F4lA& zNh{sV?Admz(c93Gg@jI6urI^4F zv;3!EU||F!mkAUP8%Kr#!%oSBqJ0!29pHVrw6p|(vf}TXh;JE1UN4L5=*(=5z4C&Dd`XxVK8=WsN=BLrt5ZW=9b-5UgME1GRC<8C z7IuKUrCi-wIsdlk!SApYoPeb|0tQc)nAQH*xbETFTP%wLlPpn!9@~4IbkFA?<^$<5 z+&--8qz7(yyoUYL#3-KiU&;SIlH-Vh_<=|Lll&h@%EPN70xTLcLBZR~o(CNEXM77w zVEoqVyFV4E=-+zu6IG>rN?3})S%~-=bH(_ZB@V-)7sZ1A9wV@C+Wvk|0O|~YJ|O9I zr^jagUhVJ48p7bpz%%T|{m$BurQTnoka`FqXdH)P&;+)E_|KZbH22av<)3;ekaZr~ z!XQ}(=o{q&=lSdv%r#=Shn7D1$A}bH8{hwLX5l0Pe*vVACj}=D&%+12(1?@W!+B)% z^I@YGo3h%K+Uq*;Y~W8x&$(6jTPwU9v7sj;aNS#!_1p4JUG z0-FR8!PT3twCjPuv4`lTe5^6%?O6R&O6uJ>9bVsp7H4c2-qe&7hRjRZ#CN9=78lE1 zQ(Wdi`>$n=Zts(ML3>$oL2B1hOp^@hb0c4d>^Q3+v2JVPrcm2&u0Tl7&%a`NR)A=TEPghr-NwsW`gnUD#iDJDl z$?RWl*XgeFhK*_&iV&vU4fMxm>G~4_t+qh*2J3vr_sZVabzDQL!`0cv#fN)mXR^$g zo4a`T7iAXq3u5*ZnvSamhdtPqT$=xn3!Zj^R zTZCq<0aR?EkO{Ov=-PW0wV8Rza(Bq$+u?iD_eJ1(j)a7yr)MiL+Z}+E#rikf8r4-* z+}zv+K);b4D<@~b4HyjlaYLK}{N4Vem^)Aq0swM8^5|alyLU)$Y((X8B4Om(ji9-7 z3MfS*(Xfg~=~XR^7 zr8XAtt(Ol~LkC3I&?Mz95N34FJ+NEDJYRf^j=27O&G$~ExcT^~Sz1vi^>DnB2rO}Y z;vyh_wh@^xft@cKy)ZYk@uev7`WwuIP51ixWy(sgf+WSd?9#1N(@+cwhxdBOH?P)~ zO}Dqm3VZanK5rJbhb(-= zIsw22c0s}3LYOI-l+lfi4J|UFp8!%1*j1Hg-Jh!u_8cuTEgl8>CrTyZxR|oUK0L<` z+%>u(zw8jJx=44dpXlSu;tEiNI=yut@z z4On1hM2m35c+nQg9h$n+WxI|9e9>vEenIm5Xf~RJ|0AZGpSq6&w1mx9?m!O0gn=BO_A)6y!zb)@|9Gfrn#|kZec+oBXjbgO54$Z-}5)nJKl@(Xkfo9m%A>Yk+QnqS%5O1W)>Y$-019z_-gE?*DZ4@ zaKKH#NB+Z@Yu!L2*Gg{&RjHgeaf?e*6jV(IoXxMh#oF9jBqtAycS+SK$?@bJqb_Nl zB(>~3OHUbVxW!apnXyUhAANFkKLoTS5otn$bO072*B`>}*nGy8QHc&f7Wie;HuB@W zZ)sR`iV9pmM#xI0&*c`-8Y(qt-f1|m3<=GKH++Z@U!#+h8I1EMtW-5Q|l*Ij=# z0TS$0lb}SGl!swwfS$;lqMAy^^*PaAXTfe4A#l#;EY_ zj7zh*J$H?sA_7L4YD=l}yhrji{Pfhuoru?k5iL$A*m@85Fl; zUP0Hx*^SoFB-tNcxT*I|tSxOj+=7T)bits6cA(E4OEdXFjKu(RLbsY4y+Ov!u{ol} zLTY^tV|qFvGxjBb=6S0H5Ntm5Cn$ZN%E92?->`wMWll2X)tIU1-c<48l7&m{v}8x- z!LR)K)t?A8NAVk}2YtcNOVP-UZ}ivCtO-m*?W4)*oFWYYs-gkF`z$m7Qy?zHN0FLb zuLnC+Pg6uoEIyPK%DCW8KQ>L`klTvDPkSy&sc0sxiNC<#2kPEC0X?1uxo$^@?$>M&1-xwhgfDQrnJu4jz z`sH~0iaM}|7{B`@1=F#@K(Bx%{=<)!$7mP&4A70Ri7%cRr?HwX7C2@@iXjDu927^t zF~=g-I)d^!CbXXiNYt za*dv6kt71@DHErz;q69c!$G7Ls>~bw(ZQsfs9 z@LfdkX$hK6g9kDz4HnXZSykE~DHrkv{E)>PXJ_25m(FO6(OE&?FEJ|OoSuy;njyXyE50ljyxJ|lxw)ZZ<$~u!pPHRTm0z`{ z_>O1=VNt<=z};dPGtuy{-p8OU|7QRZtE#L_7xqqBOrjHf8f9;5`}zed&?hmM8RDTR zKkKzOUHRjO#FZ3$ND;sikVce_f=zY+G+>bYJdoT%*y}tx8;KL(3(FJsvo)^=Ym=(; z^Mm7qkg0iRGuLj;7DNDYh*sVUz;uzbS*L@>vsdzR*OvD7=2<`=TZQNB6;T#ev6V}*<JCGpc^U;JI+TV?en!j*))j;(|qiJu;Gm5vXEA zP1RxPOUBQMv_B*+J;%a{sr)5!A;%x>`LZMTiP#C&dMjDEq5~zTATJ8`2ZON!ym4_+ z$OB3NftZ)+Fh%1}T@E6(c)*&}{y!^a7Lf1wwVoY65QWF0i$Bx69=LQ{sy+t~1MrQF z2#<|b*D=IR>KY1*z%mN_FwS8}2PuwwN$x{AOHl!KHSEDqs3Gb!ti;ss*Z_DL5?GwV z@DBFtO|GOwe{E?gE1(3Gm#^$leuL2nEjrgz#b8HqQqm|yfpPo@>0(iRT!i|ULaBm5 z0M%Pw{&@E}^qRosUfwY(OJbqVYsHX#%`Bh%T2Jk{_2zH!KrWQWM!h3pl_9@!Mxo9r!nD`k(AEro`igp4FJE1Qh6IcE0!-7lm* zpYP|7-{ax&etT4$b6)p-KgadFuIv88Q&E*pqm2m;NT;BB%t~tnC)y?WR9-%NO#I=2 z$XHKLX>qZ5=pWkB+Tq@w&^jPY0hrUs|1TfPoN1sR>htG6v4ZB8$Eu&a zR8xp0y=pG#wrmN_epF{WpZz$C%Bc9w)DyKzR<^bg?cTJ0K@YeBGr!l@qowRM3$hih z1I#F0z$vKsK0LEC}_oV4Smp`CJ9 z=KyT>29Z{Z7DD6&3Tk8`J*s>jOzD=_26OM*94sC-5ER?JEEg^H*QL%j5UYGZz*vpa zqiP>)CKQT`IF~PjF`M(WA`0l8i3^TeP_9w0tgKkpW?;9giqEA8IZw&uX({2K0Mf~u z9oG8LsyH&kcwS>^CIyVvs#F>}ZMWq!3Q`!E6!74WV7p>=s~z1ujPo`W`UY_y zY>b4){5!Ju|H(?!Xi8Gx84->u-Z;U}SZ^cxr81LYGP1IU-b-cH9n>S>x9f&bb=X{F)*64pQ2oy}ayUFRW?@yL7{ zh>wp)TGl?>!0`i~9;2KJLQ?o0j`~)Pvai4AIBHNiX!!Ai z0{fvlzzJ7VZKN%llt1u>03s{Et7tWhq+)UKtnZ${Hp#bcB+EgIj2j;?Q-BrN>9!@`}nwke#6tndCglBC?~ z#`pAICm9x*=40eCkH5SA2q6ePe@PY7)4U8`MuExbR62pTJDp_6{vFR)Xn&DttCx%m zFwN5>LK!9ah5>jM1DJ}Svo*VIdp!y*+5NVt75B**ru|G^P5Se(c`{CxYSwc*5}Hsl zG6@bhp4W7r>j7ux)%3J^a&NFkri?gY+r|sU(gWIyw5re=Ne9K9qX-BFxXA8Q`~@ZS zXX(`Y!(}#r&^mS5_B~-q;k$P~Tx#>Wq-KA9c?d?nbI7Ho(Q0pgUGAG-z5ZYU1kp|4 z)Q~oyVr<%`fYzyD(e-9O$#v5V8X{JnSWA@<5ZmH4Lx$jsPT%)de?Fvw+J^Dpx5k6f zLVtc{aUK>|0uTgbUw4UmzzAE3<|NMnUV_X7$jKZ>JavX6jg2zV`{oQhRh5;)KRCdz z^>B4n&85zfhs&#uoK88InGB8_`D=bABgMH5jUVwoQex%l#(6Gn(gt8LdD7;aHH%d> z7u%RM@&?C{Z_DCuBuSmVD(U)vq3WXVFP9U$Dy6w8WkxXN77X+ycg&Y%TWqJrB-bUI zSiWqzCRViqK3o2$RbpPgnVS-fh~}f+PXBZv1_9PyGpw!Wp%S4Ms)CvjEa&YloQn#C zNlAd^Tf0?O7<*OZaHEH&tcxgXEH1^1{I-l%eccHP8?s+ObAE~+N*aM!x0;;4Ek`dw zK@}+jHo@Rzu4>l32wecjTgbP9%!r`(AJMu8A%nD}SwIimpa72YuE?q&HP|4xOH#+|dCH!nDqF`1?m>o5oxeJ!v!?Tl_9B>C6r&gO*;U^#BLv?^otsC^aU%{|jB8{KzO&dY6~WM1j!{&oxxf z)3OQh^r1XpI?Cb6cDBE0C-AkkwdYO-A_i46Vz@j2Q0=IzLlLF2N=T5-9EN^3gtvIBQLLEWd$%+D4Or+uTmnm3RA`rxP=I!n2y2K-E_Tl>C&yUve<}ZpY+F% zAM|TRMA+J^y!a+CUTkNtZgHv$K9q(*jC(vP8KOX_(iCHT8XOMKwHv-48{k`f{852`Ek^FxD!nG*1M zZ933t9zEx_1bUjAhf5>n8NitnDC{_L8Sz0HuPRf-Evl)Bnm*0!)bIOC^i2$a5H=<><{L1?b4YB507ma3&4UKXZhXJ;qJX|lNWSPB%xk3o8qbSytZl=Z@0@&}TLRy?k4^=%Jnj3S!n7ctMP9mHFSzSgl>UF+c@czk&-+Q#{6TKFfTVQL@Y zYiH7zf1%!Me1VmzhAPVXV^0avzT3!#PbiQ#Vv1coh6e1OXxpgJq>c80go_sokMD10 zx-3q%wnZ^B9Z(9l7nF=Z5Iw;E{)>npFiT0Q)dm;2$2XIa?oNb z;Kxv&|274PjLZ^Oa$aAldZLv8qOj^6aLnI4)@nRYcF?ZK^n>^6a;et7oC1@jQ-~aa z4jn|mN7h>tBzdAnb@l4V=;*8009XW@m}?^YWFv3Ec(S1%(0qcDj}u@2 z*?J0d*TxNv%kDQDsP^kGa_s2y?d67xV#>?i)zVsnQAy~F@4Nntdkks$ftRXd|Mm0e zH%%~yMEU@i+rbqk?ONY`K1jFfU}Lj5R_(2!p#hd0e##t>v^|_x@lR$PC?EIR2Y3_g z7l8TJS65fT+J3om<;rWqhHTS|aP<4=IaXsIoq`MgFbaUUz_(51)~04&Thv!nR)O#} z(4NZBz$<0RIRykBF*UP%F33Fma;v6}F!3p^YCBLAiTfQGRotXZ8h9vk2BciJ^HTaz z1o1PG`E5Tx8VEZhUPsN%=Vma5G4{_Yq#32{RMpmI2&B;e1fyL5N{6iMGD6L>J`NLu zfbSlUTWt(ys({mr{QStD&!itx9;T;X+gi_$auBR)i!!Gc!pRDvlrd(SyGDj4p(~-k z?`1Db?9$hxjq^%j{zMm==tXKK3)4vtKskx&oA5!^eQ<(lCUTUG3)AY76$3~Qn`7xh z^^HdSfLbFPJuLnVYJ_}mfp4E8q?O_B&{>xRW1R`rA{+Jd13~}?yOeLZf-8MBc`%tV zG|~s#vmWFbeLcc(n3ghnp0@9l?QyB3!#Iaj4+qxJdJ`_@%_pL6x_K9FoC*}`MN2ua zl=i%g3SmbYT>@!fMndk^xjq4WARPi`KC--5DyJjI)9A~pY~!)%QLalsofCIQ5XgxI1Q}cuBw4y?e^(;$A$|DS}&;p`_k^i(m4Z~(Qf(^u*E(H%= zn?K-VKO-j}-%~}#3e1~>hJxfzz6MEva4wFOu$U7EMv&I$wi1XNaJ;h8Di!0D4A0!$ z+}D5)AT>BLA`B)cW=2LvmX=Jha@1MjXl@~TEH$T69}^Oc6w*vZ4xyp`XJ|@9%qunf z?4B6ecsfd9KoJ;-3AsT8+CSAJMs31JhgRIm4z1w_KxcLJ^xj`RXqaYWW7~!**3ZAe z00m4STtTn`&F?AT1{5a?8aPO^x!V5Pkea|WO(j|n%d^A1y-~RbdIDTE!Fq_Z_*7e z079J+_qAgUgst5d0c;ue@ZnG}m_|yCh3qu^vYi4;_iZag5+z}1-f|*W?C}0uJR*Xw zo>1NpBBDn-WI-Wbc_9>;Ni?a~jIye$(`K?jja2pt;{@%!K}8DOKFDCbr-%%5AOcLk zN75zqIvi`|LC4jBI=I8s09C)S*@`rG5K%EUs?E%mst7rR3N9k8UR$A=ycJ;7SxJmc z1fV$CI+T!mC+ZBkaV(H^$ba#*$e*13z%Zsv3?3@f{kIPV$R2F=X|>!+b_mo_V*}Br z3)|zK&6&swAVMOj$E1PifvvhpUon}wn3vhetP+G7#QrmL)`NmHB=CRp7AQcpo(lM0 z^rFkA37B8-2?^;0_Sv#?a5&5qmsWgw{wbwED>t>bxd>*#Lka+9-WUc`0S+KxZZnqz`Ki#Feu~WFEpTtlu}L?x$n)L z97yxJ0`d_74rHKM+!m%kJ`GpoK>-<<6HAbch*#zLv)hdEE`&Cv<+gcCuN*uv0XHG;8E*sT8m-;$%@s9o2L}!E&Qic+ zjnc%tnh|gpGzQdYo>)bP$0@bDDQWi%Iea#MT|TJ0%sIz#_^P&{@9E0%(Mm>gvNL~o zb=xt)Ax5SB*yDC!{h{yVf?B^=Gr73^z*^(tY-&jO@XgW?%Ep|&s^9Ym zLPCnw?_E1~q_f|xhjRFQ)^$mz5=&WJ>~J_xCK>LaI{HcF2N06Bb(vsrDAqw;7?Tm{ zb-HtO0_#u*cD{1>t*I9j8t#`F(k&mp&e&r?-r5>8)$5*0z7r)|c)cC4r@b7SJyIug z+4VlI_NNKkyT`nWGMcJa1g0bhN|rPA;^>=7ztb}f+}G`11QsJr+-f+gzI?tz@R;pN zUa!j2kB8uS>Pkw*s&vmx6%Yv3l<`^~ zkiGCq-cJJho@bF#v`m(1;d6qs3v=?n*nPC6L;gEH zhob}}Nu52CKp}eSBrb5cny0d!p4He1N&yKFn$pR~&BtdU>Tz2Z5YTlOMLj}?zkgTf za^vjcKO%`UbDq~L@BVdRFQFzvug?9D+CmSbhX+%p;F*(EG->G+LkmMg&cL1QFHVoU zoyyeNF&^%_=kJp#;iQnY^Rv0leup~F^JCpgpZOE25P;VZUtv`8r6Xt-%)g^B1z9yw zLS6bperfhof@XqTX7@vk?FkK=n_Sze)C^rJPz24R0J$#Fl;!q$0S&(4rO$i~M0aM? z-&X$@lLu>xI+Ka%0#Ue5N`OHSoAdF1!Si=7!=}y~JO~nHfWi>H(wR&hyyU%8!Frw$ zc=0>3@8D<4dC~-J(PbWefeJjwk2ZJz6nH1SlYlEYX7%M0Yo-?tbytipT3?aQ3{N=I zPluDdeAW91s;vV1c_Jkd9G{6Rw15~TLYn+O)SW&4G@yPOgP4N@!p__T7{KehdFoCj z{OOYbp23dlIrgua*x7@LkvDR6-6$I{XiIp_fOY{G1x%n6y-bb{m^y!Q4<(H3?3O?< zkCl^R`JA=t5G0@YmpYr71LJ|1e){-3Q}HURvMi{0nhKEL0L+T~jJf*)Rx|5eGuxl9 zE0(^=%d^2hlzb`ifruX%y2YrrqrpTEuN-Lv8n{iN&6C?gd zViZ!G{rjDdzm)971`n?EXCUdjRqzjB9cxAlu0S~ME4 zgc69}lfmax+RU#I4T}%;#R0_c=b3=;Sw9BAtuitSu-S?U4Goo}ujd_<=GJ`_#IenW zby(lYk{snpTt!+<$?ezvNE-$kWXkTw6WxXqR%W_DI zhKJQkr1zCYB-oKy=*Ry*HfDeqCK7{R#U{qX)6gh(LSjdDX`GYT0Z@HkhlbdF&DY$Y zed;c%xtCp*eXw70ghbQ8W%6=~1hp?%qElBx%b9@;em_~RDB{?RE}}$c?v+XVFEJ(T zWadnnG#UYj^fGNUihF&1_=FTkyW(d0rp4y({s*e7Q-ByC$G`}}V`Xg94W11#HHRnH zK622L82;nz5C$F=vrAZAVzG8+$%(;3HAEk-A=hW#X_OSreC|T@rM$enGyT>R@r8a= z;iuXeYM_dCRm?IHbU2Khs5>!BvF>9w@DE3u}hQ#+37vcE$kz#Y$PKW6Y;R zt^Y#p+q%5dc@mnI6De=lj_G~cmrno^kD%Oi>b+^~?4VXGG^j0jqBuYfL&=0mR{bt9stE6C*$ zHtQ_44{a;bfh^2-BQ3g~=cao-*1%O}0_u6CBtuL1-CF4ap!A=Tv3Y)eqG`b#3%2~_ z%NMCtWb*zOtmIh7b`LIutQl$Pjr+n?ecSEdr82sm zKNG<6<(A?~hD@ehdj}CX&TK7NZ5Xb0?6dY;!BKU;f4nvru& zlTOKZtr)Oadb@M$`?L^b;Wwm?pG{k*T{$RK*@8c%Xr=6X`S&SDW2zMUxYS(#!dDMEN zp+D?t@t=VUI4X?sdY5?=TK~R87Zz+P8;ge^(2;9m-wzny5ClVlY?))^w2KwO9uZe` z<~>BYU{Xqz21e{?haTg;9-wFS@ktHTGJD{IT zZ78d#^xV#>cb+%Dm=SpZIFaN^*aXM7s9m!xp7a@C$8PR6!+Poq2g8~fFdzyJWcV^< zzcn8C>$21?Wc?vS8rPX4#)8X@$2(r0x}pMCZupe12cDVj>X&I?(9*C*fq|D34@Vh@ z9-hDC=0qM)luEz_6gLJ@ypdL|ELq_}qVIKXrQ*b7#J-As;IxwT^l6=J5C+i>?UV&xCIGHA#34|Mqg3;U z5&XypzZl$ z)2()ZwM*)mKGGXe;`WFDP7bFWnk>(L2QmuIaAmblTB??COo6W*fuzY(`SA7b@)~2R zi&(EGnARUB0Xrt{IMkFDCp5&*_o0d?W>UvLQ0R%G?#ILws4)#;yLI3G9)|Clr+bkH z|9b;DK_hKeJu!{GrtOvX7;$9pM=PPfC)mmht)U4x8QP+bS9>Q_|L*cSG$2yV4iC!j zF)L4J{HlQLTcCdFh=x#=wd%Mg){YF~az_Fv3p?Txe7j6uL|#)q)pLJq3GA@Yuvr@m zl?0X2@M$tI_b$}94HaL3wrFuDr^SXq08)v70lxqKQu&Zn?zQV7P_*1o2l^4GUohcC zK%-1gOEWJV)v^$~M!7ZmdPl;Nq&)%q7%jE zVw5jWJSV+s!jY~w+`#2a>gBpm&Jiqz%*1^dGKCB1U%(j~N?}U2K+ZxDM7P*$2 zB!LlLSPKYp-=q7| zIAde`CGkoV`c^W3;E$E-`ub8kGYR;c#HjvU7%`*Xyvq&}%I>nw(s>-pmkS6OJaWFq z*UMp8V;ekq7na2R)sc>#i183~+RoEi9BRe}{vMoYM3UBYS~J zemR`;F3j|%%%h_rE=#Jk`bN*2k`h7@wAn^Yx9Alecy76qC(p-CxbZinD zE7#C!hg9xKCzU!Au3)~9&ysLZHqSBY?ds|RBO`a!oRI5-KeaO@3PE%Q1h6h^(2MHC z)g6n7{)k)be<2MNJ}={ev6V8~1|Xt22bghHoap~XK{#IxXG-5$UO8HRc$uo>+qW*8 z>Hiyb;d8G`lj~b+(C!5u>)sD0$pZs;qf}6PNNS!I%izDs@d)cX#{+#CDt9(Z$*_uM z+d2x-Duim2=FL!K!AMYPRVm^~e|%FPQp_x0A+8h>5&~{=d4m$2r{Wdn*7*Ew`{4_To)z^I{>Rx%Wh$b3df}a8AlqV zej6U|4ZOZk6=Z4%a!+lW%rQ@Q6ZJ?#Hq+zfKUlz#u&ji>;L_qDwfR{8_b^L(bSC9g zqqpfj0UZ9jZJ=%eHY5Yhm(l?**S5zD>olA?3Wtw$XSMKFg(Lu?UR=H@LqFOcg|H(` z8~~3HaCp93A>&%)Li#Q5EigRMu5y~hJfR!KlI|Dr;1BbA;mv@EsgcDl42*m<=8?$v zD-^~=1ho9N7PT()y(fN3vPDghozm-@fe~f_BfAZRDn7qc#_8FpFYNS}*L+3)ywtMQ zl4I{0d$5)eBM~E_TD%@pL%i3TWiW!HLmf>|I6W8$ICg!435%zgn-{YN!2PXo(R4yj znl{kp3IG<-+O_=(m^bW~loQNN$_6>D>u|X3Wacqcj3f1sMziP(Gcx8HUak4b`|`=a z3Z$(bG$xYKv{t01BQvsh=8AG_#Nh_8^UN8l#adTZ{i~ zZR_-tSh+h)T!i5AxQi`2e81#yJ;UNgnqS`!@%+K3|Mi}Gd({>0oJ{V0){hn?92R=l zEEToV8O%mr=Qq?G*BihahDMnr=$NS}C~ySTGud8-Q-+y_;6)+2JMmO6x?zknNhr#h z$x%E2u@~^K6Yf~cQ>WteIxp{~E`kOgSO-0X=sIH)i^}*MqkolM2q2DD+WPphdCK;W#jtuO-E~r#IHTiXv%# zDpIwBT-2b^X7ZU`$iUfQ6Mt6NHc(8AR29$AOro*9u)jh9cgvc&QrR`Djba|QvmTb0 zuOVy!GFhC4q`}B>D9!m8OZ2OLHS8w*~l49qeie$5%Z9%aAd?{-`zl|{5 z^Qxk3f;}p5Dt+?FXKe~eey?7>bj0?<)6pect*^ZiX08ceFkBBS$!vvuOUY#P=G%YE zu$V-AcW);{y=#^8_~wH(+6(ZU#b0;g2+B=<6ZhOtX1Q83l3l!?9w=f_ZD3mGb>F(5 zko(uKk_vK_Ete5)qJy8w&Z8wcku2r+msVEHtPx(s{hw~yDkUeES-1itgJp_FOApNQ zU;Pn3s>SB+Ns6zNUEFBb(>t>9Un+bacwsl{1T}2Cwl>u1$H!Lb`3pb;8OH~+`C?nG zkXhE05udfCY5IwqoKZPAW(oIo1hp1iohqkmpM)^F(YxZennI%}|LX-|m2;a*Jx@e2 zF5;+x!_=>Y>9?unzxmK_?%`P5-d_3f*7(t3?eTWSrHsAa+>gWSbYkmtVBRT`!%EvV z414bdf;~2#w)YN(f!+*L_%Ftl&L8)L2C6+b50J@%tRJ~29aTAag|AF z{DA&*;feV$CGs{>;vJ_WC2&yqXu$rjgF-6>{G(UssQVo*_8|ko(b=cp^y^nV2kfHl zM45aud_#L-p4CeopA1=i)dN@c?6oj=C(x20=i+j~mdK;S@nhe~P1M@saXpBZcCWkL z8(1ca?PZHC>}5Z}GQ+bPHABG}->)9`78w|rZcEy2lG}*xfba-nrGoRV#ex>n7xH#X zNdmYNN12wOyU;owXVZ#GO&iodP^GVm<<@mH6+$)NwBkuBKP1lO5ilhoBefq4v#*}zn zVApM2LLpG?!_@Eo!Ea@Xbe*YfWv0^hyuTqo1_!$>GqQ}?X5z0?i5N^^UqI$&!4iGC z@rc_@=%Eg~kV}-p>cQ_$F~B4XmSV({S8q2EI*Ve|NB zLt)h{;@x8CGF**QsW&C*-WrEJnE8~+e`DyNxkjm3u7o7cxTRrW+hMd)%xU2V_oS~4 ze`nLoeq+XbN0(EO&E?II6pKd1LOGt)@QRn-F9O|lXrkj)Hn%)~zak=?P81mw4WhT9;hV{m$nK47K>ogkwQh^A@hC9E?I-qK8u zV!H&e04n_ru*&M^G#5RxqlE>cF@WV zIilUQO++12(9_elX#c&h@Yf@6Y4QE_px_I1!wUOJatFvkXj^##=BY>y$ z(4li2xs^f~tgZ-DUYd^(-4QRQNRfcb4p#~k71YCwA1(dUV$$;VP6-?hhY3JQ zh$N6H?N-+7SQb@kJ=N@sa|q$cnq1t>eC;E=)?Y(e&pXb%wLfoSK#@)G9x2?inw#O5 zJV(V;q0W&RUIM1mgsA!d)Q}3)F@RcYS8At%qr`}i^POsrjggMR0`pu|M7z6_31Z5w z5T{(CptoDRCmDOGflijGYU--F3cY}eQQI+h#;#(QbC-NreEcWC_0fB+v3}n|4UEMD#zX$_CW~Vh#Z&7bq$UdSg{{yKo8u1F?X|uydGc1|Wmhs4;Vg)DOnsW|Ka_ z5XQg;AfK@k(chvTD!DTg;b!#c`8X0bR{|?(sAbQ}aOiJj04&vu^*-lpO#}5{^d}wb z7a%R1;GDNvo}oUw>!Q7wB>%-zFbwXEb1$3S0~m5yt`){UaB!)*9zpzB`(M1aof(cSs7(mLLC_G54UB6hn zdNG=KJ9GvTd!|YirLNDVTU8zq$EQDO7tUm-2aY)z4Iff62Ea$MucHVp7v9 zIC`5&u99(s#+Pksq3!16N?Vl#Y(vA|NyI2;9iv&XWwi+_HDxx^pkXKJ=l7gR9iB9M z_Q~>hIpf0QRV%~t`dulh9^c|djh9&?@f|SBm-2@ZC0|Gr`q!=&pq3)u9s^j+>B^L~ zs*68uL72>42_8y>aQ z9xc||qd(uP{b0|v+T?%G1j&0VV!smc_|I7Sgkl!*KEzi)CF=%3!EaQHyA-9OWL4p{g%dUJ&<(KR2Xdnn)R`4KXyGBt0uV`^4_ z{$T3c`1)Ayl)}>v!T-41AxW~sA;&A;A@VBsyhEym!9xZYPQ-grL5B!>Fp&QpZ~rao z4S%S=J)T~y!%)fTo+NS4W4e-r%l6Lej#$Q#T~f8*008M3%G7qLkV#1hugH?GLZ0B;J#mHMfkoK{ z7PUP~T;Co+$U`cXcks}Wu4OM>fH&w}Re9$<-Emj`$>#$j-*f3yODI?Aq2h{Lj4Yhg z1OAyi?Pm4e;(eDU^cji1_sQ|jjaGxPITp3SY+RB-zsWDD)I#Qs%KA^0*g20BJy4an z^Lia3Plt8@UjR)k%9yOpYbZID;krEwjIW8y&Ic+?WS7}x~2oC9zdaXt#rnK6zCHjT=_ADQF8h>?v>x}Gk2f|h^lRh6y zu04&&y2mx81L-e0-wJZKG$-Qpm8IP5z-Y$!u|Ydw9A?p4bZQ}IjJ5z_QGnQKWn%-V zOY}kEGTDhG_gI7hcR}jT zwu9UF1ktIrI0xyQ`5i8E$$T3LeOlk~pz<^Eq0!8z^swzO0<|ALSdc6W z&|JPndIU6hT*pj5eT4yi zn)`ac1aQHobA2L@K`Mi=P1ch6^!Pr#lC~`$vR%cM*`Sag3-xosK0dnPc=NY%vLuom zo!hsqQo3a(_`2j%_|ca7Xp4G`3e}AYvoJ0Ej?WFt5TrLZx3^u;U*Wxv+VshRVoJzz_VrTE8I#3hs8& z1@>(H6nIZ1rmpqgW*4m6+uH*g2q57E7Cfo|Wd^V@5=c4obm7F_Q$si*a6gU< zXdqdbf&)n=@Z4)x$Hjj4DfKO2(9PX{cZz06;>E;6g^y}2On{bJp1KS4{O=8%P3qtQ z)p7LNG+GODnevPX?C!C#sG@IRB&Qfa{^w?~lb_RqEp(98d+qaO&Ck!<&SI^FBcVh+ zm)QabwkDcZ!zPPX&T(~kY}bFk`9D7mh%4~HXjoB_N_f*G1saeu(g8_~BA{tycQgM@ z!nA%Lj$>746)53u*zKP*uDz55cY9gMmy5!i+5YSoXsgEoq8l?jr?ip$48QC-mW-ju1+0P__bM^N7{CGOnF6|`(&qCcdD(_C%IrLH(#nM1;uho>d!!`sxdTzIa{ zjX!LUM}pCQ`LP1zT5JP+5sIx5f&q2NBvBZ!=gF1%QTCPY@Ai)@?U0$LvDRt6C~RAUjm0-x6UXVHq;%}WL%#rAd-9iWIaV2YOy+5t!? zt%J!1U_(+!>;QgsR8$n8PR6+8(b%5oooMpphaG=4z$@_xx-`VEq!|?pHP3%V1Hc=2 z+h#0!WKrCo13$Jpp@u4`7VH@lb{(8G1kez10fwcz?R$Jt2;>0p@PQFM#BQ+OJxZcz zL99mMdzPZAU@0>OBq7%>cpnN`Q&1NTsesj3VXZM~8fGJ5eg+}Wgi(u&i>a7DYJ9KK zJavj->vfCUmD zSb=Q}5mK%N)YYu`f4(bQ;4w~J=-wiSiR|}w^amd=Eg(h71KSzf`*+??lAjM~W2t3w zm1#W0A3_4Cc%kGu?wFrIjJDSF0`jW^iKPLJn~Q@$vHPVeoNozH;CJQU8;zjQ@(>1^ zAYd$pOjL4zY_}I@o3XTSSTaqV1ZrdAa`xbMNk9NT5Bw0Y4;cNf{{H?o+qJO#5+;MT z?b_bkTy;*UmKdp|uzPu@0mxAR#&c>&03jO+-brPj z`5bO#ceXELqU&hrp%xu@H`$10v?i>a+|Qxe>Gyp3{)6B6XJ%b9x=D96?Le9IM(N|z zEx?V{YnxAYV+XFNX~cl*VM|H(Fa`&*C5x9Y$d|mw_IJ<-1sre^K_isXk-i7S3c}l( zgC?@yPw353$eOAuVW23y68czQ@j6Miyf`67Zyq=OPd~M+aISnK%1JrWAiT*rYGa{s zG>J;pqEbg`s_;VjD@wgS;bL2-_j};nk9=|GKjIY!>WjO-?i3K|vJ4HZ-!M~2i^b^z zyTW%1IRtFC)M^1JKX5wk;Peo=W(aOl0YnNOl%Hib1IUa^jF%9`Ha1VjAofQ>dU`rg zl(Q1mMk&>5Pij;9KzYx)AC1_s>25O#P{J1X11~Puch8FZPmR2~{@#ot!+?07dbmtl zm8PVp|Lu*l4zq<3YMLXL(H;!WqXk4$Jqc1A&DPTsw7Pv zD_2z2o(P`vZL`}U6p=D;d%O3xgy>Tsp>Et}*pVR(>6dU%5+m`I$x@}Sa^$MRz2*IT zOJR!gJ+P45t(L0qz%Qf1mnA=LzTCkWbK9 z?oK4of1zpB>#)8=g_Xm;^|CY+A640|(rhr2)BC~Qc1>UJ2z}M8Z$U9_RvR3(bV$cm zs-0kXjj0a?3*j9=y9iB|FeNt}Ht(g3zpTgZ79T3~-|NZPvI}Fi>_`<4Is{6U6mIMV zzt7h-92q>#xK|T0g6#_;;6Dxue0}nxTc1HMD##qc!7Cl_M4OUIZdA>}q_* zJc)qbEx+&F|ycY+)w4g8!23va3u7G%5@9VxH7+rK%ju)&STprvvq_&}M^+GaK zf)VeXX?wa;rfDP)Vbf;aXTuQ$L-pCq*Bd+PiRqfbh}5~J7-A4*)iDD@9>6O!L|Z@Q ziexgt@o)&=5p)5!sFX)nFhkb$_WGXk@Rt0DUeuy+lqw|}9GUWCE(FZ(bOl%vi zs-%QW{!QFBZAl!GZ-%*B|GE66Hz;5RFOUdT8S|k0TM;pIyU|>9Oc1v5t!lZvD~1Q# z&1SW=IYqHx+BE7~@8rWOlH*k^bvz|cFG-a8dQ7T+9pbri>EE0NEzo5o9^|X@PqKZ0 z*6^OMgrSS!`wY2H8;-jGEdteePM;5lUQs$@A~g^q+bG=XHFqnDKRhDOVO9n36Pj21 zz-BhUpFw7w1P=PYFEW*yO zOkx5SXuEhI&R=_euf*Sv13ub>5$w$~9B>ay!m8Sw?RIg`i5RP-%9^2DF+z zR5U82^&-b7BQ}7iX0g-HjhN7R^DJTd^N<6^j5)1sSmis>flOQ_>{e|f_3N`&Lv?ni zzvjgSuCsi28k&2Q%an-ozW+vHm-Xh9CYG80vy*FCzt9-Jd$i(aCPn(ziQ8}Wg8ypg z<@LV278KM&5D`hPTv=+035O*A%N2&2R%jlljtQp*oJ_5NSO*HD)hg9o;Ge+E4BSs> z08#rzR2qRm0ifWIYbinVKd8zCyWwrBVr`dzGVTa7hVQW@Dys+jV$&L2l*nd=Ipk6 z^|Z3Pdt}7C|FNP7m>wEg3(DtPPJCaNoUKaiRFKM$V&u1eesVQt;%gV8fO}l1xKzy~ zMTmHtytmcWlv#$r!()RjRCd?;)b-br(}|?!a(XREQ7; zjeA#aR6CO0%Wln$v|jI9kcM=|Mcs}1NS?H#0UCb=uHgzb4~1@qsIIk@ih*W z4=G4Hk(p_x#iI%xaBM==Q&Sm`ToC4P&Ff@&?{rJfeeuuk{0k(9RBPzX$rAqr1z@}t zi#Vi5-;|RTJ;&fq$`nGtk?(5Y*Sbx3YHs=`l;CE7CIiw_9TOqe{7$b40Ib6Q9M|5!CD8mvK!S)7fa49|bw>?f6J0Rx;UunNcxjl2mZ+E}2ce$^s~ zk_+w(t7m-F*_n^4^|R34h83 z3hcliEeIZR;5NPB0UQ2Wl|$8M`b_PK!PCcZb^^|hf+{z3 zth|qp9AJ-|CVN>lSHpY)z-$&oC-9ysN=izAY_SfMv-Ene>9S1RtcI~$Ej!bvNweO* zy_nVa89?(8xjTmY{AlG%f)kMUy+oeCYjj+y;buLUMYa&W;feir9=jv|rv*#`ei24&<4uKTl7G?Yx z^S^(~45^7}gci;Kw7^ChY85EkP1M1T{83O)j*quLG53IPFrh7jjuL72Vp7qTQyu&X OioCS4RDpz1(EkHS#(G8o literal 0 HcmV?d00001 diff --git a/docs/source/user_guide/feature_guide/index.md b/docs/source/user_guide/feature_guide/index.md index 049e496f..61c333b0 100644 --- a/docs/source/user_guide/feature_guide/index.md +++ b/docs/source/user_guide/feature_guide/index.md @@ -11,5 +11,6 @@ sleep_mode structured_output lora eplb_swift_balancer +netloader dynamic_batch ::: diff --git a/docs/source/user_guide/feature_guide/netloader.md b/docs/source/user_guide/feature_guide/netloader.md new file mode 100644 index 00000000..857f3473 --- /dev/null +++ b/docs/source/user_guide/feature_guide/netloader.md @@ -0,0 +1,97 @@ +# Netloader Guide + +This guide provides instructions for using **Netloader** as a weight-loader plugin for acceleration in **vLLM Ascend**. + +--- + +## Overview + +Netloader leverages high-bandwidth peer-to-peer (P2P) transfers between NPU cards to load model weights. It is implemented as a plugin (via the `register_model_loader` API added in vLLM 0.10). The workflow is: + +1. A **server** preloads a model. +2. A new **client** instance requests weight transfer. +3. After validating that the model and partitioning match, the client uses HCCL collective communication (send/recv) to receive weights in the same order as stored in the model. + +The server runs alongside normal inference tasks via sub-threads and via `stateless_init_torch_distributed_process_group` in vLLM. The client thus takes over weight initialization without needing to load from storage. + +### Flowchart + +![netloader flowchart](./images/netloader_flowchart.png) + +### Timing Diagram + +![netloader timing diagram](./images/netloader_timing_diagram.png) + +### Application Scenarios + +- **Reduce startup latency**: By reusing already loaded weights and transferring them directly between NPU cards, Netloader cuts down model loading time vs conventional remote/local pull strategies. +- **Relieve network & storage load**: Avoid repeated downloads of weight files from remote repositories, thus reducing pressure on central storage and network traffic. +- **Improve resource utilization & lower cost**: Faster loading allows less reliance on standby compute nodes; resources can be scaled up/down more flexibly. +- **Enhance business continuity & high availability**: In failure recovery, new instances can quickly take over without long downtime, improving system reliability and user experience. + +--- + +## Usage + +To enable Netloader, pass `--load-format=netloader` and provide configuration via `--model-loader-extra-config` (as a JSON string). Below are the supported configuration fields: + +| Field Name | Type | Description | Allowed Values / Notes | +|--------------------|---------|------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------| +| **SOURCE** | List | Weighted data sources. Each item is a map with `device_id` and `sources`, specifying the rank and its endpoints (IP:port).
Example: `{"SOURCE": [{"device_id": 0, "sources": ["10.170.22.152:19374"]}, {"device_id": 1, "sources": ["10.170.22.152:11228"]}]}`
If omitted or empty, fallback to default loader. The SOURCE here is second priority. | A list of objects with keys `device_id: int` and `sources: List[str]` | +| **MODEL** | String | The model name, used to verify consistency between client and server. | Defaults to the `--model` argument if not specified. | +| **LISTEN_PORT** | Integer | Base port for the server listener. | The actual port = `LISTEN_PORT + RANK`. If omitted, a random valid port is chosen. Valid range: 1024–65535. If out of range, that server instance won’t open a listener. | +| **INT8_CACHE** | String | Behavior for handling int8 parameters in quantized models. | One of `["hbm", "dram", "no"]`.
- `hbm`: copy original int8 parameters to high-bandwidth memory (HBM) (may cost a lot of HBM).
- `dram`: copy to DRAM.
- `no`: no special handling (may lead to divergence or unpredictable behavior). Default: `"no"`. | +| **INT8_CACHE_NAME** | List | Names of parameters to which `INT8_CACHE` is applied (i.e. filtering). | Default: `None` (means no filtering—all parameters). | +| **OUTPUT_PREFIX** | String | Prefix for writing per-rank listener address/port files in server mode. | If set, each rank writes to `{OUTPUT_PREFIX}{RANK}.txt` (text), content = `IP:Port`. | +| **CONFIG_FILE** | String | Path to a JSON file specifying the above configuration. | If provided, the SOURCE inside this file has **first priority** (overrides SOURCE in other configs). | + +--- + +## Example Commands & Placeholders + +> Replace parts in `` `<...>` `` before running. + +### Server + +```shell +VLLM_SLEEP_WHEN_IDLE=1 vllm serve `` \ + --tensor-parallel-size 1 \ + --served-model-name `` \ + --enforce-eager \ + --port `` \ + --load-format netloader +``` + +### Client + +```shell +export NETLOADER_CONFIG='{"SOURCE":[{"device_id":0, "sources": ["``:``"]}]}' + +VLLM_SLEEP_WHEN_IDLE=1 ASCEND_RT_VISIBLE_DEVICES=`` \ + vllm serve `` \ + --tensor-parallel-size 1 \ + --served-model-name `` \ + --enforce-eager \ + --port `` \ + --load-format netloader \ + --model-loader-extra-config="${NETLOADER_CONFIG}" +``` + +#### Placeholder Descriptions + +- ``: Path to the model file +- ``: Model name (must match between server & client) +- ``: Base listening port on server +- `` + ``: IP and port of the Netloader server (from server log) +- ``: Client device ID (must differ from server’s) +- ``: Port on which client listens + +After startup, you can test consistency by issuing inference requests with temperature = 0 and comparing outputs. + +--- + +## Note & Caveats + +- If Netloader is used, **each worker process** must bind a listening port. That port may be user-specified or assigned randomly. If user-specified, ensure it is available. +- Netloader requires extra HBM memory to establish HCCL connections (i.e. `HCCL_BUFFERSIZE`, default ~200 MB). Users should reserve sufficient capacity (e.g. via `--gpu-memory-utilization`). +- It is recommended to set `VLLM_SLEEP_WHEN_IDLE=1` to mitigate unstable or slow connections/transmissions. Related info: [vLLM Issue #16660](https://github.com/vllm-project/vllm/issues/16660), [vLLM PR #16226](https://github.com/vllm-project/vllm/pull/16226). diff --git a/setup.py b/setup.py index 0795304b..5a823e7a 100644 --- a/setup.py +++ b/setup.py @@ -393,7 +393,8 @@ setup( "vllm.platform_plugins": ["ascend = vllm_ascend:register"], "vllm.general_plugins": [ "ascend_enhanced_model = vllm_ascend:register_model", - "ascend_kv_connector = vllm_ascend:register_connector" + "ascend_kv_connector = vllm_ascend:register_connector", + "ascend_model_loader = vllm_ascend:register_model_loader" ], }, ) diff --git a/tests/ut/model_loader/netloader/test_netloader.py b/tests/ut/model_loader/netloader/test_netloader.py new file mode 100644 index 00000000..6658823a --- /dev/null +++ b/tests/ut/model_loader/netloader/test_netloader.py @@ -0,0 +1,215 @@ +# +# Copyright (c) 2025 Huawei Technologies Co., Ltd. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import json +from unittest.mock import MagicMock, patch + +import pytest +import torch +from torch import nn + +from vllm_ascend.model_loader.netloader.netloader import ModelNetLoaderElastic + + +class DummyDeviceConfig: + device = 'cuda' + device_type = 'cuda' + + +class DummyParallelConfig: + tensor_parallel_size = 1 + pipeline_parallel_size = 1 + + +class DummyVllmConfig: + device_config = DummyDeviceConfig() + parallel_config = DummyParallelConfig() + additional_config = None + + +class DummyModelConfig: + model = 'dummy-model' + dtype = torch.float32 + + +@pytest.fixture +def default_load_config(): + + class DummyLoadConfig: + model_loader_extra_config = None + load_format = "default" + + return DummyLoadConfig() + + +def make_loader_with_config(extra): + + class DummyLoadConfig: + model_loader_extra_config = extra + load_format = "default" + + return ModelNetLoaderElastic(DummyLoadConfig()) + + +def test_init_with_extra_config_file(tmp_path, monkeypatch): + # Generate test JSON file + config_content = { + "SOURCE": [{ + "device_id": 0 + }], + "MODEL": "foo-model", + "LISTEN_PORT": 5001, + "INT8_CACHE": "hbm", + "OUTPUT_PREFIX": str(tmp_path), + } + config_file = tmp_path / "config.json" + config_file.write_text(json.dumps(config_content)) + + dummy_logger = MagicMock() + monkeypatch.setattr("vllm.logger.logger", dummy_logger) + monkeypatch.setattr( + "vllm_ascend.model_loader.netloader.utils.is_valid_path_prefix", + lambda x: True) + + extra = {"CONFIG_FILE": str(config_file)} + loader = make_loader_with_config(extra) + assert loader.model_path == "foo-model" + assert loader.source == [{"device_id": 0}] + assert loader.listen_port == 5001 + assert loader.int8_cache == "hbm" + assert loader.output_prefix == str(tmp_path) + + +def test_init_with_extra_config(monkeypatch): + dummy_logger = MagicMock() + monkeypatch.setattr("vllm.logger.logger", dummy_logger) + monkeypatch.setattr( + "vllm_ascend.model_loader.netloader.utils.is_valid_path_prefix", + lambda x: True) + + extra = { + "SOURCE": [{ + "device_id": 0 + }], + "MODEL": "foo", + "LISTEN_PORT": "4000", + "INT8_CACHE": "dram", + "OUTPUT_PREFIX": "/tmp/" + } + loader = make_loader_with_config(extra) + assert loader.model_path == "foo" + assert loader.listen_port == 4000 + assert loader.int8_cache == "dram" + assert loader.output_prefix == "/tmp/" + assert loader.source == [{"device_id": 0}] + + +def test_init_with_invalid_config(monkeypatch): + dummy_logger = MagicMock() + monkeypatch.setattr("vllm.logger.logger", dummy_logger) + monkeypatch.setattr( + "vllm_ascend.model_loader.netloader.utils.is_valid_path_prefix", + lambda x: False) + # c + extra = { + "SOURCE": None, + "MODEL": None, + "LISTEN_PORT": None, + "INT8_CACHE": "something", + "OUTPUT_PREFIX": None, + } + loader = make_loader_with_config(extra) + assert loader.model_path is None + assert loader.listen_port is None + assert loader.int8_cache == "no" + assert loader.output_prefix is None + + +@patch("vllm_ascend.model_loader.netloader.netloader.logger") +def test_load_model_elastic_success(mock_logger, monkeypatch, tmp_path): + monkeypatch.setattr("torch.distributed.get_rank", lambda: 0) + + class FakeContext: + + def __enter__(self): + pass + + def __exit__(self, a, b, c): + pass + + monkeypatch.setattr("torch.device", lambda d: FakeContext()) + # patch deep copy + monkeypatch.setattr( + "vllm_ascend.model_loader.netloader.netloader.deepcopy", lambda x: x) + # patch set_default_torch_dtype + monkeypatch.setattr( + "vllm_ascend.model_loader.netloader.netloader.set_default_torch_dtype", + lambda dtype: FakeContext()) + # patch initialize_model + dummy_model = MagicMock(spec=nn.Module) + dummy_model.eval.return_value = dummy_model + monkeypatch.setattr( + "vllm_ascend.model_loader.netloader.netloader.initialize_model", + lambda **kwargs: dummy_model) + # patch elastic_load + monkeypatch.setattr( + "vllm_ascend.model_loader.netloader.netloader.elastic_load", + lambda **kwargs: dummy_model) + # patch process_weights_after_loading + monkeypatch.setattr( + "vllm_ascend.model_loader.netloader.netloader.process_weights_after_loading", + lambda *a, **k: None) + # patch get_ip + monkeypatch.setattr("vllm.utils.get_ip", lambda: "127.0.0.1") + # patch find_free_port + monkeypatch.setattr( + "vllm_ascend.model_loader.netloader.netloader.find_free_port", + lambda: 8888) + + # patch ElasticServer + class DummyElasticServer: + + def __init__(*a, **k): + pass + + def start(self): + pass + + monkeypatch.setattr( + "vllm_ascend.model_loader.netloader.netloader.ElasticServer", + DummyElasticServer) + # write output_prefix to the temporary directory + extra = { + "SOURCE": [{ + "device_id": 0 + }], + "MODEL": "foo", + "LISTEN_PORT": 5555, + "OUTPUT_PREFIX": str(tmp_path) + "/output_", + "INT8_CACHE": "no" + } + loader = make_loader_with_config(extra) + vllm_config = DummyVllmConfig() + model_config = DummyModelConfig() + result = loader.load_model(vllm_config, model_config) + assert isinstance(result, nn.Module) + # Check file + written_file = tmp_path / "output_0.txt" + assert written_file.exists() + + +if __name__ == "__main__": + pytest.main() diff --git a/tests/ut/model_loader/netloader/test_netloader_elastic.py b/tests/ut/model_loader/netloader/test_netloader_elastic.py new file mode 100644 index 00000000..127f1dd6 --- /dev/null +++ b/tests/ut/model_loader/netloader/test_netloader_elastic.py @@ -0,0 +1,432 @@ +# +# Copyright (c) 2025 Huawei Technologies Co., Ltd. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import io +import json +import logging +import socket +from unittest.mock import MagicMock, patch + +import pytest +import torch +import vllm.logger + +from vllm_ascend.model_loader.netloader.interaction.elastic import ( + ElasticClient, ElasticServer) + + +# Simulate server's normal response +def mock_server_response(data): + return json.dumps({ + "label": "JOIN_ACK", + "content": { + "name": "mocked_name" + } + }).encode("utf-8") + + +# Simulate server's error response +def mock_server_error_response(data): + return json.dumps({"label": "JOIN_ACK", "content": None}).encode("utf-8") + + +# Simulated server's abnormal response +def mock_server_exception_response(data): + raise Exception("Mocked server exception") + + +# Test the initialization of ElasticClient +def test_elastic_client_init(): + sources = ["127.0.0.1:12345"] + device_id = 0 + model_path = "mocked_model_path" + tp = 1 + pp = 1 + + with patch('socket.socket') as mock_socket: + mock_socket_instance = MagicMock() + mock_socket.return_value = mock_socket_instance + mock_socket_instance.recv.return_value = mock_server_response(None) + + mock_socket_instance.getsockname.return_value = ('127.0.0.1', 12346) + mock_socket_instance.__enter__.return_value = mock_socket_instance + + with ElasticClient(sources, device_id, model_path, tp, pp) as client: + assert client.server_addr == "127.0.0.1" + assert client.server_port == 12345 + assert client.ack == ("mocked_name", 12346) + mock_socket_instance.close.assert_called_once() + + +# Test the register method of ElasticClient +def test_elastic_client_register(): + sources = ["127.0.0.1:12345"] + device_id = 0 + model_path = "mocked_model_path" + tp = 1 + pp = 1 + + with patch('socket.socket') as mock_socket: + mock_socket_instance = MagicMock() + mock_socket.return_value = mock_socket_instance + mock_socket_instance.connect.return_value = None + mock_socket_instance.recv.return_value = mock_server_response(None) + + mock_socket_instance.getsockname.return_value = ('127.0.0.1', 12346) + mock_socket_instance.__enter__.return_value = mock_socket_instance + + client = ElasticClient(sources, device_id, model_path, tp, pp) + assert client.register(device_id, model_path, tp, + pp) == ("mocked_name", 12346) + + +# Test the behavior of the `register` method of ElasticClient when the server returns an error response. +def test_elastic_client_register_error_response(): + sources = ["127.0.0.1:12345"] + device_id = 0 + model_path = "mocked_model_path" + tp = 1 + pp = 1 + + with patch('socket.socket') as mock_socket: + mock_socket_instance = MagicMock() + mock_socket.return_value = mock_socket_instance + mock_socket_instance.connect.return_value = None + mock_socket_instance.recv.return_value = mock_server_error_response( + None) + + with ElasticClient(sources, device_id, model_path, tp, pp) as client: + with pytest.raises(RuntimeError): + client.register(device_id, model_path, tp, pp) + mock_socket_instance.close.assert_called_once() + + +# Test the behavior of the `register` method of ElasticClient when an exception is thrown on the server. +def test_elastic_client_register_exception(): + sources = ["127.0.0.1:12345"] + device_id = 0 + model_path = "mocked_model_path" + tp = 1 + pp = 1 + + with patch('socket.socket') as mock_socket: + mock_socket_instance = MagicMock() + mock_socket.return_value = mock_socket_instance + mock_socket_instance.connect.return_value = None + mock_socket_instance.recv.side_effect = mock_server_exception_response + mock_socket_instance.__enter__.return_value = mock_socket_instance + mock_socket_instance.__exit__.return_value = None + + with ElasticClient(sources, device_id, model_path, tp, pp) as client: + with pytest.raises(RuntimeError): + client.register(device_id, model_path, tp, pp) + mock_socket_instance.close.assert_called_once() + + +class FakeInt8Param: + + def __init__(self, name="param", device="npu", dtype=torch.int8): + self.dtype = dtype + self.device = torch.device(device) + + @property + def data(self): + return self # Simulate .data returning self so .cpu() etc. can be chained + + def clone(self): + return self + + def detach(self): + return self + + def cpu(self): + self.device = torch.device("cpu") + return self + + +class FakeModel: + + def __init__(self): + self.params = { + "param1": MagicMock(dtype=torch.float32), # This will be ignored + "param2": FakeInt8Param(), # This simulates a real int8 param + } + + def named_parameters(self): + return self.params.items() + + +@pytest.fixture +def mock_model(): + return FakeModel() + + +@pytest.fixture +def server_config(): + return { + "addr": "127.0.0.1", + "port": 8080, + "model": MagicMock(), + "device_id": 0, + "model_path": "/test/model", + "tp": 1, + "pp": 1, + "int8_cache": "dram", + 'int8_cache_name': None + } + + +# Test server initialization +def test_server_initialization(server_config, mock_model): + server_config["model"] = mock_model + with patch("socket.socket") as mock_socket: + log_capture_string = io.StringIO() + ch = logging.StreamHandler(log_capture_string) + ch.setLevel(logging.DEBUG) + vllm.logger.logger.addHandler(ch) + + server = ElasticServer(**server_config) + + # Check the socket configuration + mock_socket.assert_called_with(socket.AF_INET, socket.SOCK_STREAM) + mock_socket.return_value.setsockopt.assert_called_with( + socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + mock_socket.return_value.bind.assert_called_with(("127.0.0.1", 8080)) + mock_socket.return_value.listen.assert_called_with(256) + + # Check int8 cache + assert "param2" in server.original_int8 + assert server.original_int8[ + "param2"].device.type == "cpu" # Verifying DRAM Cache + + assert server.addr == server_config['addr'] + assert server.port == server_config['port'] + assert server.device_id == server_config['device_id'] + assert server.model_path == server_config['model_path'] + assert server.tp == server_config['tp'] + assert server.pp == server_config['pp'] + + # Get captured logs + log_output = log_capture_string.getvalue() + vllm.logger.logger.removeHandler(ch) + log_capture_string.close() + + # Check output + assert "Server 127.0.0.1:8080 starts" in log_output + + +# Test the int8 cache option +@pytest.mark.parametrize("cache_option,expected_device", [("dram", "cpu"), + ("no", None), + ("invalid", None)]) +def test_int8_cache_handling(server_config, mock_model, cache_option, + expected_device, caplog): + server_config["int8_cache"] = cache_option + server_config["model"] = mock_model + + with patch("socket.socket"): + log_capture_string = io.StringIO() + ch = logging.StreamHandler(log_capture_string) + ch.setLevel(logging.DEBUG) + vllm.logger.logger.addHandler(ch) + + server = ElasticServer(**server_config) + + log_output = log_capture_string.getvalue() + vllm.logger.logger.removeHandler(ch) + log_capture_string.close() + + if cache_option == "invalid": + assert "int8_cache should be selected in [HBM, DRAM]" in log_output + + if expected_device is None: + assert len(server.original_int8) == 0 + else: + assert server.original_int8[ + "param2"].device.type == expected_device + + +# Test client processing +def test_client_handler_valid_join(server_config, mock_model): + server_config["model"] = mock_model + with patch("vllm_ascend.model_loader.netloader.interaction.elastic.P2PSend" + ) as mock_p2p_send: + + # Create a simulated connection + mock_conn = MagicMock() + mock_addr = ("192.168.1.1", 12345) + + # Configuring Client Data + valid_data = { + "label": "JOIN", + "content": { + "device_id": 0, + "model_path": "/test/model", + "tp": 1, + "pp": 1, + "port": 9090 + } + } + mock_conn.recv.return_value = json.dumps(valid_data).encode("utf-8") + + # Start the server + server = ElasticServer(**server_config) + server.register_handler(mock_conn, mock_addr) + + # Verify response + expected_ack = { + "label": "JOIN_ACK", + "content": { + "name": "192.168.1.1:12345" + } + } + mock_conn.send.assert_called_once_with( + json.dumps(expected_ack).encode("utf-8")) + mock_p2p_send.assert_called_once_with("127.0.0.1", 9090, + "192.168.1.1:12345") + mock_conn.close.assert_called_once() + + +# Test mismatched JOIN requests +def test_client_handler_mismatch(server_config): + with patch("socket.socket"): + server = ElasticServer(**server_config) + mock_conn = MagicMock() + mock_addr = ("192.168.1.1", 12345) + + # Send mismatched data + mismatch_data = { + "label": "JOIN", + "content": { + "device_id": 1, # 不匹配的ID + "model_path": "/wrong/model", + "tp": 2, + "pp": 2, + "port": 9090 + } + } + mock_conn.recv.return_value = json.dumps(mismatch_data).encode("utf-8") + + server.register_handler(mock_conn, mock_addr) + + assert isinstance(mismatch_data["content"], dict) + + # Verify response + expected_ack = { + "label": + "JOIN_NACK", + "content": + f"Received data {(mismatch_data['content']['device_id'], mismatch_data['content']['model_path'], mismatch_data['content']['tp'], mismatch_data['content']['pp'])} does not consist with this server {(server_config['device_id'], server_config['model_path'], server_config['tp'], server_config['pp'])}" + } + mock_conn.send.assert_called_once_with( + json.dumps(expected_ack).encode("utf-8")) + mock_conn.close.assert_called_once() + + +# Test Invalid Request +@pytest.mark.parametrize( + "invalid_data,should_send", + [ + ( + { + "label": "WRONG_LABEL" + }, True + ), # Incorrect label, can be decoded as JSON, but the content is invalid. + ( + { + "content": { + "missing_fields": True + } + }, True + ), # Missing field, can be decoded as JSON, but the content is invalid. + ("plain text", False), # Non-JSON data, json.loads failed + (b"invalid_bytes", False) # Invalid byte, decode or json.loads failed + ]) +def test_client_handler_invalid_requests(server_config, invalid_data, + should_send): + with patch("socket.socket"): + log_capture_string = io.StringIO() + ch = logging.StreamHandler(log_capture_string) + ch.setLevel(logging.DEBUG) + vllm.logger.logger.addHandler(ch) + + with patch("socket.socket"): + server = ElasticServer(**server_config) + mock_conn = MagicMock() + mock_addr = ("192.168.1.1", 12345) + + if isinstance(invalid_data, (str, bytes)): + mock_conn.recv.return_value = invalid_data if isinstance( + invalid_data, bytes) else invalid_data.encode() + else: + mock_conn.recv.return_value = json.dumps(invalid_data).encode( + "utf-8") + + server.register_handler(mock_conn, mock_addr) + + if should_send: + expected_ack = { + "label": + "JOIN_NACK", + "content": + f"Received data does not contain required fields: {invalid_data}" + } + mock_conn.send.assert_called_once_with( + json.dumps(expected_ack).encode("utf-8")) + else: + mock_conn.send.assert_not_called() + + log_output = log_capture_string.getvalue() + vllm.logger.logger.removeHandler(ch) + log_capture_string.close() + + # Any warning in the log is acceptable + assert "Failed to load" in log_output or "does not contain" in log_output + mock_conn.close.assert_called_once() + + +# Test the thread startup. +def test_server_start(server_config): + with patch("socket.socket"), \ + patch("threading.Thread") as mock_thread: + + handler_thread_instance = mock_thread.return_value + + server = ElasticServer(**server_config) + server.start() + + # Assert that the correct target parameter was passed when instantiating the Thread instance. + mock_thread.assert_called_once() + args, kwargs = mock_thread.call_args + assert kwargs['target'] == server.elastic_client_handler + + # Check that the daemon attribute is set to True (the attribute value will be recorded after MagicMock assignment). + assert handler_thread_instance.daemon is True + + # Check if the start() method is called. + handler_thread_instance.start.assert_called_once() + + +# Test resource clearing +def test_server_cleanup(server_config): + with patch("socket.socket") as mock_socket: + server = ElasticServer(**server_config) + del server + mock_socket.return_value.close.assert_called_once() + + +if __name__ == "__main__": + pytest.main() diff --git a/tests/ut/model_loader/netloader/test_netloader_load.py b/tests/ut/model_loader/netloader/test_netloader_load.py new file mode 100644 index 00000000..77c3486f --- /dev/null +++ b/tests/ut/model_loader/netloader/test_netloader_load.py @@ -0,0 +1,114 @@ +# +# Copyright (c) 2025 Huawei Technologies Co., Ltd. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from unittest.mock import MagicMock, patch + +import pytest + +from vllm_ascend.model_loader.netloader.load import elastic_load + + +@pytest.fixture +def mock_sources(): + return [ + { + "device_id": 0, + "sources": ["a", "b"] + }, + { + "device_id": 1, + "sources": ["c"] + }, + ] + + +@patch("vllm_ascend.model_loader.netloader.interaction.elastic.ElasticClient") +@patch("vllm_ascend.model_loader.netloader.executor.elastic_load.P2PLoad") +def test_sources_this_device_empty(mock_p2p, mock_client): + sources = [{"device_id": 1, "sources": ["c"]}] + result = elastic_load("model", 0, "model_path", sources, 1, 1) + assert result is None + mock_client.assert_not_called() + mock_p2p.assert_not_called() + + +@patch("vllm_ascend.model_loader.netloader.interaction.elastic.ElasticClient") +@patch("vllm_ascend.model_loader.netloader.executor.elastic_load.P2PLoad") +def test_client_s_none(mock_p2p, mock_client, mock_sources): + # Simulate ElasticClient.s as None + mock_instance = MagicMock() + mock_instance.s = None + mock_client.return_value = mock_instance + result = elastic_load("model", 0, "model_path", mock_sources, 1, 1) + assert result is None + + +@patch("vllm_ascend.model_loader.netloader.interaction.elastic.ElasticClient") +@patch("vllm_ascend.model_loader.netloader.executor.elastic_load.P2PLoad") +def test_client_ack_none(mock_p2p, mock_client, mock_sources): + # Simulate ElasticClient.ack as None + mock_instance = MagicMock() + mock_instance.s = True + mock_instance.ack = None + mock_client.return_value = mock_instance + result = elastic_load("model", 0, "model_path", mock_sources, 1, 1) + assert result is None + + +@patch("vllm_ascend.model_loader.netloader.load.P2PLoad") +@patch("vllm_ascend.model_loader.netloader.load.logger") +def test_model_load_fail(mock_logger, mock_p2p): + mock_client = MagicMock() + mock_client.s = True + mock_client.ack = ["foo", "bar"] + mock_client.server_addr = "addr" + + with patch("vllm_ascend.model_loader.netloader.load.ElasticClient", + return_value=mock_client): + # P2PLoad.load returns None + mock_p2p_instance = MagicMock() + mock_p2p_instance.load.return_value = None + mock_p2p.return_value = mock_p2p_instance + + sources = [{"device_id": 0, "sources": ["whatever"]}] + result = elastic_load("model", 0, "model_path", sources, 1, 1) + assert result is None + mock_logger.error.assert_called_once() + + +@patch("vllm_ascend.model_loader.netloader.load.P2PLoad") +@patch("vllm_ascend.model_loader.netloader.load.logger") +def test_model_load_success(mock_logger, mock_p2p): + mock_client = MagicMock() + mock_client.s = True + mock_client.ack = ["foo", "bar"] + mock_client.server_addr = "addr" + + with patch("vllm_ascend.model_loader.netloader.load.ElasticClient", + return_value=mock_client): + expected_model = object() + mock_p2p_instance = MagicMock() + mock_p2p_instance.load.return_value = expected_model + mock_p2p.return_value = mock_p2p_instance + + sources = [{"device_id": 0, "sources": ["whatever"]}] + result = elastic_load("model", 0, "model_path", sources, 1, 1) + assert result is expected_model + mock_logger.info.assert_called_once() + + +if __name__ == "__main__": + pytest.main() diff --git a/tests/ut/model_loader/netloader/test_netloader_utils.py b/tests/ut/model_loader/netloader/test_netloader_utils.py new file mode 100644 index 00000000..66198449 --- /dev/null +++ b/tests/ut/model_loader/netloader/test_netloader_utils.py @@ -0,0 +1,61 @@ +# +# Copyright (c) 2025 Huawei Technologies Co., Ltd. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import os +import tempfile +from unittest.mock import patch + +import pytest + +from vllm_ascend.model_loader.netloader.utils import (find_free_port, + is_valid_path_prefix) + + +def test_find_free_port(): + port = find_free_port() + assert isinstance(port, int) + assert port > 0 + + +def test_is_valid_path_prefix_empty(): + assert not is_valid_path_prefix('') + + +def test_is_valid_path_prefixIllegal_characters(): + assert not is_valid_path_prefix('test<>:"|?*') + + +def test_is_valid_path_prefixRelative_path(): + assert is_valid_path_prefix('test') + + +def test_is_valid_path_prefixAbsolute_path(): + with tempfile.TemporaryDirectory() as tmpdir: + assert is_valid_path_prefix(os.path.join(tmpdir, 'test')) + + +@patch('os.path.exists', return_value=False) +def test_is_valid_path_prefix_no_directory(mock_exists): + assert not is_valid_path_prefix('/nonexistent_dir/test') + + +@patch('os.path.exists', return_value=True) +def test_is_valid_path_prefix_directory_exists(mock_exists): + assert is_valid_path_prefix('/existing_dir/test') + + +if __name__ == "__main__": + pytest.main() diff --git a/vllm_ascend/__init__.py b/vllm_ascend/__init__.py index 74a81537..aa72f74b 100644 --- a/vllm_ascend/__init__.py +++ b/vllm_ascend/__init__.py @@ -31,3 +31,8 @@ def register_model(): def register_connector(): from vllm_ascend.distributed import register_connector register_connector() + + +def register_model_loader(): + from .model_loader.netloader import register_netloader + register_netloader() \ No newline at end of file diff --git a/vllm_ascend/model_loader/__init__.py b/vllm_ascend/model_loader/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/vllm_ascend/model_loader/netloader/__init__.py b/vllm_ascend/model_loader/netloader/__init__.py new file mode 100644 index 00000000..f3b7c50a --- /dev/null +++ b/vllm_ascend/model_loader/netloader/__init__.py @@ -0,0 +1,20 @@ +# +# Copyright (c) 2025 Huawei Technologies Co., Ltd. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +def register_netloader(): + """Register the NetLoader plugin.""" + from .netloader import ModelNetLoaderElastic # noqa diff --git a/vllm_ascend/model_loader/netloader/executor/__init__.py b/vllm_ascend/model_loader/netloader/executor/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/vllm_ascend/model_loader/netloader/executor/elastic_load.py b/vllm_ascend/model_loader/netloader/executor/elastic_load.py new file mode 100644 index 00000000..850bfaf9 --- /dev/null +++ b/vllm_ascend/model_loader/netloader/executor/elastic_load.py @@ -0,0 +1,170 @@ +# +# Copyright (c) 2025 Huawei Technologies Co., Ltd. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import torch +import torch_npu +from vllm.distributed.utils import ( + stateless_destroy_torch_distributed_process_group, + stateless_init_torch_distributed_process_group) +from vllm.logger import logger + + +class P2PLoad: + """ + Class for receiving model parameters in a distributed manner using HCCL backend. + """ + + def __init__( + self, + world_name: str, + source_ip: str, + source_port: int, + ): + """ + Initializes the P2PLoad instance. + + Parameters: + - world_name: The name of the distributed group. + - source_ip: The IP address of the source node. + - source_port: The port number for the source node. + """ + self.world_name = world_name + self.source_ip = source_ip + self.source_port = source_port + + def load(self, model): + """ + Loads the model parameters using HCCL backend. + + Parameters: + - model: The model whose parameters are to be loaded. + + Returns: + - The model if loading is successful, otherwise None. + """ + model_device = next(model.parameters()).device + logger.info( + f"Start init_process_group, name: {self.world_name}, addr: {self.source_ip}:{self.source_port}" + ) + receiver_pg = None + loaded_model = None + try: + receiver_pg = stateless_init_torch_distributed_process_group( + host=self.world_name.split(":")[0], + port=self.source_port, + rank=0, + world_size=2, + backend='hccl', + ) + logger.info( + f"Finish init_process_group, name: {self.world_name}, addr: {self.source_ip}:{self.source_port}" + ) + + logger.info( + f"Start recv, name: {self.world_name}, addr: {self.source_ip}:{self.source_port}" + ) + logger.info(f"Model device: {model_device}") + + trans_stream = torch_npu.npu.Stream() + with torch_npu.npu.stream(trans_stream): + for name, param in model.named_parameters(): + if len(param.shape) == 0: + continue + receiver_pg.recv([param], 1, 0).wait() + torch.distributed.barrier(group=receiver_pg, + device_ids=[model_device.index]) + + torch_npu.npu.synchronize(trans_stream) + + logger.info( + f"Finish recv, name: {self.world_name}, addr: {self.source_ip}:{self.source_port}" + ) + loaded_model = model + except Exception as e: + logger.error("Failed to recv model: {}".format(e)) + finally: + if receiver_pg: + stateless_destroy_torch_distributed_process_group(receiver_pg) + return loaded_model + + +class P2PSend: + """ + Class for sending model parameters in a distributed manner using HCCL backend. + """ + + def __init__(self, listen_ip: str, listen_port: int, comm_name: str): + """ + Initializes the P2PSend instance. + + Parameters: + - listen_ip: The IP address to listen on. + - listen_port: The port number to listen on. + - comm_name: The name of the communication group. + """ + self.listen_ip = listen_ip + self.listen_port = listen_port + self.comm_name = comm_name + + def send(self, model, int8_params: dict): + """ + Sends the model parameters using HCCL backend. + + Parameters: + - model: The model whose parameters are to be sent. + - int8_params: Dictionary of parameters that are in int8 format. + """ + model_device = next(model.parameters()).device + torch.npu.set_device(model_device) + logger.info( + f"Start init_process_group, name: {self.comm_name}, addr: {self.listen_ip}:{self.listen_port}" + ) + sender_pg = None + try: + sender_pg = stateless_init_torch_distributed_process_group( + host=self.comm_name.split(":")[0], + port=self.listen_port, + rank=1, + world_size=2, + backend='hccl', + ) + logger.info( + f"Finish init_process_group, name: {self.comm_name}, addr: {self.listen_ip}:{self.listen_port}" + ) + logger.info( + f"Start send, name: {self.comm_name}, addr: {self.listen_ip}:{self.listen_port}" + ) + logger.info(f"Model device: {model_device}") + + trans_stream = torch_npu.npu.Stream() + with torch_npu.npu.stream(trans_stream): + for name, param in model.named_parameters(): + if "aclnn_input_scale" in name: + continue + if name in int8_params: + sender_pg.send([int8_params[name].to(model_device)], 0, + 0).wait() + else: + sender_pg.send([param.contiguous()], 0, 0).wait() + torch.distributed.barrier(group=sender_pg, + device_ids=[model_device.index]) + torch_npu.npu.synchronize(trans_stream) + logger.info( + f"Finish send, name: {self.comm_name}, addr: {self.listen_ip}:{self.listen_port}" + ) + finally: + if sender_pg: + stateless_destroy_torch_distributed_process_group(sender_pg) diff --git a/vllm_ascend/model_loader/netloader/interaction/__init__.py b/vllm_ascend/model_loader/netloader/interaction/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/vllm_ascend/model_loader/netloader/interaction/elastic.py b/vllm_ascend/model_loader/netloader/interaction/elastic.py new file mode 100644 index 00000000..1000bd7c --- /dev/null +++ b/vllm_ascend/model_loader/netloader/interaction/elastic.py @@ -0,0 +1,408 @@ +# +# Copyright (c) 2025 Huawei Technologies Co., Ltd. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import json +import re +import socket +import threading +from typing import List, Optional, Tuple + +import torch +from vllm.logger import logger + +from ..executor.elastic_load import P2PSend +from ..utils import find_free_port + + +class ElasticClient: + """ + Class for handling the client-side logic of Netloader of models. + """ + + def __init__(self, sources: list[str], device_id: int, model_path: str, + tp: int, pp: int): + """ + Initializes the ElasticClient instance. + + Parameters: + - sources: List of source addresses in the format IP:port. + - device_id: The ID of the current device. + - model_path: The path to the model. + - tp: Tensor parallel size. + - pp: Pipeline parallel size. + """ + self.sources = sources + self.device_id = device_id + self.model_path = model_path + self.tp = tp + self.pp = pp + + self.s: Optional[socket.socket] = None + self.ack: Optional[Tuple[str, int]] = None + self.server_addr: Optional[str] = None + self.server_port: Optional[int] = None + + for source in self.sources: + try: + ip, port_str = source.split(':') + port = int(port_str) + except Exception as e: + logger.error(f"IP format error: {source}, detail: {e}") + continue + + self.server_addr = ip + self.server_port = port + + try: + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + logger.info( + f"Start connection to server: {self.server_addr}:{self.server_port}" + ) + sock.connect((self.server_addr, self.server_port)) + logger.info( + f"Finish connection to server: {self.server_addr}:{self.server_port}" + ) + sock.settimeout(60) + + self.s = sock + self.ack = self.register(device_id, model_path, tp, pp) + break + except Exception as e: + logger.error(f"Connect to {source} fails, detail: {e}") + if sock is not None: + try: + sock.close() + except Exception: + pass + self.s = None + self.ack = None + self.server_addr = None + self.server_port = None + + def close(self) -> None: + """ + Closes the socket connection. + """ + if self.s is not None: + try: + self.s.close() + except Exception as e: + logger.error(f"Error closing socket: {e}") + finally: + self.s = None + + def __enter__(self) -> "ElasticClient": + """ + Context manager enter method. + """ + return self + + def __exit__(self, exc_type, exc_val, exc_tb) -> None: + """ + Context manager exit method. + """ + self.close() + + def __del__(self): + """ + Destructor method to ensure socket is closed. + """ + try: + self.close() + except Exception: + pass + + def send_str(self, data_str: str) -> None: + """ + Sends a string over the socket connection. + + Parameters: + - data_str: The string to be sent. + """ + if self.s is None: + raise RuntimeError("Socket was not created correctly.") + self.s.send(data_str.encode("utf-8")) + + def recv_str(self, buffer_size: int = 1024) -> str: + """ + Receives a string over the socket connection. + + Parameters: + - buffer_size: The size of the buffer for receiving data. + + Returns: + - The received string. + """ + if self.s is None: + raise RuntimeError("Socket was not created correctly.") + data_str = self.s.recv(buffer_size).decode("utf-8") + return data_str + + def register(self, device_id: int, model_path: str, tp: int, + pp: int) -> Tuple[str, int]: + """ + Registers the client with the server. + + Parameters: + - device_id: The ID of the current device. + - model_path: The path to the model. + - tp: Tensor parallel size. + - pp: Pipeline parallel size. + + Returns: + - A tuple containing the communication name and port. + """ + free_port = find_free_port() + data = { + "label": "JOIN", + "content": { + 'device_id': device_id, + 'model_path': model_path, + 'tp': tp, + 'pp': pp, + 'port': free_port + } + } + + try: + self.send_str(json.dumps(data)) + except Exception as e: + raise RuntimeError( + f"Send data {data} to server fails, detail: {e}") + + try: + ack_str = self.recv_str() + except Exception as e: + raise RuntimeError(f"Receive data from server fails, detail: {e}") + + try: + ack = json.loads(ack_str) + except Exception as e: + raise RuntimeError( + f"Receive data {ack_str} cannot be converted to JSON format, detail: {e}" + ) + + logger.info(f"Receive ack: {ack}") + + if ("label" in ack and ack["label"] == 'JOIN_ACK' and "content" in ack + and ack["content"] is not None and "name" in ack["content"]): + return (ack["content"]["name"], free_port) + elif ("label" in ack and ack["label"] == 'JOIN_NACK' + and "content" in ack): + raise RuntimeError( + f"Receive nack from server, reason: {ack['content']}") + else: + raise RuntimeError( + f"Receive ack {ack} from server does not contain required fields" + ) + + +class ElasticServer: + """ + Class for handling the server-side logic of Netloader of models. + """ + + def __init__(self, addr: str, port: int, model, device_id: int, + model_path: str, tp: int, pp: int, int8_cache: str, + int8_cache_name: Optional[List[str]]): + """ + Initializes the ElasticServer instance. + + Parameters: + - addr: The IP address to listen on. + - port: The port number to listen on. + - model: The model to be served. + - device_id: The ID of the current device (i.e. global rank). + - model_path: The path to the model. + - tp: Tensor parallel size. + - pp: Pipeline parallel size. + - int8_cache: The type of caching for int8 parameters (HBM, DRAM, or no). + - int8_cache_name: List of parameter names to be cached. + """ + self.addr = addr + self.port = port + self.s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + self.s.bind((self.addr, self.port)) + self.s.listen(256) + + self.model = model + self.device_id = device_id + self.model_path = model_path + self.tp = tp + self.pp = pp + + self.original_int8 = {} + int8_pattern = "|".join( + map(re.escape, + int8_cache_name)) if int8_cache_name is not None else "(?:)" + for name, param in self.model.named_parameters(): + if param.dtype == torch.int8: + if int8_cache == 'hbm': + if int8_cache_name is None or ( + int8_cache_name is not None + and re.search(int8_pattern, name) is not None): + try: + self.original_int8[name] = param.data.clone( + ).detach() + except RuntimeError as e: + logger.error( + f"Failed to cache int8 tensor {name} to HBM, change to DRAM, due to {e}" + ) + self.original_int8[name] = param.data.cpu() + + elif int8_cache == 'dram': + if int8_cache_name is None or ( + int8_cache_name is not None + and re.search(int8_pattern, name) is not None): + self.original_int8[name] = param.data.cpu() + elif int8_cache == 'no': + pass + else: + logger.warning( + f"int8_cache should be selected in [HBM, DRAM], but got {int8_cache}, change to no cache" + ) + + logger.info( + f"Server {self.addr}:{self.port} starts, device id: {self.device_id}, model path: {self.model_path}, tp: {self.tp}, pp: {self.pp}, int8 params {self.original_int8.keys()} are saved to {int8_cache}" + ) + + def __del__(self): + """ + Destructor method to ensure socket is closed. + """ + self.s.close() + + def start(self): + """ + Starts the server to handle incoming connections. + """ + handler_thread = threading.Thread(target=self.elastic_client_handler) + handler_thread.daemon = True + handler_thread.start() + + def elastic_client_handler(self): + """ + Handles incoming client connections. + """ + while True: + conn, addr = self.s.accept() + logger.info("Accept new connection from {}:{}...".format(*addr)) + self.register_handler(conn, addr) + + def register_handler(self, conn, addr, buffer_size=1024): + """ + Handles the registration of a client. + + Parameters: + - conn: The connection socket. + - addr: The address of the client. + - buffer_size: The size of the buffer for receiving data. + """ + data_str = conn.recv(buffer_size).decode("utf-8") + if not data_str: + return + try: + data = json.loads(data_str) + except Exception: + logger.error(f"Failed to load {data_str} as JSON string") + conn.close() + return + + def is_valid_data(data): + """ + Validates the received data. + + Parameters: + - data: The data to be validated. + + Returns: + - True if the data is valid, otherwise False. + """ + if not isinstance(data, dict): + return False + if data.get("label") != "JOIN": + return False + content = data.get("content") + if not isinstance(content, dict): + return False + required_keys = ["device_id", "model_path", "tp", "pp", "port"] + if not all(k in content for k in required_keys): + return False + port = content["port"] + if not (isinstance(port, int) or + (isinstance(port, str) and port.isdigit())): + return False + return True + + comm_name = None + if is_valid_data(data): + device_id = int(data["content"]["device_id"]) + model_path = data["content"]["model_path"] + tp = int(data["content"]["tp"]) + pp = int(data["content"]["pp"]) + + if int(self.device_id + ) == device_id and self.model_path == model_path and int( + self.tp) == tp and int(self.pp) == pp: + comm_name = str(addr[0]) + ":" + str(addr[1]) + ack = {"label": "JOIN_ACK", "content": {"name": comm_name}} + else: + logger.warning( + f"Received data {(device_id, model_path, tp, pp)} does not consist with this server {(int(self.device_id), self.model_path, int(self.tp), int(self.pp))}" + ) + ack = { + "label": + "JOIN_NACK", + "content": + f"Received data {(device_id, model_path, tp, pp)} does not consist with this server {(int(self.device_id), self.model_path, int(self.tp), int(self.pp))}" + } + else: + logger.warning( + f"Received data does not contain required fields: {data}") + ack = { + "label": + "JOIN_NACK", + "content": + f"Received data does not contain required fields: {data}" + } + + try: + ack_str = json.dumps(ack).encode("utf-8") + except Exception as e: + logger.error( + f"Failed to convert {ack} to JSON format, details: {e}") + conn.close() + return + + try: + conn.send(ack_str) + except Exception as e: + logger.error(f"Failed to send {ack} to {addr}, details: {e}") + conn.close() + return + + if ack["content"] and isinstance(ack["content"], + dict) and 'name' in ack["content"]: + try: + p2psend = P2PSend(self.addr, data["content"]["port"], + ack["content"]["name"]) + p2psend.send(self.model, self.original_int8) + except Exception as e: + logger.error( + f"P2PSend Failed to send model to {self.addr}, details: {e}" + ) + conn.close() diff --git a/vllm_ascend/model_loader/netloader/load.py b/vllm_ascend/model_loader/netloader/load.py new file mode 100644 index 00000000..90000d58 --- /dev/null +++ b/vllm_ascend/model_loader/netloader/load.py @@ -0,0 +1,84 @@ +# +# Copyright (c) 2025 Huawei Technologies Co., Ltd. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import time + +from vllm.logger import logger + +from .executor.elastic_load import P2PLoad +from .interaction.elastic import ElasticClient + + +def elastic_load( + model, + device_id: int, + model_path: str, + sources: list, + tp: int, + pp: int, +): + """ + Loads a model using elastic loading across multiple devices. + + Parameters: + - model: The model instance to be loaded. + - device_id: The ID of the current device (i.e. global rank). + - model_path: The path to the model file. + - sources: A list of source configurations, each containing device_id and sources. + - tp: Tensor parallel size, indicating the number of devices for tensor parallelism. + - pp: Pipeline parallel size, indicating the number of devices for pipeline parallelism. + + Returns: + - The loaded model if successful, otherwise None. + """ + + # Filter sources for the current device + sources_this_device = [] + for s in sources: + if isinstance( + s, dict + ) and "device_id" in s and s["device_id"] == device_id and isinstance( + s["sources"], list): + sources_this_device += s["sources"] + if len(sources_this_device) == 0: + return None + + try: + # Initialize the interaction layer with the ElasticClient + with ElasticClient(sources_this_device, device_id, model_path, tp, + pp) as client_interaction_layer: + if client_interaction_layer.s is None or client_interaction_layer.server_addr is None: + raise RuntimeError( + "Failed to initialize ElasticClient: socket or server_addr is None" + ) + ack = client_interaction_layer.ack + if ack is None: + raise RuntimeError("ElasticClient.register did not return ack") + + t0 = time.perf_counter() + elastic_loader = P2PLoad(ack[0], + client_interaction_layer.server_addr, + ack[1]) + model_loaded = elastic_loader.load(model=model) + if model_loaded is None: + logger.error("Failed to load model") + return None + logger.info("Finish elastic load (duration: {}s)".format( + time.perf_counter() - t0)) + return model_loaded + except Exception as e: + logger.error(f"elastic_load error: {e}") + return None diff --git a/vllm_ascend/model_loader/netloader/netloader.py b/vllm_ascend/model_loader/netloader/netloader.py new file mode 100644 index 00000000..9c2d8307 --- /dev/null +++ b/vllm_ascend/model_loader/netloader/netloader.py @@ -0,0 +1,324 @@ +# +# Copyright (c) 2025 Huawei Technologies Co., Ltd. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import gc +import json +import time +from copy import deepcopy +from typing import List, Optional, Tuple + +import torch +from torch import nn +from vllm.config import LoadConfig, ModelConfig, VllmConfig +from vllm.logger import logger +from vllm.model_executor.model_loader import register_model_loader +from vllm.model_executor.model_loader.base_loader import BaseModelLoader +from vllm.model_executor.model_loader.default_loader import DefaultModelLoader +from vllm.model_executor.model_loader.utils import ( + initialize_model, process_weights_after_loading, set_default_torch_dtype) + +from .interaction.elastic import ElasticServer +from .load import elastic_load +from .utils import find_free_port, is_valid_path_prefix + + +@register_model_loader("netloader") +class ModelNetLoaderElastic(BaseModelLoader): + """ + A model loader that uses elastic loading for loading weights. + """ + source: Optional[List[dict]] + model_path: Optional[str] + listen_port: Optional[int] + int8_cache: str + int8_cache_name: Optional[List[str]] + output_prefix: Optional[str] + + def __init__(self, load_config: LoadConfig): + """ + Initializes the ModelNetLoaderElastic with configuration. + + Parameters: + - load_config: Configuration for loading the model. + """ + super().__init__(load_config) + + config = None + + # Try to read config file at first + extra = load_config.model_loader_extra_config + if extra and "CONFIG_FILE" in extra: + try: + logger.info( + f"Reading configs in file {load_config.model_loader_extra_config['CONFIG_FILE']} ..." + ) + with open(extra["CONFIG_FILE"], 'r') as f: + config = json.load(f) + except FileNotFoundError: + logger.error("CONFIG_FILE not found") + except json.JSONDecodeError: + logger.error("CONFIG_FILE is not a valid JSON file") + except Exception as e: + logger.error( + f"Unexpected error while reading CONFIG_FILE: {e}") + + if config is None and extra: + logger.info("Reading configs in model_loader_extra_config ...") + config = extra + config = config or {} + + for key, attr, checker, caster, default in [ + ("SOURCE", "source", lambda v: isinstance(v, list), lambda v: v, + None), + ("MODEL", "model_path", lambda v: isinstance(v, str), lambda v: v, + None), + ("LISTEN_PORT", "listen_port", lambda v: isinstance(v, int) or + (isinstance(v, str) and v.isdigit()), lambda v: int(v), None), + ("INT8_CACHE", "int8_cache", lambda v: isinstance(v, str) and v. + lower() in ['hbm', 'dram', 'no'], lambda v: v.lower(), 'no'), + ("INT8_CACHE_NAME", "int8_cache_name", + lambda v: isinstance(v, list), lambda v: v, None), + ("OUTPUT_PREFIX", "output_prefix", + lambda v: isinstance(v, str) and is_valid_path_prefix(v), + lambda v: v, None), + ]: + v = config.get(key, default) + if not checker(v): + v = default + else: + v = caster(v) + setattr(self, attr, v) + + logger.info( + "Initializing elastic Netloader with config: " + "MODEL=%s, LISTEN_PORT=%s," + "SOURCE=%s, INT8_CACHE=%s, INT8_CACHE_NAME=%s," + "OUTPUT_PREFIX=%s)", + self.model_path, + self.listen_port, + self.source, + self.int8_cache, + self.int8_cache_name, + self.output_prefix, + ) + + def load_model(self, vllm_config: VllmConfig, + model_config: ModelConfig) -> nn.Module: + """ + Loads the model using the specified configuration. + + Parameters: + - vllm_config: Configuration for the VLLM. + - model_config: Configuration for the model. + + Returns: + - The loaded model. + """ + + device_config = vllm_config.device_config + parallel_config = vllm_config.parallel_config + + need_process_weights_after_loading = False + + if self.model_path is None: + self.model_path = model_config.model + logger.info(f"model_path is set to {self.model_path}") + + device_id = torch.distributed.get_rank() + + if (self.source is None or not isinstance(self.source, list) + or device_id not in [ + one_device["device_id"] for one_device in self.source if + isinstance(one_device, dict) and "device_id" in one_device + ]): + logger.warning( + "Did not get valid source info, use DefaultModelLoader") + model, need_process_weights_after_loading = self.revert_to_default( + model_config, vllm_config, device_config) + + else: + target_device = torch.device(device_config.device) + + vllm_config_backup = deepcopy(vllm_config) + model_config_backup = deepcopy(model_config) + + with set_default_torch_dtype(model_config.dtype): + with target_device: + model = initialize_model(vllm_config=vllm_config, + model_config=model_config) + + start_elastic_load = time.perf_counter() + model = elastic_load( + model=model, + device_id=device_id, + model_path=self.model_path, + sources=self.source, + tp=parallel_config.tensor_parallel_size, + pp=parallel_config.pipeline_parallel_size, + ) + end_elastic_load = time.perf_counter() + logger.info( + f"Elastic load time: {end_elastic_load - start_elastic_load}, rank: {device_id}" + ) + need_process_weights_after_loading = True + + if model is None: + logger.warning( + "Netloader elastic loading fails, use load format DefaultModelLoader" + ) + + vllm_config = vllm_config_backup + model_config = model_config_backup + + del model + gc.collect() + if device_config.device_type == 'npu': + logger.info("Empty NPU cache") + torch.npu.empty_cache() + elif device_config.device_type == 'cuda': + logger.info("Empty CUDA cache") + torch.cuda.empty_cache() + + model, need_process_weights_after_loading = self.revert_to_default( + model_config, vllm_config, device_config) + + start_elastic_server = time.perf_counter() + # start elastic server + if model is not None and ( + (self.listen_port and self.listen_port in range(1024, 65535)) or + (self.listen_port is None)): + from vllm.utils import get_ip + driver_ip = get_ip() + + if driver_ip == '0.0.0.0': + logger.error( + "Driver IP is not set, skip to start Netloader server") + else: + if self.listen_port is None: + self.listen_port = find_free_port() + else: + self.listen_port += device_id + + logger.info( + f"Start elastic Netloader server, rank: {device_id}, listen port: {driver_ip}:{self.listen_port}" + ) + + if self.output_prefix is not None: + try: + with open(self.output_prefix + str(device_id) + '.txt', + 'w') as file: + file.write(f"{driver_ip}:{self.listen_port}") + logger.info( + f"Successfully wrote server address to file: {self.output_prefix + str(device_id)}" + ) + except FileNotFoundError: + logger.error( + f"File path {self.output_prefix + str(device_id)} does not exist." + ) + except PermissionError: + logger.error( + f"No permission to write to file {self.output_prefix + str(device_id)}." + ) + except IOError as e: + logger.error( + f"I/O error occurred while writing to file {self.output_prefix + str(device_id)}: {e}" + ) + except Exception as e: + logger.error(f"Unknown error: {e}") + + try: + assert isinstance( + self.listen_port, int + ), f"listen port should be int but get {self.listen_port}" + + elastic_server = ElasticServer( + driver_ip, self.listen_port, model, device_id, + self.model_path, parallel_config.tensor_parallel_size, + parallel_config.pipeline_parallel_size, + self.int8_cache, self.int8_cache_name) + elastic_server.start() + except Exception as e: + logger.error( + f"Failed to start Netloader server for rank: {device_id}, details: {e}" + ) + else: + logger.info("Skip to start Netloader server") + + end_elastic_server = time.perf_counter() + logger.info( + f"Elastic server start time: {end_elastic_server - start_elastic_server}, rank: {device_id}" + ) + + if need_process_weights_after_loading: + process_weights_after_loading(model, model_config, + torch.device(device_config.device)) + + if model is None: + logger.error("NetLoader elastic loads model fails") + return None + + return model.eval() + + def revert_to_default(self, model_config, vllm_config, + device_config) -> Tuple[nn.Module, bool]: + """ + Reverts to the default model loading logic when elastic loading fails or is not applicable. + + This method resets the loader's extra config and load format to defaults, + then delegates model loading to a DefaultModelLoader. + If quantization is enabled, it will load the model and then run the + processing of weights (i.e. applying quantization adjustments) before returning. + + Parameters: + - model_config: Configuration describing model architecture, quantization, etc. + - vllm_config: Configuration for vLLM (device, parallelism, dtype, etc). + - device_config: Configuration for the target device (device type, device id, etc). + + Returns: + - A tuple (model, need_process_weights_after_loading): + * model: The loaded `nn.Module` under default loading logic. + * need_process_weights_after_loading: A boolean flag indicating whether + weights post-processing (e.g. quantization adjustments) still needs to be applied. + """ + self.load_config.model_loader_extra_config = {} + self.load_config.load_format = "auto" + default_model_loader = DefaultModelLoader(self.load_config) + + if model_config.quantization is None: + model = default_model_loader.load_model(vllm_config=vllm_config, + model_config=model_config) + need_process_weights_after_loading = False + else: + logger.warning( + "Quantization is set, netloader use DefaultModelLoader with process_weights_after_loading " + ) + need_process_weights_after_loading = True + target_device = torch.device(device_config.device) + with set_default_torch_dtype(model_config.dtype): + with target_device: + model = initialize_model(vllm_config=vllm_config, + model_config=model_config) + default_model_loader.load_weights(model, model_config) + model = model.eval() + + return model, need_process_weights_after_loading + + def download_model(self, model_config: ModelConfig) -> None: + pass + + def load_weights(self, model: nn.Module, + model_config: ModelConfig) -> None: + pass diff --git a/vllm_ascend/model_loader/netloader/utils.py b/vllm_ascend/model_loader/netloader/utils.py new file mode 100644 index 00000000..fba5a58c --- /dev/null +++ b/vllm_ascend/model_loader/netloader/utils.py @@ -0,0 +1,66 @@ +# +# Copyright (c) 2025 Huawei Technologies Co., Ltd. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import os +import re +import socket + +from vllm.logger import logger + + +def find_free_port(): + """ + Finds a free port on the local machine. + + Returns: + - A free port number. + """ + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.bind(('', 0)) + return s.getsockname()[1] + + +def is_valid_path_prefix(path_prefix): + """ + Checks if the provided path prefix is valid. + + Parameters: + - path_prefix: The path prefix to validate. + + Returns: + - True if the path prefix is valid, otherwise False. + """ + if not path_prefix: + return False + + if re.search(r'[<>:"|?*]', path_prefix): + logger.warning( + f'The path prefix {path_prefix} contains illegal characters.') + return False + + if path_prefix.startswith('/') or path_prefix.startswith('\\'): + if not os.path.exists(os.path.dirname(path_prefix)): + logger.warning( + f'The directory for the path prefix {os.path.dirname(path_prefix)} does not exist.' + ) + return False + else: + if not os.path.exists(os.path.dirname(os.path.abspath(path_prefix))): + logger.warning( + f'The directory for the path prefix {os.path.dirname(os.path.abspath(path_prefix))} does not exist.' + ) + return False + return True \ No newline at end of file