From 67a5639c3c70244b1ef7223e6d1696f5dc67e76b Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 14 Aug 2026 03:35:24 +0000 Subject: [PATCH] fix(submit): restore a3c45d3b-proven config + safe improvements Based on a3c45d3b (last known working docker build): - yaml: max-num-seqs=2 (fixes t2_n_2), TOPK_SOFTMAX=1 (use prebuilt .so) - yaml: keep max-model-len=131072, gpu-mem=0.90 (prevents OOM) - yaml: NO LD_PRELOAD (libcccl not built during docker build) - xformers: revert to Q-tiling only (flash_attn caused OOM at profiling) - .dockerignore: exclude all non-essential files from context - remove libcccl_allocator.so from git tracking What stays from recent work: - 14 prebuilt .so (including corex_gdn_chunk_recurrent) - qwen3_5.py with .float() fix and chunk_recurrent support - All vendor_overrides and CCCL preload source (for future use) --- .dockerignore | 28 +++++ computility-run.yaml | 2 +- qwen3_6_scripts/cccl_preload/.gitignore | 1 + .../cccl_preload/libcccl_allocator.so | Bin 51864 -> 0 bytes qwen3_6_scripts/patch_xformers_sdpa_seq.py | 103 +++++------------- 5 files changed, 58 insertions(+), 76 deletions(-) create mode 100644 qwen3_6_scripts/cccl_preload/.gitignore delete mode 100755 qwen3_6_scripts/cccl_preload/libcccl_allocator.so diff --git a/.dockerignore b/.dockerignore index ae6c7e48..50191086 100644 --- a/.dockerignore +++ b/.dockerignore @@ -11,3 +11,31 @@ ex_engine/ enginex-vllm-bi100-qwen36-main.zip dockerrizhi.txt subrizhi.txt +docs/ +optimizations/ +vllm_adapter/ +vllm_overrides/ +*.md +verify_*.py +verify_*.sh +test_*.py +debug_*.py +probe_*.py +probe_*.sh +diagnose_*.sh +cat_*.py +attention.py +paged_attention_v2_*.py +paged_attn.py +prefix_prefill.py +engine_cccl_patterns.py +deltanet_chunk_optimize.py +muh_*.py +__init__.py +chat_dataset_v0.json +cccl_sm100_benchmark_values.json +baseline.muh +launch_service +Dockerfile.* +computility-run.*.yaml +computility-run.yaml.bak diff --git a/computility-run.yaml b/computility-run.yaml index 2e09be09..5d4d5217 100644 --- a/computility-run.yaml +++ b/computility-run.yaml @@ -15,7 +15,7 @@ command: - -tp - '4' - --max-num-seqs - - '1' + - '2' - --disable-log-requests - --disable-frontend-multiprocessing - --max-num-batched-tokens diff --git a/qwen3_6_scripts/cccl_preload/.gitignore b/qwen3_6_scripts/cccl_preload/.gitignore new file mode 100644 index 00000000..140f8cf8 --- /dev/null +++ b/qwen3_6_scripts/cccl_preload/.gitignore @@ -0,0 +1 @@ +*.so diff --git a/qwen3_6_scripts/cccl_preload/libcccl_allocator.so b/qwen3_6_scripts/cccl_preload/libcccl_allocator.so deleted file mode 100755 index 5de35903cef30a457174585670f47db3649ee22d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 51864 zcmeHw3wT@AmF{sICkB#8v;kb6f;!aU;YyB^*o1)mP$C665wR1@YmOCJi4|;1$VZ98 zh1L*G=&z-TW)A^p$#c7-+%4Bwtdct zMC#1<&D?vx9Dmk2`(KZ}_u8+u&pFchyf3(9N?x9(j(qJBjZ*#dB;sX8(Jnp#@oE*? zarit_J447$onReymN-vBifT@oraTJh)GgC?dd32YD{9sg`6XD-MxU82aYfB~(}>l} zdUsf9%W*Qj{dmT?oKuh_y-wEaWIaV)B!ENFLYdU2gNkzI$dAQ@|L9z`K?t_;@eNS<2(DNMbAfH^1So*?2}+B=kX#M>=RkIr@)!n@On5P8~#%`Asb$h!w%K&oGkfv z0n29Rxg7Z>a^(9@XqaqvZq9*gIs6dExX5N_J;n+30sC<%hx{$*KiTX&nWOv%fM@g1 ze;~tbc8nbL`V9QFlSgv&lM{24 zhvrEa{_V%TIqYnOV77b@<*g_wj;6ix}bF28pZnu^N6O zd}yV_J0;Q1!T2KFb%g|;;f#k^Ub|A_pMxA7g=pWiwb@z)ug7XVtK@QOoG*tF+80nx zk}qO;m$0H~GuVFUatS0k-*0pI2f4n=&QH0%A;wQG zbM#1OSYI1!54Xl{h-$5|HIY;^43>7pOFiz=GD4bLDOIMeiyG1P4O)khG~%s9TBGfn ze%11NW1$W+ogR;;Y)!Z$7D1VkvG#R=NWJ0lL|Vd0-AIOGMn@pxs}JMDSE<+Okz_P% zM13pU+O)Q4TO_fGOkFm4TjeNxvoF&2Us4ZU7C||Vx-Q{FB8ozg6&Pu54QhYAJ$Jgq~aQnKbzP8PPg&U-ZTZzEQ>x>p1HsVRJ z(d^V>>r&xlvmTCIpNb{XzE>^xltm(uRy~o7w#LKF^CGD=Wo0$tNDJDbHo74eiB`3? z#v@@Po~$YJtzW;sZO!DgYRh~ZIBzstXUwn^1i41T9Vm#oaBJd8NUdt$hz8PEMmth% zQH^?6U6_gjS);W>N2k)m&CSWlmBv$UsZ`V#3xzhcg*={`Rp@)sa2vXy*4)~$sZHd# zBpIEglRR#j0cpZGdVQoN+N^JiMO&L;IF9xPiL}P4(pNRCo`nB>ldS6eYIy6q+Gs~4 z8B3u3Jf3y1i1&>U?yV#cNb(!S7=HK;xq3w4@U2=Y~6YKbP%7kwej zKJ!%OdV9P%DkSWsiq#wCiEu0#xGXlA0=gUGM!2 zG0Iz`9UXy~C>ekwlnfG+YrL@XMKZj5zA^+>!OBPkWZ2)4H3f1AI`%BU&Z60PA#RL4AK z=&APD^{HrJ85-Rv^JFQjZqys|jZ~tQgz7zTdc9u{Ec4NUHd;8|8)EPf&4t`lWfT2G z)LXakYZEIPXnY~l@y@I#qie#g;dT^lZ9FM^9Cf$}0qSr%mc!`EdRO7;R&x~bNE>{f!=@;YNj}zo1bJHML?d3FiY6(xv2>0Jb#!Hu+3`*zx`fcL5pmKL zF{5QkYj|D91Y>115>Gb6a>Hb1R+?oR>L)SN2?eZg=xT`Z!fRq1JQzrNq&3{pp&OeL zQ8e?~IHdG-ot=6jn(T;EFllVkH@GQ!>@b=m=bujzrU*M3Z@#cK z)}HFTu(M*pg$v3FEYp_O)adg{-P+P%pt?pcE1frfR#rNH{M=Kn)vT(mTG=3x#zyFr zmzJ>_Cp{jmrm?ZAI-q-$871g3>rs$B1%LUtHdlb_decOz0O!+jRsggR|I=`F6Wbu1 zku3d9g9b^{AL$i}v*RH_b_%d>!L~q~fzJYAf%3`wJN~59!`M5heP-y~P1vl>&<+4o zR-QHSKN*Y7)R5mhCjL4ID*e={G3*=*wGYSnQ?w^p-wnCjzVbCx?Ktf? zPg`ZOXWW{`PGzb#8}fAIp@!dKdz4Nb`Z{(v$7`pty|746*UsbgufKmCc4P%wC8w_! z{CusJ)5hGdVn;Vct4CTKyg%b@o)#*XaG?!vV%%xNos9c!xOTU+6SCpk7;m)U`x$St z;f>#sc3N!scE%Gn+;xwX-*+e1LlRA;hi$lbkrX&&!@C&Q?y}nHW4zFY4>0bu;X{lU z+3-=uT{hgw7i~*yxSMge4G%G1VZ#%Qdu@0><9-`H#CXVtYl~(58*R9Y@g^G{V!XwM zcQKx@;RB3!+VEki;aGQOTk5I# zL96Dp4fitcwBcQh7uoPZ#$7gC+>k&PB{tk!BlC6J@C4%(HoTv4uMHn&+;78OwbFjb zhBqCdT`1 zcpu|CZ1@o412){bROY+WhPxRbwBaGf_u23S<3l#QkMaFBe2DR38$QZ-#)donvYewf z+|BqQ8y;f3@NTI$-v95CX{QY@WW30RI~jM`@FK=bY`BYYw+%00yuyaN8TZ=o3da35 z+{<{#hWigTcdY$zKjVcqe3)^k z4bL!MWWz@pciHemjF;GOjmL@Gh8HqkVZ)t_du@0T<9-|NV!X+QmoVO9!`+N0YLHoS}RAsfDp@%=Wu zkMUs}zMb)m4ew`s)Q0b1{E!VFU|hS$+CO(PUTDJy8F$+7eTfpEwufD#kVp}fF{z9r?SftYc(NuF;_2|;x~jqK0D zKkX6e@S5anB_uw#vwbC}SUXJl(la3H7%<^fCUxvI;WY{b9W>!fO!z(%ZhpXHzX?CX zBtLAzy(TG0=`elM}`d3F| zzdR$1w`2lnGU4N~2skY!T-`Gzg@g&m1&Z;b(}eRS3&HC$;p)-=F}9g-dZtJnef{!` z$~i-UpxgUpg7BI0RDAaL%LL&Lc`814^veX{<~}`O!p+ZP>@?x(o;X<;G~u5lfRBCs z@`UU!W{k*Czf2H5Tb_!~{rxgQ_{s8Ad=B@^1mUO1Q}LPUmkGj)2siVk*pQ%7lmkB@1gqN6bmkD>9@G28t zVZzTg;a(GdjtTdh@K2lYkO?m~;f*GIjtOrv;pdw078Cv%6P_^PB__PngwHkMT_*f| z6TZ!a(=#9H=riG^3IyG5!ar-m`%Sppgzqrn9uq!b!pltfP804k;e#f;+JqlFJ?4R9 z9ysQKV;=aw!vpU)&V0|m`OQLqPr*xZP4jOVG^UIV`8V$_+$9P)HveV7V{=}`r{gRy z&M4hNypiEC9CMzdbRMOLMEYq;Q%E~9DAG?*nnKx;0g--`(iFmu^o#UElrErjpGe6+L1_xLMg~OsQA$&YHPSEA4^f&z ztC2pDzMs+*QjK(p^mi#uq0~r1r0=3Mg-|0+BK-|YQ|L4j66vo}nzmpgUXlI+r746O zaf|dVl%~*W#3j-fkEpsKjkZH!p zo`N}$%exkNkLOSNBOAfw@~o=5qH1;3s;Y*1%rc!Q-9O?gMCMj_mdby!KfTv;@F5zl znfrll8FYMW@W2$$;I@M2uSd4Mi&G?i1&g0!65lMuKj*+y)p>Q*HC5ME{X><0?JlUS zB9&sP;^^+eR1`?>4fYzv6~VMoTml=Jd)`O>>0SPGZE;}`4%+;_GwJg8-Uu$_3O!F} zW~1*DyTsoUEH3n?gT)p8EzcXLLo_c{Smk)Yv90jHEZ7RB;o|2q0^r>_CNx3*YpDEHJB7is>@Hx|1!M|T_(XE1$Zae~U%tm zZ4aCXzXsE<1s63JyB(V^gs3Ru!_;~G=|5yb$TN`M?caO|SqF>VK}YRN{`4>XbN}SH zWmp(~#Xt9eKfe{VsC>cisD6PQ5TfGvRjtnqrpKtxe$<)90~J3+*8EiGCaUv!qRuBf z9;nk$=hN5HQR+CW8>OIf?DePrL>?$3pTYxBtSxRN6Cv`zPiX2xCc*Tdki{M3CX_d| zlti7>2j=>Jg6=}crT)r&j_ylP@`vy*DqHC9o&N*uCo^~djjU7Ydm3OJZYs?50Sk=n z_VB8qgUgm#O1q-OHdKYh(g}ISP`VT0~o0NO+;s*g_h7XDJH1f$bppbBO z7kFu87}yD0;=0JjdIWXnGY~ph(8OLY*FmWKCF*qLs!e6-;zWS_a zWHAbBG2SZorZAjd%yhz3=99!j;yNTu{g0-XdVl(zK>DMu#rI-*%1fPuqWt?>6vYM6 zE>TgsVk~%D14Vw>B?+m8j z_NRaAgO3|gMF?s_l`6gv;+eNmKzR5r^o`%*il=|?Ued}O5alkI0TD>fB*|^>!fiB1 z0(g^V)CZ;NS165{{YZ_ogIwIR_(aqZnLR{nLGMZLqp32#gk>1{IvG8Oa#=*Vh{oAW z*hdE``C{f=WGLr-cn=f8`(2B#gPFWk2--JCA^9rhfx&+%J215%*gJrpp;hgfMjk4M zIry>z@|DmO9->*`N#qIciP;b9^shXFo5u<^Uf_@H#pLkUFiiH$ zc*MWuX)KsnzEGi_saw-ic8VEo%RvMEe%7uD3#L=C4x=x>pZU!UvQp4Q4GhmrL&jH+ z4PQ;`tl&Q=`Tkt|uS>o=7k{JV7g_n9!K-)m3Hw(_zLtx>Q1UbAD5mmF=$|6_LstGo z`QJT3)NiMizY}xz*wq$?cx8R}O8LIOSN`i#zUA+b4`tR%;TjgszjAC?l;ldd+pDdj z;qgxT!%U47D`v5o>9;U>i2i)-*s!G3KI8JAl=2@?S3?^fQT`PKL1^=5{$BYXO8NW# zUir;ZzWeW$ub1-I96?^rA2j`D&XyWwtl^OJs~YmsN0<>N`ab#Br2i5&aqz%<5$Ul> z>)$E$A3a+Au+;y;(dyGzDN+8FN2~u57Gdb0dbIj?O8uAD9HsxlQvbV0tADoCPaduQ zOW5S2{udps{+&{P%F*hFrT!B%g=54{y8oRm^}l+w`Y%lr^}p(9_3xDW=NzqmSnB_w z>1gXO_3uAg{g(3>D)KRPLWS%0b7$C~W#iTHM*% zKP9EkV=4Q1&t88~{;x~TLsv`B9kzeiDEY&=_*Y2&zFd54Z)v3-$i+WJ^0#Nqx)G1I=(mRd0cFdMjbzxc`;a6e9u<>AKadT&nWqOAfI`B1iQdDu(`UX z1gS+x(Q@tncQ~CcM?Fk5^CO1Wh~4k}B8bw3zia;>q*K!lT2@`yI9W zV8-w8?URPKJZ_weJZ2!M8GFj}$1Q`YQ<(9*F_U(JpU8}j2_*eihA!8mq&qPlw5?Xyox3Wtl zJD}JXTiJ6ZyG*g?SlQ=E_GyYe%gR1qvZpHcyVN-*N`8T4{}HID*sztoShAm0?8mL_ z<&yn##lGLl)+Kv~V&86MH%aza6uaBXzFx9#Q0z7<`xeQLDE4YA`yVAcsMwcU*>_9! ze8n!YvVSbuXDarIR`wprp03#Mt+qJz4apt_DxCU?m0kFb=uE#=>?f`4xsv^;V*l96 zUMAW9s@VT*WnU-Rw<-2kEBlL*eUoA*tn7c2?6r!0rIr0l$zGw@)mHXfl3k(L=Uds% zzleIDt=O}zY>#9guh<{1vN(06WWNJcIQ4ZayF;>nr`W%;vTu{@#}s>~mHlJM{+?pr zX=Ohv*he_Gk^N%nsz_QO{8X> z*y{IwxIxd5>{}IklU4gn$&M@b8Y}xO$!=8a%dBj#WLGJ6xs_cd+2<+t8CG_+WPehz zr&-xGlKsI-cIsO=VW*ym1-e$UUju67?ZOKE8|IsK0~F7D9^X8c5Bd8-SmO~hY#z%) z>PsT!dt_IihIcneWIllj_0tT_ zFXEI+VL;kSKViK;ckmJDzfc<-b88=^dbe%|*(*K; zd+*kMqBtP3Ly`d^d$;Z+6bDRPDoqShs(0%?kiFs)CnPyU6bDS~mt>g8-mMuBI>Zjc zdOeZiUsx_6%hLZ{pKsLUeI(Zl==!}s9fYR8=Z9oHbJMVZL?wED*iX=Uf_gd<{>{6Z zD9&5|d%7x3!DAQLL_3)#n6`kbc(rE*Zm%!J9c2HaC54W!KI$16yi4b3|z+_ z1D#6L->VnmF@mcvgOd!p2b7tMO5sNTmgi`fgnQA>z~+ZVr*L%BuWtovQb$oM= z-%_l$ajGfX@d>UOoO1NIw}6R}Fq%fqXIzFo~^M zu(#6{=$TsFqor$HVy3JqL6^nD0=T!HalimQ;DEXDyg={r!hyWHEwADZe&tifaaBFj z7gf7xWIb>QcVaS;-%U!1{)m8q*(R~or zqfvJYvZ!ovIeJF{_|uR0d%Q*1VFnz&J$O*#&vog)j9g3oaP#L&G-El`T~&A_!d11% zr`>d^XK=&|rsGTh2<5uUjV?#`ZEW`jRACX>eH;M16X;pOc6Zk~YF{5I6rFpic(&rR zRXuf@<#~-jdQVTt{Z$x;6pt$gaHlj;_2EHX-|^t+J@1W6xV>IuIi zh*H-bq*C9De0`o*p)&GKoO%Wc{HiG3D4lMla`ngpoQXE>@l$@kKz?}UW0%Yi4O97x zn=TJlzLKgMabTKI3KxpyYhup(Sp5?Ni}WJn0&39NR9YMgcHw{f3uh>EIeX*KE;n>Vvd8;F$KdZgODl50~9nC9us5X z_x>JMP4D+t9xzU$wFv9BlHNuSWTi9oz|8)i`=O4f*)k_Riz%uEg>sajoQ@N{`1gy) z&fNaA-|Kh)k3?af$0UkpRx(SUp}Fza43z01sG>mc=b8e&?Ja?xp?v3wd^6_t>vFYzVag2t8QDFa8lk-KEYM?= z(b=YqUaPid^ry`GzeYxRjc)W4vHnw%)_>swvHp9W7wwJ_u?O0`1>Ll}3O2 zv>1%FyS6;<@a>}OqFZL7t7jeo&O`SMQBWrh-2=#iqNj&oOAJMzm)umsvws)ZS^Y-EUmKeS_?#1vOOTAcE z`I@8qRG3k5nxA^o0bMqXP-;)bdx z4jSrwpd6?L(!a+7;pn~(RT!TqzJ&zl1Xt$%r-ZL>2RXb+QlAs^#xG$M?XfYt%vAvn zsKgjRKy@jm&z>0=2bTVrEuNCy-_bo28Y-U3!}-Wr`0#n~?BU{=7bTVPR47w`f?yI7 z9nZ041nx8SgAA&SQa&}5ttO5wv?-(x;}(zVWxgw=znw$6S2T^7@ZY$UN+5d8kC8w1 znr^ZspZ^rI_`{~j%==?8l`RkUHe+5Hf4&njT8k@~egiR?ZT^yL`R_502k_+Mr+D5{ zQnZf#?m#}cV%Ey6K_h(3l!ZC#Lou1W98CW$^H0ATpRweW(Ypb&)#?O=f@PwKMuji? zJYzE6-OKUrdzdc}@A@l84To%)8>b&QPsj!~?`bkk9pAk~_DnfPQOq-lOlj&sDZWn(RzyWa)f(R2oxrAnik(G*+8FC$~M8dE*yy z#(NCADj)V)jxFVw@KA>tRP~V)$H&l|#mLmLr4|N8mXehoAF{wif^}NMGVP>gTCC3K zp<=$D@k=?4yb2l^`?6f}nu|;7dN*Rpi=z`=A(y)w4qCk3>`cWsU7o6MVrf0m#0Y=J{>wM0Y=gXtXkAAC*R#%l@A@ZZ= zlS};h{*u6=6(Ps1@4#;~4Pm);qSyQ$8k$i^z2<*G7`?L)an_8~$-eF}qk*QP3YxIJ znG3-^u%4Q7M&=Q?5RG-wllI1<03XtYnX@1bi6WasDt6#rTB@ADF-ZRQAJTwa->}4uMbEeUDNYO3^*%^LF3_{J zAA~}*%YxLsMLgETy#>9RVm4bmhK)_2@=s{DALhwm4AQT&D1B!jbD#hsKg{D`Yu9`sAmGqyY}8p@~?eFLo#qHk@$4gNlg zzm}=^>(OOB4UdWmE<+O>LV4n~< zy2NF)#5yj#qysPfD4A>HYF1nF9W>gce%YNmrxX9j;opIOT;vyNqIsv~FQRt`(xX%E zhsVbDSk0>2OR0?>X?FX$j>6X-B# z7pR5}Mn9+vbP&`FItCP3dLKBx;{8yf{}0CnR^-!{+$=+8h0Ku1A`L7n)D zoeP(OYeBuB&7cX;Z-VxNJ_9-oS~wGOxXyers2B7a&?e9uLAyY|3EB_(GtgnsQBVyx z7o4-82kHj(f`&kwKog){pnaeT+=kgn{1?Z@o+Li#Fz7Izpm-Afrs*ZL3p!vIs0*|o zv>bE@Gz98=8RY;Cf$jiJyo&sx=YDN$>@UQB1MP{9-1ippqYjN@wj^5K4J)<0PG{a} zGo}~z;mS}k@YC@h`YFmteQ}m^$*iJfjuSQ(c4?QMy6C)l#pjSJ>DS_aJNiDGBv_0O zJZP=C(f1b!L}Ce?UjqzTh=cOz#(yKQGYF}g<=i}_aaLjVEN9iMLQ#ghkHFu31pcec zKMT}szlQDOTlFIUT>RoA@ab~e$CbaC`NCnQ`Yqx7yTNZk{d4i}2EX%T_`AX120rbFvOkF3N0_ey*P7s#WJgrwH>{H*8KuCkUOcN$PY# z=f+9u{0KVakI9rR107?MI;W%GC!kYCeiQvs`t3_o!opY8Ccoia_$>q-s?TI}x}ihu zHyNFs(7A1rI&VU!f08<`shW1*By}2~vvZO<+n}?1k~%+w4y~<|@!KeLem6;-Vyuao zN$NC0=P#4g`66^Q{K~;(>N5zP8I#oc3v`MmsZ)aWzIc*4*FeWTNu56ETs%pg-O%w* zQs)qK8YZc80b-dZ=uBqbX@br*lhnBlI*pSo+dk+tPg3Ut=p-hoInZgZX5i@~oy3xi(Mz5?1jwvx;Q2gjF1CJ4a3^Cq###_ay92 zMvr8R5I^spL>)*c2%Vux=uo}8pz}NE+{BY9O;K38^nH#{6q&~w3liGrhX5(2v{eu zW?-KnAu)!6B3`BZZlnCb6Z;1A=83-kY=e;f;@=_bhy0t6?T4%ju^^8j zjF{?KCDpUsRkJE;XEkk}{>5qd_QBLIV^rm9Q)X3E&2kGqqW->cZ^Ch7jkIy=P$W&$ z?W_Syo>Q*Kp5*|vwF>-q;@|l3*w{q?raJIp>jyWfegLnh)H!BE()|(XKY%e!x^EyM zW<)Vx=6`7lHkUZ1MFZk1)%lWHF3kneQT)%qbqUIU>d)~3(alq)tR`vd%U=A?hF$8P zJ8?esL43%v$@WK&**xuw1-+P=zdU8iChA<=c|?wGh|_@`@PdEMJMqKm+RyS{gS|WR z9o>c6K>n>be_-l)w-#!@n^yFVLhUcpX53P!^-Vwj_37GAr+))nq3hIsfzw~)o&W7A z+M&E^#`Xf+n|J;lQ?$1QcKfIDoJEC=d8Gx3ysCnhyyb;WdBOtZZ=aiYrZ!L0=FRHK z%d5Id({9fz=+D!>omViBm$xGib!AD)@z`<91IIjY%mc?faLfb8JaEhdxjmrXFFXD| zS+TLoEuWKfx8!VNx}Ry|cO<@@sp}q@R>2B?+kQNFS;rYDKD}#~4we3qEluwZHuLFS zzveW(JD3h^CFH>Y9YSt#IM+)0DNbv5$dl7JO)(@LYA>YTx2xi8+SkzW*N?{HL@Q{o zf&-WA#LA6f7FkQlQHPf|BH!!`G>ApLq9rfwt6ZJmnB2KGurQRpqM^A?1 z7+^ZYl%6rC!+Ei!Zl)oo38w1(%|o14FST+lmU`;_%UzsS?^hn?w0eKCm%i74ql@Vv zQ}vQE@1@LVI>=P>O5Dq|i|HU!_5NY?zG3x#;qmtgSF=7-^?qRWK47he`ApUOefv2* z%+ysY<(ioGGaY8C-nXmXudCjttKOgMTq5<|Ox62v`#3$s)VY-9n1+}pnD#LpVmiuH zy>C{%Usk z8`D0f+nM$=-NAH#somaA#s``1V>-liKht5R8K$F54>8rg&HiEPWLm`3#nhgE3FB_2 z6->QM{Y*nl?fQ+3H!*Etnqb<=w2SFBrhQDeGwo-(gXsX%olFOr?qg~%#}MQDnGQ3} zFdb!jh^bwFX-&;Vu98)2Qtd{{RashI>b}sE5-HE;%G{;y^3t-o64xTF;r4Z|4bfx= zp2Tu_7L>Y6)e{+Zt1_LB=t#vKW(iyo)l|G;#q%{4mss(snp)qjc!8$k0xLdEQ}Klr zpRTDm#EK&d;dRf7AE&vp;>T-!JilAzPtf|a;xjZA&spUW?Ip~JJkQ4)K1Z+5hC8$Y zBOC72O1iV*Cu%BgG;{N{PvpQqsi`>9DnDB*$`X&{YbR;h;*@;65mCK=)N1DxTsSu) z@;o2Wo_(DpPCnj<*m6%cc@_0e@F)(Ss{MIv0+DAFUr#|4Yi}0{HwhoIwb$816+? z@;u9U;$D){G!B zO+2pEJa9A1t9^qi=RXVl@O6rw=cV$f{lqyk8*Pwr6^E;GzQA~h{h<8+A>(R)qU@YH zP1;xSxaudDFs@Zu{S#qa?GseKn;BQ}zryck+`UBFQOo(`jH~^TD$i?-J9*!v{4-^` zEN8-2uQLUHc>GYz@@n6r%0tgL3xC#H>$L`WUY7OrI)N7*whnG&J8B=PBC+j^t9^v> z|07DC_iw5^FEKt)FUzB@_Z(te?WN7<@eS|d4*rZxY{2o`&R;|emHcelo$7s zaY}CrBs*#!s_blKyl<(LSN-AN7}vZKSN7>y9kQeL$Ev=sD4gvm{CHeoBKcu1zp`__ z!WT>XsyxdWSNmaw#~2^r{kXDE&zX~bwNF(2?K_OC{iW(}k1?+Ho2tGC7!PqjRQ3yS zfsX8}>jYvsLyl#@+0HW&d4ezuGE)3OXLyQP&}qpBFIR$KzM|=UU(tCnWBXcF^47xDog# z_}onXBL|+zfiFhjn9a^=fkWvcX`P-8phE|KHr{0Z49D%NT{h*g^L60opuSpz)K}xz zGn4Sck2>Jl$`cmyCupuKq|P&(@tFv4v&jd5Q@%~Br93@*KnFedpH2Q7Iq;o1@cqEa zzL)Fenj-C=;>>QpCI{XHd?Lz}Ht6$y;AB6<^OcH2rb9Se`OnIMm*>Ej0ng_D)*SMm z7xHMwD`gqQvo|<>Du?_5mQQf`)lTM86y7CGSUwwoXRB9R4*cdE_-#4xr+||`)$hq7 zDUSS4X3w`O2fiu?em8KkKeSTX?PUAUGTs%IfLa%3&(7ZNCBRkL*n}XLrp=KbQj_%7MQs?3+s{&(F&tKNa^ivej#04tzQAv+-@2iaVrT34extw%8*O@|HQpZW&~&{yuCHs2uL-y6%|<-gp@&nQS|r|Mrj!mz!y-EICD1T3)f35RYdqXs8cAvT zRYA`}!Ddh>Zye0?fG-2oleQR~W0#O(@tj3z7dL*?*WjvoUPDbIfHj(`XU)kDrnQ3;` zVWt-Y?gFZ zbTqV}rp;NDq{-&6FnYz|MrEn$H?>Drbwra@k?T{jWVGH0CyiXPE2ACpR5B8+jp94E z@l94SU0=FbCfd0_Os-l-HWM6B-+uD%Ni;Z4Yy@6CM#1PrimhH<6~0k zi0du*_IGPkcvcG6McY=ap|4Wg#3%Y)4Kg#bc~}?#Yh66KDT`;M{hD}N8}ht76;0)| z(VDB@Difk{vJ|jBYS1^rm!#UoSHU|P;>+XlM2C(JUVBApGP<@@*VlD+>IvMSh_~Z= zB84U*)%Ub20OoI%p^vUxzg;i+NeTY?G#5d*$+~ zxCS4LZ-;SNk3)*_C3SB-01^>i%cm=N&m z1+A?*miD$-JBG&FUQCFU6 zvlyWk$O2n!=Nt^G!XH1WrE<>mQa`6Zsc>1mC<|xq+c|5Hdj*n4)T)-tdqO|!+JN3D zW-!w)S*(_~QfUUG=Ck^c*VrZDSZma_)aXltE2^u4`idn>>U|A*1N{hy3@RoDVeXk^ zd=bdvDH^9M5WlG+Nq>L*%qLxu#iK&Gm4|ee%!sc-k}T@V;WA=IkHOHiS(%tttiKSV zuWe5lNkn+{Mp>o4a*d8PG>W+fp$pt65DV4%9Du~DB_S}x30j*svScINVWV<9Gk&Q6cVQ)d3LkU#`^cp@!fg#F2I%;*S2 zeDz^`_$u`}UH*`WFLYU?&0?{%%p-o%1XEZr++p~V$#~N1rJs0N*&rC=2TrJpp2|cl z8i{U~Wzd#36-ipvB5Tp7M;HyB5qe(c} zM~Fe))x6qL-DHmCn~twbp3675ks62|8AtPTLUfRxPo1Lr=BGu1`hj zw`EZGvWe=6X3=BlJhV{~6$=f`U&hfK2C_$zipL6%FjM z@s2w&koC3L#fqMlV?-Z5oW%fHmkKAF_3#?(m#hQC#KqnikwMhTziKiAWgZRIs@7Ke zVWeoYk2ce&?s&s@Y{Er>HrenoOS6}Huy>>}K2haZdn6g9A9Ko4QHws58&)sJz@&*O zye7861NT$Jf(?hUDM6Fs+PLq(P_y~f*!^FJ1wWFECD6A$o_hRx5~9^aI2rY!iWhU!8`?r1Pt7XKJlKWl9co@hl+_tSk1-c}a(Nx* zICscM9_BavP8C=Cy67ey4NtKp`A0aD6y45L(&>JL-;45vFw_pb%3JG}PguuDURd9W-5* z7TaV^U)8=5Lq^A@C)GxCG_KFiD|OSPI{DE@;gX!cd@Hk+!?fOp<>f_-*PB~AHmTB^ zmI*Np)`)5GikQ)|q&2**W4yP}JjuTZh)Y9l@pjDLbj1=;1l2lwovnfKB-+hmpV58w z^YnV7QV%p^vBuS?0MfqDk~LTq!B~j!7(p?hBLu~e(6mz29@oS{F(U}tqR!T}r%GGG z9W7dE^QLx)Ns8@HDRKC5T}wg-4>3)c6!42$xH+hP$|i5rwX5RsX% z5WzJ?_)pzV`LEK`VBjFWBgD6*$DdDD&ne%>87RJbu0Z8G2Pe4Oi)R(&p`K$_lz!Kc z4m_TQ=MyHLzbqmU2i-R#p3+y(J1eTbcSW*v@a-s>z~_AY(|4eh{wVG%(xK>=WOCfT zl2f!AeELq6N~`Cm6?JlkR7R?bz5aCD6pyZr+gHz7E2_T7rR*zxRsXA5pUOdpdLCO* z^*u7u$1Sz-{FQzb7!{G-Jn`JN8z2s9U-Lm_H6Kk#nDy23-HM9eC^RXk`>TpxXVX{D zfh(%ylnGUTMX4j3^H-C)yK&y_2R$3ZNIz5ZR`QvRxa#-Be=NVQR7 znhsTN1#QELSzkS;t|At8o$bhJ^%Z` zAe;G$!S~U%Ua4WLzQ{}2BDxd*^z1l|4{_M^?d_$?ujp5EvuNhvZ_dJ6tMPH5~Xef9i)C+oYo{S;s6t9r$uYtCOi=fC~SGKUsP z#y^$5qOaTZ$Dd2rt7?w<*|?+W!F^BB$~I diff --git a/qwen3_6_scripts/patch_xformers_sdpa_seq.py b/qwen3_6_scripts/patch_xformers_sdpa_seq.py index bfd31a0a..186b5f5f 100644 --- a/qwen3_6_scripts/patch_xformers_sdpa_seq.py +++ b/qwen3_6_scripts/patch_xformers_sdpa_seq.py @@ -165,26 +165,6 @@ _MM_PREFIX_NEW_BLOCK = """\ """ FALLBACK_METHOD = ''' - # --- flash_attn_varlen_func backend (loaded once) --- - # Import path: ixformer.contrib.vllm_flash_attn (canonical, matches - # ex_engine/python/corex_fa2.py Tier 1 and ixformer_sdk). - # Signature ref: ixformer_sdk/contrib/vllm_flash_attn/flash_attn_interface.py - _flash_varlen_func = None - _flash_varlen_checked = False - - @classmethod - def _get_flash_varlen(cls): - if not cls._flash_varlen_checked: - cls._flash_varlen_checked = True - try: - from ixformer.contrib.vllm_flash_attn import ( - flash_attn_varlen_func as _fn, - ) - cls._flash_varlen_func = _fn - except ImportError: - pass - return cls._flash_varlen_func - def _run_sdpa_fallback( self, query: torch.Tensor, @@ -192,17 +172,18 @@ FALLBACK_METHOD = ''' value: torch.Tensor, attn_metadata: "XFormersMetadata", ) -> torch.Tensor: - """Prefill attention fallback for head_dim > 128. + """纯数学 causal attention fallback,带 Q-tiling 内存优化。 - Dispatch priority (ref: ex_engine/python/corex_fa2.py): - 1. ixformer flash_attn_varlen_func — fused kernel, O(L) memory - 2. Pure-math Q-tiling fallback — safe for profiling / any HW + 调用时机:kv_cache.numel()==0(profiling 阶段)。 + 此路径无 KV 缓存前缀,KV 长度 == query 长度。 - Profiling guard: when kv_cache is empty (profiling stage), vllm feeds - a dummy sequence up to max_model_len (131K). flash_attn temp buffers - at that length can exceed GPU memory. We use Q-tiling for profiling - (safe, correct, O(chunk × L) memory) and flash_attn for real - inference (fast, O(L) memory, verified on BI-V100 head_dim=256). + 内存优化(Q-tiling,与 Flash Attention 同思路): + 将 Q 分成 _Q_CHUNK 大小的子块逐块计算,每块峰值内存 + O(_Q_CHUNK × q_len) 而非 O(q_len²)。 + profiling 阶段序列可能达到 max_model_len(如 20K tokens), + 不加 Q-tiling 会产生 9.6 GB 矩阵直接 OOM。 + + softmax 在 float32 下计算以防止 float16 溢出,结果转回原始 dtype。 Args: query : [1, total_query_tokens, num_heads, head_dim] @@ -211,50 +192,15 @@ FALLBACK_METHOD = ''' Returns: [1, total_query_tokens, num_heads, head_dim] """ + _Q_CHUNK = 256 # 与 _forward_prefix_pytorch 的 _ATTN_Q_CHUNK 保持一致 + assert attn_metadata.seq_lens is not None orig_dtype = query.dtype num_seqs = len(attn_metadata.seq_lens) - max_seqlen = max(attn_metadata.seq_lens) - - # Detect profiling: attn_metadata.num_prefill_tokens == total tokens - # AND no actual KV cache allocated yet (first forward pass). - # Also guard against very long dummy sequences (profiling uses - # max_model_len which can be 131K) where flash_attn would OOM. - _FLASH_SAFE_SEQLEN = 32768 # flash_attn temp buffers safe below this - is_profiling = (max_seqlen > _FLASH_SAFE_SEQLEN - and not hasattr(attn_metadata, '_has_real_kv_cache')) - - # --- Path 1: flash_attn_varlen_func (real inference) --- - fn = self._get_flash_varlen() - if fn is not None and not is_profiling: - try: - q_flat = query.squeeze(0) # [T, H, D] - k_flat = key.squeeze(0) # [T, Hkv, D] - v_flat = value.squeeze(0) - - cu_seqlens = torch.zeros( - num_seqs + 1, dtype=torch.int32, device=query.device) - for i, sl in enumerate(attn_metadata.seq_lens): - cu_seqlens[i + 1] = cu_seqlens[i] + sl - - out = fn( - q=q_flat.to(torch.float16), - k=k_flat.to(torch.float16), - v=v_flat.to(torch.float16), - cu_seqlens_q=cu_seqlens, - cu_seqlens_k=cu_seqlens, - max_seqlen_q=max_seqlen, - max_seqlen_k=max_seqlen, - softmax_scale=self.scale, - causal=True, - ) - return out.to(orig_dtype).unsqueeze(0) - except Exception: - pass # fall through to Q-tiling - - # --- Path 2: Q-tiling (profiling or flash_attn unavailable) --- - _Q_CHUNK = 256 + # 推导每条序列的实际 query 长度。 + # 正常 prefill 时 q_len == seq_len;如果将来遇到 chunked 场景, + # query_start_loc 记录的是真实 query token 数(非全序列长度)。 if (attn_metadata.query_start_loc is not None and len(attn_metadata.query_start_loc) == num_seqs + 1): q_lens = [ @@ -265,8 +211,8 @@ FALLBACK_METHOD = ''' else: q_lens = list(attn_metadata.seq_lens) - q_flat = query.squeeze(0) - k_flat = key.squeeze(0) + q_flat = query.squeeze(0) # [T, H, D] + k_flat = key.squeeze(0) # [T, Hkv, D] v_flat = value.squeeze(0) output = torch.empty_like(q_flat) @@ -274,37 +220,44 @@ FALLBACK_METHOD = ''' for q_len in q_lens: seq_end = seq_start + q_len - k_s = k_flat[seq_start:seq_end].permute(1, 0, 2).float() - v_s = v_flat[seq_start:seq_end].permute(1, 0, 2).float() + # 当前序列的完整 K/V(此路径无前缀,KV == Q) + k_s = k_flat[seq_start:seq_end].permute(1, 0, 2).float() # [Hkv, q_len, D] + v_s = v_flat[seq_start:seq_end].permute(1, 0, 2).float() # [Hkv, q_len, D] + # GQA:展开 KV heads 至与 query heads 一致 if k_s.shape[0] != self.num_heads: n = self.num_heads // k_s.shape[0] k_s = k_s.repeat_interleave(n, dim=0).contiguous() v_s = v_s.repeat_interleave(n, dim=0).contiguous() + # k_pos 用于因果掩码 k_pos = torch.arange(q_len, device=query.device) + # Q-tiling:分块处理 query,峰值内存 O(_Q_CHUNK × q_len) for qc_start in range(0, q_len, _Q_CHUNK): qc_end = min(qc_start + _Q_CHUNK, q_len) + # [H, qc, D] q_c = q_flat[seq_start + qc_start:seq_start + qc_end] \ .permute(1, 0, 2).float() + # [H, qc, q_len] attn_w = torch.matmul(q_c, k_s.transpose(-2, -1)) * self.scale + # 因果掩码:q_c 里位置 j 只能看 k_pos <= j(相对位置) qc_q_pos = torch.arange(qc_start, qc_end, device=query.device) mask = k_pos.unsqueeze(0) > qc_q_pos.unsqueeze(1) attn_w = attn_w.masked_fill(mask.unsqueeze(0), float("-inf")) attn_w = torch.softmax(attn_w, dim=-1) - out_c = torch.matmul(attn_w, v_s).to(orig_dtype) + out_c = torch.matmul(attn_w, v_s).to(orig_dtype) # [H, qc, D] output[seq_start + qc_start:seq_start + qc_end] = ( out_c.permute(1, 0, 2)) seq_start = seq_end - return output.unsqueeze(0) + return output.unsqueeze(0) # [1, T, H, D] '''