From b63ba28c9ef01d182471f7783b7047221c07dcb5 Mon Sep 17 00:00:00 2001 From: feixiao15 <467534404@qq.com> Date: Tue, 3 Mar 2026 15:46:50 +0800 Subject: [PATCH 1/3] add some loss functions --- .../__pycache__/__init__.cpython-310.pyc | Bin 21989 -> 0 bytes .../forward_extractor.cpython-310.pyc | Bin 8033 -> 0 bytes .../__pycache__/graph_tracker.cpython-310.pyc | Bin 3696 -> 0 bytes .../ms_autograd_function.cpython-310.pyc | Bin 14144 -> 0 bytes src/torch4ms/autograd/ms_autograd_function.py | 169 +++++++++++++----- .../ops/__pycache__/mappings.cpython-310.pyc | Bin 2846 -> 0 bytes .../ops/__pycache__/maten.cpython-310.pyc | Bin 46747 -> 0 bytes .../ops/__pycache__/mtorch.cpython-310.pyc | Bin 18827 -> 0 bytes .../ops/__pycache__/op_base.cpython-310.pyc | Bin 3924 -> 0 bytes .../__pycache__/ops_registry.cpython-310.pyc | Bin 1345 -> 0 bytes .../__pycache__/__init__.cpython-310.pyc | Bin 796 -> 0 bytes .../__pycache__/optimizer.cpython-310.pyc | Bin 17077 -> 0 bytes tests/torch4ms/test_loss_interception.py | 113 ++++++++++++ 13 files changed, 238 insertions(+), 44 deletions(-) delete mode 100644 src/torch4ms/autograd/__pycache__/__init__.cpython-310.pyc delete mode 100644 src/torch4ms/autograd/__pycache__/forward_extractor.cpython-310.pyc delete mode 100644 src/torch4ms/autograd/__pycache__/graph_tracker.cpython-310.pyc delete mode 100644 src/torch4ms/autograd/__pycache__/ms_autograd_function.cpython-310.pyc delete mode 100644 src/torch4ms/ops/__pycache__/mappings.cpython-310.pyc delete mode 100644 src/torch4ms/ops/__pycache__/maten.cpython-310.pyc delete mode 100644 src/torch4ms/ops/__pycache__/mtorch.cpython-310.pyc delete mode 100644 src/torch4ms/ops/__pycache__/op_base.cpython-310.pyc delete mode 100644 src/torch4ms/ops/__pycache__/ops_registry.cpython-310.pyc delete mode 100644 src/torch4ms/optim/__pycache__/__init__.cpython-310.pyc delete mode 100644 src/torch4ms/optim/__pycache__/optimizer.cpython-310.pyc create mode 100644 tests/torch4ms/test_loss_interception.py diff --git a/src/torch4ms/autograd/__pycache__/__init__.cpython-310.pyc b/src/torch4ms/autograd/__pycache__/__init__.cpython-310.pyc deleted file mode 100644 index e424a9528aee27dc72c5e75f11b4766fdb48ce04..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 21989 zcmdsf3y@sZdFFlfb9#DudPdUdfpif9)L677zzH4@F$jrZTLMeMgaNkc>3c`&(RBCd z+}j#W@9kJ1utqTg{9v)o!y{q{Y#ak+r|b|O_GX=Alg(CbdDoS_-pZECj4ZNMp|;4z zju)f-{{P&2yL&W}iK*;v?MQw4-t+j+fBtjMfBx6$Mt^@=!C&>(C#zRJpeR44m+rp{ z@bVFSoTqd}v6L~zQY~#t9aHhGPiZxMOs^SZMlCiLtHsCSwZvGWmK;mgQe&xFdMvHV z`h8;=)HkO3YuT{@&WqtWHgT8Czo~?CeTq>;gM&r|h-X zz=XC^v6G!=yk#eJ)L3WbtU%eMp0x(w7ocVcH>{boF0j_3=7yApf9p7 z9{^1@;S-i$j{0kvE=B8usB;P9ycIa7P>U%Id{d|eDh2Ns0!PAgLsy-p`psFJtu26g zDbL0RJI*lDfCUUe+bgUqL9vZYt97ZX@VzdioOK;2-b=L&i{3v3I6XChUpEzh4H*79 z>JU|Fwywr!F_ueFkmf%a`NS&g!`2W+P{2I?1~7_BtPi~hn*4fKj|8=gh7&n!)Vcg6dvxyL@(wCfeykL_(vqgw2?I!X$vAKz=&orb_u>ejL- z*KF6O7rim<;H&>8bbrgVEy@H48O!$BZmH6!O*dW6DgnRg{U{i@K0tOyL(G(gwf(Rw z$`xSi$A#@SMZKaC#=##w$MD&PkFy(zt9Xj59@Rv`Q})O=4R3>tA?7!zl@qwhgGo_S!a0ta!KA*ATN`44<==g^1QmM5Es<1q*1J7PuVq*r;K4rmQk{j5*n;b=%n#8gg+$8s8%-Ar*EeWW`8hP92d6Y%cG1?V5V^uXTAdO3F2M_krnGfWbCRAq zsnp|?S~xx*z%kJ_Jl!)UXxr+otFad=&8f0$o8`J?(q<_O+jRDqr)|e_D&p0 z+!#@ls<@2cC4!#)_>^7W=k7oE`XN$!7;aM$NlWw8;c%1EcFcLo1Z_}_HXxJWgWDSQ zL$+{(p_$-YPQZYO0719lWu{BqsJKh4Lk9)DdzEoXK@%T9i6HbBk8yRdBT5YsJUP5y z2;zrO-=Q6yG?G|maT;U`3Rom7viD-=s#p&%#8&jvp$C^dRY6I>cw0s7vj0!I8-hLMmkJbgYz7^up@gcq}t znAfiWcYQK#sRuOYXz-jmV2BgZvo21c?=<7kH=m*WlWIFfurT*2y!Vrob^^Hs*Ueh} z?tm8qQ_cmL4|+PUkZfzqCuxU6%mrEnWMT$Oo1 zr)tsMi}p^nCATaA6E^YeYJFd)mSfV+whvF+6*^(Co@>NP7$EOc>ao%g!gi(`b&J}{ zUN@QJP1h`&wW=dq%#Ql4kOu6owPNQZ=`>a|w@97%yJ%s4E4a7hd%ND>q;DDQ=JhO?Dai4jpR-P`l*6j?@ z1SB3;FN(g39ZWp3^=s6PSt-{Eezi6|RjpKA;8YPd=Gu}7k{IKFKb1v54QI#m5H$oi zL=}``w$|FxSqlL;yNX#5=3YJq!AOOU)pVJ6`)OW=oGplv`e~jFCghmxhcXuPx4u%Xe885_K7wY> z6cQtz2S3a6sc)zn{$*m?|I*^x&*KU0AGM_RvrJ0+sg~CMA>OC`Pc5VUy>Uj%qn4_j z(KPMdxUT&|GqiUWw=<`jYC1n$eSLMjKJ{8aO8Py2H;fw&78c!QR3)Ahf3U`x!a0 z!CRJV9x96}iK31luhwzgI)1;DUm@A>YZ?oH_H}8Z?#E?OYkbK^&qb_%Uh+{zH+sGYK)qHv3G`P@!}4X3R* zV2txqf}Z9iCbxO%$>ec~fr5jngTXjTQtgzmBZDOS=-ozNoa4|PF*((9WaIH|NVZu}U=1!b@A{-l?=5#s`!m08m z6X&r!@5M2C9V3sUygz7_;a2^!)p6~ZdQ5`~PO5jZo%OQlIn`DGBgQcP(u)B`I-vIV z0Sgoks2Fo|o(lY^f2}>>4&pc+0R96f^!9+K@vQx=r?)jXPbHoPNh&s{f;zbq2{Hsg zpcpI*h%RXpf7@8~08=K8>@;h{4r$fNQ-fZrhwOU9`DTiu=oYEQ^nP>7E+4X;BE#Wi zSHp!KaC6<<2{y0?Y67?k&XBq-c{%uc$iJpV;}BRSju%?Nd^&}_vVa$8Lkv}N&YOT# zq&RId!NB=ws@YbRPCf9s<`f`Vjryf7`1>Ks>f(RI-lzxckuHR<7yF!1?u2L&Y{^mc z78Yy6cjE3+L$kX`gFP2DWWZ5t!j#Bo2E^hMQl2dV$f#lW-Us0(%pK(^2SlUE08r3) zraCo6ynjM~h*7(4F2U{Sz}^%z!K6`p97NSa&?!NQ5C|Pmu4vMk7=dEkBNj7i-e}{r zuh8ML31?O`L!{Q(aWvUHfx%E`_E!UGK(3##<8r0atg#4hN$L~1!#@n04jw*& z1I4`Rn86AG77T(_4qL>O2AVqw7rY2*&9N{v(X3ZU z8RtX4)2q85NQC&K_s-Bmyy-h23+)`F2E7yUSVAK5K|-mw?5Rz`ln0n>-FSFYp}>GR2_SUs@awL-&b-LnX}kLymbqor+*z(S%Ts3| zV{WW8A!E`OF3(DUL{)0ti;NuM);kYxGH37J6vDx5Y@I6C#;vkR1*7K1!{!If*+LQg zbz^`a!Hb66gVq(JCYgSCuETjIS(_JCK;eyDlh%tL)5Y8hdc(>=$N$Hxb-OG!ZrQxa z+|pZ>hJu>hV&mZeyB?f+2P^xQlrNZ|V}xe+l4NEcNF%(ZC5;&FuGTe|_VSxcX%$)# z%PE?-VV}!70Y60z$$E;d=Ar^x;b+0y*nw7nHnFdXrHyl8yy5O=y(d_Ufhx4IP&9AF zIfh?rSRQ%%b*jW;JIY~fT$EerqH*n*cU@p*d{l6#>&-<@o$(4y5j<*4>a1)8B|!Fs?N z|MSvr1*4f(-0a8LLWevu6-jBBjTxzH_t|yuL2=!82rgNKv?C)*YjwEEiq}pxz-OH6 zic!sTq9r~~z{%WzjxMAxP27SnUx&>|e1ZzuwnRM(vP^!D9`+!q;1m7Ps7V|ph2U*6 zh0VcQ;3-e5bL!LTZsnnHgbWPY4H*`BcnYK@i zspS`P($sT3b$0_L+bFq_lI@i2pd=d4SbwnLN^B_dGg8TtnSLrbaA2TKGuffZ!5}`= zOLJ~1?gCuDpZ9qPQqGz#7b16gCw0!NF&QE}i!6tbN*2+-GFy{0Pg4zwu#hM`KIkDr8u z-cEQ1G#Qwi4k#$c8;n{oRm~?)Di2?zDsE~rJ>Q3Cxa+()OhD>nKbe4TS1ip?4CSb< zD)rwCWfj*l*e*)xK#c9-_xsGI}GSG!wC9A#=K?$GRi4Cz55no$Zy^IGg@K4unP{Iop( z;MOVbW%GvllP}+Y>gg92UVe7rkq1xx!7~eUFP?ht zb>#OLiUjrSeW#v!;?%>Byz$!OZ@m2J&~dczxd)~E3CRQYj<;TU5PdF!zwp8zExh~% z>bI+-A0N3-!hPh~g}E>WQzNe);zoUVQ1)7hmWc z&@*x{n5oFdFz$Ag@<1_ee_lI(v9Lv?HMiFfRYjvtN3RJwoPOzK1X6O0Q4@W?dE__|{?^&OB&?W(#FLNF=)UmW z8!ta}>MPHjI{sQjMLAIsn6Ay+1uK}%Z$7>7%A*mWg(sd1=k1Me|G~MzvO$#G1xXm` zrirf?hW!;%kZQWsDW^!Mw_W0AKPJCmRO;kqBD%oaCj|J1Z!n*fJd<33jwn-hU%gSX zs?Ic7H`(dqYo%@e@Qqc6bQ`9}jzBN*4RV~Z?afc18O@3GVUP>S*XniAM-Qp(hbuN~ zIWVI&kRzrk;j^C(EOC%(r90>{f?^W6W7&qN?t@-6B@dM`$tQU<@)V7ZaDSxa%+@Q6 z1$hVLJP=Ml1R&=+B>60iV_Htjz(|%p)HCAwu;vHM}) z3hXrrSGVGvqXaog1()7VWkv^)!DkSsnW^7V~O~7LluphyKbV6(Qd;Mh9X=l;TZLK~p z<|Q6dV9!aykuZQWAalD?Pl3~=XWJpCtE)4LmxWlAcv|%`pJlPAo#WG~uRZ7uPG;u& zt;|W)>IY@sXx#*dK^C}!Hb*s_HE_7g(2al^GB0lpQ8;B zhBM($0AV@cp@i~V8N9EsM!0n@Y#m|!*WFl0xj*jCm6V*!xAVamhP!>p~>O- zWjM{g0sNQKNZ>Yl7Ve*A?d9I`$rYH7XS`(+rvnPV<-BDGM!-2tZ|;c4?_QpQOkg2f z;SD4I!>xYczS>)6t#Mb*ukw~#7o60N8Z=AwG|~JKb@u9ROf9WF(n@yfwntDi6V$v# z)}%UjC_G0aLI17R5ZbIJ_`qaja*f1hg%aX(66ZT0jRcTpC8V&8w}Q$6GdZL5=D8Pm z%e@s|9`3MZ*4h)0NO)Y&MB`f5!Bvjy;F+Kg2sHHX4&~sP8GLA6M(1?@7SPansrTFs zPu}S4CF=Z8RoW}D1_pVBo8HRD5g76$@4MRp8?;#Y+bd-Y`8o;S zmE-b@7}kcxuxK}TAQdideiIylPlKDuLj~4lhNc8k%peHE2h-*53j8mTVNx117{D8^ zzK#>;^tWF4$;+=_@zyI}e)GtePrrW5ya_xlGB}#2zkc#3Uw=%Vf#HdJYW|N-o%p&e zS$O=l)8G2a>2Lq*g)cq#=CKnCfBbT24KNpuy$TT{Y$Q*a)(>|zLhFnJZ&!7q3dIz% zeuOn_Ks|O;a!37gLn^alI%mw8{V>~-v54GGAt^Q%-k;^vJcm(6>Rf>xq)>s(9Q~1! zMIQ&Bx5?+jDr~&1lE43K_0>d;NP`5}MaK_okZS=y&{~I;mf55Aycg9qL`U3*&RBI7 zNAU#hjultcU8>V{=u~_f)p|y!BZKs39vwNvd31SjD76i@zNJ7f=;!zbxVh&wPn$%9 z&Eiq(e7LilOpF2s{Pq%&ec~8im%_&EkYJM{U4?INYcyeBBO^p$ii}oZv@_0On3VRw z8jOF4set3otV$EIw0d(~5*F*B#uUu`W+RMVa=0&wN;+U$MR*dV_cO5E(z1iw@|@<@ z#df`(CG=iRX?tPmNRU>Hb?&Dfi28v5vtsWpySM%3RXg{TZrpyuZ8w!}xPIHmZohu_ zjlR02wdT%HOuPHeaAVvZY{t>nW#=g^_P7*AdsYH1DmYbOr@(3;k0zNfLQ{XYnoIM0u%sd3zsyu{*v^GH;P~`v^N1z<}ZX@&$FG62@4nRH{;=LSubxEX1dW0>S zg2?LG-Cqs@Hx@ZN;GGDW1uT~E zdSFw-tsof(9DLDcmd-aqd}E<=!pCgq zHhR5Pm#E17c~%S z0=GyJ`m94vi!RnjaVnr0<_{i7*p4G-A(N(uPjEC%v9OjkKT)EUbPpwD8WdMiLg9x9 zP7K&~m&s4`Q!&L)@WCKHMHRJ%<0qhIlZNf~tM*L_RPa2{= zWT|l^1B;|~CT%2*9P)DN21K01QGXTcQ3;?S7nVz^rE63h|6tqvMK+d)Qj9bPy9Lyd+?NkJP>S*;kN*v6sa#wbG1Qjcpecp?}As#zw8 z%u9N4sKldQ;y4t=x_YoUM}Y~dxCvnoDirVp!X8pyidA@8)#wc}&6uzASPH9%ocLrENowoW&IR|z+K1PILF2b-67#;_Px`%KM zJU@(Q5+JtsqAo<61@Gma%DbSu-FM@-f%xZ;6M~r{NR{^pb+Y(zta!RjKs(Jk;I2aEOYR*!pj_w zxrwhmYwc=d@lKmQsmj8Y$i zJ@Ad!J_CCzRJ<1ap6rvoh8?R~XudW2Kj{p8YM$&Ta;MB>8Og&lhV0)?>qA)abtO)G}AqqH@cB>JG7 ztFx;E(+2Va(*`Rsq{1JBxr~I_T@*{v1F3a46wyC*dc}H6N4mX+S;_lY1z^J7J>40c-~p@t^i*2 zhYpxGk|4{G^c4Q)!SlG1FGxtIrY6G|(}?P9R_qc0!TV^yu}s%F^r{ueyetvY;rAzm zHYo#HwRP!vkqjml(E>eKQCpfX#ww48IVch^4Z`OO?D%?y)LCCb8N@|0g!q~6U??D# z1)kkr*|La60|&&&BiR4PJk1YM&5POqb>pOiTh^`)?Q2UxI$BMGsfhY~30 z;J~XRnt(ZIXi@Lzn8X<=v=RNE5BiO9ze{yWW*Tufs-C+;3LNbWcCOPMlbh#R`~`K} zyWVs_W^enV!4N^C)pQm?!>t&uDA~uKq77*H9qQ%JC|P0>Gc=MvN8wo`iFyl3M??#@ zeALgsBd`I@zC+J{Ny!rZV0KC9 zehv%Tc@PiDY+Arq|p zNDi$GEvK)7u$;p`!!Rz+|BbvKMK`37BWs{m8Yc;Psd{q_oe8-&2-KD|bNoIE5DJg7 z3=X`;OsQF?;5L5{!CHv6#0`vA5MN#)oh|$$K1rvqj*s)F7%+mINye;@EY6qovT%iH zD{+ggo|U1!mIfJnWt6MC#(a#!m*a3S!8sI!FK_ZPlL<(Ykk2`M88Rp2(%jx3-00)D)A3s9xO9Jamk86Peo4QUvC2W5vBVuVj|c<_xb*kW7N*;!@MDB~fz|p$ zFh?}Rda~*emLDxw8spV+{c@R478(d+Fc<|mZ9)hkhfjg#4H#p>pmaj}mxow~@u|`x zL`E}7mrnRE2VZw6^8TZ*JFvQ#N>?B^$0CMv13YBZy_`(GB6apObu*;yP;%*Ank935 z(4LKgGgvlk?c&jMgfSwk5&cLud%B3*L2>czur55ir;hPrF_f;0VKu^>7F>hsz&eXj zb^(*jG+L7KE?8FshKTD#jo``w&2C4A?g~Rbr_W(gT=oCYA>PsUMZ5>Dx(DZ07m;Ls zNWgGygnNNQ+0FCDYI6^}7i+6~>9%O^C6b=iE0+Sd{guPBqp?xk^kKFpBFvhR*83rk3lZ!F&B_rPKwTz z_;liX#ZG$NMad^9=}4@*@w%HN);)M=tso>jy7)gOgjnn)z_(GdL>IS*U7RJ>lVnA@ zp}3urb(Ao6zeP`D(JM=xcj5I8f<&rLYi%#GivxdKcT+hI5qXse9J78o$9IP(wz*y2 zq6{vMg!?FDLGV?;rymvudoFU)7h7dI?qE#dD0TQaq-i|7KyTKXxNZvv4POgF=QF0& z!~unC-e@)j$1Cl$z=^_+CAwb<9B>b|ZsCfourF-w-yUA4?LHz12msT)A;OUT9&nq0 zZFAs27u%I6u@fV@8#+pNaU%J5E2CE|ww~!a&ALa`72z+h1mNRn#6DDbT|0WZxha&mG#N@7{_|Nc%Afg*2Me#2% z@dX&8@zt~JX%&&`@6dH25}#ds1y-1pOAZ>Svg}~Ih%z3agcok_-fCgF^6h)=h@r%_^=Hy&3PT-5 zPkMUF)R=aLuqYXz#HM6|lFO0!=}%JRGds)#?QfB!q(sR*lu!VJ_y`g|#SUg#`|P1A z49knTtV({LYTZD|HcI{zCI6X{iz)d5B}7{3$J{|rRZ1o)IY0^RK>Yeg$0dZH1!4I# zh#C*%;}RK(=vJR3NsXHF4&H#L)kKmT@sRxB70?pN6@#=V9Grs3Gq|x!`G$Ilb_r>G zYF3v4;>&gDlPF2PYs4xE01E$zkI*!za2(kXf=KO%K{o*KIMz51FhW% zLWFh+Yg*1lP`Z~x5Mn5em)DGY#2HiuD%ApYw)-i6W^v1E6H}{$K9MY;0+<5EcYZ| zI>zrqt;+{shCxjfLrxj68BfYIUJ$3g}Soupr)k3v{Hk-UPDSht$ zkA$lAa7gTiPB0glHJC%j+6Z*&aKFy^3nL^kH(xC+#nnCsXh9fPUTLj~IL>CFZCpOAyEL6td^vq19YybOhh0HAqo(**n8Nqn*Z0G| z3WL&8PLhz6n69EHioKDB^BH>T2t)n!N@m*Z1rzx4I94N6B-PY)0ZIf}b^z26>Wuq>T7AdRoLk{tNQ% z!(g4e@GvrfpEc3)xEhy7AWB?HK?q@ir5rNQFo&=pf&~&#Cxpd-%^deNU`0oW3T|&? z(bm)uVDgJ}Y*@?TZLN;CcLeoRNbcITV;B9w1-z?}#@pUqJ1B>@Y2gDD$!9exh#Ml3 z`f-H!!3HIOpdxfi3`$~@kjyPe1e5Y8naceb;*{`Kr)bz$&lM>D|Eo8C_sD-p;lctMO^bat)K)D+!>lp^+8 z>gq!7sucdnfD8|Scesu#lBGTLLuC!I%f>G#So|W^Q+;E+(U{_2(&-4^d})!dz;Gy! zT$nuXMD(sH=gB8&*iT+7)ynHsyg5ar2Ar20F_q-nke6^E=myVOJ(*6X5I3a5p8{uk z_(^N&ceNB23+M1#A0m`+T}OLY!*ATAjr2R(H58T9`DgqTQBG$lJ}LYaoIHLzCy(FG z$>XMHG>f_c=@LN0SpDCYOn^iug^>_LR xpNzk!zNl>;*u>FRTl70U_#M!n#)IGK$>Vo=(*5bTwffS1hQIHVO=f@gzX8T*Lp}fi diff --git a/src/torch4ms/autograd/__pycache__/forward_extractor.cpython-310.pyc b/src/torch4ms/autograd/__pycache__/forward_extractor.cpython-310.pyc deleted file mode 100644 index 2a633c9eea380d9ce86bbed1f7f5f88aeafa9035..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 8033 zcmb_hTaXmjdG6a>cXoDJmK91AyKSkY9V{jwId(#Th=A~&5VMZT6P&JLx@YNSre`_b z17d5Y3TYAS%E02tmTVAs*A*m)B1wJ&P$$BdFWT~%q~w<%2i3ll?wU(bGmzG zxY;SCr~1q}|2gMBxBqz3+Jd64L%sO1RndKUEJS>4x{VdH0J5sFFt*a7SgZkfW z-?e3bsb*U4e$EmNH&%!VzT5*mpO zP(mvb$q;h>0G}H_Bcoy;vetv>Do93THcSoTRW9E5%_)vbIy`qP5?h*8@Zzp|(gY6B zR6NyGO|2Y1C7qH_DP`4+neh{uS$RfoNM>SMk);`BMk>?Hll6ZCi?oiX{qeQd;ybrr zInz3OQiycx+NJjKcOkRldbQ?W#T6pSJy0)~E#{J7@~%YV$5VWn%ktN4&!g=QZYM6$ z^3oE?{?hsO?0fCGxo-aG+$Pjki|@5BPTxLtrZw|k>)aWd7A?q^tA2cvS!MfWMu@kkM0*=P zBz<&bbm;C35<1&yd6)FP4NYY#;bE^D%Cyo{Uy_`ZCV9%S5m}<&ccsJHQK=zcK>Y>j zqSDknwV_VS?CFL&qcpUwsP{BZ$%7&ZN<*5DElA*=-LEub%{a~?&UwTx;N&hsj%4ZZ z^QOeFlP~9UabGoU<|oE2+-Q&asTW@^S(5|~Tuz06WF1^B(LKeG8>C$7aJ^VDsHv~WkUE~+=y62HtQbC5| zQ%t#_nva>PrA@>p;-+RLpn~YS#0p7sy_vKUX39!|mNxr9OZU)7RmBQ@U9CuEX@v~> z^n+s-3QaKw%{8D6aM~cJ44G@WQlwUbGHk8`Wyl=C@jvdbf0#O|{|jI{xFDZG3DV`@ zTw1UHbandXou%`(bn7tt3Q8#<;JwQajm~S9o~4}?m4u(bfR^mqv-C{vNXW+ zd41*Q3oGYN3MRJ~7F&yRca~-XCEcU$?kvr6&OBN9yT#>O-w!86thR68t^*0JioUI* z`M+s9R*{Wv-t?@#ImbJOO#>l;_TKhT#XidU_4W((N7+Hgh)Et7(CAc+_V#%)T)Kf? z?Xv)+xAf7YmOXyZdsaVS7u{#|4Yo5`_lzPNcXRsk&9})y;7H0RJ;nvyA2cnqJ}J}_ z=}Nam_#QrlUXdR92!#gF$$+|U$n(kqPzxoMm$aZ1vw>ztyrHn2w$xO;1k|-UpQKt~ z(38L*tr_!Djo7r@j01_(BQo360BSYjz^~8J|Kn1Vdd26{jl{ULpv?C*v;|oP-wCy$ zZ9}VoYbF@8pO<6vVL5BkyasqWC=7-A-(d35VYOy1w%+|dtf@f1{Nzeh6=il{+qOg{ zI%1q0OujAmnqO2!&z{^Gv89?x2GXEFNRDWHW7$)fg5NL50X(tK6K$Qa91OEwodl?g zQ*?c;!c8pOv#PGI*ruO$Ezh*d#d^i_Es~OjtV^!8|DSb;*#w zzN=sywY`HOPUe!n=314qK)($%E0isVeHK{`=7l6Tj||L1J5Y43du@uOVc3rC8OB%9 z!=-h_6Il(0j;aB$M>tdiu$5uJ_MkcldRm_P+`|ATA9CC657sQm+8s|B?R(E)|A`n3@7=xAXN zz{-!z>+tG@wmmhWCz6)?W#^tXY|-04{-Aa49a#1u!u8zH<)!)7YnLB3ju@>c-KdYT zI%Km8FE$97@VJ>ra1e+^ham7FD0kqchp{MZ)OSOP{uchd>|jv^LXMS@Hz5qIz^ZSE}GbLlz@+7fIx{F`u@_a4BWv8yuM1_9WeQ`sNm-(Lf%fr-*3_Uy}` z=M?rQ_~UKw736F$?jAxrZnJBdG^`KU>BI|f@Ju2-1&SzvU(_yaOdIO>bUC`wQ3Ks|` z);jsx?ddD9mLs1s(0ihV-kyDTb>Vbt_Dt*C51>+87v_mkx$cz6K7_3YTdShqe*5_9 zJFh^uhE_F=+xq24}td&67W~4oSaBu%-N|?FAZv+ zN&f72C^wODG_Rj>Lg|^>A%z{LRwjA4Rq8#F@UoPr+)Dr`Z@`RC%dg6C0{Y+t40>zI z%CTlR14ljzaO+x|Hb2yeEx;#k#2_|_Jq()z_s~yPp(ku-9NNS?HVHEv4sp?s8SvDr zQ+IY29jE5$rJ8fZVxCTQx_|>5d)L_rT3#PJXuEo~cuYqC!7jny*S&*94~Z4s$WRt^ z;rd&qUhL+Z*Y{Q|Xb&BzR7U`U#HAJnA6&f%5=yxA@Gf+OBbZgJ1XFVJ`q#?(v0D8Z zG^)ep9~ZI-&-__T>X4;#jx-s|IbyS#Q?(pV$BZrRf}8o=TI`4WoCoi>CcI|yko`>P z;OA?TF53YTd!FbTwPm}g9C_D%;$YF`Zc*H{j@TuO{W;Y?L+w*l*AVx_&wRV?cy`r# zkuiipvOC~M*$3)gP)?2EC`6U!Jy&%R!$_bn8ES=u_eGIO^I7q{ucC@i%vTY_@cYEj z2A$|trH~cp{po}vLDdH#YrnR^N|4RD|YDP48 zRrqOlwciY8G)4KJjFy%)Wor8)F%L`V4AybHIy5*}FJO2$UmSeLkQWI29m??-#Xib$ zmun5dXb;QdsB(JB7RZk)9cP2pe1=H+n)pNf72Gp>m!a2 zVrQk7Bn9&cfqtI^pg-CT^nv$)h;--B=|q34E!cDHya#u^LT^+! zzC@>IquzSwRp>?FYLBVAO&E2fpURYbDD%LiyPtcnAs5LQ&2&#L<*OC=1z&(r`0A8B z$z2$hN0DtJ+JL~VC>#v$hFO6Z#Ox`{5N?#82yh{n_BBF2Kj{cB)Gef((AM|U!iXi0 zs?Zl0u-v^$cbr>4$pH|Kke?`4;Q9h0Q#@l3ZHTh~ygFvc_G*lMi6%^@qVRbBh;kg2 zI1CX2F+!z(#_Li5N=s3!X+-@!c?eF-h|1%)D)O)i5slKaYEpjc?jsQLK11|@2&pnY z6F$3j7#4PH$55pS379k~$0zw`$lbupBT?~m9H!~8qCNcxaJ+9K_eFUL7&$mnEcldE zmIVaPz*gObo6lkJ7OEo25O%3B``SPI0MXniYTBB4YjxoYY{f{)3KbbfUP!1SK8U!s zPasxv9v@3?pE|wz$q!*8f)zPSXMYNpU1)q=gfYkfMZjqKCaME76(45yhEadLJ@aNj zj)v?uFMF0vE+iS}JSa<{%yB^2_YzH+7Tu5^mwuoe2LAKihZ=hdEpm#lDUca!yO@L;}S_-P}+apW_GL2-WKk5a-g%v#AXIstY@_=%uVtyW?5-H6cv za7&Q=73B!R7+pT`{puj)$e?EAdb27e{6sqmN4^{?$eMMjl$11i{lM^(!|R6rbofg{ z$&nPJ`rKpeF#h~FOl3F>Vq}6gF&h?UFs_I8Jvu~@Pfo7 zx#N(k!VhxkLn`|^wWnLq`9>l>#L?QtYDOtXlX}o<{Nz@l7q`5BN)%S%Bz;lUT;QO2NfjeXfr4-g!x)RV7avyBBMe5Y?<$vsJMM{41d!PE9S&9-V6&+#@&zzZ^ zJ@cL0Y%nxrD`-bg{j~P=u%i5%2K$?a!CUCzItW+F3TIsPSzRrwb*-$`^|D?!%0}HR zn~df(KU25LmgIFmTh2me_;x*4&Pkc!=fMw2-UMGL7bKqnKU^MW%DW1;cy?Lg*(J3+ zf|1R0809b;^+uNzcZ}z~(W=Uacwt#DKf{Ol$g)y?mg_T0X*6Djn)c4^jqTfO?Jpng z-22ncH-GJHZnXcnRy4Q(U|FGP? zeRKQzUF>kT{cyGO*|n{QH`>>K)Bg1DPaphl>)Wrk9zAZ~*euSvmHCfc!Hb=>e?GZ- zt^Mu&Vl@y_(!TnzvwGKl0zH(Hky!6HX>-?4^!I9Elo%hzP2Wqjw;Pue>x}FB?yT>X zSd!yj)s6j#*DBF8#^#4!BMbz%>{(a1buaR0paspaLR;@|76xyjhf^S)Qf9D|%2`?C zs;6_!Gq~=V-0(70i<>+H)GTg!SuZoL0LLu1Apy$3FP{we@c6*11cJveV$T$f_O;b5 zfGIHnQd^HVI={M;WP+yCfPRvx#*Ip1)*8(?3KKJkBQO;MJM1>%RrE9H;W3aFYblP> zQbAcu165l(sK%8U=Y8bCZnIt7M>r# zXJU70VhS&cMWe#7e7EUfsLSKN2WC!L#DOm;?SYwgQy*Ne!7$+t;v%dC$Sx;9`M*F zKJPdeW7qG_WE_VF6^OMk62im_1pzG2B7huHUWaLax2a+-gAtN=<$E5B+4r_n0+2>j0WxYV6bJr9Kq=>?h0N&+$>s$dWiMoU9H99uLarDaCs`7gGNb+u)ZuOnJCM1;A-#udnP>HqrP{6#%7 zkVMqTY#BJYzN9Z^S{Xc*!BhGYn?h9r%xV`@O6yzST5?U$gBqy=Jx?4RHc@L_ zdPQPHl(5A~2(gqxWZ;nUjAc7TQh3!+P|BoqK>2e3&157M1MhCx`92nhWG7vxaIx7@ zHLs4TV+fi9Fb_T}j^McpIs9&;fS@t;@XsKClr1qzB7lTlVj@Ry5Z}_ql~0rvfV!b9 z>XAlZfx>;}HF2C)ttc0Cu3l2ao2i^@^Qt&~RjD$rb7R?p;fxujJ7;3Shg>R+g&xCYU5|L3vHQa3$GpUI?C01YbU1s z0JSB2V`5MCgtF0r^h9AU^?Lauy(!>@X zk1L*o=fw{p7>Hd#FLKf4i)@C{N+*-4AuTws(~zuDGSf^!mzA{sX1gi4#*?AG!YOx6 z-z3jPh$V%7%zp$~Nbj=pHq)7{T5Lqazm`X#)m2axWfvn0BVAp1>HpYpx-=xdL#iDo z@)D6E5m`T9CXVvRzQS6fAwej}OT;7*X@EUKdW#bvC($L!R9!80!aOk|3@2_a|@zuOOyoVs5a_=-rsDmq{PcIVD2J|d<|YI+M#+q zhbz`$nq9;LQZripjb-o5=!q!+dH@|UgG8a!+U!b`IC&`EENB1;?6p=IUUcL;(>YINvvtf};__67kL Zz5t>t$8>T=a!>jdw89bdCj~8U{1;{=wN(HB diff --git a/src/torch4ms/autograd/__pycache__/ms_autograd_function.cpython-310.pyc b/src/torch4ms/autograd/__pycache__/ms_autograd_function.cpython-310.pyc deleted file mode 100644 index 0bb949a21a1558ea263ff31cec122a1eab519a49..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 14144 zcmb_jYmgJymF|~XEsdm^5tvtq-F{^efm!3YoDIRmtZnSP#*QH-t2nF3(>vf-j)R?n*okor*p;k<0f+zlV}ESbR&Djat&B9o{#e)6R{o^6?ETKY z-FnQ!WGgGIJ6HF4?z!il^PO|AE6HSB!sp{NKPXi8NYWqZN9QMk9}nW|{0|aK%1V}O zDMdLehG9Cm9lFe zbmN~B1WYd^pn3RQK|!*MODJCx?$yo;JCJAABL`iUAEI5Nc{@KU zemsb;vjK@MLC&SDV#!$*GND-NQE5WXMv&L62+!-54%vwM{jF^Z39CLv;$q^{sxs%M z>diB66R)_0420Yo2s`oiJelL1>l7yH_LclWG!^{3xqJ|3i2$B7e)q=g2WOi9czEv3 z7f9@=0-`r8II`7jPW@{B>~SH6kkh$~r`s~v#ZPmw)KSYtVnq``8lTAz?I)gZHD|A$ z3}wy~vehY47(>WeZ4clf%x6~RCKgi#^jY&mmzFM>!Tp2;p^c0-*+K7EP2v*k~P_dw7D`QBH_s+N?G>grywWKDwecM zIxi2NSN(XwDU=;IUmmg(RV>?inyZ)l zO3vm|zEIxM`uTLYSx=Xoma5JT2li`V^pGFhw%0D(2S?fV`-oB^SsA}W8r#%TVxg)U z2=;bhTd^{fFFM-?T3s2fd~o-9#gFc_VH1i)Kkh(B=SK4^UxGePGW*$Tf!R)uwm6gY zBQ9xomPS9ZMlENTpYx4y9e$5x59g~zmx>uF06*TAEkDsIDZW~?%YM9EE#(U3(W>kC z{Z9TlJBRkUP#O6#FhH1%OpmWZpxK=?H|>nCk2Vi9Fjn(}t4h*hdC2F#UE%V(^N z2AScquZv2)6CznneWC%GG_o^*B;9AKhD`r-H7>9JI1xck)8#n*iC}VJmBO?NlZJ!crFpn%>@ZE#y*18`zd% z5;KynV1G?5Nmq%QY^tLZ)KE6X%efYHNG-`W+eH1iy6Wku%nF57auzVeH^OcDd-QYe zMWQElH9-nWRTSf+mAay?l2dY@e5-7XD<7FqSM*QgifWadP!h`6y5)C&5a8Iz<;wYz zoy+<0T&`5Hszu5ta=B-#`C?EL&E>4h5EaAJxrL#U?e42sYzwVsJ0-NQ9SJE{Mp|zz zzD@#(A!)J_kITvzQZldMeC!e<+H^vak5iw|Ac6G-VgwEYdb8v+3hBxb)dP~;t9bG` zuJNEPlbRBzd_UcOlYw@-7WRdDx5<~%; zEi*?($tK$?&J`?1`yOaf zA*iMI>9UR^z{u4Y!nf-~Egr+nNuV^Xo5LFvKhOHwP8CKRx!|)WXC?X6|D{aNC zo})72BNVBSvM~{-$i}Tcs~^96ECVvP!e8-qepDXkCSeaF^}(uH%q0Yo3tjWgvyE3z z3v1Ex!a}DG*x7{l54}n-$Q1rRS_#lRKlxVU2XD4CE+3TLgW);Pd|mV!7@|WHez?}e z4-PZvL~@4*h8%7V+%2RWxN=RLQQM=jFQY9sNptMt@PgundjV}xC>PvZ?jrg)IudLG zz|q+HCHOHAHXhvB?s9<<)qvk652Xg3d|*gG6|F=>&gsRepcQk)s+$CI9>7ym2}h~ zJOV9$u;BQy{kA=tvkD~&hQyg|Rfp`HQze&yYzar<4kMz^Yrcwhew2D3!^o^>S5#PG zZvjyk!p!8mLy*KSaY(zOU79;kaQD&BSBPRd5M&$Ta-TAG-%`7UmTfquO``vyi~ zPa??}AQ#~_hr(@~GI5$oMtv1S`RUM_w1?u7=BL66A;g&o`wETP8k+5*F?4t!@F~R7 z=nyk*RlbSbZ}4@<)F%{OO)Ew;fi>x{d73=7dYP4V0dm?(GBy09ka;3gGeU-jf{Ai< zJ&IfCIiQ2QW#w}Jj5?373+aycnb?bSZhC0#> zKn9k&jg}3~VCe-g>2>)e&LlL!XEC5qtP6aIP94zJ$!#VG(I%D+^2I}BpT-Z)Ai*!L znh6R8qQG`ieK(^xRQflRb9|mw;&K{{N-v{3L+FRi+L=kPFVT1pQnG`RzomqZG$W~C zq?Z^mjnVmF-=>5x2N)_)C`PfaB9Bg zMQ4(p!QITS@=|?-sB|%dQB0M3$xZfRGkw$s!8)|YiHc+NwyXM8w;v8_+A}6(a-%2Y zBQoad@np>PnOXv)Ciq+@;RGkW1fT0LFBNm0^OTttR1TMzS|~Fsz2sS#3b?_CyZYSX zMFDwl@p{M=uBF^t1Pt+dL?B>FCc}tp61qGwYQ63%uXiupe*uPTeeP{C5f@6yChH*i09?j_1w>X3;$s8-NMDwjSHt57hjt@ z`HuOuLfOu<^U4kEyTqXktd(BMyQON;kKt;i43q8H8P$&+unT+liHO-Vg*^9BwS(|f zIiW&%aRuLBGiCSQFw_x%5~{+nf|ItrlSA`Za;lI_~ebPtW%Tg@&771|Gn*4`aB zVdOq9)X2p1zrtD=IjUp#{@E*OMN@~+uk|60)0Ps`I6kRMh|fvql=UxfRMzEVacR>&vYP0iv| z*-v3;Am8@!@mEq2gbFYh7mXn%{sOg5v|LYIC$jxiq`CJZx)R$Bad?EW3PS<1kaZO2 zhAIf08{C5|;2x4kiO6B(MlRFOax_A+OPh2=X<`9qDva%=hPq&YpX7f-hWiozQ&5As z1i^l?YggFk_yDLyv2Q>f@(4C76EPe<(4GNq(T=hbCBz<%di~pOvP1ulmK-l>DGl); zM2LVc42lA&Da4Bs2xq4hL)8%6j^h{pp?;N|R@x7F@Fm*7Bmb@P{y8r@YAYnjpzlWHOsMGdqYz@k^wG!gTks8c7;@pa- zx+;wE-5$_dEdp?$;srv4-}&iu=jTQ_IEcdB8m-pbJ!8i3coez3mY2pxXp|?t$V{Bd zao0yJA~QW+Oqkhnq}>N&#XZcA$n;_K);I0I( zRv$OPBM1N;fuWYXIQt1o5gkVyKQ2mX#E~4mj03`uMr(~vf(DcVPO{g*d z4^;heNvcBsKW(i5)aiHEc>RDn$pGr4y?!s9Tk9sgB<^w3xpnTRJd}yw9Qv%R+7L}{ ze+ABITs+?V$!|&1l28X|2S`EKaY7KUy%5}fA}+(*gm>*z6qwU1+Pz<7k+`E=YRL}`O9y8LVWTjFe^CyZ5Xx$_kMXKz*Xh) zz(Wk2eWdI%Xx*=m zcicrZif~gNBMkny8FWYde4Pwx%_4xz{1-QZQo+5mkmcmk{aDD+H zM=+|uaIA>zP-_xo$Ymw$%fZqU$PnU%!Jtcb_=A=Lg8^rsyh^A|wFXqo>(#mpbWAC^ z`Wj(mT%5w#K%IXE7pMySCS0V#_lZ;~`iut*9ntjpOBZI}pCTz1I3xsJ9h<I2OGE51@>A;q@xo!imK8SeB*)WJ1`OTp<0^3OH;4ewhaGtGV2 zTPbQZ7NMxpguDmWvEosUr_ePqxT;hMNH3#HU+L7MaQA2-rck276^RL?xB)|C1-5YS z6LAaz6|6#uO9>%6Rz-qq0?v5ibkGI(ma0)heE0jfIEG-1(C<+cvq?(Y+mRTq0&)eM zrPzf-vYb{mt&h->hWja8`Rmw}v4NYPozB5B>$$A>@u%{|s(_%MjE5N^S+UlGlsiNT zN2a5cdzO+GK4#pP>E;Q0j+z{xgi*o)E$z98NIiyJW(9|~f)!&_MNxiV=e%;-5SRS} z)s0hf7)dtS#mp?BS#jDuYEq-*dzA3~`99^QkoY~Eu|kat4<#ZFMV*?o_oEGI*DS~6 z`)F^75s7}_R~!`$`$9QFvy>&}&>ttLtRKyJ&|V0n)`^Lb`owDq-**)Soc2 zC{IWFHqc`aAl?(S5+gWk<*q9Wx(%QcrsC`T7KymBsHym?7UdFj5Wtr@qn?$f@Gz|w zspCl^G}0G&J3PUy;Vqo2*EJ6ju3^NVD3Zl%BSoVWtKnfTY85v=(lbI&(C+u5b)3x! zWlAeY+~iEk(h2x=wRyZAUX-YSaGDjh3;-oOciSZe*C$jOYnNu?)iTsBH;Rq=&l*>+ z%wN1PcXgt9;f)(_y)NQ9O9CCUl;3X)hb-r~_GsIzHje$~#)}tQ?`7!%Hsl`kM=@>_z;=DSD4V+znKLb?L#bONa^ z*wcLRQuFxXRsd}F+RJ#6*7)h0KwCG)Uzxu?Y3`;Ma|mG3tJYFs3;_o5D3&*xqOfr2 zdYY}?T~jOyV3GF6+d4rpkRn0GPLwGg1rxvtKP^BcLNm@LGl+_8GU?uyBO`Gy!ub_; zLu_k){FsOiwPJAtz++$MG2B+$U^o6B;arPgLDjMdt`ag8FHJjG-WZ=ZZJw$s!C&XQ}0f1dl*xFONgYI}c{pTrGw#Eg>QD<&v_2_Is1Lu%$r7 zA+sqi+8VSQK6eg+Q;nA)850a@zILX4$VuEfq>R)0$iaLGF9tgvWNzQS-7J(Sf*1G- zoV8}&F?pu7LTbnLhxvdB@f8?t!fMPF%%($`vHEi z2ksfTXF+#jJb6|REYxgb2=oHy&|p)0qVD+xXG2nh;gFqx#F7nfZ+p6NP6DNk5X*3lt0AV9;mSWa6uTehuq?! zfvzwP*)s!QLSD{3Bz0+e18fYguj`0E0~czz-BIX~BEmCD9XB~AWEr@UTejQk+}hD^ zx{fF7S67CWJ(2>h3oK&Wt0H>3o@EdyUxLj5-e566wp!D`ZhHC2 zLBWhf%mkBVz=vDGTv_~{Q8<%vS?0;ZDrd9&32detPo;W4=DsPLTlT3XY2f%EJRo8) z=LGv9av9x^2CBr524;zEq$b2`u_ZsIcAXqPNmVaWvWgONPWXsso3{u>s)jCjYbba9 zXQo%Ykh?Q9x9$HEBMcD-&=F#tqyg_h;`i|8LTCFCzNZ|8`?1GE+vBHNLyG!o>dk!) z!kfM>6dC(G75@#AOpNgbaQwjQ&+x0DavstY`2Gdz2a4K$qBg zzxOdSZ$92{nU>2%22WIbFZ#tKV;`Y1{1ia>9=z&DoUwofyv-V%q-aJhFew8z+N&;67mC$=(YGY?a;TjZ;A(C_Z3~7lVaBhcGU+#wHH^h|HIYr~Ar(-Guu*Xhma3DoBq zd4yDCo7tHs-MPU83mlu-?7qd>v)VdaV(OhcoWlj<>?XW7hGGMU{J$FZqpgz5PQxxf z`u#gzZNHw%*0%TP54}%+{_t;q{M{$N{LRN7eCS<`l3=})%J9%*>^Ri^-7i-BBuTZ` zN|U`%Y7dQ`4ilA1Pp3n@`nIZf^_u|YZTownm!w|cYrpEfx#jJr*~>763d1P5?dk1E zd7+f4Jncm=w!iN|7E#OBA^3dl4W_!|E3YL(a0FhOfHUl85`=qEE3A68cId0ngS;{r zzMqToe85A!d=Exwd;LD%D=W2)SHEA|sCc)+EnIhhi7HBy>Kpa7%7Q7zX%IFu9krDi zNz!HzsgAE(+s4tjg+?@2%=Buc{9EmiHua4xX=&(ha{W1st@k^jDP7MJ9mU}lT&Zya zGTOJ&v~7e=jU6TWf^juW0QrZe9H}PcqNCecnkiEp!ed+Q)Yli9;ko$iR+8N{1H)_a zG8yL9Hm=rgHaFJRH)^$O*XvhSo2&JkwJYn!YIQP9O#6^}o(4L*Y@8&EJNrh&%H-;F ztS09x%Fw|Ko0Xj5+7vStHiLG@Z|#~ql+8JZcZBm@`S z{@Qj++~y_j3i$DoRpjF9QF}p*i=rr6Cu=IB z+kLC_Fc#0d-<(9hjlC`dQghJE(AnkS{Knx#@f1h#v_6yH24@Qhe(>O8a4jqFsu^e^ zpzI&#H^n^;ry;uhUGY<~J~Um$GNO9MSi9lAarXRn7CxU!@e&Y=Zl!VOvc^&IOEbgd zt01148q;_>^S- zP0~9+SHJ=ny}MJ$m@zP%BA!L->54FO^=sExe$X$|X8l_8>YHn;>rlH_B>pjZpZ8Dj zxt%Ryh4gp+XAt!mLtWU-B2@Iu;h3FO@|%!&$=@}!IgDSVR^B}o~{ zv(W9i?Uk>9?QOKbh7D>JUa!1t3WN319h7(vLhdEliL4!7#*3-i(715W_?zUeh0^{{u~4jvNmq#fx{mm_n-fpm zy!e;vim%*)_|h$j&)u?k?2d}h+%fS_cS`)joe_U`=fz*$1@Vb{R6KGQ#mDY3@sWF+ PM&!A_OrCP{rTqT@L7?BM diff --git a/src/torch4ms/ops/__pycache__/maten.cpython-310.pyc b/src/torch4ms/ops/__pycache__/maten.cpython-310.pyc deleted file mode 100644 index 89e4d2c06a697ade2e23daeda0f538ac7b80536e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 46747 zcmcJ23w+#FmH+&H^UP$DrfJ%yrNHoRpcGmVup&^Nf)EhOD*_?SZ<0=$Ov0T>Ure;n zmbTE+@)jtV%Cjmj<=OJk)pgZfUDtJW9o_ZSNm|!c*ShMuDF5&G-21zkOeSgTf1P}O zH}`k$x%Zs=I``gl&%M9inwoe3zgPeJmh?+M4FrD1m(pJ~Ue3XjJ+>hb@B*IG9cbZy zx7+D)Til*tORy)@66y)JgnJ?_5l7=hTcUy+Yl+GKs+PF?uWqT9{|WQIrlnTiC-u~| z)S0-Gd#YOM#UJdR($mn=ApQ{iQ(LC?Olz4Y@8Rz0J&i4m;*WIC=$Y9v(+Q*lEr)nf zFSa(&GAlJJRozwBb!gXN=Kta5f71LvVok`a^5Sa)Yn+xNE%s4soDqN5(Q6#9+DmM6 zyc(}|t-Hnn?lCs}SZ|V7hrh|2Lt7nhxmWK^Sxb!LEXM4T)Ml5Y*5FMwsWn;L<4bUl z7u;#ybaICjPq0{XO0eb-3zS!Qjou7W2Ih$tb8ZRdTuE=HcZf;vB#U)2ayZL7R7zo< zh0X``Fz;}Io?@Y=3bmD9(mO)(`92$Rnh815J4!-Mw;^Xp$SUt>?-&VrzYSSnLXP#0 zlaLSCkTXrlY_CZ|K4?SEG9kx%CrHTIHsl;|JjXjxI9_O>=K?y{J4vADS?KwIp6tyN z=mi#fA)xcUQv`aEg&qOusowhp`XLLwSopZrJIy;?__)M|d>HNFjLojp%KN+z&le?YqrqKCAGZwLGLWVzru!m#Dtvfog*Pv+K{VE$U^U230Y)At~Md(dFM;W zH8$j0P`|*tP^e#Lq1OX?k@q2iwpi##0lnC}M4&fV=#7AW*t=AqH(6*apqF{g0=?Nn z+W@`XyF#FgEwmlbk9b!K)U(hOpjUZ|1lnPtoq%5LT_eyX7Mce1TJJi6c3J3BK(F^& z1lnz(J%E1HyFs8C3+)B;M(-wp_F3pHfVO%!3shNX7SJ|tu|RVc+An21;CWt3%6OR# zSq}UTuT$_>Sm;W@|CpEdx&(if4Y}2XEcLo2ByU3oB;?~>#_N@kkJ*rquW_oJsvfkl z)!r?hlF-#Q^frlio0s$YCFFJ+vIe{@^Og&5Yb|u0;NR}8^i~P}9X4dWgskyy_3{$3 z!G>&2Hd&m_V{q1ZYX#?S zi*u}@w$8gl_}F4Ywj%a=Z-eCM9t*t}(2d@m0=>^d?+5fQZ<9bDu+Rqq-R#{h(1$Ga z6M$~08U$@XV0DZ~(oIrP4=q^A%@4bvs_!aL}X&Z0au(u?~dG9sv zb;Xwv z-))O|U-iBw_}{en|7hy=>)ziB&2QO|e*%yH;C(}Q{I-SuGoat}{!yUcvCw}3^jqFP z3G}-b`n{65!oLFJ+ulDD13mTo7N-FCcf5a*m_M-49|HPa?|TCMHw*oDK>yYIzCeFu zp+5$+;Qc_LKe5oCni~9}_ivJ~pV^Sz!2fseM}l9p&>=v7?EOTb!xp+nQd#N!)ccvF zve${D%$sg$Wt*h9%@bZOAVrWRM?in;{kK5>*FxU~ z^mpFx1^Ooo{WG9{@cu`j@A=TJP8aGDk|6K>(feO=z13;4vs{EiB1`DI-k(fpuq1BC zg#Ouk&xD3cLL)j9@dJ1qgApykhzZ69gmP9CG#FJS7;(Yi96C(S!CiGpXhK3G2#uQ1 znv&3335_AN%7jiT39XaRI6|vU=;V^ndI{Z#&;*_u!`YM)j0VA|Md&0Gf2tqa1qtT+ zQHIk5vrbDi^cySF5H?-H?nEk+@zfhSjehJdNIk%(jFEegvVb>?k+YD=fKM%#%Ahnu zLZ%^9Rt_l1nU)HqGvJMtXJ49mKh0vLJ4l#V&l@%}!6%3Z(EIfxAjKh4I zz&PAu90rWT4MwsA;|Rf6h0r7(Sy_N@yCOpxM_G&`fgvjlU>seN#xa6%G(wLt^>l1W z=y4KyEJBYnp|eXun(4(6dTH&z8`$5PG%=J*Ome zp@g1;(1j-ST=@B%w-z#_+T{H^o zVqjfjus&3RrB`s95c**}mzuJ@qy*!`!Y45d7_-$q$=1)h%> zy3HkVE}tOIm56hdiE~9soR3TpXA$CDZQ@*666Y$E!l=17@UFpgt-)PXf_wD@>0gI9 z*PA%kl*GBVLi#Ph{iwmct_1h`QR$mm4sdVS7KM8wo|{apmXcT>9TiJvM!?ji+X~;! zCe95dac&$HhxxZND#U2Rv)IJCsU%jbj)hQvMN=NL9bq0kM`%n7sfLMPxeSS8v^GCZcGd~xj>hb zck+P_HyzmEJn7u+IDuQ-eBi3U4yU=vRZ-v;W@>&@sx{ZE+LyF?>1X>C8O)dIL)1h91V6 z+qbe1PG>SH)kq#AaAM9hC*-V}+}k(z%Usz_V!K3|7-|OtrH5XY4EhOB>W%!3_TCez66|HU=U@IcebMl6#>XzuqV@3h^ka} zNn2kEd3073!s6HTgs9M)+{%2k_Ghwf9jVr3sI??g$dX7Rst(1!y{r;cX}GPsZ%LbF zg-}+G&r0@8ypWB&+Yp!@7zpO@mUGs)**f?f`qZqPI~dG`2E*F}+ns?>J~$Z3hu{n5 zgFfrgLOh*KXR^6AX^|b>y=^&Fg`^Ad-29$wtCw5ZhbAgDQ<&VD%C&l_j<)`8{FjhO zTbBQu!s-x4a#pIt`9FxFC{$%q%QaegDOBlFX!E?|ft{u9JrQhhYMnUhQZ?cA!bMHN z!qjxO6^-1>_Vub%D_aoz5IL5TL(3wKl$UKyWtO$GWc(lfZ2hdX?0h(R%>KI|&z8EX z8s*@k`W!UuAj&L+lA4|m42B7FP>NRp8@K_gg~7AZ8)_E{&wh}23sWm5~;{>2M{w@=z&P`wX+OAh`3 z;e4blWAKn%!zfZa+!9jK*hgBtq0-aeeG%dcl~_=n0e?F);s5BjZF?48@~+fYytKQK z@*$wix(TDL_A2&0h=)Rzyg}$JE6sMHM)me*JP6Fz9!QvR3ILHV13@iLLJ244sI%}s zrVxDg6eWh~`RW7k7lM7g%hlDX+gh)tc?^*`d2oA&Bro zl$i8{MdFOZ9Ab5Vq%c?Ks*g;>oMJ;%AA;l4t~RuLQf(QSm-C?x4ko261$)|7?9ftGHIv%qa0-!SZQcD4 ztx-rtNTh5zpMGa+ThRyfeeA6<$dLxN=@z_<%dWZ_p=AuDGwNDGD>pj`<`oS6D4iSO zj7pmg&M%6wC8GyVKFJN zq5D7j$M%27J!!iU2zLm$iXy;J&=_|j8EK;lSEb+|(@}jIG7RWuTav&H+i?iwR0S`1o4b$Erie=PZoDx zd3jDU1bwdVCrzoe9unSvkqRN??hv30i7{=wMw94Esh<8^>R}R)oUz3p0D&f5XKDr` z&ZJY;Bux5dGWWPm{}!gtcJ2S@_v>ryn2=bkdfU8q%#Sc3wk4NMufi-o-P6~dZcpdR zNt?EkTG7{*@t#2PI%oaxH96{d{2MoeSvHMI@@C9rRX5pa7ORldDoj@O*dl$fa!DA> zXF))Zk&q|}N3r*h%g)&HsKT6AlT^zQvpnmj;aH>>h$Hp7Y9c!OY7y!SRSYu_qa0=& zbqV06stb|!-b{O24uek=lkaS=%4(5XmR%z}f%!5bOE;}`)ol}zSGOWm|xPo6Q} zCS~cC>ko`Pnu$odaQl+}%u=6xMBXMOCm3I{j{UV&bOM5&T1&_5%l*c6;xHvrGj4#HRhwL8gsW>_mJcc*v&B zf#V=*G}Ja;p_me9_bC9X4RCoVGCByJqoB9r0tuDp%_pkS4<%6<$3M@EaU}D9^p8)k8WOtI&{UrT zYB?RFDJ(%d($abb_{0CTiE*ujVXoKDmu(UWO*vxxg=KzEl#&(p~UJNGbi^e+$MG$c$FDcy{p~? zFT;;K%->#P5nf zuBieN%E$B3=}@2^5*UE$P|~|0UoE&n;x-T-68I7W4-pQfY<0dG%Bn~+=1VA#z+MW~ zy=)hY)WvXRIR|~AtmX36Q#q&}DSEUMYmKx2RlGLU+Q1>d$0Ew*|FCq1u02 z)~1w9FLyFV3=70Gv9Mpz+mp)V`g_zaNs7vhLaZI26x1JuAmSAwE2Jruw?$LEi$$N$ z<}nA^ku8Eys4f_Dp zaAIyPbkFLW@E**JAe)|8`Z9bGXy}pSTS$Ev4}L#Bdt5=0Z9sV5RcwoPmw_tjQL4JA zhi#wUe!C2C`qY;ZuRO0tR2k&!{p^O^1og`7Z)5oId!t!b`I><3cEiA|1EL+>Z8`_l(1hiFh*$Mg-^g!3R14Wo;n>afE!SxqPWn*eOe69p2ih zu#)y?B$TPFHJ?>!haw09Dv}SF?JChb_SNRxF4&&=+DAvIfY=9^elWE%Rd9QtO_Obp zQ|3d}?_>&2k6@_+aD)-{1NzE_IDY}MGL`bnHRgt}8Kr&(Y{*g>JId0Sk09H^$UpNj zdW_)2+FxwNS7CcE+uEI8n%WW3%k=S5Eoaa4*pURfZF#d19t0Cr=(w5(HH4Vugv`3# zBHd!?{G466mWz2a4ysp<~$2?ZidDfnv3M4Fs2D%WIMKZb6WpcsL|=`E}2!=Gvx?o z4=`4s23Qcw+MlUE9r10HsBC~VI1$WSvcYj-Ls`V$_++Rt!S)6+B8yaALkscznoZb{Y zyp4+R9nzIylxygN5QEhwnT~XK_YSj}Hyue7!Xg*TX`3Z&Nu=+Svhn~n_1atMqX+~InjEZVF@z@^zWAH5v0n#@BiqJ;GqcKR|m(RhjabBJ!IzLJ{Qp} zHVZJ6?Ff;0A11D3L?w}GA(&O|<-GX110CyUWaCOkm^~32O4j{cD*I%s4cS&0(5PdW zN?D^a1zm!+$I?1sB3jvLl4UHku?XpK@Y__uRsfj6 zr!t+nCFLBMo|eJdw0HS`f`u$w_cT|XM0Um-c+GyKW=9yoI)po+juf=dO(@s4G^Gw_ z7Rw64kk$R-H$;%~o)2HaO?P1x8Iy*A^>I7+ksrcrce@Dw5hhR)`9K=xZ^DE+6_`E? zu^!ldb*i)n0O4viVwQ7LYx!?1GL_HJJ?RC=oL87AUB*$VBOQNWb5cOE*U&9u2>!OOhobG&(Mu zH;f?*!48F;S{M{5PxcShhmnR~!=Vx#nK}p5bYH)m#AQ`h*7SV+T^dH8ouXB#2D3)r z5OV4rb#8gNu?$&m=P^xj*z0`;f@v6!Eo4v`C13fbDFLgVrE(!L&nF2c>Hk;!J%e*GDJiP$) z3i0;d9>~H}YyYx>(_e711!s)a+i0GV!gLl6b%PC9MT;5WhmjVD2B8mV=j4VzfMDzx zfPIuvak^2cOClAwUBBL>=nt6Bm6kRj}Ez=u`NHh z(?8${KTu*K<|Z4*=SM6ZNAVK{KPWAqA1FtIP?G)vDZ7TR&j??x;p=Vi1qCWneDYuJ8qgDq{X?-jCz+<)WP2QGCU~7fQ(T<#ffpDyV)FNIRIT zMk?>hh?IcJHJC{XG_eMD;3}G{$-^!k(5V8gAv8M+d)KV;$+xBNa%)gn(lR=8!~jS(ios? zQvhAWF?5$$x=YaE$X5tC2BbBnZ{%^dOugn+XVAl-D6h`rJRAOp@=?Rt z*YZ)rSu|5Ga9u**0bMK>2BH}(8c-)c@Dh@GRC;J#&*Xs`*xa)0-L0;=Bv(H;1%0v# z>iAmm<3t<$lRy#6m>K9r19gb$=Iiox9if5A`I@cb~z3jixxPj|yG^$G|&Ve@(s)KD2{TIiH%JDmm8{(SFW%0XmZN zsg!904Vn9q*Gb?os6JGZ*KZ1!9~_m}g3bZ*I?up$JPb7~f4InN2t%OccOTwSYvTW7 z#&rUju#G?267J~AaIZ?|vsPdSWpEjmEg6cd#N6KTGDGzKZd3E{3qXBO+ zN<5DL^^c)^Z^z~pRQa1^mvkQ7{Df2Byc?-`3C(oP3eCbqkKGxU-9|Jhwpyzij|9a3S;<@wLEVy7Yos4 zz1{tsSq(q`Q1Q8~zj|}io_8J|`os%E8&?l)+gRNE+VC?k4BfYR_gmW#r?~Th;!C&d zlyst7Zr`(dP4be3*PV3Tr3;fp5i)yM@7nYBJxFZ#Tkj0t|7`LyOggWkwSIEp6&FiP zafUws?9kdRzk2hL;`+CS9(Zy1xwng3Hy7Vow|D!l-LK!BG|V#DB-vfCpoCv!Xj5_B zv&Bts>|M7RNl1dTWwkXYsW&F)B)Ks;CyCQCX)iS=*@txzX7qECUK+a;$n%_}HIww_ zBr&1D^rY2db;3xzFWXds-QoLR+P(8`owwaP@2Gej;e(4*e+nfa9Q|zdZJH%`$kXt1 z8;e_RN0ue=;?Dc`yzo@tj&kk+=)X;{lpfpmB zi^O0}SJ&S4n|8mxdGEH>d!D|nc*o|U&u-UoMmS%HBQ*=;l}$ldT9DlH_I<^5PZi%B z+`AriuwiW48*L2D{DS1rV{7)V-+Vw(n9T*r;%!@o-+F%B2+5%h4he2S+jch?)5foqO@OsJ8WhrPP9I1q;pYu~D|+6P$79t#oOj0!PC zlT#jrsA)lknBQayVX3x4Y)M9%f25|noqq9mJ~t&3zXRC_%i{G-#)T}aq! zpPR`P+)PF*X{xmyv7E9fM3~q4o|=Ue#XO)8%knfAY_k*%1;ofeW1Zp^5_&VvY;R0g z$C9EoDo|JR^;$aDk&F09_bb$n?qG$8?l9_7VpH4OVvD3z7ez}=roTs9n#i_sCoo%} zmZ6NE@3mCv;)Yi$mMpbISl0kh!HP`bAXp1Ey#T4>NFrh2V%(zNPVrjht$gupmg$UO z2FSy>66M-Fba>2(g%eI9=wiowCM_JG501NTuofEPdct7`Jrh53b1 zBEL`6kOr(Ug1`($!Uc91GFHz2x+>xNLLi8os)|VVxhX-!BpIZjZb_qp0CDY z7Is2{dj)n+0ObLzd>p!yIQ5lc<0B4=q_Vm!i-3KNaBjm1up6pY=beoJNIP0uTZ8BsydV948!TOF2$s9AJskX>S+*nTK!0^T4v<1tVOPyuDfgR zBX{U25_D%wv=E{h!wz!i$Cx`UMyg^{7Z0jCNEMI$(eG2;r**D0-r4S8 zBM+NeQ8M@WV?e0u1?=k!b*^UC6jcMQF-W%ui!F0NZM{PJLN>pgoPeYLpd!J!9U zhM|mb8P37BtX&FTSH?mlGY+SMy2R(SKB#CXFlvj%S3#27>?$}L+%9PE&~c+Ymn?7W z#*=g89h?sYH!vKhbv8)o>4A-Mnti?75yT2Ov^JItGS(W0of!HPrweOtFYHCAtLOV* zcmuUfVQ6;3)Xl7M&V;EUN+jBUIv6N!dv<8k`r`AC4DWimxD!*yJMD^Q_~lO(-+W;A zuDi3n{i;1xd~$Pf!>h&BZx%PaxI(5Y(y>R?1cnb$JhO-Fn+>Ov-MMZ6+V2g4r9p-B z*@F)F#Tn!osGp3p!MVr1h3Eem13g8`97p8ZGMy>uHfSI#QHlj8jZoQa;rYMR3Y`Bt z1W|NN&~U5}Ka&##>R-=V>DR&Rz^bE5_rTcXO1i&|p!Y3K0ihjHwHW_|5j`&mYi8a@ z=vq4Ls=m^f9I1N&K+l#^lD$_|M?7Xp-9w-7XVkvJeJdHsfJ@nQ!geeA6nYh&Ss2nd ze8EWV6NoydcliozIb>vd*YGEI?ES}TJNFkKeY|-0Cv_*+ z#1|r3IU8mV!%N|;(2_g+iVB6)qm`IJc&RX16^Bjm?0Lx(k}H~gIU?sxDxZ`w5(IJP z^=)RZ*Mo+7{D25y5ShKADS2XYo+SiO&%BD;5QdN-+-C_5PDp42LVfYL^`70Y-!)3` zs0A!L%KQC_9XbB}Fsa_f>}@|Ns&|f}3Za}zckhR+v0NeMCoR<{NR{2+|IuHmu@)bF zbkBy(yLaA?b-68BIjW8~c4;hKQthy8?H;rJ&`Rvoh zEw5?e?^6!Z?yh2$`t~^p{|sqK)1jr#m=437l8kKOaGO*g)9;T1R$P5AmdeR_C#_He zh&z6_Fw#LG>#&ExT^=uy==+g`+Cic$O8>`cARi%mDLl~XI+&;>upfhaF!aa)nU5(H z;21k{L5vE}9B<}L7}(RH0ZTayxMFnN6zb|?-`-Ac&z(MwN$RU`d_zeJ*-T9PrPsM1Ym zVRtvCTQZ!>c*=d{7MM9nqruHU721p`P6U1jS&*w8Vsd?jG?zCf%9?r&p(s?cxqH_x zC};KTZ~ts&R!+cXX5|EIW>)?PY^t3sfH%q9{v;)9Tq~HNB=)PA=VsCn4Xv1+R~+K~ zefJTxpOdVJ$+r)VER>c$S&q3v7A?He#KPatwphQmM880yc1+qi z%Rnu;5ZL8n+F0=TI5A8AtA8o|hX#jtyjFbj8J~Wb*D+PL^R+n#Lan9zby6O;E8U7r zG`am$JPd8w%6ZO)2ZtVCyLa_q@r5_`Jag~x&Uf_8d52SQmikQKf`_)w9d;^$7hYXf zUKnP~B5niNk)>&EP<`t_7&-zBVO2G>@wVa{pEny68y+q0+G*I5*}VFDB<1r2Wo}z1 z503yKQ=~|5M+a1xv!&ceIpBJEEgecG4-o&GCdjB zMg})i8VpZbn%_Nens-e|vwJ@@yN|Ur|CKcHW`Fei^8|Ayu*CXFjQh|;m~-7kY|Cg0 zs((gexQYx{_RQ1&=9~IZG*j9T-Mfy zV+dpB*Jf#^L+682vDngTTlJ%IeksO%QC@P~67x>nu*dbi@OLz@Y-2BOdQfwT1ta>S zRxs$%*Kg`rZ7)l8qx5AukOa{}ge|{yjB%^q=wSc2fV|=f#GzJJKP9Jp^M7mrd?&Zr zJK>0)g;T|<4bZYo2d4s|=4FaMf3}*JS^wXf7o&HxS75!aEuN`taNn=+fJcZFv50bmp+X&Y+UI>csTb|MQ}jd~TiV}Gp;sUdPwx*Vb3=VZiY zRwmFqfPkJ5lxa@!73}9f1^WW6KjU0j@fbCFt?V$dtP0g;508fm3iU8#?oP=mK%Tkj zz_~!En)=#s5};hvnDR{ZEKZ3HK#R|DU0D5`9OK>oIKhnKIS)^^3yv6j(_SvbbAymD zSWJ<3SweARKo(G52#`n~HgQWsJWP?t7Q%EaczIO03k$Frfx#$J!XXN5H$=7vv6&EE z=X%i&*tp1wlu{ukwFId(^x%udx9;8j`f3UWDH^@`UQ#+7y-{iBfPYz^b$;lDHy`=U z_g0-H>*;13mXPIHl;Nydbr^iG#1?~4wU93^ZXb-{imF_?uY0AM0uW-26gTb_)hB;( zp(2j=w#r3^wdOzt8Yk^NTVlFYvD#yjYC6*mOVyOu!gSqnY%SpK07-T+YlBPMIJD^X zZVa1W$7171{RT;msu{K;)|AxDTzIs7CoG7uEOzk+S+d5(YM%OBZ1+&JD>B#Z zXNe4ddaaeV*n%%^SU!|Rc7f;YbP>jE*);~GzxLalKm zqd3hj#(&5rG6vPP$tIei$!1V*EJ(hz@s6~bW=J7s4QCKm;&dzOmKOx+9T>^CWZ9VP z%4(BY=IxXoPMGIXeU&oqZ$*gZ|E+#SaucK;VyX^*&{6yW>5LLjtVD_^DUGAyDXYjJ zE74XT4AM;X9Rbw6t2YkawWCxsPrIYd4`8oCkDASDAL#j$5Xp#$+)C9pADHWHwQ=Xw z8O11>w9<o>qbj^^0`Ay>w4?$*0UphU%x5drhDtnMqwQI=tQXn99a4T;tG64^mO=_W;4y z5(r_ed>D7q!lqjGr-J?5og2x8-F6vz;ECevYmgn&yklD6{aErt=NEM1aM0wjB$$C@ zq`5am^$1T{B_nZ56^o~pv#FM9J)6)kxcNYU32b1cT8+pnmun4Lu9*W&5PdxS^BC)m z3u)}`qbJllv)L2=2#TXb8nA3}%8?#3an*S5TX8}b`>=};O5XvydXBn zXRpHEo%9#95(+Pjm|_0_Vm{IVtM98I-g6Lk9qvlxj(^t&GR|`Lx-iF7Y~%;kUEFe) zo!?ws9~i{@e)A{+QIjIU@%v|ODLaB*NU?_wMP>`ZEaVRMfoj4rGtgX%`S$6;NG z6@%sj7k>z(T2>K0Z`BaE`B3&&#JDwA-Iaj#wChE1AQJZ1ctbAbEh4FhCH1iQBjQJ^ zXYR1=A5;%Y`X#9wia5G3!bLu8D87ssFHc0#PZ>LwJo~Vyx9`Lc>o(J@UIYM(o8CU& z%SI_!h<4+G%@iz!60BdbIuKb@qi!ZZWYM4KYlBm$T?o^o^kSUY&_+kHMb6HocjX_X zFU@3e4Rl}YQmvs0b>IYaQ(Skie;|s;-#^nQRIft}8Ao6eq|%+7iSeZYGcWHN_6~d*;99Lmt4B!k9wTIS;86esWkIKNFeh}?} zJM6@13d_#H(B1bIcW%+SLWu;_zroo7`*!>*#BtX-u3e#tD&&)tHf1!{*Gjd?1?Rcb zPSsQN5YNBLj>D*h1&=+^jDi+^$`pJd8A|Fjx?_FvyhM zH9!}~gcpK*5cV`(AraMXKNWa-!&7Hg4}vO;bTN~c1xA!Ny!#o@B0Hvo4?$$zVB`c> z9MFn$4s&4Ar1NX$0uSUJBy0_Hz+v0}F@z6=lim-}TR)?(-^H|1Hi1NoU!t*}idaLR zdqGXrwyTh0N?ty zUfldG>ifwP#ICCNlo|Qn0+PXx@iRlB|*BP&gYr2gQqJ6G(s23-u=BsO!OZ1QfAxb5lB?@x8-+I_5SEW+? ztO9O9uSnA@oGsE}mhhny5wzZoYYIHY>952qDx5U@+q9Z-w(j?V?r=dh&SWfqQ@sHFUUz;;o5< zpMwrdFK&u&eQNm5S4~AJMRGGM&@?C6OYaq)Z)@quO0+rjP#UU7;KO|LO!&3T-t*4m z!@Hi-B~WUe4m}Z0CKO+PU7u3G-i*e?$_oo;T^Dk<%;@Q8!R&v*j5e3_&>&TM-TM*A>9Q@a7LxCvwSL3p=O>sdaTct{;xsR(>B zqIU3CEBgBdAjsA<3)>ToRNezBE&RS zZ61|PnCH@@fm}h01|PpDV?%&xLf}J4sFR8*PM~mx4W9@)0dwI9R&7|TVfWw+%!skZ z=ef08j>npbdkp6a46A&BMFn9Uk(Ugw+F!X!)Itc6_ctGU@RJ7#sQWr*FV<17b-`ru5gc(}Hq3d?^QU~sQaXYB*U z{y5Z8i^C_&<0#loL)}X0$2zIi((l~!siT#nSknR~cC=_Cw4sI3ykm^xUDP}32Q}U> z05zqw8|cVw>wpif13GeCsTr2KJ{mD1(D-0VfLQ0#Znpzpu%yjyJ}6XZXdeQNsJ!9Y zLowrLo4GL`8H9RN(m5OL7VB+JEX2gWhhbRhL)TOxJf!luh=jukFZh7 z=A#|khaY{txOEfTs5DklT~)Rgri8ePbuZ0xV+|9eK^ao8vl$`$*NO;*HXisF#jsvtQ2~I=br#mr!pZ+yf$Wa3N zmRdQwM8$s{rr)^kRl;i#jw?K{TZQD&U^s)aLn8bljO>tjBmfjbRI02A;{Fk+RG>WO zCvn0J<`!QWBsmTbZG2|W_DxCMhUX=ZH@^As$#qkhJ;m+6lhZg+#lEJmby&IBM|yJUsi7 zaP;h%^Ox<^exag7=Px=>bH0RQt)5$z8PPxtY6TG{HnP%!o;}@f_3Ud9Z$}iLjq6>G zkFNE0i`|Jf`sVaeah_gq7MJJ6tcXgJ`jV{(G)mPxQKr;~5Q+&{Dbu5@a4It0J&I{{ z3Cpgmh)3A6$X{nUUryYQkZ45bt-!oV27f@GJ*s3!an9YSb7#xC2+zK|;p8K7=q)OT z-Y|~YLvN9B4!sH7jN#FeLvIluh8s7cu$`Wt3+el7GIG=n_j|;;s$>^C1|4`6wBf|M z0P{BJ(plrYwVJ{yTq?HFuR+rqx9O-W+3?CL3u~UgPG$Xv zUxcd;)q7^VAo#CQSO4h~(skvJy-C)FjJ+~r8n>=Oi=)G4C|JvT9Xa`eh7*W|bF)YCeuzlxp-YS}xv&yLki-ISm7X`1P1uOFBH3 znkYGz8WF)~`mJ2GV6;)I-{hoK`tbjEr~Q*(J$^gj6=HvlcJQ80$h3nRl|)5o6Tpev zfmZyHFX_JOI@U2eFWOQTX%yn3>d`k^PJF_Y6LU3gNA}gap!9oiE6X_=>(%1>eC)?A z?$M86@??7R4W-Qt__Fb8o7q|b&_H*72~r0`;Ru%7d~ z2Xq-F-2)bW=pN2MWSs618Mk{xrE@@gz)pau`f&NP_3cY2P&A~ubSeC#HReT=~lFAb~9 z%CT)qAzao7=`|yOtLQYnOKb`_B<^7p? z_M)3Z12hLAW-nAxv3x?N!wEErSpQKdB;6!<{nKmDLz!IM@Q|+bcoIKxN^aEf0b_N;kp_pYaQzqOe+Fv$&A7z~PU zZrStV8r-iWNsJsjY?kYQ)ECe|Rvo32Nc!LYz=dE*d`BvqOU^@)boTSwBbk*$x{Hm9 zC{soT2pJn_WuVn6ceBCDU|w?ojThghhs8p{e9QzRzW$gAJuwyqFP|BId^bl2bOz35 zCpsEAN+ft23e7}gCI64b8!nUZXYL*twh9CG7)~kqUxe8?@R;MR+Li&vCTN#1QN~x1 ztWgRkeJiigC!b=$QTjAe$E_DOoh%-J@7wZ1G$*tl3)iJRF+zXmyjI}jC9aY^w8<{hcfydSt zH?77E1Tcl%`^c7|Z96GgWSXA*=*e?%M*shMey|Kxq2~u?5yz`z{u-0_6+WS6m1l>Q zW8%^v>vbnjq7)8fITwY=IeE0Zh?dLas&%X3i-gxL3iXsYUp zl1!^Uv;?Fj_B)-{u*8D5ucR$7@NE8 zUQLtrY?FCWEC6j9I`eWkBlyfzs$(k$5vGy1S$S_-{zC!;i43{Fw(swDH z3FnR44$^k2mTG>U&c7ipj)VKgU*5mWpsNe;F7_}=OwL>*&MaJhHnO~)gjl^H z7FXCXbH9_jmE2>+Fbqx3c6TEzo@M_NA6dbjMP+z-t|uaLDu{=oH@=O7;MAqc^#N8B zExU-8Nw6gCkD`HvEz(7953AO9+yd~UORJ!JX-_FwTe~fF@G%q5QauB!Ey9#^FgAVSO4`n6^ zj550cFjru(sNktqNGfc9VQCS-joUNx(8wc{rzg>-VSfpGE@O_QI&^hkFLIrAc&Gz}D!aoQht(-Ne)2iEBV^N8hnhi zHlN5(;$7d3+D_m_!Zp;3Ob;+*ERN8`cw>2FesB{yW^LHSs?SUXR(%Ew;jSqx0btM0 zp^b>uIqtIOqHv@I)hDwRg#Zq{}U6ehBuTO`L3 zw5&fFoo>|)<`f}J1Mzu!TtU4om0X_AE#Z}_xT#sS^`u}!B^R#B`NR_e*H^9L+Ert( z;9q`AUEIpj>u;N;>tU;(otEsVZaOk1aao{ej}&QzYA}EUJE;sF=;H&bmQFLG^D{HN&otB5Qloux zo*S?JAxo?*vkb1s_dEEBD}R@fFkExWuUF9vJ82wW!h{o7G<0-O95Pvaa|hR|mK)4K=5Z(dxfo?( z3lPF~6s!ZV?b;ZehVeB785zPKm%Ro5i%jn!SqP2!FJ=|*t%C}zUJ z2;O+0Fzy`YDpIryV-S_G*v^EH1wgcXVtlMW6Qs?JOAasKBf@CYA-&BjHe}|$L%q+j zekZO=$5q-`Oz|5hyh2%j!U&(EiR!y&vnmfZU<4uqOAKGukqYq#kO2e=MpOQMA;-fgVEfELC9w}fz+_1pX|V19!%v_@WGvBzB`9?ih* z_dW!(5JK`WV8z&B4s7u7hLpkaNC5UqMoG_W4DT4){wmZvu*DpHU=yxD-2KKz>ZCVO zHK1+-2e=pn7le5!+&!G?$n6N5?fY!14fo6QbO($|q$f$Q;kV3j{6j1);){7nS({2@ zhix^>h{=_Vw69W8l#_*B5OgqX8 zU!CgMPsWX_pc``lkHlhX1#Fh}&ND+*rQLL+=_=FAIkx<9NDi3>x)NU|5j>E_}l4}nmS7Xi! zy$4QQgTk3m<oFN(SXdmTf&?tumV z@TVUtZh3+`8)!Cq{4j?Cs`S}xS%k=%L`3u)$|RA~GM|%~>l(yts?ja3%8ng+?IG%F zk-xZ2oPtseqFSmc8JA-w*}FUoV=I2jOv^=I^T$gEqEM*v_5s~sD@enFS$gzmI(oak zb6IWVG!&AH8?d0KnG}T{?`Ea|L=(QJyc*fHS*=oy7s6xoe24O6IE>K_Ejurb1GmKB zp?A+k71%@X1DFVwT#6FxKLv^DqqOD(HTEDxt*ckMd!D(Y_~_fHiQ(tv<^lPf18PE# z=jg&x)*SV;rJ9sA&()gVtqb^SgcoYFxY4<-Gu3J?2P7-|LFIHpMQR14q*5SLh<>H_$mZN6&CA{J*3YV_DO@*?WR`+WZCiamySdxktiJ;rY`zee zRW)DOTME}S4aK5M*AoH!^bmq!+(*m(4q^G zW(fF5dC?;H7Qv^#96!=O03h~-7h>hcP6IkLu>nCu(a6n+#iU$am$PqSCH|H*6~Vm@ zICaP+PC&76d67Xtv4|ChbzD8)fs1}~xKln`z}F-38NZP^)lm#@lI!f}dc1{RPKe?iSZ?tHl$he-o1CP|F`-a{Ayt2sU@kTaWDsqcGy>CeNzpMU>}~l(MA4(S z)1evyU$W;HO-}8}DmI%!jsH4#Svs{`QDNQ^)feSu)ilPMOtdP7%2l(45WVQNrhX+F zTeLcqh(VZC79v;SrV;4zng|KYH>TC85}NH_tllR+%u>}E;)DI1ItxB^Hl1?>mPvK0 zbNPBcoeKoU&&Pg*uV10Fh|V>1TIk$B=SDg=(YcvU8=b{;JUS^lVz}8w9~ZBx2M#X8 z$4Oe8+f{u8-9k_Vr$?ya>?dRyofULe(pd!u=5_df+iiSZLuW0Wb#&I#*+}OuI-BTh zrgJx)Ep)chxrffZbnc^bKb;5YJV@suIvkJG!*m{@^C+EdbRMJgIGs<@8KkqF&Zp=+ zLFY+2Ptkdr&NFnLrL%+1XXrdf=Xp9W(0Pf@=jpsm=M_4y(RmY2A-=dR*SZV`4*jT)Aho|O6nJ>hnK2U`6Re)Yd}-lg*=I)A3~MLO@%2{>@DUJK|2OOU=0 z9e({+K1(ZCxu~gpokpjT&I~%U;1m)(@VCNz_Ec_}P)9RxHl5??oItdQ{w}B7ZGr(& zz9OsUGx&XUPNQ==oipgXpUwh0AE0vRNXbU0U z)+*Ps%FU>9jj2|AzDoQcgpqnf`!Xx#%1iYYQ+u1vJ9NH4=ZkdyhR&Dhe3{PQ()kLV zzoYY2I$xvnbvoam^DR2xrt=*-|3c@xbiPN2m%zytZR&^c@jJ+Uxa~|XFjMmw_EUnX zO+)j5< z4umlFs46--{3X6De2Xp3+$0@rUGCoUq^#dK(stgfI#Q(#5o4YAzRhG#S` z)OJiaz{dd@#|&wHBvwX+u$XBm8ow0cBPJVSpP^*4n_I~$MW>U_QaapEQma|*)OV{a z9qKI$^%YbZTym?A(cuEMP-RzTvH(+eFoesI0zT-?nXgQ03-OVuoy_A3iPCA5;_Rtp zx-~K}jGc*7&i&mfN)hN+gS6wB#c4CN2KtZ3;S$c5?D)P=PXw1 zgx;$SCgOY}j1x+pN=QVtAIIu>w4NdU!w^35?uVEDM|_51PtL~i<4RhkxCi4W&GIzm z<1Ozck|p#a#Dhj9(un_j{CJgURSF{vPb4w9QjAg<%4t;UjHNkrBV5k7PUuf{;lRfO zAFpe4aR&$9&%pEfF^DsN-d!v}5TqcGlxT?53${d26s-qENt8&@dO{CUlC_e&TJBtcyWE%N zUXTR5aAbjc#iA@XZobx*ZP2k3vuR^BkG5&!=KIpr{lkf4yG@$j#%U`@l=bytr)nbE zk){5AXYMW*Aeq$ZKm7`RcXsZX*O@bC&YW}RT($-W^A`So@b)LFm8UK1dwl8t8^p^l zJkcNfmZhwsrEKNY?V>HePSKTLujt{|t^18kG1JHvvyGq_*rsl-n3p;O#e)0}Hin9; z(z4-3uDCkQj}%8x?$y^c))ve@YCsi^yWwcsEVV9Ku=~m#HT<%zR;!U?j?}qS>P#%y>X zo>uGC2C210U{?V4j5<%9FR-lwyAmT3zoFW;8uDtRdLMg1KUd|P+;}dJG_Rgj7og7e zDMxKmn~(X$4@#S>YuD7SMVsf;h3X>d^V%N#I>4V-m#9kxeq9f~qeqACs6EKicsWP? zY;VK~_5NeNV6;Pe*#QkH-K2IM%gXHTmim)t;DBqxtkibY z&FU8Rjq&f1+V}Q)?_$ucxm)c?Yu+n0@9Sae01pSNc|U&h%9#q(t!l4O;XbLkZ&}Se z-ahoQuWnPflM9Rcq|Sa&=E^-z)C6XalnqN$J9rpbvcC|0Mt<>?8C=ZSui~yC9M6TI-n|2`*DF) z0Q;nxQquxc0t*2YYU+?crUbI5HVxPds;(N+@}NMuyQ%_js+PcOJ@_He ziClec7=*WxFV|VB4W6pw_ldBfbl6l8o;Yky71WG6e9XhzwWR&_THCb7E+bDK!Q76j z*<&thJkc9LckATr+#IApUcIC~syc#4)TM4QGDrcWH@ptCV+aETgmL9LPno+E0Dlzh{!=cty6-ICHM3|g#li;3qTx~VW^(5FA zJ~0zED`DdApJ~^_2|HP@!|7@ihq{Eum4l_JnP!ExORe^VlVn@%(t&amCPAazZdaSr z5rE!<)$mA?pDb(LI&yn+s)cS=m&zbn35XnS)o0j2#pd|wpMzfy&z*RpwMb&CX2(v) znsTaE$NrN2qP@>LX%pL=V9;vSqa@od>$qC3Yn);h2F@>`m}CweDeLLzq@{;Z_~R{m zc05{XVb+gIo$Zb2@T*CbX||JWQ(^VWvx(c_TRAds*%B5J z43z|3Dkd2NneY=&mE*D=!wiy)3J+JoMJ^}+)gdbBFp_mJ$!gZgm&#Gv-Fi;FfTZ9L z*#-V{hHQNa0Qy`cJx(FVkaPha;grXbiG^6P3zB)Uk2Di!3_jyA=4kaHj| zOp&0QIu;&xaWEe0K#^I80~sW=(P|Y**uFd-UR)iIEUq~L(kkaTN$5^$4lJlltWCTF z;6435KoXzXQsP#dapEJndVAte)m!B_$;!`F+mdXh)xbtNnmCOp$)zt>Z`bSC-m7D0 zC4S7UmblYloOoo9#20GoEbHXsZJ?hlpeCJ?)u^|eohCeeAxebwdKGfT4ei!2?{gG=#5WbjNTWAmT)j+Ys6Mr;pH<~^zN`kx>C-gCe4_MhGO`i>X2 zzWUeN&c z=&){bex=@u!s38*S(2{0FVf)(A}2#e{9$Nmcmt4%C`1bG5GXusdv@N*gYomWz760W z$0H97xKBWEr8-s((oxxm91SjBbYgeWQqBod$ciBz7Bi%qq042cueTtX$eOL`HOq}K z$&@29gRrsQ$~r|DF3``H9O58IDu%RwJOn>{)G9t-5tfU8;aQ6hr>4yQV@Z{+Jbx%9lD?@|fV{4C=n?oSsF z1xvI8$+OlJ_((a&f_cXrI}1*<>JCdmWqAs_^EvB9Yu=5W8WyHwP1_y!i}wA10Y`7L zf)09r-L5IGBeJj&uMn-+_U1H$D270Wp2D?X+p*`J4#aB;a{=4bu^)Bj9f)nG<4&RM zIWv$f_#4~?Z*XQ1ca)xHNjD&uxb?6(;U#t%2VAoWRYl~geRE>pg3~W(fziPT=?P>G z9|}=u59LNwMGvwU10FnBFRopHf1W+Jp%mi0oM~*?S#MR!_2|YeOYP>y?rGL%$IF#U z*pA1cUX4?3P_vovjd*tf>|?x)=?jPk-I{uu6FG>axO%Euhn5v8C|3RGPO*mIoK2A- zBnpre+>wF6hB{Km%j<2Hhnos6<(EIp*KK#SnL8jOf_ zkQy7TrrpWR2aqXPacoE*iiL~Khz;5a)>@Rt`Ua`*ufx)!9ey2WK0iNzHW@tGPQEiR zW#mx-??I=~$)Ts*fOWz}dlw_y`c_DC_gwau)kmbiOxhoQ*IV<0J(z;0b5Mr&w9JX~HQ)f1JPQ>m*YQ+T&~ZANv<5n%~GQr9ZGU(#Zo)TK|`&SXCi3# zegqy4lYH2ubSq6iQBDd1DovGX8Wablha+Ki`e2;ccO~||;(%!@-4+Kh8dSM@s&P@?>K^Lx4a9boFsm zJxylvI1vusiVW8uLSw-2FSRvr6KsZ>D9l)Ea;>-MLALsG8v2v6gl1v6TsT4kC`l>x zT2oUdi}OIVI~OF^A45G@InA(iuode&@TPgBLK<&o$v|_aQL5C-QFNMut3QPbC-KNJ z89<|5Zwo95#2hO(<7HC-pxu$J*YOoLkZisT zR45PUXr`0p#V|z8~WH=g=_8f*0yxB~EhSpnA0edk~bU>dH*Lta+L@V4fwby0kHC z5#dPpv00`e64^-xRwmARy`3+9y>$e(tSph+E~|;WEQK0I=rmlY3cVB^cbCo#0%EiqtM)jT*q^4{>!{G zoDE#h9&x2+C5KwwKj!%e_fpa3X-l)NwL{2}2H@(1w6VXpI zd4|ceOvoepc_!RfE09gjlbsFir-g$~w-k(j=%6sxctU*#4TLS7QRtrU=`Z0OEWHLl zv=K{(VZh@YTDAd+K6%u|6MY1n8smJ9Z5UO<%+Z*NX<`jO2c_b9tC{PSuZ!L1U}!y$ zI`A@k4S(LTtr%PeHN(*@$$8ApO^6cW)v~n!?>le=Fq`Z|qa-yO-?dU-V}BYHy~m_V>4Jq_0Zvb~EY2@?S2;5`BJSZiAE;YJ;HD zH{t=#9qPZ=^q2C>TY(yO^Q&EI%=?(7-TWmPdx&#*$<}|G&Vnn0^#G&-JVE;Y?i8cGJ=ff6 z=F$<0EgSQ|3j8mb+ei)(Ovt^EL52gG@U8XpA9-?l+lQ=fjb8^VzXOa zJvO&VS{uCoKsep!UXlGXkiPfrp)=0mgXzv8mc+x=1H@w4o_F-S%je!)ch-GXFyo#+ z*27kOF0dctNg$d9{F!jXt$PmmJ};b18TK6X8RNO^Y8|kI2fEHvcoP$kUe&}eh4rwJ z_)x#vv&P2aAy~oF`|DcSJk_FWkx)}4I~~Jub;2oFa5%uijd2JStoFIFM~#tXdw}8s zyn-F8wV^{lia)u`jHd3$Q*a+UroR*Gl6q|0P)EgVg8bEDvo^we+Dby(DIKE^-&@3GBA&oP7 zVT{eZ`ZQYoJn>`&4{lBrpxy zpu8nnASzvw<)K5vt4M=R)8N-s(?C8tjjK^>A_p@xtWW7*V;K<=tP@YRPX-&WS#`X zS$Z2nKEwL>9ipY@vMzJf`~&}iBA1Gz+8>?ZMOxGaeaeGZOG_}0Xd zJts~*w~JXAVdB~DLbxd>jp3Oy{3e_u+(ZGSZ)YM_D~wC6E3I(K7^`&s>VLpQmQ)g1 z(i|>(OvT^kj>w?O7|@3U`gf6q4YCG5i3{gfg|e>u$GnG!lX;U@Hpcw!*mRp1^Tjdu z6i%ao+d8^Ju9S2e9aAtvR+?3~2#UamxFc&cz7ABxds%>1i_VwbFi?c`d{}gzcZW7s z)CstUy9fv+veihO?QqBKLT!CBlUtY&dA*0p3Ze`eH+|ah8zc;c=ZK@fiJ!BsYD$Qj zxU%VMFnG9TMKbj&+_ls2FNd+d>kOmjO$A))#NWb@`uA9eCK~o`buO$RCx0jtMk4*` zb9ysnCi+zA3Ry!>0o{1UVGFsw#=Jw))2hdTrd+ww@>2H}do=f2S+TE$w~~QMt4>QD zvL@oyNhJwxhig(~3@NKWisU3T5a$0a6S@~y($X){sr82#;!QHwM(k2t>LtjYbwc{z zv7bLgg4>ER^Wy0JBj)}+lRsu6i|fyYx)tgFK){Mc%#@_;PcX9nk1YQ#k_0A|-RK## zoH)ryp1Kpyo9v_Oh3~+7cpU19rlF3LPzH3v*$}RbKLtqtCnhrcGff;C%53;Oj3B%- z?CMuh{j4MF|IE(%NAjj-a6{z@Ws^MI!gb>dIs1q{Zl3FPddSU?o_>u(;*md2*2UB1 zo1ccvVWS<-o{7x*6ngw2hm_S7zn#EN^Lwy-H=c;H2n!L;)hXl?w^aQrXP zA7+j#t|u@mu{T~P-Ys;OBLm+(yh1qNd|cMR&x)&Nh5ZtTZCnD*#6qkjQ!mf*v&lvDFqz1qgf~wha0ixAenuIxl?B%$NwPD|Ds;JqxD?GzsdILr=J6tF zoRk)>hfEuH^xNoXB1m%6A?}`Vg{<{|W%p-e5`8gBdn6;N$QIQ8cW#UuUx{WIk3lNsA446&Q@bSx*OUt*Q{Y_~3^*PEIpYMz)oNaL z990<<*K!>+112E0*FCTAE;zk`CEvKjt3sH|#9$wn(OfbHz!HOHn&2*6*NXioJdtIsYna@D1ff&Wwpf()g8mVqe>3+O z29?dZIqf115m~-h#dofnmy%&Qv^SWsH`>ti+!3(udMMF>n|C}{yO4H0mPAESw8-MJ zi|3H(+r{k7S}~7OLC}-TZX4m|52Av_^@o4;Th^E z^pfJ2cmCcKyaHfEr-plq;~s-VaFJ*(Zi4=uFB5zm2)#khGMnswAvY12b)Zfpc9lfr zT(wPhxqgdK(S=EJ#tASA!00W1$+Ev-BDxIeR2)jBLV63ak{zL)FyJ7*n}nq#FL*p4 zXi)@6N1P#N48+y&=$&0_)PKM+$%$aBK7OgUrq7}&YIvNiG8yyt*pzR}_?+83*PQ4*`djEme;Wyv(P@Ow z06l#Jfn?~UU@n_Yq!Df^4f4=lcA>PG#Y9L#odc>#4uy!N;pAi$B@ts}DAj<|=2IlA z2u6=)-mtBQI8`oq_mBDRR7HiNfusTKDhLB&(89E+S7Xq37mAWK>9JO7=~9#`xcUK< zti-2~`WU2j8`!*w#~OC@YU06s_mBDR!TRP9Ub-ajvxuN7Q-u^#LS1Rq4~M|=06>X- z;IyH;q+qofbgw_cwuqHqJ}?LE4lEa|{R3l-#j51}J)_u`LMrs7XqVn_nzcO#JsbAh zz=060#Rv;xr$<@zVdN4|93CrHZB;+2rSRw+u$6Oh$Qg!s(d$;?jPfO%*%@)Jfj@bd zOTz?<2tUpswOQQ6K^uY&g2h{+o*NxjDed$IVz&aNt|^6TPmy*btaCmR98?smy|n|V zrxSC#=rydFZpNkY3WCMaQkyr8z(ldC*Z?A=myfO)Sn(`}l_lfbGM=S8?ec8&0$w@} zB!5Z*Iwp@hA|pLWMYsjv8h{!Me`xwU1wvst5sHB=+b!o!Fw61VSMHl@#VLsy?UJ|v zsl7A;Yd{FV?|Z^#y@?DWtLWO1&;~IIO~gZXiToO0W?1@9NCP>0R>HXB#o>q}RQOwr+6xHf(rJ8fV~2F}^);Z_{! zF&3?0{SN@PND}1HWW*bBMj-=m0usJ#6L4L6q%d9_AqCb4>HsddQf*M3dDq*w@+zf1&|qml&mnUVmPFU)o7EY z;{_YHSB7~qP9<1`a3yt-=nE5w^euKlnU6gyt1SdZh+!it=GeL<@lA_tQqDi_Buq8nhN;O+ZOwABG zLX3`4-0Cy5M_2=!7UI8zmWIm_C(Jx8IZZ*Nk0Jrrd<{QI)U zLSz=AE>N``RmgDNzXCFp3_^`BA_J2zizOGMPT3Nv<933>wm#p`Ng+)7vzE#tW-?&( z&Vv0}>vOpGN2q3|J+1<1&0hOL{4&PYS2<|od(C$ODi4b0_#II_XyoSeh=|DJUNHyO z3_3Vt@YS4)06yy618RuA+;l|f`ZXV84u1}k=_z|+^(2I2mgET$x=z<7vPp2Dn&uGN zmS*)%(&`2#H!`^i$=nrpLj~K%k0r)cD~!gQErd?av;m06p-+dp-Kt9r)NnjPHGe8ykuWdbRZ9OgxUV#2WbQq1q_JMFfRnS=Zf0OZC}&Zqx0{ z#>Zg;jBt#NV{71}2RK8==hh0sRYvp!76YZ(FrFCDn>hYvB+2kSjNXI~V6qkOZOtH5 zN%pCS$hk;!CmOLVytd7%3mb$DeF*4)yaRBtab$5;yXb&_D3Wlt-Z|Vo_IM797mSCVD_i{1jt4D} zQqzUZycFIGur{KNy_$!JQy=fJO%@y+bDYng%Bs0BhY_;s^#aE~kU12%Lvh2DUcILy%RHPHIx-xNNMiRHRU zVhb#MdI6FnY9XWVifIhu`U!D?_}qeV*d2tXK|eHm?F&FuLQbpjw@&W9?I8#R1cpKa z)`bJ~4_HA$?6`+^Yb9#O?i03X$tBrmRf z2;@T{j=WuLdoO7zW8m{I#jtcm%D`5f|+-MfS+%`{C7P+590@mUIe1c z)|;}PYcA{kb!2+$f>_HbC>+gI0!iI_(b^R1L2ZS?!SfN!&vm?j{t|dsSRbXv9T;7f zLRlYQB}xnjK8Qm7>pDK_`%)ijOj>_W3T1t4C~P#G8-|-0lt_!DlUc~kXE93ZpG99~ zoCs81vf>8JQtzDk4bE&Ff3Lmr%(!fjEzRbZB{8#w-~C;w#=rlye|YNn7ys;^{qu2t z4Jgp`Ye3WblTUsB$pzCv`RMrTpE>s0uYCUX7hgH`^S}ApZ+-FyzxBfFpZ)byFMj#^ zuPnU&>94%@Pk;5)uYKjUXI?apAwWp82{*V8~ zYoGtbsb6^V)EB<|#;5=O_rLm88F8ix_;fF#LG>()#=(svQEg3Vq%n+hw~YzS!PZayn=1CI^5w@J*MrTjGs-`$~Q~?r8jN zx#Oz;54hth_4y_4C_qZ!?18*EJ=f!oGPnaif=PTtt*@hK+ks>+UT9VB-3t#HDlphh zZanr*62c-FTtA1ONooMR3qc9*ef~AF#$je7Tx>r>#x|3yFXS-jgq?}bgRb$~Q_M|{ zuM=}gwuAunN~xr0@Xp}>Uf{tN#2x5AVjnb<@R{)}KQGtNQ7w&$L)~X0!BXclC(86z zES7^S}HJcgH$oZja^(PhZepHXA?fq?D*L`?uMzrCWZcTO{L9z zox7?BHwLjB7s+4`%3!g+xU4l&um#`0*P|_^G}g_RSCGQI>sN^DNUsYb1l1{Qn)7Ic z=;V5zPA7vrJVKp*S#Ca>-c0;1V8x*O#i@K&s_B1{Pc`*E<{oAuexb*ht0Cbxuo$Y& zcku@wC(vm&5s8p~`44ygr&#<2CSPLlK&g}lh+!pvhv0w35T8uQK@_lh>G>Vj}LWYfMaM z97qQGEGls(>MAjvU_yI||FZy1lRle4%!4Gqq-~1om}HDvAm_Y!*`_o_ym`4wY3}Gg zy+Sk$(PD$0P=ROaAq{UUI*Qb@0_YOvGphhng3^M)YzVGHTm$u1I>{VZb@&7;Z*n7U zuo@++pzCJvTfl=@$%5^iW2{5ee%s6C-zsbc6g8X+0`vgrPaIf_X)d)S)_Fa2e=vF`r@ke*>SNY$5;v diff --git a/src/torch4ms/ops/__pycache__/op_base.cpython-310.pyc b/src/torch4ms/ops/__pycache__/op_base.cpython-310.pyc deleted file mode 100644 index 4f35ab3ee8e20568e3bb5bd3e37f4a3cb9e76dfa..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 3924 zcmai1OLH5?5uVu>79ha49@NWg#i;~|L`RY%$&N~~i?SRYEIFp4!xpQSOU!}-gx!T_ z7E;JsUP7j%Q+`1XWGWZ^k^G3c=8*DVNV%kZJxc(TRVgoPdV2=FGd=zIx|z3H4Tk5^ z7i+!SEyn&$lao&klg}~KCC->&f)7|&y?JPvxgFZ(?S!s*duc7K@k+)|d002RHE5(W z;f&$!K{IWIEyFv5*>ogh=dp)7SV$Mc#dIlLN|(ds^i+6?v#*%&MD2iy+P)Q@ z#>^LW%<7n(NtQ)ppND6~dC^Q(x|V2(*#kE`C+3o+WUhaHg^77^ZvVmx>pEgVEFL)F zMX{7DCa17^DLLDc(>)*sOPS=M? zN6)O~Stl-%45LFEo_uUfKF3fWfg~*Cf`wL{*D+uaTQ~7z(PmVpj2JgA7sNp z+({k{?@Kt5bzG3~L%fBthM_Kil)PjSD=ko7+MrhHfWpV?>@|M~{Ttm;);ZR@CKI!_ zUL5ACM~hLBN9m}5@_M1CqO_L@HOyra(N}F(J1QA;!`5_@XDe*aF6}+1fGrAZ-;(pCeP|!CJ*RZ~&YlYy zSKcb!Ll0KsrPZ~M_y*+n_#Q0Sua(X=FS&r5{)))@2mhA!VB3H4zwtc}U-_kX#P@31 z{7=Z%Cg-o?{CZhC;+&QCI>UDjxlz{oGo>A2V8uRcPJeG(55K2twp+%$y3td;Ocim~ zNhHM*qLNIjohZqk8{=!gxSp!0r*b(%0)DEl7c|x0Xei)T<1Fn)$skF!+sTJJQQMbu zB-@;u)dmmSb`}2j5ro(_i_;{EbR&w=T#N?9 zH>2pgQ9P)Ak&8G{en^CZ@dP2oWF3U=tSEGSW;qtOc;km=!~F7I-cpKvPU6cT%xMUh8@<*-NNrV0d<)A1ha*!r&UM zD)avu`xzfhqJ=V*Zc5_=40RWz1d_et(x;;VJgfk);QltiXt{6>0YMZA8CLh~SAZ=6 zsOkZV2xLqoNIIzP$y0dgh3y13K51)7A!%eJSz0$t{u@0JEk?;nv=ACp9%@_iSM*(* zhzTE)ueBEvl`oNNSgumT@sXlU?pTY~?zK+koob8KJA)i?r0%YYq?5}SE}1SJ@6+~m z-8e4K2THrNs^l-}OrsOUr@2?9Ck6Cav5&#O=0?9ZuXHsU)+dL30SQH_`<7)j-}>Bo z3oYL`&E4Rq^u!_LBi(vT=(%=W&#`LEL1A9m%JB`&5#a3gEz@bHonaBlv&cT4#lKSS!IFfiF?G+kE%a zaj!Ah8DCEY3igjB&8a37zfghkdoV(k*bX+cQ99h&1iDj*XbVW~Jx(%I27Muoceavf ze5vsUm1A`+R_0Uf^a>OSD;sJzmNMRfh3FEMw|)dVpW=r=~0+(8nR6N@s=bTPU%P;I~Y_H`TM5aSrkpzCO!QLSctTy$mr~}^*iqN0o2LI3T=g^_ zCT&;#0lUZtAlf(WE|3F_7!qtp^Im1wDIzza87USG@_hIVA1P{V8h~Z$nr89m^Vv6! zE&qt+Hh&DT{ED~_jX$+(5?S8CQdL0ela9VpuQi#kq8mro0WvpTt8lChs~x}&0vd=U$-mNoK4rh8(ro%I(@U6O)6N!3MFpv~uh)$+ zwey(rfqpZTy-RmiBAHNl^?X6@OJe1G0kjW^4D_U-=w;OLFl diff --git a/src/torch4ms/ops/__pycache__/ops_registry.cpython-310.pyc b/src/torch4ms/ops/__pycache__/ops_registry.cpython-310.pyc deleted file mode 100644 index fe73bd15fef4bf01f1c9997cd5c29c0f64a7be78..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1345 zcmZ`&J8u**5VpPE$8GKfd58pruAn&~A|XWxp%aRf=%@%ql4cdj+N_;q(Y~?p`S*X9Tb#xnCc6N zUBnT`2@0AO2N-mckkksSlm-+V+)C`!37pgoT#N*I@4Z2sa{C-{`;467pcSLQ7p|b9 zbwHMpnB&d}0oMKtb4RG zTho7Ai4u}= zaS&&M>sBTNS4?Eb+6UWlEKb15mhsJ47N&wR?K76J$a&7a~UX*j8_UeGwQuQniNC&}Wd@s}tKBId-HFp8 zVbQel?s`j;ytqxWyl-5tG%rG}n@sRn6=7NT$P(Dq0}#X^`1W%yJ=Y|bV)+n^|IgO| ztas*XHZNpLF2cDXX=pB(y&L$qWs%DO-Ni<(7{g7^+)d}Yuu$+J99PEwzjT1gBQOl< z&Uic?_VgSN%P>kprG%0u4MF$14_C|60zlKZOaYW49#YXK9xsfB+<7Mor7>JNiFFyH_H diff --git a/src/torch4ms/optim/__pycache__/__init__.cpython-310.pyc b/src/torch4ms/optim/__pycache__/__init__.cpython-310.pyc deleted file mode 100644 index b34b09b4bce93b80161021cff7f72dce4a69bc2c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 796 zcmZWn&rcIU6rTQ~?LuP=iRZa;fT;0gjM0R|6G|+J$)?F<*;&gZJG-0NHCj)IfS43Y zB0p6^L{apD92A5t{Xgc)&X%6Idvc~Fg~Unr%e={Z-}m0P&n*-*AP*VcAcH9YPE>ID zBxP_bbu2$X(wn>Ptug0qG>%%GkS*h=sh&6XK7B)(Y&6gny%4ay*B~M04KK+ zdLK>|!9e<%p8!A$TxL$(qCSkmHBns=>pStzC$X^#2Vr#7XwH0t?F{1uY`&j{t{2E# zRBtqwcH)&kBCKiJX=6P)_|d6~Ym3qGhv;y&xpx$W;d#ES6mRPB@=uw6A+xY8YM;-- zYE+*Uhs$DbszVD$h=Yrs%Pc4sHk(VE-HMaRDJ@>wjjOL(?~Wx^^JrQu&c$21;_$1e zt;O?OXW?9#8JJj300U+?{!^DxBOtEhpM`Iw*5$T_%n5@yic}Me)p+%Hw|O{rZptGs zAe6kc7;H^WwPxm}JMD%WaJ16Sxp3|DbzOYfj1Cq$eo04%#w^Eo8H^4lPk-%;{q3m! zQK_}0?O;IT`7S$86(0=q^nGklo>BOOXCIFgJ?45m)9#Y@n>O*#cm#luXu diff --git a/src/torch4ms/optim/__pycache__/optimizer.cpython-310.pyc b/src/torch4ms/optim/__pycache__/optimizer.cpython-310.pyc deleted file mode 100644 index 7b5a3de7a37b39e7ef9f7231d83c54467496edd8..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 17077 zcmdU1dvILUdB4wnY4xxyOMZK?3Gt>x#UyDOLW~_#9&Hc~CQY)1ZkD~*w#4pAI(HYa zvT6jfu?>DDA?9UZ6`p2-F+&3e+c2H!bS8b|kIeLs-kJQ7M!S+`+M$g*&7&mk@B7Z~ z-MbGNho*mYJ@@Fm?>XQ3-ru<%?d*&w_ziD=YB>C-itB}xlEUS*|PYm3sSRpICSFys=T4Fg$5vv2G4wP0{ zQG12eX|LF6SY1~4UL&y*?LF2Kv@fw&OzVkNDEC@RQC`aB)hPE_%TQj%~smy%-xE%ct&( z%{{Sq{?t43r)DCN%F$!9?>t%Ff4=ga)8%tV(c%rS9C^Nc^x5~$9jZL?YUR-V3*UZ+ zI-K2GK6aq;-B-(RzfeBxa{M zSKpf3%MXv{QW@9$QVtn6baUPosk<_^8`_r2WH@KuUKq>Ran;pwIi%t3DUr(B z=!Zh!c01>cij%4v%BIG~hI6}|G`-FLGf~)r$JvO)R)AGXLbKF_j&s&+!!}namSG2Q zrlu9xYbJtL5O55;U9`CD@PsXPW z-zLQh?5sDRRKzm$`qd|{zvhngsAW5MZiHxk-3uO*}y>S~qiT245G zPFtwsb|eiatX0of=kvu0gJy$P0JWi!FwQ+_2Y>^SS{c2x70S*4a{wdJVMTxm9qy7F zMamksbNSmvD(CDR71=@&Z;Hh5^W%Q`nYV!s%}CKQ7}&u><@i^R&YpSMKg+`O`O1ld zF*=J_`CEr#g)s|<9jhFFtMdAHh@R-ReCBRI*!hCUr3s?7ALZ#`#3IP#fm+ilHlh{? z%H;!3m0x_ASOU|lL}Z*KK|8LAKKzjgRX-9s4cDLrtwZht%^uPw`dc$tp@{ON^2X20`0&wX7qeR3D~MOh^);gWnk2AM1viU|!?q5E31aiWEYTUbF zvsgV~BY#a)YJ&%Z5;fc0X)pkE%h(%D1S}IR!I2Q!0=9|vaIK791I+bxY4{oEq+m=VO607+f2==ufodGdB=6Y8lvrC!O4k*_vcSa`lqE=Fh zdI{$+gMX?rZsfJ1dH`(NQcq}>c9@V?A3CXtk6@%5V4QbAO*9+$a{xlg&RL95lEMrh zg?*I9guNp;sq!omyBW6#4?ls2am!FcYA^mxT>b$YWik{VwKR|*;W7!t9W?70lA=1P zW5))#gf;@~TOt1fz|$IFu;dPH(wqzw&5~Bsi-8#pWqmSOG)j6g$Pnqy8yIQg0V^GV zC_SuAhP=7Ja!+W*P*TO{5R-afNbZt0zd%9~&>}7`uAqF-)hC9>7<`iCHc}Jurn;1i zhnx68qBC6(0{l5CSx+Qq-z!C=BjOmE==NYNs}oD9M^mL6_J%bv7+%R|Fp)!o*ia31 zt=gkT)jq8kXEJfc2c1Wa_jEM#o;GYpTuys!$9G8osN;s)(~RF&(*g;TL&{2ZFd%*( zS-_S!5%@acCs)ZScrxbeCa<2r><}sRMFPEo$e=C#Zb8F^n~*e1*`1n2bZaC`&~3n@ z+b|?-q*k`WZKdndbY^%gdDouguFU9NsZ26Q^7S6PgV{)V-%I7!rX+WnfA;?J!IxWj zTs{E=9$K7?NGki|>q*ShN)ED@v@mPsyS5O>G>km`{Sieo}L_!K++@ z7y{lMW>BX`?VUafid>t4J(3>Hj&1%Ly@6gAQzi!69g8I4x^Srvjhk*b3v0_EYfF3@ zbGQbfFm&^Ak(WpjYGPQjTO>c4uj0G-9L6OgY%MT)f~5hi$81ZoG`Z@{=p8RWD}_RcA@eh%c@W z5gWP@&jrF#(ucYNgSw#FMLCL-Kx;Ecv6+wTQmfLQBz}#@gur9Mc4VZ3b%Q8)OlPfZ zM;mxdS5<@Wcc2s4#AGsdO5|WCCI#7a58=g<&Ajr$>|3R|Bm3vhovb|aPBV{bKl~zW zrr|YEt!mR(4jlLL^QWiDY-VFyvYyU{VT4xSBomt!+3ZD#S$9Lg6m~oah9;>d@v72x z8O^*NNg~QKHxTEaP@_kj<0v^1BoVDk4QRa@^n^7RUCvOvyIDP8(}X0tAc7J^P=W|v zn#{jFY2J*n#A!WKB}Sx4wTqIb*o0P?z)Mq>m9*@1Y7bb_z4q{~-Fc@lfi_0m%A04) z50xrUO)u;}JU?@~y#Hu30ks)^5$cgIkne+__RTx@`dLp0yUmqaNw0lf`VF>~_^Xx6r2MSxy(*0ERCbZkq9P}@*6Sr`ZUlmUe`ZL_Lr zcR}kmEuFP)O4@-_#7m4VdylJghB4IB9(lPF-45jcO#2+5xqyA&VPmUG1!F(woTVB5tCD zN-ZcGMCBGd&Lc?LNs|$_CLvcKJM0dU6C_h0Lv%iPFR}(>Itb5LM>@O2E55Z zyW8#=S;A6?JixygCiSv!J`(17{PN(ItHbIwiyf zo1K$-j*V+*nYsl@zZZr#>74L{8l?9nP4Wf8XwUp zP5}Ru1`e-Itn@*t>DU_3l~{zwCVqqhBqK<4gnVzQNqE>M^*p3xUIUaqPtdR^cLn68 zVX2UmS5WH+mXVd^)hOr$#Mdaf zoDy~&u~heKt#q8SdtJH`al5@qjwq$2?~Itb^LMCs)*~^1`-U3P`_$EHA7}uSplMTm zkmP+t6_D%`t9@tPG70!4H;j4x zwS{^uq)_*aa&$fBgC0@pMZOdHrKP?ijf#93K5zr{FpYiZLYoxl=)HJ1{rf7Ome!~J zt@)nPGR$`~<_i^#d_Q3D)9U-;zW4P(IvOJb;;-`IVi=OuNG^x;4b$J2`G4r2zoqrZ z{?;6%?Z}GK%3_3|)zbHZzwh_}N2yJp$4F8@<;lOS96v3IgH_id)?HnbB}IIdJHVmL z!9spG;|x-a(N1#SHRYcUUtP1h%alY;21!nYfx@C>CnX-)xFN^RCy5KX9l6n@HSCO~ z^6A~K$**>`G-d|>ta zRHscv1Ni$3ZOZs@1LtpgZ3E{g$>?!@GDx7xDKnD$t8i746wo==K;dxs6H2gl2)Qic zsTw0s<=~6+Glyo+OjW)&Bh|LbE6*>y{M=C76gH-m%E1Q85ib)3BsJkiJUOw%^Zz$` z*_c7H;EGaX5Y9l%6C#78%e68YPMpBt!?Ui8}zJNyfO7)oANIsF%u3lf(#fU&1HwG8wugC5ggQBo=@O zKq{(1?ms4$)&+z&Zs??XLwsOyZwFa&B0~vzP^6wh&~$a^ae@$sqZ-l;LYjz--+FW` z&z}xlUYaoO8Xe7uJyfpmrGCs`M8k1BtoxD2QVXorwBKlYZ|0)*XV8vXAv|F`5xdjsu)FLiEX^qU-=kLN z-p)k#eo#!J2k77J>N{?{zR-y`;&1(a&)nDhPTlyEiM`he^3B~aB#rzpAP=>7Gz-aI z$g<7vhH^G=SB&0M^^23cI!!>Y{m8>hU?!k*ln|O31SD%d5S}Fk2ppK^h$w;v+TDg& z&b3<6sMb;+)?bNgi$U=bAb0Qxe1ozUg4Uc?N&!%MXa+(bojp>hmelK>GPTON|bwhM8(`M3={2EgyS9+UBvLDtN2& zHwNG6p&&Rtm0vO$!|~T;_EaCQ~HgRW7m&KIa-g~SzT$` z&SYw<fD137hW?xokur=F-xy;weTrsjI4SJ3Yg*~EvX`IC#&ym$Z*k`#Z4a&+kv z-Ru4jrg*QZe$;h}7fdXTl%52|h<%uAqLZ-APu2(NH4jnpFeM~ohEDztNSkDh@JL%c zNJU;$6G0z9!=1#4E=JIiW`d?sixRZhkJfmPYtml7Tb>iHMC#K6%~WP= zcZ!*Xe3j5R%%Dv~ExRBRc>R03MX8`WIsLFIeUEH3>kq@U>KHsZ>}0LahFU zeCl7=Q~$y~^^YY$5&oWJxT*xM_&=i+h6LomTd!BnezOg0|J_P}OHesKEWyMhiz$VeJ-oQMxHv`RNa~i+*Q6M-M|h&Ailudu3`q6xJG2~6j!0n= z@A=>BiI*Tp9>J@>N9!I$rcp73YWRzGs-YItei;b$|3c8%7M4JivIUQG6B0j5kWs9J zj4Ud+8Mg#%0XNJ!rbyoz@RuctkVO`rk_huRCAbA-k(Jl$E@2-NoWh<3UARxHijG8OTA9vvU3s@nZx7q73KYhOZ+%vOhj>OnmglPw5G#~~S%KZ1=EWi2u?D-R{L~$ac ziQcsHFFsp-@b%fZ>oEv#=E|cbTuh&X|LzE;px|W7E2J9&@vDVLPc3}+v05~-d}@03 z+!N&!FU81}vN0Z;ee2=!!Nc?WUxr~u@s8%DdiRXIut(+bQ;RF*zLF8w(X{^UrBDe^_J~hZ@YQNtwV!Z3kdNF0UF;M%p8bqt${PG1HEfM z$^#drAD?8RybUS8Ua30i-bS?t2rt6wVU*Y8Wm@%oZ;RKoB6RuG)3fh9OcwJC5BSfJ z4yRceVaV3!kO5l^W&9e<#dcmVoKnUC)>v@A``WU2WFB z+uwc(YoGG>AMXRUBThzMa7%}53=~LR@%TnH62RX%${2vx$(PUJ=XMcdcA@`_ zc$$TA(Lk@-hX7F@LO}iS1J(jVAVo(R><&%D5lLBL z;i^C~FdhZ|o=VXXOr}NM)M?s|ZirCyM z6eaMVJT#WzE_iq8jtHGX?V{@mO!Gy9x3&u+qYsz$#`?_l4SGquRt}t&44%ZAj29 zYFr6ePSdTRWF;j8Kk45k{w)I_L=xf|N=lT_%X|x9xUm-cC>s8eI5uhKWa5UFggSaQ zh1P^ZQA1E`{BrpMk|Ay=Pxlp}26MR84dnOGhx9V0O2i`LFl<@}s7R`+yo}Di+s{++ zawPbl8Jr|$q3acHH&04tX7vIYQJcIJ@dV{a61irskj0nhT5e2!;|Z-Lctk94qbN5u`$k@X$J#0bS*S?e1=Z{rG&Kg4JBko)s5gAbf1s@;a0NV73%Ho WS=STmS=O~u`3C-O#J`XB#{UPL>7s-H diff --git a/tests/torch4ms/test_loss_interception.py b/tests/torch4ms/test_loss_interception.py new file mode 100644 index 000000000..c8fcbbe51 --- /dev/null +++ b/tests/torch4ms/test_loss_interception.py @@ -0,0 +1,113 @@ +import torch +import torch.nn as nn + +import torch4ms +from torch4ms.autograd.ms_autograd_function import extract_and_wrap_loss_fn + + +class SimpleRegNet(nn.Module): + def __init__(self): + super().__init__() + self.fc = nn.Linear(3, 1) + + def forward(self, x): + return self.fc(x) + + +class SimpleClsNet(nn.Module): + def __init__(self, num_classes: int = 4): + super().__init__() + self.fc = nn.Linear(3, num_classes) + + def forward(self, x): + return self.fc(x) + + +def _init_linear(module: nn.Module): + with torch.no_grad(): + for p in module.parameters(): + p.fill_(0.5) + + +def _compare_loss(loss_name: str, model_ctor, loss_ctor, inputs, targets, atol=1e-5): + print("=" * 60) + print(f"Testing loss interception for {loss_name}") + + # Torch reference + model_ref = model_ctor() + _init_linear(model_ref) + loss_ref_fn = loss_ctor() + + out_ref = model_ref(inputs) + loss_ref = loss_ref_fn(out_ref, targets) + loss_ref_val = float(loss_ref.item()) + print(f"[torch] loss = {loss_ref_val}") + + # torch4ms path + model = model_ctor() + _init_linear(model) + loss_torch_fn = loss_ctor() + + env = torch4ms.default_env() + with env: + loss_wrapper = extract_and_wrap_loss_fn(model, loss_torch_fn, inputs, targets) + loss_t4 = loss_wrapper.output + + # 转回 torch 标量比较数值 + loss_t4_val = float(loss_t4.torch().item()) + print(f"[torch4ms] loss = {loss_t4_val}") + + diff = abs(loss_t4_val - loss_ref_val) + print(f"[compare] |diff| = {diff}") + if diff < atol: + print(f"[OK] {loss_name} forward loss matches within atol={atol}") + else: + print(f"[WARN] {loss_name} forward loss mismatch (>|{atol}|)") + + +def main(): + # 公共输入 + x_reg = torch.tensor([[1.0, 2.0, 3.0], + [4.0, 5.0, 6.0]], dtype=torch.float32) + y_reg = torch.tensor([[10.0], + [20.0]], dtype=torch.float32) + + y_bce = torch.tensor([[0.0], + [1.0]], dtype=torch.float32) + + # 回归类损失 + _compare_loss("MSELoss", + SimpleRegNet, + lambda: nn.MSELoss(reduction="mean"), + x_reg, + y_reg) + + _compare_loss("L1Loss", + SimpleRegNet, + lambda: nn.L1Loss(reduction="mean"), + x_reg, + y_reg) + + _compare_loss("SmoothL1Loss", + SimpleRegNet, + lambda: nn.SmoothL1Loss(beta=1.0, reduction="mean"), + x_reg, + y_reg) + + _compare_loss("HuberLoss", + SimpleRegNet, + lambda: nn.HuberLoss(delta=1.0, reduction="mean"), + x_reg, + y_reg) + + # 二分类 BCEWithLogits + _compare_loss("BCEWithLogitsLoss", + SimpleRegNet, + lambda: nn.BCEWithLogitsLoss(reduction="mean"), + x_reg, + y_bce) + + +if __name__ == "__main__": + main() + From 133d3ee76233cc68beffedb993011345b5e6b5da Mon Sep 17 00:00:00 2001 From: feixiao15 <467534404@qq.com> Date: Sun, 8 Mar 2026 20:32:03 +0800 Subject: [PATCH 2/3] move module forward to dispatch-based path and add some mmissing lowering and tests --- src/torch4ms/autograd/forward_extractor.py | 65 ++---- src/torch4ms/ops/maten.py | 123 +++++------ src/torch4ms/ops/mten.py | 30 ++- src/torch4ms/ops/mtorch.py | 234 ++++++++++++++++++++- src/torch4ms/tensor.py | 38 +++- tests/torch4ms/test_train_cnn.py | 162 ++++++++++++++ tests/torch4ms/test_train_transformer.py | 161 ++++++++++++++ 7 files changed, 685 insertions(+), 128 deletions(-) create mode 100644 tests/torch4ms/test_train_cnn.py create mode 100644 tests/torch4ms/test_train_transformer.py diff --git a/src/torch4ms/autograd/forward_extractor.py b/src/torch4ms/autograd/forward_extractor.py index b76d17b09..0e2ce1c08 100644 --- a/src/torch4ms/autograd/forward_extractor.py +++ b/src/torch4ms/autograd/forward_extractor.py @@ -6,7 +6,6 @@ """ from typing import Callable, Dict, List, Tuple import torch -from torch.nn.utils import stateless as torch_stateless from torch4ms.tensor import Tensor as Torch4msTensor from mindspore import Tensor as ms_Tensor @@ -129,58 +128,22 @@ def get_ms_forward_fn(self, env, include_buffers: bool = False) -> Callable: """ 获取用于 MindSpore GradOperation 的 forward 函数。 - 输入为 MindSpore Tensor，内部会转换为 torch.Tensor 执行前向（因为 functional_call 需要）， - 并将结果转换回 MindSpore Tensor。 + 输入为 MindSpore Tensor，内部先转为 torch4ms.Tensor 视图，再在 env 下执行 + functional forward。这样模块内部算子仍通过 torch4ms dispatch 路由到 MindSpore， + 避免退化为纯 PyTorch 计算图。 """ base_forward = self.get_forward_fn(include_buffers=include_buffers) def forward_fn(*ms_args): - # 将 MindSpore Tensor 转为 torch.Tensor（functional_call 需要） - # 先转为 torch4ms.Tensor，再转为 torch.Tensor + # 仅做同构包装：MindSpore Tensor -> torch4ms.Tensor（不复制，不降级为纯 torch.Tensor） t4_args = env.ms2t_iso(ms_args) - torch_args = [] - for arg in t4_args: - if isinstance(arg, Torch4msTensor): - # 转换为 torch.Tensor（用于 functional_call） - from torch4ms.ops import mappings - torch_args.append(mappings.ms2t(arg._elem)) - else: - torch_args.append(arg) - - # 调用 base_forward（它期望 torch.Tensor） + + # 在 env 下调用 base_forward，让模块内部 op 继续走 torch4ms dispatch with env: - res = base_forward(*torch_args) - - # 将结果转换回 MindSpore Tensor - if isinstance(res, Torch4msTensor): - # 如果已经是 torch4ms.Tensor，直接返回其 _elem - return res._elem - elif isinstance(res, torch.Tensor): - # 检查是否是 meta 设备上的 tensor - # 注意：torch4ms.Tensor 的 device 属性返回字符串，不是 torch.device - # 所以我们需要检查 res 是否是普通的 torch.Tensor 且在 meta 设备上 - try: - device_type = res.device.type if hasattr(res.device, 'type') else str(res.device) - if device_type == 'meta': - # meta tensor 无法直接转换 - # 这种情况可能发生在某些操作没有被正确拦截的情况下 - # 尝试检查是否有 _elem 属性（可能是 torch4ms.Tensor 但没有被正确识别） - if hasattr(res, '_elem') and isinstance(res._elem, ms_Tensor): - return res._elem - raise RuntimeError( - "Cannot convert meta tensor to MindSpore tensor. " - "This may indicate that the forward function returned a meta tensor. " - "Please ensure all operations are performed on real tensors. " - "If you're using torch4ms, make sure the environment is enabled." - ) - except (AttributeError, TypeError): - # 如果 device 属性不是标准的 torch.device，尝试直接转换 - pass - - from torch4ms.ops import mappings - return mappings.t2ms(res) - else: - return res + res = base_forward(*t4_args) + + # 同构回收：torch4ms / torch 结果统一映射回 MindSpore + return env.t2ms_iso(res) return forward_fn @@ -232,9 +195,11 @@ def forward_fn(*args): # 合并参数和 buffers params_and_buffers = {**param_dict, **buffer_dict} - # 使用 functional_call 调用模块 - with torch_stateless._reparametrize_module(module, params_and_buffers): - return module.forward(*input_args) + # 使用 torch.func.functional_call（与 torchax 路径一致） + # tie_weights=False 避免共享参数在 functional_call 中被强制绑定。 + return torch.func.functional_call( + module, params_and_buffers, input_args, {}, tie_weights=False + ) return forward_fn diff --git a/src/torch4ms/ops/maten.py b/src/torch4ms/ops/maten.py index debd494bb..b80460f97 100644 --- a/src/torch4ms/ops/maten.py +++ b/src/torch4ms/ops/maten.py @@ -294,7 +294,29 @@ def _aten_log_softmax(x, dim=-1, dtype=None): @op(torch.ops.aten.layer_norm) def _aten_layer_norm(x, normalized_shape, weight=None, bias=None, eps=1e-5): - return ops.layer_norm(x, normalized_shape, weight, bias, eps) + # 手工实现 layer_norm，避免部分 MindSpore 版本 layer_norm 内核不可用 + if isinstance(normalized_shape, int): + normalized_shape = (normalized_shape,) + else: + normalized_shape = tuple(normalized_shape) + + ndim = int(x.ndim) + k = len(normalized_shape) + axes = tuple(range(ndim - k, ndim)) + + mean = ops.mean(x, axis=axes, keep_dims=True) + centered = x - mean + var = ops.mean(centered * centered, axis=axes, keep_dims=True) + y = centered / ops.sqrt(var + eps) + + if weight is not None: + w_shape = (1,) * (ndim - k) + normalized_shape + y = y * ops.reshape(weight, w_shape) + if bias is not None: + b_shape = (1,) * (ndim - k) + normalized_shape + y = y + ops.reshape(bias, b_shape) + + return y @op(torch.ops.aten.batch_norm) @@ -303,11 +325,11 @@ def _aten_batch_norm(x, running_mean, running_var, weight=None, bias=None, # MindSpore的BatchNorm接口略有不同，这里使用MindSpore的reduce操作实现 if training: # 训练模式下的BatchNorm - 计算当前batch的均值和方差 - # 使用MindSpore的reduce_mean计算均值 - mean = ops.reduce_mean(x, axis=(0, 2, 3), keep_dims=True) + # 使用 MindSpore 的 mean 计算均值（该版本参数名为 keep_dims） + mean = ops.mean(x, axis=(0, 2, 3), keep_dims=True) # 计算方差：E[(x - E[x])^2] centered = x - mean - var = ops.reduce_mean(centered * centered, axis=(0, 2, 3), keep_dims=True) + var = ops.mean(centered * centered, axis=(0, 2, 3), keep_dims=True) # 更新running_mean和running_var（简化实现） if running_mean is not None: running_mean = running_mean * (1 - momentum) + mean.squeeze() * momentum @@ -345,7 +367,11 @@ def _aten_silu(x): @op(torch.ops.aten.dropout) def _aten_dropout(x, p=0.5, training=True): if training: - return ops.dropout(x, p) + out = ops.dropout(x, p) + # 部分 MindSpore 版本返回 (output, mask) + if isinstance(out, tuple): + return out[0] + return out return x @@ -684,49 +710,20 @@ def _aten_convolution(input, weight, bias, stride, padding, dilation, transposed output_padding = (0,) * num_shape_dim # 根据空间维度数量选择对应的卷积操作 - # 使用MindSpore的nn层API（参考 torchax 的实现方式，统一使用nn层以确保兼容性） - import mindspore.nn as nn + # 直接使用 ops.convXd，让 weight/bias 作为显式输入参与图计算，避免 set_data 造成梯度链断裂。 if num_shape_dim == 1: - # 1D卷积 - 通过转换为2D卷积实现（参考 MindSpore 内部实现） - # 输入: [N, C_in, L] -> [N, C_in, 1, L] - # 权重: [C_out, C_in, K] -> [C_out, C_in, 1, K] - # 使用 Conv2D 进行卷积 - # 输出: [N, C_out, 1, L_out] -> [N, C_out, L_out] - - # 扩展输入维度：在位置2插入维度1 - input_2d = ops.expand_dims(input, 2) # [N, C_in, L] -> [N, C_in, 1, L] - - # 扩展权重维度：在位置2插入维度1 - weight_2d = ops.expand_dims(weight, 2) # [C_out, C_in, K] -> [C_out, C_in, 1, K] - - # 准备padding：对于2D，padding格式为(上,下,左,右)，1D只需要左右padding - pad_val = int(padding[0]) - pad_2d = (0, 0, pad_val, pad_val) # (上=0, 下=0, 左=pad, 右=pad) - - # 使用Conv2D进行卷积 - conv2d_op = nn.Conv2d( - in_channels=int(weight.shape[1] * groups), - out_channels=int(weight.shape[0]), - kernel_size=(1, int(weight.shape[2])), # (height=1, width=kernel_size) - stride=(1, int(stride[0])), # (height_stride=1, width_stride=stride) + pad_val = int(padding[0]) if len(padding) > 0 else 0 + res = ops.conv1d( + input, + weight, + bias=bias, + stride=int(stride[0]), pad_mode='pad', - padding=pad_2d, - dilation=(1, int(dilation[0])), # (height_dilation=1, width_dilation=dilation) - group=groups, - has_bias=(bias is not None) + padding=pad_val, + dilation=int(dilation[0]), + groups=groups, ) - - # 设置权重和偏置 - conv2d_op.weight.set_data(weight_2d) - if bias is not None: - conv2d_op.bias.set_data(bias) - - # 执行2D卷积 - res_2d = conv2d_op(input_2d) # [N, C_out, 1, L_out] - - # 压缩输出维度：移除位置2的维度 - res = ops.squeeze(res_2d, 2) # [N, C_out, 1, L_out] -> [N, C_out, L_out] elif num_shape_dim == 2: # 2D卷积 - 使用 nn.Conv2d @@ -741,25 +738,16 @@ def _aten_convolution(input, weight, bias, stride, padding, dilation, transposed else: pad_val = tuple(padding[:2]) - # 创建临时Conv2d层 - conv2d_layer = nn.Conv2d( - in_channels=int(weight.shape[1] * groups), - out_channels=int(weight.shape[0]), - kernel_size=tuple(weight.shape[2:]), + res = ops.conv2d( + input, + weight, + bias=bias, stride=tuple(stride), pad_mode='pad', padding=pad_val, dilation=tuple(dilation), - group=groups, - has_bias=(bias is not None) + groups=groups, ) - - # 设置权重和偏置 - conv2d_layer.weight.set_data(weight) - if bias is not None: - conv2d_layer.bias.set_data(bias) - - res = conv2d_layer(input) elif num_shape_dim == 3: # 3D卷积 - 使用 nn.Conv3d @@ -777,25 +765,16 @@ def _aten_convolution(input, weight, bias, stride, padding, dilation, transposed else: pad_val = tuple(padding[:3]) - # 创建临时Conv3d层 - conv3d_layer = nn.Conv3d( - in_channels=int(weight.shape[1] * groups), - out_channels=int(weight.shape[0]), - kernel_size=tuple(weight.shape[2:]), + res = ops.conv3d( + input, + weight, + bias=bias, stride=tuple(stride), pad_mode='pad', padding=pad_val, dilation=tuple(dilation), - group=groups, - has_bias=(bias is not None) + groups=groups, ) - - # 设置权重和偏置 - conv3d_layer.weight.set_data(weight) - if bias is not None: - conv3d_layer.bias.set_data(bias) - - res = conv3d_layer(input) else: raise NotImplementedError(f"Convolution not supported for {num_shape_dim}D spatial dimensions (weight.ndim={weight.ndim})") diff --git a/src/torch4ms/ops/mten.py b/src/torch4ms/ops/mten.py index 335e25b15..48f46e3e8 100644 --- a/src/torch4ms/ops/mten.py +++ b/src/torch4ms/ops/mten.py @@ -295,7 +295,28 @@ def _aten_log_softmax(x, dim=-1, dtype=None): @op(torch.ops.aten.layer_norm) def _aten_layer_norm(x, normalized_shape, weight=None, bias=None, eps=1e-5): - return ops.layer_norm(x, normalized_shape, weight, bias, eps) + if isinstance(normalized_shape, int): + normalized_shape = (normalized_shape,) + else: + normalized_shape = tuple(normalized_shape) + + ndim = int(x.ndim) + k = len(normalized_shape) + axes = tuple(range(ndim - k, ndim)) + + mean = ops.mean(x, axis=axes, keep_dims=True) + centered = x - mean + var = ops.mean(centered * centered, axis=axes, keep_dims=True) + y = centered / ops.sqrt(var + eps) + + if weight is not None: + w_shape = (1,) * (ndim - k) + normalized_shape + y = y * ops.reshape(weight, w_shape) + if bias is not None: + b_shape = (1,) * (ndim - k) + normalized_shape + y = y + ops.reshape(bias, b_shape) + + return y @op(torch.ops.aten.batch_norm) @@ -305,7 +326,7 @@ def _aten_batch_norm(x, running_mean, running_var, weight=None, bias=None, if training: # 训练模式下的BatchNorm mean = ops.mean(x, (0, 2, 3), keep_dims=True) - var = ops.var(x, (0, 2, 3), keep_dims=True) + var = ops.var(x, (0, 2, 3), keepdims=True) # 更新running_mean和running_var（简化实现） if running_mean is not None: running_mean = running_mean * (1 - momentum) + mean.squeeze() * momentum @@ -343,7 +364,10 @@ def _aten_silu(x): @op(torch.ops.aten.dropout) def _aten_dropout(x, p=0.5, training=True): if training: - return ops.dropout(x, p) + out = ops.dropout(x, p) + if isinstance(out, tuple): + return out[0] + return out return x diff --git a/src/torch4ms/ops/mtorch.py b/src/torch4ms/ops/mtorch.py index 9de8b6771..783868e39 100644 --- a/src/torch4ms/ops/mtorch.py +++ b/src/torch4ms/ops/mtorch.py @@ -16,7 +16,7 @@ import torch4ms.tensor from torch4ms.view import View, NarrowInfo import torch.utils._pytree as pytree -from torch4ms.ops.maten import _aten_convolution +from torch4ms.ops.maten import _aten_convolution, _aten_batch_norm, _aten_dropout, _aten_layer_norm def register_function(torch_func, **kwargs): @@ -173,6 +173,96 @@ def one_hot(tensor, num_classes=-1): return mops.one_hot(tensor, num_classes, on_value=1, off_value=0).astype(mnp.int64) +@register_function(torch.transpose) +def functional_transpose(input, dim0, dim1): + rank = int(input.ndim) + d0 = int(dim0) + d1 = int(dim1) + if d0 < 0: + d0 += rank + if d1 < 0: + d1 += rank + perm = list(range(rank)) + perm[d0], perm[d1] = perm[d1], perm[d0] + return mops.transpose(input, tuple(perm)) + + +@register_function(torch.permute) +def functional_permute(input, dims): + return mops.transpose(input, tuple(dims)) + + +@register_function(torch.reshape) +def functional_reshape(input, shape): + return mops.reshape(input, tuple(shape)) + + +@register_function(torch.flatten) +def functional_flatten(input, start_dim=0, end_dim=-1): + return mops.flatten(input, start_dim, end_dim) + + +@register_function(torch.chunk) +def functional_chunk(input, chunks, dim=0): + return mops.chunk(input, chunks, axis=dim) + + +@register_function(torch.split) +def functional_split(input, split_size_or_sections, dim=0): + return mops.split(input, split_size_or_sections, axis=dim) + + +@register_function(torch.matmul) +def functional_matmul(input, other): + return mops.matmul(input, other) + + +@register_function(torch.softmax) +def functional_softmax(input, dim=None, dtype=None): + axis = -1 if dim is None else dim + return mops.softmax(input, axis=axis) + + +@register_function(torch.mul) +def functional_mul(input, other): + return mops.mul(input, other) + + +@register_function(torch.mean) +def functional_mean(input, dim=None, keepdim=False, dtype=None): + x = input + if dtype is not None: + x = x.astype(mappings.t2ms_dtype(dtype)) + if dim is None: + return mops.mean(x) + axis = tuple(dim) if isinstance(dim, (list, tuple)) else dim + return mops.mean(x, axis=axis, keep_dims=keepdim) + + +# 补充 Tensor 方法入口，避免 x.transpose()/x.reshape() 走回退路径 +if hasattr(torch.Tensor, "transpose"): + register_function(torch.Tensor.transpose)(functional_transpose) +if hasattr(torch.Tensor, "permute"): + register_function(torch.Tensor.permute)(functional_permute) +if hasattr(torch.Tensor, "reshape"): + register_function(torch.Tensor.reshape)(functional_reshape) +if hasattr(torch.Tensor, "flatten"): + register_function(torch.Tensor.flatten)(functional_flatten) +if hasattr(torch.Tensor, "chunk"): + register_function(torch.Tensor.chunk)(functional_chunk) +if hasattr(torch.Tensor, "split"): + register_function(torch.Tensor.split)(functional_split) +if hasattr(torch.Tensor, "mean"): + register_function(torch.Tensor.mean)(functional_mean) + + +@register_function(torch.relu) +@register_function(torch.nn.functional.relu) +def functional_relu(input, inplace=False): + # MindSpore relu 没有 inplace 语义，忽略该参数。 + return mops.relu(input) + + @register_function(torch.nn.functional.pad) def pad(tensor, pad, mode="constant", value=None): # MindSpore的pad接口需要不同的参数格式 @@ -234,6 +324,107 @@ def scaled_dot_product_attention( ) +@register_function(torch.nn.functional.multi_head_attention_forward, is_mindspore_function=False) +def functional_multi_head_attention_forward( + query, + key, + value, + embed_dim_to_check, + num_heads, + in_proj_weight, + in_proj_bias, + bias_k, + bias_v, + add_zero_attn, + dropout_p, + out_proj_weight, + out_proj_bias, + training=True, + key_padding_mask=None, + need_weights=True, + attn_mask=None, + use_separate_proj_weight=False, + q_proj_weight=None, + k_proj_weight=None, + v_proj_weight=None, + static_k=None, + static_v=None, + average_attn_weights=True, + is_causal=False, +): + """ + 轻量 MHA 前向实现，优先覆盖 TransformerEncoderLayer 的常见路径： + - use_separate_proj_weight=False + - bias_k/bias_v/static_k/static_v/add_zero_attn 不启用 + - need_weights=False + """ + if use_separate_proj_weight: + raise torch4ms.tensor.OperatorNotFound( + "multi_head_attention_forward with separate proj weights is not supported yet." + ) + if bias_k is not None or bias_v is not None or static_k is not None or static_v is not None: + raise torch4ms.tensor.OperatorNotFound( + "multi_head_attention_forward with bias_k/bias_v/static_k/static_v is not supported yet." + ) + if add_zero_attn: + raise torch4ms.tensor.OperatorNotFound( + "multi_head_attention_forward with add_zero_attn=True is not supported yet." + ) + if key_padding_mask is not None: + raise torch4ms.tensor.OperatorNotFound( + "multi_head_attention_forward with key_padding_mask is not supported yet." + ) + + # query/key/value shape: [L, N, E] + # in-projection: [L, N, 3E] -> chunk to q/k/v + qkv = torch.nn.functional.linear(query, in_proj_weight, in_proj_bias) + q, k, v = torch.chunk(qkv, 3, dim=-1) + + L, N, E = q.shape + head_dim = E // num_heads + if head_dim * num_heads != E: + raise ValueError(f"embed_dim {E} not divisible by num_heads {num_heads}") + + def _to_heads(x): + # [L, N, E] -> [N, H, L, D] + x = torch.transpose(x, 0, 1) + x = torch.reshape(x, (N, L, num_heads, head_dim)) + x = torch.transpose(x, 1, 2) + return x + + qh = _to_heads(q) + kh = _to_heads(k) + vh = _to_heads(v) + + if attn_mask is not None: + raise torch4ms.tensor.OperatorNotFound( + "multi_head_attention_forward with attn_mask is not supported yet." + ) + if is_causal: + raise torch4ms.tensor.OperatorNotFound( + "multi_head_attention_forward with is_causal=True is not supported yet." + ) + + scale = 1.0 / math.sqrt(head_dim) + attn_scores = torch.matmul(qh, torch.transpose(kh, -2, -1)) * scale + attn_probs = torch.softmax(attn_scores, dim=-1) + if training and dropout_p > 0: + attn_probs = torch.nn.functional.dropout(attn_probs, p=dropout_p, training=True) + attn_out = torch.matmul(attn_probs, vh) + + # [N, H, L, D] -> [L, N, E] + attn_out = torch.transpose(attn_out, 1, 2) + attn_out = torch.reshape(attn_out, (N, L, E)) + attn_out = torch.transpose(attn_out, 0, 1) + + out = torch.nn.functional.linear(attn_out, out_proj_weight, out_proj_bias) + + if need_weights: + # 暂不返回 attention 权重（测试路径中 need_weights=False） + return out, None + return out, None + + @register_function( torch.Tensor.__getitem__, is_mindspore_function=False, @@ -845,6 +1036,7 @@ def _functional_max_pool2d( return output +@register_function(torch.conv1d) @register_function(torch.nn.functional.conv1d) def functional_conv1d(input, weight, bias=None, stride=1, padding=0, dilation=1, groups=1): """ @@ -884,6 +1076,7 @@ def functional_conv1d(input, weight, bias=None, stride=1, padding=0, dilation=1, return result +@register_function(torch.conv2d) @register_function(torch.nn.functional.conv2d) def functional_conv2d(input, weight, bias=None, stride=1, padding=0, dilation=1, groups=1): """ @@ -923,6 +1116,7 @@ def functional_conv2d(input, weight, bias=None, stride=1, padding=0, dilation=1, return result +@register_function(torch.conv3d) @register_function(torch.nn.functional.conv3d) def functional_conv3d(input, weight, bias=None, stride=1, padding=0, dilation=1, groups=1): """ @@ -960,3 +1154,41 @@ def functional_conv3d(input, weight, bias=None, stride=1, padding=0, dilation=1, if isinstance(input, torch4ms.tensor.Tensor): return input._env.ms2t_iso(result) return result + + +@register_function(torch.nn.functional.batch_norm) +def functional_batch_norm(input, running_mean, running_var, weight=None, bias=None, + training=False, momentum=0.1, eps=1e-5): + result = _aten_batch_norm( + input, running_mean, running_var, weight, bias, training, momentum, eps + ) + if isinstance(input, torch4ms.tensor.Tensor): + return input._env.ms2t_iso(result) + return result + + +@register_function(torch.nn.functional.dropout) +def functional_dropout(input, p=0.5, training=True, inplace=False): + # MindSpore 路径无 inplace 语义，这里忽略 inplace 参数。 + result = _aten_dropout(input, p, training) + if isinstance(input, torch4ms.tensor.Tensor): + return input._env.ms2t_iso(result) + return result + + +if hasattr(torch, "batch_norm"): + register_function(torch.batch_norm)(functional_batch_norm) +if hasattr(torch, "dropout"): + register_function(torch.dropout)(functional_dropout) + + +@register_function(torch.nn.functional.layer_norm) +def functional_layer_norm(input, normalized_shape, weight=None, bias=None, eps=1e-5): + result = _aten_layer_norm(input, normalized_shape, weight, bias, eps) + if isinstance(input, torch4ms.tensor.Tensor): + return input._env.ms2t_iso(result) + return result + + +if hasattr(torch, "layer_norm"): + register_function(torch.layer_norm)(functional_layer_norm) diff --git a/src/torch4ms/tensor.py b/src/torch4ms/tensor.py index 45e04c941..d0e94ca7a 100644 --- a/src/torch4ms/tensor.py +++ b/src/torch4ms/tensor.py @@ -160,8 +160,42 @@ def size(self, dim=None): return self._elem.shape[dim] def flatten(self, start_dim=0, end_dim=-1): - # 使用MindSpore的reshape操作 - return self._env.ms2t_iso(mnp.reshape(self._elem, (-1,))) + # 对齐 PyTorch 语义：仅展平 [start_dim, end_dim] 区间 + shape = tuple(int(s) for s in self._elem.shape) + ndim = len(shape) + if ndim == 0: + # 标量 flatten 后为一维长度 1 + res = mnp.reshape(self._elem, (1,)) + out = self._env.ms2t_iso(res) + if self._requires_grad: + out._requires_grad = True + return out + + start = int(start_dim) + end = int(end_dim) + if start < 0: + start += ndim + if end < 0: + end += ndim + + start = max(0, min(start, ndim - 1)) + end = max(0, min(end, ndim - 1)) + if end < start: + # 与 PyTorch 行为一致：空区间不改变形状 + out = self._env.ms2t_iso(self._elem) + if self._requires_grad: + out._requires_grad = True + return out + + flat_dim = 1 + for s in shape[start : end + 1]: + flat_dim *= s + new_shape = shape[:start] + (flat_dim,) + shape[end + 1 :] + res = mnp.reshape(self._elem, new_shape) + out = self._env.ms2t_iso(res) + if self._requires_grad: + out._requires_grad = True + return out # ========= 基本算术运算支持 ========= def _binary_op(self, other, ms_op, op_name=None): diff --git a/tests/torch4ms/test_train_cnn.py b/tests/torch4ms/test_train_cnn.py new file mode 100644 index 000000000..9ea457c93 --- /dev/null +++ b/tests/torch4ms/test_train_cnn.py @@ -0,0 +1,162 @@ +import os +os.environ.setdefault("TORCHDYNAMO_DISABLE", "1") + +import torch +import torch.nn as nn +import torch.optim as optim +import math + +import torch4ms +from torch4ms.autograd.ms_autograd_function import extract_and_wrap_loss_fn +from torch4ms.optim import Torch4msOptimizer + +NUM_EPOCHS = 2 +LR = 0.01 + + +def get_dataloader(): + torch.manual_seed(0) + x = torch.randn(4, 1, 8, 8, dtype=torch.float32) + y = torch.randn(4, 1, dtype=torch.float32) + return [(x, y)] + + +class SimpleCNN(nn.Module): + def __init__(self): + super().__init__() + self.conv = nn.Conv2d(1, 4, kernel_size=3, padding=1) + self.bn = nn.BatchNorm2d(4) + self.relu = nn.ReLU() + self.drop = nn.Dropout(p=0.2) + self.fc = nn.Linear(4 * 8 * 8, 1) + + def forward(self, x): + x = self.conv(x) + x = self.bn(x) + x = self.relu(x) + x = x.flatten(1) + x = self.drop(x) + x = self.fc(x) + return x + + +def init_model(model: nn.Module): + torch.manual_seed(42) + with torch.no_grad(): + for p in model.parameters(): + p.uniform_(-0.1, 0.1) + + +def _print_grad_stats(prefix: str, model: nn.Module): + for name, param in model.named_parameters(): + if param.grad is None: + print(f"{prefix} {name}.grad = None") + else: + print(f"{prefix} {name}.grad norm = {param.grad.norm().item():.8f}") + + +def _collect_grad_norms(model: nn.Module): + result = {} + for name, param in model.named_parameters(): + if param.grad is None: + result[name] = None + else: + result[name] = float(param.grad.norm().item()) + return result + + +def _is_valid_norm(v): + return (v is not None) and (not math.isnan(v)) and (not math.isinf(v)) + + +def _status_by_pair(torch4ms_v, torch_v): + if not _is_valid_norm(torch4ms_v): + return "BAD_INVALID" + if torch_v is None: + return "UNKNOWN" + # torch 明显有梯度而 torch4ms 几乎为 0，判为不一致 + if abs(torch_v) > 1e-6 and abs(torch4ms_v) < 1e-8: + return "MISMATCH_ZERO_GRAD" + # 二者都非零时给一个粗粒度相对差异判断 + denom = max(abs(torch_v), 1e-12) + rel_diff = abs(torch4ms_v - torch_v) / denom + if rel_diff < 0.15: + return "OK" + return "WARN_DIFF" + + +def train_with_torch4ms_cnn(): + print("=== torch4ms CNN path ===") + dataloader = get_dataloader() + criterion = nn.MSELoss() + + model = SimpleCNN() + init_model(model) + optimizer = Torch4msOptimizer(optim.SGD(model.parameters(), lr=LR), model) + + env = torch4ms.default_env() + last_grad_norms = {} + with env: + for epoch in range(NUM_EPOCHS): + for step, (inputs, labels) in enumerate(dataloader): + print(f"[torch4ms] epoch={epoch}, step={step}") + loss_wrapper = extract_and_wrap_loss_fn(model, criterion, inputs, labels) + loss_t4 = loss_wrapper.output + print(f"[torch4ms] loss = {loss_t4}") + + loss_t4.backward(module=model) + _print_grad_stats("[torch4ms]", model) + last_grad_norms = _collect_grad_norms(model) + + optimizer.step() + optimizer.zero_grad() + + return last_grad_norms + + +def train_with_torch_cnn(): + print("=== torch CNN ===") + dataloader = get_dataloader() + criterion = nn.MSELoss() + + model = SimpleCNN() + init_model(model) + optimizer = optim.SGD(model.parameters(), lr=LR) + + last_grad_norms = {} + for epoch in range(NUM_EPOCHS): + for step, (inputs, labels) in enumerate(dataloader): + optimizer.zero_grad() + out = model(inputs) + loss = criterion(out, labels) + print(f"[torch] epoch={epoch}, step={step}, loss = {loss.item()}") + loss.backward() + _print_grad_stats("[torch]", model) + last_grad_norms = _collect_grad_norms(model) + optimizer.step() + + return last_grad_norms + + +def main(): + t4_norms = train_with_torch4ms_cnn() + print() + torch_norms = train_with_torch_cnn() + + print() + print("=== stability regression summary ===") + for name in sorted(set(t4_norms.keys()) | set(torch_norms.keys())): + t4_v = t4_norms.get(name) + th_v = torch_norms.get(name) + status = _status_by_pair(t4_v, th_v) + print(f"[summary] {name}: torch4ms={t4_v}, torch={th_v}, torch4ms_status={status}") + + +if __name__ == "__main__": + try: + main() + except Exception as exc: + import traceback + + print(f"[ERROR] {exc}") + traceback.print_exc() diff --git a/tests/torch4ms/test_train_transformer.py b/tests/torch4ms/test_train_transformer.py new file mode 100644 index 000000000..a28d1f167 --- /dev/null +++ b/tests/torch4ms/test_train_transformer.py @@ -0,0 +1,161 @@ +import os +os.environ.setdefault("TORCHDYNAMO_DISABLE", "1") + +import math +import torch +import torch.nn as nn +import torch.optim as optim + +import torch4ms +from torch4ms.autograd.ms_autograd_function import extract_and_wrap_loss_fn +from torch4ms.optim import Torch4msOptimizer + +NUM_EPOCHS = 2 +LR = 0.01 +ATOL_REL = 0.2 + + +def get_dataloader(): + torch.manual_seed(0) + # [batch, seq, in_dim] + x = torch.randn(4, 8, 16, dtype=torch.float32) + y = torch.randn(4, 1, dtype=torch.float32) + return [(x, y)] + + +class TinyTransformerRegressor(nn.Module): + def __init__(self, in_dim=16, d_model=32, nhead=4, ff_dim=64, num_layers=2): + super().__init__() + self.in_proj = nn.Linear(in_dim, d_model) + layer = nn.TransformerEncoderLayer( + d_model=d_model, + nhead=nhead, + dim_feedforward=ff_dim, + dropout=0.0, + batch_first=True, + activation="relu", + ) + self.encoder = nn.TransformerEncoder(layer, num_layers=num_layers) + self.head = nn.Linear(d_model, 1) + + def forward(self, x): + x = self.in_proj(x) + x = self.encoder(x) + x = torch.mean(x, dim=1) + x = self.head(x) + return x + + +def init_model(model: nn.Module): + torch.manual_seed(42) + with torch.no_grad(): + for p in model.parameters(): + p.uniform_(-0.1, 0.1) + + +def _grad_norms(model: nn.Module): + norms = {} + for name, p in model.named_parameters(): + if p.grad is None: + norms[name] = None + else: + norms[name] = float(p.grad.norm().item()) + return norms + + +def _print_grad_norms(prefix: str, norms): + for name, v in norms.items(): + if v is None: + print(f"{prefix} {name}.grad = None") + else: + print(f"{prefix} {name}.grad norm = {v:.8f}") + + +def _is_valid(v): + return (v is not None) and (not math.isnan(v)) and (not math.isinf(v)) + + +def _status_pair(t4, th): + if not _is_valid(t4): + return "BAD_INVALID" + if th is None: + return "UNKNOWN" + if abs(th) > 1e-6 and abs(t4) < 1e-8: + return "MISMATCH_ZERO_GRAD" + rel = abs(t4 - th) / max(abs(th), 1e-12) + return "OK" if rel < ATOL_REL else "WARN_DIFF" + + +def train_with_torch4ms(): + print("=== torch4ms TinyTransformer ===") + dataloader = get_dataloader() + criterion = nn.MSELoss() + + model = TinyTransformerRegressor() + init_model(model) + optimizer = Torch4msOptimizer(optim.SGD(model.parameters(), lr=LR), model) + + last_norms = {} + env = torch4ms.default_env() + with env: + for epoch in range(NUM_EPOCHS): + for step, (inputs, labels) in enumerate(dataloader): + print(f"[torch4ms] epoch={epoch}, step={step}") + loss_wrapper = extract_and_wrap_loss_fn(model, criterion, inputs, labels) + loss_t4 = loss_wrapper.output + print(f"[torch4ms] loss = {loss_t4}") + + loss_t4.backward(module=model) + last_norms = _grad_norms(model) + _print_grad_norms("[torch4ms]", last_norms) + + optimizer.step() + optimizer.zero_grad() + return last_norms + + +def train_with_torch(): + print("=== torch TinyTransformer ===") + dataloader = get_dataloader() + criterion = nn.MSELoss() + + model = TinyTransformerRegressor() + init_model(model) + optimizer = optim.SGD(model.parameters(), lr=LR) + + last_norms = {} + for epoch in range(NUM_EPOCHS): + for step, (inputs, labels) in enumerate(dataloader): + optimizer.zero_grad() + out = model(inputs) + loss = criterion(out, labels) + print(f"[torch] epoch={epoch}, step={step}, loss = {loss.item()}") + loss.backward() + last_norms = _grad_norms(model) + _print_grad_norms("[torch]", last_norms) + optimizer.step() + return last_norms + + +def main(): + t4_norms = train_with_torch4ms() + print() + th_norms = train_with_torch() + + print() + print("=== transformer stability summary ===") + all_names = sorted(set(t4_norms.keys()) | set(th_norms.keys())) + for name in all_names: + t4 = t4_norms.get(name) + th = th_norms.get(name) + status = _status_pair(t4, th) + print(f"[summary] {name}: torch4ms={t4}, torch={th}, status={status}") + + +if __name__ == "__main__": + try: + main() + except Exception as exc: + import traceback + print(f"[ERROR] {exc}") + traceback.print_exc() From ec4c9680fe129b0d16d31050bdea6d0f52bc2fab Mon Sep 17 00:00:00 2001 From: feixiao15 <467534404@qq.com> Date: Sun, 8 Mar 2026 23:45:22 +0800 Subject: [PATCH 3/3] fix: add missing pow/sqrt tensor lowerings to align Gemma gradients end-to-end --- src/torch4ms/ops/maten.py | 14 ++++-- src/torch4ms/ops/mten.py | 6 +++ src/torch4ms/ops/mtorch.py | 93 ++++++++++++++++++++++++++++++++++++-- src/torch4ms/tensor.py | 14 +++++- 4 files changed, 119 insertions(+), 8 deletions(-) diff --git a/src/torch4ms/ops/maten.py b/src/torch4ms/ops/maten.py index b80460f97..a5c7f8162 100644 --- a/src/torch4ms/ops/maten.py +++ b/src/torch4ms/ops/maten.py @@ -86,6 +86,12 @@ def _aten_div(x, y, rounding_mode=None): @op(torch.ops.aten.pow) @op(torch.ops.aten.pow.Scalar) def _aten_pow(x, y): + if not isinstance(y, ms.Tensor): + y = ms.Tensor(float(y), dtype=x.dtype) + else: + y_dtype = getattr(y, "dtype", None) + if y_dtype is not None and "Int" in str(y_dtype): + y = ops.cast(y, x.dtype) return ops.pow(x, y) @@ -1093,9 +1099,11 @@ def _aten_gather(x, dim, index): @op(torch.ops.aten.embedding) def _aten_embedding(weight, indices, padding_idx=-1, scale_grad_by_freq=False, sparse=False): """嵌入层""" - # 使用 MindSpore 的 embedding lookup - # 注意：MindSpore 的 embedding 可能需要不同的实现方式 - return ops.gather(weight, indices, axis=0) + vocab_size = weight.shape[0] + on_value = ms.Tensor(1.0, dtype=weight.dtype) + off_value = ms.Tensor(0.0, dtype=weight.dtype) + one_hot = ops.one_hot(indices, vocab_size, on_value, off_value) + return ops.matmul(one_hot, weight) @op(torch.ops.aten.embedding_renorm_) diff --git a/src/torch4ms/ops/mten.py b/src/torch4ms/ops/mten.py index 48f46e3e8..2871a7c7f 100644 --- a/src/torch4ms/ops/mten.py +++ b/src/torch4ms/ops/mten.py @@ -87,6 +87,12 @@ def _aten_div(x, y, rounding_mode=None): @op(torch.ops.aten.pow) @op(torch.ops.aten.pow.Scalar) def _aten_pow(x, y): + if not isinstance(y, ms.Tensor): + y = ms.Tensor(float(y), dtype=x.dtype) + else: + y_dtype = getattr(y, "dtype", None) + if y_dtype is not None and "Int" in str(y_dtype): + y = ops.cast(y, x.dtype) return ops.pow(x, y) diff --git a/src/torch4ms/ops/mtorch.py b/src/torch4ms/ops/mtorch.py index 783868e39..c907afedc 100644 --- a/src/torch4ms/ops/mtorch.py +++ b/src/torch4ms/ops/mtorch.py @@ -193,8 +193,32 @@ def functional_permute(input, dims): @register_function(torch.reshape) -def functional_reshape(input, shape): - return mops.reshape(input, tuple(shape)) +def functional_reshape(input, *shape): + if len(shape) == 1 and isinstance(shape[0], (list, tuple)): + target_shape = tuple(shape[0]) + else: + target_shape = tuple(shape) + return mops.reshape(input, target_shape) + + +@register_function(torch.Tensor.view) +def tensor_view(input, *shape): + if len(shape) == 1 and isinstance(shape[0], (list, tuple)): + target_shape = tuple(shape[0]) + else: + target_shape = tuple(shape) + return mops.reshape(input, target_shape) + + +@register_function(torch.Tensor.contiguous) +def tensor_contiguous(input, memory_format=None): + # MindSpore tensors are contiguous in this path; keep semantic no-op. + return input + + +@register_function(torch.Tensor.float) +def tensor_float(input): + return mops.cast(input, ms.float32) @register_function(torch.flatten) @@ -212,6 +236,16 @@ def functional_split(input, split_size_or_sections, dim=0): return mops.split(input, split_size_or_sections, axis=dim) +@register_function(torch.stack) +def functional_stack(tensors, dim=0): + return mops.stack(tuple(tensors), axis=dim) + + +@register_function(torch.cat) +def functional_cat(tensors, dim=0): + return mops.concat(tuple(tensors), axis=dim) + + @register_function(torch.matmul) def functional_matmul(input, other): return mops.matmul(input, other) @@ -220,7 +254,19 @@ def functional_matmul(input, other): @register_function(torch.softmax) def functional_softmax(input, dim=None, dtype=None): axis = -1 if dim is None else dim - return mops.softmax(input, axis=axis) + x = input + if dtype is not None: + x = x.astype(mappings.t2ms_dtype(dtype)) + return mops.softmax(x, axis=axis) + + +@register_function(torch.nn.functional.softmax) +def functional_softmax_f(input, dim=None, _stacklevel=3, dtype=None): + axis = -1 if dim is None else dim + x = input + if dtype is not None: + x = x.astype(mappings.t2ms_dtype(dtype)) + return mops.softmax(x, axis=axis) @register_function(torch.mul) @@ -228,6 +274,43 @@ def functional_mul(input, other): return mops.mul(input, other) +@register_function(torch.pow) +def functional_pow(input, exponent): + exp = exponent + if not isinstance(exp, ms.Tensor): + exp = ms.Tensor(float(exp), dtype=input.dtype) + else: + exp_dtype = getattr(exp, "dtype", None) + if exp_dtype is not None and "Int" in str(exp_dtype): + exp = mops.cast(exp, input.dtype) + return mops.pow(input, exp) + + +@register_function(torch.rsqrt) +def functional_rsqrt(input): + return mops.rsqrt(input) + + +@register_function(torch.sqrt) +def functional_sqrt(input): + return mops.sqrt(input) + + +@register_function(torch.nn.functional.gelu) +def functional_gelu(input, approximate="none"): + approx = "tanh" if approximate == "tanh" else "none" + return mops.gelu(input, approximate=approx) + + +@register_function(torch.nn.functional.embedding) +def functional_embedding(input, weight, padding_idx=-1, max_norm=None, norm_type=2.0, scale_grad_by_freq=False, sparse=False): + vocab_size = weight.shape[0] + on_value = ms.Tensor(1.0, dtype=weight.dtype) + off_value = ms.Tensor(0.0, dtype=weight.dtype) + one_hot = mops.one_hot(input, vocab_size, on_value, off_value) + return mops.matmul(one_hot, weight) + + @register_function(torch.mean) def functional_mean(input, dim=None, keepdim=False, dtype=None): x = input @@ -252,8 +335,12 @@ def functional_mean(input, dim=None, keepdim=False, dtype=None): register_function(torch.Tensor.chunk)(functional_chunk) if hasattr(torch.Tensor, "split"): register_function(torch.Tensor.split)(functional_split) +if hasattr(torch.Tensor, "pow"): + register_function(torch.Tensor.pow)(functional_pow) if hasattr(torch.Tensor, "mean"): register_function(torch.Tensor.mean)(functional_mean) +if hasattr(torch.Tensor, "sqrt"): + register_function(torch.Tensor.sqrt)(functional_sqrt) @register_function(torch.relu) diff --git a/src/torch4ms/tensor.py b/src/torch4ms/tensor.py index d0e94ca7a..d0a81d999 100644 --- a/src/torch4ms/tensor.py +++ b/src/torch4ms/tensor.py @@ -308,12 +308,22 @@ def __rfloordiv__(self, other): def __pow__(self, other): """支持 x ** y。""" + if not isinstance(other, (Tensor, ms_Tensor, torch.Tensor, np.ndarray)): + try: + other = ms_Tensor(float(other), dtype=self._elem.dtype) + except Exception: + pass return self._binary_op(other, ops.pow) def __rpow__(self, other): """支持 y ** x。""" if isinstance(other, Tensor): return other._binary_op(self, ops.pow) + if not isinstance(other, (ms_Tensor, torch.Tensor, np.ndarray)): + try: + other = ms_Tensor(float(other), dtype=self._elem.dtype) + except Exception: + pass return self._binary_op(other, lambda a, b: ops.pow(b, a)) # 比较运算，返回布尔 Tensor @@ -345,9 +355,9 @@ def type_as(self, other): # 确保other是Tensor类型 if not isinstance(other, Tensor): raise TypeError(f"Expected Tensor, got {type(other).__name__}") - # 获取目标数据类型（MindSpore dtype，供 mnp.astype 使用） + # 获取目标数据类型并使用 MindSpore cast target_ms_dtype = other._elem.dtype - return self._env.ms2t_iso(mnp.astype(self._elem, target_ms_dtype)) + return self._env.ms2t_iso(ops.cast(self._elem, target_ms_dtype)) __torch_function__ = torch._C._disabled_torch_function_impl