From 6b0b0d59f68cd10828d3d64874a7a64e1c123319 Mon Sep 17 00:00:00 2001
From: linuofan2025 <2531980518@qq.com>
Date: Thu, 19 Mar 2026 20:23:36 +0800
Subject: [PATCH] NPU KV Cache

---
 engine/50e8005addb244c88d30f9daccaabf8a.png | Bin 0 -> 18791 bytes
 engine/BlockManager.py                      |  56 ++++++
 engine/Design and test.md                   |  87 +++++++++
 engine/KVCache.py                           |  56 ++++++
 engine/TransformerDecoderOnly.py            | 185 ++++++++++++++++++++
 engine/test_kv_cache.py                     |  51 ++++++
 6 files changed, 435 insertions(+)
 create mode 100644 engine/50e8005addb244c88d30f9daccaabf8a.png
 create mode 100644 engine/BlockManager.py
 create mode 100644 engine/Design and test.md
 create mode 100644 engine/KVCache.py
 create mode 100644 engine/TransformerDecoderOnly.py
 create mode 100644 engine/test_kv_cache.py

diff --git a/engine/50e8005addb244c88d30f9daccaabf8a.png b/engine/50e8005addb244c88d30f9daccaabf8a.png
new file mode 100644
index 0000000000000000000000000000000000000000..3faf8a260d837173f74f22e12bb1b0ee26fdedd5
GIT binary patch
literal 18791
zcmX_o1yogC*ENU_-72MYN_R;}cSuXONOzZ{bayvMcXvs5cb9bczwi5v@vA7~a=d4s
zbJkvKt~uwrLGrR<D2RB7P*6}P65_%N;NLs&H3Sa}{ubibh6n!;8Hg*$KtZ`uKtXx?
zK|wu$m%R6(pd1*XpbmARptuvEps=k|o4)aYAHeEMi3vl!LVjnq7Q}#8;Or%2MBomg
zv0%{Ox}PoOgO{MKY-N?8$FiN0p`gUMC4>c(oEMMN+??MjKiu&*vdIXhu+y~)oPI^3
z9rBpSCLuBVC4wQMdDTqwBCMwOK}6uIlS7`k7K5|E=aXQPp+Dvr!ZpF(RBzES)UZta
z>uUV-$Js3=DSjsB-alio)W~OkrKcO&bGeW<-W~pODt$4^b(wzKBDQVMo%R+UJyT}i
zo?G9*;JH*Z1>;fU7Zfx%;{E--bPD^(nyDsM_E!`FqOVI0_q!<zZ6065A_&NNc~{mB
zYl37od=p}084dc7cSlnCliAEhmrkuY$4zS!Vq+2#oEG**2M3!9TcV>e^>`^M1D~GU
zYt-quxVSh!p<Y~G4h#(m6-?bfvOD70bK}eLKeO8Hjf#}dJ!+(}PyIxBDk?}fyua8n
z{PDwXcNn+Rq{GP2?dG4}`DSlyY;1+)3U@rc7WF?ERXaL<{tPa=UHE|fsRO^N6S$tq
zJgFkK>FAgkAz4{eC?g}I?d@%Nw8)kgo~o)UDk`du4ePUmf(*pHqq%ZI*H(g@3GsH%
z=lK>F7O(rEYBF;2ES2Jcu`x;-nh>ydbM}v5&82)5Z#3~MD=Rgfo*n!G0-~d$Hmn>L
zr3+ZC(`3b7Uts*Mhliy-9}XzltyYmc9<GlLrs~nQ`XlzHixgQ|S<lY8(PHA_$YUY*
z(6lxg9Nc-Jps6WsWJHc8?cq%>6<?v(4SUjZDZ8q3&qpd6s@wA$11wyZJ;h}|D{#Kf
zB1K+<|8Cv4<z~@LoD6evYO3*a^vl4&fZkG_1r%gSDJfE~FV9Jg8?umf3=R&ewKy}e
z{21AW+{&%KF#F@hxZGT_VdLs{t4p?^36raO!~~H7*0aBV#Ssw^{jN#H!Xsm2?K>}g
zVNnE&l<JCp&X()&cTMZ<U%Ltl2}wvx_ZTuqB_~U|x^gQkEBEyEkwl5_uQYQM$foIo
zRea)cMvs&H@9XJmD>yc{u&~hIyhsW0tF<*_XJ@C9s%nlB^+Kx|$2*GU0lFMZfg+_U
z(-~^8&?k?SatUyT=h7}C;_Lc1FJCEQm>_XAoa8H-1FgH+6J$PLA>h1+a)8;@7xwW0
zJh%{S1`0a5u%A=;=Tmb0mG%b;ZTGt$Db~IuRaGfSN=k}LOTX1Un4AN5?&DOx3@HZ(
zc3^j>meWcv%NP$#2WbRPR|rF!I0zB`*C&>-m!0n_rRx6L<pot$nBE+#&M-wvgOGy`
zXNo6GSi8DKN>wf=#9ocN{Ju!Y%8Hh%<jfs$IiC?pNJ#u!s*kw4bN2D|eFhhTm@*>j
z;=+X$@?Vs=-_(IH8Rorm!Qr9ll`AhfIXQ@hJ7w(KXKU`X!kI%P#}x-thNkX~4%6Cf
zC2D^Q<Lbs<S8Z*AtgI|#eD3#2(y(7kv*H;Y5{Y2Gl0;?Bm@?soh2pUxCP@c$Z-7^z
z%J^xC{+f8E+ud?~LQPIi9*x;}v8-M|59#f?*?mYHF*amQA}S``wAAYD?<b8J@O#{n
zGi1gEkFVu+)s0rOp<<=w&43}ZwDG$SO~EQCt%|ejmSmU`NCvQ@8to?!M^NfGp-YL&
z%*$5?O9dheyN{!64ZC|>P8q@^7~vN5-Ota<7bDkPs;}VzD#dwnYkY49e4rNnZ8!>A
zTYZ<z*?kNJa`W>Qw6#~W*$~oNVA00gK#D-dVeD8lO%0T$$f>A6Z`B^l5(?Z0PZh+o
zu&8KVGhAe+Zf0fWH^vL^UvMVaV9RHsDk8Btxd~9jR2mIoJ=~tIW#6Y{WT1UDH8pK!
zxu~jQ-rU;yMje+n6+)2dv0=?wp+<W=*>HQZG9<&XV!zsK0iqV{_LzmY-kS8c^-XU+
zfBqaBAO9w;CWZFv<L3523>g{Ookkea=8YM+Z^UW{M5Ls?q=6AoXS*p@3hL^?3Pr;7
z32xUQiWv@tYZuI^-Xof`YwS5z-3L1y%|e%zH9+kdRH`k8)wgg_R7>oOj({w}!I7ZV
z?1+Jh*%kX)5sK4h^KEw^8us8_SM%z)t{#Y{B-Wpk|D7grOUr<V($V1oiP8TW9_V3G
zBgDx}o%&GF$&fHN(Ohr-4fvw*!FPa#`ZI&5i%m$V22r0cn<hDy&im8U^debSzcLZz
zoXd>Yr@iQ=k9a81Ft_aLrST#mSO{xh6h2imm_!5|=ew||kwl??_|W@&yT)Lz8Xp@w
zUaG-CLQ47_Bn*|}55+Tw?^c$-MPN_0W@Qndp`KVUGc)7xI5Yafp`qe9(JVWy#X5d;
z_))!p!>H@c*db051-~L!8=x+m#x0_$`H4Mc<VS!<<6F}+D9@bCOz&?wme=)o3ETN8
zmfTkR6{0zM`{P;1P5-@tW^je#kB*P;yE>SvRHJ=S*@*^GA8WbNR0$FTIH|a}xXEoF
zN?HS~6Xz$jYkt9)TR4=fFeew4k%Ici-No2v1O$Z3fyT1;)eF+!zvKS=`SV33Pv);7
zbGpaPcfO}Hq1~|zxw$f}=Te1~{Tf)y(#2m2S#Ok7>5derwpzWycR9@itvQ+1dn)TW
zAcup*y=TR?7%E#}aCf#r6(@OfywrbpzV$^IW%YB){%jNTc9q%fB)P(WVF5nH*xMq4
zf`Z(!Q%uav(F3=#6nuQ?LGLIX;kN>0P1m8HBEHDy=IB;0R3y^jgo*0?wiYAOU59ob
z1r_RxG|K!00?u6=CRw++gKix+k+@^$bLq)Z&orLtJt$Ay_JYCT!UYahS|0?M+1)FB
zGf-bb^EwC5^qw0G{-MwXyEnRU1YS!0-v6U_2;=lsh=yq}tgadj>h^Yb$P7JBLB!Fq
zu6<^o^)2!f;zEn^&^wk>$~-5u;BW%C^>T)`K=5AOQxq2`gAU8CLk_fFKj=JMbE{7~
zv}I${@|{izKJmJyMsXk;_Bg=&_1qDnMNvt2v!%A$E7=<0m)qUp;NKlMnA`1mU#Yg7
zmH%R6XF30e16_iQllRG!knxNrYobqsVQlBvtHhu``@_>(Ytkof^&Z005i)YF@J26>
z!1MYD<BNumT;Cj%^*FA!uAN>6lD+7R20g03UzX<$C)aw{5Fk9AuS#Fk{WE?%fsuY%
z3A@7Q@eg}-Sy)ru{{X;5AS%B8!_|Ry6hpK)BMx2S*tE+rUbMc#XYPNkp|wl{7zAur
z-oJDZ1ekYUuJN@8)4NHleH!-CDREgG)rv(%^0_mkPF8Y$iL!1;W@6+~`?b(DOUQ=i
zXlTaDYr|(Yz@{8DHa7A+Cj4k6(IAV;{Nf24(+2Gz{CloUZ`q>WO@{sJMw_C&%xN&U
z(gp@88hD)JCI)>lPK;)=c2LsK?Haf&`iyqfFVG#Ka3|~I-&AEU`3`vRY`0XRH*>N&
zr6(DDKY;JRISp<j#NZFIv#nf72;F1*>(|<bmR-uvCc6TkJmLuU;#9TIgEi}Iq(qpf
zG4PjM3c}&DnA@Y*b)2rn15#v@Vn)<e1Ew9{1S#3<2##Kd>kR8TS|-`m*Hi3Kn^}cu
z1qn{|!#P!3Z13$&RsR@MS<|_^yey~A2@xmRxI@d+E^YJ(Tlq;^C#F<f>kKxMj^{l&
z%a73(U8fw7q6-VD#$_CPSr^qkldbH$`0JPb>vRf-*?I_H@5IWv3UW6)*6sw~o=8KL
zE1DJV-0x-F*5l!$%C-19Ny1>m_1|gySH4-Hfj+b-XU8UuFKTR@4<a+xUXn6Jnxb3q
zt-y_`SMlbDl5nEfWM{8p=myi%QjI!TOxcGZ*`MoBh%$r#g~_0N(B_}}<qHRmGv3}l
zH#Q><Pn^tHo>5TR(th#$_4rtfyYyE#!-NF$>6|?S?fpBb46m0)aGdAlX>$X>N*cp)
zA3EyoO<`{TQ!iCX#A;wMR=%#cQB9rADtE}^oAvwoJIGQav1H|^5Os7KWsYjOc9)~v
z!JJB3nV;-|;U#%@0LoVFoHpdkV7@|6m7xO0W{fxHiJO)+R|?+Ct@i_0!JI165`Ax!
zPxrsoRw*j}+x!0iaFu`jStQG$Q%aq!1ly~8)%PpPi4hrQ5aW)yn{3whPk?{V&k;Nc
zvHl<*`cMqBeUXs1rzPBZLGqsZA$z(0Dr0c|acqL$0{w8D(KYvKJEWYyV1Js4S?ozR
z5+g4N!|+P_xcBI^Mbci0wO}mO_@9+m|C+W-{K-l)%SN5V`wS{FuFoG@+wj6Dd{+2P
zq9n<4$*wc^du(J?;+eq#|6T;7v<&EYo7^}kwK_U9vi{V)X`Z)ZldBz3c2mOP{UT0G
zQRCq}s6ookK5Ld$ln~OU!Z5u~Xr<P@{Uu-C{tzdn?LnF<rvBN^HBEf4lApJ;vMYm2
z+g^J9hh+)2;U}7>r2RsZlhz}?_|`}62X)>p(G~g<7^S1)iY0uWn*z%5qhcl;v-^8z
zXoVs_Z0wdUa5aOzuN)}NQ+=vBUI#N~XgJO^pNe?{r2jQ0|5%S~gGw#o-NA5?{LTn#
zrq#@T=Huhz7{lZ+zkXC$>=ecrGTCE&s&$)%iXxN~1cg3}(zkdS`cbpgdzabqc&)S5
zX{CRR{Ql`Cp6R_Je8<3`f+e0S5<ejW5jS-*1Bca{^}4liy_v(?$|d~inK=yn4<q1_
z@-^Og|El+qrWL~7zW<7m2Up$fd;z`v^Wx%y>htIG=3jF&b4+^6!kvuJ=}(tD+FpCg
zWsfqj-LN`=+BzeUOPXJM2~XTtQ+a;mC1H(1CSbhsVK$cWNS#}AiTkcdNJ+sRzn!ae
zczEP&U1f%29ub=k<-_BB<Y_nC$bC|`<Ui~C$g`TQh|1pho<IJijTT0g>Vw*FJW`n(
zjF4~oQH<r)5^ocZ*=z{_76@H6%?X>Pa4Db~UN~hm?82t>kcrhaF)+^e1X9H!YU^BZ
zBj%YICTI0^?1rWIU)v37n?I}T4aR=9U_XIcCD&aiSeKS4I@7{0%fBsmQ?xGh#@BGH
zFc?C8FZth0+>CqG;k!RHTBGUU)w6*=a!^!sv_}2$_&BD>FKw~K#w$$@d`YBVzAnMz
zGz2nD<BVI*RlUSqU%fb3g@57`>@7lkxsLE+dO10NGF!m=LsKjvE}8k-@nZfoCVIiX
z6BQZj&8>NCbzP1Kfcr<Q7L3ByB@BRQh(=PDJOAi5Q92d?xQTNGD~pw`56(;WN{tXR
z(eofTtLVCEK9Vt8?%|wT9+00;!FRU~<5i52i3`op1ZPr_=;9GnjK*r0YNjjVd+;we
zz%(qGpk4OA%sVxvm3LbRD*q`eD*Bdb!mT?Lq%uSf7<!_~w8ozaU-W`vLXK7KFJjfc
zsIMVt>QiwV@U6`KAcFT92fRG7C1${+fx~I<-L8B~#hn%o5O#z=686K>@=ZR4mrk21
zJem9c*9J2axtZzXQo^U!+tsa;=F^*{d*x(ZueCQ3dWB^|Q!6Z5p3pF-ne)kus;hO7
z|4&!Zdti;eM2aX>dwY@fh$!TMPc6fEp*v=u1)IEInu&}EU9gAzSN`abT_MnAw7Dd{
zJ|hzqG^U1zxjJz1ocO4&PZXU;?+BlsF2}AUjzU&^Un6#%nJKy7BDW|%^juvRwv^mf
zK|Nh~9wOz-r1Vi{xNqJ@8OR3u$}6L1wGX3iO8Dd|-27YU)=8k%1#nZXM5SEIm$HBX
z5vL&mblVrhEMriNcC*r;TYOO&$eA~1?~5Q@J9%j1<?Y8-_zXI*t-n#E+dDf<rqhL}
zO#j(-bhL(M%@XCBGEd``als@E7;2}OmGIhtX5ZG%ZeV1D{PX7lKr<wH$BYb-QBZ(Z
zGg6#vPLr^5#Ml6Y){9CvW#NqJw5e*Yo~x^CU|}n<yl?#=TK^$0wcIp$AC0ZoRCUPM
ziG-Q?a>GRV-YuMH4n|->%Bwb+?m9LRZI9jaA3fssiw4hH2NYF?ZR&%idIr!Eyr__N
zh`{4Gd(-r-A>cm=n3zyu2KAK7_O7qjsq#U?SdpsO(b*c!E90ZVGnc8Pw+{>sZg#>E
zlCjUaefC-+9pkw~hC?GbD@Oe>s>BTTA$uGOj`K%&74h$^w+L9Rjlr-XTJy7uw@9V0
z4Q_<x?RVIGD@}nyk7(ALX9RuAZnKZex|K-%|Iv)Teor2{<UkGd1r3B&_45O^!O3zX
zAVlhsC^a<f!5TmQTqUbcHXFEq3=z*$u*7|JYkRp8+I>3yz~|!hImfa+BelK#H^AFH
z^FdDy&oqE`!R53js$cW`@=<kf{FZ|Q12e6sjn{3y)d(Nf_t$?5LoVBGYc@xv_1RMg
z@F*yGB_*L(SN7oleUEHyS;1)(CeZ^`5Zrq!&QIZvdg~U%02ufqx1HDL=H{Ml-N-<`
zegEwKK25eiYegq#!US~K##>^i2Fyunk*%qb`(y#3IAY(vVdS+Ndf%I_ND@3su{*8u
z`{Jj}j@Mi1oFuV-^zH1JUO@U*q_oUp!A>3bX9b7jD`<EM2G3Wc0Ax18EQzc#8mfo7
zdLEGhxY`oO<>MI73uW0yHjG#dvRnKvx}5&W*9Q$oyf7b|<PjBP`oFsjb_ABh)lG*r
z<NdDmH#Y`z-a`#*o{tugv=H_-dEX+lQ=gKTHw|rVu*-zi_VudZ?%x`hZdv~8bv;pB
zlMf;6D>K_fQ=F8G#hl7X#!D;_DG4)PoTRL8?`Rd8=Ezc#9rpRd+HO>IbQj94Y`<=d
zetox0l9PUYvpNC_))!x^r_y$-UmdS6xBSLq>BpeS@SB(xH%g<-h-B7xL*5aW2ot-O
z4m(d5<BMc({NTx{vS!NPuypTGCiHJ8!WIh0?(_<YdzD^TV<=6^Ld$mzMvJF+CSRrv
z-T%X`<C>yg<zkQuJNh5Qi$YDO#qF;sp17<W!nd~|+Rg&%>t1aQ>zn{&1`T#vnOO<x
zJH|64>+eze2{@j#XtpcHW(NICyR&K)y3iVR=I8Zzo_mrSS}&koDP?4o2w(Bt=v!)c
z!0p|i$oX00F#MzkqH<>_!O7&*r9yD<$SbA$s-S9Y$&=WZmt(b4UaL1PNAxUjmaTq%
z(}hL-xXYYQwu=xg;D6<lqC2J$2~ayQ9UI+&-|XdKqQuE^3kn`J!ixJ0nQPV7>NYZz
zl$E2Fo#3H_%0BwfXiB-<OZ+L8Rdl}-1JMK55wXMp`6*~ecf0GW61gAEGshOanh5Y>
zUe<U#ms|aPo9iF}z-hNhMG+IU-6q7@U`F$LdQO|6ZIziaJY-U<4vR)Gyx!?MVOsm5
zvKMU4`Dr+jse8@T**~|f8;8k}#O&pq0wD<Dslg3NgHIz&lAO|Nr5*AED|(lJ!%0#Q
zXM*iIXvUyTgn|TooMdkT;SQtzKVi$)XZzU~mXy;AKBHY9^F!GT-M_z=eusvVkuWE}
zJU=SBKY8)Bwt4?@O9~A|HXcsw1kAwSzkeHCe|hSH-DMH(aRUFACKvos8>xLV^QYOV
zRVX6LQ?<JBc&(Kmaynl8;Qblx8Q0rSIH92Z&!6C;YV4{_HdWS~UVVYmlVENV%$$jN
zl6>&Jq_{mMb=;>|jD(zgF?iMq5AXlceyY8or+rSh8d2=&@4p0LtIM3!;Pqet8eY`b
zvouCkXLom=ZxXE%;s;?MeLP|xL6%R(EfI-<iyO3pipI~E5k}ZLR=9M!_iV)Q=B#7*
zz6=L}h^*%|hzaI-osWot0nHnboL<GY3!tI>_E}k*&06QimxJ)H-APCKWOh!;sEd|$
zGFyaO1Nt5+dXwf4^-RKLUqHM-_&~=OAD=fF87$ls4}UH?TF>gSov$Xu%9HFB_wckY
z9cr6sSoY{mF}B84TvRb(Hjuc(ejV9!hl-M(pP%#b*MB(*Di<RHtk&Sr5ECdb^IzHU
zM%WCN6S7;yTD1_;vV%^nIXNBw;S?1W;TtDrP!^Sx<eJTu9n;rVezann;3W(})u?kH
zD*VRbrj*a2Kk-HR-80}_dV2coCAVOIhlGHZXunF4wp61&95nn78qYKoN*}f>oirLX
zNBarAhPTkx#0(4!0`D0df-S280$_a%6K;i?eY8eB%dos$vn<$`jCRi7sJ?pq{CQx6
zZ%3!7s?&c|-Y#n@`3ABO##Y3%Y}j_po0F9#eO@lEdmn>32LDdobr7=n#_Y)6zkkCM
zBtCez&zPu|=(MLpkVfTJpJU=$rQ-B@LI`N9Ko;BVK>)SX-_yKNGhCX&tdhs6`O*2d
zNug*Pv}lx{KNpFFba;d|Q<xN+-!|1tNJ%yC8tXOUDM@~Sa&1j#bWqELJ^c$xxo8-k
zh@Kv?gM)*iv2k=lLO&q<arxXi<Kp7fXtBTpb3VvRP^&UTfUxbf1KNUs!OY9c1G)l%
z`<Vc!9ur5`hDJu;R8;z&?k`;)kLwXoP>jyjJFiYwH2y||UxLcYA4GGa{{1tzwY9BK
z)3gf)AS1=)ayM)0;P2>Y+a^|To?5jr3Sg$2DD=m81#(KGVBk=p%7U;~i2H>CQ7vxS
zTry$Xq4r*^jjNNIPntr&b79u+frn7)xeA|ujHYG*Dhi;fi$MOTUx3!D_%TKRSag@O
zPB?Jo24FRYhKAN>>(KEGI->;{p8sQiyZn*9w|nwph4{zDzOSmTUTCx{h%Bwi&GmbJ
zyk#*S#*wCY2gDK_WLUr+@33vbs#WTXbToKnP0ap1(P|#}`xinYf)Wa0&=_^0Kyv*F
zR@%|Q023zayI^j7FqxOi?I0=cKHl;;V8Y7mc4H2RaDXWU3#Qn<tbRcK$(#g_Xv_7i
zM6EhChCjO>?gg7*Cts?vWzUVvWt+>$A75WEdcPUM@Z?dv^dEClD7Z9ujNhn%LU+7z
zlH_2S1%-v;&K7@4O3a;L2~Dzf-h}k}bZ=y4b;dzuPt5+-){&8sK~X=J?F+>PE3&j1
zb~|kqv$bUgs6!-63EmfbdFQ0<(K6lZ#RV4ilQ_r>0E=<XYRy{KPvwZA{d4>x7XC@^
z`Qe&Yr|pw(=laZ1DcdycYOB}AvWljb2R@<KQ)OffJv)2+=B9pdNJtXP57g7P$Fea8
zb8X@p(E@n;&s0=r&TmsXb&05{gPUAcxf6C*tZf@pV<_biIoc;?#Up9O)dmfj&Dm3k
zx);rv@9l<~j6`~S*6lg{juWJejCROLCrwdmd3Y@FGTf_X&DurGoN2~)On=s}DP$F+
z+fKDVo-{*f`ZZ90yJ80*9a3-b2lKVIG{BACE47dDJ%%b5tJFTMohbdHU;SFDVo<BD
z!Ny)Zv$#2MNWZ1sWCmLwx|^yoOE<)u^&E%eq`_r-2Vebz|D;@F@kIW!GSEmm;@EEe
zv9;H`(fG}o2wfEvFag4#Q!m8|6BTlGZIUx4j1wp8I$G(4G(^_zZnaM3hrRoSTQjZx
zkM;KLbyRVKSbe24rV!}}1Tmn*-=y9)?b0nilo482X_146J2|P)_VQo``9g+;v2E1c
zS82ovtLt-oN2ek$4^UDN$Oujl5*36&s*@F{_!t-%bm~UO=Nkl!XT`#_STQLnA;KiY
zJUppjdwo;Y5wo<U{~7}IAyY&*q}QC7gQH#x`HjoX!VmdCm-3_19R2DAzs2vxGgZbT
zcv-tQ{eOFg$j60m(E%a@_<M445<dDr;yB5up&i=qRd3@Y{Q-9hR66&k^ZxhP*gz$5
z0jWDEG*q~F29l5{WRkb*)pc+F^~0=QA^~y~WYaAml7akSb9bhf!PZ+I4@x6coTLcI
zB8dJzfZ4~h2!s0c#W55JRrzum^uk^EJkF6IR{-K9EG!J*`>5<}60kd;9|{m!ymUTQ
zstwotoZ__IhIess!8g_v=;+|}9x&A3ElKh#>IU+I`ASvX`1p7T!PNZx?_T|lhoka-
z6IOtETkz5xnE*EFrYUdPOt5o$Kbp!V1a1(Z0m&#SA=>0SMr*W8g(Bw-D+Z!C$%zAd
z5cR2PU!0<olLrUmXdr?dP~m7b>Oz5d2sBDbPfxzfy)m&oh4Z_u2o|7fz1)v^J+!ER
zWn3N346W6VoBi%d526hT4SlN_6$~hBH9T{Im}B#cRVxB``*SGfB;?BZo$^?5GEg12
zZMgtL->4}IHWsm#px^bbl+$>K+lq!J)p)OQH8EDmGq*ud{TnS=oDe3CS*_MgW2Lwm
zbX>v{CYL&jkL|(xfx@=;_lWUy8hwdO25&&H=hv#wS(KdUChi`MB%Bzu{#;I&Y$lZx
zv;Q0`kS?Q@8hQBLWw(#JBpfz2;pe(Vo}zt>e?S0Yc=_mWkp1)zQmV}tsxC1#-Zs?j
zzlSCG_)*HgEE{mO=a6jF)zzhvV#A$Azs_qFkd?plHT`f9kbU+u_J}ZWzk|ZEDk{d;
zOz%>`1DrN>DDGrkSXfXcI$zvrrd#`_?#y0QaV3n4nD@qC!AAT0Z5<Nb5>nbOJ`W%f
z#S_qF(|Hc8l-+^aCs|CFAd|w*WVy`gRXleh@81M#4LFXQlZRjMVF4{NF06@o4X0U;
zRJ`B%2BMkmcen}M9+)5$NTp3#3l!-37p^9}aGXw8kUw#EtR=j$CxL-WU@595J8#DU
zoqPoU3#aaSLT2am*II44-3IAtJhnhj?w%j7tACH%3CA`F!>y~f$%M*ZQ78;}*F2-d
zZhM?x&i;1iUQ2cB*WGD&WWHmYfJx1(tc=3ecJ5p=t&x%zRV-dQcAR^Qm}QEa(;u;0
zCsD7*oP76^a*D(2X&+K`+}l75NSI!8cDu_F4x}Tg1bPc5N@uVwzJLEg>7Bu0MF2#X
zjH9H)L>!ZW8o}3J&jU-b{lOe6@_?%mXvY=y6ZKWk5v7icj*RWdNcT*gOK1rnP?J+{
zXN`8FxCER{gNr!@242{Bj}?RcABv`WAfVA|w<LnhVAGH9l)9Ja$oYxqa+qb=VNMHR
zbT{5-1+xZ1PUzG7d$L@Lv-znA1U73y)1mc`4h=N1^ICoSb&P<}c~N=Hersxt784Z(
zX`q2t%Qq%vQLBD`F~o4cMfkeU@f=U@5e5wddv{M?tFt)SVkswcCWTSh+4>c*&j42>
zj_h7G6@XHBzwV2c$YGVP1DjCde7oYv^s-FJX_06FdiMU1`4*f&R6q%Fe*gYGl#oCD
z{&LT5jQ92oz?=GcD^@)J-E%31)}E*TE<N%!eheih=brd?Z#2U9J-8w@p_m=;I&j~J
zj?HW)d<^vD0$m^h888!ogab5YF@1d!pr!p7NoMQvY}BMjCe;G!SkdRU5i;(LaD<mO
z5k4^zgp0poMaTm6dUQL*ul}{2n@o+Xtd{W8>kyMf8fcJ^jz->HLDM~?kz*qCBY7mi
zK->yqgdWpt^?wU~^Kky|OC!)=*~BuvIU(QeOQF{Yn1WkS83Wu~n%~vcl`I>6lsHM&
z%eWVYva+kRv7iE)C;b%0J4~OSOYuY8v$zX!^v|BhcBH5Fu-ayY$MV|a#R6GM==rEQ
zv+^`#XQ$?c+1X5KT%g@7EH4#t&&CmHvC<g=aQVw8)NuSS0qUiePB3p;{Cmx_1XVjh
z{z~F@_|raf{!zy93$e{gnRcYxa=UZ@HT6Rksxox`CJAzuX%{PA#>{qFU&rmgg(>2E
zw`($iL!#{LY;g&RgV<UpBy;vd>M^2t?G+t9PQII9jkn;vPy%1YJk%1$l8@55Ivlw1
zA#Lbj!-`s|3Irhn_kp;S)MY)lXSY%gWv<zKO(R+F)r9YM8B*}@Xz)?On+~L>?_n)j
z@xBO0yISd0FTkUs-qhnA`!roBs%nL}y_u?`<Fje`@1c{yTbe?MJ^?ACSb#pk=`i-&
zF+!!1>%!^Ss-Ti`(7kY_>e$As*!M5O6fx&lQ*wZo0Nis>gxvas7-62<?FlOzTP>AY
z^@2K?l65<5Lw!)M{;K<>3>4_u>4x<jP7M7=<`{nHc++$ngx446)y1nbaIe0BT}5P1
z2~*W}@dtfN+w+YEC{wZm0??rR_yq<67GxgSK31BYC|Ouyc-?N~zkL%B5`u}LkjZXo
zNd?9f%y;h&*4p{k+%HiW+Mk$#WD6iI)BWmz`BGgNKtZ+uGQ-GgY7PTT0CD2n-z-@{
zOb`%70(5e8v*1yX{yp1O>YC%$FPerdEo;ECAggTDEW93&mQd9Tp<j$zx;NI(HoBEm
zRG6OvHh^m40j^3$MFk6>YJE_$TL4IcoUqmH7GliM_PAEAvseN#k`Hdo)xU+9Z#kk-
zF)`h<bQ+-AOSd^ON4(sB!=l#|2Ne?Hps;*Awt&0=E{idvZU>aM+c6se)3V?U;CVyx
zN`j6jm+mq_hf<CecCE8re+sDAzV*;ft7jObqUe+4Z~X-Yz|o+KfU-YbO`D%r1HA#D
zafUPaGbq!YeE@7k1=a`2JVBDE$n<nsb#**oNHV%Z=|)ftjEavJb#P$o4s4#aYS@5{
zl_UqX6e#@q>?z-y>_1b-Ap)n*TAK%qmUj9!p4SWlAkb{~KJeS^B0t`q{@Cd92lW;2
z<Cs%=sdj6!s*Xn-I6op15>t|TJ~SQx?>goWZvd$Zo-4pqKJHH6Tean+%|p#5@A;n*
ztWnr%myS7~(5F6S8j!f0H14He)?=>bH#I#3_iOj}^fY9zy+@ojW&8H+8?ZQB0PP);
zs)%9+z`h}-pn#1M7Y0H#F0&y#XaJ4IGX;=8aiRm{2n6-0sHmQvp7H7*ascu0Kb^tI
zjd&misriWJ2$ZT!)taepZf*j$4lt%cAf-k=X1p?*j1xhi8F-PN{)FepS{`G{h@Zdz
z{bPYi+@C+x5fNe7LA|W$Tr!d3!UkISB$bIo{%<sV{fU&Z&D(jzecvORz_B#XOH=qn
zD`tkV1WAxVdQmCRm1(p(VFvjDt4YVQp{k&n_K98V;+|_;3WG+B4cA^XCOs?1cejp@
zb}@}`DfSp4{%uyu4X}z~Z6fYX)S7lPoeb$6_PeZqqH-4h86pkK?23%C)yyK7-?QJ|
z&emxS+`oBiLi5YX8g0VFEO%noc}9Q<`i*lmgn0)Y!3P?Zv&~KH?>Do|#-pi0pp#wr
zk@26V4Eb$;nA?gjV}*V%;<l|zK`IBG_bmcW0~}+L^so+N2I%{u)6>Ic3xufSNMZ)&
zo1GXKTCXrcI|kB(BIrv2p?C*Sp#Rp<;n949jX}p>80Xag!=b%Xm+8i!IdTU@xTmja
z_Ye3*0&E?iM@;s3ys>$@vw=i9WIZ#cy}FEmOL4Y24_RhDjPGc6go=P5M(~<ibgLhz
z2rOo^wCom(!k{en_VywZ@_h;t?gG>}m-`(fU@w4JbJDPd3gV>M@wiKu5q+|^`rQzm
z$rr@D=n=PDg{(QGCC8Ur^kBP5sne%ZVkqh>*A2UqwxJD8+*Lr=5$NxQrvZ8kIJe|X
zO{qZ}&*$-2;rZ+eBBxlTbQ4@-0U&h&HpQY<`)XsuRKMy<0~BH7foO7QxHQ56S1a;0
zd~PzF$6z@=zVcy?FKq>a<&=9l5$fiuRB`UrTW*Ib$%R%}8S?V#6z=8YN1v(agtC;f
z<*AFtkM2*#iIJ}<S|mC;$}x*)23MTc!hlbxO9@!_1^^@ntwK~<+DMsJb0WVN9|&te
z7Id|FJUW9W92e}cq@*NBo{1tA)2~&3_TWm3k9zhLIT%ocid9vL#Ki9Z_4Gz{#Ju{v
zvoB0=0Z$4h5E1D!kBemCju|b=zs359B(HlUsIm6vx{4C88q+;lR4nZCg_~pA*E<8$
z4_H+{FOAD}Q~}cj1BkDf=;${M4<)54#pn&d^M}jt*}^EzDvm`~0t2i-y0z+Gw783I
zM8ooag=X<`Me2Ojlt!i;9Gz_&fzi`v#e=;$3{p6e%w&4@7coS}m2MzOToMb-Y{~ZV
znPd(g99r-=gM!lX3?icCbwO-$+7kYb%R(WCW#}9u`_1=}BqeCco1x}Z{|I|=_cL&5
zp&t=sI+&VXg%WVb{&HF!*v}Fws$|VP$=-AxKwv*zS<3$Rk`bX-*L3n@w0i8UpUI%G
zGPSk+ke<4$n{AoR@)NZKPGbtYjn*sl)#=Q;kcr=t_@d_nF(oL|>*>Vw^x>dH%cj&7
z{rMxA_>H{dUTxSIi-!l_Ut!=yjvqAo7X=bA;wP(V1VIwQ0sa#VY|2#pKd(#_nw@f_
z)PTH&>s?U5=iO_*Ter~-%!AIZhcNEv&$r9`6frnq0U$Of8Q9`Hi;BpcPDg(%)n7v1
zPU`H$t4a~4L7IY;lM~0c^DCfO3QQ3>7V(&Hkbe>9ClJvQ6ND_duiD;~Kc=bH)mkv&
zjkpZCiH4%zL=bQ*-B~iFXq33c$m*@HCtqn7Onp;R`z=m(anQK3<?>{sU8^pmr>ED^
z@#QmsBhbLa>QD4uXEWW~8i<&ljVEVk*8jhkyL1xfn{SB1jiEGPN&zuz@YNNs7w`*g
zSN>E~bfk2|>1nUL)21?~X59WUBN@P?3Y0lM$X7eLDPU*!?cbqkqY#=e|DEj)tV|r4
z=3uLI7D0u(KvJu^@aQn=Ir)m?^jg<E=JpW6ps+SAZ0e*+T?pQj>}|Eu{<ZmPbG?X8
zV|~<{>6salpHyeJ9@pGe6#{$_w|tKO5*fGlD-HUeW|bV<v!OhH_S(OH4;W`);beEB
zhn$nJKZ24+bL%3M#%JMvO(^xx#1uc5|5%;Xk13}$$gOt%ajcHT%a_svk8)jG>t1U|
z8P~_%x8_V*+&;YZ!jvSf727-b1M6bgOlrX82DejSPbL>N5GL;!9rM1HUne2Qk^|Kt
zAktt28^C-!(5G)S^M9lIuza($1R7PoRyh11nEU_U$0bxhtr4EzhvN??@+4V{Ff!qx
zNSvR4V&D?Ut3(Z%5C>aiBUl$Uw;+nikP+&$jZKdUE0v%S*Zn0EY;Lo03?xIG;JdI4
ztTaDfx%Pwwydj%3TWf$OgE*?p+VgCQfr97`(m|Kz9JD&fDlXu{_|$5C|0v+?A-$E|
zO<8fHqGl*)D+(%gyRinr4%O4!Q^Pr{&-zAQNn2LnCoWb_)UQ_83$W|mukJ-H59cBz
zb52j2|7v>77e%_Z7maII%*Ro&dbqoe0_gP#6%Ke~FK>5{08b52#JvVerbnLnpYbCN
z&PhnBD=#Gn%Ha(lcyJD7kj3S4s@l@&^_i|^WV$A`w6m0KiQ2s_SDMpQ^Ftc)9)~nO
zr@WD6XHYOJ6e*s;#KV(xcIE_j6KAv=j@vaSeqaH8QOQ}cDNASxQCl`8tZ&X(wkQOB
zx9!Qx)$HJ1aP^9zz#eA?9s?#aCYjwazte&WeZ9NJwwB}%!knoCDtdYfQqr8>8(^mn
z70p@u@u2x!KJ#z6vU-I=|LXl}vV3OFx@)Z8A6)iLy_bhgw_(={auv^-1tZoIv>LUw
zvtHC;6*wCH#n6Sr^`Vhw2KATaYuw9K%lofCw`@w|QIuJZ1`?>K=?nIicJd-_F@Ymh
zgk!$b7w*5Ux&`w@uk*zGf&xV?t>fF}UA;p_&?z~Mh+cEyqbhkkWaDxD(Hf>WWbmVr
zJI3h09OH-X=m|n&aaj)IxxJ8l-)Wnk@k;cuZN5_u&TF@c*aJRVqNs54@?uSuSzk>D
zDmZxP`^yU~a74LmX9Yc7_ufJ^Sc(&hYI1o0Q|0TK)ZJ;MEWpHDX3ieca$GI>?A~&s
z@rUMmdM3s1@?_cWH?XUbm5RL?T3h?7lp8>q0T`KE4tcEw+m5HDSGE3wLG|qBfb+j4
zk1qG8G|=+YhCCPJ!WF)GodMMY)n5NZHArul?a-=?N3y+MUm)D{e7)1DHjouSUGg^D
zgz<7rV*7EtSo;<Zj>TqE7<iW04s)56P_Vp;H?ZHrFz_pPFS{_<UTpu=9!(xu-dkHQ
ztxwO{-?sq|AjyOOWbgIlnOBRH%zVu*5o7LW!{doYP|$RR$Cw_wM{RQbs`~KY?VN&=
zW;pQXNiAF^v6(^hx=~*CyRj^^?J=gbzha1-F$AIb7?wPlut*G?_Xzt%KF=oWgr{^b
zS(WA&HlzZR;IPz!9Tm50s%k>%E;#+xP?*N|`xjW+YX_eDX?x=l^{<`h3X`s6)?}}o
zR;{&(jhEWn;FyDy^uLMc^$_P8dWoUKDB=zNbqj}LsY;C36}mxRsNO`j2+zyi7L@s7
zO%NKNGhy|D58$tXH5-UVSK6c76Ds-ANnI^2mknFtU&;o`SPHrYhQm0%70(I%+`Ht0
zhb5S!OHHWS`|6A96Sr?R>N-mYmc(@FNnMQ3Q*H7eppx4Q#|KUO8AO+VD}3nMcqM=#
zq9E}TO=%VRF6Dtg5M+~kZSC@{{&;@v&#=!{-X8uOo}@DTxbqG1WGXDn?dzISzDs9_
zC3^~>CO5JI3A#*cfo&J!RtGzrl!xcjSCgtaJky}xzt0*_RV5+*NaZtX*+3Y7g<o8>
zp-oNPN-rvZ0~=O5KMr60i&|$s5&ejRLhHJ#XFTE>zH@4|9>pI@X*FcoHs6vo;ItA<
zab@0qf3pI3EO_7t<Y01I4N+1(#APQ--!Ib}*QNN(m+aZ9IN!L3-b=m%Rpt@KZc=mJ
zv0Z7tRG9(ZjtIo6#QDUyZoY=*d!{J7=;*DaFJm8so_mHLlfImOaC~*{nOTN?d6-iL
zYkp6F4i4_(>I-cP@4hD&WLZf8cOEpF;ld=qN{`q{x6b@;ceHM$+3|5@z>ywMLqJ>H
zuwrWwjiiM)Qef8V>f(;uIyUEbHcLO=T|Jm>`L}0@q>}C-&U#zHu{QT~hm^LrywljP
zEm@!wzuN>`wWcnlV9S#%|DRK6D|E>B-55{nl#xC_9X1@~r^vrV&-QOvLu8bPtK%ul
zhs=~1FmnM$DMl}!+&)@e`RX8EI<37)`+1l>+mx7W)jaL#E!utm=$H2kX>!penSsIn
zU$4DsO;)T_B8M5`4lE3Blh|w37zckozzd*L1}QF!9StZ&5W6t@>pgG@KbQK*0h4K3
z-?`%mTocs}rmOJJr&+Od&<g^d6(}(PVAN=ASwbwf<qIb)oJY<vSO5HrL{F1RTS^HV
zS}rV}SQm~}oxWjbDmtKZ|HJy~GGj{*SG_>%JWc3%r<c{Znm#Z*yglaiQq72XqTX|0
zo%vg?<Bg_l=FOgtd*^9i_;mvxCWFi|t+L63E?+BtqqMsW8n?|notlLVRgTBUKy46n
zmX>8_t-MzrI>A8ufe;8FL%fTX7_gwmCX2f`?<c(McxT7R&pbubRCE3OXdC&4Do0Fr
zf)M(&26OR0{qL-jVW%<lbp;#Q3BKTF@beS;3);v8R@coG9^%oKs8#w*$G+to88Mp7
zmB8O*6E(?xboEpAzL66Yd;`Wf48b%8{^m^AQps9Hx{(!~0kDJNcQs_iwEDs|@PN)3
zjJ}+rhj$`vShKnc%Yyju?F7m&u(unqyV2t*0Ag$L8V;GKL&T=QX>e@pRK6SsiV~UM
zpu&}%)}<@Mvmeb!J!0@5?(93GNT-WgaOl^RyCI$Q9V;)lECbv|@c?c$yKCE)-;EQ^
z-?h1Ge`#EaS#!5!h!&`3(=aqO+D6EgNtNMlp8p~}ZGVM^L;Z9X^a9NzR={R`d2mei
zfjZLF>#ee6eHmXXd!PVP@<(23#Udpzb@NNMpxL;(Djq97d$#afU@^cbCiSNz20hnm
z!}uJV$H@_fW?@ZD>cFP`I9cjoq{@){0gxD?va;dr3}ZB(mc7t@S3h7`FlqcAH+j2N
zR#qaKrIaa8O+{zURaJT4l-1{*)i-_<<C<jlsN)Ik(@<9(ESZa#Y`)qk>NPUDNu!ii
z)|rc#=X0G=0?1#O5@&<|7LV0ymYmKXqUWHUw|Y57HBQ#tywT#lh<L~rVbbb?2-XK}
z&?Id*Kg}6N!~0hm3IF&%0=f55&iboc<a~pgk1`mKQHfb*N%DO}(=XwM>F{%-h>Sd3
zin77Z0TUT}Hstq<qrCYpC-or%W<!nYF)O3v2lmow+%v$kkec(#8|?T<Pc5rlGjCbH
zROB=pwaOosBDx}Mu1QOggW&NUg(Ivf=t7b0MWt1l4A2hCb}uSjbDVf#zf(*xLbL*^
zH;T314(#}e1^g1;eBeyh4A^;DO(nwBv0RlI%jMMb*o<xb_jqc<ZoU{GPL}nfyr;U#
z^FDEeUB#;@>{s`L?+o>)rOi)SrYdDGJ~gq<hLT)`f_FzqT&(l|#0{(1k4&thi4~p4
z8nH1kef~ET1B?VN%2lA1mM@+mn5&#scl*9$pl`sKq$vb~Sm^FLQ|BE9u7JHJt7IIe
zLqjq~j@VhwjTkJ-M`6kG_kvwQRQ!2cqo3dX{vArBh9O-+(lD}D!-a<s6G$BMcgY>u
z8(5XVAQEUNt7qIOqr?k~iV#>Q5Tdg<nm&C7mo>8Sgz1{^xD~DuSb32%l1fEy01$<e
z(m_y={dJK_>Ywd_f;DgQ2xj9e2dK9+u`RK@gw`Um9*s6MAeq~Tga}&IWFjatKJ*cr
zueQV-O7gz*aw7daVAJv(;Q$2z768rMug9TrIkv<Wy|Xgz#{R;o>=|%)NDYcF`n#f^
zQBStRcN%`n_Mo!SYq@jM-l$6fZ=5QN^}ct=30#BTG>Z2SMnYurcv<rH9;Qh)$JR>R
zk@H1Sk@@PMojY7!Kz;4q@BLu9z31SxulXHJ&HF*w6)`-)_I9N{ovl!p91iVi7yuIH
zt;&SY01?ROc=W@>^jEnQ#}8~-8t$bFL78;L0aT!*tyPy?|5_Q|2@?>Viqk(B!G3vI
zHN*4zMBC&Pk-x7*-j67UP*jA9PnW%{yY4%aasG5ES8-5$3bC^5lL-3NsJ`d}ibPO-
zAL=;dE6QqU1E;DY3rSb)zm;aVb=hra47}lG=OYw4Q|mnHw??XA6-sZ1K6Bvg$9Ssj
z231YjhtLp<Q{>4-*oPDF(x=w1JAd|=dPl6bu=&L6O0gwtykJs2Rqlj17`Z+hBmM9j
z5dD|H%UD0O9TpI96V-a7|CijtYEp+D8U~IyD0eARu&JR*GQPb!9z2-t!0Q~H)A!oy
zZRz8>7^GG>;`VclZObhwOr)?v717VlXlHZnZNUI^{rw2rgk081ODoh=_*=2~ew%-Z
zNyFS{7!+$L-98uuWWw2qgx9v{F^-@2?57UKy!V)ams&-hzJ_HPgBE{-b-6&L`~{hA
zC=`DUFrc8IsFT`~MU)YJKRZNPm5WJ5O}z;StBn}BS^b}9%f*oec2^w#8^qL(S(UAv
zjU%p2<K*uYsrC!&mrLJ0YDp7(hs()7VPF<Snv&4mF{wG{%}myyoYd6@B=E5rY1wZ!
zTuZl~fIZSL_>en)^@}>yUW5cztvVfE*lyL<%yX3exh)WNmhHyKX3L(@K*(G-w<ToQ
zGkp#53$rn0Lz5<0lGK-2>W#1vnQfHH)3ojMi>JHMDlyLqi{0kbyG)$0@ytlE0DgBc
zcNRR7!r`<}oL~A)L7}Vl_9gx9?yYPprvRA23dDH_1ZJenSSN3&aM2W+(UqwfE^T7W
zpzUN<OYepTPT<hybiJYiP#V9lX#R8Sewd$TYvfH2SKl#YG~mfD3<#aT=?na)Padi~
z!OeB-cwwFyoENtHv24!b;yAm5MB8fa2_}<bwIz|7lW6yQ%U}?)&T<8`SPf&8-#D?Y
zYAArMUTnv;x@he)%BQaj4me<92d|ZtDu=6SieApyyoMnfo7y9ttBy_E^?)3RyIZRG
zh&01$pVI&NAa2B;x9<fSTocT&5D&{14u1KW0!9j8{tCG(4$%!gP*TJkB#*Lq(gXSv
zFvINyu82tI=7e@?f6IBO$MkErM(q32F7THVfbRvuodYKem^MAD=jJP@sE8~nq4g@B
z_<1nR)=)m*Xh#X$F8hflpoOqYmdjWs;LQ0Vs$?z~F>vO`V}%KHu+T59QNWFnz!Lo6
z18icz`moj<@AE-HrR9dC6Y*z=g^Ha;hz0iIo_4#9JfYrW@SJw8)bFe8J@ez{a-cv#
zyF)xR5Z^6m!)}WSB=e$SLypoo)QFAFTX2`{r)0pmW+b={(G^erogK=eGVYPK)AoE-
z2Gv3{;2Lu|dJ);}c~q-0!SM1rL5OduM$A)?xIUncZ@oeW6RaA2ujz8C3_#bOC{|?%
z4h{zO2pNwRzSNO&jNS#Vk$L#X+{eOB4`Cr8UjVad!0aroT7@?t9=(c5rgMQMlEY|K
zLQ4z3w^t;z>DjX;#qQqbkpdI*9Tkn;j`|X~MTx~Ocjxz4kJq|N)E|IHG{%af1Q6br
z3+;B!n~n8}jYKJ>A78`NON~d@w4v8u4&<V^&iMjD8^RD239?(Ba^=d|bl3SvIOp9+
z1Go`!adCl<_<>ixM_vn<3Wso4m{3m^2k#yW>~^Ked7Jzzyk0%QnM@s2H@Dv5c)i*O
zpb?@D!f!ao4D?iNbkEmY5dxDea1&DTz3@whgkygE7_wqn&}-hYYuc`5RxrH3X$H(t
zh2`Z1PN%D25KYS(=r$~l%W4~0d!?cDY}f3$Rn=%J&8-OED^g#N7bI;uy$)7hU8zyn
zgHb)@`bxKgLgjMW9^S(QM36LTkeJG_61G0#aV$H+9Wt_kb!`ta#%AdiHVHEu-kBJ7
z+Cr?a{FX~!56<2DR=*&L{DzegMi%#lM?mN$jt9a3ZD%KQlVg}gQA|z{J@c1~%#c7)
zi21`8wT3rGs^F8%t{HHMeE9HTP7^kkMx<cs96(bR;6dte%NYRs!oqV6MMa#;bstK{
z>w!4pc_$9*^x))FHH1W5II<^b>fkM<I628D8JV|w<ZARHWSDxuH4_#cBbhfnJDaUi
z42JqZkFx6dc-&!9b>pLX((Z?R90r1}v!OLeke*zcJiDkUh@v1S*%eMdC|2GdC+IH%
zrwuDt5ffL1#>PWuyYb%+^MC9g1v=%;stlOEL`jGU`~Utt$1&~)2G690v6%|YO2l&&
zvQ+*<56TSDd@HLO&Ql40NQ?SUmMI^WGB552<MiyhJ@-M<`;ksk5h@XiY?5yDAYtJ8
zaJ@eCpSmCNUB^z9r3C)4PUGrts;Xl4_N*2~DKM}nFo0iaS2jr~0VY47us1mz?b<AW
zQO8>9-@I^06RN6{bljg@<k1yGa3qU@@gFaL*07_0Eb3c*tB1j)n{akrr$rS@4SWqf
zqW+2oQJDEBf-y6{D)WN}xiK~{w7y9iJ=MHI4e(Az4)}O-QTF)5eN*|?KM+$;v1IXE
zgv6CxOwquUL>wbnjX+XO30v^J0_z5IXg=)2l~lflbnaOum?18sExW!3bcCXYhLpHC
zJh%#C5)vOU@_-C0l;VyH4THAMK_Q#^m-kkB(;pklMYB$vOz_a0D7VOmcrFjvc8bc%
z`ZfxMbCkw&XnK0{fTaZx#Mx5VNLor-`|rwgolfn^FcI`Cah56#a)qx1$uJdll*iJ`
z=PrSI3KNNASzwi#m`RjSu1<>uEK`xmQaD_kxIm**qwNFg1&J9MRbhHX1!Ips6Btf?
zcyQgEag2VqS+QU)o>B|D+z<3sFc?i87u?e$4E7$FBqyPu$d}<dfYG)m9B+2A^1P-Q
z>tocRalaT)FCSpXLH6_WL;v(i8Vr>KB@L*n2J9)oM2CdvFE~$theypCWyFjN#?9aU
z_n%(<QYAPK874%8OrY1|bUxE99x;d#?>U$%I9P4{0!&uGcS21~4eBxpD{Cy6(gwN~
z7&8F&V?E|1C{?5}{YoI+n3$SkuJ(S!!z(N<7Rr)ex90pE91M|e!HouGO}zy#HDtsX
zP~5;H(niF@6xAcF6H0cJ$eAV^fp%nWYT9*RkNN(+pBk+w876VU5U{p`XGR()2@!q#
z{r!P`uya0{G)f#iQDC?HpR?;z%z!X(H59|HPEY&In2Kb{%WG@PYHFsQK|$|>>s2U<
z1p|8W3JN~JU}pc*#H7Qr-heSR?S_h-9V<^E2xz2v3J7T4)6+^NDm3gV$3CYUVRfQ&
zwU3v9DCkgMv*dM|Q_8gES-=$o2tI%8m^EwvF;j_}oR&6pwbkt%Az_3x1;m-f%9fVK
zBS?$2WzG4vyiPD{{TCcTW?5PIM2;A^_ZT6)e|Ky*NU=hKf`U5rE5Tf`zb3sHbzHVW
z5hD&V*#1Cy0vQGX@pt}$zrc@*lpu*yoNN=kH!uKzs}&Fs0J;|tF(W1OQu#c9tsa=j
zw&p7h1n0h+rIvBZC@Cqe<MVjGf{zura#l?e-@Zr>3=9M}tYYaHD7=%?)55~SZ%Lwl
zM@FIpy=i7<rp(E)C>Ao5@8biW9xzj8$y0$z9v@<lrjvL2`uaFPXFYAXL<ME3tGNPn
zBt=S-1#*PoW<cIDAt5FsLk@(PWFBVo3WQ)GiDGp8r>0V-DFH-U|5;$5)V<oD0O&RW
zIJz%qRCThHsKE^YcBV+6l4IcE4Fa0v4%jkr_&uLo=1yi+Nuu69bi&l20ke{PU=c#J
z3;p@86-zOoX?*`4Bu+*`MfH1o+YoFvpneOU27t?Wh(;si3k6f>AZUP03_el;IDmkB
z@J(5{H^r)Lpt!tEa$$g3Qc4Ok)gd78!vVN;KrDe94lJEuA_UlHf`WrFk>%i3Fr@Bm
z)*`NsmsBH(1>cfDym5dfo1U9{mt`X|!KU_6o*E3<$Bye-5S6Mfx~!Cg`0DBsR8Ulm
z8Zi8Y$Mf|O#5G2#%8)lS3=f4ObsQ!(wwQg5Sz=jpJXVk}_|yyp8CziFpYCeEAF+B1
zW3J6dMh0;Ms?p|yRrfeWP_k2CB}vatE|9~xL~1fhTC?41Tsi{-6zc}wwt#IX+=e|j
z2xB51+D|T#sfPik;JQF?`RH_l&GQc5cgFMs0U5A>pqPR*gUxsr0zMk!9_TZFhlkBC
znZP^(@JU1`CjJKFK?_wzU2WjwC$KKf$~Efg(N|_e0fYjT&g0?cTutW05;Y@4Y4Jhl
zRS;U@q>Rq`xlg~cL1e@us|p+F!-#Z<93Yg|*L6Y42YHNwm^e$M^z`K<zFfY+77_SR
zfvx<fxw-kqI11+iy&$k60~uJ4Jq1h;2^7z~DU+8fRiOzDwPgTC7a)9Y9M`YGYI1l0
zDGGdI05c0q5|0b8c&J2PUeD>kl*!9j``0wJH;)0Uvmf)xR+$m-3je<uT#?ey)fIi0
zSPjg@l_h>9FD)k)6l`;Sm<P%!OM)`H-@j|`a9VgE#R%9GcXCNNQn}Z(QECq_X!%9*
z@jlMG;MrdO`X9`z!`DAsxBDGv#u~Kx8Mp$=IQNzbFln?V->na5bvoF_E8X%IxKahU
zniJS$l`u?Vd3|jya19M;76@22bBpOD01slZ`F3ORpTF<*Hy1ti0?knGe{S)a^_^h4
zy8pZ*NgHFXM8C<tb1Y?J#Gh5}pQU2ghQ)ud7c}@_W&~U#4Llg*Ca~=UoZWV^dN9B4
z7w6PdrqgGJXaSGZh%dNuW1ej_=%f;0s~Xrw2Bux$aywx8{23UvTqju>fcrFjfFrnn
zfAfP1R^TdZ&_e6U67RkNJNTL98=2EKOKy$2`|iswiAJ?5=?zh9b+ZH(UpxWqMVtT*
zkG<Kk@ei<~0#1kn7alik_I>{hcmRh%(G!n<AN%Vk0N1hrXWl_;-hgX3foI4VRDH<+
zZs-R#pBxuo+|p5`rK!0PxV?f?SZzb?@3Q*)eSY4H*IvA@6=QdfLHtPGcI)h{>^W+a
zPu{$7<HeTSZ_gb&=Jx!~&f>YiamaJCZb>b_{IcZQm1|of&YwSDe(P<Sqn~eX@8*pg
zfs011=BoS6DVS$b2&@^b=01J;RCn6ix5cM*^!3Z-Y^#8)!<GQIAVxo)V_m*(|J5Bj
zOn)DQH-xSJ`nyM#@lW&NhaXl<K70F5y^HXg>wb?PUjZH|q*~${QIe8al4_M)lnSI6
zj0}v-bPWu3jf_GJjI9jKtPD)F4GgRd4C>k5p=!v@PsvQH#I1oZIxGWdg9hA&lFZ!H
b;*!MN0^E8c{mQok^)Pt4`njxgN@xNAzHJ<M

literal 0
HcmV?d00001

diff --git a/engine/BlockManager.py b/engine/BlockManager.py
new file mode 100644
index 000000000..28ceed333
--- /dev/null
+++ b/engine/BlockManager.py
@@ -0,0 +1,56 @@
+from mindspore import tensor, Parameter, float32
+from mindspore.ops import ScatterNdUpdate, gather, stack, MatMul
+from mindspore.common.initializer import initializer, Zero
+import torch
+
+class BlockAllocator:
+    def __init__(self, num_blocks, block_size, head_dim):
+        self.num_blocks = num_blocks
+        self.pool = []
+        for _ in range(num_blocks):
+            self.pool.append(Parameter(initializer(Zero(), [block_size, head_dim], float32)))
+
+    def alloc(self, indices, A):
+        for index, a in list(zip(indices, A)):
+            self.pool[index[0]] = ScatterNdUpdate()(self.pool[index[0]], tensor([[index[1]]]), tensor(a.detach().numpy()).unsqueeze(0))
+    
+    def get(self, indices):
+        A = []
+        for index in indices:
+            A.append(gather(self.pool[index[0]], tensor(index[1]), 0))
+        return stack(A)
+
+class MetadataEngine:
+    def __init__(self, num_blocks, num_heads, block_size):
+        self.num_blocks = num_blocks
+        self.num_heads = num_heads
+        self.block_size = block_size
+        self.block_table = []
+        self.num_token = 0
+        self.i = 0
+        self.j = 0
+
+    def alloc(self):
+        list = []
+        self.num_token = self.num_token + 1
+        for _ in range(self.num_heads):
+            if self.i == self.num_blocks:
+                raise ValueError("Out of Memory")
+            list.append([self.i, self.j])
+            self.j = self.j + 1
+            if self.j == self.block_size:
+                self.i = self.i + 1
+                self.j = 0
+        self.block_table.append(list)
+        return list
+
+    def get(self, x):
+        list = []
+        for k in range(self.num_token):
+            list.append(self.block_table[k][x])
+        return list
+
+class Multiplication:
+    def matmul(x, A, trans):
+        matmul = MatMul(transpose_b = trans)
+        return torch.from_numpy(matmul(tensor([x.detach().numpy()]), A).asnumpy())
\ No newline at end of file
diff --git a/engine/Design and test.md b/engine/Design and test.md
new file mode 100644
index 000000000..0a5e7ebf9
--- /dev/null
+++ b/engine/Design and test.md	
@@ -0,0 +1,87 @@
+#### 一、技术调研
+
+KV Cache 是一种用于加速 Transformer 等大型语言模型（LLM）自回归推理的关键优化技术，通常用于加速 Transformer 的 Decoder 部分，本次任务在 Decoder-only 结构的 Transformer 的基础之上，合理设计单机八卡 KV Cache 管理代码，对 Transformer 进行加速。
+
+##### 1. Transformer 的多头注意力机制
+
+在 Transformer 被提出的论文 《Attention is All You Need》 中详细介绍了多头注意力机制(Multi-Head Attention)层的基本结构。
+
+一般而言，Transformer 可以对多批(batch)数据进行处理。在这里，我们认为多头注意力机制层的输入是 $b$ 个 $l$ 行 $d_m$ 列矩阵组成的矩阵组 $(\textbf X_0,\textbf X_1,\dots,\textbf X_{b-1})$，每个矩阵代表一批数据，矩阵的每一行代表数据的一个词元(token)。
+
+首先，代码会对输入的数据进行“分头”操作。具体而言，是训练三组矩阵 $(\textbf W_{Q,0},\textbf W_{Q,1},\dots,\textbf W_{Q,h-1}),(\textbf W_{K,0},\textbf W_{K,1},\dots,\textbf W_{K,h-1}),(\textbf W_{V,0},\textbf W_{V,1},\dots,\textbf W_{V,h-1})$，其中第一组和第二组是 $d_m$ 行 $d_k$ 列矩阵，第三组是 $d_m$ 行 $d_v$ 列矩阵，并对每一个在 $[0,b)$ 内的自然数 $i$ 和 $[0,h)$ 的自然数 $j$，计算
+
+$$\textbf Q_{ib+j}=\textbf X_i\textbf W_{Q,j}\\\textbf K_{ib+j}=\textbf X_i\textbf W_{K,j}\\\textbf V_{ib+j}=\textbf X_i\textbf W_{V,j}$$
+
+接着，对每一个在 $[0,b)$ 内的自然数 $i$ 和 $[0,h)$ 的自然数 $j$，计算 $\textbf C_{i,j}=Att(\textbf Q_{ib+j},\textbf K_{ib+j},\textbf V_{ib+j},\textbf M)$，其中 $\textbf M$ 被称为掩码矩阵，对于掩码多头注意力层而言， $\textbf M$ 是主对角线上方（不包括主对角线）全为 $-\infty$，其余部分全为 $0$ 的矩阵，而函数 $Att(\textbf Q,\textbf K,\textbf V,\textbf M)$ 的表达式为
+
+$$Att(\textbf Q,\textbf K,\textbf V,\textbf M)=softmax\left(\frac{\textbf Q\textbf K^T}{\sqrt{d_k}}+\textbf M\right)\textbf V$$
+
+其中 $softmax(\textbf A)$ 的第 $i$ 行是行向量 $softmax(\textbf a_i)$，其中 $\textbf a_i$ 是 $\textbf A$ 的第 $i$ 行向量，而
+
+$$softmax(a_0,a_1,\dots,a_m)=\frac{(e^{a_0},e^{a_1},\dots,e^{a_{m-1}})}{\overset{m-1}{\underset{i=0}\sum}e^{a_i}}$$
+
+然后，生成 $b$ 个 $l$ 行 $h\times d_v$ 列矩阵 $\textbf O_{0},\textbf O_{1},\dots,\textbf O_{b-1}$，其中，矩阵 $\textbf O_{i}$ 的第 $j$ 行是分块矩阵 $[\textbf C_{i,0}\vdots \textbf C_{i,1}\vdots \cdots\vdots \textbf C_{i,h-1}]$ 的第 $j$ 行
+
+最后，训练 $h\times d_v$ 行 $d$ 列矩阵 $\textbf W$ 并输出 $b$ 个 $l$ 行 $d$ 列矩阵组成的矩阵组 $(\textbf Y_0,\textbf Y_1,\dots,\textbf Y_{b-1})$，其中，对每一个在 $[0,b)$ 内的自然数 $i$，都有 $\textbf Y_i=\textbf O_i\textbf W$
+
+##### 2. Decoder-only 结构下 Transformer 的优化可行性
+
+《Attention is All You Need》 的 Transformer 是 Encoder-Decoder 的，而 Encoder 的掩码矩阵无法屏蔽未来的信息，所以这里直接使用 Decoder-only 结构下的 Transformer
+
+我们认为，一个句子是由若干词元(token)组成的，每个 token 可以赋予一个编码，句子就变成了一个以编码为字符的字符串。Decoder-only 结构下的 Transformer 以这样的 $b$ 批字符串组 $(s_0,s_1,\dots,s_{b-1})$ 为输入，其中每个字符串的长度都是 $l$，输出是 $b$ 个 $l$ 行 $v$ 列矩阵组成的矩阵组，第 $i(0\leq i<b)$ 个矩阵的第 $j(0\leq j<l)$ 行第 $k(0\leq k<v)$ 列代表 $s_i$ 的长度为 $j$ 的前缀的下一个字符编码为 $k$ 的概率，其中 $v$ 代表 token 总数
+
+Decoder-only 结构下的 Transformer 可以概括成下图所示
+
+<img src=50e8005addb244c88d30f9daccaabf8a.png>
+
+其中 Input Embedding 层是用于将 token 编码转化为向量，该层训练定义域为 $\{0,1,\dots,v\}$，值域为 $d_m$ 维向量的函数 $Emb(x)$，其时间复杂度是线性的。Input Embedding 层输出 $b$ 个 $l$ 行 $d_m$ 列矩阵组成的矩阵组，矩阵组的第 $i(0\leq i<b)$ 个矩阵的第 $j(0\leq j<l)$ 行是 $Emb(c_{i,j})$，其 $c_{i,j}$ 是 $s_i$ 的第 $j$ 个字符。Linear 层、LayerNorm 层、Feed Forward 层和 Softmax 层以 $b$ 个 $l$ 行的矩阵组成的矩阵组作为输入，输出是 $b$ 个 $l$ 行矩阵组成的矩阵组，其第 $i(0\leq i<b)$ 个矩阵的第 $j(0\leq j<l)$ 行只由输入的第 $i(0\leq i<b)$ 个矩阵的第 $j(0\leq j<l)$ 行经线性时间复杂度的运算得到。
+
+通常对 Transformer 推理时，先将起始 token 对应编码作为输入字符串的开头输入到 Transformer 中，再用输出的相应矩阵的第 $1$ 行中的最大的那一列的列号更新输入的相应字符串的第 $2$ 个字符，然后再次输入到 Transformer 中，接着用输出的相应矩阵的第 $2$ 行中的最大的那一列的列号更新输入的相应字符串的第 $3$ 个字符，再次输入到 Transformer 中，依次类推，直到字符串的最后一个字符被用于更新。
+
+不难得出，从第 $2$ 次调用 Transformer 时，对每一个在 $[0,b)$ 内的自然数 $i$ 和 $[0,h)$ 的自然数 $j$，每一个 Transformer Block 的多头注意力机制层的 $\textbf Q_{ib+j},\textbf K_{ib+j},\textbf V_{ib+j}$ 只比上一次调用 Transformer 多出最后一行，并且这一行只与 $\textbf X_i$ 的最后一行有关。不难想到上述推理过程中，第 $i(0\leq i<l)$ 次调用 Transformer 只要输入每一批次的第 $i$ 个词元编码，并把多头注意力机制层的 $\textbf K_{ib+j},\textbf V_{ib+j}$ 存起来，就能快速得到结果。这便是 KV Cache 技术的核心原理。
+
+Transformer 的训练需要输入完整的字符串，所以无法通过 KV Cache 技术优化。
+
+##### 3. vLLM 和 LightLLM 的内存管理方案
+
+vLLM 和 LightLLM 都致力于解决大语言模型推理中 KV Cache 的内存管理问题，但两者采取了截然不同的技术路径。vLLM 的 PagedAttention 和 LightLLM 的 TokenAttention 是解决 KV Cache 的内存管理问题通用的两种方案。
+
+PagedAttention 核心思想是将连续的 KV Cache 逻辑空间，映射到非连续的物理内存块中，就像操作系统的虚拟内存管理一样。PagedAttention 将若干个 token 对应的 $\textbf K_{ib+j}$ 矩阵行向量、$\textbf V_{ib+j}$ 矩阵行向量存储在一个块中，并维护一个映射表(block table)存储每个行向量在块中的位置。这种方案大幅减少外部碎片，但块内可能仍有少量内部碎片。
+
+TokenAttention 预先分配一大块连续的显存存储 $\textbf K_{ib+j},\textbf V_{ib+j}$，直接通过 token 的编号找到对应的 $\textbf K_{ib+j}$ 矩阵行向量、$\textbf V_{ib+j}$ 矩阵行向量的位置。这种方案零内存浪费，实现理论上的最优内存利用，但是可能会产生无法找到足够大的空间的情况。
+
+#### 二、方案设计
+
+##### 1. Decoder-only 结构下 Transformer 设计
+
+代码改编自 graykode 的代码。代码除了将原来 Transformer 改成 Decoder-only 的结构之外，还将 `Transformer` 类分成训练模式和推理模式。代码中的 `Transformer` 类用于打包各个训练层并用于训练，其 `forward` 函数输入 $b$ 行 $l$ 列的二维张量 `dec_inputs` 和变量 `block_size`，`dec_inputs` 的第 $i(0\leq i<b)$ 行第 $j(0\leq j<l)$ 列代表 $s_i$ 的第 $j$ 个字符。当 `block_size` 为 $0$ 时，开启训练模式，训练过程大致和前文所述的一致。当 `block_size` 大于 $0$ 时，开启推理模式。
+
+推理模式为 Transformer Block 的每一个多头注意力机制层使用 `Cache` 类开辟两个内存空间 `k_cache` 和 `v_cache` 中，分别存储 $\textbf K_{ib+j}$ 和 $\textbf V_{ib+j}$，其中的参数 `num_blocks`、`num_heads`、`block_size` 和 `head_dim` 分别代表申请的内存块总数 $n=\lceil\frac{blh}t\rceil$、批次数与每层的分头数之积 $bh$、每块存储的行向量数 $t$、每个行向量对应的维数 $d$。这些内存空间会在使用完毕之后通过 `delete` 函数释放。行向量的每一位都是 $32$ 位浮点数，每个浮点数占用的空间为 $f=4B$。假设 Transformer 共有 $L$ 层 Transformer Block，那么 Transformer 推理模式因 KV Cache 占用的总空间为
+
+$$m=2Lntdf$$
+
+推理模式下 `Transformer` 类的 `forward` 函数调用 Transformer $l$ 次，第 $i(0\leq i<l)$ 次输入 `dec_inputs` 的第 $i$ 列向量 $(c_0,c_1,\dots,c_b)$，如果这一列的第 $j(0\leq j<b)$ 列是 $0$，那么这个位置会被替换为上一次输出的第 $i$ 行中的最大的那一列的列号。Input Embedding 层改为输出一个 $b$ 行 $d_m$ 列矩阵，矩阵的第 $i(0\leq i<b)$ 行是 $Emb(c_{i})$。Linear 层、LayerNorm 层、Feed Forward 层和 Softmax 层基本不变，只是输入和输出都变为 $b$ 行矩阵了。此时 Transformer 最终会输出一个 $b$ 行 $v$ 列矩阵。`forward` 函数维护 $b$ 个 $v$ 列矩阵组成的矩阵组，每次调用 Transformer 时会将输出的第 $i(0\leq i<b)$ 行接在矩阵组的第 $i$ 个矩阵后面，并输出这个矩阵组。
+
+推理模式下多头注意力机制层先输入 $b$ 行 $d_m$ 列矩阵 $X$，并训练 $b$ 行 $h\times d_m$ 列矩阵 $W_Q,W_K,W_V$，并对每一个在 $[0,h)$ 内的自然数 $i$，计算
+
+$$\textbf Q=\textbf X\textbf W_{Q}\\\textbf K=\textbf X\textbf W_{K}\\\textbf V=\textbf X\textbf W_{V}$$
+
+再计算 $flat(\textbf Q),flat(\textbf K),flat(\textbf V)$，其中 $flat(\textbf A)$ 是将 $b$ 行 $h\times d_m$ 列 $\textbf A$ 变为 $b\times h$ 行 $d_m$ 列向量，对所有在 $[0,b)$ 内的自然数 $i$ 和 $[0,h)$ 的自然数 $j$，$\textbf A$ 的第 $i$ 行第 $jd_m,(jd_m+1),\dots,[jd_m+(d_m-1)]$ 个元素分别是 $flat(\textbf A)$ 第 $ih+j$ 行第 $0,1,\dots,(d_m-1)$ 个元素。
+
+然后将 $flat(\textbf K)$ 和 $flat(\textbf V)$ 通过 `Cache` 的 `write` 函数分别存入 `k_cache` 和 `v_cache` 中，其效果是将 $\textbf K$ 和 $\textbf V$ 的第 $(ib+j)(0\leq i<b,0\leq j<h)$ 行并入到 $\textbf K_{ib+j}$ 和 $\textbf V_{ib+j}$ 的末尾。
+
+接着，对每一个在 $[0,b)$ 内的自然数 $i$ 和 $[0,h)$ 的自然数 $j$，计算 $C=Att(flat(\textbf Q),\textbf K_{ib+j},\textbf V_{ib+j},\textbf M)$，计算时用到 `Cache` 的 `transmatmul` 函数和 `matmul` 函数，这两个函数用于从内存中取出需要的行向量组成若干个矩阵参与矩阵乘法运算。
+
+最后，训练 $h\times d_v$ 行 $d$ 列矩阵 $\textbf W$ 并输出 $\textbf Y=flat^{-1}(\textbf C)\textbf W$，其中 $flat^{-1}(\textbf A)$ 是 $flat(\textbf A)$ 的反函数。
+
+##### 2. KV Cache 的管理逻辑
+
+`Cache` 的 `write`、`transmatmul` 和 `matmul` 函数需要写入或读出相应的矩阵。参考 vLLM 的 PagedAttention，通过按块存储的方式，设计了 `BlockAllocator` 类申请、写入、读出向量和 `MetadataEngine` 类记录请求与物理块的映射关系。
+
+`BlockAllocator` 类初始化时统一使用 `mindspore.Parameter` 算子申请 $n$ 个物理块，每个块是一个 $t$ 行 $d$ 列矩阵，使每个物理块能够存储在不同的位置上。定义 `alloc` 函数使用 `mindspore.ops.ScatterNdUpdate` 算子逐行写入矩阵，其输入 $bh$ 行 $2$ 列的矩阵 `indices` 和待存储的矩阵 `A`，对每一个在 $[0,b)$ 内的自然数 $i$，`indices[i][0]` 和 `indices[i][1]` 分别是 `A` 的第 $i$ 行存储的物理块号和行号。定义 `get` 函数使用 `mindspore.ops.gather` 算子逐行读出矩阵，函数输入 $bh$ 行 $2$ 列的矩阵 `indices`，对每一个在 $[0,b)$ 内的自然数 $i$，`indices[i][0]` 和 `indices[i][1]` 分别输出矩阵的第 $i$ 行存储的物理块号和行号。
+
+`MetadataEngine` 类维护一张映射表 `block_table`，对每一个在 $[0,l)$ 内的自然数 $x$ 和在 $[0,bh)$ 内的自然数 $y$ ，`block_table[x][y]` 表示矩阵 $K_y$ 或 $V_y$ 的第 $x$ 行。两个变量 `i` 和 `j`，代表第 `i` 个物理块的第 `j` 行是空的。定义 `alloc` 函数依次为 $K_0,K_1,\dots,K_{bh}$ 或 $V_0,V_1,\dots,V_{bh}$ 分配一行空间，每分配一行空间，就更新 `block_table` 的内容，并让 `i` 和 `j` 指向下一个空行。定义 `get` 函数输入一个变量 `x`，输出 `block_table[0][x]`、`block_table[1][x]`、`block_table[2][x]` 等 `list` 合并后的矩阵，代表 $K_x$ 或 $V_x$ 存储在内存中的位置。
+
+#### 三、代码测试
+
+之后补充。
\ No newline at end of file
diff --git a/engine/KVCache.py b/engine/KVCache.py
new file mode 100644
index 000000000..d93dd831a
--- /dev/null
+++ b/engine/KVCache.py
@@ -0,0 +1,56 @@
+import torch
+from BlockManager import BlockAllocator, MetadataEngine, Multiplication
+
+class CacheWithTorch:
+    def __init__(self, num_blocks, num_heads, block_size, head_dim):
+        self.num_heads = num_heads
+        self.head_dim = head_dim
+        self.A = [torch.empty(0, head_dim)] * num_heads
+    def write(self, A):
+        for i in range(0, self.num_heads):
+            self.A[i] = torch.cat([self.A[i], A[i].unsqueeze(0)], 0)
+    def transmatmul(self, x):
+        B = [torch.empty(0)] * self.num_heads
+        for i in range(0, self.num_heads):
+            B[i] = x[i].matmul(self.A[i].transpose(-1, -2)).unsqueeze(0)
+        output = torch.cat(B)
+        return output
+    def matmul(self, x):
+        B = [torch.empty(0)] * self.num_heads
+        for i in range(0, self.num_heads):
+            B[i] = x[i].matmul(self.A[i]).unsqueeze(0)
+        output = torch.cat(B)
+        return output
+
+class CacheWithMindspore:
+    def __init__(self, num_blocks, num_heads, block_size, head_dim):
+        self.block_allocator = BlockAllocator(num_blocks, block_size, head_dim)
+        self.meta_data_engine = MetadataEngine(num_blocks, num_heads, block_size)
+        self.num_heads = num_heads
+
+    def write(self, A):
+        self.block_allocator.alloc(self.meta_data_engine.alloc(), A)
+
+    def transmatmul(self, x):
+        B = []
+        for i in range(0, self.num_heads):
+            B.append(Multiplication.matmul(x[i], self.block_allocator.get(self.meta_data_engine.get(i)), True))
+        return torch.cat(B)
+
+    def matmul(self, x):
+        B = []
+        for i in range(0, self.num_heads):
+            B.append(Multiplication.matmul(x[i], self.block_allocator.get(self.meta_data_engine.get(i)), False))
+        return torch.cat(B)
+
+class Cache:
+    def __init__(self, num_blocks, num_heads, block_size, head_dim):
+        self.cache = CacheWithMindspore(num_blocks, num_heads, block_size, head_dim)
+    def write(self, A):
+        self.cache.write(A)
+    def transmatmul(self, x):
+        return self.cache.transmatmul(x)
+    def matmul(self, x):
+        return self.cache.matmul(x)
+    def delete(self):
+        del self.cache
\ No newline at end of file
diff --git a/engine/TransformerDecoderOnly.py b/engine/TransformerDecoderOnly.py
new file mode 100644
index 000000000..3fb19eb9d
--- /dev/null
+++ b/engine/TransformerDecoderOnly.py
@@ -0,0 +1,185 @@
+# code adapted from Tae Hwan Jung(Jeff Jung) @graykode
+# Reference : https://github.com/graykode/nlp-tutorial/blob/master/5-1.Transformer/Transformer.py
+import numpy as np
+import torch
+import torch.nn as nn
+import math
+from KVCache import Cache
+
+# S: Symbol that shows starting of decoding input
+# E: Symbol that shows starting of decoding output
+# P: Symbol that will fill in blank sequence if current batch data size is short than time steps
+
+def get_attn_pad_mask(seq_q):
+    batch_size, len_q = seq_q.size()
+    # eq(zero) is PAD token
+    pad_attn_mask = seq_q.data.eq(0).unsqueeze(1)  # batch_size x 1 x len_k(=len_q), one is masking
+    return pad_attn_mask.expand(batch_size, len_q, len_q)  # batch_size x len_q x len_k
+
+def get_attn_subsequent_mask(seq):
+    attn_shape = [seq.size(0), seq.size(1), seq.size(1)]
+    subsequent_mask = np.triu(np.ones(attn_shape), k=1)
+    subsequent_mask = torch.from_numpy(subsequent_mask).byte()
+    return subsequent_mask
+
+class PositionalEncoding(nn.Module):
+    def __init__(self, tgt_vocab_size, d_model, dropout=0.1):
+        super(PositionalEncoding, self).__init__()
+        self.dropout = nn.Dropout(p = dropout)
+        pe = torch.zeros(tgt_vocab_size, d_model)
+        position = torch.arange(0, tgt_vocab_size, dtype=torch.float).unsqueeze(1)
+        div_term = torch.exp(torch.arange(0, d_model, 2).float() * (-math.log(2 * d_model) / d_model))
+        pe[:, 0::2] = torch.sin(position * div_term)
+        pe[:, 1::2] = torch.cos(position * div_term)
+        ## pe:[tgt_len + 1, d_model]
+        pe = pe.unsqueeze(0).transpose(0,1)
+        ## pe:[tgt_len + 1, 1, d_model]
+        self.register_buffer('pe', pe)
+    def forward(self, x, pos):
+        if pos > -1:
+            x = x + self.pe[pos][:x.size(0), :] # x: [batch_size, d_model]
+        else:
+            x = x + self.pe[:x.size(0), :] # x: [tgt_len + 1, batch_size, d_model]
+        return self.dropout(x)
+
+class ScaledDotProductAttention(nn.Module):
+    def __init__(self, d_k):
+        super(ScaledDotProductAttention, self).__init__()
+        self.d_k=d_k
+
+    def forward(self, Q, K, V, attn_mask):
+        scores = torch.matmul(Q, K.transpose(-1, -2)) / np.sqrt(self.d_k) # scores : [batch_size x n_heads x len_q(=len_k) x len_k(=len_q)]
+        scores.masked_fill_(attn_mask, -1e9) # Fills elements of self tensor with value where mask is one.
+        attn = nn.Softmax(dim=-1)(scores)
+        context = torch.matmul(attn, V)
+        return context
+    
+class MultiHeadAttention(nn.Module):
+    def __init__(self, d_model, d_k, d_v, n_heads):
+        super(MultiHeadAttention, self).__init__()
+        self.W_Q = nn.Linear(d_model, d_k * n_heads)
+        self.W_K = nn.Linear(d_model, d_k * n_heads)
+        self.W_V = nn.Linear(d_model, d_v * n_heads)
+        self.linear = nn.Linear(n_heads * d_v, d_model)
+        self.layer_norm = nn.LayerNorm(d_model)
+        self.d_model = d_model
+        self.d_k = d_k
+        self.d_v = d_v
+        self.n_heads = n_heads
+
+    def forward(self, Q, batch_size, attn_mask, k_cache = None, v_cache = None):
+        residual = Q
+        if attn_mask is None:
+            q_s = self.W_Q(Q).view(batch_size, self.n_heads, self.d_k).flatten(0,1)  # q_s: [batch_size x n_heads, d_k]
+            k_s = self.W_K(Q).view(batch_size, self.n_heads, self.d_k).flatten(0,1)  # k_s: [batch_size x n_heads, d_k]
+            v_s = self.W_V(Q).view(batch_size, self.n_heads, self.d_v).flatten(0,1)  # v_s: [batch_size x n_heads, d_v]
+            k_cache.write(k_s)
+            v_cache.write(v_s)
+            context = k_cache.transmatmul(q_s)
+            context = v_cache.matmul(nn.Softmax(dim=-1)(context)) # context: [batch_size x n_heads, d_v]
+            context = context.contiguous().view(batch_size, -1) # context: [batch_size, n_heads x d_v]
+        else:
+            q_s = self.W_Q(Q).view(batch_size, -1, self.n_heads, self.d_k).transpose(1,2)  # q_s: [batch_size, n_heads, len_q, d_k]
+            k_s = self.W_K(Q).view(batch_size, -1, self.n_heads, self.d_k).transpose(1,2)  # k_s: [batch_size, n_heads, len_q, d_k]
+            v_s = self.W_V(Q).view(batch_size, -1, self.n_heads, self.d_v).transpose(1,2)  # v_s: [batch_size, n_heads, len_q, d_v]
+
+            attn_mask = attn_mask.unsqueeze(1).repeat(1, self.n_heads, 1, 1) # attn_mask : [batch_size, n_heads, len_q, len_k]
+
+            context = ScaledDotProductAttention(self.d_k)(q_s, k_s, v_s, attn_mask) # context: [batch_size x n_heads x len_q x d_v]
+            context = context.transpose(1, 2).contiguous().view(batch_size, -1, self.n_heads * self.d_v) # context: [batch_size, len_q, n_heads x d_v]
+        output = self.linear(context)
+        return self.layer_norm(output + residual) # output: [batch_size, len_q, d_model]
+
+class PoswiseFeedForwardNet(nn.Module):
+    def __init__(self, d_model, d_ff):
+        super(PoswiseFeedForwardNet, self).__init__()
+        self.conv1 = nn.Conv1d(in_channels=d_model, out_channels=d_ff, kernel_size=1)
+        self.conv2 = nn.Conv1d(in_channels=d_ff, out_channels=d_model, kernel_size=1)
+        self.layer_norm = nn.LayerNorm(d_model)
+
+    def forward(self, inputs):
+        residual = inputs # inputs : [batch_size, len_q, d_model]
+        output = nn.ReLU()(self.conv1(inputs.transpose(len(inputs.size()) - 2, len(inputs.size()) - 1)))
+        output = self.conv2(output).transpose(len(inputs.size()) - 2, len(inputs.size()) - 1)
+        return self.layer_norm(output + residual)
+
+class DecoderLayer(nn.Module):
+    def __init__(self, tgt_len, d_model, d_ff, d_k, d_v, n_heads):
+        super(DecoderLayer, self).__init__()
+        self.dec_enc_attn = MultiHeadAttention(d_model, d_k, d_v, n_heads)
+        self.pos_ffn = PoswiseFeedForwardNet(d_model, d_ff)
+        self.n_heads = n_heads
+        self.tgt_len = tgt_len
+        self.d_k = d_k
+        self.d_v = d_v
+
+    def forward(self, dec_inputs, batch_size, dec_enc_attn_mask, block_size, k_cache, v_cache):
+        if block_size > 0:
+            dec_outputs = self.dec_enc_attn(dec_inputs, batch_size, None, k_cache, v_cache)
+        else:
+            dec_outputs = self.dec_enc_attn(dec_inputs, batch_size, dec_enc_attn_mask)
+        dec_outputs = self.pos_ffn(dec_outputs)
+        return dec_outputs
+
+class Decoder(nn.Module):
+    def __init__(self, tgt_vocab_size, tgt_len, d_model, d_ff, d_k, d_v, n_layers, n_heads):
+        super(Decoder, self).__init__()
+        self.tgt_emb = nn.Embedding(tgt_vocab_size, d_model)
+        self.pos_emb = PositionalEncoding(tgt_len + 1, d_model)
+        self.layers = nn.ModuleList([DecoderLayer(tgt_len, d_model, d_ff, d_k, d_v, n_heads) for _ in range(n_layers)])
+
+    def forward(self, dec_inputs, batch_size, k_caches, v_caches, block_size = 0, pos = -1): # dec_inputs : [batch_size] or [batch_size, tgt_len]
+        dec_outputs = self.tgt_emb(dec_inputs)
+        if block_size > 0:
+            dec_outputs = self.pos_emb(dec_outputs, pos)  # [batch_size, d_model]
+            dec_enc_attn_mask = None
+        else:
+            dec_outputs = self.pos_emb(dec_outputs.transpose(0, 1), pos).transpose(0, 1)  # [batch_size, tgt_len, d_model]
+            dec_enc_attn_mask = get_attn_pad_mask(dec_inputs)
+
+        for layer, k_cache, v_cache in list(zip(self.layers, k_caches, v_caches)):
+            dec_outputs = layer(dec_outputs, batch_size, dec_enc_attn_mask, block_size, k_cache, v_cache)
+        return dec_outputs
+
+class Transformer(nn.Module):
+    def __init__(self, tgt_vocab_size, tgt_len = 10, d_model = 512, d_ff = 2048, d_k = 64, d_v = 64, n_layers = 6, n_heads = 8):
+        super(Transformer, self).__init__()
+        self.decoder = Decoder(tgt_vocab_size, tgt_len, d_model, d_ff, d_k, d_v, n_layers, n_heads)
+        self.projection = nn.Linear(d_model, tgt_vocab_size, bias=False)
+        self.tgt_vocab_size = tgt_vocab_size
+        self.d_model = d_model
+        self.tgt_len = tgt_len
+        self.n_layers = n_layers
+        self.n_heads = n_heads
+        self.d_k = d_k
+        self.d_v = d_v
+    def forward(self, dec_inputs, block_size = 0):
+        batch_size = dec_inputs.size()[0]
+        if block_size > 0:
+            k_caches = []
+            v_caches = []
+            dec_inputs = dec_inputs.transpose(0, 1)
+            for i in range(self.n_layers):
+                k_caches.append(Cache(math.ceil(batch_size * len(dec_inputs) * self.n_heads / block_size), batch_size * self.n_heads, block_size, self.d_k))
+                v_caches.append(Cache(math.ceil(batch_size * len(dec_inputs) * self.n_heads / block_size), batch_size * self.n_heads, block_size, self.d_v))
+            dec_output = torch.empty(self.tgt_vocab_size, self.d_model)
+            dec_outputs = torch.empty(batch_size, 0, self.d_model)
+            pos = 0
+            dec_logits = torch.empty(batch_size, 0, self.tgt_vocab_size)
+            for dec_input in dec_inputs:
+                for i in range(0, dec_input.shape[0]):
+                    if dec_input[i] == 0:
+                        dec_input[i] = dec_logit[:, :self.tgt_vocab_size - 1].data.max(1, keepdim=True).indices[i]
+                dec_output = self.decoder(dec_input, batch_size, k_caches, v_caches, block_size, pos)
+                dec_logit = self.projection(dec_output) # dec_logits : [batch_size, tgt_vocab_size]
+                pos = pos + 1
+                dec_logits = torch.cat([dec_logits, dec_logit.unsqueeze(1)], 1)
+            for i in range(self.n_layers):
+                k_caches[i].delete()
+                v_caches[i].delete()
+        else:
+            k_caches = [None] * self.n_layers
+            v_caches = [None] * self.n_layers
+            dec_outputs = self.decoder(dec_inputs, batch_size, k_caches, v_caches)
+            dec_logits = self.projection(dec_outputs) # dec_logits : [batch_size, tgt_len, tgt_vocab_size]
+        return dec_logits
\ No newline at end of file
diff --git a/engine/test_kv_cache.py b/engine/test_kv_cache.py
new file mode 100644
index 000000000..a90c2315a
--- /dev/null
+++ b/engine/test_kv_cache.py
@@ -0,0 +1,51 @@
+from torch import nn, optim, LongTensor
+from TransformerDecoderOnly import Transformer
+
+def make_training_batch(sentences):
+    output_batch = []
+    target_batch = []
+    for sentence in sentences:
+        output_batch.append([tgt_vocab[n] for n in sentence[0].split()])
+        target_batch.append([tgt_vocab[n] for n in sentence[1].split()])
+    return LongTensor(output_batch), LongTensor(target_batch)
+
+def make_learning_batch(sentences):
+    output_batch = []
+    for sentence in sentences:
+        output_batch.append([tgt_vocab[n] for n in sentence.split()])
+    return LongTensor(output_batch)
+
+if __name__ == '__main__':
+
+    sentences = [['S i want a beer', 'i want a beer E'], ['S i want a beer', 'i want a beer E']]
+
+    # Transformer Parameters
+    # Padding Should be 0
+    # Unknown Should be tgt_vocab_size - 1
+    tgt_vocab = {'P': 0, 'i': 1, 'want': 2, 'a': 3, 'beer': 4, 'S': 5, 'E': 6, 'U': 7}
+
+    number_dict = {i: w for i, w in enumerate(tgt_vocab)}
+
+    model = Transformer(len(tgt_vocab))
+
+    criterion = nn.CrossEntropyLoss()
+    optimizer = optim.Adam(model.parameters(), lr=0.001)
+
+    dec_inputs, target_batch = make_training_batch(sentences)
+    
+    for epoch in range(20):
+        optimizer.zero_grad()
+        outputs = model(dec_inputs)
+        loss = criterion(outputs.view(-1, outputs.size(-1)), target_batch.contiguous().view(-1))
+        print('Epoch:', '%04d' % (epoch + 1), 'cost =', '{:.6f}'.format(loss))
+        loss.backward()
+        optimizer.step()
+
+    # Test
+    sentences = ['S P P P P'] * 10
+    dec_inputs = make_learning_batch(sentences)
+    for epoch in range(1):
+        predict = model(dec_inputs, 16)
+    predict = predict[:, :, :len(tgt_vocab) - 1].data.max(2, keepdim=True)[1]
+    for sentence, output in list(zip(sentences, predict)):
+        print(sentence, '->', [number_dict[n.item()] for n in output.squeeze()])
\ No newline at end of file