From 51fc3f3b27fd95eb27fee852108edf60c763ed95 Mon Sep 17 00:00:00 2001 From: tphlru <35965490+tphlru@users.noreply.github.com> Date: Thu, 15 Aug 2024 22:48:02 +0300 Subject: [PATCH 1/2] Update code for the newer versions --- .../inference_preprocess.cpython-39.pyc | Bin 0 -> 2797 bytes code/__pycache__/model.cpython-39.pyc | Bin 0 -> 10578 bytes code/inference_preprocess.py | 50 ++- code/model.py | 366 ++++++++++++------ 4 files changed, 273 insertions(+), 143 deletions(-) create mode 100644 code/__pycache__/inference_preprocess.cpython-39.pyc create mode 100644 code/__pycache__/model.cpython-39.pyc diff --git a/code/__pycache__/inference_preprocess.cpython-39.pyc b/code/__pycache__/inference_preprocess.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..130929953a1d02421a59697eb7992ccad049aea6 GIT binary patch literal 2797 zcmbVO&2JmW6`$EJE|(vY9odqqv{(c|5xTaNDmTM$D$BH`!j=SzO^ZbWdc+x$OYc`^ zhmvJ1PfCvgf*x`xdLVo2KhZyAu083}TQ5aW_025pC{BO^U1Db6n|bfe`4@mUju~po12c^)MdxNZuPnDG|U|@{ERL zm|uPf0)0b?EuLj~S7GeJ$iIc1A(|Q#PfdYlxIl9}wS-YvGvLS89ADt6E$p#V*aa>y zH3k-N6>5HMOr3?43p$krT?N_&+MA=#jcIl272dcut%*9dipqt75m0vFDypb{gZXa@ zW7b&cHs^SbK?04J#B!nXfc|KJFD`X9&;MLherY{G)YG*05tu<_K2S3*!RQ${LzBbWR&T%4t*ltY|2S3;Y;; z@$Ga8e1Mv-s*ZUZ+af}19qHYl+*vF4{twMnw1J8yoca@XAE0mWmwzsr!k%puP_GR% zZ59r2?-$M-Y63l>xDCa@9&i*Dwu48ckOa$Xu8W9nuiLB_m z@yYgZSwGpQuh)-BoJGt(;wKq>Ro^*Dew|l#XzdPtkBgv%wn(;K}3NM0N??~;n_n+={f292T z{m0!sX>mrVwDS?kn6xvl&T>eWv4jRU-WobOJnlU|ZtqHmvpj^eRN#TPf6#4rdOMx= zPT=-W#cmp<9JF(lyP&aq`0TLLd-Sxk-Ff;*npp&v!#E$MXVMm-h?ulAV@52IHYdpt zV$B7WR?J8uYyC9o6TuP~t>vI`s&euqjMxzoBWcnwmRLv}1`g~To{WQLi51- zLq0i?&Pdly+PYi-Qv^|(d~bK>;9kPcw)!b$TVXO_oF#qM%ib1xGdq_n-?KeV z&U!kY8d%OpFd@sr7WVKmUc+_$bAau$+ zt@A$2AO*uU>tJaa-^cf_e6wjbVf91vCgykHtREn>>KWyRM(95QQJ^t=OW>{;0?&-X z0E{((GBssUwW{eA1Xzt{R$&2JTSF7%9K{deOY&F1)@eo9fEbm+pcdeRabeAzIhwj4 zQz^jW*qv1i6HwlI59u{`>=o|ZSV60RXrO@rFRMirZl_I}&~cOrNG$y**LU)+|BS>Z zl(hVN5B)(H!R3R}m3y8KPj?jN{$P^y)qV7#r=yVNKJgc3!eqEm+x%W!RnP~u6QV9B zsEHBt2Wb?gXG)%M5%%E30b~C1Ju6p9!Btn|J;FcDwC&;VR_n>_;Iq91Z&bH|=Byf()+b@UFl8@yFYx z&v%Z9!Vd^NzoQTF{TDCF6ufxhfAnkWCzCik2fsNvZ{1&rsMIjO6!XtUkT?H4o%m-Y z5k7dyS)86S9l6eTd6cqTCR|wtn0VJ*oTg%=l9aviY2jms0gNTr9DJu(|I^b&!64P8 z&!ycGum5v(b=5L8&0T0-UU@UMYW$|6mtc}EfJXwLb1tnkVY#$*Mz}(ISrMt`08>(c zs7T@aP@(Rr#NC!Bo$^%al;H=Bl2<4(1SGEH!bwuuiIHi+_Jyc)HTouK|E+ADo JU^cyn{{>y;`y2oO literal 0 HcmV?d00001 diff --git a/code/__pycache__/model.cpython-39.pyc b/code/__pycache__/model.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..1101728a455eb2f8dbc7e24a062060658d4bc82a GIT binary patch literal 10578 zcmeHN%aa^OUanWas-GH-<{^!w9zET=)?-G1jah7KHEVm<)>wFkW$85(?arD}kNPp2 zRWp)?YIYZC!@RV>0tasPjrkjhAP5eeIoF8@4ulTeHUij-`M%8Rr^nc<4TBG^j`~&R zFS9a#nVnyL`OBlU7N+w9nN z+oaBEJDpTLt-`5xrjxB_JGpwUV;0RUhI_WrB1nC?o{d(l{ei!)|si# zn8xdd$cXG?Lu7qx$KE-?7B=EOWg zi{h(dK`bJ4QamqCic<)k5?90-aTcK^aaB~sGD4@tHE~XyN9c^WE-r|R2%SZJm&9e% zciFdhsyoZD;heBGjM~bdETuy>{5Tve^jkRX6?4ZcjGa)s4HYt+0AC z4E=7{>UFE@et5qp@97ub?p|5*_ghWBdaKtB8_lro2SN4Qt+0U-Z~Kj4Px>9a=-%85 z@Al+N)$g>r)wf!E?o;%;v0gLd)IDD|0t7PeHJbPQuBheWGz!~aUDMASYjNtE-QB$~ zuH4*5-fjO*uh(vMw~-;+p~1WDhjD7n?*@LHS(Clp9`a^hYd2`}^h-C^Sf^@olO~K` zb`l9LAmAHy69a<-!g_4f?e~o>3=QEt&eomx4acab(0!@6H0&6!gKleDV+A@0r*PlG z9lQn+8arlWvP5A&vHO;aTf!N|Kd>V6L$hy3W@Mc*B3qb?#w?!>pCUDDpw{&IQxm0a z)~wiRJZJ@R_MY$WidH9h%Ig0eH?DturDn(Go3S12b>dvB8+yUr#;#v;Vkhw1Te10| z?mR$RJ^er>92#;Ful?-WwY$BJe=Tfv_T;rYvbWH7DtH8EnY zbvSIU?mmp2W~1F+z^g%)^_ms@T=U@gWY1h>t|sjsT(p&4&O4Za2#wH;j7Qc7)`vzP z9kpQ`n(JspD%jigr7R&!>~8y^*X(t-THA+~oTn^mJOjZy z;NM}GCG%i$%CfN{({a}Gx{Z$Sd9mwxXyaa+;ezLVZ?DnTDe@EwQsu_Qq{gP-ZVzf@ zFUn;KRX@$9ovQw$ckRC0&-V+V{Xr^nF<^^PQD7eZ5#gerV{$9CX!MK7 zn?^1Rbs>y0U13EQYubiXrCR+`-$hy`a-(9RwWvWYdDK#l%1Pc0aH3bqZb7mHBUb0>DoLMlJ%$ZThnl)#wgDghiGhtgU zp`My8Pvaq)|B5sazYCK@`^l;7~0GUSBa*YwGrdRjvu!3 z#~9I}0sNRx%zkzl%5j$F96>rnYqMyro9JwjdSXW|H2JAqIp_)#xeCw}LsO)&Lb60w zl~shU6k6!pk_y?tZ)w{?xEwlB`jHux5w3*jX@n~XA4~Lnb0*QxAbcEknPXNSgKjp_ zX)9-;IWd*%ICOIpx#pmmpR#fux&>wBf(qHuiD*GtxtLh92>r>#nv)2h3e#xQDTJ27 zOq4;-rJh*P65^+kD}}b4M)*va5p3fbgwKY0#!l2wP$$w!~^|NjxQ<+beN;8^tR94W#`N^ms}<#V_Oj6@>AWcy6yIdGVfj z0r!ize>Jwn%5{nr`88UbQmdJ1NZDBMtRp#i5yEg?)5V-OliaRZunNijajlD46RD80 zhj%AH#tC)^vVZy#WOnp_4l=r50H6liBp|B^Agci3G65;_D(^Dl=SHB6&SB|L05H!- z=K;*(x_kqF>l7^IH>o2i<+mu_gov{mf8NTW89Q824mGRtE#`U~qIN;vVfZ&F-k~6> z;PiiF|R7%<9J5&6WzP;%IPd`LbKr(+8JXI{BssaSc1C-F@ zA;1YtvENTLs<2!Ib>%=^Iz*bvk+ww513Twn(HUsuDf@k z>_T(_I4d2AGx_`YyN^WKE)%{-5l|==iV&(@{DPt=q?vmZ`xN&n9#A}_IEsF<+`%HO z{u?<0uIhe4SS3_6B&0t;iHG*cmsBArhK`o@wx%g-xDZUdt)nPwoQmj2(YK;b`!RWF z=vj^D(u6ghOB2?lx1|Y--j*gTdyKH0F~ZUwiY6?2x&!ibqJgJHowi2Pmwub&pGcNd zClID*HU4(8>{B}Zwx+8|D>YrwN=;XGG&`g#U5}Ndw;5;h}6h}E%pF3TRR(hk= z->0wnJ`-I5RNjrXUjuz;M@i8YD9Xl-{RhyLa+63}bcu#q6&s_cN9df7(bEtgiKFx- zJ{G2SBR*GrB#x4%rb#R<1KA^L%0x!uC~0cKZn0keDQcplq^aps9i<-2qrK55*qkNC!*cb!3MVdfx41w@qiw*}BL)%6~LNAEhjhaR<-0%i#TY#6& zw-aE1UPd^1Qim13B7?-8L>o-ErZ>!_nK<`23nVkz7 zy+Js=Sd5EfyMj1=n24^%#j$N|rjAEfM>`#Hez;eNTw{^dXWjmIMs&B?(Er_YbKiLQ zo@@y&Gyjfwehse}C3tVxh_^m+?3v+{v{*6=c0!25C-rlK6r8oFT|_K3 z7~=Y;U*eh@?q7IZ!+V35mSB5w4!Dra0T(bITnsS>h?2^C5%Ejma2|**!P6^)PA^B7 z;ptUAWr;f4cbuLJj$+pzVNBJYoA&wMhgSX(#UE4r3B_n5@=qE4Gm2;2jeJHp{`_QH z1lf)N`X2-+C&XU}pwnL9k5L{QG;!HUEFfkFXZz}d5`J8gt=p}^$0g>($0fNp{Dus_ zj*0hUhmkCjcW@nI{KE;8yPJk*gW(L%!J(w#;jj!6yNi$+fn!O1^qSR2uYZmL_|-$J z+Y0feLoSq!R@d9cr!(2m$FVX)PD5_vGn3}Nr_ZpnL1W*?X|%+dCBJQue@XEZioc@x zYl^?2Sf)@r!RmbZFA#mvg4qQGc)pxYeg`wdwq70B(j4>~^D>MV_4Y2`4)J}~BNJa+ z@toCKKPcaM=k3?u`Nqb^OV#UFuV25&?|gP8zI(wLGi{7h{%)^{!`*goJDA>_F7&#d zO7`#t(yKI(bs6Tt+{mo)ENJ%yq)RTYu1Vz zKF)Za=r#G#4Zhh5dUC7XyDzWO3jPoE%1C<^%Y;TCIULkC7WMh*w5#pLLmZpyonp1h zNi%bXLalAoie==4b1)rIiaHSt)XFAK>muXSTP#{Hn22R{w5|^L)Om#D2CL*okbFeJ zwYVZ~$!Q_Z@e?S1y5N&ll{u!rZd) qky|#*wf^>am8c+ literal 0 HcmV?d00001 diff --git a/code/inference_preprocess.py b/code/inference_preprocess.py index 517049b..9b269ad 100644 --- a/code/inference_preprocess.py +++ b/code/inference_preprocess.py @@ -7,15 +7,16 @@ import scipy.io from scipy.sparse import spdiags + def preprocess_raw_video(videoFilePath, dim=36): ######################################################################### # set up t = [] i = 0 - vidObj = cv2.VideoCapture(videoFilePath); - totalFrames = int(vidObj.get(cv2.CAP_PROP_FRAME_COUNT)) # get total frame size - Xsub = np.zeros((totalFrames, dim, dim, 3), dtype = np.float32) + vidObj = cv2.VideoCapture(videoFilePath) + totalFrames = int(vidObj.get(cv2.CAP_PROP_FRAME_COUNT)) # get total frame size + Xsub = np.zeros((totalFrames, dim, dim, 3), dtype=np.float32) height = vidObj.get(cv2.CAP_PROP_FRAME_HEIGHT) width = vidObj.get(cv2.CAP_PROP_FRAME_WIDTH) success, img = vidObj.read() @@ -25,35 +26,50 @@ def preprocess_raw_video(videoFilePath, dim=36): ######################################################################### # Crop each frame size into dim x dim while success: - t.append(vidObj.get(cv2.CAP_PROP_POS_MSEC))# current timestamp in milisecond - vidLxL = cv2.resize(img_as_float(img[:, int(width/2)-int(height/2 + 1):int(height/2)+int(width/2), :]), (dim, dim), interpolation = cv2.INTER_AREA) - vidLxL = cv2.rotate(vidLxL, cv2.ROTATE_90_CLOCKWISE) # rotate 90 degree - vidLxL = cv2.cvtColor(vidLxL.astype('float32'), cv2.COLOR_BGR2RGB) + t.append(vidObj.get(cv2.CAP_PROP_POS_MSEC)) # current timestamp in milisecond + vidLxL = cv2.resize( + img_as_float( + img[ + :, + int(width / 2) + - int(height / 2 + 1) : int(height / 2) + + int(width / 2), + :, + ] + ), + (dim, dim), + interpolation=cv2.INTER_AREA, + ) + # vidLxL = cv2.rotate(vidLxL, cv2.ROTATE_90_CLOCKWISE) # rotate 90 degree + vidLxL = cv2.cvtColor(vidLxL.astype("float32"), cv2.COLOR_BGR2RGB) vidLxL[vidLxL > 1] = 1 - vidLxL[vidLxL < (1/255)] = 1/255 + vidLxL[vidLxL < (1 / 255)] = 1 / 255 Xsub[i, :, :, :] = vidLxL - success, img = vidObj.read() # read the next one + success, img = vidObj.read() # read the next one i = i + 1 plt.imshow(Xsub[0]) - plt.title('Sample Preprocessed Frame') + plt.title("Sample Preprocessed Frame") plt.show() ######################################################################### # Normalized Frames in the motion branch normalized_len = len(t) - 1 - dXsub = np.zeros((normalized_len, dim, dim, 3), dtype = np.float32) + dXsub = np.zeros((normalized_len, dim, dim, 3), dtype=np.float32) for j in range(normalized_len - 1): - dXsub[j, :, :, :] = (Xsub[j+1, :, :, :] - Xsub[j, :, :, :]) / (Xsub[j+1, :, :, :] + Xsub[j, :, :, :]) + dXsub[j, :, :, :] = (Xsub[j + 1, :, :, :] - Xsub[j, :, :, :]) / ( + Xsub[j + 1, :, :, :] + Xsub[j, :, :, :] + ) dXsub = dXsub / np.std(dXsub) ######################################################################### # Normalize raw frames in the apperance branch Xsub = Xsub - np.mean(Xsub) - Xsub = Xsub / np.std(Xsub) - Xsub = Xsub[:totalFrames-1, :, :, :] + Xsub = Xsub / np.std(Xsub) + Xsub = Xsub[: totalFrames - 1, :, :, :] ######################################################################### # Plot an example of data after preprocess - dXsub = np.concatenate((dXsub, Xsub), axis = 3); + dXsub = np.concatenate((dXsub, Xsub), axis=3) return dXsub + def detrend(signal, Lambda): """detrend(signal, Lambda) -> filtered_signal This function applies a detrending filter. @@ -80,5 +96,7 @@ def detrend(signal, Lambda): diags_data = np.array([ones, minus_twos, ones]) diags_index = np.array([0, 1, 2]) D = spdiags(diags_data, diags_index, (signal_length - 2), signal_length).toarray() - filtered_signal = np.dot((H - np.linalg.inv(H + (Lambda ** 2) * np.dot(D.T, D))), signal) + filtered_signal = np.dot( + (H - np.linalg.inv(H + (Lambda**2) * np.dot(D.T, D))), signal + ) return filtered_signal diff --git a/code/model.py b/code/model.py index c41fe87..f6319d3 100755 --- a/code/model.py +++ b/code/model.py @@ -1,14 +1,23 @@ -''' +""" Models for Multi-Task Temporal Shift Attention Networks for On-Device Contactless Vitals Measurement Author: Xin Liu -''' +""" import tensorflow as tf from tensorflow import keras -from tensorflow.python.keras import backend as K -from tensorflow.python.keras.layers import Conv2D, Conv3D, Input, AveragePooling2D, \ - multiply, Dense, Dropout, Flatten, AveragePooling3D -from tensorflow.python.keras.models import Model +from tensorflow.keras import backend as K +from tensorflow.keras.layers import ( + Conv2D, + Conv3D, + Input, + AveragePooling2D, + multiply, + Dense, + Dropout, + Flatten, + AveragePooling3D, +) +from tensorflow.keras.models import Model class Attention_mask(tf.keras.layers.Layer): @@ -55,8 +64,10 @@ def get_config(self): return config -def TSM_Cov2D(x, n_frame, nb_filters=128, kernel_size=(3, 3), activation='tanh', padding='same'): - x = TSM()(x, n_frame) +def TSM_Cov2D( + x, n_frame, nb_filters=128, kernel_size=(3, 3), activation="tanh", padding="same" +): + x = TSM()(x, n_frame=n_frame) x = Conv2D(nb_filters, kernel_size, padding=padding, activation=activation)(x) return x @@ -64,18 +75,26 @@ def TSM_Cov2D(x, n_frame, nb_filters=128, kernel_size=(3, 3), activation='tanh', # %% -def CAN(nb_filters1, nb_filters2, input_shape, kernel_size=(3, 3), dropout_rate1=0.25, dropout_rate2=0.5, - pool_size=(2, 2), nb_dense=128): +def CAN( + nb_filters1, + nb_filters2, + input_shape, + kernel_size=(3, 3), + dropout_rate1=0.25, + dropout_rate2=0.5, + pool_size=(2, 2), + nb_dense=128, +): diff_input = Input(shape=input_shape) rawf_input = Input(shape=input_shape) - d1 = Conv2D(nb_filters1, kernel_size, padding='same', activation='tanh')(diff_input) - d2 = Conv2D(nb_filters1, kernel_size, activation='tanh')(d1) + d1 = Conv2D(nb_filters1, kernel_size, padding="same", activation="tanh")(diff_input) + d2 = Conv2D(nb_filters1, kernel_size, activation="tanh")(d1) - r1 = Conv2D(nb_filters1, kernel_size, padding='same', activation='tanh')(rawf_input) - r2 = Conv2D(nb_filters1, kernel_size, activation='tanh')(r1) + r1 = Conv2D(nb_filters1, kernel_size, padding="same", activation="tanh")(rawf_input) + r2 = Conv2D(nb_filters1, kernel_size, activation="tanh")(r1) - g1 = Conv2D(1, (1, 1), padding='same', activation='sigmoid')(r2) + g1 = Conv2D(1, (1, 1), padding="same", activation="sigmoid")(r2) g1 = Attention_mask()(g1) gated1 = multiply([d2, g1]) @@ -85,13 +104,13 @@ def CAN(nb_filters1, nb_filters2, input_shape, kernel_size=(3, 3), dropout_rate1 r3 = AveragePooling2D(pool_size)(r2) r4 = Dropout(dropout_rate1)(r3) - d5 = Conv2D(nb_filters2, kernel_size, padding='same', activation='tanh')(d4) - d6 = Conv2D(nb_filters2, kernel_size, activation='tanh')(d5) + d5 = Conv2D(nb_filters2, kernel_size, padding="same", activation="tanh")(d4) + d6 = Conv2D(nb_filters2, kernel_size, activation="tanh")(d5) - r5 = Conv2D(nb_filters2, kernel_size, padding='same', activation='tanh')(r4) - r6 = Conv2D(nb_filters2, kernel_size, activation='tanh')(r5) + r5 = Conv2D(nb_filters2, kernel_size, padding="same", activation="tanh")(r4) + r6 = Conv2D(nb_filters2, kernel_size, activation="tanh")(r5) - g2 = Conv2D(1, (1, 1), padding='same', activation='sigmoid')(r6) + g2 = Conv2D(1, (1, 1), padding="same", activation="sigmoid")(r6) g2 = Attention_mask()(g2) gated2 = multiply([d6, g2]) @@ -99,7 +118,7 @@ def CAN(nb_filters1, nb_filters2, input_shape, kernel_size=(3, 3), dropout_rate1 d8 = Dropout(dropout_rate1)(d7) d9 = Flatten()(d8) - d10 = Dense(nb_dense, activation='tanh')(d9) + d10 = Dense(nb_dense, activation="tanh")(d9) d11 = Dropout(dropout_rate2)(d10) out = Dense(1)(d11) model = Model(inputs=[diff_input, rawf_input], outputs=out) @@ -107,18 +126,26 @@ def CAN(nb_filters1, nb_filters2, input_shape, kernel_size=(3, 3), dropout_rate1 # %% MT_CAN -def MT_CAN(nb_filters1, nb_filters2, input_shape, kernel_size=(3, 3), dropout_rate1=0.25, dropout_rate2=0.5, - pool_size=(2, 2), nb_dense=128): +def MT_CAN( + nb_filters1, + nb_filters2, + input_shape, + kernel_size=(3, 3), + dropout_rate1=0.25, + dropout_rate2=0.5, + pool_size=(2, 2), + nb_dense=128, +): diff_input = Input(shape=input_shape) rawf_input = Input(shape=input_shape) - d1 = Conv2D(nb_filters1, kernel_size, padding='same', activation='tanh')(diff_input) - d2 = Conv2D(nb_filters1, kernel_size, activation='tanh')(d1) + d1 = Conv2D(nb_filters1, kernel_size, padding="same", activation="tanh")(diff_input) + d2 = Conv2D(nb_filters1, kernel_size, activation="tanh")(d1) - r1 = Conv2D(nb_filters1, kernel_size, padding='same', activation='tanh')(rawf_input) - r2 = Conv2D(nb_filters1, kernel_size, activation='tanh')(r1) + r1 = Conv2D(nb_filters1, kernel_size, padding="same", activation="tanh")(rawf_input) + r2 = Conv2D(nb_filters1, kernel_size, activation="tanh")(r1) - g1 = Conv2D(1, (1, 1), padding='same', activation='sigmoid')(r2) + g1 = Conv2D(1, (1, 1), padding="same", activation="sigmoid")(r2) g1 = Attention_mask()(g1) gated1 = multiply([d2, g1]) @@ -128,13 +155,13 @@ def MT_CAN(nb_filters1, nb_filters2, input_shape, kernel_size=(3, 3), dropout_ra r3 = AveragePooling2D(pool_size)(r2) r4 = Dropout(dropout_rate1)(r3) - d5 = Conv2D(nb_filters2, kernel_size, padding='same', activation='tanh')(d4) - d6 = Conv2D(nb_filters2, kernel_size, activation='tanh')(d5) + d5 = Conv2D(nb_filters2, kernel_size, padding="same", activation="tanh")(d4) + d6 = Conv2D(nb_filters2, kernel_size, activation="tanh")(d5) - r5 = Conv2D(nb_filters2, kernel_size, padding='same', activation='tanh')(r4) - r6 = Conv2D(nb_filters2, kernel_size, activation='tanh')(r5) + r5 = Conv2D(nb_filters2, kernel_size, padding="same", activation="tanh")(r4) + r6 = Conv2D(nb_filters2, kernel_size, activation="tanh")(r5) - g2 = Conv2D(1, (1, 1), padding='same', activation='sigmoid')(r6) + g2 = Conv2D(1, (1, 1), padding="same", activation="sigmoid")(r6) g2 = Attention_mask()(g2) gated2 = multiply([d6, g2]) @@ -142,13 +169,13 @@ def MT_CAN(nb_filters1, nb_filters2, input_shape, kernel_size=(3, 3), dropout_ra d8 = Dropout(dropout_rate1)(d7) d9 = Flatten()(d8) - d10_y = Dense(nb_dense, activation='tanh')(d9) + d10_y = Dense(nb_dense, activation="tanh")(d9) d11_y = Dropout(dropout_rate2)(d10_y) - out_y = Dense(1, name='output_1')(d11_y) + out_y = Dense(1, name="output_1")(d11_y) - d10_r = Dense(nb_dense, activation='tanh')(d9) + d10_r = Dense(nb_dense, activation="tanh")(d9) d11_r = Dropout(dropout_rate2)(d10_r) - out_r = Dense(1, name='output_2')(d11_r) + out_r = Dense(1, name="output_2")(d11_r) model = Model(inputs=[diff_input, rawf_input], outputs=[out_y, out_r]) return model @@ -156,18 +183,32 @@ def MT_CAN(nb_filters1, nb_filters2, input_shape, kernel_size=(3, 3), dropout_ra # %% TS_CAN -def TS_CAN(n_frame, nb_filters1, nb_filters2, input_shape, kernel_size=(3, 3), dropout_rate1=0.25, dropout_rate2=0.5, - pool_size=(2, 2), nb_dense=128): + +def TS_CAN( + n_frame, + nb_filters1, + nb_filters2, + input_shape, + kernel_size=(3, 3), + dropout_rate1=0.25, + dropout_rate2=0.5, + pool_size=(2, 2), + nb_dense=128, +): diff_input = Input(shape=input_shape) rawf_input = Input(shape=input_shape) - d1 = TSM_Cov2D(diff_input, n_frame, nb_filters1, kernel_size, padding='same', activation='tanh') - d2 = TSM_Cov2D(d1, n_frame, nb_filters1, kernel_size, padding='valid', activation='tanh') + d1 = TSM_Cov2D( + diff_input, n_frame, nb_filters1, kernel_size, padding="same", activation="tanh" + ) + d2 = TSM_Cov2D( + d1, n_frame, nb_filters1, kernel_size, padding="valid", activation="tanh" + ) - r1 = Conv2D(nb_filters1, kernel_size, padding='same', activation='tanh')(rawf_input) - r2 = Conv2D(nb_filters1, kernel_size, activation='tanh')(r1) + r1 = Conv2D(nb_filters1, kernel_size, padding="same", activation="tanh")(rawf_input) + r2 = Conv2D(nb_filters1, kernel_size, activation="tanh")(r1) - g1 = Conv2D(1, (1, 1), padding='same', activation='sigmoid')(r2) + g1 = Conv2D(1, (1, 1), padding="same", activation="sigmoid")(r2) g1 = Attention_mask()(g1) gated1 = multiply([d2, g1]) @@ -177,13 +218,17 @@ def TS_CAN(n_frame, nb_filters1, nb_filters2, input_shape, kernel_size=(3, 3), d r3 = AveragePooling2D(pool_size)(r2) r4 = Dropout(dropout_rate1)(r3) - d5 = TSM_Cov2D(d4, n_frame, nb_filters2, kernel_size, padding='same', activation='tanh') - d6 = TSM_Cov2D(d5, n_frame, nb_filters2, kernel_size, padding='valid', activation='tanh') + d5 = TSM_Cov2D( + d4, n_frame, nb_filters2, kernel_size, padding="same", activation="tanh" + ) + d6 = TSM_Cov2D( + d5, n_frame, nb_filters2, kernel_size, padding="valid", activation="tanh" + ) - r5 = Conv2D(nb_filters2, kernel_size, padding='same', activation='tanh')(r4) - r6 = Conv2D(nb_filters2, kernel_size, activation='tanh')(r5) + r5 = Conv2D(nb_filters2, kernel_size, padding="same", activation="tanh")(r4) + r6 = Conv2D(nb_filters2, kernel_size, activation="tanh")(r5) - g2 = Conv2D(1, (1, 1), padding='same', activation='sigmoid')(r6) + g2 = Conv2D(1, (1, 1), padding="same", activation="sigmoid")(r6) g2 = Attention_mask()(g2) gated2 = multiply([d6, g2]) @@ -191,7 +236,7 @@ def TS_CAN(n_frame, nb_filters1, nb_filters2, input_shape, kernel_size=(3, 3), d d8 = Dropout(dropout_rate1)(d7) d9 = Flatten()(d8) - d10 = Dense(nb_dense, activation='tanh')(d9) + d10 = Dense(nb_dense, activation="tanh")(d9) d11 = Dropout(dropout_rate2)(d10) out = Dense(1)(d11) model = Model(inputs=[diff_input, rawf_input], outputs=out) @@ -200,18 +245,32 @@ def TS_CAN(n_frame, nb_filters1, nb_filters2, input_shape, kernel_size=(3, 3), d # %% MTTS-CAN -def MTTS_CAN(n_frame, nb_filters1, nb_filters2, input_shape, kernel_size=(3, 3), dropout_rate1=0.25, - dropout_rate2=0.5, pool_size=(2, 2), nb_dense=128): + +def MTTS_CAN( + n_frame, + nb_filters1, + nb_filters2, + input_shape, + kernel_size=(3, 3), + dropout_rate1=0.25, + dropout_rate2=0.5, + pool_size=(2, 2), + nb_dense=128, +): diff_input = Input(shape=input_shape) rawf_input = Input(shape=input_shape) - d1 = TSM_Cov2D(diff_input, n_frame, nb_filters1, kernel_size, padding='same', activation='tanh') - d2 = TSM_Cov2D(d1, n_frame, nb_filters1, kernel_size, padding='valid', activation='tanh') + d1 = TSM_Cov2D( + diff_input, n_frame, nb_filters1, kernel_size, padding="same", activation="tanh" + ) + d2 = TSM_Cov2D( + d1, n_frame, nb_filters1, kernel_size, padding="valid", activation="tanh" + ) - r1 = Conv2D(nb_filters1, kernel_size, padding='same', activation='tanh')(rawf_input) - r2 = Conv2D(nb_filters1, kernel_size, activation='tanh')(r1) + r1 = Conv2D(nb_filters1, kernel_size, padding="same", activation="tanh")(rawf_input) + r2 = Conv2D(nb_filters1, kernel_size, activation="tanh")(r1) - g1 = Conv2D(1, (1, 1), padding='same', activation='sigmoid')(r2) + g1 = Conv2D(1, (1, 1), padding="same", activation="sigmoid")(r2) g1 = Attention_mask()(g1) gated1 = multiply([d2, g1]) @@ -221,13 +280,17 @@ def MTTS_CAN(n_frame, nb_filters1, nb_filters2, input_shape, kernel_size=(3, 3), r3 = AveragePooling2D(pool_size)(r2) r4 = Dropout(dropout_rate1)(r3) - d5 = TSM_Cov2D(d4, n_frame, nb_filters2, kernel_size, padding='same', activation='tanh') - d6 = TSM_Cov2D(d5, n_frame, nb_filters2, kernel_size, padding='valid', activation='tanh') + d5 = TSM_Cov2D( + d4, n_frame, nb_filters2, kernel_size, padding="same", activation="tanh" + ) + d6 = TSM_Cov2D( + d5, n_frame, nb_filters2, kernel_size, padding="valid", activation="tanh" + ) - r5 = Conv2D(nb_filters2, kernel_size, padding='same', activation='tanh')(r4) - r6 = Conv2D(nb_filters2, kernel_size, activation='tanh')(r5) + r5 = Conv2D(nb_filters2, kernel_size, padding="same", activation="tanh")(r4) + r6 = Conv2D(nb_filters2, kernel_size, activation="tanh")(r5) - g2 = Conv2D(1, (1, 1), padding='same', activation='sigmoid')(r6) + g2 = Conv2D(1, (1, 1), padding="same", activation="sigmoid")(r6) g2 = Attention_mask()(g2) gated2 = multiply([d6, g2]) @@ -236,51 +299,60 @@ def MTTS_CAN(n_frame, nb_filters1, nb_filters2, input_shape, kernel_size=(3, 3), d9 = Flatten()(d8) - d10_y = Dense(nb_dense, activation='tanh')(d9) + d10_y = Dense(nb_dense, activation="tanh")(d9) d11_y = Dropout(dropout_rate2)(d10_y) - out_y = Dense(1, name='output_1')(d11_y) + out_y = Dense(1, name="output_1")(d11_y) - d10_r = Dense(nb_dense, activation='tanh')(d9) + d10_r = Dense(nb_dense, activation="tanh")(d9) d11_r = Dropout(dropout_rate2)(d10_r) - out_r = Dense(1, name='output_2')(d11_r) + out_r = Dense(1, name="output_2")(d11_r) model = Model(inputs=[diff_input, rawf_input], outputs=[out_y, out_r]) return model # %% -def CAN_3D(n_frame, nb_filters1, nb_filters2, input_shape, kernel_size=(3, 3, 3), dropout_rate1=0.25, dropout_rate2=0.5, - pool_size=(2, 2, 2), nb_dense=128): +def CAN_3D( + n_frame, + nb_filters1, + nb_filters2, + input_shape, + kernel_size=(3, 3, 3), + dropout_rate1=0.25, + dropout_rate2=0.5, + pool_size=(2, 2, 2), + nb_dense=128, +): diff_input = Input(shape=input_shape) rawf_input = Input(shape=input_shape) - d1 = Conv3D(nb_filters1, kernel_size, padding='same', activation='tanh')(diff_input) - d2 = Conv3D(nb_filters1, kernel_size, activation='tanh')(d1) + d1 = Conv3D(nb_filters1, kernel_size, padding="same", activation="tanh")(diff_input) + d2 = Conv3D(nb_filters1, kernel_size, activation="tanh")(d1) # Appearance Branch - r1 = Conv3D(nb_filters1, kernel_size, padding='same', activation='tanh')(rawf_input) - r2 = Conv3D(nb_filters1, kernel_size, activation='tanh')(r1) - g1 = Conv3D(1, (1, 1, 1), padding='same', activation='sigmoid')(r2) + r1 = Conv3D(nb_filters1, kernel_size, padding="same", activation="tanh")(rawf_input) + r2 = Conv3D(nb_filters1, kernel_size, activation="tanh")(r1) + g1 = Conv3D(1, (1, 1, 1), padding="same", activation="sigmoid")(r2) g1 = Attention_mask()(g1) gated1 = multiply([d2, g1]) d3 = AveragePooling3D(pool_size)(gated1) d4 = Dropout(dropout_rate1)(d3) - d5 = Conv3D(nb_filters2, kernel_size, padding='same', activation='tanh')(d4) - d6 = Conv3D(nb_filters2, kernel_size, activation='tanh')(d5) + d5 = Conv3D(nb_filters2, kernel_size, padding="same", activation="tanh")(d4) + d6 = Conv3D(nb_filters2, kernel_size, activation="tanh")(d5) r3 = AveragePooling3D(pool_size)(r2) r4 = Dropout(dropout_rate1)(r3) - r5 = Conv3D(nb_filters2, kernel_size, padding='same', activation='tanh')(r4) - r6 = Conv3D(nb_filters2, kernel_size, activation='tanh')(r5) - g2 = Conv3D(1, (1, 1, 1), padding='same', activation='sigmoid')(r6) + r5 = Conv3D(nb_filters2, kernel_size, padding="same", activation="tanh")(r4) + r6 = Conv3D(nb_filters2, kernel_size, activation="tanh")(r5) + g2 = Conv3D(1, (1, 1, 1), padding="same", activation="sigmoid")(r6) g2 = Attention_mask()(g2) gated2 = multiply([d6, g2]) d7 = AveragePooling3D(pool_size)(gated2) d8 = Dropout(dropout_rate1)(d7) d9 = Flatten()(d8) - d10 = Dense(nb_dense, activation='tanh')(d9) + d10 = Dense(nb_dense, activation="tanh")(d9) d11 = Dropout(dropout_rate2)(d10) out = Dense(n_frame)(d11) model = Model(inputs=[diff_input, rawf_input], outputs=out) @@ -293,44 +365,53 @@ def CAN_3D(n_frame, nb_filters1, nb_filters2, input_shape, kernel_size=(3, 3, 3) # %% -def MT_CAN_3D(n_frame, nb_filters1, nb_filters2, input_shape, kernel_size=(3, 3, 3), dropout_rate1=0.25, - dropout_rate2=0.5, pool_size=(2, 2, 2), nb_dense=128): +def MT_CAN_3D( + n_frame, + nb_filters1, + nb_filters2, + input_shape, + kernel_size=(3, 3, 3), + dropout_rate1=0.25, + dropout_rate2=0.5, + pool_size=(2, 2, 2), + nb_dense=128, +): diff_input = Input(shape=input_shape) rawf_input = Input(shape=input_shape) - d1 = Conv3D(nb_filters1, kernel_size, padding='same', activation='tanh')(diff_input) - d2 = Conv3D(nb_filters1, kernel_size, activation='tanh')(d1) + d1 = Conv3D(nb_filters1, kernel_size, padding="same", activation="tanh")(diff_input) + d2 = Conv3D(nb_filters1, kernel_size, activation="tanh")(d1) # Appearance Branch - r1 = Conv3D(nb_filters1, kernel_size, padding='same', activation='tanh')(rawf_input) - r2 = Conv3D(nb_filters1, kernel_size, activation='tanh')(r1) - g1 = Conv3D(1, (1, 1, 1), padding='same', activation='sigmoid')(r2) + r1 = Conv3D(nb_filters1, kernel_size, padding="same", activation="tanh")(rawf_input) + r2 = Conv3D(nb_filters1, kernel_size, activation="tanh")(r1) + g1 = Conv3D(1, (1, 1, 1), padding="same", activation="sigmoid")(r2) g1 = Attention_mask()(g1) gated1 = multiply([d2, g1]) d3 = AveragePooling3D(pool_size)(gated1) d4 = Dropout(dropout_rate1)(d3) - d5 = Conv3D(nb_filters2, kernel_size, padding='same', activation='tanh')(d4) - d6 = Conv3D(nb_filters2, kernel_size, activation='tanh')(d5) + d5 = Conv3D(nb_filters2, kernel_size, padding="same", activation="tanh")(d4) + d6 = Conv3D(nb_filters2, kernel_size, activation="tanh")(d5) r3 = AveragePooling3D(pool_size)(r2) r4 = Dropout(dropout_rate1)(r3) - r5 = Conv3D(nb_filters2, kernel_size, padding='same', activation='tanh')(r4) - r6 = Conv3D(nb_filters2, kernel_size, activation='tanh')(r5) - g2 = Conv3D(1, (1, 1, 1), padding='same', activation='sigmoid')(r6) + r5 = Conv3D(nb_filters2, kernel_size, padding="same", activation="tanh")(r4) + r6 = Conv3D(nb_filters2, kernel_size, activation="tanh")(r5) + g2 = Conv3D(1, (1, 1, 1), padding="same", activation="sigmoid")(r6) g2 = Attention_mask()(g2) gated2 = multiply([d6, g2]) d7 = AveragePooling3D(pool_size)(gated2) d8 = Dropout(dropout_rate1)(d7) d9 = Flatten()(d8) - d10_y = Dense(nb_dense, activation='tanh')(d9) + d10_y = Dense(nb_dense, activation="tanh")(d9) d11_y = Dropout(dropout_rate2)(d10_y) - out_y = Dense(n_frame, name='output_1')(d11_y) + out_y = Dense(n_frame, name="output_1")(d11_y) - d10_r = Dense(nb_dense, activation='tanh')(d9) + d10_r = Dense(nb_dense, activation="tanh")(d9) d11_r = Dropout(dropout_rate2)(d10_r) - out_r = Dense(n_frame, name='output_2')(d11_r) + out_r = Dense(n_frame, name="output_2")(d11_r) model = Model(inputs=[diff_input, rawf_input], outputs=[out_y, out_r]) @@ -339,22 +420,38 @@ def MT_CAN_3D(n_frame, nb_filters1, nb_filters2, input_shape, kernel_size=(3, 3, # %% -def Hybrid_CAN(n_frame, nb_filters1, nb_filters2, input_shape_1, input_shape_2, kernel_size_1=(3, 3, 3), - kernel_size_2=(3, 3), dropout_rate1=0.25, dropout_rate2=0.5, pool_size_1=(2, 2, 2), pool_size_2=(2, 2), - nb_dense=128): + +def Hybrid_CAN( + n_frame, + nb_filters1, + nb_filters2, + input_shape_1, + input_shape_2, + kernel_size_1=(3, 3, 3), + kernel_size_2=(3, 3), + dropout_rate1=0.25, + dropout_rate2=0.5, + pool_size_1=(2, 2, 2), + pool_size_2=(2, 2), + nb_dense=128, +): diff_input = Input(shape=input_shape_1) rawf_input = Input(shape=input_shape_2) # Motion branch - d1 = Conv3D(nb_filters1, kernel_size_1, padding='same', activation='tanh')(diff_input) - d2 = Conv3D(nb_filters1, kernel_size_1, activation='tanh')(d1) + d1 = Conv3D(nb_filters1, kernel_size_1, padding="same", activation="tanh")( + diff_input + ) + d2 = Conv3D(nb_filters1, kernel_size_1, activation="tanh")(d1) # App branch - r1 = Conv2D(nb_filters1, kernel_size_2, padding='same', activation='tanh')(rawf_input) - r2 = Conv2D(nb_filters1, kernel_size_2, activation='tanh')(r1) + r1 = Conv2D(nb_filters1, kernel_size_2, padding="same", activation="tanh")( + rawf_input + ) + r2 = Conv2D(nb_filters1, kernel_size_2, activation="tanh")(r1) # Mask from App (g1) * Motion Branch (d2) - g1 = Conv2D(1, (1, 1), padding='same', activation='sigmoid')(r2) + g1 = Conv2D(1, (1, 1), padding="same", activation="sigmoid")(r2) g1 = Attention_mask()(g1) g1 = K.expand_dims(g1, axis=-1) gated1 = multiply([d2, g1]) @@ -362,17 +459,17 @@ def Hybrid_CAN(n_frame, nb_filters1, nb_filters2, input_shape_1, input_shape_2, # Motion Branch d3 = AveragePooling3D(pool_size_1)(gated1) d4 = Dropout(dropout_rate1)(d3) - d5 = Conv3D(nb_filters2, kernel_size_1, padding='same', activation='tanh')(d4) - d6 = Conv3D(nb_filters2, kernel_size_1, activation='tanh')(d5) + d5 = Conv3D(nb_filters2, kernel_size_1, padding="same", activation="tanh")(d4) + d6 = Conv3D(nb_filters2, kernel_size_1, activation="tanh")(d5) # App branch r3 = AveragePooling2D(pool_size_2)(r2) r4 = Dropout(dropout_rate1)(r3) - r5 = Conv2D(nb_filters2, kernel_size_2, padding='same', activation='tanh')(r4) - r6 = Conv2D(nb_filters2, kernel_size_2, activation='tanh')(r5) + r5 = Conv2D(nb_filters2, kernel_size_2, padding="same", activation="tanh")(r4) + r6 = Conv2D(nb_filters2, kernel_size_2, activation="tanh")(r5) # Mask from App (g2) * Motion Branch (d6) - g2 = Conv2D(1, (1, 1), padding='same', activation='sigmoid')(r6) + g2 = Conv2D(1, (1, 1), padding="same", activation="sigmoid")(r6) g2 = Attention_mask()(g2) g2 = K.repeat_elements(g2, d6.shape[3], axis=-1) g2 = K.expand_dims(g2, axis=-1) @@ -384,7 +481,7 @@ def Hybrid_CAN(n_frame, nb_filters1, nb_filters2, input_shape_1, input_shape_2, # Motion Branch d9 = Flatten()(d8) - d10 = Dense(nb_dense, activation='tanh')(d9) + d10 = Dense(nb_dense, activation="tanh")(d9) d11 = Dropout(dropout_rate2)(d10) out = Dense(n_frame)(d11) @@ -393,22 +490,37 @@ def Hybrid_CAN(n_frame, nb_filters1, nb_filters2, input_shape_1, input_shape_2, # %% -def MT_Hybrid_CAN(n_frame, nb_filters1, nb_filters2, input_shape_1, input_shape_2, kernel_size_1=(3, 3, 3), - kernel_size_2=(3, 3), dropout_rate1=0.25, dropout_rate2=0.5, pool_size_1=(2, 2, 2), - pool_size_2=(2, 2), nb_dense=128): +def MT_Hybrid_CAN( + n_frame, + nb_filters1, + nb_filters2, + input_shape_1, + input_shape_2, + kernel_size_1=(3, 3, 3), + kernel_size_2=(3, 3), + dropout_rate1=0.25, + dropout_rate2=0.5, + pool_size_1=(2, 2, 2), + pool_size_2=(2, 2), + nb_dense=128, +): diff_input = Input(shape=input_shape_1) rawf_input = Input(shape=input_shape_2) # Motion branch - d1 = Conv3D(nb_filters1, kernel_size_1, padding='same', activation='tanh')(diff_input) - d2 = Conv3D(nb_filters1, kernel_size_1, activation='tanh')(d1) + d1 = Conv3D(nb_filters1, kernel_size_1, padding="same", activation="tanh")( + diff_input + ) + d2 = Conv3D(nb_filters1, kernel_size_1, activation="tanh")(d1) # App branch - r1 = Conv2D(nb_filters1, kernel_size_2, padding='same', activation='tanh')(rawf_input) - r2 = Conv2D(nb_filters1, kernel_size_2, activation='tanh')(r1) + r1 = Conv2D(nb_filters1, kernel_size_2, padding="same", activation="tanh")( + rawf_input + ) + r2 = Conv2D(nb_filters1, kernel_size_2, activation="tanh")(r1) # Mask from App (g1) * Motion Branch (d2) - g1 = Conv2D(1, (1, 1), padding='same', activation='sigmoid')(r2) + g1 = Conv2D(1, (1, 1), padding="same", activation="sigmoid")(r2) g1 = Attention_mask()(g1) g1 = K.expand_dims(g1, axis=-1) gated1 = multiply([d2, g1]) @@ -416,17 +528,17 @@ def MT_Hybrid_CAN(n_frame, nb_filters1, nb_filters2, input_shape_1, input_shape_ # Motion Branch d3 = AveragePooling3D(pool_size_1)(gated1) d4 = Dropout(dropout_rate1)(d3) - d5 = Conv3D(nb_filters2, kernel_size_1, padding='same', activation='tanh')(d4) - d6 = Conv3D(nb_filters2, kernel_size_1, activation='tanh')(d5) + d5 = Conv3D(nb_filters2, kernel_size_1, padding="same", activation="tanh")(d4) + d6 = Conv3D(nb_filters2, kernel_size_1, activation="tanh")(d5) # App branch r3 = AveragePooling2D(pool_size_2)(r2) r4 = Dropout(dropout_rate1)(r3) - r5 = Conv2D(nb_filters2, kernel_size_2, padding='same', activation='tanh')(r4) - r6 = Conv2D(nb_filters2, kernel_size_2, activation='tanh')(r5) + r5 = Conv2D(nb_filters2, kernel_size_2, padding="same", activation="tanh")(r4) + r6 = Conv2D(nb_filters2, kernel_size_2, activation="tanh")(r5) # Mask from App (g2) * Motion Branch (d6) - g2 = Conv2D(1, (1, 1), padding='same', activation='sigmoid')(r6) + g2 = Conv2D(1, (1, 1), padding="same", activation="sigmoid")(r6) g2 = Attention_mask()(g2) g2 = K.repeat_elements(g2, d6.shape[3], axis=-1) g2 = K.expand_dims(g2, axis=-1) @@ -439,13 +551,13 @@ def MT_Hybrid_CAN(n_frame, nb_filters1, nb_filters2, input_shape_1, input_shape_ # Motion Branch d9 = Flatten()(d8) - d10_y = Dense(nb_dense, activation='tanh')(d9) + d10_y = Dense(nb_dense, activation="tanh")(d9) d11_y = Dropout(dropout_rate2)(d10_y) - out_y = Dense(n_frame, name='output_1')(d11_y) + out_y = Dense(n_frame, name="output_1")(d11_y) - d10_r = Dense(nb_dense, activation='tanh')(d9) + d10_r = Dense(nb_dense, activation="tanh")(d9) d11_r = Dropout(dropout_rate2)(d10_r) - out_r = Dense(n_frame, name='output_2')(d11_r) + out_r = Dense(n_frame, name="output_2")(d11_r) model = Model(inputs=[diff_input, rawf_input], outputs=[out_y, out_r]) return model @@ -462,4 +574,4 @@ def __init__(self, train_gen, test_gen, args, cv_split, save_dir): self.save_dir = save_dir def on_epoch_end(self, epoch, logs={}): - print('PROGRESS: 0.00%') + print("PROGRESS: 0.00%") From 21fe284cdba4adaf180f5159f8d0ebbf860a5a44 Mon Sep 17 00:00:00 2001 From: tphlru <35965490+tphlru@users.noreply.github.com> Date: Thu, 15 Aug 2024 23:01:10 +0300 Subject: [PATCH 2/2] Refactor code for consistency and readability - Standardize string quotes to double quotes across all files. - Refactor long lines to improve readability. - Extract repeated code into helper functions for better modularity. - Simplify conditional statements and assertions. - Remove unnecessary imports and variables. - Update function and variable names for clarity. - Add TODO comments for further refactoring. - Improve formatting and indentation for better code structure. --- code/inference_preprocess.py | 9 +- code/model.py | 128 +++-- code/predict_vitals.py | 47 +- code/tvm-ops-mtts-can.py | 971 ++++++++++++++++++++--------------- 4 files changed, 652 insertions(+), 503 deletions(-) diff --git a/code/inference_preprocess.py b/code/inference_preprocess.py index 9b269ad..fc18691 100644 --- a/code/inference_preprocess.py +++ b/code/inference_preprocess.py @@ -1,10 +1,7 @@ import numpy as np import cv2 from skimage.util import img_as_float -import tensorflow as tf import matplotlib.pyplot as plt -import time -import scipy.io from scipy.sparse import spdiags @@ -20,7 +17,6 @@ def preprocess_raw_video(videoFilePath, dim=36): height = vidObj.get(cv2.CAP_PROP_FRAME_HEIGHT) width = vidObj.get(cv2.CAP_PROP_FRAME_WIDTH) success, img = vidObj.read() - dims = img.shape print("Orignal Height", height) print("Original width", width) ######################################################################### @@ -96,7 +92,4 @@ def detrend(signal, Lambda): diags_data = np.array([ones, minus_twos, ones]) diags_index = np.array([0, 1, 2]) D = spdiags(diags_data, diags_index, (signal_length - 2), signal_length).toarray() - filtered_signal = np.dot( - (H - np.linalg.inv(H + (Lambda**2) * np.dot(D.T, D))), signal - ) - return filtered_signal + return np.dot((H - np.linalg.inv(H + (Lambda**2) * np.dot(D.T, D))), signal) diff --git a/code/model.py b/code/model.py index f6319d3..09430be 100755 --- a/code/model.py +++ b/code/model.py @@ -28,8 +28,7 @@ def call(self, x): return x / xsum * xshape[1] * xshape[2] * 0.5 def get_config(self): - config = super(Attention_mask, self).get_config() - return config + return super(Attention_mask, self).get_config() class TSM(tf.keras.layers.Layer): @@ -40,17 +39,11 @@ def call(self, x, n_frame, fold_div=3): last_fold = c - (fold_div - 1) * fold out1, out2, out3 = tf.split(x, [fold, fold, last_fold], axis=-1) - # Shift left - padding_1 = tf.zeros_like(out1) - padding_1 = padding_1[:, -1, :, :, :] - padding_1 = tf.expand_dims(padding_1, 1) + padding_1 = self._extracted_from_call_9(out1, -1) _, out1 = tf.split(out1, [1, n_frame - 1], axis=1) out1 = tf.concat([out1, padding_1], axis=1) - # Shift right - padding_2 = tf.zeros_like(out2) - padding_2 = padding_2[:, 0, :, :, :] - padding_2 = tf.expand_dims(padding_2, 1) + padding_2 = self._extracted_from_call_9(out2, 0) out2, _ = tf.split(out2, [n_frame - 1, 1], axis=1) out2 = tf.concat([padding_2, out2], axis=1) @@ -59,9 +52,15 @@ def call(self, x, n_frame, fold_div=3): return out + # TODO Rename this here and in `call` + def _extracted_from_call_9(self, arg0, arg1): + # Shift left + result = tf.zeros_like(arg0) + result = result[:, arg1, :, :, :] + return tf.expand_dims(result, 1) + def get_config(self): - config = super(TSM, self).get_config() - return config + return super(TSM, self).get_config() def TSM_Cov2D( @@ -121,8 +120,7 @@ def CAN( d10 = Dense(nb_dense, activation="tanh")(d9) d11 = Dropout(dropout_rate2)(d10) out = Dense(1)(d11) - model = Model(inputs=[diff_input, rawf_input], outputs=out) - return model + return Model(inputs=[diff_input, rawf_input], outputs=out) # %% MT_CAN @@ -169,16 +167,16 @@ def MT_CAN( d8 = Dropout(dropout_rate1)(d7) d9 = Flatten()(d8) - d10_y = Dense(nb_dense, activation="tanh")(d9) - d11_y = Dropout(dropout_rate2)(d10_y) - out_y = Dense(1, name="output_1")(d11_y) + out_y = _extracted_from_MT_CAN_44(nb_dense, d9, dropout_rate2, "output_1") + out_r = _extracted_from_MT_CAN_44(nb_dense, d9, dropout_rate2, "output_2") + return Model(inputs=[diff_input, rawf_input], outputs=[out_y, out_r]) - d10_r = Dense(nb_dense, activation="tanh")(d9) - d11_r = Dropout(dropout_rate2)(d10_r) - out_r = Dense(1, name="output_2")(d11_r) - model = Model(inputs=[diff_input, rawf_input], outputs=[out_y, out_r]) - return model +# TODO Rename this here and in `MT_CAN` +def _extracted_from_MT_CAN_44(nb_dense, d9, dropout_rate2, name): + d10_y = Dense(nb_dense, activation="tanh")(d9) + d11_y = Dropout(dropout_rate2)(d10_y) + return Dense(1, name=name)(d11_y) # %% TS_CAN @@ -239,8 +237,7 @@ def TS_CAN( d10 = Dense(nb_dense, activation="tanh")(d9) d11 = Dropout(dropout_rate2)(d10) out = Dense(1)(d11) - model = Model(inputs=[diff_input, rawf_input], outputs=out) - return model + return Model(inputs=[diff_input, rawf_input], outputs=out) # %% MTTS-CAN @@ -299,16 +296,16 @@ def MTTS_CAN( d9 = Flatten()(d8) - d10_y = Dense(nb_dense, activation="tanh")(d9) - d11_y = Dropout(dropout_rate2)(d10_y) - out_y = Dense(1, name="output_1")(d11_y) + out_y = _extracted_from_MTTS_CAN_54(nb_dense, d9, dropout_rate2, "output_1") + out_r = _extracted_from_MTTS_CAN_54(nb_dense, d9, dropout_rate2, "output_2") + return Model(inputs=[diff_input, rawf_input], outputs=[out_y, out_r]) - d10_r = Dense(nb_dense, activation="tanh")(d9) - d11_r = Dropout(dropout_rate2)(d10_r) - out_r = Dense(1, name="output_2")(d11_r) - model = Model(inputs=[diff_input, rawf_input], outputs=[out_y, out_r]) - return model +# TODO Rename this here and in `MTTS_CAN` +def _extracted_from_MTTS_CAN_54(nb_dense, d9, dropout_rate2, name): + d10_y = Dense(nb_dense, activation="tanh")(d9) + d11_y = Dropout(dropout_rate2)(d10_y) + return Dense(1, name=name)(d11_y) # %% @@ -355,8 +352,7 @@ def CAN_3D( d10 = Dense(nb_dense, activation="tanh")(d9) d11 = Dropout(dropout_rate2)(d10) out = Dense(n_frame)(d11) - model = Model(inputs=[diff_input, rawf_input], outputs=out) - return model + return Model(inputs=[diff_input, rawf_input], outputs=out) # input_shape = (36, 36, 10, 3) @@ -413,9 +409,7 @@ def MT_CAN_3D( d11_r = Dropout(dropout_rate2)(d10_r) out_r = Dense(n_frame, name="output_2")(d11_r) - model = Model(inputs=[diff_input, rawf_input], outputs=[out_y, out_r]) - - return model + return Model(inputs=[diff_input, rawf_input], outputs=[out_y, out_r]) # %% @@ -453,12 +447,7 @@ def Hybrid_CAN( # Mask from App (g1) * Motion Branch (d2) g1 = Conv2D(1, (1, 1), padding="same", activation="sigmoid")(r2) g1 = Attention_mask()(g1) - g1 = K.expand_dims(g1, axis=-1) - gated1 = multiply([d2, g1]) - - # Motion Branch - d3 = AveragePooling3D(pool_size_1)(gated1) - d4 = Dropout(dropout_rate1)(d3) + d4 = _extracted_from_Hybrid_CAN_33(g1, d2, pool_size_1, dropout_rate1) d5 = Conv3D(nb_filters2, kernel_size_1, padding="same", activation="tanh")(d4) d6 = Conv3D(nb_filters2, kernel_size_1, activation="tanh")(d5) @@ -472,21 +461,24 @@ def Hybrid_CAN( g2 = Conv2D(1, (1, 1), padding="same", activation="sigmoid")(r6) g2 = Attention_mask()(g2) g2 = K.repeat_elements(g2, d6.shape[3], axis=-1) - g2 = K.expand_dims(g2, axis=-1) - gated2 = multiply([d6, g2]) - - # Motion Branch - d7 = AveragePooling3D(pool_size_1)(gated2) - d8 = Dropout(dropout_rate1)(d7) - + d8 = _extracted_from_Hybrid_CAN_33(g2, d6, pool_size_1, dropout_rate1) # Motion Branch d9 = Flatten()(d8) d10 = Dense(nb_dense, activation="tanh")(d9) d11 = Dropout(dropout_rate2)(d10) out = Dense(n_frame)(d11) - model = Model(inputs=[diff_input, rawf_input], outputs=out) - return model + return Model(inputs=[diff_input, rawf_input], outputs=out) + + +# TODO Rename this here and in `Hybrid_CAN` +def _extracted_from_Hybrid_CAN_33(arg0, arg1, pool_size_1, dropout_rate1): + arg0 = K.expand_dims(arg0, axis=-1) + gated1 = multiply([arg1, arg0]) + + # Motion Branch + d3 = AveragePooling3D(pool_size_1)(gated1) + return Dropout(dropout_rate1)(d3) # %% @@ -522,12 +514,7 @@ def MT_Hybrid_CAN( # Mask from App (g1) * Motion Branch (d2) g1 = Conv2D(1, (1, 1), padding="same", activation="sigmoid")(r2) g1 = Attention_mask()(g1) - g1 = K.expand_dims(g1, axis=-1) - gated1 = multiply([d2, g1]) - - # Motion Branch - d3 = AveragePooling3D(pool_size_1)(gated1) - d4 = Dropout(dropout_rate1)(d3) + d4 = _extracted_from_MT_Hybrid_CAN_33(g1, d2, pool_size_1, dropout_rate1) d5 = Conv3D(nb_filters2, kernel_size_1, padding="same", activation="tanh")(d4) d6 = Conv3D(nb_filters2, kernel_size_1, activation="tanh")(d5) @@ -541,13 +528,7 @@ def MT_Hybrid_CAN( g2 = Conv2D(1, (1, 1), padding="same", activation="sigmoid")(r6) g2 = Attention_mask()(g2) g2 = K.repeat_elements(g2, d6.shape[3], axis=-1) - g2 = K.expand_dims(g2, axis=-1) - gated2 = multiply([d6, g2]) - - # Motion Branch - d7 = AveragePooling3D(pool_size_1)(gated2) - d8 = Dropout(dropout_rate1)(d7) - + d8 = _extracted_from_MT_Hybrid_CAN_33(g2, d6, pool_size_1, dropout_rate1) # Motion Branch d9 = Flatten()(d8) @@ -559,8 +540,17 @@ def MT_Hybrid_CAN( d11_r = Dropout(dropout_rate2)(d10_r) out_r = Dense(n_frame, name="output_2")(d11_r) - model = Model(inputs=[diff_input, rawf_input], outputs=[out_y, out_r]) - return model + return Model(inputs=[diff_input, rawf_input], outputs=[out_y, out_r]) + + +# TODO Rename this here and in `MT_Hybrid_CAN` +def _extracted_from_MT_Hybrid_CAN_33(arg0, arg1, pool_size_1, dropout_rate1): + arg0 = K.expand_dims(arg0, axis=-1) + gated1 = multiply([arg1, arg0]) + + # Motion Branch + d3 = AveragePooling3D(pool_size_1)(gated1) + return Dropout(dropout_rate1)(d3) # %% @@ -573,5 +563,7 @@ def __init__(self, train_gen, test_gen, args, cv_split, save_dir): self.cv_split = cv_split self.save_dir = save_dir - def on_epoch_end(self, epoch, logs={}): + def on_epoch_end(self, epoch, logs=None): + if logs is None: + logs = {} print("PROGRESS: 0.00%") diff --git a/code/predict_vitals.py b/code/predict_vitals.py index 1a2b74e..3964a5e 100644 --- a/code/predict_vitals.py +++ b/code/predict_vitals.py @@ -1,63 +1,68 @@ -import tensorflow as tf +from model import MTTS_CAN +import matplotlib.pyplot as plt +from scipy.signal import butter +from inference_preprocess import preprocess_raw_video, detrend + import numpy as np import scipy.io -import os import sys import argparse -sys.path.append('../') -from model import Attention_mask, MTTS_CAN -import h5py -import matplotlib.pyplot as plt -from scipy.signal import butter -from inference_preprocess import preprocess_raw_video, detrend + +sys.path.append("../") + def predict_vitals(args): img_rows = 36 img_cols = 36 frame_depth = 10 - model_checkpoint = './mtts_can.hdf5' + model_checkpoint = "./mtts_can.hdf5" batch_size = args.batch_size fs = args.sampling_rate sample_data_path = args.video_path dXsub = preprocess_raw_video(sample_data_path, dim=36) - print('dXsub shape', dXsub.shape) + print("dXsub shape", dXsub.shape) - dXsub_len = (dXsub.shape[0] // frame_depth) * frame_depth + dXsub_len = (dXsub.shape[0] // frame_depth) * frame_depth dXsub = dXsub[:dXsub_len, :, :, :] model = MTTS_CAN(frame_depth, 32, 64, (img_rows, img_cols, 3)) model.load_weights(model_checkpoint) - yptest = model.predict((dXsub[:, :, :, :3], dXsub[:, :, :, -3:]), batch_size=batch_size, verbose=1) + yptest = model.predict( + (dXsub[:, :, :, :3], dXsub[:, :, :, -3:]), batch_size=batch_size, verbose=1 + ) pulse_pred = yptest[0] pulse_pred = detrend(np.cumsum(pulse_pred), 100) - [b_pulse, a_pulse] = butter(1, [0.75 / fs * 2, 2.5 / fs * 2], btype='bandpass') + [b_pulse, a_pulse] = butter(1, [0.75 / fs * 2, 2.5 / fs * 2], btype="bandpass") pulse_pred = scipy.signal.filtfilt(b_pulse, a_pulse, np.double(pulse_pred)) resp_pred = yptest[1] resp_pred = detrend(np.cumsum(resp_pred), 100) - [b_resp, a_resp] = butter(1, [0.08 / fs * 2, 0.5 / fs * 2], btype='bandpass') + [b_resp, a_resp] = butter(1, [0.08 / fs * 2, 0.5 / fs * 2], btype="bandpass") resp_pred = scipy.signal.filtfilt(b_resp, a_resp, np.double(resp_pred)) - ########## Plot ################## + # ---------- Plot ---------------- plt.subplot(211) plt.plot(pulse_pred) - plt.title('Pulse Prediction') + plt.title("Pulse Prediction") plt.subplot(212) plt.plot(resp_pred) - plt.title('Resp Prediction') + plt.title("Resp Prediction") plt.show() if __name__ == "__main__": parser = argparse.ArgumentParser() - parser.add_argument('--video_path', type=str, help='processed video path') - parser.add_argument('--sampling_rate', type=int, default = 30, help='sampling rate of your video') - parser.add_argument('--batch_size', type=int, default = 100, help='batch size (multiplier of 10)') + parser.add_argument("--video_path", type=str, help="processed video path") + parser.add_argument( + "--sampling_rate", type=int, default=30, help="sampling rate of your video" + ) + parser.add_argument( + "--batch_size", type=int, default=100, help="batch size (multiplier of 10)" + ) args = parser.parse_args() predict_vitals(args) - diff --git a/code/tvm-ops-mtts-can.py b/code/tvm-ops-mtts-can.py index db331f9..b8986c6 100755 --- a/code/tvm-ops-mtts-can.py +++ b/code/tvm-ops-mtts-can.py @@ -32,13 +32,17 @@ from .common import ExprTable, new_var from tvm.relay.frontend.common import infer_shape -__all__ = ['from_keras'] +__all__ = ["from_keras"] def _check_data_format(keras_layer): - if hasattr(keras_layer, ('data_format')): - if keras_layer.data_format != 'channels_last': - raise ValueError("Keras frontend currently supports data_format = channels_last only.") + if ( + hasattr(keras_layer, ("data_format")) + and keras_layer.data_format != "channels_last" # noqa:W503 + ): + raise ValueError( + "Keras frontend currently supports data_format = channels_last only." + ) def _get_pad_pair(input1d, kernel1d, stride1d): @@ -51,23 +55,28 @@ def _get_pad_pair(input1d, kernel1d, stride1d): def _get_elu(inexpr, alpha): """A helper method for elu.""" - return _op.negative(alpha) * _op.nn.relu(_expr.const(1., dtype='float32') - \ - _op.exp(inexpr)) + _op.nn.relu(inexpr) + return _op.negative(alpha) * _op.nn.relu( + _expr.const(1.0, dtype="float32") - _op.exp(inexpr) + ) + _op.nn.relu(inexpr) def _as_list(arr): """Force being a list, ignore if already is.""" - if isinstance(arr, list): - return arr - return [arr] + return arr if isinstance(arr, list) else [arr] def _convert_attention_mask(inexpr, keras_layer, etab): - xsum = _op.reduce.sum(_op.reduce.sum(inexpr, axis=2, keepdims=True), axis=3, keepdims=True) + xsum = _op.reduce.sum( + _op.reduce.sum(inexpr, axis=2, keepdims=True), axis=3, keepdims=True + ) xshape = infer_shape(inexpr) - out = inexpr / xsum * tvm.relay.expr.const(xshape[2], dtype='float32') \ - * tvm.relay.expr.const(xshape[3], dtype='float32') * tvm.relay.expr.const(0.5, dtype='float32') - return out + return ( + (inexpr) + / xsum # noqa:W503 + * tvm.relay.expr.const(xshape[2], dtype="float32") # noqa:W503 + * tvm.relay.expr.const(xshape[3], dtype="float32") # noqa:W503 + * tvm.relay.expr.const(0.5, dtype="float32") # noqa:W503 + ) def _convert_expand_dim(inexpr, keras_layer, etab): @@ -78,21 +87,27 @@ def _convert_tsm(inexpr, keras_layer, etab): nt, c, h, w = infer_shape(inexpr) n = int(nt / 10) - fold_div = 3 x = _op.transform.reshape(inexpr, [n, 10, c, h, w]) fold = c // 3 # last_fold = c - (fold_div - 1) * fold - split_out = _op.split(x, (fold, fold*2, ), axis=2) + split_out = _op.split( + x, + ( + fold, + fold * 2, + ), + axis=2, + ) out1, out2, out3 = split_out[0], split_out[1], split_out[2] padding_1 = _op.zeros((n, 1, fold, h, w), dtype="float32") - out1 = _op.split(out1, (1, ), axis=1)[1] + out1 = _op.split(out1, (1,), axis=1)[1] out1 = _op.concatenate([out1, padding_1], axis=1) # Shift right padding_2 = _op.zeros((n, 1, fold, h, w), dtype="float32") - out2 = _op.split(out2, (10 - 1, ), axis=1)[0] + out2 = _op.split(out2, (10 - 1,), axis=1)[0] out2 = _op.concatenate([padding_2, out2], axis=1) out = _op.concatenate([out1, out2, out3], axis=2) @@ -109,143 +124,173 @@ def _convert_recurrent_activation(inexpr, keras_layer): def _convert_activation(inexpr, keras_layer, _): if isinstance(keras_layer, str): act_type = keras_layer + elif sys.version_info.major < 3: + act_type = keras_layer.activation.func_name else: - if sys.version_info.major < 3: - act_type = keras_layer.activation.func_name - else: - act_type = keras_layer.activation.__name__ - if act_type == 'linear': - if isinstance(keras_layer, str): - return inexpr - alpha = keras_layer.alpha if hasattr(keras_layer, 'alpha') else 1. - beta = keras_layer.beta if hasattr(keras_layer, 'beta') else 0. - alpha = _expr.const(alpha, dtype='float32') - beta = _expr.const(beta, dtype='float32') - return _op.add(_op.multiply(inexpr, alpha), beta) - if act_type == 'softmax': + act_type = keras_layer.activation.__name__ + if act_type == "linear": + return _extracted_from__convert_activation_9(keras_layer, inexpr) + if act_type == "softmax": return _op.nn.softmax(inexpr, axis=1) - if act_type == 'sigmoid': + if act_type == "sigmoid": return _op.sigmoid(inexpr) - if act_type == 'tanh': + if act_type == "tanh": return _op.tanh(inexpr) - if act_type == 'relu': + if act_type == "relu": return _op.nn.relu(inexpr) - if act_type == 'softplus': - return _op.log(_op.add(_op.exp(inexpr), _expr.const(1., dtype='float32'))) - if act_type == 'elu': - alpha = keras_layer.alpha if hasattr(keras_layer, 'alpha') else 1. - alpha = _expr.const(alpha, dtype='float32') + if act_type == "softplus": + return _op.log(_op.add(_op.exp(inexpr), _expr.const(1.0, dtype="float32"))) + if act_type == "elu": + alpha = keras_layer.alpha if hasattr(keras_layer, "alpha") else 1.0 + alpha = _expr.const(alpha, dtype="float32") return _get_elu(inexpr, alpha) - if act_type == 'selu': + if act_type == "selu": # Alpha, Gamma values obtained from https://arxiv.org/abs/1706.02515 - alpha = keras_layer.alpha if hasattr(keras_layer, 'alpha') \ + alpha = ( + keras_layer.alpha + if hasattr(keras_layer, "alpha") else 1.6732632423543772848170429916717 - gamma = keras_layer.gamma if hasattr(keras_layer, 'gamma') \ + ) + gamma = ( + keras_layer.gamma + if hasattr(keras_layer, "gamma") else 1.0507009873554804934193349852946 - alpha = _expr.const(alpha, dtype='float32') - gamma = _expr.const(gamma, dtype='float32') + ) + alpha = _expr.const(alpha, dtype="float32") + gamma = _expr.const(gamma, dtype="float32") return gamma * _get_elu(inexpr, alpha) - if act_type == 'relu6': - return _op.clip(inexpr, a_min=0., a_max=6.) - if act_type == 'softsign': - return inexpr / (_expr.const(1., dtype='float32') + _op.abs(inexpr)) - if act_type == 'hard_sigmoid': - x = (_expr.const(0.2, dtype='float32') * inexpr) + _expr.const(0.5, dtype='float32') - return _op.clip(x, a_min=0., a_max=1.) + if act_type == "relu6": + return _op.clip(inexpr, a_min=0.0, a_max=6.0) + if act_type == "softsign": + return inexpr / (_expr.const(1.0, dtype="float32") + _op.abs(inexpr)) + if act_type == "hard_sigmoid": + x = (_expr.const(0.2, dtype="float32") * inexpr) + _expr.const( + 0.5, dtype="float32" + ) + return _op.clip(x, a_min=0.0, a_max=1.0) raise tvm.error.OpNotImplemented( - 'Operator {} is not supported in frontend Keras.'.format(act_type)) + f"Operator {act_type} is not supported in frontend Keras." + ) + + +# TODO Rename this here and in `_convert_activation` +def _extracted_from__convert_activation_9(keras_layer, inexpr): + if isinstance(keras_layer, str): + return inexpr + alpha = keras_layer.alpha if hasattr(keras_layer, "alpha") else 1.0 + beta = keras_layer.beta if hasattr(keras_layer, "beta") else 0.0 + alpha = _expr.const(alpha, dtype="float32") + beta = _expr.const(beta, dtype="float32") + return _op.add(_op.multiply(inexpr, alpha), beta) def _convert_advanced_activation(inexpr, keras_layer, etab): act_type = type(keras_layer).__name__ - if act_type == 'Softmax': + if act_type == "Softmax": axis = keras_layer.axis dims = len(keras_layer.input_shape) if isinstance(axis, list): raise tvm.error.OpAttributeUnImplemented( - 'Softmax with axes {} is not supported.'.format(axis)) + f"Softmax with axes {axis} is not supported." + ) if axis == -1: axis = 1 else: axis = axis + 1 if axis < dims - 1 else 1 return _op.nn.softmax(inexpr, axis=axis) - if act_type == 'ReLU': - threshold = _expr.const(keras_layer.threshold, dtype='float32') + if act_type == "ReLU": + threshold = _expr.const(keras_layer.threshold, dtype="float32") if keras_layer.max_value and float(keras_layer.threshold) == 0: # f(x) = max_value, for x >= max_value # f(x) = x, for threshold <= x < max_value - return _op.clip(inexpr, a_min=0., a_max=float(keras_layer.max_value)) - if keras_layer.max_value and _op.greater(threshold, inexpr).astype('float32'): + return _op.clip(inexpr, a_min=0.0, a_max=float(keras_layer.max_value)) + if keras_layer.max_value and _op.greater(threshold, inexpr).astype("float32"): # f(x) = negative_slope * (inexpr - threshold) - negative_slope = _expr.const(keras_layer.negative_slope, dtype='float32') + negative_slope = _expr.const(keras_layer.negative_slope, dtype="float32") return _op.multiply(negative_slope, _op.subtract(inexpr, threshold)) return _op.nn.relu(inexpr) - if act_type == 'LeakyReLU': + if act_type == "LeakyReLU": return _op.nn.leaky_relu(inexpr, alpha=float(keras_layer.alpha)) - if act_type == 'ELU': - alpha = keras_layer.alpha if hasattr(keras_layer, 'alpha') else 1. - alpha = _expr.const(alpha, dtype='float32') + if act_type == "ELU": + alpha = keras_layer.alpha if hasattr(keras_layer, "alpha") else 1.0 + alpha = _expr.const(alpha, dtype="float32") return _get_elu(inexpr, alpha) - if act_type == 'PReLU': - assert hasattr(keras_layer, 'alpha'), "alpha required for PReLU." + if act_type == "PReLU": + assert hasattr(keras_layer, "alpha"), "alpha required for PReLU." _check_data_format(keras_layer) size = len(keras_layer.alpha.shape) - alpha = etab.new_const(keras_layer.get_weights()[0] \ - .transpose(np.roll(range(size), 1))) - return _op.negative(alpha) * _op.nn.relu(_op.negative(inexpr)) + _op.nn.relu(inexpr) - if act_type == 'ThresholdedReLU': - theta = keras_layer.theta if hasattr(keras_layer, 'theta') else 1. - return _op.multiply(inexpr, _op.greater(inexpr, \ - _expr.const(theta, dtype='float32')).astype('float32')) + alpha = etab.new_const( + keras_layer.get_weights()[0].transpose(np.roll(range(size), 1)) + ) + return _op.negative(alpha) * _op.nn.relu(_op.negative(inexpr)) + _op.nn.relu( + inexpr + ) + if act_type == "ThresholdedReLU": + theta = keras_layer.theta if hasattr(keras_layer, "theta") else 1.0 + return _op.multiply( + inexpr, + _op.greater(inexpr, _expr.const(theta, dtype="float32")).astype("float32"), + ) raise tvm.error.OpNotImplemented( - 'Operator {} is not supported in frontend Keras.'.format(act_type)) + f"Operator {act_type} is not supported in frontend Keras." + ) def _convert_merge(inexpr, keras_layer, _): merge_type = type(keras_layer).__name__ ret = inexpr[0] - if merge_type == 'Dot': - axes = keras_layer.axes - if isinstance(keras_layer.axes, int): - axes = [keras_layer.axes, keras_layer.axes] - if isinstance(axes, list): - if len(axes) != 2: - raise tvm.error.OpAttributeUnImplemented( - 'Dot with axes {} is not supported.'.format(keras_layer.axes)) - for i, axis in enumerate(axes): - if axis not in [1, 2]: - raise tvm.error.OpAttributeUnImplemented( - 'Dot with axes {} is not supported.'.format(keras_layer.axes)) - if axes[i] == 2: - inexpr[i] = _op.transpose(inexpr[i], axes=[0, 2, 1]) - else: - raise tvm.error.OpAttributeUnImplemented( - 'Dot with axes {} is not supported.'.format(keras_layer.axes)) - ret_dot = _op.nn.batch_matmul(inexpr[0], inexpr[1]) - ret = _op.transpose(ret_dot, axes=[0, 2, 1]) - elif merge_type == 'Subtract': + if merge_type == "Dot": + ret = _extracted_from__convert_merge_(keras_layer, inexpr) + elif merge_type == "Subtract": assert len(inexpr) == 2, "Subtract merge takes 2 inputs." ret = _op.subtract(ret, inexpr[1]) - elif merge_type in ['Add', 'Multiply', 'Minimum', 'Maximum']: - op_map = {'Add': _op.add, - 'Multiply': _op.multiply, - 'Minimum': _op.minimum, - 'Maximum': _op.maximum} + elif merge_type in ["Add", "Multiply", "Minimum", "Maximum"]: + op_map = { + "Add": _op.add, + "Multiply": _op.multiply, + "Minimum": _op.minimum, + "Maximum": _op.maximum, + } for i in range(1, len(inexpr)): ret = op_map[merge_type](ret, inexpr[i]) - elif merge_type == 'Average': + elif merge_type == "Average": for i in range(1, len(inexpr)): ret = _op.add(ret, inexpr[i]) - ret = ret / _expr.const(len(inexpr), dtype='float32') + ret = ret / _expr.const(len(inexpr), dtype="float32") else: raise tvm.error.OpNotImplemented( - 'Operator {} is not supported in frontend Keras.'.format(merge_type)) + f"Operator {merge_type} is not supported in frontend Keras." + ) return ret +# TODO Rename this here and in `_convert_merge` +def _extracted_from__convert_merge_(keras_layer, inexpr): + axes = keras_layer.axes + if isinstance(keras_layer.axes, int): + axes = [keras_layer.axes, keras_layer.axes] + if not isinstance(axes, list): + raise tvm.error.OpAttributeUnImplemented( + f"Dot with axes {keras_layer.axes} is not supported." + ) + if len(axes) != 2: + raise tvm.error.OpAttributeUnImplemented( + f"Dot with axes {keras_layer.axes} is not supported." + ) + for i, axis in enumerate(axes): + if axis not in [1, 2]: + raise tvm.error.OpAttributeUnImplemented( + f"Dot with axes {keras_layer.axes} is not supported." + ) + if axes[i] == 2: + inexpr[i] = _op.transpose(inexpr[i], axes=[0, 2, 1]) + ret_dot = _op.nn.batch_matmul(inexpr[0], inexpr[1]) + return _op.transpose(ret_dot, axes=[0, 2, 1]) + + def _convert_permute(inexpr, keras_layer, _): return _op.transpose(inexpr, axes=(0,) + keras_layer.dims) @@ -254,22 +299,22 @@ def _convert_embedding(inexpr, keras_layer, etab): indices = inexpr weightList = keras_layer.get_weights() weight = etab.new_const(weightList[0]) - out = _op.take(weight, indices.astype('int32'), axis=0) + return _op.take(weight, indices.astype("int32"), axis=0) - return out def _convert_dense(inexpr, keras_layer, etab): weightList = keras_layer.get_weights() weight = etab.new_const(weightList[0].transpose([1, 0])) - params = {'weight': weight, 'units': weightList[0].shape[1]} + params = {"weight": weight, "units": weightList[0].shape[1]} input_shape = keras_layer.input_shape input_dim = len(input_shape) # In case of RNN dense, input shape will be (1, 1, n) if input_dim > 2: - input_shape = tuple(dim if dim else 1 for dim in _as_list(input_shape)[0]) + input_shape = tuple(dim or 1 for dim in _as_list(input_shape)[0]) if input_dim != 3 or input_shape[0] != 1 or input_shape[1] != 1: raise tvm.error.OpAttributeInvalid( - 'Input shape {} is not valid for operator Dense.'.format(input_shape)) + f"Input shape {input_shape} is not valid for operator Dense." + ) inexpr = _op.squeeze(inexpr, axis=0) out = _op.nn.dense(data=inexpr, **params) if keras_layer.use_bias: @@ -280,7 +325,7 @@ def _convert_dense(inexpr, keras_layer, etab): act_type = keras_layer.activation.func_name else: act_type = keras_layer.activation.__name__ - if act_type != 'linear': + if act_type != "linear": out = _convert_activation(out, act_type, etab) if input_dim > 2: out = _op.expand_dims(out, axis=0) @@ -289,28 +334,24 @@ def _convert_dense(inexpr, keras_layer, etab): def _convert_convolution(inexpr, keras_layer, etab): _check_data_format(keras_layer) - is_deconv = type(keras_layer).__name__ == 'Conv2DTranspose' - is_depthconv = type(keras_layer).__name__ == 'DepthwiseConv2D' + is_deconv = type(keras_layer).__name__ == "Conv2DTranspose" + is_depthconv = type(keras_layer).__name__ == "DepthwiseConv2D" weightList = keras_layer.get_weights() weight = weightList[0] - if etab.data_layout == 'NHWC': - if is_depthconv: - kernel_layout = 'HWOI' - else: - kernel_layout = 'HWIO' + if etab.data_layout == "NHWC": + kernel_layout = "HWOI" if is_depthconv else "HWIO" else: - kernel_layout = 'OIHW' - + kernel_layout = "OIHW" if is_deconv: kernel_h, kernel_w, n_filters, in_channels = weight.shape - if kernel_layout == 'OIHW': + if kernel_layout == "OIHW": weight = weight.transpose([3, 2, 0, 1]) elif is_depthconv: kernel_h, kernel_w, in_channels, depth_mult = weight.shape - if kernel_layout == 'OIHW': + if kernel_layout == "OIHW": weight = weight.transpose([2, 3, 0, 1]) - elif etab.data_layout == 'NCHW': + elif etab.data_layout == "NCHW": kernel_h, kernel_w, in_channels, n_filters = weight.shape weight = weight.transpose([3, 2, 0, 1]) else: @@ -322,30 +363,34 @@ def _convert_convolution(inexpr, keras_layer, etab): dilated_kernel_h = (kernel_h - 1) * dilation[0] + 1 dilated_kernel_w = (kernel_w - 1) * dilation[1] + 1 stride_h, stride_w = keras_layer.strides - params = {'weight': etab.new_const(weight, dtype=str(weight.dtype)), - 'kernel_size': [kernel_h, kernel_w], - 'strides': [stride_h, stride_w], - 'dilation': dilation, - 'padding': [0, 0], - 'data_layout': etab.data_layout, - 'kernel_layout': kernel_layout} + params = { + "weight": etab.new_const(weight, dtype=str(weight.dtype)), + "kernel_size": [kernel_h, kernel_w], + "strides": [stride_h, stride_w], + "dilation": dilation, + "padding": [0, 0], + "data_layout": etab.data_layout, + "kernel_layout": kernel_layout, + } if is_depthconv: - params['channels'] = in_channels * depth_mult - params['groups'] = in_channels + params["channels"] = in_channels * depth_mult + params["groups"] = in_channels else: - params['channels'] = n_filters - if keras_layer.padding == 'valid': + params["channels"] = n_filters + if keras_layer.padding == "valid": pass # we insert a separate pad operator - elif keras_layer.padding == 'same': + elif keras_layer.padding == "same": in_h = keras_layer.input_shape[1] in_w = keras_layer.input_shape[2] pad_t, pad_b = _get_pad_pair(in_h, dilated_kernel_h, stride_h) pad_l, pad_r = _get_pad_pair(in_w, dilated_kernel_w, stride_w) - params['padding'] = (pad_t, pad_l, pad_b, pad_r) + params["padding"] = (pad_t, pad_l, pad_b, pad_r) else: - msg = 'Padding with {} is not supported for operator Convolution ' \ - 'in frontend Keras.' + msg = ( + "Padding with {} is not supported for operator Convolution " + "in frontend Keras." + ) raise tvm.error.OpAttributeUnImplemented(msg.format(keras_layer.padding)) if is_deconv: out = _op.nn.conv2d_transpose(data=inexpr, **params) @@ -354,7 +399,7 @@ def _convert_convolution(inexpr, keras_layer, etab): if keras_layer.use_bias: bias = etab.new_const(weightList[1]) - if etab.data_layout == 'NCHW': + if etab.data_layout == "NCHW": out = _op.nn.bias_add(out, bias) else: out = _op.nn.bias_add(out, bias, axis=-1) @@ -363,21 +408,24 @@ def _convert_convolution(inexpr, keras_layer, etab): act_type = keras_layer.activation.func_name else: act_type = keras_layer.activation.__name__ - if act_type != 'linear': + if act_type != "linear": out = _convert_activation(out, act_type, etab) return out + def _convert_convolution3d(inexpr, keras_layer, etab): _check_data_format(keras_layer) weightList = keras_layer.get_weights() weight = weightList[0] - if etab.data_layout == 'NDHWC': - kernel_layout = 'DHWIO' + if etab.data_layout == "NDHWC": + kernel_layout = "DHWIO" else: - kernel_layout = 'OIDHW' - msg = 'Kernel layout with {} is not supported for operator Convolution3D ' \ - 'in frontend Keras.' + kernel_layout = "OIDHW" + msg = ( + "Kernel layout with {} is not supported for operator Convolution3D " + "in frontend Keras." + ) raise tvm.error.OpAttributeUnImplemented(msg.format(etab.data_layout)) dilation_rate = keras_layer.dilation_rate @@ -396,29 +444,39 @@ def _convert_convolution3d(inexpr, keras_layer, etab): dilated_kernel_d2 = (kernel_d2 - 1) * dilation[1] + 1 dilated_kernel_d3 = (kernel_d3 - 1) * dilation[2] + 1 stride_d1, stride_d2, stride_d3 = keras_layer.strides - params = {'weight': etab.new_const(weight), - 'kernel_size': [kernel_d1, kernel_d2, kernel_d3], - 'strides': [stride_d1, stride_d2, stride_d3], - 'dilation': dilation, - 'padding': [0, 0, 0], - 'data_layout': etab.data_layout, - 'kernel_layout': kernel_layout} - params['channels'] = n_filters - - if keras_layer.padding == 'valid': + params = { + "weight": etab.new_const(weight), + "kernel_size": [kernel_d1, kernel_d2, kernel_d3], + "strides": [stride_d1, stride_d2, stride_d3], + "dilation": dilation, + "padding": [0, 0, 0], + "data_layout": etab.data_layout, + "kernel_layout": kernel_layout, + "channels": n_filters, + } + if keras_layer.padding == "valid": pass # calculate the padding values - elif keras_layer.padding == 'same': + elif keras_layer.padding == "same": in_d1 = keras_layer.input_shape[1] in_d2 = keras_layer.input_shape[2] in_d3 = keras_layer.input_shape[3] pad_d1 = _get_pad_pair(in_d1, dilated_kernel_d1, stride_d1) pad_d2 = _get_pad_pair(in_d2, dilated_kernel_d2, stride_d2) pad_d3 = _get_pad_pair(in_d3, dilated_kernel_d3, stride_d3) - params['padding'] = [pad_d1[0], pad_d2[0], pad_d3[0], pad_d1[1], pad_d2[1], pad_d3[1]] + params["padding"] = [ + pad_d1[0], + pad_d2[0], + pad_d3[0], + pad_d1[1], + pad_d2[1], + pad_d3[1], + ] else: - msg = 'Padding with {} is not supported for operator Convolution ' \ - 'in frontend Keras.' + msg = ( + "Padding with {} is not supported for operator Convolution " + "in frontend Keras." + ) raise tvm.error.OpAttributeUnImplemented(msg.format(keras_layer.padding)) out = _op.nn.conv3d(data=inexpr, **params) @@ -432,65 +490,69 @@ def _convert_convolution3d(inexpr, keras_layer, etab): act_type = keras_layer.activation.func_name else: act_type = keras_layer.activation.__name__ - if act_type != 'linear': + if act_type != "linear": out = _convert_activation(out, act_type, etab) return out + def _convert_separable_convolution(inexpr, keras_layer, etab): _check_data_format(keras_layer) - if etab.data_layout == 'NHWC': - kernel_layout = 'HWOI' - else: - kernel_layout = 'OIHW' + kernel_layout = "HWOI" if etab.data_layout == "NHWC" else "OIHW" weightList = keras_layer.get_weights() # depthwise conv kernel_h, kernel_w, in_channels, depth_mult = weightList[0].shape stride_h, stride_w = keras_layer.strides - if kernel_layout == 'OIHW': + if kernel_layout == "OIHW": weight0 = weightList[0].transpose([2, 3, 0, 1]) else: weight0 = weightList[0] - params0 = {'weight': etab.new_const(weight0), - 'channels': in_channels * depth_mult, - 'groups': in_channels, - 'kernel_size': [kernel_h, kernel_w], - 'strides': [stride_h, stride_w], - 'dilation': [1, 1], - 'padding': [0, 0], - 'data_layout': etab.data_layout, - 'kernel_layout': kernel_layout} - if keras_layer.padding == 'valid': + params0 = { + "weight": etab.new_const(weight0), + "channels": in_channels * depth_mult, + "groups": in_channels, + "kernel_size": [kernel_h, kernel_w], + "strides": [stride_h, stride_w], + "dilation": [1, 1], + "padding": [0, 0], + "data_layout": etab.data_layout, + "kernel_layout": kernel_layout, + } + if keras_layer.padding == "valid": pass # we insert a separate pad operator - elif keras_layer.padding == 'same': + elif keras_layer.padding == "same": in_h = keras_layer.input_shape[1] in_w = keras_layer.input_shape[2] pad_t, pad_b = _get_pad_pair(in_h, kernel_h, stride_h) pad_l, pad_r = _get_pad_pair(in_w, kernel_w, stride_w) - params0['padding'] = (pad_t, pad_l, pad_b, pad_r) + params0["padding"] = (pad_t, pad_l, pad_b, pad_r) else: - msg = 'Padding with {} is not supported for operator Separable ' \ - 'Convolution in frontend Keras.' + msg = ( + "Padding with {} is not supported for operator Separable " + "Convolution in frontend Keras." + ) raise tvm.error.OpAttributeUnImplemented(msg.format(keras_layer.padding)) depthconv = _op.nn.conv2d(data=inexpr, **params0) # pointwise conv - if kernel_layout == 'OIHW': + if kernel_layout == "OIHW": weight1 = weightList[1].transpose([3, 2, 0, 1]) else: weight1 = weightList[1] kernel_layout = "HWIO" - params1 = {'weight': etab.new_const(weight1), - 'channels': weightList[1].shape[3], - 'groups': 1, - 'kernel_size': [1, 1], - 'strides': [1, 1], - 'dilation': [1, 1], - 'data_layout': etab.data_layout, - 'kernel_layout': kernel_layout} + params1 = { + "weight": etab.new_const(weight1), + "channels": weightList[1].shape[3], + "groups": 1, + "kernel_size": [1, 1], + "strides": [1, 1], + "dilation": [1, 1], + "data_layout": etab.data_layout, + "kernel_layout": kernel_layout, + } out = _op.nn.conv2d(data=depthconv, **params1) if keras_layer.use_bias: bias = etab.new_const(weightList[2]) - if etab.data_layout == 'NCHW': + if etab.data_layout == "NCHW": out = _op.nn.bias_add(out, bias) else: out = _op.nn.bias_add(out, bias, axis=-1) @@ -499,7 +561,7 @@ def _convert_separable_convolution(inexpr, keras_layer, etab): act_type = keras_layer.activation.func_name else: act_type = keras_layer.activation.__name__ - if act_type != 'linear': + if act_type != "linear": out = _convert_activation(out, act_type, etab) return out @@ -507,7 +569,7 @@ def _convert_separable_convolution(inexpr, keras_layer, etab): def _convert_flatten(inexpr, keras_layer, etab): _check_data_format(keras_layer) # NCHW -> NHWC so that dense can be correctly converted - if etab.data_layout == 'NCHW': + if etab.data_layout == "NCHW": inexpr = _op.transpose(inexpr, axes=[0, 2, 3, 1]) return _op.nn.batch_flatten(inexpr) @@ -516,161 +578,187 @@ def _convert_pooling(inexpr, keras_layer, etab): _check_data_format(keras_layer) pool_type = type(keras_layer).__name__ # global pool in keras = global pool + flatten in relay - global_pool_params = {'layout': etab.data_layout} - if pool_type == 'GlobalMaxPooling2D': + global_pool_params = {"layout": etab.data_layout} + if pool_type == "GlobalMaxPooling2D": return _convert_flatten( - _op.nn.global_max_pool2d(inexpr, **global_pool_params), keras_layer, etab) - if pool_type == 'GlobalAveragePooling2D': + _op.nn.global_max_pool2d(inexpr, **global_pool_params), keras_layer, etab + ) + if pool_type == "GlobalAveragePooling2D": return _convert_flatten( - _op.nn.global_avg_pool2d(inexpr, **global_pool_params), keras_layer, etab) + _op.nn.global_avg_pool2d(inexpr, **global_pool_params), keras_layer, etab + ) pool_h, pool_w = keras_layer.pool_size stride_h, stride_w = keras_layer.strides - params = {'pool_size': [pool_h, pool_w], - 'strides': [stride_h, stride_w], - 'padding': [0, 0], - 'layout': etab.data_layout} - if keras_layer.padding == 'valid': + params = { + "pool_size": [pool_h, pool_w], + "strides": [stride_h, stride_w], + "padding": [0, 0], + "layout": etab.data_layout, + } + if keras_layer.padding == "valid": pass - elif keras_layer.padding == 'same': + elif keras_layer.padding == "same": in_h = keras_layer.input_shape[1] in_w = keras_layer.input_shape[2] pad_t, pad_b = _get_pad_pair(in_h, pool_h, stride_h) pad_l, pad_r = _get_pad_pair(in_w, pool_w, stride_w) - params['padding'] = [pad_t, pad_l, pad_b, pad_r] + params["padding"] = [pad_t, pad_l, pad_b, pad_r] else: raise tvm.error.OpAttributeUnImplemented( - 'Padding with {} is not supported in operator Pooling.'.format(keras_layer.padding)) - if pool_type == 'MaxPooling2D': + f"Padding with {keras_layer.padding} is not supported in operator Pooling." + ) + if pool_type == "MaxPooling2D": return _op.nn.max_pool2d(inexpr, **params) - if pool_type == 'AveragePooling2D': - params['count_include_pad'] = False + if pool_type == "AveragePooling2D": + params["count_include_pad"] = False return _op.nn.avg_pool2d(inexpr, **params) raise tvm.error.OpNotImplemented( - 'Operator {} is not supported for frontend Keras.'.format(keras_layer)) + f"Operator {keras_layer} is not supported for frontend Keras." + ) + def _convert_pooling3d(inexpr, keras_layer, etab): _check_data_format(keras_layer) pool_type = type(keras_layer).__name__ - if pool_type not in ['MaxPooling3D', 'AveragePooling3D']: + if pool_type not in ["MaxPooling3D", "AveragePooling3D"]: raise tvm.error.OpNotImplemented( - 'Operator {} is not supported for frontend Keras.'.format(keras_layer)) + f"Operator {keras_layer} is not supported for frontend Keras." + ) pool_d1, pool_d2, pool_d3 = keras_layer.pool_size stride_d1, stride_d2, stride_d3 = keras_layer.strides - params = {'pool_size': [pool_d1, pool_d2, pool_d3], - 'strides': [stride_d1, stride_d2, stride_d3], - 'padding': [0, 0, 0], - 'layout': etab.data_layout} - - if keras_layer.padding == 'valid': + params = { + "pool_size": [pool_d1, pool_d2, pool_d3], + "strides": [stride_d1, stride_d2, stride_d3], + "padding": [0, 0, 0], + "layout": etab.data_layout, + } + + if keras_layer.padding == "valid": pass - elif keras_layer.padding == 'same': + elif keras_layer.padding == "same": in_d1 = keras_layer.input_shape[1] in_d2 = keras_layer.input_shape[2] in_d3 = keras_layer.input_shape[3] pad_d1 = _get_pad_pair(in_d1, pool_d1, stride_d1) pad_d2 = _get_pad_pair(in_d2, pool_d2, stride_d2) pad_d3 = _get_pad_pair(in_d3, pool_d3, stride_d3) - params['padding'] = [pad_d1[0], pad_d2[0], pad_d3[0], pad_d1[1], pad_d2[1], pad_d3[1]] + params["padding"] = [ + pad_d1[0], + pad_d2[0], + pad_d3[0], + pad_d1[1], + pad_d2[1], + pad_d3[1], + ] else: raise tvm.error.OpAttributeUnImplemented( - 'Padding with {} is not supported in operator Pooling3D.'.format(keras_layer.padding)) + f"Padding with {keras_layer.padding} is not supported in operator Pooling3D." + ) out = _op.transpose(inexpr, axes=(0, 4, 1, 2, 3)) - params['layout'] = "NCDHW" - if pool_type == 'MaxPooling3D': + params["layout"] = "NCDHW" + if pool_type == "MaxPooling3D": out = _op.nn.max_pool3d(out, **params) - elif pool_type == 'AveragePooling3D': + elif pool_type == "AveragePooling3D": out = _op.nn.avg_pool3d(out, **params) return _op.transpose(out, axes=(0, 2, 3, 4, 1)) + def _convert_upsample(inexpr, keras_layer, etab): _check_data_format(keras_layer) upsample_type = type(keras_layer).__name__ params = {} - if upsample_type == 'UpSampling1D': + if upsample_type == "UpSampling1D": h = keras_layer.size - params['scale_h'] = h - elif upsample_type == 'UpSampling2D': + params["scale_h"] = h + elif upsample_type == "UpSampling2D": h, w = keras_layer.size if h != w: raise tvm.error.OpAttributeInvalid( - 'Height must equal width for operator Upsample.') - params['scale_h'] = h - params['scale_w'] = h + "Height must equal width for operator Upsample." + ) + params["scale_h"] = h + params["scale_w"] = h - if hasattr(keras_layer, 'interpolation'): + if hasattr(keras_layer, "interpolation"): interpolation = keras_layer.interpolation - if interpolation == 'nearest': - params['method'] = 'nearest_neighbor' + if interpolation == "nearest": + params["method"] = "nearest_neighbor" else: - params['method'] = 'bilinear' + params["method"] = "bilinear" else: raise tvm.error.OpNotImplemented( - 'Operator {} is not supported for frontend Keras.'.format(upsample_type)) - params['layout'] = etab.data_layout - out = _op.nn.upsampling(inexpr, **params) - return out + f"Operator {upsample_type} is not supported for frontend Keras." + ) + params["layout"] = etab.data_layout + return _op.nn.upsampling(inexpr, **params) def _convert_upsample3d(inexpr, keras_layer, etab): _check_data_format(keras_layer) - params = {} d, h, w = keras_layer.size - params['scale_d'] = d - params['scale_h'] = h - params['scale_w'] = w - params['layout'] = etab.data_layout - out = _op.nn.upsampling3d(inexpr, **params) - return out + params = {"scale_d": d, "scale_h": h, "scale_w": w, "layout": etab.data_layout} + return _op.nn.upsampling3d(inexpr, **params) def _convert_cropping(inexpr, keras_layer, _): _check_data_format(keras_layer) crop_type = type(keras_layer).__name__ - if crop_type == 'Cropping2D': - (_, in_h, in_w, _) = keras_layer.input_shape - ((crop_t, crop_b), (crop_l, crop_r)) = keras_layer.cropping - else: + if crop_type != "Cropping2D": raise tvm.error.OpNotImplemented( - 'Operator {} is not supported for frontend Keras.'.format(crop_type)) + f"Operator {crop_type} is not supported for frontend Keras." + ) + (_, in_h, in_w, _) = keras_layer.input_shape + ((crop_t, crop_b), (crop_l, crop_r)) = keras_layer.cropping int32_max = np.iinfo(np.int32).max - return _op.strided_slice(inexpr, begin=[0, 0, crop_t, crop_l], \ - end=[int32_max, int32_max, in_h-crop_b, in_w-crop_r]) + return _op.strided_slice( + inexpr, + begin=[0, 0, crop_t, crop_l], + end=[int32_max, int32_max, in_h - crop_b, in_w - crop_r], + ) def _convert_batchnorm(inexpr, keras_layer, etab): - if etab.data_layout == 'NCHW' or len(keras_layer.input_shape) < 4: + if etab.data_layout == "NCHW" or len(keras_layer.input_shape) < 4: axis = 1 else: axis = 3 - params = {'scale': False, - 'center': False, - 'epsilon': keras_layer.epsilon, - 'axis': axis} + params = { + "scale": False, + "center": False, + "epsilon": keras_layer.epsilon, + "axis": axis, + } idx = 0 if keras_layer.scale: - params['scale'] = True + params["scale"] = True gamma = keras_layer.get_weights()[idx] - params['gamma'] = etab.new_const(gamma) + params["gamma"] = etab.new_const(gamma) idx += 1 if keras_layer.center: - params['center'] = True + params["center"] = True beta = keras_layer.get_weights()[idx] - params['beta'] = etab.new_const(beta) + params["beta"] = etab.new_const(beta) idx += 1 moving_mean = keras_layer.get_weights()[idx] moving_var = keras_layer.get_weights()[idx + 1] - params['moving_mean'] = etab.new_const(moving_mean) - params['moving_var'] = etab.new_const(moving_var) + params["moving_mean"] = etab.new_const(moving_mean) + params["moving_var"] = etab.new_const(moving_var) # in case beta or gamma is not defined - params['beta'] = etab.new_const(np.zeros(moving_mean.shape)) if \ - 'beta' not in params else params['beta'] - params['gamma'] = etab.new_const(np.ones(moving_mean.shape)) if \ - 'gamma' not in params else params['gamma'] + params["beta"] = ( + etab.new_const(np.zeros(moving_mean.shape)) + if "beta" not in params + else params["beta"] + ) + params["gamma"] = ( + etab.new_const(np.ones(moving_mean.shape)) + if "gamma" not in params + else params["gamma"] + ) result, moving_mean, moving_var = _op.nn.batch_norm(inexpr, **params) return result @@ -680,30 +768,44 @@ def _convert_padding(inexpr, keras_layer, etab): padding_type = type(keras_layer).__name__ padding = keras_layer.padding top = left = bottom = right = 0 - if padding_type == 'ZeroPadding2D': - if isinstance(padding, int): - top = left = bottom = right = padding - elif isinstance(padding, tuple): - if isinstance(padding[0], int): - top, left = padding - bottom, right = padding - elif isinstance(padding[0], tuple): - top, bottom = padding[0] - left, right = padding[1] - else: - msg = 'Value {} in attribute "padding" of operator Padding ' \ - 'is not valid.' - raise tvm.error.OpAttributeInvalid(msg.format(str(padding))) - else: - msg = 'Value {} in attribute "padding" of operator Padding is ' \ - 'not valid.' + if padding_type == "ZeroPadding2D": + if ( + not isinstance(padding, int) + and isinstance(padding, tuple) + and isinstance(padding[0], int) + ): + top, left = padding + bottom, right = padding + elif ( + not isinstance(padding, int) + and isinstance(padding, tuple) + and isinstance(padding[0], tuple) + ): + top, bottom = padding[0] + left, right = padding[1] + elif ( + not isinstance(padding, int) + and isinstance(padding, tuple) + or not isinstance(padding, int) + ): + msg = ( + 'Value {} in attribute "padding" of operator Padding ' + "is not valid." + ) raise tvm.error.OpAttributeInvalid(msg.format(str(padding))) + else: + top = left = bottom = right = padding else: - msg = 'Operator {} is not supported in frontend Keras.' + msg = "Operator {} is not supported in frontend Keras." raise tvm.error.OpNotImplemented(msg.format(padding_type)) - if etab.data_layout == 'NCHW': - return _op.nn.pad(data=inexpr, pad_width=((0, 0), (0, 0), (top, bottom), (left, right))) - return _op.nn.pad(data=inexpr, pad_width=((0, 0), (top, bottom), (left, right), (0, 0))) + if etab.data_layout == "NCHW": + return _op.nn.pad( + data=inexpr, pad_width=((0, 0), (0, 0), (top, bottom), (left, right)) + ) + return _op.nn.pad( + data=inexpr, pad_width=((0, 0), (top, bottom), (left, right), (0, 0)) + ) + def _convert_padding3d(inexpr, keras_layer, etab): _check_data_format(keras_layer) @@ -719,26 +821,39 @@ def _convert_padding3d(inexpr, keras_layer, etab): h_pad = padding[1] w_pad = padding[2] else: - msg = 'Value {} in attribute "padding" of operator ZeroPadding3D is ' \ - 'not valid.' + msg = ( + 'Value {} in attribute "padding" of operator ZeroPadding3D is ' "not valid." + ) raise tvm.error.OpAttributeInvalid(msg.format(str(padding))) - if etab.data_layout == 'NCDHW': - out = _op.nn.pad(data=inexpr, pad_width=((0, 0), (0, 0), - (d_pad[0], d_pad[1]), - (h_pad[0], h_pad[1]), - (w_pad[0], w_pad[1]))) - else: - out = _op.nn.pad(data=inexpr, pad_width=((0, 0), - (d_pad[0], d_pad[1]), - (h_pad[0], h_pad[1]), - (w_pad[0], w_pad[1]), - (0, 0))) - return out + return ( + _op.nn.pad( + data=inexpr, + pad_width=( + (0, 0), + (0, 0), + (d_pad[0], d_pad[1]), + (h_pad[0], h_pad[1]), + (w_pad[0], w_pad[1]), + ), + ) + if etab.data_layout == "NCDHW" + else _op.nn.pad( + data=inexpr, + pad_width=( + (0, 0), + (d_pad[0], d_pad[1]), + (h_pad[0], h_pad[1]), + (w_pad[0], w_pad[1]), + (0, 0), + ), + ) + ) + def _convert_concat(inexpr, keras_layer, etab): _check_data_format(keras_layer) - if etab.data_layout == 'NHWC' or len(keras_layer.input_shape[0]) < 4: + if etab.data_layout == "NHWC" or len(keras_layer.input_shape[0]) < 4: axis = -1 else: axis = 1 @@ -747,26 +862,28 @@ def _convert_concat(inexpr, keras_layer, etab): def _convert_reshape(inexpr, keras_layer, etab): _check_data_format(keras_layer) - inshape = keras_layer.input_shape # includes batch - tshape = keras_layer.target_shape # no batch + inshape = keras_layer.input_shape # includes batch + tshape = keras_layer.target_shape # no batch if len(inshape) == 3 and len(tshape) == 1: # (?, a, b) -> (-1, ab) shape = (-1, tshape[0]) - elif len(inshape) in [2, 3] and len(tshape) == 2: + elif len(inshape) in {2, 3} and len(tshape) == 2: # (?, cc) -> (-1, c, c) # (?, a, b) -> (-1, c, c) - assert tshape[0] == tshape[1], \ - "Only supports square target shapes, but got {}".format(tshape) - shape = (-1, ) + tshape + assert ( + tshape[0] == tshape[1] + ), f"Only supports square target shapes, but got {tshape}" + shape = (-1,) + tshape else: # (?, h, w, c) -> (-1, c, H, W) # (?, h, w, c) -> (-1, c, hw) # (?, hw, c) -> (-1, c, h, w) ch = inshape[-1] - assert ch == tshape[-1], \ - "Only supports last dimension in target shape being equal to " \ + assert ch == tshape[-1], ( + "Only supports last dimension in target shape being equal to " "the channel number of input tensor." - if etab.data_layout == 'NCHW': + ) + if etab.data_layout == "NCHW": shape = (-1, ch) + tshape[:-1] else: shape = (-1,) + tshape[:-1] + (ch,) @@ -776,7 +893,7 @@ def _convert_reshape(inexpr, keras_layer, etab): def _convert_lstm(inexpr, keras_layer, etab): _check_data_format(keras_layer) if not isinstance(inexpr, list): - buf = np.zeros((1, keras_layer.units), 'float32') + buf = np.zeros((1, keras_layer.units), "float32") c_op = etab.new_const(buf) h_op = etab.new_const(buf) inexpr = [inexpr, h_op, c_op] @@ -784,7 +901,7 @@ def _convert_lstm(inexpr, keras_layer, etab): next_h = inexpr[1] next_c = inexpr[2] weightList = keras_layer.get_weights() - in_shape = tuple(dim if dim else 1 for dim in _as_list(keras_layer.input_shape)[0]) + in_shape = tuple(dim or 1 for dim in _as_list(keras_layer.input_shape)[0]) kernel_weight = etab.new_const(weightList[0].transpose([1, 0])) recurrent_weight = etab.new_const(weightList[1].transpose([1, 0])) in_bias = etab.new_const(weightList[2]) @@ -795,15 +912,19 @@ def _convert_lstm(inexpr, keras_layer, etab): # loop for the number of time_steps for data in in_data: ixh1 = _op.nn.dense(data, kernel_weight, units=units) - ixh2 = _op.nn.bias_add(_op.nn.dense(next_h, recurrent_weight, units=units), bias=in_bias) + ixh2 = _op.nn.bias_add( + _op.nn.dense(next_h, recurrent_weight, units=units), bias=in_bias + ) gate = ixh1 + ixh2 gates = _op.split(gate, indices_or_sections=4, axis=1) in_gate = _convert_recurrent_activation(gates[0], keras_layer) in_transform = _convert_recurrent_activation(gates[1], keras_layer) - next_c = in_transform * next_c + in_gate * _convert_activation(gates[2], keras_layer, None) + next_c = in_transform * next_c + in_gate * _convert_activation( + gates[2], keras_layer, None + ) out_gate = _convert_recurrent_activation(gates[3], keras_layer) next_h = out_gate * _convert_activation(next_c, keras_layer, None) - out_shape = tuple(dim if dim else 1 for dim in _as_list(keras_layer.output_shape)[0]) + out_shape = tuple(dim or 1 for dim in _as_list(keras_layer.output_shape)[0]) out = _op.reshape(next_h, newshape=out_shape) return [out, next_h, next_c] @@ -811,7 +932,7 @@ def _convert_lstm(inexpr, keras_layer, etab): def _convert_simple_rnn(inexpr, keras_layer, etab): _check_data_format(keras_layer) if not isinstance(inexpr, list): - buf = np.zeros((1, keras_layer.units), 'float32') + buf = np.zeros((1, keras_layer.units), "float32") prev_op = etab.new_const(buf) inexpr = [inexpr, prev_op] in_data = inexpr[0] @@ -822,12 +943,16 @@ def _convert_simple_rnn(inexpr, keras_layer, etab): in_bias = etab.new_const(weightList[2]) units = list(weightList[0].shape)[1] in_data = _op.nn.batch_flatten(in_data) - ixh = _op.nn.bias_add(_op.nn.dense(in_data, kernel_weight, units=units), bias=in_bias) + ixh = _op.nn.bias_add( + _op.nn.dense(in_data, kernel_weight, units=units), bias=in_bias + ) prev_op = _op.nn.batch_flatten(prev_op) ixh2 = _op.nn.dense(prev_op, recurrent_weight, units=units) output = ixh + ixh2 output = _convert_activation(output, keras_layer, None) - out_shape = tuple(dim if dim else 1 for dim in _as_list(keras_layer.output_shape)[0]) + out_shape = tuple( + dim or 1 for dim in _as_list(keras_layer.output_shape)[0] + ) output = _op.reshape(output, newshape=out_shape) return [output, output] @@ -835,7 +960,7 @@ def _convert_simple_rnn(inexpr, keras_layer, etab): def _convert_gru(inexpr, keras_layer, etab): _check_data_format(keras_layer) if not isinstance(inexpr, list): - buf = np.zeros((1, keras_layer.units), 'float32') + buf = np.zeros((1, keras_layer.units), "float32") h_tm1 = etab.new_const(buf) inexpr = [inexpr, h_tm1] in_data = inexpr[0] @@ -846,7 +971,9 @@ def _convert_gru(inexpr, keras_layer, etab): in_bias = etab.new_const(weightList[2]) units = list(weightList[0].shape)[1] in_data = _op.nn.batch_flatten(in_data) - matrix_x = _op.nn.bias_add(_op.nn.dense(in_data, kernel_weight, units=units), in_bias) + matrix_x = _op.nn.bias_add( + _op.nn.dense(in_data, kernel_weight, units=units), in_bias + ) # inputs projected by all gate matrices at once split_indices = [keras_layer.units, 2 * keras_layer.units] gates = _op.split(matrix_x, indices_or_sections=split_indices, axis=1) @@ -869,54 +996,53 @@ def _convert_gru(inexpr, keras_layer, etab): recurrent_h = _op.nn.dense(rec_act_r * h_tm1_op, rec_weights[1], units=units) act_hh = _convert_activation(x_h + recurrent_h, keras_layer, None) # previous and candidate state mixed by update gate - output = rec_act_z * h_tm1_op + (_expr.const(1., dtype='float32') - rec_act_z) * act_hh - out_shape = tuple(dim if dim else 1 for dim in _as_list(keras_layer.output_shape)[0]) + output = ( + rec_act_z * h_tm1_op + (_expr.const(1.0, dtype="float32") - rec_act_z) * act_hh + ) + out_shape = tuple( + dim or 1 for dim in _as_list(keras_layer.output_shape)[0] + ) output = _op.reshape(output, newshape=out_shape) return [output, output] -def _default_skip(inexpr, keras_layer, _): # pylint: disable=unused-argument +def _default_skip(inexpr, keras_layer, _): # pylint: disable=unused-argument """Layers that can be skipped because they are train time only.""" return inexpr _convert_map = { - 'TSM': _convert_tsm, - 'Attention_mask': _convert_attention_mask, - 'Expand_dim': _convert_expand_dim, - 'Dense' : _convert_dense, - 'Activation' : _convert_activation, - 'Softmax' : _convert_advanced_activation, - 'ReLU' : _convert_advanced_activation, - 'LeakyReLU' : _convert_advanced_activation, - 'PReLU' : _convert_advanced_activation, - 'ELU' : _convert_advanced_activation, - 'ThresholdedReLU' : _convert_advanced_activation, - - 'AveragePooling2D' : _convert_pooling, - 'MaxPooling2D' : _convert_pooling, - 'GlobalAveragePooling2D' : _convert_pooling, - 'GlobalMaxPooling2D' : _convert_pooling, - 'Conv2D' : _convert_convolution, - 'Conv2DTranspose' : _convert_convolution, - 'DepthwiseConv2D' : _convert_convolution, - 'SeparableConv2D' : _convert_separable_convolution, - - 'Flatten' : _convert_flatten, - 'Reshape' : _convert_reshape, - 'Concatenate' : _convert_concat, - 'BatchNormalization' : _convert_batchnorm, - + "TSM": _convert_tsm, + "Attention_mask": _convert_attention_mask, + "Expand_dim": _convert_expand_dim, + "Dense": _convert_dense, + "Activation": _convert_activation, + "Softmax": _convert_advanced_activation, + "ReLU": _convert_advanced_activation, + "LeakyReLU": _convert_advanced_activation, + "PReLU": _convert_advanced_activation, + "ELU": _convert_advanced_activation, + "ThresholdedReLU": _convert_advanced_activation, + "AveragePooling2D": _convert_pooling, + "MaxPooling2D": _convert_pooling, + "GlobalAveragePooling2D": _convert_pooling, + "GlobalMaxPooling2D": _convert_pooling, + "Conv2D": _convert_convolution, + "Conv2DTranspose": _convert_convolution, + "DepthwiseConv2D": _convert_convolution, + "SeparableConv2D": _convert_separable_convolution, + "Flatten": _convert_flatten, + "Reshape": _convert_reshape, + "Concatenate": _convert_concat, + "BatchNormalization": _convert_batchnorm, # Specific tf.Keras terminology for batch normalization - 'BatchNormalizationV1' : _convert_batchnorm, - - 'Add' : _convert_merge, - 'Subtract' : _convert_merge, - 'Multiply' : _convert_merge, - 'ZeroPadding2D' : _convert_padding, - 'UpSampling2D' : _convert_upsample, - 'Cropping2D' : _convert_cropping, - + "BatchNormalizationV1": _convert_batchnorm, + "Add": _convert_merge, + "Subtract": _convert_merge, + "Multiply": _convert_merge, + "ZeroPadding2D": _convert_padding, + "UpSampling2D": _convert_upsample, + "Cropping2D": _convert_cropping, # 'ZeroPadding1D' : _convert_padding, # 'AveragePooling1D' : _convert_pooling, # 'MaxPooling1D' : _convert_pooling, @@ -925,38 +1051,34 @@ def _default_skip(inexpr, keras_layer, _): # pylint: disable=unused-argument # 'Cropping1D' : _convert_cropping, # 'UpSampling1D' : _convert_upsample, # 'Conv1D' : _convert_convolution1d, - - 'Conv3D' : _convert_convolution3d, + "Conv3D": _convert_convolution3d, # 'Conv3DTranspose' : _convert_convolution3d, # 'SeparableConv3D' : _convert_convolution3d, - 'MaxPooling3D' : _convert_pooling3d, - 'AveragePooling3D' : _convert_pooling3d, + "MaxPooling3D": _convert_pooling3d, + "AveragePooling3D": _convert_pooling3d, # 'GlobalMaxPooling3D' : _convert_pooling3d, # 'GlobalAveragePooling3D' : _convert_pooling3d, - 'UpSampling3D' : _convert_upsample3d, - 'ZeroPadding3D' : _convert_padding3d, - - 'SimpleRNN' : _convert_simple_rnn, - 'LSTM' : _convert_lstm, - 'GRU' : _convert_gru, + "UpSampling3D": _convert_upsample3d, + "ZeroPadding3D": _convert_padding3d, + "SimpleRNN": _convert_simple_rnn, + "LSTM": _convert_lstm, + "GRU": _convert_gru, # 'Bidirectional' : _convert_bidirectional, # 'TimeDistributed' : _default_skip, - - 'Average' : _convert_merge, - 'Minimum' : _convert_merge, - 'Maximum' : _convert_merge, - 'Dot' : _convert_merge, - 'Permute' : _convert_permute, - 'Embedding' : _convert_embedding, + "Average": _convert_merge, + "Minimum": _convert_merge, + "Maximum": _convert_merge, + "Dot": _convert_merge, + "Permute": _convert_permute, + "Embedding": _convert_embedding, # 'RepeatVector' : _convert_repeat_vector, - - 'InputLayer' : _default_skip, - 'Dropout' : _default_skip, - 'AlphaDropout' : _default_skip, - 'SpatialDropout2D' : _default_skip, - 'SpatialDropout1D' : _default_skip, - 'GaussianDropout' : _default_skip, - 'GaussianNoise' : _default_skip, + "InputLayer": _default_skip, + "Dropout": _default_skip, + "AlphaDropout": _default_skip, + "SpatialDropout2D": _default_skip, + "SpatialDropout1D": _default_skip, + "GaussianDropout": _default_skip, + "GaussianNoise": _default_skip, } @@ -968,8 +1090,9 @@ def _check_unsupported_layers(model): missing_ops.add(op_name) if missing_ops: - raise NotImplementedError( \ - "The following operators are not implemented: {}".format(missing_ops)) + raise NotImplementedError( + f"The following operators are not implemented: {missing_ops}" + ) def keras_op_to_relay(inexpr, keras_layer, outname, etab): @@ -992,15 +1115,16 @@ def keras_op_to_relay(inexpr, keras_layer, outname, etab): op_name = type(keras_layer).__name__ if op_name not in _convert_map: raise tvm.error.OpNotImplemented( - 'Operator {} is not supported for frontend Keras.'.format(op_name)) + f"Operator {op_name} is not supported for frontend Keras." + ) outs = _convert_map[op_name](inexpr, keras_layer, etab) outs = _as_list(outs) for t_idx, out in enumerate(outs): - name = outname + ":" + str(t_idx) + name = f"{outname}:{str(t_idx)}" etab.set_expr(name, out) -def from_keras(model, shape=None, layout='NCHW', dtype='float32'): +def from_keras(model, shape=None, layout="NCHW", dtype="float32"): """Convert keras model to relay Function. Parameters @@ -1024,14 +1148,15 @@ def from_keras(model, shape=None, layout='NCHW', dtype='float32'): params : dict of str to tvm.nd.NDArray The parameter dict to be used by Relay. """ + def _check_model_is_tf_keras(): return type(model).__module__.startswith("tensorflow.python.keras") def _convert_input_layer(keras_layer): - if len(shape) == 2: # For mix model + if len(shape) == 2: # For mix model input_name = keras_layer.name - if input_name == 'app': + if input_name == "app": var = new_var(input_name, shape=shape[0], dtype=dtype) etab.set_expr(input_name, var) else: @@ -1052,10 +1177,14 @@ def _convert_input_layer(keras_layer): import keras except ImportError: raise ImportError("Keras must be installed") - if keras.backend.backend() != 'tensorflow': - raise ValueError("Keras frontend currently supports tensorflow backend only.") - if keras.backend.image_data_format() != 'channels_last': - raise ValueError("Keras frontend currently supports data_format = channels_last only.") + if keras.backend.backend() != "tensorflow": + raise ValueError( + "Keras frontend currently supports tensorflow backend only." + ) + if keras.backend.image_data_format() != "channels_last": + raise ValueError( + "Keras frontend currently supports data_format = channels_last only." + ) expected_model_class = keras.engine.training.Model input_layer_class = keras.engine.InputLayer else: @@ -1071,7 +1200,11 @@ def _convert_input_layer(keras_layer): etab = ExprTable() # Set global data format. - assert layout in ['NCHW', 'NHWC', 'NDHWC'], "Layout must be one of 'NCHW', NHWC or NDHWC" + assert layout in [ + "NCHW", + "NHWC", + "NDHWC", + ], "Layout must be one of 'NCHW', NHWC or NDHWC" etab.data_layout = layout for keras_layer in model.layers: if isinstance(keras_layer, input_layer_class): @@ -1081,25 +1214,41 @@ def _convert_input_layer(keras_layer): # etab.data_layout = layout _convert_input_layer(keras_layer) else: - if len(shape) == 2 and 'conv2d' in keras_layer.name or len(shape) == 2 and 'pooling2d' in keras_layer.name: - print('Processing layer: ', keras_layer.name) - etab.data_layout = 'NCHW' + if ( + len(shape) == 2 + and "conv2d" in keras_layer.name + or len(shape) == 2 + and "pooling2d" in keras_layer.name + ): + print("Processing layer: ", keras_layer.name) + etab.data_layout = "NCHW" else: etab.data_layout = layout - inbound_nodes = keras_layer.inbound_nodes if hasattr(keras_layer, 'inbound_nodes') \ - else keras_layer._inbound_nodes if hasattr(keras_layer, '_inbound_nodes') \ - else None + inbound_nodes = ( + keras_layer.inbound_nodes + if hasattr(keras_layer, "inbound_nodes") + else ( + keras_layer._inbound_nodes + if hasattr(keras_layer, "_inbound_nodes") + else None + ) + ) if inbound_nodes is None: - raise TypeError("Unknown layer type or unsupported Keras version : {}" - .format(keras_layer)) + raise TypeError( + "Unknown layer type or unsupported Keras version : {}".format( + keras_layer + ) + ) for node_idx, node in enumerate(inbound_nodes): # If some nodes in imported model are not relevant to the current model, # skip such layers. # - In Keras, model._network_nodes contains keys of all nodes relevant to the # current model; # - In tf.Keras, this is already done as part of tensorflow.keras.network.get_config - if not is_tf_keras and \ - not model._node_key(keras_layer, node_idx) in model._network_nodes: + if ( + not is_tf_keras + and model._node_key(keras_layer, node_idx) not in model._network_nodes + ): continue inexpr = [] # Since Keras allows creating multiple layers from the same name instance, @@ -1110,24 +1259,34 @@ def _convert_input_layer(keras_layer): zip_node = zip( _as_list(node.node_indices), _as_list(node.tensor_indices), - _as_list(node.inbound_layers)) + _as_list(node.inbound_layers), + ) for n_idx, t_idx, inbound_layer in zip_node: if isinstance(inbound_layer, input_layer_class): expr_name = inbound_layer.name _convert_input_layer(inbound_layer) else: - expr_name = inbound_layer.name + ':' + str(n_idx) + ':' + str(t_idx) + expr_name = f"{inbound_layer.name}:{str(n_idx)}:{str(t_idx)}" expr = etab.get_expr(expr_name) inexpr.append(expr) if len(inexpr) == 1: inexpr = inexpr[0] - keras_op_to_relay(inexpr, keras_layer, keras_layer.name + ':' + str(node_idx), etab) + keras_op_to_relay( + inexpr, + keras_layer, + f"{keras_layer.name}:{str(node_idx)}", + etab, + ) # model._output_coordinates contains out_node(oc[0]), node_index(oc[1]) and tensor_index(oc[2]) # Get all output nodes in etab using the name made from above values. # The out exprs were added to etab in keras_op_to_relay using this name. - outexpr = [etab.get_expr(oc[0].name + ":" + str(oc[1]) + ":" + str(oc[2])) \ - for oc in model._output_coordinates] + outexpr = [ + etab.get_expr(f"{oc[0].name}:{str(oc[1])}:{str(oc[2])}") + for oc in model._output_coordinates + ] outexpr = outexpr[0] if len(outexpr) == 1 else _expr.Tuple(outexpr) func = _function.Function(analysis.free_vars(outexpr), outexpr) - params = {k:_nd.array(np.array(v, dtype=np.float32)) for k, v in etab.params.items()} + params = { + k: _nd.array(np.array(v, dtype=np.float32)) for k, v in etab.params.items() + } return IRModule.from_expr(func), params