From 7f6ea0584555e75f64123409c73edb29430d827f Mon Sep 17 00:00:00 2001 From: AHLashkari Date: Wed, 5 Feb 2020 14:50:02 -0400 Subject: [PATCH 01/25] --- ReadMe.txt | 43 +++++++++++++++++++++---------------------- 1 file changed, 21 insertions(+), 22 deletions(-) diff --git a/ReadMe.txt b/ReadMe.txt index efe484ca..04a9c3c2 100644 --- a/ReadMe.txt +++ b/ReadMe.txt @@ -92,8 +92,8 @@ Bwd Packet Length Min Minimum size of packet in backward direction Bwd Packet Length Max Maximum size of packet in backward direction Bwd Packet Length Mean Mean size of packet in backward direction Bwd Packet Length Std Standard deviation size of packet in backward direction -Flow Byte/s Number of flow packets per second -Flow Packets/s Number of flow bytes per second +Flow Bytes/s Number of flow bytes per second +Flow Packets/s Number of flow packets per second Flow IAT Mean Mean time between two packets sent in the flow Flow IAT Std Standard deviation time between two packets sent in the flow Flow IAT Max Maximum time between two packets sent in the flow @@ -108,16 +108,16 @@ Bwd IAT Max Maximum time between two packets sent in the backward direction Bwd IAT Mean Mean time between two packets sent in the backward direction Bwd IAT Std Standard deviation time between two packets sent in the backward direction Bwd IAT Total Total time between two packets sent in the backward direction -Fwd PSH flag Number of times the PSH flag was set in packets travelling in the forward direction (0 for UDP) -Bwd PSH Flag Number of times the PSH flag was set in packets travelling in the backward direction (0 for UDP) -Fwd URG Flag Number of times the URG flag was set in packets travelling in the forward direction (0 for UDP) -Bwd URG Flag Number of times the URG flag was set in packets travelling in the backward direction (0 for UDP) +Fwd PSH flags Number of times the PSH flag was set in packets travelling in the forward direction (0 for UDP) +Bwd PSH Flags Number of times the PSH flag was set in packets travelling in the backward direction (0 for UDP) +Fwd URG Flags Number of times the URG flag was set in packets travelling in the forward direction (0 for UDP) +Bwd URG Flags Number of times the URG flag was set in packets travelling in the backward direction (0 for UDP) Fwd Header Length Total bytes used for headers in the forward direction Bwd Header Length Total bytes used for headers in the backward direction FWD Packets/s Number of forward packets per second Bwd Packets/s Number of backward packets per second -Min Packet Length Minimum length of a packet -Max Packet Length Maximum length of a packet +Packet Length Min Minimum length of a packet +Packet Length Max Maximum length of a packet Packet Length Mean Mean length of a packet Packet Length Std Standard deviation length of a packet Packet Length Variance Variance length of a packet @@ -127,27 +127,26 @@ RST Flag Count Number of packets with RST PSH Flag Count Number of packets with PUSH ACK Flag Count Number of packets with ACK URG Flag Count Number of packets with URG -CWR Flag Count Number of packets with CWE +CWE Flag Count Number of packets with CWE ECE Flag Count Number of packets with ECE down/Up Ratio Download and upload ratio Average Packet Size Average size of packet -Avg Fwd Segment Size Average size observed in the forward direction -AVG Bwd Segment Size Average number of bytes bulk rate in the backward direction -Fwd Header Length Length of the forward packet header -Fwd Avg Bytes/Bulk Average number of bytes bulk rate in the forward direction -Fwd AVG Packet/Bulk Average number of packets bulk rate in the forward direction -Fwd AVG Bulk Rate Average number of bulk rate in the forward direction -Bwd Avg Bytes/Bulk Average number of bytes bulk rate in the backward direction -Bwd AVG Packet/Bulk Average number of packets bulk rate in the backward direction -Bwd AVG Bulk Rate Average number of bulk rate in the backward direction +Fwd Segment Size Avg Average size observed in the forward direction +Bwd Segment Size Avg Average number of bytes bulk rate in the backward direction +Fwd Bytes/Bulk Avg Average number of bytes bulk rate in the forward direction +Fwd Packet/Bulk Avg Average number of packets bulk rate in the forward direction +Fwd Bulk Rate Avg Average number of bulk rate in the forward direction +Bwd Bytes/Bulk Avg Average number of bytes bulk rate in the backward direction +Bwd Packet/Bulk Avg Average number of packets bulk rate in the backward direction +Bwd Bulk Rate Avg Average number of bulk rate in the backward direction Subflow Fwd Packets The average number of packets in a sub flow in the forward direction Subflow Fwd Bytes The average number of bytes in a sub flow in the forward direction Subflow Bwd Packets The average number of packets in a sub flow in the backward direction Subflow Bwd Bytes The average number of bytes in a sub flow in the backward direction -Init_Win_bytes_forward The total number of bytes sent in initial window in the forward direction -Init_Win_bytes_backward The total number of bytes sent in initial window in the backward direction -Act_data_pkt_forward Count of packets with at least 1 byte of TCP data payload in the forward direction -min_seg_size_forward Minimum segment size observed in the forward direction +Fwd Init Win bytes The total number of bytes sent in initial window in the forward direction +Bwd Init Win bytes The total number of bytes sent in initial window in the backward direction +Fwd Act Data Pkts Count of packets with at least 1 byte of TCP data payload in the forward direction +Fwd Seg Size Min Minimum segment size observed in the forward direction Active Min Minimum time a flow was active before becoming idle Active Mean Mean time a flow was active before becoming idle Active Max Maximum time a flow was active before becoming idle From 9ffd41bcfb6e9db9851c22027e15a357b2252e7e Mon Sep 17 00:00:00 2001 From: AHLashkari Date: Wed, 5 Feb 2020 14:50:28 -0400 Subject: [PATCH 02/25] --- .../cic/cs/unb/ca/jnetpcap/FlowFeature.java | 154 +++++++++--------- 1 file changed, 77 insertions(+), 77 deletions(-) diff --git a/src/main/java/cic/cs/unb/ca/jnetpcap/FlowFeature.java b/src/main/java/cic/cs/unb/ca/jnetpcap/FlowFeature.java index 4fe7d779..3438d62b 100644 --- a/src/main/java/cic/cs/unb/ca/jnetpcap/FlowFeature.java +++ b/src/main/java/cic/cs/unb/ca/jnetpcap/FlowFeature.java @@ -10,91 +10,91 @@ public enum FlowFeature { - fid("Flow ID","FID",false), //1 this index is for feature not for ordinal - src_ip("Src IP","SIP",false), //2 + fid("Flow ID","FID",false), //1 this index is for feature not for ordinal + src_ip("Src IP","SIP",false), //2 src_port("Src Port","SPT"), //3 - dst_ip("Dst IP","DIP",false), //4 - dst_pot("Dst Port","DPT"), //5 - prot("Protocol","PROT"), //6 - tstp("Timestamp","TSTP",false), //7 + dst_ip("Dst IP","DIP",false), //4 + dst_pot("Dst Port","DPT"), //5 + prot("Protocol","PROT"), //6 + tstp("Timestamp","TSTP",false), //7 fl_dur("Flow Duration","DUR"), //8 - tot_fw_pkt("Tot Fwd Pkts","TFwP"), //9 - tot_bw_pkt("Tot Bwd Pkts","TBwP"), //10 - tot_l_fw_pkt("TotLen Fwd Pkts","TLFwP"), //11 - tot_l_bw_pkt("TotLen Bwd Pkts","TLBwP"), //12 - fw_pkt_l_max("Fwd Pkt Len Max","FwPLMA"), //13 - fw_pkt_l_min("Fwd Pkt Len Min","FwPLMI"), //14 - fw_pkt_l_avg("Fwd Pkt Len Mean","FwPLAG"), //15 - fw_pkt_l_std("Fwd Pkt Len Std","FwPLSD"), //16 - bw_pkt_l_max("Bwd Pkt Len Max","BwPLMA"), //17 - bw_pkt_l_min("Bwd Pkt Len Min","BwPLMI"), //18 - bw_pkt_l_avg("Bwd Pkt Len Mean","BwPLAG"), //19 - bw_pkt_l_std("Bwd Pkt Len Std","BwPLSD"), //20 - fl_byt_s("Flow Byts/s","FB/s"), //21 - fl_pkt_s("Flow Pkts/s","FP/s"), //22 - fl_iat_avg("Flow IAT Mean","FLIATAG"), //23 - fl_iat_std("Flow IAT Std","FLIATSD"), //24 - fl_iat_max("Flow IAT Max","FLIATMA"), //25 - fl_iat_min("Flow IAT Min","FLIATMI"), //26 - fw_iat_tot("Fwd IAT Tot","FwIATTO"), //27 - fw_iat_avg("Fwd IAT Mean","FwIATAG"), //28 - fw_iat_std("Fwd IAT Std","FwIATSD"), //29 - fw_iat_max("Fwd IAT Max","FwIATMA"), //30 - fw_iat_min("Fwd IAT Min","FwIATMI"), //31 - bw_iat_tot("Bwd IAT Tot","BwIATTO"), //32 - bw_iat_avg("Bwd IAT Mean","BwIATAG"), //33 - bw_iat_std("Bwd IAT Std","BwIATSD"), //34 - bw_iat_max("Bwd IAT Max","BwIATMA"), //35 - bw_iat_min("Bwd IAT Min","BwIATMI"), //36 - fw_psh_flag("Fwd PSH Flags","FwPSH"), //37 - bw_psh_flag("Bwd PSH Flags","BwPSH"), //38 - fw_urg_flag("Fwd URG Flags","FwURG"), //39 - bw_urg_flag("Bwd URG Flags","BwURG"), //40 - fw_hdr_len("Fwd Header Len","FwHL"), //41 - bw_hdr_len("Bwd Header Len","BwHL"), //42 - fw_pkt_s("Fwd Pkts/s","FwP/s"), //43 - bw_pkt_s("Bwd Pkts/s","Bwp/s"), //44 - pkt_len_min("Pkt Len Min","PLMI"), //45 - pkt_len_max("Pkt Len Max","PLMA"), //46 - pkt_len_avg("Pkt Len Mean","PLAG"), //47 - pkt_len_std("Pkt Len Std","PLSD"), //48 - pkt_len_var("Pkt Len Var","PLVA"), //49 - fin_cnt("FIN Flag Cnt","FINCT"), //50 - syn_cnt("SYN Flag Cnt","SYNCT"), //51 - rst_cnt("RST Flag Cnt","RSTCT"), //52 - pst_cnt("PSH Flag Cnt","PSHCT"), //53 - ack_cnt("ACK Flag Cnt","ACKCT"), //54 - urg_cnt("URG Flag Cnt","URGCT"), //55 + tot_fw_pkt("Total Fwd Packet","TFwP"), //9 + tot_bw_pkt("Total Bwd packets","TBwP"), //10 + tot_l_fw_pkt("Total Length of Fwd Packet","TLFwP"), //11 + tot_l_bw_pkt("Total Length of Bwd Packet","TLBwP"), //12 + fw_pkt_l_max("Fwd Packet Length Max","FwPLMA"), //13 + fw_pkt_l_min("Fwd Packet Length Min","FwPLMI"), //14 + fw_pkt_l_avg("Fwd Packet Length Mean","FwPLAG"), //15 + fw_pkt_l_std("Fwd Packet Length Std","FwPLSD"), //16 + bw_pkt_l_max("Bwd Packet Length Max","BwPLMA"), //17 + bw_pkt_l_min("Bwd Packet Length Min","BwPLMI"), //18 + bw_pkt_l_avg("Bwd Packet Length Mean","BwPLAG"), //19 + bw_pkt_l_std("Bwd Packet Length Std","BwPLSD"), //20 + fl_byt_s("Flow Bytes/s","FB/s"), //21 + fl_pkt_s("Flow Packets/s","FP/s"), //22 + fl_iat_avg("Flow IAT Mean","FLIATAG"), //23 + fl_iat_std("Flow IAT Std","FLIATSD"), //24 + fl_iat_max("Flow IAT Max","FLIATMA"), //25 + fl_iat_min("Flow IAT Min","FLIATMI"), //26 + fw_iat_tot("Fwd IAT Total","FwIATTO"), //27 + fw_iat_avg("Fwd IAT Mean","FwIATAG"), //28 + fw_iat_std("Fwd IAT Std","FwIATSD"), //29 + fw_iat_max("Fwd IAT Max","FwIATMA"), //30 + fw_iat_min("Fwd IAT Min","FwIATMI"), //31 + bw_iat_tot("Bwd IAT Total","BwIATTO"), //32 + bw_iat_avg("Bwd IAT Mean","BwIATAG"), //33 + bw_iat_std("Bwd IAT Std","BwIATSD"), //34 + bw_iat_max("Bwd IAT Max","BwIATMA"), //35 + bw_iat_min("Bwd IAT Min","BwIATMI"), //36 + fw_psh_flag("Fwd PSH Flags","FwPSH"), //37 + bw_psh_flag("Bwd PSH Flags","BwPSH"), //38 + fw_urg_flag("Fwd URG Flags","FwURG"), //39 + bw_urg_flag("Bwd URG Flags","BwURG"), //40 + fw_hdr_len("Fwd Header Length","FwHL"), //41 + bw_hdr_len("Bwd Header Length","BwHL"), //42 + fw_pkt_s("Fwd Packets/s","FwP/s"), //43 + bw_pkt_s("Bwd Packets/s","Bwp/s"), //44 + pkt_len_min("Packet Length Min","PLMI"), //45 + pkt_len_max("Packet Length Max","PLMA"), //46 + pkt_len_avg("Packet Length Mean","PLAG"), //47 + pkt_len_std("Packet Length Std","PLSD"), //48 + pkt_len_var("Packet Length Variance","PLVA"), //49 + fin_cnt("FIN Flag Count","FINCT"), //50 + syn_cnt("SYN Flag Count","SYNCT"), //51 + rst_cnt("RST Flag Count","RSTCT"), //52 + pst_cnt("PSH Flag Count","PSHCT"), //53 + ack_cnt("ACK Flag Count","ACKCT"), //54 + urg_cnt("URG Flag Count","URGCT"), //55 cwe_cnt("CWE Flag Count","CWECT"), //56 - ece_cnt("ECE Flag Cnt","ECECT"), //57 - down_up_ratio("Down/Up Ratio","D/URO"), //58 - pkt_size_avg("Pkt Size Avg","PSAG"), //59 - fw_seg_avg("Fwd Seg Size Avg","FwSgAG"), //60 - bw_seg_avg("Bwd Seg Size Avg","BwSgAG"), //61 - fw_byt_blk_avg("Fwd Byts/b Avg","FwB/BAG"), //63 62 is duplicated with 41,so has been deleted - fw_pkt_blk_avg("Fwd Pkts/b Avg","FwP/BAG"), //64 - fw_blk_rate_avg("Fwd Blk Rate Avg","FwBRAG"), //65 - bw_byt_blk_avg("Bwd Byts/b Avg","BwB/BAG"), //66 - bw_pkt_blk_avg("Bwd Pkts/b Avg","BwP/BAG"), //67 - bw_blk_rate_avg("Bwd Blk Rate Avg","BwBRAG"), //68 - subfl_fw_pkt("Subflow Fwd Pkts","SFFwP"), //69 - subfl_fw_byt("Subflow Fwd Byts","SFFwB"), //70 - subfl_bw_pkt("Subflow Bwd Pkts","SFBwP"), //71 - subfl_bw_byt("Subflow Bwd Byts","SFBwB"), //72 - fw_win_byt("Init Fwd Win Byts","FwWB"), //73 - bw_win_byt("Init Bwd Win Byts","BwWB"), //74 + ece_cnt("ECE Flag Count","ECECT"), //57 + down_up_ratio("Down/Up Ratio","D/URO"), //58 + pkt_size_avg("Average Packet Size","PSAG"), //59 + fw_seg_avg("Fwd Segment Size Avg","FwSgAG"), //60 + bw_seg_avg("Bwd Segment Size Avg","BwSgAG"), //61 + fw_byt_blk_avg("Fwd Bytes/Bulk Avg","FwB/BAG"), //63 62 is duplicated with 41,so has been deleted + fw_pkt_blk_avg("Fwd Packet/Bulk Avg","FwP/BAG"), //64 + fw_blk_rate_avg("Fwd Bulk Rate Avg","FwBRAG"), //65 + bw_byt_blk_avg("Bwd Bytes/Bulk Avg","BwB/BAG"), //66 + bw_pkt_blk_avg("Bwd Packet/Bulk Avg","BwP/BAG"), //67 + bw_blk_rate_avg("Bwd Bulk Rate Avg","BwBRAG"), //68 + subfl_fw_pkt("Subflow Fwd Packets","SFFwP"), //69 + subfl_fw_byt("Subflow Fwd Bytes","SFFwB"), //70 + subfl_bw_pkt("Subflow Bwd Packets","SFBwP"), //71 + subfl_bw_byt("Subflow Bwd Bytes","SFBwB"), //72 + fw_win_byt("FWD Init Win Bytes","FwWB"), //73 + bw_win_byt("Bwd Init Win Bytes","BwWB"), //74 Fw_act_pkt("Fwd Act Data Pkts","FwAP"), //75 - fw_seg_min("Fwd Seg Size Min","FwSgMI"), //76 - atv_avg("Active Mean","AcAG"), //77 - atv_std("Active Std","AcSD"), //78 - atv_max("Active Max","AcMA"), //79 - atv_min("Active Min","AcMI"), //80 - idl_avg("Idle Mean","IlAG"), //81 + fw_seg_min("Fwd Seg Size Min","FwSgMI"), //76 + atv_avg("Active Mean","AcAG"), //77 + atv_std("Active Std","AcSD"), //78 + atv_max("Active Max","AcMA"), //79 + atv_min("Active Min","AcMI"), //80 + idl_avg("Idle Mean","IlAG"), //81 idl_std("Idle Std","IlSD"), //82 idl_max("Idle Max","IlMA"), //83 idl_min("Idle Min","IlMI"), //84 - Label("Label","LBL",new String[]{"CIC"}); //85 + Label("Label","LBL",new String[]{"CIC"}); //85 protected static final Logger logger = LoggerFactory.getLogger(FlowFeature.class); From fc3d539192a026793b38f76d808efb78e4842a1a Mon Sep 17 00:00:00 2001 From: AHLashkari Date: Wed, 5 Feb 2020 14:52:10 -0400 Subject: [PATCH 03/25] --- src/main/java/cic/cs/unb/ca/jnetpcap/FlowFeature.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/cic/cs/unb/ca/jnetpcap/FlowFeature.java b/src/main/java/cic/cs/unb/ca/jnetpcap/FlowFeature.java index 3438d62b..cfc419f9 100644 --- a/src/main/java/cic/cs/unb/ca/jnetpcap/FlowFeature.java +++ b/src/main/java/cic/cs/unb/ca/jnetpcap/FlowFeature.java @@ -94,7 +94,7 @@ public enum FlowFeature { idl_max("Idle Max","IlMA"), //83 idl_min("Idle Min","IlMI"), //84 - Label("Label","LBL",new String[]{"CIC"}); //85 + Label("Label","LBL",new String[]{"NeedManualLabel"}); //85 protected static final Logger logger = LoggerFactory.getLogger(FlowFeature.class); From be803cb7f8fb43f356e0dfb71f3ad24b4d99a68f Mon Sep 17 00:00:00 2001 From: AHLashkari Date: Wed, 5 Feb 2020 15:11:05 -0400 Subject: [PATCH 04/25] --- src/main/java/cic/cs/unb/ca/jnetpcap/BasicFlow.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/cic/cs/unb/ca/jnetpcap/BasicFlow.java b/src/main/java/cic/cs/unb/ca/jnetpcap/BasicFlow.java index 9c0bb3b9..2e84d849 100644 --- a/src/main/java/cic/cs/unb/ca/jnetpcap/BasicFlow.java +++ b/src/main/java/cic/cs/unb/ca/jnetpcap/BasicFlow.java @@ -1060,7 +1060,7 @@ public String getLabel() { else{ return "BENIGN"; }*/ - return "No Label"; + return "NeedManualLabel"; } public String dumpFlowBasedFeaturesEx() { From 06c5ec33ac67a355717b5aced542de4c94a50b6e Mon Sep 17 00:00:00 2001 From: AHLashkari Date: Wed, 29 Apr 2020 15:54:55 -0300 Subject: [PATCH 05/25] Update ReadMe.txt --- ReadMe.txt | 67 ++---------------------------------------------------- 1 file changed, 2 insertions(+), 65 deletions(-) diff --git a/ReadMe.txt b/ReadMe.txt index 04a9c3c2..7ac56469 100644 --- a/ReadMe.txt +++ b/ReadMe.txt @@ -5,77 +5,12 @@ CICFlowMeter is a network traffic flow generator available from here . It can be NOTE: TCP flows are usually terminated upon connection teardown (by FIN packet) while UDP flows are terminated by a flow timeout. The flow timeout value can be assigned arbitrarily by the individual scheme e.g., 600 seconds for both TCP and UDP. -For citation in your works and also understanding CICFlowMeter (formerly ISCXFlowMeter) completely, you can find below published paper: -Gerard Drapper Gil, Arash Habibi Lashkari, Mohammad Mamun, Ali A. Ghorbani, "Characterization of Encrypted and VPN Traffic Using Time-Related Features", In Proceedings of the 2nd International Conference on Information Systems Security and Privacy(ICISSP 2016) , pages 407-414, Rome , Italy - - ----------------------------------------- - -# Installation and executing: - -Extract CICFlowMeterV3.zip - -___Note: The only prerequisite is that "libpcap" library or WinPcap on windows systems, be pre-installed___ - - -For Linux - -> $ sudo apt-get install libpcap-dev - - -For windows -> download [winpcap]() - -## executing -Go to the extracted directory,enter the 'bin' folder - -### linux -Open a terminal and run this command -``` -//For GUI: -sudo ./CICFlowMeter - -//For Command line: -./cfm "inputFolder" "outputFolder" -``` -### windows -Lanunch the Comand Prompt and run this command -``` -//for GUI: -CICFlowMeter.bat - -//for Commond line: -cfm.bat "inputFolder" "outputFolder" -``` - -## Get started -for offline -``` -1.Select the folder that include your PCAP files -2.Select the folder that you would like to save you CSV files -3.Click OK button -``` - -for realtime -``` -1 CLick Load button to find the list of network interfaces -2 Select the interface you would like to monitor -3 Click start button and wait for a while -4 Click stop button to stop the process and save the csv in same applcation folder/data/daily -``` - --------------------------------------------------------------- - -Contact us at A.Habibi.L@unb.ca if there are any problems. - - For citation in your works and also understanding CICFlowMeter (formerly ISCXFlowMeter) completely, you can find below published papers: Arash Habibi Lashkari, Gerard Draper-Gil, Mohammad Saiful Islam Mamun and Ali A. Ghorbani, "Characterization of Tor Traffic Using Time Based Features", In the proceeding of the 3rd International Conference on Information System Security and Privacy, SCITEPRESS, Porto, Portugal, 2017 Gerard Drapper Gil, Arash Habibi Lashkari, Mohammad Mamun, Ali A. Ghorbani, "Characterization of Encrypted and VPN Traffic Using Time-Related Features", In Proceedings of the 2nd International Conference on Information Systems Security and Privacy(ICISSP 2016) , pages 407-414, Rome , Italy - -------------------------------------------------------------- List of extracted features and descriptions: Feature Name Description @@ -156,3 +91,5 @@ Idle Mean Mean time a flow was idle before becoming active Idle Max Maximum time a flow was idle before becoming active Idle Std Standard deviation time a flow was idle before becoming active +-------------------------------------------------------------------------------------- +Contact at A.Habibi.L@unb.ca if there are any problems. From 76038e0bddd4ed8a56ec75c6d8dd309b5f710504 Mon Sep 17 00:00:00 2001 From: AHLashkari Date: Wed, 29 Apr 2020 15:55:12 -0300 Subject: [PATCH 06/25] Update ReadMe.txt --- ReadMe.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/ReadMe.txt b/ReadMe.txt index 7ac56469..b0a4ab22 100644 --- a/ReadMe.txt +++ b/ReadMe.txt @@ -1,4 +1,3 @@ -# Intro The CICFlowMeter is an open source tool that generates Biflows from pcap files, and extracts features from these flows. CICFlowMeter is a network traffic flow generator available from here . It can be used to generate bidirectional flows, where the first packet determines the forward (source to destination) and backward (destination to source) directions, hence the statistical time-related features can be calculated separately in the forward and backward directions. Additional functionalities include, selecting features from the list of existing features, adding new features, and controlling the duration of flow timeout. From 2b5b932bfc347f3cfbdd7e6fea3c3de822f7d33e Mon Sep 17 00:00:00 2001 From: AHLashkari Date: Wed, 29 Apr 2020 23:30:27 -0300 Subject: [PATCH 07/25] Update ReadMe.txt --- ReadMe.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ReadMe.txt b/ReadMe.txt index b0a4ab22..ea3b2e2d 100644 --- a/ReadMe.txt +++ b/ReadMe.txt @@ -91,4 +91,4 @@ Idle Max Maximum time a flow was idle before becoming active Idle Std Standard deviation time a flow was idle before becoming active -------------------------------------------------------------------------------------- -Contact at A.Habibi.L@unb.ca if there are any problems. + From ab1dc02975f37934b6dce1de3b4621f2e1dbee4d Mon Sep 17 00:00:00 2001 From: AHLashkari Date: Thu, 28 May 2020 13:31:47 -0300 Subject: [PATCH 08/25] --- src/main/java/cic/cs/unb/ca/jnetpcap/BasicFlow.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/cic/cs/unb/ca/jnetpcap/BasicFlow.java b/src/main/java/cic/cs/unb/ca/jnetpcap/BasicFlow.java index 2e84d849..8eaa9f97 100644 --- a/src/main/java/cic/cs/unb/ca/jnetpcap/BasicFlow.java +++ b/src/main/java/cic/cs/unb/ca/jnetpcap/BasicFlow.java @@ -1191,7 +1191,7 @@ public String dumpFlowBasedFeaturesEx() { dump.append(fAvgBytesPerBulk()).append(separator); //63 dump.append(fAvgPacketsPerBulk()).append(separator); //64 dump.append(fAvgBulkRate()).append(separator); //65 - dump.append(fAvgBytesPerBulk()).append(separator); //66 + dump.append(bAvgBytesPerBulk()).append(separator); //66 dump.append(bAvgPacketsPerBulk()).append(separator); //67 dump.append(bAvgBulkRate()).append(separator); //68 From 2b7be2690c1e4b83b2b4295303788d7cd3245108 Mon Sep 17 00:00:00 2001 From: AHLashkari Date: Thu, 28 May 2020 16:38:18 -0300 Subject: [PATCH 09/25] --- src/main/java/cic/cs/unb/ca/jnetpcap/FlowFeature.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/cic/cs/unb/ca/jnetpcap/FlowFeature.java b/src/main/java/cic/cs/unb/ca/jnetpcap/FlowFeature.java index cfc419f9..35a8849b 100644 --- a/src/main/java/cic/cs/unb/ca/jnetpcap/FlowFeature.java +++ b/src/main/java/cic/cs/unb/ca/jnetpcap/FlowFeature.java @@ -65,7 +65,7 @@ public enum FlowFeature { pst_cnt("PSH Flag Count","PSHCT"), //53 ack_cnt("ACK Flag Count","ACKCT"), //54 urg_cnt("URG Flag Count","URGCT"), //55 - cwe_cnt("CWE Flag Count","CWECT"), //56 + CWR_cnt("CWR Flag Count","CWRCT"), //56 ece_cnt("ECE Flag Count","ECECT"), //57 down_up_ratio("Down/Up Ratio","D/URO"), //58 pkt_size_avg("Average Packet Size","PSAG"), //59 From 90c965587aee4f35ae403895373b42e963c61666 Mon Sep 17 00:00:00 2001 From: AHLashkari Date: Thu, 28 May 2020 16:38:41 -0300 Subject: [PATCH 10/25] --- src/main/java/cic/cs/unb/ca/jnetpcap/FlowGenerator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/cic/cs/unb/ca/jnetpcap/FlowGenerator.java b/src/main/java/cic/cs/unb/ca/jnetpcap/FlowGenerator.java index 7759e18b..a2c7fd3e 100644 --- a/src/main/java/cic/cs/unb/ca/jnetpcap/FlowGenerator.java +++ b/src/main/java/cic/cs/unb/ca/jnetpcap/FlowGenerator.java @@ -29,7 +29,7 @@ public class FlowGenerator { + "Fwd PSH Flags, Bwd PSH Flags, Fwd URG Flags, Bwd URG Flags, Fwd Header Length, Bwd Header Length," + "Fwd Packets/s, Bwd Packets/s, Min Packet Length, Max Packet Length, Packet Length Mean, Packet Length Std, Packet Length Variance," + "FIN Flag Count, SYN Flag Count, RST Flag Count, PSH Flag Count, ACK Flag Count, URG Flag Count, " - + "CWE Flag Count, ECE Flag Count, Down/Up Ratio, Average Packet Size, Avg Fwd Segment Size, Avg Bwd Segment Size, Fwd Header Length," + + "CWR Flag Count, ECE Flag Count, Down/Up Ratio, Average Packet Size, Avg Fwd Segment Size, Avg Bwd Segment Size, Fwd Header Length," + "Fwd Avg Bytes/Bulk, Fwd Avg Packets/Bulk, Fwd Avg Bulk Rate, Bwd Avg Bytes/Bulk, Bwd Avg Packets/Bulk," + "Bwd Avg Bulk Rate," + "Subflow Fwd Packets, Subflow Fwd Bytes, Subflow Bwd Packets, Subflow Bwd Bytes," From 7aebc1bb25f9c777a90cc10cd6733c8288d62310 Mon Sep 17 00:00:00 2001 From: AHLashkari Date: Thu, 28 May 2020 16:41:04 -0300 Subject: [PATCH 11/25] Update ReadMe.txt --- ReadMe.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ReadMe.txt b/ReadMe.txt index ea3b2e2d..399bd7bf 100644 --- a/ReadMe.txt +++ b/ReadMe.txt @@ -61,7 +61,7 @@ RST Flag Count Number of packets with RST PSH Flag Count Number of packets with PUSH ACK Flag Count Number of packets with ACK URG Flag Count Number of packets with URG -CWE Flag Count Number of packets with CWE +CWR Flag Count Number of packets with CWR ECE Flag Count Number of packets with ECE down/Up Ratio Download and upload ratio Average Packet Size Average size of packet From 1d35837be700b9de3f4475697d64f85d0f8da047 Mon Sep 17 00:00:00 2001 From: AHLashkari Date: Wed, 17 Jun 2020 00:06:36 -0300 Subject: [PATCH 12/25] --- src/main/java/cic/cs/unb/ca/jnetpcap/BasicFlow.java | 13 ++++++++----- .../java/cic/cs/unb/ca/jnetpcap/FlowGenerator.java | 4 ++-- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/src/main/java/cic/cs/unb/ca/jnetpcap/BasicFlow.java b/src/main/java/cic/cs/unb/ca/jnetpcap/BasicFlow.java index 8eaa9f97..79ac0055 100644 --- a/src/main/java/cic/cs/unb/ca/jnetpcap/BasicFlow.java +++ b/src/main/java/cic/cs/unb/ca/jnetpcap/BasicFlow.java @@ -55,10 +55,11 @@ public class BasicFlow { private long flowLastSeen; private long forwardLastSeen; private long backwardLastSeen; - + private long activityTimeout; - public BasicFlow(boolean isBidirectional,BasicPacketInfo packet, byte[] flowSrc, byte[] flowDst, int flowSrcPort, int flowDstPort) { + public BasicFlow(boolean isBidirectional,BasicPacketInfo packet, byte[] flowSrc, byte[] flowDst, int flowSrcPort, int flowDstPort, long activityTimeout) { super(); + this.activityTimeout = activityTimeout; this.initParameters(); this.isBidirectional = isBidirectional; this.firstPacket(packet); @@ -68,15 +69,17 @@ public BasicFlow(boolean isBidirectional,BasicPacketInfo packet, byte[] flowSrc, this.dstPort = flowDstPort; } - public BasicFlow(boolean isBidirectional,BasicPacketInfo packet) { + public BasicFlow(boolean isBidirectional,BasicPacketInfo packet, long activityTimeout) { super(); + this.activityTimeout = activityTimeout; this.initParameters(); this.isBidirectional = isBidirectional; this.firstPacket(packet); } - public BasicFlow(BasicPacketInfo packet) { + public BasicFlow(BasicPacketInfo packet, long activityTimeout) { super(); + this.activityTimeout = activityTimeout; this.initParameters(); this.isBidirectional = true; firstPacket(packet); @@ -346,7 +349,7 @@ void detectUpdateSubflows( BasicPacketInfo packet ){ if( (packet.getTimeStamp() - (sfLastPacketTS)/(double)1000000) > 1.0 ){ sfCount ++ ; long lastSFduration = packet.getTimeStamp() - sfAcHelper; - updateActiveIdleTime(packet.getTimeStamp() - sfLastPacketTS, 5000000L); + updateActiveIdleTime(packet.getTimeStamp(), this.activityTimeout); sfAcHelper = packet.getTimeStamp(); } diff --git a/src/main/java/cic/cs/unb/ca/jnetpcap/FlowGenerator.java b/src/main/java/cic/cs/unb/ca/jnetpcap/FlowGenerator.java index a2c7fd3e..60566578 100644 --- a/src/main/java/cic/cs/unb/ca/jnetpcap/FlowGenerator.java +++ b/src/main/java/cic/cs/unb/ca/jnetpcap/FlowGenerator.java @@ -99,7 +99,7 @@ public void addPacket(BasicPacketInfo packet){ //flow.endActiveIdleTime(currentTimestamp,this.flowActivityTimeOut, this.flowTimeOut, false); } currentFlows.remove(id); - currentFlows.put(id, new BasicFlow(bidirectional,packet,flow.getSrc(),flow.getDst(),flow.getSrcPort(),flow.getDstPort())); + currentFlows.put(id, new BasicFlow(bidirectional,packet,flow.getSrc(),flow.getDst(),flow.getSrcPort(),flow.getDstPort(), this.flowActivityTimeOut)); int cfsize = currentFlows.size(); if(cfsize%50==0) { @@ -126,7 +126,7 @@ public void addPacket(BasicPacketInfo packet){ currentFlows.put(id,flow); } }else{ - currentFlows.put(packet.fwdFlowId(), new BasicFlow(bidirectional,packet)); + currentFlows.put(packet.fwdFlowId(), new BasicFlow(bidirectional,packet, this.flowActivityTimeOut)); } } From c0e1039ebc2384fd059d1329cdbe23320bf4cdf8 Mon Sep 17 00:00:00 2001 From: AHLashkari Date: Tue, 30 Jun 2020 14:07:50 -0300 Subject: [PATCH 13/25] Update README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 4d866cc7..af397280 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ open a Terminal in the IDE ``` //linux: $ sudo bash -$ gradle execute +$ ./gradlew execute //windows: $ gradlew execute @@ -35,7 +35,7 @@ Run eclipse with sudo open a Terminal in the IDE ``` //linux: -$ gradle distZip +$ ./gradlew distZip //window $ gradlew distZip ``` From 718524a27438c5c379b22bc6c090506c4bcedd80 Mon Sep 17 00:00:00 2001 From: Arash Habibi Lashkari Date: Sun, 7 Mar 2021 23:35:55 -0400 Subject: [PATCH 14/25] Update LICENSE.txt --- LICENSE.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LICENSE.txt b/LICENSE.txt index 5f9ab32b..847c6649 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -1,4 +1,4 @@ -Copyright (c) 2016 Canadian Institute for Cybersecurity (CIC) +Copyright (c) 2016 Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (CICFlowMeter), to deal From fdc02524a4cbe2cc1362c0046d243d5fe83008c1 Mon Sep 17 00:00:00 2001 From: Arash Habibi Lashkari Date: Sun, 7 Mar 2021 23:37:49 -0400 Subject: [PATCH 15/25] Update LICENSE.txt --- LICENSE.txt | 26 ++++++-------------------- 1 file changed, 6 insertions(+), 20 deletions(-) diff --git a/LICENSE.txt b/LICENSE.txt index 847c6649..befceff6 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -1,32 +1,18 @@ -Copyright (c) 2016 +# Copyright (c) 2016 -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (CICFlowMeter), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (CICFlowMeter), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -For citation in your works and also understanding CICFlowMeter (formerly ISCXFlowMeter) completely, you can find below published papers: +# For citation in your works and also understanding CICFlowMeter (formerly ISCXFlowMeter) completely, you can find below published papers: Arash Habibi Lashkari, Gerard Draper-Gil, Mohammad Saiful Islam Mamun and Ali A. Ghorbani, "Characterization of Tor Traffic Using Time Based Features", In the proceeding of the 3rd International Conference on Information System Security and Privacy, SCITEPRESS, Porto, Portugal, 2017 Gerard Drapper Gil, Arash Habibi Lashkari, Mohammad Mamun, Ali A. Ghorbani, "Characterization of Encrypted and VPN Traffic Using Time-Related Features", In Proceedings of the 2nd International Conference on Information Systems Security and Privacy(ICISSP 2016) , pages 407-414, Rome , Italy -Contributors: +# Contributors: Arash Habibi Lashkari: Researcher and Developer (Founder) - Gerard Drapper: Researcher and Developer (Co-funder) - muhammad saiful islam: Researcher and Developer (Co-funder) From b5b6085ecd0b889e12b46b520fd5d9269a946c49 Mon Sep 17 00:00:00 2001 From: Arash Habibi Lashkari Date: Sun, 7 Mar 2021 23:38:26 -0400 Subject: [PATCH 16/25] Rename LICENSE.txt to LICENSE.me --- LICENSE.txt => LICENSE.me | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename LICENSE.txt => LICENSE.me (100%) diff --git a/LICENSE.txt b/LICENSE.me similarity index 100% rename from LICENSE.txt rename to LICENSE.me From 9d91f721cc3ef588f0d82ec0f19970660904812b Mon Sep 17 00:00:00 2001 From: Arash Habibi Lashkari Date: Sun, 7 Mar 2021 23:39:01 -0400 Subject: [PATCH 17/25] Rename LICENSE.me to LICENSE.txt --- LICENSE.me => LICENSE.txt | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename LICENSE.me => LICENSE.txt (100%) diff --git a/LICENSE.me b/LICENSE.txt similarity index 100% rename from LICENSE.me rename to LICENSE.txt From 5df1a62e9cf16380c79c87779067f3c630ca8578 Mon Sep 17 00:00:00 2001 From: Arash Habibi Lashkari Date: Sun, 6 Jun 2021 11:49:32 -0300 Subject: [PATCH 18/25] --- src/main/java/cic/cs/unb/ca/jnetpcap/BasicFlow.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/cic/cs/unb/ca/jnetpcap/BasicFlow.java b/src/main/java/cic/cs/unb/ca/jnetpcap/BasicFlow.java index 79ac0055..f18471d4 100644 --- a/src/main/java/cic/cs/unb/ca/jnetpcap/BasicFlow.java +++ b/src/main/java/cic/cs/unb/ca/jnetpcap/BasicFlow.java @@ -346,7 +346,7 @@ void detectUpdateSubflows( BasicPacketInfo packet ){ sfAcHelper = packet.getTimeStamp(); } //System.out.print(" - "+(packet.timeStamp - sfLastPacketTS)); - if( (packet.getTimeStamp() - (sfLastPacketTS)/(double)1000000) > 1.0 ){ + if(((packet.getTimeStamp() - sfLastPacketTS)/(double)1000000) > 1.0){ sfCount ++ ; long lastSFduration = packet.getTimeStamp() - sfAcHelper; updateActiveIdleTime(packet.getTimeStamp(), this.activityTimeout); From 97d56406447c0fe4473a33ce54e2623be63f7adb Mon Sep 17 00:00:00 2001 From: Arash Habibi Lashkari Date: Mon, 7 Jun 2021 09:55:46 -0300 Subject: [PATCH 19/25] Update ReadMe.txt --- ReadMe.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ReadMe.txt b/ReadMe.txt index 399bd7bf..1fecdda8 100644 --- a/ReadMe.txt +++ b/ReadMe.txt @@ -66,7 +66,7 @@ ECE Flag Count Number of packets with ECE down/Up Ratio Download and upload ratio Average Packet Size Average size of packet Fwd Segment Size Avg Average size observed in the forward direction -Bwd Segment Size Avg Average number of bytes bulk rate in the backward direction +Bwd Segment Size Avg Average size observed in the backward direction Fwd Bytes/Bulk Avg Average number of bytes bulk rate in the forward direction Fwd Packet/Bulk Avg Average number of packets bulk rate in the forward direction Fwd Bulk Rate Avg Average number of bulk rate in the forward direction From dec65b9685c50c4c3fb5804ea356f5e32a839929 Mon Sep 17 00:00:00 2001 From: Arash Habibi Lashkari Date: Fri, 2 Jul 2021 22:43:59 -0300 Subject: [PATCH 20/25] --- .../cic/cs/unb/ca/jnetpcap/BasicFlow.java | 22 ++++ .../cic/cs/unb/ca/jnetpcap/FlowGenerator.java | 118 ++++++++++++++++-- 2 files changed, 129 insertions(+), 11 deletions(-) diff --git a/src/main/java/cic/cs/unb/ca/jnetpcap/BasicFlow.java b/src/main/java/cic/cs/unb/ca/jnetpcap/BasicFlow.java index f18471d4..f7b1b4cc 100644 --- a/src/main/java/cic/cs/unb/ca/jnetpcap/BasicFlow.java +++ b/src/main/java/cic/cs/unb/ca/jnetpcap/BasicFlow.java @@ -28,6 +28,8 @@ public class BasicFlow { private int bPSH_cnt; private int fURG_cnt; private int bURG_cnt; + private int fFIN_cnt; + private int bFIN_cnt; private long Act_data_pkt_forward; private long min_seg_size_forward; @@ -108,6 +110,8 @@ public void initParameters(){ this.bPSH_cnt=0; this.fURG_cnt=0; this.bURG_cnt=0; + this.fFIN_cnt=0; + this.bFIN_cnt=0; this.fHeaderBytes=0L; this.bHeaderBytes=0L; @@ -975,6 +979,24 @@ public int getBwdURGFlags() { return bURG_cnt; } + public int getFwdFINFlags() { + return fFIN_cnt; + } + + public int getBwdFINFlags() { + return bFIN_cnt; + } + + public int setFwdFINFlags() { + fFIN_cnt++; + return fFIN_cnt; + } + + public int setBwdFINFlags() { + bFIN_cnt++; + return bFIN_cnt; + } + public long getFwdHeaderLength() { return fHeaderBytes; } diff --git a/src/main/java/cic/cs/unb/ca/jnetpcap/FlowGenerator.java b/src/main/java/cic/cs/unb/ca/jnetpcap/FlowGenerator.java index 60566578..6f546975 100644 --- a/src/main/java/cic/cs/unb/ca/jnetpcap/FlowGenerator.java +++ b/src/main/java/cic/cs/unb/ca/jnetpcap/FlowGenerator.java @@ -4,6 +4,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.Arrays; + import java.io.File; import java.io.FileOutputStream; import java.io.IOException; @@ -106,24 +108,118 @@ public void addPacket(BasicPacketInfo packet){ logger.debug("Timeout current has {} flow",cfsize); } - // Flow finished due FIN flag (tcp only): +// // Flow finished due FIN flag (tcp only): +// // 1.- we add the packet-in-process to the flow (it is the last packet) +// // 2.- we move the flow to finished flow list +// // 3.- we eliminate the flow from the current flow list +// }else if(packet.hasFlagFIN()){ +// logger.debug("FlagFIN current has {} flow",currentFlows.size()); +// flow.addPacket(packet); +// if (mListener != null) { +// mListener.onFlowGenerated(flow); +// } else { +// finishedFlows.put(getFlowCount(), flow); +// } +// currentFlows.remove(id); + }else if(packet.hasFlagFIN()){ + // + // Forward Flow + // + if (Arrays.equals(flow.getSrc(), packet.getSrc())) { + // How many forward FIN received? + if (flow.setFwdFINFlags() == 1) { + // Flow finished due FIN flag (tcp only)?: + // 1.- we add the packet-in-process to the flow (it is the last packet) + // 2.- we move the flow to finished flow list + // 3.- we eliminate the flow from the current flow list + if ((flow.getBwdFINFlags() + flow.getBwdFINFlags()) == 2) { + logger.debug("FlagFIN current has {} flow",currentFlows.size()); + flow.addPacket(packet); + if (mListener != null) { + mListener.onFlowGenerated(flow); + } else { + finishedFlows.put(getFlowCount(), flow); + } + currentFlows.remove(id); + // Forward Flow Finished. + } else { + logger.info("Forward flow closed due to FIN Flag"); + flow.updateActiveIdleTime(currentTimestamp,this.flowActivityTimeOut); + flow.addPacket(packet); + currentFlows.put(id,flow); + } + }else{ + // some error + // TODO: review what to do with the packet + logger.warn("Forward flow received {} FIN packets", flow.getFwdFINFlags()); + } + // + // Backward Flow + // + } else { + // How many backward FIN packets received? + if (flow.setBwdFINFlags() == 1) { + // Flow finished due FIN flag (tcp only)?: + // 1.- we add the packet-in-process to the flow (it is the last packet) + // 2.- we move the flow to finished flow list + // 3.- we eliminate the flow from the current flow list + if ((flow.getBwdFINFlags() + flow.getBwdFINFlags()) == 2) { + logger.debug("FlagFIN current has {} flow",currentFlows.size()); + flow.addPacket(packet); + if (mListener != null) { + mListener.onFlowGenerated(flow); + } else { + finishedFlows.put(getFlowCount(), flow); + } + currentFlows.remove(id); + // Backward Flow Finished. + } else { + logger.info("Backwards flow closed due to FIN Flag"); + flow.updateActiveIdleTime(currentTimestamp,this.flowActivityTimeOut); + flow.addPacket(packet); + currentFlows.put(id,flow); + } + }else{ + // some error + // TODO: review what to do with the packet + logger.warn("Backward flow received {} FIN packets", flow.getBwdFINFlags()); + } + } + // Flow finished due RST flag (tcp only): // 1.- we add the packet-in-process to the flow (it is the last packet) // 2.- we move the flow to finished flow list - // 3.- we eliminate the flow from the current flow list - }else if(packet.hasFlagFIN()){ - logger.debug("FlagFIN current has {} flow",currentFlows.size()); - flow.addPacket(packet); + // 3.- we eliminate the flow from the current flow list + }else if(packet.hasFlagRST()){ + logger.debug("FlagRST current has {} flow",currentFlows.size()); + flow.addPacket(packet); if (mListener != null) { mListener.onFlowGenerated(flow); - } - else { + } else { finishedFlows.put(getFlowCount(), flow); } - currentFlows.remove(id); + currentFlows.remove(id); }else{ - flow.updateActiveIdleTime(currentTimestamp,this.flowActivityTimeOut); - flow.addPacket(packet); - currentFlows.put(id,flow); + // + // Forward Flow and fwdFIN = 0 + // + if (Arrays.equals(flow.getSrc(), packet.getSrc()) && (flow.getFwdFINFlags() == 0)) { + flow.updateActiveIdleTime(currentTimestamp,this.flowActivityTimeOut); + flow.addPacket(packet); + currentFlows.put(id,flow); + // + // Backward Flow and bwdFIN = 0 + // + } else if (flow.getBwdFINFlags() == 0) { + flow.updateActiveIdleTime(currentTimestamp,this.flowActivityTimeOut); + flow.addPacket(packet); + currentFlows.put(id,flow); + // + // FLOW already closed!!! + // + } else { + logger.warn("FLOW already closed! fwdFIN {} bwdFIN {}", flow.getFwdFINFlags(), flow.getBwdFINFlags()); + // TODO: we just discard the packet? + } } }else{ currentFlows.put(packet.fwdFlowId(), new BasicFlow(bidirectional,packet, this.flowActivityTimeOut)); From d212d6c403934f1c721163a4c103171a1d2a3514 Mon Sep 17 00:00:00 2001 From: Arash Habibi Lashkari Date: Mon, 5 Jul 2021 12:41:05 -0300 Subject: [PATCH 21/25] --- src/main/java/cic/cs/unb/ca/jnetpcap/BasicFlow.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/cic/cs/unb/ca/jnetpcap/BasicFlow.java b/src/main/java/cic/cs/unb/ca/jnetpcap/BasicFlow.java index f7b1b4cc..d8f05dee 100644 --- a/src/main/java/cic/cs/unb/ca/jnetpcap/BasicFlow.java +++ b/src/main/java/cic/cs/unb/ca/jnetpcap/BasicFlow.java @@ -681,7 +681,7 @@ public String dumpFlowBasedFeatures(){ dump+=fAvgBytesPerBulk()+","; dump+=fAvgPacketsPerBulk()+","; dump+=fAvgBulkRate()+","; - dump+=fAvgBytesPerBulk()+","; + dump+=bAvgBytesPerBulk()+","; dump+=bAvgPacketsPerBulk()+","; dump+=bAvgBulkRate()+","; From e2c2ff25ea17444449b1380a1d1c8bf8f647103f Mon Sep 17 00:00:00 2001 From: Arash Habibi Lashkari Date: Wed, 5 Jan 2022 10:39:20 -0500 Subject: [PATCH 22/25] Update LICENSE.txt --- LICENSE.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LICENSE.txt b/LICENSE.txt index befceff6..44631c14 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -13,6 +13,6 @@ Arash Habibi Lashkari, Gerard Draper-Gil, Mohammad Saiful Islam Mamun and Ali A. Gerard Drapper Gil, Arash Habibi Lashkari, Mohammad Mamun, Ali A. Ghorbani, "Characterization of Encrypted and VPN Traffic Using Time-Related Features", In Proceedings of the 2nd International Conference on Information Systems Security and Privacy(ICISSP 2016) , pages 407-414, Rome , Italy # Contributors: -Arash Habibi Lashkari: Researcher and Developer (Founder) +* [**Arash Habibi Lashkari:**](http://ahlashkari.com/index.asp): Researcher and Developer (Founder) Gerard Drapper: Researcher and Developer (Co-funder) muhammad saiful islam: Researcher and Developer (Co-funder) From 98a5ebad0df579cc8b43eedd3421b3ae87699901 Mon Sep 17 00:00:00 2001 From: Arash Habibi Lashkari Date: Wed, 5 Jan 2022 10:41:44 -0500 Subject: [PATCH 23/25] Update LICENSE.txt --- LICENSE.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LICENSE.txt b/LICENSE.txt index 44631c14..befceff6 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -13,6 +13,6 @@ Arash Habibi Lashkari, Gerard Draper-Gil, Mohammad Saiful Islam Mamun and Ali A. Gerard Drapper Gil, Arash Habibi Lashkari, Mohammad Mamun, Ali A. Ghorbani, "Characterization of Encrypted and VPN Traffic Using Time-Related Features", In Proceedings of the 2nd International Conference on Information Systems Security and Privacy(ICISSP 2016) , pages 407-414, Rome , Italy # Contributors: -* [**Arash Habibi Lashkari:**](http://ahlashkari.com/index.asp): Researcher and Developer (Founder) +Arash Habibi Lashkari: Researcher and Developer (Founder) Gerard Drapper: Researcher and Developer (Co-funder) muhammad saiful islam: Researcher and Developer (Co-funder) From 719b08079da48cdf2ad2012389f9296341c35de9 Mon Sep 17 00:00:00 2001 From: ekharisma Date: Sun, 3 Jul 2022 19:09:50 +0700 Subject: [PATCH 24/25] add kafka support --- .gitignore | 5 + build.gradle | 5 +- src/main/java/cic/cs/unb/ca/ifm/App.java | 14 +- .../java/cic/cs/unb/ca/ifm/CICFlowMeter.java | 5 - .../cic/cs/unb/ca/jnetpcap/BasicFlow.java | 189 +++++-- .../cic/cs/unb/ca/jnetpcap/FlowGenerator.java | 1 + .../worker/LoadPcapInterfaceWorker.java | 1 - .../ca/jnetpcap/worker/TrafficFlowWorker.java | 2 +- src/main/java/pens/ce/kafka/BaseKafka.java | 29 + src/main/java/pens/ce/model/FlowModel.java | 511 ++++++++++++++++++ 10 files changed, 708 insertions(+), 54 deletions(-) create mode 100644 .gitignore create mode 100644 src/main/java/pens/ce/kafka/BaseKafka.java create mode 100644 src/main/java/pens/ce/model/FlowModel.java diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..8cb990ee --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +.gradle +.idea +build +logs +data diff --git a/build.gradle b/build.gradle index d766d1f7..067ef458 100644 --- a/build.gradle +++ b/build.gradle @@ -29,6 +29,8 @@ dependencies { compile group: 'com.google.guava', name: 'guava', version: '23.6-jre' // https://mvnrepository.com/artifact/org.apache.tika/tika-core compile group: 'org.apache.tika', name: 'tika-core', version: '1.17' + compile group: 'com.google.code.gson', name: 'gson', version: '2.8.9' + compile group: 'org.apache.kafka', name: 'kafka-clients', version: '3.1.0' } sourceSets { @@ -94,8 +96,7 @@ task fatJar(type: Jar) { task execute(type: JavaExec) { println 'type JavaExec' - - + standardInput = System.in main = "cic.cs.unb.ca.ifm.App" //main class classpath = sourceSets.main.runtimeClasspath String osName = System.getProperty('os.name').toLowerCase() diff --git a/src/main/java/cic/cs/unb/ca/ifm/App.java b/src/main/java/cic/cs/unb/ca/ifm/App.java index 86b765eb..cbeb9d54 100644 --- a/src/main/java/cic/cs/unb/ca/ifm/App.java +++ b/src/main/java/cic/cs/unb/ca/ifm/App.java @@ -6,6 +6,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.awt.*; +import java.util.Scanner; public class App { public static final Logger logger = LoggerFactory.getLogger(App.class); @@ -18,19 +19,6 @@ public static void init() { * Launch the application. */ public static void main(String[] args) { - /*try { - //UIManager.setLookAndFeel(UIManager.getSystemLookAndFeelClassName()); - UIManager.setLookAndFeel("javax.swing.plaf.nimbus.NimbusLookAndFeel"); - } catch (ClassNotFoundException e1) { - e1.printStackTrace(); - } catch (InstantiationException e1) { - e1.printStackTrace(); - } catch (IllegalAccessException e1) { - e1.printStackTrace(); - } catch (UnsupportedLookAndFeelException e1) { - e1.printStackTrace(); - }*/ - EventQueue.invokeLater(() -> { try { init(); diff --git a/src/main/java/cic/cs/unb/ca/ifm/CICFlowMeter.java b/src/main/java/cic/cs/unb/ca/ifm/CICFlowMeter.java index c8f3bb6a..c2321e89 100644 --- a/src/main/java/cic/cs/unb/ca/ifm/CICFlowMeter.java +++ b/src/main/java/cic/cs/unb/ca/ifm/CICFlowMeter.java @@ -105,10 +105,5 @@ public boolean accept(File dir, String name) { } logger.info("\n\n----------------------------------------------------------------------------\n TOTAL FLOWS GENERATED: {}",totalFlows); logger.info("----------------------------------------------------------------------------\n"); -// try { -// Files.write(Paths.get("src/main/resources/executionLog.log"),flowGen.dumpFlows().getBytes()); -// } catch (IOException e) { -// e.printStackTrace(); -// } } } diff --git a/src/main/java/cic/cs/unb/ca/jnetpcap/BasicFlow.java b/src/main/java/cic/cs/unb/ca/jnetpcap/BasicFlow.java index d8f05dee..75d5f2e5 100644 --- a/src/main/java/cic/cs/unb/ca/jnetpcap/BasicFlow.java +++ b/src/main/java/cic/cs/unb/ca/jnetpcap/BasicFlow.java @@ -4,17 +4,25 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; + +import com.google.gson.Gson; import org.apache.commons.math3.stat.descriptive.SummaryStatistics; +import org.apache.kafka.clients.producer.KafkaProducer; import org.jnetpcap.packet.format.FormatUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import pens.ce.kafka.BaseKafka; +import pens.ce.model.FlowModel; public class BasicFlow { + private final static Logger logger = LoggerFactory.getLogger(BasicFlow.class); private final static String separator = ","; private SummaryStatistics fwdPktStats = null; private SummaryStatistics bwdPktStats = null; private List forward = null; private List backward = null; - + private final FlowModel flowModel = new FlowModel(); private long forwardBytes; private long backwardBytes; private long fHeaderBytes; @@ -45,7 +53,7 @@ public class BasicFlow { private long flowStartTime; private long startActiveTime; private long endActiveTime; - private String flowId = null; + private String flowId = ""; private SummaryStatistics flowIAT = null; private SummaryStatistics forwardIAT = null; @@ -1090,35 +1098,53 @@ public String getLabel() { public String dumpFlowBasedFeaturesEx() { StringBuilder dump = new StringBuilder(); - dump.append(flowId).append(separator); //1 + flowModel.setFlowId(flowId); dump.append(FormatUtils.ip(src)).append(separator); //2 - dump.append(getSrcPort()).append(separator); //3 - dump.append(FormatUtils.ip(dst)).append(separator); //4 - dump.append(getDstPort()).append(separator); //5 - dump.append(getProtocol()).append(separator); //6 - + flowModel.setSrcIp(FormatUtils.ip(src)); + dump.append(getSrcPort()).append(separator); //3 + flowModel.setSrcPort(getSrcPort()); + dump.append(FormatUtils.ip(dst)).append(separator); //4 + flowModel.setDstIp(FormatUtils.ip(dst)); + dump.append(getDstPort()).append(separator); //5 + flowModel.setDstPort(getDstPort()); + dump.append(getProtocol()).append(separator); //6 + flowModel.setProtocol(getProtocol()); + String starttime = DateFormatter.convertMilliseconds2String(flowStartTime/1000L, "dd/MM/yyyy hh:mm:ss a"); dump.append(starttime).append(separator); //7 - + flowModel.setStartTime(starttime); long flowDuration = flowLastSeen - flowStartTime; dump.append(flowDuration).append(separator); //8 - + flowModel.setFlowDuration(flowDuration); + dump.append(fwdPktStats.getN()).append(separator); //9 - dump.append(bwdPktStats.getN()).append(separator); //10 - dump.append(fwdPktStats.getSum()).append(separator); //11 - dump.append(bwdPktStats.getSum()).append(separator); //12 - + flowModel.setFwdPktCount(fwdPktStats.getN()); + dump.append(bwdPktStats.getN()).append(separator); //10 + flowModel.setBwdPktCount(bwdPktStats.getN()); + dump.append(fwdPktStats.getSum()).append(separator); //11 + flowModel.setFwdPktStats(fwdPktStats.getSum()); + dump.append(bwdPktStats.getSum()).append(separator); //12 + flowModel.setBwdPktStats(bwdPktStats.getSum()); + if(fwdPktStats.getN() > 0L) { dump.append(fwdPktStats.getMax()).append(separator); //13 - dump.append(fwdPktStats.getMin()).append(separator); //14 - dump.append(fwdPktStats.getMean()).append(separator); //15 - dump.append(fwdPktStats.getStandardDeviation()).append(separator); //16 - }else { + flowModel.setFwdPktMax(fwdPktStats.getMax()); + dump.append(fwdPktStats.getMin()).append(separator); //14 + flowModel.setFwdPktMin(fwdPktStats.getMin()); + dump.append(fwdPktStats.getMean()).append(separator); //15 + flowModel.setFwdPktMean(fwdPktStats.getMean()); + dump.append(fwdPktStats.getStandardDeviation()).append(separator); //16 + flowModel.setFwdPktStd(fwdPktStats.getStandardDeviation()); + }else { dump.append(0).append(separator); dump.append(0).append(separator); dump.append(0).append(separator); dump.append(0).append(separator); + flowModel.setFwdPktMax(0); + flowModel.setFwdPktMin(0); + flowModel.setFwdPktMean(0); + flowModel.setFwdPktStd(0); } if(bwdPktStats.getN() > 0L) { @@ -1126,11 +1152,19 @@ public String dumpFlowBasedFeaturesEx() { dump.append(bwdPktStats.getMin()).append(separator); //18 dump.append(bwdPktStats.getMean()).append(separator); //19 dump.append(bwdPktStats.getStandardDeviation()).append(separator); //20 + flowModel.setBwdPktMax(bwdPktStats.getMax()); + flowModel.setBwdPktMin(bwdPktStats.getMin()); + flowModel.setBwdPktMean(bwdPktStats.getMean()); + flowModel.setBwdPktStd(bwdPktStats.getStandardDeviation()); }else{ dump.append(0).append(separator); dump.append(0).append(separator); dump.append(0).append(separator); dump.append(0).append(separator); + flowModel.setBwdPktMax(0); + flowModel.setBwdPktMin(0); + flowModel.setBwdPktMean(0); + flowModel.setBwdPktStd(0); } dump.append(((double)(forwardBytes+backwardBytes))/((double)flowDuration/1000000L)).append(separator);//21 dump.append(((double)packetCount())/((double)flowDuration/1000000L)).append(separator);//22 @@ -1138,13 +1172,23 @@ public String dumpFlowBasedFeaturesEx() { dump.append(flowIAT.getStandardDeviation()).append(separator); //24 dump.append(flowIAT.getMax()).append(separator); //25 dump.append(flowIAT.getMin()).append(separator); //26 - + flowModel.setFlowBytePs(((double)(forwardBytes+backwardBytes))/((double)flowDuration/1000000L)); + flowModel.setFlowPktPs(((double)packetCount())/((double)flowDuration/1000000L)); + flowModel.setFlowIATMean(flowIAT.getMean()); + flowModel.setFlowIATStd(flowIAT.getStandardDeviation()); + flowModel.setFlowIATMax(flowIAT.getMax()); + flowModel.setFlowIATMin(flowIAT.getMin()); if(this.forward.size()>1){ dump.append(forwardIAT.getSum()).append(separator); //27 dump.append(forwardIAT.getMean()).append(separator); //28 dump.append(forwardIAT.getStandardDeviation()).append(separator); //29 dump.append(forwardIAT.getMax()).append(separator); //30 dump.append(forwardIAT.getMin()).append(separator); //31 + flowModel.setFwdIATSum(forwardIAT.getSum()); + flowModel.setFwdIATMean(forwardIAT.getMean()); + flowModel.setFwdIATStd(forwardIAT.getStandardDeviation()); + flowModel.setFwdIATMax(forwardIAT.getMax()); + flowModel.setFwdIATMin(forwardIAT.getMin()); }else{ dump.append(0).append(separator); @@ -1152,6 +1196,11 @@ public String dumpFlowBasedFeaturesEx() { dump.append(0).append(separator); dump.append(0).append(separator); dump.append(0).append(separator); + flowModel.setFwdIATSum(0); + flowModel.setFwdIATMean(0); + flowModel.setFwdIATStd(0); + flowModel.setFwdIATMax(0); + flowModel.setFwdIATMin(0); } if(this.backward.size()>1){ dump.append(backwardIAT.getSum()).append(separator); //32 @@ -1159,24 +1208,41 @@ public String dumpFlowBasedFeaturesEx() { dump.append(backwardIAT.getStandardDeviation()).append(separator); //34 dump.append(backwardIAT.getMax()).append(separator); //35 dump.append(backwardIAT.getMin()).append(separator); //36 - }else{ + flowModel.setBwdIATSum(backwardIAT.getSum()); + flowModel.setBwdIATMean(backwardIAT.getMean()); + flowModel.setBwdIATStd(backwardIAT.getStandardDeviation()); + flowModel.setBwdIATMax(backwardIAT.getMax()); + flowModel.setBwdIATMin(backwardIAT.getMin()); + }else{ dump.append(0).append(separator); dump.append(0).append(separator); dump.append(0).append(separator); dump.append(0).append(separator); dump.append(0).append(separator); + flowModel.setBwdIATSum(0); + flowModel.setBwdIATMean(0); + flowModel.setBwdIATStd(0); + flowModel.setBwdIATMax(0); + flowModel.setBwdIATMin(0); } dump.append(fPSH_cnt).append(separator); //37 dump.append(bPSH_cnt).append(separator); //38 dump.append(fURG_cnt).append(separator); //39 dump.append(bURG_cnt).append(separator); //40 - + flowModel.setFwdPshCount(fPSH_cnt); + flowModel.setBwdPshCount(bPSH_cnt); + flowModel.setFwdUrgCount(fURG_cnt); + flowModel.setBwdUrgCount(bURG_cnt); + dump.append(fHeaderBytes).append(separator); //41 dump.append(bHeaderBytes).append(separator); //42 dump.append(getfPktsPerSecond()).append(separator); //43 dump.append(getbPktsPerSecond()).append(separator); //44 - + flowModel.setFwdHeaderLen(getfPktsPerSecond()); + flowModel.setBwdHeaderLen(getbPktsPerSecond()); + flowModel.setFwdPktPS(getbPktsPerSecond()); + flowModel.setBwdPktPS(getbPktsPerSecond()); if(this.forward.size() > 0 || this.backward.size() > 0){ dump.append(flowLengthStats.getMin()).append(separator); //45 @@ -1184,12 +1250,23 @@ public String dumpFlowBasedFeaturesEx() { dump.append(flowLengthStats.getMean()).append(separator); //47 dump.append(flowLengthStats.getStandardDeviation()).append(separator); //48 dump.append(flowLengthStats.getVariance()).append(separator); //49 + flowModel.setPktLenMin(flowLengthStats.getMin()); + flowModel.setPktLenMax(flowLengthStats.getMax()); + flowModel.setPktLenMean(flowLengthStats.getMean()); + flowModel.setPktLenStd(flowLengthStats.getStandardDeviation()); + flowModel.setPktLenVar(flowLengthStats.getVariance()); }else{//seem to less one dump.append(0).append(separator); dump.append(0).append(separator); dump.append(0).append(separator); dump.append(0).append(separator); dump.append(0).append(separator); + flowModel.setPktLenMin(0); + flowModel.setPktLenMax(0); + flowModel.setPktLenMean(0); + flowModel.setPktLenStd(0); + flowModel.setPktLenVar(0); + } /*for(MutableInt v:flagCounts.values()) { @@ -1206,41 +1283,79 @@ public String dumpFlowBasedFeaturesEx() { dump.append(flagCounts.get("URG").value).append(separator); //55 dump.append(flagCounts.get("CWR").value).append(separator); //56 dump.append(flagCounts.get("ECE").value).append(separator); //57 - + + flowModel.setFinFlagCnt(flagCounts.get("FIN").value); + flowModel.setSynFlagCnt(flagCounts.get("SYN").value); + flowModel.setRstFlagCnt(flagCounts.get("RST").value); + flowModel.setPshFlagCnt(flagCounts.get("PSH").value); + flowModel.setAckFlagCnt(flagCounts.get("ACK").value); + flowModel.setUrgFlagCnt(flagCounts.get("URG").value); + flowModel.setCwrFlagCnt(flagCounts.get("CWR").value); + flowModel.setEceFlagCnt(flagCounts.get("ECE").value); + dump.append(getDownUpRatio()).append(separator); //58 dump.append(getAvgPacketSize()).append(separator); //59 dump.append(fAvgSegmentSize()).append(separator); //60 dump.append(bAvgSegmentSize()).append(separator); //61 //dump.append(fHeaderBytes).append(separator); //62 dupicate with 41 - + + flowModel.setUpDownRatio(getDownUpRatio()); + flowModel.setAvgPktSize(getAvgPacketSize()); + flowModel.setFwdAvgSegmentSize(fAvgSegmentSize()); + flowModel.setBwdAvgSegmentSize(bAvgSegmentSize()); + dump.append(fAvgBytesPerBulk()).append(separator); //63 dump.append(fAvgPacketsPerBulk()).append(separator); //64 dump.append(fAvgBulkRate()).append(separator); //65 dump.append(bAvgBytesPerBulk()).append(separator); //66 dump.append(bAvgPacketsPerBulk()).append(separator); //67 dump.append(bAvgBulkRate()).append(separator); //68 - + + flowModel.setFwdAvgBytesPBulk(fAvgBytesPerBulk()); + flowModel.setFwdAbgPktPBulk(fAvgPacketsPerBulk()); + flowModel.setFwdAvgBulkRate(fAvgBulkRate()); + flowModel.setBwdAvgBytesPBulk(fAvgBytesPerBulk()); + flowModel.setBwdAvgPktPBulk(fAvgPacketsPerBulk()); + flowModel.setBwdAvgBulkRate(fAvgBulkRate()); + dump.append(getSflow_fpackets()).append(separator); //69 dump.append(getSflow_fbytes()).append(separator); //70 dump.append(getSflow_bpackets()).append(separator); //71 dump.append(getSflow_bbytes()).append(separator); //72 - + + flowModel.setFwdSubFlowPkt(getSflow_fpackets()); + flowModel.setFwdSubFlowBytes(getSflow_fbytes()); + flowModel.setBwdSubFlowPkt(getSflow_bpackets()); + flowModel.setBwdSubFlowBytes(getSflow_bbytes()); + dump.append(Init_Win_bytes_forward).append(separator); //73 dump.append(Init_Win_bytes_backward).append(separator); //74 dump.append(Act_data_pkt_forward).append(separator); //75 dump.append(min_seg_size_forward).append(separator); //76 - + + flowModel.setInitWinBytesFwd(Init_Win_bytes_backward); + flowModel.setInitWinBytesBwd(Init_Win_bytes_backward); + flowModel.setFwdActData(Act_data_pkt_forward); + flowModel.setFwdSegSize(min_seg_size_forward); if(this.flowActive.getN()>0){ dump.append(flowActive.getMean()).append(separator); //77 dump.append(flowActive.getStandardDeviation()).append(separator); //78 dump.append(flowActive.getMax()).append(separator); //79 dump.append(flowActive.getMin()).append(separator); //80 - }else{ + flowModel.setFlowActiveMean(flowActive.getMean()); + flowModel.setFlowActiveStd(flowActive.getStandardDeviation()); + flowModel.setFlowActiveMax(flowActive.getMax()); + flowModel.setFlowActiveMin(flowActive.getMin()); + }else{ dump.append(0).append(separator); dump.append(0).append(separator); dump.append(0).append(separator); dump.append(0).append(separator); + flowModel.setFlowActiveMean(0); + flowModel.setFlowActiveStd(0); + flowModel.setFlowActiveMax(0); + flowModel.setFlowActiveMin(0); } if(this.flowIdle.getN()>0){ @@ -1248,16 +1363,26 @@ public String dumpFlowBasedFeaturesEx() { dump.append(flowIdle.getStandardDeviation()).append(separator); //82 dump.append(flowIdle.getMax()).append(separator); //83 dump.append(flowIdle.getMin()).append(separator); //84 - }else{ + flowModel.setFlowIdleMean(flowIdle.getMean()); + flowModel.setFlowIdleStd(flowIdle.getStandardDeviation()); + flowModel.setFlowIdleMax(flowIdle.getMax()); + flowModel.setFlowIdleMin(flowIdle.getMin()); + }else{ dump.append(0).append(separator); dump.append(0).append(separator); dump.append(0).append(separator); dump.append(0).append(separator); + flowModel.setFlowIdleMean(0); + flowModel.setFlowIdleStd(0); + flowModel.setFlowIdleMax(0); + flowModel.setFlowIdleMin(0); } - dump.append(getLabel()); - - + Gson gson = new Gson(); + String json = gson.toJson(flowModel); + BaseKafka baseKafka = new BaseKafka(); + KafkaProducer kafkaProducer = baseKafka.getKafkaProducer("localhost:9092"); + baseKafka.sendPayload(kafkaProducer, "sniffer-data", json); return dump.toString(); } } diff --git a/src/main/java/cic/cs/unb/ca/jnetpcap/FlowGenerator.java b/src/main/java/cic/cs/unb/ca/jnetpcap/FlowGenerator.java index 6f546975..f0fca4b3 100644 --- a/src/main/java/cic/cs/unb/ca/jnetpcap/FlowGenerator.java +++ b/src/main/java/cic/cs/unb/ca/jnetpcap/FlowGenerator.java @@ -152,6 +152,7 @@ public void addPacket(BasicPacketInfo packet){ // some error // TODO: review what to do with the packet logger.warn("Forward flow received {} FIN packets", flow.getFwdFINFlags()); + logger.warn("Coba dapat data : " + flow.getProtocol()); } // // Backward Flow diff --git a/src/main/java/cic/cs/unb/ca/jnetpcap/worker/LoadPcapInterfaceWorker.java b/src/main/java/cic/cs/unb/ca/jnetpcap/worker/LoadPcapInterfaceWorker.java index 87342958..1a69d196 100644 --- a/src/main/java/cic/cs/unb/ca/jnetpcap/worker/LoadPcapInterfaceWorker.java +++ b/src/main/java/cic/cs/unb/ca/jnetpcap/worker/LoadPcapInterfaceWorker.java @@ -21,7 +21,6 @@ public LoadPcapInterfaceWorker() { @Override protected List doInBackground() throws Exception { - StringBuilder errbuf = new StringBuilder(); List ifs = new ArrayList<>(); if(Pcap.findAllDevs(ifs, errbuf)!=Pcap.OK) { diff --git a/src/main/java/cic/cs/unb/ca/jnetpcap/worker/TrafficFlowWorker.java b/src/main/java/cic/cs/unb/ca/jnetpcap/worker/TrafficFlowWorker.java index b807b837..f79c1bba 100644 --- a/src/main/java/cic/cs/unb/ca/jnetpcap/worker/TrafficFlowWorker.java +++ b/src/main/java/cic/cs/unb/ca/jnetpcap/worker/TrafficFlowWorker.java @@ -28,7 +28,7 @@ public TrafficFlowWorker(String device) { @Override protected String doInBackground() { - FlowGenerator flowGen = new FlowGenerator(true,120000000L, 5000000L); + FlowGenerator flowGen = new FlowGenerator(true,120000000L, 5000000L); flowGen.addFlowListener(this); int snaplen = 64 * 1024;//2048; // Truncate packet at this size int promiscous = Pcap.MODE_PROMISCUOUS; diff --git a/src/main/java/pens/ce/kafka/BaseKafka.java b/src/main/java/pens/ce/kafka/BaseKafka.java new file mode 100644 index 00000000..3c64b959 --- /dev/null +++ b/src/main/java/pens/ce/kafka/BaseKafka.java @@ -0,0 +1,29 @@ +package pens.ce.kafka; + +import org.apache.kafka.clients.producer.KafkaProducer; +import org.apache.kafka.clients.producer.Producer; +import org.apache.kafka.clients.producer.ProducerConfig; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.apache.kafka.common.serialization.StringSerializer; + +import java.util.Properties; + +public class BaseKafka { + private KafkaProducer kafkaProducer; + + public KafkaProducer getKafkaProducer(String host) { + if (kafkaProducer == null){ + Properties properties = new Properties(); + properties.setProperty(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, host); + properties.setProperty(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName()); + properties.setProperty(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName()); + this.kafkaProducer = new KafkaProducer<>(properties); + } + return this.kafkaProducer; + } + + public void sendPayload(Producer producer, String topic, String message) { + producer.send(new ProducerRecord<>(topic, message)); + producer.close(); + } +} diff --git a/src/main/java/pens/ce/model/FlowModel.java b/src/main/java/pens/ce/model/FlowModel.java new file mode 100644 index 00000000..f9b71db1 --- /dev/null +++ b/src/main/java/pens/ce/model/FlowModel.java @@ -0,0 +1,511 @@ +package pens.ce.model; + +import com.google.gson.annotations.SerializedName; + + +public class FlowModel { + @SerializedName("flowId") + String flowId; + @SerializedName("srcIp") + String srcIp; + @SerializedName("srcPort") + int srcPort; + @SerializedName("dstIp") + String dstIp; + @SerializedName("dstPort") + int dstPort; + @SerializedName("protocol") + int protocol; + @SerializedName("starttime") + String startTime; + @SerializedName("flowDuration") + long flowDuration; + @SerializedName("fwdPktCount") + long fwdPktCount; + @SerializedName("bwdPktCount") + long bwdPktCount; + @SerializedName("fwdPktStats") + double fwdPktStats; + @SerializedName("bwdPktStats") + double bwdPktStats; + @SerializedName("fwdPktMax") + double fwdPktMax; + @SerializedName("fwdPktMin") + double fwdPktMin; + @SerializedName("fwdPktMean") + double fwdPktMean; + @SerializedName("fwdPktStd") + double fwdPktStd; + @SerializedName("bwdPktMax") + double bwdPktMax; + @SerializedName("bwdPktMin") + double bwdPktMin; + @SerializedName("bwdPktMean") + double bwdPktMean; + @SerializedName("bwdPktStd") + double bwdPktStd; + @SerializedName("flowBytePs") + double flowBytePs; + @SerializedName("flowPktPs") + double flowPktPs; + @SerializedName("flowIATMean") + double flowIATMean; + @SerializedName("flowIATStd") + double flowIATStd; + @SerializedName("flowIATMax") + double flowIATMax; + @SerializedName("flowIATMin") + double flowIATMin; + @SerializedName("fwdIATSum") + double fwdIATSum; + @SerializedName("fwdIATMean") + double fwdIATMean; + @SerializedName("fwdIATStd") + double fwdIATStd; + @SerializedName("fwdIATMax") + double fwdIATMax; + @SerializedName("fwdIATMin") + double fwdIATMin; + @SerializedName("bwdIATSum") + double bwdIATSum; + @SerializedName("bwdIATMean") + double bwdIATMean; + @SerializedName("bwdIATStd") + double bwdIATStd; + @SerializedName("bwdIATMax") + double bwdIATMax; + @SerializedName("bwdIATMin") + double bwdIATMin; + @SerializedName("fwdPshCount") + int fwdPshCount; + @SerializedName("bwdPshCount") + int bwdPshCount; + @SerializedName("fwdUrgCount") + int fwdUrgCount; + @SerializedName("bwdUrgCount") + int bwdUrgCount; + @SerializedName("fwdHeaderLen") + double fwdHeaderLen; + @SerializedName("bwdHeaderLen") + double bwdHeaderLen; + @SerializedName("fwdPktPS") + double fwdPktPS; + @SerializedName("bwdPktPS") + double bwdPktPS; + @SerializedName("pktLenMin") + double PktLenMin; + @SerializedName("pktLenMax") + double PktLenMax; + @SerializedName("pktLenMean") + double pktLenMean; + @SerializedName("pktLenStd") + double pktLenStd; + @SerializedName("pktLenVar") + double pktLenVar; + @SerializedName("finFlagCnt") + int finFlagCnt; + @SerializedName("synFlagCnt") + int synFlagCnt; + @SerializedName("rstFlagCnt") + int rstFlagCnt; + @SerializedName("pshFlagCnt") + int pshFlagCnt; + @SerializedName("ackFlagCnt") + int ackFlagCnt; + @SerializedName("urgFlagCnt") + int urgFlagCnt; + @SerializedName("cwrFlagCnt") + int cwrFlagCnt; + @SerializedName("eceFlagCnt") + int eceFlagCnt; + @SerializedName("upDownRatio") + double upDownRatio; + @SerializedName("avgPktSize") + double avgPktSize; + @SerializedName("fwdAvgSegmentSize") + double fwdAvgSegmentSize; + @SerializedName("bwdAvgSegmentSize") + double bwdAvgSegmentSize; + @SerializedName("fwdAvgBytesPBulk") + long fwdAvgBytesPBulk; + @SerializedName("fwdAvgPktPBulk") + long fwdAbgPktPBulk; + @SerializedName("fwdAvgBulkRate") + long fwdAvgBulkRate; + @SerializedName("bwdAvgBytesPBulk") + long bwdAvgBytesPBulk; + @SerializedName("bwdAvgPktPBulk") + long bwdAvgPktPBulk; + @SerializedName("bwdAvgBulkRate") + long bwdAvgBulkRate; + @SerializedName("fwdSubFlowPkt") + long fwdSubFlowPkt; + @SerializedName("fwdSubFlowBytes") + long fwdSubFlowBytes; + @SerializedName("bwdSubFlowPkt") + long bwdSubFlowPkt; + @SerializedName("bwdSubFlowBytes") + long bwdSubFlowBytes; + @SerializedName("initWinBytesFwd") + int initWinBytesFwd; + @SerializedName("initWinBytesBwd") + int initWinBytesBwd; + @SerializedName("fwdActData") + long fwdActData; + @SerializedName("fwdSegSize") + long fwdSegSize; + @SerializedName("flowActiveMean") + double flowActiveMean; + @SerializedName("flowActiveStd") + double flowActiveStd; + @SerializedName("flowActiveMax") + double flowActiveMax; + @SerializedName("flowActiveMin") + double flowActiveMin; + @SerializedName("flowIdleMean") + double flowIdleMean; + @SerializedName("flowIdleStd") + double flowIdleStd; + @SerializedName("flowIdleMax") + double flowIdleMax; + @SerializedName("flowIdleMin") + double flowIdleMin; + @SerializedName("label") + String label; + + public void setFlowId(String flowId) { + this.flowId = flowId; + } + + public void setSrcIp(String srcIp) { + this.srcIp = srcIp; + } + + public void setSrcPort(int srcPort) { + this.srcPort = srcPort; + } + + public void setDstIp(String dstIp) { + this.dstIp = dstIp; + } + + public void setDstPort(int dstPort) { + this.dstPort = dstPort; + } + + public void setProtocol(int protocol) { + this.protocol = protocol; + } + + public void setStartTime(String startTime) { + this.startTime = startTime; + } + + public void setFlowDuration(long flowDuration) { + this.flowDuration = flowDuration; + } + + public void setFwdPktCount(long fwdPktCount) { + this.fwdPktCount = fwdPktCount; + } + + public void setBwdPktCount(long bwdPktCount) { + this.bwdPktCount = bwdPktCount; + } + + public void setFwdPktStats(double fwdPktStats) { + this.fwdPktStats = fwdPktStats; + } + + public void setBwdPktStats(double bwdPktStats) { + this.bwdPktStats = bwdPktStats; + } + + public void setFwdPktMax(double fwdPktMax) { + this.fwdPktMax = fwdPktMax; + } + + public void setFwdPktMin(double fwdPktMin) { + this.fwdPktMin = fwdPktMin; + } + + public void setFwdPktMean(double fwdPktMean) { + this.fwdPktMean = fwdPktMean; + } + + public void setFwdPktStd(double fwdPktStd) { + this.fwdPktStd = fwdPktStd; + } + + public void setBwdPktMax(double bwdPktMax) { + this.bwdPktMax = bwdPktMax; + } + + public void setBwdPktMin(double bwdPktMin) { + this.bwdPktMin = bwdPktMin; + } + + public void setBwdPktMean(double bwdPktMean) { + this.bwdPktMean = bwdPktMean; + } + + public void setBwdPktStd(double bwdPktStd) { + this.bwdPktStd = bwdPktStd; + } + + public void setFlowBytePs(double flowBytePs) { + this.flowBytePs = flowBytePs; + } + + public void setFlowPktPs(double flowPktPs) { + this.flowPktPs = flowPktPs; + } + + public void setFlowIATMean(double flowIATMean) { + this.flowIATMean = flowIATMean; + } + + public void setFlowIATStd(double flowIATStd) { + this.flowIATStd = flowIATStd; + } + + public void setFlowIATMax(double flowIATMax) { + this.flowIATMax = flowIATMax; + } + + public void setFlowIATMin(double flowIATMin) { + this.flowIATMin = flowIATMin; + } + + public void setFwdIATSum(double fwdIATSum) { + this.fwdIATSum = fwdIATSum; + } + + public void setFwdIATMean(double fwdIATMean) { + this.fwdIATMean = fwdIATMean; + } + + public void setFwdIATStd(double fwdIATStd) { + this.fwdIATStd = fwdIATStd; + } + + public void setFwdIATMax(double fwdIATMax) { + this.fwdIATMax = fwdIATMax; + } + + public void setFwdIATMin(double fwdIATMin) { + this.fwdIATMin = fwdIATMin; + } + + public void setBwdIATSum(double bwdIATSum) { + this.bwdIATSum = bwdIATSum; + } + + public void setBwdIATMean(double bwdIATMean) { + this.bwdIATMean = bwdIATMean; + } + + public void setBwdIATStd(double bwdIATStd) { + this.bwdIATStd = bwdIATStd; + } + + public void setBwdIATMax(double bwdIATMax) { + this.bwdIATMax = bwdIATMax; + } + + public void setBwdIATMin(double bwdIATMin) { + this.bwdIATMin = bwdIATMin; + } + + public void setFwdPshCount(int fwdPshCount) { + this.fwdPshCount = fwdPshCount; + } + + public void setBwdPshCount(int bwdPshCount) { + this.bwdPshCount = bwdPshCount; + } + + public void setFwdUrgCount(int fwdUrgCount) { + this.fwdUrgCount = fwdUrgCount; + } + + public void setBwdUrgCount(int bwdUrgCount) { + this.bwdUrgCount = bwdUrgCount; + } + + public void setFwdHeaderLen(double fwdHeaderLen) { + this.fwdHeaderLen = fwdHeaderLen; + } + + public void setBwdHeaderLen(double bwdHeaderLen) { + this.bwdHeaderLen = bwdHeaderLen; + } + + public void setFwdPktPS(double fwdPktPS) { + this.fwdPktPS = fwdPktPS; + } + + public void setBwdPktPS(double bwdPktPS) { + this.bwdPktPS = bwdPktPS; + } + + public void setPktLenMin(double pktLenMin) { + PktLenMin = pktLenMin; + } + + public void setPktLenMax(double pktLenMax) { + PktLenMax = pktLenMax; + } + + public void setPktLenMean(double pktLenMean) { + this.pktLenMean = pktLenMean; + } + + public void setPktLenStd(double pktLenStd) { + this.pktLenStd = pktLenStd; + } + + public void setPktLenVar(double pktLenVar) { + this.pktLenVar = pktLenVar; + } + + public void setFinFlagCnt(int finFlagCnt) { + this.finFlagCnt = finFlagCnt; + } + + public void setSynFlagCnt(int synFlagCnt) { + this.synFlagCnt = synFlagCnt; + } + + public void setRstFlagCnt(int rstFlagCnt) { + this.rstFlagCnt = rstFlagCnt; + } + + public void setPshFlagCnt(int pshFlagCnt) { + this.pshFlagCnt = pshFlagCnt; + } + + public void setAckFlagCnt(int ackFlagCnt) { + this.ackFlagCnt = ackFlagCnt; + } + + public void setUrgFlagCnt(int urgFlagCnt) { + this.urgFlagCnt = urgFlagCnt; + } + + public void setCwrFlagCnt(int cwrFlagCnt) { + this.cwrFlagCnt = cwrFlagCnt; + } + + public void setEceFlagCnt(int eceFlagCnt) { + this.eceFlagCnt = eceFlagCnt; + } + + public void setUpDownRatio(double upDownRatio) { + this.upDownRatio = upDownRatio; + } + + public void setAvgPktSize(double avgPktSize) { + this.avgPktSize = avgPktSize; + } + + public void setFwdAvgSegmentSize(double fwdAvgSegmentSize) { + this.fwdAvgSegmentSize = fwdAvgSegmentSize; + } + + public void setBwdAvgSegmentSize(double bwdAvgSegmentSize) { + this.bwdAvgSegmentSize = bwdAvgSegmentSize; + } + + public void setFwdAvgBytesPBulk(long fwdAvgBytesPBulk) { + this.fwdAvgBytesPBulk = fwdAvgBytesPBulk; + } + + public void setFwdAbgPktPBulk(long fwdAbgPktPBulk) { + this.fwdAbgPktPBulk = fwdAbgPktPBulk; + } + + public void setFwdAvgBulkRate(long fwdAvgBulkRate) { + this.fwdAvgBulkRate = fwdAvgBulkRate; + } + + public void setBwdAvgBytesPBulk(long bwdAvgBytesPBulk) { + this.bwdAvgBytesPBulk = bwdAvgBytesPBulk; + } + + public void setBwdAvgPktPBulk(long bwdAvgPktPBulk) { + this.bwdAvgPktPBulk = bwdAvgPktPBulk; + } + + public void setBwdAvgBulkRate(long bwdAvgBulkRate) { + this.bwdAvgBulkRate = bwdAvgBulkRate; + } + + public void setFwdSubFlowPkt(long fwdSubFlowPkt) { + this.fwdSubFlowPkt = fwdSubFlowPkt; + } + + public void setFwdSubFlowBytes(long fwdSubFlowBytes) { + this.fwdSubFlowBytes = fwdSubFlowBytes; + } + + public void setBwdSubFlowPkt(long bwdSubFlowPkt) { + this.bwdSubFlowPkt = bwdSubFlowPkt; + } + + public void setBwdSubFlowBytes(long bwdSubFlowBytes) { + this.bwdSubFlowBytes = bwdSubFlowBytes; + } + + public void setInitWinBytesFwd(int initWinBytesFwd) { + this.initWinBytesFwd = initWinBytesFwd; + } + + public void setInitWinBytesBwd(int initWinBytesBwd) { + this.initWinBytesBwd = initWinBytesBwd; + } + + public void setFwdActData(long fwdActData) { + this.fwdActData = fwdActData; + } + + public void setFwdSegSize(long fwdSegSize) { + this.fwdSegSize = fwdSegSize; + } + + public void setFlowActiveMean(double flowActiveMean) { + this.flowActiveMean = flowActiveMean; + } + + public void setFlowActiveStd(double flowActiveStd) { + this.flowActiveStd = flowActiveStd; + } + + public void setFlowActiveMax(double flowActiveMax) { + this.flowActiveMax = flowActiveMax; + } + + public void setFlowActiveMin(double flowActiveMin) { + this.flowActiveMin = flowActiveMin; + } + + public void setFlowIdleMean(double flowIdleMean) { + this.flowIdleMean = flowIdleMean; + } + + public void setFlowIdleStd(double flowIdleStd) { + this.flowIdleStd = flowIdleStd; + } + + public void setFlowIdleMax(double flowIdleMax) { + this.flowIdleMax = flowIdleMax; + } + + public void setFlowIdleMin(double flowIdleMin) { + this.flowIdleMin = flowIdleMin; + } + + public void setLabel(String label) { + this.label = label; + } +} From c601bb1970277582b832519578823b85e8049c12 Mon Sep 17 00:00:00 2001 From: ekharisma Date: Wed, 27 Jul 2022 23:38:57 +0700 Subject: [PATCH 25/25] edit basic packet info --- src/main/java/cic/cs/unb/ca/jnetpcap/BasicPacketInfo.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/cic/cs/unb/ca/jnetpcap/BasicPacketInfo.java b/src/main/java/cic/cs/unb/ca/jnetpcap/BasicPacketInfo.java index fdc4eadf..61c81ae4 100644 --- a/src/main/java/cic/cs/unb/ca/jnetpcap/BasicPacketInfo.java +++ b/src/main/java/cic/cs/unb/ca/jnetpcap/BasicPacketInfo.java @@ -1,6 +1,7 @@ package cic.cs.unb.ca.jnetpcap; import java.util.Arrays; +import java.util.UUID; import org.jnetpcap.packet.format.FormatUtils;