diff --git a/src/main/java/cic/cs/unb/ca/jnetpcap/BasicFlow.java b/src/main/java/cic/cs/unb/ca/jnetpcap/BasicFlow.java index 35ad16b4..5e5fe967 100644 --- a/src/main/java/cic/cs/unb/ca/jnetpcap/BasicFlow.java +++ b/src/main/java/cic/cs/unb/ca/jnetpcap/BasicFlow.java @@ -5,11 +5,44 @@ import org.apache.commons.math3.stat.descriptive.SummaryStatistics; import org.jnetpcap.packet.format.FormatUtils; + +// Layer and PDU terminology +// PDU (Protocol Data Unit): the data unit at a given network layer, including +// that layer's header plus everything it encapsulates +// L3 PDU = packet = IP datagram = L3 (IP) header + L4 header + L4 payload +// L4 PDU = segment = L4 unit = L4 (TCP/UDP/SCTP) header + L4 payload +// "segment" is used loosely throughout to mean any L4 PDU regardless of protocol +// strictly speaking, the correct terms should be: TCP segment, UDP datagram, etc. + + public class BasicFlow { private final static String separator = ","; + + // L3 PDU length (i.e., full wire-level packet length) statistics + // original bug: these stats were fed only L4 payload bytes, so "packet length" + // features actually described payload size, not IP datagram size + // fix: packet length = L3 header + L4 header + L4 payload private SummaryStatistics fwdPktStats = null; private SummaryStatistics bwdPktStats = null; + private SummaryStatistics flowLengthStats = null; + + // L4 payload length statistics + // originally mislabeled as "packet" length stats + private SummaryStatistics fwdSegPayloadStats = null; + private SummaryStatistics bwdSegPayloadStats = null; + private SummaryStatistics bidirSegPayloadStats = null; + + // L4 PDU ("segment") length statistics + // original bugs: + // - bug 1: the fwd/bwd segment length mean values were the same as + // the old packet length mean values (i.e., only the L4 payloads were considered) + // - bug 2: the fwd/bwd segment length min values were calculated as the L4 header length min values + // fix: L4 PDU length = segment length = L4 header + L4 payload + // note: there were not max and std feature for the segment lengths + private SummaryStatistics fwdSegmentStats = null; + private SummaryStatistics bwdSegmentStats = null; + private List forward = null; private List backward = null; @@ -18,6 +51,12 @@ public class BasicFlow { private long fHeaderBytes; private long bHeaderBytes; + // fwd/bwd segment header (L4 header) length min + // original bug: these were stored in the values "min_seg_size_forward" and "min_seg_size_backward" + // fix: variable renaming to reflect that these are L4 header length stats, not segment length stats + private long fwdSegHeaderLenMin; + private long bwdSegHeaderLenMin; + // Is always the value true in this application private boolean isBidirectional; @@ -34,8 +73,6 @@ public class BasicFlow { private long Act_data_pkt_forward; private long Act_data_pkt_backward; - private long min_seg_size_forward; - private long min_seg_size_backward; private int Init_Win_bytes_forward = 0; private int Init_Win_bytes_backward = 0; @@ -53,7 +90,6 @@ public class BasicFlow { private SummaryStatistics flowIAT = null; private SummaryStatistics forwardIAT = null; private SummaryStatistics backwardIAT = null; - private SummaryStatistics flowLengthStats = null; private SummaryStatistics flowActive = null; private SummaryStatistics flowIdle = null; @@ -154,6 +190,11 @@ public void initParameters() { this.flowLengthStats = new SummaryStatistics(); this.fwdPktStats = new SummaryStatistics(); this.bwdPktStats = new SummaryStatistics(); + this.fwdSegPayloadStats = new SummaryStatistics(); + this.bwdSegPayloadStats = new SummaryStatistics(); + this.bidirSegPayloadStats = new SummaryStatistics(); + this.fwdSegmentStats = new SummaryStatistics(); + this.bwdSegmentStats = new SummaryStatistics(); this.flagCounts = new HashMap(); initFlags(); this.forwardBytes = 0L; @@ -170,6 +211,8 @@ public void initParameters() { this.bFIN_cnt = 0; this.fHeaderBytes = 0L; this.bHeaderBytes = 0L; + this.fwdSegHeaderLenMin = 0L; + this.bwdSegHeaderLenMin = 0L; this.cumulativeConnectionDuration = 0L; this.tcpFlowState = null; this.tcpPacketsSeen = new HashSet(); @@ -195,13 +238,41 @@ public void firstPacket(BasicPacketInfo packet) { this.flowLastSeen = packet.getTimeStamp(); this.startActiveTime = packet.getTimeStamp(); detectUpdateSubflows(packet); - this.flowLengthStats.addValue((double) packet.getPayloadBytes()); + + // track fwd and bwd packet length stats + this.flowLengthStats.addValue( + (double)(packet.getPacketHeaderBytes() + + packet.getHeaderBytes() + + packet.getPayloadBytes()) + ); + + // track fwd and bwd L4 payload stats + this.bidirSegPayloadStats.addValue((double)(packet.getPayloadBytes())); + if (Arrays.equals(this.src, packet.getSrc())) { - this.min_seg_size_forward = packet.getHeaderBytes(); + // track fwd segment header length min + this.fwdSegHeaderLenMin = packet.getHeaderBytes(); + Init_Win_bytes_forward = packet.getTCPWindow(); - this.fwdPktStats.addValue((double) packet.getPayloadBytes()); + + // track fwd packet length stats + this.fwdPktStats.addValue( + (double)(packet.getPacketHeaderBytes() + + packet.getHeaderBytes() + + packet.getPayloadBytes()) + ); + + // track fwd L4 payload only + this.fwdSegPayloadStats.addValue((double) packet.getPayloadBytes()); + + // track fwd segment length stats + this.fwdSegmentStats.addValue( + (double)(packet.getHeaderBytes() + packet.getPayloadBytes()) + ); + this.fHeaderBytes = packet.getHeaderBytes(); + this.forwardLastSeen = packet.getTimeStamp(); this.forwardBytes += packet.getPayloadBytes(); this.forward.add(packet); @@ -221,10 +292,28 @@ public void firstPacket(BasicPacketInfo packet) { this.fRST_cnt++; } } else { - this.min_seg_size_backward = packet.getHeaderBytes(); + // track bwd segment header length min + this.bwdSegHeaderLenMin = packet.getHeaderBytes(); + Init_Win_bytes_backward = packet.getTCPWindow(); - this.bwdPktStats.addValue((double) packet.getPayloadBytes()); + + // track bwd packet length stats + this.bwdPktStats.addValue( + (double)(packet.getPacketHeaderBytes() + + packet.getHeaderBytes() + + packet.getPayloadBytes()) + ); + + // track bwd L4 payload only + this.bwdSegPayloadStats.addValue((double) packet.getPayloadBytes()); + + // track bwd segment length stats + this.bwdSegmentStats.addValue( + (double)(packet.getHeaderBytes() + packet.getPayloadBytes()) + ); + this.bHeaderBytes = packet.getHeaderBytes(); + this.backwardLastSeen = packet.getTimeStamp(); this.backwardBytes += packet.getPayloadBytes(); this.backward.add(packet); @@ -283,19 +372,47 @@ public void addPacket(BasicPacketInfo packet) { handleTcpRetransmissionFields(packet); long currentTimestamp = packet.getTimeStamp(); if (isBidirectional) { - this.flowLengthStats.addValue((double) packet.getPayloadBytes()); + + // track fwd and bwd packet length stats + this.flowLengthStats.addValue( + (double)(packet.getPacketHeaderBytes() + + packet.getHeaderBytes() + + packet.getPayloadBytes()) + ); + + // track fwd and bwd L4 payload stats + this.bidirSegPayloadStats.addValue((double)(packet.getPayloadBytes())); + if (Arrays.equals(this.src, packet.getSrc())) { if (packet.getPayloadBytes() >= 1) { this.Act_data_pkt_forward++; } - this.fwdPktStats.addValue((double) packet.getPayloadBytes()); + + // track fwd packet length stats + this.fwdPktStats.addValue( + (double)(packet.getPacketHeaderBytes() + + packet.getHeaderBytes() + + packet.getPayloadBytes()) + ); + + // track L4 payload only + this.fwdSegPayloadStats.addValue((double) packet.getPayloadBytes()); + + // track fwd segment length stats + this.fwdSegmentStats.addValue( + (double)(packet.getHeaderBytes() + packet.getPayloadBytes()) + ); + this.fHeaderBytes += packet.getHeaderBytes(); + + // track fwd segment header length min + this.fwdSegHeaderLenMin = Math.min(this.fwdSegHeaderLenMin, packet.getHeaderBytes()); + this.forward.add(packet); this.forwardBytes += packet.getPayloadBytes(); if (this.forward.size() > 1) this.forwardIAT.addValue(currentTimestamp - this.forwardLastSeen); this.forwardLastSeen = currentTimestamp; - this.min_seg_size_forward = Math.min(packet.getHeaderBytes(), this.min_seg_size_forward); if (packet.hasFlagPSH()) { this.fPSH_cnt++; } @@ -312,20 +429,38 @@ public void addPacket(BasicPacketInfo packet) { if (packet.getPayloadBytes() >= 1) { this.Act_data_pkt_backward++; } - this.bwdPktStats.addValue((double) packet.getPayloadBytes()); + + // track bwd packet length stats + this.bwdPktStats.addValue( + (double)(packet.getPacketHeaderBytes() + + packet.getHeaderBytes() + + packet.getPayloadBytes()) + ); + + // track L4 payload only + this.bwdSegPayloadStats.addValue((double) packet.getPayloadBytes()); + + // track segment length stats + this.bwdSegmentStats.addValue( + (double)(packet.getHeaderBytes() + packet.getPayloadBytes()) + ); + // set Init_win_bytes_backward if not been set. The set logic isn't 100% // accurate, since it technically takes the first non-zero value, but should // be good enough for most cases. if (Init_Win_bytes_backward == 0) { Init_Win_bytes_backward = packet.getTCPWindow(); } + this.bHeaderBytes += packet.getHeaderBytes(); + // track bwd segment header length min + this.bwdSegHeaderLenMin = Math.min(this.bwdSegHeaderLenMin, packet.getHeaderBytes()); + this.backward.add(packet); this.backwardBytes += packet.getPayloadBytes(); if (this.backward.size() > 1) this.backwardIAT.addValue(currentTimestamp - this.backwardLastSeen); this.backwardLastSeen = currentTimestamp; - this.min_seg_size_backward = Math.min(packet.getHeaderBytes(), this.min_seg_size_backward); if (packet.hasFlagPSH()) { this.bPSH_cnt++; } @@ -343,14 +478,34 @@ public void addPacket(BasicPacketInfo packet) { if (packet.getPayloadBytes() >= 1) { this.Act_data_pkt_forward++; } - this.fwdPktStats.addValue((double) packet.getPayloadBytes()); - this.flowLengthStats.addValue((double) packet.getPayloadBytes()); + + this.fwdPktStats.addValue( + (double)(packet.getPacketHeaderBytes() + + packet.getHeaderBytes() + + packet.getPayloadBytes()) + ); + + this.flowLengthStats.addValue( + (double)(packet.getPacketHeaderBytes() + + packet.getHeaderBytes() + + packet.getPayloadBytes()) + ); + + this.bidirSegPayloadStats.addValue((double)(packet.getPayloadBytes())); + + this.fwdSegPayloadStats.addValue((double) packet.getPayloadBytes()); + + this.fwdSegmentStats.addValue( + (double)(packet.getHeaderBytes() + packet.getPayloadBytes()) + ); + this.fHeaderBytes += packet.getHeaderBytes(); + this.fwdSegHeaderLenMin = Math.min(this.fwdSegHeaderLenMin, packet.getHeaderBytes()); + this.forward.add(packet); this.forwardBytes += packet.getPayloadBytes(); this.forwardIAT.addValue(currentTimestamp - this.forwardLastSeen); this.forwardLastSeen = currentTimestamp; - this.min_seg_size_forward = Math.min(packet.getHeaderBytes(), this.min_seg_size_forward); } this.flowIAT.addValue(packet.getTimeStamp() - this.flowLastSeen); @@ -381,23 +536,12 @@ public double getDownUpRatio() { return 0; } - public double getAvgPacketSize() { - if (this.packetCount() > 0) { - return (this.flowLengthStats.getSum() / this.packetCount()); - } - return 0; - } - - public double fAvgSegmentSize() { - if (this.forward.size() != 0) - return (this.fwdPktStats.getSum() / (double) this.forward.size()); - return 0; + public double getFwdSegmentLengthMean() { + return (this.fwdSegmentStats.getN() > 0) ? this.fwdSegmentStats.getMean() : 0; } - public double bAvgSegmentSize() { - if (this.backward.size() != 0) - return (this.bwdPktStats.getSum() / (double) this.backward.size()); - return 0; + public double getBwdSegmentLengthMean() { + return (this.bwdSegmentStats.getN() > 0) ? this.bwdSegmentStats.getMean() : 0; } public void initFlags() { @@ -772,7 +916,6 @@ public String dumpFlowBasedFeatures() { dump += this.flowLengthStats.getMax() + ","; dump += this.flowLengthStats.getMean() + ","; dump += this.flowLengthStats.getStandardDeviation() + ","; - dump += flowLengthStats.getVariance() + ","; } else { dump += "0,0,0,0,"; } @@ -782,9 +925,6 @@ public String dumpFlowBasedFeatures() { } dump += getDownUpRatio() + ","; - dump += getAvgPacketSize() + ","; - dump += fAvgSegmentSize() + ","; - dump += bAvgSegmentSize() + ","; dump += this.fHeaderBytes + ","; //this feature is duplicated @@ -804,8 +944,6 @@ public String dumpFlowBasedFeatures() { dump += this.Init_Win_bytes_backward + ","; dump += this.Act_data_pkt_forward + ","; dump += this.Act_data_pkt_forward + ","; - dump += this.min_seg_size_forward + ","; - dump += this.min_seg_size_backward + ","; if (this.flowActive.getN() > 0) { dump += this.flowActive.getMean() + ","; @@ -1022,6 +1160,40 @@ public double getBwdPacketLengthStd() { return (bwdPktStats.getN() > 0L) ? bwdPktStats.getStandardDeviation() : 0; } + // fwd L4 payload (= segment payload) stats -- used to be called "packet" length stats in the original version + public double getFwdSegPayloadLengthTotal() { + return (this.fwdSegPayloadStats.getN() > 0) ? this.fwdSegPayloadStats.getSum() : 0; + } + public double getFwdSegPayloadLengthMax() { + return (this.fwdSegPayloadStats.getN() > 0) ? this.fwdSegPayloadStats.getMax() : 0; + } + public double getFwdSegPayloadLengthMin() { + return (this.fwdSegPayloadStats.getN() > 0) ? this.fwdSegPayloadStats.getMin() : 0; + } + public double getFwdSegPayloadLengthMean() { + return (this.fwdSegPayloadStats.getN() > 0) ? this.fwdSegPayloadStats.getMean() : 0; + } + public double getFwdSegPayloadLengthStd() { + return (this.fwdSegPayloadStats.getN() > 0) ? this.fwdSegPayloadStats.getStandardDeviation() : 0; + } + + // bwd L4 payload stats + public double getBwdSegPayloadLengthTotal() { + return (this.bwdSegPayloadStats.getN() > 0) ? this.bwdSegPayloadStats.getSum() : 0; + } + public double getBwdSegPayloadLengthMax() { + return (this.bwdSegPayloadStats.getN() > 0) ? this.bwdSegPayloadStats.getMax() : 0; + } + public double getBwdSegPayloadLengthMin() { + return (this.bwdSegPayloadStats.getN() > 0) ? this.bwdSegPayloadStats.getMin() : 0; + } + public double getBwdSegPayloadLengthMean() { + return (this.bwdSegPayloadStats.getN() > 0) ? this.bwdSegPayloadStats.getMean() : 0; + } + public double getBwdSegPayloadLengthStd() { + return (this.bwdSegPayloadStats.getN() > 0) ? this.bwdSegPayloadStats.getStandardDeviation() : 0; + } + public double getFlowBytesPerSec() { //flow duration is in microseconds, therefore packets per seconds = packets / (duration/1000000) return ((double) (forwardBytes + backwardBytes)) / ((double) getFlowDuration() / 1000000L); @@ -1107,6 +1279,14 @@ public long getBwdHeaderLength() { return bHeaderBytes; } + // fwd/bwd segment header length min getters + public long getFwdSegHeaderLenMin() { + return (this.forward.size() > 0) ? this.fwdSegHeaderLenMin : 0; + } + public long getBwdSegHeaderLenMin() { + return (this.backward.size() > 0) ? this.bwdSegHeaderLenMin : 0; + } + public double getMinPacketLength() { return (forward.size() > 0 || backward.size() > 0) ? flowLengthStats.getMin() : 0; } @@ -1123,8 +1303,18 @@ public double getPacketLengthStd() { return (forward.size() > 0 || backward.size() > 0) ? flowLengthStats.getStandardDeviation() : 0; } - public double getPacketLengthVariance() { - return (forward.size() > 0 || backward.size() > 0) ? flowLengthStats.getVariance() : 0; + // bidirectional L4 payload stat getters + public double getBidirSegPayloadLengthMin() { + return (this.forward.size() > 0 || this.backward.size() > 0) ? this.bidirSegPayloadStats.getMin() : 0; + } + public double getBidirSegPayloadLengthMax() { + return (this.forward.size() > 0 || this.backward.size() > 0) ? this.bidirSegPayloadStats.getMax() : 0; + } + public double getBidirSegPayloadLengthMean() { + return (this.forward.size() > 0 || this.backward.size() > 0) ? this.bidirSegPayloadStats.getMean() : 0; + } + public double getBidirSegPayloadLengthStd() { + return (this.forward.size() > 0 || this.backward.size() > 0) ? this.bidirSegPayloadStats.getStandardDeviation() : 0; } public int getFlagCount(String key) { @@ -1147,12 +1337,12 @@ public long getAct_data_pkt_backward() { return Act_data_pkt_backward; } - public long getmin_seg_size_forward() { - return min_seg_size_forward; + public double getFwdSegmentLengthMin() { + return (this.fwdSegmentStats.getN() > 0) ? this.fwdSegmentStats.getMin() : 0; } - public long getmin_seg_size_backward() { - return min_seg_size_backward; + public double getBwdSegmentLengthMin() { + return (this.bwdSegmentStats.getN() > 0) ? this.bwdSegmentStats.getMin() : 0; } public double getActiveMean() { @@ -1329,19 +1519,16 @@ public String dumpFlowBasedFeaturesEx() { dump.append(getfPktsPerSecond()).append(separator); //45 dump.append(getbPktsPerSecond()).append(separator); //46 - if (this.forward.size() > 0 || this.backward.size() > 0) { dump.append(flowLengthStats.getMin()).append(separator); //47 dump.append(flowLengthStats.getMax()).append(separator); //48 dump.append(flowLengthStats.getMean()).append(separator); //49 dump.append(flowLengthStats.getStandardDeviation()).append(separator); //50 - dump.append(flowLengthStats.getVariance()).append(separator); //51 } else {//seem to less one dump.append(0).append(separator); dump.append(0).append(separator); dump.append(0).append(separator); dump.append(0).append(separator); - dump.append(0).append(separator); } /*for(MutableInt v:flagCounts.values()) { @@ -1350,46 +1537,45 @@ public String dumpFlowBasedFeaturesEx() { for(String key: flagCounts.keySet()){ dump.append(flagCounts.get(key).value).append(separator); //50,51,52,53,54,55,56,57 } */ - dump.append(flagCounts.get("FIN").value).append(separator); //52 - dump.append(flagCounts.get("SYN").value).append(separator); //53 - dump.append(flagCounts.get("RST").value).append(separator); //54 - dump.append(flagCounts.get("PSH").value).append(separator); //55 - dump.append(flagCounts.get("ACK").value).append(separator); //56 - dump.append(flagCounts.get("URG").value).append(separator); //57 - dump.append(flagCounts.get("CWR").value).append(separator); //58 - dump.append(flagCounts.get("ECE").value).append(separator); //59 - - dump.append(getDownUpRatio()).append(separator); //60 - dump.append(getAvgPacketSize()).append(separator); //61 - dump.append(fAvgSegmentSize()).append(separator); //62 - dump.append(bAvgSegmentSize()).append(separator); //63 - //dump.append(fHeaderBytes).append(separator); //62 dupicate with 43 - - dump.append(fAvgBytesPerBulk()).append(separator); //64 - dump.append(fAvgPacketsPerBulk()).append(separator); //65 - dump.append(fAvgBulkRate()).append(separator); //66 - dump.append(bAvgBytesPerBulk()).append(separator); //67 - dump.append(bAvgPacketsPerBulk()).append(separator); //68 - dump.append(bAvgBulkRate()).append(separator); //69 - - dump.append(getSflow_fpackets()).append(separator); //70 - dump.append(getSflow_fbytes()).append(separator); //71 - dump.append(getSflow_bpackets()).append(separator); //72 - dump.append(getSflow_bbytes()).append(separator); //73 - - dump.append(Init_Win_bytes_forward).append(separator); //74 - dump.append(Init_Win_bytes_backward).append(separator); //75 - dump.append(Act_data_pkt_forward).append(separator); //76 - dump.append(Act_data_pkt_backward).append(separator); //77 - dump.append(min_seg_size_forward).append(separator); //78 - dump.append(min_seg_size_backward).append(separator); //79 + dump.append(flagCounts.get("FIN").value).append(separator); //51 + dump.append(flagCounts.get("SYN").value).append(separator); //52 + dump.append(flagCounts.get("RST").value).append(separator); //53 + dump.append(flagCounts.get("PSH").value).append(separator); //54 + dump.append(flagCounts.get("ACK").value).append(separator); //55 + dump.append(flagCounts.get("URG").value).append(separator); //56 + dump.append(flagCounts.get("CWR").value).append(separator); //57 + dump.append(flagCounts.get("ECE").value).append(separator); //58 + + dump.append(getDownUpRatio()).append(separator); //59 + dump.append(this.getFwdSegmentLengthMean()).append(separator); //60 + dump.append(this.getBwdSegmentLengthMean()).append(separator); //61 + //dump.append(fHeaderBytes).append(separator); + + dump.append(fAvgBytesPerBulk()).append(separator); //62 + dump.append(fAvgPacketsPerBulk()).append(separator); //63 + dump.append(fAvgBulkRate()).append(separator); //64 + dump.append(bAvgBytesPerBulk()).append(separator); //65 + dump.append(bAvgPacketsPerBulk()).append(separator); //66 + dump.append(bAvgBulkRate()).append(separator); //67 + + dump.append(getSflow_fpackets()).append(separator); //68 + dump.append(getSflow_fbytes()).append(separator); //69 + dump.append(getSflow_bpackets()).append(separator); //70 + dump.append(getSflow_bbytes()).append(separator); //71 + + dump.append(Init_Win_bytes_forward).append(separator); //72 + dump.append(Init_Win_bytes_backward).append(separator); //73 + dump.append(Act_data_pkt_forward).append(separator); //74 + dump.append(Act_data_pkt_backward).append(separator); //75 + dump.append(this.getFwdSegmentLengthMin()).append(separator); //76 + dump.append(this.getBwdSegmentLengthMin()).append(separator); //77 if (this.flowActive.getN() > 0) { - dump.append(flowActive.getMean()).append(separator); //80 - dump.append(flowActive.getStandardDeviation()).append(separator); //81 - dump.append(flowActive.getMax()).append(separator); //82 - dump.append(flowActive.getMin()).append(separator); //83 + dump.append(flowActive.getMean()).append(separator); //78 + dump.append(flowActive.getStandardDeviation()).append(separator); //79 + dump.append(flowActive.getMax()).append(separator); //80 + dump.append(flowActive.getMin()).append(separator); //81 } else { dump.append(0).append(separator); dump.append(0).append(separator); @@ -1398,10 +1584,10 @@ public String dumpFlowBasedFeaturesEx() { } if (this.flowIdle.getN() > 0) { - dump.append(flowIdle.getMean()).append(separator); //84 - dump.append(flowIdle.getStandardDeviation()).append(separator); //85 - dump.append(flowIdle.getMax()).append(separator); //86 - dump.append(flowIdle.getMin()).append(separator); //87 + dump.append(flowIdle.getMean()).append(separator); //82 + dump.append(flowIdle.getStandardDeviation()).append(separator); //83 + dump.append(flowIdle.getMax()).append(separator); //84 + dump.append(flowIdle.getMin()).append(separator); //85 } else { dump.append(0).append(separator); dump.append(0).append(separator); @@ -1409,15 +1595,36 @@ public String dumpFlowBasedFeaturesEx() { dump.append(0).append(separator); } - dump.append(icmpCode).append(separator); // 88 - dump.append(icmpType).append(separator); // 89 + dump.append(icmpCode).append(separator); //86 + dump.append(icmpType).append(separator); //87 - dump.append(fwdTcpRetransCnt).append(separator); // 88 - dump.append(bwdTcpRetransCnt).append(separator); // 89 - dump.append(fwdTcpRetransCnt+bwdTcpRetransCnt).append(separator); // 90 + dump.append(fwdTcpRetransCnt).append(separator); //88 + dump.append(bwdTcpRetransCnt).append(separator); //89 + dump.append(fwdTcpRetransCnt+bwdTcpRetransCnt).append(separator); //90 dump.append(cumulativeConnectionDuration).append(separator); //91 - dump.append(getLabel()); //92 + + // L4 payload (= segment payload) length stats + dump.append(this.getFwdSegPayloadLengthTotal()).append(separator); //92 + dump.append(this.getFwdSegPayloadLengthMax()).append(separator); //93 + dump.append(this.getFwdSegPayloadLengthMin()).append(separator); //94 + dump.append(this.getFwdSegPayloadLengthMean()).append(separator); //95 + dump.append(this.getFwdSegPayloadLengthStd()).append(separator); //96 + dump.append(this.getBwdSegPayloadLengthTotal()).append(separator); //97 + dump.append(this.getBwdSegPayloadLengthMax()).append(separator); //98 + dump.append(this.getBwdSegPayloadLengthMin()).append(separator); //99 + dump.append(this.getBwdSegPayloadLengthMean()).append(separator); //100 + dump.append(this.getBwdSegPayloadLengthStd()).append(separator); //101 + dump.append(this.getBidirSegPayloadLengthMax()).append(separator); //102 + dump.append(this.getBidirSegPayloadLengthMin()).append(separator); //103 + dump.append(this.getBidirSegPayloadLengthMean()).append(separator); //104 + dump.append(this.getBidirSegPayloadLengthStd()).append(separator); //105 + + // fwd/bwd segment header (L4 header) length min + dump.append(this.getFwdSegHeaderLenMin()).append(separator); //106 + dump.append(this.getBwdSegHeaderLenMin()).append(separator); //107 + + dump.append(getLabel()); //108 return dump.toString(); } diff --git a/src/main/java/cic/cs/unb/ca/jnetpcap/BasicPacketInfo.java b/src/main/java/cic/cs/unb/ca/jnetpcap/BasicPacketInfo.java index 749314cd..ac7bbeff 100644 --- a/src/main/java/cic/cs/unb/ca/jnetpcap/BasicPacketInfo.java +++ b/src/main/java/cic/cs/unb/ca/jnetpcap/BasicPacketInfo.java @@ -4,6 +4,17 @@ import org.jnetpcap.packet.format.FormatUtils; + +// Layer and PDU terminology +// PDU (Protocol Data Unit): the data unit at a given network layer, including +// that layer's header plus everything it encapsulates +// L3 PDU = packet = IP datagram = L3 (IP) header + L4 header + L4 payload +// L4 PDU = segment = L4 unit = L4 (TCP/UDP/SCTP) header + L4 payload +// "segment" is used loosely throughout to mean any L4 PDU regardless of protocol +// strictly speaking, the correct terms should be: TCP segment, UDP datagram, etc. + + + public class BasicPacketInfo { /* Basic Info to generate flows from packets */ @@ -14,7 +25,7 @@ public class BasicPacketInfo { private int dstPort; private ProtocolEnum protocol = ProtocolEnum.DEFAULT; private long timeStamp; - private long payloadBytes; + private long payloadBytes; // L4 payload length = segment payload length, in bytes private String flowId = null; /* ******************************************** */ private boolean flagFIN = false; @@ -26,7 +37,8 @@ public class BasicPacketInfo { private boolean flagCWR = false; private boolean flagRST = false; private int TCPWindow=0; - private long headerBytes; + private long headerBytes; // L4 header length = segment header length, in bytes + private long packetHeaderBytes; // L3 header length = packet header length, in bytes private int payloadPacket=0; /* ** ICMP FIELDS ** */ private int icmpCode = -1; @@ -171,22 +183,36 @@ public boolean isForwardPacket(byte[] sourceIP) { return Arrays.equals(sourceIP, this.src); } + // L4 payload length getter public long getPayloadBytes() { return payloadBytes; } + // L4 payload length setter public void setPayloadBytes(long payloadBytes) { this.payloadBytes = payloadBytes; } + // L4 header length getter public long getHeaderBytes() { return headerBytes; } + // L4 header length setter public void setHeaderBytes(long headerBytes) { this.headerBytes = headerBytes; } + // L3 header length getter + public long getPacketHeaderBytes() { + return this.packetHeaderBytes; + } + + // L3 header length setter + public void setPacketHeaderBytes(long packetHeaderBytes) { + this.packetHeaderBytes = packetHeaderBytes; + } + public boolean hasFlagFIN() { return flagFIN; } diff --git a/src/main/java/cic/cs/unb/ca/jnetpcap/FlowFeature.java b/src/main/java/cic/cs/unb/ca/jnetpcap/FlowFeature.java index 62508bbb..9c36b579 100644 --- a/src/main/java/cic/cs/unb/ca/jnetpcap/FlowFeature.java +++ b/src/main/java/cic/cs/unb/ca/jnetpcap/FlowFeature.java @@ -62,53 +62,70 @@ public enum FlowFeature { pkt_len_max("Packet Length Max","PLMA"), //48 pkt_len_avg("Packet Length Mean","PLAG"), //49 pkt_len_std("Packet Length Std","PLSD"), //50 - pkt_len_var("Packet Length Variance","PLVA"), //51 - fin_cnt("FIN Flag Count","FINCT"), //52 - syn_cnt("SYN Flag Count","SYNCT"), //53 - rst_cnt("RST Flag Count","RSTCT"), //54 - pst_cnt("PSH Flag Count","PSHCT"), //55 - ack_cnt("ACK Flag Count","ACKCT"), //56 - urg_cnt("URG Flag Count","URGCT"), //57 - CWR_cnt("CWR Flag Count","CWRCT"), //58 - ece_cnt("ECE Flag Count","ECECT"), //59 - down_up_ratio("Down/Up Ratio","D/URO"), //60 - pkt_size_avg("Average Packet Size","PSAG"), //61 - fw_seg_avg("Fwd Segment Size Avg","FwSgAG"), //62 - bw_seg_avg("Bwd Segment Size Avg","BwSgAG"), //63 - fw_byt_blk_avg("Fwd Bytes/Bulk Avg","FwB/BAG"), //64 62 is duplicated with 43,so has been deleted - fw_pkt_blk_avg("Fwd Packet/Bulk Avg","FwP/BAG"), //65 - fw_blk_rate_avg("Fwd Bulk Rate Avg","FwBRAG"), //66 - bw_byt_blk_avg("Bwd Bytes/Bulk Avg","BwB/BAG"), //67 - bw_pkt_blk_avg("Bwd Packet/Bulk Avg","BwP/BAG"), //68 - bw_blk_rate_avg("Bwd Bulk Rate Avg","BwBRAG"), //69 - subfl_fw_pkt("Subflow Fwd Packets","SFFwP"), //70 - subfl_fw_byt("Subflow Fwd Bytes","SFFwB"), //71 - subfl_bw_pkt("Subflow Bwd Packets","SFBwP"), //72 - subfl_bw_byt("Subflow Bwd Bytes","SFBwB"), //73 - fw_win_byt("FWD Init Win Bytes","FwWB"), //74 - bw_win_byt("Bwd Init Win Bytes","BwWB"), //75 - Fw_act_pkt("Fwd Act Data Pkts","FwAP"), - Bw_act_pkt("Bwd Act Data Pkts","BwAP"),//76 - fw_seg_min("Fwd Seg Size Min","FwSgMI"), //77 - bw_seg_min("Bwd Seg Size Min", "BwSgMI"), - atv_avg("Active Mean","AcAG"), //78 - atv_std("Active Std","AcSD"), //79 - atv_max("Active Max","AcMA"), //80 - atv_min("Active Min","AcMI"), //81 - idl_avg("Idle Mean","IlAG"), //82 - idl_std("Idle Std","IlSD"), //83 - idl_max("Idle Max","IlMA"), //84 - idl_min("Idle Min","IlMI"), //85 - icmp_code("ICMP Code", "IcmpC"), // 86 - icmp_type("ICMP Type", "IcmpT"), // 87 - - fwd_tcp_retrans("Fwd TCP Retrans. Count", "FwTcpRt"), //88 - bwd_tcp_retrans("Bwd TCP Retrans. Count", "BwTcpRt"), //89 - total_tcp_retrans("Total TCP Retrans. Count", "TotalTcpRt"), //90 - - cum_cnx_time("Total Connection Flow Time", "TCFT"), // 91 - - Label("Label","LBL",new String[]{"NeedManualLabel"}); //92 + fin_cnt("FIN Flag Count","FINCT"), //51 + syn_cnt("SYN Flag Count","SYNCT"), //52 + rst_cnt("RST Flag Count","RSTCT"), //53 + pst_cnt("PSH Flag Count","PSHCT"), //54 + ack_cnt("ACK Flag Count","ACKCT"), //55 + urg_cnt("URG Flag Count","URGCT"), //56 + CWR_cnt("CWR Flag Count","CWRCT"), //57 + ece_cnt("ECE Flag Count","ECECT"), //58 + down_up_ratio("Down/Up Ratio","D/URO"), //59 + fw_seg_avg("Fwd Segment Size Avg","FwSgAG"), //60 + bw_seg_avg("Bwd Segment Size Avg","BwSgAG"), //61 + fw_byt_blk_avg("Fwd Bytes/Bulk Avg","FwB/BAG"), //62 62 is duplicated with 43,so has been deleted + fw_pkt_blk_avg("Fwd Packet/Bulk Avg","FwP/BAG"), //63 + fw_blk_rate_avg("Fwd Bulk Rate Avg","FwBRAG"), //64 + bw_byt_blk_avg("Bwd Bytes/Bulk Avg","BwB/BAG"), //65 + bw_pkt_blk_avg("Bwd Packet/Bulk Avg","BwP/BAG"), //66 + bw_blk_rate_avg("Bwd Bulk Rate Avg","BwBRAG"), //67 + subfl_fw_pkt("Subflow Fwd Packets","SFFwP"), //68 + subfl_fw_byt("Subflow Fwd Bytes","SFFwB"), //69 + subfl_bw_pkt("Subflow Bwd Packets","SFBwP"), //70 + subfl_bw_byt("Subflow Bwd Bytes","SFBwB"), //71 + fw_win_byt("FWD Init Win Bytes","FwWB"), //72 + bw_win_byt("Bwd Init Win Bytes","BwWB"), //73 + Fw_act_pkt("Fwd Act Data Pkts","FwAP"), //74 + Bw_act_pkt("Bwd Act Data Pkts","BwAP"), //75 + fw_seg_min("Fwd Seg Size Min","FwSgMI"), //76 + bw_seg_min("Bwd Seg Size Min", "BwSgMI"), //77 + atv_avg("Active Mean","AcAG"), //78 + atv_std("Active Std","AcSD"), //79 + atv_max("Active Max","AcMA"), //80 + atv_min("Active Min","AcMI"), //81 + idl_avg("Idle Mean","IlAG"), //82 + idl_std("Idle Std","IlSD"), //83 + idl_max("Idle Max","IlMA"), //84 + idl_min("Idle Min","IlMI"), //85 + icmp_code("ICMP Code", "IcmpC"), //86 + icmp_type("ICMP Type", "IcmpT"), //87 + + fwd_tcp_retrans("Fwd TCP Retrans. Count", "FwTcpRt"), //88 + bwd_tcp_retrans("Bwd TCP Retrans. Count", "BwTcpRt"), //89 + total_tcp_retrans("Total TCP Retrans. Count", "TotalTcpRt"), //90 + + cum_cnx_time("Total Connection Flow Time", "TCFT"), //91 + + // L4 payload (= segment payload) length stats + fw_pay_l_tot("Fwd Segment Payload Length Total","FwSPaTO"), //92 + fw_pay_l_max("Fwd Segment Payload Length Max","FwSPaLMA"), //93 + fw_pay_l_min("Fwd Segment Payload Length Min","FwSPaLMI"), //94 + fw_pay_l_avg("Fwd Segment Payload Length Mean","FwSPaLAG"), //95 + fw_pay_l_std("Fwd Segment Payload Length Std","FwSPaLSD"), //96 + bw_pay_l_tot("Bwd Segment Payload Length Total","BwSPaTO"), //97 + bw_pay_l_max("Bwd Segment Payload Length Max","BwSPaLMA"), //98 + bw_pay_l_min("Bwd Segment Payload Length Min","BwSPaLMI"), //99 + bw_pay_l_avg("Bwd Segment Payload Length Mean","BwSPaLAG"), //100 + bw_pay_l_std("Bwd Segment Payload Length Std","BwSPaLSD"), //101 + pay_l_max("Segment Payload Length Max","SPaLMA"), //102 + pay_l_min("Segment Payload Length Min","SPaLMI"), //103 + pay_l_avg("Segment Payload Length Mean","SPaLAG"), //104 + pay_l_std("Segment Payload Length Std","SPaLSD"), //105 + + fw_seg_hdr_len_min("Fwd Segment Header Length Min","FwSgHLMI"), //106 + bw_seg_hdr_len_min("Bwd Segment Header Length Min","BwSgHLMI"), //107 + + Label("Label","LBL",new String[]{"NeedManualLabel"}); //108 protected static final Logger logger = LoggerFactory.getLogger(FlowFeature.class); @@ -192,7 +209,7 @@ public static List getLengthFeature(){ features.add(bw_hdr_len); features.add(fw_pkt_s); features.add(bw_pkt_s); - features.add(pkt_size_avg); + // features.add(pkt_size_avg); features.add(fw_seg_avg); features.add(bw_seg_avg); return features; diff --git a/src/main/java/cic/cs/unb/ca/jnetpcap/PacketReader.java b/src/main/java/cic/cs/unb/ca/jnetpcap/PacketReader.java index a61ec3a6..6d086c86 100644 --- a/src/main/java/cic/cs/unb/ca/jnetpcap/PacketReader.java +++ b/src/main/java/cic/cs/unb/ca/jnetpcap/PacketReader.java @@ -20,6 +20,16 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; + +// Layer and PDU terminology +// PDU (Protocol Data Unit): the data unit at a given network layer, including +// that layer's header plus everything it encapsulates +// L3 PDU = packet = IP datagram = L3 (IP) header + L4 header + L4 payload +// L4 PDU = segment = L4 unit = L4 (TCP/UDP/SCTP) header + L4 payload +// "segment" is used loosely throughout to mean any L4 PDU regardless of protocol +// strictly speaking, the correct terms should be: TCP segment, UDP datagram, etc. + + public class PacketReader { private static final Logger logger = LoggerFactory.getLogger(PacketReader.class); @@ -132,6 +142,10 @@ private BasicPacketInfo getIpv4Info(PcapPacket packet){ this.firstPacket = packet.getCaptureHeader().timestampInMillis(); this.lastPacket = packet.getCaptureHeader().timestampInMillis(); + // set L3 header length + // getHeaderLength() from class JHeader is in the inherited methods list of class Ip4 in the javadoc of jnetpcap + packetInfo.setPacketHeaderBytes(this.ipv4.getHeaderLength()); + // This check needs to be first because ICMP protocol may embed the contents of // the original packet (so it will have tcp/udp headers as well). if (packet.hasHeader(this.icmp)) { @@ -233,7 +247,11 @@ private BasicPacketInfo getIpv6Info(PcapPacket packet){ packetInfo.setSrc(this.ipv6.source()); packetInfo.setDst(this.ipv6.destination()); packetInfo.setTimeStamp(packet.getCaptureHeader().timestampInMillis()); - + + // set L3 header length + // getHeaderLength() from class JHeader is in the inherited methods list of class Ip6 in the javadoc of jnetpcap + packetInfo.setPacketHeaderBytes(this.ipv6.getHeaderLength()); + if(packet.hasHeader(this.tcp)){ packetInfo.setSrcPort(tcp.source()); packetInfo.setDstPort(tcp.destination()); @@ -404,6 +422,9 @@ private static BasicPacketInfo getIpv6Info(PcapPacket packet,Protocol protocol) packetInfo.setDst(protocol.getIpv6().destination()); packetInfo.setTimeStamp(packet.getCaptureHeader().timestampInMillis()); + // set L3 header length + packetInfo.setPacketHeaderBytes(protocol.getIpv6().getHeaderLength()); + if(packet.hasHeader(protocol.getTcp())){ packetInfo.setSrcPort(protocol.getTcp().source()); packetInfo.setDstPort(protocol.getTcp().destination()); @@ -453,6 +474,9 @@ private static BasicPacketInfo getIpv4Info(PcapPacket packet,Protocol protocol) this.firstPacket = packet.getCaptureHeader().timestampInMillis(); this.lastPacket = packet.getCaptureHeader().timestampInMillis();*/ + // set L3 header length + packetInfo.setPacketHeaderBytes(protocol.getIpv4().getHeaderLength()); + if(packet.hasHeader(protocol.getTcp())){ packetInfo.setTCPWindow(protocol.getTcp().window()); packetInfo.setSrcPort(protocol.getTcp().source());